mirror of
https://github.com/Stirling-Tools/Stirling-PDF.git
synced 2026-09-03 05:10:16 +03:00
Compare commits
22
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
89db09a4e6 | ||
|
|
6b9567cf38 | ||
|
|
c731d5fd5d | ||
|
|
beb99e273b | ||
|
|
ece1bb6865 | ||
|
|
5b9ef852ab | ||
|
|
672e81d286 | ||
|
|
8abe734f0b | ||
|
|
8a59c10f42 | ||
|
|
d62f2ad3ed | ||
|
|
f60a075443 | ||
|
|
c059e13423 | ||
|
|
575684ee4b | ||
|
|
294b616a63 | ||
|
|
f61121953e | ||
|
|
8645280ee5 | ||
|
|
bd50e2e59d | ||
|
|
77abaf5de5 | ||
|
|
503a81be50 | ||
|
|
67fc5c70b9 | ||
|
|
1e6514a471 | ||
|
|
c4874deb5d |
@@ -22,7 +22,7 @@ runs:
|
||||
steps:
|
||||
- name: Generate a GitHub App Token
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
|
||||
with:
|
||||
app-id: ${{ inputs.app-id }}
|
||||
private-key: ${{ inputs.private-key }}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-desktop
|
||||
pkgver=2.10.1
|
||||
pkgver=2.11.0
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (Tauri desktop app, official Stirling PDF Inc build)"
|
||||
arch=('x86_64')
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-server-bin
|
||||
pkgver=2.10.1
|
||||
pkgver=2.11.0
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (server JAR, prebuilt)"
|
||||
arch=('any')
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
pr_ref: ${{ steps.resolve.outputs.ref }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -111,7 +111,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -359,7 +359,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
enable_prototypes: ${{ steps.check-prototypes-flag.outputs.enable_prototypes }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -183,7 +183,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -474,7 +474,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
|
||||
- name: Publish stirling-pdf-desktop to AUR
|
||||
if: ${{ github.event_name == 'release' || inputs.dry_run == false }}
|
||||
uses: KSXGitHub/github-actions-deploy-aur@v4.1.2
|
||||
uses: KSXGitHub/github-actions-deploy-aur@v4.1.3
|
||||
with:
|
||||
pkgname: stirling-pdf-desktop
|
||||
pkgbuild: .github/aur/stirling-pdf-desktop/PKGBUILD
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
spring-security: [true, false]
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
SYSTEM_ENABLEANALYTICS: "false"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -63,7 +63,15 @@ jobs:
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Install Playwright (chromium only)
|
||||
run: task frontend:test:e2e:install -- chromium
|
||||
run: task e2e:install -- chromium
|
||||
- name: Build frontend (needed for playwright's vite preview webServer)
|
||||
# Enterprise tests target :8080 (Spring Boot's bundled frontend), but
|
||||
# playwright's webServer config still launches `vite preview --port 5173`
|
||||
# before any test run, which needs dist/ to exist. VITE_BUILD_FOR_PREVIEW
|
||||
# forces absolute asset paths so vite preview can serve deep SPA routes.
|
||||
env:
|
||||
VITE_BUILD_FOR_PREVIEW: "1"
|
||||
run: task frontend:build
|
||||
|
||||
- name: Resolve kubernetes.docker.internal to localhost
|
||||
# The compose stacks set KC_HOSTNAME=kubernetes.docker.internal so
|
||||
@@ -140,7 +148,7 @@ jobs:
|
||||
wait_for_backend
|
||||
- name: Run enterprise OAuth Playwright tests
|
||||
id: oauth-tests
|
||||
run: task frontend:test:e2e -- --project=enterprise --grep "OAuth"
|
||||
run: task e2e:enterprise -- --grep "OAuth"
|
||||
- name: Stop backend + tear down OAuth Keycloak
|
||||
if: always()
|
||||
run: |
|
||||
@@ -213,7 +221,7 @@ jobs:
|
||||
wait_for_backend
|
||||
- name: Run enterprise SAML Playwright tests
|
||||
id: saml-tests
|
||||
run: task frontend:test:e2e -- --project=enterprise --grep "SAML"
|
||||
run: task e2e:enterprise -- --grep "SAML"
|
||||
- name: Stop backend + tear down SAML Keycloak
|
||||
if: always()
|
||||
run: |
|
||||
@@ -243,7 +251,7 @@ jobs:
|
||||
wait_for_backend
|
||||
- name: Run enterprise feature Playwright tests
|
||||
id: feature-tests
|
||||
run: task frontend:test:e2e -- --project=enterprise --grep "Enterprise license"
|
||||
run: task e2e:enterprise -- --grep "Enterprise license"
|
||||
- name: Print backend log on failure
|
||||
if: failure()
|
||||
run: |
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
proprietary: ${{ steps.changes.outputs.proprietary }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
pull-requests: write # Allow writing to pull requests
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -34,58 +34,26 @@ jobs:
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Install Playwright (chromium only)
|
||||
run: task frontend:test:e2e:install -- chromium
|
||||
- name: Start Spring Boot backend (background)
|
||||
run: task e2e:install -- chromium
|
||||
- name: Build frontend (production bundle for vite preview)
|
||||
env:
|
||||
# Suppress the analytics opt-in modal that fires on first admin login when
|
||||
# enableAnalytics is null (see Onboarding.tsx). The modal renders a Mantine
|
||||
# overlay that intercepts pointer events on every tool page until dismissed,
|
||||
# which causes every "click run button" assertion in the live suite to fail.
|
||||
SYSTEM_ENABLEANALYTICS: "false"
|
||||
# NOTE: SECURITY_INITIALLOGIN_USERNAME/PASSWORD are intentionally NOT set.
|
||||
# The live-setup project's bootstrap spec performs the real first-login
|
||||
# flow against the backend's default admin/stirling user, exercising the
|
||||
# forced-password-change UI and leaving the DB at admin/adminadmin for
|
||||
# the rest of the live suite. This is both real coverage of the first-
|
||||
# login flow and a stronger seed than env-var-driven user creation.
|
||||
run: |
|
||||
nohup ./gradlew :stirling-pdf:bootRun > /tmp/backend.log 2>&1 &
|
||||
echo $! > /tmp/backend.pid
|
||||
- name: Wait for backend to become ready
|
||||
run: |
|
||||
start=$SECONDS
|
||||
# 300 iterations × 2s = 10 minute ceiling
|
||||
for i in $(seq 1 300); do
|
||||
if curl -fsS http://localhost:8080/api/v1/info/status >/dev/null 2>&1; then
|
||||
echo "Backend up after $((SECONDS - start))s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "Backend did not become ready in $((SECONDS - start))s"
|
||||
tail -200 /tmp/backend.log || true
|
||||
exit 1
|
||||
VITE_BUILD_FOR_PREVIEW: "1"
|
||||
run: task frontend:build
|
||||
- name: Run live E2E tests (chromium)
|
||||
id: live-tests
|
||||
run: task frontend:test:e2e -- --project=live
|
||||
run: task e2e:live
|
||||
- name: Print backend log on failure
|
||||
if: failure() && steps.live-tests.conclusion == 'failure'
|
||||
run: |
|
||||
echo "::group::Spring Boot backend log (last 500 lines)"
|
||||
tail -500 /tmp/backend.log || echo "no backend log found"
|
||||
tail -500 .test-state/playwright/backend.log || echo "no backend log found"
|
||||
echo "::endgroup::"
|
||||
- name: Stop backend
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f /tmp/backend.pid ]; then
|
||||
kill "$(cat /tmp/backend.pid)" 2>/dev/null || true
|
||||
fi
|
||||
- name: Upload backend log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: backend-log-live-${{ github.run_id }}
|
||||
path: /tmp/backend.log
|
||||
path: .test-state/playwright/backend.log
|
||||
retention-days: 7
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -28,9 +28,13 @@ jobs:
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Install Playwright (chromium only)
|
||||
run: task frontend:test:e2e:install -- chromium
|
||||
run: task e2e:install -- chromium
|
||||
- name: Build frontend (production bundle for vite preview)
|
||||
env:
|
||||
VITE_BUILD_FOR_PREVIEW: "1"
|
||||
run: task frontend:build
|
||||
- name: Run stubbed E2E tests (chromium)
|
||||
run: task frontend:test:e2e -- --project=stubbed --workers=3
|
||||
run: task e2e:stubbed -- --workers=3
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
licenses-backend: ${{ steps.changes.outputs.licenses-backend }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
repository-projects: write # Required for enabling automerge
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -316,7 +316,7 @@ jobs:
|
||||
repository-projects: write # Required for enabling automerge
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -85,6 +85,26 @@ jobs:
|
||||
echo "to see what still needs fixing manually."
|
||||
echo "============================================"
|
||||
exit 1
|
||||
- name: Remove frontend check comment on success
|
||||
if: steps.frontend-check.outcome == 'success' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- frontend-check -->';
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
});
|
||||
}
|
||||
- name: Upload frontend build artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
version: ${{ steps.versionNumber.outputs.versionNumber }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependencies
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/caches
|
||||
@@ -127,7 +127,7 @@ jobs:
|
||||
file_suffix: "-server"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -191,7 +191,7 @@ jobs:
|
||||
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
allowed-endpoints: >
|
||||
@@ -606,7 +606,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -35,10 +35,10 @@ jobs:
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Install all Playwright browsers
|
||||
run: task frontend:test:e2e:install
|
||||
run: task e2e:install
|
||||
|
||||
- name: Run E2E tests (all browsers)
|
||||
run: task frontend:test:e2e
|
||||
run: task e2e:cross-browser
|
||||
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
PIP_DISABLE_PIP_VERSION_CHECK: "1"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
echo "version=${VERSION}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -2,6 +2,22 @@ name: Push Docker Image with VersionNumber
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_main_app:
|
||||
description: "Build & push the main Stirling-PDF image (latest, fat, ultra-lite)."
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
build_unoserver:
|
||||
description: "Build & push the standalone stirling-unoserver image."
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
force_unoserver_rebuild:
|
||||
description: "Rebuild stirling-unoserver even if its source hash is unchanged."
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
@@ -31,9 +47,13 @@ jobs:
|
||||
permissions:
|
||||
packages: write
|
||||
id-token: write
|
||||
# On push events these stay 'true'; on workflow_dispatch they follow the inputs.
|
||||
env:
|
||||
RUN_MAIN_APP: ${{ github.event_name != 'workflow_dispatch' || inputs.build_main_app }}
|
||||
RUN_UNOSERVER: ${{ github.event_name != 'workflow_dispatch' || inputs.build_unoserver }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -46,7 +66,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependencies
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/caches
|
||||
@@ -108,6 +128,7 @@ jobs:
|
||||
|
||||
- name: Generate tags for latest
|
||||
id: meta
|
||||
if: env.RUN_MAIN_APP == 'true'
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
with:
|
||||
images: |
|
||||
@@ -122,6 +143,8 @@ jobs:
|
||||
|
||||
- name: Build and push Unified Dockerfile (latest variant)
|
||||
id: build-push-latest
|
||||
# Empty-tag guard: build-push-action errors when asked to push with no tags.
|
||||
if: env.RUN_MAIN_APP == 'true' && steps.meta.outputs.tags != ''
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
@@ -140,7 +163,7 @@ jobs:
|
||||
sbom: true
|
||||
|
||||
- name: Sign regular images
|
||||
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master'
|
||||
if: env.RUN_MAIN_APP == 'true' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master') && steps.build-push-latest.outputs.digest != ''
|
||||
env:
|
||||
DIGEST: ${{ steps.build-push-latest.outputs.digest }}
|
||||
TAGS: ${{ steps.meta.outputs.tags }}
|
||||
@@ -156,7 +179,7 @@ jobs:
|
||||
- name: Generate tags for latest-fat
|
||||
id: meta-fat
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
|
||||
@@ -170,7 +193,7 @@ jobs:
|
||||
- name: Build and push Unified Dockerfile (fat variant)
|
||||
id: build-push-fat
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain' && steps.meta-fat.outputs.tags != ''
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
@@ -186,7 +209,7 @@ jobs:
|
||||
sbom: true
|
||||
|
||||
- name: Sign fat images
|
||||
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master'
|
||||
if: env.RUN_MAIN_APP == 'true' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master') && steps.build-push-fat.outputs.digest != ''
|
||||
env:
|
||||
DIGEST: ${{ steps.build-push-fat.outputs.digest }}
|
||||
TAGS: ${{ steps.meta-fat.outputs.tags }}
|
||||
@@ -200,7 +223,7 @@ jobs:
|
||||
- name: Generate tags for ultra-lite
|
||||
id: meta-lite
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
|
||||
@@ -214,7 +237,7 @@ jobs:
|
||||
- name: Build and push Unified Dockerfile (ultra-lite variant)
|
||||
id: build-push-lite
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain' && steps.meta-lite.outputs.tags != ''
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
@@ -230,7 +253,7 @@ jobs:
|
||||
sbom: true
|
||||
|
||||
- name: Sign ultra-lite images
|
||||
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master'
|
||||
if: env.RUN_MAIN_APP == 'true' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master') && steps.build-push-lite.outputs.digest != ''
|
||||
env:
|
||||
DIGEST: ${{ steps.build-push-lite.outputs.digest }}
|
||||
TAGS: ${{ steps.meta-lite.outputs.tags }}
|
||||
@@ -240,3 +263,136 @@ jobs:
|
||||
echo "$TAGS" | tr ',' '\n' | while read -r tag; do
|
||||
cosign sign --key env://COSIGN_PRIVATE_KEY --yes "${tag}@${DIGEST}"
|
||||
done
|
||||
|
||||
# Standalone unoserver image — versioned independently via
|
||||
# docker/unoserver/VERSION. master/V2-master: publish <version>+latest
|
||||
# only when the version is new. main/testMain: republish :alpha only
|
||||
# when the source hash differs from the published image's annotation.
|
||||
- name: Read unoserver image version
|
||||
id: unoserverVersion
|
||||
if: env.RUN_UNOSERVER == 'true'
|
||||
run: |
|
||||
version=$(tr -d '[:space:]' < docker/unoserver/VERSION)
|
||||
if [ -z "$version" ]; then
|
||||
echo "docker/unoserver/VERSION is empty"; exit 1
|
||||
fi
|
||||
echo "version=${version}" >> "$GITHUB_OUTPUT"
|
||||
echo "Unoserver image version (from file): ${version}"
|
||||
|
||||
- name: Compute unoserver image source hash
|
||||
id: unoserverHash
|
||||
if: env.RUN_UNOSERVER == 'true'
|
||||
run: |
|
||||
set -eu
|
||||
hash=$(cat \
|
||||
docker/unoserver/Dockerfile \
|
||||
docker/unoserver/entrypoint.sh \
|
||||
docker/unoserver/healthcheck.sh \
|
||||
docker/unoserver/VERSION \
|
||||
| sha256sum | cut -d' ' -f1)
|
||||
echo "hash=${hash}" >> "$GITHUB_OUTPUT"
|
||||
echo "Unoserver source hash: ${hash}"
|
||||
|
||||
- name: Decide whether to publish unoserver image
|
||||
id: unoserverDecision
|
||||
if: env.RUN_UNOSERVER == 'true'
|
||||
env:
|
||||
UNOSERVER_VERSION: ${{ steps.unoserverVersion.outputs.version }}
|
||||
UNOSERVER_HASH: ${{ steps.unoserverHash.outputs.hash }}
|
||||
UNOSERVER_IMAGE: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-unoserver
|
||||
UNOSERVER_HASH_ANNOTATION: org.stirlingpdf.unoserver-source-hash
|
||||
FORCE_REBUILD: ${{ inputs.force_unoserver_rebuild }}
|
||||
GH_REF: ${{ github.ref }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
run: |
|
||||
set -eu
|
||||
mode="skip"
|
||||
tags=""
|
||||
|
||||
read_published_hash() {
|
||||
local ref="$1"
|
||||
docker buildx imagetools inspect "$ref" --raw 2>/dev/null \
|
||||
| jq -r --arg key "$UNOSERVER_HASH_ANNOTATION" \
|
||||
'.annotations[$key] // empty' \
|
||||
2>/dev/null || true
|
||||
}
|
||||
|
||||
# Manual dispatch from any branch routes to the :alpha publish path.
|
||||
EFFECTIVE_REF="$GH_REF"
|
||||
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
|
||||
EFFECTIVE_REF="refs/heads/testMain"
|
||||
fi
|
||||
|
||||
case "$EFFECTIVE_REF" in
|
||||
refs/heads/master|refs/heads/V2-master)
|
||||
if [ "${FORCE_REBUILD}" = "true" ]; then
|
||||
echo "force_unoserver_rebuild=true — building stable regardless"
|
||||
mode="stable"
|
||||
tags="${UNOSERVER_IMAGE}:${UNOSERVER_VERSION},${UNOSERVER_IMAGE}:latest"
|
||||
elif docker manifest inspect "${UNOSERVER_IMAGE}:${UNOSERVER_VERSION}" >/dev/null 2>&1; then
|
||||
echo "stirling-unoserver:${UNOSERVER_VERSION} already on GHCR — skipping"
|
||||
else
|
||||
echo "stirling-unoserver:${UNOSERVER_VERSION} is new — will publish"
|
||||
mode="stable"
|
||||
tags="${UNOSERVER_IMAGE}:${UNOSERVER_VERSION},${UNOSERVER_IMAGE}:latest"
|
||||
fi
|
||||
;;
|
||||
refs/heads/main|refs/heads/testMain)
|
||||
published_hash=$(read_published_hash "${UNOSERVER_IMAGE}:alpha")
|
||||
if [ "${FORCE_REBUILD}" = "true" ]; then
|
||||
echo "force_unoserver_rebuild=true — rebuilding :alpha regardless"
|
||||
mode="alpha"
|
||||
tags="${UNOSERVER_IMAGE}:alpha"
|
||||
elif [ -n "$published_hash" ] && [ "$published_hash" = "$UNOSERVER_HASH" ]; then
|
||||
echo "Published :alpha source hash matches (${published_hash}) — skipping"
|
||||
else
|
||||
if [ -z "$published_hash" ]; then
|
||||
echo ":alpha has no source-hash annotation (first publish or pre-tracking image) — will publish"
|
||||
else
|
||||
echo "Source hash changed (was ${published_hash}, now ${UNOSERVER_HASH}) — will publish"
|
||||
fi
|
||||
mode="alpha"
|
||||
tags="${UNOSERVER_IMAGE}:alpha"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "Branch ${GH_REF} does not publish unoserver image"
|
||||
;;
|
||||
esac
|
||||
echo "mode=${mode}" >> "$GITHUB_OUTPUT"
|
||||
echo "tags=${tags}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build and push unoserver image
|
||||
id: build-push-unoserver
|
||||
if: env.RUN_UNOSERVER == 'true' && steps.unoserverDecision.outputs.mode != 'skip'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
file: ./docker/unoserver/Dockerfile
|
||||
push: true
|
||||
cache-from: type=gha,scope=stirling-unoserver
|
||||
cache-to: type=gha,mode=max,scope=stirling-unoserver
|
||||
tags: ${{ steps.unoserverDecision.outputs.tags }}
|
||||
# Manifest annotation read by the decision step above to detect drift.
|
||||
annotations: |
|
||||
index:org.stirlingpdf.unoserver-source-hash=${{ steps.unoserverHash.outputs.hash }}
|
||||
platforms: linux/amd64,linux/arm64/v8
|
||||
provenance: true
|
||||
sbom: true
|
||||
|
||||
- name: Sign unoserver image
|
||||
if: env.RUN_UNOSERVER == 'true' && steps.unoserverDecision.outputs.mode == 'stable'
|
||||
env:
|
||||
DIGEST: ${{ steps.build-push-unoserver.outputs.digest }}
|
||||
TAGS: ${{ steps.unoserverDecision.outputs.tags }}
|
||||
COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
||||
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
|
||||
run: |
|
||||
if [ -n "$COSIGN_PRIVATE_KEY" ]; then
|
||||
echo "$TAGS" | tr ',' '\n' | while read -r tag; do
|
||||
cosign sign --key env://COSIGN_PRIVATE_KEY --yes "${tag}@${DIGEST}"
|
||||
done
|
||||
else
|
||||
echo "Warning: COSIGN_PRIVATE_KEY not set, skipping unoserver image signing"
|
||||
fi
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -75,6 +75,6 @@ jobs:
|
||||
|
||||
# Upload the results to GitHub's code scanning dashboard.
|
||||
- name: "Upload to code-scanning"
|
||||
uses: github/codeql-action/upload-sarif@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v3.29.5
|
||||
uses: github/codeql-action/upload-sarif@e46ed2cbd01164d986452f91f178727624ae40d7 # v3.29.5
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -557,7 +557,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -648,7 +648,7 @@ jobs:
|
||||
if: always()
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
cache-scope: stirling-pdf-fat
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -80,7 +80,7 @@ jobs:
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
@@ -160,3 +160,58 @@ jobs:
|
||||
build/reports/problems/
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
test-build-unoserver-image:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build docker/unoserver/Dockerfile
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
file: ./docker/unoserver/Dockerfile
|
||||
push: false
|
||||
load: true
|
||||
cache-from: type=gha,scope=stirling-unoserver
|
||||
cache-to: type=gha,mode=max,scope=stirling-unoserver
|
||||
platforms: linux/amd64
|
||||
tags: stirling-unoserver:pr-test
|
||||
provenance: false
|
||||
sbom: false
|
||||
|
||||
- name: Smoke test the built image
|
||||
run: |
|
||||
set -eu
|
||||
docker run -d --name unoserver-smoke \
|
||||
-e UNOSERVER_RECYCLE_INTERVAL_SECONDS=0 \
|
||||
stirling-unoserver:pr-test
|
||||
deadline=$((SECONDS + 60))
|
||||
while [ $SECONDS -lt $deadline ]; do
|
||||
status=$(docker inspect -f '{{.State.Health.Status}}' unoserver-smoke 2>/dev/null || echo "starting")
|
||||
if [ "$status" = "healthy" ]; then
|
||||
echo "unoserver became healthy"
|
||||
docker logs unoserver-smoke | tail -30
|
||||
docker rm -f unoserver-smoke
|
||||
exit 0
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
echo "unoserver did not become healthy in time"
|
||||
docker logs unoserver-smoke || true
|
||||
docker rm -f unoserver-smoke || true
|
||||
exit 1
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
frontend: ${{ steps.changes.outputs.frontend }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -186,7 +186,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ exampleYmlFiles/stirling/
|
||||
/testing/cucumber/junit/
|
||||
/testing/cucumber/report.html
|
||||
/testing/.failed_tests
|
||||
/.test-state/
|
||||
SwaggerDoc.json
|
||||
|
||||
# Runtime storage for uploaded files and user data (not Java source code)
|
||||
|
||||
+21
-48
@@ -1,5 +1,8 @@
|
||||
version: '3'
|
||||
|
||||
# All gradle invocations go through `bash gradlew` (the sh wrapper) so they work
|
||||
# uniformly on Linux/macOS (native bash) and Windows-with-Git-Bash.
|
||||
|
||||
tasks:
|
||||
dev:
|
||||
desc: "Start backend dev server"
|
||||
@@ -10,58 +13,43 @@ tasks:
|
||||
env:
|
||||
SERVER_PORT: '{{.PORT}}'
|
||||
cmds:
|
||||
- cmd: '{{if .AIENGINE_URL}}set "AIENGINE_URL={{.AIENGINE_URL}}" && set "AIENGINE_ENABLED=true" && {{end}}cmd /c gradlew.bat :stirling-pdf:bootRun'
|
||||
platforms: [windows]
|
||||
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true {{end}}./gradlew :stirling-pdf:bootRun'
|
||||
platforms: [linux, darwin]
|
||||
- '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true {{end}}bash gradlew :stirling-pdf:bootRun'
|
||||
|
||||
dev:bundled:
|
||||
desc: "Clean + bootRun with frontend bundled into the backend (single :8080 server)"
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash gradlew clean bootRun -PbuildWithFrontend=true
|
||||
|
||||
build:
|
||||
desc: "Full backend build"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat clean build
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew clean build
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew clean build
|
||||
|
||||
build:fast:
|
||||
desc: "Build without tests"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat clean build -x test
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew clean build -x test
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew clean build -x test
|
||||
|
||||
build:ci:
|
||||
desc: "Build for CI (formatting checked separately)"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat build -PnoSpotless
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew build -PnoSpotless
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew build -PnoSpotless
|
||||
|
||||
test:
|
||||
desc: "Run backend tests"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat test
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew test
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew test
|
||||
|
||||
format:
|
||||
desc: "Auto-fix code formatting"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat spotlessApply
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew spotlessApply
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew spotlessApply
|
||||
|
||||
format:check:
|
||||
desc: "Check code formatting"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat spotlessCheck
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew spotlessCheck
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew spotlessCheck
|
||||
|
||||
fix:
|
||||
desc: "Auto-fix backend"
|
||||
@@ -71,10 +59,7 @@ tasks:
|
||||
swagger:
|
||||
desc: "Generate OpenAPI docs"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat :stirling-pdf:copySwaggerDoc
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew :stirling-pdf:copySwaggerDoc
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew :stirling-pdf:copySwaggerDoc
|
||||
sources:
|
||||
- app/core/src/main/java/**/*.java
|
||||
- app/proprietary/src/main/java/**/*.java
|
||||
@@ -92,31 +77,19 @@ tasks:
|
||||
desc: "Print project version"
|
||||
silent: true
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat printVersion --quiet | tail -1
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew printVersion --quiet | tail -1
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew printVersion --quiet | tail -1
|
||||
|
||||
licenses:check:
|
||||
desc: "Check dependency licenses"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat checkLicense --no-parallel
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew checkLicense --no-parallel
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew checkLicense --no-parallel
|
||||
|
||||
licenses:generate:
|
||||
desc: "Check and generate dependency license report"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat checkLicense generateLicenseReport --no-parallel
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew checkLicense generateLicenseReport --no-parallel
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew checkLicense generateLicenseReport --no-parallel
|
||||
|
||||
clean:
|
||||
desc: "Clean build artifacts"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat clean
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew clean
|
||||
platforms: [linux, darwin]
|
||||
- bash gradlew clean
|
||||
|
||||
@@ -55,3 +55,9 @@ tasks:
|
||||
desc: "Tail docker compose logs"
|
||||
cmds:
|
||||
- docker compose -f {{.COMPOSE_DIR}}/docker-compose.yml logs -f
|
||||
|
||||
test:
|
||||
desc: "Run full Docker integration test suite (builds all variants and tests them)"
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash testing/test.sh {{.CLI_ARGS}}
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
version: '3'
|
||||
|
||||
tasks:
|
||||
install:
|
||||
desc: "Install Playwright browsers"
|
||||
dir: frontend
|
||||
deps: [ ':frontend:install' ]
|
||||
cmds:
|
||||
- npx playwright install {{.CLI_ARGS}} --with-deps
|
||||
|
||||
stubbed:
|
||||
desc: "Run stubbed E2E tests"
|
||||
dir: frontend
|
||||
deps: [ ':frontend:prepare' ]
|
||||
cmds:
|
||||
- npx playwright test --project=stubbed {{.CLI_ARGS}}
|
||||
|
||||
live:
|
||||
desc: "Run live E2E tests"
|
||||
summary: |
|
||||
Auto-spawns a Spring Boot backend with isolated state under
|
||||
.test-state/playwright/ (purged on every run) so the suite doesn't
|
||||
touch your local DB, settings.yml, backups or customFiles. The
|
||||
backend is stopped automatically when the runner exits.
|
||||
|
||||
Pass extra Playwright flags via -- :
|
||||
task e2e:live -- --headed --grep merge
|
||||
deps:
|
||||
- live:backend
|
||||
- live:runner
|
||||
|
||||
live:backend:
|
||||
internal: true
|
||||
ignore_error: true
|
||||
vars:
|
||||
BASE_DIR: '{{.ROOT_DIR}}/.test-state/playwright'
|
||||
env:
|
||||
STIRLING_BASE_PATH: '{{.BASE_DIR}}'
|
||||
# Suppress the analytics opt-in modal that fires on first admin login.
|
||||
# It renders a Mantine overlay that intercepts pointer events and blocks
|
||||
# FirstLoginSlide from rendering, so the bootstrap spec times out waiting
|
||||
# for the password-change prompt.
|
||||
SYSTEM_ENABLEANALYTICS: "false"
|
||||
# NOTE: SECURITY_INITIALLOGIN_USERNAME/PASSWORD are intentionally NOT set.
|
||||
# The live-setup project's bootstrap spec performs the real first-login
|
||||
# flow against the backend's default admin/stirling user, exercising the
|
||||
# forced-password-change UI and leaving the DB at admin/adminadmin for
|
||||
# the rest of the live suite. This is both real coverage of the first-
|
||||
# login flow and a stronger seed than env-var-driven user creation.
|
||||
cmds:
|
||||
# Wrapped in `bash -c` because Task's embedded shell (mvdan/sh) doesn't
|
||||
# support `&` + `$!` + `wait` reliably (gradle was never actually
|
||||
# backgrounded, so `$!` recorded a stale PID and the runner saw an
|
||||
# immediate "backend died"). Real bash backgrounds gradle properly and
|
||||
# gives us a stable PID to clean up.
|
||||
- |
|
||||
bash -c '
|
||||
set -e
|
||||
rm -rf "{{.BASE_DIR}}"
|
||||
mkdir -p "{{.BASE_DIR}}"
|
||||
# Background gradle and record its PID so the runner can clean up
|
||||
# the exact process tree (wrapper + forked Spring Boot JVM) without
|
||||
# resorting to fuzzy `pkill -f` patterns. `wait` keeps this script
|
||||
# alive for the lifetime of gradle so Task'"'"'s parallel deps stay
|
||||
# synchronised.
|
||||
bash gradlew :stirling-pdf:bootRun > "{{.BASE_DIR}}/backend.log" 2>&1 &
|
||||
GRADLE_PID=$!
|
||||
echo $GRADLE_PID > "{{.BASE_DIR}}/backend.pid"
|
||||
wait $GRADLE_PID
|
||||
'
|
||||
|
||||
live:runner:
|
||||
internal: true
|
||||
deps: [ ':frontend:prepare' ]
|
||||
dir: frontend
|
||||
vars:
|
||||
BASE_DIR: '{{.ROOT_DIR}}/.test-state/playwright'
|
||||
cmds:
|
||||
# Wrapped in `bash -c` because Task's embedded shell (mvdan/sh) only
|
||||
# supports `EXIT`/`ERR` in `trap`, not `INT`/`TERM`. CLI_ARGS are
|
||||
# passed positionally so quoted args like --grep "foo bar" survive.
|
||||
- |
|
||||
bash -c '
|
||||
set +e
|
||||
# bootRun forks a separate Spring Boot JVM as a child of the gradle
|
||||
# wrapper; killing only the wrapper leaves that JVM orphaned holding
|
||||
# :8080. Walk the recorded PID'"'"'s process tree via pgrep so we kill
|
||||
# every descendant, then lsof as a last-resort backstop.
|
||||
kill_tree() {
|
||||
local pid=$1
|
||||
for child in $(pgrep -P "$pid" 2>/dev/null); do
|
||||
kill_tree "$child"
|
||||
done
|
||||
kill "$pid" 2>/dev/null || true
|
||||
}
|
||||
cleanup() {
|
||||
if [ -f "{{.BASE_DIR}}/backend.pid" ]; then
|
||||
echo "Stopping backend..."
|
||||
kill_tree "$(cat "{{.BASE_DIR}}/backend.pid")"
|
||||
sleep 1
|
||||
lsof -ti:8080 2>/dev/null | xargs kill 2>/dev/null || true
|
||||
echo "Backend stopped"
|
||||
fi
|
||||
}
|
||||
# Trap so cleanup runs on every exit path: tests pass, tests fail,
|
||||
# backend never starts, user Ctrl-Cs. Without this the gradle JVM
|
||||
# is left orphaned holding :8080 whenever the wait loop times out.
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
echo "Waiting for backend on :8080 (up to 10min; first compile can be slow)..."
|
||||
WAITED=0
|
||||
while [ $WAITED -lt 600 ]; do
|
||||
if curl -sf http://localhost:8080/api/v1/info/status 2>/dev/null | grep -q UP; then
|
||||
echo "Backend ready, starting Playwright"
|
||||
break
|
||||
fi
|
||||
# Bail early if the backend already died: no point waiting 10min.
|
||||
if [ -f "{{.BASE_DIR}}/backend.pid" ]; then
|
||||
BACKEND_PID=$(cat "{{.BASE_DIR}}/backend.pid")
|
||||
if ! kill -0 "$BACKEND_PID" 2>/dev/null; then
|
||||
echo "Backend process exited before becoming ready; last 100 log lines:"
|
||||
tail -100 "{{.BASE_DIR}}/backend.log" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
sleep 2
|
||||
WAITED=$((WAITED + 2))
|
||||
done
|
||||
if [ $WAITED -ge 600 ]; then
|
||||
echo "Backend did not become ready within 10min, aborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
npx playwright test --project=live "$@"
|
||||
' bash {{.CLI_ARGS}}
|
||||
|
||||
enterprise:
|
||||
desc: "Run enterprise E2E tests"
|
||||
summary: |
|
||||
Requires an already-running keycloak compose stack on :8080. Bring one
|
||||
up first with:
|
||||
task e2e:oauth:up
|
||||
task e2e:saml:up
|
||||
dir: frontend
|
||||
deps: [ ':frontend:prepare' ]
|
||||
cmds:
|
||||
- npx playwright test --project=enterprise {{.CLI_ARGS}}
|
||||
|
||||
cross-browser:
|
||||
desc: "Run stubbed E2E tests on Chromium, Firefox, and WebKit"
|
||||
dir: frontend
|
||||
deps: [ ':frontend:prepare' ]
|
||||
cmds:
|
||||
- npx playwright test --project=stubbed --project=stubbed-firefox --project=stubbed-webkit {{.CLI_ARGS}}
|
||||
|
||||
check:
|
||||
desc: "Run all E2E tests not requiring Docker"
|
||||
cmds:
|
||||
- task: stubbed
|
||||
- task: live
|
||||
|
||||
check:all:
|
||||
desc: "Run all E2E tests"
|
||||
summary: |
|
||||
Includes the enterprise project, which requires a keycloak compose
|
||||
stack on :8080. Bring one up first with:
|
||||
task e2e:oauth:up
|
||||
task e2e:saml:up
|
||||
cmds:
|
||||
- task: stubbed
|
||||
- task: live
|
||||
- task: enterprise
|
||||
|
||||
oauth:up:
|
||||
desc: "Start the OAuth keycloak test environment"
|
||||
summary: |
|
||||
Set LICENSE_KEY=<KEY> to skip the interactive license prompt:
|
||||
task e2e:oauth:up LICENSE_KEY=abc123
|
||||
Pass extra flags via -- :
|
||||
task e2e:oauth:up -- --auto --nobuild
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash testing/compose/start-oauth-test.sh {{if .LICENSE_KEY}}--license-key "{{.LICENSE_KEY}}"{{end}} {{.CLI_ARGS}}
|
||||
|
||||
oauth:down:
|
||||
desc: "Stop the OAuth keycloak test environment"
|
||||
cmds:
|
||||
- docker compose -f testing/compose/docker-compose-keycloak-oauth.yml down -v
|
||||
|
||||
saml:up:
|
||||
desc: "Start the SAML keycloak test environment"
|
||||
summary: |
|
||||
Set LICENSE_KEY=<KEY> to skip the interactive license prompt:
|
||||
task e2e:saml:up LICENSE_KEY=abc123
|
||||
Pass extra flags via -- :
|
||||
task e2e:saml:up -- --auto --with-storage --nobuild --language sv-SE
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash testing/compose/start-saml-test.sh {{if .LICENSE_KEY}}--license-key "{{.LICENSE_KEY}}"{{end}} {{.CLI_ARGS}}
|
||||
|
||||
saml:down:
|
||||
desc: "Stop the SAML keycloak test environment"
|
||||
cmds:
|
||||
- docker compose -f testing/compose/docker-compose-keycloak-saml.yml down -v
|
||||
@@ -86,8 +86,9 @@ tasks:
|
||||
fix:
|
||||
desc: "Auto-fix lint + format"
|
||||
cmds:
|
||||
- task: format # Can auto-fix some things that `lint:fix` can't like line length violations
|
||||
- task: lint:fix
|
||||
- task: format
|
||||
- task: format # Ensure that after lint fixing that the code is still formatted correctly
|
||||
|
||||
check:
|
||||
desc: "Full engine quality gate"
|
||||
|
||||
@@ -107,13 +107,6 @@ tasks:
|
||||
- task: dev:_run
|
||||
vars: { MODE: prototypes, PORT: '{{.PORT}}', BACKEND_URL: '{{.BACKEND_URL}}', OPEN: '{{.OPEN}}' }
|
||||
|
||||
dev:portal:
|
||||
desc: "Start developer portal dev server"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- task: dev:_run
|
||||
vars: { MODE: portal, PORT: '{{.PORT}}', BACKEND_URL: '{{.BACKEND_URL}}', OPEN: '{{.OPEN}}' }
|
||||
|
||||
# ============================================================
|
||||
# Build
|
||||
# ============================================================
|
||||
@@ -158,24 +151,6 @@ tasks:
|
||||
cmds:
|
||||
- npx vite build --mode prototypes
|
||||
|
||||
build:portal:
|
||||
desc: "Build developer portal"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx vite build --mode portal
|
||||
|
||||
storybook:
|
||||
desc: "Start Storybook dev server"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx storybook dev -p 6006 {{.CLI_ARGS}}
|
||||
|
||||
storybook:build:
|
||||
desc: "Build static Storybook"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx storybook build {{.CLI_ARGS}}
|
||||
|
||||
# ============================================================
|
||||
# Code quality
|
||||
# ============================================================
|
||||
@@ -256,12 +231,6 @@ tasks:
|
||||
cmds:
|
||||
- npx tsc --noEmit --project src/prototypes/tsconfig.json
|
||||
|
||||
typecheck:portal:
|
||||
desc: "Typecheck developer portal build variant"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx tsc --noEmit --project src/portal/tsconfig.json
|
||||
|
||||
typecheck:all:
|
||||
desc: "Typecheck all build variants"
|
||||
cmds:
|
||||
@@ -271,7 +240,6 @@ tasks:
|
||||
- task: typecheck:desktop
|
||||
- task: typecheck:scripts
|
||||
- task: typecheck:prototypes
|
||||
- task: typecheck:portal
|
||||
|
||||
# ============================================================
|
||||
# Quality Gate
|
||||
@@ -293,7 +261,6 @@ tasks:
|
||||
- task: format:check
|
||||
- task: build
|
||||
- task: test
|
||||
- task: storybook:build
|
||||
|
||||
# ============================================================
|
||||
# Test
|
||||
@@ -317,18 +284,6 @@ tasks:
|
||||
cmds:
|
||||
- npx vitest --coverage
|
||||
|
||||
test:e2e:
|
||||
desc: "Run E2E tests"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx playwright test {{.CLI_ARGS}}
|
||||
|
||||
test:e2e:install:
|
||||
desc: "Install E2E test browsers"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx playwright install {{.CLI_ARGS}} --with-deps
|
||||
|
||||
# ============================================================
|
||||
# Code Generation
|
||||
# ============================================================
|
||||
|
||||
@@ -22,6 +22,9 @@ includes:
|
||||
desktop:
|
||||
taskfile: .taskfiles/desktop.yml
|
||||
dir: frontend
|
||||
e2e:
|
||||
taskfile: .taskfiles/e2e.yml
|
||||
dir: .
|
||||
|
||||
tasks:
|
||||
# ============================================================
|
||||
|
||||
@@ -51,4 +51,12 @@ dependencies {
|
||||
api 'org.simplejavamail:outlook-module:8.12.6' // MSG file support
|
||||
api 'jakarta.mail:jakarta.mail-api:2.1.5'
|
||||
runtimeOnly 'org.eclipse.angus:angus-mail:2.0.5'
|
||||
|
||||
// Tabula table extraction — used by the shared PDF parser and directly by downstream modules.
|
||||
// api-scoped so downstream modules (core, proprietary) retain it on their compile classpath.
|
||||
api ('technology.tabula:tabula:1.0.5') {
|
||||
exclude group: 'org.slf4j', module: 'slf4j-simple'
|
||||
exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
|
||||
exclude group: 'com.google.code.gson', module: 'gson'
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Chains table parsers in priority order: Tabula lattice → Tabula stream → {@link
|
||||
* LineAlignmentTableParser}. The first parser returning a result above {@link
|
||||
* #TABULA_CONFIDENCE_THRESHOLD} wins; results from different parsers are never mixed on one page.
|
||||
*/
|
||||
@Service
|
||||
@Primary
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class CompositeTableParser implements TableParser {
|
||||
|
||||
/** Min Tabula confidence to accept results; below this LineAlignment is tried instead. */
|
||||
static final float TABULA_CONFIDENCE_THRESHOLD = 0.5f;
|
||||
|
||||
private final TabulaTableParser tabulaParser;
|
||||
private final LineAlignmentTableParser lineAlignmentParser;
|
||||
|
||||
@Override
|
||||
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
|
||||
// Step 1: Tabula lattice mode (ruled/bordered tables).
|
||||
List<TableFragment> latticeResults = filterConfident(tabulaParser.parse(document, rawPage));
|
||||
if (!latticeResults.isEmpty()) {
|
||||
log.debug(
|
||||
"Page {}: using Tabula lattice ({} table(s))",
|
||||
rawPage.pageNumber(),
|
||||
latticeResults.size());
|
||||
return latticeResults;
|
||||
}
|
||||
|
||||
// Step 2: Tabula stream mode (borderless/whitespace-delimited tables).
|
||||
// parseStream is not on the TableParser interface — this intentionally couples to the
|
||||
// concrete TabulaTableParser since stream mode is a Tabula-specific concept.
|
||||
List<TableFragment> streamResults =
|
||||
filterConfident(tabulaParser.parseStream(document, rawPage));
|
||||
if (!streamResults.isEmpty()) {
|
||||
log.debug(
|
||||
"Page {}: using Tabula stream ({} table(s))",
|
||||
rawPage.pageNumber(),
|
||||
streamResults.size());
|
||||
return streamResults;
|
||||
}
|
||||
|
||||
// Step 3: Geometry-based line-alignment fallback.
|
||||
List<TableFragment> lineResults = lineAlignmentParser.parse(document, rawPage);
|
||||
if (!lineResults.isEmpty()) {
|
||||
log.debug(
|
||||
"Page {}: using LineAlignment ({} table(s))",
|
||||
rawPage.pageNumber(),
|
||||
lineResults.size());
|
||||
return lineResults;
|
||||
}
|
||||
|
||||
return List.of();
|
||||
}
|
||||
|
||||
private List<TableFragment> filterConfident(List<TableFragment> tables) {
|
||||
return tables.stream().filter(t -> t.confidence() >= TABULA_CONFIDENCE_THRESHOLD).toList();
|
||||
}
|
||||
}
|
||||
+528
@@ -0,0 +1,528 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.TreeMap;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Fallback {@link TableParser} for borderless financial tables using text geometry.
|
||||
*
|
||||
* <p>Identifies "anchor lines" (≥2 numeric tokens), builds a column grid from their right-edge
|
||||
* positions, groups vertically proximate anchor lines into table candidates, then scores each group
|
||||
* on column consistency and anchor density (confidence ceiling 0.85).
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class LineAlignmentTableParser implements TableParser {
|
||||
|
||||
/** Width in points of each column position bucket. */
|
||||
static final float COLUMN_BUCKET_PT = 5f;
|
||||
|
||||
/** Tolerance in buckets when matching a token's right-edge to a confirmed column position. */
|
||||
private static final int COLUMN_MATCH_BUCKETS = 2;
|
||||
|
||||
/** Maximum gap (as a multiple of modal line spacing) before splitting a group. */
|
||||
private static final float MAX_GAP_FACTOR = 2.5f;
|
||||
|
||||
/** Minimum anchor rows (numeric-heavy) to form a valid table. */
|
||||
static final int MIN_TABLE_ROWS = 3;
|
||||
|
||||
/** Minimum confirmed column positions to form a valid table. */
|
||||
static final int MIN_COLUMNS = 2;
|
||||
|
||||
/**
|
||||
* Min fraction of anchor lines a column must appear on to be confirmed (permissive for N/A
|
||||
* rows).
|
||||
*/
|
||||
private static final double COLUMN_MIN_FREQUENCY = 0.40;
|
||||
|
||||
/**
|
||||
* Matches financial numeric tokens: integers, decimals, parenthetical negatives, currency,
|
||||
* percent, nil dashes.
|
||||
*/
|
||||
private static final Pattern NUMERIC =
|
||||
Pattern.compile("^[\\(\\-\\$£€¥]?\\d[\\d,\\.]*[\\)%]?$|^[-–—]$");
|
||||
|
||||
/**
|
||||
* Lines within this y-distance are merged into one row (restores rows split by LineBuilder's
|
||||
* column-gap logic).
|
||||
*/
|
||||
static final float ROW_MERGE_TOLERANCE_PT = 2f;
|
||||
|
||||
// ── public API ───────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Override
|
||||
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
|
||||
List<RawLine> lines = rawPage.lines();
|
||||
if (lines.size() < MIN_TABLE_ROWS) return List.of();
|
||||
|
||||
float modalSpacing = computeModalSpacing(lines);
|
||||
List<TokenizedLine> tokenized =
|
||||
mergeCoincidentLines(lines.stream().map(this::tokenize).toList());
|
||||
|
||||
List<TokenizedLine> anchors = tokenized.stream().filter(TokenizedLine::isAnchor).toList();
|
||||
|
||||
if (anchors.size() < MIN_TABLE_ROWS) return List.of();
|
||||
|
||||
List<Float> columnGrid = buildColumnGrid(anchors);
|
||||
if (columnGrid.size() < MIN_COLUMNS) {
|
||||
log.debug(
|
||||
"Page {}: LineAlignment — fewer than {} confirmed columns, skipping",
|
||||
rawPage.pageNumber(),
|
||||
MIN_COLUMNS);
|
||||
return List.of();
|
||||
}
|
||||
|
||||
List<List<TokenizedLine>> groups = groupRows(tokenized, columnGrid, modalSpacing);
|
||||
|
||||
List<TableFragment> results = new ArrayList<>();
|
||||
for (int i = 0; i < groups.size(); i++) {
|
||||
buildFragment(groups.get(i), columnGrid, rawPage.pageNumber(), i)
|
||||
.ifPresent(results::add);
|
||||
}
|
||||
|
||||
log.debug(
|
||||
"Page {}: LineAlignment detected {} table(s) ({} anchor lines, {} columns)",
|
||||
rawPage.pageNumber(),
|
||||
results.size(),
|
||||
anchors.size(),
|
||||
columnGrid.size());
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── coincident-line merging ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Merges tokenised lines sharing the same y-position into one row, rejoining label/value halves
|
||||
* split by LineBuilder.
|
||||
*/
|
||||
List<TokenizedLine> mergeCoincidentLines(List<TokenizedLine> tokenized) {
|
||||
if (tokenized.size() < 2) return tokenized;
|
||||
|
||||
List<TokenizedLine> result = new ArrayList<>();
|
||||
int i = 0;
|
||||
|
||||
while (i < tokenized.size()) {
|
||||
float baseY = tokenized.get(i).line().bounds().y();
|
||||
int j = i + 1;
|
||||
while (j < tokenized.size()
|
||||
&& Math.abs(tokenized.get(j).line().bounds().y() - baseY)
|
||||
<= ROW_MERGE_TOLERANCE_PT) {
|
||||
j++;
|
||||
}
|
||||
|
||||
if (j == i + 1) {
|
||||
result.add(tokenized.get(i));
|
||||
} else {
|
||||
result.add(mergeGroup(tokenized.subList(i, j)));
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private TokenizedLine mergeGroup(List<TokenizedLine> group) {
|
||||
List<TextFragment> mergedFragments =
|
||||
group.stream()
|
||||
.flatMap(tl -> tl.line().fragments().stream())
|
||||
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
|
||||
.toList();
|
||||
|
||||
Bounds mergedBounds =
|
||||
group.stream()
|
||||
.map(tl -> tl.line().bounds())
|
||||
.reduce(Bounds::merge)
|
||||
.orElse(group.get(0).line().bounds());
|
||||
|
||||
RawLine mergedLine =
|
||||
new RawLine(
|
||||
group.get(0).line().lineId(),
|
||||
mergedFragments,
|
||||
mergedBounds,
|
||||
group.get(0).line().pageNumber());
|
||||
|
||||
return tokenize(mergedLine);
|
||||
}
|
||||
|
||||
// ── tokenisation ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Splits fragments into word-level tokens; x-positions are estimated linearly within each
|
||||
* fragment.
|
||||
*/
|
||||
TokenizedLine tokenize(RawLine line) {
|
||||
List<LineToken> tokens = new ArrayList<>();
|
||||
for (TextFragment frag : line.fragments()) {
|
||||
tokens.addAll(tokensFromFragment(frag));
|
||||
}
|
||||
List<LineToken> numeric = tokens.stream().filter(LineToken::numeric).toList();
|
||||
return new TokenizedLine(line, tokens, numeric);
|
||||
}
|
||||
|
||||
private List<LineToken> tokensFromFragment(TextFragment frag) {
|
||||
String raw = frag.text();
|
||||
if (raw == null || raw.isBlank()) return List.of();
|
||||
|
||||
float fragX = frag.bounds().x();
|
||||
float fragWidth = frag.bounds().width();
|
||||
int rawLen = raw.length();
|
||||
|
||||
List<LineToken> result = new ArrayList<>();
|
||||
int offset = 0;
|
||||
for (String part : raw.split("\\s+")) {
|
||||
if (part.isEmpty()) {
|
||||
offset++;
|
||||
continue;
|
||||
}
|
||||
int idx = raw.indexOf(part, offset);
|
||||
if (idx < 0) idx = offset;
|
||||
|
||||
float tokenX = rawLen > 0 ? fragX + ((float) idx / rawLen) * fragWidth : fragX;
|
||||
float tokenRight =
|
||||
rawLen > 0
|
||||
? fragX + ((float) (idx + part.length()) / rawLen) * fragWidth
|
||||
: fragX + fragWidth;
|
||||
|
||||
result.add(new LineToken(part, tokenX, tokenRight, NUMERIC.matcher(part).matches()));
|
||||
offset = idx + part.length();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── column grid ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Returns confirmed column right-edge positions — those appearing on ≥ {@value
|
||||
* #COLUMN_MIN_FREQUENCY} × N anchor lines.
|
||||
*/
|
||||
private List<Float> buildColumnGrid(List<TokenizedLine> anchors) {
|
||||
// bucket → set of line indices that contributed a numeric token to that bucket
|
||||
Map<Integer, List<Integer>> bucketLines = new HashMap<>();
|
||||
for (int i = 0; i < anchors.size(); i++) {
|
||||
for (LineToken t : anchors.get(i).numeric()) {
|
||||
int bucket = bucket(t.right());
|
||||
bucketLines.computeIfAbsent(bucket, k -> new ArrayList<>()).add(i);
|
||||
}
|
||||
}
|
||||
|
||||
int minHits =
|
||||
Math.max(MIN_TABLE_ROWS, (int) Math.ceil(anchors.size() * COLUMN_MIN_FREQUENCY));
|
||||
|
||||
// Confirmed buckets → average right-edge for that bucket
|
||||
TreeMap<Integer, Float> confirmed = new TreeMap<>();
|
||||
for (Map.Entry<Integer, List<Integer>> entry : bucketLines.entrySet()) {
|
||||
// Count distinct lines
|
||||
long distinctLines = entry.getValue().stream().distinct().count();
|
||||
if (distinctLines >= minHits) {
|
||||
double avg =
|
||||
entry.getValue().stream()
|
||||
.distinct() // weight each line equally regardless of token count
|
||||
.mapToDouble(
|
||||
lineIdx ->
|
||||
avgRightEdgeForBucket(
|
||||
anchors, lineIdx, entry.getKey()))
|
||||
.average()
|
||||
.orElse(entry.getKey() * (double) COLUMN_BUCKET_PT);
|
||||
confirmed.put(entry.getKey(), (float) avg);
|
||||
}
|
||||
}
|
||||
|
||||
return new ArrayList<>(confirmed.values()); // already sorted by bucket (left to right)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the average right-edge position of tokens in {@code line} whose bucket matches {@code
|
||||
* targetBucket}, falling back to the bucket's nominal centre when no tokens match.
|
||||
*/
|
||||
private double avgRightEdgeForBucket(
|
||||
List<TokenizedLine> anchors, int lineIdx, int targetBucket) {
|
||||
return anchors.get(lineIdx).numeric().stream()
|
||||
.filter(t -> bucket(t.right()) == targetBucket)
|
||||
.mapToDouble(LineToken::right)
|
||||
.average()
|
||||
.orElse(targetBucket * (double) COLUMN_BUCKET_PT);
|
||||
}
|
||||
|
||||
// ── grouping ─────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Groups anchor lines into table candidates, including adjacent label rows; a gap >
|
||||
* MAX_GAP_FACTOR × modal spacing splits groups.
|
||||
*/
|
||||
private List<List<TokenizedLine>> groupRows(
|
||||
List<TokenizedLine> all, List<Float> columnGrid, float modalSpacing) {
|
||||
float maxGap = modalSpacing > 0 ? modalSpacing * MAX_GAP_FACTOR : 30f;
|
||||
|
||||
List<List<TokenizedLine>> groups = new ArrayList<>();
|
||||
List<TokenizedLine> current = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < all.size(); i++) {
|
||||
TokenizedLine tl = all.get(i);
|
||||
boolean fits = tl.isAnchor() && matchesGrid(tl, columnGrid);
|
||||
|
||||
if (current.isEmpty()) {
|
||||
if (fits) current.add(tl);
|
||||
continue;
|
||||
}
|
||||
|
||||
float gap =
|
||||
tl.line().bounds().y()
|
||||
- current.get(current.size() - 1).line().bounds().bottom();
|
||||
|
||||
if (gap > maxGap) {
|
||||
groups.add(current);
|
||||
current = new ArrayList<>();
|
||||
if (fits) current.add(tl);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (fits) {
|
||||
current.add(tl);
|
||||
} else if (!tl.line().text().isBlank()) {
|
||||
// Include non-anchor lines (labels) only if they have text and are within
|
||||
// proximity.
|
||||
current.add(tl);
|
||||
}
|
||||
}
|
||||
|
||||
if (!current.isEmpty()) groups.add(current);
|
||||
|
||||
return groups.stream().filter(g -> hasEnoughAnchorRows(g, columnGrid)).toList();
|
||||
}
|
||||
|
||||
private boolean hasEnoughAnchorRows(List<TokenizedLine> group, List<Float> columnGrid) {
|
||||
return group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count()
|
||||
>= MIN_TABLE_ROWS;
|
||||
}
|
||||
|
||||
/** A line "matches" the grid when ≥ 60 % of its numeric tokens land in confirmed columns. */
|
||||
private boolean matchesGrid(TokenizedLine tl, List<Float> columnGrid) {
|
||||
if (tl.numeric().isEmpty()) return false;
|
||||
long matches =
|
||||
tl.numeric().stream()
|
||||
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
|
||||
.count();
|
||||
return (double) matches / tl.numeric().size() >= 0.60;
|
||||
}
|
||||
|
||||
private boolean hasInconsistentColumnMatch(TokenizedLine tl, List<Float> columnGrid) {
|
||||
if (tl.numeric().isEmpty()) return false;
|
||||
long hits =
|
||||
tl.numeric().stream()
|
||||
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
|
||||
.count();
|
||||
return (double) hits / tl.numeric().size() < 0.60;
|
||||
}
|
||||
|
||||
// ── fragment assembly ────────────────────────────────────────────────────────────────────────
|
||||
|
||||
private Optional<TableFragment> buildFragment(
|
||||
List<TokenizedLine> group, List<Float> columnGrid, int pageNumber, int tableIndex) {
|
||||
|
||||
long anchorCount =
|
||||
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
|
||||
if (anchorCount < MIN_TABLE_ROWS) return Optional.empty();
|
||||
|
||||
List<String> warnings = new ArrayList<>();
|
||||
List<List<String>> rawRows = new ArrayList<>();
|
||||
List<TableRow> rows = new ArrayList<>();
|
||||
|
||||
for (int rowIdx = 0; rowIdx < group.size(); rowIdx++) {
|
||||
TokenizedLine tl = group.get(rowIdx);
|
||||
List<String> rawRow = buildRawRow(tl, columnGrid);
|
||||
rawRows.add(Collections.unmodifiableList(rawRow));
|
||||
rows.add(buildTableRow(rowIdx, tl, rawRow, columnGrid));
|
||||
}
|
||||
|
||||
// Column count = 1 label column + confirmed numeric columns
|
||||
int colCount = columnGrid.size() + 1;
|
||||
Bounds bounds = computeGroupBounds(group);
|
||||
float confidence = computeConfidence(group, columnGrid, warnings);
|
||||
|
||||
return Optional.of(
|
||||
new TableFragment(
|
||||
"tbl-la-p" + pageNumber + "-" + tableIndex,
|
||||
pageNumber,
|
||||
bounds,
|
||||
List.of(),
|
||||
Collections.unmodifiableList(rows),
|
||||
Collections.unmodifiableList(rawRows),
|
||||
colCount,
|
||||
confidence,
|
||||
Collections.unmodifiableList(warnings),
|
||||
null));
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a raw row as a list of strings: index 0 = label text, indices 1..N = column values.
|
||||
*/
|
||||
private List<String> buildRawRow(TokenizedLine tl, List<Float> columnGrid) {
|
||||
String[] cells = new String[columnGrid.size() + 1];
|
||||
Arrays.fill(cells, "");
|
||||
|
||||
// Separate label tokens (those not landing in any confirmed column) from column tokens.
|
||||
List<String> labelParts = new ArrayList<>();
|
||||
for (LineToken token : tl.all()) {
|
||||
int col = nearestColumnIndex(token.right(), columnGrid);
|
||||
if (col >= 0 && token.numeric()) {
|
||||
int cellIdx = col + 1;
|
||||
cells[cellIdx] =
|
||||
cells[cellIdx].isEmpty()
|
||||
? token.text()
|
||||
: cells[cellIdx] + " " + token.text();
|
||||
} else {
|
||||
labelParts.add(token.text());
|
||||
}
|
||||
}
|
||||
cells[0] = String.join(" ", labelParts).trim();
|
||||
return Arrays.asList(cells);
|
||||
}
|
||||
|
||||
private TableRow buildTableRow(
|
||||
int rowIdx, TokenizedLine tl, List<String> rawRow, List<Float> columnGrid) {
|
||||
List<TableCell> cells = new ArrayList<>(rawRow.size());
|
||||
|
||||
// Label cell: use the line's full bounds as an approximation.
|
||||
cells.add(TableCell.of(0, rawRow.get(0), tl.line().bounds()));
|
||||
|
||||
for (int col = 0; col < columnGrid.size(); col++) {
|
||||
String text = col + 1 < rawRow.size() ? rawRow.get(col + 1) : "";
|
||||
float right = columnGrid.get(col);
|
||||
float left = col > 0 ? columnGrid.get(col - 1) : right - 50f;
|
||||
Bounds cellBounds =
|
||||
new Bounds(
|
||||
left,
|
||||
tl.line().bounds().y(),
|
||||
right - left,
|
||||
tl.line().bounds().height());
|
||||
cells.add(TableCell.of(col + 1, text, cellBounds));
|
||||
}
|
||||
return new TableRow(rowIdx, Collections.unmodifiableList(cells));
|
||||
}
|
||||
|
||||
// ── confidence scoring ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Heuristic score in [0.0, 0.85] (ceiling keeps results below Tabula lattice which starts at
|
||||
* 1.0). Base 0.70; +0.05/col beyond 2 (max +0.10); +0.05 at ≥5 anchors, +0.05 at ≥8; −0.15 if
|
||||
* >30 % of anchors have inconsistent columns; −0.10 if non-anchors outnumber anchors.
|
||||
*/
|
||||
private float computeConfidence(
|
||||
List<TokenizedLine> group, List<Float> columnGrid, List<String> warnings) {
|
||||
float score = 0.70f;
|
||||
|
||||
long anchorCount =
|
||||
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
|
||||
long totalRows = group.size();
|
||||
|
||||
// More columns
|
||||
int extraCols = Math.min(columnGrid.size() - MIN_COLUMNS, 2);
|
||||
score += extraCols * 0.05f;
|
||||
|
||||
// More anchor rows
|
||||
if (anchorCount >= 5) score += 0.05f;
|
||||
if (anchorCount >= 8) score += 0.05f;
|
||||
|
||||
// Inconsistent column matching
|
||||
long inconsistent =
|
||||
group.stream()
|
||||
.filter(TokenizedLine::isAnchor)
|
||||
.filter(tl -> hasInconsistentColumnMatch(tl, columnGrid))
|
||||
.count();
|
||||
if (inconsistent > anchorCount * 0.30) {
|
||||
score -= 0.15f;
|
||||
warnings.add(
|
||||
"Column match inconsistent on "
|
||||
+ inconsistent
|
||||
+ "/"
|
||||
+ anchorCount
|
||||
+ " anchor rows");
|
||||
}
|
||||
|
||||
// Label-heavy
|
||||
long nonAnchor = totalRows - anchorCount;
|
||||
if (nonAnchor > anchorCount) {
|
||||
score -= 0.10f;
|
||||
warnings.add(
|
||||
"Non-anchor rows ("
|
||||
+ nonAnchor
|
||||
+ ") outnumber anchor rows ("
|
||||
+ anchorCount
|
||||
+ ")");
|
||||
}
|
||||
|
||||
return Math.max(0f, Math.min(0.85f, score));
|
||||
}
|
||||
|
||||
// ── utility ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Returns the grid index nearest to {@code rightEdge}, or -1 if none is within {@value
|
||||
* #COLUMN_MATCH_BUCKETS} buckets.
|
||||
*/
|
||||
private int nearestColumnIndex(float rightEdge, List<Float> grid) {
|
||||
int nearest = -1;
|
||||
float minDist = COLUMN_MATCH_BUCKETS * COLUMN_BUCKET_PT + 1f;
|
||||
for (int i = 0; i < grid.size(); i++) {
|
||||
float dist = Math.abs(rightEdge - grid.get(i));
|
||||
if (dist < minDist) {
|
||||
minDist = dist;
|
||||
nearest = i;
|
||||
}
|
||||
}
|
||||
return nearest;
|
||||
}
|
||||
|
||||
private Bounds computeGroupBounds(List<TokenizedLine> group) {
|
||||
return group.stream()
|
||||
.map(tl -> tl.line().bounds())
|
||||
.reduce(Bounds::merge)
|
||||
.orElse(new Bounds(0, 0, 0, 0));
|
||||
}
|
||||
|
||||
/** Modal gap between consecutive line edges, used to calibrate the group-split threshold. */
|
||||
private float computeModalSpacing(List<RawLine> lines) {
|
||||
if (lines.size() < 2) return 0f;
|
||||
Map<Float, Long> freq = new HashMap<>();
|
||||
for (int i = 1; i < lines.size(); i++) {
|
||||
float gap = lines.get(i).bounds().y() - lines.get(i - 1).bounds().bottom();
|
||||
if (gap > 0) freq.merge(Math.round(gap / 2f) * 2f, 1L, Long::sum);
|
||||
}
|
||||
return freq.entrySet().stream()
|
||||
.max(Map.Entry.comparingByValue())
|
||||
.map(Map.Entry::getKey)
|
||||
.orElse(0f);
|
||||
}
|
||||
|
||||
private static int bucket(float x) {
|
||||
return Math.round(x / COLUMN_BUCKET_PT);
|
||||
}
|
||||
|
||||
// ── private data types ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** A word-level token with an approximate right-edge x-position. */
|
||||
record LineToken(String text, float x, float right, boolean numeric) {}
|
||||
|
||||
/** A {@link RawLine} with tokens pre-computed; an "anchor" has ≥ 2 numeric tokens. */
|
||||
record TokenizedLine(RawLine line, List<LineToken> all, List<LineToken> numeric) {
|
||||
boolean isAnchor() {
|
||||
return numeric.size() >= 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Groups {@link TextFragment} objects into visual {@link RawLine}s using baseline proximity.
|
||||
*
|
||||
* <p>Fragments are on the same line when their baselines are within a font-size-derived tolerance.
|
||||
* A new line starts whenever the horizontal gap exceeds an adaptive column-gap threshold ({@code
|
||||
* max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT)}), splitting two-column text.
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class LineBuilder {
|
||||
|
||||
/** Baseline tolerance as a fraction of font size; 0.5 keeps mixed-size text on one line. */
|
||||
private static final float BASELINE_TOLERANCE_FACTOR = 0.5f;
|
||||
|
||||
/** Absolute minimum tolerance so tiny font sizes don't collapse multi-line content. */
|
||||
private static final float MIN_BASELINE_TOLERANCE = 2f;
|
||||
|
||||
/**
|
||||
* Column-gap threshold as a fraction of page width; 0.10 clears tab stops but stays below
|
||||
* two-column gutters.
|
||||
*/
|
||||
static final float COLUMN_GAP_RATIO = 0.10f;
|
||||
|
||||
/** Floor for the column-gap threshold so narrow pages don't over-split lines. */
|
||||
static final float COLUMN_GAP_MIN_PT = 40f;
|
||||
|
||||
public List<RawLine> build(List<TextFragment> fragments, int pageNumber) {
|
||||
if (fragments.isEmpty()) return List.of();
|
||||
|
||||
float effectiveWidth = inferEffectiveWidth(fragments);
|
||||
float columnGapThreshold = Math.max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT);
|
||||
log.debug(
|
||||
"LineBuilder page {}: effectiveWidth={:.1f}pt, columnGapThreshold={:.1f}pt",
|
||||
pageNumber,
|
||||
effectiveWidth,
|
||||
columnGapThreshold);
|
||||
|
||||
// Sort top-to-bottom first, then left-to-right within the same baseline band.
|
||||
List<TextFragment> sorted =
|
||||
fragments.stream()
|
||||
.sorted(
|
||||
Comparator.comparingDouble(TextFragment::baseline)
|
||||
.thenComparingDouble(f -> f.bounds().x()))
|
||||
.toList();
|
||||
|
||||
List<List<TextFragment>> groups = groupByBaseline(sorted, columnGapThreshold);
|
||||
|
||||
List<RawLine> lines = new ArrayList<>(groups.size());
|
||||
for (int i = 0; i < groups.size(); i++) {
|
||||
List<TextFragment> group =
|
||||
groups.get(i).stream()
|
||||
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
|
||||
.toList();
|
||||
|
||||
Bounds lineBounds =
|
||||
group.stream()
|
||||
.map(TextFragment::bounds)
|
||||
.reduce(Bounds::merge)
|
||||
.orElse(new Bounds(0, 0, 0, 0));
|
||||
|
||||
lines.add(new RawLine("ln-p" + pageNumber + "-" + i, group, lineBounds, pageNumber));
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
private List<List<TextFragment>> groupByBaseline(
|
||||
List<TextFragment> sorted, float columnGapThreshold) {
|
||||
List<List<TextFragment>> groups = new ArrayList<>();
|
||||
List<TextFragment> current = new ArrayList<>();
|
||||
float currentBaseline = Float.NaN;
|
||||
|
||||
for (TextFragment fragment : sorted) {
|
||||
if (current.isEmpty()) {
|
||||
current.add(fragment);
|
||||
currentBaseline = fragment.baseline();
|
||||
continue;
|
||||
}
|
||||
|
||||
float maxFontSize =
|
||||
Math.max(
|
||||
fragment.fontSize(),
|
||||
(float)
|
||||
current.stream()
|
||||
.mapToDouble(TextFragment::fontSize)
|
||||
.max()
|
||||
.orElse(0));
|
||||
float tolerance =
|
||||
Math.max(maxFontSize * BASELINE_TOLERANCE_FACTOR, MIN_BASELINE_TOLERANCE);
|
||||
|
||||
boolean sameBaseline = Math.abs(fragment.baseline() - currentBaseline) <= tolerance;
|
||||
boolean columnGap = sameBaseline && hasColumnGap(fragment, current, columnGapThreshold);
|
||||
|
||||
if (sameBaseline && !columnGap) {
|
||||
current.add(fragment);
|
||||
// Anchor to the weighted mean baseline so long lines stay stable.
|
||||
currentBaseline =
|
||||
(currentBaseline * (current.size() - 1) + fragment.baseline())
|
||||
/ current.size();
|
||||
} else {
|
||||
groups.add(current);
|
||||
current = new ArrayList<>();
|
||||
current.add(fragment);
|
||||
currentBaseline = fragment.baseline();
|
||||
}
|
||||
}
|
||||
|
||||
if (!current.isEmpty()) groups.add(current);
|
||||
return groups;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the gap from the rightmost fragment in {@code group} to {@code next} exceeds {@code
|
||||
* threshold}.
|
||||
*/
|
||||
private static boolean hasColumnGap(
|
||||
TextFragment next, List<TextFragment> group, float threshold) {
|
||||
float lastRight = group.get(group.size() - 1).bounds().right();
|
||||
return next.bounds().x() - lastRight > threshold;
|
||||
}
|
||||
|
||||
/** Infers effective page width from the rightmost fragment right-edge plus a 10 % margin. */
|
||||
private static float inferEffectiveWidth(List<TextFragment> fragments) {
|
||||
double maxRight =
|
||||
fragments.stream().mapToDouble(f -> f.bounds().right()).max().orElse(500.0);
|
||||
return (float) maxRight * 1.10f;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Runs the per-page ingestion pipeline: {@link WordExtractingStripper} → {@link LineBuilder} →
|
||||
* {@link TableParser}, producing a {@link PdfModels.ParsedPage} per page. The caller owns the
|
||||
* {@link PDDocument} lifecycle.
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class PdfIngester {
|
||||
|
||||
private final LineBuilder lineBuilder;
|
||||
private final TableParser tableParser;
|
||||
|
||||
public List<ParsedPage> parse(PDDocument document) throws IOException {
|
||||
return parse(document, document.getNumberOfPages());
|
||||
}
|
||||
|
||||
public List<ParsedPage> parse(PDDocument document, int maxPages) throws IOException {
|
||||
int pageCount = Math.min(document.getNumberOfPages(), maxPages);
|
||||
List<ParsedPage> pages = new ArrayList<>(pageCount);
|
||||
long fragmentsMs = 0;
|
||||
long tablesMs = 0;
|
||||
long t0 = System.currentTimeMillis();
|
||||
|
||||
for (int p = 1; p <= pageCount; p++) {
|
||||
long ft = System.currentTimeMillis();
|
||||
List<TextFragment> fragments = extractFragments(document, p);
|
||||
fragmentsMs += System.currentTimeMillis() - ft;
|
||||
|
||||
PDPage page = document.getPage(p - 1);
|
||||
PDRectangle mediaBox = page.getMediaBox();
|
||||
List<RawLine> lines = lineBuilder.build(fragments, p);
|
||||
RawPage rawPage = new RawPage(p, mediaBox.getWidth(), mediaBox.getHeight(), lines);
|
||||
|
||||
long tt = System.currentTimeMillis();
|
||||
List<TableFragment> tables = tableParser.parse(document, rawPage);
|
||||
tablesMs += System.currentTimeMillis() - tt;
|
||||
|
||||
log.debug(
|
||||
"Page {}: {} fragments → {} lines, {} table(s)",
|
||||
p,
|
||||
fragments.size(),
|
||||
lines.size(),
|
||||
tables.size());
|
||||
pages.add(new ParsedPage(p, mediaBox.getWidth(), mediaBox.getHeight(), tables, lines));
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[timing] parse pages={} total={}ms fragments={}ms tables={}ms",
|
||||
pageCount,
|
||||
System.currentTimeMillis() - t0,
|
||||
fragmentsMs,
|
||||
tablesMs);
|
||||
return pages;
|
||||
}
|
||||
|
||||
private List<TextFragment> extractFragments(PDDocument document, int pageNumber)
|
||||
throws IOException {
|
||||
WordExtractingStripper stripper = new WordExtractingStripper(pageNumber);
|
||||
stripper.getText(document);
|
||||
return stripper.getFragments();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
|
||||
/**
|
||||
* All PDF parser model types and the table-parser contract in one place.
|
||||
*
|
||||
* <p>Import as {@code import static stirling.software.SPDF.pdf.parser.PdfModels.*;} to use all
|
||||
* nested types without qualification.
|
||||
*/
|
||||
public final class PdfModels {
|
||||
|
||||
private PdfModels() {}
|
||||
|
||||
// ── Geometry ──────────────────────────────────────────────────────────────
|
||||
|
||||
public record Bounds(float x, float y, float width, float height) {
|
||||
|
||||
public float right() {
|
||||
return x + width;
|
||||
}
|
||||
|
||||
public float bottom() {
|
||||
return y + height;
|
||||
}
|
||||
|
||||
public static Bounds merge(Bounds a, Bounds b) {
|
||||
float x = Math.min(a.x, b.x);
|
||||
float y = Math.min(a.y, b.y);
|
||||
return new Bounds(
|
||||
x, y, Math.max(a.right(), b.right()) - x, Math.max(a.bottom(), b.bottom()) - y);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Text fragments and lines ──────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* A contiguous run of text from a single PDF content-stream string operation. {@code baseline}
|
||||
* is preserved separately from bounds for line-grouping — characters of different sizes on the
|
||||
* same visual line share a baseline but differ in top Y.
|
||||
*/
|
||||
public record TextFragment(
|
||||
String fragmentId,
|
||||
String text,
|
||||
Bounds bounds,
|
||||
float baseline,
|
||||
float fontSize,
|
||||
String fontName,
|
||||
boolean bold) {}
|
||||
|
||||
public record RawLine(
|
||||
String lineId, List<TextFragment> fragments, Bounds bounds, int pageNumber) {
|
||||
|
||||
public String text() {
|
||||
if (fragments.isEmpty()) return "";
|
||||
StringBuilder sb = new StringBuilder();
|
||||
TextFragment prev = null;
|
||||
for (TextFragment f : fragments) {
|
||||
if (prev != null) {
|
||||
float gap = f.bounds().x() - prev.bounds().right();
|
||||
float avgCharWidth = prev.bounds().width() / Math.max(prev.text().length(), 1);
|
||||
if (gap > avgCharWidth * 0.5f) sb.append(' ');
|
||||
}
|
||||
sb.append(f.text());
|
||||
prev = f;
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public float dominantFontSize() {
|
||||
return fragments.stream()
|
||||
.collect(Collectors.groupingBy(TextFragment::fontSize, Collectors.counting()))
|
||||
.entrySet()
|
||||
.stream()
|
||||
.max(Map.Entry.comparingByValue())
|
||||
.map(Map.Entry::getKey)
|
||||
.orElse(0f);
|
||||
}
|
||||
|
||||
public boolean hasBold() {
|
||||
return fragments.stream().anyMatch(TextFragment::bold);
|
||||
}
|
||||
}
|
||||
|
||||
public record RawPage(int pageNumber, float widthPt, float heightPt, List<RawLine> lines) {}
|
||||
|
||||
// ── Table model ───────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* A single cell within a {@link TableRow}. {@code colSpan} and {@code rowSpan} are always 1 in
|
||||
* v1 — span detection is deferred but the fields keep the schema stable.
|
||||
*/
|
||||
public record TableCell(int colIndex, String text, Bounds bounds, int colSpan, int rowSpan) {
|
||||
|
||||
public static TableCell of(int colIndex, String text, Bounds bounds) {
|
||||
return new TableCell(colIndex, text, bounds, 1, 1);
|
||||
}
|
||||
}
|
||||
|
||||
public record TableRow(int rowIndex, List<TableCell> cells) {}
|
||||
|
||||
/**
|
||||
* A table as it appears on a single page. {@code headers} is empty in v1 — all rows are in
|
||||
* {@code rows}. {@code rawRows} preserves exact Tabula text output for diagnostics. {@code
|
||||
* confidence} is a heuristic score in [0.0, 1.0]. {@code continuedFromPage} is null in v1.
|
||||
*/
|
||||
public record TableFragment(
|
||||
String tableId,
|
||||
int pageNumber,
|
||||
Bounds bounds,
|
||||
List<TableRow> headers,
|
||||
List<TableRow> rows,
|
||||
List<List<String>> rawRows,
|
||||
int columnCount,
|
||||
float confidence,
|
||||
List<String> warnings,
|
||||
Integer continuedFromPage) {}
|
||||
|
||||
// ── Page output ───────────────────────────────────────────────────────────
|
||||
|
||||
public record ParsedPage(
|
||||
int pageNumber,
|
||||
float widthPt,
|
||||
float heightPt,
|
||||
List<TableFragment> tables,
|
||||
List<RawLine> layoutLines) {}
|
||||
|
||||
// ── Parser contract ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Extracts tables from a single page of a PDF. The caller owns the document lifecycle;
|
||||
* implementations must not close it.
|
||||
*/
|
||||
public interface TableParser {
|
||||
|
||||
/**
|
||||
* @param document open PDF; must not be closed by the implementation
|
||||
* @param rawPage page metadata and lines for the page to process
|
||||
* @return zero or more table fragments found on the page, never null
|
||||
*/
|
||||
List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import technology.tabula.ObjectExtractor;
|
||||
import technology.tabula.Page;
|
||||
import technology.tabula.RectangularTextContainer;
|
||||
import technology.tabula.Table;
|
||||
import technology.tabula.extractors.BasicExtractionAlgorithm;
|
||||
import technology.tabula.extractors.ExtractionAlgorithm;
|
||||
import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
|
||||
|
||||
/**
|
||||
* Primary {@link TableParser} implementation using Tabula's lattice-mode extraction.
|
||||
*
|
||||
* <h3>Algorithm</h3>
|
||||
*
|
||||
* Uses {@link SpreadsheetExtractionAlgorithm} (lattice mode), which detects tables from ruled lines
|
||||
* (horizontal and vertical PDF path operators). This is reliable for tables with visible borders.
|
||||
* Borderless or whitespace-delimited tables will not be detected — that requires stream mode, which
|
||||
* is deferred to a later iteration.
|
||||
*
|
||||
* <h3>Coordinate system</h3>
|
||||
*
|
||||
* Tabula normalises page coordinates so that (0,0) is the top-left corner and Y increases downward,
|
||||
* matching the {@link Bounds} convention used throughout this parser. No coordinate transformation
|
||||
* is needed when mapping Tabula cell bounds to {@link Bounds}.
|
||||
*
|
||||
* <h3>Known limitations</h3>
|
||||
*
|
||||
* <ul>
|
||||
* <li>Borderless tables are not detected (use stream mode, deferred).
|
||||
* <li>Colspan and rowspan are not detected; all cells report colSpan=1, rowSpan=1.
|
||||
* <li>Header rows are not identified; all rows appear in {@code rows}, headers is empty.
|
||||
* <li>Cross-page table linking is not performed; each page is independent.
|
||||
* <li>Rotated tables (90°/270° pages) may produce incorrect bounds.
|
||||
* </ul>
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class TabulaTableParser implements TableParser {
|
||||
|
||||
/** Lattice mode — reliable for tables with visible ruled borders. */
|
||||
@Override
|
||||
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
|
||||
return parseWithAlgorithm(
|
||||
document, rawPage, new SpreadsheetExtractionAlgorithm(), "lattice");
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience overload for callers that only have a page number, not a full {@link RawPage}.
|
||||
*/
|
||||
public List<TableFragment> parse(PDDocument document, int pageNumber) throws IOException {
|
||||
return parse(document, new RawPage(pageNumber, 0f, 0f, List.of()));
|
||||
}
|
||||
|
||||
/** Stream mode — whitespace-based column detection for borderless tables. */
|
||||
public List<TableFragment> parseStream(PDDocument document, RawPage rawPage)
|
||||
throws IOException {
|
||||
return parseWithAlgorithm(document, rawPage, new BasicExtractionAlgorithm(), "stream");
|
||||
}
|
||||
|
||||
private List<TableFragment> parseWithAlgorithm(
|
||||
PDDocument document, RawPage rawPage, ExtractionAlgorithm algorithm, String modeName)
|
||||
throws IOException {
|
||||
int pageNumber = rawPage.pageNumber();
|
||||
|
||||
List<Table> tabulaTables;
|
||||
try {
|
||||
// Do NOT use try-with-resources: ObjectExtractor.close() closes the underlying
|
||||
// PDDocument, which we don't own. The extractor holds no resources of its own.
|
||||
ObjectExtractor extractor = new ObjectExtractor(document);
|
||||
Page page = extractor.extract(pageNumber);
|
||||
tabulaTables = new ArrayList<>(algorithm.extract(page));
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"Tabula {} extraction failed on page {}: {}",
|
||||
modeName,
|
||||
pageNumber,
|
||||
e.getMessage());
|
||||
return List.of();
|
||||
}
|
||||
|
||||
if (tabulaTables.isEmpty()) {
|
||||
log.debug("Page {}: no tables detected by Tabula ({})", pageNumber, modeName);
|
||||
return List.of();
|
||||
}
|
||||
|
||||
log.debug(
|
||||
"Page {}: Tabula ({}) detected {} table(s)",
|
||||
pageNumber,
|
||||
modeName,
|
||||
tabulaTables.size());
|
||||
|
||||
List<TableFragment> fragments = new ArrayList<>(tabulaTables.size());
|
||||
for (int i = 0; i < tabulaTables.size(); i++) {
|
||||
fragments.add(toFragment(tabulaTables.get(i), pageNumber, i));
|
||||
}
|
||||
return fragments;
|
||||
}
|
||||
|
||||
// ── private helpers ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
private TableFragment toFragment(Table table, int pageNumber, int tableIndex) {
|
||||
List<List<RectangularTextContainer>> tabulaRows = table.getRows();
|
||||
List<String> warnings = new ArrayList<>();
|
||||
|
||||
List<List<String>> rawRows = buildRawRows(tabulaRows);
|
||||
int colCount = inferColumnCount(rawRows, warnings);
|
||||
List<TableRow> rows = buildRows(tabulaRows, colCount, warnings);
|
||||
float confidence = computeConfidence(rawRows, colCount, warnings);
|
||||
Bounds bounds = tableBounds(table);
|
||||
|
||||
String tableId = "tbl-p" + pageNumber + "-" + tableIndex;
|
||||
|
||||
if (!warnings.isEmpty()) {
|
||||
log.warn("Page {}, table {}: {}", pageNumber, tableIndex, warnings);
|
||||
}
|
||||
|
||||
return new TableFragment(
|
||||
tableId,
|
||||
pageNumber,
|
||||
bounds,
|
||||
List.of(), // headers: deferred to v2
|
||||
rows,
|
||||
rawRows,
|
||||
colCount,
|
||||
confidence,
|
||||
Collections.unmodifiableList(warnings),
|
||||
null); // continuedFromPage: deferred to v2
|
||||
}
|
||||
|
||||
private List<List<String>> buildRawRows(List<List<RectangularTextContainer>> tabulaRows) {
|
||||
List<List<String>> rawRows = new ArrayList<>(tabulaRows.size());
|
||||
for (List<RectangularTextContainer> tabulaRow : tabulaRows) {
|
||||
List<String> cells = new ArrayList<>(tabulaRow.size());
|
||||
for (RectangularTextContainer cell : tabulaRow) {
|
||||
cells.add(normaliseText(cell.getText()));
|
||||
}
|
||||
rawRows.add(Collections.unmodifiableList(cells));
|
||||
}
|
||||
return rawRows;
|
||||
}
|
||||
|
||||
private List<TableRow> buildRows(
|
||||
List<List<RectangularTextContainer>> tabulaRows, int colCount, List<String> warnings) {
|
||||
List<TableRow> rows = new ArrayList<>(tabulaRows.size());
|
||||
for (int rowIdx = 0; rowIdx < tabulaRows.size(); rowIdx++) {
|
||||
List<RectangularTextContainer> tabulaRow = tabulaRows.get(rowIdx);
|
||||
List<TableCell> cells = new ArrayList<>(tabulaRow.size());
|
||||
|
||||
for (int colIdx = 0; colIdx < tabulaRow.size(); colIdx++) {
|
||||
RectangularTextContainer c = tabulaRow.get(colIdx);
|
||||
Bounds cellBounds =
|
||||
new Bounds(
|
||||
(float) c.getX(),
|
||||
(float) c.getY(),
|
||||
(float) c.getWidth(),
|
||||
(float) c.getHeight());
|
||||
cells.add(TableCell.of(colIdx, normaliseText(c.getText()), cellBounds));
|
||||
}
|
||||
|
||||
if (tabulaRow.size() != colCount) {
|
||||
warnings.add(
|
||||
"Row "
|
||||
+ rowIdx
|
||||
+ " has "
|
||||
+ tabulaRow.size()
|
||||
+ " cells; expected "
|
||||
+ colCount);
|
||||
}
|
||||
|
||||
rows.add(new TableRow(rowIdx, Collections.unmodifiableList(cells)));
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* The canonical column count for a table is the size of the widest row. Tabula can produce
|
||||
* uneven rows when a cell's ruling lines are partially missing.
|
||||
*/
|
||||
private int inferColumnCount(List<List<String>> rawRows, List<String> warnings) {
|
||||
if (rawRows.isEmpty()) return 0;
|
||||
int max = rawRows.stream().mapToInt(List::size).max().orElse(0);
|
||||
int mode =
|
||||
rawRows.stream()
|
||||
.collect(
|
||||
java.util.stream.Collectors.groupingBy(
|
||||
List::size, java.util.stream.Collectors.counting()))
|
||||
.entrySet()
|
||||
.stream()
|
||||
.max(java.util.Map.Entry.comparingByValue())
|
||||
.map(java.util.Map.Entry::getKey)
|
||||
.orElse(0);
|
||||
if (max != mode) {
|
||||
warnings.add("Inconsistent column count: modal=" + mode + " max=" + max);
|
||||
}
|
||||
return mode > 0 ? mode : max;
|
||||
}
|
||||
|
||||
/**
|
||||
* Heuristic confidence score in [0.0, 1.0].
|
||||
*
|
||||
* <ul>
|
||||
* <li>Starts at 1.0.
|
||||
* <li>-0.3 if only one column (single-column tables are usually not real tables).
|
||||
* <li>-0.1 per row with an inconsistent column count, capped at -0.4.
|
||||
* <li>-0.3 if the empty-cell ratio across all cells exceeds 80%.
|
||||
* </ul>
|
||||
*/
|
||||
private float computeConfidence(
|
||||
List<List<String>> rawRows, int colCount, List<String> warnings) {
|
||||
if (rawRows.isEmpty() || colCount == 0) return 0f;
|
||||
|
||||
float score = 1.0f;
|
||||
|
||||
if (colCount == 1) score -= 0.3f;
|
||||
|
||||
long inconsistentRows = rawRows.stream().filter(r -> r.size() != colCount).count();
|
||||
score -= Math.min(inconsistentRows * 0.1f, 0.4f);
|
||||
|
||||
long totalCells = rawRows.stream().mapToLong(List::size).sum();
|
||||
long emptyCells =
|
||||
rawRows.stream().flatMap(Collection::stream).filter(String::isBlank).count();
|
||||
if (totalCells > 0 && (float) emptyCells / totalCells > 0.8f) {
|
||||
score -= 0.3f;
|
||||
}
|
||||
|
||||
return Math.max(0f, Math.min(1f, score));
|
||||
}
|
||||
|
||||
private Bounds tableBounds(Table table) {
|
||||
return new Bounds(
|
||||
(float) table.getX(),
|
||||
(float) table.getY(),
|
||||
(float) table.getWidth(),
|
||||
(float) table.getHeight());
|
||||
}
|
||||
|
||||
private String normaliseText(String raw) {
|
||||
if (raw == null) return "";
|
||||
// Tabula wraps multi-line cell content with \r\n — collapse to a single space.
|
||||
return raw.replace("\r\n", " ").replace("\n", " ").replace("\r", " ").trim();
|
||||
}
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
/**
|
||||
* Extends {@link PDFTextStripper} to capture per-fragment geometry and font metadata.
|
||||
*
|
||||
* <p>Overrides {@link #writeString} to split each content-stream string into word-level {@link
|
||||
* TextFragment}s with bounding boxes, baseline, font name, and bold flag. Coordinates are in
|
||||
* PDFTextStripper space: (0,0) top-left, Y increases downward, {@code getY()} is the baseline.
|
||||
*/
|
||||
class WordExtractingStripper extends PDFTextStripper {
|
||||
|
||||
private final int targetPage;
|
||||
private final List<TextFragment> fragments = new ArrayList<>();
|
||||
private int fragmentIndex = 0;
|
||||
|
||||
WordExtractingStripper(int pageNumber) throws IOException {
|
||||
this.targetPage = pageNumber;
|
||||
setStartPage(pageNumber);
|
||||
setEndPage(pageNumber);
|
||||
setSortByPosition(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void startPage(PDPage page) throws IOException {
|
||||
super.startPage(page);
|
||||
fragments.clear();
|
||||
fragmentIndex = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeString(String text, List<TextPosition> textPositions) throws IOException {
|
||||
if (text == null || text.isBlank()) return;
|
||||
|
||||
// Fast path: no whitespace → emit one fragment (most financial PDFs have each
|
||||
// number as its own string operation, so this is the common case).
|
||||
if (text.indexOf(' ') < 0) {
|
||||
emitFragment(text, textPositions);
|
||||
return;
|
||||
}
|
||||
|
||||
// Per-word splitting requires 1:1 text-char to TextPosition correspondence.
|
||||
// Fall back to one fragment when sizes differ (ligatures, encoding edge cases).
|
||||
if (textPositions.size() != text.length()) {
|
||||
emitFragment(text, textPositions);
|
||||
return;
|
||||
}
|
||||
|
||||
// Emit one TextFragment per whitespace-delimited word with accurate per-word bounds.
|
||||
int start = 0;
|
||||
for (int i = 0; i <= text.length(); i++) {
|
||||
if (i == text.length() || text.charAt(i) == ' ') {
|
||||
if (start < i) {
|
||||
emitFragment(text.substring(start, i), textPositions.subList(start, i));
|
||||
}
|
||||
start = i + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void emitFragment(String text, List<TextPosition> positions) {
|
||||
if (positions.isEmpty()) return;
|
||||
|
||||
float minX = Float.MAX_VALUE;
|
||||
float minY = Float.MAX_VALUE;
|
||||
float maxRight = -Float.MAX_VALUE;
|
||||
float maxBaseline = -Float.MAX_VALUE;
|
||||
TextPosition first = null;
|
||||
|
||||
for (TextPosition tp : positions) {
|
||||
if (tp == null) continue;
|
||||
if (first == null) first = tp;
|
||||
|
||||
float x = tp.getX();
|
||||
// getY() is the baseline; top of character = getY() - getHeight().
|
||||
float top = tp.getY() - tp.getHeight();
|
||||
float right = x + tp.getWidth();
|
||||
float baseline = tp.getY();
|
||||
|
||||
minX = Math.min(minX, x);
|
||||
minY = Math.min(minY, top);
|
||||
maxRight = Math.max(maxRight, right);
|
||||
maxBaseline = Math.max(maxBaseline, baseline);
|
||||
}
|
||||
|
||||
if (first == null) return;
|
||||
|
||||
PDFont font = first.getFont();
|
||||
String fontName = font != null ? font.getName() : "";
|
||||
boolean bold = fontName != null && fontName.toLowerCase().contains("bold");
|
||||
// getHeight() gives the rendered glyph height, which is the most reliable visual size.
|
||||
float fontSize = first.getHeight();
|
||||
|
||||
Bounds bounds = new Bounds(minX, minY, maxRight - minX, maxBaseline - minY);
|
||||
String id = "tf-p" + targetPage + "-" + fragmentIndex++;
|
||||
fragments.add(new TextFragment(id, text, bounds, maxBaseline, fontSize, fontName, bold));
|
||||
}
|
||||
|
||||
List<TextFragment> getFragments() {
|
||||
return Collections.unmodifiableList(fragments);
|
||||
}
|
||||
}
|
||||
+12
@@ -59,6 +59,18 @@ public class InstallationPathConfig {
|
||||
}
|
||||
|
||||
private static String initializeBasePath() {
|
||||
// Allow tests / harnesses to redirect the entire state tree (configs,
|
||||
// backups, customFiles, pipeline, logs) to an isolated location via
|
||||
// -Dstirling.base-path=... or STIRLING_BASE_PATH=... so a Playwright
|
||||
// run never touches a developer's working state.
|
||||
String override = System.getProperty("stirling.base-path");
|
||||
if (override == null || override.isBlank()) {
|
||||
override = System.getenv("STIRLING_BASE_PATH");
|
||||
}
|
||||
if (override != null && !override.isBlank()) {
|
||||
boolean hasTrailingSeparator = override.endsWith("/") || override.endsWith("\\");
|
||||
return hasTrailingSeparator ? override : override + File.separator;
|
||||
}
|
||||
return "." + File.separator;
|
||||
}
|
||||
|
||||
|
||||
@@ -76,6 +76,7 @@ public class ApplicationProperties {
|
||||
private ProcessExecutor processExecutor = new ProcessExecutor();
|
||||
private PdfEditor pdfEditor = new PdfEditor();
|
||||
private AiEngine aiEngine = new AiEngine();
|
||||
private InternalApi internalApi = new InternalApi();
|
||||
|
||||
@Bean
|
||||
public PropertySource<?> dynamicYamlPropertySource(ConfigurableEnvironment environment)
|
||||
@@ -246,6 +247,19 @@ public class ApplicationProperties {
|
||||
private int longRunningTimeoutSeconds = 600;
|
||||
}
|
||||
|
||||
/**
|
||||
* HTTP timeouts for loopback calls to internal Stirling API endpoints, used by the AI workflow
|
||||
* executor and the pipeline processor. A bounded read timeout prevents a hung tool (e.g. an
|
||||
* infinite loop in a PDF processing service) from stalling the entire chat workflow forever.
|
||||
* Tools that legitimately need longer than the read timeout should be invoked through the async
|
||||
* job executor instead of synchronously.
|
||||
*/
|
||||
@Data
|
||||
public static class InternalApi {
|
||||
private int connectTimeoutSeconds = 10;
|
||||
private int readTimeoutSeconds = 300;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Legal {
|
||||
private String termsAndConditions;
|
||||
|
||||
@@ -14,7 +14,6 @@ import lombok.NoArgsConstructor;
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@EqualsAndHashCode
|
||||
@Schema(description = "PDF file input - either upload a file or provide a server-side file ID")
|
||||
public class PDFFile {
|
||||
|
||||
@Schema(
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package stirling.software.common.model.api.general;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode
|
||||
public class EditTextOperation {
|
||||
|
||||
@Schema(description = "The literal text to find.", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String find;
|
||||
|
||||
@Schema(
|
||||
description = "The replacement text. May be empty to delete the matched text.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String replace;
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import java.io.UncheckedIOException;
|
||||
import java.net.URLDecoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -12,15 +13,18 @@ import org.springframework.core.env.Environment;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.*;
|
||||
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.RequestCallback;
|
||||
import org.springframework.web.client.ResourceAccessException;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import jakarta.servlet.ServletContext;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.enumeration.Role;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
@@ -50,16 +54,29 @@ public class InternalApiClient {
|
||||
private final UserServiceInterface userService;
|
||||
private final TempFileManager tempFileManager;
|
||||
private final Environment environment;
|
||||
private final Duration readTimeout;
|
||||
private final RestTemplate restTemplate;
|
||||
|
||||
public InternalApiClient(
|
||||
ServletContext servletContext,
|
||||
@Autowired(required = false) UserServiceInterface userService,
|
||||
TempFileManager tempFileManager,
|
||||
Environment environment) {
|
||||
Environment environment,
|
||||
ApplicationProperties applicationProperties) {
|
||||
this.servletContext = servletContext;
|
||||
this.userService = userService;
|
||||
this.tempFileManager = tempFileManager;
|
||||
this.environment = environment;
|
||||
ApplicationProperties.InternalApi internalApi = applicationProperties.getInternalApi();
|
||||
// A bounded read timeout is what protects the workflow when an internal tool hangs
|
||||
// (e.g. an infinite loop in a PDF processing service). The connect timeout is short
|
||||
// because this is a loopback call; if connecting takes longer than a few seconds the
|
||||
// local server is itself unhealthy.
|
||||
this.readTimeout = Duration.ofSeconds(internalApi.getReadTimeoutSeconds());
|
||||
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
|
||||
factory.setConnectTimeout(Duration.ofSeconds(internalApi.getConnectTimeoutSeconds()));
|
||||
factory.setReadTimeout(readTimeout);
|
||||
this.restTemplate = new RestTemplate(factory);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -74,7 +91,6 @@ public class InternalApiClient {
|
||||
validateUrl(endpointPath);
|
||||
String url = getBaseUrl() + endpointPath;
|
||||
|
||||
RestTemplate restTemplate = new RestTemplate();
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
String apiKey = getApiKeyForUser();
|
||||
if (apiKey != null && !apiKey.isEmpty()) {
|
||||
@@ -84,26 +100,38 @@ public class InternalApiClient {
|
||||
HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
|
||||
RequestCallback requestCallback = restTemplate.httpEntityCallback(entity, Resource.class);
|
||||
|
||||
return restTemplate.execute(
|
||||
url,
|
||||
HttpMethod.POST,
|
||||
requestCallback,
|
||||
response -> {
|
||||
try {
|
||||
TempFile tempFile = tempFileManager.createManagedTempFile("internal-api");
|
||||
Files.copy(
|
||||
response.getBody(),
|
||||
tempFile.getPath(),
|
||||
java.nio.file.StandardCopyOption.REPLACE_EXISTING);
|
||||
String filename = extractFilename(response.getHeaders());
|
||||
TempFileResource resource = new TempFileResource(tempFile, filename);
|
||||
return ResponseEntity.status(response.getStatusCode())
|
||||
.headers(response.getHeaders())
|
||||
.body(resource);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
});
|
||||
try {
|
||||
return restTemplate.execute(
|
||||
url,
|
||||
HttpMethod.POST,
|
||||
requestCallback,
|
||||
response -> {
|
||||
try {
|
||||
TempFile tempFile =
|
||||
tempFileManager.createManagedTempFile("internal-api");
|
||||
Files.copy(
|
||||
response.getBody(),
|
||||
tempFile.getPath(),
|
||||
java.nio.file.StandardCopyOption.REPLACE_EXISTING);
|
||||
String filename = extractFilename(response.getHeaders());
|
||||
TempFileResource resource = new TempFileResource(tempFile, filename);
|
||||
return ResponseEntity.status(response.getStatusCode())
|
||||
.headers(response.getHeaders())
|
||||
.body(resource);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
});
|
||||
} catch (ResourceAccessException e) {
|
||||
// RestTemplate wraps low-level I/O failures in ResourceAccessException. Only the
|
||||
// SocketTimeoutException-rooted case is a real timeout; other I/O failures (connection
|
||||
// refused, DNS, etc.) propagate as-is so the upstream generic handler can describe
|
||||
// them accurately.
|
||||
if (e.getCause() instanceof java.net.SocketTimeoutException) {
|
||||
throw new InternalApiTimeoutException(endpointPath, readTimeout, e);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* Thrown when an internal Stirling tool invocation exceeds its configured read timeout or otherwise
|
||||
* fails at the transport layer. Distinguishes a hung/timed-out tool from a tool that returned a
|
||||
* non-2xx HTTP response (which is reported via the response status itself), so callers can present
|
||||
* a clear "the tool didn't respond in time" message to the user instead of a generic stack trace.
|
||||
*/
|
||||
public class InternalApiTimeoutException extends RuntimeException {
|
||||
|
||||
private final String endpointPath;
|
||||
private final Duration readTimeout;
|
||||
|
||||
public InternalApiTimeoutException(String endpointPath, Duration readTimeout, Throwable cause) {
|
||||
super(buildMessage(endpointPath, readTimeout, cause), cause);
|
||||
this.endpointPath = endpointPath;
|
||||
this.readTimeout = readTimeout;
|
||||
}
|
||||
|
||||
public String getEndpointPath() {
|
||||
return endpointPath;
|
||||
}
|
||||
|
||||
public Duration getReadTimeout() {
|
||||
return readTimeout;
|
||||
}
|
||||
|
||||
private static String buildMessage(String endpointPath, Duration readTimeout, Throwable cause) {
|
||||
String reason = cause != null && cause.getMessage() != null ? cause.getMessage() : "";
|
||||
return String.format(
|
||||
"Internal tool %s did not respond within %ds (%s)",
|
||||
endpointPath, readTimeout.toSeconds(), reason);
|
||||
}
|
||||
}
|
||||
@@ -2289,6 +2289,85 @@ public class FormUtils {
|
||||
acroForm.getFields().add(field);
|
||||
}
|
||||
|
||||
/** Drops AcroForm fields whose widgets are no longer on any page of {@code document}. */
|
||||
public void pruneOrphanedFormFields(PDDocument document) {
|
||||
if (document == null) {
|
||||
return;
|
||||
}
|
||||
PDDocumentCatalog catalog = document.getDocumentCatalog();
|
||||
if (catalog == null) {
|
||||
return;
|
||||
}
|
||||
PDAcroForm form = catalog.getAcroForm(null);
|
||||
if (form == null) {
|
||||
return;
|
||||
}
|
||||
List<PDField> fields = form.getFields();
|
||||
if (fields.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Set<COSDictionary> liveWidgets = collectLiveWidgetDictionaries(document);
|
||||
List<PDField> kept = pruneFieldList(fields, liveWidgets);
|
||||
if (kept.isEmpty()) {
|
||||
catalog.setAcroForm(null);
|
||||
} else if (kept.size() != fields.size()) {
|
||||
form.setFields(kept);
|
||||
}
|
||||
}
|
||||
|
||||
private Set<COSDictionary> collectLiveWidgetDictionaries(PDDocument document) {
|
||||
Set<COSDictionary> live = new HashSet<>();
|
||||
int pageCount = document.getNumberOfPages();
|
||||
for (int i = 0; i < pageCount; i++) {
|
||||
try {
|
||||
for (PDAnnotation annotation : document.getPage(i).getAnnotations()) {
|
||||
if (annotation instanceof PDAnnotationWidget) {
|
||||
live.add(annotation.getCOSObject());
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.debug("Failed reading page {} annotations: {}", i, e.getMessage());
|
||||
}
|
||||
}
|
||||
return live;
|
||||
}
|
||||
|
||||
private List<PDField> pruneFieldList(List<PDField> fields, Set<COSDictionary> liveWidgets) {
|
||||
List<PDField> kept = new ArrayList<>(fields.size());
|
||||
for (PDField field : fields) {
|
||||
if (field instanceof PDNonTerminalField nonTerminal) {
|
||||
List<PDField> children = nonTerminal.getChildren();
|
||||
List<PDField> remaining = pruneFieldList(children, liveWidgets);
|
||||
if (remaining.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (remaining.size() != children.size()) {
|
||||
nonTerminal.setChildren(remaining);
|
||||
}
|
||||
kept.add(nonTerminal);
|
||||
} else if (field instanceof PDTerminalField terminal) {
|
||||
List<PDAnnotationWidget> widgets = terminal.getWidgets();
|
||||
List<PDAnnotationWidget> liveOnes = new ArrayList<>(widgets.size());
|
||||
for (PDAnnotationWidget widget : widgets) {
|
||||
if (liveWidgets.contains(widget.getCOSObject())) {
|
||||
liveOnes.add(widget);
|
||||
}
|
||||
}
|
||||
if (liveOnes.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (liveOnes.size() != widgets.size()) {
|
||||
terminal.setWidgets(liveOnes);
|
||||
}
|
||||
kept.add(terminal);
|
||||
} else {
|
||||
kept.add(field);
|
||||
}
|
||||
}
|
||||
return kept;
|
||||
}
|
||||
|
||||
// Delegation methods to GeneralFormCopyUtils for form field transformation
|
||||
public boolean hasAnyRotatedPage(PDDocument document) {
|
||||
return stirling.software.common.util.GeneralFormCopyUtils.hasAnyRotatedPage(document);
|
||||
|
||||
@@ -15,6 +15,7 @@ import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import io.github.pixee.security.BoundedLineReader;
|
||||
|
||||
@@ -202,7 +203,15 @@ public class ProcessExecutor {
|
||||
boolean useSemaphore = true;
|
||||
List<String> commandToRun = command;
|
||||
if (shouldUseUnoServerPool(command)) {
|
||||
unoLease = unoServerPool.acquireEndpoint();
|
||||
try {
|
||||
unoLease = unoServerPool.acquireEndpoint(timeoutDuration, TimeUnit.MINUTES);
|
||||
} catch (TimeoutException e) {
|
||||
throw new IOException(
|
||||
"All unoserver endpoints busy; request timed out after "
|
||||
+ timeoutDuration
|
||||
+ " minutes",
|
||||
e);
|
||||
}
|
||||
commandToRun = applyUnoServerEndpoint(command, unoLease.getEndpoint());
|
||||
useSemaphore = false;
|
||||
}
|
||||
|
||||
@@ -3,8 +3,11 @@ package stirling.software.common.util;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
@@ -42,6 +45,27 @@ public class UnoServerPool {
|
||||
return new UnoServerLease(endpoints.get(index), index, this);
|
||||
}
|
||||
|
||||
/** Fail-fast variant; non-positive timeout falls back to unbounded acquire. */
|
||||
public UnoServerLease acquireEndpoint(long timeout, TimeUnit unit)
|
||||
throws InterruptedException, TimeoutException {
|
||||
if (endpoints.isEmpty()) {
|
||||
return new UnoServerLease(defaultEndpoint(), null, this);
|
||||
}
|
||||
if (timeout <= 0) {
|
||||
return acquireEndpoint();
|
||||
}
|
||||
|
||||
Integer index = availableIndices.poll(timeout, unit);
|
||||
if (index == null) {
|
||||
throw new TimeoutException(
|
||||
"Timed out waiting for a free unoserver endpoint after "
|
||||
+ timeout
|
||||
+ " "
|
||||
+ unit.name().toLowerCase(Locale.ROOT));
|
||||
}
|
||||
return new UnoServerLease(endpoints.get(index), index, this);
|
||||
}
|
||||
|
||||
private void releaseEndpoint(Integer index) {
|
||||
if (index != null) {
|
||||
availableIndices.offer(index);
|
||||
|
||||
+18
-6
@@ -6,15 +6,18 @@ import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.JavaType;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Spring property editor that decodes a JSON string into a typed {@link ArrayList}. Used to bind
|
||||
* complex list parameters (e.g. {@code List<RedactionArea>}, {@code List<EditTextOperation>}) from
|
||||
* multipart form fields, where Spring's default binding cannot deserialize a JSON array.
|
||||
*/
|
||||
@Slf4j
|
||||
public class StringToArrayListPropertyEditor extends PropertyEditorSupport {
|
||||
public class StringToArrayListPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private final ObjectMapper objectMapper =
|
||||
JsonMapper.builder()
|
||||
@@ -22,6 +25,12 @@ public class StringToArrayListPropertyEditor extends PropertyEditorSupport {
|
||||
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.build();
|
||||
|
||||
private final Class<T> elementType;
|
||||
|
||||
public StringToArrayListPropertyEditor(Class<T> elementType) {
|
||||
this.elementType = elementType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
@@ -29,8 +38,11 @@ public class StringToArrayListPropertyEditor extends PropertyEditorSupport {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
TypeReference<ArrayList<RedactionArea>> typeRef = new TypeReference<>() {};
|
||||
List<RedactionArea> list = objectMapper.readValue(text, typeRef);
|
||||
JavaType listType =
|
||||
objectMapper
|
||||
.getTypeFactory()
|
||||
.constructCollectionType(ArrayList.class, elementType);
|
||||
List<T> list = objectMapper.readValue(text, listType);
|
||||
setValue(list);
|
||||
} catch (Exception e) {
|
||||
log.error("Exception while converting {}", e);
|
||||
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link LineAlignmentTableParser}, focused on the coincident-line merge logic and
|
||||
* column-grid construction.
|
||||
*/
|
||||
class LineAlignmentTableParserTest {
|
||||
|
||||
private final LineAlignmentTableParser parser = new LineAlignmentTableParser();
|
||||
|
||||
// ── mergeCoincidentLines ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_singleLine_unchanged() {
|
||||
var lines = List.of(tokenized(rawLine(10f, 100f, "Revenue")));
|
||||
assertThat(parser.mergeCoincidentLines(lines)).hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_distinctYLines_unchanged() {
|
||||
// Two lines at different y positions — must NOT be merged.
|
||||
var lines =
|
||||
List.of(
|
||||
tokenized(rawLine(10f, 100f, "Revenue")),
|
||||
tokenized(rawLine(10f, 115f, "Cost")));
|
||||
assertThat(parser.mergeCoincidentLines(lines)).hasSize(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_sameY_merged() {
|
||||
// Simulates a financial-table row split by LineBuilder at the column gap:
|
||||
// label fragment at x=72 → "Revenue"
|
||||
// value fragment at x=350 → "1,234"
|
||||
// Both have y=100. After merge they should form one TokenizedLine.
|
||||
var label = rawLine(72f, 100f, "Revenue");
|
||||
var value = rawLine(350f, 100f, "1,234");
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
|
||||
|
||||
assertThat(merged).hasSize(1);
|
||||
// The merged line should contain tokens from both halves.
|
||||
var tokens = merged.get(0).all();
|
||||
assertThat(tokens.stream().map(t -> t.text()).toList())
|
||||
.containsExactlyInAnyOrder("Revenue", "1,234");
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_sameY_mergedLineHasCorrectBounds() {
|
||||
var label = rawLine(72f, 100f, "Revenue"); // 7 chars × 6pt = 42pt wide → right = 114
|
||||
var value = rawLine(350f, 100f, "1,234"); // 5 chars × 6pt = 30pt wide → right = 380
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
|
||||
|
||||
var bounds = merged.get(0).line().bounds();
|
||||
assertThat(bounds.x()).isEqualTo(72f);
|
||||
assertThat(bounds.right()).isEqualTo(380f);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_withinTolerance_merged() {
|
||||
// Lines 1.5pt apart (within ROW_MERGE_TOLERANCE_PT = 2pt) should merge.
|
||||
var a = rawLine(10f, 100.0f, "Alpha");
|
||||
var b = rawLine(200f, 101.5f, "99");
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
|
||||
assertThat(merged).hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_beyondTolerance_notMerged() {
|
||||
// Lines 3pt apart (beyond ROW_MERGE_TOLERANCE_PT = 2pt) should NOT merge.
|
||||
var a = rawLine(10f, 100.0f, "Alpha");
|
||||
var b = rawLine(200f, 103.0f, "99");
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
|
||||
assertThat(merged).hasSize(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_threeCoincident_allMerged() {
|
||||
// Three fragments at the same y (e.g. wide financial table with two value columns).
|
||||
var a = rawLine(72f, 100f, "Revenue");
|
||||
var b = rawLine(300f, 100f, "1,234");
|
||||
var c = rawLine(400f, 100f, "5,678");
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
|
||||
assertThat(merged).hasSize(1);
|
||||
assertThat(merged.get(0).all()).hasSize(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_coincidentPairFollowedByDistinctLine_twoGroups() {
|
||||
var a = rawLine(72f, 100f, "Revenue");
|
||||
var b = rawLine(350f, 100f, "1,234"); // same y as a → merges with a
|
||||
var c = rawLine(10f, 115f, "Expenses"); // different y → stays separate
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
|
||||
assertThat(merged).hasSize(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_numericAnchorStatus_correctAfterMerge() {
|
||||
// After merging, the combined line should be an anchor (≥2 numeric tokens).
|
||||
// "Revenue" alone → not an anchor. "1,234 567" alone → anchor.
|
||||
// Merged → anchor with at least 2 numerics.
|
||||
var label = rawLine(72f, 100f, "Revenue");
|
||||
var values = rawLineMultiWord(350f, 100f, "1,234", 30f, "567", 30f);
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(values)));
|
||||
|
||||
assertThat(merged).hasSize(1);
|
||||
assertThat(merged.get(0).isAnchor()).isTrue();
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Creates a RawLine with a single TextFragment of the given text at the given position. */
|
||||
private static RawLine rawLine(float x, float y, String text) {
|
||||
float width = text.length() * 6f; // ~6pt per char — rough but consistent
|
||||
float height = 12f;
|
||||
Bounds bounds = new Bounds(x, y, width, height);
|
||||
TextFragment fragment =
|
||||
new TextFragment("tf-test", text, bounds, y + height, 11f, "Helvetica", false);
|
||||
return new RawLine("ln-test", List.of(fragment), bounds, 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a RawLine with two TextFragments representing two words separated by a small gap.
|
||||
* Used to simulate a values-only line with multiple numeric tokens.
|
||||
*/
|
||||
private static RawLine rawLineMultiWord(
|
||||
float x, float y, String word1, float w1, String word2, float w2) {
|
||||
float height = 12f;
|
||||
Bounds b1 = new Bounds(x, y, w1, height);
|
||||
Bounds b2 = new Bounds(x + w1 + 5f, y, w2, height);
|
||||
TextFragment f1 = new TextFragment("tf-1", word1, b1, y + height, 11f, "Helvetica", false);
|
||||
TextFragment f2 = new TextFragment("tf-2", word2, b2, y + height, 11f, "Helvetica", false);
|
||||
Bounds lineBounds = new Bounds(x, y, x + w1 + 5f + w2 - x, height);
|
||||
return new RawLine("ln-test", List.of(f1, f2), lineBounds, 1);
|
||||
}
|
||||
|
||||
/** Tokenises a RawLine via the parser's own tokenise logic (package-private access). */
|
||||
private LineAlignmentTableParser.TokenizedLine tokenized(RawLine line) {
|
||||
return parser.tokenize(line);
|
||||
}
|
||||
}
|
||||
+87
-3
@@ -22,11 +22,13 @@ import org.springframework.mock.env.MockEnvironment;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.RequestCallback;
|
||||
import org.springframework.web.client.ResourceAccessException;
|
||||
import org.springframework.web.client.ResponseExtractor;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import jakarta.servlet.ServletContext;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
@@ -42,8 +44,19 @@ class InternalApiClientTest {
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(servletContext.getContextPath()).thenReturn("");
|
||||
client = newClient();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a fresh client. Tests that use {@link org.mockito.Mockito#mockConstruction} to
|
||||
* intercept {@link RestTemplate} must call this from inside their {@code mockConstruction}
|
||||
* block, since the client now caches one RestTemplate per instance at construction time.
|
||||
*/
|
||||
private InternalApiClient newClient() {
|
||||
MockEnvironment environment = new MockEnvironment().withProperty("server.port", "8080");
|
||||
client = new InternalApiClient(servletContext, userService, tempFileManager, environment);
|
||||
ApplicationProperties applicationProperties = new ApplicationProperties();
|
||||
return new InternalApiClient(
|
||||
servletContext, userService, tempFileManager, environment, applicationProperties);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -75,7 +88,10 @@ class InternalApiClientTest {
|
||||
.thenAnswer(inv -> fakeOkResponse(inv.getArgument(3)));
|
||||
})) {
|
||||
|
||||
ResponseEntity<Resource> response = client.post("/api/v1/general/merge-pdfs", body);
|
||||
// Reconstruct the client so its cached RestTemplate is the mocked one.
|
||||
InternalApiClient mockedClient = newClient();
|
||||
ResponseEntity<Resource> response =
|
||||
mockedClient.post("/api/v1/general/merge-pdfs", body);
|
||||
|
||||
assertNotNull(response);
|
||||
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||
@@ -86,6 +102,72 @@ class InternalApiClientTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void postWrapsSocketTimeoutAsInternalApiTimeoutException() {
|
||||
// Simulates the read-timeout case: RestTemplate wraps SocketTimeoutException in
|
||||
// ResourceAccessException when the underlying socket times out. The client must repackage
|
||||
// that into a typed timeout exception that carries the failing endpoint and the configured
|
||||
// read timeout, so the workflow layer can surface a clean "tool didn't respond" message
|
||||
// to the user.
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
body.add("fileInput", namedResource("input.pdf", "data"));
|
||||
|
||||
try (var ignored =
|
||||
mockConstruction(
|
||||
RestTemplate.class,
|
||||
(rt, ctx) -> {
|
||||
when(rt.httpEntityCallback(any(), eq(Resource.class)))
|
||||
.thenAnswer(inv -> (RequestCallback) req -> {});
|
||||
when(rt.execute(anyString(), eq(HttpMethod.POST), any(), any()))
|
||||
.thenThrow(
|
||||
new ResourceAccessException(
|
||||
"I/O error on POST request: Read timed out",
|
||||
new java.net.SocketTimeoutException(
|
||||
"Read timed out")));
|
||||
})) {
|
||||
|
||||
InternalApiClient mockedClient = newClient();
|
||||
InternalApiTimeoutException thrown =
|
||||
assertThrows(
|
||||
InternalApiTimeoutException.class,
|
||||
() -> mockedClient.post("/api/v1/general/merge-pdfs", body));
|
||||
|
||||
assertEquals("/api/v1/general/merge-pdfs", thrown.getEndpointPath());
|
||||
assertNotNull(thrown.getReadTimeout());
|
||||
assertTrue(thrown.getMessage().contains("/api/v1/general/merge-pdfs"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void postRethrowsNonTimeoutResourceAccessExceptionAsIs() {
|
||||
// ResourceAccessException covers more than just timeouts (e.g. connection refused, DNS
|
||||
// failure). Only SocketTimeoutException-rooted failures are timeouts; everything else
|
||||
// must propagate so the upstream generic handler can label it correctly instead of lying
|
||||
// about a "tool didn't respond" timeout.
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
body.add("fileInput", namedResource("input.pdf", "data"));
|
||||
|
||||
try (var ignored =
|
||||
mockConstruction(
|
||||
RestTemplate.class,
|
||||
(rt, ctx) -> {
|
||||
when(rt.httpEntityCallback(any(), eq(Resource.class)))
|
||||
.thenAnswer(inv -> (RequestCallback) req -> {});
|
||||
when(rt.execute(anyString(), eq(HttpMethod.POST), any(), any()))
|
||||
.thenThrow(
|
||||
new ResourceAccessException(
|
||||
"I/O error on POST request: Connection refused",
|
||||
new java.net.ConnectException(
|
||||
"Connection refused")));
|
||||
})) {
|
||||
|
||||
InternalApiClient mockedClient = newClient();
|
||||
assertThrows(
|
||||
ResourceAccessException.class,
|
||||
() -> mockedClient.post("/api/v1/general/merge-pdfs", body));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void postRejectsDisallowedPath() {
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
@@ -125,8 +207,10 @@ class InternalApiClientTest {
|
||||
.thenAnswer(inv -> fakeOkResponse(inv.getArgument(3)));
|
||||
})) {
|
||||
|
||||
// Reconstruct the client so its cached RestTemplate is the mocked one.
|
||||
InternalApiClient mockedClient = newClient();
|
||||
ResponseEntity<Resource> response =
|
||||
client.post("/api/v1/ai/tools/pdf-comment-agent", body);
|
||||
mockedClient.post("/api/v1/ai/tools/pdf-comment-agent", body);
|
||||
|
||||
assertNotNull(response);
|
||||
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationWidget;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDField;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDNonTerminalField;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDTextField;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class FormUtilsPruneOrphanedFieldsTest {
|
||||
|
||||
@Test
|
||||
void noAcroFormIsNoOp() throws IOException {
|
||||
try (PDDocument document = new PDDocument()) {
|
||||
document.addPage(new PDPage(PDRectangle.A4));
|
||||
FormUtils.pruneOrphanedFormFields(document);
|
||||
assertNull(document.getDocumentCatalog().getAcroForm(null));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void dropsFieldsWhoseWidgetsAreAllOnRemovedPages() throws IOException {
|
||||
byte[] pdfBytes = buildPdfWithFieldPerPage(3);
|
||||
|
||||
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
|
||||
document.removePage(2);
|
||||
document.removePage(0);
|
||||
|
||||
FormUtils.pruneOrphanedFormFields(document);
|
||||
|
||||
PDAcroForm form = document.getDocumentCatalog().getAcroForm(null);
|
||||
assertNotNull(form);
|
||||
List<String> remainingNames = new ArrayList<>();
|
||||
for (PDField field : form.getFields()) {
|
||||
remainingNames.add(field.getPartialName());
|
||||
}
|
||||
assertEquals(List.of("field_1"), remainingNames);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void dropsAcroFormEntirelyWhenNoFieldsSurvive() throws IOException {
|
||||
byte[] pdfBytes = buildPdfWithFieldPerPage(2);
|
||||
|
||||
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
|
||||
document.removePage(1);
|
||||
document.removePage(0);
|
||||
document.addPage(new PDPage(PDRectangle.A4));
|
||||
|
||||
FormUtils.pruneOrphanedFormFields(document);
|
||||
|
||||
assertNull(document.getDocumentCatalog().getAcroForm(null));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void keepsLiveWidgetsAndDropsOrphanWidgetsFromMultiWidgetField() throws IOException {
|
||||
byte[] pdfBytes = buildPdfWithMultiWidgetField();
|
||||
|
||||
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
|
||||
document.removePage(0);
|
||||
|
||||
FormUtils.pruneOrphanedFormFields(document);
|
||||
|
||||
PDAcroForm form = document.getDocumentCatalog().getAcroForm(null);
|
||||
assertNotNull(form);
|
||||
assertEquals(1, form.getFields().size());
|
||||
PDField field = form.getFields().get(0);
|
||||
assertEquals("multi", field.getPartialName());
|
||||
assertEquals(2, field.getWidgets().size(), "two widgets remain after one is dropped");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void survivesRoundTripWithoutOrphanPagesInOutput() throws IOException {
|
||||
byte[] pdfBytes = buildPdfWithFieldPerPage(3);
|
||||
|
||||
byte[] writtenBytes;
|
||||
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
|
||||
document.removePage(2);
|
||||
document.removePage(1);
|
||||
FormUtils.pruneOrphanedFormFields(document);
|
||||
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||
document.save(out);
|
||||
writtenBytes = out.toByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
try (PDDocument reloaded = Loader.loadPDF(writtenBytes)) {
|
||||
assertEquals(1, reloaded.getNumberOfPages());
|
||||
PDAcroForm form = reloaded.getDocumentCatalog().getAcroForm(null);
|
||||
assertNotNull(form);
|
||||
assertEquals(1, form.getFields().size());
|
||||
assertEquals("field_0", form.getFields().get(0).getPartialName());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void prunesNestedNonTerminalFields() throws IOException {
|
||||
byte[] pdfBytes = buildPdfWithNestedFields();
|
||||
|
||||
try (PDDocument document = Loader.loadPDF(pdfBytes)) {
|
||||
document.removePage(1);
|
||||
|
||||
FormUtils.pruneOrphanedFormFields(document);
|
||||
|
||||
PDAcroForm form = document.getDocumentCatalog().getAcroForm(null);
|
||||
assertNotNull(form);
|
||||
assertEquals(1, form.getFields().size());
|
||||
PDField group = form.getFields().get(0);
|
||||
assertEquals("group", group.getPartialName());
|
||||
assertTrue(group instanceof PDNonTerminalField);
|
||||
PDNonTerminalField nonTerminal = (PDNonTerminalField) group;
|
||||
assertEquals(1, nonTerminal.getChildren().size());
|
||||
assertEquals("kept", nonTerminal.getChildren().get(0).getPartialName());
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] buildPdfWithFieldPerPage(int pageCount) throws IOException {
|
||||
try (PDDocument document = new PDDocument()) {
|
||||
PDAcroForm acroForm = new PDAcroForm(document);
|
||||
acroForm.setDefaultResources(new PDResources());
|
||||
document.getDocumentCatalog().setAcroForm(acroForm);
|
||||
|
||||
for (int i = 0; i < pageCount; i++) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
document.addPage(page);
|
||||
|
||||
PDTextField field = new PDTextField(acroForm);
|
||||
field.setPartialName("field_" + i);
|
||||
PDAnnotationWidget widget = new PDAnnotationWidget();
|
||||
widget.setRectangle(new PDRectangle(50, 50, 100, 20));
|
||||
widget.setPage(page);
|
||||
field.setWidgets(List.of(widget));
|
||||
acroForm.getFields().add(field);
|
||||
page.getAnnotations().add(widget);
|
||||
}
|
||||
|
||||
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||
document.save(out);
|
||||
return out.toByteArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] buildPdfWithMultiWidgetField() throws IOException {
|
||||
try (PDDocument document = new PDDocument()) {
|
||||
PDAcroForm acroForm = new PDAcroForm(document);
|
||||
acroForm.setDefaultResources(new PDResources());
|
||||
document.getDocumentCatalog().setAcroForm(acroForm);
|
||||
|
||||
List<PDAnnotationWidget> widgets = new ArrayList<>();
|
||||
for (int i = 0; i < 3; i++) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
document.addPage(page);
|
||||
PDAnnotationWidget widget = new PDAnnotationWidget();
|
||||
widget.setRectangle(new PDRectangle(50, 50, 100, 20));
|
||||
widget.setPage(page);
|
||||
page.getAnnotations().add(widget);
|
||||
widgets.add(widget);
|
||||
}
|
||||
|
||||
PDTextField field = new PDTextField(acroForm);
|
||||
field.setPartialName("multi");
|
||||
field.setWidgets(widgets);
|
||||
acroForm.getFields().add(field);
|
||||
|
||||
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||
document.save(out);
|
||||
return out.toByteArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] buildPdfWithNestedFields() throws IOException {
|
||||
try (PDDocument document = new PDDocument()) {
|
||||
PDAcroForm acroForm = new PDAcroForm(document);
|
||||
acroForm.setDefaultResources(new PDResources());
|
||||
document.getDocumentCatalog().setAcroForm(acroForm);
|
||||
|
||||
PDPage pageA = new PDPage(PDRectangle.A4);
|
||||
PDPage pageB = new PDPage(PDRectangle.A4);
|
||||
document.addPage(pageA);
|
||||
document.addPage(pageB);
|
||||
|
||||
PDNonTerminalField group = new PDNonTerminalField(acroForm);
|
||||
group.setPartialName("group");
|
||||
|
||||
PDTextField kept = new PDTextField(acroForm);
|
||||
kept.setPartialName("kept");
|
||||
PDAnnotationWidget keptWidget = new PDAnnotationWidget();
|
||||
keptWidget.setRectangle(new PDRectangle(50, 50, 100, 20));
|
||||
keptWidget.setPage(pageA);
|
||||
kept.setWidgets(List.of(keptWidget));
|
||||
pageA.getAnnotations().add(keptWidget);
|
||||
|
||||
PDTextField dropped = new PDTextField(acroForm);
|
||||
dropped.setPartialName("dropped");
|
||||
PDAnnotationWidget droppedWidget = new PDAnnotationWidget();
|
||||
droppedWidget.setRectangle(new PDRectangle(50, 100, 100, 20));
|
||||
droppedWidget.setPage(pageB);
|
||||
dropped.setWidgets(List.of(droppedWidget));
|
||||
pageB.getAnnotations().add(droppedWidget);
|
||||
|
||||
group.setChildren(List.of(kept, dropped));
|
||||
acroForm.getFields().add(group);
|
||||
|
||||
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||
document.save(out);
|
||||
return out.toByteArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -173,6 +174,70 @@ public class UnoServerPoolTest {
|
||||
assertEquals(port1, port2, "Should reuse the same endpoint after release");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAcquireWithTimeoutFailsFast() throws InterruptedException {
|
||||
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints =
|
||||
createEndpoints(1);
|
||||
UnoServerPool pool = new UnoServerPool(endpoints);
|
||||
|
||||
UnoServerPool.UnoServerLease held = pool.acquireEndpoint();
|
||||
|
||||
long start = System.nanoTime();
|
||||
assertThrows(
|
||||
TimeoutException.class,
|
||||
() -> pool.acquireEndpoint(100, TimeUnit.MILLISECONDS),
|
||||
"Should time out when no endpoint available");
|
||||
long elapsedMs = (System.nanoTime() - start) / 1_000_000;
|
||||
|
||||
assertTrue(
|
||||
elapsedMs >= 100 && elapsedMs < 1000,
|
||||
"Should fail-fast after roughly the timeout, got " + elapsedMs + "ms");
|
||||
|
||||
held.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAcquireWithTimeoutSucceedsWhenAvailable()
|
||||
throws InterruptedException, TimeoutException {
|
||||
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints =
|
||||
createEndpoints(1);
|
||||
UnoServerPool pool = new UnoServerPool(endpoints);
|
||||
|
||||
try (UnoServerPool.UnoServerLease lease = pool.acquireEndpoint(1, TimeUnit.SECONDS)) {
|
||||
assertNotNull(lease);
|
||||
assertEquals(2003, lease.getEndpoint().getPort());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAcquireWithZeroTimeoutBlocksUnbounded() throws InterruptedException {
|
||||
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints =
|
||||
createEndpoints(1);
|
||||
UnoServerPool pool = new UnoServerPool(endpoints);
|
||||
UnoServerPool.UnoServerLease held = pool.acquireEndpoint();
|
||||
|
||||
AtomicInteger acquired = new AtomicInteger(0);
|
||||
Thread t =
|
||||
Thread.ofVirtual()
|
||||
.start(
|
||||
() -> {
|
||||
try {
|
||||
UnoServerPool.UnoServerLease lease =
|
||||
pool.acquireEndpoint(0, TimeUnit.MILLISECONDS);
|
||||
acquired.incrementAndGet();
|
||||
lease.close();
|
||||
} catch (Exception e) {
|
||||
fail("unexpected: " + e);
|
||||
}
|
||||
});
|
||||
|
||||
Thread.sleep(150);
|
||||
assertEquals(0, acquired.get(), "Should still be blocked");
|
||||
held.close();
|
||||
t.join(1000);
|
||||
assertEquals(1, acquired.get(), "Should acquire after release");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testHostLocationAndProtocol() throws InterruptedException {
|
||||
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints = new ArrayList<>();
|
||||
|
||||
+2
-2
@@ -11,11 +11,11 @@ import stirling.software.common.model.api.security.RedactionArea;
|
||||
|
||||
class StringToArrayListPropertyEditorTest {
|
||||
|
||||
private StringToArrayListPropertyEditor editor;
|
||||
private StringToArrayListPropertyEditor<RedactionArea> editor;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
editor = new StringToArrayListPropertyEditor();
|
||||
editor = new StringToArrayListPropertyEditor<>(RedactionArea.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -65,7 +65,7 @@ dependencies {
|
||||
exclude group: 'com.fasterxml.jackson.jaxrs'
|
||||
exclude group: 'com.fasterxml.jackson.module', module: 'jackson-module-jaxb-annotations'
|
||||
}
|
||||
implementation 'commons-io:commons-io:2.21.0'
|
||||
implementation 'commons-io:commons-io:2.22.0'
|
||||
implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion"
|
||||
implementation "org.bouncycastle:bcpkix-jdk18on:$bouncycastleVersion"
|
||||
implementation 'io.micrometer:micrometer-core'
|
||||
@@ -87,12 +87,6 @@ dependencies {
|
||||
implementation 'com.sun.xml.bind:jaxb-core:4.0.7'
|
||||
implementation 'org.apache.poi:poi-ooxml:5.5.1'
|
||||
|
||||
// https://mvnrepository.com/artifact/technology.tabula/tabula
|
||||
implementation ('technology.tabula:tabula:1.0.5') {
|
||||
exclude group: 'org.slf4j', module: 'slf4j-simple'
|
||||
exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
|
||||
exclude group: 'com.google.code.gson', module: 'gson'
|
||||
}
|
||||
// CVE-2022-25647: Explicit gson 2.13.2 to prevent unsafe deserialization (tabula would pull 2.8.7)
|
||||
implementation 'com.google.code.gson:gson:2.13.2'
|
||||
implementation 'org.apache.pdfbox:jbig2-imageio:3.0.4'
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
package stirling.software.SPDF.controller.api;
|
||||
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.WebDataBinder;
|
||||
import org.springframework.web.bind.annotation.InitBinder;
|
||||
import org.springframework.web.bind.annotation.ModelAttribute;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import io.github.pixee.security.Filenames;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.config.swagger.StandardPdfResponse;
|
||||
import stirling.software.SPDF.model.api.general.EditTextRequest;
|
||||
import stirling.software.SPDF.model.json.PdfJsonDocument;
|
||||
import stirling.software.SPDF.model.json.PdfJsonPage;
|
||||
import stirling.software.SPDF.model.json.PdfJsonTextElement;
|
||||
import stirling.software.SPDF.service.PdfJsonConversionService;
|
||||
import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.GeneralApi;
|
||||
import stirling.software.common.model.api.general.EditTextOperation;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.common.util.propertyeditor.StringToArrayListPropertyEditor;
|
||||
|
||||
/**
|
||||
* Find/replace text editing for PDFs. Round-trips through {@link PdfJsonConversionService}: the
|
||||
* input PDF is parsed into the editable JSON model, find/replace operations are applied to the
|
||||
* {@code text} field of each text element, and the mutated model is rebuilt into a PDF.
|
||||
*
|
||||
* <p>Matching joins all text elements on a page into a single string before searching, so find
|
||||
* strings can span multiple visual runs (titles split per word, kerning-broken phrases, etc.).
|
||||
*
|
||||
* <p>For cross-element matches the entire replacement is written into the first matched element,
|
||||
* any intermediate elements are emptied, and the suffix of the last matched element is preserved.
|
||||
* This anchors the new text at the leftmost match position and lets the font lay it out as one run,
|
||||
* which is more visually reliable than redistributing words to the original per-element X positions
|
||||
* (those positions are calibrated to the original word widths and rarely match the replacement
|
||||
* widths). Centered or tracked layouts will become left-aligned at the original first-word position
|
||||
* when their text is replaced - the trade-off is accepted in favour of avoiding glyph overlaps and
|
||||
* gaps from variable-width word distribution.
|
||||
*/
|
||||
@Slf4j
|
||||
@GeneralApi
|
||||
@RequiredArgsConstructor
|
||||
public class EditTextController {
|
||||
|
||||
private static final Pattern FILE_EXTENSION_PATTERN = Pattern.compile("[.][^.]+$");
|
||||
|
||||
private final PdfJsonConversionService pdfJsonConversionService;
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
@InitBinder
|
||||
public void initBinder(WebDataBinder binder) {
|
||||
binder.registerCustomEditor(
|
||||
List.class,
|
||||
"edits",
|
||||
new StringToArrayListPropertyEditor<>(EditTextOperation.class));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(consumes = "multipart/form-data", value = "/edit-text")
|
||||
@StandardPdfResponse
|
||||
@Operation(
|
||||
summary = "Edit text in a PDF via find and replace",
|
||||
description =
|
||||
"Applies an ordered list of find/replace operations to the text in a PDF and"
|
||||
+ " returns the edited PDF. Useful for find-and-replace, bulk renames"
|
||||
+ " (e.g. updating a company name throughout a document), and copy"
|
||||
+ " editing where the AI agent has identified specific replacements."
|
||||
+ " Matching is performed against the joined text of each page, so"
|
||||
+ " find strings can span multiple visual runs (titles split per word,"
|
||||
+ " kerning-broken phrases). Cross-element matches are written as a"
|
||||
+ " single replacement run anchored at the leftmost matched position;"
|
||||
+ " centered or tracked text may shift left when its content changes."
|
||||
+ " Input:PDF Output:PDF Type:SISO")
|
||||
public ResponseEntity<Resource> editText(@ModelAttribute EditTextRequest request)
|
||||
throws Exception {
|
||||
MultipartFile inputFile = request.getFileInput();
|
||||
if (inputFile == null) {
|
||||
throw ExceptionUtils.createFileNullOrEmptyException();
|
||||
}
|
||||
List<EditTextOperation> edits = request.getEdits();
|
||||
if (edits == null || edits.isEmpty()) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.editText.no.edits",
|
||||
"No find/replace operations provided for text editing");
|
||||
}
|
||||
for (EditTextOperation edit : edits) {
|
||||
if (edit == null || edit.getFind() == null || edit.getFind().isEmpty()) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.editText.empty.find", "Each edit must have a non-empty find string");
|
||||
}
|
||||
}
|
||||
|
||||
boolean wholeWordSearch = Boolean.TRUE.equals(request.getWholeWordSearch());
|
||||
List<CompiledEdit> compiledEdits = compileEdits(edits, wholeWordSearch);
|
||||
|
||||
PdfJsonDocument document = pdfJsonConversionService.convertPdfToJsonDocument(inputFile);
|
||||
Set<Integer> pageFilter = resolvePageFilter(request, document);
|
||||
int modifiedSpans = applyEdits(document, compiledEdits, pageFilter);
|
||||
log.info(
|
||||
"edit-text: modified {} text span(s) using {} edit(s) on {} page(s)",
|
||||
modifiedSpans,
|
||||
compiledEdits.size(),
|
||||
pageFilter == null
|
||||
? document.getPages() == null ? 0 : document.getPages().size()
|
||||
: pageFilter.size());
|
||||
|
||||
String docName = buildOutputFilename(inputFile);
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try (OutputStream os = Files.newOutputStream(tempOut.getPath())) {
|
||||
pdfJsonConversionService.convertJsonToPdf(document, os);
|
||||
} catch (Exception e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
return WebResponseUtils.pdfFileToWebResponse(tempOut, docName);
|
||||
}
|
||||
|
||||
private List<CompiledEdit> compileEdits(
|
||||
List<EditTextOperation> edits, boolean wholeWordSearch) {
|
||||
return edits.stream().map(edit -> compileEdit(edit, wholeWordSearch)).toList();
|
||||
}
|
||||
|
||||
private CompiledEdit compileEdit(EditTextOperation edit, boolean wholeWordSearch) {
|
||||
// Always treat the user-supplied find string as a literal: Pattern.quote escapes any
|
||||
// regex metacharacters, so the constructed pattern can only ever do a literal match
|
||||
// (optionally bounded by our own word-boundary anchors). This rules out catastrophic
|
||||
// backtracking from a malicious find string.
|
||||
String findRaw = edit.getFind();
|
||||
String replacement = Objects.toString(edit.getReplace(), "");
|
||||
String regex = Pattern.quote(findRaw);
|
||||
if (wholeWordSearch) {
|
||||
// Use lookarounds rather than \b so the bound works even when the find string starts
|
||||
// or ends with a non-word character (e.g. "-foo" or "$id"). \b only fires at a
|
||||
// word/non-word transition, which would never trigger for a find that starts with a
|
||||
// non-word char and was preceded by another non-word char in the source text.
|
||||
regex = "(?<!\\w)(?:" + regex + ")(?!\\w)";
|
||||
}
|
||||
Pattern pattern = Pattern.compile(regex);
|
||||
String safeReplacement = Matcher.quoteReplacement(replacement);
|
||||
return new CompiledEdit(pattern, safeReplacement);
|
||||
}
|
||||
|
||||
private Set<Integer> resolvePageFilter(EditTextRequest request, PdfJsonDocument document) {
|
||||
String pageNumbers = request.getPageNumbers();
|
||||
int totalPages = document.getPages() == null ? 0 : document.getPages().size();
|
||||
if (totalPages == 0) {
|
||||
return Collections.emptySet();
|
||||
}
|
||||
if (pageNumbers == null || pageNumbers.isBlank() || "all".equalsIgnoreCase(pageNumbers)) {
|
||||
return null;
|
||||
}
|
||||
List<Integer> pages = GeneralUtils.parsePageList(pageNumbers, totalPages, true);
|
||||
return new HashSet<>(pages);
|
||||
}
|
||||
|
||||
private int applyEdits(
|
||||
PdfJsonDocument document, List<CompiledEdit> edits, Set<Integer> pageFilter) {
|
||||
List<PdfJsonPage> pages = document.getPages();
|
||||
if (pages == null) {
|
||||
return 0;
|
||||
}
|
||||
int modifiedSpans = 0;
|
||||
// The page filter is built from parsePageList against pages.size(), so it returns 1-based
|
||||
// positional page numbers. Match against the same positional numbering here rather than
|
||||
// the per-page pageNumber field, which can become non-sequential after split/merge round
|
||||
// trips and would misalign the filter.
|
||||
for (int i = 0; i < pages.size(); i++) {
|
||||
int positionalPageNumber = i + 1;
|
||||
if (pageFilter != null && !pageFilter.contains(positionalPageNumber)) {
|
||||
continue;
|
||||
}
|
||||
modifiedSpans += applyEditsToPage(pages.get(i), edits);
|
||||
}
|
||||
return modifiedSpans;
|
||||
}
|
||||
|
||||
private int applyEditsToPage(PdfJsonPage page, List<CompiledEdit> edits) {
|
||||
List<PdfJsonTextElement> elements = page.getTextElements();
|
||||
if (elements == null || elements.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
Set<Integer> modifiedIndices = new HashSet<>();
|
||||
for (CompiledEdit edit : edits) {
|
||||
applyEditToPage(elements, edit, modifiedIndices);
|
||||
}
|
||||
for (Integer index : modifiedIndices) {
|
||||
// Char codes were captured for the original glyph sequence; clear them so the rebuild
|
||||
// re-encodes from the new text via the font.
|
||||
elements.get(index).setCharCodes(null);
|
||||
}
|
||||
return modifiedIndices.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a single edit across the page by matching against the concatenation of all element
|
||||
* texts, then writing the replacement back into the originating element(s).
|
||||
*/
|
||||
private void applyEditToPage(
|
||||
List<PdfJsonTextElement> elements, CompiledEdit edit, Set<Integer> modifiedIndices) {
|
||||
StringBuilder joined = new StringBuilder();
|
||||
int[] starts = new int[elements.size()];
|
||||
int[] ends = new int[elements.size()];
|
||||
for (int i = 0; i < elements.size(); i++) {
|
||||
starts[i] = joined.length();
|
||||
String text = elements.get(i).getText();
|
||||
if (text != null) {
|
||||
joined.append(text);
|
||||
}
|
||||
ends[i] = joined.length();
|
||||
}
|
||||
|
||||
Matcher matcher = edit.pattern().matcher(joined);
|
||||
List<MatchSpan> spans = new ArrayList<>();
|
||||
StringBuffer interpolation = new StringBuffer();
|
||||
int previousAppendPosition = 0;
|
||||
while (matcher.find()) {
|
||||
if (matcher.start() == matcher.end()) {
|
||||
// Skip zero-length matches (e.g. /a*/ on empty input) — they cannot be applied.
|
||||
continue;
|
||||
}
|
||||
int sizeBefore = interpolation.length();
|
||||
matcher.appendReplacement(interpolation, edit.replacement());
|
||||
int prefixLength = matcher.start() - previousAppendPosition;
|
||||
String actualReplacement =
|
||||
interpolation.substring(sizeBefore + prefixLength, interpolation.length());
|
||||
spans.add(new MatchSpan(matcher.start(), matcher.end(), actualReplacement));
|
||||
previousAppendPosition = matcher.end();
|
||||
}
|
||||
|
||||
// Apply right-to-left so earlier match positions stay valid as we mutate elements.
|
||||
for (int i = spans.size() - 1; i >= 0; i--) {
|
||||
MatchSpan span = spans.get(i);
|
||||
int firstElement = findElementForCharIndex(starts, ends, span.start());
|
||||
int lastElement = findElementForCharIndex(starts, ends, span.end() - 1);
|
||||
if (firstElement < 0 || lastElement < 0) {
|
||||
continue;
|
||||
}
|
||||
applyMatchToElements(
|
||||
elements, starts, span, firstElement, lastElement, modifiedIndices);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the element whose text covers the character at {@code charIndex} in the joined string.
|
||||
* Returns -1 if no element covers that index (which should not happen for valid match spans).
|
||||
*/
|
||||
private static int findElementForCharIndex(int[] starts, int[] ends, int charIndex) {
|
||||
for (int i = 0; i < starts.length; i++) {
|
||||
if (starts[i] <= charIndex && charIndex < ends[i]) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private static void applyMatchToElements(
|
||||
List<PdfJsonTextElement> elements,
|
||||
int[] starts,
|
||||
MatchSpan span,
|
||||
int firstElement,
|
||||
int lastElement,
|
||||
Set<Integer> modifiedIndices) {
|
||||
if (firstElement == lastElement) {
|
||||
PdfJsonTextElement element = elements.get(firstElement);
|
||||
String text = nullToEmpty(element.getText());
|
||||
int matchStartInElement = span.start() - starts[firstElement];
|
||||
int matchEndInElement = span.end() - starts[firstElement];
|
||||
element.setText(
|
||||
text.substring(0, matchStartInElement)
|
||||
+ span.replacement()
|
||||
+ text.substring(matchEndInElement));
|
||||
modifiedIndices.add(firstElement);
|
||||
return;
|
||||
}
|
||||
|
||||
// Cross-element match: write the whole replacement into the first matched element, empty
|
||||
// any intermediate elements, and keep only the suffix of the last matched element. The
|
||||
// JSON->PDF rebuild concatenates per-token text, so the font lays out the replacement as
|
||||
// one continuous run anchored at the first element's X position.
|
||||
String firstText = nullToEmpty(elements.get(firstElement).getText());
|
||||
int firstSplit = span.start() - starts[firstElement];
|
||||
elements.get(firstElement).setText(firstText.substring(0, firstSplit) + span.replacement());
|
||||
modifiedIndices.add(firstElement);
|
||||
|
||||
for (int mid = firstElement + 1; mid < lastElement; mid++) {
|
||||
elements.get(mid).setText("");
|
||||
modifiedIndices.add(mid);
|
||||
}
|
||||
|
||||
String lastText = nullToEmpty(elements.get(lastElement).getText());
|
||||
int lastSplit = span.end() - starts[lastElement];
|
||||
elements.get(lastElement).setText(lastText.substring(lastSplit));
|
||||
modifiedIndices.add(lastElement);
|
||||
}
|
||||
|
||||
private static String nullToEmpty(String value) {
|
||||
return value != null ? value : "";
|
||||
}
|
||||
|
||||
private String buildOutputFilename(MultipartFile inputFile) {
|
||||
String originalName = inputFile.getOriginalFilename();
|
||||
String baseName =
|
||||
(originalName != null && !originalName.isBlank())
|
||||
? FILE_EXTENSION_PATTERN
|
||||
.matcher(Filenames.toSimpleFileName(originalName))
|
||||
.replaceFirst("")
|
||||
: "document";
|
||||
return baseName + "_edited.pdf";
|
||||
}
|
||||
|
||||
private record CompiledEdit(Pattern pattern, String replacement) {}
|
||||
|
||||
private record MatchSpan(int start, int end, String replacement) {}
|
||||
}
|
||||
+16
@@ -6,8 +6,11 @@ import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
@@ -27,6 +30,7 @@ import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.GeneralApi;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.FormUtils;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
@@ -67,6 +71,7 @@ public class RearrangePagesPDFController {
|
||||
int pageIndex = pagesToRemove.get(i);
|
||||
document.removePage(pageIndex);
|
||||
}
|
||||
FormUtils.pruneOrphanedFormFields(document);
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
document,
|
||||
GeneralUtils.generateFilename(
|
||||
@@ -265,6 +270,17 @@ public class RearrangePagesPDFController {
|
||||
rearrangedDocument.addPage(page);
|
||||
}
|
||||
|
||||
PDDocumentCatalog sourceCatalog = document.getDocumentCatalog();
|
||||
if (sourceCatalog != null) {
|
||||
PDAcroForm sourceForm = sourceCatalog.getAcroForm(null);
|
||||
if (sourceForm != null) {
|
||||
rearrangedDocument
|
||||
.getDocumentCatalog()
|
||||
.getCOSObject()
|
||||
.setItem(COSName.ACRO_FORM, sourceForm.getCOSObject());
|
||||
}
|
||||
}
|
||||
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
rearrangedDocument,
|
||||
GeneralUtils.generateFilename(
|
||||
|
||||
+82
-26
@@ -1,9 +1,13 @@
|
||||
package stirling.software.SPDF.controller.api;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
@@ -26,6 +30,7 @@ import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.GeneralApi;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.FormUtils;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
@@ -54,9 +59,21 @@ public class SplitPDFController {
|
||||
MultipartFile file = request.getFileInput();
|
||||
TempFile outputTempFile = new TempFile(tempFileManager, ".zip");
|
||||
try {
|
||||
try (PDDocument document = pdfDocumentFactory.load(file)) {
|
||||
int totalPages = document.getNumberOfPages();
|
||||
List<Integer> pageNumbers = request.getPageNumbersList(document, false);
|
||||
try (TempFile sourceTempFile = new TempFile(tempFileManager, ".pdf")) {
|
||||
Files.copy(
|
||||
file.getInputStream(),
|
||||
sourceTempFile.getPath(),
|
||||
StandardCopyOption.REPLACE_EXISTING);
|
||||
|
||||
int totalPages;
|
||||
List<Integer> pageNumbers;
|
||||
boolean hasForm;
|
||||
try (PDDocument document =
|
||||
pdfDocumentFactory.load(sourceTempFile.getFile(), true)) {
|
||||
totalPages = document.getNumberOfPages();
|
||||
pageNumbers = request.getPageNumbersList(document, false);
|
||||
hasForm = document.getDocumentCatalog().getAcroForm(null) != null;
|
||||
}
|
||||
if (!pageNumbers.contains(totalPages - 1)) {
|
||||
pageNumbers = new ArrayList<>(pageNumbers);
|
||||
pageNumbers.add(totalPages - 1);
|
||||
@@ -69,33 +86,16 @@ public class SplitPDFController {
|
||||
String baseFilename = GeneralUtils.removeExtension(file.getOriginalFilename());
|
||||
try (ZipOutputStream zipOut =
|
||||
new ZipOutputStream(Files.newOutputStream(outputTempFile.getPath()))) {
|
||||
int previousPageNumber = 0;
|
||||
for (int splitIndex = 0; splitIndex < pageNumbers.size(); splitIndex++) {
|
||||
int splitPoint = pageNumbers.get(splitIndex);
|
||||
try (PDDocument splitDocument =
|
||||
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(document)) {
|
||||
for (int i = previousPageNumber; i <= splitPoint; i++) {
|
||||
splitDocument.addPage(document.getPage(i));
|
||||
log.debug("Adding page {} to split document", i);
|
||||
}
|
||||
previousPageNumber = splitPoint + 1;
|
||||
|
||||
String fileName = baseFilename + "_" + (splitIndex + 1) + ".pdf";
|
||||
zipOut.putNextEntry(new ZipEntry(fileName));
|
||||
splitDocument.save(zipOut);
|
||||
zipOut.closeEntry();
|
||||
log.debug("Wrote split document {} to zip file", fileName);
|
||||
} catch (Exception e) {
|
||||
ExceptionUtils.logException("document splitting and saving", e);
|
||||
throw e;
|
||||
}
|
||||
if (hasForm) {
|
||||
writeSplitsViaReload(
|
||||
sourceTempFile.getFile(), pageNumbers, baseFilename, zipOut);
|
||||
} else {
|
||||
writeSplitsViaSharedSource(
|
||||
sourceTempFile.getFile(), pageNumbers, baseFilename, zipOut);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.debug(
|
||||
"Successfully created zip file with split documents: {}",
|
||||
outputTempFile.getPath().toString());
|
||||
String zipFilename =
|
||||
GeneralUtils.generateFilename(file.getOriginalFilename(), "_split.zip");
|
||||
return WebResponseUtils.zipFileToWebResponse(outputTempFile, zipFilename);
|
||||
@@ -104,4 +104,60 @@ public class SplitPDFController {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private void writeSplitsViaReload(
|
||||
File source, List<Integer> pageNumbers, String baseFilename, ZipOutputStream zipOut)
|
||||
throws IOException {
|
||||
int previousPageNumber = 0;
|
||||
for (int splitIndex = 0; splitIndex < pageNumbers.size(); splitIndex++) {
|
||||
int splitPoint = pageNumbers.get(splitIndex);
|
||||
Set<Integer> keep = new HashSet<>();
|
||||
for (int i = previousPageNumber; i <= splitPoint; i++) {
|
||||
keep.add(i);
|
||||
}
|
||||
previousPageNumber = splitPoint + 1;
|
||||
|
||||
try (PDDocument splitDoc = pdfDocumentFactory.load(source)) {
|
||||
for (int p = splitDoc.getNumberOfPages() - 1; p >= 0; p--) {
|
||||
if (!keep.contains(p)) {
|
||||
splitDoc.removePage(p);
|
||||
}
|
||||
}
|
||||
FormUtils.pruneOrphanedFormFields(splitDoc);
|
||||
writeEntry(zipOut, baseFilename, splitIndex + 1, splitDoc);
|
||||
} catch (Exception e) {
|
||||
ExceptionUtils.logException("document splitting and saving", e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void writeSplitsViaSharedSource(
|
||||
File source, List<Integer> pageNumbers, String baseFilename, ZipOutputStream zipOut)
|
||||
throws IOException {
|
||||
try (PDDocument sourceDoc = pdfDocumentFactory.load(source)) {
|
||||
int previousPageNumber = 0;
|
||||
for (int splitIndex = 0; splitIndex < pageNumbers.size(); splitIndex++) {
|
||||
int splitPoint = pageNumbers.get(splitIndex);
|
||||
try (PDDocument splitDoc =
|
||||
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDoc)) {
|
||||
for (int i = previousPageNumber; i <= splitPoint; i++) {
|
||||
splitDoc.addPage(sourceDoc.getPage(i));
|
||||
}
|
||||
previousPageNumber = splitPoint + 1;
|
||||
writeEntry(zipOut, baseFilename, splitIndex + 1, splitDoc);
|
||||
} catch (Exception e) {
|
||||
ExceptionUtils.logException("document splitting and saving", e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void writeEntry(ZipOutputStream zipOut, String baseFilename, int index, PDDocument doc)
|
||||
throws IOException {
|
||||
zipOut.putNextEntry(new ZipEntry(baseFilename + "_" + index + ".pdf"));
|
||||
doc.save(zipOut);
|
||||
zipOut.closeEntry();
|
||||
}
|
||||
}
|
||||
|
||||
+196
-391
@@ -1,8 +1,14 @@
|
||||
package stirling.software.SPDF.controller.api;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
@@ -16,7 +22,6 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@@ -26,6 +31,7 @@ import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.GeneralApi;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.FormUtils;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
@@ -54,52 +60,40 @@ public class SplitPdfBySizeController {
|
||||
public ResponseEntity<Resource> autoSplitPdf(
|
||||
@ModelAttribute SplitPdfBySizeOrCountRequest request) throws Exception {
|
||||
|
||||
log.debug("Starting PDF split process with request: {}", request);
|
||||
MultipartFile file = request.getFileInput();
|
||||
|
||||
String filename = GeneralUtils.generateFilename(file.getOriginalFilename(), "");
|
||||
log.debug("Base filename for output: {}", filename);
|
||||
|
||||
TempFile zipTempFile = new TempFile(tempFileManager, ".zip");
|
||||
try {
|
||||
log.debug("Created temporary managed zip file: {}", zipTempFile.getPath());
|
||||
log.debug("Creating ZIP output stream");
|
||||
try (ZipOutputStream zipOut =
|
||||
new ZipOutputStream(Files.newOutputStream(zipTempFile.getPath()));
|
||||
PDDocument sourceDocument = pdfDocumentFactory.load(file)) {
|
||||
log.debug(
|
||||
"Successfully loaded PDF with {} pages", sourceDocument.getNumberOfPages());
|
||||
try (TempFile sourceTempFile = new TempFile(tempFileManager, ".pdf");
|
||||
ZipOutputStream zipOut =
|
||||
new ZipOutputStream(Files.newOutputStream(zipTempFile.getPath()))) {
|
||||
Files.copy(
|
||||
file.getInputStream(),
|
||||
sourceTempFile.getPath(),
|
||||
StandardCopyOption.REPLACE_EXISTING);
|
||||
|
||||
int type = request.getSplitType();
|
||||
String value = request.getSplitValue();
|
||||
log.debug("Split type: {}, Split value: {}", type, value);
|
||||
try (PDDocument sourceDocument =
|
||||
pdfDocumentFactory.load(sourceTempFile.getFile(), true)) {
|
||||
boolean hasForm = sourceDocument.getDocumentCatalog().getAcroForm(null) != null;
|
||||
List<List<Integer>> ranges = computeRanges(request, sourceDocument);
|
||||
|
||||
if (type == 0) {
|
||||
log.debug("Processing split by size");
|
||||
long maxBytes = GeneralUtils.convertSizeToBytes(value);
|
||||
log.debug("Max bytes per document: {}", maxBytes);
|
||||
handleSplitBySize(sourceDocument, maxBytes, zipOut, filename);
|
||||
} else if (type == 1) {
|
||||
log.debug("Processing split by page count");
|
||||
int pageCount = Integer.parseInt(value);
|
||||
log.debug("Pages per document: {}", pageCount);
|
||||
handleSplitByPageCount(sourceDocument, pageCount, zipOut, filename);
|
||||
} else if (type == 2) {
|
||||
log.debug("Processing split by document count");
|
||||
int documentCount = Integer.parseInt(value);
|
||||
log.debug("Total number of documents: {}", documentCount);
|
||||
handleSplitByDocCount(sourceDocument, documentCount, zipOut, filename);
|
||||
} else {
|
||||
log.error("Invalid split type: {}", type);
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.invalidArgument",
|
||||
"Invalid argument: {0}",
|
||||
"split type: " + type);
|
||||
int fileIndex = 1;
|
||||
for (List<Integer> range : ranges) {
|
||||
if (range.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (hasForm) {
|
||||
writeRangeViaReload(
|
||||
sourceTempFile.getFile(), range, zipOut, filename, fileIndex++);
|
||||
} else {
|
||||
writeRangeViaSharedSource(
|
||||
sourceDocument, range, zipOut, filename, fileIndex++);
|
||||
}
|
||||
}
|
||||
}
|
||||
log.debug("PDF splitting completed successfully");
|
||||
}
|
||||
|
||||
log.debug("Returning streaming response for zip file");
|
||||
return WebResponseUtils.zipFileToWebResponse(zipTempFile, filename + ".zip");
|
||||
} catch (Exception e) {
|
||||
ExceptionUtils.logException("PDF splitting process", e);
|
||||
@@ -108,387 +102,198 @@ public class SplitPdfBySizeController {
|
||||
}
|
||||
}
|
||||
|
||||
private void handleSplitBySize(
|
||||
PDDocument sourceDocument, long maxBytes, ZipOutputStream zipOut, String baseFilename)
|
||||
throws IOException {
|
||||
log.debug("Starting handleSplitBySize with maxBytes={}", maxBytes);
|
||||
|
||||
@Getter
|
||||
class DocHolder implements AutoCloseable {
|
||||
private PDDocument doc;
|
||||
|
||||
public DocHolder(PDDocument doc) {
|
||||
this.doc = doc;
|
||||
}
|
||||
|
||||
public void setDoc(PDDocument doc) {
|
||||
if (this.doc != null) {
|
||||
try {
|
||||
this.doc.close();
|
||||
} catch (IOException e) {
|
||||
log.error("Error closing document", e);
|
||||
}
|
||||
}
|
||||
this.doc = doc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (doc != null) {
|
||||
doc.close();
|
||||
}
|
||||
}
|
||||
private List<List<Integer>> computeRanges(
|
||||
SplitPdfBySizeOrCountRequest request, PDDocument sourceDocument) throws IOException {
|
||||
int type = request.getSplitType();
|
||||
String value = request.getSplitValue();
|
||||
if (type == 0) {
|
||||
return computeSizeRanges(sourceDocument, GeneralUtils.convertSizeToBytes(value));
|
||||
} else if (type == 1) {
|
||||
return computePageCountRanges(sourceDocument, Integer.parseInt(value));
|
||||
} else if (type == 2) {
|
||||
return computeDocCountRanges(sourceDocument, Integer.parseInt(value));
|
||||
}
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.invalidArgument", "Invalid argument: {0}", "split type: " + type);
|
||||
}
|
||||
|
||||
int fileIndex = 1;
|
||||
try (DocHolder holder =
|
||||
new DocHolder(
|
||||
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument))) {
|
||||
int totalPages = sourceDocument.getNumberOfPages();
|
||||
int pageAdded = 0;
|
||||
private void writeRangeViaReload(
|
||||
File sourceFile,
|
||||
List<Integer> keepIndices,
|
||||
ZipOutputStream zipOut,
|
||||
String baseFilename,
|
||||
int fileIndex)
|
||||
throws IOException {
|
||||
Set<Integer> keep = new HashSet<>(keepIndices);
|
||||
try (PDDocument doc = pdfDocumentFactory.load(sourceFile)) {
|
||||
for (int i = doc.getNumberOfPages() - 1; i >= 0; i--) {
|
||||
if (!keep.contains(i)) {
|
||||
doc.removePage(i);
|
||||
}
|
||||
}
|
||||
FormUtils.pruneOrphanedFormFields(doc);
|
||||
writeEntry(zipOut, baseFilename, fileIndex, doc);
|
||||
}
|
||||
}
|
||||
|
||||
// Smart size check frequency - check more often with larger documents
|
||||
int baseCheckFrequency = 5;
|
||||
private void writeRangeViaSharedSource(
|
||||
PDDocument sourceDocument,
|
||||
List<Integer> keepIndices,
|
||||
ZipOutputStream zipOut,
|
||||
String baseFilename,
|
||||
int fileIndex)
|
||||
throws IOException {
|
||||
try (PDDocument doc =
|
||||
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument)) {
|
||||
for (int p : keepIndices) {
|
||||
doc.addPage(sourceDocument.getPage(p));
|
||||
}
|
||||
writeEntry(zipOut, baseFilename, fileIndex, doc);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeEntry(
|
||||
ZipOutputStream zipOut, String baseFilename, int fileIndex, PDDocument doc)
|
||||
throws IOException {
|
||||
zipOut.putNextEntry(new ZipEntry(baseFilename + "_" + fileIndex + ".pdf"));
|
||||
doc.save(zipOut);
|
||||
zipOut.closeEntry();
|
||||
}
|
||||
|
||||
/** Page-index ranges each output should contain. AcroForm overhead isn't modeled. */
|
||||
private List<List<Integer>> computeSizeRanges(PDDocument sourceDocument, long maxBytes)
|
||||
throws IOException {
|
||||
List<List<Integer>> ranges = new ArrayList<>();
|
||||
List<Integer> currentRange = new ArrayList<>();
|
||||
int totalPages = sourceDocument.getNumberOfPages();
|
||||
int baseCheckFrequency = 5;
|
||||
|
||||
PDDocument scratch = new PDDocument();
|
||||
try {
|
||||
for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
|
||||
PDPage page = sourceDocument.getPage(pageIndex);
|
||||
log.debug("Processing page {} of {}", pageIndex + 1, totalPages);
|
||||
scratch.addPage(new PDPage(page.getCOSObject()));
|
||||
currentRange.add(pageIndex);
|
||||
|
||||
// Add the page to current document
|
||||
PDPage newPage = new PDPage(page.getCOSObject());
|
||||
holder.getDoc().addPage(newPage);
|
||||
pageAdded++;
|
||||
|
||||
// Dynamic size checking based on document size and page count
|
||||
int pageAdded = currentRange.size();
|
||||
boolean shouldCheckSize =
|
||||
(pageAdded % baseCheckFrequency == 0)
|
||||
|| (pageIndex == totalPages - 1)
|
||||
|| (pageAdded >= 20); // Always check after 20 pages
|
||||
|| (pageAdded >= 20);
|
||||
if (!shouldCheckSize) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (shouldCheckSize) {
|
||||
log.debug("Performing size check after {} pages", pageAdded);
|
||||
long actualSize;
|
||||
try (ByteArrayOutputStream checkSizeStream = new ByteArrayOutputStream()) {
|
||||
holder.getDoc().save(checkSizeStream);
|
||||
actualSize = checkSizeStream.size();
|
||||
long actualSize;
|
||||
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||
scratch.save(out);
|
||||
actualSize = out.size();
|
||||
}
|
||||
|
||||
if (actualSize > maxBytes) {
|
||||
if (scratch.getNumberOfPages() > 1) {
|
||||
scratch.removePage(scratch.getNumberOfPages() - 1);
|
||||
currentRange.remove(currentRange.size() - 1);
|
||||
pageIndex--; // retry this page in the next chunk
|
||||
}
|
||||
log.debug(
|
||||
"Current document size: {} bytes (max: {} bytes)",
|
||||
actualSize,
|
||||
maxBytes);
|
||||
|
||||
if (actualSize > maxBytes) {
|
||||
// We exceeded the limit - remove the last page and save
|
||||
if (holder.getDoc().getNumberOfPages() > 1) {
|
||||
holder.getDoc().removePage(holder.getDoc().getNumberOfPages() - 1);
|
||||
pageIndex--; // Process this page again in the next document
|
||||
log.debug("Size limit exceeded - removed last page");
|
||||
}
|
||||
|
||||
log.debug(
|
||||
"Saving document with {} pages as part {}",
|
||||
holder.getDoc().getNumberOfPages(),
|
||||
fileIndex);
|
||||
saveDocumentToZip(holder.getDoc(), zipOut, baseFilename, fileIndex++);
|
||||
holder.setDoc(new PDDocument());
|
||||
pageAdded = 0;
|
||||
} else if (pageIndex < totalPages - 1) {
|
||||
// We're under the limit, calculate if we might fit more pages
|
||||
// Try to predict how many more similar pages might fit
|
||||
if (actualSize < maxBytes * 0.75 && pageAdded > 0) {
|
||||
// Rather than using a ratio, look ahead to test actual upcoming pages
|
||||
int pagesToLookAhead = Math.min(5, totalPages - pageIndex - 1);
|
||||
|
||||
if (pagesToLookAhead > 0) {
|
||||
log.debug(
|
||||
"Testing {} upcoming pages for potential addition",
|
||||
pagesToLookAhead);
|
||||
|
||||
// Create a temp document with current pages + look-ahead pages
|
||||
try (PDDocument testDoc = new PDDocument()) {
|
||||
// First copy existing pages
|
||||
for (int i = 0; i < holder.getDoc().getNumberOfPages(); i++) {
|
||||
testDoc.addPage(
|
||||
new PDPage(
|
||||
holder.getDoc().getPage(i).getCOSObject()));
|
||||
}
|
||||
|
||||
// Try adding look-ahead pages one by one
|
||||
int extraPagesAdded = 0;
|
||||
for (int i = 0; i < pagesToLookAhead; i++) {
|
||||
int testPageIndex = pageIndex + 1 + i;
|
||||
PDPage testPage = sourceDocument.getPage(testPageIndex);
|
||||
testDoc.addPage(new PDPage(testPage.getCOSObject()));
|
||||
|
||||
// Check if we're still under size
|
||||
long testSize;
|
||||
try (ByteArrayOutputStream testStream =
|
||||
new ByteArrayOutputStream()) {
|
||||
testDoc.save(testStream);
|
||||
testSize = testStream.size();
|
||||
}
|
||||
|
||||
if (testSize <= maxBytes) {
|
||||
extraPagesAdded++;
|
||||
log.debug(
|
||||
"Test: Can add page {} (size would be {})",
|
||||
testPageIndex + 1,
|
||||
testSize);
|
||||
} else {
|
||||
log.debug(
|
||||
"Test: Cannot add page {} (size would be {})",
|
||||
testPageIndex + 1,
|
||||
testSize);
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Add the pages we verified would fit
|
||||
if (extraPagesAdded > 0) {
|
||||
log.debug(
|
||||
"Adding {} verified pages ahead", extraPagesAdded);
|
||||
for (int i = 0; i < extraPagesAdded; i++) {
|
||||
int extraPageIndex = pageIndex + 1 + i;
|
||||
PDPage extraPage =
|
||||
sourceDocument.getPage(extraPageIndex);
|
||||
holder.getDoc()
|
||||
.addPage(new PDPage(extraPage.getCOSObject()));
|
||||
}
|
||||
pageIndex += extraPagesAdded;
|
||||
pageAdded += extraPagesAdded;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ranges.add(new ArrayList<>(currentRange));
|
||||
currentRange.clear();
|
||||
scratch.close();
|
||||
scratch = new PDDocument();
|
||||
} else if (pageIndex < totalPages - 1 && actualSize < maxBytes * 0.75) {
|
||||
int extraPagesAdded =
|
||||
lookAheadFit(scratch, sourceDocument, pageIndex, maxBytes);
|
||||
for (int i = 0; i < extraPagesAdded; i++) {
|
||||
int extra = pageIndex + 1 + i;
|
||||
scratch.addPage(new PDPage(sourceDocument.getPage(extra).getCOSObject()));
|
||||
currentRange.add(extra);
|
||||
}
|
||||
pageIndex += extraPagesAdded;
|
||||
}
|
||||
}
|
||||
|
||||
// Save final document if it has any pages
|
||||
if (holder.getDoc() != null && holder.getDoc().getNumberOfPages() > 0) {
|
||||
log.debug(
|
||||
"Saving final document with {} pages as part {}",
|
||||
holder.getDoc().getNumberOfPages(),
|
||||
fileIndex);
|
||||
saveDocumentToZip(holder.getDoc(), zipOut, baseFilename, fileIndex++);
|
||||
holder.setDoc(null);
|
||||
}
|
||||
}
|
||||
|
||||
log.debug("Completed handleSplitBySize with {} document parts created", fileIndex - 1);
|
||||
}
|
||||
|
||||
private void handleSplitByPageCount(
|
||||
PDDocument sourceDocument, int pageCount, ZipOutputStream zipOut, String baseFilename)
|
||||
throws IOException {
|
||||
log.debug("Starting handleSplitByPageCount with pageCount={}", pageCount);
|
||||
int currentPageCount = 0;
|
||||
PDDocument currentDoc = null;
|
||||
int fileIndex = 1;
|
||||
|
||||
try {
|
||||
log.debug("Creating initial output document");
|
||||
try {
|
||||
currentDoc = pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument);
|
||||
log.debug("Successfully created initial output document");
|
||||
} catch (Exception e) {
|
||||
ExceptionUtils.logException("initial output document creation", e);
|
||||
throw ExceptionUtils.createFileProcessingException("split", e);
|
||||
}
|
||||
|
||||
int pageIndex = 0;
|
||||
int totalPages = sourceDocument.getNumberOfPages();
|
||||
log.debug("Processing {} pages", totalPages);
|
||||
|
||||
try {
|
||||
for (PDPage page : sourceDocument.getPages()) {
|
||||
pageIndex++;
|
||||
log.debug("Processing page {} of {}", pageIndex, totalPages);
|
||||
|
||||
try {
|
||||
log.debug("Adding page {} to current document", pageIndex);
|
||||
currentDoc.addPage(page);
|
||||
log.debug("Successfully added page {} to current document", pageIndex);
|
||||
} catch (Exception e) {
|
||||
log.error("Error adding page {} to current document", pageIndex, e);
|
||||
throw ExceptionUtils.createFileProcessingException("split", e);
|
||||
}
|
||||
|
||||
currentPageCount++;
|
||||
log.debug("Current page count: {}/{}", currentPageCount, pageCount);
|
||||
|
||||
if (currentPageCount == pageCount) {
|
||||
log.debug(
|
||||
"Reached target page count ({}), saving current document as part {}",
|
||||
pageCount,
|
||||
fileIndex);
|
||||
try {
|
||||
saveDocumentToZip(currentDoc, zipOut, baseFilename, fileIndex++);
|
||||
currentDoc = null; // Document is closed by saveDocumentToZip
|
||||
log.debug("Successfully saved document part {}", fileIndex - 1);
|
||||
} catch (Exception e) {
|
||||
log.error("Error saving document part {}", fileIndex - 1, e);
|
||||
throw e;
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug("Creating new document for next part");
|
||||
currentDoc = new PDDocument();
|
||||
log.debug("Successfully created new document");
|
||||
} catch (Exception e) {
|
||||
log.error("Error creating new document for next part", e);
|
||||
throw ExceptionUtils.createFileProcessingException("split", e);
|
||||
}
|
||||
|
||||
currentPageCount = 0;
|
||||
log.debug("Reset current page count to 0");
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error iterating through pages", e);
|
||||
throw ExceptionUtils.createFileProcessingException("split", e);
|
||||
}
|
||||
|
||||
// Add the last document if it contains any pages
|
||||
try {
|
||||
if (currentDoc != null && currentDoc.getPages().getCount() != 0) {
|
||||
log.debug(
|
||||
"Saving final document with {} pages as part {}",
|
||||
currentDoc.getPages().getCount(),
|
||||
fileIndex);
|
||||
try {
|
||||
saveDocumentToZip(currentDoc, zipOut, baseFilename, fileIndex++);
|
||||
currentDoc = null; // Document is closed by saveDocumentToZip
|
||||
log.debug("Successfully saved final document part {}", fileIndex - 1);
|
||||
} catch (Exception e) {
|
||||
log.error("Error saving final document part {}", fileIndex - 1, e);
|
||||
throw e;
|
||||
}
|
||||
} else {
|
||||
log.debug("Final document has no pages, skipping");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error checking or saving final document", e);
|
||||
throw ExceptionUtils.createFileProcessingException("split", e);
|
||||
if (!currentRange.isEmpty()) {
|
||||
ranges.add(new ArrayList<>(currentRange));
|
||||
}
|
||||
} finally {
|
||||
if (currentDoc != null) {
|
||||
try {
|
||||
log.debug("Closing remaining document");
|
||||
currentDoc.close();
|
||||
log.debug("Successfully closed remaining document");
|
||||
} catch (Exception e) {
|
||||
log.error("Error closing remaining document", e);
|
||||
}
|
||||
}
|
||||
scratch.close();
|
||||
}
|
||||
|
||||
log.debug("Completed handleSplitByPageCount with {} document parts created", fileIndex - 1);
|
||||
return ranges;
|
||||
}
|
||||
|
||||
private void handleSplitByDocCount(
|
||||
PDDocument sourceDocument,
|
||||
int documentCount,
|
||||
ZipOutputStream zipOut,
|
||||
String baseFilename)
|
||||
/** Speculatively tries up to 5 next pages; returns how many fit under {@code maxBytes}. */
|
||||
private int lookAheadFit(PDDocument scratch, PDDocument source, int pageIndex, long maxBytes)
|
||||
throws IOException {
|
||||
log.debug("Starting handleSplitByDocCount with documentCount={}", documentCount);
|
||||
int totalPageCount = sourceDocument.getNumberOfPages();
|
||||
log.debug("Total pages in source document: {}", totalPageCount);
|
||||
int totalPages = source.getNumberOfPages();
|
||||
int pagesToLookAhead = Math.min(5, totalPages - pageIndex - 1);
|
||||
if (pagesToLookAhead == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int pagesPerDocument = totalPageCount / documentCount;
|
||||
int extraPages = totalPageCount % documentCount;
|
||||
log.debug("Pages per document: {}, Extra pages: {}", pagesPerDocument, extraPages);
|
||||
int extraPagesAdded = 0;
|
||||
try (PDDocument testDoc = new PDDocument()) {
|
||||
for (int i = 0; i < scratch.getNumberOfPages(); i++) {
|
||||
testDoc.addPage(new PDPage(scratch.getPage(i).getCOSObject()));
|
||||
}
|
||||
for (int i = 0; i < pagesToLookAhead; i++) {
|
||||
testDoc.addPage(new PDPage(source.getPage(pageIndex + 1 + i).getCOSObject()));
|
||||
long testSize;
|
||||
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||
testDoc.save(out);
|
||||
testSize = out.size();
|
||||
}
|
||||
if (testSize > maxBytes) {
|
||||
break;
|
||||
}
|
||||
extraPagesAdded++;
|
||||
}
|
||||
}
|
||||
return extraPagesAdded;
|
||||
}
|
||||
|
||||
int currentPageIndex = 0;
|
||||
int fileIndex = 1;
|
||||
private List<List<Integer>> computePageCountRanges(PDDocument sourceDocument, int pageCount) {
|
||||
if (pageCount <= 0) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.invalidArgument", "Invalid argument: {0}", "page count: " + pageCount);
|
||||
}
|
||||
int totalPages = sourceDocument.getNumberOfPages();
|
||||
List<List<Integer>> ranges = new ArrayList<>();
|
||||
List<Integer> current = new ArrayList<>(pageCount);
|
||||
for (int i = 0; i < totalPages; i++) {
|
||||
current.add(i);
|
||||
if (current.size() == pageCount) {
|
||||
ranges.add(current);
|
||||
current = new ArrayList<>(pageCount);
|
||||
}
|
||||
}
|
||||
if (!current.isEmpty()) {
|
||||
ranges.add(current);
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
private List<List<Integer>> computeDocCountRanges(
|
||||
PDDocument sourceDocument, int documentCount) {
|
||||
if (documentCount <= 0) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.invalidArgument",
|
||||
"Invalid argument: {0}",
|
||||
"document count: " + documentCount);
|
||||
}
|
||||
int totalPages = sourceDocument.getNumberOfPages();
|
||||
int pagesPerDocument = totalPages / documentCount;
|
||||
int extraPages = totalPages % documentCount;
|
||||
|
||||
List<List<Integer>> ranges = new ArrayList<>();
|
||||
int cursor = 0;
|
||||
for (int i = 0; i < documentCount; i++) {
|
||||
log.debug("Creating document {} of {}", i + 1, documentCount);
|
||||
PDDocument currentDoc = null;
|
||||
try {
|
||||
currentDoc = pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument);
|
||||
log.debug("Successfully created document {} of {}", i + 1, documentCount);
|
||||
|
||||
int pagesToAdd = pagesPerDocument + (i < extraPages ? 1 : 0);
|
||||
log.debug("Adding {} pages to document {}", pagesToAdd, i + 1);
|
||||
|
||||
for (int j = 0; j < pagesToAdd; j++) {
|
||||
try {
|
||||
log.debug(
|
||||
"Adding page {} (index {}) to document {}",
|
||||
j + 1,
|
||||
currentPageIndex,
|
||||
i + 1);
|
||||
currentDoc.addPage(sourceDocument.getPage(currentPageIndex));
|
||||
log.debug("Successfully added page {} to document {}", j + 1, i + 1);
|
||||
currentPageIndex++;
|
||||
} catch (Exception e) {
|
||||
log.error("Error adding page {} to document {}", j + 1, i + 1, e);
|
||||
throw ExceptionUtils.createFileProcessingException("split", e);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug("Saving document {} with {} pages", i + 1, pagesToAdd);
|
||||
saveDocumentToZip(currentDoc, zipOut, baseFilename, fileIndex++);
|
||||
// saveDocumentToZip closes the document
|
||||
currentDoc = null;
|
||||
log.debug("Successfully saved document {}", i + 1);
|
||||
} catch (Exception e) {
|
||||
log.error("Error saving document {}", i + 1, e);
|
||||
throw e;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error creating document {} of {}", i + 1, documentCount, e);
|
||||
throw ExceptionUtils.createFileProcessingException("split", e);
|
||||
} finally {
|
||||
if (currentDoc != null) {
|
||||
try {
|
||||
currentDoc.close();
|
||||
} catch (IOException e) {
|
||||
log.error("Error closing document {} of {}", i + 1, documentCount, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.debug("Completed handleSplitByDocCount with {} documents created", documentCount);
|
||||
}
|
||||
|
||||
private void saveDocumentToZip(
|
||||
PDDocument document, ZipOutputStream zipOut, String baseFilename, int index)
|
||||
throws IOException {
|
||||
log.debug("Starting saveDocumentToZip for document part {}", index);
|
||||
try (ByteArrayOutputStream outStream = new ByteArrayOutputStream()) {
|
||||
|
||||
try (PDDocument doc = document) {
|
||||
log.debug("Saving document part {} to byte array", index);
|
||||
doc.save(outStream);
|
||||
log.debug(
|
||||
"Successfully saved document part {} ({} bytes)", index, outStream.size());
|
||||
} catch (Exception e) {
|
||||
log.error("Error saving document part {} to byte array", index, e);
|
||||
throw ExceptionUtils.createFileProcessingException("split", e);
|
||||
}
|
||||
|
||||
try {
|
||||
// Create a new zip entry
|
||||
String entryName = baseFilename + "_" + index + ".pdf";
|
||||
log.debug("Creating ZIP entry: {}", entryName);
|
||||
ZipEntry zipEntry = new ZipEntry(entryName);
|
||||
zipOut.putNextEntry(zipEntry);
|
||||
|
||||
byte[] bytes = outStream.toByteArray();
|
||||
log.debug("Writing {} bytes to ZIP entry", bytes.length);
|
||||
zipOut.write(bytes);
|
||||
|
||||
log.debug("Closing ZIP entry");
|
||||
zipOut.closeEntry();
|
||||
log.debug("Successfully added document part {} to ZIP", index);
|
||||
} catch (Exception e) {
|
||||
log.error("Error adding document part {} to ZIP", index, e);
|
||||
throw ExceptionUtils.createFileProcessingException("split", e);
|
||||
int pagesToAdd = pagesPerDocument + (i < extraPages ? 1 : 0);
|
||||
List<Integer> range = new ArrayList<>(pagesToAdd);
|
||||
for (int j = 0; j < pagesToAdd; j++) {
|
||||
range.add(cursor++);
|
||||
}
|
||||
ranges.add(range);
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
}
|
||||
|
||||
+15
-19
@@ -4,13 +4,13 @@ import java.io.ByteArrayOutputStream;
|
||||
import java.io.StringWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
import org.apache.commons.csv.CSVPrinter;
|
||||
import org.apache.commons.csv.QuoteMode;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.http.ContentDisposition;
|
||||
@@ -26,24 +26,21 @@ import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.config.swagger.CsvConversionResponse;
|
||||
import stirling.software.SPDF.model.api.PDFWithPageNums;
|
||||
import stirling.software.SPDF.pdf.FlexibleCSVWriter;
|
||||
import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
|
||||
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
|
||||
import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.ConvertApi;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
|
||||
import technology.tabula.ObjectExtractor;
|
||||
import technology.tabula.Page;
|
||||
import technology.tabula.Table;
|
||||
import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
|
||||
|
||||
@ConvertApi
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class ExtractCSVController {
|
||||
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final TabulaTableParser tabulaTableParser;
|
||||
|
||||
@AutoJobPostMapping(value = "/pdf/csv", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@CsvConversionResponse
|
||||
@@ -58,24 +55,23 @@ public class ExtractCSVController {
|
||||
|
||||
try (PDDocument document = pdfDocumentFactory.load(request)) {
|
||||
List<Integer> pages = request.getPageNumbersList(document, true);
|
||||
SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm();
|
||||
CSVFormat format =
|
||||
CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build();
|
||||
|
||||
for (int pageNum : pages) {
|
||||
try (ObjectExtractor extractor = new ObjectExtractor(document)) {
|
||||
log.info("{}", pageNum);
|
||||
Page page = extractor.extract(pageNum);
|
||||
List<Table> tables = sea.extract(page);
|
||||
log.info("{}", pageNum);
|
||||
List<TableFragment> fragments = tabulaTableParser.parse(document, pageNum);
|
||||
|
||||
for (int i = 0; i < tables.size(); i++) {
|
||||
StringWriter sw = new StringWriter();
|
||||
FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format);
|
||||
csvWriter.write(sw, Collections.singletonList(tables.get(i)));
|
||||
|
||||
String entryName = generateEntryName(baseName, pageNum, i + 1);
|
||||
csvEntries.add(new CsvEntry(entryName, sw.toString()));
|
||||
for (int i = 0; i < fragments.size(); i++) {
|
||||
StringWriter sw = new StringWriter();
|
||||
try (CSVPrinter printer = format.print(sw)) {
|
||||
for (List<String> row : fragments.get(i).rawRows()) {
|
||||
printer.printRecord(row);
|
||||
}
|
||||
}
|
||||
csvEntries.add(
|
||||
new CsvEntry(
|
||||
generateEntryName(baseName, pageNum, i + 1), sw.toString()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+3
-1
@@ -96,7 +96,9 @@ public class RedactController {
|
||||
@InitBinder
|
||||
public void initBinder(WebDataBinder binder) {
|
||||
binder.registerCustomEditor(
|
||||
List.class, "redactions", new StringToArrayListPropertyEditor());
|
||||
List.class,
|
||||
"redactions",
|
||||
new StringToArrayListPropertyEditor<>(RedactionArea.class));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(value = "/redact", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
|
||||
@@ -682,7 +682,7 @@ public class GlobalExceptionHandler {
|
||||
*
|
||||
* @param ex the MaxUploadSizeExceededException
|
||||
* @param request the HTTP servlet request
|
||||
* @return ProblemDetail with HTTP 413 PAYLOAD_TOO_LARGE
|
||||
* @return ProblemDetail with HTTP 413 CONTENT_TOO_LARGE
|
||||
*/
|
||||
@ExceptionHandler(MaxUploadSizeExceededException.class)
|
||||
public ResponseEntity<ProblemDetail> handleMaxUploadSize(
|
||||
@@ -706,7 +706,7 @@ public class GlobalExceptionHandler {
|
||||
getLocalizedMessage("error.fileTooLarge.title", ErrorTitles.FILE_TOO_LARGE_DEFAULT);
|
||||
|
||||
ProblemDetail problemDetail =
|
||||
createBaseProblemDetail(HttpStatus.PAYLOAD_TOO_LARGE, message, request);
|
||||
createBaseProblemDetail(HttpStatus.CONTENT_TOO_LARGE, message, request);
|
||||
problemDetail.setType(URI.create(ErrorTypes.FILE_TOO_LARGE));
|
||||
problemDetail.setTitle(title);
|
||||
problemDetail.setProperty("title", title); // Ensure serialization
|
||||
@@ -724,7 +724,7 @@ public class GlobalExceptionHandler {
|
||||
problemDetail.setProperty(
|
||||
"actionRequired", "Reduce the file size to be within the upload limit.");
|
||||
|
||||
return ResponseEntity.status(HttpStatus.PAYLOAD_TOO_LARGE)
|
||||
return ResponseEntity.status(HttpStatus.CONTENT_TOO_LARGE)
|
||||
.contentType(PROBLEM_JSON)
|
||||
.body(problemDetail);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package stirling.software.SPDF.model.api.general;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.SPDF.model.api.PDFWithPageNums;
|
||||
import stirling.software.common.model.api.general.EditTextOperation;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class EditTextRequest extends PDFWithPageNums {
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Ordered list of find/replace operations. Each replaces every occurrence on"
|
||||
+ " the selected pages, in order; later operations see the result of"
|
||||
+ " earlier ones (so 'foo'->'foos' then 'foos'->'bars' turns 'foo'"
|
||||
+ " into 'bars').",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<EditTextOperation> edits;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Whether matches must be whole words (boundaries determined by non-word"
|
||||
+ " characters)",
|
||||
defaultValue = "false")
|
||||
private Boolean wholeWordSearch;
|
||||
}
|
||||
+1
-2
@@ -13,8 +13,7 @@ public class RotatePDFRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"The clockwise angle by which to rotate the PDF file. Must be a multiple of"
|
||||
+ " 90.",
|
||||
"The clockwise angle by which to rotate all pages in the PDF file. Must be a multiple of 90.",
|
||||
type = "integer",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
allowableValues = {"0", "90", "180", "270"})
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
package stirling.software.SPDF.pdf;
|
||||
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
|
||||
import technology.tabula.writers.CSVWriter;
|
||||
|
||||
public class FlexibleCSVWriter extends CSVWriter {
|
||||
|
||||
public FlexibleCSVWriter() {
|
||||
super();
|
||||
}
|
||||
|
||||
public FlexibleCSVWriter(CSVFormat csvFormat) {
|
||||
super(csvFormat);
|
||||
}
|
||||
}
|
||||
+118
-10
@@ -252,6 +252,17 @@ public class PdfJsonConversionService {
|
||||
convertPdfToJson(file, null, false, out);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a PDF to the editable {@link PdfJsonDocument} model. Convenience wrapper around
|
||||
* {@link #convertPdfToJson(MultipartFile, OutputStream)} for callers that need to inspect or
|
||||
* mutate the document in memory before round-tripping it back to PDF.
|
||||
*/
|
||||
public PdfJsonDocument convertPdfToJsonDocument(MultipartFile file) throws IOException {
|
||||
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
|
||||
convertPdfToJson(file, null, false, buffer);
|
||||
return objectMapper.readValue(buffer.toByteArray(), PdfJsonDocument.class);
|
||||
}
|
||||
|
||||
public void convertPdfToJson(MultipartFile file, boolean lightweight, OutputStream out)
|
||||
throws IOException {
|
||||
convertPdfToJson(file, null, lightweight, out);
|
||||
@@ -620,6 +631,13 @@ public class PdfJsonConversionService {
|
||||
}
|
||||
byte[] jsonBytes = file.getBytes();
|
||||
PdfJsonDocument pdfJson = objectMapper.readValue(jsonBytes, PdfJsonDocument.class);
|
||||
convertJsonToPdf(pdfJson, out);
|
||||
}
|
||||
|
||||
public void convertJsonToPdf(PdfJsonDocument pdfJson, OutputStream out) throws IOException {
|
||||
if (pdfJson == null) {
|
||||
throw ExceptionUtils.createNullArgumentException("document");
|
||||
}
|
||||
|
||||
List<PdfJsonFont> fontModels = pdfJson.getFonts();
|
||||
if (fontModels == null) {
|
||||
@@ -2076,14 +2094,29 @@ public class PdfJsonConversionService {
|
||||
byte[] toUnicodeBytes = Base64.getDecoder().decode(toUnicodeBase64);
|
||||
String toUnicodeStr = new String(toUnicodeBytes, StandardCharsets.UTF_8);
|
||||
|
||||
// Parse ToUnicode CMap for bfchar and bfrange
|
||||
// Parse ToUnicode CMap for bfchar and bfrange. Both sides go through
|
||||
// parseToUnicodeCodepoint so an 8-hex-char surrogate-pair value on either side
|
||||
// (e.g. "D837DF0E") is decoded as a single supplementary codepoint instead of
|
||||
// overflowing Integer.parseInt and aborting the whole mapping build.
|
||||
java.util.regex.Pattern bfcharPattern =
|
||||
java.util.regex.Pattern.compile("<([0-9A-Fa-f]+)>\\s*<([0-9A-Fa-f]+)>");
|
||||
java.util.regex.Matcher matcher = bfcharPattern.matcher(toUnicodeStr);
|
||||
while (matcher.find()) {
|
||||
int charCode = Integer.parseInt(matcher.group(1), 16);
|
||||
int unicode = Integer.parseInt(matcher.group(2), 16);
|
||||
charCodeToUnicode.put(charCode, unicode);
|
||||
try {
|
||||
int charCode = parseToUnicodeCodepoint(matcher.group(1));
|
||||
int unicode = parseToUnicodeCodepoint(matcher.group(2));
|
||||
charCodeToUnicode.put(charCode, unicode);
|
||||
} catch (NumberFormatException entryEx) {
|
||||
// Tolerate a single malformed entry: log and skip rather than aborting the
|
||||
// entire ToUnicode CMap (which would force the whole font onto the slow
|
||||
// raw-bytes fallback path).
|
||||
log.debug(
|
||||
"Skipping malformed ToUnicode entry <{}> <{}> in font {}: {}",
|
||||
matcher.group(1),
|
||||
matcher.group(2),
|
||||
font.getName(),
|
||||
entryEx.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// Build JSON mapping: CharCode → CID → GID → Unicode
|
||||
@@ -2136,6 +2169,42 @@ public class PdfJsonConversionService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a hex string from a PDF ToUnicode CMap into a single Unicode codepoint. Handles three
|
||||
* cases: a single BMP code unit (4 hex chars), a UTF-16 surrogate pair encoding a supplementary
|
||||
* codepoint above U+FFFF (8 hex chars, e.g. {@code D837DF0E} for U+1F40E), and multi-codepoint
|
||||
* mappings (longer; returns the first codepoint as a best-effort representative).
|
||||
*
|
||||
* <p>Without this, {@code Integer.parseInt("D837DF0E", 16)} overflows because the value is ~3.6
|
||||
* billion, throwing {@link NumberFormatException} and forcing the conversion to fall back to a
|
||||
* raw ToUnicode payload that the JSON→PDF rebuild then fails to use efficiently.
|
||||
*/
|
||||
static int parseToUnicodeCodepoint(String hex) {
|
||||
if (hex == null || hex.isEmpty()) {
|
||||
throw new NumberFormatException("Empty ToUnicode hex value");
|
||||
}
|
||||
if (hex.length() <= 4) {
|
||||
return Integer.parseInt(hex, 16);
|
||||
}
|
||||
// Treat the hex string as UTF-16BE: pairs of hex digits form bytes, four hex digits form
|
||||
// one UTF-16 code unit. The PDF ToUnicode CMap convention requires an even number of bytes
|
||||
// (i.e. a multiple of four hex characters) for multi-unit values.
|
||||
if (hex.length() % 4 != 0) {
|
||||
throw new NumberFormatException(
|
||||
"ToUnicode hex value not a multiple of 4 chars: " + hex);
|
||||
}
|
||||
int unitCount = hex.length() / 4;
|
||||
char[] units = new char[unitCount];
|
||||
for (int i = 0; i < unitCount; i++) {
|
||||
units[i] = (char) Integer.parseInt(hex.substring(i * 4, i * 4 + 4), 16);
|
||||
}
|
||||
// codePointAt assembles a surrogate pair into a supplementary codepoint when the
|
||||
// high/low surrogates appear in sequence; for any other multi-unit sequence it returns
|
||||
// the first BMP codepoint, which is the right best-effort fallback for ligature
|
||||
// decompositions (one charCode -> several Unicode chars).
|
||||
return new String(units).codePointAt(0);
|
||||
}
|
||||
|
||||
private PdfJsonFontCidSystemInfo extractCidSystemInfo(COSDictionary fontDictionary) {
|
||||
if (fontDictionary == null) {
|
||||
return null;
|
||||
@@ -4256,12 +4325,8 @@ public class PdfJsonConversionService {
|
||||
return 0;
|
||||
}
|
||||
if (font != null) {
|
||||
try (InputStream inputStream = new ByteArrayInputStream(value.getBytes())) {
|
||||
int count = 0;
|
||||
int code;
|
||||
while ((code = font.readCode(inputStream)) != -1) {
|
||||
count++;
|
||||
}
|
||||
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(value.getBytes())) {
|
||||
int count = countCodesProtected(inputStream, font::readCode);
|
||||
if (count > 0) {
|
||||
return count;
|
||||
}
|
||||
@@ -4273,6 +4338,49 @@ public class PdfJsonConversionService {
|
||||
return Math.max(1, bytes.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Functional accessor for {@link PDFont#readCode(InputStream)} so the bounded counting loop can
|
||||
* be exercised in isolation without instantiating a {@link PDFont}.
|
||||
*/
|
||||
@FunctionalInterface
|
||||
interface CodeReader {
|
||||
int readCode(InputStream stream) throws IOException;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count how many codes the supplied {@code reader} can extract from {@code inputStream}, with
|
||||
* two safety nets that PDFBox's raw {@link PDFont#readCode(InputStream)} loop lacks:
|
||||
*
|
||||
* <ol>
|
||||
* <li>Stop when the stream is empty (a corrupt CMap can otherwise loop forever returning
|
||||
* successfully-matched zero-bytes from an exhausted {@link ByteArrayInputStream}).
|
||||
* <li>Stop when a {@code readCode} call did not consume any bytes, even if it returned a
|
||||
* non-{@code -1} value.
|
||||
* </ol>
|
||||
*
|
||||
* <p>Both conditions were observed in the wild on round-tripped fallback fonts where the
|
||||
* embedded ToUnicode CMap matched 0x00 sequences, hanging the JSON→PDF rebuild.
|
||||
*/
|
||||
static int countCodesProtected(ByteArrayInputStream inputStream, CodeReader reader)
|
||||
throws IOException {
|
||||
int count = 0;
|
||||
int previousAvailable = inputStream.available();
|
||||
while (previousAvailable > 0) {
|
||||
int code = reader.readCode(inputStream);
|
||||
if (code == -1) {
|
||||
break;
|
||||
}
|
||||
int currentAvailable = inputStream.available();
|
||||
if (currentAvailable >= previousAvailable) {
|
||||
// No progress made; break to avoid infinite loop on corrupt CMaps.
|
||||
break;
|
||||
}
|
||||
count++;
|
||||
previousAvailable = currentAvailable;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private MergedText mergeText(List<PdfJsonTextElement> elements) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
List<Integer> combinedCodes = new ArrayList<>();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+664
@@ -0,0 +1,664 @@
|
||||
package stirling.software.SPDF.controller.api;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import stirling.software.SPDF.model.api.general.EditTextRequest;
|
||||
import stirling.software.SPDF.model.json.PdfJsonDocument;
|
||||
import stirling.software.SPDF.model.json.PdfJsonPage;
|
||||
import stirling.software.SPDF.model.json.PdfJsonTextElement;
|
||||
import stirling.software.SPDF.service.PdfJsonConversionService;
|
||||
import stirling.software.common.model.api.general.EditTextOperation;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class EditTextControllerTest {
|
||||
|
||||
@Mock private PdfJsonConversionService pdfJsonConversionService;
|
||||
@Mock private TempFileManager tempFileManager;
|
||||
|
||||
@InjectMocks private EditTextController controller;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
lenient()
|
||||
.when(tempFileManager.createManagedTempFile(anyString()))
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
File f =
|
||||
Files.createTempFile("test", inv.<String>getArgument(0))
|
||||
.toFile();
|
||||
TempFile tf = mock(TempFile.class);
|
||||
lenient().when(tf.getFile()).thenReturn(f);
|
||||
lenient().when(tf.getPath()).thenReturn(f.toPath());
|
||||
return tf;
|
||||
});
|
||||
}
|
||||
|
||||
private static MultipartFile pdfFile() {
|
||||
return new MockMultipartFile(
|
||||
"fileInput", "doc.pdf", "application/pdf", "stub-pdf-bytes".getBytes());
|
||||
}
|
||||
|
||||
private static EditTextOperation edit(String find, String replace) {
|
||||
EditTextOperation op = new EditTextOperation();
|
||||
op.setFind(find);
|
||||
op.setReplace(replace);
|
||||
return op;
|
||||
}
|
||||
|
||||
private static PdfJsonTextElement textElement(String text) {
|
||||
PdfJsonTextElement el = new PdfJsonTextElement();
|
||||
el.setText(text);
|
||||
el.setCharCodes(new int[] {1, 2, 3});
|
||||
return el;
|
||||
}
|
||||
|
||||
private static PdfJsonDocument documentWithElements(List<PdfJsonTextElement> elements) {
|
||||
PdfJsonDocument doc = new PdfJsonDocument();
|
||||
PdfJsonPage page = new PdfJsonPage();
|
||||
page.setPageNumber(1);
|
||||
page.setTextElements(new ArrayList<>(elements));
|
||||
doc.setPages(new ArrayList<>(List.of(page)));
|
||||
return doc;
|
||||
}
|
||||
|
||||
private static PdfJsonDocument documentWithText(String... textsByPage) {
|
||||
PdfJsonDocument doc = new PdfJsonDocument();
|
||||
List<PdfJsonPage> pages = new ArrayList<>();
|
||||
for (int i = 0; i < textsByPage.length; i++) {
|
||||
PdfJsonPage page = new PdfJsonPage();
|
||||
page.setPageNumber(i + 1);
|
||||
PdfJsonTextElement el = new PdfJsonTextElement();
|
||||
el.setText(textsByPage[i]);
|
||||
el.setCharCodes(new int[] {1, 2, 3});
|
||||
page.setTextElements(new ArrayList<>(List.of(el)));
|
||||
pages.add(page);
|
||||
}
|
||||
doc.setPages(pages);
|
||||
return doc;
|
||||
}
|
||||
|
||||
/** Build a minimal real PDF so tests can run end-to-end without mocking the conversion. */
|
||||
private static byte[] buildEmptyPdf() throws Exception {
|
||||
try (PDDocument doc = new PDDocument();
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
|
||||
doc.addPage(new PDPage());
|
||||
doc.save(baos);
|
||||
return baos.toByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_nullFileInputThrows() {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(null);
|
||||
request.setEdits(List.of(edit("foo", "bar")));
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> controller.editText(request));
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_emptyEditsThrows() {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of());
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> controller.editText(request));
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_nullEditsThrows() {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(null);
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> controller.editText(request));
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_emptyFindStringThrows() {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("", "replacement")));
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> controller.editText(request));
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_findStringWithRegexMetacharsIsTreatedLiterally() throws Exception {
|
||||
// Find strings are always treated as literals (Pattern.quote'd internally) — no ReDoS
|
||||
// exposure from regex metacharacters supplied by the caller.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("(unclosed", "fixed")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("text with (unclosed paren");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
assertEquals(
|
||||
"text with fixed paren",
|
||||
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_literalFindReplace_mutatesMatchingSpansAndClearsCharCodes() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo", "bar")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("foo and foo", "no match here");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
ResponseEntity<Resource> response = controller.editText(request);
|
||||
|
||||
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||
assertNotNull(response.getBody());
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
PdfJsonDocument mutated = captor.getValue();
|
||||
|
||||
assertEquals("bar and bar", mutated.getPages().get(0).getTextElements().get(0).getText());
|
||||
assertNull(mutated.getPages().get(0).getTextElements().get(0).getCharCodes());
|
||||
|
||||
assertEquals("no match here", mutated.getPages().get(1).getTextElements().get(0).getText());
|
||||
// Char codes preserved on unmodified spans.
|
||||
assertNotNull(mutated.getPages().get(1).getTextElements().get(0).getCharCodes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_wholeWordSearch_doesNotMatchInsideWords() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("cat", "dog")));
|
||||
request.setWholeWordSearch(true);
|
||||
|
||||
PdfJsonDocument input = documentWithText("the cat is in the catalogue");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
assertEquals(
|
||||
"the dog is in the catalogue",
|
||||
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_wholeWordSearch_matchesFindStartingWithNonWordChar() throws Exception {
|
||||
// Regression: \b only fires on a word/non-word transition. A find that starts with a
|
||||
// non-word char (e.g. "-foo") preceded by another non-word char in the source (a space)
|
||||
// would never match under \b. The lookaround-based bound handles this correctly.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("-foo", "-bar")));
|
||||
request.setWholeWordSearch(true);
|
||||
|
||||
PdfJsonDocument input = documentWithText("space then -foo here");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
assertEquals(
|
||||
"space then -bar here",
|
||||
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_wholeWordSearch_doesNotMatchWhenAdjacentToWordChar() throws Exception {
|
||||
// The lookaround bound must still reject matches that are part of a larger word.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("-foo", "-bar")));
|
||||
request.setWholeWordSearch(true);
|
||||
|
||||
PdfJsonDocument input = documentWithText("inline-foo should not match");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
// "inline-foo" has 'e' (word) before '-foo' so the lookbehind blocks the match. The
|
||||
// trailing 'o' is followed by a space (non-word) so the trailing lookahead would pass on
|
||||
// its own; the leading lookbehind is what rejects it.
|
||||
assertEquals(
|
||||
"inline-foo should not match",
|
||||
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_pageFilter_onlyAffectsListedPages() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo", "bar")));
|
||||
request.setPageNumbers("2");
|
||||
|
||||
PdfJsonDocument input = documentWithText("foo on page 1", "foo on page 2", "foo on page 3");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
PdfJsonDocument mutated = captor.getValue();
|
||||
|
||||
assertEquals("foo on page 1", mutated.getPages().get(0).getTextElements().get(0).getText());
|
||||
assertEquals("bar on page 2", mutated.getPages().get(1).getTextElements().get(0).getText());
|
||||
assertEquals("foo on page 3", mutated.getPages().get(2).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_pageRange_appliesToAllPagesInRange() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo", "bar")));
|
||||
request.setPageNumbers("2-3");
|
||||
|
||||
PdfJsonDocument input =
|
||||
documentWithText("foo page 1", "foo page 2", "foo page 3", "foo page 4");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
PdfJsonDocument mutated = captor.getValue();
|
||||
|
||||
assertTrue(mutated.getPages().get(0).getTextElements().get(0).getText().startsWith("foo"));
|
||||
assertTrue(mutated.getPages().get(1).getTextElements().get(0).getText().startsWith("bar"));
|
||||
assertTrue(mutated.getPages().get(2).getTextElements().get(0).getText().startsWith("bar"));
|
||||
assertTrue(mutated.getPages().get(3).getTextElements().get(0).getText().startsWith("foo"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_orderedEdits_applyInSequence() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo", "bar"), edit("bar", "baz")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("foo");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
assertEquals("baz", captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_replaceWithEmptyString_deletesMatch() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("DRAFT ", "")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("DRAFT Confidential Memo");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
assertEquals(
|
||||
"Confidential Memo",
|
||||
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_noMatches_returnsPdfWithoutMutation() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("notfound", "replacement")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("nothing to match");
|
||||
int[] originalCodes = input.getPages().get(0).getTextElements().get(0).getCharCodes();
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
ResponseEntity<Resource> response = controller.editText(request);
|
||||
|
||||
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||
// Char codes left intact when nothing was replaced.
|
||||
assertEquals(
|
||||
originalCodes, input.getPages().get(0).getTextElements().get(0).getCharCodes());
|
||||
assertEquals(
|
||||
"nothing to match", input.getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_dollarInLiteralReplacement_isQuoted() throws Exception {
|
||||
// Without quoting, '$1' would be interpreted as a backreference and crash.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("price", "$100")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("the price is set");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
assertEquals(
|
||||
"the $100 is set",
|
||||
captor.getValue().getPages().get(0).getTextElements().get(0).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_emptyDocument_returnsResponseWithoutErrors() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo", "bar")));
|
||||
|
||||
PdfJsonDocument input = new PdfJsonDocument();
|
||||
input.setPages(new ArrayList<>());
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
ResponseEntity<Resource> response = controller.editText(request);
|
||||
|
||||
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||
assertNotNull(response.getBody());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_textElementWithNullText_isSkipped() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo", "bar")));
|
||||
|
||||
PdfJsonDocument input = new PdfJsonDocument();
|
||||
PdfJsonPage page = new PdfJsonPage();
|
||||
page.setPageNumber(1);
|
||||
PdfJsonTextElement nullText = new PdfJsonTextElement();
|
||||
nullText.setText(null);
|
||||
PdfJsonTextElement realText = new PdfJsonTextElement();
|
||||
realText.setText("foo here");
|
||||
page.setTextElements(new ArrayList<>(List.of(nullText, realText)));
|
||||
input.setPages(new ArrayList<>(List.of(page)));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
PdfJsonDocument mutated = captor.getValue();
|
||||
assertNull(mutated.getPages().get(0).getTextElements().get(0).getText());
|
||||
assertEquals("bar here", mutated.getPages().get(0).getTextElements().get(1).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_crossElement_matchSpansTwoElements() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("Hello World", "Goodbye Earth")));
|
||||
|
||||
PdfJsonDocument input =
|
||||
documentWithElements(List.of(textElement("Hello "), textElement("World")));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
|
||||
|
||||
// Whole replacement lands in the first matched element; the second is emptied. The
|
||||
// JSON->PDF rebuild concatenates per-token text so the font lays out the replacement as
|
||||
// one run anchored at the first element's X position.
|
||||
assertEquals("Goodbye Earth", elements.get(0).getText());
|
||||
assertEquals("", elements.get(1).getText());
|
||||
assertNull(elements.get(0).getCharCodes());
|
||||
assertNull(elements.get(1).getCharCodes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_crossElement_matchSpansFiveElementsLikeFragmentedTitle() throws Exception {
|
||||
// Reproduces the real-world case: a multi-word title fragmented one word per text span.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(
|
||||
List.of(edit("The Free Adobe Acrobat Alternative", "The PDF automation pipeline")));
|
||||
|
||||
PdfJsonDocument input =
|
||||
documentWithElements(
|
||||
List.of(
|
||||
textElement("The "),
|
||||
textElement("Free "),
|
||||
textElement("Adobe "),
|
||||
textElement("Acrobat "),
|
||||
textElement("Alternative")));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
|
||||
|
||||
// Whole replacement is written into the first matched element; the remaining four are
|
||||
// emptied. Centered titles will become left-aligned at the original first-word X position.
|
||||
assertEquals("The PDF automation pipeline", elements.get(0).getText());
|
||||
for (int i = 1; i < elements.size(); i++) {
|
||||
assertEquals("", elements.get(i).getText(), "element " + i + " should be empty");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_crossElement_preservesPrefixAndSuffix() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("Hello World", "Goodbye Earth")));
|
||||
|
||||
PdfJsonDocument input =
|
||||
documentWithElements(
|
||||
List.of(textElement("Greeting: Hello "), textElement("World! And more")));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
|
||||
|
||||
// First element keeps its prefix and gets the entire replacement; last element keeps its
|
||||
// suffix only.
|
||||
assertEquals("Greeting: Goodbye Earth", elements.get(0).getText());
|
||||
assertEquals("! And more", elements.get(1).getText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_matchInOneElementOfMany_onlyTouchesThatElement() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("World", "Earth")));
|
||||
|
||||
PdfJsonDocument input =
|
||||
documentWithElements(
|
||||
List.of(
|
||||
textElement("Hello "),
|
||||
textElement("World!"),
|
||||
textElement(" Goodbye")));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
|
||||
|
||||
assertEquals("Hello ", elements.get(0).getText());
|
||||
assertEquals("Earth!", elements.get(1).getText());
|
||||
assertEquals(" Goodbye", elements.get(2).getText());
|
||||
// Only the modified element's char codes get cleared.
|
||||
assertNotNull(elements.get(0).getCharCodes());
|
||||
assertNull(elements.get(1).getCharCodes());
|
||||
assertNotNull(elements.get(2).getCharCodes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_crossElement_multipleMatchesAppliedRightToLeft() throws Exception {
|
||||
// Two matches in the same page text. Right-to-left application keeps earlier indices
|
||||
// valid as later matches are written.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("foo bar", "X")));
|
||||
|
||||
PdfJsonDocument input =
|
||||
documentWithElements(
|
||||
List.of(
|
||||
textElement("foo "),
|
||||
textElement("bar baz "),
|
||||
textElement("foo "),
|
||||
textElement("bar")));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
|
||||
|
||||
// Joined was "foo bar baz foo bar"; both "foo bar" runs replaced with "X".
|
||||
StringBuilder joined = new StringBuilder();
|
||||
for (PdfJsonTextElement el : elements) {
|
||||
joined.append(el.getText() == null ? "" : el.getText());
|
||||
}
|
||||
assertEquals("X baz X", joined.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_subWordFragmentation_writesIntoFirstElement() throws Exception {
|
||||
// When the matched text is split into many character-level spans (typical of Type3 glyph
|
||||
// runs), the whole replacement is dumped into the first matched element and the rest are
|
||||
// emptied. The font lays out the replacement as one run at the first glyph's X position.
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(pdfFile());
|
||||
request.setEdits(List.of(edit("Hello World", "Goodbye Earth")));
|
||||
|
||||
// 11 sub-word elements covering "Hello World".
|
||||
PdfJsonDocument input =
|
||||
documentWithElements(
|
||||
List.of(
|
||||
textElement("H"),
|
||||
textElement("e"),
|
||||
textElement("l"),
|
||||
textElement("l"),
|
||||
textElement("o"),
|
||||
textElement(" "),
|
||||
textElement("W"),
|
||||
textElement("o"),
|
||||
textElement("r"),
|
||||
textElement("l"),
|
||||
textElement("d")));
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
controller.editText(request);
|
||||
|
||||
ArgumentCaptor<PdfJsonDocument> captor = ArgumentCaptor.forClass(PdfJsonDocument.class);
|
||||
org.mockito.Mockito.verify(pdfJsonConversionService)
|
||||
.convertJsonToPdf(captor.capture(), any(OutputStream.class));
|
||||
List<PdfJsonTextElement> elements = captor.getValue().getPages().get(0).getTextElements();
|
||||
|
||||
// Entire replacement goes into the first matched element; the other 10 are emptied.
|
||||
assertEquals("Goodbye Earth", elements.get(0).getText());
|
||||
for (int i = 1; i < elements.size(); i++) {
|
||||
assertEquals("", elements.get(i).getText(), "element " + i + " should be empty");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void editText_outputFilenameDerivedFromInput() throws Exception {
|
||||
EditTextRequest request = new EditTextRequest();
|
||||
request.setFileInput(
|
||||
new MockMultipartFile(
|
||||
"fileInput", "report.pdf", "application/pdf", buildEmptyPdf()));
|
||||
request.setEdits(List.of(edit("anything", "x")));
|
||||
|
||||
PdfJsonDocument input = documentWithText("nothing matches");
|
||||
when(pdfJsonConversionService.convertPdfToJsonDocument(any(MultipartFile.class)))
|
||||
.thenReturn(input);
|
||||
|
||||
ResponseEntity<Resource> response = controller.editText(request);
|
||||
String contentDisposition =
|
||||
response.getHeaders()
|
||||
.getFirst(org.springframework.http.HttpHeaders.CONTENT_DISPOSITION);
|
||||
assertNotNull(contentDisposition);
|
||||
assertTrue(contentDisposition.contains("report_edited.pdf"));
|
||||
assertFalse(contentDisposition.contains(".pdf_edited.pdf"));
|
||||
}
|
||||
}
|
||||
+6
-3
@@ -3,8 +3,10 @@ package stirling.software.SPDF.controller.api;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
@@ -24,7 +26,6 @@ import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import stirling.software.SPDF.model.api.SplitPagesRequest;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
@@ -60,8 +61,10 @@ class SplitPDFControllerTest {
|
||||
}
|
||||
|
||||
private void setupFactory() throws IOException {
|
||||
when(pdfDocumentFactory.load(any(MultipartFile.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
|
||||
when(pdfDocumentFactory.load(any(File.class), eq(true)))
|
||||
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
|
||||
when(pdfDocumentFactory.load(any(File.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(any(PDDocument.class)))
|
||||
.thenAnswer(inv -> new PDDocument());
|
||||
}
|
||||
|
||||
+6
-12
@@ -3,8 +3,10 @@ package stirling.software.SPDF.controller.api;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
@@ -25,7 +27,6 @@ import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import stirling.software.SPDF.model.api.general.SplitPdfBySizeOrCountRequest;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
@@ -70,9 +71,8 @@ class SplitPdfBySizeControllerTest {
|
||||
request.setSplitType(1); // Page count
|
||||
request.setSplitValue("2");
|
||||
|
||||
when(pdfDocumentFactory.load(any(MultipartFile.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
|
||||
|
||||
when(pdfDocumentFactory.load(any(File.class), eq(true)))
|
||||
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(any(PDDocument.class)))
|
||||
.thenAnswer(inv -> new PDDocument());
|
||||
|
||||
@@ -104,14 +104,8 @@ class SplitPdfBySizeControllerTest {
|
||||
request.setSplitType(2); // Document count
|
||||
request.setSplitValue("3"); // Split into 3 docs (2 pages each)
|
||||
|
||||
when(pdfDocumentFactory.load(any(org.springframework.web.multipart.MultipartFile.class)))
|
||||
.thenAnswer(
|
||||
inv ->
|
||||
Loader.loadPDF(
|
||||
((org.springframework.web.multipart.MultipartFile)
|
||||
inv.getArgument(0))
|
||||
.getBytes()));
|
||||
|
||||
when(pdfDocumentFactory.load(any(File.class), eq(true)))
|
||||
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(any(PDDocument.class)))
|
||||
.thenAnswer(inv -> new PDDocument());
|
||||
|
||||
|
||||
+2
@@ -20,6 +20,7 @@ import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
|
||||
import stirling.software.SPDF.model.api.PDFWithPageNums;
|
||||
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
|
||||
@@ -27,6 +28,7 @@ import stirling.software.common.util.GeneralUtils;
|
||||
class ExtractCSVControllerTest {
|
||||
|
||||
@Mock private CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
@Mock private TabulaTableParser tabulaTableParser;
|
||||
|
||||
@InjectMocks private ExtractCSVController controller;
|
||||
|
||||
|
||||
+2
-2
@@ -177,14 +177,14 @@ class GlobalExceptionHandlerTest {
|
||||
void handleMaxUploadSize_returns_413() {
|
||||
MaxUploadSizeExceededException ex = new MaxUploadSizeExceededException(10485760);
|
||||
ResponseEntity<ProblemDetail> resp = handler.handleMaxUploadSize(ex, request);
|
||||
assertEquals(HttpStatus.PAYLOAD_TOO_LARGE, resp.getStatusCode());
|
||||
assertEquals(HttpStatus.CONTENT_TOO_LARGE, resp.getStatusCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void handleMaxUploadSize_unknown_limit() {
|
||||
MaxUploadSizeExceededException ex = new MaxUploadSizeExceededException(-1);
|
||||
ResponseEntity<ProblemDetail> resp = handler.handleMaxUploadSize(ex, request);
|
||||
assertEquals(HttpStatus.PAYLOAD_TOO_LARGE, resp.getStatusCode());
|
||||
assertEquals(HttpStatus.CONTENT_TOO_LARGE, resp.getStatusCode());
|
||||
assertNull(resp.getBody().getProperties().get("maxSizeBytes"));
|
||||
}
|
||||
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
package stirling.software.SPDF.pdf;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class FlexibleCSVWriterTest {
|
||||
|
||||
@Test
|
||||
void testDefaultConstructor() {
|
||||
FlexibleCSVWriter writer = new FlexibleCSVWriter();
|
||||
assertNotNull(writer, "The FlexibleCSVWriter instance should not be null");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testConstructorWithCSVFormat() {
|
||||
CSVFormat csvFormat = CSVFormat.DEFAULT;
|
||||
FlexibleCSVWriter writer = new FlexibleCSVWriter(csvFormat);
|
||||
assertNotNull(
|
||||
writer,
|
||||
"The FlexibleCSVWriter instance should not be null when initialized with"
|
||||
+ " CSVFormat");
|
||||
}
|
||||
}
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
package stirling.software.SPDF.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.time.Duration;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link PdfJsonConversionService#parseToUnicodeCodepoint(String)}.
|
||||
*
|
||||
* <p>The function exists because PDF ToUnicode CMap entries can encode supplementary-plane
|
||||
* codepoints (above U+FFFF) as UTF-16 surrogate pairs, e.g. {@code <D837DF0E>} for U+1F40E. The
|
||||
* naive implementation that called {@link Integer#parseInt(String, int)} on the whole hex string
|
||||
* threw {@link NumberFormatException} for any 8-char value because they overflow {@code int}. That
|
||||
* triggered a fallback that left fonts with no proper CID mapping and made downstream JSON→PDF
|
||||
* rebuilds slow/hung when rendering text with those fonts.
|
||||
*/
|
||||
class PdfJsonConversionServiceUnicodeParsingTest {
|
||||
|
||||
@Test
|
||||
void parsesSingleBmpCodeUnit() {
|
||||
// Latin capital A.
|
||||
assertEquals(0x41, PdfJsonConversionService.parseToUnicodeCodepoint("0041"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesShortBmpHexValue() {
|
||||
// Some ToUnicode entries omit leading zeros for low codepoints.
|
||||
assertEquals(0x41, PdfJsonConversionService.parseToUnicodeCodepoint("41"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesSupplementaryCodepointFromSurrogatePair() {
|
||||
// U+1F40E HORSE encoded as UTF-16 surrogate pair D83D DC0E. The bug we are fixing was
|
||||
// that Integer.parseInt("D83DDC0E", 16) overflows because D83DDC0E > Integer.MAX_VALUE.
|
||||
assertEquals(0x1F40E, PdfJsonConversionService.parseToUnicodeCodepoint("D83DDC0E"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesSurrogatePairFromUserReportedHang() {
|
||||
// The exact hex value from the user's hang reproducer log:
|
||||
// "Failed to build Unicode mapping for font NotoSans-Regular: For input string:
|
||||
// \"D837DF0E\" under radix 16"
|
||||
// D837 DF0E decodes to U+1DF0E (CJK supplementary). Important assertion: it returns a
|
||||
// valid codepoint instead of overflowing Integer.parseInt and throwing.
|
||||
int expected = new String(new char[] {(char) 0xD837, (char) 0xDF0E}).codePointAt(0);
|
||||
assertEquals(0x1DF0E, expected); // sanity check on the test setup itself
|
||||
assertEquals(expected, PdfJsonConversionService.parseToUnicodeCodepoint("D837DF0E"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesLigatureDecompositionAsFirstCodepoint() {
|
||||
// A ToUnicode entry can map one charCode to multiple Unicode chars (a ligature). PDF
|
||||
// spec allows e.g. <0041 0042> for "AB". Our best-effort behavior is to return the
|
||||
// first codepoint so the mapping is at least roughly meaningful for search/copy.
|
||||
assertEquals(0x41, PdfJsonConversionService.parseToUnicodeCodepoint("00410042"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsEmptyHex() {
|
||||
assertThrows(
|
||||
NumberFormatException.class,
|
||||
() -> PdfJsonConversionService.parseToUnicodeCodepoint(""));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsNullHex() {
|
||||
assertThrows(
|
||||
NumberFormatException.class,
|
||||
() -> PdfJsonConversionService.parseToUnicodeCodepoint(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsOddByteCountAboveBmp() {
|
||||
// 6 hex chars is 3 bytes — not a valid UTF-16BE sequence.
|
||||
assertThrows(
|
||||
NumberFormatException.class,
|
||||
() -> PdfJsonConversionService.parseToUnicodeCodepoint("D83DDC"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void countCodesProtectedTerminatesWhenReaderMakesNoProgress() {
|
||||
// Reproduces the user's hang: PDFBox's CMap.readCode can return a successful code (0)
|
||||
// from a stream where no bytes were consumed (corrupt codespace matching 0x00 bytes from
|
||||
// the buffer's uninitialized region after EOF). Without the no-progress guard, the
|
||||
// counting loop in countGlyphs ran forever.
|
||||
ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3, 4});
|
||||
PdfJsonConversionService.CodeReader reader = in -> 0; // never reads, always "succeeds"
|
||||
|
||||
int count =
|
||||
assertTimeoutPreemptively(
|
||||
Duration.ofSeconds(2),
|
||||
() -> PdfJsonConversionService.countCodesProtected(stream, reader));
|
||||
|
||||
// First iteration sees no progress and breaks immediately.
|
||||
assertEquals(0, count);
|
||||
}
|
||||
|
||||
@Test
|
||||
void countCodesProtectedTerminatesOnEmptyStream() {
|
||||
ByteArrayInputStream stream = new ByteArrayInputStream(new byte[0]);
|
||||
PdfJsonConversionService.CodeReader reader =
|
||||
in -> {
|
||||
throw new AssertionError("reader must not be called when stream is empty");
|
||||
};
|
||||
|
||||
int count =
|
||||
assertTimeoutPreemptively(
|
||||
Duration.ofSeconds(2),
|
||||
() -> PdfJsonConversionService.countCodesProtected(stream, reader));
|
||||
|
||||
assertEquals(0, count);
|
||||
}
|
||||
|
||||
@Test
|
||||
void countCodesProtectedHonorsExplicitMinusOneReturn() throws IOException {
|
||||
ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3});
|
||||
PdfJsonConversionService.CodeReader reader =
|
||||
in -> {
|
||||
int b = in.read();
|
||||
return b == -1 ? -1 : b;
|
||||
};
|
||||
|
||||
int count = PdfJsonConversionService.countCodesProtected(stream, reader);
|
||||
|
||||
assertEquals(3, count);
|
||||
}
|
||||
|
||||
@Test
|
||||
void countCodesProtectedTerminatesIfReaderReadsThenStops() throws IOException {
|
||||
// A reader that consumes one byte then hits a corrupt-CMap pattern returning 0 without
|
||||
// consuming further must still terminate after counting the consumed bytes.
|
||||
ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3, 4});
|
||||
PdfJsonConversionService.CodeReader reader =
|
||||
new PdfJsonConversionService.CodeReader() {
|
||||
boolean firstCall = true;
|
||||
|
||||
@Override
|
||||
public int readCode(InputStream in) throws IOException {
|
||||
if (firstCall) {
|
||||
firstCall = false;
|
||||
return in.read();
|
||||
}
|
||||
return 0; // simulates corrupt CMap thereafter
|
||||
}
|
||||
};
|
||||
|
||||
int count =
|
||||
assertTimeoutPreemptively(
|
||||
Duration.ofSeconds(2),
|
||||
() -> PdfJsonConversionService.countCodesProtected(stream, reader));
|
||||
|
||||
assertEquals(1, count);
|
||||
}
|
||||
}
|
||||
@@ -59,12 +59,6 @@ dependencies {
|
||||
// https://mvnrepository.com/artifact/com.bucket4j/bucket4j_jdk17
|
||||
implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion"
|
||||
|
||||
// Tabula table extraction — used by MathAuditorOrchestrator
|
||||
implementation ('technology.tabula:tabula:1.0.5') {
|
||||
exclude group: 'org.slf4j', module: 'slf4j-simple'
|
||||
exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
|
||||
exclude group: 'com.google.code.gson', module: 'gson'
|
||||
}
|
||||
implementation 'com.google.code.gson:gson:2.13.2'
|
||||
|
||||
api 'io.micrometer:micrometer-registry-prometheus'
|
||||
|
||||
+38
-4
@@ -30,6 +30,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import stirling.software.common.model.job.ResultFile;
|
||||
import stirling.software.common.service.JobOwnershipService;
|
||||
import stirling.software.common.service.TaskManager;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowProgressEvent;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowRequest;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowResponse;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowResultFile;
|
||||
@@ -140,13 +141,32 @@ public class AiEngineController {
|
||||
}
|
||||
|
||||
private void runOrchestrationStream(AiWorkflowRequest request, SseEmitter emitter) {
|
||||
AiWorkflowService.ProgressListener listener =
|
||||
new AiWorkflowService.ProgressListener() {
|
||||
@Override
|
||||
public void onProgress(AiWorkflowProgressEvent event) {
|
||||
sendEvent(emitter, "progress", event);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onHeartbeat() {
|
||||
// Forward upstream heartbeats so the SSE pipe stays visibly alive between
|
||||
// real progress events; if the frontend has gone away, sendEvent throws,
|
||||
// which propagates up through the stream consumer and closes our upstream
|
||||
// engine connection so the engine can cancel its in-flight workflow.
|
||||
sendEvent(emitter, "heartbeat", Map.of());
|
||||
}
|
||||
};
|
||||
try {
|
||||
AiWorkflowResponse result =
|
||||
aiWorkflowService.orchestrate(
|
||||
request, progress -> sendEvent(emitter, "progress", progress));
|
||||
AiWorkflowResponse result = aiWorkflowService.orchestrate(request, listener);
|
||||
registerFileResultAsJob(result);
|
||||
sendEvent(emitter, "result", result);
|
||||
emitter.complete();
|
||||
} catch (ClientDisconnectedException e) {
|
||||
// The frontend gave up mid-stream. The exception unwinding through orchestrate()
|
||||
// already closed the upstream engine connection (engine sees disconnect and cancels).
|
||||
// The emitter is already toast; nothing useful left to send.
|
||||
log.debug("Client disconnected mid-stream; aborting workflow", e);
|
||||
} catch (Exception e) {
|
||||
log.error("AI orchestration stream failed", e);
|
||||
// Emit an error frame for the frontend and then complete normally. Using
|
||||
@@ -192,7 +212,21 @@ public class AiEngineController {
|
||||
try {
|
||||
emitter.send(SseEmitter.event().name(name).data(data, MediaType.APPLICATION_JSON));
|
||||
} catch (IOException e) {
|
||||
log.debug("Failed to send SSE event (client may have disconnected)", e);
|
||||
// Surface the disconnect so the streaming pipeline unwinds: callers higher up close
|
||||
// the upstream engine connection, which lets the engine cancel its in-flight workflow.
|
||||
// Without this, the engine would keep producing (and billing for) tokens whose results
|
||||
// nobody is reading.
|
||||
throw new ClientDisconnectedException("Client disconnected from SSE stream", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Thrown by {@link #sendEvent} when the SSE emitter's underlying connection is gone. Treated as
|
||||
* a signal to abort the workflow, not as an error to report.
|
||||
*/
|
||||
private static final class ClientDisconnectedException extends RuntimeException {
|
||||
ClientDisconnectedException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* Body for {@code POST /api/v1/documents} on the AI engine. Sent by Java when the engine reports
|
||||
* {@code need_ingest} and the requested document's extracted content must be stored before the
|
||||
* workflow can continue.
|
||||
*/
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AiDocumentIngestRequest {
|
||||
|
||||
private String documentId;
|
||||
|
||||
private String source;
|
||||
|
||||
private List<AiPageText> pageText;
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonSubTypes;
|
||||
import com.fasterxml.jackson.annotation.JsonTypeInfo;
|
||||
|
||||
/**
|
||||
* Typed engine-emitted progress detail, mirroring the Python {@code ProgressEvent} discriminated
|
||||
* union (see {@code engine/src/stirling/contracts/progress.py}). Carried inside {@link
|
||||
* AiWorkflowProgressEvent#getEngineDetail()} for {@link AiWorkflowPhase#ENGINE_PROGRESS} events.
|
||||
*
|
||||
* <p>Sealed so adding a new engine-side phase forces a matching subtype on the Java side instead of
|
||||
* silently passing through as an opaque map. The {@code phase} string is the discriminator and
|
||||
* stays on the wire so the frontend (which doesn't know about Java's class hierarchy) can switch on
|
||||
* it.
|
||||
*/
|
||||
@JsonTypeInfo(
|
||||
use = JsonTypeInfo.Id.NAME,
|
||||
include = JsonTypeInfo.As.EXISTING_PROPERTY,
|
||||
property = "phase",
|
||||
visible = true)
|
||||
@JsonSubTypes({
|
||||
@JsonSubTypes.Type(
|
||||
value = AiEngineProgressDetail.WholeDocReadStarted.class,
|
||||
name = "whole_doc_read_started"),
|
||||
@JsonSubTypes.Type(
|
||||
value = AiEngineProgressDetail.WholeDocSliceDone.class,
|
||||
name = "whole_doc_slice_done"),
|
||||
@JsonSubTypes.Type(
|
||||
value = AiEngineProgressDetail.WholeDocCompressionRound.class,
|
||||
name = "whole_doc_compression_round"),
|
||||
@JsonSubTypes.Type(
|
||||
value = AiEngineProgressDetail.WholeDocReadDone.class,
|
||||
name = "whole_doc_read_done"),
|
||||
})
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public sealed interface AiEngineProgressDetail {
|
||||
|
||||
String phase();
|
||||
|
||||
record WholeDocReadStarted(String phase, String question, int pages, int slices)
|
||||
implements AiEngineProgressDetail {}
|
||||
|
||||
record WholeDocSliceDone(
|
||||
String phase,
|
||||
int completed,
|
||||
int total,
|
||||
String pages,
|
||||
int durationMs,
|
||||
int excerpts,
|
||||
int facts)
|
||||
implements AiEngineProgressDetail {}
|
||||
|
||||
record WholeDocCompressionRound(String phase, int roundNumber, int notesIn, int groups)
|
||||
implements AiEngineProgressDetail {}
|
||||
|
||||
record WholeDocReadDone(String phase, int completed, int slices, double durationSeconds)
|
||||
implements AiEngineProgressDetail {}
|
||||
}
|
||||
+2
-2
@@ -4,11 +4,11 @@ import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/** A single page of extracted text for RAG ingest requests. */
|
||||
/** A single page of extracted text for document ingest requests. */
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AiRagPageText {
|
||||
public class AiPageText {
|
||||
|
||||
private int pageNumber;
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* Body for {@code POST /api/v1/rag/documents} on the AI engine. Sent by Java when the engine
|
||||
* reports {@code need_ingest} and the requested document's extracted content must be stored before
|
||||
* the workflow can continue.
|
||||
*/
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AiRagIngestRequest {
|
||||
|
||||
private String documentId;
|
||||
|
||||
private String source;
|
||||
|
||||
private List<AiRagPageText> pageText;
|
||||
}
|
||||
+2
-1
@@ -20,7 +20,8 @@ public enum AiWorkflowOutcome {
|
||||
TOOL_CALL("tool_call"),
|
||||
COMPLETED("completed"),
|
||||
UNSUPPORTED_CAPABILITY("unsupported_capability"),
|
||||
CANNOT_CONTINUE("cannot_continue");
|
||||
CANNOT_CONTINUE("cannot_continue"),
|
||||
GENERATE_FILE("generate_file");
|
||||
|
||||
private final String value;
|
||||
|
||||
|
||||
+7
-1
@@ -9,7 +9,13 @@ public enum AiWorkflowPhase {
|
||||
CALLING_ENGINE("calling_engine"),
|
||||
EXTRACTING_CONTENT("extracting_content"),
|
||||
EXECUTING_TOOL("executing_tool"),
|
||||
PROCESSING("processing");
|
||||
PROCESSING("processing"),
|
||||
/**
|
||||
* Generic engine-emitted progress event (e.g. chunked-reasoner slice progress). The original
|
||||
* engine event JSON is carried in {@link AiWorkflowProgressEvent#getEngineDetail()}, including
|
||||
* a specific {@code phase} string the frontend can switch on.
|
||||
*/
|
||||
ENGINE_PROGRESS("engine_progress");
|
||||
|
||||
private final String value;
|
||||
|
||||
|
||||
+22
-2
@@ -23,8 +23,17 @@ public class AiWorkflowProgressEvent {
|
||||
/** Total number of plan steps, for {@link AiWorkflowPhase#EXECUTING_TOOL} events. */
|
||||
private Integer stepCount;
|
||||
|
||||
/**
|
||||
* Engine-emitted event payload, for {@link AiWorkflowPhase#ENGINE_PROGRESS} events. The payload
|
||||
* is a typed subtype keyed on its {@code phase} string (e.g. {@code "whole_doc_slice_done"})
|
||||
* carrying phase-specific fields (slice index, page range, durations, etc.) that the frontend
|
||||
* can render as detailed progress.
|
||||
*/
|
||||
private AiEngineProgressDetail engineDetail;
|
||||
|
||||
public static AiWorkflowProgressEvent of(AiWorkflowPhase phase) {
|
||||
return new AiWorkflowProgressEvent(phase, System.currentTimeMillis(), null, null, null);
|
||||
return new AiWorkflowProgressEvent(
|
||||
phase, System.currentTimeMillis(), null, null, null, null);
|
||||
}
|
||||
|
||||
public static AiWorkflowProgressEvent executingTool(String tool, int stepIndex, int stepCount) {
|
||||
@@ -33,6 +42,17 @@ public class AiWorkflowProgressEvent {
|
||||
System.currentTimeMillis(),
|
||||
tool,
|
||||
stepIndex,
|
||||
stepCount);
|
||||
stepCount,
|
||||
null);
|
||||
}
|
||||
|
||||
public static AiWorkflowProgressEvent engineProgress(AiEngineProgressDetail detail) {
|
||||
return new AiWorkflowProgressEvent(
|
||||
AiWorkflowPhase.ENGINE_PROGRESS,
|
||||
System.currentTimeMillis(),
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
detail);
|
||||
}
|
||||
}
|
||||
|
||||
+10
@@ -4,6 +4,8 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
@@ -20,6 +22,14 @@ public class AiWorkflowResponse {
|
||||
@Schema(description = "Answer returned by the AI workflow when applicable")
|
||||
private String answer;
|
||||
|
||||
@JsonProperty("content")
|
||||
@Schema(description = "Text content to package as a file (generate_file outcomes)")
|
||||
private String generatedContent;
|
||||
|
||||
@JsonProperty("filename")
|
||||
@Schema(description = "Desired output filename for generate_file outcomes")
|
||||
private String generatedFilename;
|
||||
|
||||
@Schema(description = "Summary returned by the AI workflow when applicable")
|
||||
private String summary;
|
||||
|
||||
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
|
||||
import technology.tabula.writers.CSVWriter;
|
||||
|
||||
/** Exposes Tabula's protected {@link CSVWriter#CSVWriter(CSVFormat)} constructor. */
|
||||
public class FlexibleCSVWriter extends CSVWriter {
|
||||
|
||||
public FlexibleCSVWriter() {
|
||||
super();
|
||||
}
|
||||
|
||||
public FlexibleCSVWriter(CSVFormat csvFormat) {
|
||||
super(csvFormat);
|
||||
}
|
||||
}
|
||||
+66
@@ -7,6 +7,8 @@ import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.net.http.HttpTimeoutException;
|
||||
import java.time.Duration;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.HttpStatus;
|
||||
@@ -83,6 +85,70 @@ public class AiEngineClient {
|
||||
return response.body();
|
||||
}
|
||||
|
||||
/**
|
||||
* POST a JSON body and consume the response as a stream of NDJSON lines. Each line is passed to
|
||||
* {@code lineConsumer} in arrival order; the call returns when the engine closes the stream.
|
||||
*
|
||||
* <p>This is the right shape for long-running orchestrator calls that emit incremental
|
||||
* progress. The total HTTP timeout is the long-running timeout (typically 600s+), but in
|
||||
* practice line arrival keeps the connection logically alive: as long as the engine emits
|
||||
* events, the work is progressing. Genuine engine hangs still hit the total timeout.
|
||||
*/
|
||||
public void streamPost(String path, String jsonBody, Consumer<String> lineConsumer)
|
||||
throws IOException {
|
||||
ApplicationProperties.AiEngine config = applicationProperties.getAiEngine();
|
||||
if (!config.isEnabled()) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.SERVICE_UNAVAILABLE, "AI engine is not enabled");
|
||||
}
|
||||
|
||||
String url = config.getUrl().stripTrailing() + path;
|
||||
Duration timeout = Duration.ofSeconds(config.getLongRunningTimeoutSeconds());
|
||||
log.debug(
|
||||
"Proxying AI engine streaming request to {} (timeout {}s)",
|
||||
url,
|
||||
timeout.toSeconds());
|
||||
|
||||
HttpRequest request =
|
||||
HttpRequest.newBuilder()
|
||||
.uri(URI.create(url))
|
||||
.header("Content-Type", "application/json")
|
||||
.header("Accept", "application/x-ndjson")
|
||||
.timeout(timeout)
|
||||
.POST(HttpRequest.BodyPublishers.ofString(jsonBody))
|
||||
.build();
|
||||
|
||||
HttpResponse<Stream<String>> response;
|
||||
try {
|
||||
response = httpClient.send(request, HttpResponse.BodyHandlers.ofLines());
|
||||
} catch (HttpTimeoutException e) {
|
||||
throw new ResponseStatusException(HttpStatus.GATEWAY_TIMEOUT, "AI engine timed out", e);
|
||||
} catch (IOException e) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.SERVICE_UNAVAILABLE, "AI engine unreachable: " + e.getMessage(), e);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.SERVICE_UNAVAILABLE, "AI engine request was interrupted");
|
||||
}
|
||||
|
||||
int status = response.statusCode();
|
||||
if (status >= 400) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.valueOf(status >= 500 ? 502 : status),
|
||||
"AI engine returned error: " + status);
|
||||
}
|
||||
|
||||
try (Stream<String> lines = response.body()) {
|
||||
lines.forEach(
|
||||
line -> {
|
||||
if (!line.isEmpty()) {
|
||||
lineConsumer.accept(line);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public String get(String path) throws IOException {
|
||||
ApplicationProperties.AiEngine config = applicationProperties.getAiEngine();
|
||||
if (!config.isEnabled()) {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user