From 6f7f28946c6643aecd96da043e9a1d1549193437 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:19:26 +0000 Subject: [PATCH 01/45] Set deployment: false on environment jobs that do not deploy (#7562) # Description of Changes thanks ludy for the tip :P --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details. --- .github/workflows/PR-Auto-Deploy-V2.yml | 5 ++++- .github/workflows/PR-Demo-cleanup.yml | 5 ++++- .github/workflows/backend-build.yml | 4 +++- .github/workflows/build-enterprise.yml | 8 ++++++-- .github/workflows/build.yml | 4 +++- .github/workflows/check-licence.yml | 4 +++- .github/workflows/check-openapi.yml | 4 +++- .github/workflows/db-migration-test.yml | 4 +++- .github/workflows/docker-compose-tests.yml | 4 +++- .github/workflows/e2e-live.yml | 4 +++- .github/workflows/frontend-backend-licenses-update.yml | 8 ++++++-- .github/workflows/multiOSReleases.yml | 8 ++++++-- .github/workflows/nightly.yml | 4 +++- .github/workflows/tauri-build.yml | 4 +++- .github/workflows/test-build-docker.yml | 4 +++- 15 files changed, 56 insertions(+), 18 deletions(-) diff --git a/.github/workflows/PR-Auto-Deploy-V2.yml b/.github/workflows/PR-Auto-Deploy-V2.yml index 0f07aabbe5..4375b1b8b0 100644 --- a/.github/workflows/PR-Auto-Deploy-V2.yml +++ b/.github/workflows/PR-Auto-Deploy-V2.yml @@ -462,7 +462,10 @@ jobs: }); cleanup-v2-deployment: - environment: pr-preview + # Tearing a preview down is not a deployment - no deployment object. + environment: + name: pr-preview + deployment: false if: github.event.action == 'closed' runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/PR-Demo-cleanup.yml b/.github/workflows/PR-Demo-cleanup.yml index 1407939994..098f8d7803 100644 --- a/.github/workflows/PR-Demo-cleanup.yml +++ b/.github/workflows/PR-Demo-cleanup.yml @@ -9,7 +9,10 @@ permissions: jobs: cleanup: - environment: pr-preview + # Tearing a preview down is not a deployment - no deployment object. + environment: + name: pr-preview + deployment: false if: github.event.action == 'closed' runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/backend-build.yml b/.github/workflows/backend-build.yml index 6623940bce..596be96e8c 100644 --- a/.github/workflows/backend-build.yml +++ b/.github/workflows/backend-build.yml @@ -20,7 +20,9 @@ permissions: jobs: build: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/build-enterprise.yml b/.github/workflows/build-enterprise.yml index b4a8373ccc..194d86d9da 100644 --- a/.github/workflows/build-enterprise.yml +++ b/.github/workflows/build-enterprise.yml @@ -42,7 +42,9 @@ jobs: uses: ./.github/workflows/_runner-pick.yml playwright-e2e-enterprise: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false needs: pick # Skip on fork PRs / untrusted authors: they have no PREMIUM_KEY_ENTERPRISE, # so the suite can't boot premium and would fail. See the header comment. @@ -325,7 +327,9 @@ jobs: # Multi-node regression: builds + seeds the clustered stack (testing/compose/docker-compose-multinode.yml) # and runs behave features/multinode. Licence-gated, so it runs after the Playwright job (not in parallel). multinode-e2e: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false needs: [pick, playwright-e2e-enterprise] # Nightly cron + manual dispatch only (heavy build), fork-gated for the licence secret. if: >- diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 566262d24f..1feaff2560 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,7 +61,9 @@ jobs: filters: .github/config/.files.yaml gradle-cache-prime: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false name: Prime shared Gradle cache needs: [files-changed] runs-on: ubuntu-latest diff --git a/.github/workflows/check-licence.yml b/.github/workflows/check-licence.yml index 17c64d5c64..4e04a83656 100644 --- a/.github/workflows/check-licence.yml +++ b/.github/workflows/check-licence.yml @@ -10,7 +10,9 @@ permissions: jobs: check-licence: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false runs-on: ubuntu-latest steps: - name: Harden Runner diff --git a/.github/workflows/check-openapi.yml b/.github/workflows/check-openapi.yml index ed83447335..bc9b302857 100644 --- a/.github/workflows/check-openapi.yml +++ b/.github/workflows/check-openapi.yml @@ -11,7 +11,9 @@ permissions: jobs: check-generate-openapi-docs: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false runs-on: ubuntu-latest steps: - name: Harden Runner diff --git a/.github/workflows/db-migration-test.yml b/.github/workflows/db-migration-test.yml index ccb46d3988..785073944e 100644 --- a/.github/workflows/db-migration-test.yml +++ b/.github/workflows/db-migration-test.yml @@ -13,7 +13,9 @@ permissions: jobs: migration-test: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false runs-on: ubuntu-latest timeout-minutes: 30 steps: diff --git a/.github/workflows/docker-compose-tests.yml b/.github/workflows/docker-compose-tests.yml index 039c73e5db..439d4240b2 100644 --- a/.github/workflows/docker-compose-tests.yml +++ b/.github/workflows/docker-compose-tests.yml @@ -17,7 +17,9 @@ permissions: jobs: docker-compose-tests: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false runs-on: ubuntu-latest permissions: actions: write diff --git a/.github/workflows/e2e-live.yml b/.github/workflows/e2e-live.yml index 43d66dd1cf..844d26a3d1 100644 --- a/.github/workflows/e2e-live.yml +++ b/.github/workflows/e2e-live.yml @@ -11,7 +11,9 @@ permissions: jobs: playwright-e2e-live: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false runs-on: ubuntu-latest timeout-minutes: 30 steps: diff --git a/.github/workflows/frontend-backend-licenses-update.yml b/.github/workflows/frontend-backend-licenses-update.yml index 7fd0136718..9aef2316d2 100644 --- a/.github/workflows/frontend-backend-licenses-update.yml +++ b/.github/workflows/frontend-backend-licenses-update.yml @@ -43,7 +43,9 @@ jobs: generate-frontend-license-report: # ci-bot, not bot-identity: this job runs on PRs too, and bot-identity is main-only. - environment: ci-bot + environment: + name: ci-bot + deployment: false if: needs.files-changed.outputs.licenses-frontend == 'true' name: Generate Frontend License Report needs: files-changed @@ -319,7 +321,9 @@ jobs: generate-backend-license-report: # ci-bot, not bot-identity: this job runs on PRs too, and bot-identity is main-only. - environment: ci-bot + environment: + name: ci-bot + deployment: false if: needs.files-changed.outputs.licenses-backend == 'true' needs: files-changed name: Generate Backend License Report diff --git a/.github/workflows/multiOSReleases.yml b/.github/workflows/multiOSReleases.yml index 0f0b2d3585..0ac94ffe68 100644 --- a/.github/workflows/multiOSReleases.yml +++ b/.github/workflows/multiOSReleases.yml @@ -38,7 +38,9 @@ permissions: jobs: determine-matrix: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false if: ${{ vars.CI_PROFILE != 'lite' }} runs-on: ubuntu-latest outputs: @@ -116,7 +118,9 @@ jobs: env: INPUT_PLATFORM: ${{ github.event.inputs.platform }} build-jars: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false needs: determine-matrix runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c92d17f027..65c25b7b66 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -127,7 +127,9 @@ jobs: # Runs the @nightly tag (conversion scenarios) plus a 10-shard concurrency run # of every other feature. cucumber-nightly: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false name: Cucumber (nightly scenarios + full concurrency) runs-on: ubuntu-latest # Fork pull requests get no MAVEN_* secrets, so the image build cannot work. diff --git a/.github/workflows/tauri-build.yml b/.github/workflows/tauri-build.yml index e3f3122773..0a82647690 100644 --- a/.github/workflows/tauri-build.yml +++ b/.github/workflows/tauri-build.yml @@ -63,7 +63,9 @@ jobs: determine-matrix: # Only probes APPLE_CERTIFICATE for presence, so it stays on the unrestricted # signing environment - release-signing would block every PR run. - environment: ci-signing + environment: + name: ci-signing + deployment: false if: ${{ vars.CI_PROFILE != 'lite' }} runs-on: ubuntu-latest outputs: diff --git a/.github/workflows/test-build-docker.yml b/.github/workflows/test-build-docker.yml index 12d5a35a1f..37cb7cb546 100644 --- a/.github/workflows/test-build-docker.yml +++ b/.github/workflows/test-build-docker.yml @@ -37,7 +37,9 @@ jobs: # spring-security=true matrix entry if `task backend:build` and # `task backend:build:ci` produce equivalent JARs (verify before wiring). test-build-docker-images: - environment: ci-unsigned + environment: + name: ci-unsigned + deployment: false runs-on: ubuntu-latest strategy: fail-fast: false From 0f8803f35f14cf8e9c191cfc5b6101a33586674c Mon Sep 17 00:00:00 2001 From: EthanHealy01 <80844253+EthanHealy01@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:27:39 +0000 Subject: [PATCH 02/45] Require the policy-management role to run a policy against its sources (#7565) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Running a stored policy against its **configured sources** (`POST /api/v1/policies/{id}/trigger`, the manual "run now") now requires the policy-management role — global admin self-hosted, team leader on SaaS — alongside the existing team scoping. ## Why A source sweep operates on the team's configured sources using the server's stored connection credentials, so it belongs with the other policy-management capabilities rather than with ordinary use. Team scoping on its own didn't express that distinction. ## Not changed - `POST /{id}/run` — running a policy over documents the **caller supplied** stays open to every team member. That's ordinary editor enforcement on upload and export, and gating it would break it. - Ad-hoc pipelines (`/run`, `/run/stream`). - The scheduled, folder-watch and webhook triggers. - Single-user deployments (login disabled), which have no roles. ## Implementation `PolicyManagementAuthority` gains `canTriggerPolicies()`, kept separate from `canEditPolicies()` so the two capabilities can diverge later. Both current implementations grant it to the same principals that may edit policies. ## Tests - role absent → 403, rejected before any run starts - role present → 202 - login disabled → check skipped entirely - `/{id}/run` asserted to consult neither authority method, so the gate can't quietly extend to the editor path later --- .../AdminPolicyManagementAuthority.java | 5 ++ .../config/PolicyManagementAuthority.java | 11 +++ .../policy/controller/PolicyController.java | 30 +++++++- .../AdminPolicyManagementAuthorityTest.java | 12 +++ .../controller/PolicyControllerTest.java | 74 +++++++++++++++++++ .../TeamLeaderPolicyManagementAuthority.java | 5 ++ ...amLeaderPolicyManagementAuthorityTest.java | 12 +++ 7 files changed, 145 insertions(+), 4 deletions(-) diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/AdminPolicyManagementAuthority.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/AdminPolicyManagementAuthority.java index a49c8e5aa9..6d8229aa26 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/AdminPolicyManagementAuthority.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/AdminPolicyManagementAuthority.java @@ -26,6 +26,11 @@ public class AdminPolicyManagementAuthority implements PolicyManagementAuthority return userService.isCurrentUserAdmin(); } + @Override + public boolean canTriggerPolicies() { + return userService.isCurrentUserAdmin(); + } + @Override public Long currentUserTeamId() { String username = userService.getCurrentUsername(); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/PolicyManagementAuthority.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/PolicyManagementAuthority.java index 0ea3c298ad..d7e4f50ad1 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/PolicyManagementAuthority.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/PolicyManagementAuthority.java @@ -12,6 +12,17 @@ public interface PolicyManagementAuthority { /** Whether the current user may create, edit, or delete policies (for their own team). */ boolean canEditPolicies(); + /** + * Whether the current user may run a policy against its configured sources (the manual + * "run now" sweep). Kept separate from {@link #canEditPolicies()} because the two are distinct + * capabilities, even where a deployment grants both to the same people: a sweep operates on the + * team's configured sources using the server's stored connection credentials, which makes it a + * policy-management capability rather than ordinary use. Running a policy over the caller's + * own uploaded files is not covered by this and stays open to every team member — that + * is ordinary editor enforcement. + */ + boolean canTriggerPolicies(); + /** * The team that scopes the current user's policies — the team a new policy is stamped with and * the only team whose policies the user may see/run/edit. {@code null} when it can't be diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java index 506bb75578..778a04e169 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java @@ -432,9 +432,10 @@ public class PolicyController { * admin gets no say on SaaS. Team scoping (which team's policies) is enforced separately by * {@link PolicyAccessGuard}. Every mutation routes through {@link #savePolicy} (pause/resume * re-save with a flipped {@code enabled} flag) or {@link #deletePolicy}, so gating those two - * covers them all; runs ({@code /run}) stay open to the team. Single-user deployments (login - * disabled) have no such role, so they trust the local operator. The path allowlist for folder - * sources/outputs is enforced separately by {@link PolicyValidator} at validation time. + * covers them all; runs over the caller's own files ({@code /{id}/run}) stay open to the team, + * while source sweeps are gated by {@link #requirePolicySweepAllowed}. Single-user deployments + * (login disabled) have no such role, so they trust the local operator. The path allowlist for + * folder sources/outputs is enforced separately by {@link PolicyValidator} at validation time. */ private void requirePolicyEditingAllowed() { if (!applicationProperties.getSecurity().isEnableLogin()) { @@ -447,6 +448,25 @@ public class PolicyController { } } + /** + * Sweeping a policy's configured sources requires the same role as managing policies: the sweep + * operates on the team's configured sources using the server's stored connection credentials, + * which makes it a policy-management capability rather than ordinary use, and team scoping on + * its own does not express that. Deliberately narrower than it looks: it gates only the sweep, + * not {@link #runStoredPolicy}, because running a policy over documents the caller supplied is + * ordinary editor enforcement that every member performs on upload and export. + */ + private void requirePolicySweepAllowed() { + if (!applicationProperties.getSecurity().isEnableLogin()) { + return; + } + if (!policyManagementAuthority.canTriggerPolicies()) { + throw new ResponseStatusException( + HttpStatus.FORBIDDEN, + "Not permitted to run this policy against its configured sources"); + } + } + @GetMapping @Operation( summary = "List policies", @@ -571,8 +591,10 @@ public class PolicyController { + " the enabled flag (which only gates automatic triggering). Returns" + " the ids of the runs started (poll the run-status endpoint for each)" + " plus what the sweep skipped - already-processed, parked-by-failure," - + " and in-flight counts - so an empty result explains itself.") + + " and in-flight counts - so an empty result explains itself. Requires" + + " the policy-management role.") public ResponseEntity trigger(@PathVariable String policyId) { + requirePolicySweepAllowed(); Policy policy = policyStore .get(policyId) diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/config/AdminPolicyManagementAuthorityTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/config/AdminPolicyManagementAuthorityTest.java index 811aae6b4f..0f97fcaf62 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/config/AdminPolicyManagementAuthorityTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/config/AdminPolicyManagementAuthorityTest.java @@ -39,6 +39,18 @@ class AdminPolicyManagementAuthorityTest { assertFalse(authority().canEditPolicies()); } + @Test + void adminMayTriggerPolicies() { + when(userService.isCurrentUserAdmin()).thenReturn(true); + assertTrue(authority().canTriggerPolicies()); + } + + @Test + void nonAdminMayNotTriggerPolicies() { + when(userService.isCurrentUserAdmin()).thenReturn(false); + assertFalse(authority().canTriggerPolicies()); + } + @Test void currentUserTeamIdResolvesFromTheCurrentUsersTeam() { Team team = new Team(); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java index 84e9998b90..2fa675597c 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java @@ -2,6 +2,7 @@ package stirling.software.proprietary.policy.controller; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -738,5 +739,78 @@ class PolicyControllerTest { assertThat(((ResponseStatusException) e).getStatusCode()) .isEqualTo(HttpStatus.NOT_FOUND)); } + + @Test + @DisplayName("trigger is forbidden for a team member who cannot manage policies") + void triggerForbiddenForMember() { + // Sweeping a policy's configured sources is a policy-management capability, so being + // in the policy's team is not on its own enough to perform it. + applicationProperties.getSecurity().setEnableLogin(true); + when(policyManagementAuthority.canTriggerPolicies()).thenReturn(false); + + assertThatThrownBy(() -> controller.trigger("a")) + .isInstanceOf(ResponseStatusException.class) + .satisfies( + e -> + assertThat(((ResponseStatusException) e).getStatusCode()) + .isEqualTo(HttpStatus.FORBIDDEN)); + // Rejected before the policy is looked up, so no run starts. + verify(policyRunner, never()).run(any()); + verify(policyStore, never()).get(any()); + } + + @Test + @DisplayName("trigger runs for a caller who may manage policies") + void triggerAllowedForLeader() { + applicationProperties.getSecurity().setEnableLogin(true); + when(policyManagementAuthority.canTriggerPolicies()).thenReturn(true); + Policy p = policy("a", 1L); + when(policyStore.get("a")).thenReturn(Optional.of(p)); + when(policyAccessGuard.canAccess(p)).thenReturn(true); + SweepOutcome outcome = new SweepOutcome(List.of("run-a"), 1, 0, 0, 0); + when(policyRunner.run(p)).thenReturn(outcome); + + ResponseEntity response = controller.trigger("a"); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.ACCEPTED); + assertThat(response.getBody()).isEqualTo(outcome); + } + + @Test + @DisplayName("trigger skips the role check when login is disabled") + void triggerTrustsTheLocalOperator() { + // Single-user deployments have no roles at all; the gate must not lock them out of + // their + // own sweeps. + applicationProperties.getSecurity().setEnableLogin(false); + Policy p = policy("a", null); + when(policyStore.get("a")).thenReturn(Optional.of(p)); + when(policyAccessGuard.canAccess(p)).thenReturn(true); + SweepOutcome outcome = new SweepOutcome(List.of("run-a"), 1, 0, 0, 0); + when(policyRunner.run(p)).thenReturn(outcome); + + assertThat(controller.trigger("a").getStatusCode()).isEqualTo(HttpStatus.ACCEPTED); + verify(policyManagementAuthority, never()).canTriggerPolicies(); + } + + @Test + @DisplayName("running a policy over the caller's own files stays open to any member") + void storedRunIsNotGatedByRole() { + // Editor enforcement: every member's upload/export runs the team's stored policies on + // their own documents. Gating this the way the sweep is gated would break the editor. + applicationProperties.getSecurity().setEnableLogin(true); + Policy p = policy("a", 1L); + when(policyStore.get("a")).thenReturn(Optional.of(p)); + when(policyAccessGuard.canAccess(p)).thenReturn(true); + when(policyRunner.runWith(eq(p), any(), eq(PolicyProgressListener.NOOP))) + .thenReturn(handle("run-9")); + + ResponseEntity> response = + assertDoesNotThrow(() -> controller.runStoredPolicy("a", new PolicyRunFiles())); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.ACCEPTED); + verify(policyManagementAuthority, never()).canTriggerPolicies(); + verify(policyManagementAuthority, never()).canEditPolicies(); + } } } diff --git a/app/saas/src/main/java/stirling/software/saas/security/TeamLeaderPolicyManagementAuthority.java b/app/saas/src/main/java/stirling/software/saas/security/TeamLeaderPolicyManagementAuthority.java index e2f5b65b47..0ce5e8e308 100644 --- a/app/saas/src/main/java/stirling/software/saas/security/TeamLeaderPolicyManagementAuthority.java +++ b/app/saas/src/main/java/stirling/software/saas/security/TeamLeaderPolicyManagementAuthority.java @@ -25,6 +25,11 @@ public class TeamLeaderPolicyManagementAuthority implements PolicyManagementAuth return teamSecurity.isCurrentUserTeamLeader(); } + @Override + public boolean canTriggerPolicies() { + return teamSecurity.isCurrentUserTeamLeader(); + } + @Override public Long currentUserTeamId() { return teamSecurity.currentUserTeamId(); diff --git a/app/saas/src/test/java/stirling/software/saas/security/TeamLeaderPolicyManagementAuthorityTest.java b/app/saas/src/test/java/stirling/software/saas/security/TeamLeaderPolicyManagementAuthorityTest.java index 70cd360d7c..2c37980a5c 100644 --- a/app/saas/src/test/java/stirling/software/saas/security/TeamLeaderPolicyManagementAuthorityTest.java +++ b/app/saas/src/test/java/stirling/software/saas/security/TeamLeaderPolicyManagementAuthorityTest.java @@ -32,6 +32,18 @@ class TeamLeaderPolicyManagementAuthorityTest { assertFalse(authority().canEditPolicies()); } + @Test + void teamLeaderMayTriggerPolicies() { + when(teamSecurity.isCurrentUserTeamLeader()).thenReturn(true); + assertTrue(authority().canTriggerPolicies()); + } + + @Test + void nonLeaderMayNotTriggerPolicies() { + when(teamSecurity.isCurrentUserTeamLeader()).thenReturn(false); + assertFalse(authority().canTriggerPolicies()); + } + @Test void currentUserTeamIdDelegatesToTeamSecurity() { when(teamSecurity.currentUserTeamId()).thenReturn(9L); From 6bae9d516dc029869fee80ec5c2e92da89a5d541 Mon Sep 17 00:00:00 2001 From: ConnorYoh <40631091+ConnorYoh@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:46:00 +0000 Subject: [PATCH 03/45] chore(saas): one task per environment, and make the frontend follow it (#7483) ## The problem The `dev` profile hardcoded one project ref (`qacaivhsjtftfwtgjvva`) in five places: the ref, the Supabase URL, the publishable key, the datasource host and the meter endpoint. That made it both the shared environment everyone relies on *and* the only thing you could point the backend at. Testing an open SaaS PR meant hand-overriding all five via env just to reach that PR's Supabase preview branch, which is the only place the PR's migrations have actually been applied. Get it wrong and you see `relation "stirling_pdf." does not exist` for a table the PR added, which is what happened on [#7414](https://github.com/Stirling-Tools/Stirling-PDF/pull/7414). ## One task per environment ```bash task dev:saas # backend + frontend + engine, against this PR's preview branch task staging:saas # backend + frontend + engine, against the shared v3 project task backend:dev:saas # backend only, preview branch task backend:staging:saas # backend only, v3 ``` | | how | vars | project | |---|---|---|---| | prod | `PROFILES=none` | `SAAS_DB_*` | the live one | | staging | `PROFILES=staging` | `SAAS_STAGING_*` | pinned to v3, always there | | dev | `PROFILES=dev` | `SAAS_DEV_*` | follows a SaaS PR's preview branch | `PROFILES` is still the underlying switch, so the old spelling keeps working. Production deliberately has no named task: reaching it should take a conscious `PROFILES=none`, not a tab-complete. **staging** is the old `dev` configuration, moved and kept pinned. The value of a shared environment is that it is still there tomorrow: reproduce a bug, paste a link to a colleague, share data. **dev** is parameterised by `SAAS_DEV_PROJECT_REF` and derives the Supabase URL, JWT issuer, JWKS, meter endpoint and (unless overridden) the database host from it. Switching which PR you are testing is one variable instead of five. With no ref set, `task backend:dev:saas` stops and says what to set rather than falling back. ## The frontend was the real gap `frontend/editor/.env` is committed and pins the **production** Supabase project, and nothing in the frontend knew about dev or staging. So `task dev:saas` gave you a backend on a preview branch and a login against prod, unless you happened to have hand-written `frontend/editor/.env.saas.local`. The dev tasks now read the backend's env files and derive `VITE_SUPABASE_URL` and `VITE_SUPABASE_PUBLISHABLE_DEFAULT_KEY` from the same project ref the backend resolved, so the two halves cannot point at different projects. Nothing to keep in sync by hand, no new vite mode, and `SAAS_ENV=prod` opts back out to the committed values. ## Where to put your local values Two files, both gitignored, neither ever committed: **`app/.env.saas.local`** is the only one you normally need. The tasks load it for the backend *and* the frontend. ```bash # staging: everything else is already defaulted, so this is all it takes SAAS_STAGING_DB_PASSWORD=... # dev: the preview branch of the PR you are testing, from its "Supabase Preview" check. # A branch has its OWN password and API keys; the parent project's will not authenticate. SAAS_DEV_PROJECT_REF=... SAAS_DEV_DB_PASSWORD=... SAAS_DEV_PUBLISHABLE_KEY=... # prod, if you ever need it SAAS_DB_PROJECT_REF=... SAAS_DB_URL=... SAAS_DB_PASSWORD=... SUPABASE_EDGE_FUNCTION_SECRET=... ``` **`frontend/editor/.env.saas.local`** is no longer needed for choosing a Supabase project, and is best left empty or deleted. If you have one from before this PR, note that the task-supplied values now win, which is the point: the frontend follows the backend. **A blank is not the same as absent.** A dotenv line with an empty value still *sets* the variable, and Spring's `${VAR:default}` only falls back when a variable is absent. So `.env.saas` lists what you must set as blanks, and leaves out the two `*_DB_URL` overrides, which have real defaults to fall back to. This is not theoretical, see below. Committed `app/.env.saas` holds non-secret defaults only. Real secrets are passwords, the edge-function secret and service-role keys. Project refs and publishable keys are neither: a ref is the public `.supabase.co` subdomain and a publishable key ships in the browser bundle by design, which is why `frontend/editor/.env` has always carried prod's. ## Three bugs found while building the tasks All three were in this PR's own earlier commits, and all three were caught by actually booting things rather than by reading the config. **staging could not boot at all.** A blank `SAAS_STAGING_DB_URL=` in `.env.saas` set the variable to empty, so `${SAAS_STAGING_DB_URL:jdbc:...}` resolved to `""` and startup failed with `spring.datasource.url is required when the saas profile is active`. The file already carried a comment warning about exactly this; it had only been applied to the dev block. The original verification for this PR was "placeholders resolve" and "the task parses", neither of which boots anything. **The dev to staging fallback ran `ddl-auto=update` against shared v3.** The dev profile sets `update`, which is right for a disposable preview branch, and separately fell back to staging's project ref. Together that meant Hibernate was free to reconcile tables that RLS policies depend on. `application-staging.properties` pins `none`, but that only applies when the staging profile is the active one, which it was not on the fallback path. There is no fallback now: with no ref the task stops before gradle, and the frontend fails the same way, both naming the variable. **`PROFILES=` never selected production.** Go template `default` treats `""` as absent, so it silently resolved back to `dev`. It is `PROFILES=none` now. ## Two choices worth reviewing **Staging keeps its committed project ref**, now as a `${SAAS_STAGING_PROJECT_REF:...}` default in one place, with the URL, database host and meter endpoint all derived from it. So staging still works with zero setup, and repointing it is one variable. Nothing in CI referenced the ref or the profile. Its publishable key default carries no inline `gitleaks:allow`: a trailing comment in a `.properties` file is part of the value, so the pragma ended up inside the key. It is in `.gitleaksignore` instead. **`SAAS_DEV_DB_URL` still overrides the whole URL**, so a branch needing the pooler host rather than the direct one is reachable without touching committed config. ## Verification - `task backend:staging:saas` boots against v3 and serves `200`. It could not boot before this commit. - `task backend:dev:saas` with no ref stops before gradle naming the variable, and `PROFILES=none` still reaches production. `task frontend:dev:saas` fails the same way; `SAAS_ENV=staging` still resolves with no local config. - Frontend routing picks the SaaS runner for dev/staging and the plain runner for prod; the derivation returns the right URL and key for each. - Vite's `process.env` precedence and Task's dotenv/env semantics were measured, not assumed. That is how one trap surfaced: Task sets an `env:` key even when its value resolves to empty, and Vite treats an empty `process.env` `VITE_*` as authoritative over a committed `.env`. Putting the Supabase vars on the shared `dev:_run` would have blanked Supabase config for the core, proprietary and desktop dev servers, so the SaaS path has its own runner. - `:saas:spotlessApply` and `:saas:compileJava` green. `DevProfileProjectNotice` becomes `SaasProjectNotice` and covers both profiles, stating the project ref and `ddl-auto` at startup so which environment you are on is never a guess. No behaviour change for prod: the `saas` profile is untouched. --- .gitleaksignore | 5 ++ .taskfiles/backend.yml | 55 ++++++++++++-- .taskfiles/frontend.yml | 74 ++++++++++++++++--- Taskfile.yml | 21 +++++- app/.env.saas | 52 ++++++++----- .../saas/config/SaasProjectNotice.java | 53 +++++++++++++ .../main/resources/application-dev.properties | 38 ++++++---- .../resources/application-staging.properties | 39 ++++++++++ 8 files changed, 286 insertions(+), 51 deletions(-) create mode 100644 app/saas/src/main/java/stirling/software/saas/config/SaasProjectNotice.java create mode 100644 app/saas/src/main/resources/application-staging.properties diff --git a/.gitleaksignore b/.gitleaksignore index 12d98aebeb..c3917e985f 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -27,3 +27,8 @@ app/core/src/main/java/stirling/software/SPDF/pdf/signature/CreateSignatureBase. # Supabase publishable key (public by design, RLS-protected) used as a CI fallback # default in the tauri-build workflow when the GitHub secret is unset - not a real secret. .github/workflows/tauri-build.yml:generic-api-key:402 + +# Staging Supabase publishable key (public by design). Ignored here rather than with an +# inline gitleaks:allow because a trailing comment in a .properties file is part of the +# value, so the pragma would end up inside the key. +app/saas/src/main/resources/application-staging.properties:generic-api-key:16 diff --git a/.taskfiles/backend.yml b/.taskfiles/backend.yml index 63773f61fc..08a12b9535 100644 --- a/.taskfiles/backend.yml +++ b/.taskfiles/backend.yml @@ -57,16 +57,57 @@ tasks: - cmd: ./gradlew clean bootRun -PbuildWithFrontend=true platforms: [linux, darwin] + # SaaS backend. dev:saas -> the PR's preview branch, staging:saas -> shared v3, + # PROFILES=none -> production against your own SAAS_DB_*. Production has no named + # task on purpose. Use `none`, not an empty value: Go template `default` treats "" + # as absent and would resolve back to dev. + dev:saas: - desc: "Start backend in SaaS flavor against Supabase" - # `dotenv:` reads from the root Taskfile's directory (".") because this - # subtaskfile is included with `dir: .`. + desc: "Start SaaS backend against the current PR's Supabase preview branch" + dotenv: ['app/.env.saas.local', 'app/.env.saas'] + vars: + PROFILES: '{{.PROFILES | default "dev"}}' + cmds: + # Don't move this check into a `sh:` var: dotenv is visible in cmds but not + # during var evaluation, so the test would always see an empty value. + - cmd: | + if [ "{{.PROFILES}}" = "dev" ] && [ -z "${SAAS_DEV_PROJECT_REF:-}" ]; then + echo ">> SAAS_DEV_PROJECT_REF is not set." + echo ">> Testing a SaaS PR? Put its ref, DB password and publishable key in app/.env.saas.local." + echo ">> Wanted the shared v3 project? Use 'task backend:staging:saas' instead." + exit 1 + fi + - task: _run:saas + vars: + PORT: '{{.PORT}}' + PROFILES: '{{.PROFILES}}' + AIENGINE_URL: '{{.AIENGINE_URL}}' + AIENGINE_ENABLED: '{{.AIENGINE_ENABLED}}' + AIENGINE_TIMEOUTSECONDS: '{{.AIENGINE_TIMEOUTSECONDS}}' + + staging:saas: + desc: "Start SaaS backend against the shared v3 staging project" + cmds: + - task: _run:saas + vars: + PORT: '{{.PORT}}' + PROFILES: staging + AIENGINE_URL: '{{.AIENGINE_URL}}' + AIENGINE_ENABLED: '{{.AIENGINE_ENABLED}}' + AIENGINE_TIMEOUTSECONDS: '{{.AIENGINE_TIMEOUTSECONDS}}' + + _run:saas: + internal: true dotenv: ['app/.env.saas.local', 'app/.env.saas'] ignore_error: true vars: PORT: '{{.PORT | default "8080"}}' - # Override to "" to run the pure `saas` profile against your own SAAS_DB_*. PROFILES: '{{.PROFILES | default "dev"}}' + # Built here rather than inline in the cmds below: the Windows line is an + # unquoted YAML scalar wrapping a cmd.exe string, so a nested {{if ne .X + # "none"}} needs escaped quotes that reach the Go template as literal + # backslashes and fail with `unexpected "\" in operand`. + PROFILE_ARGS: '{{if ne .PROFILES "none"}}--spring.profiles.include={{.PROFILES}}{{end}}' AIENGINE_URL: '{{.AIENGINE_URL | default ""}}' AIENGINE_ENABLED: '{{.AIENGINE_ENABLED | default "false"}}' AIENGINE_TIMEOUTSECONDS: '{{.AIENGINE_TIMEOUTSECONDS | default "120"}}' @@ -77,9 +118,11 @@ tasks: AIENGINE_ENABLED: '{{.AIENGINE_ENABLED}}' AIENGINE_TIMEOUTSECONDS: '{{.AIENGINE_TIMEOUTSECONDS}}' cmds: - - cmd: cmd /c ".\gradlew.bat :stirling-pdf:bootRun {{if .PROFILES}}--args=\"--spring.profiles.include={{.PROFILES}}\"{{end}}" + # PROFILE_ARGS is empty when PROFILES=none, i.e. the bare `saas` profile + # against SAAS_DB_* (production). + - cmd: cmd /c ".\gradlew.bat :stirling-pdf:bootRun {{if .PROFILE_ARGS}}--args=\"{{.PROFILE_ARGS}}\"{{end}}" platforms: [windows] - - cmd: ./gradlew :stirling-pdf:bootRun {{if .PROFILES}}--args='--spring.profiles.include={{.PROFILES}}'{{end}} + - cmd: ./gradlew :stirling-pdf:bootRun {{if .PROFILE_ARGS}}--args='{{.PROFILE_ARGS}}'{{end}} platforms: [linux, darwin] build: diff --git a/.taskfiles/frontend.yml b/.taskfiles/frontend.yml index 481a225ce1..f5325c9ed7 100644 --- a/.taskfiles/frontend.yml +++ b/.taskfiles/frontend.yml @@ -5,6 +5,14 @@ version: '3' # mode flag) or use `--project editor/...` for tsc — so the editor lives # under frontend/editor/ without each task needing a cd. +vars: + # Dev-only browser-tab label so concurrent worktrees are distinguishable. Only + # the worktree folder basename (e.g. "wt1") is exposed — never the full path, + # hostname, or user. Dropped from production builds. + DEV_LABEL: + sh: >- + {{if eq OS "windows"}}powershell -NoProfile -Command '$root = git rev-parse --show-toplevel 2>$null; if (-not $root) { $root = (Get-Location).Path }; Split-Path -Leaf $root'{{else}}basename "$(git rev-parse --show-toplevel 2>/dev/null || pwd)"{{end}} + tasks: install: desc: "Install dependencies" @@ -80,16 +88,52 @@ tasks: OPEN: '{{.OPEN | default ""}}' env: BACKEND_URL: '{{.BACKEND_URL}}' - # Dev-only browser-tab label so concurrent worktrees are distinguishable. - # Only the worktree folder basename (e.g. "wt1") is exposed — never the - # full path, hostname, or user. Consumed at dev-serve time by vite.config - # and dropped from production builds. - STIRLING_DEV_LABEL: - sh: >- - {{if eq OS "windows"}}powershell -NoProfile -Command '$root = git rev-parse --show-toplevel 2>$null; if (-not $root) { $root = (Get-Location).Path }; Split-Path -Leaf $root'{{else}}basename "$(git rev-parse --show-toplevel 2>/dev/null || pwd)"{{end}} + STIRLING_DEV_LABEL: '{{.DEV_LABEL}}' cmds: - npx vite editor --mode {{.MODE}} --port {{.PORT}}{{if .OPEN}} --open{{end}} + # Separate from dev:_run rather than a flag on it: Task sets an `env:` key even + # when its value resolves to empty, and Vite treats an empty process.env VITE_* as + # authoritative over the committed editor/.env, so folding these in blanks Supabase + # config for the core, proprietary and desktop dev servers. + dev:_run:saas: + internal: true + ignore_error: true + # The backend's own env files, so both halves target one project. Paths are + # relative to this taskfile's dir, `frontend`. + dotenv: ['../app/.env.saas.local', '../app/.env.saas'] + vars: + PORT: '{{.PORT | default "5173"}}' + BACKEND_URL: '{{.BACKEND_URL | default "http://localhost:8080"}}' + OPEN: '{{.OPEN | default ""}}' + SAAS_ENV: '{{.SAAS_ENV | default "dev"}}' + env: + BACKEND_URL: '{{.BACKEND_URL}}' + STIRLING_DEV_LABEL: '{{.DEV_LABEL}}' + SAAS_ENV: '{{.SAAS_ENV}}' + # A real process.env VITE_* beats a committed .env in Vite (loadEnv applies + # process.env last), which is what lets this override editor/.env. + # + # These must stay `sh:`, not Go templates: dotenv values are visible to Task's + # embedded shell but not to templates, where {{.SAAS_DEV_PROJECT_REF}} is + # always empty. + VITE_SUPABASE_URL: + sh: | + case "${SAAS_ENV:-dev}" in + staging) ref="${SAAS_STAGING_PROJECT_REF:?set it in app/.env.saas.local}" ;; + *) ref="${SAAS_DEV_PROJECT_REF:?set it in app/.env.saas.local, or run task staging:saas}" ;; + esac + echo "https://${ref}.supabase.co" + VITE_SUPABASE_PUBLISHABLE_DEFAULT_KEY: + sh: | + case "${SAAS_ENV:-dev}" in + staging) echo "${SAAS_STAGING_PUBLISHABLE_KEY:?set it in app/.env.saas.local}" ;; + *) echo "${SAAS_DEV_PUBLISHABLE_KEY:?set it in app/.env.saas.local}" ;; + esac + cmds: + - 'echo ">> frontend Supabase target: $VITE_SUPABASE_URL"' + - npx vite editor --mode saas --port {{.PORT}}{{if .OPEN}} --open{{end}} + dev: desc: "Start frontend dev server" cmds: @@ -111,13 +155,23 @@ tasks: vars: { MODE: proprietary, PORT: '{{.PORT}}', BACKEND_URL: '{{.BACKEND_URL}}', OPEN: '{{.OPEN}}' } dev:saas: - desc: "Start frontend dev server in SaaS mode" + desc: "Start frontend dev server in SaaS mode (SAAS_ENV=dev|staging|prod)" deps: - task: prepare vars: { MODE: saas } + vars: + SAAS_ENV: '{{.SAAS_ENV | default "dev"}}' + # prod routes to the plain runner, which sets no VITE_SUPABASE_* and so leaves + # the committed editor/.env alone. + RUNNER: '{{if eq .SAAS_ENV "prod"}}dev:_run{{else}}dev:_run:saas{{end}}' cmds: - - task: dev:_run - vars: { MODE: saas, PORT: '{{.PORT}}', BACKEND_URL: '{{.BACKEND_URL}}', OPEN: '{{.OPEN}}' } + - task: '{{.RUNNER}}' + vars: + MODE: saas + PORT: '{{.PORT}}' + BACKEND_URL: '{{.BACKEND_URL}}' + OPEN: '{{.OPEN}}' + SAAS_ENV: '{{.SAAS_ENV}}' dev:desktop: desc: "Start frontend dev server in desktop mode" diff --git a/Taskfile.yml b/Taskfile.yml index fc7a564032..92dcdcc742 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -99,11 +99,22 @@ tasks: BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}' OPEN: "true" + # Set SAAS_DEV_PROJECT_REF in app/.env.saas.local to pick the PR. dev:saas: - desc: "Start SaaS backend + frontend concurrently on free ports" + desc: "Start SaaS backend + frontend + engine against the current PR's preview branch" cmds: - task: dev:_all - vars: { FRONTEND: saas, BACKEND: saas } + vars: { FRONTEND: saas, BACKEND: saas, SAAS_ENV: dev } + + staging:saas: + desc: "Start SaaS backend + frontend + engine against the shared v3 staging project" + cmds: + - task: dev:_all + vars: + FRONTEND: saas + BACKEND: saas + BACKEND_TASK: backend:staging:saas + SAAS_ENV: staging dev:all: desc: "Start backend + frontend + engine concurrently on free ports" @@ -115,6 +126,9 @@ tasks: vars: FRONTEND: '{{.FRONTEND | default "proprietary"}}' BACKEND: '{{.BACKEND | default "proprietary"}}' + BACKEND_TASK: '{{.BACKEND_TASK | default (printf "backend:dev:%s" .BACKEND)}}' + # Only meaningful to the saas frontend; every other flavor ignores it. + SAAS_ENV: '{{.SAAS_ENV | default ""}}' PORTS: sh: '{{if eq OS "windows"}}{{.FIND_FREE_PORT_PS}} 8080 5173 5001{{else}}{{.FIND_FREE_PORT_SH}} 8080 5173 5001{{end}}' BACKEND_PORT: '{{index (splitList "\n" .PORTS) 0}}' @@ -124,7 +138,7 @@ tasks: - task: engine:dev vars: PORT: '{{.ENGINE_PORT}}' - - task: 'backend:dev:{{.BACKEND}}' + - task: '{{.BACKEND_TASK}}' vars: PORT: '{{.BACKEND_PORT}}' AIENGINE_URL: 'http://localhost:{{.ENGINE_PORT}}' @@ -134,6 +148,7 @@ tasks: PORT: '{{.FRONTEND_PORT}}' BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}' OPEN: "true" + SAAS_ENV: '{{.SAAS_ENV}}' # ============================================================ # Build diff --git a/app/.env.saas b/app/.env.saas index fb5feec559..25eefb84c5 100644 --- a/app/.env.saas +++ b/app/.env.saas @@ -1,15 +1,16 @@ -############################################################################### -# Stirling-PDF SaaS environment defaults. +# Stirling-PDF SaaS environment defaults. Committed, non-secret. Real values for secrets go in +# .env.saas.local, which is loaded first and wins. Do not commit that file. # -# This file is committed and provides non-secret defaults loaded by -# `task backend:dev:saas`. Put real values for secrets (passwords, project -# refs, edge function secrets) in `.env.saas.local` - any variable set there -# takes precedence over what's defined here. +# Three environments, each deriving its Supabase URLs, JWT issuer and JWKS from one project ref: # -# DO NOT commit `.env.saas.local`. Only `.env.saas` is checked in. -############################################################################### +# prod PROFILES=none SAAS_DB_* the live project +# staging PROFILES=staging SAAS_STAGING_* pinned to v3, always there +# dev PROFILES=dev SAAS_DEV_* follows a SaaS PR's preview branch +# +# dev is the default for `task backend:dev:saas`. Use staging for somewhere stable; use dev when +# testing an open SaaS PR, since its preview branch is the only place those migrations are applied. -# ---------- Supabase project ---------- +# ---------- Supabase project (prod / no-profile) ---------- # Project reference (the subdomain part of .supabase.co). Required. # Set in .env.saas.local. SAAS_DB_PROJECT_REF= @@ -17,18 +18,35 @@ SAAS_DB_PROJECT_REF= # Edge function secret used by billing/license rollup calls. Set in .env.saas.local. SUPABASE_EDGE_FUNCTION_SECRET= -# ---------- Database (saas profile) ---------- -# Direct JDBC URL to the Supabase Postgres. Required when running the plain -# `saas` profile (i.e. without `--spring.profiles.include=dev`). +# ---------- Database (no profile) ---------- +# Direct JDBC URL to the Supabase Postgres. Required when running without +# `--spring.profiles.include=...`. # Example: jdbc:postgresql://db..supabase.co:5432/postgres SAAS_DB_URL= SAAS_DB_USERNAME=postgres SAAS_DB_PASSWORD= -# ---------- Database (dev profile overrides) ---------- -# Used when `--spring.profiles.include=dev` is active. The dev profile -# defaults the URL/username to the shared dev Supabase project, but the -# password must still be provided in .env.saas.local. -SAAS_DEV_DB_URL= +# ---------- staging profile ---------- +# The shared long-lived v3 project. application-staging.properties defaults the ref, +# URL, database host and meter endpoint, so staging needs only the password, in +# .env.saas.local. Set SAAS_STAGING_PROJECT_REF to repoint it; everything derives. +# +# The ref and publishable key are duplicated here because the task derives the +# frontend's VITE_SUPABASE_* from them and a shell cannot read a Spring default. +# Neither is secret: the ref is a public subdomain, the key ships in the bundle. +SAAS_STAGING_PROJECT_REF=qacaivhsjtftfwtgjvva +SAAS_STAGING_PUBLISHABLE_KEY=sb_publishable_nIM8y-9ARPE7EzQwAQHKMg_40fCN6kY # gitleaks:allow +SAAS_STAGING_DB_USERNAME=postgres +SAAS_STAGING_DB_PASSWORD= + +# ---------- dev profile ---------- +# The SaaS PR's Supabase preview branch. Take the ref from that PR's "Supabase +# Preview" check; the profile derives URL, JWT issuer, JWKS, meter endpoint and +# database host from it, so this one value follows a different PR. +# +# A preview branch has its own password and keys; the parent project's will not +# authenticate. Both go in .env.saas.local, along with the ref. +SAAS_DEV_PROJECT_REF= +SAAS_DEV_PUBLISHABLE_KEY= SAAS_DEV_DB_USERNAME=postgres SAAS_DEV_DB_PASSWORD= diff --git a/app/saas/src/main/java/stirling/software/saas/config/SaasProjectNotice.java b/app/saas/src/main/java/stirling/software/saas/config/SaasProjectNotice.java new file mode 100644 index 0000000000..39305fcbcd --- /dev/null +++ b/app/saas/src/main/java/stirling/software/saas/config/SaasProjectNotice.java @@ -0,0 +1,53 @@ +package stirling.software.saas.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.annotation.Profile; +import org.springframework.context.event.EventListener; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; + +import lombok.extern.slf4j.Slf4j; + +/** Logs which Supabase project this backend is talking to, and its schema policy. */ +@Slf4j +@Component +@Profile({"dev", "staging"}) +public class SaasProjectNotice { + + private final Environment environment; + private final String projectRef; + private final String ddlAuto; + + public SaasProjectNotice( + Environment environment, + @Value("${app.supabase.project-ref:unknown}") String projectRef, + @Value("${spring.jpa.hibernate.ddl-auto:none}") String ddlAuto) { + this.environment = environment; + this.projectRef = projectRef; + this.ddlAuto = ddlAuto; + } + + @EventListener(ApplicationReadyEvent.class) + public void announceProject() { + boolean staging = environment.matchesProfiles("staging"); + if (staging) { + log.info( + """ + SaaS staging profile: Supabase project {}, ddl-auto={}. This is the SHARED \ + long-lived environment, so its data and schema are not yours alone. Testing an \ + open SaaS PR? Use that PR's preview branch instead \ + (SAAS_DEV_PROJECT_REF in app/.env.saas.local); staging will not have its \ + migrations.\ + """, + projectRef, + ddlAuto); + return; + } + log.info( + "SaaS dev profile: Supabase preview branch {}, ddl-auto={}. Disposable, so Hibernate" + + " is allowed to add the inherited tables the migrations do not create.", + projectRef, + ddlAuto); + } +} diff --git a/app/saas/src/main/resources/application-dev.properties b/app/saas/src/main/resources/application-dev.properties index ee8bf80ff2..b289fc95cc 100644 --- a/app/saas/src/main/resources/application-dev.properties +++ b/app/saas/src/main/resources/application-dev.properties @@ -1,32 +1,40 @@ -# SaaS dev profile. Points at the dev Supabase project. -# Boot: java -jar stirling-pdf.jar --spring.profiles.include=dev +# SaaS dev profile: follows the Supabase preview branch of the SaaS PR under test. +# One variable switches PR, SAAS_DEV_PROJECT_REF; everything else derives from it. +# Want a stable shared environment instead? Use the staging profile. + spring.config.import=optional:classpath:application-dev-local.properties -app.supabase.project-ref=qacaivhsjtftfwtgjvva +# Let Hibernate reconcile the entity tables so a fresh preview branch heals itself. A branch is built +# from the Supabase migrations, which cover the SaaS-owned tables but not the ~28 inherited from the +# self-hosted app -- those have only ever been created by ddl-auto. Safe here because a preview branch +# is disposable and `update` only ever adds; staging pins `none`, so keep this profile-scoped. +spring.jpa.hibernate.ddl-auto=update -stirling.supabase.url=https://qacaivhsjtftfwtgjvva.supabase.co -stirling.supabase.publishable-key=sb_publishable_nIM8y-9ARPE7EzQwAQHKMg_40fCN6kY # gitleaks:allow +# From the PR's "Supabase Preview" check. Required with no fallback: ddl-auto=update above must never +# be aimed at the shared project. +app.supabase.project-ref=${SAAS_DEV_PROJECT_REF} -spring.datasource.url=${SAAS_DEV_DB_URL:jdbc:postgresql://db.qacaivhsjtftfwtgjvva.supabase.co:5432/postgres?ApplicationName=stirling-consolidation-${user.name}} +stirling.supabase.url=https://${app.supabase.project-ref}.supabase.co +# Per-branch, not derivable. Dashboard > Settings > API. +stirling.supabase.publishable-key=${SAAS_DEV_PUBLISHABLE_KEY} + +# Override the whole URL if the branch needs the pooler host rather than the direct one. +spring.datasource.url=${SAAS_DEV_DB_URL:jdbc:postgresql://db.${app.supabase.project-ref}.supabase.co:5432/postgres?ApplicationName=stirling-dev-${user.name}} spring.datasource.username=${SAAS_DEV_DB_USERNAME:postgres} -# Password not committed; export SAAS_DEV_DB_PASSWORD or pass --spring.datasource.password=... +# A preview branch has its own password; the parent project's will not authenticate. spring.datasource.password=${SAAS_DEV_DB_PASSWORD:} -# Conservative dev pool sizing. spring.datasource.hikari.maximum-pool-size=2 spring.datasource.hikari.minimum-idle=1 spring.datasource.hikari.idle-timeout=60000 spring.datasource.hikari.max-lifetime=1800000 spring.datasource.hikari.keepalive-time=300000 -spring.datasource.hikari.data-source-properties.ApplicationName=stirling-consolidation-${user.name} +spring.datasource.hikari.data-source-properties.ApplicationName=stirling-dev-${user.name} logging.level.stirling.software.saas=DEBUG logging.level.org.springframework.security.oauth2.jwt=WARN logging.level.org.springframework.security.oauth2.server.resource=WARN -# Supabase meter edge fn the Java backend calls (server-to-server, on job close). -# URL is not a secret; auth rides the existing SUPABASE_EDGE_FUNCTION_SECRET (same -# shared secret the team-invitation flow uses — no service-role key in the Java env). -# Blank secret → the meter service no-ops with a WARN, so the app still boots. -# The billing portal is NOT here — the FE calls create-customer-portal-session directly. -payg.meter.endpoint=https://qacaivhsjtftfwtgjvva.supabase.co/functions/v1/meter-payg-units +# Server-to-server meter call. Auth rides SUPABASE_EDGE_FUNCTION_SECRET; blank secret means the meter +# service no-ops with a WARN rather than failing the boot. +payg.meter.endpoint=https://${app.supabase.project-ref}.supabase.co/functions/v1/meter-payg-units diff --git a/app/saas/src/main/resources/application-staging.properties b/app/saas/src/main/resources/application-staging.properties new file mode 100644 index 0000000000..4874ed3fb5 --- /dev/null +++ b/app/saas/src/main/resources/application-staging.properties @@ -0,0 +1,39 @@ +# SaaS staging profile: the long-lived shared v3 project, pinned so it is still there tomorrow. +# For work on an open SaaS PR use the dev profile, which follows that PR's preview branch. + +spring.config.import=optional:classpath:application-staging-local.properties + +# Stated rather than inherited: application-saas.properties defaults to `update`, and staging's +# schema is shared and RLS-dependent, so it must not be reconciled by Hibernate. +spring.jpa.hibernate.ddl-auto=none + +# Committed as a default rather than a literal, so staging needs no setup but stays repointable. +# Neither the ref nor the publishable key is secret: the ref is a public subdomain, the key ships in +# the browser bundle. Everything below derives from the ref, so an override follows through. +app.supabase.project-ref=${SAAS_STAGING_PROJECT_REF:qacaivhsjtftfwtgjvva} + +stirling.supabase.url=https://${app.supabase.project-ref}.supabase.co +stirling.supabase.publishable-key=${SAAS_STAGING_PUBLISHABLE_KEY:sb_publishable_nIM8y-9ARPE7EzQwAQHKMg_40fCN6kY} + +spring.datasource.url=${SAAS_STAGING_DB_URL:jdbc:postgresql://db.${app.supabase.project-ref}.supabase.co:5432/postgres?ApplicationName=stirling-staging-${user.name}} +spring.datasource.username=${SAAS_STAGING_DB_USERNAME:postgres} +# Password not committed; export SAAS_STAGING_DB_PASSWORD or pass --spring.datasource.password=... +spring.datasource.password=${SAAS_STAGING_DB_PASSWORD:} + +# Conservative pool sizing: this is a shared project, so don't hold connections others need. +spring.datasource.hikari.maximum-pool-size=2 +spring.datasource.hikari.minimum-idle=1 +spring.datasource.hikari.idle-timeout=60000 +spring.datasource.hikari.max-lifetime=1800000 +spring.datasource.hikari.keepalive-time=300000 +spring.datasource.hikari.data-source-properties.ApplicationName=stirling-staging-${user.name} + +logging.level.stirling.software.saas=DEBUG +logging.level.org.springframework.security.oauth2.jwt=WARN +logging.level.org.springframework.security.oauth2.server.resource=WARN + +# Supabase meter edge fn the Java backend calls (server-to-server, on job close). +# URL is not a secret; auth rides the existing SUPABASE_EDGE_FUNCTION_SECRET (same +# shared secret the team-invitation flow uses — no service-role key in the Java env). +# Blank secret → the meter service no-ops with a WARN, so the app still boots. +payg.meter.endpoint=https://${app.supabase.project-ref}.supabase.co/functions/v1/meter-payg-units From 088e0ef4e25e06ae5f911f8be74912fef8673240 Mon Sep 17 00:00:00 2001 From: Ludy Date: Wed, 19 Aug 2026 18:29:02 +0000 Subject: [PATCH 04/45] deps(frontend): upgrade Cantoo PDF library to 2.8.2 (#7493) # Description of Changes This pull request upgrades the frontend PDF dependency from `@cantoo/pdf-lib` 2.6.5 to 2.8.2. - Updated `frontend/package.json` to require `@cantoo/pdf-lib` `^2.8.2`. - Regenerated `frontend/package-lock.json` with `@cantoo/pdf-lib@2.8.2`, `pako@2.2.0`, and `node-html-better-parser@1.5.9`. - Added the root npm `pako` override recommended by the upstream release. - The upgrade brings upstream parser, object-stream, encryption, form, PNG, and PDF serialization fixes into the frontend dependency. - No application API migration was required because the project does not use the newly added PDF/A, XFA, Factur-X, incremental-update, fontkit, or page-content-extraction APIs. The main challenge was validating the broad upstream change set against the project's actual usage. The frontend typecheck and a direct PDF create/save/load smoke test passed. The complete `frontend:check` and `frontend:test` tasks exceeded the available execution timeout without reporting a test failure. No related issue. --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/HowToAddNewLanguage.md) (if applicable) - [x] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [x] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#6-testing) for more details. --- frontend/package-lock.json | 38 ++++++++++++++++++++++++++------------ frontend/package.json | 3 ++- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index d50868bd60..37bca97b95 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -10,7 +10,7 @@ "license": "SEE LICENSE IN https://raw.githubusercontent.com/Stirling-Tools/Stirling-PDF/refs/heads/main/proprietary/LICENSE", "dependencies": { "@atlaskit/pragmatic-drag-and-drop": "^1.7.7", - "@cantoo/pdf-lib": "^2.5.3", + "@cantoo/pdf-lib": "^2.8.2", "@dnd-kit/core": "^6.3.1", "@embedpdf/core": "^2.14.4", "@embedpdf/engines": "^2.14.4", @@ -606,18 +606,22 @@ } }, "node_modules/@cantoo/pdf-lib": { - "version": "2.6.5", - "resolved": "https://registry.npmjs.org/@cantoo/pdf-lib/-/pdf-lib-2.6.5.tgz", - "integrity": "sha512-3eMHEaqKHt/G/q+6QjT06A3lz0S/a8x3+myiSN7FNeL3uWcedO0lpfs6TWofa4C03Z1wz3tWeHoa4CsI7DrTSA==", + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/@cantoo/pdf-lib/-/pdf-lib-2.8.2.tgz", + "integrity": "sha512-f0BJM3uPOjbPR3YriSEUIaTM0qnqthjFmTZX9NGI0NDM2Tj4a8xv7Z5Hb6jzUrhfb3/9Y77+xxoOln0IIiYq+w==", "license": "MIT", "dependencies": { "@pdf-lib/standard-fonts": "^1.0.0", "@pdf-lib/upng": "^1.0.1", "color": "^4.2.3", "crypto-js": "^4.2.0", - "node-html-better-parser": ">=1.4.0", - "pako": "^1.0.11", + "html-entities": "^2.3.2", + "node-html-better-parser": ">=1.5.9", + "pako": "^2.2.0", "tslib": ">=2" + }, + "peerDependencies": { + "html-entities": "^2.3.2" } }, "node_modules/@csstools/color-helpers": { @@ -12459,9 +12463,9 @@ } }, "node_modules/node-html-better-parser": { - "version": "1.5.8", - "resolved": "https://registry.npmjs.org/node-html-better-parser/-/node-html-better-parser-1.5.8.tgz", - "integrity": "sha512-t/wAKvaTSKco43X+yf9+76RiMt18MtMmzd4wc7rKj+fWav6DV4ajDEKdWlLzSE8USDF5zr/06uGj0Wr/dGAFtw==", + "version": "1.5.9", + "resolved": "https://registry.npmjs.org/node-html-better-parser/-/node-html-better-parser-1.5.9.tgz", + "integrity": "sha512-z1I5UINMezJXYL9cH3h0a9KBth2G978gSLlfkpQ+CQzzVHVQy9gpARgm9eDsz1O4gn1HtgUqjdAIYxKFZm6uHQ==", "license": "MIT", "dependencies": { "html-entities": "^2.3.2" @@ -12751,9 +12755,19 @@ "license": "MIT" }, "node_modules/pako": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.2.0.tgz", + "integrity": "sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "(MIT AND Zlib)" }, "node_modules/parent-module": { diff --git a/frontend/package.json b/frontend/package.json index 90a0e10b05..f874dedfa3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -7,7 +7,7 @@ "proxy": "http://localhost:8080", "dependencies": { "@atlaskit/pragmatic-drag-and-drop": "^1.7.7", - "@cantoo/pdf-lib": "^2.5.3", + "@cantoo/pdf-lib": "^2.8.2", "@dnd-kit/core": "^6.3.1", "@embedpdf/core": "^2.14.4", "@embedpdf/engines": "^2.14.4", @@ -171,6 +171,7 @@ }, "overrides": { "devalue": "^5.8.1", + "pako": "^2.2.0", "tsconfck": { "typescript": "$typescript" } From 1690cc25ccbf100f3be1699ac098bca419f1ca6a Mon Sep 17 00:00:00 2001 From: "stirlingbot[bot]" <195170888+stirlingbot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 05:15:37 +0000 Subject: [PATCH 05/45] Update Frontend 3rd Party Licenses (#7573) Auto-generated by stirlingbot[bot] This PR updates the frontend license report based on changes to package.json dependencies. Signed-off-by: stirlingbot[bot] Co-authored-by: stirlingbot[bot] <195170888+stirlingbot[bot]@users.noreply.github.com> --- frontend/editor/src/assets/3rdPartyLicenses.json | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/frontend/editor/src/assets/3rdPartyLicenses.json b/frontend/editor/src/assets/3rdPartyLicenses.json index 110a6c88a5..e17cef8f8b 100644 --- a/frontend/editor/src/assets/3rdPartyLicenses.json +++ b/frontend/editor/src/assets/3rdPartyLicenses.json @@ -10,7 +10,7 @@ { "moduleName": "@cantoo/pdf-lib", "moduleUrl": "https://github.com/cantoo-scribe/pdf-lib", - "moduleVersion": "2.6.5", + "moduleVersion": "2.8.2", "moduleLicense": "MIT", "moduleLicenseUrl": "https://opensource.org/licenses/MIT" }, @@ -255,14 +255,14 @@ { "moduleName": "@stripe/react-stripe-js", "moduleUrl": "https://github.com/stripe/react-stripe-js", - "moduleVersion": "4.0.2", + "moduleVersion": "6.8.0", "moduleLicense": "MIT", "moduleLicenseUrl": "https://opensource.org/licenses/MIT" }, { "moduleName": "@stripe/stripe-js", "moduleUrl": "https://github.com/stripe/stripe-js", - "moduleVersion": "7.9.0", + "moduleVersion": "9.10.0", "moduleLicense": "MIT", "moduleLicenseUrl": "https://opensource.org/licenses/MIT" }, @@ -287,6 +287,13 @@ "moduleLicense": "MIT", "moduleLicenseUrl": "https://opensource.org/licenses/MIT" }, + { + "moduleName": "@tanstack/react-table", + "moduleUrl": "https://github.com/TanStack/table", + "moduleVersion": "9.1.2", + "moduleLicense": "MIT", + "moduleLicenseUrl": "https://opensource.org/licenses/MIT" + }, { "moduleName": "@tanstack/react-virtual", "moduleUrl": "https://github.com/TanStack/virtual", From a744102cb68441a5dcf9b436a77919cf73c7eaab Mon Sep 17 00:00:00 2001 From: James Brunton Date: Thu, 20 Aug 2026 08:23:20 +0000 Subject: [PATCH 06/45] Support Supporting Files in Pipelines (#7547) # Description of Changes Currently in the Processor's Pipelines page, none of the tools which require supporting files are usable because it's never been hooked up to the new API to upload supporting files. This PR hooks it up to that so all tools using supporting files work in the processor. I had to tweak the type generation a little for this so we have a static map of which params are for supporting files so we know to handle them differently. The `Test with a file` button has to work a little differently than the main run since it's running an ad-hoc pipeline so the files haven't necessarily been saved to the server yet. In this case, it'll use whatever local changes the user has made for those pipeline steps, and for all other steps, it'll just use what's saved in the server. --- .../policy/controller/PolicyController.java | 42 +++- .../controller/PolicyControllerTest.java | 48 +++- .../public/locales/en-US/translation.toml | 4 +- .../scripts/generate-tool-api-types.mts | 64 ++++- .../hooks/tools/shared/toolApiMapping.test.ts | 19 +- .../core/hooks/tools/shared/toolApiMapping.ts | 22 +- .../hooks/tools/shared/toolAutomation.test.ts | 178 +++++++++++++- .../core/hooks/tools/shared/toolAutomation.ts | 223 +++++++++++++++++- .../hooks/tools/shared/toolOperationTypes.ts | 47 +++- .../editor/src/core/types/toolApiTypes.ts | 46 ++-- .../editor/src/portal/api/pipelineAssets.ts | 41 ++++ frontend/editor/src/portal/api/pipelines.ts | 28 ++- .../pipelines/PipelineStepSettings.css | 17 ++ .../PipelineStepSettings.stories.tsx | 2 + .../pipelines/PipelineStepSettings.test.tsx | 10 + .../pipelines/PipelineStepSettings.tsx | 146 +++++++++--- .../src/portal/mocks/handlers/pipelines.ts | 42 ++++ .../src/portal/views/PipelineBuilder.test.tsx | 122 +++++++++- .../src/portal/views/PipelineBuilder.tsx | 201 +++++++++++++--- 19 files changed, 1148 insertions(+), 154 deletions(-) create mode 100644 frontend/editor/src/portal/api/pipelineAssets.ts create mode 100644 frontend/editor/src/portal/components/pipelines/PipelineStepSettings.css diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java index 778a04e169..9b9fca4133 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java @@ -21,6 +21,7 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.RequestContextHolder; @@ -51,6 +52,7 @@ import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; import stirling.software.proprietary.audit.AuditContext; import stirling.software.proprietary.policy.asset.PolicyAssetCleaner; +import stirling.software.proprietary.policy.asset.PolicyAssetResolver; import stirling.software.proprietary.policy.config.PolicyAccessGuard; import stirling.software.proprietary.policy.config.PolicyManagementAuthority; import stirling.software.proprietary.policy.engine.PolicyRunHandle; @@ -106,6 +108,7 @@ public class PolicyController { private final PolicyTriggerManager policyTriggerManager; private final PolicyOverviewService policyOverviewService; private final PolicyAssetCleaner assetCleaner; + private final PolicyAssetResolver assetResolver; private final ProcessedLedger processedLedger; private final List policyTriggers; private final ApplicationProperties applicationProperties; @@ -125,12 +128,13 @@ public class PolicyController { + " endpoint and download outputs via /api/v1/general/files/{id}.") public ResponseEntity> run( @RequestPart("json") PipelineDefinition definition, + @RequestParam(value = "policyId", required = false) String policyId, @Valid @ModelAttribute PolicyRunFiles files) throws IOException { stampPolicyAudit(definition); requireRunnable(definition); validateAdHocRun(definition); - PolicyInputs inputs = toInputs(files); + PolicyInputs inputs = resolveStoredAssets(policyId, toInputs(files)); PolicyRunHandle handle = policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP); recordEditorDocs(inputs); @@ -146,12 +150,13 @@ public class PolicyController { + " 'cancelled', or 'waiting' event carrying the final run view.") public SseEmitter runStream( @RequestPart("json") PipelineDefinition definition, + @RequestParam(value = "policyId", required = false) String policyId, @Valid @ModelAttribute PolicyRunFiles files) throws IOException { stampPolicyAudit(definition); requireRunnable(definition); validateAdHocRun(definition); - PolicyInputs inputs = toInputs(files); + PolicyInputs inputs = resolveStoredAssets(policyId, toInputs(files)); SseEmitter emitter = new SseEmitter(applicationProperties.getPolicies().getStreamTimeoutMs()); @@ -438,10 +443,7 @@ public class PolicyController { * folder sources/outputs is enforced separately by {@link PolicyValidator} at validation time. */ private void requirePolicyEditingAllowed() { - if (!applicationProperties.getSecurity().isEnableLogin()) { - return; - } - if (!policyManagementAuthority.canEditPolicies()) { + if (!policyEditingAllowed()) { throw new ResponseStatusException( HttpStatus.FORBIDDEN, "Policies may only be created or modified by a team leader"); @@ -467,6 +469,15 @@ public class PolicyController { } } + /** + * Whether the caller may create/modify policies (a team leader, or any operator when login is + * off). + */ + private boolean policyEditingAllowed() { + return !applicationProperties.getSecurity().isEnableLogin() + || policyManagementAuthority.canEditPolicies(); + } + @GetMapping @Operation( summary = "List policies", @@ -672,6 +683,25 @@ public class PolicyController { inputs.primary().size()); } + /** + * Resolve a test run's stored {@code asset:} bindings from the saved policy the builder is + * editing, so their bytes need not be re-uploaded. Scoped to that policy (the resolver loads + * only the assets it references, in its own team) and gated to policy editors - the same + * authority that can read asset bytes - so a member can't rebind a policy's stored asset into + * an ad-hoc step to read it back. A blank id (an unsaved pipeline has no stored bindings) or an + * inaccessible policy leaves the run-supplied inputs untouched. + */ + private PolicyInputs resolveStoredAssets(String policyId, PolicyInputs inputs) { + if (policyId == null || policyId.isBlank() || !policyEditingAllowed()) { + return inputs; + } + return policyStore + .get(policyId) + .filter(policyAccessGuard::canAccess) + .map(policy -> assetResolver.resolve(policy, inputs)) + .orElse(inputs); + } + /** * Turn the typed run files into engine {@link PolicyInputs}: the primary documents plus the * named supporting-file store, where each asset's {@code key} is the name a step references diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java index 2fa675597c..36f5cc221b 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java @@ -83,6 +83,8 @@ class PolicyControllerTest { @Mock private stirling.software.proprietary.policy.asset.PolicyAssetCleaner assetCleaner; + @Mock private stirling.software.proprietary.policy.asset.PolicyAssetResolver assetResolver; + @Mock private ProcessedLedger processedLedger; @Mock private TempFileManager tempFileManager; @@ -115,6 +117,7 @@ class PolicyControllerTest { policyTriggerManager, policyOverviewService, assetCleaner, + assetResolver, processedLedger, policyTriggers, applicationProperties, @@ -232,7 +235,7 @@ class PolicyControllerTest { .thenReturn(handle("run-1")); ResponseEntity> response = - controller.run(definitionWithStep(), new PolicyRunFiles()); + controller.run(definitionWithStep(), null, new PolicyRunFiles()); assertThat(response.getStatusCode()).isEqualTo(HttpStatus.ACCEPTED); assertThat(response.getBody().getJobId()).isEqualTo("run-1"); @@ -245,7 +248,7 @@ class PolicyControllerTest { .thenReturn(handle("run-1")); when(sourceAccessGuard.currentTeamId()).thenReturn(3L); - controller.run(definitionWithStep(), new PolicyRunFiles()); + controller.run(definitionWithStep(), null, new PolicyRunFiles()); verify(docCounter).record(EditorSource.counterKey(3L), 0L); } @@ -255,7 +258,7 @@ class PolicyControllerTest { void rejectsEmptyPipeline() { PipelineDefinition empty = new PipelineDefinition("pipe", List.of(), List.of()); - assertThatThrownBy(() -> controller.run(empty, new PolicyRunFiles())) + assertThatThrownBy(() -> controller.run(empty, null, new PolicyRunFiles())) .isInstanceOf(ResponseStatusException.class) .satisfies( e -> @@ -277,7 +280,7 @@ class PolicyControllerTest { .when(policyValidator) .validateOutput(any()); - assertThatThrownBy(() -> controller.run(definition, new PolicyRunFiles())) + assertThatThrownBy(() -> controller.run(definition, null, new PolicyRunFiles())) .isInstanceOf(ResponseStatusException.class) .satisfies( e -> @@ -285,6 +288,38 @@ class PolicyControllerTest { .isEqualTo(HttpStatus.BAD_REQUEST)); verify(policyRunner, never()).runAdHoc(any(), any(), any()); } + + @Test + @DisplayName("resolves stored assets from the supplied policy when the caller may edit it") + void resolvesStoredAssetsForEditor() throws Exception { + applicationProperties.getSecurity().setEnableLogin(false); // editing allowed + Policy p = policy("pol-1", 1L); + when(policyStore.get("pol-1")).thenReturn(Optional.of(p)); + when(policyAccessGuard.canAccess(p)).thenReturn(true); + when(assetResolver.resolve(eq(p), any())).thenAnswer(inv -> inv.getArgument(1)); + when(policyRunner.runAdHoc(any(), any(), eq(PolicyProgressListener.NOOP))) + .thenReturn(handle("run-1")); + + controller.run(definitionWithStep(), "pol-1", new PolicyRunFiles()); + + verify(assetResolver).resolve(eq(p), any()); + } + + @Test + @DisplayName("does not resolve a policy's stored assets for a caller who cannot edit it") + void skipsStoredAssetsForNonEditor() throws Exception { + // Gating asset resolution to editors keeps a member from rebinding a policy's stored + // asset into an ad-hoc step to read it back. + applicationProperties.getSecurity().setEnableLogin(true); + when(policyManagementAuthority.canEditPolicies()).thenReturn(false); + when(policyRunner.runAdHoc(any(), any(), eq(PolicyProgressListener.NOOP))) + .thenReturn(handle("run-1")); + + controller.run(definitionWithStep(), "pol-1", new PolicyRunFiles()); + + verify(assetResolver, never()).resolve(any(), any()); + verify(policyStore, never()).get(any()); + } } @Nested @@ -296,7 +331,8 @@ class PolicyControllerTest { void returnsEmitter() throws Exception { when(policyRunner.runAdHoc(any(), any(), any())).thenReturn(handle("run-2")); - SseEmitter emitter = controller.runStream(definitionWithStep(), new PolicyRunFiles()); + SseEmitter emitter = + controller.runStream(definitionWithStep(), null, new PolicyRunFiles()); assertThat(emitter).isNotNull(); } @@ -306,7 +342,7 @@ class PolicyControllerTest { void rejectsEmpty() { PipelineDefinition empty = new PipelineDefinition("pipe", List.of(), List.of()); - assertThatThrownBy(() -> controller.runStream(empty, new PolicyRunFiles())) + assertThatThrownBy(() -> controller.runStream(empty, null, new PolicyRunFiles())) .isInstanceOf(ResponseStatusException.class); } } diff --git a/frontend/editor/public/locales/en-US/translation.toml b/frontend/editor/public/locales/en-US/translation.toml index 12ad26ceec..01db314031 100644 --- a/frontend/editor/public/locales/en-US/translation.toml +++ b/frontend/editor/public/locales/en-US/translation.toml @@ -7705,7 +7705,6 @@ moreActions = "More actions" needsConfiguring = "Needs setting up" needsDestination = "No destination chosen" needsSource = "No source chosen" -needsUpload = "Needs an uploaded file" noToolMatches = "No tools match your search." pause = "Pause" rename = "Rename pipeline" @@ -7713,11 +7712,11 @@ searchTools = "Search tools" sendToSystem = "Send to another system" stepsIncompatible = "These steps can't run on what their prior step produces: {{tools}}." stepsNeedSetup = "These steps still need setting up before saving: {{tools}}." +supportingFiles = "Supporting files" testRun = "Test with a file" unknownStep = "Unrecognized operation, kept as-is." unsavedBody = "You have unsaved changes. Save them before leaving, or discard them?" unsavedTitle = "Unsaved changes" -uploadUnsupported = "Uploaded files aren't supported in pipelines yet, so these steps can't be saved: {{tools}}." usesDefaults = "Runs with default settings" viewDefinition = "View definition" @@ -7730,7 +7729,6 @@ saveHeading = "To save your changes:" schedule = "Set how often it runs" setup = "Finish setting up: {{tools}}" source = "Choose an input source" -upload = "Remove steps that need an uploaded file: {{tools}}" [portal.pipelines.builder.diagnostic] fan-in = "Combines every incoming file" diff --git a/frontend/editor/scripts/generate-tool-api-types.mts b/frontend/editor/scripts/generate-tool-api-types.mts index d5990096af..38eb7fe7b6 100644 --- a/frontend/editor/scripts/generate-tool-api-types.mts +++ b/frontend/editor/scripts/generate-tool-api-types.mts @@ -28,10 +28,11 @@ const ALLOWED_PATH_PREFIXES = [ "/api/v1/integration/", ]; -// File plumbing, not user parameters: `fileInput` is the uploaded document and -// `fileId` a server-side handle. Stripped from every generated request model. -// Named file fields (stampImage, attachments, ...) are real parameters and kept. -const BASE_FILE_FIELDS = new Set(["fileInput", "fileId"]); +// File plumbing, not user parameters: `fileInput` and `file` are the uploaded primary document +// (endpoints use one name or the other - `file` is never a second, supporting upload) and `fileId` +// a server-side handle. Stripped from every generated request model. Named supporting-file fields +// (stampImage, attachments, ...) are real parameters and kept. +const BASE_FILE_FIELDS = new Set(["fileInput", "file", "fileId"]); // The shared "upload a file or provide a file ID" wrapper schema and its two // branches. An endpoint whose body is exactly this has no parameters, so it must @@ -73,6 +74,19 @@ function isObject(value: unknown): value is Json { return typeof value === "object" && value !== null && !Array.isArray(value); } +/** A single file upload: `type: string, format: binary` (a Java MultipartFile param). */ +function isBinaryField(schema: unknown): schema is Json { + return ( + isObject(schema) && schema.type === "string" && schema.format === "binary" + ); +} + +/** A multi file upload: an array of binary items (some specs also flag the array itself binary). */ +function isBinaryArrayField(schema: unknown): schema is Json { + if (!isObject(schema) || schema.type !== "array") return false; + return schema.format === "binary" || isBinaryField(schema.items); +} + /** * Recursively sort object keys so the output is byte-stable regardless of the * key ordering springdoc happens to emit. @@ -358,6 +372,9 @@ async function main(): Promise { const usedClassNames = new Set(); const pendingComponents = new Set(); const skipped: string[] = []; + // Named file fields (as File uploads) per model, so a caller can tell a file param from a scalar + // string param - which `format: binary` -> `string` would otherwise erase. + const fileFieldsByClass: Record = {}; for (const path of Object.keys(paths).sort()) { if ( @@ -408,7 +425,32 @@ async function main(): Promise { const query = queryParameters(pathItem); // Body wins over query on a name collision. const properties: Json = { ...query.props, ...bodyProps }; + // `file` is stripped as a primary-document alias (see BASE_FILE_FIELDS). That only holds while + // no endpoint uses `file` as a *supporting* upload beside a primary `fileInput`; if one ever + // does, blanket-stripping would silently drop it. Fail generation so the assumption is fixed + // here rather than shipping a lost file. + if ("file" in properties && "fileInput" in properties) { + throw new Error( + `${path} has both 'fileInput' and 'file' uploads. 'file' is stripped as a primary-document` + + " alias, which would drop it as a supporting file. Rename the supporting param or revise" + + " BASE_FILE_FIELDS handling in this generator.", + ); + } for (const field of BASE_FILE_FIELDS) delete properties[field]; + // Type each named file upload as File/File[] (not the `string` a binary format yields) via + // json-schema-to-typescript's `tsType` override, and record it. Base file fields are already + // stripped, so what remains is the real supporting-file params. + const fileFields: string[] = []; + for (const [name, prop] of Object.entries(properties)) { + if (isBinaryField(prop)) { + prop.tsType = "File"; + fileFields.push(name); + } else if (isBinaryArrayField(prop)) { + prop.tsType = "File[]"; + fileFields.push(name); + } + } + fileFieldsByClass[className] = fileFields; modelSchema.properties = properties; const required = new Set(computeRequired(modelSchema, properties)); for (const name of query.required) { @@ -464,6 +506,7 @@ async function main(): Promise { await compileAndWrite( tools, definitions, + fileFieldsByClass, outputPath, values.check ?? false, skipped, @@ -473,6 +516,7 @@ async function main(): Promise { async function compileAndWrite( tools: DiscoveredTool[], definitions: Record, + fileFieldsByClass: Record, outputPath: string, check: boolean, skipped: string[], @@ -525,6 +569,15 @@ async function compileAndWrite( const endpointList = tools .map((t) => ` ${JSON.stringify(t.path)},`) .join("\n"); + // Endpoints that take supporting files, mapped to those file params' names. Only endpoints with at + // least one are listed, so membership answers "does this tool take extra files". + const fileFieldEntries = tools + .filter((t) => (fileFieldsByClass[t.className] ?? []).length > 0) + .map( + (t) => + ` ${JSON.stringify(t.path)}: ${JSON.stringify(fileFieldsByClass[t.className])},`, + ) + .join("\n"); const footer = [ "/** Endpoint path for a generated tool operation (the operation identity across languages). */", @@ -536,6 +589,9 @@ async function compileAndWrite( "/** Every generated tool endpoint, for iteration. */", `export const TOOL_ENDPOINTS = [\n${endpointList}\n] as const satisfies readonly ToolEndpoint[];`, "", + "/** The supporting-file parameters each endpoint accepts beyond its primary fileInput, by name. */", + `export const TOOL_FILE_FIELDS = {\n${fileFieldEntries}\n} as const satisfies Partial<\n Record\n>;`, + "", "/** Union of every generated tool request model. */", `export type ToolApiRequest = ToolApiParams[ToolEndpoint];`, ].join("\n"); diff --git a/frontend/editor/src/core/hooks/tools/shared/toolApiMapping.test.ts b/frontend/editor/src/core/hooks/tools/shared/toolApiMapping.test.ts index ba8c0bc2be..f3443dfb51 100644 --- a/frontend/editor/src/core/hooks/tools/shared/toolApiMapping.test.ts +++ b/frontend/editor/src/core/hooks/tools/shared/toolApiMapping.test.ts @@ -41,12 +41,13 @@ describe("objectToFormData", () => { }); test("expands arrays into repeated fields", () => { - const request: ToolApiParams["/api/v1/misc/add-attachments"] = { - attachments: ["a.png", "b.png", "c.png"], + const request: ToolApiParams["/api/v1/misc/ocr-pdf"] = { + ocrType: "Normal", + languages: ["eng", "fra", "deu"], }; const formData = objectToFormData(request); - expect(formData.getAll("attachments")).toEqual(["a.png", "b.png", "c.png"]); + expect(formData.getAll("languages")).toEqual(["eng", "fra", "deu"]); }); test("throws on a non-primitive field value rather than dropping it", () => { @@ -70,6 +71,18 @@ describe("objectToFormData", () => { expect(formData.get("optimizeLevel")).toBe("5"); }); + test("sends a File-valued model field as a file part, not stringified", () => { + const stamp = new File(["s"], "stamp.png", { type: "image/png" }); + const request: ToolApiParams["/api/v1/misc/add-stamp"] = { + stampType: "image", + stampImage: stamp, + }; + const formData = objectToFormData(request); + + expect(formData.get("stampImage")).toBe(stamp); + expect(formData.get("stampType")).toBe("image"); + }); + test("appends multiple files under the same field name", () => { const files = [ new File(["1"], "a.pdf", { type: "application/pdf" }), diff --git a/frontend/editor/src/core/hooks/tools/shared/toolApiMapping.ts b/frontend/editor/src/core/hooks/tools/shared/toolApiMapping.ts index 9c6f5d9e4b..81663fbf69 100644 --- a/frontend/editor/src/core/hooks/tools/shared/toolApiMapping.ts +++ b/frontend/editor/src/core/hooks/tools/shared/toolApiMapping.ts @@ -47,26 +47,28 @@ function appendPrimitive( formData.append(key, value); } else if (typeof value === "number" || typeof value === "boolean") { formData.append(key, `${value}`); + } else if (typeof Blob !== "undefined" && value instanceof Blob) { + // A File upload (models type binary params as File): send it as the file part, not stringified. + formData.append(key, value); } else { - // A non-primitive here means a mapper produced a value the backend cannot - // receive as a form field. Fail loudly rather than silently drop it: - // structured fields must be JSON-encoded in the mapper, and Files passed via - // the `files` argument. + // Any other non-primitive means a mapper produced a value the backend cannot receive as a form + // field. Fail loudly rather than silently drop it: structured fields must be JSON-encoded first. throw new Error( `objectToFormData: field "${key}" has an unsupported value of type ` + - `"${typeof value}"; expected a string, number, or boolean.`, + `"${typeof value}"; expected a string, number, boolean, or File.`, ); } } /** * Serialize a backend request model (the output of a `toApiParams` function) - * into multipart FormData: primitives become string fields, arrays become - * repeated fields, and `undefined`/`null` are omitted. Files are appended - * separately via `files`, keeping file plumbing out of the parameter mapper. + * into multipart FormData: primitives become string fields, `File` values become + * file parts, arrays become repeated fields, and `undefined`/`null` are omitted. + * Extra files may still be passed via `files` (the primary `fileInput`, or a + * field the mapper doesn't carry). * - * Throws if a field holds a non-primitive value, since that cannot be sent as a - * form field: structured fields must be JSON-encoded by the mapper. + * Throws if a field holds any other non-primitive value, since that cannot be + * sent as a form field: structured fields must be JSON-encoded by the mapper. */ export function objectToFormData( params: ToolApiRequest, diff --git a/frontend/editor/src/core/hooks/tools/shared/toolAutomation.test.ts b/frontend/editor/src/core/hooks/tools/shared/toolAutomation.test.ts index 5be4831907..0f7184f848 100644 --- a/frontend/editor/src/core/hooks/tools/shared/toolAutomation.test.ts +++ b/frontend/editor/src/core/hooks/tools/shared/toolAutomation.test.ts @@ -10,11 +10,14 @@ import { asRegistryConfig, ToolType, } from "@app/hooks/tools/shared/toolOperationTypes"; +import { objectToFormData } from "@app/hooks/tools/shared/toolApiMapping"; import { + activeFileFields, deserializeToolStep, + extractStepFiles, getExecutableTools, serializeToolStep, - stepRequiresUpload, + stepNeedsConfiguring, type WorkingToolStep, } from "@app/hooks/tools/shared/toolAutomation"; import { compressOperationConfig } from "@app/hooks/tools/compress/useCompressOperation"; @@ -28,6 +31,10 @@ import { addPasswordOperationConfig } from "@app/hooks/tools/addPassword/useAddP import { changePermissionsOperationConfig } from "@app/hooks/tools/changePermissions/useChangePermissionsOperation"; import { convertOperationConfig } from "@app/hooks/tools/convert/useConvertOperation"; import { defaultParameters as convertDefaults } from "@app/hooks/tools/convert/useConvertParameters"; +import { overlayPdfsOperationConfig } from "@app/hooks/tools/overlayPdfs/useOverlayPdfsOperation"; +import { defaultParameters as overlayDefaults } from "@app/hooks/tools/overlayPdfs/useOverlayPdfsParameters"; +import { certSignOperationConfig } from "@app/hooks/tools/certSign/useCertSignOperation"; +import { defaultParameters as certSignDefaults } from "@app/hooks/tools/certSign/useCertSignParameters"; function entry(over: Partial): ToolRegistryEntry { return { @@ -419,18 +426,167 @@ describe("convert (format-routed custom tool)", () => { }); }); -describe("stepRequiresUpload", () => { - const step = (params: Record): WorkingToolStep => ({ - toolId: "compress" as ToolId, - operation: "/api/v1/misc/compress-pdf", - params, +describe("supporting files", () => { + const fileRegistry: Partial = { + overlayPdfs: entry({ + name: "Overlay", + automationSettings: NoopSettings, + operationConfig: asRegistryConfig(overlayPdfsOperationConfig), + }), + certSign: entry({ + name: "Cert sign", + automationSettings: NoopSettings, + operationConfig: asRegistryConfig(certSignOperationConfig), + }), + }; + + const overlayStep = ( + params: Record, + fileParameters?: Record, + ): WorkingToolStep => ({ + toolId: "overlayPdfs" as ToolId, + operation: "/api/v1/general/overlay-pdfs", + params: { ...overlayDefaults, ...params }, support: "editable", + fileParameters, }); - test("detects a File (or list of Files) among the parameters", () => { - const image = new File(["x"], "logo.png", { type: "image/png" }); - expect(stepRequiresUpload(step({ level: 5 }))).toBe(false); - expect(stepRequiresUpload(step({ watermarkImage: image }))).toBe(true); - expect(stepRequiresUpload(step({ attachments: [image] }))).toBe(true); + const certStep = ( + params: Record, + fileParameters?: Record, + ): WorkingToolStep => ({ + toolId: "certSign" as ToolId, + operation: "/api/v1/security/cert-sign", + params: { ...certSignDefaults, signMode: "MANUAL", ...params }, + support: "editable", + fileParameters, + }); + + test("extractStepFiles groups fresh picks by their backend file field", () => { + const a = new File(["1"], "a.pdf", { type: "application/pdf" }); + const b = new File(["2"], "b.pdf", { type: "application/pdf" }); + expect( + extractStepFiles(overlayStep({ overlayFiles: [a, b] }), fileRegistry), + ).toEqual({ overlayFiles: [a, b] }); + }); + + test("extractStepFiles respects a tool's file selection (certSign by certType)", () => { + const p12 = new File(["k"], "key.p12"); + expect( + extractStepFiles( + certStep({ certType: "PKCS12", p12File: p12 }), + fileRegistry, + ), + ).toEqual({ p12File: [p12] }); + }); + + test("serialize/deserialize round-trips fileParameters", () => { + const step = certStep({ certType: "PKCS12" }, { p12File: "asset:abc" }); + const api = serializeToolStep(step, fileRegistry); + expect(api.fileParameters).toEqual({ p12File: "asset:abc" }); + expect(deserializeToolStep(api, fileRegistry).fileParameters).toEqual({ + p12File: "asset:abc", + }); + }); + + test("stepNeedsConfiguring: a stored binding satisfies the file requirement", () => { + expect( + stepNeedsConfiguring( + certStep({ certType: "PKCS12" }, { p12File: "asset:abc" }), + fileRegistry, + ), + ).toBe(false); + // Without the binding the keystore is still owed. + expect( + stepNeedsConfiguring(certStep({ certType: "PKCS12" }), fileRegistry), + ).toBe(true); + }); + + test("activeFileFields drops a stored binding the tool no longer emits", () => { + // Still PKCS12: the p12File binding is what the tool sends. + expect( + activeFileFields( + certStep({ certType: "PKCS12" }, { p12File: "asset:abc" }), + fileRegistry, + ), + ).toEqual(["p12File"]); + // Switched to PEM: certSign wants privateKeyFile/certFile, so the p12File binding is stale. + expect( + activeFileFields( + certStep({ certType: "PEM" }, { p12File: "asset:abc" }), + fileRegistry, + ), + ).toEqual([]); + }); + + test("activeFileFields is null (not empty) when the tool can't be probed", () => { + // A buildFormData that throws can't be probed; returning null (vs []) tells callers to keep the + // step's stored bindings rather than drop them and let the server GC the assets. + const config = asRegistryConfig<{ signingCert?: File }>({ + toolType: ToolType.singleFile, + operationType: "certSign", + endpoint: "/api/v1/security/cert-sign", + defaultParameters: {}, + buildFormData: () => { + throw new Error("cannot build"); + }, + }); + const registry: Partial = { + certSign: entry({ name: "Boom", operationConfig: config }), + }; + const step: WorkingToolStep = { + toolId: "certSign" as ToolId, + operation: "/api/v1/security/cert-sign", + params: {}, + support: "editable", + fileParameters: { certFile: "asset:x" }, + }; + expect(activeFileFields(step, registry)).toBeNull(); + }); + + test("the overlay sentinel is sized to the binding's asset count", () => { + // Two ids -> two files, matching two counts, so FixedRepeat validation passes. + const step = overlayStep( + { overlayMode: "FixedRepeatOverlay", counts: [1, 2] }, + { overlayFiles: "asset:one,two" }, + ); + expect(activeFileFields(step, fileRegistry)).toEqual(["overlayFiles"]); + expect(stepNeedsConfiguring(step, fileRegistry)).toBe(false); + }); + + test("a rename override binds a backend field to a differently-named param", () => { + // The cert-sign endpoint's `certFile` is held by a frontend param named `signingCert`. + const config = asRegistryConfig<{ signingCert?: File }>({ + toolType: ToolType.singleFile, + operationType: "certSign", + endpoint: "/api/v1/security/cert-sign", + defaultParameters: {}, + validateParams: (p) => p.signingCert !== undefined, + // Sends the File under the backend field `certFile`, like real tools do via objectToFormData + // (which sends a param's File or File[] under a named field, iterating arrays). + buildFormData: (p, file) => + objectToFormData({}, { fileInput: file, certFile: p.signingCert }), + fileParamOverrides: [{ field: "certFile", param: "signingCert" }], + }); + const registry: Partial = { + certSign: entry({ name: "Sign", operationConfig: config }), + }; + const step = ( + fileParameters?: Record, + ): WorkingToolStep => ({ + toolId: "certSign" as ToolId, + operation: "/api/v1/security/cert-sign", + params: {}, + support: "editable", + fileParameters, + }); + // The stored binding is keyed by the backend field, but satisfies the frontend param on reload. + expect(stepNeedsConfiguring(step({ certFile: "asset:x" }), registry)).toBe( + false, + ); + expect(stepNeedsConfiguring(step(), registry)).toBe(true); + expect(activeFileFields(step({ certFile: "asset:x" }), registry)).toEqual([ + "certFile", + ]); }); }); diff --git a/frontend/editor/src/core/hooks/tools/shared/toolAutomation.ts b/frontend/editor/src/core/hooks/tools/shared/toolAutomation.ts index 0485791f15..a9368ccdb7 100644 --- a/frontend/editor/src/core/hooks/tools/shared/toolAutomation.ts +++ b/frontend/editor/src/core/hooks/tools/shared/toolAutomation.ts @@ -18,11 +18,13 @@ import { type ToolRegistryEntry, } from "@app/data/toolsTaxonomy"; import { type ToolId } from "@app/types/toolId"; +import { TOOL_FILE_FIELDS } from "@app/types/toolApiTypes"; import { isToolEndpoint, type ToolEndpoint, } from "@app/hooks/tools/shared/toolApiMapping"; import { + ToolType, type ErasedToolParams, type RegistryToolOperationConfig, } from "@app/hooks/tools/shared/toolOperationTypes"; @@ -62,6 +64,12 @@ export interface ExecutableTool { export interface ToolApiStep { operation: string; parameters: Record; + /** + * Supporting-file bindings: a backend file field (e.g. `stampImage`, `overlayFiles`) mapped to + * `asset:[,]` (stored supporting files) or a run-supplied key. Absent when the step needs + * no supporting file. Mirrors the wire {@code PipelineStep.fileParameters}. + */ + fileParameters?: SupportingFileBindings; } /** A step being edited in a UI that maps to a known tool: parameters are in the tool's frontend shape. */ @@ -70,6 +78,12 @@ export interface KnownToolStep { operation: ToolEndpoint; params: ErasedToolParams; support: ToolStepSupport; + /** + * Stored supporting-file bindings carried from a saved step (field -> `asset:`), so an edit + * round-trips them without the user re-picking. A field the user re-picks lands in `params` as a + * File and takes precedence on save. + */ + fileParameters?: SupportingFileBindings; } /** A stored step whose endpoint maps to no known tool: preserved verbatim, not editable. */ @@ -78,6 +92,8 @@ export interface UnknownToolStep { operation: string; params: ErasedToolParams; support: "unknown"; + /** Supporting-file bindings preserved verbatim, so an unknown step's files round-trip untouched. */ + fileParameters?: SupportingFileBindings; } /** A step being edited in a UI, discriminated by whether its endpoint maps to a known tool. */ @@ -135,12 +151,176 @@ function isFileValue(value: unknown): boolean { } /** - * True if any of a step's parameters is an uploaded file (or list of files). Such a step cannot be - * saved into a stored pipeline yet: the file bytes are not persisted with the policy, so a later - * (e.g. scheduled) run would have nothing to send for that named file field. + * A stored supporting-file id, as returned by the asset store. */ -export function stepRequiresUpload(step: WorkingToolStep): boolean { - return Object.values(step.params).some(isFileValue); +declare const ASSET_ID_BRAND: unique symbol; +export type AssetId = string & { readonly [ASSET_ID_BRAND]: never }; + +/** + * A step's supporting-file bindings: each backend file field (e.g. `stampImage`) mapped to its file. + * A value of `asset:[,]` names stored assets loaded at run time; any other value is a key for + * a file supplied with the run itself. + */ +export type SupportingFileBindings = Record; + +/** + * The `fileParameters` binding format shared with the backend (see PolicyAssetRefs). This module owns + * the frontend side of the step contract, so the format lives here and the builder/settings reuse it. + */ +export const ASSET_REF_PREFIX = "asset:"; + +/** A `fileParameters` value binding one tool file field to the given stored asset ids. */ +export function assetRef(ids: readonly AssetId[]): string { + return ASSET_REF_PREFIX + ids.join(","); +} + +/** The stored asset ids inside a binding value, or none when it isn't an `asset:` ref. */ +export function assetRefIds(binding: string): AssetId[] { + if (!binding.startsWith(ASSET_REF_PREFIX)) return []; + return binding + .slice(ASSET_REF_PREFIX.length) + .split(",") + .map((id) => id.trim()) + .filter(Boolean) as AssetId[]; +} + +/** A throwaway primary document for probing a tool's buildFormData; never sent anywhere. */ +function dummyPrimaryFile(): File { + return new File([], "input.pdf", { type: "application/pdf" }); +} + +/** + * Run a tool's buildFormData so we can read the request it would produce. + * Returns null when File is unavailable or buildFormData throws. + */ +function probeFormData( + config: RegistryToolOperationConfig, + params: ErasedToolParams, +): FormData | null { + if (typeof File === "undefined") return null; + const dummy = dummyPrimaryFile(); + try { + switch (config.toolType) { + case ToolType.singleFile: + return config.buildFormData(params, dummy); + case ToolType.multiFile: + return config.buildFormData(params, [dummy]); + default: + return null; + } + } catch { + return null; + } +} + +/** Defaults merged under the step's params - the shape a tool's mappers and buildFormData expect. */ +function mergedStepParams( + step: WorkingToolStep, + config: RegistryToolOperationConfig, +): ErasedToolParams { + return { ...(config.defaultParameters ?? {}), ...step.params }; +} + +/** The backend file fields an endpoint accepts, from the generated spec-sourced table. */ +function backendFileFields(operation: string): readonly string[] { + return ( + (TOOL_FILE_FIELDS as Partial>)[ + operation + ] ?? [] + ); +} + +/** + * Each backend file field the step's endpoint accepts (from {@link TOOL_FILE_FIELDS}), mapped to the + * tool param that holds it - the same name unless the tool declared a rename override. + */ +function fileFieldMappings( + operation: string, + config: RegistryToolOperationConfig, +): { field: string; param: string }[] { + // The override's erased type collapses `param` to `never`; restore the real runtime shape. + const overrides = (config.fileParamOverrides ?? []) as readonly { + field: string; + param: string; + }[]; + const paramByField = new Map(overrides.map((o) => [o.field, o.param])); + return backendFileFields(operation).map((field) => ({ + field, + param: paramByField.get(field) ?? field, + })); +} + +/** + * The step's params with a stand-in File array injected for each stored binding whose param has no + * fresh pick, so a tool's buildFormData/validateParams sees the supporting file as present. Stored + * bindings are keyed by the backend field (from {@link TOOL_FILE_FIELDS}), so each field finds its + * binding and the sentinel lands on its param - the two coincide unless the tool declared a rename + * override. The array is sized to the binding's asset count (overlay validates count == file count). + * Sentinels are empty and live only in this local object - never written back to step.params, so they + * can never be uploaded. + */ +function withStoredFileSentinels( + step: WorkingToolStep, + config: RegistryToolOperationConfig, +): ErasedToolParams { + const merged = mergedStepParams(step, config); + const bindings = step.fileParameters; + if (!bindings || typeof File === "undefined") return merged; + for (const { param, field } of fileFieldMappings(step.operation, config)) { + const binding = bindings[field]; + if (binding == null || isFileValue(merged[param])) continue; // unbound, or a fresh pick stands in + const count = Math.max(1, assetRefIds(binding).length); + merged[param] = Array.from({ length: count }, () => new File([], "stored")); + } + return merged; +} + +/** + * The fresh File picks on a step, grouped by the backend file field its buildFormData sends them + * under (excluding the primary `fileInput`). buildFormData is the source of truth for the field name + * and for tool-specific selection (certSign picks files by certType), so probing it - rather than + * scanning params - keeps the field mapping correct. These are the files to upload on save. + */ +export function extractStepFiles( + step: WorkingToolStep, + registry: Partial, +): Record { + if (step.toolId === null) return {}; + const config = registry[step.toolId]?.operationConfig; + if (!config) return {}; + const formData = probeFormData(config, mergedStepParams(step, config)); + if (!formData) return {}; + const files: Record = {}; + formData.forEach((value, key) => { + if (key !== "fileInput" && value instanceof File) { + (files[key] ??= []).push(value); + } + }); + return files; +} + +/** + * The backend file fields this step actually uses right now, per its own buildFormData: fresh picks + * plus any stored binding the tool still emits (a stale one - e.g. a PKCS12 keystore after switching + * to PEM - is dropped, because buildFormData no longer sends it). Drives the stored-file chips, the + * save-time binding set, and the test run. + */ +export function activeFileFields( + step: WorkingToolStep, + registry: Partial, +): string[] | null { + if (step.toolId === null) { + return step.fileParameters ? Object.keys(step.fileParameters) : []; + } + const config = registry[step.toolId]?.operationConfig; + if (!config) return null; + const formData = probeFormData(config, withStoredFileSentinels(step, config)); + if (!formData) return null; + const fields = new Set(); + formData.forEach((value, key) => { + if (key !== "fileInput" && value instanceof File) fields.add(key); + }); + return [...fields]; } /** @@ -158,9 +338,10 @@ export function stepNeedsConfiguring( ): boolean { if (step.toolId === null) return false; const config = registry[step.toolId]?.operationConfig; - if (!config?.validateParams) return false; - const merged = { ...(config.defaultParameters ?? {}), ...step.params }; - return !config.validateParams(merged); + if (!config || !config.validateParams) return false; + // Stored supporting files satisfy their field just as a fresh pick would, so validate against the + // sentinel-injected params rather than the bare ones (which drop the file on reload). + return !config.validateParams(withStoredFileSentinels(step, config)); } /** @@ -240,14 +421,27 @@ export function serializeToolStep( step.toolId !== null ? registry[step.toolId]?.operationConfig : undefined; if (!config) { // Unmapped step (unknown endpoint on edit): round-trip it unchanged. - return { operation: step.operation, parameters: step.params }; + return withFileParameters( + { operation: step.operation, parameters: step.params }, + step, + ); } const merged = { ...(config.defaultParameters ?? {}), ...step.params }; const operation = resolveEndpoint(config, merged) ?? step.operation; const parameters = config.toApiParams ? (config.toApiParams(merged) as Record) : {}; - return { operation, parameters }; + return withFileParameters({ operation, parameters }, step); +} + +/** Attach the step's supporting-file bindings to a serialized step, omitting the field when empty. */ +function withFileParameters( + serialized: ToolApiStep, + step: WorkingToolStep, +): ToolApiStep { + const bindings = step.fileParameters; + if (!bindings || Object.keys(bindings).length === 0) return serialized; + return { ...serialized, fileParameters: bindings }; } /** @@ -308,6 +502,7 @@ function unmappedStep(step: ToolApiStep): UnknownToolStep { operation: step.operation, params: { ...step.parameters }, support: "unknown", + fileParameters: step.fileParameters, }; } @@ -345,5 +540,11 @@ export function deserializeToolStep( resolveEndpoint(config, params) ?? (isToolEndpoint(step.operation) ? step.operation : undefined); if (operation === undefined) return unmappedStep(step); - return { toolId, operation, params, support: classifyToolStepSupport(entry) }; + return { + toolId, + operation, + params, + support: classifyToolStepSupport(entry), + fileParameters: step.fileParameters, + }; } diff --git a/frontend/editor/src/core/hooks/tools/shared/toolOperationTypes.ts b/frontend/editor/src/core/hooks/tools/shared/toolOperationTypes.ts index d6dbf3b355..d01d897002 100644 --- a/frontend/editor/src/core/hooks/tools/shared/toolOperationTypes.ts +++ b/frontend/editor/src/core/hooks/tools/shared/toolOperationTypes.ts @@ -3,7 +3,11 @@ import { StirlingFile } from "@app/types/fileContext"; import type { ResponseHandler } from "@app/utils/toolResponseProcessor"; import { ToolId } from "@app/types/toolId"; import type { ProcessingProgress } from "@app/hooks/tools/shared/useToolState"; -import type { ToolApiParams, ToolEndpoint } from "@app/types/toolApiTypes"; +import { + TOOL_FILE_FIELDS, + type ToolApiParams, + type ToolEndpoint, +} from "@app/types/toolApiTypes"; export type { ProcessingProgress, ResponseHandler }; @@ -45,6 +49,39 @@ export interface CustomProcessorResult { consumedAllInputs?: boolean; } +/** + * The parameter keys that carry a supporting file - a `File` or `File[]` value the tool sends + * beyond its primary document. Derived from the tool's own parameter type, so a file field can only + * ever be declared against a param that genuinely holds a file. + */ +export type FileParamKey = { + [K in keyof TParams]-?: NonNullable extends File | File[] + ? K + : never; +}[keyof TParams] & + string; + +/** + * The backend multipart file fields an endpoint accepts, from the generated {@link TOOL_FILE_FIELDS} + * (which the spec derives from the Java MultipartFile params). `never` for an endpoint that takes no + * supporting files. This is what makes a rename override's `field` a checked name, not a free string. + */ +export type BackendFileField = + TEndpoint extends keyof typeof TOOL_FILE_FIELDS + ? (typeof TOOL_FILE_FIELDS)[TEndpoint][number] + : never; + +/** + * A remap for the rare case where a tool's frontend file param has a different name from the backend + * field it is sent under. Both sides are checked: `field` must be one of the endpoint's generated + * backend file fields, and `param` a real file param of the tool. Same-name fields need no entry - + * they are derived from {@link TOOL_FILE_FIELDS} directly. + */ +export interface FileParamOverride { + field: BackendFileField; + param: FileParamKey; +} + /** * Configuration for tool operations defining processing behavior and API integration. * @@ -79,6 +116,14 @@ interface BaseToolOperationConfig { /** Default parameter values for automation */ defaultParameters?: TParams; + /** + * Rename overrides for supporting-file params. The set of a tool's file fields is derived from the + * generated {@link TOOL_FILE_FIELDS} (spec-sourced), keyed by the backend field name; declare an + * override only when a backend field maps to a differently-named frontend param, so a step composer + * can bind the stored file to the right param. Omitted by the common case where field == param. + */ + fileParamOverrides?: readonly FileParamOverride[]; + /** * Whether these parameters are complete enough to run. The same predicate a tool gives * `useBaseParameters` as its `validateFn`, so the Run button in the editor and anything composing diff --git a/frontend/editor/src/core/types/toolApiTypes.ts b/frontend/editor/src/core/types/toolApiTypes.ts index bf498eb450..b2bdf7fb4b 100644 --- a/frontend/editor/src/core/types/toolApiTypes.ts +++ b/frontend/editor/src/core/types/toolApiTypes.ts @@ -7,7 +7,7 @@ export interface AddAttachmentRequest { /** * The image file to be overlaid onto the PDF. */ - attachments: string[]; + attachments: File[]; /** * Convert the resulting PDF to PDF/A-3b format after adding attachments */ @@ -148,7 +148,7 @@ export interface AddStampRequest { * The rotation of the stamp in degrees */ rotation?: number; - stampImage?: string; + stampImage?: File; /** * The stamp text */ @@ -187,7 +187,7 @@ export interface AddWatermarkRequest { * The rotation of the watermark in degrees */ rotation?: number; - watermarkImage?: string; + watermarkImage?: File; /** * The watermark text */ @@ -525,9 +525,7 @@ export interface FlattenRequest { */ renderDpi?: number; } -export interface GeneralExtractBookmarksRequest { - file: string; -} +export type GeneralExtractBookmarksRequest = Record; export type GeneralFile = Record; export type GeneralPdfToSinglePageRequest = Record; export type GeneralRemoveImagePdfRequest = Record; @@ -788,7 +786,7 @@ export interface OverlayImageRequest { * Whether to overlay the image onto every page of the PDF. */ everyPage?: boolean; - imageFile: string; + imageFile: File; /** * The x-coordinate at which to place the top-left corner of the image. */ @@ -806,7 +804,7 @@ export interface OverlayPdfsRequest { /** * An array of PDF files to be used as overlays on the base PDF. The order in these files is applied based on the selected mode. */ - overlayFiles: string[]; + overlayFiles: File[]; /** * The mode of overlaying: 'SequentialOverlay' for sequential application, 'InterleavedOverlay' for round-robin application, 'FixedRepeatOverlay' for fixed repetition based on provided counts */ @@ -1276,7 +1274,6 @@ export interface ScannerEffectRequest { yellowish?: boolean; } export interface SecurityCertSignSessionsRequest { - file: string; request?: WorkflowCreationRequest; } export interface WorkflowCreationRequest { @@ -1291,8 +1288,8 @@ export interface WorkflowCreationRequest { } export interface SecurityCertSignValidateCertificateRequest { certType: string; - jksFile?: string; - p12File?: string; + jksFile?: File; + p12File?: File; password?: string; } export type SecurityGetInfoOnPdfRequest = Record; @@ -1302,7 +1299,7 @@ export interface SignPDFWithCertRequest { * The alias of the certificate to sign with. Required for WINDOWS_STORE and recommended for PKCS11 tokens holding multiple certificates. */ alias?: string; - certFile?: string; + certFile?: File; /** * The type of the digital certificate. WINDOWS_STORE and PKCS11 are hardware-backed and only available in the desktop app. */ @@ -1314,7 +1311,7 @@ export interface SignPDFWithCertRequest { | "SERVER" | "WINDOWS_STORE" | "PKCS11"; - jksFile?: string; + jksFile?: File; /** * The location where the PDF is signed */ @@ -1323,7 +1320,7 @@ export interface SignPDFWithCertRequest { * The name of the signer */ name?: string; - p12File?: string; + p12File?: File; /** * The page number where the signature should be visible. This is required if showSignature is set to true */ @@ -1340,7 +1337,7 @@ export interface SignPDFWithCertRequest { * Optional PKCS#11 slot index. When omitted the first slot with a token is used. */ pkcs11Slot?: number; - privateKeyFile?: string; + privateKeyFile?: File; /** * The reason for signing the PDF */ @@ -1355,7 +1352,7 @@ export interface SignPDFWithCertRequest { showSignature?: boolean; } export interface SignatureValidationRequest { - certFile?: string; + certFile?: File; } export interface SplitPagesRequest { /** @@ -1741,5 +1738,22 @@ export const TOOL_ENDPOINTS = [ "/api/v1/security/verify-pdf", ] as const satisfies readonly ToolEndpoint[]; +/** The supporting-file parameters each endpoint accepts beyond its primary fileInput, by name. */ +export const TOOL_FILE_FIELDS = { + "/api/v1/general/overlay-pdfs": ["overlayFiles"], + "/api/v1/misc/add-attachments": ["attachments"], + "/api/v1/misc/add-image": ["imageFile"], + "/api/v1/misc/add-stamp": ["stampImage"], + "/api/v1/security/add-watermark": ["watermarkImage"], + "/api/v1/security/cert-sign": [ + "privateKeyFile", + "certFile", + "p12File", + "jksFile", + ], + "/api/v1/security/cert-sign/validate-certificate": ["p12File", "jksFile"], + "/api/v1/security/validate-signature": ["certFile"], +} as const satisfies Partial>; + /** Union of every generated tool request model. */ export type ToolApiRequest = ToolApiParams[ToolEndpoint]; diff --git a/frontend/editor/src/portal/api/pipelineAssets.ts b/frontend/editor/src/portal/api/pipelineAssets.ts new file mode 100644 index 0000000000..b68c95fda5 --- /dev/null +++ b/frontend/editor/src/portal/api/pipelineAssets.ts @@ -0,0 +1,41 @@ +import { apiClient } from "@portal/api/http"; +import { type AssetId } from "@app/hooks/tools/shared/toolAutomation"; + +export { type AssetId }; + +/** + * Stored supporting files for pipeline steps (backend PolicyAssetController). + * + * A pipeline step that needs more than the document stream - a signing + * certificate, a watermark/stamp image, overlay PDFs, attachments - references + * its file by id from the step's `fileParameters` as `asset:`. The bytes are + * uploaded here first (the save-time validator rejects a policy that binds an + * asset id that doesn't yet exist), then a triggered or scheduled run loads the + * file server-side without anyone re-supplying it. Assets are team-scoped exactly + * like the policies that reference them, and unreferenced uploads are cleaned up + * server-side, so the builder never has to delete what a cancelled edit left. + */ + +/** Metadata for one stored supporting file. Mirrors the Java `PolicyAsset` record. */ +export interface PolicyAsset { + id: AssetId; + fileName: string; + contentType: string | null; + size: number; + createdAt: number; +} + +/** POST /api/v1/policies/assets: store a supporting file, returning its metadata (with the id). */ +export async function uploadPipelineAsset(file: File): Promise { + const form = new FormData(); + form.append("file", file); + return apiClient.local.multipart( + "/api/v1/policies/assets", + form, + ); +} + +/** GET /api/v1/policies/assets: the team's stored supporting files (metadata only). */ +export async function listPipelineAssets(): Promise { + return apiClient.local.json("/api/v1/policies/assets"); +} diff --git a/frontend/editor/src/portal/api/pipelines.ts b/frontend/editor/src/portal/api/pipelines.ts index 50bdc173c4..e441fcdac8 100644 --- a/frontend/editor/src/portal/api/pipelines.ts +++ b/frontend/editor/src/portal/api/pipelines.ts @@ -1,5 +1,8 @@ import { apiClient } from "@portal/api/http"; -import { type ToolApiStep } from "@app/hooks/tools/shared/toolAutomation"; +import { + type SupportingFileBindings, + type ToolApiStep, +} from "@app/hooks/tools/shared/toolAutomation"; /** * Pipelines service layer: the backend contract. @@ -15,7 +18,7 @@ import { type ToolApiStep } from "@app/hooks/tools/shared/toolAutomation"; export interface PipelineStep { operation: string; parameters: Record; - fileParameters?: Record; + fileParameters?: SupportingFileBindings; } /** When a policy input fires automatically. `type` keys a trigger bean (e.g. "schedule"). */ @@ -217,14 +220,28 @@ export interface TestRunDefinition { output: OutputSpec; } +/** + * A fresh, in-memory supporting file sent inline with a test run, bound to the run key a test step's + * `fileParameters` references. Only unsaved picks ride along here; a stored file keeps its + * `asset:` binding, which the backend resolves from the saved policy (see `runPipelineTest`). + */ +export interface TestRunAsset { + key: string; + file: File; +} + /** * POST /api/v1/policies/run: run a definition against one uploaded file now. The builder's test * path - callers force an inline output so nothing reaches the pipeline's real destination, and - * the pipeline need not be saved first. + * the pipeline need not be saved first. Fresh supporting files travel as keyed `assets[i]` parts; + * a stored file keeps its `asset:` binding, and `policyId` lets the backend resolve it from that + * saved policy (so its bytes need not be re-sent). */ export async function runPipelineTest( definition: TestRunDefinition, file: File, + assets: TestRunAsset[] = [], + policyId?: string, ): Promise<{ runId: string }> { const form = new FormData(); form.append( @@ -232,6 +249,11 @@ export async function runPipelineTest( new Blob([JSON.stringify(definition)], { type: "application/json" }), ); form.append("fileInput", file); + if (policyId) form.append("policyId", policyId); + assets.forEach((asset, i) => { + form.append(`assets[${i}].key`, asset.key); + form.append(`assets[${i}].file`, asset.file); + }); // The POST returns the identifier as `jobId`, but it is the same run id every other endpoint // (fetchRun, fetchRunOutput) calls `runId`; normalise to that here so callers see one name. const res = await apiClient.local.multipart<{ jobId: string }>( diff --git a/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.css b/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.css new file mode 100644 index 0000000000..46bb72a736 --- /dev/null +++ b/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.css @@ -0,0 +1,17 @@ +.portal-step-settings__files { + display: flex; + flex-direction: column; + gap: 0.375rem; + margin-bottom: 0.75rem; +} + +.portal-step-settings__files-label { + font-size: 0.75rem; + color: var(--c-text-muted); +} + +.portal-step-settings__files-chips { + display: flex; + flex-wrap: wrap; + gap: 0.375rem; +} diff --git a/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.stories.tsx b/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.stories.tsx index d2bd6bc6f2..823d0ae4fe 100644 --- a/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.stories.tsx +++ b/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.stories.tsx @@ -52,6 +52,8 @@ const meta = { step: editableStep, registry, onChange: () => {}, + assetNames: {}, + onClearBinding: () => {}, }, } satisfies Meta; export default meta; diff --git a/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.test.tsx b/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.test.tsx index 55a52b496e..0b15b7233a 100644 --- a/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.test.tsx +++ b/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.test.tsx @@ -116,6 +116,8 @@ describe("PipelineStepSettings", () => { step={step} registry={registry} onChange={() => {}} + assetNames={{}} + onClearBinding={() => {}} /> , ), @@ -131,6 +133,8 @@ describe("PipelineStepSettings", () => { step={convertStep} registry={convertRegistry} onChange={() => {}} + assetNames={{}} + onClearBinding={() => {}} /> , ), @@ -146,6 +150,8 @@ describe("PipelineStepSettings", () => { step={changeMetadataStep} registry={changeMetadataRegistry} onChange={() => {}} + assetNames={{}} + onClearBinding={() => {}} /> , ), @@ -161,6 +167,8 @@ describe("PipelineStepSettings", () => { step={overlayStep} registry={overlayRegistry} onChange={() => {}} + assetNames={{}} + onClearBinding={() => {}} /> , ), @@ -205,6 +213,8 @@ describe("PipelineStepSettings", () => { typeof update === "function" ? update(prev) : update, ) } + assetNames={{}} + onClearBinding={() => {}} /> {JSON.stringify(params)} diff --git a/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.tsx b/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.tsx index 58fb2a96fa..3d17dbf652 100644 --- a/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.tsx +++ b/frontend/editor/src/portal/components/pipelines/PipelineStepSettings.tsx @@ -1,15 +1,22 @@ import { Suspense } from "react"; import { useTranslation } from "react-i18next"; -import { Banner } from "@app/ui"; +import InsertDriveFileOutlinedIcon from "@mui/icons-material/InsertDriveFileOutlined"; +import { Banner, Chip } from "@app/ui"; import { PreferencesProvider } from "@app/contexts/PreferencesContext"; import { SidebarProvider } from "@app/contexts/SidebarContext"; import { type ToolRegistry } from "@app/data/toolsTaxonomy"; import { type ErasedToolParams } from "@app/hooks/tools/shared/toolOperationTypes"; -import { type WorkingToolStep } from "@app/hooks/tools/shared/toolAutomation"; +import { + activeFileFields, + assetRefIds, + extractStepFiles, + type WorkingToolStep, +} from "@app/hooks/tools/shared/toolAutomation"; import { PolicyExternalApiConfig } from "@portal/components/policies/PolicyExternalApiConfig"; import { isIntegrationStep } from "@portal/components/pipelines/integrationStep"; import type { ExternalApiStepParams } from "@portal/components/policies/stepOperations"; +import "@portal/components/pipelines/PipelineStepSettings.css"; /** * A params update: the next params outright, or a merge from the latest params. Settings UIs fire @@ -25,17 +32,61 @@ interface PipelineStepSettingsProps { step: WorkingToolStep; registry: Partial; onChange: (update: ParamsUpdate) => void; + /** Stored asset id -> file name, for labelling the supporting-file chips on a reopened pipeline. */ + assetNames: Record; + /** Drop a field's stored supporting-file binding (the user re-picks a file if the step still needs one). */ + onClearBinding: (field: string) => void; +} + +/** One reopened supporting file shown as a chip: the field it binds and the stored file name(s). */ +interface StoredFileChip { + field: string; + label: string; +} + +/** + * The supporting files this step is reusing from a previous save: an active binding whose field has + * no fresh pick (a fresh pick shows in the tool's own file picker instead). Labelled by the resolved + * asset name so the user sees "using cert.pfx" rather than an empty picker. + */ +function storedFileChips( + step: WorkingToolStep, + registry: Partial, + assetNames: Record, +): StoredFileChip[] { + const bindings = step.fileParameters; + if (!bindings) return []; + // A null active set means the tool couldn't be probed; show every stored binding rather than hide + // the user's files (mirrors the save path, which keeps them too). + const active = activeFileFields(step, registry); + const activeSet = active === null ? null : new Set(active); + const fresh = extractStepFiles(step, registry); + return Object.entries(bindings) + .filter( + ([field]) => + (activeSet === null || activeSet.has(field)) && !fresh[field], + ) + .map(([field, binding]) => ({ + field, + label: + assetRefIds(binding) + .map((id) => assetNames[id] ?? id) + .join(", ") || binding, + })); } /** * Renders the parameter editor for one pipeline step, chosen by the tool's capability: * the tool's own settings UI when editable, an explanatory note when it has no parameters, - * or a "not supported yet" fallback for tools not yet migrated to the mapper seam. + * or a "not supported yet" fallback for tools not yet migrated to the mapper seam. Reopened + * supporting files appear as removable chips above the tool's own settings. */ export function PipelineStepSettings({ step, registry, onChange, + assetNames, + onClearBinding, }: PipelineStepSettingsProps) { // Hooks first: selecting a different step re-renders this same instance, so an early return // above useTranslation would change the hook count between renders and crash. @@ -52,41 +103,70 @@ export function PipelineStepSettings({ ); } - if (step.support === "noSettings") { - return ( - - ); - } + const chips = storedFileChips(step, registry, assetNames); - const entry = step.toolId ? registry[step.toolId] : undefined; - const Settings = - step.support === "editable" ? entry?.automationSettings : null; - - if (!Settings) { + function toolBody() { + if (step.support === "noSettings") { + return ( + + ); + } + const entry = step.toolId ? registry[step.toolId] : undefined; + const Settings = + step.support === "editable" ? entry?.automationSettings : null; + if (!Settings) { + return ( + + ); + } return ( - + + + + + onChange((prev) => ({ ...prev, [key]: value })) + } + disabled={false} + /> + + + ); } return ( - - - - - onChange((prev) => ({ ...prev, [key]: value })) - } - disabled={false} - /> - - - + <> + {chips.length > 0 && ( +
+ + {t("portal.pipelines.builder.supportingFiles")} + +
+ {chips.map((chip) => ( + + } + onRemove={() => onClearBinding(chip.field)} + > + {chip.label} + + ))} +
+
+ )} + {toolBody()} + ); } diff --git a/frontend/editor/src/portal/mocks/handlers/pipelines.ts b/frontend/editor/src/portal/mocks/handlers/pipelines.ts index ed7cec1626..c5505f38ff 100644 --- a/frontend/editor/src/portal/mocks/handlers/pipelines.ts +++ b/frontend/editor/src/portal/mocks/handlers/pipelines.ts @@ -116,6 +116,21 @@ function nextId(): string { return `plc_${Date.now().toString(36)}_${idCounter}`; } +/** Stored supporting files a step binds as `asset:` (PolicyAssetController), for mock mode. */ +interface StoredAsset { + id: string; + fileName: string; + contentType: string | null; + size: number; + createdAt: number; +} +let assetStore: StoredAsset[] = []; +let assetCounter = 0; +function nextAssetId(): string { + assetCounter += 1; + return `ast_${Date.now().toString(36)}_${assetCounter}`; +} + function deriveStatus(policy: StoredPolicy): PipelineStatus { return policy.enabled ? "active" : "paused"; } @@ -197,6 +212,33 @@ export const pipelinesHandlers = [ ]); }), + // Supporting files. Registered before the `/policies/:id` matcher so "assets" isn't read as an id. + http.get("/api/v1/policies/assets", async () => { + await delay(80); + return HttpResponse.json(assetStore); + }), + + http.post("/api/v1/policies/assets", async ({ request }) => { + const form = await request.formData(); + const file = form.get("file"); + if (!(file instanceof File)) { + return HttpResponse.json( + { detail: "Uploaded file is empty" }, + { status: 400 }, + ); + } + await delay(120); + const asset: StoredAsset = { + id: nextAssetId(), + fileName: file.name || "asset", + contentType: file.type || null, + size: file.size, + createdAt: Date.now(), + }; + assetStore = [...assetStore, asset]; + return HttpResponse.json(asset); + }), + // Run status: the mock completes runs immediately, so polling resolves at once. http.get("/api/v1/policies/run/:runId", async ({ params }) => { await delay(120); diff --git a/frontend/editor/src/portal/views/PipelineBuilder.test.tsx b/frontend/editor/src/portal/views/PipelineBuilder.test.tsx index 52fb5d516c..2737411c74 100644 --- a/frontend/editor/src/portal/views/PipelineBuilder.test.tsx +++ b/frontend/editor/src/portal/views/PipelineBuilder.test.tsx @@ -43,6 +43,13 @@ vi.mock("@portal/api/pipelines", () => ({ fetchRun: (runId: string) => fetchRun(runId), })); +const uploadPipelineAsset = vi.fn(); +const listPipelineAssets = vi.fn(); +vi.mock("@portal/api/pipelineAssets", () => ({ + uploadPipelineAsset: (file: File) => uploadPipelineAsset(file), + listPipelineAssets: () => listPipelineAssets(), +})); + const fetchSources = vi.fn(); vi.mock("@portal/api/sources", () => ({ fetchSources: () => fetchSources(), @@ -149,8 +156,21 @@ vi.mock("@app/contexts/ToolRegistryContext", () => { toolType: 0, endpoint: "/api/v1/misc/compress-pdf", defaultParameters: {}, - buildFormData: () => new FormData(), - toApiParams: (params: Record) => ({ ...params }), + // Sends the supporting file under a named field, like a real file tool, so the upload path + // has a field to bind. The scalar mapper drops the File (files never ride in parameters). + buildFormData: (params: Record, file: File | File[]) => { + const fd = new FormData(); + fd.append("fileInput", Array.isArray(file) ? file[0] : file); + if (params.watermarkImage instanceof File) { + fd.append("watermarkImage", params.watermarkImage); + } + return fd; + }, + toApiParams: (params: Record) => { + const scalars = { ...params }; + delete scalars.watermarkImage; + return scalars; + }, fromApiParams: (params: Record) => ({ ...params }), }, } as unknown as ToolRegistryEntry; @@ -203,10 +223,32 @@ vi.mock("@app/contexts/ToolRegistryContext", () => { fromApiParams: (params: Record) => ({ ...params }), }, } as unknown as ToolRegistryEntry; + // A tool whose buildFormData throws, so it can't be probed: exercises the "activeFileFields is + // null" path where a reopened step's stored binding must be kept, not dropped. + const sign = { + name: "Sign", + icon: null, + component: null, + description: "", + categoryId: "recommendedTools", + subcategoryId: "general", + operationConfig: { + operationType: "certSign", + toolType: 0, + endpoint: "/api/v1/security/cert-sign", + defaultParameters: {}, + buildFormData: () => { + throw new Error("cannot build"); + }, + toApiParams: (params: Record) => ({ ...params }), + fromApiParams: (params: Record) => ({ ...params }), + }, + } as unknown as ToolRegistryEntry; const allTools = { compress, extractImages, ocr, + sign, } as unknown as ToolRegistryCatalog["allTools"]; const catalog: ToolRegistryCatalog = { regularTools: allTools, @@ -288,6 +330,16 @@ describe("PipelineBuilder", () => { fetchS3Connections.mockReset(); fetchS3Connections.mockResolvedValue([]); createIntegration.mockReset(); + uploadPipelineAsset.mockReset(); + uploadPipelineAsset.mockResolvedValue({ + id: "ast-1", + fileName: "logo.png", + contentType: "image/png", + size: 1, + createdAt: 0, + }); + listPipelineAssets.mockReset(); + listPipelineAssets.mockResolvedValue([]); }); // The settings of a node are reached by selecting it in the graph, so every helper below opens @@ -798,7 +850,7 @@ describe("PipelineBuilder", () => { ).toBeInTheDocument(); }); - it("blocks saving a step that needs an uploaded file", async () => { + it("uploads a step's supporting file and saves it as an asset binding", async () => { renderBuilder("/processor/pipelines/new"); fireEvent.change( @@ -810,15 +862,65 @@ describe("PipelineBuilder", () => { }, ); await addTool("Compress"); - // The tool's settings upload a file, which a stored pipeline can't persist yet. + // The tool's settings attach a supporting file. fireEvent.click(await screen.findByText("upload logo")); - expect( - await screen.findByText("portal.pipelines.builder.uploadUnsupported"), - ).toBeInTheDocument(); - expect( - screen.getByText("portal.pipelines.composer.create").closest("button"), - ).toBeDisabled(); + await pickInputSource("Claims intake"); + await pickDestination(); + + fireEvent.click(screen.getByText("portal.pipelines.composer.create")); + + // The file is uploaded to the asset store first, then the policy is saved binding that asset. + await waitFor(() => expect(uploadPipelineAsset).toHaveBeenCalledTimes(1)); + expect(uploadPipelineAsset.mock.calls[0][0]).toBeInstanceOf(File); + await waitFor(() => expect(savePipeline).toHaveBeenCalledTimes(1)); + expect(savePipeline).toHaveBeenCalledWith( + expect.objectContaining({ + steps: [ + expect.objectContaining({ + operation: "/api/v1/misc/compress-pdf", + fileParameters: { watermarkImage: "asset:ast-1" }, + }), + ], + }), + ); + }); + + it("keeps a step's stored file binding on save when the tool can't be probed", async () => { + // buildFormData throws for `sign`, so activeFileFields is null. The stored binding must survive + // the save unchanged - dropping it would let the server GC the user's uploaded file - and no + // re-upload should happen. + fetchPipeline.mockResolvedValue({ + id: "plc-sign", + name: "Signed", + enabled: true, + inputs: [{ sourceId: "src-in", trigger: null }], + steps: [ + { + operation: "/api/v1/security/cert-sign", + parameters: {}, + fileParameters: { certFile: "asset:x" }, + }, + ], + output: { type: "inline", options: {} }, + outputIds: ["src-1"], + }); + renderBuilder("/processor/pipelines/plc-sign"); + + fireEvent.click(await screen.findByText("portal.pipelines.composer.save")); + + await waitFor(() => expect(savePipeline).toHaveBeenCalledTimes(1)); + expect(savePipeline).toHaveBeenCalledWith( + expect.objectContaining({ + steps: [ + expect.objectContaining({ + operation: "/api/v1/security/cert-sign", + fileParameters: { certFile: "asset:x" }, + }), + ], + }), + ); + expect(uploadPipelineAsset).not.toHaveBeenCalled(); }); it("blocks saving an integration step with no account chosen", async () => { diff --git a/frontend/editor/src/portal/views/PipelineBuilder.tsx b/frontend/editor/src/portal/views/PipelineBuilder.tsx index d5092f5c88..975a8e5cfe 100644 --- a/frontend/editor/src/portal/views/PipelineBuilder.tsx +++ b/frontend/editor/src/portal/views/PipelineBuilder.tsx @@ -17,14 +17,17 @@ import { } from "@app/ui"; import { useToolRegistry } from "@app/contexts/ToolRegistryContext"; import { + activeFileFields, + assetRef, deserializeToolStep, + extractStepFiles, getExecutableTools, newWorkingToolStep, serializeToolStep, stepNeedsConfiguring, - stepRequiresUpload, updateWorkingStepParams, type ExecutableTool, + type SupportingFileBindings, type WorkingToolStep, } from "@app/hooks/tools/shared/toolAutomation"; import { @@ -48,13 +51,20 @@ import { runPipelineTest, savePipeline, triggerPipeline, + type PipelineStep, type Policy, type PolicyRunView, type RunOutputFile, + type TestRunAsset, type TriggerConfig, type TriggerInfo, type TriggerOutcome, } from "@portal/api/pipelines"; +import { + listPipelineAssets, + uploadPipelineAsset, + type PolicyAsset, +} from "@portal/api/pipelineAssets"; import { clearProcessedHistory } from "@portal/api/policies"; import { DestinationPicker } from "@portal/components/pipelines/DestinationPicker"; import { availableOutputModes } from "@portal/components/pipelines/outputModes"; @@ -207,6 +217,17 @@ export function PipelineBuilder() { [allTools], ); + // Stored supporting files from earlier saves, so a reopened step can label its bindings by name. + const assetsState = useAsync( + async () => await listPipelineAssets(), + [], + ); + const assetNames = useMemo(() => { + const map: Record = {}; + for (const asset of assetsState.data ?? []) map[asset.id] = asset.fileName; + return map; + }, [assetsState.data]); + const policyState = useAsync( async () => (id ? await fetchPipeline(id) : null), [id], @@ -486,6 +507,21 @@ export function PipelineBuilder() { ); } + /** Drop a step's stored supporting-file binding for one field (the chip's remove action). */ + function clearStepBinding(index: number, field: string) { + setSteps((current) => + current.map((step, i) => { + if (i !== index || !step.fileParameters) return step; + const next = { ...step.fileParameters }; + delete next[field]; + return { + ...step, + fileParameters: Object.keys(next).length > 0 ? next : undefined, + }; + }), + ); + } + function stepLabel(step: WorkingToolStep): string { // An integration step's endpoint is the same for every vendor, so the raw path would read // "External api call" for all of them. Name it by the operation instead. @@ -512,11 +548,6 @@ export function PipelineBuilder() { return step.toolId ? allTools[step.toolId]?.icon : undefined; } - // Steps whose params carry an uploaded file can't be saved: the bytes aren't persisted with the - // policy, so a later run would send null for that field (see stepRequiresUpload). - const uploadStepLabels = steps.filter(stepRequiresUpload).map(stepLabel); - const hasUploadSteps = uploadStepLabels.length > 0; - // A step still missing a choice - an integration with no operation or account, a tool whose // mandatory parameters are unset - would fail at run time with a raw backend rejection, so block // saving on it here where the fix is one click away. @@ -601,11 +632,30 @@ export function PipelineBuilder() { // seeding, so leaving the builder can prompt to save or discard. `enabled` is deliberately left // out: in edit it is toggled and persisted at once (never an unsaved edit), and in create it is // chosen at submit - so it can never be the thing that makes the form dirty. + // Per-step dirty signature: the serialized step plus a stable identity (name/size/mtime) of its + // fresh file picks - a raw File JSON-stringifies to `{}`, so serializeToolStep (which excludes + // Files) can't see a file added or swapped. Memoized on the steps because it probes each tool's + // buildFormData; without this it would re-run for every step on any render (e.g. each keystroke in + // the name field). Stored bindings are covered by the serialized step. + const stepSnapshot = useMemo( + () => + steps.map((step) => { + const files: Record = {}; + for (const [field, picks] of Object.entries( + extractStepFiles(step, allTools), + )) { + files[field] = picks.map( + (file) => `${file.name}:${file.size}:${file.lastModified}`, + ); + } + return { step: serializeToolStep(step, allTools), files }; + }), + [steps, allTools], + ); const snapshot = JSON.stringify({ name: name.trim(), input, - steps: steps.map((step) => serializeToolStep(step, allTools)), - uploads: steps.map(stepRequiresUpload), + steps: stepSnapshot, outputIds: [...outputIds].sort(), }); const baseline = useRef(null); @@ -639,12 +689,6 @@ export function PipelineBuilder() { tools: unconfiguredStepLabels.join(", "), }), ); - if (hasUploadSteps) - blockers.push( - t("portal.pipelines.builder.blocker.upload", { - tools: uploadStepLabels.join(", "), - }), - ); if (hasIncompatibleSteps) blockers.push( t("portal.pipelines.builder.blocker.incompatible", { @@ -667,23 +711,84 @@ export function PipelineBuilder() { else navigate(destination); } + /** + * The active supporting-file fields of a step, each paired with its fresh in-memory pick(s) and its + * stored `asset:` binding (either may be absent). The single source both saving and test-running + * read, so the two agree on which fields are active and how a binding is chosen; they differ only in + * how a fresh pick is emitted - uploaded as an asset vs. sent inline. + */ + function stepFileFields( + step: WorkingToolStep, + ): { field: string; fresh: File[] | null; stored: string | null }[] { + const fresh = extractStepFiles(step, allTools); + const stored = step.fileParameters ?? {}; + const fields = activeFileFields(step, allTools) ?? Object.keys(stored); + return fields.map((field) => ({ + field, + fresh: fresh[field] ?? null, + stored: stored[field] ?? null, + })); + } + + /** A wire step, attaching fileParameters only when it has any. */ + function toWireStep( + operation: string, + parameters: Record, + bindings: SupportingFileBindings, + ): PipelineStep { + return Object.keys(bindings).length > 0 + ? { operation, parameters, fileParameters: bindings } + : { operation, parameters }; + } + + /** + * The wire steps for saving: scalar params from serialization, plus supporting-file bindings. A + * fresh pick is uploaded to the asset store (the save-time validator rejects a policy that binds an + * asset id that doesn't yet exist); a stored binding the tool still uses is kept when the user + * didn't replace it. Uploads run in parallel; any abandoned by a later failure are GC'd server-side. + */ + async function serializeStepsForSave(): Promise { + return Promise.all( + steps.map(async (step) => { + const { operation, parameters } = serializeToolStep(step, allTools); + const entries = await Promise.all( + stepFileFields(step).map(async ({ field, fresh, stored }) => { + if (fresh?.length) { + const ids = await Promise.all( + fresh.map((file) => + uploadPipelineAsset(file).then((a) => a.id), + ), + ); + return [field, assetRef(ids)] as const; + } + return stored ? ([field, stored] as const) : null; + }), + ); + const bindings: SupportingFileBindings = Object.fromEntries( + entries.filter((e): e is readonly [string, string] => e !== null), + ); + return toWireStep(operation, parameters, bindings); + }), + ); + } + async function save(destination: string, enabledOverride?: boolean) { if (!canSave) return; setSubmitting(true); setError(null); - const policy: Policy = { - id: policyState.data?.id ?? undefined, - name: name.trim(), - enabled: enabledOverride ?? enabled, - // The wire shape stays a list; canSave guarantees the one input has a source. - inputs: [{ sourceId: input.sourceId, trigger: buildTriggerFor(input) }], - steps: steps.map((step) => serializeToolStep(step, allTools)), - // Destinations are the referenced saved sources; the inline output field is - // preserved as-is (e.g. an editor policy's membership metadata) or defaults to inline. - output: policyState.data?.output ?? { type: "inline", options: {} }, - outputIds, - }; try { + const policy: Policy = { + id: policyState.data?.id ?? undefined, + name: name.trim(), + enabled: enabledOverride ?? enabled, + // The wire shape stays a list; canSave guarantees the one input has a source. + inputs: [{ sourceId: input.sourceId, trigger: buildTriggerFor(input) }], + steps: await serializeStepsForSave(), + // Destinations are the referenced saved sources; the inline output field is + // preserved as-is (e.g. an editor policy's membership metadata) or defaults to inline. + output: policyState.data?.output ?? { type: "inline", options: {} }, + outputIds, + }; await savePipeline(policy); await invalidatePipelines(); navigate(destination); @@ -741,6 +846,32 @@ export function PipelineBuilder() { return null; } + /** + * The steps + inline supporting files for a test run. A fresh (in-memory) pick rides along as a + * keyed `assets[i]` under a per-step run key; a stored file keeps its `asset:` binding, which + * the backend resolves from the pipeline's saved policy (passed as policyId) - no re-fetch needed. + */ + function buildTestSteps(): { steps: PipelineStep[]; assets: TestRunAsset[] } { + const assets: TestRunAsset[] = []; + const outSteps = steps.map((step, i) => { + const { operation, parameters } = serializeToolStep(step, allTools); + const bindings: SupportingFileBindings = {}; + for (const { field, fresh, stored } of stepFileFields(step)) { + if (fresh?.length) { + // In-memory pick: inline the bytes under a run key. + const key = `s${i}_${field}`; + bindings[field] = key; + for (const file of fresh) assets.push({ key, file }); + } else if (stored) { + // Already an asset: keep its ref for the backend to resolve from the saved policy. + bindings[field] = stored; + } + } + return toWireStep(operation, parameters, bindings); + }); + return { steps: outSteps, assets }; + } + /** * Run the steps as they stand against one uploaded file. Output is forced inline so nothing * reaches the pipeline's real destination, and the pipeline need not be saved first - this is @@ -752,13 +883,17 @@ export function PipelineBuilder() { setTestRun(null); setRunResult(null); try { + const { steps: testSteps, assets } = buildTestSteps(); const { runId } = await runPipelineTest( { name: name.trim() || t("portal.pipelines.builder.testRun"), - steps: steps.map((step) => serializeToolStep(step, allTools)), + steps: testSteps, output: { type: "inline", options: {} }, }, file, + assets, + // Lets the backend resolve any stored `asset:` refs from this saved policy. + policyState.data?.id, ); const final = await awaitRun(runId, (view) => { if (mounted.current) setTestRun(view); @@ -934,8 +1069,6 @@ export function PipelineBuilder() { return t("portal.pipelines.builder.chooseAccount"); return undefined; } - if (stepRequiresUpload(step)) - return t("portal.pipelines.builder.needsUpload"); if (stepNeedsConfiguring(step, allTools)) return t("portal.pipelines.builder.needsConfiguring"); return undefined; @@ -1120,6 +1253,8 @@ export function PipelineBuilder() { step={selectedStep} registry={allTools} onChange={(params) => updateStepParams(chosenSteps[0], params)} + assetNames={assetNames} + onClearBinding={(field) => clearStepBinding(chosenSteps[0], field)} /> ); } @@ -1165,14 +1300,6 @@ export function PipelineBuilder() { {runResult && ( )} - {hasUploadSteps && ( - - )} {hasUnconfiguredSteps && ( Date: Thu, 20 Aug 2026 09:33:26 +0000 Subject: [PATCH 07/45] Float the editor search when no file is open (#7575) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What The super-search work pinned the editor's `WorkbenchBar` visible on every view except My Files, even with no file open. That left an empty workbench showing a fully painted bar whose only live control was the search — download / close / print / save were all disabled, because those actions only make sense with a file open. This stops forcing the bar. When nothing is open, only the global search floats (unpainted, centered), mirroring how the Processor already works. When a file **is** open, the `WorkbenchBar` renders exactly as before. Also fixes a smaller Processor issue: its floating search strip was shorter than the sidebar logo row, so the search sat higher than the brand. Its height now matches the logo row (51px) so they line up. ## Changes - **`Workbench.tsx`** — render the `WorkbenchBar` only when a file is open (or a custom view supplies content); otherwise render the new floating search. My Files and `hideTopControls` custom views are unchanged. - **`WorkbenchFloatingSearch.tsx` / `.css`** (new) — the editor's `SuperSearch` floated in an unpainted strip, mirroring `PortalSearchBar`. Its vertical band matches the bar's so opening a file swaps in the bar without a shift. - **`PortalSearchBar.css`** — strip height matched to `.portal-sidebar__logo` (51px) so the Processor search aligns with the logo. The notification bell is intentionally out of scope — it ships in a separate PR. ## Before / after (ignore the bell icon in the after that’s not live yet) Screenshot 2026-08-20 at 2 28
17 AM Screenshot 2026-08-20 at 2 28
31 AM - **Editor, no file:** painted bar with disabled buttons → just a floating search. - **Editor, file open:** unchanged. - **Processor:** search now vertically aligned with the logo. ## Testing - `task frontend:check` — lint (incl. colour linters) + typecheck + tests (247 files / 2137 tests) all pass. - Processor alignment verified in Storybook (`Portal/Shell/AppShell`): logo row, search strip, and search pill share the same vertical center. - Editor float not verified in-browser (local backend is behind a login gate); covered by types/tests and reuses the verified Processor pattern. --- .../src/core/components/layout/Workbench.tsx | 77 +++++++++++-------- .../shared/WorkbenchFloatingSearch.css | 31 ++++++++ .../shared/WorkbenchFloatingSearch.tsx | 15 ++++ .../src/portal/components/PortalSearchBar.css | 7 +- 4 files changed, 94 insertions(+), 36 deletions(-) create mode 100644 frontend/editor/src/core/components/shared/WorkbenchFloatingSearch.css create mode 100644 frontend/editor/src/core/components/shared/WorkbenchFloatingSearch.tsx diff --git a/frontend/editor/src/core/components/layout/Workbench.tsx b/frontend/editor/src/core/components/layout/Workbench.tsx index 7dbc6f75be..8bc0794140 100644 --- a/frontend/editor/src/core/components/layout/Workbench.tsx +++ b/frontend/editor/src/core/components/layout/Workbench.tsx @@ -18,6 +18,7 @@ import { useCookieConsent } from "@app/hooks/useCookieConsent"; import styles from "@app/components/layout/Workbench.module.css"; import WorkbenchBar from "@app/components/shared/WorkbenchBar"; +import WorkbenchFloatingSearch from "@app/components/shared/WorkbenchFloatingSearch"; import LandingPage from "@app/components/shared/LandingPage"; import DismissAllErrorsButton from "@app/components/shared/DismissAllErrorsButton"; import { ChatFAB } from "@app/components/chat/ChatFAB"; @@ -77,6 +78,22 @@ export default function Workbench() { const [viewerToolbarCollapsed, setViewerToolbarCollapsed] = useState(false); const showReopenTab = currentView === "viewer" && viewerToolbarCollapsed; + // The WorkbenchBar carries file-scoped actions, so it only shows once a file + // is open or a custom view supplies content; otherwise the search floats. + const activeCustomView = customWorkbenchViews.find( + (v) => v.workbenchId === currentView, + ); + const topControlsAvailable = + currentView !== "myFiles" && !activeCustomView?.hideTopControls; + const hasWorkbenchContent = + hasFiles || + fileIds.length > 0 || + !isBaseWorkbench(currentView) || + // Shared signing drives the viewer from the sidebar with no file in context. + (currentView === "viewer" && !!signingOverlay?.file); + const showWorkbenchBar = topControlsAvailable && hasWorkbenchContent; + const showFloatingSearch = topControlsAvailable && !hasWorkbenchContent; + const handlePreviewClose = () => { setPreviewFile(null); const previousMode = sessionStorage.getItem("previousMode"); @@ -231,41 +248,35 @@ export default function Workbench() { data-tour="workbench" style={{ backgroundColor: "var(--c-bg)", minWidth: 0 }} > - {/* Workbench Bar — always visible outside My Files (it hosts the - global search), even with no files loaded. */} - {currentView !== "myFiles" && - !customWorkbenchViews.find((v) => v.workbenchId === currentView) - ?.hideTopControls && ( -
-
-
- -
-
- {/* Reopen tab: a little handle hanging off the bar's bottom-right - while the viewer tool row is retracted. */} - {showReopenTab && ( -
+ )} + {showFloatingSearch && } {/* Dismiss All Errors Button */} diff --git a/frontend/editor/src/core/components/shared/WorkbenchFloatingSearch.css b/frontend/editor/src/core/components/shared/WorkbenchFloatingSearch.css new file mode 100644 index 0000000000..47c7e9eed4 --- /dev/null +++ b/frontend/editor/src/core/components/shared/WorkbenchFloatingSearch.css @@ -0,0 +1,31 @@ +/* Unpainted floating search for the empty workbench. Height + top margin match + the WorkbenchBar's band so opening a file swaps it in without a shift. */ +.workbench-floating-search { + display: flex; + align-items: center; + justify-content: center; + min-height: 38px; + margin-top: var(--nav-gutter); + padding: 0 1rem; + flex-shrink: 0; +} + +.workbench-floating-search .super-search { + flex: 0 1 24rem; + width: min(100%, 24rem); + max-width: 24rem; +} + +.workbench-floating-search .super-search input { + background-color: transparent; + padding-top: 4px; + padding-bottom: 4px; + font-size: 12.5px; +} + +[data-mantine-color-scheme="dark"] + .workbench-floating-search + .super-search + input { + background-color: transparent; +} diff --git a/frontend/editor/src/core/components/shared/WorkbenchFloatingSearch.tsx b/frontend/editor/src/core/components/shared/WorkbenchFloatingSearch.tsx new file mode 100644 index 0000000000..37e858adff --- /dev/null +++ b/frontend/editor/src/core/components/shared/WorkbenchFloatingSearch.tsx @@ -0,0 +1,15 @@ +import SuperSearch from "@app/components/shared/superSearch/SuperSearch"; +import { useEditorSearchScopes } from "@app/hooks/useSuperSearch"; +import "@app/components/shared/WorkbenchFloatingSearch.css"; + +// The editor's global search, floated while no file is open (mirrors the +// processor's PortalSearchBar). Renders only when the WorkbenchBar doesn't, so +// reusing the default input id is safe. +export default function WorkbenchFloatingSearch() { + const scopes = useEditorSearchScopes(); + return ( +
+ +
+ ); +} diff --git a/frontend/editor/src/portal/components/PortalSearchBar.css b/frontend/editor/src/portal/components/PortalSearchBar.css index cf1d9971c7..8e253b6136 100644 --- a/frontend/editor/src/portal/components/PortalSearchBar.css +++ b/frontend/editor/src/portal/components/PortalSearchBar.css @@ -1,10 +1,11 @@ -/* Slim strip at the top of the main column hosting the shared search bar. - Deliberately unpainted: only the input itself shows, on the page ground. */ +/* Unpainted strip at the top of the main column. Height matches the sidebar's + logo row (.portal-sidebar__logo, 51px) so the search lines up with the brand. */ .portal-searchbar { display: flex; align-items: center; justify-content: center; - padding: 0.22rem 1rem; + min-height: 3.1875rem; + padding: 0 1rem; flex-shrink: 0; } From 91fc26f10c21690af6d7c78c7f247ea1d5acfcaa Mon Sep 17 00:00:00 2001 From: Ludy Date: Thu, 20 Aug 2026 09:53:28 +0000 Subject: [PATCH 08/45] chore: Bump version to 2.14.3 (#7554) # Description of Changes This PR bumps the Stirling PDF application version from `2.14.2` to `2.14.3` across the project. Changes include: - Updated the Gradle project version in `build.gradle` to `2.14.3`. - Updated the Tauri desktop application version in `frontend/editor/src-tauri/tauri.conf.json`. - Updated the AUR package version for `stirling-pdf-desktop`. - Updated the AUR package version for `stirling-pdf-server-bin`. - Updated the mocked `appVersion` used by the core frontend server experience simulations. - Updated the mocked `appVersion` used by the proprietary frontend server experience simulations. - Kept all application, desktop, packaging, and test/simulation version references synchronized for the `2.14.3` release. The change prepares the project metadata and packaging configuration for the `2.14.3` release and prevents different components from reporting or packaging the previous `2.14.2` version. --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details. --- .github/aur/stirling-pdf-desktop/PKGBUILD | 2 +- .github/aur/stirling-pdf-server-bin/PKGBUILD | 2 +- build.gradle | 2 +- frontend/editor/src-tauri/tauri.conf.json | 2 +- frontend/editor/src/core/testing/serverExperienceSimulations.ts | 2 +- .../src/proprietary/testing/serverExperienceSimulations.ts | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/aur/stirling-pdf-desktop/PKGBUILD b/.github/aur/stirling-pdf-desktop/PKGBUILD index fb6a99cfca..c47c8e3ee7 100644 --- a/.github/aur/stirling-pdf-desktop/PKGBUILD +++ b/.github/aur/stirling-pdf-desktop/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: Stirling PDF Inc pkgname=stirling-pdf-desktop -pkgver=2.14.2 +pkgver=2.14.3 pkgrel=1 pkgdesc="Locally hosted, web-based PDF manipulation tool (Tauri desktop app, official Stirling PDF Inc build)" arch=('x86_64') diff --git a/.github/aur/stirling-pdf-server-bin/PKGBUILD b/.github/aur/stirling-pdf-server-bin/PKGBUILD index 70bcee0423..d6159ddce3 100644 --- a/.github/aur/stirling-pdf-server-bin/PKGBUILD +++ b/.github/aur/stirling-pdf-server-bin/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: Stirling PDF Inc pkgname=stirling-pdf-server-bin -pkgver=2.14.2 +pkgver=2.14.3 pkgrel=1 pkgdesc="Locally hosted, web-based PDF manipulation tool (server JAR, prebuilt)" arch=('any') diff --git a/build.gradle b/build.gradle index cadf4bafa6..30a9123ebb 100644 --- a/build.gradle +++ b/build.gradle @@ -108,7 +108,7 @@ springBoot { allprojects { group = 'stirling.software' - version = '2.14.2' + version = '2.14.3' configurations.configureEach { exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat" diff --git a/frontend/editor/src-tauri/tauri.conf.json b/frontend/editor/src-tauri/tauri.conf.json index dee2cc7023..c308e3ad4e 100644 --- a/frontend/editor/src-tauri/tauri.conf.json +++ b/frontend/editor/src-tauri/tauri.conf.json @@ -2,7 +2,7 @@ "$schema": "../node_modules/@tauri-apps/cli/config.schema.json", "productName": "Stirling PDF", "mainBinaryName": "Stirling-PDF", - "version": "2.14.2", + "version": "2.14.3", "identifier": "stirling.pdf.dev", "build": { "frontendDist": "../dist", diff --git a/frontend/editor/src/core/testing/serverExperienceSimulations.ts b/frontend/editor/src/core/testing/serverExperienceSimulations.ts index 97da95c730..55ab79c111 100644 --- a/frontend/editor/src/core/testing/serverExperienceSimulations.ts +++ b/frontend/editor/src/core/testing/serverExperienceSimulations.ts @@ -38,7 +38,7 @@ const FREE_LICENSE_INFO: LicenseInfo = { const BASE_NO_LOGIN_CONFIG: AppConfig = { enableAnalytics: true, - appVersion: "2.14.2", + appVersion: "2.14.3", serverCertificateEnabled: false, enableAlphaFunctionality: false, serverPort: 8080, diff --git a/frontend/editor/src/proprietary/testing/serverExperienceSimulations.ts b/frontend/editor/src/proprietary/testing/serverExperienceSimulations.ts index d92cf8fdf9..0519b9afb8 100644 --- a/frontend/editor/src/proprietary/testing/serverExperienceSimulations.ts +++ b/frontend/editor/src/proprietary/testing/serverExperienceSimulations.ts @@ -48,7 +48,7 @@ const FREE_LICENSE_INFO: LicenseInfo = { const BASE_NO_LOGIN_CONFIG: AppConfig = { enableAnalytics: true, - appVersion: "2.14.2", + appVersion: "2.14.3", serverCertificateEnabled: false, enableAlphaFunctionality: false, enableDesktopInstallSlide: true, From 4791d558c515c08bfbb954bb2af5315a68953b93 Mon Sep 17 00:00:00 2001 From: "stirlingbot[bot]" <195170888+stirlingbot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:16:18 +0000 Subject: [PATCH 09/45] Update Backend 3rd Party Licenses (#7579) Auto-generated by stirlingbot[bot] This PR updates the backend license report based on dependency changes. Signed-off-by: stirlingbot[bot] Co-authored-by: stirlingbot[bot] <195170888+stirlingbot[bot]@users.noreply.github.com> --- .../resources/static/3rdPartyLicenses.json | 226 ++++++++++++++---- 1 file changed, 185 insertions(+), 41 deletions(-) diff --git a/app/core/src/main/resources/static/3rdPartyLicenses.json b/app/core/src/main/resources/static/3rdPartyLicenses.json index fa852846b3..fbdce0a158 100644 --- a/app/core/src/main/resources/static/3rdPartyLicenses.json +++ b/app/core/src/main/resources/static/3rdPartyLicenses.json @@ -14,6 +14,13 @@ "moduleLicense": "GNU Lesser General Public License", "moduleLicenseUrl": "https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html" }, + { + "moduleName": "ch.qos.logback:logback-classic", + "moduleUrl": "http://www.qos.ch", + "moduleVersion": "1.6.1", + "moduleLicense": "LGPL-2.1-only", + "moduleLicenseUrl": "https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html" + }, { "moduleName": "ch.qos.logback:logback-core", "moduleUrl": "http://www.qos.ch", @@ -21,6 +28,13 @@ "moduleLicense": "GNU Lesser General Public License", "moduleLicenseUrl": "https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html" }, + { + "moduleName": "ch.qos.logback:logback-core", + "moduleUrl": "http://www.qos.ch", + "moduleVersion": "1.6.1", + "moduleLicense": "LGPL-2.1-only", + "moduleLicenseUrl": "https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html" + }, { "moduleName": "com.adobe.xmp:xmpcore", "moduleUrl": "https://www.adobe.com/devnet/xmp/library/eula-xmp-library-java.html", @@ -182,7 +196,7 @@ { "moduleName": "com.github.mwiede:jsch", "moduleUrl": "https://github.com/mwiede/jsch", - "moduleVersion": "0.2.23", + "moduleVersion": "2.28.6", "moduleLicense": "Revised BSD", "moduleLicenseUrl": "https://github.com/mwiede/jsch/blob/master/LICENSE.txt" }, @@ -758,7 +772,7 @@ { "moduleName": "commons-net:commons-net", "moduleUrl": "https://commons.apache.org/proper/commons-net/", - "moduleVersion": "3.11.1", + "moduleVersion": "3.13.0", "moduleLicense": "Apache-2.0", "moduleLicenseUrl": "https://www.apache.org/licenses/LICENSE-2.0.txt" }, @@ -1213,24 +1227,48 @@ "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "net.shibboleth:shib-networking", + "moduleVersion": "9.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "net.shibboleth:shib-security", "moduleVersion": "9.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "net.shibboleth:shib-security", + "moduleVersion": "9.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "net.shibboleth:shib-support", "moduleVersion": "9.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "net.shibboleth:shib-support", + "moduleVersion": "9.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "net.shibboleth:shib-velocity", "moduleVersion": "9.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "net.shibboleth:shib-velocity", + "moduleVersion": "9.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.antlr:antlr4-runtime", "moduleUrl": "https://www.antlr.org/", @@ -1325,13 +1363,6 @@ "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://www.apache.org/licenses/LICENSE-2.0.txt" }, - { - "moduleName": "org.apache.httpcomponents:httpclient", - "moduleUrl": "http://hc.apache.org/httpcomponents-client", - "moduleVersion": "4.5.13", - "moduleLicense": "Apache License, Version 2.0", - "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" - }, { "moduleName": "org.apache.httpcomponents:httpclient", "moduleUrl": "http://hc.apache.org/httpcomponents-client-ga", @@ -1456,6 +1487,13 @@ "moduleLicense": "Apache-2.0", "moduleLicenseUrl": "https://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.apache.santuario:xmlsec", + "moduleUrl": "https://www.apache.org/", + "moduleVersion": "3.0.6", + "moduleLicense": "Apache-2.0", + "moduleLicenseUrl": "https://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.apache.tomcat.embed:tomcat-embed-el", "moduleUrl": "https://tomcat.apache.org/", @@ -1470,6 +1508,13 @@ "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.apache.velocity:velocity-engine-core", + "moduleUrl": "https://www.apache.org/", + "moduleVersion": "2.4.1", + "moduleLicense": "Apache-2.0", + "moduleLicenseUrl": "https://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.apache.xmlbeans:xmlbeans", "moduleUrl": "https://xmlbeans.apache.org/", @@ -1647,6 +1692,13 @@ "moduleLicense": "GNU Lesser General Public License", "moduleLicenseUrl": "http://www.gnu.org/licenses/lgpl-3.0.txt" }, + { + "moduleName": "org.cryptacular:cryptacular", + "moduleUrl": "https://www.cryptacular.org", + "moduleVersion": "1.3.0", + "moduleLicense": "GNU Lesser General Public License", + "moduleLicenseUrl": "https://www.gnu.org/licenses/lgpl-3.0.txt" + }, { "moduleName": "org.eclipse.angus:angus-activation", "moduleUrl": "https://www.eclipse.org", @@ -2016,78 +2068,156 @@ "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-core-api", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.opensaml:opensaml-core-impl", "moduleVersion": "5.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-core-impl", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.opensaml:opensaml-messaging-api", "moduleVersion": "5.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-messaging-api", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.opensaml:opensaml-profile-api", "moduleVersion": "5.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-profile-api", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.opensaml:opensaml-saml-api", "moduleVersion": "5.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-saml-api", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.opensaml:opensaml-saml-impl", "moduleVersion": "5.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-saml-impl", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.opensaml:opensaml-security-api", "moduleVersion": "5.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-security-api", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.opensaml:opensaml-security-impl", "moduleVersion": "5.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-security-impl", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.opensaml:opensaml-soap-api", "moduleVersion": "5.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-soap-api", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.opensaml:opensaml-soap-impl", "moduleVersion": "5.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-soap-impl", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.opensaml:opensaml-storage-api", "moduleVersion": "5.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-storage-api", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.opensaml:opensaml-xmlsec-api", "moduleVersion": "5.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-xmlsec-api", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.opensaml:opensaml-xmlsec-impl", "moduleVersion": "5.1.6", "moduleLicense": "The Apache Software License, Version 2.0", "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" }, + { + "moduleName": "org.opensaml:opensaml-xmlsec-impl", + "moduleVersion": "5.2.2", + "moduleLicense": "The Apache Software License, Version 2.0", + "moduleLicenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, { "moduleName": "org.ow2.asm:asm", "moduleUrl": "http://asm.ow2.org", @@ -2564,6 +2694,13 @@ "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://www.apache.org/licenses/LICENSE-2.0" }, + { + "moduleName": "org.springframework.security:spring-security-core", + "moduleUrl": "https://spring.io/projects/spring-security", + "moduleVersion": "7.1.0", + "moduleLicense": "Apache License, Version 2.0", + "moduleLicenseUrl": "https://www.apache.org/licenses/LICENSE-2.0" + }, { "moduleName": "org.springframework.security:spring-security-crypto", "moduleUrl": "https://spring.io/projects/spring-security", @@ -2606,6 +2743,13 @@ "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://www.apache.org/licenses/LICENSE-2.0" }, + { + "moduleName": "org.springframework.security:spring-security-saml2-service-provider", + "moduleUrl": "https://spring.io/projects/spring-security", + "moduleVersion": "7.1.0", + "moduleLicense": "Apache License, Version 2.0", + "moduleLicenseUrl": "https://www.apache.org/licenses/LICENSE-2.0" + }, { "moduleName": "org.springframework.security:spring-security-web", "moduleUrl": "https://spring.io/projects/spring-security", @@ -2805,207 +2949,207 @@ }, { "moduleName": "software.amazon.awssdk:annotations", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { - "moduleName": "software.amazon.awssdk:apache-client", - "moduleVersion": "2.44.12", + "moduleName": "software.amazon.awssdk:apache5-client", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:arns", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:auth", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:aws-core", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:aws-query-protocol", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:aws-xml-protocol", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:checksums", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:checksums-spi", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:crt-core", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:endpoints-spi", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:http-auth", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:http-auth-aws", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:http-auth-aws-eventstream", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:http-auth-spi", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:http-client-spi", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:identity-spi", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:json-utils", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:metrics-spi", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:netty-nio-client", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:profiles", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:protocol-core", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:regions", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:retries", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:retries-spi", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:s3", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:sdk-core", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:third-party-jackson-core", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:url-connection-client", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:utils", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, { "moduleName": "software.amazon.awssdk:utils-lite", "moduleUrl": "https://aws.amazon.com/sdkforjava", - "moduleVersion": "2.44.12", + "moduleVersion": "2.51.3", "moduleLicense": "Apache License, Version 2.0", "moduleLicenseUrl": "https://aws.amazon.com/apache2.0" }, From 50d34fcca5e778ec9dd66ec5c6d2048a23a35a2e Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:47:05 +0000 Subject: [PATCH 10/45] Add download, rename and duplicate to the file actions menu (#7536) # Description of Changes Adds expanded dropdown menu for download, rename and duplicate image image --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details. --- .../public/locales/en-US/translation.toml | 15 + .../core/components/filesPage/FileGrid.tsx | 552 +++++++++--------- .../components/filesPage/FileManagerView.tsx | 101 ++++ .../core/components/shared/FileSidebar.tsx | 137 +++++ .../components/shared/FileSidebarFileItem.css | 22 + .../components/shared/FileSidebarFileItem.tsx | 351 +++++++---- .../shared/RenameFileDialog.stories.tsx | 39 ++ .../components/shared/RenameFileDialog.tsx | 140 +++++ .../core/components/shared/WorkbenchBar.tsx | 6 +- .../editor/src/core/hooks/useFileHandler.ts | 4 + .../tests/stubbed/file-actions-menu.spec.ts | 167 ++++++ .../src/core/utils/duplicateFile.test.ts | 130 +++++ .../editor/src/core/utils/duplicateFile.ts | 85 +++ frontend/editor/src/core/utils/fileUtils.ts | 11 + 14 files changed, 1373 insertions(+), 387 deletions(-) create mode 100644 frontend/editor/src/core/components/shared/RenameFileDialog.stories.tsx create mode 100644 frontend/editor/src/core/components/shared/RenameFileDialog.tsx create mode 100644 frontend/editor/src/core/tests/stubbed/file-actions-menu.spec.ts create mode 100644 frontend/editor/src/core/utils/duplicateFile.test.ts create mode 100644 frontend/editor/src/core/utils/duplicateFile.ts diff --git a/frontend/editor/public/locales/en-US/translation.toml b/frontend/editor/public/locales/en-US/translation.toml index 01db314031..c2287ace9c 100644 --- a/frontend/editor/public/locales/en-US/translation.toml +++ b/frontend/editor/public/locales/en-US/translation.toml @@ -3897,8 +3897,10 @@ collapse = "Collapse sidebar" customizeGroups = "Customize groups" dataLostBody = "This browser lost this file's contents. Upload it again to keep working with it." dataLostTitle = "File data is unavailable" +downloadFailed = "Download failed" dropHint = "Open files to get started" dropToAdd = "Drop files to add" +duplicateFailed = "Could not duplicate file" expand = "Expand sidebar" googleDrive = "Google Drive" googleDriveDisabled = "Google Drive is not configured" @@ -3918,8 +3920,10 @@ closeViewer = "Close viewer" dataLost = "Data lost" dataLostTooltip = "This browser lost this file's contents. Upload it again to keep working with it." delete = "Delete" +duplicate = "Duplicate" moreActions = "More actions" openInViewer = "Open in viewer" +rename = "Rename" savedToServer = "Saved to server" updateOnServer = "Update on server" uploadToServer = "Upload to server" @@ -3931,6 +3935,14 @@ reset = "Show all" subtitle = "Show or hide categories in the files sidebar." title = "Sidebar categories" +[fileSidebar.rename] +cancel = "Cancel" +error = "Could not rename the file." +illegalCharacters = "A file name can't contain \\ / : * ? \" < > |" +label = "File name" +save = "Rename" +title = "Rename file" + [filesPage] addToWorkspace = "Add to workspace" addToWorkspaceCount = "Add {{count}} to workspace" @@ -3972,6 +3984,7 @@ downloadAll = "Download all" downloadVersion = "Download this version" dropOverlay = "Drop files to upload" dropOverlaySub = "Files start in Local. Use 'Move to' or 'Save to cloud' to organize them into a folder." +duplicate = "Duplicate" file = "File" fileInfo = "File info" fileMenu = "File actions" @@ -4087,6 +4100,8 @@ cloudDeleteFailed_one = "Couldn't delete 1 file from the cloud." cloudDeleteFailed_other = "Couldn't delete {{count}} files from the cloud." deleteFolderFailed = "Could not delete folder." deleteFolderFailedDetail = "Could not delete folder: {{message}}" +downloadFailed = "Could not download the file." +duplicateFailed = "Could not duplicate the file." folderAppearanceFailed = "Could not update folder appearance." folderAppearanceFailedDetail = "Could not update folder appearance: {{message}}" moveFilesFailed = "Could not move files." diff --git a/frontend/editor/src/core/components/filesPage/FileGrid.tsx b/frontend/editor/src/core/components/filesPage/FileGrid.tsx index e54964c592..7fd2eda133 100644 --- a/frontend/editor/src/core/components/filesPage/FileGrid.tsx +++ b/frontend/editor/src/core/components/filesPage/FileGrid.tsx @@ -13,6 +13,7 @@ import DeleteIcon from "@mui/icons-material/Delete"; import HistoryIcon from "@mui/icons-material/History"; import OpenInNewIcon from "@mui/icons-material/OpenInNew"; import DriveFileRenameOutlineIcon from "@mui/icons-material/DriveFileRenameOutline"; +import ContentCopyOutlinedIcon from "@mui/icons-material/ContentCopyOutlined"; import CloudUploadIcon from "@mui/icons-material/CloudUpload"; import UploadFileIcon from "@mui/icons-material/UploadFile"; import CreateNewFolderIcon from "@mui/icons-material/CreateNewFolder"; @@ -36,6 +37,8 @@ import { FolderThumbnail } from "@app/components/filesPage/FolderThumbnail"; import { findFolderIcon } from "@app/components/filesPage/folderIcons"; import { FolderAppearancePicker } from "@app/components/filesPage/FolderAppearancePicker"; import { useLazyThumbnail } from "@app/hooks/useLazyThumbnail"; +import { useFileActionIcons } from "@app/hooks/useFileActionIcons"; +import { useFileActionTerminology } from "@app/hooks/useFileActionTerminology"; import type { FilesPageSortMode } from "@app/contexts/FilesPageContext"; import { OpenInNewWindowMenuItem } from "@app/components/filesPage/OpenInNewWindowMenuItem"; @@ -83,6 +86,12 @@ interface FileGridProps { onSaveToServer?: (file: StirlingFileStub) => void; /** Open the version-history modal for a file (only when it has >1 version). */ onVersionHistory?: (file: StirlingFileStub) => void; + /** Download a copy (desktop: save a copy). */ + onDownloadFile?: (file: StirlingFileStub) => void; + /** Open the rename dialog for a file. */ + onRenameFile?: (file: StirlingFileStub) => void; + /** Save a second copy of the file into the library. */ + onDuplicateFile?: (file: StirlingFileStub) => void; /** When set, the Save to server item renders disabled with this tooltip. */ saveToServerDisabledReason?: string | null; /** When supplied the list-view column headers become sortable. */ @@ -366,24 +375,21 @@ function EmptyState({ ); } -function GridView({ - entries, - selectedFileIds, - activeWorkspaceFileIds, - onSelectFile, - onOpenFolder, - onOpenFile, - onMoveFiles, - onMoveFolder, - onRenameFolder, - onDeleteFolder, - onChangeFolderAppearance, - onRemoveFiles, - onPromptMoveFiles, - onSaveToServer, - onVersionHistory, - saveToServerDisabledReason, -}: FileGridProps) { +function GridView(props: FileGridProps) { + const { + entries, + selectedFileIds, + activeWorkspaceFileIds, + onSelectFile, + onOpenFolder, + onOpenFile, + onMoveFiles, + onMoveFolder, + onRenameFolder, + onDeleteFolder, + onChangeFolderAppearance, + } = props; + const menuHandlersFor = useFileMenuHandlers(props); return (
{entries.map((entry) => { @@ -424,22 +430,7 @@ function GridView({ onSelectFile(entry.file!.id, e.shiftKey, e.metaKey || e.ctrlKey) } onDoubleClick={() => onOpenFile(entry.file!)} - onRemove={() => onRemoveFiles([entry.file!.id])} - onMove={() => { - const target = selectedFileIds.has(entry.file!.id) - ? Array.from(selectedFileIds) - : [entry.file!.id]; - onPromptMoveFiles(target); - }} - onSaveToServer={ - onSaveToServer ? () => onSaveToServer(entry.file!) : undefined - } - onVersionHistory={ - onVersionHistory - ? () => onVersionHistory(entry.file!) - : undefined - } - saveToServerDisabledReason={saveToServerDisabledReason} + {...menuHandlersFor(entry.file)} /> ); } @@ -626,7 +617,222 @@ function PolicyBadges({ fileId }: { fileId: string }) { return ; } -interface FileCardProps { +/** Per-file actions. Shared verbatim by the grid card and the list row, and + * kept in step with the file sidebar's kebab so both surfaces offer the same. */ +interface FileActionsMenuProps { + file: StirlingFileStub; + triggerRef: React.RefObject; + onOpen: () => void; + onMove: () => void; + onRemove: () => void; + onDownload?: () => void; + onRename?: () => void; + onDuplicate?: () => void; + onSaveToServer?: () => void; + onVersionHistory?: () => void; + saveToServerDisabledReason?: string | null; +} + +function FileActionsMenu({ + file, + triggerRef, + onOpen, + onMove, + onRemove, + onDownload, + onRename, + onDuplicate, + onSaveToServer, + onVersionHistory, + saveToServerDisabledReason, +}: FileActionsMenuProps) { + const { t } = useTranslation(); + const terminology = useFileActionTerminology(); + const DownloadIcon = useFileActionIcons().download; + const showSaveToServer = + Boolean(onSaveToServer) && file.remoteStorageId == null; + const showVersionHistory = + Boolean(onVersionHistory) && (file.versionNumber ?? 1) > 1; + return ( + + + e.stopPropagation()} + aria-label={t("filesPage.fileMenu", "File actions")} + data-testid="file-card-actions" + > + + + + + } + onClick={(e) => { + e.stopPropagation(); + onOpen(); + }} + > + {t("filesPage.addToWorkspace", "Add to workspace")} + + + } + onClick={(e) => { + e.stopPropagation(); + onMove(); + }} + data-testid="file-menu-move-to" + > + {t("filesPage.moveTo", "Move to…")} + + + {(onDownload || onRename || onDuplicate) && } + {onDownload && ( + } + onClick={(e) => { + e.stopPropagation(); + onDownload(); + }} + data-testid="file-menu-download" + > + {terminology.download} + + )} + {onRename && ( + } + onClick={(e) => { + e.stopPropagation(); + onRename(); + }} + data-testid="file-menu-rename" + > + {t("filesPage.rename", "Rename")} + + )} + {onDuplicate && ( + } + onClick={(e) => { + e.stopPropagation(); + onDuplicate(); + }} + data-testid="file-menu-duplicate" + > + {t("filesPage.duplicate", "Duplicate")} + + )} + + {(showSaveToServer || showVersionHistory) && } + {/* Per-file Save to server; shown for local-only files. When + storage is off it stays visible but disabled with a tooltip. */} + {showSaveToServer && onSaveToServer && ( + + } + disabled={Boolean(saveToServerDisabledReason)} + onClick={(e) => { + e.stopPropagation(); + onSaveToServer(); + }} + style={ + saveToServerDisabledReason + ? { pointerEvents: "auto" } + : undefined + } + > + {t("filesPage.saveToServer", "Save to server")} + + + )} + {showVersionHistory && onVersionHistory && ( + } + onClick={(e) => { + e.stopPropagation(); + onVersionHistory(); + }} + > + {t("filesPage.versionHistory", "Version history")} + + )} + + + } + onClick={(e) => { + e.stopPropagation(); + onRemove(); + }} + > + {t("filesPage.remove", "Delete")} + + + + ); +} + +/** Binds one file's kebab handlers, so grid and list wire them identically. */ +function useFileMenuHandlers( + props: FileGridProps, +): (file: StirlingFileStub) => FileMenuHandlers { + const { + selectedFileIds, + onRemoveFiles, + onPromptMoveFiles, + onSaveToServer, + onVersionHistory, + onDownloadFile, + onRenameFile, + onDuplicateFile, + saveToServerDisabledReason, + } = props; + return (file: StirlingFileStub) => ({ + onRemove: () => onRemoveFiles([file.id]), + // A move acts on the whole selection when this file is part of it. + onMove: () => + onPromptMoveFiles( + selectedFileIds.has(file.id) ? Array.from(selectedFileIds) : [file.id], + ), + onDownload: onDownloadFile ? () => onDownloadFile(file) : undefined, + onRename: onRenameFile ? () => onRenameFile(file) : undefined, + onDuplicate: onDuplicateFile ? () => onDuplicateFile(file) : undefined, + onSaveToServer: onSaveToServer ? () => onSaveToServer(file) : undefined, + onVersionHistory: onVersionHistory + ? () => onVersionHistory(file) + : undefined, + saveToServerDisabledReason, + }); +} + +/** Per-file kebab handlers, shared by the card and row wrappers. */ +interface FileMenuHandlers { + onRemove: () => void; + onMove: () => void; + onDownload?: () => void; + onRename?: () => void; + onDuplicate?: () => void; + /** Kebab Save to server; only fires when file is local-only. */ + onSaveToServer?: () => void; + /** Open the version-history modal; shown only when file has >1 version. */ + onVersionHistory?: () => void; + /** When set, the kebab Save to server is disabled with this tooltip. */ + saveToServerDisabledReason?: string | null; +} + +interface FileCardProps extends FileMenuHandlers { file: StirlingFileStub; isSelected: boolean; isInWorkspace: boolean; @@ -637,14 +843,6 @@ interface FileCardProps { multiSelectActive: boolean; onClick: (e: React.MouseEvent) => void; onDoubleClick: () => void; - onRemove: () => void; - onMove: () => void; - /** Kebab Save to server; only fires when file is local-only. */ - onSaveToServer?: () => void; - /** Open the version-history modal; shown only when file has >1 version. */ - onVersionHistory?: () => void; - /** When set, the kebab Save to server is disabled with this tooltip. */ - saveToServerDisabledReason?: string | null; } function FileCard({ @@ -656,11 +854,7 @@ function FileCard({ multiSelectActive, onClick, onDoubleClick, - onRemove, - onMove, - onSaveToServer, - onVersionHistory, - saveToServerDisabledReason, + ...menuHandlers }: FileCardProps) { const { t } = useTranslation(); const cardRef = useRef(null); @@ -790,120 +984,40 @@ function FileCard({
- - - e.stopPropagation()} - aria-label={t("filesPage.fileMenu", "File actions")} - data-testid="file-card-actions" - > - - - - - } - onClick={(e) => { - e.stopPropagation(); - onDoubleClick(); - }} - > - {t("filesPage.addToWorkspace", "Add to workspace")} - - - } - onClick={(e) => { - e.stopPropagation(); - onMove(); - }} - data-testid="file-menu-move-to" - > - {t("filesPage.moveTo", "Move to…")} - - {/* Per-file Save to server; shown for local-only files. When - storage is off it stays visible but disabled with a tooltip. */} - {onSaveToServer && file.remoteStorageId == null && ( - - } - disabled={Boolean(saveToServerDisabledReason)} - onClick={(e) => { - e.stopPropagation(); - onSaveToServer(); - }} - style={ - saveToServerDisabledReason - ? { pointerEvents: "auto" } - : undefined - } - > - {t("filesPage.saveToServer", "Save to server")} - - - )} - {onVersionHistory && (file.versionNumber ?? 1) > 1 && ( - } - onClick={(e) => { - e.stopPropagation(); - onVersionHistory(); - }} - > - {t("filesPage.versionHistory", "Version history")} - - )} - - } - onClick={(e) => { - e.stopPropagation(); - onRemove(); - }} - > - {t("filesPage.remove", "Delete")} - - - +
); } -function ListView({ - entries, - selectedFileIds, - activeWorkspaceFileIds, - onSelectFile, - onSetSelection, - onOpenFolder, - onOpenFile, - onMoveFiles, - onMoveFolder, - onRenameFolder, - onDeleteFolder, - onSaveToServer, - onVersionHistory, - saveToServerDisabledReason, - onChangeFolderAppearance, - onRemoveFiles, - onPromptMoveFiles, - sortMode, - onChangeSortMode, -}: FileGridProps & { - sortMode?: FilesPageSortMode; - onChangeSortMode?: (next: FilesPageSortMode) => void; -}) { +function ListView( + props: FileGridProps & { + sortMode?: FilesPageSortMode; + onChangeSortMode?: (next: FilesPageSortMode) => void; + }, +) { + const { + entries, + selectedFileIds, + activeWorkspaceFileIds, + onSelectFile, + onSetSelection, + onOpenFolder, + onOpenFile, + onMoveFiles, + onMoveFolder, + onRenameFolder, + onDeleteFolder, + onChangeFolderAppearance, + sortMode, + onChangeSortMode, + } = props; + const menuHandlersFor = useFileMenuHandlers(props); const { t } = useTranslation(); // Tri-state header checkbox state - computed from current entries. @@ -1029,22 +1143,7 @@ function ListView({ onSelectFile(entry.file!.id, e.shiftKey, e.metaKey || e.ctrlKey) } onOpen={() => onOpenFile(entry.file!)} - onRemove={() => onRemoveFiles([entry.file!.id])} - onMove={() => { - const target = selectedFileIds.has(entry.file!.id) - ? Array.from(selectedFileIds) - : [entry.file!.id]; - onPromptMoveFiles(target); - }} - onSaveToServer={ - onSaveToServer ? () => onSaveToServer(entry.file!) : undefined - } - onVersionHistory={ - onVersionHistory - ? () => onVersionHistory(entry.file!) - : undefined - } - saveToServerDisabledReason={saveToServerDisabledReason} + {...menuHandlersFor(entry.file)} /> ); } @@ -1241,7 +1340,7 @@ function FolderRow({ ); } -interface FileRowProps { +interface FileRowProps extends FileMenuHandlers { file: StirlingFileStub; isSelected: boolean; isInWorkspace: boolean; @@ -1251,14 +1350,6 @@ interface FileRowProps { multiSelectActive: boolean; onClick: (e: React.MouseEvent) => void; onOpen: () => void; - onRemove: () => void; - onMove: () => void; - /** Kebab Save to server; only fires when file is local-only. */ - onSaveToServer?: () => void; - /** Open the version-history modal; shown only when file has >1 version. */ - onVersionHistory?: () => void; - /** When set, the kebab Save to server is disabled with this tooltip. */ - saveToServerDisabledReason?: string | null; } function FileRow({ @@ -1270,11 +1361,7 @@ function FileRow({ multiSelectActive, onClick, onOpen, - onRemove, - onMove, - onSaveToServer, - onVersionHistory, - saveToServerDisabledReason, + ...menuHandlers }: FileRowProps) { const { t } = useTranslation(); const kebabRef = useRef(null); @@ -1414,91 +1501,12 @@ function FileRow({ {fileSize} {fileDate} - - - e.stopPropagation()} - aria-label={t("filesPage.fileMenu", "File actions")} - data-testid="file-card-actions" - > - - - - - } - onClick={(e) => { - e.stopPropagation(); - onOpen(); - }} - > - {t("filesPage.addToWorkspace", "Add to workspace")} - - - } - onClick={(e) => { - e.stopPropagation(); - onMove(); - }} - > - {t("filesPage.moveTo", "Move to…")} - - {/* Per-file Save to server; shown for local-only files. When - storage is off it stays visible but disabled with a tooltip. */} - {onSaveToServer && file.remoteStorageId == null && ( - - } - disabled={Boolean(saveToServerDisabledReason)} - onClick={(e) => { - e.stopPropagation(); - onSaveToServer(); - }} - style={ - saveToServerDisabledReason - ? { pointerEvents: "auto" } - : undefined - } - > - {t("filesPage.saveToServer", "Save to server")} - - - )} - {onVersionHistory && (file.versionNumber ?? 1) > 1 && ( - } - onClick={(e) => { - e.stopPropagation(); - onVersionHistory(); - }} - > - {t("filesPage.versionHistory", "Version history")} - - )} - - } - onClick={(e) => { - e.stopPropagation(); - onRemove(); - }} - > - {t("filesPage.remove", "Delete")} - - - + ); diff --git a/frontend/editor/src/core/components/filesPage/FileManagerView.tsx b/frontend/editor/src/core/components/filesPage/FileManagerView.tsx index a0b0c6cfa1..e87b348780 100644 --- a/frontend/editor/src/core/components/filesPage/FileManagerView.tsx +++ b/frontend/editor/src/core/components/filesPage/FileManagerView.tsx @@ -71,6 +71,10 @@ import { FolderNameDialog } from "@app/components/filesPage/FolderNameDialog"; import { DeleteFolderDialog } from "@app/components/filesPage/DeleteFolderDialog"; import { DeleteFilesDialog } from "@app/components/filesPage/DeleteFilesDialog"; import { VersionHistoryModal } from "@app/components/filesPage/VersionHistoryModal"; +import { RenameFileDialog } from "@app/components/shared/RenameFileDialog"; +import { duplicateStoredFile } from "@app/utils/duplicateFile"; +import { downloadFileFromStorage } from "@app/utils/downloadUtils"; +import { fileStorage } from "@app/services/fileStorage"; import { materializeServerStubs } from "@app/services/fileSyncService"; import { FILES_PAGE_DRAG_TYPE, @@ -794,6 +798,92 @@ export default function FileManagerView() { [removeFiles], ); + // ─── per-file kebab: download / rename / duplicate ─────────────────────── + // Same actions the file sidebar's kebab offers, so both surfaces match. + + /** Cloud-only rows hold no bytes; pull them local before acting on them. */ + const localCopyOf = useCallback( + async (file: StirlingFileStub): Promise => { + const [materialized] = await materializeServerStubs([file], { + addFiles: fileActions.addFilesWithOptions, + updateStub: fileActions.updateStirlingFileStub, + }); + return materialized ?? null; + }, + [fileActions], + ); + + const handleDownloadFile = useCallback( + async (file: StirlingFileStub) => { + try { + const local = await localCopyOf(file); + if (!local) return; + await downloadFileFromStorage(local); + } catch (err) { + console.error("[FilesPage] Download failed", err); + folders.setError( + t("filesPage.error.downloadFailed", "Could not download the file."), + ); + } + }, + [localCopyOf, folders, t], + ); + + const handleDuplicateFile = useCallback( + async (file: StirlingFileStub) => { + try { + const local = await localCopyOf(file); + if (!local) return; + const copyId = await duplicateStoredFile( + local, + allFiles.map((f) => f.name), + addFiles, + ); + if (!copyId) { + throw new Error(`File "${local.name}" not found in storage`); + } + await refresh(); + } catch (err) { + console.error("[FilesPage] Duplicate failed", err); + folders.setError( + t("filesPage.error.duplicateFailed", "Could not duplicate the file."), + ); + } + }, + [localCopyOf, allFiles, addFiles, refresh, folders, t], + ); + + const [renameTarget, setRenameTarget] = useState( + null, + ); + + // The stub name is what the UI and exports read, so a rename is a metadata + // write; the workbench copy (if any) is updated in the same breath. + const handleConfirmRename = useCallback( + async (name: string) => { + const file = renameTarget; + if (!file) return; + const local = await localCopyOf(file); + if (!local) return; + // quickKey is name|size|lastModified; a stale one would make a re-upload + // of the original look like a duplicate of the renamed file. + const quickKey = `${name}|${local.size}|${local.lastModified}`; + const saved = await fileStorage.updateFileMetadata(local.id, { + name, + quickKey, + }); + if (!saved) { + throw new Error( + t("fileSidebar.rename.error", "Could not rename the file."), + ); + } + fileActions.updateStirlingFileStub(local.id, { name, quickKey }); + setRenameTarget(null); + await refresh(); + }, + [renameTarget, localCopyOf, fileActions, refresh, t], + ); + // ─── derived UI bits ──────────────────────────────────────────────────── const currentFolderRecord = currentFolderId ? (foldersById.get(currentFolderId) ?? null) @@ -1503,6 +1593,9 @@ export default function FileManagerView() { onPromptMoveFiles={promptMoveFiles} onSaveToServer={(file) => setSaveToServerTarget([file])} onVersionHistory={(file) => setVersionHistoryFile(file)} + onDownloadFile={handleDownloadFile} + onRenameFile={setRenameTarget} + onDuplicateFile={handleDuplicateFile} saveToServerDisabledReason={saveToServerDisabledReason} // Center-of-grid CTAs when the empty state shows - same // handlers the corner header buttons use so behaviour @@ -1662,6 +1755,14 @@ export default function FileManagerView() { onConfirm={confirmRemoveFiles} /> + {/* Rename (opened from the card kebab). */} + setRenameTarget(null)} + onSubmit={handleConfirmRename} + /> + {/* Version journey in a modal (opened from the card kebab). */} ( const [deleteTarget, setDeleteTarget] = useState( null, ); + // Kebab "Rename" target; drives RenameFileDialog. + const [renameTarget, setRenameTarget] = useState( + null, + ); // Storage gate: only offer Save-to-cloud when the server allows it and // the user is signed in (guests have no cloud library). const storageEnabled = config?.storageEnabled === true && !isAnonymous; @@ -417,6 +425,121 @@ const FileSidebar = forwardRef( [allFileStubs], ); + const warnDataUnavailable = useCallback(() => { + alert({ + alertType: "warning", + title: t("fileSidebar.dataLostTitle", "File data is unavailable"), + body: t( + "fileSidebar.dataLostBody", + "This browser lost this file's contents. Upload it again to keep working with it.", + ), + expandable: false, + durationMs: 6000, + }); + }, [t]); + + // Kebab: download a copy (desktop saves via the native dialog). Routed + // through the policy wrapper so export policies enforce here too. + const handleDownload = useCallback( + async (fileId: FileId) => { + const stub = allFileStubs.find((s) => s.id === fileId); + const file = await fileStorage.getStirlingFile(fileId); + if (!file) { + warnDataUnavailable(); + return; + } + try { + await downloadFileWithPolicy({ + data: file, + filename: stub?.name ?? file.name, + fileId: fileId as string, + }); + } catch (error) { + console.error("[FileSidebar] Download failed:", error); + alert({ + alertType: "error", + title: t("fileSidebar.downloadFailed", "Download failed"), + body: error instanceof Error ? error.message : String(error), + expandable: false, + }); + } + }, + [allFileStubs, warnDataUnavailable, t], + ); + + // Kebab: copy the file into the library under a free "(copy)" name. + const handleDuplicate = useCallback( + async (fileId: FileId) => { + const stub = allFileStubs.find((s) => s.id === fileId); + if (!stub) return; + try { + const copyId = await duplicateStoredFile( + stub, + allFileStubs.map((s) => s.name), + addFiles, + ); + if (!copyId) { + warnDataUnavailable(); + return; + } + await refreshStubs(); + } catch (error) { + console.error("[FileSidebar] Duplicate failed:", error); + alert({ + alertType: "error", + title: t("fileSidebar.duplicateFailed", "Could not duplicate file"), + body: error instanceof Error ? error.message : String(error), + expandable: false, + }); + } + }, + [allFileStubs, addFiles, refreshStubs, warnDataUnavailable, t], + ); + + // Kebab: open the rename dialog for this one file. + const handleRename = useCallback( + (fileId: FileId) => { + const stub = allFileStubs.find((s) => s.id === fileId); + if (stub) setRenameTarget(stub); + }, + [allFileStubs], + ); + + // The stub name is what the UI and exports read, so a rename is a metadata + // write - storage first, then the workbench copy if the file is open. + const handleConfirmRename = useCallback( + async (name: string) => { + const stub = renameTarget; + if (!stub) return; + // quickKey is name|size|lastModified; a stale one would make a re-upload + // of the original look like a duplicate of the renamed file. + const quickKey = `${name}|${stub.size}|${stub.lastModified}`; + const saved = await fileStorage.updateFileMetadata(stub.id, { + name, + quickKey, + }); + if (!saved) { + throw new Error( + t("fileSidebar.rename.error", "Could not rename the file."), + ); + } + fileActions.updateStirlingFileStub(stub.id, { name, quickKey }); + setRenameTarget(null); + await refreshStubs(); + }, + [renameTarget, fileActions, refreshStubs, t], + ); + + // Desktop-only; a no-op stub on web, where this stays hidden. + const { canOpenInNewWindow, openInNewWindow } = useOpenInNewWindow(); + const handleOpenInNewWindow = useCallback( + (fileId: FileId) => { + const stub = allFileStubs.find((s) => s.id === fileId); + if (stub) openInNewWindow(stub); + }, + [allFileStubs, openInNewWindow], + ); + // Once a pending file lands in state, open it in the viewer. useEffect(() => { if (!pendingViewFileId) return; @@ -773,6 +896,12 @@ const FileSidebar = forwardRef( onFolderClick={openWatchedFolder} policies={policyFileBadges.get(stub.id as string) ?? NO_POLICIES} onDelete={isWatchedFoldersActive ? undefined : handleSidebarDelete} + onDownload={handleDownload} + onRename={isWatchedFoldersActive ? undefined : handleRename} + onDuplicate={isWatchedFoldersActive ? undefined : handleDuplicate} + onOpenInNewWindow={ + canOpenInNewWindow(stub) ? handleOpenInNewWindow : undefined + } onSaveToCloud={isWatchedFoldersActive ? undefined : handleSaveToCloud} canSaveToCloud={storageEnabled && fileOrigin !== "shared-with-me"} isUploadedToCloud={fileOrigin === "cloud"} @@ -1222,6 +1351,14 @@ const FileSidebar = forwardRef( onChanged={refreshStubs} /> + {/* Kebab "Rename" dialog. */} + setRenameTarget(null)} + onSubmit={handleConfirmRename} + /> + {/* Cloud-aware delete choice (only opened for cloud-uploaded files). */} void; + /** Download a copy (desktop: save a copy) from the kebab menu. */ + onDownload?: (fileId: FileId) => void; + /** Rename the file from the kebab menu. */ + onRename?: (fileId: FileId) => void; + /** Save a second copy of the file into the library. */ + onDuplicate?: (fileId: FileId) => void; + /** Desktop only: open the file in its own window. Omit where unsupported. */ + onOpenInNewWindow?: (fileId: FileId) => void; /** Save to cloud from the kebab menu. */ onSaveToCloud?: (fileId: FileId) => void; /** Whether the upload-to-server menu item is offered (storage on, signed in). */ @@ -171,6 +184,46 @@ export interface FileItemProps { const MAX_VISIBLE_FOLDER_TAGS = 2; +/** One kebab row. `disabledReason`, when set, greys the row out and says why. */ +function FileMenuItem({ + disabledReason, + icon, + color, + onClick, + children, +}: { + disabledReason?: React.ReactNode; + icon: React.ReactNode; + color?: string; + onClick: (e: React.MouseEvent) => void; + children: React.ReactNode; +}) { + return ( + + {/* Disabled items swallow pointer events, so the tooltip needs a live wrapper. */} +
+ { + e.stopPropagation(); + onClick(e); + }} + > + {children} + +
+
+ ); +} + // Memoized: sidebar rows bail out unless THEIR props change, so one file's // update (e.g. a new version landing) re-renders one row, not the whole list. export const FileItem = React.memo(function FileItem({ @@ -192,6 +245,10 @@ export const FileItem = React.memo(function FileItem({ policies = [], primaryLabel, onDelete, + onDownload, + onRename, + onDuplicate, + onOpenInNewWindow, onSaveToCloud, canSaveToCloud = false, isUploadedToCloud = false, @@ -199,9 +256,14 @@ export const FileItem = React.memo(function FileItem({ hasVersionHistory = false, }: FileItemProps) { const { t } = useTranslation(); + const terminology = useFileActionTerminology(); + const DownloadIcon = useFileActionIcons().download; const ext = getFileExtension(name); const dateLabel = lastModified ? formatFileDate(lastModified) : ""; const typeLabel = ext ? ext.toUpperCase() : "File"; + const metaLine = [typeLabel, size ? formatFileSize(size) : null, dateLabel] + .filter(Boolean) + .join(" · "); const policyEnforcing = policies.some((p) => p.enforcing); const enforcingTooltip = (action: string): React.ReactNode => ( @@ -220,6 +282,25 @@ export const FileItem = React.memo(function FileItem({ ); + // Why an action can't run right now: a policy is rewriting the file, or its + // bytes are gone. `needsBytes` actions are the ones that read the file. + const blockedReason = ( + action: string, + needsBytes = true, + ): React.ReactNode | null => { + if (policyEnforcing) return enforcingTooltip(action); + if (needsBytes && dataUnavailable) + return t( + "fileSidebar.fileItem.dataLostTooltip", + "This browser lost this file's contents. Upload it again to keep working with it.", + ); + return null; + }; + + const viewerLabel = isViewedInViewer + ? t("fileSidebar.fileItem.closeViewer", "Close viewer") + : t("fileSidebar.fileItem.openInViewer", "Open in viewer"); + const visibleFolders = folders.slice(0, MAX_VISIBLE_FOLDER_TAGS); const overflowFolders = folders.slice(MAX_VISIBLE_FOLDER_TAGS); @@ -233,6 +314,7 @@ export const FileItem = React.memo(function FileItem({ const itemRef = useRef(null); const [hoverRect, setHoverRect] = useState(null); + const [menuOpened, setMenuOpened] = useState(false); const handleMouseEnter = useCallback(() => { setHoverRect(itemRef.current?.getBoundingClientRect() ?? null); @@ -240,9 +322,10 @@ export const FileItem = React.memo(function FileItem({ const handleMouseLeave = useCallback(() => setHoverRect(null), []); - // Reactive: tooltip appears as soon as both hover rect and thumbnail are ready + // Reactive: tooltip appears as soon as both hover rect and thumbnail are ready. + // The kebab suppresses it - two cards floating off one row read as a glitch. const thumbPos = - hoverRect && resolvedThumbnail + hoverRect && resolvedThumbnail && !menuOpened ? { top: hoverRect.top + hoverRect.height / 2, left: hoverRect.right + 10, @@ -389,11 +472,7 @@ export const FileItem = React.memo(function FileItem({ onEyeClick(fileId, e); }} tabIndex={-1} - aria-label={ - isViewedInViewer - ? t("fileSidebar.fileItem.closeViewer", "Close viewer") - : t("fileSidebar.fileItem.openInViewer", "Open in viewer") - } + aria-label={viewerLabel} > - {(onDelete || - (canSaveToCloud && onSaveToCloud) || - (hasVersionHistory && onVersionHistory)) && ( - - - e.stopPropagation()} - tabIndex={-1} - aria-label={t( - "fileSidebar.fileItem.moreActions", - "More actions", - )} - > - - - - e.stopPropagation()}> - {hasVersionHistory && onVersionHistory && ( - } - onClick={(e) => { - e.stopPropagation(); - onVersionHistory(fileId); - }} - > - {t( - "fileSidebar.fileItem.versionHistory", - "Version history", - )} - + + + e.stopPropagation()} + tabIndex={-1} + aria-label={t( + "fileSidebar.fileItem.moreActions", + "More actions", )} - {canSaveToCloud && - onSaveToCloud && - (() => { - const uploadLabel = isUploadedToCloud - ? t( - "fileSidebar.fileItem.updateOnServer", - "Update on server", - ) - : t( - "fileSidebar.fileItem.uploadToServer", - "Upload to server", - ); - return ( - -
- - } - onClick={(e) => { - e.stopPropagation(); - onSaveToCloud(fileId); - }} - > - {uploadLabel} - -
-
- ); - })()} - {onDelete && - (() => { - const deleteLabel = t( - "fileSidebar.fileItem.delete", - "Delete", - ); - return ( - -
- - } - onClick={(e) => { - e.stopPropagation(); - onDelete(fileId); - }} - > - {deleteLabel} - -
-
- ); - })()} - -
- )} + > + + + + e.stopPropagation()}> + {/* Rows truncate long names; the menu header is where the whole + name (and the size the row has no space for) is readable. */} + + {name} + + {metaLine} + + + + + ) : ( + + ) + } + onClick={(e) => onEyeClick(fileId, e)} + > + {viewerLabel} + + + {onOpenInNewWindow && ( + } + onClick={() => onOpenInNewWindow(fileId)} + > + {t("openInNewWindow", "Open in new window")} + + )} + + {(onDownload || onRename || onDuplicate) && } + + {onDownload && ( + } + onClick={() => onDownload(fileId)} + > + {terminology.download} + + )} + + {onRename && ( + } + onClick={() => onRename(fileId)} + > + {t("fileSidebar.fileItem.rename", "Rename")} + + )} + + {onDuplicate && ( + } + onClick={() => onDuplicate(fileId)} + > + {t("fileSidebar.fileItem.duplicate", "Duplicate")} + + )} + + {((canSaveToCloud && onSaveToCloud) || + (hasVersionHistory && onVersionHistory)) && } + + {canSaveToCloud && + onSaveToCloud && + (() => { + const uploadLabel = isUploadedToCloud + ? t( + "fileSidebar.fileItem.updateOnServer", + "Update on server", + ) + : t( + "fileSidebar.fileItem.uploadToServer", + "Upload to server", + ); + return ( + } + onClick={() => onSaveToCloud(fileId)} + > + {uploadLabel} + + ); + })()} + + {hasVersionHistory && onVersionHistory && ( + } + onClick={() => onVersionHistory(fileId)} + > + {t("fileSidebar.fileItem.versionHistory", "Version history")} + + )} + + {onDelete && ( + <> + + } + onClick={() => onDelete(fileId)} + > + {t("fileSidebar.fileItem.delete", "Delete")} + + + )} + +
diff --git a/frontend/editor/src/core/components/shared/RenameFileDialog.stories.tsx b/frontend/editor/src/core/components/shared/RenameFileDialog.stories.tsx new file mode 100644 index 0000000000..9413db41b0 --- /dev/null +++ b/frontend/editor/src/core/components/shared/RenameFileDialog.stories.tsx @@ -0,0 +1,39 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { RenameFileDialog } from "@app/components/shared/RenameFileDialog"; + +const meta = { + title: "Shared/RenameFileDialog", + component: RenameFileDialog, +} satisfies Meta; +export default meta; + +type Story = StoryObj; + +export const Default: Story = { + args: { + opened: true, + fileName: "Q3 invoice bundle.pdf", + onClose: () => {}, + onSubmit: () => {}, + }, +}; + +export const NoExtension: Story = { + args: { + opened: true, + fileName: "scanned-document", + onClose: () => {}, + onSubmit: () => {}, + }, +}; + +export const SaveFails: Story = { + args: { + opened: true, + fileName: "contract.pdf", + onClose: () => {}, + onSubmit: () => { + throw new Error("Could not rename the file."); + }, + }, +}; diff --git a/frontend/editor/src/core/components/shared/RenameFileDialog.tsx b/frontend/editor/src/core/components/shared/RenameFileDialog.tsx new file mode 100644 index 0000000000..83e00f6ba3 --- /dev/null +++ b/frontend/editor/src/core/components/shared/RenameFileDialog.tsx @@ -0,0 +1,140 @@ +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Alert, Group, Modal, Stack, TextInput } from "@mantine/core"; +import ErrorOutlineIcon from "@mui/icons-material/ErrorOutlined"; + +import { Button } from "@app/ui/Button"; +import { splitFileName } from "@app/utils/fileUtils"; + +/** Characters Windows/macOS reject in a filename, which is also what a download saves as. */ +const ILLEGAL_NAME_CHARS = /[\\/:*?"<>|]/; + +interface RenameFileDialogProps { + opened: boolean; + /** Current name, including its extension. */ + fileName: string; + onClose: () => void; + /** Gets the new full name. Throwing keeps the dialog open with the message. */ + onSubmit: (name: string) => void | Promise; +} + +/** + * Renames one library file. Only the base name is editable - the extension is + * shown but fixed, so a rename can't leave the file claiming the wrong type. + */ +export function RenameFileDialog({ + opened, + fileName, + onClose, + onSubmit, +}: RenameFileDialogProps) { + const { t } = useTranslation(); + const [base, extension] = splitFileName(fileName); + const [value, setValue] = useState(base); + const [submitting, setSubmitting] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + if (opened) { + setValue(base); + setSubmitting(false); + setError(null); + } + }, [opened, base]); + + const submit = async () => { + const nextBase = value.trim(); + if (!nextBase) return; + if (ILLEGAL_NAME_CHARS.test(nextBase)) { + setError( + t( + "fileSidebar.rename.illegalCharacters", + "A file name can't contain \\ / : * ? \" < > |", + ), + ); + return; + } + const nextName = `${nextBase}${extension}`; + if (nextName === fileName) { + onClose(); + return; + } + setSubmitting(true); + setError(null); + try { + await onSubmit(nextName); + onClose(); + } catch (err) { + // Stay open on failure: closing would look like the rename worked. + setError( + err instanceof Error + ? err.message + : t("fileSidebar.rename.error", "Could not rename the file."), + ); + } finally { + setSubmitting(false); + } + }; + + return ( + + + setValue(e.currentTarget.value)} + onFocus={(e) => e.currentTarget.select()} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + void submit(); + } + }} + maxLength={200} + aria-label={t("fileSidebar.rename.label", "File name")} + rightSection={ + extension ? ( + + {extension} + + ) : undefined + } + rightSectionWidth={extension ? extension.length * 8 + 12 : undefined} + rightSectionPointerEvents="none" + /> + {error && ( + } + variant="light" + role="alert" + > + {error} + + )} + + + + + + + ); +} diff --git a/frontend/editor/src/core/components/shared/WorkbenchBar.tsx b/frontend/editor/src/core/components/shared/WorkbenchBar.tsx index 241d658bad..ad38fb1974 100644 --- a/frontend/editor/src/core/components/shared/WorkbenchBar.tsx +++ b/frontend/editor/src/core/components/shared/WorkbenchBar.tsx @@ -229,7 +229,8 @@ export default function WorkbenchBar({ try { const result = await downloadFile({ data: new Blob([buffer], { type: "application/pdf" }), - filename: fileToExport.name, + // Stub name, not File.name: a rename only writes the stub. + filename: stub?.name ?? fileToExport.name, localPath: forceNewFile ? undefined : stub?.localFilePath, fileId: stub?.id, }); @@ -281,7 +282,8 @@ export default function WorkbenchBar({ try { const result = await downloadRaw({ data: enforced[idx], - filename: file.name, + // Stub name, not File.name: a rename only writes the stub. + filename: stub?.name ?? file.name, localPath: forceNewFile ? undefined : stub?.localFilePath, fileId: stub?.id, }); diff --git a/frontend/editor/src/core/hooks/useFileHandler.ts b/frontend/editor/src/core/hooks/useFileHandler.ts index 2f26648fd1..6fcbcb2350 100644 --- a/frontend/editor/src/core/hooks/useFileHandler.ts +++ b/frontend/editor/src/core/hooks/useFileHandler.ts @@ -13,6 +13,10 @@ export const useFileHandler = () => { selectFiles?: boolean; /** Persist to IDB without dispatching to workspace state. */ skipWorkspaceDispatch?: boolean; + /** Defaults to true; false keeps an archive intact (e.g. duplicating one). */ + autoUnzip?: boolean; + /** Skip the upload metric - the file isn't new to the system (e.g. a copy). */ + skipUploadTracking?: boolean; } = {}, ): Promise => { // Merge default options with passed options - passed options take precedence diff --git a/frontend/editor/src/core/tests/stubbed/file-actions-menu.spec.ts b/frontend/editor/src/core/tests/stubbed/file-actions-menu.spec.ts new file mode 100644 index 0000000000..cafc188462 --- /dev/null +++ b/frontend/editor/src/core/tests/stubbed/file-actions-menu.spec.ts @@ -0,0 +1,167 @@ +import path from "path"; +import type { Page } from "@playwright/test"; +import { test, expect } from "@app/tests/helpers/stub-test-base"; +import { uploadFiles } from "@app/tests/helpers/ui-helpers"; + +// Per-file actions live behind a kebab on two surfaces - the file sidebar and +// the My Files grid. They must offer the same file actions on both. + +const SAMPLE = path.join(import.meta.dirname, "../test-fixtures/sample.pdf"); + +const rows = (page: Page) => page.locator(".file-sidebar-file-item"); + +/** Hover the first sidebar row (the kebab only shows on hover) and open it. */ +async function openKebab(page: Page): Promise { + const row = rows(page).first(); + await row.hover(); + await row.locator(".file-sidebar-kebab-btn").click(); + await expect(page.getByRole("menu")).toBeVisible(); +} + +test("the kebab lists the file's actions under its full name", async ({ + page, +}) => { + await uploadFiles(page, SAMPLE); + await openKebab(page); + + const menu = page.getByRole("menu"); + await expect(menu.locator(".file-sidebar-kebab-header-name")).toHaveText( + "sample.pdf", + ); + // Type · size · date - the row itself has no space for the size. + await expect(menu.locator(".file-sidebar-kebab-header-meta")).toContainText( + "PDF", + ); + // A lone upload lands in the viewer, so the toggle offers the way out. + await expect( + menu.getByRole("menuitem", { name: "Close viewer" }), + ).toBeVisible(); + await expect(menu.getByRole("menuitem", { name: "Download" })).toBeVisible(); + await expect(menu.getByRole("menuitem", { name: "Rename" })).toBeVisible(); + await expect(menu.getByRole("menuitem", { name: "Duplicate" })).toBeVisible(); + await expect(menu.getByRole("menuitem", { name: "Delete" })).toBeVisible(); +}); + +test("Download saves the file under its current name", async ({ page }) => { + await uploadFiles(page, SAMPLE); + await openKebab(page); + + const download = page.waitForEvent("download"); + await page.getByRole("menuitem", { name: "Download" }).click(); + expect((await download).suggestedFilename()).toBe("sample.pdf"); +}); + +test("Rename updates the row and survives a reload", async ({ page }) => { + await uploadFiles(page, SAMPLE); + await openKebab(page); + await page.getByRole("menuitem", { name: "Rename" }).click(); + + // Only the base name is editable; the extension is re-applied on submit. + const input = page.getByLabel("File name"); + await expect(input).toHaveValue("sample"); + await input.fill("quarterly report"); + await page.getByRole("button", { name: "Rename" }).click(); + + await expect(rows(page).first()).toContainText("quarterly report.pdf"); + + // The name is metadata in IndexedDB, so it must outlive the page. + await page.reload(); + await expect(rows(page).first()).toContainText("quarterly report.pdf"); +}); + +test("Duplicate adds a copy to the library", async ({ page }) => { + await uploadFiles(page, SAMPLE); + await openKebab(page); + await page.getByRole("menuitem", { name: "Duplicate" }).click(); + + await expect(rows(page)).toHaveCount(2); + await expect(rows(page).filter({ hasText: "sample (copy).pdf" })).toHaveCount( + 1, + ); +}); + +test("a duplicate inherits the original's classification", async ({ page }) => { + // The copy is byte-identical, so it must land in the same category group - + // it inherits the label rather than waiting on the idle backfill to re-parse. + await uploadFiles( + page, + path.join( + import.meta.dirname, + "../test-fixtures/classification/classified_invoice.pdf", + ), + ); + const financial = page + .locator(".file-sidebar-group") + .filter({ hasText: "Financial" }); + await expect(financial).toBeVisible({ timeout: 15_000 }); + + await openKebab(page); + await page.getByRole("menuitem", { name: "Duplicate" }).click(); + + // The copy carries the label straight away - its row shows the label chip... + const copy = rows(page) + .filter({ hasText: "classified_invoice (copy).pdf" }) + .first(); + await expect(copy).toContainText("Invoice", { timeout: 5_000 }); + // ...and it counts towards the same category group. + await expect(financial.locator(".file-sidebar-group-count")).toHaveText("2"); +}); + +// ─── My Files grid: the same actions, same behaviour ──────────────────────── + +const cards = (page: Page) => page.locator(".files-page-card:not(.is-folder)"); + +/** Upload a file, cross to My Files, and open the card's kebab. */ +async function openCardKebab(page: Page): Promise { + await uploadFiles(page, SAMPLE); + await page.getByTestId("my-files-button").click(); + const card = cards(page).filter({ hasText: "sample.pdf" }).first(); + await expect(card).toBeVisible(); + await card.getByRole("button", { name: /File actions/i }).click(); + await expect(page.getByRole("menu")).toBeVisible(); +} + +test("My Files offers the same file actions as the sidebar", async ({ + page, +}) => { + await openCardKebab(page); + + const menu = page.getByRole("menu"); + await expect( + menu.getByRole("menuitem", { name: "Add to workspace" }), + ).toBeVisible(); + await expect(menu.getByRole("menuitem", { name: "Move to…" })).toBeVisible(); + await expect(menu.getByRole("menuitem", { name: "Download" })).toBeVisible(); + await expect(menu.getByRole("menuitem", { name: "Rename" })).toBeVisible(); + await expect(menu.getByRole("menuitem", { name: "Duplicate" })).toBeVisible(); + await expect(menu.getByRole("menuitem", { name: "Delete" })).toBeVisible(); +}); + +test("My Files Download saves the file under its current name", async ({ + page, +}) => { + await openCardKebab(page); + + const download = page.waitForEvent("download"); + await page.getByRole("menuitem", { name: "Download" }).click(); + expect((await download).suggestedFilename()).toBe("sample.pdf"); +}); + +test("My Files Rename updates the card", async ({ page }) => { + await openCardKebab(page); + await page.getByRole("menuitem", { name: "Rename" }).click(); + + await page.getByLabel("File name").fill("statement"); + await page.getByRole("button", { name: "Rename" }).click(); + + await expect(cards(page).filter({ hasText: "statement.pdf" })).toHaveCount(1); +}); + +test("My Files Duplicate adds a copy", async ({ page }) => { + await openCardKebab(page); + await page.getByRole("menuitem", { name: "Duplicate" }).click(); + + await expect( + cards(page).filter({ hasText: "sample (copy).pdf" }), + ).toHaveCount(1); +}); diff --git a/frontend/editor/src/core/utils/duplicateFile.test.ts b/frontend/editor/src/core/utils/duplicateFile.test.ts new file mode 100644 index 0000000000..88c7cce686 --- /dev/null +++ b/frontend/editor/src/core/utils/duplicateFile.test.ts @@ -0,0 +1,130 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import type { StirlingFile, StirlingFileStub } from "@app/types/fileContext"; +import type { FileId } from "@app/types/file"; +import type { FolderId } from "@app/types/folder"; + +const getStirlingFile = vi.fn(); +const updateFileMetadata = vi.fn(); + +vi.mock("@app/services/fileStorage", () => ({ + fileStorage: { + getStirlingFile: (...args: unknown[]) => getStirlingFile(...args), + updateFileMetadata: (...args: unknown[]) => updateFileMetadata(...args), + }, +})); + +const { copyNameFor, duplicateStoredFile } = + await import("@app/utils/duplicateFile"); + +/** + * A duplicate is byte-identical to its source, so everything derived from those + * bytes (classification labels, thumbnail) and its place in the library (folder) + * is inherited rather than re-derived. These lock that contract. + */ + +const stub = (extra: Partial = {}): StirlingFileStub => + ({ + id: "src-id" as FileId, + name: "report.pdf", + size: 10, + type: "application/pdf", + lastModified: 1, + ...extra, + }) as StirlingFileStub; + +const asStirlingFile = (name: string): StirlingFile => + Object.assign(new File(["%PDF-1.7"], name, { type: "application/pdf" }), { + fileId: "new-id" as FileId, + quickKey: "k", + }) as StirlingFile; + +type AddFilesOptions = { + selectFiles?: boolean; + skipWorkspaceDispatch?: boolean; + autoUnzip?: boolean; + skipUploadTracking?: boolean; +}; +const addFiles = vi.fn< + (files: File[], options: AddFilesOptions) => Promise +>(async () => [asStirlingFile("report (copy).pdf")]); + +beforeEach(() => { + vi.clearAllMocks(); + getStirlingFile.mockResolvedValue(asStirlingFile("report.pdf")); + updateFileMetadata.mockResolvedValue(true); + addFiles.mockResolvedValue([asStirlingFile("report (copy).pdf")]); +}); + +describe("copyNameFor", () => { + it("keeps the extension and counts up until the name is free", () => { + expect(copyNameFor("report.pdf", [])).toBe("report (copy).pdf"); + expect(copyNameFor("report.pdf", ["report (copy).pdf"])).toBe( + "report (copy 2).pdf", + ); + expect( + copyNameFor("report.pdf", ["report (copy).pdf", "report (copy 2).pdf"]), + ).toBe("report (copy 3).pdf"); + }); + + it("handles a name with no extension", () => { + expect(copyNameFor("scan", [])).toBe("scan (copy)"); + }); +}); + +describe("duplicateStoredFile", () => { + it("inherits labels, folder and a persisted thumbnail", async () => { + const id = await duplicateStoredFile( + stub({ + classificationLabels: ["invoice"], + folderId: "folder-1" as FolderId, + thumbnailUrl: "data:image/png;base64,AAA", + }), + [], + addFiles, + ); + + expect(id).toBe("new-id"); + const [, updates] = updateFileMetadata.mock.calls[0]; + expect(updates.classificationLabels).toEqual(["invoice"]); + expect(updates.folderId).toBe("folder-1"); + expect(updates.thumbnail).toBe("data:image/png;base64,AAA"); + expect(updates.thumbnailStoredAt).toEqual(expect.any(Number)); + }); + + it("drops a blob: thumbnail - it would be dead on the next load", async () => { + await duplicateStoredFile( + stub({ classificationLabels: ["invoice"], thumbnailUrl: "blob:abc" }), + [], + addFiles, + ); + + const [, updates] = updateFileMetadata.mock.calls[0]; + expect(updates).not.toHaveProperty("thumbnail"); + }); + + it("writes nothing when the source has no derived metadata", async () => { + await duplicateStoredFile(stub(), [], addFiles); + + expect(updateFileMetadata).not.toHaveBeenCalled(); + }); + + it("copies the library entry without touching the workbench or metrics", async () => { + await duplicateStoredFile(stub(), ["report (copy).pdf"], addFiles); + + const [files, options] = addFiles.mock.calls[0]; + expect(files[0].name).toBe("report (copy 2).pdf"); + expect(options).toMatchObject({ + selectFiles: false, + skipWorkspaceDispatch: true, + autoUnzip: false, + skipUploadTracking: true, + }); + }); + + it("returns null when the source bytes are gone", async () => { + getStirlingFile.mockResolvedValue(null); + + expect(await duplicateStoredFile(stub(), [], addFiles)).toBeNull(); + expect(addFiles).not.toHaveBeenCalled(); + }); +}); diff --git a/frontend/editor/src/core/utils/duplicateFile.ts b/frontend/editor/src/core/utils/duplicateFile.ts new file mode 100644 index 0000000000..5bad75b513 --- /dev/null +++ b/frontend/editor/src/core/utils/duplicateFile.ts @@ -0,0 +1,85 @@ +import type { StirlingFile, StirlingFileStub } from "@app/types/fileContext"; +import type { FileId } from "@app/types/file"; +import { fileStorage } from "@app/services/fileStorage"; +import { splitFileName } from "@app/utils/fileUtils"; + +/** The subset of `useFileHandler().addFiles` a duplicate needs. */ +type AddFilesFn = ( + files: File[], + options: { + selectFiles?: boolean; + skipWorkspaceDispatch?: boolean; + autoUnzip?: boolean; + skipUploadTracking?: boolean; + }, +) => Promise; + +/** "report.pdf" → "report (copy).pdf", counting up until the name is free. */ +export function copyNameFor(name: string, taken: Iterable): string { + const [base, extension] = splitFileName(name); + const used = new Set(taken); + let candidate = `${base} (copy)${extension}`; + for (let n = 2; used.has(candidate); n++) { + candidate = `${base} (copy ${n})${extension}`; + } + return candidate; +} + +/** + * Copies a stored file into the library under a free "(copy)" name. + * + * The copy stays out of the workbench - it's an archive of the current bytes, + * not a file the user asked to work on. That skips the ingest side-effects that + * hang off the workspace dispatch, so the derived metadata is inherited from + * the source instead: the bytes are identical, so its classification labels and + * thumbnail are too, and re-deriving them would only re-parse the same PDF. The + * copy also lands in the source's folder rather than back at the root. + * + * @returns the new file's id, or null if the source has no readable bytes. + */ +export async function duplicateStoredFile( + stub: StirlingFileStub, + existingNames: Iterable, + addFiles: AddFilesFn, +): Promise { + const source = await fileStorage.getStirlingFile(stub.id); + if (!source) return null; + + const [copy] = await addFiles( + [ + new File([source], copyNameFor(stub.name, existingNames), { + type: source.type, + }), + ], + { + selectFiles: false, + skipWorkspaceDispatch: true, + // Duplicating an archive must yield one copy, not its contents scattered + // across the library. + autoUnzip: false, + // A local copy is not a new document entering the system. + skipUploadTracking: true, + }, + ); + if (!copy) return null; + + const inherited: Parameters[1] = {}; + if (stub.classificationLabels) { + inherited.classificationLabels = stub.classificationLabels; + } + // A copy belongs beside its original. Safe to set on a local-only file: the + // server is authoritative for folderId only on files it actually holds. + if (stub.folderId) { + inherited.folderId = stub.folderId; + } + // Blob URLs die with the session, so only a persisted thumbnail is worth + // carrying over; without one the row falls back to lazy regeneration. + if (stub.thumbnailUrl && !stub.thumbnailUrl.startsWith("blob:")) { + inherited.thumbnail = stub.thumbnailUrl; + inherited.thumbnailStoredAt = Date.now(); + } + if (Object.keys(inherited).length > 0) { + await fileStorage.updateFileMetadata(copy.fileId, inherited); + } + return copy.fileId; +} diff --git a/frontend/editor/src/core/utils/fileUtils.ts b/frontend/editor/src/core/utils/fileUtils.ts index 6a256a9073..13002db52e 100644 --- a/frontend/editor/src/core/utils/fileUtils.ts +++ b/frontend/editor/src/core/utils/fileUtils.ts @@ -73,6 +73,17 @@ export function getFilenameWithoutExtension( return preserveCase ? withoutExtension : withoutExtension.toLowerCase(); } +/** + * Splits a filename into its base and extension, keeping the dot on the + * extension so `base + extension` round-trips. A name with no extension (or a + * leading-dot name like ".env") gets an empty extension. + * @example splitFileName('report.pdf') // ['report', '.pdf'] + */ +export function splitFileName(name: string): [string, string] { + const dot = name.lastIndexOf("."); + return dot > 0 ? [name.slice(0, dot), name.slice(dot)] : [name, ""]; +} + /** * Checks if a file is a PDF based on extension and MIME type * @param file - File or file-like object with name and type properties From 96a00cebd18ba703f7c5719fa348d31885cd6543 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:00:03 +0000 Subject: [PATCH 11/45] Pdf ua converter testing (#7301) # Description of Changes Adds a PDF/UA converter, an accessibility report, and PDF/A conformance level A. **New: `POST /api/v1/convert/pdf/ua`** (Convert tool, "PDF/UA" target). Tags an untagged PDF, marks decorative content as artifacts, embeds missing fonts and applies the document-level PDF/UA requirements (title, language, tab order, form-field descriptions), then validates with veraPDF. The `pdfuaid` declaration is written only if validation passes, so a returned file never claims more than it delivers; response headers report whether it was declared, how many checks still fail and how many images still need a description. **New: `POST /api/v1/security/accessibility-report`.** Reports what fails, what the converter can fix on its own, what needs a person, and lists the figures needing a description with the keys the conversion accepts back. Read-only; does not modify the file. Capped at 100 MB / 2000 pages and weighted `LARGE_WEIGHT`, since it runs a full veraPDF pass plus the converter's layout analysis over every page. **PDF/A level A.** `pdfa-1a`, `pdfa-2a` and `pdfa-3a` output formats on the existing `/api/v1/convert/pdf/pdfa` endpoint. Level A is level B plus tagging, so the document is tagged after Ghostscript (which discards any structure tree it is given) and the level A claim is written only if veraPDF agrees. Optional `pdfUa=true` additionally declares PDF/UA alongside PDF/A, again only if it validates. Honesty rules the implementation holds to: - **Never claim a level that was not reached.** If tagging fails, the file is returned at level B and is named `_PDFA-2b.pdf`, not `_PDFA-2a.pdf`. With `strict=true` the request fails outright rather than returning a level B file against a level A request, and a level B pass no longer satisfies a strict level A request. - **Never relabel a document's language.** The requested language (default `en-GB`) is applied only when the document declares none; a French PDF stays French unless the caller sets `overrideLanguage`, and ignoring a requested language is reported as a warning. - **Never invent alternative text.** Descriptions come from the caller. The Convert panel can list the images needing one (via the report endpoint) and send them back per figure; any image left undescribed blocks the conformance claim rather than being papered over. - **Never certify hidden content.** Marking images decorative, or suppressing text that could not be tagged reliably, withdraws the claim instead of passing the checker by hiding content. PDF/UA-1 and PDF/UA-2 are both offered; UA-2 raises the file to PDF 2.0 and namespaces the structure tree, and its test asserts conformance rather than merely reporting it. Convert steps saved in Automations/Pipelines round-trip their PDF/UA settings (profile, language, override, title, font embedding, descriptions). --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details. --- .../SPDF/config/EndpointConfiguration.java | 19 +- .../service/PdfaLevelAServiceInterface.java | 22 + .../config/EndpointConfigurationGapTest.java | 32 +- .../api/converters/ConvertPDFToPDFA.java | 171 +++- .../api/converters/PdfToPdfARequest.java | 12 +- .../software/SPDF/service/VeraPDFService.java | 2 + .../converters/ConvertPDFToPDFAGapTest.java | 136 ++- .../converters/ConvertPDFToPDFAMoreTest.java | 5 +- .../VeraPDFServicePdfaFixtureTest.java | 17 +- app/proprietary/build.gradle | 9 + .../api/converters/ConvertPdfToPdfUa.java | 176 ++++ .../AccessibilityReportController.java | 67 ++ .../api/converters/PdfToPdfUaRequest.java | 73 ++ .../model/api/ua/AccessibilityIssue.java | 38 + .../model/api/ua/AccessibilityReport.java | 63 ++ .../api/ua/AccessibilityReportRequest.java | 19 + .../model/api/ua/FigureDescriptor.java | 18 + .../model/api/ua/PdfUaConversionOutcome.java | 26 + .../model/api/ua/UaValidationResult.java | 18 + .../proprietary/pdf/ua/ArtifactType.java | 23 + .../software/proprietary/pdf/ua/BBox.java | 48 + .../proprietary/pdf/ua/DocumentStructure.java | 84 ++ .../proprietary/pdf/ua/LayoutAnalyzer.java | 831 ++++++++++++++++++ .../proprietary/pdf/ua/MarkableOp.java | 44 + .../pdf/ua/MarkedContentInjector.java | 284 ++++++ .../proprietary/pdf/ua/PageContent.java | 33 + .../pdf/ua/PdfUaIdentificationSchema.java | 47 + .../pdf/ua/PdfUaMetadataWriter.java | 224 +++++ .../proprietary/pdf/ua/PdfUaProfile.java | 47 + .../proprietary/pdf/ua/PdfUaTagger.java | 303 +++++++ .../proprietary/pdf/ua/SourceFacts.java | 59 ++ .../proprietary/pdf/ua/StructBlock.java | 134 +++ .../proprietary/pdf/ua/StructTreeWriter.java | 295 +++++++ .../proprietary/pdf/ua/StructType.java | 62 ++ .../pdf/ua/TaggedContentExtractor.java | 630 +++++++++++++ .../proprietary/pdf/ua/TaggingOptions.java | 63 ++ .../proprietary/pdf/ua/TaggingResult.java | 42 + .../proprietary/pdf/ua/TextLineInfo.java | 42 + .../software/proprietary/pdf/ua/WordInfo.java | 18 + .../service/ua/AccessibilityAuditService.java | 187 ++++ .../service/ua/FontEmbeddingService.java | 254 ++++++ .../service/ua/PdfUaConversionService.java | 251 ++++++ .../service/ua/PdfUaValidationService.java | 245 ++++++ .../service/ua/PdfaAccessibilityService.java | 297 +++++++ .../pdf/ua/LayoutAnalyzerTest.java | 286 ++++++ .../pdf/ua/MarkedContentInjectorTest.java | 164 ++++ .../pdf/ua/MarkedContentSafetyTest.java | 193 ++++ .../pdf/ua/PdfUaFormAndDeclarationTest.java | 171 ++++ .../proprietary/pdf/ua/PdfUaLanguageTest.java | 79 ++ .../pdf/ua/PdfUaMetadataWriterTest.java | 137 +++ .../proprietary/pdf/ua/PdfUaModelTest.java | 160 ++++ .../pdf/ua/VectorAndHeadingTest.java | 155 ++++ .../service/ua/AltTextRoundTripTest.java | 115 +++ .../service/ua/PdfUa2ProfileTest.java | 92 ++ .../service/ua/PdfUaBenchmarkTest.java | 341 +++++++ .../ua/PdfUaConversionIntegrationTest.java | 231 +++++ .../service/ua/PdfUaHardeningTest.java | 322 +++++++ .../service/ua/PdfUaHttpEndpointTest.java | 183 ++++ .../service/ua/PdfUaRealCorpusTest.java | 249 ++++++ .../service/ua/PdfUaSampleDumpTest.java | 103 +++ .../service/ua/PdfUaServicesTest.java | 319 +++++++ .../service/ua/PdfUaTestDocuments.java | 390 ++++++++ .../service/ua/PdfaLevelATest.java | 202 +++++ .../TaggedContentExtractorRealFilesTest.java | 99 +++ engine/src/stirling/models/tool_io.py | 4 + engine/src/stirling/models/tool_models.py | 89 ++ .../public/locales/en-US/translation.toml | 23 + .../tools/convert/ConvertSettings.tsx | 15 + .../ConvertToPdfUaSettings.selection.test.tsx | 149 ++++ .../convert/ConvertToPdfUaSettings.test.ts | 34 + .../tools/convert/ConvertToPdfUaSettings.tsx | 280 ++++++ .../tools/convert/ConvertToPdfaSettings.tsx | 11 + .../src/core/constants/convertConstants.ts | 6 + .../tools/convert/convertPdfUaAltText.test.ts | 92 ++ .../tools/convert/useConvertOperation.ts | 55 +- .../tools/convert/useConvertParameters.ts | 17 + .../hooks/tools/shared/toolAutomation.test.ts | 46 + .../tests/convert/ConvertIntegration.test.tsx | 96 ++ .../editor/src/core/types/toolApiTypes.ts | 53 ++ frontend/editor/src/core/types/toolIO.ts | 10 + 80 files changed, 10373 insertions(+), 68 deletions(-) create mode 100644 app/common/src/main/java/stirling/software/common/service/PdfaLevelAServiceInterface.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/controller/api/converters/ConvertPdfToPdfUa.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/controller/api/security/AccessibilityReportController.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/model/api/converters/PdfToPdfUaRequest.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/AccessibilityIssue.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/AccessibilityReport.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/AccessibilityReportRequest.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/FigureDescriptor.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/PdfUaConversionOutcome.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/UaValidationResult.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/ArtifactType.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/BBox.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/DocumentStructure.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/LayoutAnalyzer.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/MarkableOp.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/MarkedContentInjector.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PageContent.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaIdentificationSchema.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaMetadataWriter.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaProfile.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaTagger.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/SourceFacts.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/StructBlock.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/StructTreeWriter.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/StructType.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TaggedContentExtractor.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TaggingOptions.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TaggingResult.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TextLineInfo.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/WordInfo.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/service/ua/AccessibilityAuditService.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/service/ua/FontEmbeddingService.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/service/ua/PdfUaConversionService.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/service/ua/PdfUaValidationService.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/service/ua/PdfaAccessibilityService.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/LayoutAnalyzerTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/MarkedContentInjectorTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/MarkedContentSafetyTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaFormAndDeclarationTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaLanguageTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaMetadataWriterTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaModelTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/VectorAndHeadingTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/ua/AltTextRoundTripTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUa2ProfileTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaBenchmarkTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaConversionIntegrationTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaHardeningTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaHttpEndpointTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaRealCorpusTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaSampleDumpTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaServicesTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaTestDocuments.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfaLevelATest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/ua/TaggedContentExtractorRealFilesTest.java create mode 100644 frontend/editor/src/core/components/tools/convert/ConvertToPdfUaSettings.selection.test.tsx create mode 100644 frontend/editor/src/core/components/tools/convert/ConvertToPdfUaSettings.test.ts create mode 100644 frontend/editor/src/core/components/tools/convert/ConvertToPdfUaSettings.tsx create mode 100644 frontend/editor/src/core/hooks/tools/convert/convertPdfUaAltText.test.ts diff --git a/app/common/src/main/java/stirling/software/SPDF/config/EndpointConfiguration.java b/app/common/src/main/java/stirling/software/SPDF/config/EndpointConfiguration.java index ff1a880010..5e8f7fe336 100644 --- a/app/common/src/main/java/stirling/software/SPDF/config/EndpointConfiguration.java +++ b/app/common/src/main/java/stirling/software/SPDF/config/EndpointConfiguration.java @@ -6,6 +6,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; @@ -13,6 +14,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.PdfaLevelAServiceInterface; @Service @Slf4j @@ -51,12 +53,16 @@ public class EndpointConfiguration { private Map groupDisableReasons = new ConcurrentHashMap<>(); private Map> endpointAlternatives = new ConcurrentHashMap<>(); private final boolean runningProOrHigher; + private final boolean pdfUaAvailable; public EndpointConfiguration( ApplicationProperties applicationProperties, - @Qualifier("runningProOrHigher") boolean runningProOrHigher) { + @Qualifier("runningProOrHigher") boolean runningProOrHigher, + @Autowired(required = false) PdfaLevelAServiceInterface pdfaLevelAService) { this.applicationProperties = applicationProperties; this.runningProOrHigher = runningProOrHigher; + // The PDF/UA tagger ships in the proprietary module, and so do its endpoints. + this.pdfUaAvailable = pdfaLevelAService != null; init(); processEnvironmentConfigs(); } @@ -356,6 +362,7 @@ public class EndpointConfiguration { addEndpointToGroup("Convert", "pdf-to-img"); addEndpointToGroup("Convert", "img-to-pdf"); addEndpointToGroup("Convert", "pdf-to-pdfa"); + addEndpointToGroup("Convert", "pdf-to-ua"); addEndpointToGroup("Convert", "file-to-pdf"); addEndpointToGroup("Convert", "pdf-to-word"); addEndpointToGroup("Convert", "pdf-to-presentation"); @@ -395,6 +402,7 @@ public class EndpointConfiguration { // Backend-only endpoints (not in frontend tool registry endpoints) addEndpointToGroup("Security", "redact"); addEndpointToGroup("Security", "verify-pdf"); + addEndpointToGroup("Security", "accessibility-report"); addEndpointToGroup("Security", "sign"); // Adding endpoints to "Other" group @@ -529,6 +537,8 @@ public class EndpointConfiguration { addEndpointToGroup("Java", "json-to-pdf"); addEndpointToGroup("Java", "pdf-to-video"); addEndpointToGroup("Java", "verify-pdf"); + addEndpointToGroup("Java", "pdf-to-ua"); + addEndpointToGroup("Java", "accessibility-report"); addEndpointToGroup("Java", "flatten"); addEndpointToGroup("Java", "unlock-pdf-forms"); addEndpointToGroup("Java", "validate-signature"); @@ -600,6 +610,8 @@ public class EndpointConfiguration { // veraPDF dependent endpoints addEndpointToGroup("veraPDF", "verify-pdf"); + addEndpointToGroup("veraPDF", "pdf-to-ua"); + addEndpointToGroup("veraPDF", "accessibility-report"); // Pdftohtml dependent endpoints addEndpointToGroup("Pdftohtml", "pdf-to-html"); @@ -630,6 +642,11 @@ public class EndpointConfiguration { disableGroup("enterprise"); } + if (!pdfUaAvailable) { + disableEndpoint("pdf-to-ua"); + disableEndpoint("accessibility-report"); + } + if (!applicationProperties.getSystem().isEnableUrlToPDF()) { disableEndpoint("url-to-pdf"); } diff --git a/app/common/src/main/java/stirling/software/common/service/PdfaLevelAServiceInterface.java b/app/common/src/main/java/stirling/software/common/service/PdfaLevelAServiceInterface.java new file mode 100644 index 0000000000..2ee55719b2 --- /dev/null +++ b/app/common/src/main/java/stirling/software/common/service/PdfaLevelAServiceInterface.java @@ -0,0 +1,22 @@ +package stirling.software.common.service; + +import java.util.List; + +/** + * Raises a converted PDF/A file from conformance level B to level A, which needs the tagging the + * PDF/UA tagger does. Implemented only in the proprietary module; core builds convert at level B. + */ +public interface PdfaLevelAServiceInterface { + + /** + * @param levelA true only when the file was tagged and validated, so the claim is never a guess + */ + record Result(byte[] pdfBytes, boolean levelA, List warnings) {} + + /** + * @param part PDF/A part, 1 to 3; part 1 keeps its PDF 1.4 version + * @param alsoDeclareUa additionally claim PDF/UA, but only if it validates + */ + Result upgradeToLevelA( + byte[] pdfBytes, int part, String language, String title, boolean alsoDeclareUa); +} diff --git a/app/common/src/test/java/stirling/software/SPDF/config/EndpointConfigurationGapTest.java b/app/common/src/test/java/stirling/software/SPDF/config/EndpointConfigurationGapTest.java index 6275b49343..afc6fa7020 100644 --- a/app/common/src/test/java/stirling/software/SPDF/config/EndpointConfigurationGapTest.java +++ b/app/common/src/test/java/stirling/software/SPDF/config/EndpointConfigurationGapTest.java @@ -17,6 +17,7 @@ import org.junit.jupiter.api.Test; import stirling.software.SPDF.config.EndpointConfiguration.DisableReason; import stirling.software.SPDF.config.EndpointConfiguration.EndpointAvailability; import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.service.PdfaLevelAServiceInterface; /** * Unit tests for {@link EndpointConfiguration}. The class wires up its endpoint/group registry in @@ -32,7 +33,14 @@ class EndpointConfigurationGapTest { * Construct an EndpointConfiguration with the given pro flag and current applicationProperties. */ private EndpointConfiguration build(boolean runningProOrHigher) { - return new EndpointConfiguration(applicationProperties, runningProOrHigher); + return build(runningProOrHigher, null); + } + + /** The PDF/UA service is only present in proprietary builds, so it is injected separately. */ + private EndpointConfiguration build( + boolean runningProOrHigher, PdfaLevelAServiceInterface pdfaLevelAService) { + return new EndpointConfiguration( + applicationProperties, runningProOrHigher, pdfaLevelAService); } /** Default config: not pro, no removals, url-to-pdf disabled (default System flag is false). */ @@ -177,6 +185,28 @@ class EndpointConfigurationGapTest { } } + @Nested + @DisplayName("PDF/UA availability") + class PdfUaTests { + + @Test + @DisplayName("the PDF/UA endpoints are off when the proprietary tagger is absent") + void disabledWithoutTagger() { + EndpointConfiguration config = build(false, null); + assertFalse(config.isEndpointEnabled("pdf-to-ua")); + assertFalse(config.isEndpointEnabled("accessibility-report")); + } + + @Test + @DisplayName("they are on once the tagger is on the classpath") + void enabledWithTagger() { + EndpointConfiguration config = + build(false, (pdfBytes, part, language, title, alsoDeclareUa) -> null); + assertTrue(config.isEndpointEnabled("pdf-to-ua")); + assertTrue(config.isEndpointEnabled("accessibility-report")); + } + } + @Nested @DisplayName("group enable / disable") class GroupTests { diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFA.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFA.java index 49bf4e4895..0354817315 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFA.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFA.java @@ -11,6 +11,7 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.*; +import java.util.Locale; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -71,6 +72,7 @@ import org.apache.xmpbox.schema.PDFAIdentificationSchema; import org.apache.xmpbox.schema.XMPBasicSchema; import org.apache.xmpbox.xml.DomXmpParser; import org.apache.xmpbox.xml.XmpSerializer; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.Resource; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -83,7 +85,6 @@ import io.github.pixee.security.Filenames; import io.swagger.v3.oas.annotations.Operation; import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import stirling.software.SPDF.model.api.converters.PdfToPdfARequest; @@ -93,6 +94,7 @@ import stirling.software.common.configuration.RuntimePathConfig; import stirling.software.common.enumeration.ResourceWeight; import stirling.software.common.model.tool.ToolFormat; import stirling.software.common.model.tool.ToolIO; +import stirling.software.common.service.PdfaLevelAServiceInterface; import stirling.software.common.util.ExceptionUtils; import stirling.software.common.util.ProcessExecutor; import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult; @@ -102,14 +104,26 @@ import stirling.software.common.util.WebResponseUtils; @ConvertApi @Slf4j -@RequiredArgsConstructor public class ConvertPDFToPDFA { private static final Pattern NON_PRINTABLE_ASCII = Pattern.compile("[^\\x20-\\x7E]"); private final RuntimePathConfig runtimePathConfig; private final stirling.software.SPDF.service.VeraPDFService veraPDFService; + // Level A needs the proprietary tagger; core builds convert at level B instead. + private final PdfaLevelAServiceInterface pdfaLevelAService; private final TempFileManager tempFileManager; + public ConvertPDFToPDFA( + RuntimePathConfig runtimePathConfig, + stirling.software.SPDF.service.VeraPDFService veraPDFService, + @Autowired(required = false) PdfaLevelAServiceInterface pdfaLevelAService, + TempFileManager tempFileManager) { + this.runtimePathConfig = runtimePathConfig; + this.veraPDFService = veraPDFService; + this.pdfaLevelAService = pdfaLevelAService; + this.tempFileManager = tempFileManager; + } + private static final String ICC_RESOURCE_PATH = "/icc/sRGB2014.icc"; private static final int PDFA_COMPATIBILITY_POLICY = 1; @@ -604,7 +618,10 @@ public class ConvertPDFToPDFA { return handlePdfXConversion(inputFile, outputFormat); } else { return handlePdfAConversion( - inputFile, outputFormat, request.getStrict() != null && request.getStrict()); + inputFile, + outputFormat, + request.getStrict() != null && request.getStrict(), + request.getPdfUa() != null && request.getPdfUa()); } } @@ -1815,8 +1832,64 @@ public class ConvertPDFToPDFA { return Files.readAllBytes(outputPdf); } + /** Tags a converted PDF/A for level A; must run after Ghostscript, which discards tags. */ + private PdfaLevelAServiceInterface.Result applyLevelA( + byte[] converted, + Path original, + PdfaProfile profile, + String baseFileName, + boolean declarePdfUa) { + if (!profile.requiresTagging()) { + return new PdfaLevelAServiceInterface.Result(converted, true, List.of()); + } + if (pdfaLevelAService == null) { + return new PdfaLevelAServiceInterface.Result( + converted, + false, + List.of( + "Level A tagging is not available in this build, so the file was left" + + " at conformance level B.")); + } + // Prefer the document's own title/language; hardcoding "en" mislabelled German reports. + // Read the original, not the converted bytes: Ghostscript discards /Lang, so probing its + // output always yields null and every document would be relabelled with the default. + String language = null; + String title = null; + try (PDDocument probe = Loader.loadPDF(original.toFile())) { + language = probe.getDocumentCatalog().getLanguage(); + title = probe.getDocumentInformation().getTitle(); + } catch (IOException e) { + log.debug("Could not read original title/language: {}", e.getMessage()); + } + if (language == null || language.isBlank()) { + try (PDDocument probe = Loader.loadPDF(converted)) { + language = probe.getDocumentCatalog().getLanguage(); + if (title == null || title.isBlank()) { + title = probe.getDocumentInformation().getTitle(); + } + } catch (IOException e) { + log.debug("Could not read converted title/language: {}", e.getMessage()); + } + } + PdfaLevelAServiceInterface.Result result = + pdfaLevelAService.upgradeToLevelA( + converted, + profile.getPart(), + language, + title != null && !title.isBlank() ? title : baseFileName, + declarePdfUa); + result.warnings().forEach(warning -> log.info("PDF/A level A: {}", warning)); + if (!result.levelA()) { + log.warn( + "{} requested but the document could not be tagged; returning level B", + profile.getDisplayName()); + } + return result; + } + private ResponseEntity handlePdfAConversion( - MultipartFile inputFile, String outputFormat, boolean strict) throws Exception { + MultipartFile inputFile, String outputFormat, boolean strict, boolean declarePdfUa) + throws Exception { PdfaProfile profile = PdfaProfile.fromRequest(outputFormat); // Get the original filename without extension @@ -1841,12 +1914,15 @@ public class ConvertPDFToPDFA { log.info("Using Ghostscript for PDF/A conversion to {}", profile.getDisplayName()); try { converted = convertWithGhostscript(inputPath, workingDir, profile); - String outputFilename = baseFileName + profile.outputSuffix(); + var levelA = + applyLevelA(converted, inputPath, profile, baseFileName, declarePdfUa); + converted = levelA.pdfBytes(); + String outputFilename = baseFileName + profile.outputSuffix(levelA.levelA()); validateAndWarnPdfA(converted, profile, "Ghostscript"); if (strict) { - verifyStrictCompliance(converted); + verifyStrictCompliance(converted, profile, levelA.levelA()); } TempFile tempOut = tempFileManager.createManagedTempFile(".pdf"); @@ -1867,13 +1943,15 @@ public class ConvertPDFToPDFA { } converted = convertWithPdfBoxMethod(inputPath, profile); - String outputFilename = baseFileName + profile.outputSuffix(); + var levelA = applyLevelA(converted, inputPath, profile, baseFileName, declarePdfUa); + converted = levelA.pdfBytes(); + String outputFilename = baseFileName + profile.outputSuffix(levelA.levelA()); // Validate with PDFBox preflight and warn if issues found validateAndWarnPdfA(converted, profile, "PDFBox/LibreOffice"); if (strict) { - verifyStrictCompliance(converted); + verifyStrictCompliance(converted, profile, levelA.levelA()); } TempFile tempOut = tempFileManager.createManagedTempFile(".pdf"); @@ -1889,11 +1967,56 @@ public class ConvertPDFToPDFA { } } - private void verifyStrictCompliance(byte[] pdfBytes) throws IOException { + /** True for a PDF/UA or WCAG result, which says nothing about archival conformance. */ + private static boolean isAccessibilityProfile( + stirling.software.SPDF.model.api.security.PDFVerificationResult result) { + String profile = result.getValidationProfile(); + if (profile == null) { + return false; + } + String normalised = profile.toLowerCase(Locale.ROOT); + return normalised.contains("ua") || normalised.contains("wcag"); + } + + /** + * True when a result speaks for the requested profile. Only archival results count, and a level + * B pass must never satisfy a level A request. + */ + private static boolean answersRequest( + PdfaProfile profile, + stirling.software.SPDF.model.api.security.PDFVerificationResult result) { + if (isAccessibilityProfile(result)) { + return false; + } + String standard = result.getStandard(); + if (standard == null || standard.length() < 2) { + return false; + } + if (standard.charAt(0) != Character.forDigit(profile.getPart(), 10)) { + return false; + } + return !profile.requiresTagging() || Character.toLowerCase(standard.charAt(1)) == 'a'; + } + + private void verifyStrictCompliance(byte[] pdfBytes, PdfaProfile profile, boolean levelAReached) + throws IOException { + // Tagging is the only route to level A, so an untagged file cannot answer a strict request. + if (!levelAReached) { + throw new ResponseStatusException( + HttpStatus.BAD_REQUEST, + "Strict PDF/A mode enabled: the document could not be tagged, so " + + profile.getDisplayName() + + " was not reached. It is valid at level B."); + } try (InputStream is = new ByteArrayInputStream(pdfBytes)) { List results = veraPDFService.validatePDF(is); - boolean isCompliant = results.stream().anyMatch(result -> result.isCompliant()); + boolean isCompliant = + results.stream() + .filter(result -> answersRequest(profile, result)) + .anyMatch( + stirling.software.SPDF.model.api.security.PDFVerificationResult + ::isCompliant); if (!isCompliant) { String details = results.stream() @@ -1901,7 +2024,9 @@ public class ConvertPDFToPDFA { .collect(Collectors.joining("; ")); throw new ResponseStatusException( HttpStatus.BAD_REQUEST, - "Strict PDF/A mode enabled: Conversion is not perfectly compliant. Details: " + "Strict PDF/A mode enabled: the output is not perfectly compliant with " + + profile.getDisplayName() + + ". Details: " + details); } } catch (Exception e) { @@ -2466,11 +2591,16 @@ public class ConvertPDFToPDFA { @Getter private enum PdfaProfile { - PDF_A_1B(1, "PDF/A-1b", "_PDFA-1b.pdf", "1.4", Format.PDF_A1B, "pdfa-1"), - PDF_A_2B(2, "PDF/A-2b", "_PDFA-2b.pdf", "1.7", null, "pdfa", "pdfa-2", "pdfa-2b"), - PDF_A_3B(3, "PDF/A-3b", "_PDFA-3b.pdf", "1.7", null, "pdfa-3", "pdfa-3b"); + PDF_A_1B(1, "B", "PDF/A-1b", "_PDFA-1b.pdf", "1.4", Format.PDF_A1B, "pdfa-1"), + PDF_A_2B(2, "B", "PDF/A-2b", "_PDFA-2b.pdf", "1.7", null, "pdfa", "pdfa-2", "pdfa-2b"), + PDF_A_3B(3, "B", "PDF/A-3b", "_PDFA-3b.pdf", "1.7", null, "pdfa-3", "pdfa-3b"), + // Level A = level B plus tagging, declared language and Unicode text; tagged post-convert. + PDF_A_1A(1, "A", "PDF/A-1a", "_PDFA-1a.pdf", "1.4", Format.PDF_A1B, "pdfa-1a"), + PDF_A_2A(2, "A", "PDF/A-2a", "_PDFA-2a.pdf", "1.7", null, "pdfa-2a"), + PDF_A_3A(3, "A", "PDF/A-3a", "_PDFA-3a.pdf", "1.7", null, "pdfa-3a"); private final int part; + private final String conformanceLevel; private final String displayName; private final String suffix; private final String compatibilityLevel; @@ -2479,12 +2609,14 @@ public class ConvertPDFToPDFA { PdfaProfile( int part, + String conformanceLevel, String displayName, String suffix, String compatibilityLevel, Format preflightFormat, String... requestTokens) { this.part = part; + this.conformanceLevel = conformanceLevel; this.displayName = displayName; this.suffix = suffix; this.compatibilityLevel = compatibilityLevel; @@ -2495,6 +2627,10 @@ public class ConvertPDFToPDFA { .toList(); } + boolean requiresTagging() { + return "A".equals(conformanceLevel); + } + static PdfaProfile fromRequest(String requestToken) { if (requestToken == null) { return PDF_A_2B; @@ -2508,8 +2644,11 @@ public class ConvertPDFToPDFA { return match.orElse(PDF_A_2B); } - String outputSuffix() { - return suffix; + /** + * Names the file at the level actually reached; a level A name over level B content lies. + */ + String outputSuffix(boolean levelAReached) { + return levelAReached ? suffix : "_PDFA-" + part + "b.pdf"; } Optional preflightFormat() { diff --git a/app/core/src/main/java/stirling/software/SPDF/model/api/converters/PdfToPdfARequest.java b/app/core/src/main/java/stirling/software/SPDF/model/api/converters/PdfToPdfARequest.java index bb0520a4ba..921663912b 100644 --- a/app/core/src/main/java/stirling/software/SPDF/model/api/converters/PdfToPdfARequest.java +++ b/app/core/src/main/java/stirling/software/SPDF/model/api/converters/PdfToPdfARequest.java @@ -14,9 +14,19 @@ public class PdfToPdfARequest extends PDFFile { @Schema( description = "The output format type (PDF/A or PDF/X)", requiredMode = Schema.RequiredMode.REQUIRED, - allowableValues = {"pdfa", "pdfa-1", "pdfa-2", "pdfa-2b", "pdfa-3", "pdfa-3b", "pdfx"}) + allowableValues = { + "pdfa", "pdfa-1", "pdfa-2", "pdfa-2b", "pdfa-3", "pdfa-3b", "pdfa-1a", "pdfa-2a", + "pdfa-3a", "pdfx" + }) private String outputFormat; + @Schema( + description = + "Also declare PDF/UA accessibility alongside PDF/A. Only applies to the level A" + + " formats, and the claim is written only if it validates.", + defaultValue = "false") + private Boolean pdfUa; + @Schema( description = "If true, the conversion will fail if the output is not perfectly compliant") diff --git a/app/core/src/main/java/stirling/software/SPDF/service/VeraPDFService.java b/app/core/src/main/java/stirling/software/SPDF/service/VeraPDFService.java index bb3c84534c..6361157b21 100644 --- a/app/core/src/main/java/stirling/software/SPDF/service/VeraPDFService.java +++ b/app/core/src/main/java/stirling/software/SPDF/service/VeraPDFService.java @@ -285,6 +285,8 @@ public class VeraPDFService { } } + // Never force PDF/UA here - it flags every ordinary document as non-compliant and doubles + // verify cost; /accessibility-report checks PDF/UA on demand. if (!hasPdfaDeclaration) { results.add(createNoPdfaDeclarationResult()); } diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFAGapTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFAGapTest.java index b64776665b..ea693ddd27 100644 --- a/app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFAGapTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFAGapTest.java @@ -46,6 +46,7 @@ import stirling.software.SPDF.model.api.converters.PdfToPdfARequest; import stirling.software.SPDF.model.api.security.PDFVerificationResult; import stirling.software.SPDF.service.VeraPDFService; import stirling.software.common.configuration.RuntimePathConfig; +import stirling.software.common.service.PdfaLevelAServiceInterface; import stirling.software.common.util.TempFileManager; /** @@ -62,10 +63,12 @@ class ConvertPDFToPDFAGapTest { @Mock private RuntimePathConfig runtimePathConfig; @Mock private VeraPDFService veraPDFService; + @Mock private PdfaLevelAServiceInterface pdfaLevelAService; @Mock private TempFileManager tempFileManager; private ConvertPDFToPDFA newController() { - return new ConvertPDFToPDFA(runtimePathConfig, veraPDFService, tempFileManager); + return new ConvertPDFToPDFA( + runtimePathConfig, veraPDFService, pdfaLevelAService, tempFileManager); } // ---- reflection helpers ---------------------------------------------------------------- @@ -161,9 +164,21 @@ class ConvertPDFToPDFAGapTest { } private String suffixOf(Object profile) throws Exception { - Method m = profile.getClass().getDeclaredMethod("outputSuffix"); + return suffixOf(profile, true); + } + + private String suffixOf(Object profile, boolean levelAReached) throws Exception { + Method m = profile.getClass().getDeclaredMethod("outputSuffix", boolean.class); m.setAccessible(true); - return (String) m.invoke(profile); + return (String) m.invoke(profile, levelAReached); + } + + @Test + @DisplayName("a level A profile falls back to the level B name when tagging failed") + void levelANotReachedIsNamedLevelB() throws Exception { + assertThat(suffixOf(resolveProfile("pdfa-1a"), false)).isEqualTo("_PDFA-1b.pdf"); + assertThat(suffixOf(resolveProfile("pdfa-2a"), false)).isEqualTo("_PDFA-2b.pdf"); + assertThat(suffixOf(resolveProfile("pdfa-3a"), true)).isEqualTo("_PDFA-3a.pdf"); } @Test @@ -717,6 +732,30 @@ class ConvertPDFToPDFAGapTest { @DisplayName("verifyStrictCompliance (VeraPDFService mocked)") class StrictCompliance { + private Object profile(String token) throws Exception { + Class enumClass = null; + for (Class inner : ConvertPDFToPDFA.class.getDeclaredClasses()) { + if (inner.getSimpleName().equals("PdfaProfile")) { + enumClass = inner; + } + } + Method m = enumClass.getDeclaredMethod("fromRequest", String.class); + m.setAccessible(true); + return m.invoke(null, token); + } + + private Throwable verify(String token, boolean levelAReached) throws Exception { + ConvertPDFToPDFA controller = newController(); + return catchThrowable( + () -> + invokeInstance( + controller, + "verifyStrictCompliance", + (Object) "dummy".getBytes(), + profile(token), + levelAReached)); + } + @Test @DisplayName("compliant result passes without throwing") void compliantPasses() throws Exception { @@ -726,14 +765,7 @@ class ConvertPDFToPDFAGapTest { ok.setComplianceSummary("PDF/A-1b compliant"); when(veraPDFService.validatePDF(any())).thenReturn(List.of(ok)); - ConvertPDFToPDFA controller = newController(); - assertThatCode( - () -> - invokeInstance( - controller, - "verifyStrictCompliance", - (Object) "dummy".getBytes())) - .doesNotThrowAnyException(); + assertThat(verify("pdfa-1", true)).isNull(); } @Test @@ -745,34 +777,70 @@ class ConvertPDFToPDFAGapTest { bad.setComplianceSummary("PDF/A-1b with errors"); when(veraPDFService.validatePDF(any())).thenReturn(List.of(bad)); - ConvertPDFToPDFA controller = newController(); - ResponseStatusException ex = - (ResponseStatusException) - catchThrowable( - () -> - invokeInstance( - controller, - "verifyStrictCompliance", - (Object) "dummy".getBytes())); + ResponseStatusException ex = (ResponseStatusException) verify("pdfa-1", true); assertThat(ex).isNotNull(); assertThat(ex.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST); assertThat(ex.getReason()).contains("PDF/A-1b with errors"); } + @Test + @DisplayName("a level B pass does not satisfy a level A request") + void levelBDoesNotSatisfyLevelA() throws Exception { + PDFVerificationResult ok = new PDFVerificationResult(); + ok.setCompliant(true); + ok.setStandard("1b"); + ok.setComplianceSummary("PDF/A-1b compliant"); + when(veraPDFService.validatePDF(any())).thenReturn(List.of(ok)); + + ResponseStatusException ex = (ResponseStatusException) verify("pdfa-1a", true); + assertThat(ex).isNotNull(); + assertThat(ex.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST); + assertThat(ex.getReason()).contains("PDF/A-1a"); + } + + @Test + @DisplayName("a level A result satisfies a level A request") + void levelASatisfiesLevelA() throws Exception { + PDFVerificationResult ok = new PDFVerificationResult(); + ok.setCompliant(true); + ok.setStandard("2a"); + ok.setComplianceSummary("PDF/A-2a compliant"); + when(veraPDFService.validatePDF(any())).thenReturn(List.of(ok)); + + assertThat(verify("pdfa-2a", true)).isNull(); + } + + @Test + @DisplayName("untagged output fails a level A request before validation runs") + void untaggedLevelARequestFails() throws Exception { + ResponseStatusException ex = (ResponseStatusException) verify("pdfa-2a", false); + assertThat(ex).isNotNull(); + assertThat(ex.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST); + assertThat(ex.getReason()).contains("could not be tagged"); + verifyNoInteractions(veraPDFService); + } + + @Test + @DisplayName("a compliant PDF/UA result never satisfies a strict PDF/A request") + void accessibilityResultIsIgnored() throws Exception { + PDFVerificationResult ua = new PDFVerificationResult(); + ua.setCompliant(true); + ua.setStandard("ua1"); + ua.setValidationProfile("ua1"); + ua.setComplianceSummary("PDF/UA-1 compliant"); + when(veraPDFService.validatePDF(any())).thenReturn(List.of(ua)); + + ResponseStatusException ex = (ResponseStatusException) verify("pdfa-2b", true); + assertThat(ex).isNotNull(); + assertThat(ex.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST); + } + @Test @DisplayName("empty result list is treated as non-compliant -> 400") void emptyResultsTreatedNonCompliant() throws Exception { when(veraPDFService.validatePDF(any())).thenReturn(Collections.emptyList()); - ConvertPDFToPDFA controller = newController(); - ResponseStatusException ex = - (ResponseStatusException) - catchThrowable( - () -> - invokeInstance( - controller, - "verifyStrictCompliance", - (Object) "dummy".getBytes())); + ResponseStatusException ex = (ResponseStatusException) verify("pdfa-1", true); assertThat(ex).isNotNull(); assertThat(ex.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST); } @@ -782,15 +850,7 @@ class ConvertPDFToPDFAGapTest { void serviceErrorWrappedAs500() throws Exception { when(veraPDFService.validatePDF(any())).thenThrow(new IOException("boom")); - ConvertPDFToPDFA controller = newController(); - ResponseStatusException ex = - (ResponseStatusException) - catchThrowable( - () -> - invokeInstance( - controller, - "verifyStrictCompliance", - (Object) "dummy".getBytes())); + ResponseStatusException ex = (ResponseStatusException) verify("pdfa-1", true); assertThat(ex).isNotNull(); assertThat(ex.getStatusCode()).isEqualTo(HttpStatus.INTERNAL_SERVER_ERROR); } diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFAMoreTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFAMoreTest.java index e9d9b9ce1d..d56a283464 100644 --- a/app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFAMoreTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFAMoreTest.java @@ -42,6 +42,7 @@ import org.springframework.mock.web.MockMultipartFile; import stirling.software.SPDF.model.api.converters.PdfToPdfARequest; import stirling.software.SPDF.service.VeraPDFService; import stirling.software.common.configuration.RuntimePathConfig; +import stirling.software.common.service.PdfaLevelAServiceInterface; import stirling.software.common.util.ProcessExecutor; import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult; import stirling.software.common.util.TempFile; @@ -63,10 +64,12 @@ class ConvertPDFToPDFAMoreTest { @Mock private RuntimePathConfig runtimePathConfig; @Mock private VeraPDFService veraPDFService; + @Mock private PdfaLevelAServiceInterface pdfaLevelAService; @Mock private TempFileManager tempFileManager; private ConvertPDFToPDFA newController() { - return new ConvertPDFToPDFA(runtimePathConfig, veraPDFService, tempFileManager); + return new ConvertPDFToPDFA( + runtimePathConfig, veraPDFService, pdfaLevelAService, tempFileManager); } private static ResponseEntity streamingOk(byte[] bytes) { diff --git a/app/core/src/test/java/stirling/software/SPDF/service/VeraPDFServicePdfaFixtureTest.java b/app/core/src/test/java/stirling/software/SPDF/service/VeraPDFServicePdfaFixtureTest.java index 38043780d9..e071ece3fb 100644 --- a/app/core/src/test/java/stirling/software/SPDF/service/VeraPDFServicePdfaFixtureTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/service/VeraPDFServicePdfaFixtureTest.java @@ -90,7 +90,9 @@ class VeraPDFServicePdfaFixtureTest { () -> service.validatePDF(new ByteArrayInputStream(pdfBytes)), "Empty veraPDF flavour list must not surface as IndexOutOfBoundsException"); - assertEquals(1, results.size()); + // One result: PDF/UA is checked by the dedicated accessibility-report endpoint, not here. + assertEquals(1, results.size(), () -> "Expected a single PDF/A result, got: " + results); + PDFVerificationResult result = results.get(0); assertEquals("not-pdfa", result.getStandard()); assertFalse(result.isDeclaredPdfa()); @@ -161,13 +163,22 @@ class VeraPDFServicePdfaFixtureTest { } } + /** The PDF/A result; every document is also checked against PDF/UA, so filter that one out. */ private PDFVerificationResult onlyResult(byte[] pdfBytes) throws Exception { List results = service.validatePDF(new ByteArrayInputStream(pdfBytes)); assertNotNull(results); - assertEquals(1, results.size(), () -> "Expected a single result, got: " + results); - return results.get(0); + List pdfaResults = + results.stream().filter(r -> !isUaResult(r)).toList(); + assertEquals( + 1, pdfaResults.size(), () -> "Expected a single PDF/A result, got: " + results); + return pdfaResults.get(0); + } + + private static boolean isUaResult(PDFVerificationResult result) { + String profile = result.getValidationProfile(); + return profile != null && profile.toLowerCase().contains("ua"); } private static String messages(PDFVerificationResult result) { diff --git a/app/proprietary/build.gradle b/app/proprietary/build.gradle index c20241dbb5..b884cb18be 100644 --- a/app/proprietary/build.gradle +++ b/app/proprietary/build.gradle @@ -37,6 +37,15 @@ dependencies { // https://mvnrepository.com/artifact/com.bucket4j/bucket4j_jdk17 implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion" + // PDF/UA tagging and its validation oracle. + implementation 'org.verapdf:validation-model:1.30.2' + // CVE-2025-66453: Explicit rhino 1.7.15 to override verapdf's 1.7.13 + implementation "org.mozilla:rhino:${rhinoVersion}" + // veraPDF still uses javax.xml.bind, not the new jakarta namespace + implementation 'javax.xml.bind:jaxb-api:2.3.1' + runtimeOnly 'com.sun.xml.bind:jaxb-impl:2.3.9' + runtimeOnly 'com.sun.xml.bind:jaxb-core:4.0.9' + implementation "com.google.code.gson:gson:${gsonVersion}" // jinjava/jjwt transitively request older Jackson 2 versions; declare the current diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/converters/ConvertPdfToPdfUa.java b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/converters/ConvertPdfToPdfUa.java new file mode 100644 index 0000000000..7f5241388f --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/converters/ConvertPdfToPdfUa.java @@ -0,0 +1,176 @@ +package stirling.software.proprietary.controller.api.converters; + +import java.io.IOException; +import java.nio.file.Files; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.regex.Pattern; + +import org.springframework.core.io.Resource; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ModelAttribute; +import org.springframework.web.multipart.MultipartFile; + +import io.github.pixee.security.Filenames; +import io.swagger.v3.oas.annotations.Operation; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.annotations.AutoJobPostMapping; +import stirling.software.common.annotations.api.ConvertApi; +import stirling.software.common.enumeration.ResourceWeight; +import stirling.software.common.model.tool.ToolFormat; +import stirling.software.common.model.tool.ToolIO; +import stirling.software.common.util.ExceptionUtils; +import stirling.software.common.util.TempFile; +import stirling.software.common.util.TempFileManager; +import stirling.software.common.util.WebResponseUtils; +import stirling.software.proprietary.model.api.converters.PdfToPdfUaRequest; +import stirling.software.proprietary.model.api.ua.PdfUaConversionOutcome; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.pdf.ua.TaggingOptions; +import stirling.software.proprietary.service.ua.PdfUaConversionService; + +/** Converts a PDF to PDF/UA; response headers say whether the result actually conforms. */ +@ConvertApi +@Slf4j +@RequiredArgsConstructor +public class ConvertPdfToPdfUa { + + private static final String HEADER_DECLARED = "X-Stirling-UA-Declared"; + private static final String HEADER_FAILURES = "X-Stirling-UA-Failures"; + private static final String HEADER_ALT_NEEDED = "X-Stirling-UA-Figures-Needing-Alt"; + private static final String HEADER_WARNINGS = "X-Stirling-UA-Warnings"; + + /** Any line ending, so descriptions pasted from any platform parse the same. */ + private static final Pattern NEWLINE = Pattern.compile("\\R"); + + private final PdfUaConversionService conversionService; + private final TempFileManager tempFileManager; + + @AutoJobPostMapping( + consumes = MediaType.MULTIPART_FORM_DATA_VALUE, + value = "/pdf/ua", + resourceWeight = ResourceWeight.LARGE_WEIGHT) + @ToolIO(produces = ToolFormat.PDF) + @Operation( + summary = "Convert a PDF to PDF/UA-1 or PDF/UA-2", + description = + "Tags the document, marks decorative content as artifacts, embeds fonts and" + + " applies the document-level requirements of PDF/UA, then validates" + + " the result. A conformance declaration is written only if validation" + + " passes, so the returned file never claims more than it delivers.") + public ResponseEntity pdfToPdfUa(@ModelAttribute PdfToPdfUaRequest request) + throws IOException { + + MultipartFile input = request.getFileInput(); + if (input == null || input.isEmpty()) { + throw ExceptionUtils.createPdfFileRequiredException(); + } + + String originalName = Filenames.toSimpleFileName(input.getOriginalFilename()); + String stem = stripExtension(originalName == null ? "document" : originalName); + PdfUaProfile profile = PdfUaProfile.fromRequest(request.getProfile()); + + TaggingOptions options = + TaggingOptions.builder() + .profile(profile) + .title(request.getTitle()) + .fallbackTitle(stem) + // Only used when the document declares no language of its own. + .language( + request.getLanguage() == null || request.getLanguage().isBlank() + ? "en-GB" + : request.getLanguage()) + .overrideLanguage( + request.getOverrideLanguage() != null + && request.getOverrideLanguage()) + .existingTags(existingTags(request.getExistingTags())) + .figurePolicy(figurePolicy(request.getFigurePolicy())) + .embedFonts(request.getEmbedFonts() == null || request.getEmbedFonts()) + .altTextByFigure(parseAltText(request.getAltText())) + .build(); + + PdfUaConversionOutcome outcome = conversionService.convert(input.getBytes(), options); + + log.info( + "Converted '{}' to {}: declared={}, {} remaining failure(s)", + originalName, + profile.displayName(), + outcome.declared(), + outcome.validation().totalFailures()); + + outcome.warnings().forEach(warning -> log.info("PDF/UA warning: {}", warning)); + + // Streamed from a temp file so a large conversion does not hold a second heap copy. + String suffix = outcome.declared() ? "_pdfua" + profile.part() : "_tagged"; + TempFile tempOut = tempFileManager.createManagedTempFile(".pdf"); + try { + Files.write(tempOut.getPath(), outcome.pdfBytes()); + } catch (IOException e) { + tempOut.close(); + throw e; + } + ResponseEntity response = + WebResponseUtils.pdfFileToWebResponse(tempOut, stem + suffix + ".pdf"); + + return ResponseEntity.status(response.getStatusCode()) + .headers(response.getHeaders()) + .header(HEADER_DECLARED, String.valueOf(outcome.declared())) + .header(HEADER_FAILURES, String.valueOf(outcome.validation().totalFailures())) + .header( + HEADER_ALT_NEEDED, + String.valueOf(outcome.tagging().figuresNeedingAltText())) + // Count only: warning text is multi-line prose, which HTTP headers mangle. + .header(HEADER_WARNINGS, String.valueOf(outcome.warnings().size())) + .body(response.getBody()); + } + + /** + * Parses newline-separated {@code key=description} pairs, keyed as the report hands them out. + * Only the first "=" splits, since a description may contain one. + */ + public static Map parseAltText(String raw) { + if (raw == null || raw.isBlank()) { + return Map.of(); + } + Map parsed = new LinkedHashMap<>(); + for (String line : NEWLINE.split(raw)) { + int split = line.indexOf('='); + if (split <= 0) { + continue; + } + String key = line.substring(0, split).strip(); + String description = line.substring(split + 1).strip(); + if (!key.isEmpty() && !description.isEmpty()) { + parsed.put(key, description); + } + } + return parsed; + } + + private static TaggingOptions.ExistingTags existingTags(String value) { + if (value == null) { + return TaggingOptions.ExistingTags.AUTO; + } + return switch (value.trim().toLowerCase()) { + case "keep" -> TaggingOptions.ExistingTags.KEEP; + case "rebuild" -> TaggingOptions.ExistingTags.REBUILD; + default -> TaggingOptions.ExistingTags.AUTO; + }; + } + + private static TaggingOptions.FigurePolicy figurePolicy(String value) { + if (value != null && value.trim().equalsIgnoreCase("mark-decorative")) { + return TaggingOptions.FigurePolicy.MARK_DECORATIVE; + } + return TaggingOptions.FigurePolicy.REQUIRE_ALT; + } + + private static String stripExtension(String filename) { + int dot = filename.lastIndexOf('.'); + return dot > 0 ? filename.substring(0, dot) : filename; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/security/AccessibilityReportController.java b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/security/AccessibilityReportController.java new file mode 100644 index 0000000000..043734dad8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/controller/api/security/AccessibilityReportController.java @@ -0,0 +1,67 @@ +package stirling.software.proprietary.controller.api.security; + +import java.io.IOException; + +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ModelAttribute; +import org.springframework.web.multipart.MultipartFile; + +import io.swagger.v3.oas.annotations.Operation; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.annotations.AutoJobPostMapping; +import stirling.software.common.annotations.api.SecurityApi; +import stirling.software.common.enumeration.ResourceWeight; +import stirling.software.common.model.tool.ToolFormat; +import stirling.software.common.model.tool.ToolIO; +import stirling.software.common.util.ExceptionUtils; +import stirling.software.proprietary.model.api.ua.AccessibilityReport; +import stirling.software.proprietary.model.api.ua.AccessibilityReportRequest; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.service.ua.AccessibilityAuditService; + +/** Reports how accessible a document is, without modifying it. */ +@SecurityApi +@RequiredArgsConstructor +@Slf4j +public class AccessibilityReportController { + + private final AccessibilityAuditService auditService; + + @ToolIO(produces = ToolFormat.JSON) + @Operation( + summary = "Report a document's accessibility standing", + description = + "Validates the document against PDF/UA and reports what fails, which failures" + + " can be fixed automatically, and which checks still need a person." + + " Does not modify the file.") + // Costs a full veraPDF pass plus the converter's own layout analysis over every page. + @AutoJobPostMapping( + value = "/accessibility-report", + consumes = MediaType.MULTIPART_FORM_DATA_VALUE, + resourceWeight = ResourceWeight.LARGE_WEIGHT) + public ResponseEntity report( + @ModelAttribute AccessibilityReportRequest request) { + + MultipartFile file = request.getFileInput(); + if (file == null || file.isEmpty()) { + throw ExceptionUtils.createPdfFileRequiredException(); + } + PdfUaProfile profile = PdfUaProfile.fromRequest(request.getProfile()); + try { + AccessibilityReport report = auditService.audit(file.getBytes(), profile); + log.info( + "Accessibility report for '{}': tagged={}, {} issue(s)", + file.getOriginalFilename(), + report.isTagged(), + report.getIssues().size()); + return ResponseEntity.ok(report); + } catch (IOException e) { + throw ExceptionUtils.createRuntimeException( + "error.ioException", "Could not read the PDF: {0}", e, e.getMessage()); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/converters/PdfToPdfUaRequest.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/converters/PdfToPdfUaRequest.java new file mode 100644 index 0000000000..8981398001 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/converters/PdfToPdfUaRequest.java @@ -0,0 +1,73 @@ +package stirling.software.proprietary.model.api.converters; + +import io.swagger.v3.oas.annotations.media.Schema; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import stirling.software.common.model.api.PDFFile; + +@Data +@EqualsAndHashCode(callSuper = true) +public class PdfToPdfUaRequest extends PDFFile { + + @Schema( + description = "PDF/UA conformance level to target", + defaultValue = "ua1", + allowableValues = {"ua1", "ua2"}) + private String profile; + + @Schema( + description = + "Document title, required by PDF/UA. Falls back to the first heading, then the" + + " filename.") + private String title; + + @Schema( + description = + "Document language as a BCP-47 tag, for example en-GB. Applied only when the" + + " document does not already declare one, unless overrideLanguage is" + + " set.", + defaultValue = "en-GB") + private String language; + + @Schema( + description = + "Replace the language the document already declares. Off by default, so a" + + " document is never relabelled into a language it is not written in.", + defaultValue = "false") + private Boolean overrideLanguage; + + @Schema( + description = + "What to do with an existing structure tree: keep it, rebuild it, or decide" + + " automatically", + defaultValue = "auto", + allowableValues = {"auto", "keep", "rebuild"}) + private String existingTags; + + @Schema( + description = + "How to treat images with no description. require-alt leaves them undescribed so" + + " the report asks for input; mark-decorative treats every image as" + + " decoration.", + defaultValue = "require-alt", + allowableValues = {"require-alt", "mark-decorative"}) + private String figurePolicy; + + @Schema( + description = + "Embed fonts the document references but does not carry. Required for" + + " conformance and needs Ghostscript.", + defaultValue = "true") + private Boolean embedFonts; + + @Schema( + description = + "Alternative descriptions for figures, as key=text pairs separated by newlines." + + " Keys come from the accessibility-report endpoint's" + + " figuresNeedingDescription list, for example \"0:12=Bar chart of" + + " quarterly revenue\". Descriptions are never invented, so without" + + " these an illustrated document cannot claim conformance.") + private String altText; +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/AccessibilityIssue.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/AccessibilityIssue.java new file mode 100644 index 0000000000..0c2a86ad99 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/AccessibilityIssue.java @@ -0,0 +1,38 @@ +package stirling.software.proprietary.model.api.ua; + +import io.swagger.v3.oas.annotations.media.Schema; + +import lombok.Data; + +/** One accessibility problem, grouped across all of its occurrences. */ +@Data +@Schema(description = "A single accessibility issue found in a document") +public class AccessibilityIssue { + + @Schema(description = "ISO 14289 clause, e.g. 7.3") + private String clause; + + @Schema(description = "Test number within the clause") + private String testNumber; + + @Schema(description = "Plain-English description of the problem") + private String message; + + @Schema(description = "The validator's own wording, for support and debugging") + private String technicalMessage; + + @Schema(description = "error or warning") + private String severity = "error"; + + @Schema(description = "Standard the check came from, e.g. PDF/UA-1") + private String specification; + + @Schema(description = "Where the problem was found, when the validator reports it") + private String location; + + @Schema(description = "How many times this issue occurs") + private int occurrences; + + @Schema(description = "True when the converter can fix this without human input") + private boolean autoFixable; +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/AccessibilityReport.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/AccessibilityReport.java new file mode 100644 index 0000000000..bc810635d1 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/AccessibilityReport.java @@ -0,0 +1,63 @@ +package stirling.software.proprietary.model.api.ua; + +import java.util.List; + +import io.swagger.v3.oas.annotations.media.Schema; + +import lombok.Data; + +/** + * A document's accessibility standing. The machine/human split is load-bearing: veraPDF covers only + * about half of the Matterhorn Protocol, so a clean automated pass is not "accessible". + */ +@Data +@Schema(description = "Accessibility standing of a document") +public class AccessibilityReport { + + @Schema(description = "Profile the document was checked against, e.g. PDF/UA-1") + private String profile; + + @Schema(description = "Whether the document has a structure tree at all") + private boolean tagged; + + @Schema(description = "Whether the document declares PDF/UA conformance in its metadata") + private boolean declaresConformance; + + @Schema(description = "Whether every automated check passed") + private boolean passesAutomatedChecks; + + @Schema(description = "Automated checks that failed, grouped by rule") + private List issues = List.of(); + + @Schema(description = "Things a person still has to verify; automation cannot decide these") + private List humanChecks = List.of(); + + @Schema(description = "How many of the failing checks the converter can fix on its own") + private int automaticallyFixable; + + @Schema(description = "How many need information from the user, such as alternative text") + private int needsInput; + + @Schema( + description = + "Figures that need an alternative description. Each carries the key to pass" + + " back in the conversion request's altTextByFigure map, so a caller" + + " can enumerate what is missing and then supply it.") + private List figuresNeedingDescription = List.of(); + + @Schema(description = "Document-level facts that drive most failures") + private Summary summary = new Summary(); + + @Data + @Schema(description = "Quick document-level facts") + public static class Summary { + private int pages; + private boolean hasTitle; + private boolean displaysDocTitle; + private boolean hasLanguage; + private boolean allFontsEmbedded; + private int unembeddedFonts; + private int figures; + private boolean encrypted; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/AccessibilityReportRequest.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/AccessibilityReportRequest.java new file mode 100644 index 0000000000..178d2637b2 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/AccessibilityReportRequest.java @@ -0,0 +1,19 @@ +package stirling.software.proprietary.model.api.ua; + +import io.swagger.v3.oas.annotations.media.Schema; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import stirling.software.common.model.api.PDFFile; + +@Data +@EqualsAndHashCode(callSuper = true) +public class AccessibilityReportRequest extends PDFFile { + + @Schema( + description = "Profile to check against", + defaultValue = "ua1", + allowableValues = {"ua1", "ua2"}) + private String profile; +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/FigureDescriptor.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/FigureDescriptor.java new file mode 100644 index 0000000000..1c960d87e7 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/FigureDescriptor.java @@ -0,0 +1,18 @@ +package stirling.software.proprietary.model.api.ua; + +import io.swagger.v3.oas.annotations.media.Schema; + +/** + * One figure needing an alternative description, which is never invented. key is the + * altTextByFigure key "pageIndex:ordinal"; page is 1-based; kind is "figure" or "formula". + */ +@Schema(description = "A figure that needs an alternative description") +public record FigureDescriptor( + String key, + int page, + String kind, + float x, + float y, + float width, + float height, + String existingAlt) {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/PdfUaConversionOutcome.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/PdfUaConversionOutcome.java new file mode 100644 index 0000000000..dcef97c916 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/PdfUaConversionOutcome.java @@ -0,0 +1,26 @@ +package stirling.software.proprietary.model.api.ua; + +import java.util.List; + +import io.swagger.v3.oas.annotations.media.Schema; + +/** + * Result of a PDF/UA conversion. + * + * @param declared whether a {@code pdfuaid} conformance claim was written into {@code pdfBytes} + */ +@Schema(description = "Result of converting a document to PDF/UA") +public record PdfUaConversionOutcome( + byte[] pdfBytes, + boolean declared, + UaValidationResult validation, + TaggingSummary tagging, + List warnings) { + + @Schema(description = "What the tagging pass produced") + public record TaggingSummary( + boolean rebuiltStructure, + int taggedElements, + int artifacts, + int figuresNeedingAltText) {} +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/UaValidationResult.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/UaValidationResult.java new file mode 100644 index 0000000000..5e494c35be --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ua/UaValidationResult.java @@ -0,0 +1,18 @@ +package stirling.software.proprietary.model.api.ua; + +import java.util.List; + +import io.swagger.v3.oas.annotations.media.Schema; + +/** + * Outcome of validating against one PDF/UA profile. compliant means every automated check passed, + * which is not the same as usable by assistive technology; totalFailures is ungrouped. + */ +@Schema(description = "Result of validating a document against a PDF/UA profile") +public record UaValidationResult( + String profile, boolean compliant, List issues, int totalFailures) { + + public boolean hasIssues() { + return !issues.isEmpty(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/ArtifactType.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/ArtifactType.java new file mode 100644 index 0000000000..78df1fbc27 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/ArtifactType.java @@ -0,0 +1,23 @@ +package stirling.software.proprietary.pdf.ua; + +/** Artifact subtypes (ISO 32000-1 14.8.2.2). Artifacts are excluded from the structure tree. */ +public enum ArtifactType { + /** Running heads, folios, page numbers. Required by PDF/UA-1 clause 7.8. */ + PAGINATION("Pagination"), + /** Rules, boxes, and other layout ornamentation. */ + LAYOUT("Layout"), + /** Cut marks and colour bars. */ + PAGE("Page"), + /** Background graphics with no informational content. */ + BACKGROUND("Background"); + + private final String subtype; + + ArtifactType(String subtype) { + this.subtype = subtype; + } + + public String subtype() { + return subtype; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/BBox.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/BBox.java new file mode 100644 index 0000000000..f2fbf97438 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/BBox.java @@ -0,0 +1,48 @@ +package stirling.software.proprietary.pdf.ua; + +/** An axis-aligned rectangle in PDF user space, with y increasing upwards. */ +public record BBox(float x0, float y0, float x1, float y1) { + + public static final BBox EMPTY = new BBox(0, 0, 0, 0); + + public static BBox of(float x, float y, float width, float height) { + return new BBox(x, y, x + width, y + height); + } + + public float width() { + return x1 - x0; + } + + public float height() { + return y1 - y0; + } + + public float centreX() { + return (x0 + x1) / 2f; + } + + public BBox union(BBox other) { + if (other == null || other.isEmpty()) { + return this; + } + if (isEmpty()) { + return other; + } + return new BBox( + Math.min(x0, other.x0), + Math.min(y0, other.y0), + Math.max(x1, other.x1), + Math.max(y1, other.y1)); + } + + public boolean isEmpty() { + return x1 <= x0 || y1 <= y0; + } + + /** Horizontal overlap with another box as a fraction of the narrower box's width. */ + public float horizontalOverlap(BBox other) { + float overlap = Math.min(x1, other.x1) - Math.max(x0, other.x0); + float narrower = Math.min(width(), other.width()); + return narrower <= 0 ? 0 : Math.max(0, overlap) / narrower; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/DocumentStructure.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/DocumentStructure.java new file mode 100644 index 0000000000..5922895453 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/DocumentStructure.java @@ -0,0 +1,84 @@ +package stirling.software.proprietary.pdf.ua; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import lombok.Getter; +import lombok.Setter; + +/** The derived logical structure of a document, ready for serialisation into a structure tree. */ +@Getter +@Setter +public class DocumentStructure { + + /** Top-level blocks in document reading order. */ + private final List blocks = new ArrayList<>(); + + /** Warnings raised during analysis, surfaced in the conversion report. */ + private final List warnings = new ArrayList<>(); + + private String title; + private String language; + + /** True when real text was wrapped as artifacts, which blocks any conformance claim. */ + private boolean textSuppressed; + + /** Body text size used as the baseline for heading detection, in points. */ + private float bodyFontSize; + + public void add(StructBlock block) { + blocks.add(block); + } + + public void warn(String message) { + if (!warnings.contains(message)) { + warnings.add(message); + } + } + + public void visit(Consumer visitor) { + blocks.forEach(block -> block.visit(visitor)); + } + + public int count(StructType type) { + int[] total = {0}; + visit( + block -> { + if (block.getType() == type) { + total[0]++; + } + }); + return total[0]; + } + + public int artifactCount() { + int[] total = {0}; + visit( + block -> { + if (block.isArtifact()) { + total[0]++; + } + }); + return total[0]; + } + + /** Figures with no alternative description, the most common PDF/UA failure. */ + public List figuresWithoutAlt() { + List missing = new ArrayList<>(); + visit( + block -> { + if ((block.getType() == StructType.FIGURE + || block.getType() == StructType.FORMULA) + && (block.getAlt() == null || block.getAlt().isBlank()) + && (block.getActualText() == null || block.getActualText().isBlank())) { + missing.add(block); + } + }); + return missing; + } + + public boolean isEmpty() { + return blocks.isEmpty(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/LayoutAnalyzer.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/LayoutAnalyzer.java new file mode 100644 index 0000000000..11c510d6ea --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/LayoutAnalyzer.java @@ -0,0 +1,831 @@ +package stirling.software.proprietary.pdf.ua; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import lombok.extern.slf4j.Slf4j; + +/** + * Derives a logical structure from extracted lines and graphics, reusing {@code HeadingDetector}'s + * heuristics. Degrades to paragraphs rather than guessing, since a wrong tag misleads readers. + */ +@Slf4j +public class LayoutAnalyzer { + + private static final Pattern BULLET = Pattern.compile("^[•‣◦⁃∙·▪●■o\\-\\*\\+]\\s+.*"); + private static final Pattern ORDERED = + Pattern.compile("^(\\d{1,3}|[a-zA-Z]|[ivxlcIVXLC]{1,5})[\\.\\)]\\s+.*"); + private static final Pattern PAGE_NUMBER = + Pattern.compile( + "^(page\\s+)?\\d{1,4}(\\s*(of|/)\\s*\\d{1,4})?$", Pattern.CASE_INSENSITIVE); + private static final Pattern DIGITS = Pattern.compile("\\d+"); + + /** Fraction of page height treated as the running head / foot band. */ + private static final float MARGIN_BAND = 0.10f; + + /** A line must exceed the body size by this ratio before it can be a heading. */ + private static final float HEADING_RATIO = 1.10f; + + /** Sizes within this many points are treated as the same heading tier. */ + private static final float TIER_TOLERANCE = 0.4f; + + private static final int MAX_HEADING_WORDS = 12; + + /** Word gap beyond this multiple of the font size separates table cells. */ + private static final float CELL_GAP_RATIO = 1.2f; + + /** Images smaller than this in either dimension are decoration, not content. */ + private static final float MIN_FIGURE_SIZE = 12f; + + /** A size used by more than this share of lines is body text, however large the median says. */ + private static final float MAX_HEADING_LINE_SHARE = 0.2f; + + /** Consecutive lines sharing a size are a text block; headings appear alone. */ + private static final int MAX_HEADING_RUN = 3; + + /** A vector thinner than this in either dimension is a rule or border, not a drawing. */ + private static final float MIN_VECTOR_THICKNESS = 3f; + + /** Vector clusters smaller than this are ornament; larger ones are probably a chart. */ + private static final float MIN_VECTOR_FIGURE_SIZE = 40f; + + /** A drawing is built from several strokes; one big rectangle is a panel, not a chart. */ + private static final int MIN_VECTOR_FIGURE_OPS = 4; + + /** More text than this inside the region means shading behind content, not a drawing. */ + private static final int MAX_LINES_INSIDE_FIGURE = 2; + + public DocumentStructure analyse(List pages) { + DocumentStructure structure = new DocumentStructure(); + float bodySize = bodyFontSize(pages); + structure.setBodyFontSize(bodySize); + Map tiers = headingTiers(pages, bodySize); + Map> artifactLines = repeatedMarginLines(pages, bodySize); + + for (PageContent page : pages) { + analysePage( + page, + structure, + bodySize, + tiers, + artifactLines.getOrDefault(page.pageIndex(), List.of())); + } + + List suppressedPages = + pages.stream() + .filter(PageContent::linesDropped) + .map(PageContent::pageIndex) + .toList(); + if (!suppressedPages.isEmpty()) { + structure.setTextSuppressed(true); + structure.warn( + "Text on page(s) " + + suppressedPages.stream() + .map(i -> String.valueOf(i + 1)) + .collect(Collectors.joining(", ")) + + " could not be tagged reliably and was marked as artifacts. The" + + " converter will not claim conformance while real text is hidden" + + " from assistive technology."); + } + + normaliseHeadingLevels(structure); + structure.setTitle(deriveTitle(structure)); + return structure; + } + + // --- Document-wide statistics ----------------------------------------- + + /** Character-weighted median line size, which is far more stable than a plain median. */ + static float bodyFontSize(List pages) { + Map weights = new HashMap<>(); + for (PageContent page : pages) { + for (TextLineInfo line : page.lines()) { + if (line.dominantFontSize() > 0 && !line.isBlank()) { + weights.merge(line.dominantFontSize(), line.charCount(), Integer::sum); + } + } + } + if (weights.isEmpty()) { + return 0f; + } + int total = weights.values().stream().mapToInt(Integer::intValue).sum(); + List> sorted = + weights.entrySet().stream().sorted(Map.Entry.comparingByKey()).toList(); + int seen = 0; + for (Map.Entry entry : sorted) { + seen += entry.getValue(); + if (seen >= total / 2) { + return entry.getKey(); + } + } + return sorted.get(sorted.size() - 1).getKey(); + } + + /** Maps each distinct heading size to a 1-based level, largest size first. */ + static Map headingTiers(List pages, float bodySize) { + if (bodySize <= 0) { + return Map.of(); + } + // A size used by a large share of the lines is body text, whatever the median says. + Map lineCounts = new HashMap<>(); + int totalLines = 0; + for (PageContent page : pages) { + for (TextLineInfo line : page.lines()) { + if (!line.isBlank()) { + lineCounts.merge(line.dominantFontSize(), 1, Integer::sum); + totalLines++; + } + } + } + int headingLineCeiling = Math.max(1, (int) (totalLines * MAX_HEADING_LINE_SHARE)); + + // Headings do not cluster; a run of same-size lines is a text block, not headings. + Map longestRun = new HashMap<>(); + for (PageContent page : pages) { + Float runSize = null; + int runLength = 0; + for (TextLineInfo line : page.lines()) { + if (line.isBlank()) { + continue; + } + float size = line.dominantFontSize(); + if (runSize != null && Float.compare(size, runSize) == 0) { + runLength++; + } else { + runSize = size; + runLength = 1; + } + int seen = longestRun.getOrDefault(size, 0); + if (runLength > seen) { + longestRun.put(size, runLength); + } + } + } + + List sizes = new ArrayList<>(); + for (PageContent page : pages) { + for (TextLineInfo line : page.lines()) { + if (isHeadingCandidate(line) + && line.dominantFontSize() > bodySize * HEADING_RATIO + && lineCounts.getOrDefault(line.dominantFontSize(), 0) <= headingLineCeiling + && longestRun.getOrDefault(line.dominantFontSize(), 0) < MAX_HEADING_RUN) { + sizes.add(line.dominantFontSize()); + } + } + } + List distinct = sizes.stream().distinct().sorted(Comparator.reverseOrder()).toList(); + + Map tiers = new LinkedHashMap<>(); + int level = 0; + Float previous = null; + for (Float size : distinct) { + if (previous == null || previous - size > TIER_TOLERANCE) { + level = Math.min(level + 1, 6); + previous = size; + } + tiers.put(size, level); + } + return tiers; + } + + /** + * Claims a line's operators word run by word run; claiming the whole ordinal interval would + * swallow anything drawn between them, an image included. + */ + private static void claimLine(StructBlock block, TextLineInfo line) { + // Sort by ordinal, not position: merging out-of-order runs silently drops them to + // /Artifact, hiding them from assistive technology while the file still validates. + List words = + line.words().stream() + .filter(w -> !w.isBlank()) + .sorted(Comparator.comparingInt(WordInfo::startOrdinal)) + .toList(); + if (words.isEmpty()) { + block.addRange(line.startOrdinal(), line.endOrdinal()); + return; + } + int start = words.get(0).startOrdinal(); + int end = words.get(0).endOrdinal(); + for (int i = 1; i < words.size(); i++) { + WordInfo word = words.get(i); + if (word.startOrdinal() <= end + 1) { + end = Math.max(end, word.endOrdinal()); + } else { + block.addRange(start, end); + start = word.startOrdinal(); + end = word.endOrdinal(); + } + } + block.addRange(start, end); + } + + static boolean isHeadingCandidate(TextLineInfo line) { + String text = line.text().strip(); + if (text.isEmpty() || line.wordCount() > MAX_HEADING_WORDS) { + return false; + } + char last = text.charAt(text.length() - 1); + return last != '.' && last != '!' && last != '?'; + } + + /** + * Finds lines in the head/foot bands whose text repeats across pages. Digits are masked first + * so that "Page 4" and "Page 5" count as the same running foot. + */ + static Map> repeatedMarginLines(List pages) { + return repeatedMarginLines(pages, bodyFontSize(pages)); + } + + static Map> repeatedMarginLines( + List pages, float bodySize) { + Map> result = new HashMap<>(); + if (pages.isEmpty()) { + return result; + } + Map counts = new HashMap<>(); + Map> candidates = new HashMap<>(); + + for (PageContent page : pages) { + float height = page.mediaBox().height(); + if (height <= 0) { + continue; + } + float topEdge = page.mediaBox().y1() - height * MARGIN_BAND; + float bottomEdge = page.mediaBox().y0() + height * MARGIN_BAND; + List inBand = new ArrayList<>(); + for (TextLineInfo line : page.lines()) { + if (line.bbox().y0() >= topEdge || line.bbox().y1() <= bottomEdge) { + inBand.add(line); + counts.merge(mask(line.text()), 1, Integer::sum); + } + } + candidates.put(page.pageIndex(), inBand); + } + + int threshold = Math.max(2, pages.size() / 2); + for (Map.Entry> entry : candidates.entrySet()) { + List artifacts = new ArrayList<>(); + for (TextLineInfo line : entry.getValue()) { + boolean repeats = + pages.size() >= 3 && counts.getOrDefault(mask(line.text()), 0) >= threshold; + boolean pageNumber = PAGE_NUMBER.matcher(line.text().strip()).matches(); + // Masked digits merge "Section 1" and "Section 2"; size is the tie-break that stops + // a real heading being demoted, as running heads are never larger than body text. + boolean looksLikeChrome = + bodySize <= 0 || line.dominantFontSize() <= bodySize * 1.05f; + if (pageNumber || (repeats && looksLikeChrome)) { + artifacts.add(line); + } + } + result.put(entry.getKey(), artifacts); + } + return result; + } + + private static String mask(String text) { + return DIGITS.matcher(text.strip().toLowerCase()).replaceAll("#").replaceAll("\\s+", " "); + } + + // --- Per-page analysis ------------------------------------------------- + + private void analysePage( + PageContent page, + DocumentStructure structure, + float bodySize, + Map tiers, + List marginArtifacts) { + + for (TextLineInfo line : marginArtifacts) { + StructBlock artifact = StructBlock.artifact(ArtifactType.PAGINATION, page.pageIndex()); + claimLine(artifact, line); + artifact.setBbox(line.bbox()); + artifact.setText(line.text()); + structure.add(artifact); + } + + // Identity set, not List.contains: TextLineInfo is a record whose equals walks its word + // list, so a linear scan per line is quadratic with a deep comparison inside it. + java.util.Set marginSet = Collections.newSetFromMap(new IdentityHashMap<>()); + marginSet.addAll(marginArtifacts); + List body = + page.lines().stream() + .filter(line -> !line.isBlank() && !marginSet.contains(line)) + .sorted(readingOrder(page)) + .toList(); + + List blocks = new ArrayList<>(); + int index = 0; + while (index < body.size()) { + TextLineInfo line = body.get(index); + + int tableEnd = tableRunEnd(body, index); + if (tableEnd > index) { + StructBlock table = buildTable(body.subList(index, tableEnd + 1), page.pageIndex()); + if (table != null) { + blocks.add(table); + index = tableEnd + 1; + continue; + } + } + + int listEnd = listRunEnd(body, index); + if (listEnd > index) { + blocks.add(buildList(body.subList(index, listEnd + 1), page.pageIndex())); + index = listEnd + 1; + continue; + } + + Integer level = headingLevel(line, tiers); + if (level != null) { + StructBlock heading = new StructBlock(StructType.heading(level), page.pageIndex()); + claimLine(heading, line); + heading.setBbox(line.bbox()); + heading.setText(line.text()); + blocks.add(heading); + index++; + continue; + } + + int paragraphEnd = paragraphRunEnd(body, index, tiers, bodySize); + blocks.add(buildParagraph(body.subList(index, paragraphEnd + 1), page.pageIndex())); + index = paragraphEnd + 1; + } + + // Form XObject text is attributed to its Do, so a Figure too would double-claim it. + Set claimed = new HashSet<>(); + for (StructBlock block : blocks) { + block.visit( + node -> + node.getRanges() + .forEach( + range -> { + for (int i = range.start(); i <= range.end(); i++) { + claimed.add(i); + } + })); + } + blocks.addAll(buildGraphics(page, structure, claimed)); + blocks.forEach(structure::add); + } + + /** + * Orders lines top-to-bottom, splitting into columns first when the page is clearly + * multi-column. Without this, a two-column page reads as interleaved half-sentences. + */ + private Comparator readingOrder(PageContent page) { + Float gutter = detectGutter(page); + if (gutter == null) { + return Comparator.comparingDouble((TextLineInfo l) -> -l.bbox().y1()) + .thenComparingDouble(l -> l.bbox().x0()); + } + return Comparator.comparingInt((TextLineInfo l) -> l.bbox().centreX() < gutter ? 0 : 1) + .thenComparingDouble(l -> -l.bbox().y1()) + .thenComparingDouble(l -> l.bbox().x0()); + } + + /** + * Returns the x of a vertical gutter when the page is two-column, else null. A gutter must sit + * near the middle, be crossed by almost no line, and have substantial text on both sides. + */ + static Float detectGutter(PageContent page) { + List lines = page.lines().stream().filter(line -> !line.isBlank()).toList(); + if (lines.size() < 8) { + return null; + } + float pageWidth = page.mediaBox().width(); + if (pageWidth <= 0) { + return null; + } + float centre = page.mediaBox().x0() + pageWidth / 2f; + long crossing = + lines.stream() + .filter( + line -> + line.bbox().x0() < centre - 5 + && line.bbox().x1() > centre + 5) + .count(); + if (crossing > lines.size() * 0.1) { + return null; + } + long left = lines.stream().filter(line -> line.bbox().centreX() < centre).count(); + long right = lines.size() - left; + boolean balanced = left > lines.size() * 0.25 && right > lines.size() * 0.25; + return balanced ? centre : null; + } + + private static Integer headingLevel(TextLineInfo line, Map tiers) { + if (!isHeadingCandidate(line)) { + return null; + } + return tiers.get(line.dominantFontSize()); + } + + // --- Paragraphs -------------------------------------------------------- + + private static int paragraphRunEnd( + List lines, int start, Map tiers, float bodySize) { + int end = start; + for (int i = start + 1; i < lines.size(); i++) { + TextLineInfo previous = lines.get(i - 1); + TextLineInfo current = lines.get(i); + if (headingLevel(current, tiers) != null || startsListItem(current)) { + break; + } + float gap = previous.bbox().y0() - current.bbox().y1(); + float leading = Math.max(bodySize, current.bbox().height()); + boolean sameBlock = gap < leading * 0.8f && gap > -leading; + boolean sentenceEnded = endsSentence(previous.text()); + if (!sameBlock || (sentenceEnded && gap > leading * 0.4f)) { + break; + } + end = i; + } + return end; + } + + private static boolean endsSentence(String text) { + String stripped = text.strip(); + if (stripped.isEmpty()) { + return false; + } + char last = stripped.charAt(stripped.length() - 1); + return last == '.' || last == '!' || last == '?'; + } + + private static StructBlock buildParagraph(List lines, int pageIndex) { + StructBlock paragraph = new StructBlock(StructType.P, pageIndex); + BBox box = BBox.EMPTY; + StringBuilder text = new StringBuilder(); + for (TextLineInfo line : lines) { + claimLine(paragraph, line); + box = box.union(line.bbox()); + if (text.length() > 0) { + text.append(' '); + } + text.append(line.text().strip()); + } + paragraph.setBbox(box); + paragraph.setText(text.toString()); + return paragraph; + } + + // --- Lists ------------------------------------------------------------- + + static boolean startsListItem(TextLineInfo line) { + String text = line.text().strip(); + return BULLET.matcher(text).matches() || ORDERED.matcher(text).matches(); + } + + private static int listRunEnd(List lines, int start) { + if (!startsListItem(lines.get(start))) { + return start; + } + float indent = lines.get(start).bbox().x0(); + int end = start; + for (int i = start + 1; i < lines.size(); i++) { + TextLineInfo line = lines.get(i); + boolean isItem = startsListItem(line) && Math.abs(line.bbox().x0() - indent) < 6f; + boolean isContinuation = !startsListItem(line) && line.bbox().x0() > indent + 2f; + if (!isItem && !isContinuation) { + break; + } + end = i; + } + // A single marker is a stray character, not a list. + long items = + lines.subList(start, end + 1).stream() + .filter(LayoutAnalyzer::startsListItem) + .count(); + return items >= 2 ? end : start; + } + + private static StructBlock buildList(List lines, int pageIndex) { + StructBlock list = new StructBlock(StructType.L, pageIndex); + list.setListNumbering(listNumbering(lines.get(0))); + BBox box = BBox.EMPTY; + StructBlock currentBody = null; + + for (TextLineInfo line : lines) { + box = box.union(line.bbox()); + if (startsListItem(line) || currentBody == null) { + StructBlock item = new StructBlock(StructType.LI, pageIndex); + StructBlock body = new StructBlock(StructType.LBODY, pageIndex); + claimLine(body, line); + body.setBbox(line.bbox()); + body.setText(line.text()); + item.addChild(body); + item.setBbox(line.bbox()); + list.addChild(item); + currentBody = body; + } else { + claimLine(currentBody, line); + currentBody.setBbox(currentBody.getBbox().union(line.bbox())); + currentBody.setText(currentBody.getText() + " " + line.text().strip()); + } + } + list.setBbox(box); + return list; + } + + private static String listNumbering(TextLineInfo first) { + String text = first.text().strip(); + if (BULLET.matcher(text).matches()) { + return "Disc"; + } + char c = text.charAt(0); + if (Character.isDigit(c)) { + return "Decimal"; + } + if ("ivxlc".indexOf(Character.toLowerCase(c)) >= 0 && text.length() > 1) { + return Character.isUpperCase(c) ? "UpperRoman" : "LowerRoman"; + } + return Character.isUpperCase(c) ? "UpperAlpha" : "LowerAlpha"; + } + + // --- Tables ------------------------------------------------------------ + + /** Splits a line into cells wherever the gap between words exceeds the cell threshold. */ + static List> splitCells(TextLineInfo line) { + List words = line.words().stream().filter(w -> !w.isBlank()).toList(); + List> cells = new ArrayList<>(); + if (words.isEmpty()) { + return cells; + } + float threshold = Math.max(line.dominantFontSize(), 1f) * CELL_GAP_RATIO; + List current = new ArrayList<>(); + current.add(words.get(0)); + for (int i = 1; i < words.size(); i++) { + float gap = words.get(i).bbox().x0() - words.get(i - 1).bbox().x1(); + if (gap > threshold) { + cells.add(List.copyOf(current)); + current = new ArrayList<>(); + } + current.add(words.get(i)); + } + cells.add(List.copyOf(current)); + return cells; + } + + /** + * Index of the last line of a table run starting at {@code start}, or {@code start} if none. + */ + private static int tableRunEnd(List lines, int start) { + int end = start; + for (int i = start; i < lines.size(); i++) { + if (splitCells(lines.get(i)).size() < 2) { + break; + } + end = i; + } + return end > start ? end : start; + } + + /** + * Builds a Table when the run really looks tabular and each cell owns its own operators. + * Returns null when it does not, so the caller falls back to paragraphs. + */ + private static StructBlock buildTable(List rows, int pageIndex) { + if (rows.size() < 2) { + return null; + } + List>> grid = new ArrayList<>(); + for (TextLineInfo row : rows) { + if (!row.wordsAreSeparable()) { + log.debug("Table row shares operators between cells; falling back to paragraphs"); + return null; + } + grid.add(splitCells(row)); + } + int columns = grid.get(0).size(); + long consistent = grid.stream().filter(row -> row.size() == columns).count(); + if (columns < 2 || consistent < Math.max(2, grid.size() * 0.6)) { + return null; + } + + boolean headerRow = looksLikeHeader(rows, grid); + StructBlock table = new StructBlock(StructType.TABLE, pageIndex); + BBox box = BBox.EMPTY; + + for (int r = 0; r < grid.size(); r++) { + List> cells = grid.get(r); + if (cells.size() != columns) { + continue; + } + StructBlock tr = new StructBlock(StructType.TR, pageIndex); + boolean isHeader = headerRow && r == 0; + for (List cell : cells) { + StructBlock td = + new StructBlock(isHeader ? StructType.TH : StructType.TD, pageIndex); + if (isHeader) { + td.setScope("Column"); + } + BBox cellBox = BBox.EMPTY; + StringBuilder text = new StringBuilder(); + int from = cell.get(0).startOrdinal(); + int to = cell.get(cell.size() - 1).endOrdinal(); + for (WordInfo word : cell) { + cellBox = cellBox.union(word.bbox()); + if (text.length() > 0) { + text.append(' '); + } + text.append(word.text()); + } + td.addRange(from, to); + td.setBbox(cellBox); + td.setText(text.toString()); + tr.addChild(td); + box = box.union(cellBox); + } + tr.setBbox(box); + table.addChild(tr); + } + table.setBbox(box); + if (table.getChildren().size() < 2) { + return null; + } + // Clause 7.5 needs equal cell counts per row; a ragged table fails validation outright. + long distinctWidths = + table.getChildren().stream() + .map(row -> row.getChildren().size()) + .distinct() + .count(); + if (distinctWidths != 1) { + log.debug("Discarding a table whose rows have different cell counts"); + return null; + } + return table; + } + + /** The first row is a header when it is bold, or when only later rows carry numbers. */ + private static boolean looksLikeHeader( + List rows, List>> grid) { + if (rows.get(0).bold()) { + return true; + } + boolean firstHasDigits = DIGITS.matcher(rows.get(0).text()).find(); + boolean laterHasDigits = + rows.subList(1, rows.size()).stream() + .anyMatch(row -> DIGITS.matcher(row.text()).find()); + return !firstHasDigits && laterHasDigits; + } + + // --- Graphics ---------------------------------------------------------- + + private List buildGraphics( + PageContent page, DocumentStructure structure, java.util.Set claimed) { + List blocks = new ArrayList<>(); + boolean warnedForms = false; + + // Vectors cluster: a chart is many strokes in one region, a rule is a single thin one. + java.util.Set vectorFigureOrdinals = vectorFigureOrdinals(page, claimed); + + for (MarkableOp op : page.ops()) { + if (op.kind() == MarkableOp.Kind.TEXT || claimed.contains(op.ordinal())) { + continue; + } + BBox box = op.bbox(); + + if (op.kind() == MarkableOp.Kind.VECTOR) { + StructBlock block; + if (vectorFigureOrdinals.contains(op.ordinal())) { + block = new StructBlock(StructType.FIGURE, page.pageIndex()); + } else { + block = StructBlock.artifact(ArtifactType.LAYOUT, page.pageIndex()); + } + block.addRange(op.ordinal(), op.ordinal()); + block.setBbox(box); + blocks.add(block); + continue; + } + + boolean decorative = box.width() < MIN_FIGURE_SIZE || box.height() < MIN_FIGURE_SIZE; + if (decorative) { + StructBlock artifact = StructBlock.artifact(ArtifactType.LAYOUT, page.pageIndex()); + artifact.addRange(op.ordinal(), op.ordinal()); + artifact.setBbox(box); + blocks.add(artifact); + continue; + } + + if (op.kind() == MarkableOp.Kind.FORM && !warnedForms) { + structure.warn( + "Content inside form XObjects was tagged as a single region because its" + + " text is not separately addressable; review those areas."); + warnedForms = true; + } + + StructBlock figure = new StructBlock(StructType.FIGURE, page.pageIndex()); + figure.addRange(op.ordinal(), op.ordinal()); + figure.setBbox(box); + blocks.add(figure); + } + return blocks; + } + + /** + * Finds vector operators belonging to a substantial drawing rather than page furniture; thin + * paths are rules and table borders, and a short run is ornament. + */ + private static Set vectorFigureOrdinals( + PageContent page, java.util.Set claimed) { + // A chart's plot area is mostly empty, while shading sits behind the text it decorates. + Set result = new HashSet<>(); + List run = new ArrayList<>(); + BBox extent = BBox.EMPTY; + + for (MarkableOp op : page.ops()) { + boolean substantial = + op.kind() == MarkableOp.Kind.VECTOR + && !claimed.contains(op.ordinal()) + && !op.bbox().isEmpty() + && op.bbox().width() >= MIN_VECTOR_THICKNESS + && op.bbox().height() >= MIN_VECTOR_THICKNESS; + if (substantial) { + run.add(op); + extent = extent.isEmpty() ? op.bbox() : extent.union(op.bbox()); + continue; + } + flushVectorRun(run, extent, page.lines(), result); + run = new ArrayList<>(); + extent = BBox.EMPTY; + } + flushVectorRun(run, extent, page.lines(), result); + return result; + } + + private static void flushVectorRun( + List run, + BBox extent, + List lines, + java.util.Set result) { + if (run.size() < MIN_VECTOR_FIGURE_OPS + || extent.width() < MIN_VECTOR_FIGURE_SIZE + || extent.height() < MIN_VECTOR_FIGURE_SIZE) { + return; + } + if (overlappingLines(extent, lines) > MAX_LINES_INSIDE_FIGURE) { + return; + } + run.forEach(op -> result.add(op.ordinal())); + } + + /** How many text lines sit within the region a vector cluster covers. */ + private static int overlappingLines(BBox extent, List lines) { + int count = 0; + for (TextLineInfo line : lines) { + BBox box = line.bbox(); + boolean inside = + box.x0() >= extent.x0() - 2 + && box.x1() <= extent.x1() + 2 + && box.y0() >= extent.y0() - 2 + && box.y1() <= extent.y1() + 2; + if (inside) { + count++; + } + } + return count; + } + + // --- Post-processing --------------------------------------------------- + + /** + * Rewrites heading levels so no level is skipped, which PDF/UA-1 clause 7.4 requires. A + * document that jumps H1 to H3 is remapped to H1, H2 while preserving relative depth. + */ + static void normaliseHeadingLevels(DocumentStructure structure) { + List headings = new ArrayList<>(); + structure.visit( + block -> { + if (block.getType().isHeading()) { + headings.add(block); + } + }); + int previous = 0; + for (StructBlock heading : headings) { + int level = heading.getType().headingLevel(); + int adjusted = level > previous + 1 ? previous + 1 : level; + heading.setType(StructType.heading(adjusted)); + previous = adjusted; + } + } + + /** Uses the first top-level heading as the title when the document has no metadata title. */ + private static String deriveTitle(DocumentStructure structure) { + for (StructBlock block : structure.getBlocks()) { + if (block.getType().isHeading() && !block.getText().isBlank()) { + return block.getText().strip(); + } + } + return null; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/MarkableOp.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/MarkableOp.java new file mode 100644 index 0000000000..c48319260f --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/MarkableOp.java @@ -0,0 +1,44 @@ +package stirling.software.proprietary.pdf.ua; + +/** + * One operator in a page content stream that may be wrapped in a marked-content sequence. The + * ordinal counts only markable operators, joining text extraction to token rewriting. + */ +public record MarkableOp(int ordinal, Kind kind, BBox bbox, String resourceName) { + + public enum Kind { + /** Tj, TJ, ' or " */ + TEXT, + /** Do referencing an image XObject */ + IMAGE, + /** Do referencing a form XObject */ + FORM, + /** BI ... ID ... EI */ + INLINE_IMAGE, + /** A path-painting or shading operator: rules, borders, fills, logos */ + VECTOR; + + public boolean isGraphic() { + return this == IMAGE || this == INLINE_IMAGE; + } + } + + /** + * Operator names counted as markable; both passes must agree on this set. Path painting is + * included because clause 7.1 needs visible rules and borders tagged or artifacted. + */ + public static boolean isMarkableOperator(String name) { + return switch (name) { + case "Tj", "TJ", "'", "\"", "Do", "BI" -> true; + default -> isPathPainting(name); + }; + } + + /** Painting operators only: {@code n} ends a path without marking the page. */ + public static boolean isPathPainting(String name) { + return switch (name) { + case "S", "s", "f", "F", "f*", "B", "B*", "b", "b*", "sh" -> true; + default -> false; + }; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/MarkedContentInjector.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/MarkedContentInjector.java new file mode 100644 index 0000000000..c9d6330a60 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/MarkedContentInjector.java @@ -0,0 +1,284 @@ +package stirling.software.proprietary.pdf.ua; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.pdfbox.contentstream.operator.Operator; +import org.apache.pdfbox.cos.COSBase; +import org.apache.pdfbox.cos.COSDictionary; +import org.apache.pdfbox.cos.COSInteger; +import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdfparser.PDFStreamParser; +import org.apache.pdfbox.pdfwriter.ContentStreamWriter; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.common.PDStream; + +import lombok.extern.slf4j.Slf4j; + +/** + * Rewrites a page stream so every markable operator sits inside a marked-content sequence: claimed + * content gets an MCID, everything else /Artifact, satisfying PDF/UA-1 clause 7.1 by construction. + */ +@Slf4j +public class MarkedContentInjector { + + private static final COSName ARTIFACT = COSName.getPDFName("Artifact"); + private static final COSName MCID = COSName.getPDFName("MCID"); + private static final COSName ACTUAL_TEXT = COSName.getPDFName("ActualText"); + private static final COSName ALT = COSName.getPDFName("Alt"); + + /** Operators that force an open sequence to close so nesting stays legal. */ + private static boolean isBoundary(String name) { + return "BT".equals(name) || "ET".equals(name) || "q".equals(name) || "Q".equals(name); + } + + private static boolean isMarkedContentOperator(String name) { + return "BDC".equals(name) || "BMC".equals(name) || "EMC".equals(name); + } + + /** + * Path-construction operators; ISO 32000-1 forbids marked content inside a path object, so a + * sequence wrapping a fill or stroke must open before the path starts. + */ + private static boolean isPathConstruction(String name) { + return switch (name) { + case "m", "l", "c", "v", "y", "h", "re" -> true; + default -> false; + }; + } + + private static boolean opensMarkedContent(String name) { + return "BDC".equals(name) || "BMC".equals(name); + } + + /** + * True for an optional-content sequence; stripping an {@code /OC} wrapper would make hidden + * layers such as watermarks or redaction overlays visible. + */ + private static boolean isOptionalContent(String name, List operands) { + return opensMarkedContent(name) + && !operands.isEmpty() + && operands.get(0) instanceof COSName tag + && "OC".equals(tag.getName()); + } + + /** + * True when a sequence supplies replacement text for its glyphs; dropping it leaves a screen + * reader with the font's own mapping, which for a ligature says nothing useful. + */ + private static boolean carriesReplacementText(String name, List operands) { + if (!opensMarkedContent(name)) { + return false; + } + for (COSBase operand : operands) { + if (operand instanceof COSDictionary properties + && (properties.containsKey(ACTUAL_TEXT) + || properties.containsKey(ALT) + || properties.containsKey(COSName.E))) { + return true; + } + } + return false; + } + + /** The source's own ids mean nothing once the tree is rebuilt, so they are dropped. */ + private static void stripStaleMcid(List operands) { + for (COSBase operand : operands) { + if (operand instanceof COSDictionary properties) { + properties.removeItem(MCID); + } + } + } + + /** Wraps every markable operator on the page; returns the next unused marked content id. */ + public int inject( + PDDocument document, + PDPage page, + List blocks, + int nextMcid, + boolean stripExisting) + throws IOException { + + Map owners = ownersByOrdinal(blocks); + List tokens = parse(page); + List output = new ArrayList<>(tokens.size() + owners.size() * 4); + + List operands = new ArrayList<>(); + // Tracks, for each surviving source sequence, whether its closer should be kept. + Deque keptSequences = new ArrayDeque<>(); + StructBlock openBlock = null; + boolean open = false; + int ordinal = -1; + int mcid = nextMcid; + int pathStart = -1; + + for (Object token : tokens) { + if (!(token instanceof Operator operator)) { + operands.add((COSBase) token); + continue; + } + String name = operator.getName(); + + if (stripExisting && isMarkedContentOperator(name)) { + boolean keep; + if (opensMarkedContent(name)) { + keep = + isOptionalContent(name, operands) + || carriesReplacementText(name, operands); + if (keep) { + stripStaleMcid(operands); + } + keptSequences.push(keep); + } else { + // A closer is kept exactly when its matching opener was. + keep = !keptSequences.isEmpty() && keptSequences.pop(); + } + if (!keep) { + operands.clear(); + continue; + } + // Close our own sequence first so the two never interleave illegally. + if (open) { + output.add(Operator.getOperator("EMC")); + open = false; + openBlock = null; + } + output.addAll(operands); + output.add(operator); + operands.clear(); + continue; + } + + if (isBoundary(name) && open) { + output.add(Operator.getOperator("EMC")); + open = false; + openBlock = null; + } + + // Remember where the current path object began so a sequence wrapping its painting + // operator can be opened before it rather than inside it. + if (isPathConstruction(name)) { + if (pathStart < 0) { + pathStart = output.size(); + } + } else if (!MarkableOp.isPathPainting(name) && !"n".equals(name)) { + pathStart = -1; + } + + if (MarkableOp.isMarkableOperator(name)) { + ordinal++; + StructBlock owner = owners.get(ordinal); + if (!open || owner != openBlock) { + boolean insidePath = MarkableOp.isPathPainting(name) && pathStart >= 0; + if (open) { + // Close before the path began, so the EMC also stays outside the path. + output.add( + insidePath ? pathStart : output.size(), + Operator.getOperator("EMC")); + if (insidePath) { + pathStart++; + } + } + int at = insidePath ? pathStart : output.size(); + mcid = openSequenceAt(output, at, owner, mcid); + open = true; + openBlock = owner; + } + } + + output.addAll(operands); + output.add(operator); + operands.clear(); + + if (MarkableOp.isPathPainting(name) || "n".equals(name)) { + pathStart = -1; + } + } + + if (open) { + output.add(Operator.getOperator("EMC")); + } + + write(document, page, output); + return mcid; + } + + /** Emits the opening BDC/BMC at a given position and records the id on the owning block. */ + private int openSequenceAt(List output, int at, StructBlock owner, int mcid) { + List opening = new ArrayList<>(3); + if (owner == null) { + opening.add(ARTIFACT); + opening.add(Operator.getOperator("BMC")); + } else if (owner.isArtifact()) { + COSDictionary properties = new COSDictionary(); + if (owner.getArtifactType() != null) { + properties.setName(COSName.TYPE, owner.getArtifactType().subtype()); + } + opening.add(ARTIFACT); + opening.add(properties); + opening.add(Operator.getOperator("BDC")); + } else { + COSDictionary properties = new COSDictionary(); + properties.setItem(MCID, COSInteger.get(mcid)); + opening.add(COSName.getPDFName(owner.getType().tag())); + opening.add(properties); + opening.add(Operator.getOperator("BDC")); + owner.getMcids().add(mcid); + mcid++; + } + output.addAll(at, opening); + return mcid; + } + + /** + * Maps each claimed ordinal to its block. Overlapping claims are dropped rather than merged: + * two structure elements sharing content would make the reading order ambiguous. + */ + static Map ownersByOrdinal(List blocks) { + Map owners = new HashMap<>(); + for (StructBlock block : blocks) { + block.visit( + node -> { + for (StructBlock.OrdinalRange range : node.getRanges()) { + for (int i = range.start(); i <= range.end(); i++) { + StructBlock existing = owners.putIfAbsent(i, node); + if (existing != null && existing != node) { + log.debug( + "Ordinal {} claimed by both {} and {}; keeping the first", + i, + existing, + node); + } + } + } + }); + } + return owners; + } + + private static List parse(PDPage page) throws IOException { + PDFStreamParser parser = new PDFStreamParser(page); + List tokens = new ArrayList<>(); + Object token; + while ((token = parser.parseNextToken()) != null) { + tokens.add(token); + } + return tokens; + } + + private static void write(PDDocument document, PDPage page, List tokens) + throws IOException { + PDStream stream = new PDStream(document); + try (OutputStream out = stream.createOutputStream(COSName.FLATE_DECODE)) { + new ContentStreamWriter(out).writeTokens(tokens); + } + page.setContents(stream); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PageContent.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PageContent.java new file mode 100644 index 0000000000..6c74212621 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PageContent.java @@ -0,0 +1,33 @@ +package stirling.software.proprietary.pdf.ua; + +import java.util.List; + +/** + * Everything the layout analyser needs about one page. carriesTextSemantics: existing marked + * content has ActualText/Alt/expansion a rebuild would discard. linesDropped: text became + * artifacts. + */ +public record PageContent( + int pageIndex, + List lines, + List ops, + int markableCount, + boolean preExistingMarkedContent, + boolean carriesTextSemantics, + boolean linesDropped, + BBox mediaBox) { + + public boolean hasText() { + return lines.stream().anyMatch(line -> !line.isBlank()); + } + + /** Markable operators that draw graphics rather than text. */ + public List graphics() { + return ops.stream().filter(op -> op.kind().isGraphic()).toList(); + } + + /** Form XObject invocations, which are tagged as a unit because their text is opaque here. */ + public List forms() { + return ops.stream().filter(op -> op.kind() == MarkableOp.Kind.FORM).toList(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaIdentificationSchema.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaIdentificationSchema.java new file mode 100644 index 0000000000..d37c2569c9 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaIdentificationSchema.java @@ -0,0 +1,47 @@ +package stirling.software.proprietary.pdf.ua; + +import org.apache.xmpbox.XMPMetadata; +import org.apache.xmpbox.schema.XMPSchema; +import org.apache.xmpbox.type.IntegerType; +import org.apache.xmpbox.type.StructuredType; + +/** + * The {@code pdfuaid} XMP conformance schema, which XMPBox does not ship. Only write it once + * validation has passed - it is a compliance claim. + */ +@StructuredType( + preferedPrefix = PdfUaIdentificationSchema.PREFERRED_PREFIX, + namespace = PdfUaIdentificationSchema.NAMESPACE) +public class PdfUaIdentificationSchema extends XMPSchema { + + public static final String PREFERRED_PREFIX = "pdfuaid"; + public static final String NAMESPACE = "http://www.aiim.org/pdfua/ns/id/"; + + public static final String PART = "part"; + public static final String REV = "rev"; + + public PdfUaIdentificationSchema(XMPMetadata metadata) { + super(metadata); + } + + public PdfUaIdentificationSchema(XMPMetadata metadata, String prefix) { + super(metadata, prefix); + } + + /** Sets {@code pdfuaid:part}, the conformance level (1 or 2). */ + public void setPart(int part) { + addProperty(new IntegerType(getMetadata(), getNamespace(), getPrefix(), PART, part)); + } + + /** Sets {@code pdfuaid:rev}, the four-digit revision year used by PDF/UA-2. */ + public void setRevision(int year) { + addProperty(new IntegerType(getMetadata(), getNamespace(), getPrefix(), REV, year)); + } + + public Integer getPart() { + if (getProperty(PART) instanceof IntegerType part) { + return part.getValue(); + } + return null; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaMetadataWriter.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaMetadataWriter.java new file mode 100644 index 0000000000..7f58654c87 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaMetadataWriter.java @@ -0,0 +1,224 @@ +package stirling.software.proprietary.pdf.ua; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDDocumentCatalog; +import org.apache.pdfbox.pdmodel.PDDocumentInformation; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.common.PDMetadata; +import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm; +import org.apache.pdfbox.pdmodel.interactive.form.PDField; +import org.apache.pdfbox.pdmodel.interactive.viewerpreferences.PDViewerPreferences; +import org.apache.xmpbox.XMPMetadata; +import org.apache.xmpbox.schema.DublinCoreSchema; +import org.apache.xmpbox.schema.XMPSchema; +import org.apache.xmpbox.xml.DomXmpParser; +import org.apache.xmpbox.xml.XmpSerializer; + +import lombok.extern.slf4j.Slf4j; + +/** Applies the document-level PDF/UA requirements: title, language, tab order, declaration. */ +@Slf4j +public class PdfUaMetadataWriter { + + private static final COSName TABS = COSName.getPDFName("Tabs"); + private static final COSName SUSPECTS = COSName.getPDFName("Suspects"); + + /** + * Applies everything except the conformance declaration. Clause 7.1 requires a title, so a + * blank one falls back to the existing metadata title. + */ + public List applyDocumentRequirements( + PDDocument document, String title, String language, PdfUaProfile profile) + throws IOException { + return applyDocumentRequirements(document, title, language, profile, false); + } + + public List applyDocumentRequirements( + PDDocument document, + String title, + String language, + PdfUaProfile profile, + boolean preserveVersion) + throws IOException { + + List warnings = new ArrayList<>(); + PDDocumentCatalog catalog = document.getDocumentCatalog(); + + if (language != null && !language.isBlank()) { + catalog.setLanguage(language); + } + + String effectiveTitle = resolveTitle(document, title); + if (effectiveTitle != null) { + PDDocumentInformation info = document.getDocumentInformation(); + info.setTitle(effectiveTitle); + document.setDocumentInformation(info); + } + + // Without this a viewer shows the filename instead of the title, which defeats the point. + PDViewerPreferences preferences = catalog.getViewerPreferences(); + if (preferences == null) { + preferences = new PDViewerPreferences(catalog.getCOSObject()); + } + preferences.setDisplayDocTitle(true); + catalog.setViewerPreferences(preferences); + + // Clause 7.18.1: every page needs an explicit tab order. + for (PDPage page : document.getPages()) { + page.getCOSObject().setName(TABS, "S"); + } + + // A structure tree flagged as suspect is not conforming. + if (catalog.getMarkInfo() != null) { + catalog.getMarkInfo().getCOSObject().removeItem(SUSPECTS); + } + + if (!preserveVersion && document.getVersion() < profile.pdfVersion()) { + document.setVersion(profile.pdfVersion()); + } + + warnings.addAll(describeFormFields(document)); + writeXmp(document, effectiveTitle, language, null); + return warnings; + } + + /** + * Gives every form field the {@code /TU} description clause 7.18.1 requires, reusing its + * authored partial name. Unnamed fields are reported, never given a useless placeholder. + */ + private static List describeFormFields(PDDocument document) { + List warnings = new ArrayList<>(); + PDAcroForm form = document.getDocumentCatalog().getAcroForm(); + if (form == null) { + return warnings; + } + int unnamed = 0; + for (PDField field : form.getFieldTree()) { + String existing = field.getAlternateFieldName(); + if (existing != null && !existing.isBlank()) { + continue; + } + String partialName = field.getPartialName(); + if (partialName == null || partialName.isBlank()) { + unnamed++; + continue; + } + field.setAlternateFieldName(partialName); + } + if (unnamed > 0) { + warnings.add( + unnamed + + " form field(s) have neither a description nor a name, so no tooltip" + + " could be derived. Add one for each before claiming conformance."); + } + return warnings; + } + + /** + * Strips the {@code pdfuaid} declaration when validation fails after it was written, so the + * returned file does not assert conformance it lacks. + */ + public void removeConformanceDeclaration(PDDocument document) throws IOException { + PDDocumentCatalog catalog = document.getDocumentCatalog(); + XMPMetadata metadata = loadOrCreate(catalog); + XMPSchema identification = metadata.getSchema(PdfUaIdentificationSchema.NAMESPACE); + if (identification == null) { + return; + } + metadata.removeSchema(identification); + serialiseInto(document, metadata); + } + + /** Writes the {@code pdfuaid:part} declaration. Only call this after validation has passed. */ + public void declareConformance(PDDocument document, PdfUaProfile profile) throws IOException { + writeXmp(document, resolveTitle(document, null), documentLanguage(document), profile); + } + + private String resolveTitle(PDDocument document, String preferred) { + if (preferred != null && !preferred.isBlank()) { + return preferred.strip(); + } + String existing = document.getDocumentInformation().getTitle(); + return existing != null && !existing.isBlank() ? existing.strip() : null; + } + + private static String documentLanguage(PDDocument document) { + return document.getDocumentCatalog().getLanguage(); + } + + /** + * Rewrites the XMP packet, preserving what was there. A malformed packet is replaced, since an + * unparseable one fails validation on its own. + */ + private void writeXmp(PDDocument document, String title, String language, PdfUaProfile profile) + throws IOException { + + PDDocumentCatalog catalog = document.getDocumentCatalog(); + XMPMetadata metadata = loadOrCreate(catalog); + + if (title != null) { + DublinCoreSchema dublinCore = metadata.getDublinCoreSchema(); + if (dublinCore == null) { + dublinCore = metadata.createAndAddDublinCoreSchema(); + } + dublinCore.setTitle(title); + if (language != null + && !language.isBlank() + && (dublinCore.getLanguages() == null + || !dublinCore.getLanguages().contains(language))) { + dublinCore.addLanguage(language); + } + } + + if (profile != null) { + // Re-converting an already-declared file must not leave two pdfuaid schemas. + XMPSchema stale = metadata.getSchema(PdfUaIdentificationSchema.NAMESPACE); + if (stale != null) { + metadata.removeSchema(stale); + } + PdfUaIdentificationSchema identification = new PdfUaIdentificationSchema(metadata); + identification.setPart(profile.part()); + if (profile.revision() > 0) { + identification.setRevision(profile.revision()); + } + metadata.addSchema(identification); + } + + serialiseInto(document, metadata); + } + + private static void serialiseInto(PDDocument document, XMPMetadata metadata) + throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + new XmpSerializer().serialize(metadata, out, true); + } catch (javax.xml.transform.TransformerException e) { + throw new IOException("Could not serialise XMP metadata", e); + } + PDMetadata pdMetadata = new PDMetadata(document); + pdMetadata.importXMPMetadata(out.toByteArray()); + document.getDocumentCatalog().setMetadata(pdMetadata); + } + + private XMPMetadata loadOrCreate(PDDocumentCatalog catalog) { + PDMetadata existing = catalog.getMetadata(); + if (existing != null) { + try { + DomXmpParser parser = new DomXmpParser(); + // Strict parsing rejects pdfuaid, silently discarding a packet we just wrote. + parser.setStrictParsing(false); + return parser.parse(new ByteArrayInputStream(existing.toByteArray())); + } catch (Exception e) { + log.debug("Replacing unparseable XMP packet: {}", e.getMessage()); + } + } + return XMPMetadata.createXMPMetadata(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaProfile.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaProfile.java new file mode 100644 index 0000000000..9523f5c790 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaProfile.java @@ -0,0 +1,47 @@ +package stirling.software.proprietary.pdf.ua; + +/** The PDF/UA conformance level a conversion targets. */ +public enum PdfUaProfile { + /** ISO 14289-1, layered on PDF 1.7. */ + UA1(1, 1.7f, 0), + /** ISO 14289-2: needs PDF 2.0, namespaced structure types and a revision year. */ + UA2(2, 2.0f, 2024); + + private final int part; + private final float pdfVersion; + private final int revision; + + PdfUaProfile(int part, float pdfVersion, int revision) { + this.part = part; + this.pdfVersion = pdfVersion; + this.revision = revision; + } + + public int part() { + return part; + } + + public float pdfVersion() { + return pdfVersion; + } + + /** The {@code pdfuaid:rev} year, or 0 when the profile does not use one. */ + public int revision() { + return revision; + } + + public String displayName() { + return "PDF/UA-" + part; + } + + public static PdfUaProfile fromRequest(String value) { + if (value == null || value.isBlank()) { + return UA1; + } + String normalised = value.trim().toLowerCase().replace("/", "").replace("-", ""); + return switch (normalised) { + case "ua2", "pdfua2", "2" -> UA2; + default -> UA1; + }; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaTagger.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaTagger.java new file mode 100644 index 0000000000..7f5dcc3739 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/PdfUaTagger.java @@ -0,0 +1,303 @@ +package stirling.software.proprietary.pdf.ua; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDDocumentCatalog; +import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureElement; +import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureTreeRoot; + +import lombok.extern.slf4j.Slf4j; + +/** + * Tags an untagged PDF and applies the document-level PDF/UA requirements. Content must be marked + * before the tree can reference it, and conformance is declared elsewhere, only after validation. + */ +@Slf4j +public class PdfUaTagger { + + private final TaggedContentExtractor extractor = new TaggedContentExtractor(); + private final LayoutAnalyzer analyzer = new LayoutAnalyzer(); + private final MarkedContentInjector injector = new MarkedContentInjector(); + private final PdfUaMetadataWriter metadataWriter = new PdfUaMetadataWriter(); + + public TaggingResult tag(PDDocument document, TaggingOptions options) throws IOException { + boolean alreadyTagged = hasUsableStructureTree(document); + boolean rebuild = + switch (options.getExistingTags()) { + case KEEP -> false; + case REBUILD -> true; + case AUTO -> !alreadyTagged; + }; + + List languageWarnings = new ArrayList<>(); + String language = resolveLanguage(document, options, languageWarnings); + + if (!rebuild) { + log.info("Keeping existing structure tree; applying document requirements only"); + DocumentStructure kept = new DocumentStructure(); + languageWarnings.forEach(kept::warn); + metadataWriter + .applyDocumentRequirements( + document, + options.getTitle(), + language, + options.getProfile(), + options.isPreservePdfVersion()) + .forEach(kept::warn); + return new TaggingResult(kept, false); + } + + // Types the old tree carried, so a rebuild that cannot reproduce them can say so. Font + // embedding may already have deleted the tree, so fall back to what the source had. + Set discardedTypes = + alreadyTagged + ? structureTypes(document) + : options.getSourceFacts().structureTypes(); + + if (alreadyTagged) { + stripStructure(document); + } + + List pages = extractor.extract(document); + DocumentStructure structure = analyzer.analyse(pages); + structure.setLanguage(language); + languageWarnings.forEach(structure::warn); + applyFigurePolicy(structure, options); + + if (structure.isEmpty()) { + structure.warn( + "No taggable content was found; the document may be a scan with no text layer."); + } + + injectMarkedContent(document, structure, pages); + new StructTreeWriter().write(document, structure, options.getProfile()); + // Losing the tree to the embedder is a different problem from a requested rebuild, and + // the advice that helps differs too, so tell them apart. + boolean lostToEmbedder = !alreadyTagged && options.getSourceFacts().hasUsableTree(); + warnAboutFlattenedStructure( + discardedTypes, structureTypes(document), structure, lostToEmbedder); + + String title = resolveTitle(options, structure); + if (title == null) { + structure.warn( + "No document title could be derived. PDF/UA requires one, so supply a title."); + } + metadataWriter + .applyDocumentRequirements( + document, + title, + language, + options.getProfile(), + options.isPreservePdfVersion()) + .forEach(structure::warn); + + return new TaggingResult(structure, true); + } + + /** + * Keeps the language the document already declares. Overwriting it relabels, say, a French file + * as English, and no validator can catch that. + */ + private static String resolveLanguage( + PDDocument document, TaggingOptions options, List warnings) { + String existing = document.getDocumentCatalog().getLanguage(); + if (existing == null || existing.isBlank()) { + // Font embedding discards /Lang, so without this a rewritten French document would + // silently take the caller's default language. + existing = options.getSourceFacts().language(); + } + String requested = options.getLanguage(); + if (existing == null || existing.isBlank() || options.isOverrideLanguage()) { + return requested; + } + if (requested != null && !requested.isBlank() && !requested.equalsIgnoreCase(existing)) { + warnings.add( + "The document already declares its language as '" + + existing + + "', so the requested '" + + requested + + "' was ignored. Ask to override the language to change it."); + } + return existing; + } + + /** Explicit title first, then the first heading, then the caller's fallback. */ + private static String resolveTitle(TaggingOptions options, DocumentStructure structure) { + for (String candidate : + new String[] { + options.getTitle(), structure.getTitle(), options.getFallbackTitle() + }) { + if (candidate != null && !candidate.isBlank()) { + return candidate.strip(); + } + } + return null; + } + + /** Writes the conformance declaration. Separate from tagging so validation can gate it. */ + public void declareConformance(PDDocument document, PdfUaProfile profile) throws IOException { + metadataWriter.declareConformance(document, profile); + } + + /** Withdraws the conformance claim, for a document that turned out not to validate. */ + public void withdrawConformance(PDDocument document) throws IOException { + metadataWriter.removeConformanceDeclaration(document); + } + + /** Wraps content page by page; marked content ids restart on each page. */ + private void injectMarkedContent( + PDDocument document, DocumentStructure structure, List pages) + throws IOException { + Map markableCounts = new LinkedHashMap<>(); + pages.forEach(page -> markableCounts.put(page.pageIndex(), page.markableCount())); + Map> byPage = new LinkedHashMap<>(); + for (StructBlock block : structure.getBlocks()) { + byPage.computeIfAbsent(block.getPageIndex(), k -> new ArrayList<>()).add(block); + } + for (int pageIndex = 0; pageIndex < document.getNumberOfPages(); pageIndex++) { + List blocks = byPage.getOrDefault(pageIndex, List.of()); + // Nothing to wrap, and rewriting costs a parse and recompress for an identical stream. + if (blocks.isEmpty() && markableCounts.getOrDefault(pageIndex, 0) == 0) { + continue; + } + injector.inject(document, document.getPage(pageIndex), blocks, 0, true); + } + } + + /** Applies alt text supplied by the caller, or demotes images to artifacts on request. */ + private static void applyFigurePolicy(DocumentStructure structure, TaggingOptions options) { + int[] suppressed = {0}; + structure.visit( + block -> { + if (block.getType() != StructType.FIGURE) { + return; + } + if (options.getFigurePolicy() == TaggingOptions.FigurePolicy.MARK_DECORATIVE) { + block.setType(StructType.ARTIFACT); + block.setArtifactType(ArtifactType.LAYOUT); + suppressed[0]++; + return; + } + int ordinal = + block.getRanges().isEmpty() ? -1 : block.getRanges().get(0).start(); + String alt = options.altTextFor(block.getPageIndex(), ordinal); + if (alt != null && !alt.isBlank()) { + block.setAlt(alt); + } + }); + // Marking images decorative validates by hiding content, so never report it as clean. + if (suppressed[0] > 0) { + structure.warn( + suppressed[0] + + " image(s) were marked as decoration and are now hidden from" + + " assistive technology. Confirm none of them carried meaning."); + } + int missing = structure.figuresWithoutAlt().size(); + if (missing > 0) { + structure.warn( + missing + + " figure(s) have no alternative description. PDF/UA requires one for" + + " every image that carries meaning."); + } + } + + /** + * A tree is only worth keeping when wired up: kids, a parent tree, and a marked catalog. + * Keeping one that fails any of those leaves the document permanently unfixable. + */ + public static boolean hasUsableStructureTree(PDDocument document) { + PDDocumentCatalog catalog = document.getDocumentCatalog(); + PDStructureTreeRoot root = catalog.getStructureTreeRoot(); + if (root == null) { + return false; + } + try { + boolean hasKids = root.getKids() != null && !root.getKids().isEmpty(); + boolean hasParentTree = root.getParentTree() != null; + boolean marked = catalog.getMarkInfo() != null && catalog.getMarkInfo().isMarked(); + return hasKids && hasParentTree && marked; + } catch (RuntimeException e) { + log.debug("Unreadable structure tree, treating as absent: {}", e.getMessage()); + return false; + } + } + + /** + * A rebuild derives structure from layout, so semantics the old tree carried can vanish - a + * table becomes loose paragraphs. Validators cannot see that loss, so it has to be reported. + */ + private static void warnAboutFlattenedStructure( + Set before, + Set after, + DocumentStructure structure, + boolean lostToEmbedder) { + List lost = + MEANINGFUL_TYPES.stream() + .filter(type -> before.contains(type) && !after.contains(type)) + .toList(); + if (lost.isEmpty()) { + return; + } + // Keeping the tags cannot help once the embedder has deleted them, so do not suggest it. + String remedy = + lostToEmbedder + ? " Embedding the missing fonts rewrote the document and deleted its" + + " original tags. Turn off font embedding to keep them." + : " Keep the existing tags instead to preserve it."; + structure.warn( + "Rebuilding the tags could not reproduce " + + String.join(", ", lost) + + " structure, so that content is now plain paragraphs." + + remedy); + } + + /** Structure whose loss changes what a screen reader conveys, not just how it is nested. */ + private static final List MEANINGFUL_TYPES = + List.of("Table", "TH", "Formula", "L", "LI", "TOC", "Note"); + + private static Set structureTypes(PDDocument document) { + Set types = new HashSet<>(); + try { + PDStructureTreeRoot root = document.getDocumentCatalog().getStructureTreeRoot(); + if (root != null) { + collectTypes(root.getKids(), types, 0); + } + } catch (RuntimeException e) { + log.debug("Could not read structure types: {}", e.getMessage()); + } + return types; + } + + private static void collectTypes(Object node, Set types, int depth) { + // Structure trees can be deep or, in damaged files, cyclic; cap rather than overflow. + if (node == null || depth > 64) { + return; + } + if (node instanceof List list) { + list.forEach(child -> collectTypes(child, types, depth + 1)); + } else if (node instanceof PDStructureElement element) { + types.add(element.getStructureType()); + collectTypes(element.getKids(), types, depth + 1); + } + } + + private static void stripStructure(PDDocument document) { + PDDocumentCatalog catalog = document.getDocumentCatalog(); + catalog.getCOSObject().removeItem(COSName.getPDFName("StructTreeRoot")); + catalog.getCOSObject().removeItem(COSName.getPDFName("MarkInfo")); + document.getPages() + .forEach( + page -> + page.getCOSObject() + .removeItem(COSName.getPDFName("StructParents"))); + log.info("Removed existing structure tree before rebuilding"); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/SourceFacts.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/SourceFacts.java new file mode 100644 index 0000000000..4f058d6c48 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/SourceFacts.java @@ -0,0 +1,59 @@ +package stirling.software.proprietary.pdf.ua; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureElement; +import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureTreeRoot; + +import lombok.extern.slf4j.Slf4j; + +/** + * What the document said about itself before anything rewrote it. Font embedding shells out to + * Ghostscript, which returns a file with no structure tree, no {@code /Lang} and no XMP, so a + * tagger reading the rewritten document sees an untagged, language-less file and cannot tell that + * anything was lost. These facts are captured from the original and carried past that stage. + * + * @param language the catalog {@code /Lang} the author declared, or null + * @param structureTypes every structure element type the original tree contained + * @param hasUsableTree whether the original had a structure tree worth preserving + */ +@Slf4j +public record SourceFacts(String language, Set structureTypes, boolean hasUsableTree) { + + private static final int MAX_DEPTH = 64; + + /** Facts for a document nothing has rewritten, used when font embedding did not run. */ + public static final SourceFacts NONE = new SourceFacts(null, Set.of(), false); + + public static SourceFacts of(PDDocument document) { + String language = null; + Set types = new HashSet<>(); + boolean usable = false; + try { + language = document.getDocumentCatalog().getLanguage(); + usable = PdfUaTagger.hasUsableStructureTree(document); + PDStructureTreeRoot root = document.getDocumentCatalog().getStructureTreeRoot(); + if (root != null) { + collect(root.getKids(), types, 0); + } + } catch (RuntimeException e) { + log.debug("Could not read source facts: {}", e.getMessage()); + } + return new SourceFacts(language, Set.copyOf(types), usable); + } + + private static void collect(Object node, Set types, int depth) { + // Damaged files can present a cyclic tree; cap rather than overflow the stack. + if (node == null || depth > MAX_DEPTH) { + return; + } + if (node instanceof java.util.List list) { + list.forEach(child -> collect(child, types, depth + 1)); + } else if (node instanceof PDStructureElement element) { + types.add(element.getStructureType()); + collect(element.getKids(), types, depth + 1); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/StructBlock.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/StructBlock.java new file mode 100644 index 0000000000..d2f3e452b1 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/StructBlock.java @@ -0,0 +1,134 @@ +package stirling.software.proprietary.pdf.ua; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import lombok.Getter; +import lombok.Setter; + +/** + * One node of the derived logical structure: either page content (ranges of markable operator + * ordinals) or child blocks. Containers with no content are pruned before serialisation. + */ +@Getter +@Setter +public class StructBlock { + + /** A contiguous, inclusive run of markable operator ordinals within one page stream. */ + public record OrdinalRange(int start, int end) { + public boolean contains(int ordinal) { + return ordinal >= start && ordinal <= end; + } + + public int size() { + return end - start + 1; + } + } + + private StructType type; + private ArtifactType artifactType; + private int pageIndex; + private BBox bbox = BBox.EMPTY; + private String text = ""; + + private final List ranges = new ArrayList<>(); + private final List children = new ArrayList<>(); + + /** {@code /Alt} - required on Figure and Formula for PDF/UA. */ + private String alt; + + /** {@code /ActualText} - replacement text for content whose glyphs do not spell the word. */ + private String actualText; + + /** {@code /Lang} - set only where it differs from the document default. */ + private String lang; + + /** {@code /Scope} on a TH: Row, Column or Both. */ + private String scope; + + /** {@code /ListNumbering} on an L. */ + private String listNumbering; + + /** Unique {@code /ID}, required on Note and FENote elements. */ + private String id; + + /** + * Marked content ids assigned during injection; one block yields several when split, since a + * sequence must nest inside BT/ET and q/Q rather than straddle them. + */ + private final List mcids = new ArrayList<>(); + + /** True when the source content was already inside a marked-content sequence. */ + private boolean preMarked; + + public StructBlock(StructType type, int pageIndex) { + this.type = type; + this.pageIndex = pageIndex; + } + + public static StructBlock artifact(ArtifactType artifactType, int pageIndex) { + StructBlock block = new StructBlock(StructType.ARTIFACT, pageIndex); + block.artifactType = artifactType; + return block; + } + + public StructBlock addChild(StructBlock child) { + children.add(child); + return this; + } + + public StructBlock addRange(int start, int end) { + ranges.add(new OrdinalRange(start, end)); + return this; + } + + public boolean isArtifact() { + return type == StructType.ARTIFACT; + } + + /** Depth-first walk over this block and all descendants. */ + public void visit(Consumer visitor) { + visitor.accept(this); + for (StructBlock child : children) { + child.visit(visitor); + } + } + + /** Total number of ordinals owned by this block and its descendants. */ + public int contentCount() { + int total = ranges.stream().mapToInt(OrdinalRange::size).sum(); + for (StructBlock child : children) { + total += child.contentCount(); + } + return total; + } + + /** Concatenated text of this block and its descendants, in tree order. */ + public String collectText() { + StringBuilder sb = new StringBuilder(); + visit( + block -> { + if (!block.text.isBlank()) { + if (sb.length() > 0) { + sb.append(' '); + } + sb.append(block.text.strip()); + } + }); + return sb.toString(); + } + + @Override + public String toString() { + return type.tag() + + (artifactType != null ? "[" + artifactType.subtype() + "]" : "") + + "(p" + + pageIndex + + ", " + + ranges.size() + + " ranges, " + + children.size() + + " kids)"; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/StructTreeWriter.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/StructTreeWriter.java new file mode 100644 index 0000000000..f6b5d89b4c --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/StructTreeWriter.java @@ -0,0 +1,295 @@ +package stirling.software.proprietary.pdf.ua; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.pdfbox.cos.COSArray; +import org.apache.pdfbox.cos.COSBase; +import org.apache.pdfbox.cos.COSDictionary; +import org.apache.pdfbox.cos.COSInteger; +import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.common.PDNumberTreeNode; +import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDMarkInfo; +import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDObjectReference; +import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureElement; +import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureTreeRoot; +import org.apache.pdfbox.pdmodel.documentinterchange.taggedpdf.PDListAttributeObject; +import org.apache.pdfbox.pdmodel.documentinterchange.taggedpdf.PDTableAttributeObject; +import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation; +import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationLink; +import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationWidget; + +import lombok.extern.slf4j.Slf4j; + +/** + * Serialises a {@link DocumentStructure} into a PDF structure tree. Must run after {@link + * MarkedContentInjector}, which assigns the marked content ids this writer references. + */ +@Slf4j +public class StructTreeWriter { + + private static final COSName STRUCT_PARENT = COSName.getPDFName("StructParent"); + private static final COSName NUMS = COSName.getPDFName("Nums"); + private static final String PDF2_STANDARD_NAMESPACE = "http://iso.org/pdf2/ssn"; + + /** Per-page marked content id to owning element, built while walking the tree. */ + private final Map> mcidOwners = new LinkedHashMap<>(); + + private COSDictionary standardNamespace; + private int nextParentKey; + + public void write(PDDocument document, DocumentStructure structure, PdfUaProfile profile) + throws IOException { + PDStructureTreeRoot root = new PDStructureTreeRoot(); + PDStructureElement documentElement = + new PDStructureElement(StructType.DOCUMENT.tag(), root); + if (structure.getLanguage() != null) { + documentElement.setLanguage(structure.getLanguage()); + } + if (profile == PdfUaProfile.UA2) { + applyNamespace(documentElement, document); + } + + for (StructBlock block : structure.getBlocks()) { + if (block.isArtifact()) { + continue; + } + PDStructureElement child = buildElement(document, block, documentElement, profile); + if (child != null) { + documentElement.appendKid(child); + } + } + + root.appendKid(documentElement); + buildParentTree(document, root); + registerNamespaces(root); + + PDMarkInfo markInfo = new PDMarkInfo(); + markInfo.setMarked(true); + document.getDocumentCatalog().setMarkInfo(markInfo); + document.getDocumentCatalog().setStructureTreeRoot(root); + } + + /** Recursively builds an element, returning null when the block carries no content at all. */ + private PDStructureElement buildElement( + PDDocument document, + StructBlock block, + PDStructureElement parent, + PdfUaProfile profile) { + + // Prune on assigned MCIDs, not claimed ranges: form-XObject lines all resolve to one Do, + // and emitting the losers would announce empty paragraphs to a screen reader. + if (!carriesContent(block)) { + return null; + } + StructType type = effectiveType(block, profile); + PDStructureElement element = new PDStructureElement(type.tag(), parent); + PDPage page = document.getPage(block.getPageIndex()); + element.setPage(page); + + if (profile == PdfUaProfile.UA2) { + applyNamespace(element, document); + } + applyAttributes(block, element); + + for (int mcid : block.getMcids()) { + element.appendKid(mcid); + mcidOwners + .computeIfAbsent(block.getPageIndex(), k -> new LinkedHashMap<>()) + .put(mcid, element); + } + + for (StructBlock child : block.getChildren()) { + PDStructureElement childElement = buildElement(document, child, element, profile); + if (childElement != null) { + element.appendKid(childElement); + } + } + return element; + } + + /** True when this block, or something beneath it, was actually given marked content. */ + private static boolean carriesContent(StructBlock block) { + if (!block.getMcids().isEmpty()) { + return true; + } + return block.getChildren().stream().anyMatch(StructTreeWriter::carriesContent); + } + + /** PDF/UA-2 replaces Note with FENote for footnotes. */ + private static StructType effectiveType(StructBlock block, PdfUaProfile profile) { + if (profile == PdfUaProfile.UA2 && block.getType() == StructType.NOTE) { + return StructType.FENOTE; + } + return block.getType(); + } + + private static void applyAttributes(StructBlock block, PDStructureElement element) { + if (block.getAlt() != null && !block.getAlt().isBlank()) { + element.setAlternateDescription(block.getAlt()); + } + if (block.getActualText() != null && !block.getActualText().isBlank()) { + element.setActualText(block.getActualText()); + } + if (block.getLang() != null && !block.getLang().isBlank()) { + element.setLanguage(block.getLang()); + } + if (block.getId() != null && !block.getId().isBlank()) { + element.setElementIdentifier(block.getId()); + } + if (block.getScope() != null) { + PDTableAttributeObject table = new PDTableAttributeObject(); + table.setScope(block.getScope()); + element.addAttribute(table); + } + if (block.getListNumbering() != null) { + PDListAttributeObject list = new PDListAttributeObject(); + list.setListNumbering(block.getListNumbering()); + element.addAttribute(list); + } + } + + /** PDF/UA-2 requires every element to declare the standard structure namespace. */ + private void applyNamespace(PDStructureElement element, PDDocument document) { + element.getCOSObject().setItem(COSName.getPDFName("NS"), standardNamespace()); + } + + /** The PDF 2.0 standard structure namespace, created once per document. */ + private COSDictionary standardNamespace() { + if (standardNamespace == null) { + standardNamespace = new COSDictionary(); + standardNamespace.setName(COSName.TYPE, "Namespace"); + standardNamespace.setString(COSName.getPDFName("NS"), PDF2_STANDARD_NAMESPACE); + } + return standardNamespace; + } + + private void registerNamespaces(PDStructureTreeRoot root) { + if (standardNamespace == null) { + return; + } + COSArray namespaces = new COSArray(); + namespaces.add(standardNamespace); + root.getCOSObject().setItem(COSName.getPDFName("Namespaces"), namespaces); + } + + /** + * Builds {@code /ParentTree}: per page, an array indexed by marked content id keyed on {@code + * /StructParents}, plus one entry per annotation keyed on {@code /StructParent}. + */ + private void buildParentTree(PDDocument document, PDStructureTreeRoot root) { + COSArray nums = new COSArray(); + nextParentKey = 0; + + for (int pageIndex = 0; pageIndex < document.getNumberOfPages(); pageIndex++) { + Map owners = mcidOwners.get(pageIndex); + if (owners == null || owners.isEmpty()) { + continue; + } + PDPage page = document.getPage(pageIndex); + int key = nextParentKey++; + page.setStructParents(key); + + int maxMcid = owners.keySet().stream().mapToInt(Integer::intValue).max().orElse(-1); + COSArray entries = new COSArray(); + for (int mcid = 0; mcid <= maxMcid; mcid++) { + PDStructureElement owner = owners.get(mcid); + entries.add( + owner != null ? owner.getCOSObject() : org.apache.pdfbox.cos.COSNull.NULL); + } + nums.add(COSInteger.get(key)); + nums.add(entries); + } + + List annotationEntries = tagAnnotations(document, root); + for (int i = 0; i + 1 < annotationEntries.size(); i += 2) { + nums.add(annotationEntries.get(i)); + nums.add(annotationEntries.get(i + 1)); + } + + COSDictionary parentTreeDict = new COSDictionary(); + parentTreeDict.setItem(NUMS, nums); + root.setParentTree(new PDNumberTreeNode(parentTreeDict, PDStructureElement.class)); + root.setParentTreeNextKey(nextParentKey); + } + + /** + * Clause 7.18: every visible annotation needs a structure element so it is reachable from the + * tree. Links become Link elements, anything else an Annot. + */ + private List tagAnnotations(PDDocument document, PDStructureTreeRoot root) { + List entries = new ArrayList<>(); + PDStructureElement documentElement = firstDocumentElement(root); + if (documentElement == null) { + return entries; + } + for (int pageIndex = 0; pageIndex < document.getNumberOfPages(); pageIndex++) { + PDPage page = document.getPage(pageIndex); + List annotations; + try { + annotations = page.getAnnotations(); + } catch (IOException e) { + log.debug("Could not read annotations on page {}: {}", pageIndex, e.getMessage()); + continue; + } + for (PDAnnotation annotation : annotations) { + if (annotation == null + || annotation.isHidden() + || annotation.isNoView() + || "Popup".equals(annotation.getSubtype())) { + continue; + } + PDStructureElement element = + new PDStructureElement(annotationType(annotation), documentElement); + element.setPage(page); + + PDObjectReference reference = new PDObjectReference(); + reference.setReferencedObject(annotation); + element.appendKid(reference); + documentElement.appendKid(element); + + int key = nextParentKey++; + annotation.getCOSObject().setInt(STRUCT_PARENT, key); + entries.add(COSInteger.get(key)); + entries.add(element.getCOSObject()); + + if (annotation.getContents() == null || annotation.getContents().isBlank()) { + annotation.setContents(defaultContents(annotation)); + } + } + } + return entries; + } + + /** Clause 7.18.4: widgets need a Form element, links a Link element, everything else Annot. */ + private static String annotationType(PDAnnotation annotation) { + if (annotation instanceof PDAnnotationWidget) { + return StructType.FORM.tag(); + } + if (annotation instanceof PDAnnotationLink) { + return StructType.LINK.tag(); + } + return "Annot"; + } + + private static String defaultContents(PDAnnotation annotation) { + if (annotation instanceof PDAnnotationLink link && link.getAction() != null) { + return "Link"; + } + return annotation.getSubtype() == null ? "Annotation" : annotation.getSubtype(); + } + + private static PDStructureElement firstDocumentElement(PDStructureTreeRoot root) { + for (Object kid : root.getKids()) { + if (kid instanceof PDStructureElement element) { + return element; + } + } + return null; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/StructType.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/StructType.java new file mode 100644 index 0000000000..b4634a58c6 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/StructType.java @@ -0,0 +1,62 @@ +package stirling.software.proprietary.pdf.ua; + +/** + * PDF standard structure types emitted by the tagger (ISO 32000-1 14.8.4), limited to the PDF/UA + * subset. {@link #ARTIFACT} is not one: it marks content in the stream and stays out of the tree. + */ +public enum StructType { + DOCUMENT("Document"), + PART("Part"), + SECT("Sect"), + H1("H1"), + H2("H2"), + H3("H3"), + H4("H4"), + H5("H5"), + H6("H6"), + P("P"), + L("L"), + LI("LI"), + LBL("Lbl"), + LBODY("LBody"), + TABLE("Table"), + TR("TR"), + TH("TH"), + TD("TD"), + FIGURE("Figure"), + CAPTION("Caption"), + FORMULA("Formula"), + NOTE("Note"), + FENOTE("FENote"), + LINK("Link"), + /** Wraps a widget annotation; PDF/UA-1 clause 7.18.4 requires widgets to sit inside one. */ + FORM("Form"), + SPAN("Span"), + ARTIFACT("Artifact"); + + private final String tag; + + StructType(String tag) { + this.tag = tag; + } + + /** The name written into the PDF {@code /S} entry. */ + public String tag() { + return tag; + } + + public boolean isHeading() { + return this == H1 || this == H2 || this == H3 || this == H4 || this == H5 || this == H6; + } + + /** Heading level 1-6, or 0 when this is not a heading. */ + public int headingLevel() { + return isHeading() ? ordinal() - H1.ordinal() + 1 : 0; + } + + /** The heading type for a 1-based level, clamped to the H1-H6 range. */ + public static StructType heading(int level) { + int clamped = Math.max(1, Math.min(6, level)); + return values()[H1.ordinal() + clamped - 1]; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TaggedContentExtractor.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TaggedContentExtractor.java new file mode 100644 index 0000000000..01c86215d4 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TaggedContentExtractor.java @@ -0,0 +1,630 @@ +package stirling.software.proprietary.pdf.ua; + +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.pdfbox.contentstream.operator.Operator; +import org.apache.pdfbox.cos.COSBase; +import org.apache.pdfbox.cos.COSDictionary; +import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdfparser.PDFStreamParser; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDResources; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.apache.pdfbox.pdmodel.font.PDFontDescriptor; +import org.apache.pdfbox.pdmodel.font.PDType3Font; +import org.apache.pdfbox.pdmodel.graphics.PDXObject; +import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject; +import org.apache.pdfbox.pdmodel.graphics.form.PDTransparencyGroup; +import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject; +import org.apache.pdfbox.text.PDFTextStripper; +import org.apache.pdfbox.text.TextPosition; +import org.apache.pdfbox.util.Matrix; +import org.apache.pdfbox.util.Vector; + +import lombok.extern.slf4j.Slf4j; + +/** + * Extracts text lines and graphic ops from page streams, tagging each with its operator ordinal. + * Both passes count the same operators in the same order, so ordinals cross-reference. + */ +@Slf4j +public class TaggedContentExtractor { + + /** Glyph size below which a run is treated as noise rather than a line. */ + private static final float MIN_FONT_SIZE = 0.5f; + + public List extract(PDDocument document) throws IOException { + LineCollector collector = new LineCollector(); + collector.setSortByPosition(true); + collector.setStartPage(1); + collector.setEndPage(document.getNumberOfPages()); + collector.writeText(document, Writer.nullWriter()); + + List pages = new ArrayList<>(document.getNumberOfPages()); + for (int i = 0; i < document.getNumberOfPages(); i++) { + PDPage page = document.getPage(i); + List ops = collector.opsFor(i); + List lines = collector.linesFor(i); + boolean dropped = false; + if (ops.size() < maxOrdinal(lines) + 1) { + // Untrusted ordinals: drop the lines so the page is untaggable rather than + // mis-tagged, and flag it so the caller refuses to declare conformance. + log.warn( + "Ordinal mismatch on page {} (ops={}, text={}); skipping page", + i, + ops.size(), + maxOrdinal(lines) + 1); + dropped = !lines.isEmpty(); + lines = List.of(); + } + pages.add( + new PageContent( + i, + lines, + ops, + ops.size(), + collector.preMarkedOn(i), + collector.textSemanticsOn(i), + dropped, + normalisedBox(page))); + } + return pages; + } + + /** + * The page box in the space of extracted line coordinates: origin-zero, width and height + * swapped for 90/270 rotations, because the text engine reports in the rotated frame. + */ + static BBox normalisedBox(PDPage page) { + PDRectangle mediaBox = page.getMediaBox(); + boolean sideways = page.getRotation() % 180 != 0; + float width = sideways ? mediaBox.getHeight() : mediaBox.getWidth(); + float height = sideways ? mediaBox.getWidth() : mediaBox.getHeight(); + return new BBox(0, 0, width, height); + } + + /** Counts images with the token scan alone, skipping the expensive text pass. */ + public int countGraphics(PDDocument document) { + int total = 0; + for (int i = 0; i < document.getNumberOfPages(); i++) { + try { + PDResources resources = document.getPage(i).getResources(); + PDFStreamParser parser = new PDFStreamParser(document.getPage(i)); + List operands = new ArrayList<>(); + Object token; + while ((token = parser.parseNextToken()) != null) { + if (!(token instanceof Operator operator)) { + operands.add((COSBase) token); + continue; + } + if (isGraphicOperator(operator.getName(), operands, resources)) { + total++; + } + operands.clear(); + } + } catch (IOException e) { + log.debug("Could not scan page {} for graphics: {}", i, e.getMessage()); + } + } + return total; + } + + /** True for an inline image, or a Do that resolves to an image XObject. */ + private static boolean isGraphicOperator( + String name, List operands, PDResources resources) { + if ("BI".equals(name)) { + return true; + } + if (!"Do".equals(name) || resources == null || operands.size() != 1) { + return false; + } + if (!(operands.get(0) instanceof COSName resourceName)) { + return false; + } + try { + return resources.getXObject(resourceName) instanceof PDImageXObject; + } catch (IOException e) { + return false; + } + } + + private static int maxOrdinal(List lines) { + return lines.stream().mapToInt(TextLineInfo::endOrdinal).max().orElse(-1); + } + + static BBox toBBox(PDRectangle rect) { + return new BBox( + rect.getLowerLeftX(), + rect.getLowerLeftY(), + rect.getUpperRightX(), + rect.getUpperRightY()); + } + + // --- Operator classification ------------------------------------------- + + private static boolean isPathConstruction(String name) { + return switch (name) { + case "m", "l", "c", "v", "y", "re" -> true; + default -> false; + }; + } + + /** True when a sequence carries replacement or alternative text, which a rebuild would drop. */ + private static boolean carriesTextSemantics(List operands) { + for (COSBase operand : operands) { + if (operand instanceof COSDictionary dictionary + && (dictionary.containsKey(COSName.getPDFName("ActualText")) + || dictionary.containsKey(COSName.getPDFName("Alt")) + || dictionary.containsKey(COSName.E))) { + return true; + } + } + return false; + } + + /** + * Describes one markable operator, placed with the engine's own matrix rather than a + * hand-rolled q/Q/cm stack that would get nesting and form matrices wrong. + */ + private static MarkableOp classify( + String name, + List operands, + PDResources resources, + Matrix ctm, + BBox pathBox, + int ordinal) { + + if ("BI".equals(name)) { + return new MarkableOp(ordinal, MarkableOp.Kind.INLINE_IMAGE, unitSquare(ctm), null); + } + if (MarkableOp.isPathPainting(name)) { + return new MarkableOp(ordinal, MarkableOp.Kind.VECTOR, pathBox, null); + } + if (!"Do".equals(name)) { + return new MarkableOp(ordinal, MarkableOp.Kind.TEXT, BBox.EMPTY, null); + } + COSName resourceName = + operands.size() == 1 && operands.get(0) instanceof COSName n ? n : null; + if (resourceName == null || resources == null) { + return new MarkableOp(ordinal, MarkableOp.Kind.FORM, unitSquare(ctm), null); + } + try { + PDXObject xobject = resources.getXObject(resourceName); + if (xobject instanceof PDImageXObject) { + return new MarkableOp( + ordinal, MarkableOp.Kind.IMAGE, unitSquare(ctm), resourceName.getName()); + } + if (xobject instanceof PDFormXObject form) { + return new MarkableOp( + ordinal, MarkableOp.Kind.FORM, formBox(form, ctm), resourceName.getName()); + } + } catch (IOException e) { + log.debug("Could not resolve XObject {}: {}", resourceName.getName(), e.getMessage()); + } + return new MarkableOp( + ordinal, MarkableOp.Kind.FORM, unitSquare(ctm), resourceName.getName()); + } + + /** + * Extends the running path box with one path-construction operator's points; without it every + * vector had an empty box and charts and vector logos vanished from the structure tree. + */ + private static BBox extendPath(BBox current, String name, List operands, Matrix ctm) { + int pairs = + switch (name) { + case "m", "l" -> 1; + case "re" -> 2; + case "v", "y" -> 2; + case "c" -> 3; + default -> 0; + }; + if (pairs == 0 || operands.size() < pairs * 2) { + return current; + } + + // Deliberately allocation-free; the obvious version cost a third of the extraction budget. + float minX = current.isEmpty() ? Float.MAX_VALUE : current.x0(); + float minY = current.isEmpty() ? Float.MAX_VALUE : current.y0(); + float maxX = current.isEmpty() ? -Float.MAX_VALUE : current.x1(); + float maxY = current.isEmpty() ? -Float.MAX_VALUE : current.y1(); + + for (int pair = 0; pair < pairs; pair++) { + Float x = numberAt(operands, pair * 2); + Float y = numberAt(operands, pair * 2 + 1); + if (x == null || y == null) { + continue; + } + float px = x; + float py = y; + // "re" gives origin plus size, so the second pair is a corner offset from the first. + if ("re".equals(name) && pair == 1) { + Float ox = numberAt(operands, 0); + Float oy = numberAt(operands, 1); + if (ox == null || oy == null) { + continue; + } + px = ox + x; + py = oy + y; + } + float tx = ctm.getScaleX() * px + ctm.getShearX() * py + ctm.getTranslateX(); + float ty = ctm.getShearY() * px + ctm.getScaleY() * py + ctm.getTranslateY(); + minX = Math.min(minX, tx); + minY = Math.min(minY, ty); + maxX = Math.max(maxX, tx); + maxY = Math.max(maxY, ty); + } + return maxX < minX ? current : new BBox(minX, minY, maxX, maxY); + } + + private static Float numberAt(List operands, int index) { + return index < operands.size() + && operands.get(index) instanceof org.apache.pdfbox.cos.COSNumber number + ? number.floatValue() + : null; + } + + /** The unit square mapped through the CTM, which is how images are placed. */ + private static BBox unitSquare(Matrix ctm) { + return transformBox(new BBox(0, 0, 1, 1), ctm); + } + + private static BBox formBox(PDFormXObject form, Matrix ctm) { + PDRectangle box = form.getBBox(); + if (box == null) { + return unitSquare(ctm); + } + Matrix combined = form.getMatrix() != null ? form.getMatrix().multiply(ctm) : ctm; + return transformBox(toBBox(box), combined); + } + + private static BBox transformBox(BBox box, Matrix m) { + float[] xs = new float[4]; + float[] ys = new float[4]; + float[][] corners = { + {box.x0(), box.y0()}, {box.x1(), box.y0()}, + {box.x0(), box.y1()}, {box.x1(), box.y1()} + }; + for (int i = 0; i < 4; i++) { + Vector v = m.transform(new Vector(corners[i][0], corners[i][1])); + xs[i] = v.getX(); + ys[i] = v.getY(); + } + float minX = Math.min(Math.min(xs[0], xs[1]), Math.min(xs[2], xs[3])); + float maxX = Math.max(Math.max(xs[0], xs[1]), Math.max(xs[2], xs[3])); + float minY = Math.min(Math.min(ys[0], ys[1]), Math.min(ys[2], ys[3])); + float maxY = Math.max(Math.max(ys[0], ys[1]), Math.max(ys[2], ys[3])); + return new BBox(minX, minY, maxX, maxY); + } + + // --- Text pass --------------------------------------------------------- + + /** Marker recorded for each glyph so a finished line knows where it came from. */ + private record GlyphOrigin(int ordinal, boolean marked) {} + + private static final class LineCollector extends PDFTextStripper { + + private final Map> byPage = new HashMap<>(); + private final Map> opsByPage = new HashMap<>(); + private final Map preMarkedByPage = new HashMap<>(); + private final Map textSemanticsByPage = new HashMap<>(); + private final Map origins = new IdentityHashMap<>(); + private final List lineBuffer = new ArrayList<>(); + private final List lineWords = new ArrayList<>(); + private final StringBuilder lineText = new StringBuilder(); + + private int ordinal = -1; + private int markedDepth; + private int nestedDepth; + private BBox pathBox = BBox.EMPTY; + private int syntheticDepth; + private float pageHeight; + private int pageIndex; + + LineCollector() throws IOException { + super(); + } + + List linesFor(int index) { + return byPage.getOrDefault(index, List.of()); + } + + List opsFor(int index) { + return opsByPage.getOrDefault(index, List.of()); + } + + boolean preMarkedOn(int index) { + return preMarkedByPage.getOrDefault(index, false); + } + + boolean textSemanticsOn(int index) { + return textSemanticsByPage.getOrDefault(index, false); + } + + @Override + protected void startPage(PDPage page) throws IOException { + ordinal = -1; + markedDepth = 0; + nestedDepth = 0; + syntheticDepth = 0; + pathBox = BBox.EMPTY; + origins.clear(); + lineBuffer.clear(); + lineWords.clear(); + lineText.setLength(0); + // Dir-adjusted glyph coordinates live in the rotated frame, so the flip must too. + pageHeight = normalisedBox(page).height(); + pageIndex = getCurrentPageNo() - 1; + super.startPage(page); + } + + @Override + protected void endPage(PDPage page) throws IOException { + flushLine(); + super.endPage(page); + } + + /** + * Counts only operators physically present in the page's own stream: PDFBox re-enters here + * with synthetic calls for {@code '} and {@code "}, and descends into form XObjects. + */ + @Override + protected void processOperator(Operator operator, List operands) + throws IOException { + String name = operator.getName(); + if (nestedDepth == 0 && syntheticDepth == 0) { + if (isPathConstruction(name)) { + pathBox = + extendPath( + pathBox, + name, + operands, + getGraphicsState().getCurrentTransformationMatrix()); + } + if (MarkableOp.isMarkableOperator(name)) { + ordinal++; + // Classified here rather than in a second parse of the same stream: the engine + // already has the operands and the live transformation matrix. + opsByPage + .computeIfAbsent(pageIndex, k -> new ArrayList<>()) + .add( + classify( + name, + operands, + getResources(), + getGraphicsState().getCurrentTransformationMatrix(), + pathBox, + ordinal)); + if (MarkableOp.isPathPainting(name)) { + pathBox = BBox.EMPTY; + } + } else if ("n".equals(name)) { + pathBox = BBox.EMPTY; + } else if ("BDC".equals(name) || "BMC".equals(name)) { + markedDepth++; + preMarkedByPage.put(pageIndex, true); + if (carriesTextSemantics(operands)) { + textSemanticsByPage.put(pageIndex, true); + } + } else if ("EMC".equals(name) && markedDepth > 0) { + markedDepth--; + } + } + boolean synthesises = "'".equals(name) || "\"".equals(name); + if (synthesises) { + syntheticDepth++; + } + try { + super.processOperator(operator, operands); + } finally { + if (synthesises) { + syntheticDepth--; + } + } + } + + @Override + public void showForm(PDFormXObject form) throws IOException { + nestedDepth++; + try { + super.showForm(form); + } finally { + nestedDepth--; + } + } + + @Override + public void showTransparencyGroup(PDTransparencyGroup group) throws IOException { + nestedDepth++; + try { + super.showTransparencyGroup(group); + } finally { + nestedDepth--; + } + } + + @Override + protected void showType3Glyph( + Matrix textRenderingMatrix, + PDType3Font font, + int code, + org.apache.pdfbox.util.Vector displacement) + throws IOException { + nestedDepth++; + try { + super.showType3Glyph(textRenderingMatrix, font, code, displacement); + } finally { + nestedDepth--; + } + } + + @Override + protected void processChildStream( + org.apache.pdfbox.contentstream.PDContentStream contentStream, PDPage page) + throws IOException { + nestedDepth++; + try { + super.processChildStream(contentStream, page); + } finally { + nestedDepth--; + } + } + + @Override + protected void processTextPosition(TextPosition text) { + origins.put(text, new GlyphOrigin(ordinal, markedDepth > 0)); + super.processTextPosition(text); + } + + @Override + protected void writeString(String text, List positions) { + lineText.append(text); + lineBuffer.addAll(positions); + WordInfo word = buildWord(text, positions); + if (word != null) { + lineWords.add(word); + } + } + + private WordInfo buildWord(String text, List positions) { + if (text == null || text.isBlank() || positions.isEmpty()) { + return null; + } + Bounds bounds = new Bounds(); + for (TextPosition tp : positions) { + bounds.accept(tp, pageHeight, origins.get(tp)); + } + if (bounds.end < 0) { + return null; + } + return new WordInfo( + text, + bounds.box(), + bounds.start, + bounds.end, + bounds.dominantSize(), + bounds.bold); + } + + @Override + protected void writeWordSeparator() { + lineText.append(' '); + } + + @Override + protected void writeLineSeparator() { + flushLine(); + } + + @Override + protected void writeParagraphSeparator() { + flushLine(); + } + + private void flushLine() { + if (lineBuffer.isEmpty()) { + lineText.setLength(0); + lineWords.clear(); + return; + } + TextLineInfo line = buildLine(); + lineBuffer.clear(); + lineWords.clear(); + lineText.setLength(0); + if (line != null) { + byPage.computeIfAbsent(pageIndex, k -> new ArrayList<>()).add(line); + } + } + + private TextLineInfo buildLine() { + String text = lineText.toString(); + if (text.isBlank()) { + return null; + } + Bounds bounds = new Bounds(); + for (TextPosition tp : lineBuffer) { + bounds.accept(tp, pageHeight, origins.get(tp)); + } + if (bounds.end < 0) { + return null; + } + return new TextLineInfo( + pageIndex, + text, + bounds.box(), + bounds.dominantSize(), + bounds.bold, + bounds.start, + bounds.end, + bounds.marked, + List.copyOf(lineWords)); + } + } + + /** Accumulates glyph geometry, ordinals and font signals for a word or a line. */ + private static final class Bounds { + private float minX = Float.MAX_VALUE; + private float maxX = -Float.MAX_VALUE; + private float minY = Float.MAX_VALUE; + private float maxY = -Float.MAX_VALUE; + private int start = Integer.MAX_VALUE; + private int end = -1; + private boolean marked; + private boolean bold; + private final Map sizeCounts = new HashMap<>(); + + void accept(TextPosition tp, float pageHeight, GlyphOrigin origin) { + float top = pageHeight - tp.getYDirAdj(); + float bottom = top - Math.max(tp.getHeightDir(), 0); + minX = Math.min(minX, tp.getXDirAdj()); + maxX = Math.max(maxX, tp.getXDirAdj() + tp.getWidthDirAdj()); + minY = Math.min(minY, bottom); + maxY = Math.max(maxY, top); + + if (origin != null) { + start = Math.min(start, origin.ordinal()); + end = Math.max(end, origin.ordinal()); + marked |= origin.marked(); + } + float size = tp.getFontSizeInPt(); + if (size > MIN_FONT_SIZE) { + sizeCounts.merge(round(size), 1, Integer::sum); + } + bold |= isBold(tp); + } + + BBox box() { + return new BBox(minX, minY, maxX, maxY); + } + + float dominantSize() { + return sizeCounts.entrySet().stream() + .max(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .orElse(0f); + } + + private static float round(float value) { + return Math.round(value * 10f) / 10f; + } + + private static boolean isBold(TextPosition tp) { + if (tp.getFont() == null) { + return false; + } + String name = tp.getFont().getName(); + if (name != null && name.toLowerCase().contains("bold")) { + return true; + } + PDFontDescriptor descriptor = tp.getFont().getFontDescriptor(); + return descriptor != null + && (descriptor.getFontWeight() >= 600 || descriptor.isForceBold()); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TaggingOptions.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TaggingOptions.java new file mode 100644 index 0000000000..bba15ed09f --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TaggingOptions.java @@ -0,0 +1,63 @@ +package stirling.software.proprietary.pdf.ua; + +import java.util.Map; + +import lombok.Builder; +import lombok.Getter; + +/** Inputs that change how a document is tagged. */ +@Getter +@Builder(toBuilder = true) +public class TaggingOptions { + + /** What to do when the source already has a structure tree. */ + public enum ExistingTags { + /** Leave the tree alone and fix only document-level requirements. */ + KEEP, + /** Discard the tree and derive a new one. */ + REBUILD, + /** Keep a usable tree, rebuild an empty or trivially broken one. */ + AUTO + } + + /** How images with no alternative description are handled. */ + public enum FigurePolicy { + /** Leave undescribed so validation fails honestly; a faked {@code /Alt} helps nobody. */ + REQUIRE_ALT, + /** Treat every image as decoration and mark it as an artifact. */ + MARK_DECORATIVE + } + + @Builder.Default private PdfUaProfile profile = PdfUaProfile.UA1; + + /** BCP-47 language tag for the document, for example {@code en-GB}. */ + private String language; + + /** Replace a language the document already declares. Off, so a French file stays French. */ + @Builder.Default private boolean overrideLanguage = false; + + private String title; + + /** Last resort when no title is given and none can be derived; pass the uploaded filename. */ + private String fallbackTitle; + + /** Embed any font the document references but does not carry, which clause 7.21 requires. */ + @Builder.Default private boolean embedFonts = true; + + /** Leave the PDF version alone; raising it would break PDF/A-1, defined on PDF 1.4. */ + @Builder.Default private boolean preservePdfVersion = false; + + @Builder.Default private ExistingTags existingTags = ExistingTags.AUTO; + + @Builder.Default private FigurePolicy figurePolicy = FigurePolicy.REQUIRE_ALT; + + /** Alternative descriptions supplied by the caller, keyed by "pageIndex:ordinal". */ + @Builder.Default private Map altTextByFigure = Map.of(); + + /** What the document said before font embedding rewrote it; see {@link SourceFacts}. */ + @Builder.Default private SourceFacts sourceFacts = SourceFacts.NONE; + + public String altTextFor(int pageIndex, int ordinal) { + return altTextByFigure.get(pageIndex + ":" + ordinal); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TaggingResult.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TaggingResult.java new file mode 100644 index 0000000000..d54fad5de8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TaggingResult.java @@ -0,0 +1,42 @@ +package stirling.software.proprietary.pdf.ua; + +import java.util.ArrayList; +import java.util.List; + +import lombok.Getter; + +/** What a tagging run produced, for the conversion report. */ +@Getter +public class TaggingResult { + + private final List warnings = new ArrayList<>(); + private final DocumentStructure structure; + private final boolean rebuilt; + private final int taggedElements; + private final int artifacts; + private final int figuresNeedingAlt; + + /** True when text was hidden as artifacts; the caller must not declare conformance. */ + private final boolean contentSuppressed; + + public TaggingResult(DocumentStructure structure, boolean rebuilt) { + this.structure = structure; + this.rebuilt = rebuilt; + this.warnings.addAll(structure.getWarnings()); + int[] elements = {0}; + structure.visit( + block -> { + if (!block.isArtifact()) { + elements[0]++; + } + }); + this.taggedElements = elements[0]; + this.artifacts = structure.artifactCount(); + this.figuresNeedingAlt = structure.figuresWithoutAlt().size(); + this.contentSuppressed = structure.isTextSuppressed(); + } + + public boolean needsHumanReview() { + return figuresNeedingAlt > 0 || !warnings.isEmpty(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TextLineInfo.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TextLineInfo.java new file mode 100644 index 0000000000..67f37d1c8d --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/TextLineInfo.java @@ -0,0 +1,42 @@ +package stirling.software.proprietary.pdf.ua; + +import java.util.List; + +/** + * A run of text on one baseline, with the operator ordinals that produced it. {@code preMarked} + * means the source stream already wrapped this text in BDC/EMC. + */ +public record TextLineInfo( + int pageIndex, + String text, + BBox bbox, + float dominantFontSize, + boolean bold, + int startOrdinal, + int endOrdinal, + boolean preMarked, + List words) { + + public boolean isBlank() { + return text == null || text.isBlank(); + } + + public int charCount() { + return text == null ? 0 : text.strip().length(); + } + + public int wordCount() { + return (int) words.stream().filter(w -> !w.isBlank()).count(); + } + + /** True when every word occupies its own operator run, so cells can be tagged separately. */ + public boolean wordsAreSeparable() { + List real = words.stream().filter(w -> !w.isBlank()).toList(); + for (int i = 1; i < real.size(); i++) { + if (!real.get(i - 1).isSeparableFrom(real.get(i))) { + return false; + } + } + return true; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/WordInfo.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/WordInfo.java new file mode 100644 index 0000000000..515775b6f8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/ua/WordInfo.java @@ -0,0 +1,18 @@ +package stirling.software.proprietary.pdf.ua; + +/** + * A whitespace-delimited run of glyphs, with the operator ordinals that produced it. Cell detection + * needs both: geometry to find cells, ordinals to tell whether they can be tagged separately. + */ +public record WordInfo( + String text, BBox bbox, int startOrdinal, int endOrdinal, float fontSize, boolean bold) { + + public boolean isBlank() { + return text == null || text.isBlank(); + } + + /** True when this word shares no operator with the other, so both can carry their own MCID. */ + public boolean isSeparableFrom(WordInfo other) { + return endOrdinal < other.startOrdinal || other.endOrdinal < startOrdinal; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/AccessibilityAuditService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/AccessibilityAuditService.java new file mode 100644 index 0000000000..869d489a04 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/AccessibilityAuditService.java @@ -0,0 +1,187 @@ +package stirling.software.proprietary.service.ua; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDDocumentCatalog; +import org.apache.pdfbox.pdmodel.interactive.viewerpreferences.PDViewerPreferences; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.util.ExceptionUtils; +import stirling.software.proprietary.model.api.ua.AccessibilityIssue; +import stirling.software.proprietary.model.api.ua.AccessibilityReport; +import stirling.software.proprietary.model.api.ua.FigureDescriptor; +import stirling.software.proprietary.model.api.ua.UaValidationResult; +import stirling.software.proprietary.pdf.ua.BBox; +import stirling.software.proprietary.pdf.ua.DocumentStructure; +import stirling.software.proprietary.pdf.ua.LayoutAnalyzer; +import stirling.software.proprietary.pdf.ua.PageContent; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.pdf.ua.StructBlock; +import stirling.software.proprietary.pdf.ua.StructType; +import stirling.software.proprietary.pdf.ua.TaggedContentExtractor; + +/** Produces an accessibility report without changing the document. */ +@Service +@Slf4j +@RequiredArgsConstructor +public class AccessibilityAuditService { + + /** Checks no validator can make; omitting them implies the work does not exist. */ + private static final List HUMAN_CHECKS = + List.of( + "Is the reading order correct for someone who cannot see the layout?", + "Does each alternative description convey what the image is for, not just what" + + " it looks like?", + "Are headings used for structure rather than for visual emphasis?", + "Is any information conveyed by colour alone also available another way?", + "Do tables have headers that identify the right rows and columns?", + "Is the document language correct, including for quoted passages?", + "Do links describe their destination rather than saying 'click here'?"); + + /** The report walks every page and validates, so it carries the conversion's own caps. */ + private static final long MAX_INPUT_BYTES = 100L * 1024 * 1024; + + private static final int MAX_PAGES = 2000; + + private final PdfUaValidationService validationService; + + public AccessibilityReport audit(byte[] pdfBytes, PdfUaProfile profile) throws IOException { + enforceLimits(pdfBytes); + AccessibilityReport report = new AccessibilityReport(); + report.setProfile(profile.displayName()); + + UaValidationResult validation = validationService.validate(pdfBytes, profile); + report.setIssues(validation.issues()); + report.setPassesAutomatedChecks(validation.compliant()); + report.setHumanChecks(HUMAN_CHECKS); + + int fixable = 0; + int needsInput = 0; + for (AccessibilityIssue issue : validation.issues()) { + if (issue.isAutoFixable()) { + fixable++; + } else { + needsInput++; + } + } + report.setAutomaticallyFixable(fixable); + report.setNeedsInput(needsInput); + + try (PDDocument document = Loader.loadPDF(pdfBytes)) { + populateSummary(document, report); + report.setFiguresNeedingDescription(figuresNeedingDescription(document)); + } catch (IOException e) { + log.debug("Could not inspect document for the summary: {}", e.getMessage()); + } + return report; + } + + /** + * Rejects before the expensive pass. An unreadable file is left to the report itself to say. + */ + private static void enforceLimits(byte[] pdfBytes) { + if (pdfBytes.length > MAX_INPUT_BYTES) { + throw ExceptionUtils.createIllegalArgumentException( + "error.fileTooLarge", + "This PDF is {0} MB. The accessibility report is limited to {1} MB.", + pdfBytes.length / (1024 * 1024), + MAX_INPUT_BYTES / (1024 * 1024)); + } + int pages; + try (PDDocument document = Loader.loadPDF(pdfBytes)) { + pages = document.getNumberOfPages(); + } catch (IOException e) { + return; + } + if (pages > MAX_PAGES) { + throw ExceptionUtils.createIllegalArgumentException( + "error.tooManyPages", + "This PDF has {0} pages. The accessibility report is limited to {1} pages;" + + " split it first.", + pages, + MAX_PAGES); + } + } + + private void populateSummary(PDDocument document, AccessibilityReport report) + throws IOException { + PDDocumentCatalog catalog = document.getDocumentCatalog(); + AccessibilityReport.Summary summary = report.getSummary(); + + report.setTagged(catalog.getStructureTreeRoot() != null); + report.setDeclaresConformance(declaresUa(document)); + + summary.setPages(document.getNumberOfPages()); + summary.setEncrypted(document.isEncrypted()); + summary.setHasLanguage(catalog.getLanguage() != null && !catalog.getLanguage().isBlank()); + + String title = document.getDocumentInformation().getTitle(); + summary.setHasTitle(title != null && !title.isBlank()); + + PDViewerPreferences preferences = catalog.getViewerPreferences(); + summary.setDisplaysDocTitle(preferences != null && preferences.displayDocTitle()); + + Set unembedded = FontEmbeddingService.findUnembeddedFonts(document); + summary.setUnembeddedFonts(unembedded.size()); + summary.setAllFontsEmbedded(unembedded.isEmpty()); + + try { + summary.setFigures(new TaggedContentExtractor().countGraphics(document)); + } catch (Exception e) { + log.debug("Could not count figures: {}", e.getMessage()); + } + } + + /** + * Lists the figures a conversion would leave undescribed, running the converter's own analysis + * because counting raster images would miss vector charts and existing descriptions. + */ + private List figuresNeedingDescription(PDDocument document) { + try { + List pages = new TaggedContentExtractor().extract(document); + DocumentStructure structure = new LayoutAnalyzer().analyse(pages); + List figures = new ArrayList<>(); + for (StructBlock block : structure.figuresWithoutAlt()) { + int ordinal = block.getRanges().isEmpty() ? -1 : block.getRanges().get(0).start(); + BBox box = block.getBbox(); + figures.add( + new FigureDescriptor( + block.getPageIndex() + ":" + ordinal, + block.getPageIndex() + 1, + block.getType() == StructType.FORMULA ? "formula" : "figure", + box.x0(), + box.y0(), + box.width(), + box.height(), + block.getAlt())); + } + return figures; + } catch (Exception e) { + log.debug("Could not enumerate figures: {}", e.getMessage()); + return List.of(); + } + } + + /** True when the XMP packet carries a pdfuaid identifier. */ + private static boolean declaresUa(PDDocument document) { + try { + var metadata = document.getDocumentCatalog().getMetadata(); + if (metadata == null) { + return false; + } + String xmp = + new String(metadata.toByteArray(), java.nio.charset.StandardCharsets.UTF_8); + return xmp.contains("pdfuaid"); + } catch (IOException e) { + return false; + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/FontEmbeddingService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/FontEmbeddingService.java new file mode 100644 index 0000000000..9c38b54f6b --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/FontEmbeddingService.java @@ -0,0 +1,254 @@ +package stirling.software.proprietary.service.ua; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDResources; +import org.apache.pdfbox.pdmodel.font.PDFont; +import org.springframework.stereotype.Service; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.util.ProcessExecutor; +import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult; + +/** + * Embeds any font the document references but does not carry, as PDF/UA-1 clause 7.21 requires. + * Ghostscript does the embedding and discards the structure tree, so this must run before tagging. + */ +@Service +@Slf4j +public class FontEmbeddingService { + + public boolean hasUnembeddedFonts(byte[] pdfBytes) { + try (PDDocument document = Loader.loadPDF(pdfBytes)) { + return !findUnembeddedFonts(document).isEmpty(); + } catch (IOException e) { + log.debug("Could not inspect fonts: {}", e.getMessage()); + return false; + } + } + + public static Set findUnembeddedFonts(PDDocument document) { + Set missing = new HashSet<>(); + for (PDPage page : document.getPages()) { + PDResources resources = page.getResources(); + if (resources == null) { + continue; + } + for (COSName name : resources.getFontNames()) { + try { + PDFont font = resources.getFont(name); + if (font != null && !font.isEmbedded()) { + missing.add(font.getName()); + } + } catch (IOException e) { + log.debug("Could not read font {}: {}", name.getName(), e.getMessage()); + } + } + } + return missing; + } + + /** + * Returns the document with all fonts embedded, or the input unchanged. Never throws: failing + * to embed is a reportable shortfall, not a reason to abandon the conversion. + */ + public Result embedFonts(byte[] pdfBytes) { + Set missing; + try (PDDocument document = Loader.loadPDF(pdfBytes)) { + missing = findUnembeddedFonts(document); + } catch (IOException e) { + return new Result( + pdfBytes, false, Set.of(), "Could not inspect fonts: " + e.getMessage()); + } + if (missing.isEmpty()) { + return new Result(pdfBytes, false, Set.of(), null); + } + if (!isGhostscriptAvailable()) { + return new Result( + pdfBytes, + false, + missing, + "Ghostscript is not installed, so " + + missing.size() + + " unembedded font(s) could not be embedded. PDF/UA requires every font" + + " to be embedded."); + } + + Path workingDir = null; + try { + workingDir = Files.createTempDirectory("pdfua_fonts_"); + Path input = workingDir.resolve("input.pdf"); + Path output = workingDir.resolve("output.pdf"); + Files.write(input, pdfBytes); + + ProcessExecutorResult result = + ProcessExecutor.getInstance(ProcessExecutor.Processes.GHOSTSCRIPT) + .runCommandWithOutputHandling(command(input, output, workingDir)); + + if (result.getRc() != 0 || !Files.exists(output)) { + return new Result( + pdfBytes, + false, + missing, + "Font embedding failed with code " + result.getRc()); + } + byte[] embedded = Files.readAllBytes(output); + + // Ghostscript can exit 0 having written a blank page, so keep the original rather than + // return an empty document. + if (!survived(pdfBytes, embedded)) { + log.warn("Ghostscript produced a degenerate document; keeping the original"); + return new Result( + pdfBytes, + false, + missing, + "Font embedding was skipped because the embedder returned a document that" + + " had lost content. " + + missing.size() + + " font(s) remain unembedded."); + } + + // It can also exit 0 while simply leaving fonts unembedded. + Set remaining; + try (PDDocument check = Loader.loadPDF(embedded)) { + remaining = findUnembeddedFonts(check); + } + if (!remaining.isEmpty()) { + return new Result( + embedded, + true, + remaining, + remaining.size() + + " font(s) could not be embedded (" + + String.join(", ", remaining) + + "). PDF/UA requires every font to be embedded."); + } + log.info("Embedded {} previously unembedded font(s)", missing.size()); + return new Result(embedded, true, missing, null); + + } catch (Exception e) { + log.warn("Font embedding failed: {}", e.getMessage()); + return new Result(pdfBytes, false, missing, "Font embedding failed: " + e.getMessage()); + } finally { + deleteQuietly(workingDir); + } + } + + /** + * True when the rewritten document still holds the original's content. A collapse in page count + * or content-stream size is the only signature of a failed rewrite the exit code hides. + */ + private static boolean survived(byte[] original, byte[] rewritten) { + try (PDDocument before = Loader.loadPDF(original); + PDDocument after = Loader.loadPDF(rewritten)) { + if (after.getNumberOfPages() != before.getNumberOfPages()) { + return false; + } + long beforeBytes = contentBytes(before); + long afterBytes = contentBytes(after); + if (beforeBytes == 0) { + return true; + } + return afterBytes * 20L >= beforeBytes; + } catch (IOException e) { + log.debug("Could not compare documents after embedding: {}", e.getMessage()); + return false; + } + } + + private static long contentBytes(PDDocument document) { + long total = 0; + for (PDPage page : document.getPages()) { + try (InputStream in = page.getContents()) { + if (in != null) { + byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) > 0) { + total += read; + } + } + } catch (IOException e) { + log.debug("Could not measure page content: {}", e.getMessage()); + } + } + return total; + } + + private static List command(Path input, Path output, Path workingDir) { + List command = new ArrayList<>(); + command.add("gs"); + command.add("--permit-file-read=" + workingDir.toAbsolutePath()); + command.add("--permit-file-write=" + workingDir.toAbsolutePath()); + command.add("-sDEVICE=pdfwrite"); + command.add("-dEmbedAllFonts=true"); + command.add("-dSubsetFonts=true"); + command.add("-dCompressFonts=true"); + command.add("-dNOSUBSTFONTS=false"); + command.add("-dPDFSETTINGS=/prepress"); + command.add("-dNOPAUSE"); + command.add("-dBATCH"); + command.add("-sOutputFile=" + output.toAbsolutePath()); + command.add(input.toAbsolutePath().toString()); + return command; + } + + /** Cached after the first probe: availability does not change mid-process. */ + private volatile Boolean ghostscriptAvailable; + + private boolean isGhostscriptAvailable() { + Boolean cached = ghostscriptAvailable; + if (cached != null) { + return cached; + } + boolean available; + try { + ProcessExecutorResult result = + ProcessExecutor.getInstance(ProcessExecutor.Processes.GHOSTSCRIPT) + .runCommandWithOutputHandling(List.of("gs", "--version")); + available = result.getRc() == 0; + } catch (Exception e) { + log.debug("Ghostscript availability check failed: {}", e.getMessage()); + available = false; + } + ghostscriptAvailable = available; + return available; + } + + private static void deleteQuietly(Path directory) { + if (directory == null) { + return; + } + try (Stream stream = Files.walk(directory)) { + stream.sorted(Comparator.reverseOrder()) + .forEach( + path -> { + try { + Files.deleteIfExists(path); + } catch (IOException e) { + log.debug("Could not delete {}", path); + } + }); + } catch (IOException e) { + log.debug("Could not clean {}", directory); + } + } + + /** + * @param warning non-null when fonts remain unembedded, for the conversion report + */ + public record Result(byte[] pdfBytes, boolean changed, Set fonts, String warning) {} +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/PdfUaConversionService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/PdfUaConversionService.java new file mode 100644 index 0000000000..1d0920c109 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/PdfUaConversionService.java @@ -0,0 +1,251 @@ +package stirling.software.proprietary.service.ua; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.encryption.InvalidPasswordException; +import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.proprietary.model.api.ua.PdfUaConversionOutcome; +import stirling.software.proprietary.model.api.ua.UaValidationResult; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.pdf.ua.PdfUaTagger; +import stirling.software.proprietary.pdf.ua.SourceFacts; +import stirling.software.proprietary.pdf.ua.TaggingOptions; +import stirling.software.proprietary.pdf.ua.TaggingResult; + +/** + * Converts a PDF to PDF/UA. The declaration is written first and withdrawn unless validation + * passes, so a returned file either conforms or does not claim to. + */ +@Service +@Slf4j +@RequiredArgsConstructor +public class PdfUaConversionService { + + private final PdfUaValidationService validationService; + private final FontEmbeddingService fontEmbeddingService; + private final stirling.software.common.service.CustomPDFDocumentFactory pdfDocumentFactory; + + /** Matches the cap GetInfoOnPDF already applies to comparable whole-document work. */ + private static final long MAX_INPUT_BYTES = 100L * 1024 * 1024; + + /** Beyond this the structure model alone runs to hundreds of megabytes. */ + private static final int MAX_PAGES = 2000; + + public PdfUaConversionOutcome convert(byte[] input, TaggingOptions options) throws IOException { + if (input.length > MAX_INPUT_BYTES) { + throw new IOException( + "This PDF is " + + (input.length / (1024 * 1024)) + + " MB. PDF/UA conversion is limited to " + + (MAX_INPUT_BYTES / (1024 * 1024)) + + " MB."); + } + PdfUaProfile profile = options.getProfile(); + List warnings = new ArrayList<>(); + + // Read the document's own facts before anything rewrites it. Font embedding runs + // Ghostscript over the whole file, which discards the structure tree, /Lang and XFA, so + // every guard and every "what did the source say" question must be answered from here. + SourceFacts facts; + try (PDDocument original = load(input)) { + rejectUnsupportedSource(original); + warnSignatures(original, warnings); + facts = SourceFacts.of(original); + } + + byte[] source = input; + if (options.isEmbedFonts()) { + // Must precede tagging: the embedder rewrites the file and drops any structure tree. + FontEmbeddingService.Result fonts = fontEmbeddingService.embedFonts(input); + source = fonts.pdfBytes(); + if (fonts.warning() != null) { + warnings.add(fonts.warning()); + } + source = keepTagsOverFonts(input, source, facts, options, warnings); + } + + TaggingOptions effective = options.toBuilder().sourceFacts(facts).build(); + + // Tag and declare in one pass; the claim is withdrawn below if validation disagrees. + byte[] declared; + TaggingResult taggingResult; + PdfUaTagger tagger = new PdfUaTagger(); + try (PDDocument document = load(source)) { + rejectEncrypted(document); + taggingResult = tagger.tag(document, effective); + warnings.addAll(taggingResult.getWarnings()); + tagger.declareConformance(document, profile); + declared = save(document); + } + + UaValidationResult validation = validationService.validate(declared, profile); + + // A validator cannot see text hidden behind artifact markers, so a clean verdict over + // suppressed content would be a false claim. + boolean honest = !taggingResult.isContentSuppressed(); + + if (validation.compliant() && honest) { + log.info("{} conversion passed validation", profile.displayName()); + return new PdfUaConversionOutcome( + declared, true, validation, summary(taggingResult), warnings); + } + + byte[] undeclared; + try (PDDocument document = load(declared)) { + tagger.withdrawConformance(document); + undeclared = save(document); + } + + if (!validation.compliant()) { + warnings.add( + "The document could not be made " + + profile.displayName() + + " conformant, so no conformance claim was written. " + + validation.totalFailures() + + " automated check(s) still fail."); + } + log.info( + "{} conversion left undeclared: {} failures, suppressedText={}", + profile.displayName(), + validation.totalFailures(), + !honest); + return new PdfUaConversionOutcome( + undeclared, false, validation, summary(taggingResult), warnings); + } + + private static PdfUaConversionOutcome.TaggingSummary summary(TaggingResult result) { + return new PdfUaConversionOutcome.TaggingSummary( + result.isRebuilt(), + result.getTaggedElements(), + result.getArtifacts(), + result.getFiguresNeedingAlt()); + } + + /** + * Tagging rewrites the content streams a signature covers, so the conversion still runs but the + * caller has to know the signature will no longer verify. + */ + private static void warnSignatures(PDDocument document, List warnings) { + int signatures = document.getSignatureDictionaries().size(); + if (signatures > 0) { + warnings.add( + signatures + + " digital signature(s) will stop verifying: tagging rewrites the" + + " content streams they cover. Convert first, then re-sign."); + } + } + + /** Replaces PDFBox's "incorrect password" wording, baffling when the caller supplied none. */ + private PDDocument load(byte[] bytes) throws IOException { + try { + // The factory spills large documents to a temp-file cache instead of the heap. + return pdfDocumentFactory.load(bytes); + } catch (IOException | RuntimeException e) { + // The factory wraps the parse failure, so check the cause chain rather than the type. + if (mentionsPassword(e)) { + throw new IOException( + "This PDF is encrypted. Remove the password before converting it to" + + " PDF/UA.", + e); + } + throw e; + } + } + + private static boolean mentionsPassword(Throwable error) { + for (Throwable cause = error; cause != null; cause = cause.getCause()) { + if (cause instanceof InvalidPasswordException) { + return true; + } + String message = cause.getMessage(); + if (message != null) { + String lower = message.toLowerCase(Locale.ROOT); + if (lower.contains("password") || lower.contains("decrypt")) { + return true; + } + } + } + return false; + } + + /** XFA is forbidden by PDF/UA-1 clause 7.15; encrypted or huge files cannot be restructured. */ + /** + * Under KEEP nothing rebuilds a tree, so if the embedder deleted one we would hand back an + * untagged document. Fonts are not worth the whole structure; give the tags back instead. + */ + private byte[] keepTagsOverFonts( + byte[] input, + byte[] embedded, + SourceFacts facts, + TaggingOptions options, + List warnings) + throws IOException { + if (options.getExistingTags() != TaggingOptions.ExistingTags.KEEP + || !facts.hasUsableTree() + || embedded == input) { + return embedded; + } + boolean survived; + try (PDDocument rewritten = load(embedded)) { + survived = PdfUaTagger.hasUsableStructureTree(rewritten); + } + if (survived) { + return embedded; + } + warnings.add( + "Embedding the missing fonts would have deleted the document's existing tags, so" + + " the tags were kept and the fonts left unembedded. Turn off font" + + " embedding to silence this, or rebuild the tags to embed them."); + return input; + } + + /** + * Checks that must see the document as the author wrote it. Font embedding strips XFA, so + * running this afterwards would let a dynamic form through unnoticed, and it would push a + * document we are about to reject through the whole embedder first. + */ + private static void rejectUnsupportedSource(PDDocument document) throws IOException { + if (document.getNumberOfPages() > MAX_PAGES) { + throw new IOException( + "This PDF has " + + document.getNumberOfPages() + + " pages. PDF/UA conversion is limited to " + + MAX_PAGES + + " pages; split it first."); + } + PDAcroForm form = document.getDocumentCatalog().getAcroForm(); + if (form != null && form.xfaIsDynamic()) { + throw new IOException( + "Dynamic XFA forms are not permitted by PDF/UA. Flatten the form first."); + } + } + + /** + * Deliberately checked on the working document rather than the source. Permissions-only + * encryption with an empty user password is common in published documents, the embedder + * resolves it, and those files convert usefully; rejecting them up front would fail a document + * for a password its author never set. + */ + private static void rejectEncrypted(PDDocument document) throws IOException { + if (document.isEncrypted()) { + throw new IOException( + "Encrypted PDFs cannot be converted to PDF/UA. Remove the password first."); + } + } + + private static byte[] save(PDDocument document) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + return out.toByteArray(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/PdfUaValidationService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/PdfUaValidationService.java new file mode 100644 index 0000000000..18d9fcaee7 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/PdfUaValidationService.java @@ -0,0 +1,245 @@ +package stirling.software.proprietary.service.ua; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Service; +import org.verapdf.gf.foundry.VeraGreenfieldFoundryProvider; +import org.verapdf.pdfa.Foundries; +import org.verapdf.pdfa.PDFAParser; +import org.verapdf.pdfa.PDFAValidator; +import org.verapdf.pdfa.flavours.PDFAFlavour; +import org.verapdf.pdfa.results.TestAssertion; +import org.verapdf.pdfa.results.ValidationResult; + +import jakarta.annotation.PostConstruct; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.proprietary.model.api.ua.AccessibilityIssue; +import stirling.software.proprietary.model.api.ua.UaValidationResult; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; + +/** + * Validates a document against a PDF/UA profile using veraPDF, the oracle a conversion is declared + * against. It checks only the machine-verifiable subset, so a clean result is not "accessible". + */ +@Service +@Slf4j +public class PdfUaValidationService { + + /** Plain-English text and remediability for the clauses users actually hit. */ + private static final Map CLAUSES = buildClauseTable(); + + record ClauseInfo(String message, boolean autoFixable) {} + + @PostConstruct + public void initialise() { + try { + VeraGreenfieldFoundryProvider.initialise(); + } catch (Exception e) { + log.error("Failed to initialise veraPDF for PDF/UA validation", e); + } + } + + public UaValidationResult validate(byte[] pdfBytes, PdfUaProfile profile) { + PDFAFlavour flavour = flavourFor(profile); + try (PDFAParser parser = + Foundries.defaultInstance() + .createParser(new ByteArrayInputStream(pdfBytes), flavour)) { + + PDFAValidator validator = Foundries.defaultInstance().createValidator(flavour, false); + ValidationResult result = validator.validate(parser); + return toResult(profile, result); + + } catch (Exception e) { + log.warn("PDF/UA validation failed for {}: {}", profile.displayName(), e.getMessage()); + AccessibilityIssue issue = new AccessibilityIssue(); + issue.setMessage("Validation could not run: " + e.getMessage()); + issue.setSeverity("error"); + issue.setClause("n/a"); + return new UaValidationResult(profile.displayName(), false, List.of(issue), 0); + } + } + + public static PDFAFlavour flavourFor(PdfUaProfile profile) { + return profile == PdfUaProfile.UA2 ? PDFAFlavour.PDFUA_2 : PDFAFlavour.PDFUA_1; + } + + /** + * Whether the bytes really validate as PDF/A level A for the given part. Tagging is necessary + * for level A but not sufficient, so the claim is only written once veraPDF agrees. + */ + public boolean validatesAsPdfaLevelA(byte[] pdfBytes, int part) { + PDFAFlavour flavour = + switch (part) { + case 1 -> PDFAFlavour.PDFA_1_A; + case 2 -> PDFAFlavour.PDFA_2_A; + case 3 -> PDFAFlavour.PDFA_3_A; + default -> null; + }; + if (flavour == null) { + log.warn("No PDF/A level A flavour for part {}", part); + return false; + } + try (PDFAParser parser = + Foundries.defaultInstance() + .createParser(new ByteArrayInputStream(pdfBytes), flavour)) { + PDFAValidator validator = Foundries.defaultInstance().createValidator(flavour, false); + return validator.validate(parser).isCompliant(); + } catch (Exception e) { + log.warn("Level A validation could not run: {}", e.getMessage()); + return false; + } + } + + /** + * Groups repeated failures of the same rule so a report lists issues, not thousands of lines. + */ + private static UaValidationResult toResult(PdfUaProfile profile, ValidationResult result) { + Map grouped = new LinkedHashMap<>(); + int total = 0; + + for (TestAssertion assertion : result.getTestAssertions()) { + if (assertion.getStatus() != TestAssertion.Status.FAILED) { + continue; + } + total++; + String clause = + assertion.getRuleId() != null ? assertion.getRuleId().getClause() : "unknown"; + int test = assertion.getRuleId() != null ? assertion.getRuleId().getTestNumber() : 0; + String key = clause + "-" + test; + + AccessibilityIssue issue = + grouped.computeIfAbsent( + key, + k -> { + AccessibilityIssue created = new AccessibilityIssue(); + created.setClause(clause); + created.setTestNumber(String.valueOf(test)); + created.setSeverity("error"); + ClauseInfo info = lookupClause(clause); + created.setMessage( + info != null ? info.message() : assertion.getMessage()); + created.setTechnicalMessage(assertion.getMessage()); + created.setAutoFixable(info != null && info.autoFixable()); + created.setSpecification(profile.displayName()); + return created; + }); + issue.setOccurrences(issue.getOccurrences() + 1); + if (issue.getLocation() == null && assertion.getLocation() != null) { + issue.setLocation(assertion.getLocation().toString()); + } + } + + List issues = new ArrayList<>(grouped.values()); + return new UaValidationResult( + profile.displayName(), result.isCompliant() && total == 0, issues, total); + } + + /** + * Finds the most specific entry covering a clause by walking up the dotted hierarchy. String + * prefixes would be wrong: {@code 7.1} prefixes {@code 7.18.1} without being its ancestor. + */ + static ClauseInfo lookupClause(String clause) { + if (clause == null) { + return null; + } + String current = clause; + while (!current.isEmpty()) { + ClauseInfo info = CLAUSES.get(current); + if (info != null) { + return info; + } + int dot = current.lastIndexOf('.'); + if (dot < 0) { + return null; + } + current = current.substring(0, dot); + } + return null; + } + + private static Map buildClauseTable() { + Map table = new LinkedHashMap<>(); + table.put( + "7.1", + new ClauseInfo( + "Document is not tagged, or some content is neither tagged nor marked as an artifact.", + true)); + table.put( + "7.2", + new ClauseInfo( + "Text cannot be mapped to Unicode, or the document language is not declared.", + true)); + table.put( + "7.3", + new ClauseInfo("An image or graphic has no alternative description.", false)); + table.put( + "7.4", + new ClauseInfo( + "Heading levels skip a level, or headings are nested incorrectly.", true)); + table.put( + "7.5", + new ClauseInfo("A table is missing header cells or header associations.", false)); + table.put( + "7.6", new ClauseInfo("A list is not structured as list items with bodies.", true)); + table.put( + "7.7", + new ClauseInfo("A mathematical expression has no alternative description.", false)); + table.put( + "7.8", + new ClauseInfo("Running heads or page numbers are not marked as artifacts.", true)); + table.put("7.9", new ClauseInfo("A note is missing a unique identifier.", true)); + // Tagging does not touch optional content groups, so this needs the authoring tool. + table.put("7.10", new ClauseInfo("An optional content group has no name.", false)); + // The attachment's own /AFRelationship and /Desc are not something tagging can supply. + table.put( + "7.11", + new ClauseInfo( + "An embedded file is missing its relationship or description.", false)); + table.put( + "7.15", + new ClauseInfo( + "The document uses a dynamic XFA form, which PDF/UA does not allow.", + false)); + table.put( + "7.16", + new ClauseInfo( + "Security settings prevent assistive technology from reading the content.", + true)); + table.put("7.17", new ClauseInfo("Navigation aids such as page labels are missing.", true)); + table.put( + "7.18", + new ClauseInfo( + "An annotation is missing a description, tab order, or structure entry.", + true)); + table.put( + "7.20", + new ClauseInfo( + "A form or group XObject is not marked as content or as an artifact.", + false)); + // Most font defects (CIDFont, CMap, metrics, encoding) need the font itself repaired. + table.put( + "7.21", + new ClauseInfo("A font in the document does not meet PDF/UA rules.", false)); + // The one font defect embedding does fix. + table.put("7.21.4.1", new ClauseInfo("A font used in the document is not embedded.", true)); + // ToUnicode gaps need the font itself repaired, which embedding does not do. + table.put( + "7.21.7", + new ClauseInfo( + "A font does not map every character it uses to Unicode, so extracted text" + + " may be wrong.", + false)); + table.put( + "5", + new ClauseInfo( + "The document does not declare PDF/UA conformance in its XMP metadata.", + true)); + return table; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/PdfaAccessibilityService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/PdfaAccessibilityService.java new file mode 100644 index 0000000000..5a13b4ab65 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/ua/PdfaAccessibilityService.java @@ -0,0 +1,297 @@ +package stirling.software.proprietary.service.ua; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdfwriter.compress.CompressParameters; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.common.PDMetadata; +import org.apache.xmpbox.XMPMetadata; +import org.apache.xmpbox.schema.PDFAExtensionSchema; +import org.apache.xmpbox.schema.PDFAIdentificationSchema; +import org.apache.xmpbox.type.AbstractStructuredType; +import org.apache.xmpbox.type.ArrayProperty; +import org.apache.xmpbox.type.Cardinality; +import org.apache.xmpbox.type.PDFAPropertyType; +import org.apache.xmpbox.type.PDFASchemaType; +import org.apache.xmpbox.xml.DomXmpParser; +import org.apache.xmpbox.xml.XmpSerializer; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.service.PdfaLevelAServiceInterface; +import stirling.software.proprietary.pdf.ua.PdfUaIdentificationSchema; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.pdf.ua.PdfUaTagger; +import stirling.software.proprietary.pdf.ua.TaggingOptions; +import stirling.software.proprietary.pdf.ua.TaggingResult; + +/** + * Raises a PDF/A file from conformance level B to level A, which adds the tagging the PDF/UA tagger + * already does. Must run after Ghostscript, which discards any structure tree it is given. + */ +@Service +@Slf4j +@RequiredArgsConstructor +public class PdfaAccessibilityService implements PdfaLevelAServiceInterface { + + /** + * Matches the PDF/UA converter's own cap; beyond this the structure model exhausts the heap. + */ + private static final int MAX_TAGGABLE_PAGES = 2000; + + private final PdfUaValidationService validationService; + + /** + * Tags a converted PDF/A and marks it conformance A, or returns it unchanged rather than + * claiming level A over untagged content. part is 1 to 3; part 1 keeps its PDF 1.4 version. + */ + public Result upgradeToLevelA(byte[] pdfBytes, int part, String language, String title) { + return upgradeToLevelA(pdfBytes, part, language, title, false); + } + + /** + * @param alsoDeclareUa additionally claim PDF/UA, but only if it validates + */ + @Override + public Result upgradeToLevelA( + byte[] pdfBytes, int part, String language, String title, boolean alsoDeclareUa) { + List warnings = new ArrayList<>(); + try { + byte[] tagged; + TaggingResult taggingResult; + + try (PDDocument document = Loader.loadPDF(pdfBytes)) { + // Tagging holds a model of the whole document; without a cap a large file exhausts + // the heap, and OutOfMemoryError is an Error, so the catch below never sees it. + if (document.getNumberOfPages() > MAX_TAGGABLE_PAGES) { + warnings.add( + "This document has " + + document.getNumberOfPages() + + " pages, more than the " + + MAX_TAGGABLE_PAGES + + " that can be tagged, so it was left at conformance level B."); + return new Result(pdfBytes, false, warnings); + } + TaggingOptions options = + TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language(language) + .title(title) + .fallbackTitle(title) + // Fonts were embedded on the PDF/A pass; a rewrite would undo it. + .embedFonts(false) + // PDF/A-1 is defined on PDF 1.4; raising it breaks conformance. + .preservePdfVersion(part == 1) + .existingTags(TaggingOptions.ExistingTags.AUTO) + .build(); + + taggingResult = new PdfUaTagger().tag(document, options); + warnings.addAll(taggingResult.getWarnings()); + tagged = save(document, part); + } + + if (taggingResult.getTaggedElements() == 0 && taggingResult.isRebuilt()) { + warnings.add( + "No taggable content was found, so the file cannot claim PDF/A level A." + + " It remains valid at level B."); + return new Result(pdfBytes, false, warnings); + } + if (taggingResult.isContentSuppressed()) { + warnings.add( + "Some text could not be tagged reliably and was marked as an artifact, so" + + " no level A claim was written. The file remains valid at level B."); + return new Result(tagged, false, warnings); + } + + byte[] declared = setConformance(tagged, part, "A"); + + // Tagging is necessary for level A but not sufficient: Unicode mappings are too. + if (!validationService.validatesAsPdfaLevelA(declared, part)) { + warnings.add( + "The document was tagged but does not validate as PDF/A-" + + part + + "a, so it was left at conformance level B."); + return new Result(setConformance(tagged, part, "B"), false, warnings); + } + + if (alsoDeclareUa) { + byte[] withUa = declarePdfUaAlongsidePdfa(declared, part); + var uaResult = validationService.validate(withUa, PdfUaProfile.UA1); + if (uaResult.compliant()) { + log.info("Upgraded PDF/A-{} to level A and declared PDF/UA", part); + return new Result(withUa, true, warnings); + } + // The archival upgrade stands on its own; only the accessibility claim is dropped. + warnings.add( + "PDF/UA was requested alongside PDF/A but " + + uaResult.totalFailures() + + " accessibility check(s) still fail, so no PDF/UA claim was" + + " written. The file is valid PDF/A-" + + part + + "a."); + } + + log.info("Upgraded PDF/A-{} to conformance level A", part); + return new Result(declared, true, warnings); + + } catch (Exception e) { + log.warn("Could not upgrade to PDF/A level A: {}", e.getMessage()); + warnings.add( + "Level A upgrade failed (" + + e.getMessage() + + "), so the file was left at conformance level B."); + return new Result(pdfBytes, false, warnings); + } + } + + /** + * Declares PDF/UA alongside PDF/A in one file. The extension schema is required: PDF/A forbids + * XMP properties no schema describes, and XMPBox has none for {@code pdfuaid}. + */ + static byte[] declarePdfUaAlongsidePdfa(byte[] pdfBytes, int part) throws Exception { + try (PDDocument document = Loader.loadPDF(pdfBytes)) { + XMPMetadata xmp = parseOrCreate(document); + + PdfUaIdentificationSchema identification = new PdfUaIdentificationSchema(xmp); + identification.setPart(1); + xmp.addSchema(identification); + + addPdfUaExtensionSchema(xmp); + writeMetadata(document, xmp); + return save(document, part); + } + } + + /** + * Describes the pdfuaid namespace so a PDF/A validator accepts it. Fields are set individually, + * not by subclassing: XMPBox reads the namespace from an annotation, which is not inherited. + */ + private static void addPdfUaExtensionSchema(XMPMetadata xmp) { + PDFAExtensionSchema extension = + (PDFAExtensionSchema) xmp.getSchema(PDFAExtensionSchema.class); + if (extension == null) { + extension = xmp.createAndAddPDFAExtensionSchemaWithDefaultNS(); + } + + PDFAPropertyType partProperty = new PDFAPropertyType(xmp); + addField(xmp, partProperty, PDFAPropertyType.NAME, "part"); + addField(xmp, partProperty, PDFAPropertyType.VALUETYPE, "Integer"); + addField(xmp, partProperty, PDFAPropertyType.CATEGORY, "internal"); + addField( + xmp, + partProperty, + PDFAPropertyType.DESCRIPTION, + "Indicates which part of ISO 14289 the document conforms to"); + + PDFASchemaType schema = new PDFASchemaType(xmp); + addField(xmp, schema, PDFASchemaType.SCHEMA, "PDF/UA Universal Accessibility Schema"); + addField(xmp, schema, PDFASchemaType.NAMESPACE_URI, PdfUaIdentificationSchema.NAMESPACE); + addField(xmp, schema, PDFASchemaType.PREFIX, PdfUaIdentificationSchema.PREFERRED_PREFIX); + + ArrayProperty properties = + xmp.getTypeMapping() + .createArrayProperty( + schema.getNamespace(), + schema.getPrefix(), + PDFASchemaType.PROPERTY, + Cardinality.Seq); + properties.getContainer().addProperty(partProperty); + schema.getContainer().addProperty(properties); + + // A freshly created extension schema has no schemas bag yet, so make one. + ArrayProperty schemas = extension.getSchemasProperty(); + if (schemas == null) { + schemas = + xmp.getTypeMapping() + .createArrayProperty( + extension.getNamespace(), + extension.getPrefix(), + PDFAExtensionSchema.SCHEMAS, + Cardinality.Bag); + extension.addProperty(schemas); + } + schemas.getContainer().addProperty(schema); + } + + /** Adds one text field to a structured type, in that type's own namespace. */ + private static void addField( + XMPMetadata xmp, AbstractStructuredType target, String name, String value) { + target.getContainer() + .addProperty( + xmp.getTypeMapping() + .createText( + target.getNamespace(), target.getPrefix(), name, value)); + } + + private static XMPMetadata parseOrCreate(PDDocument document) throws Exception { + PDMetadata existing = document.getDocumentCatalog().getMetadata(); + if (existing == null) { + return XMPMetadata.createXMPMetadata(); + } + try (InputStream in = new ByteArrayInputStream(existing.toByteArray())) { + DomXmpParser parser = new DomXmpParser(); + parser.setStrictParsing(false); + return parser.parse(in); + } + } + + private static void writeMetadata(PDDocument document, XMPMetadata xmp) throws Exception { + ByteArrayOutputStream serialised = new ByteArrayOutputStream(); + new XmpSerializer().serialize(xmp, serialised, true); + PDMetadata metadata = new PDMetadata(document); + metadata.importXMPMetadata(serialised.toByteArray()); + document.getDocumentCatalog().setMetadata(metadata); + } + + /** Rewrites {@code pdfaid:conformance} without disturbing the rest of the packet. */ + static byte[] setConformance(byte[] pdfBytes, int part, String conformance) throws Exception { + try (PDDocument document = Loader.loadPDF(pdfBytes)) { + PDMetadata existing = document.getDocumentCatalog().getMetadata(); + XMPMetadata xmp; + if (existing != null) { + try (InputStream in = new ByteArrayInputStream(existing.toByteArray())) { + DomXmpParser parser = new DomXmpParser(); + parser.setStrictParsing(false); + xmp = parser.parse(in); + } + } else { + xmp = XMPMetadata.createXMPMetadata(); + } + + PDFAIdentificationSchema identification = + (PDFAIdentificationSchema) xmp.getSchema(PDFAIdentificationSchema.class); + if (identification == null) { + identification = xmp.createAndAddPDFAIdentificationSchema(); + } + identification.setPart(part); + identification.setConformance(conformance); + + ByteArrayOutputStream serialised = new ByteArrayOutputStream(); + new XmpSerializer().serialize(xmp, serialised, true); + PDMetadata metadata = new PDMetadata(document); + metadata.importXMPMetadata(serialised.toByteArray()); + document.getDocumentCatalog().setMetadata(metadata); + + return save(document, part); + } + } + + /** + * Part 1 is saved uncompressed: PDFBox's default object streams need PDF 1.5, which would push + * a PDF/A-1 file off its required 1.4 version. + */ + private static byte[] save(PDDocument document, int part) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save( + out, part == 1 ? CompressParameters.NO_COMPRESSION : new CompressParameters()); + return out.toByteArray(); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/LayoutAnalyzerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/LayoutAnalyzerTest.java new file mode 100644 index 0000000000..e9aaab0a1a --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/LayoutAnalyzerTest.java @@ -0,0 +1,286 @@ +package stirling.software.proprietary.pdf.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** Unit tests for the heuristics that decide what a run of text means. */ +class LayoutAnalyzerTest { + + private static final BBox A4 = new BBox(0, 0, 595, 842); + + private static TextLineInfo line(String text, float size, float x, float y) { + return line(text, size, x, y, false, 0, 0); + } + + private static TextLineInfo line( + String text, float size, float x, float y, boolean bold, int start, int end) { + List words = new ArrayList<>(); + float cursor = x; + for (String token : text.strip().split("\\s+")) { + float width = token.length() * size * 0.5f; + words.add( + new WordInfo( + token, + new BBox(cursor, y, cursor + width, y + size), + start, + end, + size, + bold)); + cursor += width + size * 0.3f; + } + return new TextLineInfo( + 0, text, new BBox(x, y, cursor, y + size), size, bold, start, end, false, words); + } + + private static PageContent page(List lines) { + return new PageContent(0, lines, List.of(), lines.size(), false, false, false, A4); + } + + @Nested + @DisplayName("body font size") + class BodyFontSize { + + @Test + @DisplayName("weights by characters so one huge title does not skew the baseline") + void weightsByCharacterCount() { + List lines = + List.of( + line("A Very Large Title", 32, 50, 700), + line("Body text line one which is long", 11, 50, 650), + line("Body text line two which is long", 11, 50, 630), + line("Body text line three also long", 11, 50, 610)); + assertEquals(11f, LayoutAnalyzer.bodyFontSize(List.of(page(lines)))); + } + + @Test + @DisplayName("returns zero when there is no text") + void handlesEmptyDocument() { + assertEquals(0f, LayoutAnalyzer.bodyFontSize(List.of(page(List.of())))); + } + } + + @Nested + @DisplayName("heading detection") + class Headings { + + @Test + @DisplayName("assigns distinct sizes to descending levels") + void assignsTiers() { + List lines = + List.of( + line("Title", 24, 50, 800), + line("Chapter", 18, 50, 750), + line("Section", 14, 50, 700), + line("Body text that is long enough to set a baseline", 11, 50, 650)); + Map tiers = LayoutAnalyzer.headingTiers(List.of(page(lines)), 11f); + assertEquals(1, tiers.get(24f)); + assertEquals(2, tiers.get(18f)); + assertEquals(3, tiers.get(14f)); + assertNull(tiers.get(11f), "body size must not be a heading tier"); + } + + @Test + @DisplayName("rejects long lines and full sentences whatever their size") + void rejectsProse() { + assertFalse( + LayoutAnalyzer.isHeadingCandidate( + line("This line ends like a sentence does.", 20, 50, 700)), + "a line ending in a full stop reads as prose"); + assertFalse( + LayoutAnalyzer.isHeadingCandidate( + line( + "one two three four five six seven eight nine ten eleven twelve" + + " thirteen", + 20, + 50, + 700)), + "a long line is body text however large"); + assertTrue(LayoutAnalyzer.isHeadingCandidate(line("Financial Results", 20, 50, 700))); + } + + @Test + @DisplayName("boldness alone never promotes a line to a heading") + void boldIsNotAHeadingSignal() { + List lines = + List.of( + line("Bold Label", 11, 50, 700, true, 0, 0), + line("Body text long enough to set the baseline here", 11, 50, 650)); + assertTrue( + LayoutAnalyzer.headingTiers(List.of(page(lines)), 11f).isEmpty(), + "a bold line at body size is emphasis, not a heading"); + } + + @Test + @DisplayName("rewrites skipped levels so H1 is never followed by H3") + void normalisesSkippedLevels() { + DocumentStructure structure = new DocumentStructure(); + structure.add(new StructBlock(StructType.H1, 0)); + structure.add(new StructBlock(StructType.H3, 0)); + structure.add(new StructBlock(StructType.H4, 0)); + LayoutAnalyzer.normaliseHeadingLevels(structure); + + assertEquals(StructType.H1, structure.getBlocks().get(0).getType()); + assertEquals(StructType.H2, structure.getBlocks().get(1).getType()); + assertEquals(StructType.H3, structure.getBlocks().get(2).getType()); + } + } + + @Nested + @DisplayName("lists") + class Lists { + + @Test + @DisplayName("recognises bullet and ordered markers") + void recognisesMarkers() { + assertTrue(LayoutAnalyzer.startsListItem(line("• First item", 11, 50, 700))); + assertTrue(LayoutAnalyzer.startsListItem(line("- First item", 11, 50, 700))); + assertTrue(LayoutAnalyzer.startsListItem(line("1. First item", 11, 50, 700))); + assertTrue(LayoutAnalyzer.startsListItem(line("a) First item", 11, 50, 700))); + assertFalse(LayoutAnalyzer.startsListItem(line("Ordinary prose here", 11, 50, 700))); + } + } + + @Nested + @DisplayName("table cells") + class Tables { + + @Test + @DisplayName("splits a row at wide gaps but not at ordinary word spacing") + void splitsOnWideGaps() { + List words = + List.of( + new WordInfo("Region", new BBox(50, 700, 90, 711), 0, 0, 11, false), + new WordInfo("name", new BBox(93, 700, 125, 711), 0, 0, 11, false), + new WordInfo("Units", new BBox(250, 700, 285, 711), 1, 1, 11, false)); + TextLineInfo row = + new TextLineInfo( + 0, + "Region name Units", + new BBox(50, 700, 285, 711), + 11, + false, + 0, + 1, + false, + words); + List> cells = LayoutAnalyzer.splitCells(row); + assertEquals(2, cells.size(), "the small gap is a word space, the large one is a cell"); + assertEquals(2, cells.get(0).size()); + assertEquals("Units", cells.get(1).get(0).text()); + } + + @Test + @DisplayName("words sharing an operator cannot become separate cells") + void detectsInseparableWords() { + List shared = + List.of( + new WordInfo("A", new BBox(50, 700, 60, 711), 3, 3, 11, false), + new WordInfo("B", new BBox(250, 700, 260, 711), 3, 3, 11, false)); + TextLineInfo row = + new TextLineInfo( + 0, "A B", new BBox(50, 700, 260, 711), 11, false, 3, 3, false, shared); + assertFalse( + row.wordsAreSeparable(), + "cells drawn by one operator cannot carry separate marked content ids"); + } + } + + @Nested + @DisplayName("running heads") + class RunningHeads { + + @Test + @DisplayName("treats text repeating in the margin band across pages as an artifact") + void findsRepeatedMarginText() { + List pages = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + List lines = + List.of( + line("Confidential Report", 9, 50, 800), + line("Body content for the page", 11, 50, 400), + line("Page " + (i + 1), 9, 300, 20)); + pages.add(new PageContent(i, lines, List.of(), 3, false, false, false, A4)); + } + Map> artifacts = LayoutAnalyzer.repeatedMarginLines(pages); + assertEquals( + 2, artifacts.get(0).size(), "the running head and the folio are artifacts"); + assertTrue( + artifacts.get(0).stream().noneMatch(l -> l.text().contains("Body content")), + "body text must never be demoted to an artifact"); + } + + @Test + @DisplayName("does not treat a one-off margin line as a running head") + void ignoresUniqueMarginText() { + List titles = List.of("Alpha", "Beta", "Gamma", "Delta"); + List pages = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + List lines = + List.of( + line(titles.get(i) + " overview", 9, 50, 800), + line("Body content", 11, 50, 400)); + pages.add(new PageContent(i, lines, List.of(), 2, false, false, false, A4)); + } + assertTrue(LayoutAnalyzer.repeatedMarginLines(pages).get(0).isEmpty()); + } + + @Test + @DisplayName("a large heading high on the page stays a heading, not chrome") + void doesNotDemoteHeadingsNearTheTop() { + List pages = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + List lines = + List.of( + // Masking digits makes these look identical across pages. + line("Section " + (i + 1), 20, 50, 800), + line("Body text long enough to set the baseline", 11, 50, 400)); + pages.add(new PageContent(i, lines, List.of(), 2, false, false, false, A4)); + } + assertTrue( + LayoutAnalyzer.repeatedMarginLines(pages, 11f).get(0).isEmpty(), + "a heading larger than body text is content, wherever it sits"); + } + } + + @Nested + @DisplayName("columns") + class Columns { + + @Test + @DisplayName("detects a gutter when text sits in two balanced blocks") + void detectsTwoColumns() { + List lines = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + lines.add(line("Left column text", 10, 50, 700 - i * 14)); + lines.add(line("Right column text", 10, 320, 700 - i * 14)); + } + assertNotNull(LayoutAnalyzer.detectGutter(page(lines))); + } + + @Test + @DisplayName("does not split a page whose lines span the full width") + void ignoresSingleColumn() { + List lines = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + lines.add( + line( + "A full width line of prose that crosses the centre of the page", + 10, + 50, + 700 - i * 14)); + } + assertNull(LayoutAnalyzer.detectGutter(page(lines))); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/MarkedContentInjectorTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/MarkedContentInjectorTest.java new file mode 100644 index 0000000000..73d4c64817 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/MarkedContentInjectorTest.java @@ -0,0 +1,164 @@ +package stirling.software.proprietary.pdf.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.apache.pdfbox.pdmodel.font.PDFont; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.pdfbox.pdmodel.font.Standard14Fonts; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** Tests the content-stream rewriting that makes tagging possible. */ +class MarkedContentInjectorTest { + + private static byte[] threeLinePdf() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + PDFont font = new PDType1Font(Standard14Fonts.FontName.HELVETICA); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + for (int i = 0; i < 3; i++) { + cs.beginText(); + cs.setFont(font, 12); + cs.newLineAtOffset(50, 700 - i * 20); + cs.showText("Line " + i); + cs.endText(); + } + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + return out.toByteArray(); + } + } + + private static String contentOf(PDDocument document) throws IOException { + try (InputStream in = document.getPage(0).getContents()) { + return new String(in.readAllBytes(), StandardCharsets.ISO_8859_1); + } + } + + @Test + @DisplayName("wraps claimed content in BDC/EMC with a marked content id") + void wrapsClaimedContent() throws Exception { + try (PDDocument document = Loader.loadPDF(threeLinePdf())) { + StructBlock paragraph = new StructBlock(StructType.P, 0); + paragraph.addRange(0, 1); + + int next = + new MarkedContentInjector() + .inject(document, document.getPage(0), List.of(paragraph), 0, true); + + String content = contentOf(document); + assertTrue(content.contains("/P"), "the structure type was not written"); + assertTrue(content.contains("/MCID"), "no marked content id was written"); + assertTrue(content.contains("BDC"), "no marked content sequence was opened"); + assertTrue(content.contains("EMC"), "no marked content sequence was closed"); + assertFalse(paragraph.getMcids().isEmpty(), "the block was given no marked content id"); + assertTrue(next > 0, "the id counter did not advance"); + } + } + + @Test + @DisplayName("marks unclaimed content as an artifact so nothing is left untagged") + void unclaimedContentBecomesArtifact() throws Exception { + try (PDDocument document = Loader.loadPDF(threeLinePdf())) { + StructBlock paragraph = new StructBlock(StructType.P, 0); + paragraph.addRange(0, 0); + + new MarkedContentInjector() + .inject(document, document.getPage(0), List.of(paragraph), 0, true); + + String content = contentOf(document); + assertTrue( + content.contains("/Artifact"), + "content nobody claimed must be marked as an artifact, or PDF/UA clause 7.1" + + " fails"); + } + } + + @Test + @DisplayName("opens and closes sequences in balanced pairs") + void sequencesAreBalanced() throws Exception { + try (PDDocument document = Loader.loadPDF(threeLinePdf())) { + StructBlock first = new StructBlock(StructType.P, 0); + first.addRange(0, 0); + StructBlock second = new StructBlock(StructType.H1, 0); + second.addRange(2, 2); + + new MarkedContentInjector() + .inject(document, document.getPage(0), List.of(first, second), 0, true); + + String content = contentOf(document); + int opens = count(content, "BDC") + count(content, "BMC"); + int closes = count(content, "EMC"); + assertEquals(opens, closes, "every opened sequence must be closed"); + } + } + + @Test + @DisplayName("rewriting does not change what a reader extracts") + void textIsUnchanged() throws Exception { + byte[] original = threeLinePdf(); + String before = extract(original); + + byte[] rewritten; + try (PDDocument document = Loader.loadPDF(original)) { + StructBlock paragraph = new StructBlock(StructType.P, 0); + paragraph.addRange(0, 2); + new MarkedContentInjector() + .inject(document, document.getPage(0), List.of(paragraph), 0, true); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + rewritten = out.toByteArray(); + } + assertEquals(before, extract(rewritten), "marked content operators must not render"); + } + + @Test + @DisplayName("two blocks claiming the same content keep the first, not both") + void overlappingClaimsAreResolved() { + StructBlock first = new StructBlock(StructType.P, 0); + first.addRange(0, 2); + StructBlock second = new StructBlock(StructType.H1, 0); + second.addRange(1, 1); + + Map owners = + MarkedContentInjector.ownersByOrdinal(List.of(first, second)); + assertSame(first, owners.get(1), "the first claim wins so reading order stays unambiguous"); + assertEquals(3, owners.size()); + } + + private static int count(String haystack, String needle) { + int total = 0; + int index = 0; + while ((index = haystack.indexOf(needle, index)) >= 0) { + total++; + index += needle.length(); + } + return total; + } + + private static String extract(byte[] pdf) throws IOException { + try (PDDocument document = Loader.loadPDF(pdf)) { + return new org.apache.pdfbox.text.PDFTextStripper() + .getText(document) + .replaceAll("\\s+", " ") + .strip(); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/MarkedContentSafetyTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/MarkedContentSafetyTest.java new file mode 100644 index 0000000000..29cc18c525 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/MarkedContentSafetyTest.java @@ -0,0 +1,193 @@ +package stirling.software.proprietary.pdf.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.apache.pdfbox.pdmodel.common.PDStream; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.pdfbox.pdmodel.font.Standard14Fonts; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** Regression tests for rewriter damage a validator cannot see, so it still passes validation. */ +class MarkedContentSafetyTest { + + private static String contentOf(PDDocument document) throws IOException { + try (InputStream in = document.getPage(0).getContents()) { + return new String(in.readAllBytes(), StandardCharsets.ISO_8859_1); + } + } + + private static void setContent(PDDocument document, String content) throws IOException { + PDStream stream = new PDStream(document); + try (var out = stream.createOutputStream()) { + out.write(content.getBytes(StandardCharsets.ISO_8859_1)); + } + document.getPage(0).setContents(stream); + } + + private static PDDocument onePage() throws IOException { + PDDocument document = new PDDocument(); + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + cs.beginText(); + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12); + cs.newLineAtOffset(50, 700); + cs.showText("visible"); + cs.endText(); + } + return document; + } + + @Test + @DisplayName("an optional-content layer survives the rebuild, so hidden content stays hidden") + void optionalContentIsPreserved() throws Exception { + try (PDDocument document = onePage()) { + String original = contentOf(document); + setContent(document, "/OC /MC0 BDC\n" + original + "\nEMC\n"); + + StructBlock paragraph = new StructBlock(StructType.P, 0); + paragraph.addRange(0, 0); + new MarkedContentInjector() + .inject(document, document.getPage(0), List.of(paragraph), 0, true); + + String rewritten = contentOf(document); + assertTrue( + rewritten.contains("/OC"), + "the optional-content wrapper was stripped, which would make a hidden" + + " DRAFT/CONFIDENTIAL or redaction layer permanently visible:\n" + + rewritten); + assertEquals( + countOf(rewritten, "BDC") + countOf(rewritten, "BMC"), + countOf(rewritten, "EMC"), + "marked content is unbalanced after preserving the layer"); + } + } + + @Test + @DisplayName("replacement text survives the rebuild so ligatures still read correctly") + void actualTextIsPreserved() throws Exception { + try (PDDocument document = onePage()) { + String original = contentOf(document); + // A generator marks an ffi ligature with what it really spells. + setContent( + document, "/Span <> BDC\n" + original + "\nEMC\n"); + + StructBlock paragraph = new StructBlock(StructType.P, 0); + paragraph.addRange(0, 0); + new MarkedContentInjector() + .inject(document, document.getPage(0), List.of(paragraph), 0, true); + + String rewritten = contentOf(document); + assertTrue( + rewritten.contains("ActualText"), + "dropping ActualText leaves a screen reader announcing the raw glyph:\n" + + rewritten); + assertFalse( + rewritten.contains("/MCID 7"), + "the source's own marked content id is meaningless after a rebuild"); + assertEquals( + countOf(rewritten, "BDC") + countOf(rewritten, "BMC"), + countOf(rewritten, "EMC"), + "marked content is unbalanced after preserving replacement text"); + } + } + + @Test + @DisplayName("a sequence wrapping a fill opens before the path, not inside it") + void markedContentNeverOpensInsideAPathObject() throws Exception { + try (PDDocument document = onePage()) { + setContent(document, "0 0 0 rg\n10 10 50 5 re\nf\n"); + + new MarkedContentInjector().inject(document, document.getPage(0), List.of(), 0, true); + + String rewritten = contentOf(document); + int reAt = rewritten.indexOf(" re"); + int openAt = Math.max(rewritten.indexOf("BMC"), rewritten.indexOf("BDC")); + assertTrue(openAt >= 0, "no sequence was opened at all: " + rewritten); + assertTrue( + openAt < reAt, + "ISO 32000-1 does not permit a marked-content operator inside a path object;" + + " the sequence must open before the path construction:\n" + + rewritten); + } + } + + @Test + @DisplayName("words drawn out of stream order are still claimed, not silently artifacted") + void outOfOrderWordsAreClaimed() { + // A line whose second word on the page was painted first: ordinals 1 then 0. + WordInfo right = new WordInfo("label", new BBox(50, 700, 90, 712), 1, 1, 11, false); + WordInfo left = new WordInfo("value", new BBox(200, 700, 240, 712), 0, 0, 11, false); + TextLineInfo line = + new TextLineInfo( + 0, + "label value", + new BBox(50, 700, 240, 712), + 11, + false, + 0, + 1, + false, + List.of(right, left)); + + PageContent page = + new PageContent( + 0, + List.of(line), + List.of(), + 2, + false, + false, + false, + new BBox(0, 0, 595, 842)); + DocumentStructure structure = new LayoutAnalyzer().analyse(List.of(page)); + + boolean[] claimed = new boolean[2]; + structure.visit( + block -> { + if (block.isArtifact()) { + return; + } + block.getRanges() + .forEach( + r -> { + for (int i = r.start(); i <= r.end() && i < 2; i++) { + claimed[i] = true; + } + }); + }); + assertTrue( + claimed[0] && claimed[1], + "an out-of-order word was left unclaimed and would be hidden from assistive" + + " technology while the file still validated"); + } + + private static int countOf(String haystack, String needle) { + int total = 0; + int index = 0; + while ((index = haystack.indexOf(needle, index)) >= 0) { + total++; + index += needle.length(); + } + return total; + } + + private static byte[] bytes(PDDocument document) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + return out.toByteArray(); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaFormAndDeclarationTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaFormAndDeclarationTest.java new file mode 100644 index 0000000000..bae4ebed9a --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaFormAndDeclarationTest.java @@ -0,0 +1,171 @@ +package stirling.software.proprietary.pdf.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureElement; +import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureNode; +import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationWidget; +import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm; +import org.apache.pdfbox.pdmodel.interactive.form.PDTextField; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** Covers form-field descriptions, widget nesting and withdrawing a conformance claim. */ +class PdfUaFormAndDeclarationTest { + + /** A document with one named text field and one unnamed one. */ + private static PDDocument formDocument(boolean nameTheSecondField) throws IOException { + PDDocument document = new PDDocument(); + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + + PDAcroForm form = new PDAcroForm(document); + document.getDocumentCatalog().setAcroForm(form); + + PDTextField named = new PDTextField(form); + named.setPartialName("EmailAddress"); + addWidget(named, page, 700); + form.getFields().add(named); + + PDTextField second = new PDTextField(form); + if (nameTheSecondField) { + second.setPartialName("PostCode"); + } + addWidget(second, page, 650); + form.getFields().add(second); + + return document; + } + + private static void addWidget(PDTextField field, PDPage page, float y) throws IOException { + PDAnnotationWidget widget = field.getWidgets().get(0); + PDRectangle rectangle = new PDRectangle(); + rectangle.setLowerLeftX(50); + rectangle.setLowerLeftY(y); + rectangle.setUpperRightX(250); + rectangle.setUpperRightY(y + 18); + widget.setRectangle(rectangle); + widget.setPage(page); + page.getAnnotations().add(widget); + } + + private static String xmpOf(PDDocument document) throws IOException { + var metadata = document.getDocumentCatalog().getMetadata(); + assertNotNull(metadata, "no XMP packet"); + return new String(metadata.toByteArray(), StandardCharsets.UTF_8); + } + + @Test + @DisplayName("a form field gets its tooltip from its own name, not an invented one") + void derivesTooltipFromFieldName() throws Exception { + try (PDDocument document = formDocument(true)) { + List warnings = + new PdfUaMetadataWriter() + .applyDocumentRequirements(document, "Form", "en", PdfUaProfile.UA1); + + PDAcroForm form = document.getDocumentCatalog().getAcroForm(); + assertEquals("EmailAddress", form.getField("EmailAddress").getAlternateFieldName()); + assertEquals("PostCode", form.getField("PostCode").getAlternateFieldName()); + assertTrue(warnings.isEmpty(), "nothing needed reporting: " + warnings); + } + } + + @Test + @DisplayName("a field with no name is reported rather than given a placeholder tooltip") + void reportsUnnameableField() throws Exception { + try (PDDocument document = formDocument(false)) { + List warnings = + new PdfUaMetadataWriter() + .applyDocumentRequirements(document, "Form", "en", PdfUaProfile.UA1); + assertEquals(1, warnings.size()); + assertTrue(warnings.get(0).contains("form field"), warnings.get(0)); + } + } + + @Test + @DisplayName("an existing description is never overwritten") + void keepsExistingDescription() throws Exception { + try (PDDocument document = formDocument(true)) { + PDAcroForm form = document.getDocumentCatalog().getAcroForm(); + form.getField("EmailAddress").setAlternateFieldName("Your email address"); + + new PdfUaMetadataWriter() + .applyDocumentRequirements(document, "Form", "en", PdfUaProfile.UA1); + assertEquals( + "Your email address", form.getField("EmailAddress").getAlternateFieldName()); + } + } + + @Test + @DisplayName("widget annotations are nested inside a Form structure element") + void widgetsAreNestedInFormElements() throws Exception { + try (PDDocument document = formDocument(true)) { + DocumentStructure structure = new DocumentStructure(); + StructBlock paragraph = new StructBlock(StructType.P, 0); + paragraph.getMcids().add(0); + structure.add(paragraph); + + new StructTreeWriter().write(document, structure, PdfUaProfile.UA1); + + var root = document.getDocumentCatalog().getStructureTreeRoot(); + assertTrue( + typesUnder(root).contains("Form"), + "clause 7.18.4 requires a widget to sit inside a Form element, found: " + + typesUnder(root)); + } + } + + private static List typesUnder(PDStructureNode node) { + List types = new java.util.ArrayList<>(); + for (Object kid : node.getKids()) { + if (kid instanceof PDStructureElement element) { + types.add(element.getStructureType()); + types.addAll(typesUnder(element)); + } + } + return types; + } + + @Test + @DisplayName("withdrawing conformance removes the claim but keeps the other metadata") + void withdrawingConformanceRemovesOnlyTheClaim() throws Exception { + try (PDDocument document = new PDDocument()) { + document.addPage(new PDPage()); + PdfUaTagger tagger = new PdfUaTagger(); + PdfUaMetadataWriter writer = new PdfUaMetadataWriter(); + + writer.applyDocumentRequirements(document, "Kept Title", "en-GB", PdfUaProfile.UA1); + writer.declareConformance(document, PdfUaProfile.UA1); + assertTrue(xmpOf(document).contains("pdfuaid")); + + tagger.withdrawConformance(document); + + String xmp = xmpOf(document); + assertFalse(xmp.contains("pdfuaid"), "the conformance claim should be gone"); + assertTrue(xmp.contains("Kept Title"), "the title should survive"); + assertEquals("en-GB", document.getDocumentCatalog().getLanguage()); + } + } + + @Test + @DisplayName("withdrawing conformance on a document that never claimed it is harmless") + void withdrawingIsIdempotent() throws Exception { + try (PDDocument document = new PDDocument()) { + document.addPage(new PDPage()); + new PdfUaMetadataWriter() + .applyDocumentRequirements(document, "Title", "en", PdfUaProfile.UA1); + new PdfUaTagger().withdrawConformance(document); + assertFalse(xmpOf(document).contains("pdfuaid")); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaLanguageTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaLanguageTest.java new file mode 100644 index 0000000000..ed05e8de0b --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaLanguageTest.java @@ -0,0 +1,79 @@ +package stirling.software.proprietary.pdf.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * A relabelled language is invisible to every validator, so the tagger must not guess over one the + * document already declares. + */ +class PdfUaLanguageTest { + + private static PDDocument documentWithLanguage(String language) { + PDDocument document = new PDDocument(); + document.addPage(new PDPage()); + if (language != null) { + document.getDocumentCatalog().setLanguage(language); + } + return document; + } + + private static TaggingOptions.TaggingOptionsBuilder options() { + return TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language("en-GB") + .title("Rapport") + .embedFonts(false); + } + + @Test + @DisplayName("keeps the language the document already declares") + void keepsExistingLanguage() throws Exception { + try (PDDocument document = documentWithLanguage("fr-FR")) { + TaggingResult result = new PdfUaTagger().tag(document, options().build()); + + assertEquals("fr-FR", document.getDocumentCatalog().getLanguage()); + assertTrue( + result.getWarnings().stream().anyMatch(w -> w.contains("fr-FR")), + "ignoring the requested language must be reported: " + result.getWarnings()); + } + } + + @Test + @DisplayName("applies the requested language when the document declares none") + void fillsInMissingLanguage() throws Exception { + try (PDDocument document = documentWithLanguage(null)) { + new PdfUaTagger().tag(document, options().build()); + + assertEquals("en-GB", document.getDocumentCatalog().getLanguage()); + } + } + + @Test + @DisplayName("replaces the declared language only when the caller asks") + void overridesOnRequest() throws Exception { + try (PDDocument document = documentWithLanguage("fr-FR")) { + new PdfUaTagger().tag(document, options().overrideLanguage(true).build()); + + assertEquals("en-GB", document.getDocumentCatalog().getLanguage()); + } + } + + @Test + @DisplayName("keeps the existing language when an existing structure tree is left alone") + void keepsExistingLanguageWithoutRebuilding() throws Exception { + try (PDDocument document = documentWithLanguage("de-DE")) { + new PdfUaTagger() + .tag( + document, + options().existingTags(TaggingOptions.ExistingTags.KEEP).build()); + + assertEquals("de-DE", document.getDocumentCatalog().getLanguage()); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaMetadataWriterTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaMetadataWriterTest.java new file mode 100644 index 0000000000..f953488b03 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaMetadataWriterTest.java @@ -0,0 +1,137 @@ +package stirling.software.proprietary.pdf.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** Tests the document-level requirements that have nothing to do with tagging. */ +class PdfUaMetadataWriterTest { + + private static PDDocument twoPageDocument() { + PDDocument document = new PDDocument(); + document.addPage(new PDPage()); + document.addPage(new PDPage()); + return document; + } + + private static String xmpOf(PDDocument document) throws IOException { + var metadata = document.getDocumentCatalog().getMetadata(); + assertNotNull(metadata, "no XMP packet was written"); + return new String(metadata.toByteArray(), StandardCharsets.UTF_8); + } + + @Test + @DisplayName("sets title, language, tab order and the display-title flag") + void appliesDocumentRequirements() throws Exception { + try (PDDocument document = twoPageDocument()) { + new PdfUaMetadataWriter() + .applyDocumentRequirements( + document, "Annual Report", "en-GB", PdfUaProfile.UA1); + + assertEquals("en-GB", document.getDocumentCatalog().getLanguage()); + assertEquals("Annual Report", document.getDocumentInformation().getTitle()); + assertTrue( + document.getDocumentCatalog().getViewerPreferences().displayDocTitle(), + "without DisplayDocTitle a viewer shows the filename instead of the title"); + + for (PDPage page : document.getPages()) { + assertEquals( + "S", + page.getCOSObject().getNameAsString(COSName.getPDFName("Tabs")), + "clause 7.18.1 requires an explicit tab order on every page"); + } + } + } + + @Test + @DisplayName("writes dc:title into the XMP packet, not just the info dictionary") + void writesDublinCoreTitle() throws Exception { + try (PDDocument document = twoPageDocument()) { + new PdfUaMetadataWriter() + .applyDocumentRequirements( + document, "Annual Report", "en-GB", PdfUaProfile.UA1); + assertTrue(xmpOf(document).contains("Annual Report")); + } + } + + @Test + @DisplayName("does not declare conformance as part of applying requirements") + void doesNotDeclareEarly() throws Exception { + try (PDDocument document = twoPageDocument()) { + new PdfUaMetadataWriter() + .applyDocumentRequirements(document, "Report", "en", PdfUaProfile.UA1); + assertFalse( + xmpOf(document).contains("pdfuaid"), + "the conformance claim must wait until validation has passed"); + } + } + + @Test + @DisplayName("declaring conformance writes pdfuaid with the right part") + void declaresConformance() throws Exception { + try (PDDocument document = twoPageDocument()) { + PdfUaMetadataWriter writer = new PdfUaMetadataWriter(); + writer.applyDocumentRequirements(document, "Report", "en", PdfUaProfile.UA1); + writer.declareConformance(document, PdfUaProfile.UA1); + + String xmp = xmpOf(document); + assertTrue(xmp.contains("pdfuaid"), "no PDF/UA identifier was written"); + assertTrue(xmp.contains("part"), "no conformance part was written"); + } + } + + @Test + @DisplayName("UA-2 raises the PDF version to 2.0") + void ua2RaisesVersion() throws Exception { + try (PDDocument document = twoPageDocument()) { + new PdfUaMetadataWriter() + .applyDocumentRequirements(document, "Report", "en", PdfUaProfile.UA2); + assertEquals(2.0f, document.getVersion()); + } + } + + @Test + @DisplayName("keeps an existing title when none is supplied") + void keepsExistingTitle() throws Exception { + try (PDDocument document = twoPageDocument()) { + var info = document.getDocumentInformation(); + info.setTitle("Original Title"); + document.setDocumentInformation(info); + + new PdfUaMetadataWriter() + .applyDocumentRequirements(document, null, "en", PdfUaProfile.UA1); + assertEquals("Original Title", document.getDocumentInformation().getTitle()); + } + } + + @Test + @DisplayName("survives a round trip through save and reload") + void survivesRoundTrip() throws Exception { + byte[] saved; + try (PDDocument document = twoPageDocument()) { + PdfUaMetadataWriter writer = new PdfUaMetadataWriter(); + writer.applyDocumentRequirements(document, "Round Trip", "fr-FR", PdfUaProfile.UA1); + writer.declareConformance(document, PdfUaProfile.UA1); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + saved = out.toByteArray(); + } + try (PDDocument reloaded = Loader.loadPDF(saved)) { + assertEquals("fr-FR", reloaded.getDocumentCatalog().getLanguage()); + assertEquals("Round Trip", reloaded.getDocumentInformation().getTitle()); + assertTrue(xmpOf(reloaded).contains("pdfuaid")); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaModelTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaModelTest.java new file mode 100644 index 0000000000..de189f2f74 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/PdfUaModelTest.java @@ -0,0 +1,160 @@ +package stirling.software.proprietary.pdf.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** Tests for the small types the tagger is built from. */ +class PdfUaModelTest { + + @Nested + @DisplayName("structure types") + class Types { + + @Test + @DisplayName("maps levels to heading tags and back") + void headingLevelsRoundTrip() { + for (int level = 1; level <= 6; level++) { + assertEquals(level, StructType.heading(level).headingLevel()); + assertEquals("H" + level, StructType.heading(level).tag()); + } + } + + @Test + @DisplayName("clamps out-of-range levels rather than throwing") + void clampsLevels() { + assertEquals(StructType.H1, StructType.heading(0)); + assertEquals(StructType.H1, StructType.heading(-3)); + assertEquals(StructType.H6, StructType.heading(9)); + } + + @Test + @DisplayName("reports zero for types that are not headings") + void nonHeadingsHaveNoLevel() { + assertEquals(0, StructType.P.headingLevel()); + assertFalse(StructType.TABLE.isHeading()); + } + } + + @Nested + @DisplayName("markable operators") + class Markable { + + @ParameterizedTest + @ValueSource(strings = {"Tj", "TJ", "'", "\"", "Do", "BI", "S", "f", "f*", "B", "sh"}) + @DisplayName("counts text, XObjects and path painting") + void counted(String operator) { + assertTrue(MarkableOp.isMarkableOperator(operator), operator + " should be markable"); + } + + @ParameterizedTest + @ValueSource(strings = {"q", "Q", "cm", "BT", "ET", "Tf", "Td", "n", "W", "gs", "re"}) + @DisplayName("ignores operators that paint nothing") + void notCounted(String operator) { + assertFalse( + MarkableOp.isMarkableOperator(operator), operator + " should not be markable"); + } + + @Test + @DisplayName("n ends a path without painting, so it is not content") + void pathEndIsNotPainting() { + assertFalse(MarkableOp.isPathPainting("n")); + assertTrue(MarkableOp.isPathPainting("f")); + } + } + + @Nested + @DisplayName("profiles") + class Profiles { + + @Test + @DisplayName("parses the shapes a caller might send") + void parsesRequestValues() { + assertEquals(PdfUaProfile.UA1, PdfUaProfile.fromRequest("ua1")); + assertEquals(PdfUaProfile.UA1, PdfUaProfile.fromRequest(null)); + assertEquals(PdfUaProfile.UA1, PdfUaProfile.fromRequest("")); + assertEquals(PdfUaProfile.UA1, PdfUaProfile.fromRequest("nonsense")); + assertEquals(PdfUaProfile.UA2, PdfUaProfile.fromRequest("ua2")); + assertEquals(PdfUaProfile.UA2, PdfUaProfile.fromRequest("PDF/UA-2")); + } + + @Test + @DisplayName("UA-2 requires PDF 2.0") + void ua2NeedsPdf2() { + assertEquals(2.0f, PdfUaProfile.UA2.pdfVersion()); + assertEquals(1.7f, PdfUaProfile.UA1.pdfVersion()); + } + } + + @Nested + @DisplayName("bounding boxes") + class Boxes { + + @Test + @DisplayName("union of an empty box is the other box") + void unionWithEmpty() { + BBox box = new BBox(10, 10, 20, 20); + assertEquals(box, box.union(BBox.EMPTY)); + assertEquals(box, BBox.EMPTY.union(box)); + } + + @Test + @DisplayName("union covers both boxes") + void unionCoversBoth() { + BBox union = new BBox(0, 0, 10, 10).union(new BBox(20, 5, 30, 25)); + assertEquals(new BBox(0, 0, 30, 25), union); + } + + @Test + @DisplayName("reports horizontal overlap as a fraction of the narrower box") + void overlapIsRelative() { + BBox wide = new BBox(0, 0, 100, 10); + BBox narrow = new BBox(40, 0, 60, 10); + assertEquals(1.0f, wide.horizontalOverlap(narrow)); + assertEquals(0f, wide.horizontalOverlap(new BBox(200, 0, 220, 10))); + } + } + + @Nested + @DisplayName("structure blocks") + class Blocks { + + @Test + @DisplayName("counts content across the whole subtree") + void countsDescendantContent() { + StructBlock table = new StructBlock(StructType.TABLE, 0); + StructBlock row = new StructBlock(StructType.TR, 0); + StructBlock cell = new StructBlock(StructType.TD, 0); + cell.addRange(3, 5); + row.addChild(cell); + table.addChild(row); + assertEquals(3, table.contentCount()); + } + + @Test + @DisplayName("collects text in tree order") + void collectsText() { + StructBlock list = new StructBlock(StructType.L, 0); + StructBlock first = new StructBlock(StructType.LI, 0); + first.setText("one"); + StructBlock second = new StructBlock(StructType.LI, 0); + second.setText("two"); + list.addChild(first).addChild(second); + assertEquals("one two", list.collectText()); + } + + @Test + @DisplayName("an artifact is not a structure element") + void artifactsAreDistinct() { + StructBlock artifact = StructBlock.artifact(ArtifactType.PAGINATION, 0); + assertTrue(artifact.isArtifact()); + assertEquals("Pagination", artifact.getArtifactType().subtype()); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/VectorAndHeadingTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/VectorAndHeadingTest.java new file mode 100644 index 0000000000..aa1b13e3b9 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/pdf/ua/VectorAndHeadingTest.java @@ -0,0 +1,155 @@ +package stirling.software.proprietary.pdf.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** Tests the heuristics that decide what counts as a drawing and what counts as a heading. */ +class VectorAndHeadingTest { + + private static final BBox A4 = new BBox(0, 0, 595, 842); + + private static TextLineInfo line(String text, float size, float x, float y) { + List words = new ArrayList<>(); + float cursor = x; + for (String token : text.strip().split("\\s+")) { + float width = token.length() * size * 0.5f; + words.add( + new WordInfo( + token, + new BBox(cursor, y, cursor + width, y + size), + 0, + 0, + size, + false)); + cursor += width + size * 0.3f; + } + return new TextLineInfo( + 0, text, new BBox(x, y, cursor, y + size), size, false, 0, 0, false, words); + } + + private static MarkableOp vector(int ordinal, BBox box) { + return new MarkableOp(ordinal, MarkableOp.Kind.VECTOR, box, null); + } + + private static DocumentStructure analyse(List lines, List ops) { + PageContent page = new PageContent(0, lines, ops, ops.size(), false, false, false, A4); + return new LayoutAnalyzer().analyse(List.of(page)); + } + + private static long countOf(DocumentStructure structure, StructType type) { + long[] total = {0}; + structure.visit( + block -> { + if (block.getType() == type) { + total[0]++; + } + }); + return total[0]; + } + + @Test + @DisplayName("a cluster of substantial strokes becomes a figure, not silent decoration") + void chartBecomesAFigure() { + List bars = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + bars.add(vector(i, new BBox(100 + i * 20, 400, 115 + i * 20, 400 + 30 + i * 10))); + } + DocumentStructure structure = analyse(List.of(), bars); + + assertTrue( + countOf(structure, StructType.FIGURE) > 0, + "a bar chart drawn with path operators must not vanish as decoration"); + assertTrue( + structure.figuresWithoutAlt().size() > 0, + "the report must say the chart needs a description"); + } + + @Test + @DisplayName("thin rules and table borders stay artifacts") + void tableRulesStayDecoration() { + List rules = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + rules.add(vector(i, new BBox(60, 700 - i * 20, 540, 701 - i * 20))); + } + DocumentStructure structure = analyse(List.of(), rules); + + assertEquals( + 0, + countOf(structure, StructType.FIGURE), + "horizontal rules are page furniture and must not demand alt text"); + assertEquals(0, structure.figuresWithoutAlt().size()); + } + + @Test + @DisplayName("a lone box is ornament, not a chart") + void singleBoxIsNotAFigure() { + DocumentStructure structure = + analyse(List.of(), List.of(vector(0, new BBox(60, 400, 500, 700)))); + assertEquals(0, countOf(structure, StructType.FIGURE)); + } + + @Test + @DisplayName("shaded table rows behind text are not mistaken for a chart") + void shadedTableRowsAreNotFigures() { + List shading = new ArrayList<>(); + List rows = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + float y = 600 - i * 20; + // A filled row background, tall enough to pass the thinness test. + shading.add(vector(i, new BBox(60, y, 540, y + 16))); + rows.add(line("Expense line item " + i + " amount", 10, 64, y + 3)); + } + DocumentStructure structure = analyse(rows, shading); + + assertEquals( + 0, + countOf(structure, StructType.FIGURE), + "row shading sits behind the text it decorates and is not a drawing"); + } + + @Test + @DisplayName("small print dominating an invoice does not promote addresses to headings") + void smallPrintDoesNotCreateHeadings() { + List lines = new ArrayList<>(); + // Address block at ordinary 11pt. + lines.add(line("Acme Industries Limited", 11, 60, 780)); + lines.add(line("14 Example Street", 11, 60, 765)); + lines.add(line("Manchester M1 2AB", 11, 60, 750)); + // 40 lines of 9pt line-item small print, which dominates the character count. + for (int i = 0; i < 40; i++) { + lines.add(line("Item " + i + " widget assembly part number " + i, 9, 60, 700 - i * 12)); + } + + Map tiers = + LayoutAnalyzer.headingTiers( + List.of(new PageContent(0, lines, List.of(), 0, false, false, false, A4)), + 9f); + assertNull( + tiers.get(11f), + "11pt address lines are body text on an invoice, not headings: " + tiers); + } + + @Test + @DisplayName("a genuinely rare large size is still a heading") + void realHeadingsSurvive() { + List lines = new ArrayList<>(); + lines.add(line("Annual Report", 24, 60, 780)); + for (int i = 0; i < 40; i++) { + lines.add( + line("Body prose line number " + i + " continues here", 11, 60, 700 - i * 12)); + } + Map tiers = + LayoutAnalyzer.headingTiers( + List.of(new PageContent(0, lines, List.of(), 0, false, false, false, A4)), + 11f); + assertEquals(1, tiers.get(24f), "a rare large size is exactly what a heading looks like"); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/AltTextRoundTripTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/AltTextRoundTripTest.java new file mode 100644 index 0000000000..0f8e52d9ec --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/AltTextRoundTripTest.java @@ -0,0 +1,115 @@ +package stirling.software.proprietary.service.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Map; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.proprietary.controller.api.converters.ConvertPdfToPdfUa; +import stirling.software.proprietary.model.api.ua.AccessibilityReport; +import stirling.software.proprietary.model.api.ua.FigureDescriptor; +import stirling.software.proprietary.model.api.ua.PdfUaConversionOutcome; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.pdf.ua.TaggingOptions; + +/** + * The alt-text loop end to end: the report hands out keys the conversion accepts. The converter + * never invents descriptions, so a caller must be able to supply them. + */ +class AltTextRoundTripTest { + + private static PdfUaConversionService conversion; + private static AccessibilityAuditService audit; + + @BeforeAll + static void setUp() { + PdfUaValidationService validation = new PdfUaValidationService(); + validation.initialise(); + conversion = + new PdfUaConversionService( + validation, + new FontEmbeddingService(), + new CustomPDFDocumentFactory( + org.mockito.Mockito.mock(PdfMetadataService.class))); + audit = new AccessibilityAuditService(validation); + } + + private static TaggingOptions.TaggingOptionsBuilder options() { + return TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language("en-GB") + .title("Illustrated") + .embedFonts(false); + } + + @Test + @DisplayName("the report names the figures that need describing, with usable keys") + void reportEnumeratesFigures() throws Exception { + byte[] input = PdfUaTestDocuments.imageDocument(); + AccessibilityReport report = audit.audit(input, PdfUaProfile.UA1); + + assertFalse( + report.getFiguresNeedingDescription().isEmpty(), + "a document with an undescribed image must say which figure needs text"); + + FigureDescriptor figure = report.getFiguresNeedingDescription().get(0); + assertTrue(figure.key().matches("\\d+:\\d+"), "key should be pageIndex:ordinal: " + figure); + assertEquals(1, figure.page(), "pages are reported 1-based for humans"); + assertTrue(figure.width() > 0 && figure.height() > 0, "figure should carry its box"); + } + + @Test + @DisplayName("feeding the report's key back makes the document conform") + void suppliedDescriptionClosesTheLoop() throws Exception { + byte[] input = PdfUaTestDocuments.imageDocument(); + + PdfUaConversionOutcome before = conversion.convert(input, options().build()); + assertFalse(before.declared(), "an undescribed image must block the claim"); + + String key = + audit.audit(input, PdfUaProfile.UA1).getFiguresNeedingDescription().get(0).key(); + PdfUaConversionOutcome after = + conversion.convert( + input, options().altTextByFigure(Map.of(key, "A blue rectangle")).build()); + + assertEquals( + 0, + after.tagging().figuresNeedingAltText(), + "the description supplied against the report's own key was not applied"); + assertTrue(after.declared(), "with every figure described the document should conform"); + } + + @Test + @DisplayName("the request's key=text form parses the way the report emits keys") + void parsesTheWireFormat() { + Map parsed = + ConvertPdfToPdfUa.parseAltText( + "0:12=Bar chart of quarterly revenue\r\n" + + "1:3=Company logo\n" + + " \n" + + "malformed-line\n" + + "2:7=Diagram showing the approval flow = end to end"); + + assertEquals(3, parsed.size(), "blank and malformed lines are skipped: " + parsed); + assertEquals("Bar chart of quarterly revenue", parsed.get("0:12")); + assertEquals("Company logo", parsed.get("1:3")); + assertEquals( + "Diagram showing the approval flow = end to end", + parsed.get("2:7"), + "only the first equals splits, so descriptions may contain one"); + } + + @Test + @DisplayName("no descriptions supplied means none invented") + void emptyInputInventsNothing() { + assertTrue(ConvertPdfToPdfUa.parseAltText(null).isEmpty()); + assertTrue(ConvertPdfToPdfUa.parseAltText(" ").isEmpty()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUa2ProfileTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUa2ProfileTest.java new file mode 100644 index 0000000000..9cf342824f --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUa2ProfileTest.java @@ -0,0 +1,92 @@ +package stirling.software.proprietary.service.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.proprietary.model.api.ua.PdfUaConversionOutcome; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.pdf.ua.TaggingOptions; + +/** PDF/UA-2 is not just a metadata number: it needs PDF 2.0 and namespaced structure types. */ +class PdfUa2ProfileTest { + + private static PdfUaConversionService conversion; + + @BeforeAll + static void setUp() { + PdfUaValidationService validation = new PdfUaValidationService(); + validation.initialise(); + conversion = + new PdfUaConversionService( + validation, + new FontEmbeddingService(), + new CustomPDFDocumentFactory( + org.mockito.Mockito.mock(PdfMetadataService.class))); + } + + private static PdfUaConversionOutcome convertUa2(byte[] input) throws Exception { + return conversion.convert( + input, + TaggingOptions.builder() + .profile(PdfUaProfile.UA2) + .language("en-GB") + .title("UA-2 Document") + .embedFonts(false) + .build()); + } + + @Test + @DisplayName("raises the file to PDF 2.0 and namespaces the structure tree") + void producesPdf2WithNamespaces() throws Exception { + PdfUaConversionOutcome outcome = convertUa2(PdfUaTestDocuments.headingHierarchy()); + + try (PDDocument document = Loader.loadPDF(outcome.pdfBytes())) { + assertEquals(2.0f, document.getVersion(), "UA-2 is defined on PDF 2.0"); + + var root = document.getDocumentCatalog().getStructureTreeRoot(); + assertNotNull(root, "no structure tree was written"); + assertNotNull( + root.getCOSObject().getDictionaryObject(COSName.getPDFName("Namespaces")), + "UA-2 requires the standard structure namespace to be declared"); + } + } + + @Test + @DisplayName("validates against the PDF/UA-2 profile, not the UA-1 one") + void validatesAgainstUa2() throws Exception { + PdfUaConversionOutcome outcome = convertUa2(PdfUaTestDocuments.simpleDocument()); + assertEquals("PDF/UA-2", outcome.validation().profile()); + } + + @Test + @DisplayName("reaches UA-2 conformance and declares it") + void reachesUa2Conformance() throws Exception { + PdfUaConversionOutcome outcome = convertUa2(PdfUaTestDocuments.simpleDocument()); + + String failures = + outcome.validation().issues().stream() + .map(issue -> issue.getClause() + ": " + issue.getTechnicalMessage()) + .collect(java.util.stream.Collectors.joining("; ")); + assertEquals(0, outcome.validation().totalFailures(), "UA-2 checks failed: " + failures); + assertTrue(outcome.declared(), "a conforming UA-2 file must carry the declaration"); + assertTrue(outcome.pdfBytes().length > 0); + } + + @Test + @DisplayName("an illustrated document still cannot claim UA-2 without descriptions") + void undescribedImageBlocksTheUa2Claim() throws Exception { + PdfUaConversionOutcome outcome = convertUa2(PdfUaTestDocuments.imageDocument()); + assertFalse(outcome.declared(), "an undescribed image must block the claim"); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaBenchmarkTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaBenchmarkTest.java new file mode 100644 index 0000000000..2c4ca6db2a --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaBenchmarkTest.java @@ -0,0 +1,341 @@ +package stirling.software.proprietary.service.ua; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.apache.pdfbox.pdmodel.font.PDFont; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.proprietary.pdf.ua.DocumentStructure; +import stirling.software.proprietary.pdf.ua.LayoutAnalyzer; +import stirling.software.proprietary.pdf.ua.PageContent; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.pdf.ua.PdfUaTagger; +import stirling.software.proprietary.pdf.ua.TaggedContentExtractor; +import stirling.software.proprietary.pdf.ua.TaggingOptions; + +/** + * Measures where conversion time and memory go; meant to be read, not to gate CI. Assertions catch + * only order-of-magnitude regressions - wall-clock numbers are no contract. + */ +class PdfUaBenchmarkTest { + + private static PdfUaConversionService service; + private static PdfUaValidationService validation; + + @BeforeAll + static void setUp() { + validation = new PdfUaValidationService(); + validation.initialise(); + service = + new PdfUaConversionService( + validation, + new FontEmbeddingService(), + new CustomPDFDocumentFactory( + org.mockito.Mockito.mock(PdfMetadataService.class))); + } + + /** A realistic page: heading, prose, a small table, a bullet list. */ + private static byte[] document(int pages) throws IOException { + try (PDDocument document = new PDDocument()) { + PDFont font = null; + for (int p = 0; p < pages; p++) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + if (font == null) { + font = PdfUaTestDocuments.font(document); + } + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + float y = 790; + write(cs, font, 9, 60, 810, "Benchmark Corpus Running Head"); + write(cs, font, 18, 60, y, "Section " + (p + 1)); + y -= 30; + for (int line = 0; line < 22; line++) { + write( + cs, + font, + 11, + 60, + y, + "Body line " + line + " of section " + (p + 1) + " with prose."); + y -= 15; + } + for (int row = 0; row < 4; row++) { + cs.beginText(); + cs.setFont(font, 11); + cs.newLineAtOffset(60, y); + cs.showText("Row " + row); + cs.newLineAtOffset(160, 0); + cs.showText(String.valueOf(row * 120)); + cs.newLineAtOffset(140, 0); + cs.showText(String.valueOf(row * 480)); + cs.endText(); + y -= 16; + } + write(cs, font, 11, 60, y - 10, "• First bullet point"); + write(cs, font, 11, 60, y - 25, "• Second bullet point"); + write(cs, font, 9, 300, 30, "Page " + (p + 1)); + } + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + return out.toByteArray(); + } + } + + private static void write( + PDPageContentStream cs, PDFont font, float size, float x, float y, String text) + throws IOException { + cs.beginText(); + cs.setFont(font, size); + cs.newLineAtOffset(x, y); + cs.showText(text); + cs.endText(); + } + + private static TaggingOptions options() { + return TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language("en-GB") + .title("Benchmark") + .embedFonts(false) + .existingTags(TaggingOptions.ExistingTags.REBUILD) + .build(); + } + + private static long usedHeap() { + Runtime runtime = Runtime.getRuntime(); + System.gc(); + return runtime.totalMemory() - runtime.freeMemory(); + } + + @Test + @DisplayName("reports throughput and memory across document sizes") + void throughputAcrossSizes() throws Exception { + int[] sizes = {1, 10, 50, 150}; + StringBuilder report = + new StringBuilder("\nPDF/UA conversion throughput\n") + .append( + String.format( + " %-7s %-10s %-12s %-12s %-10s %s%n", + "pages", + "input", + "convert ms", + "ms/page", + "pages/s", + "heap MB")); + + // Warm up so the first timed run is not measuring class loading and JIT. + service.convert(document(5), options()); + + for (int pages : sizes) { + byte[] input = document(pages); + long heapBefore = usedHeap(); + long start = System.nanoTime(); + var outcome = service.convert(input, options()); + long elapsedMs = (System.nanoTime() - start) / 1_000_000; + long heapDelta = (usedHeap() - heapBefore) / (1024 * 1024); + + assertTrue(outcome.pdfBytes().length > 0); + report.append( + String.format( + Locale.ROOT, + " %-7d %-10s %-12d %-12.2f %-10.1f %d%n", + pages, + humanBytes(input.length), + elapsedMs, + elapsedMs / (double) pages, + pages * 1000.0 / Math.max(elapsedMs, 1), + Math.max(heapDelta, 0))); + } + System.out.println(report); + } + + @Test + @DisplayName("breaks conversion down by phase so optimisation has a target") + void phaseBreakdown() throws Exception { + byte[] input = document(60); + + // Warm up. + try (PDDocument warm = Loader.loadPDF(input)) { + new TaggedContentExtractor().extract(warm); + } + + long parseMs; + long extractMs; + long analyseMs; + long tagMs; + List pages; + DocumentStructure structure; + + long t0 = System.nanoTime(); + try (PDDocument document = Loader.loadPDF(input)) { + parseMs = ms(t0); + + long t1 = System.nanoTime(); + pages = new TaggedContentExtractor().extract(document); + extractMs = ms(t1); + + long t2 = System.nanoTime(); + structure = new LayoutAnalyzer().analyse(pages); + analyseMs = ms(t2); + } + + long t3 = System.nanoTime(); + try (PDDocument document = Loader.loadPDF(input)) { + new PdfUaTagger().tag(document, options()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + } + tagMs = ms(t3); + + long t4 = System.nanoTime(); + var outcome = service.convert(input, options()); + long totalMs = ms(t4); + + long t5 = System.nanoTime(); + validation.validate(outcome.pdfBytes(), PdfUaProfile.UA1); + long validateMs = ms(t5); + + System.out.printf( + Locale.ROOT, + "%nPhase breakdown over %d pages (%d blocks)%n" + + " parse %5d ms%n" + + " extract %5d ms (text pass + token scan)%n" + + " analyse %5d ms%n" + + " tag end-to-end %5d ms (includes parse, extract, analyse, inject, write)%n" + + " validate %5d ms (veraPDF)%n" + + " full convert %5d ms (tag + declare + validate)%n", + 60, + structure.getBlocks().size(), + parseMs, + extractMs, + analyseMs, + tagMs, + validateMs, + totalMs); + + assertTrue(pages.size() == 60, "extractor lost pages"); + } + + @Test + @DisplayName("splits the tagging pass into its own sub-phases") + void taggingSubPhases() throws Exception { + byte[] input = document(60); + try (PDDocument warm = Loader.loadPDF(input)) { + new TaggedContentExtractor().extract(warm); + } + + long extractMs; + long analyseMs; + long injectMs; + long treeMs; + long saveMs; + + try (PDDocument document = Loader.loadPDF(input)) { + long t = System.nanoTime(); + List pages = new TaggedContentExtractor().extract(document); + extractMs = ms(t); + + t = System.nanoTime(); + DocumentStructure structure = new LayoutAnalyzer().analyse(pages); + analyseMs = ms(t); + + t = System.nanoTime(); + var injector = new stirling.software.proprietary.pdf.ua.MarkedContentInjector(); + var byPage = + new java.util.LinkedHashMap< + Integer, List>(); + structure + .getBlocks() + .forEach( + b -> + byPage.computeIfAbsent(b.getPageIndex(), k -> new ArrayList<>()) + .add(b)); + for (int p = 0; p < document.getNumberOfPages(); p++) { + injector.inject( + document, document.getPage(p), byPage.getOrDefault(p, List.of()), 0, true); + } + injectMs = ms(t); + + t = System.nanoTime(); + new stirling.software.proprietary.pdf.ua.StructTreeWriter() + .write(document, structure, PdfUaProfile.UA1); + treeMs = ms(t); + + t = System.nanoTime(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + saveMs = ms(t); + } + + System.out.printf( + Locale.ROOT, + "%nTagging sub-phases over 60 pages%n" + + " extract %5d ms%n" + + " analyse %5d ms%n" + + " inject %5d ms%n" + + " struct tree %5d ms%n" + + " save %5d ms%n", + extractMs, + analyseMs, + injectMs, + treeMs, + saveMs); + } + + @Test + @DisplayName("memory stays proportional to document size, not quadratic") + void memoryScales() throws Exception { + List rows = new ArrayList<>(); + long previousPerPage = 0; + boolean blewUp = false; + + for (int pages : new int[] {20, 80, 200}) { + byte[] input = document(pages); + long before = usedHeap(); + var outcome = service.convert(input, options()); + long after = usedHeap(); + long perPageKb = Math.max(after - before, 0) / 1024 / pages; + rows.add( + String.format( + Locale.ROOT, + " %-6d pages in %-9s out %-9s ~%d KB/page retained", + pages, + humanBytes(input.length), + humanBytes(outcome.pdfBytes().length), + perPageKb)); + // Per-page cost should stay roughly flat; a big jump means something accumulates. + if (previousPerPage > 0 && perPageKb > previousPerPage * 4 && perPageKb > 200) { + blewUp = true; + } + previousPerPage = Math.max(perPageKb, 1); + } + System.out.println("\nMemory scaling\n" + String.join("\n", rows)); + assertTrue(!blewUp, "per-page memory grew superlinearly: " + rows); + } + + private static long ms(long startNanos) { + return (System.nanoTime() - startNanos) / 1_000_000; + } + + private static String humanBytes(int bytes) { + return bytes < 1024 * 1024 + ? (bytes / 1024) + " KB" + : String.format(Locale.ROOT, "%.1f MB", bytes / 1024.0 / 1024.0); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaConversionIntegrationTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaConversionIntegrationTest.java new file mode 100644 index 0000000000..e785957b7b --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaConversionIntegrationTest.java @@ -0,0 +1,231 @@ +package stirling.software.proprietary.service.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.Callable; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.text.PDFTextStripper; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.proprietary.model.api.ua.AccessibilityIssue; +import stirling.software.proprietary.model.api.ua.PdfUaConversionOutcome; +import stirling.software.proprietary.model.api.ua.UaValidationResult; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.pdf.ua.TaggingOptions; + +/** + * End-to-end conversion over the fixture corpus, validated with veraPDF. The corpus is deliberately + * varied: what breaks a tagger is rarely the simple case. + */ +class PdfUaConversionIntegrationTest { + + private static PdfUaConversionService service; + + @BeforeAll + static void setUp() { + PdfUaValidationService validation = new PdfUaValidationService(); + validation.initialise(); + service = + new PdfUaConversionService( + validation, + new FontEmbeddingService(), + new CustomPDFDocumentFactory( + org.mockito.Mockito.mock(PdfMetadataService.class))); + } + + /** Fixtures already embed fonts, so the Ghostscript pass is off to keep tests hermetic. */ + private static TaggingOptions options() { + return TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language("en-GB") + .title("Test Document") + .embedFonts(false) + .build(); + } + + private static PdfUaConversionOutcome convert(byte[] input) throws IOException { + return service.convert(input, options()); + } + + private static String extractText(byte[] pdf) throws IOException { + try (PDDocument document = Loader.loadPDF(pdf)) { + PDFTextStripper stripper = new PDFTextStripper(); + stripper.setSortByPosition(true); + return PdfUaRealCorpusTest.normalise(stripper.getText(document)); + } + } + + @Test + @DisplayName("every fixture converts without error and gains a structure tree") + void corpusConverts() throws Exception { + Map> corpus = corpus(); + StringBuilder report = new StringBuilder("\nPDF/UA conversion over the fixture corpus\n"); + + for (Map.Entry> entry : corpus.entrySet()) { + byte[] input = + entry.getKey().equals("empty") + ? entry.getValue().call() + : entry.getValue().call(); + PdfUaConversionOutcome outcome = convert(input); + + assertNotNull(outcome.pdfBytes(), entry.getKey() + " produced no output"); + report.append( + String.format( + " %-18s declared=%-5s failures=%-3d elements=%-3d artifacts=%-3d altNeeded=%d%n", + entry.getKey(), + outcome.declared(), + outcome.validation().totalFailures(), + outcome.tagging().taggedElements(), + outcome.tagging().artifacts(), + outcome.tagging().figuresNeedingAltText())); + for (AccessibilityIssue issue : outcome.validation().issues()) { + report.append( + String.format( + " clause %-6s x%-4d %s%n", + issue.getClause(), + issue.getOccurrences(), + issue.getTechnicalMessage())); + } + } + System.out.println(report); + } + + @Test + @DisplayName("tagging never changes the text content of a page") + void textIsPreserved() throws Exception { + for (Map.Entry> entry : corpus().entrySet()) { + byte[] input = entry.getValue().call(); + String before = extractText(input); + String after = extractText(convert(input).pdfBytes()); + assertEquals(before, after, "text changed for fixture " + entry.getKey()); + } + } + + @Test + @DisplayName("a simple document gains a structure tree with headings and paragraphs") + void simpleDocumentIsTagged() throws Exception { + PdfUaConversionOutcome outcome = convert(PdfUaTestDocuments.simpleDocument()); + try (PDDocument document = Loader.loadPDF(outcome.pdfBytes())) { + assertNotNull( + document.getDocumentCatalog().getStructureTreeRoot(), + "no structure tree was written"); + assertTrue( + document.getDocumentCatalog().getMarkInfo() != null + && document.getDocumentCatalog().getMarkInfo().isMarked(), + "MarkInfo/Marked was not set"); + assertEquals("en-GB", document.getDocumentCatalog().getLanguage()); + assertEquals("Test Document", document.getDocumentInformation().getTitle()); + } + assertTrue(outcome.tagging().taggedElements() > 0, "nothing was tagged"); + } + + @Test + @DisplayName("running heads and page numbers become artifacts, not content") + void runningHeadersBecomeArtifacts() throws Exception { + PdfUaConversionOutcome outcome = convert(PdfUaTestDocuments.runningHeadersDocument()); + assertTrue( + outcome.tagging().artifacts() >= 4, + "expected the repeated header on each page to become an artifact, got " + + outcome.tagging().artifacts()); + } + + @Test + @DisplayName("an image is tagged as a figure and reported as needing alt text") + void imagesNeedAltText() throws Exception { + PdfUaConversionOutcome outcome = convert(PdfUaTestDocuments.imageDocument()); + assertEquals(1, outcome.tagging().figuresNeedingAltText()); + assertFalse( + outcome.declared(), + "a document with an undescribed image must not claim conformance"); + } + + @Test + @DisplayName("supplying alt text lets an illustrated document conform") + void suppliedAltTextIsApplied() throws Exception { + byte[] input = PdfUaTestDocuments.imageDocument(); + PdfUaConversionOutcome probe = convert(input); + assertEquals(1, probe.tagging().figuresNeedingAltText()); + + TaggingOptions withAlt = + options().toBuilder() + .altTextByFigure(Map.of(figureKey(input), "A blue rectangle")) + .build(); + PdfUaConversionOutcome outcome = service.convert(input, withAlt); + assertEquals( + 0, + outcome.tagging().figuresNeedingAltText(), + "alt text supplied by the caller was not applied"); + } + + @Test + @DisplayName("an empty document does not crash the converter") + void emptyDocumentIsHandled() throws Exception { + PdfUaConversionOutcome outcome = convert(PdfUaTestDocuments.emptyDocument()); + assertNotNull(outcome.pdfBytes()); + assertFalse(outcome.warnings().isEmpty(), "an empty document should warn"); + } + + @Test + @DisplayName("an un-OCRed scan is reported rather than silently declared conformant") + void scannedDocumentIsNotDeclared() throws Exception { + PdfUaConversionOutcome outcome = convert(PdfUaTestDocuments.scannedDocument()); + assertFalse(outcome.declared(), "a scan with no text layer must not claim conformance"); + } + + @Test + @DisplayName("validation of an untagged document reports the missing structure") + void untaggedDocumentFailsValidation() throws Exception { + PdfUaValidationService validation = new PdfUaValidationService(); + validation.initialise(); + UaValidationResult result = + validation.validate(PdfUaTestDocuments.simpleDocument(), PdfUaProfile.UA1); + assertFalse(result.compliant(), "an untagged document cannot be PDF/UA compliant"); + assertTrue(result.hasIssues()); + } + + /** The key the tagger uses for figure alt text is "pageIndex:firstOrdinal". */ + private static String figureKey(byte[] input) throws IOException { + try (PDDocument document = Loader.loadPDF(input)) { + var pages = + new stirling.software.proprietary.pdf.ua.TaggedContentExtractor() + .extract(document); + for (var page : pages) { + for (var op : page.graphics()) { + return page.pageIndex() + ":" + op.ordinal(); + } + } + } + return "0:0"; + } + + private static Map> corpus() { + Map> corpus = new LinkedHashMap<>(); + corpus.put("simple", PdfUaTestDocuments::simpleDocument); + corpus.put("headings", PdfUaTestDocuments::headingHierarchy); + corpus.put("lists", PdfUaTestDocuments::listDocument); + corpus.put("table", PdfUaTestDocuments::tableDocument); + corpus.put("image", PdfUaTestDocuments::imageDocument); + corpus.put("runningHeads", PdfUaTestDocuments::runningHeadersDocument); + corpus.put("twoColumn", PdfUaTestDocuments::twoColumnDocument); + corpus.put("link", PdfUaTestDocuments::linkDocument); + corpus.put("empty", PdfUaTestDocuments::emptyDocument); + corpus.put("scanned", PdfUaTestDocuments::scannedDocument); + corpus.put("formXObject", PdfUaTestDocuments::formXObjectDocument); + corpus.put("multiStream", PdfUaTestDocuments::multiStreamDocument); + corpus.put("rotated", PdfUaTestDocuments::rotatedDocument); + corpus.put("offsetMediaBox", PdfUaTestDocuments::offsetMediaBoxDocument); + return corpus; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaHardeningTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaHardeningTest.java new file mode 100644 index 0000000000..cf72013a13 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaHardeningTest.java @@ -0,0 +1,322 @@ +package stirling.software.proprietary.service.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.contentstream.operator.Operator; +import org.apache.pdfbox.pdfparser.PDFStreamParser; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.text.PDFTextStripper; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.proprietary.model.api.ua.PdfUaConversionOutcome; +import stirling.software.proprietary.pdf.ua.PageContent; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.pdf.ua.StructBlock; +import stirling.software.proprietary.pdf.ua.StructType; +import stirling.software.proprietary.pdf.ua.TaggingOptions; + +/** Covers the document shapes and failure modes the first round of tests missed. */ +class PdfUaHardeningTest { + + private static PdfUaConversionService service; + + @BeforeAll + static void setUp() { + PdfUaValidationService validation = new PdfUaValidationService(); + validation.initialise(); + service = + new PdfUaConversionService( + validation, + new FontEmbeddingService(), + new CustomPDFDocumentFactory( + org.mockito.Mockito.mock(PdfMetadataService.class))); + } + + private static PdfUaConversionOutcome convert(byte[] input) throws IOException { + return service.convert( + input, + TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language("en-GB") + .title("Hardening") + .embedFonts(false) + .build()); + } + + private static String extract(byte[] pdf) throws IOException { + try (PDDocument document = Loader.loadPDF(pdf)) { + PDFTextStripper stripper = new PDFTextStripper(); + stripper.setSortByPosition(true); + return PdfUaRealCorpusTest.normalise(stripper.getText(document)); + } + } + + @Nested + @DisplayName("document shapes") + class Shapes { + + @Test + @DisplayName("text inside a form XObject is attributed to the Do and tagged as prose") + void formXObjectTextIsTagged() throws Exception { + byte[] input = PdfUaTestDocuments.formXObjectDocument(); + assertTrue( + extract(input).contains("inside the form XObject"), + "fixture must actually draw text inside a form"); + + PdfUaConversionOutcome outcome = convert(input); + assertTrue(outcome.declared(), warnings(outcome)); + assertEquals(extract(input), extract(outcome.pdfBytes())); + assertEquals( + 0, + outcome.tagging().figuresNeedingAltText(), + "a form whose text is reachable must not degrade to an undescribed figure"); + } + + @Test + @DisplayName("a page built from multiple content streams converts as one sequence") + void multiStreamPageConverts() throws Exception { + byte[] input = PdfUaTestDocuments.multiStreamDocument(); + PdfUaConversionOutcome outcome = convert(input); + assertTrue(outcome.declared(), warnings(outcome)); + assertEquals(extract(input), extract(outcome.pdfBytes())); + } + + @Test + @DisplayName("a rotated page keeps its text and converts") + void rotatedPageConverts() throws Exception { + byte[] input = PdfUaTestDocuments.rotatedDocument(); + PdfUaConversionOutcome outcome = convert(input); + assertEquals(extract(input), extract(outcome.pdfBytes())); + assertTrue(outcome.tagging().taggedElements() > 0, "rotated text was not tagged"); + } + + @Test + @DisplayName("a MediaBox that does not start at the origin does not break analysis") + void offsetMediaBoxConverts() throws Exception { + byte[] input = PdfUaTestDocuments.offsetMediaBoxDocument(); + PdfUaConversionOutcome outcome = convert(input); + assertTrue(outcome.declared(), warnings(outcome)); + assertEquals(extract(input), extract(outcome.pdfBytes())); + } + + private static String warnings(PdfUaConversionOutcome outcome) { + return "warnings: " + + String.join(" | ", outcome.warnings()) + + " issues: " + + outcome.validation().issues(); + } + } + + @Nested + @DisplayName("pre-marked content") + class PreMarked { + + @Test + @DisplayName("existing BDC/EMC operators are stripped before new ones are written") + void stripsExistingMarkedContent() throws Exception { + byte[] premarked = premarkedDocument(); + PdfUaConversionOutcome outcome = convert(premarked); + + try (PDDocument document = Loader.loadPDF(outcome.pdfBytes())) { + Counts counts = countMarkedContent(document.getPage(0)); + assertEquals( + counts.opens(), + counts.closes(), + "unbalanced marked content after stripping and re-injection"); + assertFalse( + contentOf(document).contains("/OldTag"), + "the source's own marked content survived the rebuild"); + } + assertEquals(extract(premarked), extract(outcome.pdfBytes())); + } + + /** A document whose stream already contains a BDC sequence under a custom tag. */ + private static byte[] premarkedDocument() throws Exception { + byte[] plain = PdfUaTestDocuments.simpleDocument(); + try (PDDocument document = Loader.loadPDF(plain)) { + PDPage page = document.getPage(0); + String content; + try (InputStream in = page.getContents()) { + content = new String(in.readAllBytes(), StandardCharsets.ISO_8859_1); + } + String wrapped = "/OldTag <> BDC\n" + content + "\nEMC\n"; + var stream = new org.apache.pdfbox.pdmodel.common.PDStream(document); + try (var out = stream.createOutputStream()) { + out.write(wrapped.getBytes(StandardCharsets.ISO_8859_1)); + } + page.setContents(stream); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + return out.toByteArray(); + } + } + + private record Counts(int opens, int closes) {} + + private static Counts countMarkedContent(PDPage page) throws IOException { + int opens = 0; + int closes = 0; + PDFStreamParser parser = new PDFStreamParser(page); + Object token; + while ((token = parser.parseNextToken()) != null) { + if (token instanceof Operator operator) { + switch (operator.getName()) { + case "BDC", "BMC" -> opens++; + case "EMC" -> closes++; + default -> {} + } + } + } + return new Counts(opens, closes); + } + + private static String contentOf(PDDocument document) throws IOException { + try (InputStream in = document.getPage(0).getContents()) { + return new String(in.readAllBytes(), StandardCharsets.ISO_8859_1); + } + } + } + + @Nested + @DisplayName("honesty rules") + class Honesty { + + @Test + @DisplayName("suppressed text blocks the conformance claim even when validation passes") + void suppressedTextBlocksDeclaration() { + var structure = new stirling.software.proprietary.pdf.ua.DocumentStructure(); + structure.setTextSuppressed(true); + var result = new stirling.software.proprietary.pdf.ua.TaggingResult(structure, true); + assertTrue( + result.isContentSuppressed(), + "the suppression flag must survive into the tagging result"); + } + + @Test + @DisplayName("dropped lines are reported per page by the analyser") + void analyserWarnsOnDroppedLines() { + PageContent dropped = + new PageContent( + 0, + List.of(), + List.of(), + 5, + false, + false, + true, + new stirling.software.proprietary.pdf.ua.BBox(0, 0, 595, 842)); + var structure = + new stirling.software.proprietary.pdf.ua.LayoutAnalyzer() + .analyse(List.of(dropped)); + assertTrue(structure.isTextSuppressed()); + assertTrue( + structure.getWarnings().stream().anyMatch(w -> w.contains("page(s) 1")), + "warning should name the affected page: " + structure.getWarnings()); + } + } + + @Nested + @DisplayName("clause table") + class Clauses { + + @Test + @DisplayName("subclauses resolve to their parent entry, not to a string prefix") + void subclauseLookupWalksSegments() { + assertNotNull(PdfUaValidationService.lookupClause("7.21.4.1"), "7.21.4.1 -> 7.21"); + assertNotNull(PdfUaValidationService.lookupClause("7.18.1"), "7.18.1 -> 7.18"); + var toUnicode = PdfUaValidationService.lookupClause("7.21.7"); + assertNotNull(toUnicode); + assertFalse( + toUnicode.autoFixable(), + "a missing ToUnicode map is not fixable by embedding fonts"); + assertNull(PdfUaValidationService.lookupClause("9.9.9")); + assertNull(PdfUaValidationService.lookupClause(null)); + } + } + + @Nested + @DisplayName("range claiming") + class Claiming { + + @Test + @DisplayName("a figure drawn between two text runs on one line stays a figure") + void interleavedFigureIsNotSwallowed() throws Exception { + // Words at ordinals 0 and 2 with an image at ordinal 1 between them. + var line = + new stirling.software.proprietary.pdf.ua.TextLineInfo( + 0, + "left right", + new stirling.software.proprietary.pdf.ua.BBox(50, 700, 400, 712), + 11, + false, + 0, + 2, + false, + List.of( + new stirling.software.proprietary.pdf.ua.WordInfo( + "left", + new stirling.software.proprietary.pdf.ua.BBox( + 50, 700, 90, 712), + 0, + 0, + 11, + false), + new stirling.software.proprietary.pdf.ua.WordInfo( + "right", + new stirling.software.proprietary.pdf.ua.BBox( + 360, 700, 400, 712), + 2, + 2, + 11, + false))); + var image = + new stirling.software.proprietary.pdf.ua.MarkableOp( + 1, + stirling.software.proprietary.pdf.ua.MarkableOp.Kind.IMAGE, + new stirling.software.proprietary.pdf.ua.BBox(150, 650, 350, 760), + "Im0"); + PageContent page = + new PageContent( + 0, + List.of(line), + List.of(image), + 3, + false, + false, + false, + new stirling.software.proprietary.pdf.ua.BBox(0, 0, 595, 842)); + + var structure = + new stirling.software.proprietary.pdf.ua.LayoutAnalyzer() + .analyse(List.of(page)); + List figures = new java.util.ArrayList<>(); + structure.visit( + block -> { + if (block.getType() == StructType.FIGURE) { + figures.add(block); + } + }); + assertEquals( + 1, + figures.size(), + "the image between the words must survive as its own figure"); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaHttpEndpointTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaHttpEndpointTest.java new file mode 100644 index 0000000000..90c40af82b --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaHttpEndpointTest.java @@ -0,0 +1,183 @@ +package stirling.software.proprietary.service.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.common.util.TempFileManager; +import stirling.software.proprietary.controller.api.converters.ConvertPdfToPdfUa; +import stirling.software.proprietary.controller.api.security.AccessibilityReportController; + +/** + * Exercises the endpoints over HTTP, not through the service layer. Covers route mapping, multipart + * binding, and the headers and JSON a client depends on. + */ +class PdfUaHttpEndpointTest { + + private static MockMvc convertMvc; + private static MockMvc reportMvc; + private static final ObjectMapper JSON = new ObjectMapper(); + + @BeforeAll + static void setUp() throws Exception { + PdfUaValidationService validation = new PdfUaValidationService(); + validation.initialise(); + PdfUaConversionService conversion = + new PdfUaConversionService( + validation, + new FontEmbeddingService(), + new CustomPDFDocumentFactory( + org.mockito.Mockito.mock(PdfMetadataService.class))); + + // A real TempFileManager, so the streamed response path is exercised rather than mocked. + TempFileManager tempFiles = + new TempFileManager( + new stirling.software.common.util.TempFileRegistry(), + new stirling.software.common.model.ApplicationProperties()); + + // Stand in for the app's advice, which lives in core; without one every rejection is a 500. + var advice = new BadRequestAdvice(); + convertMvc = + MockMvcBuilders.standaloneSetup(new ConvertPdfToPdfUa(conversion, tempFiles)) + .setControllerAdvice(advice) + .build(); + reportMvc = + MockMvcBuilders.standaloneSetup( + new AccessibilityReportController( + new AccessibilityAuditService(validation))) + .setControllerAdvice(advice) + .build(); + } + + private static MockMultipartFile upload(byte[] pdf, String name) { + return new MockMultipartFile("fileInput", name, "application/pdf", pdf); + } + + /** Mirrors the one rule these endpoints rely on: a rejected input is a 400, not a 500. */ + @org.springframework.web.bind.annotation.RestControllerAdvice + static class BadRequestAdvice { + @org.springframework.web.bind.annotation.ExceptionHandler(IllegalArgumentException.class) + org.springframework.http.ResponseEntity badRequest(IllegalArgumentException ex) { + return org.springframework.http.ResponseEntity.badRequest().body(ex.getMessage()); + } + } + + @Test + @DisplayName("POST /api/v1/convert/pdf/ua returns a PDF and reports what it did in headers") + void conversionEndpointResponds() throws Exception { + MvcResult result = + convertMvc + .perform( + multipart("/api/v1/convert/pdf/ua") + .file(upload(PdfUaTestDocuments.simpleDocument(), "in.pdf")) + .param("language", "en-GB") + .param("title", "Over The Wire") + .param("embedFonts", "false")) + .andExpect(status().isOk()) + .andExpect(header().exists("X-Stirling-UA-Declared")) + .andExpect(header().exists("X-Stirling-UA-Failures")) + .andReturn(); + + byte[] body = result.getResponse().getContentAsByteArray(); + assertTrue(body.length > 0, "no document came back"); + assertEquals( + "%PDF", + new String(body, 0, 4, java.nio.charset.StandardCharsets.ISO_8859_1), + "the response body is not a PDF"); + assertEquals( + "true", + result.getResponse().getHeader("X-Stirling-UA-Declared"), + "a simple embedded-font document should convert and be declared conformant"); + } + + @Test + @DisplayName("POST /api/v1/security/accessibility-report returns the figure inventory as JSON") + void reportEndpointResponds() throws Exception { + MvcResult result = + reportMvc + .perform( + multipart("/api/v1/security/accessibility-report") + .file(upload(PdfUaTestDocuments.imageDocument(), "in.pdf")) + .param("profile", "ua1")) + .andExpect(status().isOk()) + .andReturn(); + + JsonNode json = JSON.readTree(result.getResponse().getContentAsString()); + assertEquals("PDF/UA-1", json.get("profile").asText()); + assertNotNull(json.get("summary"), "the report should carry a summary"); + + JsonNode figures = json.get("figuresNeedingDescription"); + assertNotNull(figures, "the field a caller needs to supply alt text is missing"); + assertTrue(figures.isArray() && figures.size() > 0, "the image should be listed: " + json); + assertTrue( + figures.get(0).get("key").asText().matches("\\d+:\\d+"), + "the key must be usable in a follow-up conversion request"); + } + + @Test + @DisplayName("alt text supplied as form data reaches the converter over HTTP") + void altTextBindsFromFormData() throws Exception { + byte[] input = PdfUaTestDocuments.imageDocument(); + + // Discover the key the way a client would, through the report endpoint. + MvcResult reported = + reportMvc + .perform( + multipart("/api/v1/security/accessibility-report") + .file(upload(input, "in.pdf"))) + .andExpect(status().isOk()) + .andReturn(); + String key = + JSON.readTree(reported.getResponse().getContentAsString()) + .get("figuresNeedingDescription") + .get(0) + .get("key") + .asText(); + + MvcResult converted = + convertMvc + .perform( + multipart("/api/v1/convert/pdf/ua") + .file(upload(input, "in.pdf")) + .param("embedFonts", "false") + .param("altText", key + "=A blue rectangle")) + .andExpect(status().isOk()) + .andReturn(); + + assertEquals( + "0", + converted.getResponse().getHeader("X-Stirling-UA-Figures-Needing-Alt"), + "the description posted as form data was not applied"); + } + + @Test + @DisplayName("a request with no file is rejected rather than processed") + void missingFileIsRejected() throws Exception { + convertMvc + .perform( + multipart("/api/v1/convert/pdf/ua") + .file( + new MockMultipartFile( + "fileInput", + "e.pdf", + "application/pdf", + new byte[0]))) + .andExpect(status().is4xxClientError()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaRealCorpusTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaRealCorpusTest.java new file mode 100644 index 0000000000..9ebda08736 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaRealCorpusTest.java @@ -0,0 +1,249 @@ +package stirling.software.proprietary.service.ua; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Stream; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.text.PDFTextStripper; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.proprietary.model.api.ua.PdfUaConversionOutcome; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.pdf.ua.TaggingOptions; + +/** + * Runs the converter over every PDF in the repository, where real-tool output breaks assumptions. A + * clean refusal counts as a pass; nothing may crash or make a false conformance claim. + */ +class PdfUaRealCorpusTest { + + private static PdfUaConversionService service; + private static Path repoRoot; + + /** Files the converter is expected to refuse rather than process. */ + private static final List EXPECTED_REJECTS = List.of("encrypted.pdf", "corrupted.pdf"); + + @BeforeAll + static void setUp() { + PdfUaValidationService validation = new PdfUaValidationService(); + validation.initialise(); + service = + new PdfUaConversionService( + validation, + new FontEmbeddingService(), + new CustomPDFDocumentFactory( + org.mockito.Mockito.mock(PdfMetadataService.class))); + repoRoot = findRepoRoot(); + } + + private static Path findRepoRoot() { + Path current = Path.of("").toAbsolutePath(); + while (current != null && !Files.exists(current.resolve("settings.gradle"))) { + current = current.getParent(); + } + return current; + } + + private record Outcome(String name, String status, int failures, int elements, int artifacts) {} + + private static TaggingOptions.TaggingOptionsBuilder options(String fallbackTitle) { + return TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language("en-GB") + .fallbackTitle(fallbackTitle) + .existingTags(TaggingOptions.ExistingTags.REBUILD); + } + + @Test + @DisplayName("converts every PDF in the repository without crashing or lying about conformance") + void realCorpusConverts() throws Exception { + assertNotNull(repoRoot, "could not locate the repository root"); + List pdfs = findPdfs(); + assertTrue(pdfs.size() >= 20, "expected a substantial corpus, found " + pdfs.size()); + + List outcomes = new ArrayList<>(); + List crashes = new ArrayList<>(); + java.util.Map clauseFiles = new java.util.TreeMap<>(); + java.util.Map clauseText = new java.util.HashMap<>(); + java.util.Map clauseExamples = new java.util.HashMap<>(); + + for (Path pdf : pdfs) { + String name = repoRoot.relativize(pdf).toString().replace('\\', '/'); + byte[] input; + try { + input = Files.readAllBytes(pdf); + } catch (IOException e) { + continue; + } + try { + String stem = pdf.getFileName().toString().replaceFirst("\\.pdf$", ""); + + // Fidelity is a tagging property, so measure it with font embedding off. + PdfUaConversionOutcome taggedOnly = + service.convert(input, options(stem).embedFonts(false).build()); + assertTextPreserved(name, input, taggedOnly.pdfBytes()); + + PdfUaConversionOutcome outcome = service.convert(input, options(stem).build()); + // Full pipeline too: Ghostscript can exit 0 having blanked the document. + assertTextPreserved(name + " (with font embedding)", input, outcome.pdfBytes()); + outcomes.add( + new Outcome( + name, + outcome.declared() ? "CONFORMS" : "improved", + outcome.validation().totalFailures(), + outcome.tagging().taggedElements(), + outcome.tagging().artifacts())); + outcome.validation() + .issues() + .forEach( + issue -> { + clauseFiles.merge(issue.getClause(), 1, Integer::sum); + clauseText.putIfAbsent( + issue.getClause(), issue.getTechnicalMessage()); + clauseExamples.putIfAbsent(issue.getClause(), name); + }); + + } catch (IOException e) { + // A refusal with an explanation is an acceptable outcome. + outcomes.add(new Outcome(name, "refused: " + e.getMessage(), 0, 0, 0)); + } catch (RuntimeException e) { + crashes.add(name + " -> " + e); + outcomes.add(new Outcome(name, "CRASH: " + e, 0, 0, 0)); + } + } + + System.out.println(render(outcomes)); + + StringBuilder clauses = + new StringBuilder("\nBlocking clauses, by number of files affected\n"); + clauseFiles.entrySet().stream() + .sorted(java.util.Map.Entry.comparingByValue().reversed()) + .forEach( + e -> + clauses.append( + String.format( + " clause %-9s %-3d files e.g. %s%n %s%n", + e.getKey(), + e.getValue(), + clauseExamples.get(e.getKey()), + abbreviate(clauseText.get(e.getKey()))))); + System.out.println(clauses); + + List unexpectedCrashes = + crashes.stream() + .filter(c -> EXPECTED_REJECTS.stream().noneMatch(c::contains)) + .toList(); + assertTrue( + unexpectedCrashes.isEmpty(), + "converter crashed on: " + String.join("; ", unexpectedCrashes)); + } + + /** Tagging must not change extracted text; a diff means the rewrite corrupted the page. */ + private static void assertTextPreserved(String name, byte[] before, byte[] after) { + String textBefore = safeExtract(before); + if (textBefore == null) { + // The source itself is unreadable, so there is nothing to compare against. + return; + } + String textAfter = safeExtract(after); + assertTrue( + textAfter != null, "the converted file could not be read back at all for " + name); + assertTrue( + textBefore.equals(textAfter), + "tagging changed extracted text for " + + name + + "\n before: " + + preview(textBefore) + + "\n after: " + + preview(textAfter)); + } + + private static String abbreviate(String text) { + if (text == null) { + return ""; + } + return text.length() <= 110 ? text : text.substring(0, 110) + "..."; + } + + private static String preview(String text) { + return text.length() <= 160 ? text : text.substring(0, 160) + "..."; + } + + private static String safeExtract(byte[] pdf) { + try (PDDocument document = Loader.loadPDF(pdf)) { + PDFTextStripper stripper = new PDFTextStripper(); + stripper.setSortByPosition(true); + return normalise(stripper.getText(document)); + } catch (Exception e) { + return null; + } + } + + /** Drops invisible formatting characters: a rebuild legitimately loses soft hyphens. */ + static String normalise(String text) { + StringBuilder sb = new StringBuilder(text.length()); + text.codePoints() + .forEach( + cp -> { + if (Character.getType(cp) != Character.FORMAT && cp != 0x00AD) { + sb.appendCodePoint(cp); + } + }); + return sb.toString().replaceAll("\\s+", " ").strip(); + } + + private List findPdfs() throws IOException { + try (Stream stream = Files.walk(repoRoot)) { + return stream.filter(Files::isRegularFile) + .filter(p -> p.toString().toLowerCase().endsWith(".pdf")) + .filter(p -> !p.toString().contains("node_modules")) + .filter(p -> !p.toString().contains(File_BUILD)) + .filter(p -> !p.toString().contains(".git")) + .sorted(Comparator.comparing(Path::toString)) + .toList(); + } + } + + private static final String File_BUILD = "build" + java.io.File.separator; + + private static String render(List outcomes) { + StringBuilder sb = new StringBuilder("\nPDF/UA conversion over the repository corpus\n"); + long conforming = outcomes.stream().filter(o -> "CONFORMS".equals(o.status())).count(); + long refused = outcomes.stream().filter(o -> o.status().startsWith("refused")).count(); + sb.append( + String.format( + " %d files: %d conform, %d improved but not conformant, %d refused%n%n", + outcomes.size(), + conforming, + outcomes.size() - conforming - refused, + refused)); + for (Outcome outcome : outcomes) { + sb.append( + String.format( + " %-62s %-10s fail=%-4d el=%-5d art=%d%n", + outcome.name().length() > 60 + ? "..." + outcome.name().substring(outcome.name().length() - 57) + : outcome.name(), + outcome.status().length() > 10 + ? outcome.status().substring(0, 10) + : outcome.status(), + outcome.failures(), + outcome.elements(), + outcome.artifacts())); + } + return sb.toString(); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaSampleDumpTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaSampleDumpTest.java new file mode 100644 index 0000000000..46cdb37c76 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaSampleDumpTest.java @@ -0,0 +1,103 @@ +package stirling.software.proprietary.service.ua; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.proprietary.model.api.ua.PdfUaConversionOutcome; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.pdf.ua.TaggingOptions; + +/** + * Dumps converted output for independent checkers; validating PDFBox with PDFBox is circular. Off + * by default as it writes outside the build directory: run with {@code DUMP_UA_SAMPLES=}. + */ +@EnabledIfEnvironmentVariable(named = "DUMP_UA_SAMPLES", matches = ".+") +class PdfUaSampleDumpTest { + + private static PdfUaConversionService service; + private static Path repoRoot; + + @BeforeAll + static void setUp() { + PdfUaValidationService validation = new PdfUaValidationService(); + validation.initialise(); + service = + new PdfUaConversionService( + validation, + new FontEmbeddingService(), + new CustomPDFDocumentFactory( + org.mockito.Mockito.mock(PdfMetadataService.class))); + repoRoot = Path.of("").toAbsolutePath(); + while (repoRoot != null && !Files.exists(repoRoot.resolve("settings.gradle"))) { + repoRoot = repoRoot.getParent(); + } + } + + @Test + @DisplayName("writes original and converted pairs for external validation") + void dumpSamples() throws Exception { + Path out = Path.of(System.getenv("DUMP_UA_SAMPLES")); + Files.createDirectories(out); + + List pdfs; + try (Stream stream = Files.walk(repoRoot)) { + pdfs = + stream.filter(Files::isRegularFile) + .filter(p -> p.toString().toLowerCase().endsWith(".pdf")) + .filter(p -> !p.toString().contains("node_modules")) + .filter(p -> !p.toString().contains(java.io.File.separator + "build")) + .filter(p -> !p.toString().contains(".git")) + .sorted() + .toList(); + } + + List manifest = new ArrayList<>(); + int written = 0; + for (Path pdf : pdfs) { + String stem = pdf.getFileName().toString().replaceFirst("\\.pdf$", ""); + byte[] input; + try { + input = Files.readAllBytes(pdf); + } catch (Exception e) { + continue; + } + try { + PdfUaConversionOutcome outcome = + service.convert( + input, + TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language("en-GB") + .fallbackTitle(stem) + .existingTags(TaggingOptions.ExistingTags.REBUILD) + .build()); + Files.write(out.resolve(stem + "__before.pdf"), input); + Files.write(out.resolve(stem + "__after.pdf"), outcome.pdfBytes()); + manifest.add( + stem + + "\tdeclared=" + + outcome.declared() + + "\tfailures=" + + outcome.validation().totalFailures()); + written++; + } catch (Exception e) { + manifest.add(stem + "\tREFUSED\t" + e.getMessage()); + } + } + Files.write(out.resolve("manifest.tsv"), manifest); + System.out.println("Wrote " + written + " before/after pairs to " + out); + assertTrue(written > 10, "expected a usable sample set, wrote " + written); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaServicesTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaServicesTest.java new file mode 100644 index 0000000000..9d2e14ee0d --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaServicesTest.java @@ -0,0 +1,319 @@ +package stirling.software.proprietary.service.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.apache.pdfbox.pdmodel.encryption.AccessPermission; +import org.apache.pdfbox.pdmodel.encryption.StandardProtectionPolicy; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.pdfbox.pdmodel.font.Standard14Fonts; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.service.PdfMetadataService; +import stirling.software.proprietary.model.api.ua.AccessibilityReport; +import stirling.software.proprietary.model.api.ua.PdfUaConversionOutcome; +import stirling.software.proprietary.model.api.ua.UaValidationResult; +import stirling.software.proprietary.pdf.ua.PdfUaProfile; +import stirling.software.proprietary.pdf.ua.TaggingOptions; + +/** Tests for the services that validate, audit and convert. */ +class PdfUaServicesTest { + + private static PdfUaValidationService validation; + private static PdfUaConversionService conversion; + private static AccessibilityAuditService audit; + private static FontEmbeddingService fonts; + + @BeforeAll + static void setUp() { + validation = new PdfUaValidationService(); + validation.initialise(); + fonts = new FontEmbeddingService(); + conversion = + new PdfUaConversionService( + validation, + fonts, + new CustomPDFDocumentFactory( + org.mockito.Mockito.mock(PdfMetadataService.class))); + audit = new AccessibilityAuditService(validation); + } + + /** Uses a standard 14 font deliberately: never embedded, which clause 7.21 forbids. */ + private static byte[] unembeddedFontPdf() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + cs.beginText(); + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12); + cs.newLineAtOffset(50, 700); + cs.showText("Hello accessibility"); + cs.endText(); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + return out.toByteArray(); + } + } + + private static byte[] manyPages(int pages) throws IOException { + try (PDDocument document = new PDDocument()) { + for (int i = 0; i < pages; i++) { + document.addPage(new PDPage()); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + return out.toByteArray(); + } + } + + private static byte[] encryptedPdf() throws IOException { + try (PDDocument document = new PDDocument()) { + document.addPage(new PDPage()); + AccessPermission permissions = new AccessPermission(); + document.protect(new StandardProtectionPolicy("owner", "user", permissions)); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + return out.toByteArray(); + } + } + + @Nested + @DisplayName("validation") + class Validation { + + @Test + @DisplayName("an untagged document fails and the failures are grouped by rule") + void untaggedFails() throws Exception { + UaValidationResult result = validation.validate(unembeddedFontPdf(), PdfUaProfile.UA1); + assertFalse(result.compliant()); + assertTrue(result.hasIssues()); + assertTrue( + result.totalFailures() >= result.issues().size(), + "grouping must not invent failures"); + assertTrue( + result.issues().stream().allMatch(i -> i.getOccurrences() > 0), + "every grouped issue should count its occurrences"); + } + + @Test + @DisplayName("malformed input reports a failure instead of throwing") + void malformedInputIsReported() { + UaValidationResult result = + validation.validate("not a pdf".getBytes(), PdfUaProfile.UA1); + assertFalse(result.compliant()); + assertFalse(result.issues().isEmpty()); + } + + @Test + @DisplayName("issues carry plain-English text as well as the validator's own wording") + void issuesAreReadable() throws Exception { + UaValidationResult result = validation.validate(unembeddedFontPdf(), PdfUaProfile.UA1); + assertTrue( + result.issues().stream() + .allMatch(i -> i.getMessage() != null && !i.getMessage().isBlank())); + } + } + + @Nested + @DisplayName("auditing") + class Auditing { + + @Test + @DisplayName("reports the document facts that drive most failures") + void reportsSummary() throws Exception { + AccessibilityReport report = audit.audit(unembeddedFontPdf(), PdfUaProfile.UA1); + + assertFalse(report.isTagged(), "the fixture has no structure tree"); + assertFalse(report.isDeclaresConformance()); + assertFalse(report.isPassesAutomatedChecks()); + assertEquals(1, report.getSummary().getPages()); + assertFalse(report.getSummary().isAllFontsEmbedded()); + assertTrue(report.getSummary().getUnembeddedFonts() > 0); + assertFalse(report.getSummary().isHasLanguage()); + assertFalse(report.getSummary().isHasTitle()); + } + + @Test + @DisplayName("always lists the checks a person still has to make") + void listsHumanChecks() throws Exception { + AccessibilityReport report = audit.audit(unembeddedFontPdf(), PdfUaProfile.UA1); + assertFalse( + report.getHumanChecks().isEmpty(), + "a report showing only automated results implies the rest does not exist"); + } + + @Test + @DisplayName("splits failures into automatically fixable and needs-input") + void splitsRemediability() throws Exception { + AccessibilityReport report = audit.audit(unembeddedFontPdf(), PdfUaProfile.UA1); + assertEquals( + report.getIssues().size(), + report.getAutomaticallyFixable() + report.getNeedsInput()); + } + + @Test + @DisplayName("refuses a document past the page cap the conversion also applies") + void refusesTooManyPages() throws Exception { + byte[] oversized = manyPages(2001); + assertThrows( + IllegalArgumentException.class, + () -> audit.audit(oversized, PdfUaProfile.UA1), + "an uncapped report walks every page of any document a caller uploads"); + } + + @Test + @DisplayName("a converted document reports as tagged and conformant") + void reportsAfterConversion() throws Exception { + PdfUaConversionOutcome outcome = + conversion.convert( + unembeddedFontPdf(), + TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language("en-GB") + .title("Converted") + .build()); + AccessibilityReport report = audit.audit(outcome.pdfBytes(), PdfUaProfile.UA1); + assertTrue(report.isTagged()); + assertTrue(report.getSummary().isHasTitle()); + assertTrue(report.getSummary().isHasLanguage()); + assertTrue(report.getSummary().isDisplaysDocTitle()); + } + } + + @Nested + @DisplayName("font embedding") + class Fonts { + + @Test + @DisplayName("detects a standard 14 font as unembedded") + void detectsUnembedded() throws Exception { + assertTrue(fonts.hasUnembeddedFonts(unembeddedFontPdf())); + } + + @Test + @DisplayName("embeds fonts, or explains why it could not") + void embedsOrExplains() throws Exception { + FontEmbeddingService.Result result = fonts.embedFonts(unembeddedFontPdf()); + assertNotNull(result.pdfBytes()); + if (result.changed()) { + assertFalse( + fonts.hasUnembeddedFonts(result.pdfBytes()), + "embedding reported success but fonts are still missing"); + } else { + assertNotNull( + result.warning(), "failing to embed must be explained, not passed over"); + } + } + + @Test + @DisplayName("leaves a document alone when every font is already embedded") + void skipsWhenNothingToDo() throws Exception { + byte[] embedded = PdfUaTestDocuments.simpleDocument(); + FontEmbeddingService.Result result = fonts.embedFonts(embedded); + assertFalse(result.changed()); + assertEquals(embedded.length, result.pdfBytes().length); + } + } + + @Nested + @DisplayName("conversion") + class Conversion { + + @Test + @DisplayName("refuses an encrypted document with an explanation") + void refusesEncrypted() throws Exception { + byte[] encrypted = encryptedPdf(); + IOException error = + assertThrows( + IOException.class, + () -> + conversion.convert( + encrypted, + TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .build())); + assertTrue(error.getMessage().toLowerCase().contains("encrypted")); + } + + @Test + @DisplayName("keeping existing tags does not rebuild the tree") + void keepRespectsExistingTags() throws Exception { + byte[] tagged = + conversion + .convert( + PdfUaTestDocuments.simpleDocument(), + TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language("en-GB") + .title("First pass") + .embedFonts(false) + .build()) + .pdfBytes(); + + PdfUaConversionOutcome second = + conversion.convert( + tagged, + TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language("en-GB") + .title("Second pass") + .embedFonts(false) + .existingTags(TaggingOptions.ExistingTags.KEEP) + .build()); + + assertFalse(second.tagging().rebuiltStructure(), "KEEP must not rebuild"); + try (PDDocument document = Loader.loadPDF(second.pdfBytes())) { + assertEquals("Second pass", document.getDocumentInformation().getTitle()); + } + } + + @Test + @DisplayName("marking images decorative removes the alt-text blocker") + void decorativePolicyClearsFigures() throws Exception { + PdfUaConversionOutcome outcome = + conversion.convert( + PdfUaTestDocuments.imageDocument(), + TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language("en-GB") + .title("Decorative") + .embedFonts(false) + .figurePolicy(TaggingOptions.FigurePolicy.MARK_DECORATIVE) + .build()); + assertEquals(0, outcome.tagging().figuresNeedingAltText()); + assertTrue(outcome.declared(), "with no undescribed figures the file should conform"); + } + + @Test + @DisplayName("converting twice produces the same conformance verdict") + void conversionIsStable() throws Exception { + byte[] input = PdfUaTestDocuments.headingHierarchy(); + TaggingOptions options = + TaggingOptions.builder() + .profile(PdfUaProfile.UA1) + .language("en-GB") + .title("Stable") + .embedFonts(false) + .build(); + PdfUaConversionOutcome first = conversion.convert(input, options); + PdfUaConversionOutcome second = conversion.convert(first.pdfBytes(), options); + assertEquals(first.declared(), second.declared()); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaTestDocuments.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaTestDocuments.java new file mode 100644 index 0000000000..589edbc1a2 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfUaTestDocuments.java @@ -0,0 +1,390 @@ +package stirling.software.proprietary.service.ua; + +import java.awt.Color; +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDFormContentStream; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.PDResources; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.apache.pdfbox.pdmodel.font.PDFont; +import org.apache.pdfbox.pdmodel.font.PDType0Font; +import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject; +import org.apache.pdfbox.pdmodel.graphics.image.LosslessFactory; +import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject; +import org.apache.pdfbox.pdmodel.interactive.action.PDActionURI; +import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationLink; +import org.apache.pdfbox.util.Matrix; + +/** + * Builds the fixture corpus used by the PDF/UA tests. Fonts are embedded deliberately: the standard + * 14 fail clause 7.21 and would mask every result. + */ +final class PdfUaTestDocuments { + + private static final String FONT_RESOURCE = "/static/fonts/DejaVuSans.ttf"; + // The font ships with core's resources, which are not on this module's classpath. + private static final String FONT_REPO_PATH = + "app/core/src/main/resources/static/fonts/DejaVuSans.ttf"; + private static final float MARGIN = 60f; + + private PdfUaTestDocuments() {} + + static PDFont font(PDDocument document) throws IOException { + try (InputStream in = PdfUaTestDocuments.class.getResourceAsStream(FONT_RESOURCE)) { + if (in != null) { + return PDType0Font.load(document, in, true); + } + } + Path repoRoot = Path.of("").toAbsolutePath(); + while (repoRoot != null && !Files.exists(repoRoot.resolve("settings.gradle"))) { + repoRoot = repoRoot.getParent(); + } + Path font = repoRoot == null ? null : repoRoot.resolve(FONT_REPO_PATH); + if (font == null || !Files.exists(font)) { + throw new IOException( + "Test font not found: " + FONT_RESOURCE + " or " + FONT_REPO_PATH); + } + try (InputStream in = Files.newInputStream(font)) { + return PDType0Font.load(document, in, true); + } + } + + /** A heading followed by two paragraphs: the simplest thing that should convert cleanly. */ + static byte[] simpleDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + PDFont font = font(document); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + float y = 760; + y = text(cs, font, 20, MARGIN, y, "Quarterly Report"); + y -= 14; + y = text(cs, font, 11, MARGIN, y, "This document summarises the results for the"); + y = text(cs, font, 11, MARGIN, y, "period and outlines the outlook for next year."); + y -= 14; + text(cs, font, 11, MARGIN, y, "A second paragraph follows the first one here."); + } + return bytes(document); + } + } + + /** Three heading tiers, to exercise level assignment and the no-skipped-levels rule. */ + static byte[] headingHierarchy() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + PDFont font = font(document); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + float y = 760; + y = text(cs, font, 24, MARGIN, y, "Annual Review"); + y -= 12; + y = text(cs, font, 11, MARGIN, y, "Introductory prose sits under the title here."); + y -= 16; + y = text(cs, font, 17, MARGIN, y, "Financial Results"); + y -= 10; + y = text(cs, font, 11, MARGIN, y, "Revenue grew steadily across every region."); + y -= 16; + y = text(cs, font, 13, MARGIN, y, "Europe"); + y -= 10; + text(cs, font, 11, MARGIN, y, "European revenue rose by eleven per cent."); + } + return bytes(document); + } + } + + /** A bulleted and a numbered list. */ + static byte[] listDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + PDFont font = font(document); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + float y = 760; + y = text(cs, font, 18, MARGIN, y, "Checklist"); + y -= 14; + y = text(cs, font, 11, MARGIN, y, "• Review the source document"); + y = text(cs, font, 11, MARGIN, y, "• Check every heading level"); + y = text(cs, font, 11, MARGIN, y, "• Describe each image"); + y -= 16; + y = text(cs, font, 11, MARGIN, y, "1. Open the file"); + y = text(cs, font, 11, MARGIN, y, "2. Run the converter"); + text(cs, font, 11, MARGIN, y, "3. Validate the result"); + } + return bytes(document); + } + } + + /** A three-column table whose cells each occupy their own text-showing operator. */ + static byte[] tableDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + PDFont font = font(document); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + float y = 760; + y = text(cs, font, 18, MARGIN, y, "Regional Totals"); + y -= 20; + String[][] rows = { + {"Region", "Units", "Revenue"}, + {"North", "1200", "48000"}, + {"South", "980", "39200"}, + {"East", "1430", "57200"} + }; + float[] columns = {MARGIN, MARGIN + 160, MARGIN + 300}; + for (String[] row : rows) { + tableRow(cs, font, 11, columns, y, row); + y -= 20; + } + } + return bytes(document); + } + } + + /** A page with a real image, which must end up as a Figure needing alternative text. */ + static byte[] imageDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + PDFont font = font(document); + BufferedImage bitmap = new BufferedImage(120, 90, BufferedImage.TYPE_INT_RGB); + java.awt.Graphics2D graphics = bitmap.createGraphics(); + graphics.setColor(Color.BLUE); + graphics.fillRect(0, 0, 120, 90); + graphics.dispose(); + PDImageXObject image = LosslessFactory.createFromImage(document, bitmap); + + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + float y = 760; + y = text(cs, font, 18, MARGIN, y, "Illustrated Page"); + y -= 20; + y = text(cs, font, 11, MARGIN, y, "The chart below shows the trend."); + cs.drawImage(image, MARGIN, y - 120, 180, 100); + } + return bytes(document); + } + } + + /** Four pages sharing a running head and a page number, which must become artifacts. */ + static byte[] runningHeadersDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + PDFont font = null; + for (int i = 1; i <= 4; i++) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + if (font == null) { + font = font(document); + } + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + text(cs, font, 9, MARGIN, 810, "Confidential Internal Report"); + float y = 750; + y = text(cs, font, 16, MARGIN, y, "Section " + i); + y -= 12; + text(cs, font, 11, MARGIN, y, "Body text for section number " + i + " here."); + text(cs, font, 9, 300, 30, "Page " + i); + } + } + return bytes(document); + } + } + + /** Two columns of prose, to exercise reading order. */ + static byte[] twoColumnDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + PDFont font = font(document); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + float left = MARGIN; + float right = 320; + float y = 740; + for (int i = 1; i <= 8; i++) { + text(cs, font, 10, left, y - i * 16, "Left column line number " + i); + text(cs, font, 10, right, y - i * 16, "Right column line number " + i); + } + } + return bytes(document); + } + } + + /** A page carrying a link annotation, which must be reachable from the structure tree. */ + static byte[] linkDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + PDFont font = font(document); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + float y = 760; + y = text(cs, font, 18, MARGIN, y, "Useful Links"); + text(cs, font, 11, MARGIN, y - 20, "Visit the project home page for details."); + } + PDAnnotationLink link = new PDAnnotationLink(); + PDRectangle rectangle = new PDRectangle(); + rectangle.setLowerLeftX(MARGIN); + rectangle.setLowerLeftY(725); + rectangle.setUpperRightX(MARGIN + 200); + rectangle.setUpperRightY(740); + link.setRectangle(rectangle); + PDActionURI action = new PDActionURI(); + action.setURI("https://example.org"); + link.setAction(action); + page.getAnnotations().add(link); + return bytes(document); + } + } + + /** A page with no content at all. */ + static byte[] emptyDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + document.addPage(new PDPage(PDRectangle.A4)); + return bytes(document); + } + } + + /** A page whose only content is a full-page image, standing in for an un-OCRed scan. */ + static byte[] scannedDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + BufferedImage bitmap = new BufferedImage(600, 850, BufferedImage.TYPE_INT_RGB); + java.awt.Graphics2D graphics = bitmap.createGraphics(); + graphics.setColor(Color.WHITE); + graphics.fillRect(0, 0, 600, 850); + graphics.setColor(Color.BLACK); + graphics.drawString("scanned page", 40, 60); + graphics.dispose(); + PDImageXObject image = LosslessFactory.createFromImage(document, bitmap); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + cs.drawImage(image, 0, 0, PDRectangle.A4.getWidth(), PDRectangle.A4.getHeight()); + } + return bytes(document); + } + } + + /** Text drawn inside a form XObject, attributed to the Do operator that invoked it. */ + static byte[] formXObjectDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + PDFont font = font(document); + + PDFormXObject form = new PDFormXObject(document); + form.setBBox(new PDRectangle(220, 40)); + form.setResources(new PDResources()); + try (PDFormContentStream fcs = new PDFormContentStream(form)) { + fcs.beginText(); + fcs.setFont(font, 11); + fcs.newLineAtOffset(4, 14); + fcs.showText("Text living inside the form XObject"); + fcs.endText(); + } + + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + text(cs, font, 18, MARGIN, 760, "Page With Embedded Form"); + text(cs, font, 11, MARGIN, 730, "Ordinary page text sits above the form."); + cs.saveGraphicsState(); + cs.transform(Matrix.getTranslateInstance(MARGIN, 650)); + cs.drawForm(form); + cs.restoreGraphicsState(); + } + return bytes(document); + } + } + + /** Content split across two streams (a PDF array) - the parser must see one sequence. */ + static byte[] multiStreamDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + document.addPage(page); + PDFont font = font(document); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + text(cs, font, 18, MARGIN, 760, "First Stream Heading"); + } + try (PDPageContentStream cs = + new PDPageContentStream( + document, page, PDPageContentStream.AppendMode.APPEND, true)) { + text(cs, font, 11, MARGIN, 720, "Second stream paragraph appended later."); + } + return bytes(document); + } + } + + /** A landscape page via /Rotate 90, which flips the frame the text engine reports in. */ + static byte[] rotatedDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(PDRectangle.A4); + page.setRotation(90); + document.addPage(page); + PDFont font = font(document); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + // Drawn rotated so the text reads upright on the rotated page. + cs.transform(Matrix.getRotateInstance(Math.toRadians(90), 595, 0)); + text(cs, font, 18, MARGIN, 500, "Rotated Page Title"); + text(cs, font, 11, MARGIN, 470, "Body text on a landscape page."); + } + return bytes(document); + } + } + + /** A MediaBox whose origin is not (0,0), which some scanners produce. */ + static byte[] offsetMediaBoxDocument() throws IOException { + try (PDDocument document = new PDDocument()) { + PDPage page = new PDPage(new PDRectangle(100, 200, 595, 842)); + document.addPage(page); + PDFont font = font(document); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + text(cs, font, 18, 160, 960, "Offset Origin Title"); + text(cs, font, 11, 160, 930, "Text on a page whose MediaBox starts at 100,200."); + } + return bytes(document); + } + } + + // --- helpers ----------------------------------------------------------- + + private static float text( + PDPageContentStream cs, PDFont font, float size, float x, float y, String value) + throws IOException { + cs.beginText(); + cs.setFont(font, size); + cs.newLineAtOffset(x, y); + cs.showText(value); + cs.endText(); + return y - size * 1.35f; + } + + /** + * Emits one row with a separate show-text operator per cell, so each cell gets its own MCID. + */ + private static void tableRow( + PDPageContentStream cs, + PDFont font, + float size, + float[] columns, + float y, + String[] values) + throws IOException { + cs.beginText(); + cs.setFont(font, size); + cs.newLineAtOffset(columns[0], y); + cs.showText(values[0]); + for (int i = 1; i < values.length; i++) { + cs.newLineAtOffset(columns[i] - columns[i - 1], 0); + cs.showText(values[i]); + } + cs.endText(); + } + + private static byte[] bytes(PDDocument document) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + document.save(out); + return out.toByteArray(); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfaLevelATest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfaLevelATest.java new file mode 100644 index 0000000000..367b9d1171 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/PdfaLevelATest.java @@ -0,0 +1,202 @@ +package stirling.software.proprietary.service.ua; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.verapdf.pdfa.Foundries; +import org.verapdf.pdfa.PDFAParser; +import org.verapdf.pdfa.flavours.PDFAFlavour; +import org.verapdf.pdfa.results.TestAssertion; +import org.verapdf.pdfa.results.ValidationResult; + +/** + * Proves tagging raises a PDF/A file from level B to the accessible level A. veraPDF is the + * arbiter: the claim only counts if the validator agrees. + */ +class PdfaLevelATest { + + private static PdfaAccessibilityService service; + private static Path repoRoot; + + @BeforeAll + static void setUp() { + PdfUaValidationService uaValidation = new PdfUaValidationService(); + uaValidation.initialise(); + service = new PdfaAccessibilityService(uaValidation); + repoRoot = Path.of("").toAbsolutePath(); + while (repoRoot != null && !Files.exists(repoRoot.resolve("settings.gradle"))) { + repoRoot = repoRoot.getParent(); + } + } + + private static byte[] fixture(String name) throws Exception { + return Files.readAllBytes( + repoRoot.resolve("app/core/src/test/resources/pdfa").resolve(name)); + } + + private static String xmpOf(byte[] pdf) throws Exception { + try (PDDocument document = Loader.loadPDF(pdf)) { + var metadata = document.getDocumentCatalog().getMetadata(); + assertNotNull(metadata, "no XMP packet"); + return new String(metadata.toByteArray(), StandardCharsets.UTF_8); + } + } + + /** The flavour the file declares in its XMP, which is what a validator picks up by itself. */ + private static String declaredStandard(byte[] pdf) throws Exception { + try (PDFAParser parser = + Foundries.defaultInstance().createParser(new ByteArrayInputStream(pdf))) { + List flavours = parser.getFlavours(); + return flavours == null || flavours.isEmpty() ? null : flavours.get(0).getId(); + } + } + + private static ValidationResult validate(byte[] pdf, PDFAFlavour flavour) throws Exception { + try (PDFAParser parser = + Foundries.defaultInstance().createParser(new ByteArrayInputStream(pdf), flavour)) { + return Foundries.defaultInstance().createValidator(flavour, false).validate(parser); + } + } + + private static List failures(ValidationResult result) { + return result.getTestAssertions().stream() + .filter(assertion -> assertion.getStatus() == TestAssertion.Status.FAILED) + .map(TestAssertion::getMessage) + .toList(); + } + + @Test + @DisplayName("a level B file gains a structure tree and a conformance A claim") + void upgradesLevelBToLevelA() throws Exception { + byte[] levelB = fixture("valid-pdfa-2b.pdf"); + + try (PDDocument before = Loader.loadPDF(levelB)) { + assertEquals( + null, + before.getDocumentCatalog().getStructureTreeRoot(), + "the fixture should start untagged, or the test proves nothing"); + } + + PdfaAccessibilityService.Result result = + service.upgradeToLevelA(levelB, 2, "en-GB", "Archived Report"); + assertTrue(result.levelA(), "upgrade failed: " + result.warnings()); + + try (PDDocument after = Loader.loadPDF(result.pdfBytes())) { + assertNotNull( + after.getDocumentCatalog().getStructureTreeRoot(), "no structure tree written"); + assertTrue(after.getDocumentCatalog().getMarkInfo().isMarked()); + assertEquals("en-GB", after.getDocumentCatalog().getLanguage()); + } + + String xmp = xmpOf(result.pdfBytes()); + assertTrue(xmp.contains("part"), "pdfaid:part missing"); + assertTrue( + xmp.contains(">A<") || xmp.contains("conformance=\"A\""), + "conformance was not raised to A: " + xmp); + } + + @Test + @DisplayName("the upgraded file still validates as PDF/A, now at level A") + void upgradedFileStillValidates() throws Exception { + byte[] levelB = fixture("valid-pdfa-2b.pdf"); + PdfaAccessibilityService.Result result = + service.upgradeToLevelA(levelB, 2, "en-GB", "Archived Report"); + assertTrue(result.levelA(), "upgrade failed: " + result.warnings()); + + assertEquals( + "2a", declaredStandard(result.pdfBytes()), "the file should now declare PDF/A-2a"); + + ValidationResult pdfa = validate(result.pdfBytes(), PDFAFlavour.PDFA_2_A); + assertTrue(pdfa.isCompliant(), () -> "PDF/A-2a validation failed: " + failures(pdfa)); + } + + @Test + @DisplayName("PDF/A-1 keeps its 1.4 version, since level A must not change the part") + void partOneKeepsItsVersion() throws Exception { + byte[] levelB = fixture("valid-pdfa-1b.pdf"); + float versionBefore; + try (PDDocument document = Loader.loadPDF(levelB)) { + versionBefore = document.getVersion(); + } + + PdfaAccessibilityService.Result result = + service.upgradeToLevelA(levelB, 1, "en", "Archived"); + try (PDDocument document = Loader.loadPDF(result.pdfBytes())) { + assertEquals( + versionBefore, + document.getVersion(), + "raising the PDF version would break PDF/A-1 conformance"); + } + } + + @Test + @DisplayName("a document with nothing to tag is left at level B rather than mislabelled") + void refusesToClaimLevelAWithoutTags() throws Exception { + byte[] blank; + try (PDDocument document = new PDDocument()) { + document.addPage(new org.apache.pdfbox.pdmodel.PDPage()); + var out = new java.io.ByteArrayOutputStream(); + document.save(out); + blank = out.toByteArray(); + } + + PdfaAccessibilityService.Result result = service.upgradeToLevelA(blank, 2, "en", "Empty"); + assertFalse(result.levelA(), "an untaggable document must not claim level A"); + assertFalse(result.warnings().isEmpty(), "the refusal should be explained"); + } + + @Test + @DisplayName("setting conformance leaves the rest of the XMP packet intact") + void conformanceRewritePreservesPacket() throws Exception { + byte[] levelB = fixture("valid-pdfa-2b.pdf"); + byte[] rewritten = PdfaAccessibilityService.setConformance(levelB, 2, "A"); + + assertEquals("2a", declaredStandard(rewritten), "the rewritten packet should declare 2a"); + } + + @Test + @DisplayName("a file can declare PDF/A and PDF/UA at once without breaking either") + void combinedPdfaAndPdfUa() throws Exception { + byte[] levelB = fixture("valid-pdfa-2b.pdf"); + PdfaAccessibilityService.Result upgraded = + service.upgradeToLevelA(levelB, 2, "en-GB", "Archived and Accessible"); + assertTrue(upgraded.levelA(), "upgrade failed: " + upgraded.warnings()); + + byte[] both = PdfaAccessibilityService.declarePdfUaAlongsidePdfa(upgraded.pdfBytes(), 2); + + String xmp = xmpOf(both); + assertTrue(xmp.contains("pdfuaid"), "no PDF/UA identifier"); + assertTrue( + xmp.contains("pdfaSchema") || xmp.contains("schemas"), + "PDF/A requires an extension schema describing pdfuaid, none found: " + xmp); + + assertEquals( + "2a", declaredStandard(both), "the combined file should still declare PDF/A-2a"); + + ValidationResult pdfa = validate(both, PDFAFlavour.PDFA_2_A); + assertTrue( + pdfa.isCompliant(), + () -> "adding the PDF/UA identifier broke PDF/A: " + failures(pdfa)); + } + + @Test + @DisplayName("PDFAFlavour exposes the level A profiles the converter now targets") + void flavoursExistForLevelA() { + assertNotNull(PDFAFlavour.PDFA_1_A); + assertNotNull(PDFAFlavour.PDFA_2_A); + assertNotNull(PDFAFlavour.PDFA_3_A); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/TaggedContentExtractorRealFilesTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/TaggedContentExtractorRealFilesTest.java new file mode 100644 index 0000000000..b42a429007 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/ua/TaggedContentExtractorRealFilesTest.java @@ -0,0 +1,99 @@ +package stirling.software.proprietary.service.ua; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.text.PDFTextStripper; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import stirling.software.proprietary.pdf.ua.PageContent; +import stirling.software.proprietary.pdf.ua.TaggedContentExtractor; + +/** + * Guards the invariant the tagger rests on: the token pass and text pass must agree on ordinals. + * They can silently disagree, and the extractor then drops the page rather than mis-tag it. + */ +class TaggedContentExtractorRealFilesTest { + + private static Path repoRoot; + + @BeforeAll + static void setUp() { + repoRoot = Path.of("").toAbsolutePath(); + while (repoRoot != null && !Files.exists(repoRoot.resolve("settings.gradle"))) { + repoRoot = repoRoot.getParent(); + } + } + + @Test + @DisplayName("pages with extractable text always yield lines across the repository corpus") + void ordinalsAgreeOnRealFiles() throws Exception { + assertNotNull(repoRoot, "could not locate the repository root"); + List dropped = new ArrayList<>(); + int inspected = 0; + + for (Path pdf : findPdfs()) { + byte[] bytes; + try { + bytes = Files.readAllBytes(pdf); + } catch (Exception e) { + continue; + } + try (PDDocument document = Loader.loadPDF(bytes)) { + if (document.getNumberOfPages() > 30) { + continue; + } + inspected++; + List pages = new TaggedContentExtractor().extract(document); + + for (PageContent page : pages) { + if (page.markableCount() == 0 || !hasText(document, page.pageIndex())) { + continue; + } + if (page.lines().isEmpty() && page.forms().isEmpty()) { + dropped.add(repoRoot.relativize(pdf) + " page " + page.pageIndex()); + } + } + } catch (Exception e) { + // Unreadable files are covered by the conversion tests. + } + } + + assertTrue(inspected > 15, "expected to inspect a real corpus, saw " + inspected); + assertTrue( + dropped.isEmpty(), + "the two extraction passes disagreed, so these pages were skipped: " + dropped); + } + + private static boolean hasText(PDDocument document, int pageIndex) { + try { + PDFTextStripper stripper = new PDFTextStripper(); + stripper.setStartPage(pageIndex + 1); + stripper.setEndPage(pageIndex + 1); + return !stripper.getText(document).isBlank(); + } catch (Exception e) { + return false; + } + } + + private List findPdfs() throws Exception { + try (Stream stream = Files.walk(repoRoot)) { + return stream.filter(Files::isRegularFile) + .filter(p -> p.toString().toLowerCase().endsWith(".pdf")) + .filter(p -> !p.toString().contains("node_modules")) + .filter(p -> !p.toString().contains(java.io.File.separator + "build")) + .filter(p -> !p.toString().contains(".git")) + .toList(); + } + } +} diff --git a/engine/src/stirling/models/tool_io.py b/engine/src/stirling/models/tool_io.py index da0cd5b9ae..c9da9cce90 100644 --- a/engine/src/stirling/models/tool_io.py +++ b/engine/src/stirling/models/tool_io.py @@ -126,6 +126,7 @@ TOOL_IO: dict[ToolEndpoint, ToolIOSpec] = { ) ], ), + ToolEndpoint.PDF_TO_UA: ToolIOSpec(accepts=[ToolFormat.PDF], produces=ToolFormat.PDF, arity=ToolArity.SISO), ToolEndpoint.PDF_TO_VECTOR: ToolIOSpec( accepts=[ToolFormat.PDF], produces=ToolFormat.IMAGE, @@ -246,6 +247,9 @@ TOOL_IO: dict[ToolEndpoint, ToolIOSpec] = { ToolEndpoint.SCANNER_EFFECT: ToolIOSpec(accepts=[ToolFormat.PDF], produces=ToolFormat.PDF, arity=ToolArity.SISO), ToolEndpoint.UNLOCK_PDF_FORMS: ToolIOSpec(accepts=[ToolFormat.PDF], produces=ToolFormat.PDF, arity=ToolArity.SISO), ToolEndpoint.UPDATE_METADATA: ToolIOSpec(accepts=[ToolFormat.PDF], produces=ToolFormat.PDF, arity=ToolArity.SISO), + ToolEndpoint.ACCESSIBILITY_REPORT: ToolIOSpec( + accepts=[ToolFormat.PDF], produces=ToolFormat.JSON, arity=ToolArity.SISO + ), ToolEndpoint.ADD_PASSWORD: ToolIOSpec( accepts=[ToolFormat.PDF], produces=ToolFormat.PDF_ENCRYPTED, diff --git a/engine/src/stirling/models/tool_models.py b/engine/src/stirling/models/tool_models.py index 712e3575af..6650942b6f 100644 --- a/engine/src/stirling/models/tool_models.py +++ b/engine/src/stirling/models/tool_models.py @@ -11,6 +11,19 @@ from pydantic import Field, RootModel, SecretStr from stirling.models.base import ApiModel +class Profile(StrEnum): + """ + Profile to check against + """ + + ua1 = "ua1" + ua2 = "ua2" + + +class AccessibilityReportParams(ApiModel): + profile: Profile = Field(Profile.ua1, description="Profile to check against") + + class AddCommentsParams(ApiModel): comments: str = Field( ..., @@ -843,11 +856,18 @@ class OutputFormat1(StrEnum): pdfa_2b = "pdfa-2b" pdfa_3 = "pdfa-3" pdfa_3b = "pdfa-3b" + pdfa_1a = "pdfa-1a" + pdfa_2a = "pdfa-2a" + pdfa_3a = "pdfa-3a" pdfx = "pdfx" class PdfToPdfaParams(ApiModel): output_format: OutputFormat1 = Field(..., description="The output format type (PDF/A or PDF/X)") + pdf_ua: bool = Field( + False, + description="Also declare PDF/UA accessibility alongside PDF/A. Only applies to the level A formats, and the claim is written only if it validates.", + ) strict: bool | None = Field( None, description="If true, the conversion will fail if the output is not perfectly compliant" ) @@ -886,6 +906,65 @@ class PdfToTextParams(ApiModel): output_format: OutputFormat3 = Field(..., description="The output Text or RTF format") +class ExistingTags(StrEnum): + """ + What to do with an existing structure tree: keep it, rebuild it, or decide automatically + """ + + auto = "auto" + keep = "keep" + rebuild = "rebuild" + + +class FigurePolicy(StrEnum): + """ + How to treat images with no description. require-alt leaves them undescribed so the report asks for input; mark-decorative treats every image as decoration. + """ + + require_alt = "require-alt" + mark_decorative = "mark-decorative" + + +class Profile1(StrEnum): + """ + PDF/UA conformance level to target + """ + + ua1 = "ua1" + ua2 = "ua2" + + +class PdfToUaParams(ApiModel): + alt_text: str | None = Field( + None, + description='Alternative descriptions for figures, as key=text pairs separated by newlines. Keys come from the accessibility-report endpoint\'s figuresNeedingDescription list, for example "0:12=Bar chart of quarterly revenue". Descriptions are never invented, so without these an illustrated document cannot claim conformance.', + ) + embed_fonts: bool = Field( + True, + description="Embed fonts the document references but does not carry. Required for conformance and needs Ghostscript.", + ) + existing_tags: ExistingTags = Field( + ExistingTags.auto, + description="What to do with an existing structure tree: keep it, rebuild it, or decide automatically", + ) + figure_policy: FigurePolicy = Field( + FigurePolicy.require_alt, + description="How to treat images with no description. require-alt leaves them undescribed so the report asks for input; mark-decorative treats every image as decoration.", + ) + language: str = Field( + "en-GB", + description="Document language as a BCP-47 tag, for example en-GB. Applied only when the document does not already declare one, unless overrideLanguage is set.", + ) + override_language: bool = Field( + False, + description="Replace the language the document already declares. Off by default, so a document is never relabelled into a language it is not written in.", + ) + profile: Profile1 = Field(Profile1.ua1, description="PDF/UA conformance level to target") + title: str | None = Field( + None, description="Document title, required by PDF/UA. Falls back to the first heading, then the filename." + ) + + class OutputFormat4(StrEnum): """ Target vector format extension @@ -1451,6 +1530,7 @@ class Model( | PdfToPdfaParams | PdfToPresentationParams | PdfToTextParams + | PdfToUaParams | PdfToVectorParams | PdfToWordParams | PdfToXlsxParams @@ -1495,6 +1575,7 @@ class Model( | ScannerEffectParams | UnlockPdfFormsParams | UpdateMetadataParams + | AccessibilityReportParams | AddPasswordParams | AddWatermarkParams | AutoRedactParams @@ -1525,6 +1606,7 @@ class Model( | PdfToPdfaParams | PdfToPresentationParams | PdfToTextParams + | PdfToUaParams | PdfToVectorParams | PdfToWordParams | PdfToXlsxParams @@ -1569,6 +1651,7 @@ class Model( | ScannerEffectParams | UnlockPdfFormsParams | UpdateMetadataParams + | AccessibilityReportParams | AddPasswordParams | AddWatermarkParams | AutoRedactParams @@ -1600,6 +1683,7 @@ type ParamToolModel = ( | PdfToPdfaParams | PdfToPresentationParams | PdfToTextParams + | PdfToUaParams | PdfToVectorParams | PdfToWordParams | PdfToXlsxParams @@ -1644,6 +1728,7 @@ type ParamToolModel = ( | ScannerEffectParams | UnlockPdfFormsParams | UpdateMetadataParams + | AccessibilityReportParams | AddPasswordParams | AddWatermarkParams | AutoRedactParams @@ -1676,6 +1761,7 @@ class ToolEndpoint(StrEnum): PDF_TO_PDFA = "/api/v1/convert/pdf/pdfa" PDF_TO_PRESENTATION = "/api/v1/convert/pdf/presentation" PDF_TO_TEXT = "/api/v1/convert/pdf/text" + PDF_TO_UA = "/api/v1/convert/pdf/ua" PDF_TO_VECTOR = "/api/v1/convert/pdf/vector" PDF_TO_WORD = "/api/v1/convert/pdf/word" PDF_TO_XLSX = "/api/v1/convert/pdf/xlsx" @@ -1720,6 +1806,7 @@ class ToolEndpoint(StrEnum): SCANNER_EFFECT = "/api/v1/misc/scanner-effect" UNLOCK_PDF_FORMS = "/api/v1/misc/unlock-pdf-forms" UPDATE_METADATA = "/api/v1/misc/update-metadata" + ACCESSIBILITY_REPORT = "/api/v1/security/accessibility-report" ADD_PASSWORD = "/api/v1/security/add-password" ADD_WATERMARK = "/api/v1/security/add-watermark" AUTO_REDACT = "/api/v1/security/auto-redact" @@ -1750,6 +1837,7 @@ OPERATIONS: dict[ToolEndpoint, ParamToolModelType] = { ToolEndpoint.PDF_TO_PDFA: PdfToPdfaParams, ToolEndpoint.PDF_TO_PRESENTATION: PdfToPresentationParams, ToolEndpoint.PDF_TO_TEXT: PdfToTextParams, + ToolEndpoint.PDF_TO_UA: PdfToUaParams, ToolEndpoint.PDF_TO_VECTOR: PdfToVectorParams, ToolEndpoint.PDF_TO_WORD: PdfToWordParams, ToolEndpoint.PDF_TO_XLSX: PdfToXlsxParams, @@ -1794,6 +1882,7 @@ OPERATIONS: dict[ToolEndpoint, ParamToolModelType] = { ToolEndpoint.SCANNER_EFFECT: ScannerEffectParams, ToolEndpoint.UNLOCK_PDF_FORMS: UnlockPdfFormsParams, ToolEndpoint.UPDATE_METADATA: UpdateMetadataParams, + ToolEndpoint.ACCESSIBILITY_REPORT: AccessibilityReportParams, ToolEndpoint.ADD_PASSWORD: AddPasswordParams, ToolEndpoint.ADD_WATERMARK: AddWatermarkParams, ToolEndpoint.AUTO_REDACT: AutoRedactParams, diff --git a/frontend/editor/public/locales/en-US/translation.toml b/frontend/editor/public/locales/en-US/translation.toml index c2287ace9c..2424b8e2c5 100644 --- a/frontend/editor/public/locales/en-US/translation.toml +++ b/frontend/editor/public/locales/en-US/translation.toml @@ -3402,6 +3402,24 @@ pdfOptions = "PDF Options" pdfToCbr = "PDF → CBR" pdfToCbz = "PDF → CBZ" pdfToEpub = "PDF → EPUB" +pdfUaAltTextNotice = "Images need a written description before a document can be certified. Descriptions are never generated automatically, because an invented one passes the checker while telling a screen-reader user nothing. Any image left without one is reported, and the file comes back tagged but not certified." +pdfUaAltTextScanFailed = "The images could not be listed. Convert anyway and the response reports what is missing." +pdfUaAltTextSingleFileOnly = "Descriptions belong to one document: an image is identified by its position, which is a different image in every file. Convert these {{fileCount}} files to tag them, then convert one at a time to describe its images." +pdfUaEmbedFonts = "Embed missing fonts" +pdfUaEmbedFontsHelp = "PDF/UA requires every font to be embedded. Turning this off is faster but usually prevents conformance." +pdfUaFigureLabel = "Page {{page}} {{kind}}" +pdfUaFigurePlaceholder = "What this image tells the reader" +pdfUaFindImages = "Find images needing a description" +pdfUaLanguage = "Document language" +pdfUaLanguageHelp = "A BCP-47 tag such as en-GB. Used only when the document does not already declare its own language." +pdfUaNoImagesNeedingText = "No image is missing a description." +pdfUaOptions = "PDF/UA Options" +pdfUaOverrideLanguage = "Replace the document's own language" +pdfUaOverrideLanguageHelp = "Only tick this if the language above is right and the document's own is wrong. Relabelling a document into a language it is not written in makes a screen reader unintelligible." +pdfUaProfile = "Conformance level" +pdfUaSignatureWarning = "This PDF is digitally signed. Tagging rewrites the page content the signature covers, so the signature will stop verifying. Convert first, then re-sign." +pdfUaTitle = "Document title" +pdfUaTitleHelp = "Shown by a reader instead of the filename. Left blank, the first heading is used." selectSourceFormatFirst = "Choose a source format first" settings = "Settings" single = "Single" @@ -6053,6 +6071,11 @@ header = "PDF To PDF/A" tags = "archive,long-term,standard,conversion,storage,preservation" title = "PDF To PDF/A" +[pdfToPDFUA] +header = "PDF To PDF/UA" +tags = "accessibility,accessible,tagged,screen reader,wcag,eaa,section 508,conversion" +title = "PDF To PDF/UA" + [pdfToPDFX] tags = "print,standard,conversion,production,prepress,archive" title = "PDF To PDF/X" diff --git a/frontend/editor/src/core/components/tools/convert/ConvertSettings.tsx b/frontend/editor/src/core/components/tools/convert/ConvertSettings.tsx index c3ee0758f6..c550832c78 100644 --- a/frontend/editor/src/core/components/tools/convert/ConvertSettings.tsx +++ b/frontend/editor/src/core/components/tools/convert/ConvertSettings.tsx @@ -20,6 +20,7 @@ import ConvertFromEmailSettings from "@app/components/tools/convert/ConvertFromE import ConvertFromCbzSettings from "@app/components/tools/convert/ConvertFromCbzSettings"; import ConvertToCbzSettings from "@app/components/tools/convert/ConvertToCbzSettings"; import ConvertToPdfaSettings from "@app/components/tools/convert/ConvertToPdfaSettings"; +import ConvertToPdfUaSettings from "@app/components/tools/convert/ConvertToPdfUaSettings"; import ConvertToPdfxSettings from "@app/components/tools/convert/ConvertToPdfxSettings"; import ConvertFromCbrSettings from "@app/components/tools/convert/ConvertFromCbrSettings"; import ConvertToCbrSettings from "@app/components/tools/convert/ConvertToCbrSettings"; @@ -456,6 +457,20 @@ const ConvertSettings = ({ )} + {/* PDF to PDF/UA options */} + {parameters.fromExtension === "pdf" && + parameters.toExtension === "pdfua" && ( + <> + + + + )} + {/* PDF to PDF/X options */} {parameters.fromExtension === "pdf" && parameters.toExtension === "pdfx" && ( diff --git a/frontend/editor/src/core/components/tools/convert/ConvertToPdfUaSettings.selection.test.tsx b/frontend/editor/src/core/components/tools/convert/ConvertToPdfUaSettings.selection.test.tsx new file mode 100644 index 0000000000..5fbeace1b5 --- /dev/null +++ b/frontend/editor/src/core/components/tools/convert/ConvertToPdfUaSettings.selection.test.tsx @@ -0,0 +1,149 @@ +/** + * Which document the PDF/UA descriptions belong to. + * + * A description is keyed by an image's position inside one file, so it is only meaningful for the + * file it was written against. The panel therefore offers the description fields for a single + * selection only, and forgets what was typed as soon as the selection changes. + */ + +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { MantineProvider } from "@mantine/core"; +import ConvertToPdfUaSettings from "@app/components/tools/convert/ConvertToPdfUaSettings"; +import { defaultParameters } from "@app/hooks/tools/convert/useConvertParameters"; +import type { ConvertParameters } from "@app/hooks/tools/convert/useConvertParameters"; +import type { StirlingFile } from "@app/types/fileContext"; + +// Render the English fallbacks (the test i18n instance has no loaded locale). +vi.mock("react-i18next", () => ({ + useTranslation: () => ({ + t: (key: string, fallback?: unknown, options?: Record) => { + const text = typeof fallback === "string" ? fallback : key; + return options + ? text.replace(/\{\{(\w+)\}\}/g, (_, name) => String(options[name])) + : text; + }, + }), +})); + +const api = vi.hoisted(() => ({ post: vi.fn() })); +vi.mock("@app/services/apiClient", () => ({ default: { post: api.post } })); + +// The real hook parses the PDF in a worker, which is not what this file is about. +vi.mock("@app/hooks/usePdfSignatureDetection", () => ({ + usePdfSignatureDetection: () => ({ + hasDigitalSignatures: false, + isChecking: false, + }), +})); + +const file = (name: string, content = "%PDF-1.7") => + new File([content], name, { type: "application/pdf" }) as StirlingFile; + +const parametersWith = (altText: string): ConvertParameters => ({ + ...defaultParameters, + fromExtension: "pdf", + toExtension: "pdfua", + pdfUaOptions: { ...defaultParameters.pdfUaOptions, altText }, +}); + +function renderPanel(selectedFiles: StirlingFile[], altText = "") { + const onParameterChange = vi.fn(); + const view = render( + + + , + ); + const rerenderWith = (files: StirlingFile[], text = altText) => + view.rerender( + + + , + ); + return { onParameterChange, rerenderWith }; +} + +beforeEach(() => { + vi.clearAllMocks(); + api.post.mockResolvedValue({ + data: { + figuresNeedingDescription: [{ key: "0:1", page: 1, kind: "image" }], + }, + }); +}); + +describe("PDF/UA descriptions are scoped to one document", () => { + test("one file: the images can be listed and described", async () => { + const { onParameterChange } = renderPanel([file("report.pdf")]); + + await userEvent.click(screen.getByTestId("pdfua-find-figures")); + const field = await screen.findByTestId("pdfua-alt-text-0:1"); + await userEvent.type(field, "B"); + + expect(onParameterChange).toHaveBeenCalledWith( + "pdfUaOptions", + expect.objectContaining({ altText: "0:1=B" }), + ); + }); + + test("several files: no description fields, and a reason why", () => { + renderPanel([file("report.pdf"), file("appendix.pdf")]); + + expect( + screen.getByTestId("pdfua-alt-text-single-file-only"), + ).toHaveTextContent( + /Convert these 2 files to tag them, then convert one at a time/, + ); + expect(screen.queryByTestId("pdfua-find-figures")).toBeNull(); + }); + + test("several files: descriptions already typed are dropped, not carried over", async () => { + const { onParameterChange, rerenderWith } = renderPanel( + [file("report.pdf")], + "0:1=Bar chart of revenue", + ); + + rerenderWith([file("report.pdf"), file("appendix.pdf")]); + + await waitFor(() => + expect(onParameterChange).toHaveBeenCalledWith( + "pdfUaOptions", + expect.objectContaining({ altText: "" }), + ), + ); + }); + + test("swapping the single file clears the descriptions written for the old one", async () => { + const { onParameterChange, rerenderWith } = renderPanel( + [file("report.pdf")], + "0:1=Bar chart of revenue", + ); + + rerenderWith([file("other.pdf")]); + + await waitFor(() => + expect(onParameterChange).toHaveBeenCalledWith( + "pdfUaOptions", + expect.objectContaining({ altText: "" }), + ), + ); + }); + + test("mounting with stored descriptions keeps them, so an automation step survives editing", () => { + const { onParameterChange } = renderPanel( + [file("report.pdf")], + "0:1=Bar chart of revenue", + ); + + expect(onParameterChange).not.toHaveBeenCalled(); + }); +}); diff --git a/frontend/editor/src/core/components/tools/convert/ConvertToPdfUaSettings.test.ts b/frontend/editor/src/core/components/tools/convert/ConvertToPdfUaSettings.test.ts new file mode 100644 index 0000000000..ccef5b5142 --- /dev/null +++ b/frontend/editor/src/core/components/tools/convert/ConvertToPdfUaSettings.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, test } from "vitest"; +import { + formatAltText, + parseAltText, +} from "@app/components/tools/convert/ConvertToPdfUaSettings"; + +describe("PDF/UA alt-text wire format", () => { + test("reads the key=description lines the report's keys produce", () => { + expect(parseAltText("0:12=Bar chart\n1:3=Company logo")).toEqual({ + "0:12": "Bar chart", + "1:3": "Company logo", + }); + }); + + test("keeps a description containing an equals sign whole", () => { + expect(parseAltText("2:7=Flow: approval = sign-off")).toEqual({ + "2:7": "Flow: approval = sign-off", + }); + }); + + test("skips blank and malformed lines rather than inventing keys", () => { + expect(parseAltText("\nnot-a-pair\n3:1= \n")).toEqual({}); + }); + + test("round-trips a half-typed description, spaces and all", () => { + // Trimming here would eat the space the moment it is typed, blocking the next word. + const typed = { "0:1": "Bar chart " }; + expect(parseAltText(formatAltText(typed))).toEqual(typed); + }); + + test("drops a description the user cleared", () => { + expect(formatAltText({ "0:1": "Kept", "0:2": " " })).toBe("0:1=Kept"); + }); +}); diff --git a/frontend/editor/src/core/components/tools/convert/ConvertToPdfUaSettings.tsx b/frontend/editor/src/core/components/tools/convert/ConvertToPdfUaSettings.tsx new file mode 100644 index 0000000000..b1e5e4e5fe --- /dev/null +++ b/frontend/editor/src/core/components/tools/convert/ConvertToPdfUaSettings.tsx @@ -0,0 +1,280 @@ +import { useEffect, useRef, useState } from "react"; +import { Stack, Text, Select, Alert, Checkbox, TextInput } from "@mantine/core"; +import { useTranslation } from "react-i18next"; +import apiClient from "@app/services/apiClient"; +import { Button } from "@app/ui/Button"; +import { ConvertParameters } from "@app/hooks/tools/convert/useConvertParameters"; +import { usePdfSignatureDetection } from "@app/hooks/usePdfSignatureDetection"; +import { StirlingFile } from "@app/types/fileContext"; +import { Z_INDEX_AUTOMATE_DROPDOWN } from "@app/styles/zIndex"; + +interface ConvertToPdfUaSettingsProps { + parameters: ConvertParameters; + onParameterChange: ( + key: K, + value: ConvertParameters[K], + ) => void; + selectedFiles: StirlingFile[]; + disabled?: boolean; +} + +/** One image the backend says has no description yet, keyed as the conversion expects it back. */ +interface FigureNeedingDescription { + key: string; + page: number; + kind: string; +} + +/** + * The wire form the endpoint parses: one `pageIndex:ordinal=description` per line. Descriptions are + * kept verbatim so that typing a space does not fight the field; the backend trims them. + */ +export const parseAltText = (raw: string): Record => { + const parsed: Record = {}; + raw.split(/\r?\n/).forEach((line) => { + const split = line.indexOf("="); + if (split <= 0) return; + const key = line.slice(0, split).trim(); + const description = line.slice(split + 1); + if (key && description.trim()) parsed[key] = description; + }); + return parsed; +}; + +export const formatAltText = (descriptions: Record): string => + Object.entries(descriptions) + .filter(([, description]) => description.trim()) + .map(([key, description]) => `${key}=${description}`) + .join("\n"); + +/** PDF/UA conversion options; copy is deliberate - conformance is not guaranteed by one click. */ +const ConvertToPdfUaSettings = ({ + parameters, + onParameterChange, + selectedFiles, + disabled = false, +}: ConvertToPdfUaSettingsProps) => { + const { t } = useTranslation(); + const { hasDigitalSignatures } = usePdfSignatureDetection(selectedFiles); + const [figures, setFigures] = useState( + null, + ); + const [isScanning, setIsScanning] = useState(false); + const [scanError, setScanError] = useState(null); + + const profileOptions = [ + { value: "ua1", label: "PDF/UA-1" }, + { value: "ua2", label: "PDF/UA-2 (PDF 2.0)" }, + ]; + + const update = (patch: Partial) => + onParameterChange("pdfUaOptions", { ...parameters.pdfUaOptions, ...patch }); + + const descriptions = parseAltText(parameters.pdfUaOptions.altText); + // A key is a position inside one document, so descriptions only mean anything for one file. + const scannableFile = selectedFiles.length === 1 ? selectedFiles[0] : null; + const tooManyFiles = selectedFiles.length > 1; + const fileKey = selectedFiles + .map((file) => `${file.name}:${file.size}`) + .join("|"); + const describedFileKey = useRef(fileKey); + + // The same key names a different image in the next document, so descriptions must not outlive the + // selection. Mount is skipped so a stored automation step keeps the text it was saved with. + useEffect(() => { + if (describedFileKey.current === fileKey) return; + describedFileKey.current = fileKey; + setFigures(null); + if (parameters.pdfUaOptions.altText) update({ altText: "" }); + }, [fileKey]); + + // The keys are opaque, so they have to come from the backend's own analysis of this file. + const findFigures = async () => { + const file = scannableFile; + if (!file) return; + setIsScanning(true); + setScanError(null); + try { + const formData = new FormData(); + formData.append("fileInput", file); + formData.append("profile", parameters.pdfUaOptions.profile); + const { data } = await apiClient.post<{ + figuresNeedingDescription?: FigureNeedingDescription[]; + }>("/api/v1/security/accessibility-report", formData); + setFigures(data.figuresNeedingDescription ?? []); + } catch { + setScanError( + t( + "convert.pdfUaAltTextScanFailed", + "The images could not be listed. Convert anyway and the response reports what is missing.", + ), + ); + } finally { + setIsScanning(false); + } + }; + + return ( + + + {t("convert.pdfUaOptions", "PDF/UA Options")}: + + + {hasDigitalSignatures && ( + + + {t( + "convert.pdfUaSignatureWarning", + "This PDF is digitally signed. Tagging rewrites the page content the signature covers, so the signature will stop verifying. Convert first, then re-sign.", + )} + + + )} + + + + {t("convert.pdfUaProfile", "Conformance level")}: + + + value && + setOriginFilter(value as FilesPageOriginFilter) + } + data={[ + { + value: "all", + label: t("filesPage.origin.all", "All sources"), + }, + { + value: "local", + label: t("filesPage.origin.local", "Local"), + }, + { + value: "cloud", + label: t("filesPage.origin.cloud", "Cloud"), + }, + { + value: "shared-with-me", + label: t("filesPage.origin.shared", "Shared"), + }, + ]} + style={{ width: 140 }} + aria-label={t( + "filesPage.originFilter", + "Filter by source", )} - - - - - - - - clearSelection()} + /> + {availableTypes.length > 1 && ( + ({ + value: ext, + label: ext, + }))} + placeholder={ + typeFilter.length === 0 + ? t("filesPage.typeFilter.allTypes", "All types") + : undefined + } + clearable + hidePickedOptions + searchable={false} + style={{ width: 160 }} aria-label={t( - "filesPage.clearSelection", - "Clear selection", + "filesPage.typeFilter.label", + "Filter by type", )} - > - × - - - - ); - })()} - {selectedFiles.length > 0 && ( - + + + ); +} + +export default FilesToolbarFilterMenu; diff --git a/frontend/editor/src/core/components/filesPage/FilesToolbarSortMenu.tsx b/frontend/editor/src/core/components/filesPage/FilesToolbarSortMenu.tsx new file mode 100644 index 0000000000..d8a5ee1018 --- /dev/null +++ b/frontend/editor/src/core/components/filesPage/FilesToolbarSortMenu.tsx @@ -0,0 +1,86 @@ +import { Menu } from "@mantine/core"; +import { useTranslation } from "react-i18next"; +import CheckIcon from "@mui/icons-material/Check"; +import SwapVertIcon from "@mui/icons-material/SwapVert"; + +import { ActionIcon } from "@app/ui/ActionIcon"; +import { Tooltip } from "@app/components/shared/Tooltip"; +import type { FilesPageSortMode } from "@app/contexts/FilesPageContext"; + +interface FilesToolbarSortMenuProps { + value: FilesPageSortMode; + onChange: (mode: FilesPageSortMode) => void; +} + +/** + * Sort control collapsed to a single icon. The desktop Select needs 160px and + * still truncated its longest label ("Recent first" → "Recent fi") once the + * toolbar got tight, so on narrow viewports the options move into a menu where + * they have room to read in full. + */ +export function FilesToolbarSortMenu({ + value, + onChange, +}: FilesToolbarSortMenuProps) { + const { t } = useTranslation(); + + const options: { value: FilesPageSortMode; label: string }[] = [ + { + value: "modified-desc", + label: t("filesPage.sort.modifiedDesc", "Recent first"), + }, + { + value: "modified-asc", + label: t("filesPage.sort.modifiedAsc", "Oldest first"), + }, + { value: "name-asc", label: t("filesPage.sort.nameAsc", "Name A→Z") }, + { value: "name-desc", label: t("filesPage.sort.nameDesc", "Name Z→A") }, + { + value: "size-desc", + label: t("filesPage.sort.sizeDesc", "Largest first"), + }, + { value: "size-asc", label: t("filesPage.sort.sizeAsc", "Smallest first") }, + ]; + + const label = t("filesPage.sort.label", "Sort files"); + const current = options.find((o) => o.value === value)?.label ?? ""; + + return ( + + +
+ + + + + +
+
+ + {label} + {options.map((option) => ( + onChange(option.value)} + leftSection={ + option.value === value ? ( + + ) : ( + + ) + } + > + {option.label} + + ))} + +
+ ); +} + +export default FilesToolbarSortMenu; diff --git a/frontend/editor/src/core/components/layout/Workbench.module.css b/frontend/editor/src/core/components/layout/Workbench.module.css index fb73be8655..dd2b4a12bd 100644 --- a/frontend/editor/src/core/components/layout/Workbench.module.css +++ b/frontend/editor/src/core/components/layout/Workbench.module.css @@ -38,6 +38,13 @@ background: var(--c-hover); } +@media (max-width: 64rem) { + .workbenchBarReopenTab { + width: 3rem; + height: 1.375rem; + } +} + .workbenchBarWrapper { display: grid; grid-template-rows: 1fr; diff --git a/frontend/editor/src/core/components/shared/AppConfigModal.css b/frontend/editor/src/core/components/shared/AppConfigModal.css index 2953632d98..1f2246c8f5 100644 --- a/frontend/editor/src/core/components/shared/AppConfigModal.css +++ b/frontend/editor/src/core/components/shared/AppConfigModal.css @@ -39,41 +39,77 @@ flex-direction: column; } -/* Mobile: compact icon-only navigation */ +/* Mobile: two-level settings navigation */ @media (max-width: 1024px) { .modal-container { - height: 100vh !important; + flex-direction: column; + height: 100dvh !important; max-height: none !important; } .modal-nav { - width: 5rem; /* 80px - wider for larger icons */ - height: 100vh !important; + width: 100%; + flex: 1; + min-height: 0; + height: auto !important; max-height: none !important; - border-top-left-radius: 0; - border-bottom-left-radius: 0; + border-radius: 0; } .modal-nav-scroll { - padding: 1rem 0.5rem; + padding: 0.75rem 0.75rem 2rem; } .modal-nav-section { - margin-bottom: 1.5rem; + margin-bottom: 1.25rem; + } + + .modal-nav-section > .mantine-Text-root { + padding: 0 0.5rem; } .modal-nav-item.mobile { - padding: 1rem; - justify-content: center; - border-radius: 0.75rem; - margin-bottom: 0.75rem; + padding: 0.75rem 0.625rem; + min-height: 3rem; + border-radius: 0.625rem; + margin-bottom: 0.125rem; + gap: 0.75rem; + } + + .modal-nav-item .modal-nav-item-badge { + display: inline-flex; + } + + .modal-nav-chevron { + flex-shrink: 0; + color: var(--c-text-subtle); } .modal-content { - height: 100vh !important; + height: auto !important; + flex: 1; + min-height: 0; max-height: none !important; border-radius: 0; } + + .modal-body { + padding: 1rem; + padding-top: 0.75rem; + } +} + +@media (max-width: 48rem) { + .modal-body [id^="setting-"] { + flex-direction: column; + align-items: flex-start !important; + gap: 0.625rem; + } + + .modal-body [id^="setting-"]:has(.mantine-Switch-root) { + flex-direction: row; + align-items: center !important; + } } .modal-nav-scroll { @@ -241,6 +277,7 @@ @media (max-width: 1024px) { .settings-sticky-footer { padding: 0.75rem 1rem; + padding-bottom: calc(0.75rem + env(safe-area-inset-bottom, 0px)); margin: 0 -1rem; margin-bottom: -1rem; } diff --git a/frontend/editor/src/core/components/shared/AppConfigModal.tsx b/frontend/editor/src/core/components/shared/AppConfigModal.tsx index 12faf6eea5..8c6bf423ed 100644 --- a/frontend/editor/src/core/components/shared/AppConfigModal.tsx +++ b/frontend/editor/src/core/components/shared/AppConfigModal.tsx @@ -7,6 +7,9 @@ import React, { } from "react"; import { Badge, Modal, Text, Tooltip, Group } from "@mantine/core"; import { ActionIcon } from "@app/ui/ActionIcon"; +import { SettingsMobileBackButton } from "@app/components/shared/config/SettingsMobileBackButton"; +import { SettingsMobileNavHeader } from "@app/components/shared/config/SettingsMobileNavHeader"; +import { SettingsNavChevron } from "@app/components/shared/config/SettingsNavChevron"; import { useNavigate, useLocation } from "react-router-dom"; import { useTranslation } from "react-i18next"; import LocalIcon from "@app/components/shared/LocalIcon"; @@ -82,6 +85,7 @@ const AppConfigModalInner: React.FC = ({ "general", ); const isMobile = useIsMobile(); + const [mobilePane, setMobilePane] = useState<"nav" | "content">("nav"); const navigate = useNavigate(); const location = useLocation(); const { config } = useAppConfig(); @@ -122,6 +126,14 @@ const AppConfigModalInner: React.FC = ({ } }, [opened]); + useEffect(() => { + if (!opened) return; + const target = urlSync + ? getSectionFromPath(window.location.pathname) + : initialSection; + setMobilePane(target ? "content" : "nav"); + }, [opened, urlSync, initialSection]); + // Switch tab without forcing every `useLocation()` subscriber (HomePage and // its FileSidebar/Workbench/RightSidebar/FileManager tree) to re-render. // @@ -306,10 +318,17 @@ const AppConfigModalInner: React.FC = ({ const canProceed = await confirmIfDirty(); if (!canProceed) return; switchSection(key); + setMobilePane("content"); }, [confirmIfDirty, switchSection], ); + const handleMobileBack = useCallback(async () => { + const canProceed = await confirmIfDirty(); + if (!canProceed) return; + setMobilePane("nav"); + }, [confirmIfDirty]); + return ( = ({ className={`modal-nav ${isMobile ? "mobile" : ""}`} style={{ background: colors.navBg, - borderRight: `1px solid ${colors.headerBorder}`, + ...(isMobile + ? { display: mobilePane === "nav" ? undefined : "none" } + : { borderRight: `1px solid ${colors.headerBorder}` }), }} > +
{configNavSections.map((section) => (
- {!isMobile && ( - - {section.title} - - )} + + {section.title} +
{section.items.map((item) => { const isActive = active === item.key; @@ -355,7 +380,7 @@ const AppConfigModalInner: React.FC = ({ const color = isActive ? colors.navItemActive : colors.navItem; - const iconSize = isMobile ? 28 : 18; + const iconSize = 18; const showPlanWarning = item.key === "adminPlan" && licenseAlert.active && @@ -383,47 +408,46 @@ const AppConfigModalInner: React.FC = ({ icon={item.icon} width={iconSize} height={iconSize} - style={{ color }} + style={{ color, flexShrink: 0 }} /> - {!isMobile && ( - + - + {item.badge && ( + - {item.label} - - {item.badge && ( - - {item.badge} - - )} - {showPlanWarning && ( - - )} - - )} + {item.badge} + + )} + {showPlanWarning && ( + + )} + +
); @@ -450,7 +474,15 @@ const AppConfigModalInner: React.FC = ({
{/* Right content */} -
+
{/* Sticky header with section title and small close button */}
= ({ borderBottom: `1px solid ${colors.headerBorder}`, }} > - - {activeLabel} - + + void handleMobileBack()} + /> + + {activeLabel} + + * { +.workbench-bar[data-wrapped="true"] .workbench-bar-center-scroll > * { flex-shrink: 0; } @@ -152,16 +152,25 @@ .workbench-bar-center { order: 4; flex: 0 0 100%; + min-width: 0; + max-width: 100%; position: relative; display: flex; align-items: center; + /* Symmetric side padding leaves room for the retract handle pinned right + without knocking the centred tool icons off-centre. */ + padding: 4px 36px; + border-top: 1px solid var(--c-border-subtle); +} + +.workbench-bar-center-scroll { + flex: 1 1 auto; + min-width: 0; + display: flex; + align-items: center; justify-content: center; flex-wrap: wrap; gap: 2px; - /* Symmetric side padding leaves room for the retract handle pinned right - without knocking the centred tool icons off-centre. */ - padding: 4px 36px; - border-top: 1px solid var(--c-border-subtle); } /* Retract / reopen handle for the viewer tool row. */ @@ -297,3 +306,79 @@ text-align: right; white-space: nowrap; } + +/* ---- Mobile layout (matches useIsMobile's 1024px) ---- */ +@media (max-width: 64rem) { + .workbench-bar { + margin: var(--nav-gutter) var(--nav-gutter) 0; + } + + .workbench-bar-action-icon { + width: 40px !important; + height: 40px !important; + min-width: 40px !important; + min-height: 40px !important; + } + + .workbench-bar-views, + .workbench-bar-globals { + height: auto; + min-height: 44px; + } + + .workbench-bar-center { + padding: 2px 4px 2px 8px; + } + + .workbench-bar-center-scroll { + gap: 4px; + } + + .workbench-bar[data-wrapped="true"] .workbench-bar-search { + order: 2; + flex: 1 1 0; + min-width: 0; + padding: 4px 0; + } + + .workbench-bar[data-wrapped="true"] .workbench-bar-globals { + order: 3; + } + + .workbench-bar[data-wrapped="true"] .workbench-bar-center-scroll { + scrollbar-width: none; + /* Wider than one 40px icon plus its gap: a 2rem fade always landed + mid-glyph, which read as a clipping bug rather than "scroll me". */ + -webkit-mask-image: linear-gradient( + to right, + #000 calc(100% - 3.5rem), + transparent + ); + mask-image: linear-gradient( + to right, + #000 calc(100% - 3.5rem), + transparent + ); + } + + .workbench-bar[data-wrapped="true"] + .workbench-bar-center--expanded + .workbench-bar-center-scroll { + flex-wrap: wrap; + justify-content: center; + overflow-x: visible; + -webkit-mask-image: none; + mask-image: none; + } + + .workbench-bar[data-wrapped="true"] + .workbench-bar-center--expanded + .workbench-bar-divider { + display: none; + } + + .workbench-bar-toolbar-handle-expand { + flex-shrink: 0; + align-self: flex-start; + } +} diff --git a/frontend/editor/src/core/components/shared/WorkbenchBar.tsx b/frontend/editor/src/core/components/shared/WorkbenchBar.tsx index ad38fb1974..19026c9df0 100644 --- a/frontend/editor/src/core/components/shared/WorkbenchBar.tsx +++ b/frontend/editor/src/core/components/shared/WorkbenchBar.tsx @@ -3,9 +3,9 @@ import React, { useLayoutEffect, useMemo, useRef, + useState, useSyncExternalStore, } from "react"; -import { Group, Loader, Progress, Stack, Text } from "@mantine/core"; import { Button } from "@app/ui/Button"; import { ActionIcon } from "@app/ui/ActionIcon"; import { SegmentedControl } from "@app/ui/SegmentedControl"; @@ -31,7 +31,6 @@ import { useToolWorkflow } from "@app/contexts/ToolWorkflowContext"; import { useNavigationState } from "@app/contexts/NavigationContext"; import { ViewerContext, useViewer } from "@app/contexts/ViewerContext"; import { WorkbenchType, isBaseWorkbench } from "@app/types/workbench"; -import { Tooltip } from "@app/components/shared/Tooltip"; import LocalIcon from "@app/components/shared/LocalIcon"; import SuperSearch from "@app/components/shared/superSearch/SuperSearch"; import { useEditorSearchScopes } from "@app/hooks/useSuperSearch"; @@ -53,10 +52,12 @@ import { } from "@app/types/workbenchBar"; import InsertDriveFileOutlinedIcon from "@mui/icons-material/InsertDriveFileOutlined"; import FolderOutlinedIcon from "@mui/icons-material/FolderOutlined"; -import CloseIcon from "@mui/icons-material/Close"; -import PrintIcon from "@mui/icons-material/Print"; -import KeyboardArrowUpIcon from "@mui/icons-material/KeyboardArrowUp"; -import ShieldOutlinedIcon from "@mui/icons-material/ShieldOutlined"; +import WorkbenchBarDesktopActions from "@app/components/shared/workbenchBar/WorkbenchBarDesktopActions"; +import WorkbenchBarMobileActions from "@app/components/shared/workbenchBar/WorkbenchBarMobileActions"; +import WorkbenchBarToolbarHandle from "@app/components/shared/workbenchBar/WorkbenchBarToolbarHandle"; +import { renderWithTooltip } from "@app/components/shared/workbenchBar/workbenchBarTooltip"; +import { WorkbenchBarActionsProps } from "@app/components/shared/workbenchBar/types"; +import { useIsMobile } from "@app/hooks/useIsMobile"; import "@app/components/shared/WorkbenchBar.css"; const SECTION_ORDER: WorkbenchBarSection[] = ["top", "middle", "bottom"]; @@ -77,24 +78,6 @@ interface WorkbenchBarProps { onCollapseViewerToolbar?: (collapsed: boolean) => void; } -function renderWithTooltip( - node: React.ReactNode, - tooltip: React.ReactNode | undefined, -) { - if (!tooltip) return node; - return ( - -
{node}
-
- ); -} - export default function WorkbenchBar({ currentView, setCurrentView, @@ -132,6 +115,8 @@ export default function WorkbenchBar({ const icons = useFileActionIcons(); const { sharingEnabled } = useSharingEnabled(); const viewerContext = React.useContext(ViewerContext); + const isMobile = useIsMobile(); + const [mobileToolsExpanded, setMobileToolsExpanded] = useState(false); const selectors = useFileSelectors(); const { selectedFiles, selectedFileIds } = useFileSelection(); @@ -166,32 +151,6 @@ export default function WorkbenchBar({ enforcingRun?.currentStep != null && enforcingRun.stepCount ? Math.round((enforcingRun.currentStep / enforcingRun.stepCount) * 100) : undefined; - const makeEnforcingTooltip = (action: string): React.ReactNode => ( - - - - - {t( - "policy.blockingAction", - "{{action}} blocked while enforcing policy, please wait", - { action }, - )} - - - {enforcingProgress != null ? ( - - ) : ( - - )} - - ); const pageEditorTotalPages = pageEditorFunctions?.totalPages ?? 0; const pageEditorSelectedCount = pageEditorFunctions?.selectedPageIds?.length ?? 0; @@ -365,6 +324,33 @@ export default function WorkbenchBar({ return terminology.downloadAll; }, [currentView, selectedCount, t, terminology]); + const actionsDisabled = + totalItems === 0 || allButtonsDisabled || disableForFullscreen; + + // Shared by the mobile overflow menu and the desktop icon cluster so the two + // stay in step; each renders the same actions in its own shape. + const globalActionProps: WorkbenchBarActionsProps = { + currentView, + isCustomView, + actionsDisabled, + policyEnforcing, + downloadLabel: downloadTooltip, + downloadIconName: icons.downloadIconName, + saveAsIconName: icons.saveAsIconName, + onPrint: handlePrint, + onExport: handleExportAll, + onClose: handleClose, + }; + + const toggleMobileTools = useCallback( + () => setMobileToolsExpanded((v) => !v), + [], + ); + const handleRetractToolbar = useCallback( + () => onCollapseViewerToolbar?.(true), + [onCollapseViewerToolbar], + ); + const renderButton = useCallback( (btn: WorkbenchBarButtonConfig) => { const action = actions[btn.id]; @@ -560,38 +546,44 @@ export default function WorkbenchBar({ whole row; Workbench then shows a tab below the bar to bring it back. */} {sectionsWithButtons.length > 0 && !(isViewer && viewerToolbarCollapsed) && ( -
- {sectionsWithButtons.map( - ({ section, buttons: sectionButtons }, idx) => ( - - {idx > 0 &&
} - {sectionButtons.map((btn) => { - const content = renderButton(btn); - if (!content) return null; - return ( -
- {content} -
- ); - })} - - ), - )} - {isViewer && onCollapseViewerToolbar && ( -
- )} + ) : null} (null); + // Phones render a desktop-width document into ~400px, which reads as a blank + // column, so the iframe is opt-in there. Derived rather than seeded into + // state because useIsMobile resolves after first paint. + const [optedIn, setOptedIn] = useState(false); + const showPreview = !isMobile || optedIn; useEffect(() => { + if (!showPreview) return; const url = URL.createObjectURL(file); setObjectUrl(url); return () => URL.revokeObjectURL(url); - }, [file]); + }, [file, showPreview]); return ( - - {t("viewer.nonPdf.htmlPreviewWarning", { - size: formatFileSize(file.size), - })} - + + + {t("viewer.nonPdf.htmlPreviewWarning", { + size: formatFileSize(file.size), + })} + + {/* Opting in used to be one-way: the only way back was closing and + reopening the file. */} + {isMobile && optedIn && ( + + )} + - {objectUrl && ( -