mirror of
https://github.com/Stirling-Tools/Stirling-PDF.git
synced 2026-09-03 21:30:14 +03:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9cd461cc08 | ||
|
|
9c8be25ff0 |
@@ -20,8 +20,8 @@ set -e
|
||||
# - To build the project, use:
|
||||
# ./gradlew build
|
||||
#
|
||||
# - To run the lint/format/secret checks, use:
|
||||
# task pre-commit
|
||||
# - For running pre-commit hooks (if configured), use:
|
||||
# pre-commit run --all-files
|
||||
#
|
||||
# Make sure you are in the project root directory after this script executes.
|
||||
# =============================================================================
|
||||
@@ -70,6 +70,6 @@ echo ""
|
||||
echo " To build the project: "
|
||||
echo -e "\e[34m gradle build\e[0m"
|
||||
echo ""
|
||||
echo " To run the lint/format/secret checks:"
|
||||
echo -e "\e[34m task pre-commit\e[0m"
|
||||
echo " To run pre-commit hooks (if configured):"
|
||||
echo -e "\e[34m pre-commit run --all-files -c .pre-commit-config.yaml\e[0m"
|
||||
echo "=================================================================="
|
||||
|
||||
+2
-2
@@ -26,8 +26,8 @@ version_builds/
|
||||
node_modules/
|
||||
**/node_modules/
|
||||
frontend/node_modules/
|
||||
frontend/editor/dist/
|
||||
frontend/editor/playwright-report/
|
||||
frontend/dist/
|
||||
frontend/playwright-report/
|
||||
.npm/
|
||||
.yarn/
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ runs:
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
|
||||
with:
|
||||
client-id: ${{ inputs.app-id }}
|
||||
app-id: ${{ inputs.app-id }}
|
||||
private-key: ${{ inputs.private-key }}
|
||||
- name: Configure Git
|
||||
run: |
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-desktop
|
||||
pkgver=2.14.2
|
||||
pkgver=2.11.0
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (Tauri desktop app, official Stirling PDF Inc build)"
|
||||
arch=('x86_64')
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Stirling PDF Inc <contact@stirlingpdf.com>
|
||||
pkgname=stirling-pdf-server-bin
|
||||
pkgver=2.14.2
|
||||
pkgver=2.11.0
|
||||
pkgrel=1
|
||||
pkgdesc="Locally hosted, web-based PDF manipulation tool (server JAR, prebuilt)"
|
||||
arch=('any')
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
build: &build
|
||||
- build.gradle
|
||||
- app/(common|core|proprietary)/build.gradle
|
||||
- Taskfile.yml
|
||||
- .taskfiles/backend.yml
|
||||
|
||||
openapi: &openapi
|
||||
- *build
|
||||
@@ -40,9 +38,6 @@ project: &project
|
||||
- frontend/**
|
||||
- docker/**
|
||||
- scripts/RestartHelper.java
|
||||
- Taskfile.yml
|
||||
- .taskfiles/backend.yml
|
||||
- .taskfiles/docker.yml
|
||||
- scripts/db-migration/**
|
||||
- .github/workflows/db-migration-test.yml
|
||||
|
||||
@@ -60,9 +55,6 @@ frontend: &frontend
|
||||
- scripts/summarize_type3_signatures.py
|
||||
- scripts/type3_to_cff.py
|
||||
- scripts/update_type3_library.py
|
||||
- Taskfile.yml
|
||||
- .taskfiles/frontend.yml
|
||||
- .taskfiles/e2e.yml
|
||||
|
||||
# Files that affect the Tauri desktop bundle. Gate the multi-OS Tauri build
|
||||
# job on changes to any of these.
|
||||
@@ -74,8 +66,6 @@ tauri: &tauri
|
||||
- frontend/package-lock.json
|
||||
- frontend/editor/vite.config.ts
|
||||
- .github/workflows/tauri-build.yml
|
||||
- Taskfile.yml
|
||||
- .taskfiles/desktop.yml
|
||||
|
||||
# Files that affect the AI engine (Python tool models, fixers, tests). Gate
|
||||
# the engine validation job on changes to engine sources or to the Java
|
||||
@@ -84,23 +74,6 @@ engine: &engine
|
||||
- engine/**
|
||||
- app/(common|core|proprietary)/src/main/java/**
|
||||
- .github/workflows/ai-engine.yml
|
||||
- Taskfile.yml
|
||||
- .taskfiles/engine.yml
|
||||
|
||||
# Files that can make the committed generated API models (frontend tool API
|
||||
# types + engine tool models) go stale: the Java tool surfaces they derive from,
|
||||
# the generators, the generated files themselves (to catch a hand-edit), and the
|
||||
# tasks that drive generation. Deliberately excludes the broad frontend/docker/
|
||||
# testing globs, so a CSS-only PR does not boot the backend to rebuild the spec.
|
||||
generated-models: &generated-models
|
||||
- *openapi
|
||||
- frontend/editor/scripts/generate-tool-api-types.mts
|
||||
- frontend/editor/src/core/types/toolApiTypes.ts
|
||||
- engine/scripts/generate_tool_models.py
|
||||
- engine/src/stirling/models/tool_models.py
|
||||
- .taskfiles/frontend.yml
|
||||
- .taskfiles/engine.yml
|
||||
- .github/workflows/check-generated-models.yml
|
||||
|
||||
licenses-frontend: &licenses-frontend
|
||||
- ".github/workflows/frontend-backend-licenses-update.yml"
|
||||
@@ -129,4 +102,4 @@ proprietary: &proprietary
|
||||
- configs/settings.yml.template
|
||||
- build.gradle
|
||||
- app/proprietary/build.gradle
|
||||
- .github/workflows/build-enterprise.yml
|
||||
- .github/workflows/build-enterprise.yml
|
||||
@@ -13,7 +13,7 @@ Usage:
|
||||
"""
|
||||
|
||||
# Sample for Windows:
|
||||
# python .github/scripts/check_language_toml.py --reference-file frontend/editor/public/locales/en-US/translation.toml --branch "" --files frontend/editor/public/locales/de-DE/translation.toml frontend/editor/public/locales/fr-FR/translation.toml
|
||||
# python .github/scripts/check_language_toml.py --reference-file frontend/editor/public/locales/en-GB/translation.toml --branch "" --files frontend/editor/public/locales/de-DE/translation.toml frontend/editor/public/locales/fr-FR/translation.toml
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
@@ -184,7 +184,7 @@ def check_for_differences(reference_file, file_list, branch, actor):
|
||||
if len(file_list) == 1:
|
||||
file_arr = file_list[0].split()
|
||||
|
||||
base_dir = Path.cwd() / "frontend" / "editor" / "public" / "locales"
|
||||
base_dir = Path.cwd() / "frontend" / "public" / "locales"
|
||||
|
||||
for file_path in file_arr:
|
||||
file_path = Path(file_path)
|
||||
@@ -211,7 +211,7 @@ def check_for_differences(reference_file, file_list, branch, actor):
|
||||
)
|
||||
continue
|
||||
|
||||
if basename_current_file == basename_reference_file and locale_dir == "en-US":
|
||||
if basename_current_file == basename_reference_file and locale_dir == "en-GB":
|
||||
continue
|
||||
|
||||
if (
|
||||
@@ -308,7 +308,7 @@ def check_for_differences(reference_file, file_list, branch, actor):
|
||||
report.append("## ❌ Overall Check Status: **_Failed_**")
|
||||
report.append("")
|
||||
report.append(
|
||||
f"@{actor} please check your translation if it conforms to the standard. Follow the format of [en-US/translation.toml](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/frontend/editor/public/locales/en-US/translation.toml)"
|
||||
f"@{actor} please check your translation if it conforms to the standard. Follow the format of [en-GB/translation.toml](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/frontend/editor/public/locales/en-GB/translation.toml)"
|
||||
)
|
||||
else:
|
||||
report.append("## ✅ Overall Check Status: **_Success_**")
|
||||
@@ -372,7 +372,6 @@ if __name__ == "__main__":
|
||||
os.path.join(
|
||||
os.getcwd(),
|
||||
"frontend",
|
||||
"editor",
|
||||
"public",
|
||||
"locales",
|
||||
"*",
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
pre-commit
|
||||
@@ -0,0 +1,121 @@
|
||||
#
|
||||
# This file is autogenerated by pip-compile with Python 3.12
|
||||
# by the following command:
|
||||
#
|
||||
# pip-compile --generate-hashes --output-file='.github\scripts\requirements_pre_commit.txt' --strip-extras '.github\scripts\requirements_pre_commit.in'
|
||||
#
|
||||
cfgv==3.5.0 \
|
||||
--hash=sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0 \
|
||||
--hash=sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132
|
||||
# via pre-commit
|
||||
distlib==0.4.0 \
|
||||
--hash=sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16 \
|
||||
--hash=sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d
|
||||
# via virtualenv
|
||||
filelock==3.29.0 \
|
||||
--hash=sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90 \
|
||||
--hash=sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258
|
||||
# via
|
||||
# python-discovery
|
||||
# virtualenv
|
||||
identify==2.6.19 \
|
||||
--hash=sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a \
|
||||
--hash=sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842
|
||||
# via pre-commit
|
||||
nodeenv==1.10.0 \
|
||||
--hash=sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827 \
|
||||
--hash=sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb
|
||||
# via pre-commit
|
||||
platformdirs==4.9.6 \
|
||||
--hash=sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a \
|
||||
--hash=sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917
|
||||
# via
|
||||
# python-discovery
|
||||
# virtualenv
|
||||
pre-commit==4.6.0 \
|
||||
--hash=sha256:718d2208cef53fdc38206e40524a6d4d9576d103eb16f0fec11c875e7716e9d9 \
|
||||
--hash=sha256:e2cf246f7299edcabcf15f9b0571fdce06058527f0a06535068a86d38089f29b
|
||||
# via -r .github/scripts/requirements_pre_commit.in
|
||||
python-discovery==1.2.2 \
|
||||
--hash=sha256:876e9c57139eb757cb5878cbdd9ae5379e5d96266c99ef731119e04fffe533bb \
|
||||
--hash=sha256:e1ae95d9af875e78f15e19aed0c6137ab1bb49c200f21f5061786490c9585c7a
|
||||
# via virtualenv
|
||||
pyyaml==6.0.3 \
|
||||
--hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \
|
||||
--hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \
|
||||
--hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \
|
||||
--hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \
|
||||
--hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \
|
||||
--hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \
|
||||
--hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \
|
||||
--hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \
|
||||
--hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \
|
||||
--hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \
|
||||
--hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \
|
||||
--hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \
|
||||
--hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \
|
||||
--hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \
|
||||
--hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \
|
||||
--hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \
|
||||
--hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \
|
||||
--hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \
|
||||
--hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \
|
||||
--hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \
|
||||
--hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \
|
||||
--hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \
|
||||
--hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \
|
||||
--hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \
|
||||
--hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \
|
||||
--hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \
|
||||
--hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \
|
||||
--hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \
|
||||
--hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \
|
||||
--hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \
|
||||
--hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \
|
||||
--hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \
|
||||
--hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \
|
||||
--hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \
|
||||
--hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \
|
||||
--hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \
|
||||
--hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \
|
||||
--hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \
|
||||
--hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \
|
||||
--hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \
|
||||
--hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \
|
||||
--hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \
|
||||
--hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \
|
||||
--hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \
|
||||
--hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \
|
||||
--hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \
|
||||
--hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \
|
||||
--hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \
|
||||
--hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \
|
||||
--hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \
|
||||
--hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \
|
||||
--hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \
|
||||
--hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \
|
||||
--hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \
|
||||
--hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \
|
||||
--hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \
|
||||
--hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \
|
||||
--hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \
|
||||
--hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \
|
||||
--hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \
|
||||
--hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \
|
||||
--hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \
|
||||
--hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \
|
||||
--hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \
|
||||
--hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \
|
||||
--hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \
|
||||
--hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \
|
||||
--hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \
|
||||
--hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \
|
||||
--hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \
|
||||
--hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \
|
||||
--hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \
|
||||
--hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0
|
||||
# via pre-commit
|
||||
virtualenv==21.2.4 \
|
||||
--hash=sha256:29d21e941795206138d0f22f4e45ff7050e5da6c6472299fb7103318763861ac \
|
||||
--hash=sha256:b294ef68192638004d72524ce7ef303e9d0cf5a44c95ce2e54a7500a6381cada
|
||||
# via pre-commit
|
||||
@@ -1,83 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify Tauri updater .sig files against plugins.updater.pubkey in tauri.conf.json.
|
||||
|
||||
Usage: verify-updater-signatures.py <dir-to-scan> [tauri.conf.json]
|
||||
"""
|
||||
|
||||
import binascii
|
||||
import sys
|
||||
import json
|
||||
import base64
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
|
||||
ART_ROOT = Path(sys.argv[1])
|
||||
CONF = Path(
|
||||
sys.argv[2] if len(sys.argv) > 2 else "frontend/editor/src-tauri/tauri.conf.json"
|
||||
)
|
||||
|
||||
|
||||
def load_pubkey():
|
||||
# tauri pubkey = base64 of a minisign .pub file; last line is base64 of
|
||||
# [2 algo][8 key-id][32 ed25519 public key].
|
||||
raw = json.loads(CONF.read_text())["plugins"]["updater"]["pubkey"]
|
||||
blob = base64.b64decode(base64.b64decode(raw).decode().splitlines()[-1])
|
||||
return blob[2:10], Ed25519PublicKey.from_public_bytes(blob[10:])
|
||||
|
||||
|
||||
def hash_file(path: Path) -> bytes:
|
||||
h = hashlib.blake2b(digest_size=64)
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 16), b""):
|
||||
h.update(chunk)
|
||||
return h.digest()
|
||||
|
||||
|
||||
def verify(artifact: Path, sig_file: Path, keyid_pub, pub) -> str:
|
||||
# tauri .sig = base64 of a minisign signature file (4 lines).
|
||||
try:
|
||||
lines = base64.b64decode(sig_file.read_text()).decode().splitlines()
|
||||
sig_blob = base64.b64decode(lines[1])
|
||||
except (binascii.Error, IndexError, UnicodeDecodeError) as e:
|
||||
return f"FAIL malformed sig ({type(e).__name__})"
|
||||
algo, keyid, sig = sig_blob[:2], sig_blob[2:10], sig_blob[10:74]
|
||||
if keyid != keyid_pub:
|
||||
return f"FAIL key-id mismatch (sig {keyid.hex()} vs pub {keyid_pub.hex()})"
|
||||
# 'ED' = prehashed (BLAKE2b-512), 'Ed' = legacy (raw message).
|
||||
msg = hash_file(artifact) if algo == b"ED" else artifact.read_bytes()
|
||||
try:
|
||||
pub.verify(sig, msg)
|
||||
except InvalidSignature:
|
||||
return f"FAIL signature invalid (algo={algo.decode()})"
|
||||
# Global signature covers sig + trusted_comment.
|
||||
gc = "global-sig FAIL"
|
||||
try:
|
||||
tc = lines[2].split("trusted comment: ", 1)[1]
|
||||
pub.verify(base64.b64decode(lines[3]), sig + tc.encode())
|
||||
gc = "global-sig OK"
|
||||
except (InvalidSignature, IndexError, binascii.Error):
|
||||
pass
|
||||
return f"VALID (algo={algo.decode()}, keyid={keyid.hex()}, {gc})"
|
||||
|
||||
|
||||
keyid_pub, pub = load_pubkey()
|
||||
print(f"updater pubkey keyid={keyid_pub.hex()}\n")
|
||||
sigs = sorted(ART_ROOT.rglob("*.sig"))
|
||||
if not sigs:
|
||||
print(f"WARN: no .sig files under {ART_ROOT} - nothing to verify")
|
||||
sys.exit(0)
|
||||
bad = 0
|
||||
for sig_file in sigs:
|
||||
artifact = sig_file.with_suffix("")
|
||||
if not artifact.exists():
|
||||
print(f" ? {sig_file.name}: artifact missing")
|
||||
bad += 1
|
||||
continue
|
||||
res = verify(artifact, sig_file, keyid_pub, pub)
|
||||
print(f" {artifact.name}: {res}")
|
||||
if not res.startswith("VALID") or "global-sig FAIL" in res:
|
||||
bad += 1
|
||||
print(f"\n{'ALL SIGNATURES VALID' if bad == 0 else f'{bad} SIGNATURE(S) FAILED'}")
|
||||
sys.exit(1 if bad else 0)
|
||||
@@ -239,7 +239,7 @@ jobs:
|
||||
|
||||
- name: Build and push V2 image (Depot)
|
||||
if: env.USE_DEPOT == 'true' && steps.check-image.outputs.exists == 'false'
|
||||
uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.16.0
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
@@ -293,7 +293,7 @@ jobs:
|
||||
SECURITY_ENABLELOGIN: "true"
|
||||
SECURITY_INITIALLOGIN_USERNAME: "${{ secrets.TEST_LOGIN_USERNAME }}"
|
||||
SECURITY_INITIALLOGIN_PASSWORD: "${{ secrets.TEST_LOGIN_PASSWORD }}"
|
||||
SYSTEM_DEFAULTLOCALE: en-US
|
||||
SYSTEM_DEFAULTLOCALE: en-GB
|
||||
UI_APPNAME: "Stirling-PDF V2 PR#${{ needs.check-pr.outputs.pr_number }}"
|
||||
UI_HOMEDESCRIPTION: "V2 PR#${{ needs.check-pr.outputs.pr_number }} - Embedded Architecture"
|
||||
UI_APPNAMENAVBAR: "V2 PR#${{ needs.check-pr.outputs.pr_number }}"
|
||||
|
||||
@@ -222,10 +222,10 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Run Gradle Command
|
||||
run: |
|
||||
if [ "${{ needs.check-comment.outputs.disable_security }}" == "true" ]; then
|
||||
@@ -256,7 +256,7 @@ jobs:
|
||||
|
||||
- name: Build and push PR-specific image (Depot)
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.16.0
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
@@ -285,7 +285,7 @@ jobs:
|
||||
|
||||
- name: Build and push engine image (Depot)
|
||||
if: env.USE_DEPOT == 'true' && needs.check-comment.outputs.enable_prototypes == 'true'
|
||||
uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.16.0
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: ./engine
|
||||
@@ -388,7 +388,7 @@ jobs:
|
||||
environment:
|
||||
DISABLE_ADDITIONAL_FEATURES: "${DISABLE_ADDITIONAL_FEATURES}"
|
||||
SECURITY_ENABLELOGIN: "${LOGIN_SECURITY}"
|
||||
SYSTEM_DEFAULTLOCALE: en-US
|
||||
SYSTEM_DEFAULTLOCALE: en-GB
|
||||
UI_APPNAME: "Stirling-PDF PR#${PR_NUMBER}"
|
||||
UI_HOMEDESCRIPTION: "PR#${PR_NUMBER} for Stirling-PDF Latest"
|
||||
UI_APPNAMENAVBAR: "PR#${PR_NUMBER}"
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
name: AI Engine CI
|
||||
|
||||
# Runs the engine quality gate (lint, type-check, format-check, tests). Called
|
||||
# from build.yml on PRs and merge_group; also runs directly on push to main as
|
||||
# a post-merge safety net. Freshness of the generated tool_models.py is checked
|
||||
# by the shared check-generated-models workflow.
|
||||
# Validates the Python AI engine: regenerates tool models and runs the
|
||||
# engine quality gate (lint, type-check, format-check, tests). Called from
|
||||
# build.yml on PRs and merge_group; also runs directly on push to main as
|
||||
# a post-merge safety net.
|
||||
on:
|
||||
workflow_call:
|
||||
push:
|
||||
@@ -30,12 +30,107 @@ jobs:
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
with:
|
||||
enable-cache: true
|
||||
|
||||
- name: Set up JDK 25
|
||||
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
|
||||
with:
|
||||
java-version: "25"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.3.1
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
|
||||
- name: Regenerate tool models
|
||||
run: task engine:tool-models
|
||||
|
||||
- name: Verify tool models are up to date
|
||||
id: tool-models-check
|
||||
continue-on-error: true
|
||||
run: git diff --exit-code engine/src/stirling/models/tool_models.py
|
||||
|
||||
- name: Comment on tool models check failure
|
||||
# Only post a comment on PRs. github-script's PR helpers need an
|
||||
# issue/PR number, which doesn't exist on merge_group runs.
|
||||
if: steps.tool-models-check.outcome == 'failure' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- tool-models-check -->';
|
||||
const body = [
|
||||
marker,
|
||||
'### Tool Models Check Failed',
|
||||
'',
|
||||
'The generated `engine/src/stirling/models/tool_models.py` is out of date with the Java OpenAPI spec and will need to be regenerated before it can be merged in.',
|
||||
'',
|
||||
'Run `task engine:tool-models` to regenerate, then commit the updated file.',
|
||||
].join('\n');
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
- name: Fail if tool models check failed
|
||||
if: steps.tool-models-check.outcome == 'failure'
|
||||
run: |
|
||||
echo "============================================"
|
||||
echo " Tool Models Check Failed"
|
||||
echo "============================================"
|
||||
echo ""
|
||||
echo "The generated engine/src/stirling/models/tool_models.py"
|
||||
echo "is out of date with the Java OpenAPI spec and will"
|
||||
echo "need to be regenerated before it can be merged in."
|
||||
echo ""
|
||||
echo "Run 'task engine:tool-models' to regenerate, then"
|
||||
echo "commit the updated file."
|
||||
echo "============================================"
|
||||
exit 1
|
||||
|
||||
- name: Remove tool models check comment on success
|
||||
if: steps.tool-models-check.outcome == 'success' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- tool-models-check -->';
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
});
|
||||
}
|
||||
|
||||
- name: Quality-check engine
|
||||
id: engine-check
|
||||
|
||||
@@ -1,14 +1,8 @@
|
||||
name: Backend build, format check, and coverage
|
||||
|
||||
# Reusable workflow called from build.yml. Runs the backend build matrix
|
||||
# (JDK 25 × every flavor), Spotless formatting check, JUnit, and
|
||||
# (JDK 25 × spring-security on/off), Spotless formatting check, JUnit, and
|
||||
# posts Jacoco coverage to PRs.
|
||||
#
|
||||
# Flavor axis (maps to STIRLING_FLAVOR in settings.gradle):
|
||||
# core - DISABLE_ADDITIONAL_FEATURES=true, no proprietary, no saas
|
||||
# proprietary - default build, no saas
|
||||
# saas - proprietary + the saas subproject (build + JUnit only,
|
||||
# never any runtime/integration testing)
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
@@ -31,7 +25,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
jdk-version: [25]
|
||||
flavor: [core, proprietary, saas]
|
||||
spring-security: [true, false]
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
@@ -58,16 +52,13 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
cache-disabled: true
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Check Java formatting (Spotless)
|
||||
# Runs once per matrix combination - pick the cheapest leg
|
||||
# (core - no proprietary, no saas) so we don't wait for the
|
||||
# heavier flavors just to fail formatting.
|
||||
if: matrix.jdk-version == 25 && matrix.flavor == 'core'
|
||||
if: matrix.jdk-version == 25 && matrix.spring-security == false
|
||||
id: spotless-check
|
||||
run: task backend:format:check
|
||||
continue-on-error: true
|
||||
@@ -152,24 +143,17 @@ jobs:
|
||||
});
|
||||
}
|
||||
|
||||
- name: Build with Gradle (flavor=${{ matrix.flavor }})
|
||||
# STIRLING_FLAVOR is read by settings.gradle and expands into the
|
||||
# right combination of DISABLE_ADDITIONAL_FEATURES + ENABLE_SAAS
|
||||
# so we don't have to set them by hand. The saas flavor pulls in
|
||||
# the app/saas subproject (unit tests only - no runtime tests).
|
||||
- name: Build with Gradle and spring security ${{ matrix.spring-security }}
|
||||
run: task backend:build:ci
|
||||
env:
|
||||
MAVEN_USER: ${{ secrets.MAVEN_USER }}
|
||||
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
|
||||
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
|
||||
STIRLING_FLAVOR: ${{ matrix.flavor }}
|
||||
DISABLE_ADDITIONAL_FEATURES: ${{ matrix.spring-security }}
|
||||
|
||||
- name: Check Test Reports Exist
|
||||
if: always()
|
||||
run: |
|
||||
# Common + core + proprietary always build (proprietary is
|
||||
# excluded only at runtime, not from the gradle subproject
|
||||
# graph). Saas builds add a fourth report dir.
|
||||
declare -a dirs=(
|
||||
"app/core/build/reports/tests/"
|
||||
"app/core/build/test-results/"
|
||||
@@ -178,9 +162,6 @@ jobs:
|
||||
"app/proprietary/build/reports/tests/"
|
||||
"app/proprietary/build/test-results/"
|
||||
)
|
||||
if [ "${{ matrix.flavor }}" = "saas" ]; then
|
||||
dirs+=("app/saas/build/reports/tests/" "app/saas/build/test-results/")
|
||||
fi
|
||||
for dir in "${dirs[@]}"; do
|
||||
if [ ! -d "$dir" ]; then
|
||||
echo "Missing $dir"
|
||||
@@ -192,7 +173,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: test-reports-jdk-${{ matrix.jdk-version }}-flavor-${{ matrix.flavor }}
|
||||
name: test-reports-jdk-${{ matrix.jdk-version }}-spring-security-${{ matrix.spring-security }}
|
||||
path: |
|
||||
app/**/build/reports/jacoco/test
|
||||
app/**/build/reports/tests/
|
||||
@@ -202,47 +183,7 @@ jobs:
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Install defusedxml for coverage summary
|
||||
# coverage-summary.py parses JaCoCo XML through defusedxml to
|
||||
# silence security scanners that pattern-match on the stdlib
|
||||
# xml.etree.ElementTree.parse call.
|
||||
if: always() && matrix.flavor == 'saas'
|
||||
run: python -m pip install --quiet defusedxml
|
||||
|
||||
- name: JaCoCo coverage step summary
|
||||
# Only the saas leg posts the JUnit summary - it's a strict
|
||||
# superset of the core + proprietary legs (same .exec files plus
|
||||
# the saas subproject). Posting from all three would mean three
|
||||
# near-identical tables crowding out the aggregate report.
|
||||
if: always() && matrix.flavor == 'saas'
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Backend JUnit coverage (JDK ${{ matrix.jdk-version }})" \
|
||||
--jacoco "common=app/common/build/reports/jacoco/test/jacocoTestReport.xml" \
|
||||
--jacoco "core=app/core/build/reports/jacoco/test/jacocoTestReport.xml" \
|
||||
--jacoco "proprietary=app/proprietary/build/reports/jacoco/test/jacocoTestReport.xml" \
|
||||
--jacoco "saas=app/saas/build/reports/jacoco/test/jacocoTestReport.xml" \
|
||||
--github-step-summary
|
||||
|
||||
- name: Upload raw JUnit .exec for aggregate merge
|
||||
# Same dedup rationale as the summary step: upload from the saas
|
||||
# leg only (the most complete set, includes app/saas/.../test.exec)
|
||||
# so the aggregate workflow merges the union rather than three
|
||||
# overlapping subsets.
|
||||
#
|
||||
# Separate artifact from the HTML reports so the aggregate
|
||||
# workflow can grab just the .exec files with a name pattern
|
||||
# (`jacoco-exec-*`) instead of unpacking the whole test-reports
|
||||
# tarball.
|
||||
if: always() && matrix.flavor == 'saas'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-exec-junit-jdk-${{ matrix.jdk-version }}
|
||||
path: app/*/build/jacoco/*.exec
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Add coverage to PR (flavor=${{ matrix.flavor }}, JDK=${{ matrix.jdk-version }})
|
||||
- name: Add coverage to PR with spring security ${{ matrix.spring-security }} and JDK ${{ matrix.jdk-version }}
|
||||
# The action only supports the pull_request event (it posts a PR comment),
|
||||
# so skip it for merge_group runs and workflow_dispatch.
|
||||
if: github.event_name == 'pull_request'
|
||||
|
||||
@@ -2,7 +2,7 @@ name: Enterprise E2E (Playwright)
|
||||
|
||||
# Enterprise Playwright suite — exercises premium-key gated features (audit,
|
||||
# teams, analytics) plus full OAuth + SAML logins via the Keycloak compose
|
||||
# stacks under testing/compose. Slow and secret-gated, so it runs in four
|
||||
# stacks under testing/compose. Slow and secret-gated, so it runs in three
|
||||
# situations:
|
||||
#
|
||||
# - PRs that touch proprietary / premium / SSO compose / enterprise tests
|
||||
@@ -12,6 +12,8 @@ name: Enterprise E2E (Playwright)
|
||||
# - on a nightly cron schedule (catches Keycloak image drift, license
|
||||
# expiry, upstream proprietary changes),
|
||||
# - manual workflow_dispatch.
|
||||
#
|
||||
# Auto-skipped when secrets.PREMIUM_KEY_ENTERPRISE is missing (forks, dependabot).
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
@@ -50,10 +52,6 @@ jobs:
|
||||
|
||||
playwright-e2e-enterprise:
|
||||
needs: pick
|
||||
# Skip on fork PRs / untrusted authors: they have no PREMIUM_KEY_ENTERPRISE
|
||||
# (nor DEPOT_TOKEN), so the suite can't boot premium and would fail. See the
|
||||
# header comment. GitHub reports the skipped reusable workflow as success.
|
||||
if: needs.pick.outputs.is_fork != 'true'
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || format('depot-ubuntu-24.04-{0}', inputs.depot_cores || '8') }}
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
@@ -80,7 +78,7 @@ jobs:
|
||||
cache: "npm"
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Install Playwright (chromium only)
|
||||
run: task e2e:install -- chromium
|
||||
- name: Build frontend (needed for playwright's vite preview webServer)
|
||||
@@ -167,8 +165,6 @@ jobs:
|
||||
wait_for_backend
|
||||
- name: Run enterprise OAuth Playwright tests
|
||||
id: oauth-tests
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results-oauth.json
|
||||
run: task e2e:enterprise -- --grep "OAuth"
|
||||
- name: Stop backend + tear down OAuth Keycloak
|
||||
if: always()
|
||||
@@ -242,8 +238,6 @@ jobs:
|
||||
wait_for_backend
|
||||
- name: Run enterprise SAML Playwright tests
|
||||
id: saml-tests
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results-saml.json
|
||||
run: task e2e:enterprise -- --grep "SAML"
|
||||
- name: Stop backend + tear down SAML Keycloak
|
||||
if: always()
|
||||
@@ -274,8 +268,6 @@ jobs:
|
||||
wait_for_backend
|
||||
- name: Run enterprise feature Playwright tests
|
||||
id: feature-tests
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results-feature.json
|
||||
run: task e2e:enterprise -- --grep "Enterprise license"
|
||||
- name: Print backend log on failure
|
||||
if: failure()
|
||||
@@ -288,23 +280,10 @@ jobs:
|
||||
run: |
|
||||
source /tmp/helpers.sh
|
||||
stop_backend
|
||||
- name: Flag flaky tests
|
||||
# Runs regardless of the test outcomes: a flaky test (passed on retry)
|
||||
# leaves its step green, so this is the only place it surfaces. Merges
|
||||
# all three phase reports (some may be absent if an earlier phase hard-
|
||||
# failed and skipped the rest). Emits ::warning:: annotations + a job
|
||||
# summary; never fails the job.
|
||||
if: always()
|
||||
working-directory: frontend
|
||||
run: >
|
||||
npx tsx editor/scripts/report-flaky-tests.mts
|
||||
"${{ github.workspace }}/frontend/playwright-report/results-oauth.json"
|
||||
"${{ github.workspace }}/frontend/playwright-report/results-saml.json"
|
||||
"${{ github.workspace }}/frontend/playwright-report/results-feature.json"
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-report-enterprise-${{ github.run_id }}
|
||||
path: frontend/playwright-report/
|
||||
path: frontend/editor/playwright-report/
|
||||
retention-days: 7
|
||||
|
||||
@@ -43,7 +43,6 @@ jobs:
|
||||
docker-base: ${{ steps.changes.outputs.docker-base }}
|
||||
tauri: ${{ steps.changes.outputs.tauri }}
|
||||
engine: ${{ steps.changes.outputs.engine }}
|
||||
generated-models: ${{ steps.changes.outputs.generated-models }}
|
||||
proprietary: ${{ steps.changes.outputs.proprietary }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
@@ -172,20 +171,6 @@ jobs:
|
||||
uses: ./.github/workflows/ai-engine.yml
|
||||
secrets: inherit
|
||||
|
||||
# The generated frontend types and engine tool models are both derived from
|
||||
# the Java OpenAPI spec. This job regenerates and diffs them; it boots the
|
||||
# backend, so it is gated on the narrow generated-models filter (spec source,
|
||||
# generators, generated files, generation tasks) rather than the broad
|
||||
# frontend filter, so a CSS-only PR does not pay for a backend build.
|
||||
generated-models:
|
||||
if: needs.files-changed.outputs.generated-models == 'true'
|
||||
needs: [files-changed]
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
uses: ./.github/workflows/check-generated-models.yml
|
||||
secrets: inherit
|
||||
|
||||
pre-commit:
|
||||
needs: [files-changed]
|
||||
permissions:
|
||||
@@ -200,27 +185,6 @@ jobs:
|
||||
uses: ./.github/workflows/dependency-review.yml
|
||||
secrets: inherit
|
||||
|
||||
# Coverage aggregate: merges the JUnit + e2e:live + cucumber .exec
|
||||
# artifacts produced by the jobs above into one report, plus pulls
|
||||
# in vitest + Playwright frontend coverage for the per-area matrix.
|
||||
# `if: always()` so a producer failing partway still gets credit
|
||||
# for whatever did record. Advisory only - intentionally NOT in
|
||||
# all-checks-passed, so a flaky aggregate run never blocks merging.
|
||||
coverage-aggregate:
|
||||
if: always()
|
||||
needs:
|
||||
- build
|
||||
- playwright-e2e-live
|
||||
- docker-compose-tests
|
||||
- frontend-validation
|
||||
permissions:
|
||||
contents: read
|
||||
uses: ./.github/workflows/coverage-aggregate.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
frontend-validation-result: ${{ needs.frontend-validation.result }}
|
||||
playwright-e2e-live-result: ${{ needs.playwright-e2e-live.result }}
|
||||
|
||||
# Single status check that branch protection should mark as required.
|
||||
# Succeeds when every upstream job is either `success` or `skipped` (path-
|
||||
# gated jobs that didn't apply this run). Any `failure` or `cancelled`
|
||||
@@ -243,7 +207,6 @@ jobs:
|
||||
- test-build-docker-images
|
||||
- tauri-build
|
||||
- ai-engine
|
||||
- generated-models
|
||||
- pre-commit
|
||||
- dependency-review
|
||||
runs-on: ubuntu-latest
|
||||
@@ -269,7 +232,6 @@ jobs:
|
||||
test-build-docker-images=${{ needs.test-build-docker-images.result }}
|
||||
tauri-build=${{ needs.tauri-build.result }}
|
||||
ai-engine=${{ needs.ai-engine.result }}
|
||||
generated-models=${{ needs.generated-models.result }}
|
||||
pre-commit=${{ needs.pre-commit.result }}
|
||||
dependency-review=${{ needs.dependency-review.result }}
|
||||
run: |
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
name: Check generated models
|
||||
|
||||
# Verifies the committed generated API models are still in sync with the Java
|
||||
# OpenAPI spec: the frontend tool API types
|
||||
# (frontend/editor/src/core/types/toolApiTypes.ts) and the engine tool
|
||||
# models (engine/src/stirling/models/tool_models.py). Regenerates both with the
|
||||
# single top-level `task tool-models` and fails if either committed file is
|
||||
# out of date. Called from build.yml when the backend Java, frontend, or engine
|
||||
# changes; also runs on push to main as a post-merge safety net.
|
||||
on:
|
||||
workflow_call:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
generated-models:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
with:
|
||||
enable-cache: true
|
||||
|
||||
- name: Set up JDK 25
|
||||
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
|
||||
with:
|
||||
java-version: "25"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: "npm"
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
|
||||
# Rebuilds the OpenAPI spec from the current Java and regenerates both the
|
||||
# frontend types and the engine tool models from it.
|
||||
- name: Regenerate generated models
|
||||
run: task tool-models
|
||||
|
||||
- name: Verify generated models are up to date
|
||||
id: models-check
|
||||
continue-on-error: true
|
||||
run: |
|
||||
git diff --exit-code \
|
||||
frontend/editor/src/core/types/toolApiTypes.ts \
|
||||
engine/src/stirling/models/tool_models.py
|
||||
|
||||
- name: Comment on generated models check failure
|
||||
# Only post a comment on PRs. github-script's PR helpers need an
|
||||
# issue/PR number, which doesn't exist on merge_group runs.
|
||||
if: steps.models-check.outcome == 'failure' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- generated-models-check -->';
|
||||
const body = [
|
||||
marker,
|
||||
'### Generated Models Check Failed',
|
||||
'',
|
||||
'The generated `frontend/editor/src/core/types/toolApiTypes.ts` and/or `engine/src/stirling/models/tool_models.py` are out of date with the Java OpenAPI spec and will need to be regenerated before they can be merged in.',
|
||||
'',
|
||||
'Run `task tool-models` to regenerate both, then commit the updated files.',
|
||||
].join('\n');
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
- name: Fail if generated models check failed
|
||||
if: steps.models-check.outcome == 'failure'
|
||||
run: |
|
||||
echo "============================================"
|
||||
echo " Generated Models Check Failed"
|
||||
echo "============================================"
|
||||
echo ""
|
||||
echo "The generated frontend API types and/or engine tool"
|
||||
echo "models are out of date with the Java OpenAPI spec and"
|
||||
echo "will need to be regenerated before they can be merged in."
|
||||
echo ""
|
||||
echo "Run 'task tool-models' to regenerate both, then"
|
||||
echo "commit the updated files."
|
||||
echo "============================================"
|
||||
exit 1
|
||||
|
||||
- name: Remove generated models check comment on success
|
||||
if: steps.models-check.outcome == 'success' && github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- generated-models-check -->';
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
});
|
||||
}
|
||||
@@ -40,11 +40,11 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
cache-disabled: true
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Check licenses for compatibility
|
||||
run: task backend:licenses:check
|
||||
env:
|
||||
|
||||
@@ -45,11 +45,11 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
cache-disabled: true
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Generate OpenAPI documentation
|
||||
run: task backend:swagger
|
||||
env:
|
||||
|
||||
@@ -166,16 +166,16 @@ jobs:
|
||||
|
||||
// Determine reference file
|
||||
let referenceFilePath;
|
||||
if (changedFiles.includes("frontend/editor/public/locales/en-US/translation.toml")) {
|
||||
if (changedFiles.includes("frontend/editor/public/locales/en-GB/translation.toml")) {
|
||||
console.log("Using PR branch reference file.");
|
||||
const { data: fileContent } = await github.rest.repos.getContent({
|
||||
owner: prRepoOwner,
|
||||
repo: prRepoName,
|
||||
path: "frontend/editor/public/locales/en-US/translation.toml",
|
||||
path: "frontend/editor/public/locales/en-GB/translation.toml",
|
||||
ref: branch,
|
||||
});
|
||||
|
||||
referenceFilePath = "pr-branch-translation-en-US.toml";
|
||||
referenceFilePath = "pr-branch-translation-en-GB.toml";
|
||||
const content = Buffer.from(fileContent.content, "base64").toString("utf-8");
|
||||
fs.writeFileSync(referenceFilePath, content);
|
||||
} else {
|
||||
@@ -183,11 +183,11 @@ jobs:
|
||||
const { data: fileContent } = await github.rest.repos.getContent({
|
||||
owner: repoOwner,
|
||||
repo: repoName,
|
||||
path: "frontend/editor/public/locales/en-US/translation.toml",
|
||||
path: "frontend/editor/public/locales/en-GB/translation.toml",
|
||||
ref: "main",
|
||||
});
|
||||
|
||||
referenceFilePath = "main-branch-translation-en-US.toml";
|
||||
referenceFilePath = "main-branch-translation-en-GB.toml";
|
||||
const content = Buffer.from(fileContent.content, "base64").toString("utf-8");
|
||||
fs.writeFileSync(referenceFilePath, content);
|
||||
}
|
||||
@@ -293,6 +293,6 @@ jobs:
|
||||
run: |
|
||||
echo "Cleaning up temporary files..."
|
||||
rm -rf pr-branch
|
||||
rm -f pr-branch-translation-en-US.toml main-branch-translation-en-US.toml changed_files.txt result.txt
|
||||
rm -f pr-branch-translation-en-GB.toml main-branch-translation-en-GB.toml changed_files.txt result.txt
|
||||
echo "Cleanup complete."
|
||||
continue-on-error: true # Ensure cleanup runs even if previous steps fail
|
||||
|
||||
@@ -1,241 +0,0 @@
|
||||
name: Aggregate backend coverage
|
||||
|
||||
# Reusable workflow called from build.yml after every backend coverage
|
||||
# producer (JUnit, e2e:live, cucumber) has run. Downloads each job's raw
|
||||
# .exec, merges them into one JaCoCo report, and posts a combined step
|
||||
# summary alongside the per-source ones.
|
||||
#
|
||||
# Kept separate from the per-source jobs so:
|
||||
# - the per-source jobs stay fast and independent (no cross-job waits)
|
||||
# - this job can `if: always()` and still produce something useful when
|
||||
# one of the producers fails partway through
|
||||
# - frontend producers can be added later without touching the
|
||||
# producers themselves
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
frontend-validation-result:
|
||||
description: Result of the frontend-validation producer job
|
||||
required: false
|
||||
type: string
|
||||
default: skipped
|
||||
playwright-e2e-live-result:
|
||||
description: Result of the playwright-e2e-live producer job
|
||||
required: false
|
||||
type: string
|
||||
default: skipped
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
aggregate:
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up JDK 25
|
||||
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
|
||||
with:
|
||||
java-version: "25"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Cache Gradle dependency artifacts
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.gradle/wrapper
|
||||
~/.gradle/caches/modules-2/files-2.1
|
||||
~/.gradle/caches/modules-2/metadata-2.*
|
||||
key: gradle-deps-${{ runner.os }}-jdk-25-${{ hashFiles('**/gradle/wrapper/gradle-wrapper.properties', '**/*.gradle', '**/*.gradle.kts', 'settings.gradle', 'settings.gradle.kts', 'gradle/libs.versions.toml') }}
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
cache-disabled: true
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install defusedxml for coverage scripts
|
||||
# Both coverage-summary.py and coverage-matrix.py parse JaCoCo
|
||||
# XML through defusedxml - see the script headers for context.
|
||||
run: python -m pip install --quiet defusedxml
|
||||
|
||||
# Pattern matches every artifact this PR's producers might upload:
|
||||
# jacoco-exec-junit-jdk-25 (uploaded only by the saas
|
||||
# leg of backend-build, which
|
||||
# is a strict superset of the
|
||||
# core + proprietary legs)
|
||||
# jacoco-exec-e2e-live
|
||||
# jacoco-exec-cucumber
|
||||
# Each lands as a sibling dir under coverage-execs/, with the .exec
|
||||
# files preserving their original relative paths.
|
||||
- name: Download all .exec artifacts
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
|
||||
with:
|
||||
pattern: jacoco-exec-*
|
||||
path: coverage-execs/
|
||||
merge-multiple: false
|
||||
continue-on-error: true
|
||||
|
||||
- name: Inventory .exec files
|
||||
id: inventory
|
||||
# Splits the downloaded artifacts into two buckets:
|
||||
# * e2e-only = cucumber + Playwright live (user-flow coverage)
|
||||
# * all = the above plus JUnit (everything we test)
|
||||
#
|
||||
# Bucketing is by artifact-name prefix: download-artifact preserves
|
||||
# the artifact name as the top-level dir, so JUnit's `.exec`s live
|
||||
# under coverage-execs/jacoco-exec-junit-*/... while the others
|
||||
# are under coverage-execs/jacoco-exec-{e2e-live,cucumber}/...
|
||||
#
|
||||
# If nothing was uploaded (e.g. all producers crashed before
|
||||
# writing) we exit gracefully so this advisory job never fails CI.
|
||||
run: |
|
||||
mapfile -t all_execs < <(find coverage-execs -name '*.exec' -type f | sort)
|
||||
mapfile -t e2e_execs < <(find coverage-execs -name '*.exec' -type f -not -path '*/jacoco-exec-junit-*' | sort)
|
||||
if [ "${#all_execs[@]}" -eq 0 ]; then
|
||||
echo "::warning::No .exec artifacts found - skipping aggregate report"
|
||||
echo "found_all=false" >> "$GITHUB_OUTPUT"
|
||||
echo "found_e2e=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
printf 'All %d .exec files:\n' "${#all_execs[@]}"
|
||||
printf ' %s\n' "${all_execs[@]}"
|
||||
IFS=','; all_joined="${all_execs[*]}"
|
||||
echo "files_all=$all_joined" >> "$GITHUB_OUTPUT"
|
||||
echo "found_all=true" >> "$GITHUB_OUTPUT"
|
||||
if [ "${#e2e_execs[@]}" -eq 0 ]; then
|
||||
echo "::notice::No e2e/cucumber .exec files - e2e-only report will be skipped"
|
||||
echo "found_e2e=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
printf 'E2E-only %d .exec files:\n' "${#e2e_execs[@]}"
|
||||
printf ' %s\n' "${e2e_execs[@]}"
|
||||
unset IFS
|
||||
IFS=','; e2e_joined="${e2e_execs[*]}"
|
||||
echo "files_e2e=$e2e_joined" >> "$GITHUB_OUTPUT"
|
||||
echo "found_e2e=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Compile classes for JaCoCo class lookup
|
||||
# jacocoReportFromExec only needs the compiled .class files
|
||||
# under each subproject's build/classes/java/main/. `classes`
|
||||
# (compileJava + processResources) is enough; we skipped the
|
||||
# heavier `assemble` to avoid building bootJar / fat jars that
|
||||
# add 60+ seconds per run for no gain to the report.
|
||||
if: steps.inventory.outputs.found_all == 'true'
|
||||
run: ./gradlew classes -PnoSpotless
|
||||
|
||||
- name: Generate e2e-only JaCoCo report
|
||||
# "Real user-flow" coverage: only counts code reached by an actual
|
||||
# HTTP request from cucumber or live Playwright. Useful for
|
||||
# questions like "how much of our backend does a user actually
|
||||
# hit?". Skipped when neither producer uploaded a .exec.
|
||||
if: steps.inventory.outputs.found_e2e == 'true'
|
||||
run: |
|
||||
./gradlew jacocoReportFromExec \
|
||||
-PexecFile="${{ steps.inventory.outputs.files_e2e }}" \
|
||||
-PreportDir=build/reports/jacoco/aggregate-e2e \
|
||||
-PnoSpotless
|
||||
|
||||
- name: Generate combined JaCoCo report (everything)
|
||||
if: steps.inventory.outputs.found_all == 'true'
|
||||
run: |
|
||||
./gradlew jacocoReportFromExec \
|
||||
-PexecFile="${{ steps.inventory.outputs.files_all }}" \
|
||||
-PreportDir=build/reports/jacoco/aggregate-all \
|
||||
-PnoSpotless
|
||||
|
||||
- name: E2E-only step summary
|
||||
# Rendered first so it gets prime real estate in the Summary
|
||||
# tab - this is the number most readers actually want
|
||||
# ("how much of the backend do real user flows cover?").
|
||||
if: steps.inventory.outputs.found_e2e == 'true'
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Real user-flow backend coverage (e2e:live + cucumber)" \
|
||||
--jacoco "merged=build/reports/jacoco/aggregate-e2e/jacocoTestReport.xml" \
|
||||
--github-step-summary
|
||||
|
||||
- name: ALL-sources step summary
|
||||
# Separate call (not a multi-input one) because the helper's
|
||||
# rightmost "Aggregate" column would sum the two reports - which
|
||||
# is meaningless when one is a strict superset of the other.
|
||||
if: steps.inventory.outputs.found_all == 'true'
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Combined backend coverage (JUnit + e2e:live + cucumber)" \
|
||||
--jacoco "merged=build/reports/jacoco/aggregate-all/jacocoTestReport.xml" \
|
||||
--github-step-summary
|
||||
|
||||
- name: Upload combined aggregate report
|
||||
if: steps.inventory.outputs.found_all == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-aggregate-all-${{ github.run_id }}
|
||||
path: build/reports/jacoco/aggregate-all/
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload e2e-only aggregate report
|
||||
if: steps.inventory.outputs.found_e2e == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-aggregate-e2e-${{ github.run_id }}
|
||||
path: build/reports/jacoco/aggregate-e2e/
|
||||
retention-days: 14
|
||||
|
||||
# --------------------------------------------------------------
|
||||
# Per-area matrix: rolls backend + frontend coverage into one
|
||||
# table indexed by core/proprietary/saas/desktop. Pulls the
|
||||
# frontend artifacts now (after the JaCoCo step has done its
|
||||
# work) so the per-source backend summaries still render first
|
||||
# even if the matrix step fails.
|
||||
# --------------------------------------------------------------
|
||||
- name: Download vitest coverage artifact
|
||||
# frontend-validation uploads as `frontend-coverage`. Tolerate
|
||||
# absence on backend-only runs by skipping the download entirely
|
||||
# when the producer job was not part of this workflow run.
|
||||
if: inputs.frontend-validation-result == 'success'
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
|
||||
with:
|
||||
name: frontend-coverage
|
||||
path: matrix-inputs/vitest/
|
||||
continue-on-error: true
|
||||
|
||||
- name: Download Playwright frontend coverage artifact
|
||||
# e2e-live uploads the artifact with a stable name. Skip the
|
||||
# download entirely when the producer job did not run.
|
||||
if: inputs.playwright-e2e-live-result == 'success'
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
|
||||
with:
|
||||
name: playwright-frontend-coverage
|
||||
path: matrix-inputs/playwright/
|
||||
continue-on-error: true
|
||||
|
||||
- name: Coverage matrix step summary
|
||||
if: always()
|
||||
# Matrix references the two aggregate JaCoCo XMLs (already
|
||||
# generated above) plus whichever frontend artifacts landed.
|
||||
# Every input is optional; missing ones render as "-".
|
||||
run: |
|
||||
python scripts/coverage-matrix.py \
|
||||
${{ steps.inventory.outputs.found_all == 'true' && '--jacoco-all build/reports/jacoco/aggregate-all/jacocoTestReport.xml' || '' }} \
|
||||
${{ steps.inventory.outputs.found_e2e == 'true' && '--jacoco-e2e build/reports/jacoco/aggregate-e2e/jacocoTestReport.xml' || '' }} \
|
||||
--vitest matrix-inputs/vitest/coverage-summary.json \
|
||||
--playwright-frontend matrix-inputs/playwright/coverage-pw-summary/coverage-summary.json \
|
||||
--title "Coverage matrix (per-area, e2e vs all)" \
|
||||
--github-step-summary
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
cache-disabled: true
|
||||
|
||||
# No `-PnoSpotless` here yet because the upstream cache layer matches the
|
||||
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
|
||||
- name: Build and push frontend image (Depot)
|
||||
if: env.USE_DEPOT == 'true' && steps.check-frontend.outputs.exists == 'false'
|
||||
uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.16.0
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
@@ -136,7 +136,7 @@ jobs:
|
||||
|
||||
- name: Build and push backend image (Depot)
|
||||
if: env.USE_DEPOT == 'true' && steps.check-backend.outputs.exists == 'false'
|
||||
uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.16.0
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
@@ -188,7 +188,7 @@ jobs:
|
||||
environment:
|
||||
DISABLE_ADDITIONAL_FEATURES: "true"
|
||||
SECURITY_ENABLELOGIN: "false"
|
||||
SYSTEM_DEFAULTLOCALE: en-US
|
||||
SYSTEM_DEFAULTLOCALE: en-GB
|
||||
UI_APPNAME: "Stirling-PDF V2"
|
||||
UI_HOMEDESCRIPTION: "V2 Frontend/Backend Split"
|
||||
UI_APPNAMENAVBAR: "V2 Deployment"
|
||||
|
||||
@@ -61,22 +61,14 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
cache-disabled: true
|
||||
|
||||
# When the PR changes the base image, test.sh builds it locally
|
||||
# (stirling-pdf-base:local) into the daemon image store. A buildx
|
||||
# container builder can't see that store, so skip it here and let
|
||||
# `docker buildx build` fall back to the default docker driver, which
|
||||
# resolves the local base. The gha cache backend is also skipped (its
|
||||
# runtime token isn't exposed) since the docker driver can't use it.
|
||||
- name: Set up Docker Buildx
|
||||
if: inputs.docker-base-changed != 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
# Expose ACTIONS_RUNTIME_TOKEN / ACTIONS_RESULTS_URL for docker buildx type=gha cache backend.
|
||||
- name: Expose GitHub runtime for Buildx cache
|
||||
if: inputs.docker-base-changed != 'true'
|
||||
uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0
|
||||
|
||||
- name: Install Docker Compose
|
||||
@@ -95,12 +87,6 @@ jobs:
|
||||
run: |
|
||||
pip install --require-hashes --only-binary=:all: -r ./testing/cucumber/requirements.txt
|
||||
|
||||
- name: Extract JaCoCo agent for cucumber coverage
|
||||
# Stages build/jacoco/jacocoagent.jar where the coverage override
|
||||
# file bind-mounts it into the cucumber container. The agent jar
|
||||
# never goes into the published image - this is host-only.
|
||||
run: ./gradlew copyJacocoAgent -PnoSpotless
|
||||
|
||||
- name: Run Docker Compose Tests
|
||||
run: |
|
||||
chmod +x ./testing/test_webpages.sh
|
||||
@@ -112,62 +98,6 @@ jobs:
|
||||
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
|
||||
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
|
||||
DOCKER_BASE_CHANGED: ${{ inputs.docker-base-changed }}
|
||||
# Tells test.sh to layer testing/compose/docker-compose-coverage.override.yml
|
||||
# over the cucumber compose so the container starts with the
|
||||
# JaCoCo agent attached via JAVA_CUSTOM_OPTS.
|
||||
STIRLING_PDF_TEST_COVERAGE: "1"
|
||||
|
||||
- name: Generate cucumber JaCoCo report
|
||||
# `if: always()` so a behave failure still produces partial
|
||||
# coverage from whatever endpoints did run. The exec file only
|
||||
# exists when the container shut down cleanly - guard so the step
|
||||
# is silent on the (rare) crash path.
|
||||
if: always()
|
||||
id: cucumber-coverage
|
||||
run: |
|
||||
if [ -s testing/cucumber-coverage/cucumber.exec ]; then
|
||||
./gradlew jacocoReportFromExec \
|
||||
-PexecFile=testing/cucumber-coverage/cucumber.exec \
|
||||
-PreportDir=build/reports/jacoco/cucumber \
|
||||
-PnoSpotless
|
||||
echo "report=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "::warning::No cucumber .exec at testing/cucumber-coverage/cucumber.exec (container may have crashed before flushing)"
|
||||
echo "report=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Install defusedxml for coverage summary
|
||||
# coverage-summary.py parses JaCoCo XML through defusedxml -
|
||||
# see the script header for context.
|
||||
if: always() && steps.cucumber-coverage.outputs.report == 'true'
|
||||
run: python -m pip install --quiet defusedxml
|
||||
|
||||
- name: Cucumber coverage step summary
|
||||
if: always() && steps.cucumber-coverage.outputs.report == 'true'
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Cucumber (docker) JaCoCo coverage" \
|
||||
--jacoco "cucumber=build/reports/jacoco/cucumber/jacocoTestReport.xml" \
|
||||
--github-step-summary
|
||||
|
||||
- name: Upload cucumber JaCoCo report
|
||||
if: always() && steps.cucumber-coverage.outputs.report == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-cucumber-${{ github.run_id }}
|
||||
path: build/reports/jacoco/cucumber/
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload raw cucumber .exec for aggregate merge
|
||||
# Picked up by the coverage-aggregate workflow via the
|
||||
# `jacoco-exec-*` artifact name pattern.
|
||||
if: always() && steps.cucumber-coverage.outputs.report == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-exec-cucumber
|
||||
path: testing/cucumber-coverage/cucumber.exec
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload Cucumber Report
|
||||
if: always()
|
||||
|
||||
@@ -42,149 +42,16 @@ jobs:
|
||||
cache: "npm"
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Install Playwright (chromium only)
|
||||
run: task e2e:install -- chromium
|
||||
- name: Build frontend (production bundle for vite preview)
|
||||
env:
|
||||
VITE_BUILD_FOR_PREVIEW: "1"
|
||||
run: task frontend:build
|
||||
- name: Run live E2E tests (chromium) with coverage
|
||||
- name: Run live E2E tests (chromium)
|
||||
id: live-tests
|
||||
env:
|
||||
# Attaches the JaCoCo agent to the bootRun JVM (see
|
||||
# .taskfiles/e2e.yml live:backend). The .exec gets flushed on
|
||||
# graceful shutdown when the runner traps EXIT/INT/TERM, so the
|
||||
# report step below sees a populated file.
|
||||
COVERAGE: "1"
|
||||
# Tells the Playwright fixture (test-base.ts) to capture per-test
|
||||
# V8 JS coverage. Raw dumps land under
|
||||
# .test-state/playwright/coverage-pw/ for the post-process step
|
||||
# to aggregate. Chromium-only - other engines silently skip.
|
||||
PW_COVERAGE: "1"
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results.json
|
||||
run: task e2e:live
|
||||
- name: Flag flaky tests
|
||||
# Runs regardless of the test outcome: a flaky test (passed on retry)
|
||||
# leaves the step green, so this is the only place it surfaces. Emits
|
||||
# ::warning:: annotations + a job summary; never fails the job.
|
||||
if: always()
|
||||
working-directory: frontend
|
||||
run: npx tsx editor/scripts/report-flaky-tests.mts "$PLAYWRIGHT_JSON_OUTPUT_FILE"
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results.json
|
||||
- name: Generate JaCoCo report from e2e:live .exec
|
||||
if: always()
|
||||
id: live-coverage
|
||||
# `if: always()` so even a failed test run still produces a
|
||||
# report from whatever flows did exercise the backend before
|
||||
# the failure. The task itself tolerates a missing .exec
|
||||
# (jacoco emits an empty report rather than crashing) but we
|
||||
# guard with `test -s` to keep the job log clean.
|
||||
run: |
|
||||
if [ -s .test-state/playwright/jacoco.exec ]; then
|
||||
./gradlew jacocoReportFromExec \
|
||||
-PexecFile=.test-state/playwright/jacoco.exec \
|
||||
-PreportDir=build/reports/jacoco/e2e-live \
|
||||
-PnoSpotless
|
||||
echo "report=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "::warning::No e2e:live .exec found at .test-state/playwright/jacoco.exec; skipping report"
|
||||
echo "report=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- name: Set up Python for coverage summary
|
||||
if: always() && steps.live-coverage.outputs.report == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Install defusedxml for coverage summary
|
||||
# coverage-summary.py uses defusedxml instead of stdlib xml.etree
|
||||
# to dodge XXE / billion-laughs scanner findings.
|
||||
if: always() && steps.live-coverage.outputs.report == 'true'
|
||||
run: python -m pip install --quiet defusedxml
|
||||
- name: e2e:live coverage step summary
|
||||
if: always() && steps.live-coverage.outputs.report == 'true'
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Playwright (live backend) JaCoCo coverage" \
|
||||
--jacoco "e2e-live=build/reports/jacoco/e2e-live/jacocoTestReport.xml" \
|
||||
--github-step-summary
|
||||
- name: Upload e2e:live JaCoCo report
|
||||
if: always() && steps.live-coverage.outputs.report == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-e2e-live-${{ github.run_id }}
|
||||
path: build/reports/jacoco/e2e-live/
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload raw e2e:live .exec for aggregate merge
|
||||
# Picked up by the coverage-aggregate workflow via the
|
||||
# `jacoco-exec-*` artifact name pattern.
|
||||
if: always() && steps.live-coverage.outputs.report == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: jacoco-exec-e2e-live
|
||||
path: .test-state/playwright/jacoco.exec
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Set up Python for frontend coverage summary
|
||||
# Separate from the backend-coverage python step because the
|
||||
# frontend path doesn't depend on a JaCoCo report - it produces
|
||||
# a summary even on backend failure, as long as some Playwright
|
||||
# tests ran far enough to dump V8 coverage.
|
||||
if: always()
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install defusedxml for frontend coverage summary
|
||||
# Idempotent re-install: the backend-coverage step may have
|
||||
# installed it already, but this leg can run on its own when the
|
||||
# backend report step skips (e.g. .exec missing).
|
||||
if: always()
|
||||
run: python -m pip install --quiet defusedxml
|
||||
|
||||
- name: Aggregate Playwright frontend (V8) coverage
|
||||
# Rolls per-test V8 dumps from the test-base fixture into one
|
||||
# vitest-shaped coverage-summary.json. Tolerates a missing dump
|
||||
# dir (firefox/webkit runs, or a failure before any test got
|
||||
# far enough to dump).
|
||||
if: always()
|
||||
id: pw-frontend-coverage
|
||||
run: |
|
||||
if [ -d .test-state/playwright/coverage-pw ] && \
|
||||
find .test-state/playwright/coverage-pw -name '*.json' -type f | grep -q .; then
|
||||
python scripts/playwright-coverage-summary.py \
|
||||
.test-state/playwright/coverage-pw \
|
||||
--out .test-state/playwright/coverage-pw-summary/coverage-summary.json
|
||||
echo "summary=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "::notice::No Playwright frontend coverage dumps found (chromium-only feature)"
|
||||
echo "summary=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Playwright frontend coverage step summary
|
||||
if: always() && steps.pw-frontend-coverage.outputs.summary == 'true'
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Playwright (live) frontend coverage" \
|
||||
--vitest .test-state/playwright/coverage-pw-summary/coverage-summary.json \
|
||||
--github-step-summary
|
||||
|
||||
- name: Upload Playwright frontend coverage
|
||||
# Bundle both the aggregated summary and the raw V8 dumps so
|
||||
# someone debugging "why is this function showing as covered"
|
||||
# can trace it back to the source dump.
|
||||
if: always() && steps.pw-frontend-coverage.outputs.summary == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-frontend-coverage
|
||||
path: |
|
||||
.test-state/playwright/coverage-pw-summary/
|
||||
.test-state/playwright/coverage-pw/
|
||||
retention-days: 7
|
||||
|
||||
- name: Print backend log on failure
|
||||
if: failure() && steps.live-tests.conclusion == 'failure'
|
||||
run: |
|
||||
@@ -198,30 +65,10 @@ jobs:
|
||||
name: backend-log-live-${{ github.run_id }}
|
||||
path: .test-state/playwright/backend.log
|
||||
retention-days: 7
|
||||
- name: List Playwright output locations (debug)
|
||||
if: always()
|
||||
run: |
|
||||
echo "::group::Playwright output dirs"
|
||||
# Playwright anchors its default outputDir + HTML report to the
|
||||
# nearest package.json, which is frontend/ (frontend/editor has
|
||||
# none), so artifacts land under frontend/, not frontend/editor/.
|
||||
ls -la frontend/playwright-report 2>/dev/null \
|
||||
|| echo "no playwright-report at frontend/"
|
||||
ls -la frontend/test-results 2>/dev/null \
|
||||
|| echo "no test-results at frontend/"
|
||||
find . -name node_modules -prune -o -name 'trace.zip' -print 2>/dev/null || true
|
||||
echo "::endgroup::"
|
||||
- name: Upload Playwright report + traces
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-report-live-${{ github.run_id }}
|
||||
# test-results/ holds the per-test trace.zip (with browser console
|
||||
# logs) + screenshots/video; playwright-report/ is the HTML report.
|
||||
# Both live under frontend/ (Playwright anchors them to the nearest
|
||||
# package.json, which is frontend/; frontend/editor has none).
|
||||
path: |
|
||||
frontend/playwright-report/
|
||||
frontend/test-results/
|
||||
path: frontend/editor/playwright-report/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
cache: "npm"
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Install Playwright (chromium only)
|
||||
run: task e2e:install -- chromium
|
||||
- name: Build frontend (production bundle for vite preview)
|
||||
@@ -44,22 +44,11 @@ jobs:
|
||||
VITE_BUILD_FOR_PREVIEW: "1"
|
||||
run: task frontend:build
|
||||
- name: Run stubbed E2E tests (chromium)
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results.json
|
||||
run: task e2e:stubbed -- --workers=3
|
||||
- name: Flag flaky tests
|
||||
# Runs regardless of the test outcome: a flaky test (passed on retry)
|
||||
# leaves the step green, so this is the only place it surfaces. Emits
|
||||
# ::warning:: annotations + a job summary; never fails the job.
|
||||
if: always()
|
||||
working-directory: frontend
|
||||
run: npx tsx editor/scripts/report-flaky-tests.mts "$PLAYWRIGHT_JSON_OUTPUT_FILE"
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results.json
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-report-stubbed-${{ github.run_id }}
|
||||
path: frontend/playwright-report/
|
||||
path: frontend/editor/playwright-report/
|
||||
retention-days: 7
|
||||
|
||||
@@ -97,14 +97,7 @@ jobs:
|
||||
run: npm ci --ignore-scripts --audit=false --fund=false
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
|
||||
- name: Generate frontend license report (Push only)
|
||||
if: github.event_name == 'push'
|
||||
env:
|
||||
PR_IS_FORK: "false"
|
||||
run: task frontend:licenses:generate
|
||||
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Generate frontend license report (internal PR)
|
||||
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
|
||||
env:
|
||||
@@ -117,8 +110,8 @@ jobs:
|
||||
NPM_CONFIG_IGNORE_SCRIPTS: "true"
|
||||
working-directory: frontend
|
||||
run: |
|
||||
mkdir -p editor/src/assets
|
||||
npx --yes license-report --only=prod --output=json > editor/src/assets/3rdPartyLicenses.json
|
||||
mkdir -p src/assets
|
||||
npx --yes license-report --only=prod --output=json > src/assets/3rdPartyLicenses.json
|
||||
|
||||
- name: Postprocess with project script (BASE version)
|
||||
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true
|
||||
@@ -356,11 +349,10 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Check licenses and generate report
|
||||
id: license-check
|
||||
run: task backend:licenses:generate || echo "LICENSE_CHECK_FAILED=true" >> $GITHUB_ENV
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
cache: "npm"
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Quality-check frontend
|
||||
id: frontend-check
|
||||
run: task frontend:check:all
|
||||
@@ -109,41 +109,6 @@ jobs:
|
||||
comment_id: existing.id,
|
||||
});
|
||||
}
|
||||
- name: Vitest coverage
|
||||
# Separate from `frontend:check:all` so the quality-gate run stays
|
||||
# uninstrumented (faster signal) and coverage stays an informational
|
||||
# follow-up. Continue-on-error keeps the workflow green even when
|
||||
# a handful of test files refuse to import (e.g. missing icon
|
||||
# specifiers) - the summary still gets posted with whatever
|
||||
# vitest managed to instrument.
|
||||
id: frontend-coverage
|
||||
continue-on-error: true
|
||||
run: task frontend:test:coverage
|
||||
- name: Set up Python for coverage summary
|
||||
if: always()
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Install defusedxml for coverage summary
|
||||
# See coverage-summary.py header - it parses XML through defusedxml
|
||||
# to dodge the stdlib parser's exposure to XXE / billion-laughs.
|
||||
if: always()
|
||||
run: python -m pip install --quiet defusedxml
|
||||
- name: Vitest coverage step summary
|
||||
if: always()
|
||||
run: |
|
||||
python scripts/coverage-summary.py \
|
||||
--title "Frontend Vitest coverage" \
|
||||
--vitest frontend/editor/coverage/coverage-summary.json \
|
||||
--github-step-summary
|
||||
- name: Upload vitest coverage report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: frontend-coverage
|
||||
path: frontend/editor/coverage/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
- name: Upload frontend build artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
|
||||
@@ -73,10 +73,10 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Get version number
|
||||
id: versionNumber
|
||||
run: |
|
||||
@@ -148,7 +148,7 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
|
||||
- name: Setup Node.js
|
||||
if: matrix.variant.build_frontend == true
|
||||
@@ -159,7 +159,7 @@ jobs:
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
|
||||
- name: Build JAR
|
||||
run: ./gradlew build ${{ matrix.variant.build_frontend && '-PbuildWithFrontend=true' || '' }} -x spotlessApply -x spotlessCheck -x test -x sonarqube
|
||||
@@ -252,10 +252,10 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
|
||||
# Build the universal JRE before desktop:prepare so the jlink:runtime
|
||||
# task short-circuits on its `test -d runtime/jre` status check.
|
||||
@@ -442,6 +442,10 @@ jobs:
|
||||
echo "Generated tauri.windows.conf.json (alias masked):"
|
||||
sed "s/${KEYPAIR_ALIAS}/***/g" ./frontend/editor/src-tauri/tauri.windows.conf.json
|
||||
|
||||
- name: Sign JPDFium dylibs inside bootJar (macOS only)
|
||||
if: matrix.platform == 'macos-15' && env.APPLE_CERTIFICATE != ''
|
||||
run: bash frontend/scripts/sign-jpdfium-dylibs-in-bootjar.sh
|
||||
|
||||
- name: Import release GPG signing key (Linux)
|
||||
if: matrix.platform == 'ubuntu-22.04' && env.RELEASE_GPG_PRIVATE_KEY != '' && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.sign != 'false') || github.ref == 'refs/heads/V2-master')
|
||||
run: |
|
||||
@@ -491,7 +495,6 @@ jobs:
|
||||
projectPath: ./frontend/editor
|
||||
tauriScript: npx tauri
|
||||
args: ${{ matrix.args }}
|
||||
updaterJsonKeepUniversal: true
|
||||
|
||||
- name: Clear release GPG key from runner keyring (Linux)
|
||||
if: always() && matrix.platform == 'ubuntu-22.04' && env.RELEASE_GPG_PRIVATE_KEY != '' && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.sign != 'false') || github.ref == 'refs/heads/V2-master')
|
||||
@@ -588,32 +591,18 @@ jobs:
|
||||
mkdir -p "$DIST"
|
||||
cd ./frontend/editor/src-tauri/target
|
||||
|
||||
echo "=== tauri bundle artifacts ==="
|
||||
find . -path "*/bundle/*" \( -name "*.msi" -o -name "*.deb" \
|
||||
-o -name "*.rpm" -o -name "*.AppImage" -o -name "*.dmg" \
|
||||
-o -name "*.app.tar.gz" -o -name "*.sig" \) 2>/dev/null | sort || true
|
||||
echo "=============================="
|
||||
|
||||
# createUpdaterArtifacts:true signs the native installers in place;
|
||||
# each <bundle> ships with a sibling <bundle>.sig consumed by latest.json.
|
||||
# Find and rename artifacts based on platform
|
||||
if [ "${{ matrix.platform }}" = "windows-latest" ]; then
|
||||
# Only ship the MSI installer on Windows. The loose exe and WiX toolset exes
|
||||
# are not the user-facing installer - the MSI contains the signed inner exe.
|
||||
find . -name "*.msi" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.msi" \;
|
||||
find . -name "*.msi.sig" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.msi.sig" \;
|
||||
elif [ "${{ matrix.platform }}" = "macos-15" ]; then
|
||||
# DMG = manual install; .app.tar.gz (+ .sig) = updater payload.
|
||||
# Raw .app is intentionally not shipped (hundreds of MB of uncompressed input).
|
||||
find . -name "*.dmg" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.dmg" \;
|
||||
find . -name "*.app.tar.gz" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.app.tar.gz" \;
|
||||
find . -name "*.app.tar.gz.sig" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.app.tar.gz.sig" \;
|
||||
find . -name "*.app" -exec cp -r {} "$DIST/Stirling-PDF-${{ matrix.name }}.app" \;
|
||||
else
|
||||
# The raw .AppImage IS its updater payload (signed -> .AppImage.sig),
|
||||
# not a .tar.gz wrapper - that's only produced under v1Compatible.
|
||||
find . -name "*.deb" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.deb" \;
|
||||
find . -name "*.deb.sig" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.deb.sig" \;
|
||||
find . -name "*.rpm" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.rpm" \;
|
||||
find . -name "*.rpm.sig" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.rpm.sig" \;
|
||||
find . -name "*.AppImage" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.AppImage" \;
|
||||
find . -name "*.AppImage.sig" -exec cp {} "$DIST/Stirling-PDF-${{ matrix.name }}.AppImage.sig" \;
|
||||
fi
|
||||
|
||||
- name: Upload build artifacts
|
||||
@@ -624,7 +613,8 @@ jobs:
|
||||
path: ./dist/*
|
||||
retention-days: 1
|
||||
|
||||
collect-and-release:
|
||||
create-release:
|
||||
if: (github.event_name == 'workflow_dispatch' && github.event.inputs.test_mode != 'true') || github.event_name == 'release' || github.ref == 'refs/heads/V2-master'
|
||||
needs: [pick, determine-matrix, build, build-jars]
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
permissions:
|
||||
@@ -635,16 +625,6 @@ jobs:
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
# Sparse-check out the verifier + pubkey before the artifact downloads
|
||||
# so the checkout cannot clobber ./artifacts.
|
||||
- name: Checkout updater verifier
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github/scripts/verify-updater-signatures.py
|
||||
frontend/editor/src-tauri/tauri.conf.json
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: Download all Tauri artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
@@ -672,146 +652,17 @@ jobs:
|
||||
- name: Display structure of downloaded files
|
||||
run: ls -R ./artifacts
|
||||
|
||||
# tauri-action only emits latest.json when it also publishes the release
|
||||
# (tagName/releaseId set). We publish separately via action-gh-release,
|
||||
# so build latest.json here from the per-platform .sig files.
|
||||
- name: Generate updater latest.json
|
||||
env:
|
||||
VERSION: ${{ needs.determine-matrix.outputs.version }}
|
||||
TAG: v${{ needs.determine-matrix.outputs.version }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
python3 - << 'PYEOF'
|
||||
import json, os, sys
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
VERSION = os.environ['VERSION']
|
||||
TAG = os.environ['TAG']
|
||||
REPO = os.environ['REPO']
|
||||
|
||||
ART = Path('./artifacts/tauri')
|
||||
|
||||
# Tauri updater looks up {os}-{arch}-{installer} (e.g. linux-x86_64-deb)
|
||||
# before bare {os}-{arch}, so per-format Linux keys let deb/rpm/appimage
|
||||
# each self-update from their matching file. macOS universal serves both
|
||||
# arches from the one .app.tar.gz.
|
||||
PLATFORM_MAP = [
|
||||
{
|
||||
'bundles': ['Stirling-PDF-linux-x86_64.deb'],
|
||||
'targets': ['linux-x86_64-deb'],
|
||||
},
|
||||
{
|
||||
'bundles': ['Stirling-PDF-linux-x86_64.rpm'],
|
||||
'targets': ['linux-x86_64-rpm'],
|
||||
},
|
||||
{
|
||||
'bundles': ['Stirling-PDF-linux-x86_64.AppImage'],
|
||||
'targets': ['linux-x86_64-appimage'],
|
||||
},
|
||||
{
|
||||
'bundles': ['Stirling-PDF-windows-x86_64.msi'],
|
||||
'targets': ['windows-x86_64-msi', 'windows-x86_64'],
|
||||
},
|
||||
{
|
||||
'bundles': ['Stirling-PDF-macos-universal.app.tar.gz'],
|
||||
'targets': ['darwin-x86_64', 'darwin-aarch64'],
|
||||
},
|
||||
]
|
||||
|
||||
# rglob() because download-artifact varies layout: one artifact -> flat,
|
||||
# many -> nested under <artifact-name>/.
|
||||
def find_signed(name):
|
||||
for bundle_path in sorted(ART.rglob(name)):
|
||||
sig_path = bundle_path.with_name(bundle_path.name + '.sig')
|
||||
if sig_path.exists():
|
||||
return bundle_path, sig_path
|
||||
return None
|
||||
|
||||
platforms = {}
|
||||
skipped = []
|
||||
for entry in PLATFORM_MAP:
|
||||
picked = None
|
||||
for name in entry['bundles']:
|
||||
picked = find_signed(name)
|
||||
if picked:
|
||||
break
|
||||
if not picked:
|
||||
skipped.append(
|
||||
f"{entry['targets']} (no signed bundle among "
|
||||
f"{entry['bundles']} - TAURI_SIGNING_PRIVATE_KEY unset "
|
||||
f"or createUpdaterArtifacts disabled?)"
|
||||
)
|
||||
continue
|
||||
bundle_path, sig_path = picked
|
||||
signature = sig_path.read_text(encoding='utf-8').strip()
|
||||
url = f"https://github.com/{REPO}/releases/download/{TAG}/{bundle_path.name}"
|
||||
for target in entry['targets']:
|
||||
platforms[target] = {'signature': signature, 'url': url}
|
||||
print(f"Added {entry['targets']} from {bundle_path.name}")
|
||||
|
||||
if skipped:
|
||||
print("Skipped platforms:")
|
||||
for s in skipped:
|
||||
print(f" - {s}")
|
||||
|
||||
if not platforms:
|
||||
print(
|
||||
"WARN: no signed updater bundles found - "
|
||||
"skipping latest.json generation"
|
||||
)
|
||||
sys.exit(0)
|
||||
|
||||
manifest = {
|
||||
'version': VERSION,
|
||||
'notes': f"See https://github.com/{REPO}/releases/tag/{TAG}",
|
||||
'pub_date': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
|
||||
'platforms': platforms,
|
||||
}
|
||||
|
||||
out = Path('./artifacts/latest.json')
|
||||
out.write_text(json.dumps(manifest, indent=2) + '\n', encoding='utf-8')
|
||||
print(f"Generated {out} with platforms: {sorted(platforms.keys())}")
|
||||
PYEOF
|
||||
|
||||
- name: Upload merged artifacts for review
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: release-artifacts
|
||||
path: ./artifacts/
|
||||
retention-days: 7
|
||||
|
||||
# Gate publish on valid updater sigs. Runs after the review upload (so
|
||||
# artifacts survive for debugging) and before action-gh-release.
|
||||
- name: Verify updater signatures
|
||||
run: |
|
||||
python3 -m pip install --quiet 'cryptography==44.0.0'
|
||||
python3 .github/scripts/verify-updater-signatures.py \
|
||||
./artifacts/tauri frontend/editor/src-tauri/tauri.conf.json
|
||||
|
||||
# workflow_dispatch path requires platform=='all' so a single-platform
|
||||
# dispatch can't overwrite an existing release's full latest.json with a
|
||||
# partial one (action-gh-release defaults overwrite_files:true).
|
||||
# release / V2-master always build the full matrix so no extra guard needed.
|
||||
# fail_on_unmatched_files makes a missing latest.json or installer fail loudly
|
||||
# instead of silently shipping a broken auto-update.
|
||||
- name: Upload binaries to Release
|
||||
if: (github.event_name == 'workflow_dispatch' && github.event.inputs.test_mode != 'true' && github.event.inputs.platform == 'all') || github.event_name == 'release' || github.ref == 'refs/heads/V2-master'
|
||||
uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0
|
||||
with:
|
||||
tag_name: v${{ needs.determine-matrix.outputs.version }}
|
||||
generate_release_notes: true
|
||||
fail_on_unmatched_files: true
|
||||
# Installers + updater payloads + manifest. .sig contents are embedded
|
||||
# in latest.json so the .sig files themselves are not uploaded.
|
||||
files: |
|
||||
./artifacts/**/*.jar
|
||||
./artifacts/**/*.msi
|
||||
./artifacts/**/*.dmg
|
||||
./artifacts/**/*.app.tar.gz
|
||||
./artifacts/**/*.deb
|
||||
./artifacts/**/*.rpm
|
||||
./artifacts/**/*.AppImage
|
||||
./artifacts/latest.json
|
||||
draft: false
|
||||
prerelease: false
|
||||
|
||||
@@ -37,15 +37,10 @@ jobs:
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Install all Playwright browsers
|
||||
run: task e2e:install
|
||||
|
||||
- name: Build frontend (production bundle for vite preview)
|
||||
env:
|
||||
VITE_BUILD_FOR_PREVIEW: "1"
|
||||
run: task frontend:build
|
||||
|
||||
- name: Run E2E tests (all browsers)
|
||||
run: task e2e:cross-browser
|
||||
|
||||
@@ -53,19 +48,6 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-report-nightly-${{ github.run_id }}
|
||||
path: frontend/playwright-report/
|
||||
name: playwright-nightly-${{ github.run_id }}
|
||||
path: frontend/editor/playwright-report/
|
||||
retention-days: 14
|
||||
|
||||
# Builds all desktop platforms on a schedule so the Rust dependency cache is
|
||||
# written on main, where PR and merge-queue tauri builds can restore it.
|
||||
warm-tauri-cache:
|
||||
name: Warm Tauri Rust cache
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
uses: ./.github/workflows/tauri-build.yml
|
||||
with:
|
||||
platform: all
|
||||
sign: false
|
||||
secrets: inherit
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
name: Pre-commit
|
||||
|
||||
# Runs the repo-wide lint/format/secret checks via `task pre-commit`.
|
||||
# Called from build.yml on PRs and merge_group; also runnable on demand via workflow_dispatch.
|
||||
# Runs `pre-commit run` for ruff / codespell / gitleaks / EOF / trailing-ws.
|
||||
# Called from build.yml on PRs and merge_group; also runnable on demand via
|
||||
# workflow_dispatch for manual local-equivalent linting.
|
||||
on:
|
||||
workflow_call:
|
||||
workflow_dispatch:
|
||||
@@ -12,6 +13,10 @@ permissions:
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# Prevents sdist builds → no tar extraction
|
||||
PIP_ONLY_BINARY: ":all:"
|
||||
PIP_DISABLE_PIP_VERSION_CHECK: "1"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
@@ -24,13 +29,23 @@ jobs:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: 3.12
|
||||
cache: "pip" # caching pip dependencies
|
||||
cache-dependency-path: ./.github/scripts/requirements_pre_commit.txt
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Run Pre-Commit Hooks
|
||||
run: |
|
||||
pip install --require-hashes --only-binary=:all: -r ./.github/scripts/requirements_pre_commit.txt
|
||||
|
||||
- name: Run pre-commit checks
|
||||
run: task pre-commit
|
||||
- name: Run Pre-Commit
|
||||
run: |
|
||||
pre-commit run ruff --all-files -c .pre-commit-config.yaml
|
||||
pre-commit run ruff-format --all-files -c .pre-commit-config.yaml
|
||||
pre-commit run codespell --all-files -c .pre-commit-config.yaml
|
||||
pre-commit run gitleaks --all-files -c .pre-commit-config.yaml
|
||||
pre-commit run end-of-file-fixer --all-files -c .pre-commit-config.yaml
|
||||
pre-commit run trailing-whitespace --all-files -c .pre-commit-config.yaml
|
||||
git diff --exit-code
|
||||
|
||||
@@ -75,7 +75,7 @@ jobs:
|
||||
|
||||
- name: Generate tags for base image
|
||||
id: meta
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_HUB_ORG_USERNAME }}/stirling-pdf-base
|
||||
|
||||
@@ -78,14 +78,14 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Get version number
|
||||
id: versionNumber
|
||||
run: echo "versionNumber=$(./gradlew printVersion --quiet | tail -1)" >> $GITHUB_OUTPUT
|
||||
@@ -129,7 +129,7 @@ jobs:
|
||||
- name: Generate tags for latest
|
||||
id: meta
|
||||
if: env.RUN_MAIN_APP == 'true'
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
|
||||
@@ -155,9 +155,9 @@ jobs:
|
||||
cache-to: type=gha,mode=max,scope=stirling-pdf-latest
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
# No BASE_VERSION pin: inherit the Dockerfile ARG default (single source of truth).
|
||||
build-args: |
|
||||
VERSION_TAG=${{ steps.versionNumber.outputs.versionNumber }}
|
||||
BASE_VERSION=1.0.0
|
||||
platforms: linux/amd64,linux/arm64/v8
|
||||
provenance: true
|
||||
sbom: true
|
||||
@@ -178,7 +178,7 @@ jobs:
|
||||
|
||||
- name: Generate tags for latest-fat
|
||||
id: meta-fat
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
with:
|
||||
images: |
|
||||
@@ -222,7 +222,7 @@ jobs:
|
||||
|
||||
- name: Generate tags for ultra-lite
|
||||
id: meta-lite
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
if: env.RUN_MAIN_APP == 'true' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/testMain'
|
||||
with:
|
||||
images: |
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
egress-policy: audit
|
||||
|
||||
- name: 30 days stale issues
|
||||
uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0
|
||||
uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
days-before-stale: 30
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
|
||||
- name: Generate Swagger documentation
|
||||
run: ./gradlew :stirling-pdf:generateOpenApiDocs
|
||||
@@ -63,7 +63,7 @@ jobs:
|
||||
SWAGGERHUB_USER: "Frooodle"
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Get version number
|
||||
id: versionNumber
|
||||
run: echo "versionNumber=$(./gradlew printVersion --quiet | tail -1)" >> $GITHUB_OUTPUT
|
||||
|
||||
@@ -58,23 +58,15 @@ jobs:
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
pip install --require-hashes --only-binary=:all: -r ./.github/scripts/requirements_sync_readme.txt
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
with:
|
||||
enable-cache: true
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
pip install --require-hashes --only-binary=:all: -r ./.github/scripts/requirements_sync_readme.txt -r ./.github/scripts/requirements_pre_commit.txt
|
||||
|
||||
- name: Sync translation TOML files
|
||||
run: |
|
||||
python .github/scripts/check_language_toml.py --reference-file "frontend/editor/public/locales/en-US/translation.toml" --branch main
|
||||
python .github/scripts/check_language_toml.py --reference-file "frontend/editor/public/locales/en-GB/translation.toml" --branch main
|
||||
|
||||
- name: Sort translation TOML files
|
||||
- name: pre-commit run
|
||||
run: |
|
||||
task pre-commit:toml-sort FIX=1
|
||||
pre-commit run toml-sort-fix --all-files
|
||||
|
||||
- name: Commit translation files
|
||||
run: |
|
||||
@@ -108,7 +100,7 @@ jobs:
|
||||
This Pull Request was automatically generated to synchronize updates to translation files and documentation. Below are the details of the changes made:
|
||||
|
||||
#### **1. Synchronization of Translation Files**
|
||||
- Updated translation files (`frontend/editor/public/locales/*/translation.toml`) to reflect changes in the reference file `en-US/translation.toml`.
|
||||
- Updated translation files (`frontend/editor/public/locales/*/translation.toml`) to reflect changes in the reference file `en-GB/translation.toml`.
|
||||
- Ensured consistency and synchronization across all supported language files.
|
||||
- Highlighted any missing or incomplete translations.
|
||||
- **Format**: TOML
|
||||
|
||||
@@ -16,11 +16,6 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: "all"
|
||||
sign:
|
||||
description: "Sign and notarize the bundles."
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
platform:
|
||||
@@ -33,11 +28,6 @@ on:
|
||||
- windows
|
||||
- macos
|
||||
- linux
|
||||
sign:
|
||||
description: "Sign and notarize the bundles."
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -125,20 +115,6 @@ jobs:
|
||||
toolchain: stable
|
||||
targets: ${{ matrix.platform == 'macos-15' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }}
|
||||
|
||||
# Cache the Cargo registry and compiled dependency crates so the build
|
||||
# only recompiles the app crate. Written on main; PRs and the merge queue
|
||||
# restore from it.
|
||||
- name: Cache Rust build
|
||||
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
workspaces: frontend/editor/src-tauri
|
||||
# Stable key shared across workflows so the nightly warmer.
|
||||
# rust-cache still appends OS + rustc + Cargo.lock.
|
||||
shared-key: tauri-${{ matrix.name }}
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
# Save the dependency cache even if a later step fails
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Set up x86_64 JDK 25 (macOS universal JRE)
|
||||
if: matrix.platform == 'macos-15'
|
||||
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
|
||||
@@ -160,10 +136,10 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
|
||||
- name: Setup Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
|
||||
- name: Build universal macOS JRE
|
||||
if: matrix.platform == 'macos-15'
|
||||
@@ -187,7 +163,7 @@ jobs:
|
||||
# DigiCert KeyLocker Setup (Cloud HSM)
|
||||
- name: Setup DigiCert KeyLocker
|
||||
id: digicert-setup
|
||||
if: ${{ inputs.sign && matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && github.ref == 'refs/heads/main' }}
|
||||
if: ${{ matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && github.ref == 'refs/heads/main' }}
|
||||
uses: digicert/ssm-code-signing@1d820463733701cf1484c7eb5d7d24a15ca2c454 # v1.2.1
|
||||
env:
|
||||
SM_API_KEY: ${{ secrets.SM_API_KEY }}
|
||||
@@ -197,7 +173,7 @@ jobs:
|
||||
SM_HOST: ${{ secrets.SM_HOST }}
|
||||
|
||||
- name: Setup DigiCert KeyLocker Certificate
|
||||
if: ${{ inputs.sign && matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && github.ref == 'refs/heads/main' }}
|
||||
if: ${{ matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && github.ref == 'refs/heads/main' }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
Write-Host "Setting up DigiCert KeyLocker environment..."
|
||||
@@ -232,7 +208,7 @@ jobs:
|
||||
|
||||
# Traditional PFX Certificate Import (fallback if KeyLocker not configured)
|
||||
- name: Import Windows Code Signing Certificate
|
||||
if: ${{ inputs.sign && matrix.platform == 'windows-latest' && env.SM_API_KEY == '' && github.ref == 'refs/heads/main' }}
|
||||
if: ${{ matrix.platform == 'windows-latest' && env.SM_API_KEY == '' && github.ref == 'refs/heads/main' }}
|
||||
env:
|
||||
WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
|
||||
WINDOWS_CERTIFICATE_PASSWORD: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }}
|
||||
@@ -263,7 +239,7 @@ jobs:
|
||||
}
|
||||
|
||||
- name: Import Apple Developer Certificate
|
||||
if: inputs.sign && matrix.platform == 'macos-15' && env.APPLE_CERTIFICATE != ''
|
||||
if: matrix.platform == 'macos-15' && env.APPLE_CERTIFICATE != ''
|
||||
env:
|
||||
APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
|
||||
APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
|
||||
@@ -284,7 +260,7 @@ jobs:
|
||||
rm certificate.p12
|
||||
|
||||
- name: Verify Certificate
|
||||
if: inputs.sign && matrix.platform == 'macos-15' && env.APPLE_CERTIFICATE != ''
|
||||
if: matrix.platform == 'macos-15' && env.APPLE_CERTIFICATE != ''
|
||||
run: |
|
||||
echo "Verifying Apple Developer Certificate..."
|
||||
KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db
|
||||
@@ -295,6 +271,10 @@ jobs:
|
||||
echo "APPLE_SIGNING_IDENTITY=$CERT_ID" >> $GITHUB_ENV
|
||||
echo "Certificate imported successfully."
|
||||
|
||||
- name: Sign JPDFium dylibs inside bootJar (macOS only)
|
||||
if: matrix.platform == 'macos-15' && env.APPLE_CERTIFICATE != ''
|
||||
run: bash frontend/scripts/sign-jpdfium-dylibs-in-bootjar.sh
|
||||
|
||||
- name: Check DMG creation dependencies (macOS only)
|
||||
if: matrix.platform == 'macos-15'
|
||||
run: |
|
||||
@@ -307,7 +287,7 @@ jobs:
|
||||
ls -la /usr/bin/hd* || echo "No hd* tools found"
|
||||
|
||||
- name: Preflight smctl
|
||||
if: ${{ inputs.sign && matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && github.ref == 'refs/heads/main' }}
|
||||
if: ${{ matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && github.ref == 'refs/heads/main' }}
|
||||
shell: pwsh
|
||||
env:
|
||||
KEYPAIR_ALIAS: ${{ secrets.SM_KEYPAIR_ALIAS }}
|
||||
@@ -320,7 +300,7 @@ jobs:
|
||||
if ($LASTEXITCODE -ne 0) { Write-Host "[WARN] smctl windows certsync returned non-zero - continuing" }
|
||||
|
||||
- name: Configure Windows code signing
|
||||
if: ${{ inputs.sign && matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && github.ref == 'refs/heads/main' }}
|
||||
if: ${{ matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && github.ref == 'refs/heads/main' }}
|
||||
shell: bash
|
||||
env:
|
||||
KEYPAIR_ALIAS: ${{ secrets.SM_KEYPAIR_ALIAS }}
|
||||
@@ -339,7 +319,7 @@ jobs:
|
||||
EOF
|
||||
|
||||
- name: Import release GPG signing key (Linux)
|
||||
if: inputs.sign && matrix.platform == 'ubuntu-22.04' && env.RELEASE_GPG_PRIVATE_KEY != '' && github.ref == 'refs/heads/main'
|
||||
if: matrix.platform == 'ubuntu-22.04' && env.RELEASE_GPG_PRIVATE_KEY != '' && github.ref == 'refs/heads/main'
|
||||
run: |
|
||||
echo "$RELEASE_GPG_PRIVATE_KEY" | gpg --batch --import
|
||||
gpg --list-secret-keys --keyid-format=long
|
||||
@@ -356,8 +336,7 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Build Tauri app (signed)
|
||||
if: inputs.sign
|
||||
- name: Build Tauri app
|
||||
uses: tauri-apps/tauri-action@84b9d35b5fc46c1e45415bdb6144030364f7ebc5 # v0.6.2
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -391,26 +370,6 @@ jobs:
|
||||
# failure (#6127 onwards) does not tank deb/rpm uploads.
|
||||
args: ${{ matrix.platform == 'ubuntu-22.04' && '--bundles deb,rpm' || matrix.args }}
|
||||
|
||||
- name: Build Tauri app (unsigned)
|
||||
if: ${{ !inputs.sign }}
|
||||
uses: tauri-apps/tauri-action@84b9d35b5fc46c1e45415bdb6144030364f7ebc5 # v0.6.2
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SIGN: "0"
|
||||
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
|
||||
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
|
||||
VITE_SUPABASE_PUBLISHABLE_DEFAULT_KEY: ${{ secrets.VITE_SUPABASE_PUBLISHABLE_DEFAULT_KEY || 'sb_publishable_UHz2SVRF5mvdrPHWkRteyA_yNlZTkYb' }}
|
||||
VITE_SAAS_SERVER_URL: ${{ secrets.VITE_SAAS_SERVER_URL || 'https://app.stirlingpdf.com' }}
|
||||
VITE_SAAS_BACKEND_API_URL: ${{ secrets.VITE_SAAS_BACKEND_API_URL || 'https://api.stirlingpdf.com' }}
|
||||
CI: true
|
||||
with:
|
||||
projectPath: ./frontend/editor
|
||||
tauriScript: npx tauri
|
||||
# Linux: build deb+rpm only here. AppImage runs in its own
|
||||
# continue-on-error step below so its persistent linuxdeploy
|
||||
# failure (#6127 onwards) does not tank deb/rpm uploads.
|
||||
args: ${{ matrix.platform == 'ubuntu-22.04' && '--bundles deb,rpm' || matrix.args }}
|
||||
|
||||
# AppImage is decoupled so its linuxdeploy run gets a fresh process
|
||||
# (rpm scratch state torn down) and its failure can't tank deb/rpm.
|
||||
- name: Build Tauri app (Linux AppImage)
|
||||
@@ -419,7 +378,7 @@ jobs:
|
||||
uses: tauri-apps/tauri-action@84b9d35b5fc46c1e45415bdb6144030364f7ebc5 # v0.6.2
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SIGN: ${{ (inputs.sign && env.RELEASE_GPG_PRIVATE_KEY != '' && github.ref == 'refs/heads/main') && '1' || '0' }}
|
||||
SIGN: ${{ (env.RELEASE_GPG_PRIVATE_KEY != '' && github.ref == 'refs/heads/main') && '1' || '0' }}
|
||||
APPIMAGETOOL_SIGN_PASSPHRASE: ${{ secrets.RELEASE_GPG_PASSPHRASE }}
|
||||
SIGN_KEY: ${{ vars.RELEASE_GPG_FINGERPRINT }}
|
||||
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
|
||||
@@ -434,7 +393,7 @@ jobs:
|
||||
args: --bundles appimage
|
||||
|
||||
- name: Clear release GPG key from runner keyring (Linux)
|
||||
if: always() && inputs.sign && matrix.platform == 'ubuntu-22.04' && env.RELEASE_GPG_PRIVATE_KEY != '' && github.ref == 'refs/heads/main'
|
||||
if: always() && matrix.platform == 'ubuntu-22.04' && env.RELEASE_GPG_PRIVATE_KEY != '' && github.ref == 'refs/heads/main'
|
||||
env:
|
||||
RELEASE_GPG_FINGERPRINT: ${{ vars.RELEASE_GPG_FINGERPRINT }}
|
||||
run: |
|
||||
@@ -444,7 +403,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Verify notarization (macOS only)
|
||||
if: inputs.sign && matrix.platform == 'macos-15'
|
||||
if: matrix.platform == 'macos-15'
|
||||
run: |
|
||||
echo "🔍 Verifying notarization status..."
|
||||
cd ./frontend/editor/src-tauri/target
|
||||
@@ -482,7 +441,7 @@ jobs:
|
||||
# Verify the MSI AND the inner exe extracted from it are signed.
|
||||
# The inner exe is what gets installed on users' machines and what AV scans.
|
||||
- name: Verify Windows Code Signature
|
||||
if: inputs.sign && matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && github.ref == 'refs/heads/main'
|
||||
if: matrix.platform == 'windows-latest' && env.SM_API_KEY != '' && github.ref == 'refs/heads/main'
|
||||
shell: pwsh
|
||||
run: |
|
||||
$allSigned = $true
|
||||
|
||||
@@ -106,11 +106,11 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
cache-disabled: true
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
- name: Build application
|
||||
run: task backend:build
|
||||
env:
|
||||
@@ -155,22 +155,9 @@ jobs:
|
||||
echo "platforms=linux/amd64,linux/arm64/v8" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Base-changed PRs build the embedded image with the local docker driver
|
||||
# so the locally-built stirling-pdf-base:pr-test (in the daemon image
|
||||
# store) resolves. A buildx container builder cannot see it and would try
|
||||
# to pull it from a registry, which fails. Single-platform, no gha cache.
|
||||
- name: Build ${{ matrix.docker-rev }} against local base (PR base change)
|
||||
if: github.event_name == 'pull_request' && inputs.docker-base-changed == 'true'
|
||||
run: |
|
||||
DOCKER_BUILDKIT=1 docker build \
|
||||
--build-arg BASE_IMAGE=${{ steps.build-params.outputs.base_image }} \
|
||||
--file ./${{ matrix.docker-rev }} \
|
||||
--tag stirling-pdf-embedded:pr-test \
|
||||
.
|
||||
|
||||
- name: Build ${{ matrix.docker-rev }} (Depot)
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.16.0
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
@@ -182,10 +169,8 @@ jobs:
|
||||
provenance: true
|
||||
sbom: true
|
||||
|
||||
# Fork PRs that did NOT change the base use the buildx container builder
|
||||
# (multi-platform + gha cache) against the published base image.
|
||||
- name: Build ${{ matrix.docker-rev }} (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && inputs.docker-base-changed != 'true'
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
@@ -245,7 +230,7 @@ jobs:
|
||||
|
||||
- name: Build docker/unoserver/Dockerfile (Depot)
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.16.0
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
|
||||
with:
|
||||
gradle-version: 9.6.0
|
||||
gradle-version: 9.3.1
|
||||
|
||||
- name: Build with Gradle
|
||||
run: ./gradlew build
|
||||
@@ -83,7 +83,7 @@ jobs:
|
||||
|
||||
- name: Build and push test image (Depot)
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.16.0
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
@@ -129,7 +129,7 @@ jobs:
|
||||
environment:
|
||||
DISABLE_ADDITIONAL_FEATURES: "true"
|
||||
SECURITY_ENABLELOGIN: "false"
|
||||
SYSTEM_DEFAULTLOCALE: en-US
|
||||
SYSTEM_DEFAULTLOCALE: en-GB
|
||||
UI_APPNAME: "Stirling-PDF Test"
|
||||
UI_HOMEDESCRIPTION: "Test Deployment"
|
||||
UI_APPNAMENAVBAR: "Test"
|
||||
|
||||
+3
-13
@@ -46,12 +46,6 @@ app/core/storage/
|
||||
# These are generated by npm build and should not be committed
|
||||
app/core/src/main/resources/static/assets/
|
||||
app/core/src/main/resources/static/index.html
|
||||
# Prerendered per-route SPA pages (OG/social-preview), e.g. compress.html. api-landing.html is source.
|
||||
app/core/src/main/resources/static/*.html
|
||||
!app/core/src/main/resources/static/api-landing.html
|
||||
!app/core/src/main/resources/static/mobile-upload.html
|
||||
# Prerendered nested-route pages (e.g. settings/people.html)
|
||||
app/core/src/main/resources/static/settings/
|
||||
app/core/src/main/resources/static/locales/
|
||||
app/core/src/main/resources/static/Login/
|
||||
app/core/src/main/resources/static/classic-logo/
|
||||
@@ -59,14 +53,10 @@ app/core/src/main/resources/static/modern-logo/
|
||||
app/core/src/main/resources/static/og_images/
|
||||
app/core/src/main/resources/static/samples/
|
||||
app/core/src/main/resources/static/manifest-classic.json
|
||||
app/core/src/main/resources/static/og-metadata.json
|
||||
app/core/src/main/resources/static/sw-folder-retry.js
|
||||
app/core/src/main/resources/static/robots.txt
|
||||
app/core/src/main/resources/static/pdfium/
|
||||
app/core/src/main/resources/static/pdfjs/
|
||||
app/core/src/main/resources/static/vendor/
|
||||
app/core/src/main/resources/static/**/*.gz
|
||||
app/core/src/main/resources/static/**/*.br
|
||||
# Note: Keep backend-managed files like fonts/, css/, js/, pdfjs/, etc.
|
||||
|
||||
# Gradle
|
||||
@@ -222,7 +212,7 @@ out/
|
||||
*.asc
|
||||
|
||||
# Allow test fixture certificates (synthetic, no real credentials)
|
||||
!frontend/editor/src/core/tests/test-fixtures/certs/**
|
||||
!frontend/src/core/tests/test-fixtures/certs/**
|
||||
|
||||
# SSH Keys
|
||||
*.pub
|
||||
@@ -264,7 +254,7 @@ node_modules/
|
||||
*compact*.json
|
||||
test_batch.json
|
||||
*.backup.*.json
|
||||
frontend/editor/public/locales/*/translation.backup*.json
|
||||
frontend/public/locales/*/translation.backup*.json
|
||||
|
||||
# Development/build artifacts
|
||||
.gradle-cache/
|
||||
@@ -289,4 +279,4 @@ docs/type3/signatures/
|
||||
*.playwright-mcp.png
|
||||
|
||||
# Local screenshot artifacts from *-screenshots.spec.ts
|
||||
frontend/editor/screenshots/
|
||||
frontend/screenshots/
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
# PostHog project-level key - phc_ prefix keys are public/client-side by design
|
||||
# (PostHog client-side tracking embeds them in the browser bundle). Committed
|
||||
# intentionally in #6150 so engine/.env has a working default, with real
|
||||
# credentials overridden via engine/.env.local.
|
||||
engine/.env:generic-api-key:41
|
||||
|
||||
# MCP test fixtures / harness - no real secrets:
|
||||
# - test-only API key constant in an integration test
|
||||
# - JDBC URL + throwaway Keycloak creds in the local test compose
|
||||
# - placeholder / shell-variable Bearer headers in curl-based validation scripts
|
||||
app/proprietary/src/test/java/stirling/software/proprietary/mcp/security/McpApiKeyIntegrationTest.java:generic-api-key:40
|
||||
testing/compose/docker-compose-keycloak-mcp.yml:generic-api-key:25
|
||||
testing/compose/validate-mcp-apikey.sh:curl-auth-header:73
|
||||
testing/compose/validate-mcp-test.sh:curl-auth-header:92
|
||||
testing/compose/validate-mcp-test.sh:curl-auth-header:116
|
||||
|
||||
# Storybook example showing curl with a fake Bearer token placeholder (sk_live_a3f8...).
|
||||
frontend/editor/src/proprietary/ui/CodeBlock.stories.tsx:curl-auth-header:5
|
||||
|
||||
# Truncated placeholder API key in portal docs example (sk_live_8f2c...e10) - not a real secret.
|
||||
frontend/editor/src/portal/components/docs/GettingStartedSection.tsx:generic-api-key:30
|
||||
|
||||
# False positive: generic-api-key matches the Java type name "X509Certificate"
|
||||
# in a method signature (CreateSignatureBase.resolveSignatureAlgorithm) - not a secret.
|
||||
app/core/src/main/java/org/apache/pdfbox/examples/signature/CreateSignatureBase.java:generic-api-key:224
|
||||
|
||||
# Supabase publishable key (public by design, RLS-protected) used as a CI fallback
|
||||
# default in the tauri-build workflow when the GitHub secret is unset - not a real secret.
|
||||
.github/workflows/tauri-build.yml:generic-api-key:402
|
||||
+50
-12
@@ -1,14 +1,52 @@
|
||||
# The actual checks live in .taskfiles/pre-commit.yml (with helper scripts under
|
||||
# scripts/pre-commit/) and are driven by Task. This hook just delegates to `task
|
||||
# pre-commit` so the git pre-commit hook, CI and a manual `task pre-commit` all
|
||||
# run the exact same thing. Requires `task` and `uv` on PATH. To auto-fix instead
|
||||
# of only checking, run `task pre-commit:fix`.
|
||||
repos:
|
||||
- repo: local
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.14
|
||||
hooks:
|
||||
- id: task-pre-commit
|
||||
name: task pre-commit
|
||||
entry: task pre-commit
|
||||
language: system
|
||||
pass_filenames: false
|
||||
always_run: true
|
||||
- id: ruff
|
||||
args:
|
||||
- --fix
|
||||
- --line-length=127
|
||||
files: ^((\.github/scripts|scripts|app/core/src/main/resources/static/python)/.+)?[^/]+\.py$
|
||||
exclude: (split_photos.py)
|
||||
- id: ruff-format
|
||||
files: ^((\.github/scripts|scripts|app/core/src/main/resources/static/python)/.+)?[^/]+\.py$
|
||||
exclude: (split_photos.py)
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.4.2
|
||||
hooks:
|
||||
- id: codespell
|
||||
args:
|
||||
- --ignore-words-list=thirdParty,tabEl,tabEls,Sie,ist,fulfilment
|
||||
- --skip="./.*,*.csv,*.json,*.ambr"
|
||||
- --quiet-level=2
|
||||
files: \.(html|css|js|py|md)$
|
||||
exclude: (.vscode|.devcontainer|app/core/src/main/resources|app/proprietary/src/main/resources|frontend/editor/public/vendor|Dockerfile|.*/pdfjs.*|.*/thirdParty.*|bootstrap.*|.*\.min\..*|.*diff\.js)
|
||||
- repo: https://github.com/gitleaks/gitleaks
|
||||
rev: v8.30.0
|
||||
hooks:
|
||||
- id: gitleaks
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v6.0.0
|
||||
hooks:
|
||||
- id: end-of-file-fixer
|
||||
files: ^.*(\.js|\.java|\.py|\.yml)$
|
||||
exclude: ^(.*/pdfjs.*|.*/thirdParty.*|bootstrap.*|.*\.min\..*|.*diff\.js|\.github/workflows/.*$)
|
||||
- id: trailing-whitespace
|
||||
files: ^.*(\.js|\.java|\.py|\.yml)$
|
||||
exclude: ^(.*/pdfjs.*|.*/thirdParty.*|bootstrap.*|.*\.min\..*|.*diff\.js|\.github/workflows/.*$)
|
||||
- repo: https://github.com/pappasam/toml-sort
|
||||
rev: v0.24.4
|
||||
hooks:
|
||||
- id: toml-sort-fix
|
||||
files: frontend/editor/public/locales/.*\.toml$
|
||||
args: ['--in-place', '--all', '--ignore-case']
|
||||
# - repo: https://github.com/thibaudcolas/pre-commit-stylelint
|
||||
# rev: v16.21.1
|
||||
# hooks:
|
||||
# - id: stylelint
|
||||
# additional_dependencies:
|
||||
# - stylelint@16.21.1
|
||||
# - stylelint-config-standard@38.0.0
|
||||
# - "@stylistic/stylelint-plugin@3.1.3"
|
||||
# files: \.(css)$
|
||||
# args: [--fix]
|
||||
|
||||
+2
-27
@@ -18,36 +18,17 @@ version: '3'
|
||||
tasks:
|
||||
dev:
|
||||
desc: "Start backend dev server"
|
||||
cmds:
|
||||
- task: dev:proprietary
|
||||
vars:
|
||||
PORT: '{{.PORT}}'
|
||||
AIENGINE_URL: '{{.AIENGINE_URL}}'
|
||||
AIENGINE_ENABLED: '{{.AIENGINE_ENABLED}}'
|
||||
AIENGINE_TIMEOUTSECONDS: '{{.AIENGINE_TIMEOUTSECONDS}}'
|
||||
SECURITY_ENABLELOGIN: '{{.SECURITY_ENABLELOGIN}}'
|
||||
POLICIES_ENABLED: '{{.POLICIES_ENABLED}}'
|
||||
|
||||
dev:proprietary:
|
||||
desc: "Start backend dev server in proprietary mode"
|
||||
# `dotenv:` reads from the root Taskfile's directory (".") because this
|
||||
# subtaskfile is included with `dir: .`. Local overrides in
|
||||
# .env.proprietary.local win over the committed .env.proprietary defaults.
|
||||
dotenv: ['app/.env.proprietary.local', 'app/.env.proprietary']
|
||||
ignore_error: true
|
||||
vars:
|
||||
PORT: '{{.PORT | default "8080"}}'
|
||||
AIENGINE_URL: '{{.AIENGINE_URL | default ""}}'
|
||||
AIENGINE_ENABLED: '{{.AIENGINE_ENABLED | default "false"}}'
|
||||
AIENGINE_TIMEOUTSECONDS: '{{.AIENGINE_TIMEOUTSECONDS | default "120"}}'
|
||||
SECURITY_ENABLELOGIN: '{{.SECURITY_ENABLELOGIN | default ""}}'
|
||||
POLICIES_ENABLED: '{{.POLICIES_ENABLED | default ""}}'
|
||||
env:
|
||||
SERVER_PORT: '{{.PORT}}'
|
||||
cmds:
|
||||
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED={{.AIENGINE_ENABLED}} AIENGINE_TIMEOUTSECONDS={{.AIENGINE_TIMEOUTSECONDS}} {{end}}{{if .SECURITY_ENABLELOGIN}}SECURITY_ENABLELOGIN={{.SECURITY_ENABLELOGIN}} {{end}}{{if .POLICIES_ENABLED}}POLICIES_ENABLED={{.POLICIES_ENABLED}} {{end}}cmd /c ".\gradlew.bat :stirling-pdf:bootRun"'
|
||||
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true AIENGINE_TIMEOUTSECONDS={{.AIENGINE_TIMEOUTSECONDS}} {{end}}cmd /c ".\gradlew.bat :stirling-pdf:bootRun"'
|
||||
platforms: [windows]
|
||||
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED={{.AIENGINE_ENABLED}} AIENGINE_TIMEOUTSECONDS={{.AIENGINE_TIMEOUTSECONDS}} {{end}}{{if .SECURITY_ENABLELOGIN}}SECURITY_ENABLELOGIN={{.SECURITY_ENABLELOGIN}} {{end}}{{if .POLICIES_ENABLED}}POLICIES_ENABLED={{.POLICIES_ENABLED}} {{end}}./gradlew :stirling-pdf:bootRun'
|
||||
- cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED=true AIENGINE_TIMEOUTSECONDS={{.AIENGINE_TIMEOUTSECONDS}} {{end}}./gradlew :stirling-pdf:bootRun'
|
||||
platforms: [linux, darwin]
|
||||
|
||||
dev:bundled:
|
||||
@@ -69,15 +50,9 @@ tasks:
|
||||
PORT: '{{.PORT | default "8080"}}'
|
||||
# Override to "" to run the pure `saas` profile against your own SAAS_DB_*.
|
||||
PROFILES: '{{.PROFILES | default "dev"}}'
|
||||
AIENGINE_URL: '{{.AIENGINE_URL | default ""}}'
|
||||
AIENGINE_ENABLED: '{{.AIENGINE_ENABLED | default "false"}}'
|
||||
AIENGINE_TIMEOUTSECONDS: '{{.AIENGINE_TIMEOUTSECONDS | default "120"}}'
|
||||
env:
|
||||
SERVER_PORT: '{{.PORT}}'
|
||||
STIRLING_FLAVOR: saas
|
||||
AIENGINE_URL: '{{.AIENGINE_URL}}'
|
||||
AIENGINE_ENABLED: '{{.AIENGINE_ENABLED}}'
|
||||
AIENGINE_TIMEOUTSECONDS: '{{.AIENGINE_TIMEOUTSECONDS}}'
|
||||
cmds:
|
||||
- cmd: cmd /c ".\gradlew.bat :stirling-pdf:bootRun {{if .PROFILES}}--args=\"--spring.profiles.include={{.PROFILES}}\"{{end}}"
|
||||
platforms: [windows]
|
||||
|
||||
+13
-76
@@ -1,14 +1,7 @@
|
||||
version: '3'
|
||||
|
||||
vars:
|
||||
# jdk.dynalink is required by VeraPDF (PDF/A validation); without it the bundled JRE throws
|
||||
# NoClassDefFoundError: jdk/dynalink/Namespace at runtime in get-info-on-pdf and verify-pdf
|
||||
JLINK_MODULES: "java.base,java.compiler,java.desktop,java.instrument,java.logging,java.management,java.naming,java.net.http,java.prefs,java.rmi,java.scripting,java.security.jgss,java.security.sasl,java.sql,java.transaction.xa,java.xml,java.xml.crypto,jdk.crypto.ec,jdk.crypto.cryptoki,jdk.unsupported,jdk.dynalink"
|
||||
|
||||
# Minimum Java major the bundled JRE must be. Keep in sync with build.gradle
|
||||
# `modernJavaVersion` - the app JAR is compiled for this, so an older runtime
|
||||
# fails at launch with UnsupportedClassVersionError. Enforced by jlink:verify.
|
||||
REQUIRED_JAVA: "25"
|
||||
JLINK_MODULES: "java.base,java.compiler,java.desktop,java.instrument,java.logging,java.management,java.naming,java.net.http,java.prefs,java.rmi,java.scripting,java.security.jgss,java.security.sasl,java.sql,java.transaction.xa,java.xml,java.xml.crypto,jdk.crypto.ec,jdk.crypto.cryptoki,jdk.unsupported"
|
||||
|
||||
# Override via JPDFIUM_PLATFORMS env (csv of platform keys, or 'all').
|
||||
JPDFIUM_PLATFORMS:
|
||||
@@ -69,21 +62,21 @@ tasks:
|
||||
deps: [prepare]
|
||||
dir: editor
|
||||
cmds:
|
||||
- npx tauri build --bundles app --config '{"bundle":{"createUpdaterArtifacts":false}}'
|
||||
- npx tauri build --bundles app
|
||||
|
||||
build:dev:windows:
|
||||
desc: "Build Tauri desktop NSIS installer (Windows)"
|
||||
deps: [prepare]
|
||||
dir: editor
|
||||
cmds:
|
||||
- npx tauri build --bundles nsis --config '{"bundle":{"createUpdaterArtifacts":false}}'
|
||||
- npx tauri build --bundles nsis
|
||||
|
||||
build:dev:linux:
|
||||
desc: "Build Tauri desktop AppImage (Linux)"
|
||||
deps: [prepare]
|
||||
dir: editor
|
||||
cmds:
|
||||
- npx tauri build --bundles appimage --config '{"bundle":{"createUpdaterArtifacts":false}}'
|
||||
- npx tauri build --bundles appimage
|
||||
|
||||
test:
|
||||
desc: "Run Tauri/Cargo tests"
|
||||
@@ -107,20 +100,6 @@ tasks:
|
||||
jlink:
|
||||
desc: "Build backend JAR and create JLink runtime for Tauri"
|
||||
deps: [jlink:jar, jlink:runtime]
|
||||
# Runs after the runtime is in place. Lives here (not in jlink:runtime's
|
||||
# cmds) so it still fires when jlink:runtime short-circuits on its `status:`
|
||||
# check and reuses an existing runtime/jre - that reuse path is exactly how
|
||||
# a stale, too-old JRE slips through.
|
||||
cmds:
|
||||
- task: jlink:verify
|
||||
|
||||
jlink:verify:
|
||||
desc: "Fail the build if the bundled JRE is older than the app JAR requires"
|
||||
dir: editor
|
||||
env:
|
||||
REQUIRED_JAVA: "{{.REQUIRED_JAVA}}"
|
||||
cmds:
|
||||
- node scripts/verify-bundled-jre.mjs src-tauri/runtime/jre/release
|
||||
|
||||
jlink:jar:
|
||||
desc: "Build backend JAR for Tauri bundling (host-OS natives only by default)"
|
||||
@@ -146,58 +125,16 @@ tasks:
|
||||
cmds:
|
||||
- rm -rf runtime/jre
|
||||
- mkdir -p runtime
|
||||
# Pin jlink to JAVA_HOME so the bundled JRE matches the JDK the build
|
||||
# uses. Bare `jlink` on PATH can resolve to an older system Java (the
|
||||
# ubuntu runner ships Java 11), producing a runtime jlink:verify rejects.
|
||||
#
|
||||
# jdk.crypto.mscapi (the Windows certificate store / SunMSCAPI provider, used by
|
||||
# hardware-backed cert signing) is a Windows-only module - it only exists in a Windows
|
||||
# JDK's jmods, so it is added on Windows only or jlink fails to resolve it elsewhere.
|
||||
- cmd: |
|
||||
JLINK="${JAVA_HOME:+$JAVA_HOME/bin/}jlink"
|
||||
JLINK_COMPRESS="$("$JLINK" --help 2>&1 | grep -q 'zip-\[0-9\]' && echo zip-6 || echo 2)"
|
||||
"$JLINK" \
|
||||
--add-modules {{.JLINK_MODULES}},jdk.crypto.mscapi \
|
||||
--strip-debug \
|
||||
--compress="$JLINK_COMPRESS" \
|
||||
--no-header-files \
|
||||
--no-man-pages \
|
||||
--output runtime/jre
|
||||
platforms: [windows]
|
||||
- cmd: |
|
||||
JLINK="${JAVA_HOME:+$JAVA_HOME/bin/}jlink"
|
||||
JLINK_COMPRESS="$("$JLINK" --help 2>&1 | grep -q 'zip-\[0-9\]' && echo zip-6 || echo 2)"
|
||||
"$JLINK" \
|
||||
--add-modules {{.JLINK_MODULES}} \
|
||||
--strip-debug \
|
||||
--compress="$JLINK_COMPRESS" \
|
||||
--no-header-files \
|
||||
--no-man-pages \
|
||||
--output runtime/jre
|
||||
platforms: [linux, darwin]
|
||||
# jlink emits its files mode 444 (read-only). Tauri's build-script
|
||||
# resource copier preserves source permissions when staging
|
||||
# `runtime/jre/**/*` into `target/<profile>/runtime/jre/...`, so the
|
||||
# staged copies are read-only too. On any subsequent incremental
|
||||
# build the copier tries to overwrite them and fails with a bare
|
||||
# `Permission denied (os error 13)` (Rust's io::Error Display drops
|
||||
# the path, so the failure is opaque). Make the source writable here
|
||||
# so the staged destinations are writable and can be overwritten.
|
||||
#
|
||||
# Trade-off: this task runs for both `task desktop:dev` and
|
||||
# `task desktop:build`, so production bundles also ship mode-644
|
||||
# JRE files instead of 444. Functionally harmless on POSIX (the
|
||||
# `other` bit is `r--` either way, and on macOS code signing is the
|
||||
# real integrity check) and on Windows the DOS read-only attribute
|
||||
# isn't load-bearing for the bundled JDK. If we ever need strict
|
||||
# 444 in production, split the chmod into a dev-only step and have
|
||||
# `desktop:build` run `jlink:clean` first to force a fresh build.
|
||||
- cmd: chmod -R u+w runtime/jre
|
||||
platforms: [linux, darwin]
|
||||
- cmd: powershell -NoProfile -Command "Get-ChildItem -Recurse runtime/jre | ForEach-Object { $_.IsReadOnly = $false }"
|
||||
platforms: [windows]
|
||||
- >-
|
||||
jlink
|
||||
--add-modules {{.JLINK_MODULES}}
|
||||
--strip-debug
|
||||
--compress=zip-6
|
||||
--no-header-files
|
||||
--no-man-pages
|
||||
--output runtime/jre
|
||||
status:
|
||||
- test -f runtime/jre/release
|
||||
- test -d editor/src-tauri/runtime/jre
|
||||
|
||||
jlink:clean:
|
||||
desc: "Remove JLink runtime and bundled JARs"
|
||||
|
||||
@@ -20,11 +20,6 @@ tasks:
|
||||
cmds:
|
||||
- docker build -t stirling-pdf-ultra-lite -f {{.EMBEDDED_DIR}}/Dockerfile.ultra-lite .
|
||||
|
||||
build:backend:
|
||||
desc: "Build backend-only Docker image (no embedded frontend)"
|
||||
cmds:
|
||||
- docker build -t stirling-pdf-backend -f docker/backend/Dockerfile .
|
||||
|
||||
build:frontend:
|
||||
desc: "Build frontend-only Docker image"
|
||||
cmds:
|
||||
|
||||
+1
-63
@@ -34,9 +34,6 @@ tasks:
|
||||
ignore_error: true
|
||||
vars:
|
||||
BASE_DIR: '{{.ROOT_DIR}}/.test-state/playwright'
|
||||
# COVERAGE=1 in the calling environment attaches the JaCoCo agent to
|
||||
# the bootRun JVM and writes to BASE_DIR/jacoco.exec on shutdown.
|
||||
# Off by default to keep local dev runs uninstrumented; CI flips it.
|
||||
env:
|
||||
STIRLING_BASE_PATH: '{{.BASE_DIR}}'
|
||||
# Suppress the analytics opt-in modal that fires on first admin login.
|
||||
@@ -61,19 +58,12 @@ tasks:
|
||||
set -e
|
||||
rm -rf "{{.BASE_DIR}}"
|
||||
mkdir -p "{{.BASE_DIR}}"
|
||||
GRADLE_ARGS=":stirling-pdf:bootRun"
|
||||
if [ -n "${COVERAGE:-}" ]; then
|
||||
# copyJacocoAgent is wired as a dependency of bootRun when
|
||||
# -PjacocoAgent=true, so we do not need to invoke it separately.
|
||||
GRADLE_ARGS="$GRADLE_ARGS -PjacocoAgent=true -PjacocoExec={{.BASE_DIR}}/jacoco.exec"
|
||||
echo "JaCoCo coverage enabled, writing to {{.BASE_DIR}}/jacoco.exec"
|
||||
fi
|
||||
# Background gradle and record its PID so the runner can clean up
|
||||
# the exact process tree (wrapper + forked Spring Boot JVM) without
|
||||
# resorting to fuzzy `pkill -f` patterns. `wait` keeps this script
|
||||
# alive for the lifetime of gradle so Task'"'"'s parallel deps stay
|
||||
# synchronised.
|
||||
bash gradlew $GRADLE_ARGS > "{{.BASE_DIR}}/backend.log" 2>&1 &
|
||||
bash gradlew :stirling-pdf:bootRun > "{{.BASE_DIR}}/backend.log" 2>&1 &
|
||||
GRADLE_PID=$!
|
||||
echo $GRADLE_PID > "{{.BASE_DIR}}/backend.pid"
|
||||
wait $GRADLE_PID
|
||||
@@ -212,55 +202,3 @@ tasks:
|
||||
desc: "Stop the SAML keycloak test environment"
|
||||
cmds:
|
||||
- docker compose -f testing/compose/docker-compose-keycloak-saml.yml down -v
|
||||
|
||||
mcp:up:
|
||||
desc: "Start the MCP keycloak test environment (Stirling as OAuth resource server)"
|
||||
summary: |
|
||||
Brings up Keycloak (OAuth authorization server) + Stirling configured as an
|
||||
MCP resource server, then you can exercise /mcp with real Keycloak tokens.
|
||||
Set LICENSE_KEY=<KEY> to skip the interactive license prompt:
|
||||
task e2e:mcp:up LICENSE_KEY=abc123
|
||||
Pass extra flags via -- :
|
||||
task e2e:mcp:up -- --validate --nobuild
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash testing/compose/start-mcp-test.sh {{if .LICENSE_KEY}}--license-key "{{.LICENSE_KEY}}"{{end}} {{.CLI_ARGS}}
|
||||
|
||||
mcp:manual:
|
||||
desc: "Start the MCP keycloak test env in manual mode (prints URLs + a live token for your client)"
|
||||
summary: |
|
||||
Brings the stack up and prints copy-paste URLs/commands plus a freshly minted
|
||||
access token so you can drive your own MCP client (Inspector, curl, ...).
|
||||
task e2e:mcp:manual LICENSE_KEY=<your-license-key>
|
||||
Add --nobuild if the images are already built:
|
||||
task e2e:mcp:manual LICENSE_KEY=<your-license-key> -- --nobuild
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash testing/compose/start-mcp-test.sh --manual {{if .LICENSE_KEY}}--license-key "{{.LICENSE_KEY}}"{{end}} {{.CLI_ARGS}}
|
||||
|
||||
mcp:apikey:
|
||||
desc: "Start the MCP test env in API-KEY manual mode (no OAuth/IdP): mints a key + prints client settings"
|
||||
summary: |
|
||||
Brings Stirling up in apikey auth mode and prints copy-paste client settings with a freshly
|
||||
minted X-API-KEY - ideal for clients whose OAuth layer can't reach localhost.
|
||||
task e2e:mcp:apikey LICENSE_KEY=<your-license-key>
|
||||
Add --nobuild if images are already built:
|
||||
task e2e:mcp:apikey LICENSE_KEY=<your-license-key> -- --nobuild
|
||||
ignore_error: true
|
||||
cmds:
|
||||
- bash testing/compose/start-mcp-test.sh --apikey {{if .LICENSE_KEY}}--license-key "{{.LICENSE_KEY}}"{{end}} {{.CLI_ARGS}}
|
||||
|
||||
mcp:validate:
|
||||
desc: "Validate the running MCP keycloak test environment end-to-end (oauth mode + real MCP SDK client)"
|
||||
cmds:
|
||||
- bash testing/compose/validate-mcp-test.sh
|
||||
|
||||
mcp:validate-apikey:
|
||||
desc: "Validate the MCP server in API-KEY auth mode (mints a key + real MCP SDK client), then restore oauth"
|
||||
cmds:
|
||||
- bash testing/compose/validate-mcp-apikey.sh
|
||||
|
||||
mcp:down:
|
||||
desc: "Stop the MCP keycloak test environment"
|
||||
cmds:
|
||||
- docker compose -f testing/compose/docker-compose-keycloak-mcp.yml down -v
|
||||
|
||||
@@ -33,7 +33,7 @@ tasks:
|
||||
env:
|
||||
PYTHONUNBUFFERED: "1"
|
||||
cmds:
|
||||
- uv run uvicorn stirling.api.app:app --host 0.0.0.0 --port {{.PORT}} --workers "${STIRLING_ENGINE_WORKERS:-4}"
|
||||
- uv run uvicorn stirling.api.app:app --host 0.0.0.0 --port {{.PORT}}
|
||||
|
||||
dev:
|
||||
desc: "Start engine dev server with hot reload"
|
||||
|
||||
+13
-145
@@ -40,21 +40,6 @@ tasks:
|
||||
cmds:
|
||||
- node editor/scripts/generate-icons.js
|
||||
|
||||
prepare:og:
|
||||
internal: true
|
||||
run: when_changed
|
||||
desc: "Regenerate OG/social-preview metadata from the tool registry"
|
||||
cmds:
|
||||
- node editor/scripts/generate-og-metadata.mjs
|
||||
sources:
|
||||
- editor/src/core/types/toolId.ts
|
||||
- editor/src/core/utils/urlMapping.ts
|
||||
- editor/src/core/data/useTranslatedToolRegistry.tsx
|
||||
- editor/public/og_images/*.png
|
||||
generates:
|
||||
- editor/src/core/data/ogImageMap.json
|
||||
- editor/public/og-metadata.json
|
||||
|
||||
prepare:
|
||||
desc: "Set up dev environment"
|
||||
run: when_changed
|
||||
@@ -64,7 +49,6 @@ tasks:
|
||||
- task: prepare:env
|
||||
vars: { MODE: '{{.MODE}}' }
|
||||
- prepare:icons
|
||||
- prepare:og
|
||||
|
||||
# ============================================================
|
||||
# Development
|
||||
@@ -147,10 +131,8 @@ tasks:
|
||||
build:proprietary:
|
||||
desc: "Build for proprietary mode"
|
||||
deps: [prepare]
|
||||
vars:
|
||||
PREVIEW: '{{.PREVIEW | default ""}}'
|
||||
cmds:
|
||||
- '{{if .PREVIEW}}VITE_BUILD_FOR_PREVIEW=1 {{end}}npx vite build editor --mode proprietary'
|
||||
- npx vite build editor --mode proprietary
|
||||
|
||||
build:saas:
|
||||
desc: "Build for SaaS mode"
|
||||
@@ -174,19 +156,6 @@ tasks:
|
||||
cmds:
|
||||
- npx vite build editor --mode prototypes
|
||||
|
||||
|
||||
storybook:
|
||||
desc: "Start Storybook dev server"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx storybook dev -p 6006 {{.CLI_ARGS}}
|
||||
|
||||
storybook:build:
|
||||
desc: "Build static Storybook"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx storybook build {{.CLI_ARGS}}
|
||||
|
||||
# ============================================================
|
||||
# Code quality
|
||||
# ============================================================
|
||||
@@ -194,23 +163,9 @@ tasks:
|
||||
lint:
|
||||
desc: "Run linting"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- task: lint:eslint
|
||||
- task: lint:dpdm
|
||||
|
||||
lint:eslint:
|
||||
desc: "Run ESLint linting"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx eslint --max-warnings=0
|
||||
|
||||
lint:dpdm:
|
||||
desc: "Run circular import linting"
|
||||
deps: [install]
|
||||
cmds:
|
||||
# Globs so dpdm walks the whole tree. dpdm expands the braces itself, so this is
|
||||
# shell-agnostic. Covers the whole editor tree, including the portal layer.
|
||||
- npx dpdm "editor/src/**/*.{ts,tsx}" --circular --no-warning --no-tree --exit-code circular:1
|
||||
- npx dpdm editor/src --circular --no-warning --no-tree --exit-code circular:1
|
||||
|
||||
lint:fix:
|
||||
desc: "Auto-fix lint issues"
|
||||
@@ -241,26 +196,17 @@ tasks:
|
||||
cmds:
|
||||
- task: typecheck:proprietary
|
||||
|
||||
typecheck:_run:
|
||||
internal: true
|
||||
env:
|
||||
CI: '{{ .CI | default "false" }}'
|
||||
cmds:
|
||||
- '{{ if eq .CI "true" }}npx tsc{{ else }}npx tsgo{{ end }} --noEmit --project {{.PROJECT}}'
|
||||
|
||||
typecheck:core:
|
||||
desc: "Typecheck core build variant"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: editor/src/core/tsconfig.json }
|
||||
- npx tsc --noEmit --project editor/src/core/tsconfig.json
|
||||
|
||||
typecheck:proprietary:
|
||||
desc: "Typecheck proprietary build variant"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: editor/src/proprietary/tsconfig.json }
|
||||
- npx tsc --noEmit --project editor/src/proprietary/tsconfig.json
|
||||
|
||||
typecheck:saas:
|
||||
desc: "Typecheck SaaS build variant"
|
||||
@@ -268,8 +214,7 @@ tasks:
|
||||
- task: prepare
|
||||
vars: { MODE: saas }
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: editor/src/saas/tsconfig.json }
|
||||
- npx tsc --noEmit --project editor/src/saas/tsconfig.json
|
||||
|
||||
typecheck:desktop:
|
||||
desc: "Typecheck desktop build variant"
|
||||
@@ -277,36 +222,19 @@ tasks:
|
||||
- task: prepare
|
||||
vars: { MODE: desktop }
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: editor/src/desktop/tsconfig.json }
|
||||
|
||||
typecheck:cloud:
|
||||
desc: "Typecheck cloud shared layer (standalone)"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: editor/src/cloud/tsconfig.json }
|
||||
- npx tsc --noEmit --project editor/src/desktop/tsconfig.json
|
||||
|
||||
typecheck:scripts:
|
||||
desc: "Typecheck scripts"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: editor/scripts/tsconfig.json }
|
||||
- npx tsc --noEmit --project editor/scripts/tsconfig.json
|
||||
|
||||
typecheck:prototypes:
|
||||
desc: "Typecheck prototypes build variant"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: editor/src/prototypes/tsconfig.json }
|
||||
|
||||
typecheck:portal:
|
||||
desc: "Typecheck developer portal build variant"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: editor/src/portal/tsconfig.json }
|
||||
- npx tsc --noEmit --project editor/src/prototypes/tsconfig.json
|
||||
|
||||
typecheck:all:
|
||||
desc: "Typecheck all build variants"
|
||||
@@ -315,10 +243,8 @@ tasks:
|
||||
- task: typecheck:proprietary
|
||||
- task: typecheck:saas
|
||||
- task: typecheck:desktop
|
||||
- task: typecheck:cloud
|
||||
- task: typecheck:scripts
|
||||
- task: typecheck:prototypes
|
||||
- task: typecheck:portal
|
||||
|
||||
# ============================================================
|
||||
# Quality Gate
|
||||
@@ -332,23 +258,14 @@ tasks:
|
||||
- task: format:check
|
||||
- task: test
|
||||
|
||||
og:check:
|
||||
desc: "Fail if committed OG/social-preview metadata is out of date"
|
||||
cmds:
|
||||
- node editor/scripts/generate-og-metadata.mjs --check
|
||||
|
||||
check:all:
|
||||
desc: "Full CI quality gate"
|
||||
cmds:
|
||||
# Runs first, before prepare regenerates: guards the committed og-metadata.json /
|
||||
# ogImageMap.json that the Cloudflare Pages (plain `vite build`) deploy relies on.
|
||||
- task: og:check
|
||||
- task: typecheck:all
|
||||
- task: lint
|
||||
- task: format:check
|
||||
- task: build
|
||||
- task: test
|
||||
- task: storybook:build
|
||||
|
||||
# ============================================================
|
||||
# Test
|
||||
@@ -356,77 +273,28 @@ tasks:
|
||||
|
||||
test:
|
||||
desc: "Run tests"
|
||||
cmds:
|
||||
- task: test:editor
|
||||
|
||||
test:editor:
|
||||
desc: "Run editor tests"
|
||||
deps: [prepare]
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx vitest run --root editor
|
||||
|
||||
test:watch:
|
||||
desc: "Run tests in watch mode"
|
||||
deps: [prepare]
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx vitest --watch --root editor
|
||||
|
||||
test:coverage:
|
||||
desc: "Run tests with coverage (one-shot; CI-friendly)."
|
||||
deps: [prepare]
|
||||
desc: "Run tests with coverage"
|
||||
deps: [install]
|
||||
cmds:
|
||||
# `vitest run` makes this CI-safe (the bare `vitest` form enters watch
|
||||
# mode). Explicit reporter list because v8 + json-summary is what the
|
||||
# coverage-summary.py helper consumes; html/text are kept for humans.
|
||||
#
|
||||
# reportsDirectory is pinned to ./coverage relative to vitest's root
|
||||
# (--root editor), so output lands at frontend/editor/coverage/. The
|
||||
# CI upload step reads from that path. An earlier attempt with
|
||||
# `./editor/coverage` double-nested into frontend/editor/editor/coverage;
|
||||
# pinning future-proofs against vitest changing the default.
|
||||
- >
|
||||
npx vitest run --root editor --coverage
|
||||
--coverage.provider=v8
|
||||
--coverage.reporter=text-summary
|
||||
--coverage.reporter=json-summary
|
||||
--coverage.reporter=html
|
||||
--coverage.reportsDirectory=./coverage
|
||||
- npx vitest --coverage --root editor
|
||||
|
||||
# ============================================================
|
||||
# Code Generation
|
||||
# ============================================================
|
||||
|
||||
tool-models:
|
||||
desc: "Generate tool API types from the Java OpenAPI spec"
|
||||
deps: [install, ":backend:swagger"]
|
||||
cmds:
|
||||
- npx tsx editor/scripts/generate-tool-api-types.mts --spec ../SwaggerDoc.json --output editor/src/core/types/toolApiTypes.ts
|
||||
sources:
|
||||
- editor/scripts/generate-tool-api-types.mts
|
||||
- ../SwaggerDoc.json
|
||||
generates:
|
||||
- editor/src/core/types/toolApiTypes.ts
|
||||
|
||||
tool-models:check:
|
||||
desc: "Fail if committed tool API types are out of date"
|
||||
deps: [install, ":backend:swagger"]
|
||||
cmds:
|
||||
- npx tsx editor/scripts/generate-tool-api-types.mts --spec ../SwaggerDoc.json --output editor/src/core/types/toolApiTypes.ts --check
|
||||
|
||||
licenses:generate:
|
||||
desc: "Generate frontend license report"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- node editor/scripts/generate-licenses.js
|
||||
|
||||
# ============================================================
|
||||
# Clean
|
||||
# ============================================================
|
||||
|
||||
clean:
|
||||
desc: "Clean build artifacts and caches"
|
||||
cmds:
|
||||
- cmd: powershell rm -Recurse -Force -ErrorAction SilentlyContinue node_modules/.vite, editor/dist, dist
|
||||
platforms: [windows]
|
||||
- cmd: rm -rf node_modules/.vite editor/dist dist
|
||||
platforms: [linux, darwin]
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
version: '3'
|
||||
|
||||
# Repo-wide lint/format/secret checks - the single source of truth that the git
|
||||
# pre-commit hook (.pre-commit-config.yaml) and CI (pre_commit.yml) both call.
|
||||
|
||||
vars:
|
||||
# File selections as git pathspecs: git does the include/exclude matching, so
|
||||
# there is no grep/xargs and it behaves identically on every platform.
|
||||
PY_FILES: >-
|
||||
'scripts/*.py'
|
||||
'.github/scripts/*.py'
|
||||
'app/core/src/main/resources/static/python/*.py'
|
||||
':(exclude)*split_photos.py'
|
||||
SPELL_FILES: >-
|
||||
'*.html'
|
||||
'*.css'
|
||||
'*.js'
|
||||
'*.py'
|
||||
'*.md'
|
||||
':(exclude).vscode/*'
|
||||
':(exclude).devcontainer/*'
|
||||
':(exclude)app/core/src/main/resources/*'
|
||||
':(exclude)app/proprietary/src/main/resources/*'
|
||||
':(exclude)frontend/editor/public/vendor/*'
|
||||
':(exclude)*Dockerfile*'
|
||||
':(exclude)*pdfjs*'
|
||||
':(exclude)*thirdParty*'
|
||||
':(exclude)*bootstrap*'
|
||||
':(exclude)*.min.*'
|
||||
':(exclude)*diff.js'
|
||||
WS_FILES: >-
|
||||
'*.js'
|
||||
'*.java'
|
||||
'*.py'
|
||||
'*.yml'
|
||||
':(exclude)*pdfjs*'
|
||||
':(exclude)*thirdParty*'
|
||||
':(exclude)*bootstrap*'
|
||||
':(exclude)*.min.*'
|
||||
':(exclude)*diff.js'
|
||||
':(exclude).github/workflows/*'
|
||||
LOCALE_TOML: 'frontend/editor/public/locales/*/translation.toml'
|
||||
|
||||
# gitleaks is pinned + checksum-verified by scripts/pre-commit/install_gitleaks.py,
|
||||
# which owns the version and caches the binary here.
|
||||
GITLEAKS_BIN: '.task/bin/gitleaks{{if eq OS "windows"}}.exe{{end}}'
|
||||
|
||||
tasks:
|
||||
default:
|
||||
desc: "Check formatting, spelling, and secrets across the repo"
|
||||
cmds:
|
||||
- task: ruff
|
||||
- task: ruff-format
|
||||
- task: codespell
|
||||
- task: gitleaks
|
||||
- task: whitespace
|
||||
- task: toml-sort
|
||||
|
||||
fix:
|
||||
desc: "Auto-fix formatting, spelling, and secrets issues across the repo"
|
||||
cmds:
|
||||
# Auto-fixers first, then the report-only tools (codespell, gitleaks) so a
|
||||
# finding there does not stop the fixers from running.
|
||||
- task: ruff
|
||||
vars: { FIX: '1' }
|
||||
- task: ruff-format
|
||||
vars: { FIX: '1' }
|
||||
- task: whitespace
|
||||
vars: { FIX: '1' }
|
||||
- task: toml-sort
|
||||
vars: { FIX: '1' }
|
||||
- task: codespell
|
||||
- task: gitleaks
|
||||
|
||||
install:
|
||||
desc: "Install the pinned pre-commit Python tools (ruff, codespell, toml-sort)"
|
||||
run: once
|
||||
cmds:
|
||||
- uv sync --project scripts/pre-commit --locked
|
||||
sources:
|
||||
- scripts/pre-commit/uv.lock
|
||||
- scripts/pre-commit/pyproject.toml
|
||||
status:
|
||||
- test -d scripts/pre-commit/.venv
|
||||
|
||||
clean:
|
||||
desc: "Remove the cached gitleaks binary and the tool virtualenv"
|
||||
cmds:
|
||||
- cmd: rm -rf scripts/pre-commit/.venv .task/bin/gitleaks
|
||||
platforms: [linux, darwin]
|
||||
- cmd: cmd /c "rmdir /s /q scripts\pre-commit\.venv & del /q .task\bin\gitleaks.exe"
|
||||
platforms: [windows]
|
||||
ignore_error: true
|
||||
|
||||
# Individual checks (hidden from `task --list`, but callable, e.g.
|
||||
# `task pre-commit:toml-sort FIX=1`). Pass FIX=1 to auto-fix where supported.
|
||||
ruff:
|
||||
deps: [install]
|
||||
cmds:
|
||||
- uv run --project scripts/pre-commit --no-sync ruff check --line-length=127 {{if .FIX}}--fix {{end}}$(git ls-files {{.PY_FILES}})
|
||||
|
||||
ruff-format:
|
||||
deps: [install]
|
||||
cmds:
|
||||
- uv run --project scripts/pre-commit --no-sync ruff format {{if .FIX}}{{else}}--check {{end}}$(git ls-files {{.PY_FILES}})
|
||||
|
||||
codespell:
|
||||
deps: [install]
|
||||
cmds:
|
||||
- uv run --project scripts/pre-commit --no-sync codespell --ignore-words-list=thirdParty,tabEl,tabEls,Sie,ist,fulfilment --quiet-level=2 $(git ls-files {{.SPELL_FILES}})
|
||||
|
||||
toml-sort:
|
||||
deps: [install]
|
||||
cmds:
|
||||
- uv run --project scripts/pre-commit --no-sync toml-sort --all --ignore-case {{if .FIX}}--in-place{{else}}--check{{end}} {{.LOCALE_TOML}}
|
||||
|
||||
whitespace:
|
||||
cmds:
|
||||
- uv run --no-project python scripts/pre-commit/whitespace.py {{if .FIX}}--fix {{end}}{{.WS_FILES}}
|
||||
|
||||
gitleaks:
|
||||
deps: [gitleaks-bin]
|
||||
# Scan staged changes only, matching the old hook: the git-mode fingerprints
|
||||
# in .gitleaksignore (file:rule:line) still apply, and with nothing staged
|
||||
# this is a no-op. Secrets are never auto-fixed, so FIX has no effect.
|
||||
cmds:
|
||||
- "{{.GITLEAKS_BIN}} git --pre-commit --redact --staged --verbose"
|
||||
|
||||
gitleaks-bin:
|
||||
internal: true
|
||||
desc: "Ensure the pinned, checksum-verified gitleaks binary is cached in .task/bin"
|
||||
cmds:
|
||||
- uv run --no-project python scripts/pre-commit/install_gitleaks.py
|
||||
+3
-3
@@ -200,9 +200,9 @@ const [ToolName] = (props: BaseToolProps) => {
|
||||
```
|
||||
|
||||
## 5. Add Translations
|
||||
Update translation files. **Important: Only update `en-US` files** - other languages are handled separately.
|
||||
Update translation files. **Important: Only update `en-GB` files** - other languages are handled separately.
|
||||
|
||||
**File to update:** `frontend/editor/public/locales/en-US/translation.toml`
|
||||
**File to update:** `frontend/editor/public/locales/en-GB/translation.toml`
|
||||
|
||||
**Required Translation Keys**:
|
||||
```toml
|
||||
@@ -251,7 +251,7 @@ Update translation files. **Important: Only update `en-US` files** - other langu
|
||||
```
|
||||
|
||||
**Translation Notes:**
|
||||
- **Only update `en-US/translation.toml`** - other locale files are managed separately
|
||||
- **Only update `en-GB/translation.toml`** - other locale files are managed separately
|
||||
- Use descriptive keys that match your component's `t()` calls
|
||||
- Include tooltip translations if you created tooltip hooks
|
||||
- Add `options.*` keys if your tool has settings with descriptions
|
||||
|
||||
@@ -139,8 +139,7 @@ The project structure is defined in `engine/pyproject.toml`. Any new dependencie
|
||||
|
||||
#### Environment Variables
|
||||
- All `VITE_*` variables must be declared in the appropriate committed env file:
|
||||
- `frontend/editor/.env` — core and shared vars (base, loaded in every mode)
|
||||
- `frontend/editor/.env.proprietary` — proprietary-only vars, e.g. the admin portal's SaaS/account-link keys (layered on top of `.env` in proprietary mode)
|
||||
- `frontend/editor/.env` — core, proprietary, and shared vars
|
||||
- `frontend/editor/.env.saas` — SaaS-only vars (layered on top of `.env` in SaaS mode)
|
||||
- `frontend/editor/.env.desktop` — desktop (Tauri)-only vars (layered on top of `.env` in desktop mode)
|
||||
- These files are committed to Git and must not contain private keys
|
||||
@@ -153,7 +152,7 @@ The project structure is defined in `engine/pyproject.toml`. Any new dependencie
|
||||
#### Import Paths - CRITICAL
|
||||
**ALWAYS use `@app/*` for imports.** Do not use `@core/*` or `@proprietary/*` unless explicitly wrapping/extending a lower layer implementation.
|
||||
|
||||
For a broader explanation of the frontend layering and override architecture, read @frontend/editor/DeveloperGuide.md
|
||||
For a broader explanation of the frontend layering and override architecture, see [frontend/editor/DeveloperGuide.md](frontend/editor/DeveloperGuide.md).
|
||||
|
||||
```typescript
|
||||
// ✅ CORRECT - Use @app/* for all imports
|
||||
@@ -170,31 +169,7 @@ import { useFileContext } from "@proprietary/contexts/FileContext";
|
||||
- Building layer-specific override that wraps a lower layer's component
|
||||
- Example: `import { AppProviders as CoreAppProviders } from "@core/components/AppProviders"` when creating proprietary/AppProviders.tsx that extends the core version
|
||||
|
||||
The `@app/*` alias automatically resolves to the correct layer based on build target (core/proprietary/saas/desktop/cloud) and handles the fallback cascade — see "Frontend `cloud/` Layer" below for the full per-flavor order.
|
||||
|
||||
#### Frontend `cloud/` Layer
|
||||
|
||||
`@app/*` resolves through a per-flavor cascade — first existing file wins (shadow/override):
|
||||
|
||||
- **core** → core
|
||||
- **proprietary** → proprietary → core
|
||||
- **saas** → saas → cloud → proprietary → core
|
||||
- **desktop** → desktop → cloud → proprietary → core
|
||||
- **cloud** → cloud → proprietary → core
|
||||
|
||||
What goes where:
|
||||
|
||||
- **core** — OSS base.
|
||||
- **proprietary** — licensed / offline features.
|
||||
- **cloud** — the SHARED hosted/SaaS experience used by BOTH saas + desktop: PAYG, wallet, plan, billing, usage meters, cloud config/team/onboarding.
|
||||
- **saas** — web-only: Supabase web auth, AuthCallback, avatar canvas, `window.location`.
|
||||
- **desktop** — Tauri-only: keyring authService, tauriHttpClient, native files/windows, backend routing.
|
||||
|
||||
`cloud/` MUST NOT import `@supabase/*`, `@tauri-apps/*`, raw `fetch`, `window.location`, `localStorage`, `sessionStorage`, or `import.meta.env.VITE_*` (enforced by ESLint). It reaches platform-specific things only via `@app/*` seams: `services/apiClient`, `auth/session.getAccessToken`, `auth/supabase`, `platform/openExternal`, `services/billing`, `hooks/useSaaSMode` — each provided per-platform in `saas/` and `desktop/`.
|
||||
|
||||
Rule of thumb — **move, don't copy**: share via `cloud/`, override by shadowing the same `@app/*` path in a leaf (`saas/` or `desktop/`).
|
||||
|
||||
**Cloud feature flags on desktop.** The local `AppConfigContext` reads `/api/v1/config/app-config` from the LOCAL bundled backend, so cloud-only flags (`aiEngineEnabled`, `premiumEnabled`, …) are never seen on desktop. To read the cloud's view, use `useSaasAppConfig()` (`desktop/hooks/useSaasAppConfig.ts`, backed by the general `saasAppConfigService` — SaaS-mode-only, public endpoint, native HTTP, 5-min cache). It returns `null` outside SaaS mode, so cloud features stay off in local/self-hosted and the server keeps the on/off switch (no desktop release needed to flip a flag). Gate a feature behind a per-platform seam — e.g. `useAiEngineEnabled()` (core reads `useAppConfig()`, desktop reads `useSaasAppConfig()`) — rather than hardcoding the flag on.
|
||||
The `@app/*` alias automatically resolves to the correct layer based on build target (core/proprietary/desktop) and handles the fallback cascade.
|
||||
|
||||
#### Component Override Pattern (Stub/Shadow)
|
||||
Use this pattern for desktop-specific or proprietary-specific features WITHOUT runtime checks or conditionals.
|
||||
@@ -451,7 +426,7 @@ The frontend is organized with a clear separation of concerns:
|
||||
|
||||
## Translation Rules
|
||||
|
||||
- **CRITICAL**: Always update translations in `en-US` only - all other languages (including `en-GB`) are handled separately
|
||||
- **CRITICAL**: Always update translations in `en-GB` only, never `en-US`
|
||||
- Translation files are located in `frontend/editor/public/locales/`
|
||||
|
||||
## Important Notes
|
||||
|
||||
+4
-15
@@ -52,17 +52,6 @@ This guide focuses on developing for Stirling 2.0, including both the React fron
|
||||
- Rust and Cargo (required for Tauri desktop app development)
|
||||
- Tauri CLI (install with `cargo install tauri-cli`)
|
||||
|
||||
### Optional System Dependencies
|
||||
|
||||
These are not required to run the app but enable specific features. The app detects them at startup and disables the relevant features if they are missing.
|
||||
|
||||
| Dependency | Feature | Install |
|
||||
|---|---|---|
|
||||
| LibreOffice | File-to-PDF conversions | `brew install libreoffice` / `apt install libreoffice` |
|
||||
| Tesseract | OCR | `brew install tesseract` / `apt install tesseract-ocr` |
|
||||
| WeasyPrint | AI document creation | `brew install weasyprint` / `apt install weasyprint` |
|
||||
| qpdf | PDF optimisation | `brew install qpdf` / `apt install qpdf` |
|
||||
|
||||
### Setup Steps
|
||||
|
||||
1. Clone the repository:
|
||||
@@ -92,7 +81,7 @@ Visit the [Lombok website](https://projectlombok.org/setup/) for installation in
|
||||
|
||||
5. Add environment variable
|
||||
For local testing, you should generally be testing the full 'Security' version of Stirling PDF. To do this, you must add the environment flag DISABLE_ADDITIONAL_FEATURES=false to your system and/or IDE build/run step.
|
||||
6. **Frontend Setup (Required for Stirling 2.0)**
|
||||
5. **Frontend Setup (Required for Stirling 2.0)**
|
||||
Navigate to the frontend directory and install dependencies using npm.
|
||||
|
||||
### Verify Setup
|
||||
@@ -275,7 +264,7 @@ Stirling-PDF uses different Docker images for various configurations. The build
|
||||
1. Set the security environment variable:
|
||||
|
||||
```bash
|
||||
export DISABLE_ADDITIONAL_FEATURES=true # or false to enable login and security features for builds
|
||||
export DISABLE_ADDITIONAL_FEATURES=true # or false for to enable login and security features for builds
|
||||
```
|
||||
|
||||
2. Build the project:
|
||||
@@ -305,7 +294,7 @@ Stirling-PDF uses different Docker images for various configurations. The build
|
||||
docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t stirlingtools/stirling-pdf:latest-fat -f ./Dockerfile.fat .
|
||||
```
|
||||
|
||||
Note: The `--no-cache` and `--pull` flags ensure that the build process uses the latest base images and doesn't use cached layers, which is useful for testing and ensuring reproducible builds. However, to improve build times these can often be removed depending on your use case
|
||||
Note: The `--no-cache` and `--pull` flags ensure that the build process uses the latest base images and doesn't use cached layers, which is useful for testing and ensuring reproducible builds. however to improve build times these can often be removed depending on your usecase
|
||||
|
||||
## 7. Testing
|
||||
|
||||
@@ -587,7 +576,7 @@ When adding a new feature or modifying existing ones in Stirling-PDF, you'll nee
|
||||
Find the existing `messages.properties` files in the `stirling-pdf/src/main/resources` directory. You'll see files like:
|
||||
|
||||
- `messages.properties` (default, usually English)
|
||||
- `messages_en_US.properties`
|
||||
- `messages_en_GB.properties`
|
||||
- `messages_fr_FR.properties`
|
||||
- `messages_de_DE.properties`
|
||||
- etc.
|
||||
|
||||
@@ -10,18 +10,14 @@ if that directory exists, is licensed under the license defined in "app/propriet
|
||||
if that directory exists, is licensed under the license defined in "app/saas/LICENSE".
|
||||
* All content that resides under the "engine/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "engine/LICENSE".
|
||||
* All content that resides under the "frontend/editor/src/proprietary/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/proprietary/LICENSE".
|
||||
* All content that resides under the "frontend/editor/src/desktop/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/desktop/LICENSE".
|
||||
* All content that resides under the "frontend/editor/src/saas/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/saas/LICENSE".
|
||||
* All content that resides under the "frontend/editor/src/cloud/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/cloud/LICENSE".
|
||||
* All content that resides under the "frontend/editor/src/prototypes/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/prototypes/LICENSE".
|
||||
* All content that resides under the "frontend/editor/src/portal/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/editor/src/portal/LICENSE".
|
||||
* All content that resides under the "frontend/src/proprietary/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/src/proprietary/LICENSE".
|
||||
* All content that resides under the "frontend/src/desktop/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/src/desktop/LICENSE".
|
||||
* All content that resides under the "frontend/src/saas/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/src/saas/LICENSE".
|
||||
* All content that resides under the "frontend/src/prototypes/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "frontend/src/prototypes/LICENSE".
|
||||
* Content outside of the above mentioned directories or restrictions above is
|
||||
available under the MIT License as defined below.
|
||||
|
||||
|
||||
@@ -53,14 +53,14 @@ For full installation options (including desktop and Kubernetes), see our [Docum
|
||||
|
||||
## Support
|
||||
|
||||
- **Community**: [Discord](https://discord.gg/HYmhKj45pU)
|
||||
- **Bug Reports**: [GitHub Issues](https://github.com/Stirling-Tools/Stirling-PDF/issues)
|
||||
- **Community** [Discord](https://discord.gg/HYmhKj45pU)
|
||||
- **Bug Reports**: [Github issues](https://github.com/Stirling-Tools/Stirling-PDF/issues)
|
||||
|
||||
## Contributing
|
||||
|
||||
We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
|
||||
|
||||
This project uses [Task](https://taskfile.dev/) as a unified command runner for all build, dev, and test commands. Run `task dev` to get started running the editor, run `task` to see the most common commands, or see the [Developer Guide](DeveloperGuide.md) for full details.
|
||||
This project uses [Task](https://taskfile.dev/) as a unified command runner for all build, dev, and test commands. Run `task install` to get started, or see the [Developer Guide](DeveloperGuide.md) for full details.
|
||||
|
||||
For adding translations, see the [Translation Guide](devGuide/HowToAddNewLanguage.md).
|
||||
|
||||
|
||||
+8
-56
@@ -25,28 +25,8 @@ includes:
|
||||
e2e:
|
||||
taskfile: .taskfiles/e2e.yml
|
||||
dir: .
|
||||
pre-commit:
|
||||
taskfile: .taskfiles/pre-commit.yml
|
||||
dir: .
|
||||
|
||||
tasks:
|
||||
# ============================================================
|
||||
# Help (shown when you run `task` with no arguments)
|
||||
# ============================================================
|
||||
|
||||
default:
|
||||
desc: "List the most common commands"
|
||||
silent: true
|
||||
cmds:
|
||||
- |
|
||||
echo "Common commands (run 'task --list' to see all):"
|
||||
echo ""
|
||||
echo " task dev Start backend & frontend on free ports"
|
||||
echo " task backend:dev Start backend on default port"
|
||||
echo " task frontend:dev Start frontend on default port"
|
||||
echo " task desktop:dev Start desktop app"
|
||||
echo " task check Quality gate (lint, typecheck, test, etc.)"
|
||||
|
||||
# ============================================================
|
||||
# Setup & Prerequisites
|
||||
# ============================================================
|
||||
@@ -78,41 +58,26 @@ tasks:
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
OPEN: "true"
|
||||
|
||||
dev:portal:
|
||||
desc: "Start backend + editor; the portal is an admin route at /portal"
|
||||
dev:saas:
|
||||
desc: "Start SaaS backend + frontend concurrently on free ports"
|
||||
vars:
|
||||
PORTS:
|
||||
sh: '{{if eq OS "windows"}}{{.FIND_FREE_PORT_PS}} 8080 5173{{else}}{{.FIND_FREE_PORT_SH}} 8080 5173{{end}}'
|
||||
BACKEND_PORT: '{{index (splitList "\n" .PORTS) 0}}'
|
||||
EDITOR_PORT: '{{index (splitList "\n" .PORTS) 1}}'
|
||||
FRONTEND_PORT: '{{index (splitList "\n" .PORTS) 1}}'
|
||||
deps:
|
||||
- task: backend:dev
|
||||
- task: backend:dev:saas
|
||||
vars:
|
||||
PORT: '{{.BACKEND_PORT}}'
|
||||
SECURITY_ENABLELOGIN: "true"
|
||||
POLICIES_ENABLED: "true"
|
||||
- task: frontend:dev:proprietary
|
||||
- task: frontend:dev:saas
|
||||
vars:
|
||||
PORT: '{{.EDITOR_PORT}}'
|
||||
PORT: '{{.FRONTEND_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
OPEN: "true"
|
||||
|
||||
dev:saas:
|
||||
desc: "Start SaaS backend + frontend concurrently on free ports"
|
||||
cmds:
|
||||
- task: dev:_all
|
||||
vars: { FRONTEND: saas, BACKEND: saas }
|
||||
|
||||
dev:all:
|
||||
desc: "Start backend + frontend + engine concurrently on free ports"
|
||||
cmds:
|
||||
- task: dev:_all
|
||||
|
||||
dev:_all:
|
||||
internal: true
|
||||
vars:
|
||||
FRONTEND: '{{.FRONTEND | default "proprietary"}}'
|
||||
BACKEND: '{{.BACKEND | default "proprietary"}}'
|
||||
PORTS:
|
||||
sh: '{{if eq OS "windows"}}{{.FIND_FREE_PORT_PS}} 8080 5173 5001{{else}}{{.FIND_FREE_PORT_SH}} 8080 5173 5001{{end}}'
|
||||
BACKEND_PORT: '{{index (splitList "\n" .PORTS) 0}}'
|
||||
@@ -122,12 +87,11 @@ tasks:
|
||||
- task: engine:dev
|
||||
vars:
|
||||
PORT: '{{.ENGINE_PORT}}'
|
||||
- task: 'backend:dev:{{.BACKEND}}'
|
||||
- task: backend:dev
|
||||
vars:
|
||||
PORT: '{{.BACKEND_PORT}}'
|
||||
AIENGINE_URL: 'http://localhost:{{.ENGINE_PORT}}'
|
||||
AIENGINE_ENABLED: "true"
|
||||
- task: 'frontend:dev:{{.FRONTEND}}'
|
||||
- task: frontend:dev
|
||||
vars:
|
||||
PORT: '{{.FRONTEND_PORT}}'
|
||||
BACKEND_URL: 'http://localhost:{{.BACKEND_PORT}}'
|
||||
@@ -185,16 +149,6 @@ tasks:
|
||||
- task: frontend:format:check
|
||||
- task: engine:format:check
|
||||
|
||||
# ============================================================
|
||||
# Code generation
|
||||
# ============================================================
|
||||
|
||||
tool-models:
|
||||
desc: "Generate all API models from the Java OpenAPI spec"
|
||||
cmds:
|
||||
- task: frontend:tool-models
|
||||
- task: engine:tool-models
|
||||
|
||||
# ============================================================
|
||||
# Quality Gate
|
||||
# ============================================================
|
||||
@@ -221,6 +175,4 @@ tasks:
|
||||
desc: "Clean all build artifacts"
|
||||
cmds:
|
||||
- task: backend:clean
|
||||
- task: frontend:clean
|
||||
- task: engine:clean
|
||||
- task: pre-commit:clean
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# Committed defaults for `task backend:dev:proprietary` (self-hosted / proprietary
|
||||
# flavor). Local overrides + secrets live in app/.env.proprietary.local (ignored).
|
||||
|
||||
# Combined-billing account link (Mode A). Feature-flagged: OFF until release.
|
||||
# Flip to true in app/.env.proprietary.local to test linking locally.
|
||||
STIRLING_BILLING_ACCOUNT_LINK_ENABLED=false
|
||||
# SaaS base URL the linked instance calls (register + entitlement).
|
||||
STIRLING_BILLING_ACCOUNT_LINK_SAAS_BASE_URL=https://stirling.com/app
|
||||
@@ -1,4 +1,3 @@
|
||||
# Whitelist committed env defaults. `.env.saas.local` (and any other .env*)
|
||||
# stays ignored via the root .gitignore.
|
||||
!.env.saas
|
||||
!.env.proprietary
|
||||
|
||||
@@ -80,18 +80,10 @@
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Apache License Version 2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Apache License version 2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Apache License, Version 2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Apache License, version 2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "The Apache License, Version 2.0"
|
||||
@@ -116,10 +108,6 @@
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Mozilla Public License 2.0 (MPL-2.0)"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Mozilla Public License Version 2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "CDDL+GPL License"
|
||||
@@ -184,14 +172,6 @@
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Eclipse Public License, Version 2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "EPL-2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "LGPL-2.1-only"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Ubuntu Font Licence 1.0"
|
||||
|
||||
@@ -29,13 +29,13 @@ spotless {
|
||||
}
|
||||
}
|
||||
dependencies {
|
||||
api "com.google.guava:guava:${guavaVersion}"
|
||||
api 'com.google.guava:guava:33.6.0-jre'
|
||||
api 'org.springframework.boot:spring-boot-starter-webmvc'
|
||||
api 'org.springframework.boot:spring-boot-starter-aspectj'
|
||||
api 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20260313.1'
|
||||
api 'com.fathzer:javaluator:3.0.6'
|
||||
api 'com.posthog.java:posthog:1.2.0'
|
||||
api "org.apache.commons:commons-lang3:${commonsLang3}"
|
||||
api 'org.apache.commons:commons-lang3:3.20.0'
|
||||
api 'com.drewnoakes:metadata-extractor:2.20.0' // Image metadata extractor
|
||||
api 'com.vladsch.flexmark:flexmark-html2md-converter:0.64.8'
|
||||
api "org.apache.pdfbox:pdfbox:$pdfboxVersion"
|
||||
@@ -60,7 +60,7 @@ dependencies {
|
||||
exclude group: 'com.google.code.gson', module: 'gson'
|
||||
}
|
||||
|
||||
api "com.stirling:jpdfium:${jpdfiumVersion}"
|
||||
api 'com.stirling:jpdfium:1.0.1'
|
||||
|
||||
// -PjpdfiumPlatforms=all|<csv of linux-x64,linux-arm64,darwin-x64,darwin-arm64,windows-x64>
|
||||
def jpdfiumPlatformsProp = (project.findProperty('jpdfiumPlatforms') ?: 'all').toString().trim()
|
||||
@@ -75,12 +75,12 @@ dependencies {
|
||||
}
|
||||
logger.lifecycle("JPDFium native platforms: ${jpdfiumPlatforms.join(', ')}")
|
||||
jpdfiumPlatforms.each { platform ->
|
||||
runtimeOnly "com.stirling:jpdfium-natives-${platform}:${jpdfiumVersion}"
|
||||
runtimeOnly "com.stirling:jpdfium-natives-${platform}:1.0.1"
|
||||
}
|
||||
|
||||
// Bucket4j (local in-process token bucket for RateLimitStore default impl)
|
||||
implementation "com.bucket4j:bucket4j_jdk17-core:${bucket4jVersion}"
|
||||
implementation 'com.bucket4j:bucket4j_jdk17-core:8.19.0'
|
||||
|
||||
// ArchUnit: enforces module dependency direction (see ArchitectureTest)
|
||||
testImplementation "com.tngtech.archunit:archunit-junit5:${archunitVersion}"
|
||||
testImplementation 'com.tngtech.archunit:archunit-junit5:1.4.2'
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Chains table parsers in priority order: Tabula lattice → Tabula stream → {@link
|
||||
* LineAlignmentTableParser}. The first parser returning a result above {@link
|
||||
* #TABULA_CONFIDENCE_THRESHOLD} wins; results from different parsers are never mixed on one page.
|
||||
*/
|
||||
@Service
|
||||
@Primary
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class CompositeTableParser implements TableParser {
|
||||
|
||||
/** Min Tabula confidence to accept results; below this LineAlignment is tried instead. */
|
||||
static final float TABULA_CONFIDENCE_THRESHOLD = 0.5f;
|
||||
|
||||
private final TabulaTableParser tabulaParser;
|
||||
private final LineAlignmentTableParser lineAlignmentParser;
|
||||
|
||||
@Override
|
||||
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
|
||||
// Step 1: Tabula lattice mode (ruled/bordered tables).
|
||||
List<TableFragment> latticeResults = filterConfident(tabulaParser.parse(document, rawPage));
|
||||
if (!latticeResults.isEmpty()) {
|
||||
log.debug(
|
||||
"Page {}: using Tabula lattice ({} table(s))",
|
||||
rawPage.pageNumber(),
|
||||
latticeResults.size());
|
||||
return latticeResults;
|
||||
}
|
||||
|
||||
// Step 2: Tabula stream mode (borderless/whitespace-delimited tables).
|
||||
// parseStream is not on the TableParser interface — this intentionally couples to the
|
||||
// concrete TabulaTableParser since stream mode is a Tabula-specific concept.
|
||||
List<TableFragment> streamResults =
|
||||
filterConfident(tabulaParser.parseStream(document, rawPage));
|
||||
if (!streamResults.isEmpty()) {
|
||||
log.debug(
|
||||
"Page {}: using Tabula stream ({} table(s))",
|
||||
rawPage.pageNumber(),
|
||||
streamResults.size());
|
||||
return streamResults;
|
||||
}
|
||||
|
||||
// Step 3: Geometry-based line-alignment fallback.
|
||||
List<TableFragment> lineResults = lineAlignmentParser.parse(document, rawPage);
|
||||
if (!lineResults.isEmpty()) {
|
||||
log.debug(
|
||||
"Page {}: using LineAlignment ({} table(s))",
|
||||
rawPage.pageNumber(),
|
||||
lineResults.size());
|
||||
return lineResults;
|
||||
}
|
||||
|
||||
return List.of();
|
||||
}
|
||||
|
||||
private List<TableFragment> filterConfident(List<TableFragment> tables) {
|
||||
return tables.stream().filter(t -> t.confidence() >= TABULA_CONFIDENCE_THRESHOLD).toList();
|
||||
}
|
||||
}
|
||||
+528
@@ -0,0 +1,528 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.TreeMap;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Fallback {@link TableParser} for borderless financial tables using text geometry.
|
||||
*
|
||||
* <p>Identifies "anchor lines" (≥2 numeric tokens), builds a column grid from their right-edge
|
||||
* positions, groups vertically proximate anchor lines into table candidates, then scores each group
|
||||
* on column consistency and anchor density (confidence ceiling 0.85).
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class LineAlignmentTableParser implements TableParser {
|
||||
|
||||
/** Width in points of each column position bucket. */
|
||||
static final float COLUMN_BUCKET_PT = 5f;
|
||||
|
||||
/** Tolerance in buckets when matching a token's right-edge to a confirmed column position. */
|
||||
private static final int COLUMN_MATCH_BUCKETS = 2;
|
||||
|
||||
/** Maximum gap (as a multiple of modal line spacing) before splitting a group. */
|
||||
private static final float MAX_GAP_FACTOR = 2.5f;
|
||||
|
||||
/** Minimum anchor rows (numeric-heavy) to form a valid table. */
|
||||
static final int MIN_TABLE_ROWS = 3;
|
||||
|
||||
/** Minimum confirmed column positions to form a valid table. */
|
||||
static final int MIN_COLUMNS = 2;
|
||||
|
||||
/**
|
||||
* Min fraction of anchor lines a column must appear on to be confirmed (permissive for N/A
|
||||
* rows).
|
||||
*/
|
||||
private static final double COLUMN_MIN_FREQUENCY = 0.40;
|
||||
|
||||
/**
|
||||
* Matches financial numeric tokens: integers, decimals, parenthetical negatives, currency,
|
||||
* percent, nil dashes.
|
||||
*/
|
||||
private static final Pattern NUMERIC =
|
||||
Pattern.compile("^[\\(\\-\\$£€¥]?\\d[\\d,\\.]*[\\)%]?$|^[-–—]$");
|
||||
|
||||
/**
|
||||
* Lines within this y-distance are merged into one row (restores rows split by LineBuilder's
|
||||
* column-gap logic).
|
||||
*/
|
||||
static final float ROW_MERGE_TOLERANCE_PT = 2f;
|
||||
|
||||
// ── public API ───────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Override
|
||||
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
|
||||
List<RawLine> lines = rawPage.lines();
|
||||
if (lines.size() < MIN_TABLE_ROWS) return List.of();
|
||||
|
||||
float modalSpacing = computeModalSpacing(lines);
|
||||
List<TokenizedLine> tokenized =
|
||||
mergeCoincidentLines(lines.stream().map(this::tokenize).toList());
|
||||
|
||||
List<TokenizedLine> anchors = tokenized.stream().filter(TokenizedLine::isAnchor).toList();
|
||||
|
||||
if (anchors.size() < MIN_TABLE_ROWS) return List.of();
|
||||
|
||||
List<Float> columnGrid = buildColumnGrid(anchors);
|
||||
if (columnGrid.size() < MIN_COLUMNS) {
|
||||
log.debug(
|
||||
"Page {}: LineAlignment — fewer than {} confirmed columns, skipping",
|
||||
rawPage.pageNumber(),
|
||||
MIN_COLUMNS);
|
||||
return List.of();
|
||||
}
|
||||
|
||||
List<List<TokenizedLine>> groups = groupRows(tokenized, columnGrid, modalSpacing);
|
||||
|
||||
List<TableFragment> results = new ArrayList<>();
|
||||
for (int i = 0; i < groups.size(); i++) {
|
||||
buildFragment(groups.get(i), columnGrid, rawPage.pageNumber(), i)
|
||||
.ifPresent(results::add);
|
||||
}
|
||||
|
||||
log.debug(
|
||||
"Page {}: LineAlignment detected {} table(s) ({} anchor lines, {} columns)",
|
||||
rawPage.pageNumber(),
|
||||
results.size(),
|
||||
anchors.size(),
|
||||
columnGrid.size());
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── coincident-line merging ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Merges tokenised lines sharing the same y-position into one row, rejoining label/value halves
|
||||
* split by LineBuilder.
|
||||
*/
|
||||
List<TokenizedLine> mergeCoincidentLines(List<TokenizedLine> tokenized) {
|
||||
if (tokenized.size() < 2) return tokenized;
|
||||
|
||||
List<TokenizedLine> result = new ArrayList<>();
|
||||
int i = 0;
|
||||
|
||||
while (i < tokenized.size()) {
|
||||
float baseY = tokenized.get(i).line().bounds().y();
|
||||
int j = i + 1;
|
||||
while (j < tokenized.size()
|
||||
&& Math.abs(tokenized.get(j).line().bounds().y() - baseY)
|
||||
<= ROW_MERGE_TOLERANCE_PT) {
|
||||
j++;
|
||||
}
|
||||
|
||||
if (j == i + 1) {
|
||||
result.add(tokenized.get(i));
|
||||
} else {
|
||||
result.add(mergeGroup(tokenized.subList(i, j)));
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private TokenizedLine mergeGroup(List<TokenizedLine> group) {
|
||||
List<TextFragment> mergedFragments =
|
||||
group.stream()
|
||||
.flatMap(tl -> tl.line().fragments().stream())
|
||||
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
|
||||
.toList();
|
||||
|
||||
Bounds mergedBounds =
|
||||
group.stream()
|
||||
.map(tl -> tl.line().bounds())
|
||||
.reduce(Bounds::merge)
|
||||
.orElse(group.get(0).line().bounds());
|
||||
|
||||
RawLine mergedLine =
|
||||
new RawLine(
|
||||
group.get(0).line().lineId(),
|
||||
mergedFragments,
|
||||
mergedBounds,
|
||||
group.get(0).line().pageNumber());
|
||||
|
||||
return tokenize(mergedLine);
|
||||
}
|
||||
|
||||
// ── tokenisation ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Splits fragments into word-level tokens; x-positions are estimated linearly within each
|
||||
* fragment.
|
||||
*/
|
||||
TokenizedLine tokenize(RawLine line) {
|
||||
List<LineToken> tokens = new ArrayList<>();
|
||||
for (TextFragment frag : line.fragments()) {
|
||||
tokens.addAll(tokensFromFragment(frag));
|
||||
}
|
||||
List<LineToken> numeric = tokens.stream().filter(LineToken::numeric).toList();
|
||||
return new TokenizedLine(line, tokens, numeric);
|
||||
}
|
||||
|
||||
private List<LineToken> tokensFromFragment(TextFragment frag) {
|
||||
String raw = frag.text();
|
||||
if (raw == null || raw.isBlank()) return List.of();
|
||||
|
||||
float fragX = frag.bounds().x();
|
||||
float fragWidth = frag.bounds().width();
|
||||
int rawLen = raw.length();
|
||||
|
||||
List<LineToken> result = new ArrayList<>();
|
||||
int offset = 0;
|
||||
for (String part : raw.split("\\s+")) {
|
||||
if (part.isEmpty()) {
|
||||
offset++;
|
||||
continue;
|
||||
}
|
||||
int idx = raw.indexOf(part, offset);
|
||||
if (idx < 0) idx = offset;
|
||||
|
||||
float tokenX = rawLen > 0 ? fragX + ((float) idx / rawLen) * fragWidth : fragX;
|
||||
float tokenRight =
|
||||
rawLen > 0
|
||||
? fragX + ((float) (idx + part.length()) / rawLen) * fragWidth
|
||||
: fragX + fragWidth;
|
||||
|
||||
result.add(new LineToken(part, tokenX, tokenRight, NUMERIC.matcher(part).matches()));
|
||||
offset = idx + part.length();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── column grid ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Returns confirmed column right-edge positions — those appearing on ≥ {@value
|
||||
* #COLUMN_MIN_FREQUENCY} × N anchor lines.
|
||||
*/
|
||||
private List<Float> buildColumnGrid(List<TokenizedLine> anchors) {
|
||||
// bucket → set of line indices that contributed a numeric token to that bucket
|
||||
Map<Integer, List<Integer>> bucketLines = new HashMap<>();
|
||||
for (int i = 0; i < anchors.size(); i++) {
|
||||
for (LineToken t : anchors.get(i).numeric()) {
|
||||
int bucket = bucket(t.right());
|
||||
bucketLines.computeIfAbsent(bucket, k -> new ArrayList<>()).add(i);
|
||||
}
|
||||
}
|
||||
|
||||
int minHits =
|
||||
Math.max(MIN_TABLE_ROWS, (int) Math.ceil(anchors.size() * COLUMN_MIN_FREQUENCY));
|
||||
|
||||
// Confirmed buckets → average right-edge for that bucket
|
||||
TreeMap<Integer, Float> confirmed = new TreeMap<>();
|
||||
for (Map.Entry<Integer, List<Integer>> entry : bucketLines.entrySet()) {
|
||||
// Count distinct lines
|
||||
long distinctLines = entry.getValue().stream().distinct().count();
|
||||
if (distinctLines >= minHits) {
|
||||
double avg =
|
||||
entry.getValue().stream()
|
||||
.distinct() // weight each line equally regardless of token count
|
||||
.mapToDouble(
|
||||
lineIdx ->
|
||||
avgRightEdgeForBucket(
|
||||
anchors, lineIdx, entry.getKey()))
|
||||
.average()
|
||||
.orElse(entry.getKey() * (double) COLUMN_BUCKET_PT);
|
||||
confirmed.put(entry.getKey(), (float) avg);
|
||||
}
|
||||
}
|
||||
|
||||
return new ArrayList<>(confirmed.values()); // already sorted by bucket (left to right)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the average right-edge position of tokens in {@code line} whose bucket matches {@code
|
||||
* targetBucket}, falling back to the bucket's nominal centre when no tokens match.
|
||||
*/
|
||||
private double avgRightEdgeForBucket(
|
||||
List<TokenizedLine> anchors, int lineIdx, int targetBucket) {
|
||||
return anchors.get(lineIdx).numeric().stream()
|
||||
.filter(t -> bucket(t.right()) == targetBucket)
|
||||
.mapToDouble(LineToken::right)
|
||||
.average()
|
||||
.orElse(targetBucket * (double) COLUMN_BUCKET_PT);
|
||||
}
|
||||
|
||||
// ── grouping ─────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Groups anchor lines into table candidates, including adjacent label rows; a gap >
|
||||
* MAX_GAP_FACTOR × modal spacing splits groups.
|
||||
*/
|
||||
private List<List<TokenizedLine>> groupRows(
|
||||
List<TokenizedLine> all, List<Float> columnGrid, float modalSpacing) {
|
||||
float maxGap = modalSpacing > 0 ? modalSpacing * MAX_GAP_FACTOR : 30f;
|
||||
|
||||
List<List<TokenizedLine>> groups = new ArrayList<>();
|
||||
List<TokenizedLine> current = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < all.size(); i++) {
|
||||
TokenizedLine tl = all.get(i);
|
||||
boolean fits = tl.isAnchor() && matchesGrid(tl, columnGrid);
|
||||
|
||||
if (current.isEmpty()) {
|
||||
if (fits) current.add(tl);
|
||||
continue;
|
||||
}
|
||||
|
||||
float gap =
|
||||
tl.line().bounds().y()
|
||||
- current.get(current.size() - 1).line().bounds().bottom();
|
||||
|
||||
if (gap > maxGap) {
|
||||
groups.add(current);
|
||||
current = new ArrayList<>();
|
||||
if (fits) current.add(tl);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (fits) {
|
||||
current.add(tl);
|
||||
} else if (!tl.line().text().isBlank()) {
|
||||
// Include non-anchor lines (labels) only if they have text and are within
|
||||
// proximity.
|
||||
current.add(tl);
|
||||
}
|
||||
}
|
||||
|
||||
if (!current.isEmpty()) groups.add(current);
|
||||
|
||||
return groups.stream().filter(g -> hasEnoughAnchorRows(g, columnGrid)).toList();
|
||||
}
|
||||
|
||||
private boolean hasEnoughAnchorRows(List<TokenizedLine> group, List<Float> columnGrid) {
|
||||
return group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count()
|
||||
>= MIN_TABLE_ROWS;
|
||||
}
|
||||
|
||||
/** A line "matches" the grid when ≥ 60 % of its numeric tokens land in confirmed columns. */
|
||||
private boolean matchesGrid(TokenizedLine tl, List<Float> columnGrid) {
|
||||
if (tl.numeric().isEmpty()) return false;
|
||||
long matches =
|
||||
tl.numeric().stream()
|
||||
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
|
||||
.count();
|
||||
return (double) matches / tl.numeric().size() >= 0.60;
|
||||
}
|
||||
|
||||
private boolean hasInconsistentColumnMatch(TokenizedLine tl, List<Float> columnGrid) {
|
||||
if (tl.numeric().isEmpty()) return false;
|
||||
long hits =
|
||||
tl.numeric().stream()
|
||||
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
|
||||
.count();
|
||||
return (double) hits / tl.numeric().size() < 0.60;
|
||||
}
|
||||
|
||||
// ── fragment assembly ────────────────────────────────────────────────────────────────────────
|
||||
|
||||
private Optional<TableFragment> buildFragment(
|
||||
List<TokenizedLine> group, List<Float> columnGrid, int pageNumber, int tableIndex) {
|
||||
|
||||
long anchorCount =
|
||||
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
|
||||
if (anchorCount < MIN_TABLE_ROWS) return Optional.empty();
|
||||
|
||||
List<String> warnings = new ArrayList<>();
|
||||
List<List<String>> rawRows = new ArrayList<>();
|
||||
List<TableRow> rows = new ArrayList<>();
|
||||
|
||||
for (int rowIdx = 0; rowIdx < group.size(); rowIdx++) {
|
||||
TokenizedLine tl = group.get(rowIdx);
|
||||
List<String> rawRow = buildRawRow(tl, columnGrid);
|
||||
rawRows.add(Collections.unmodifiableList(rawRow));
|
||||
rows.add(buildTableRow(rowIdx, tl, rawRow, columnGrid));
|
||||
}
|
||||
|
||||
// Column count = 1 label column + confirmed numeric columns
|
||||
int colCount = columnGrid.size() + 1;
|
||||
Bounds bounds = computeGroupBounds(group);
|
||||
float confidence = computeConfidence(group, columnGrid, warnings);
|
||||
|
||||
return Optional.of(
|
||||
new TableFragment(
|
||||
"tbl-la-p" + pageNumber + "-" + tableIndex,
|
||||
pageNumber,
|
||||
bounds,
|
||||
List.of(),
|
||||
Collections.unmodifiableList(rows),
|
||||
Collections.unmodifiableList(rawRows),
|
||||
colCount,
|
||||
confidence,
|
||||
Collections.unmodifiableList(warnings),
|
||||
null));
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a raw row as a list of strings: index 0 = label text, indices 1..N = column values.
|
||||
*/
|
||||
private List<String> buildRawRow(TokenizedLine tl, List<Float> columnGrid) {
|
||||
String[] cells = new String[columnGrid.size() + 1];
|
||||
Arrays.fill(cells, "");
|
||||
|
||||
// Separate label tokens (those not landing in any confirmed column) from column tokens.
|
||||
List<String> labelParts = new ArrayList<>();
|
||||
for (LineToken token : tl.all()) {
|
||||
int col = nearestColumnIndex(token.right(), columnGrid);
|
||||
if (col >= 0 && token.numeric()) {
|
||||
int cellIdx = col + 1;
|
||||
cells[cellIdx] =
|
||||
cells[cellIdx].isEmpty()
|
||||
? token.text()
|
||||
: cells[cellIdx] + " " + token.text();
|
||||
} else {
|
||||
labelParts.add(token.text());
|
||||
}
|
||||
}
|
||||
cells[0] = String.join(" ", labelParts).trim();
|
||||
return Arrays.asList(cells);
|
||||
}
|
||||
|
||||
private TableRow buildTableRow(
|
||||
int rowIdx, TokenizedLine tl, List<String> rawRow, List<Float> columnGrid) {
|
||||
List<TableCell> cells = new ArrayList<>(rawRow.size());
|
||||
|
||||
// Label cell: use the line's full bounds as an approximation.
|
||||
cells.add(TableCell.of(0, rawRow.get(0), tl.line().bounds()));
|
||||
|
||||
for (int col = 0; col < columnGrid.size(); col++) {
|
||||
String text = col + 1 < rawRow.size() ? rawRow.get(col + 1) : "";
|
||||
float right = columnGrid.get(col);
|
||||
float left = col > 0 ? columnGrid.get(col - 1) : right - 50f;
|
||||
Bounds cellBounds =
|
||||
new Bounds(
|
||||
left,
|
||||
tl.line().bounds().y(),
|
||||
right - left,
|
||||
tl.line().bounds().height());
|
||||
cells.add(TableCell.of(col + 1, text, cellBounds));
|
||||
}
|
||||
return new TableRow(rowIdx, Collections.unmodifiableList(cells));
|
||||
}
|
||||
|
||||
// ── confidence scoring ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Heuristic score in [0.0, 0.85] (ceiling keeps results below Tabula lattice which starts at
|
||||
* 1.0). Base 0.70; +0.05/col beyond 2 (max +0.10); +0.05 at ≥5 anchors, +0.05 at ≥8; −0.15 if
|
||||
* >30 % of anchors have inconsistent columns; −0.10 if non-anchors outnumber anchors.
|
||||
*/
|
||||
private float computeConfidence(
|
||||
List<TokenizedLine> group, List<Float> columnGrid, List<String> warnings) {
|
||||
float score = 0.70f;
|
||||
|
||||
long anchorCount =
|
||||
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
|
||||
long totalRows = group.size();
|
||||
|
||||
// More columns
|
||||
int extraCols = Math.min(columnGrid.size() - MIN_COLUMNS, 2);
|
||||
score += extraCols * 0.05f;
|
||||
|
||||
// More anchor rows
|
||||
if (anchorCount >= 5) score += 0.05f;
|
||||
if (anchorCount >= 8) score += 0.05f;
|
||||
|
||||
// Inconsistent column matching
|
||||
long inconsistent =
|
||||
group.stream()
|
||||
.filter(TokenizedLine::isAnchor)
|
||||
.filter(tl -> hasInconsistentColumnMatch(tl, columnGrid))
|
||||
.count();
|
||||
if (inconsistent > anchorCount * 0.30) {
|
||||
score -= 0.15f;
|
||||
warnings.add(
|
||||
"Column match inconsistent on "
|
||||
+ inconsistent
|
||||
+ "/"
|
||||
+ anchorCount
|
||||
+ " anchor rows");
|
||||
}
|
||||
|
||||
// Label-heavy
|
||||
long nonAnchor = totalRows - anchorCount;
|
||||
if (nonAnchor > anchorCount) {
|
||||
score -= 0.10f;
|
||||
warnings.add(
|
||||
"Non-anchor rows ("
|
||||
+ nonAnchor
|
||||
+ ") outnumber anchor rows ("
|
||||
+ anchorCount
|
||||
+ ")");
|
||||
}
|
||||
|
||||
return Math.max(0f, Math.min(0.85f, score));
|
||||
}
|
||||
|
||||
// ── utility ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Returns the grid index nearest to {@code rightEdge}, or -1 if none is within {@value
|
||||
* #COLUMN_MATCH_BUCKETS} buckets.
|
||||
*/
|
||||
private int nearestColumnIndex(float rightEdge, List<Float> grid) {
|
||||
int nearest = -1;
|
||||
float minDist = COLUMN_MATCH_BUCKETS * COLUMN_BUCKET_PT + 1f;
|
||||
for (int i = 0; i < grid.size(); i++) {
|
||||
float dist = Math.abs(rightEdge - grid.get(i));
|
||||
if (dist < minDist) {
|
||||
minDist = dist;
|
||||
nearest = i;
|
||||
}
|
||||
}
|
||||
return nearest;
|
||||
}
|
||||
|
||||
private Bounds computeGroupBounds(List<TokenizedLine> group) {
|
||||
return group.stream()
|
||||
.map(tl -> tl.line().bounds())
|
||||
.reduce(Bounds::merge)
|
||||
.orElse(new Bounds(0, 0, 0, 0));
|
||||
}
|
||||
|
||||
/** Modal gap between consecutive line edges, used to calibrate the group-split threshold. */
|
||||
private float computeModalSpacing(List<RawLine> lines) {
|
||||
if (lines.size() < 2) return 0f;
|
||||
Map<Float, Long> freq = new HashMap<>();
|
||||
for (int i = 1; i < lines.size(); i++) {
|
||||
float gap = lines.get(i).bounds().y() - lines.get(i - 1).bounds().bottom();
|
||||
if (gap > 0) freq.merge(Math.round(gap / 2f) * 2f, 1L, Long::sum);
|
||||
}
|
||||
return freq.entrySet().stream()
|
||||
.max(Map.Entry.comparingByValue())
|
||||
.map(Map.Entry::getKey)
|
||||
.orElse(0f);
|
||||
}
|
||||
|
||||
private static int bucket(float x) {
|
||||
return Math.round(x / COLUMN_BUCKET_PT);
|
||||
}
|
||||
|
||||
// ── private data types ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** A word-level token with an approximate right-edge x-position. */
|
||||
record LineToken(String text, float x, float right, boolean numeric) {}
|
||||
|
||||
/** A {@link RawLine} with tokens pre-computed; an "anchor" has ≥ 2 numeric tokens. */
|
||||
record TokenizedLine(RawLine line, List<LineToken> all, List<LineToken> numeric) {
|
||||
boolean isAnchor() {
|
||||
return numeric.size() >= 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Groups {@link TextFragment} objects into visual {@link RawLine}s using baseline proximity.
|
||||
*
|
||||
* <p>Fragments are on the same line when their baselines are within a font-size-derived tolerance.
|
||||
* A new line starts whenever the horizontal gap exceeds an adaptive column-gap threshold ({@code
|
||||
* max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT)}), splitting two-column text.
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class LineBuilder {
|
||||
|
||||
/** Baseline tolerance as a fraction of font size; 0.5 keeps mixed-size text on one line. */
|
||||
private static final float BASELINE_TOLERANCE_FACTOR = 0.5f;
|
||||
|
||||
/** Absolute minimum tolerance so tiny font sizes don't collapse multi-line content. */
|
||||
private static final float MIN_BASELINE_TOLERANCE = 2f;
|
||||
|
||||
/**
|
||||
* Column-gap threshold as a fraction of page width; 0.10 clears tab stops but stays below
|
||||
* two-column gutters.
|
||||
*/
|
||||
static final float COLUMN_GAP_RATIO = 0.10f;
|
||||
|
||||
/** Floor for the column-gap threshold so narrow pages don't over-split lines. */
|
||||
static final float COLUMN_GAP_MIN_PT = 40f;
|
||||
|
||||
public List<RawLine> build(List<TextFragment> fragments, int pageNumber) {
|
||||
if (fragments.isEmpty()) return List.of();
|
||||
|
||||
float effectiveWidth = inferEffectiveWidth(fragments);
|
||||
float columnGapThreshold = Math.max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT);
|
||||
log.debug(
|
||||
"LineBuilder page {}: effectiveWidth={:.1f}pt, columnGapThreshold={:.1f}pt",
|
||||
pageNumber,
|
||||
effectiveWidth,
|
||||
columnGapThreshold);
|
||||
|
||||
// Sort top-to-bottom first, then left-to-right within the same baseline band.
|
||||
List<TextFragment> sorted =
|
||||
fragments.stream()
|
||||
.sorted(
|
||||
Comparator.comparingDouble(TextFragment::baseline)
|
||||
.thenComparingDouble(f -> f.bounds().x()))
|
||||
.toList();
|
||||
|
||||
List<List<TextFragment>> groups = groupByBaseline(sorted, columnGapThreshold);
|
||||
|
||||
List<RawLine> lines = new ArrayList<>(groups.size());
|
||||
for (int i = 0; i < groups.size(); i++) {
|
||||
List<TextFragment> group =
|
||||
groups.get(i).stream()
|
||||
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
|
||||
.toList();
|
||||
|
||||
Bounds lineBounds =
|
||||
group.stream()
|
||||
.map(TextFragment::bounds)
|
||||
.reduce(Bounds::merge)
|
||||
.orElse(new Bounds(0, 0, 0, 0));
|
||||
|
||||
lines.add(new RawLine("ln-p" + pageNumber + "-" + i, group, lineBounds, pageNumber));
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
private List<List<TextFragment>> groupByBaseline(
|
||||
List<TextFragment> sorted, float columnGapThreshold) {
|
||||
List<List<TextFragment>> groups = new ArrayList<>();
|
||||
List<TextFragment> current = new ArrayList<>();
|
||||
float currentBaseline = Float.NaN;
|
||||
|
||||
for (TextFragment fragment : sorted) {
|
||||
if (current.isEmpty()) {
|
||||
current.add(fragment);
|
||||
currentBaseline = fragment.baseline();
|
||||
continue;
|
||||
}
|
||||
|
||||
float maxFontSize =
|
||||
Math.max(
|
||||
fragment.fontSize(),
|
||||
(float)
|
||||
current.stream()
|
||||
.mapToDouble(TextFragment::fontSize)
|
||||
.max()
|
||||
.orElse(0));
|
||||
float tolerance =
|
||||
Math.max(maxFontSize * BASELINE_TOLERANCE_FACTOR, MIN_BASELINE_TOLERANCE);
|
||||
|
||||
boolean sameBaseline = Math.abs(fragment.baseline() - currentBaseline) <= tolerance;
|
||||
boolean columnGap = sameBaseline && hasColumnGap(fragment, current, columnGapThreshold);
|
||||
|
||||
if (sameBaseline && !columnGap) {
|
||||
current.add(fragment);
|
||||
// Anchor to the weighted mean baseline so long lines stay stable.
|
||||
currentBaseline =
|
||||
(currentBaseline * (current.size() - 1) + fragment.baseline())
|
||||
/ current.size();
|
||||
} else {
|
||||
groups.add(current);
|
||||
current = new ArrayList<>();
|
||||
current.add(fragment);
|
||||
currentBaseline = fragment.baseline();
|
||||
}
|
||||
}
|
||||
|
||||
if (!current.isEmpty()) groups.add(current);
|
||||
return groups;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the gap from the rightmost fragment in {@code group} to {@code next} exceeds {@code
|
||||
* threshold}.
|
||||
*/
|
||||
private static boolean hasColumnGap(
|
||||
TextFragment next, List<TextFragment> group, float threshold) {
|
||||
float lastRight = group.get(group.size() - 1).bounds().right();
|
||||
return next.bounds().x() - lastRight > threshold;
|
||||
}
|
||||
|
||||
/** Infers effective page width from the rightmost fragment right-edge plus a 10 % margin. */
|
||||
private static float inferEffectiveWidth(List<TextFragment> fragments) {
|
||||
double maxRight =
|
||||
fragments.stream().mapToDouble(f -> f.bounds().right()).max().orElse(500.0);
|
||||
return (float) maxRight * 1.10f;
|
||||
}
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Detects whether a page is one- or two-column from per-line bounding boxes, and classifies an
|
||||
* X-span into the column it belongs to. Detection is a midpoint vote at {@code pageWidth / 2}.
|
||||
*
|
||||
* <p>Capped at two columns by design — sufficient for the redaction target set (single-column
|
||||
* documents and IEEE-style two-column papers). 3+ column layouts (newspapers, magazines) and
|
||||
* off-centre gutters (asymmetric two-column) would need a histogram or clustering approach to
|
||||
* detect the actual gutter X. (future work)
|
||||
*
|
||||
* <p>Coordinates are PDFTextStripper screen space (top-left origin, Y increases downward).
|
||||
*/
|
||||
public final class PageColumnLayout {
|
||||
|
||||
/**
|
||||
* Slack when checking "crosses a gutter" so single-pixel overshoots don't mark a line as
|
||||
* spanning.
|
||||
*/
|
||||
public static final float SPAN_SLACK_PT = 2f;
|
||||
|
||||
/**
|
||||
* Slack on each side of the page midpoint inside which a line is considered "spanning"
|
||||
* (covering both columns) rather than belonging to one side.
|
||||
*/
|
||||
private static final float MIDPOINT_SLACK_PT = 30f;
|
||||
|
||||
/**
|
||||
* Minimum line width (points) for a line to count toward the two-column tally. Avoids false
|
||||
* positives where right-aligned dates, page numbers, or short "Link" fragments next to a
|
||||
* heading look like a second column when they're really just inline metadata.
|
||||
*/
|
||||
private static final float MIN_COLUMN_LINE_WIDTH_PT = 100f;
|
||||
|
||||
/**
|
||||
* Minimum number of clearly leftish AND clearly rightish lines (each of width ≥ {@link
|
||||
* #MIN_COLUMN_LINE_WIDTH_PT}) required to call the page two-column. Anything below this falls
|
||||
* back to single-column.
|
||||
*/
|
||||
private static final int MIN_SIDE_LINES = 3;
|
||||
|
||||
private final List<float[]> columns;
|
||||
private final List<float[]> gutters;
|
||||
|
||||
private PageColumnLayout(List<float[]> columns, List<float[]> gutters) {
|
||||
this.columns = columns;
|
||||
this.gutters = gutters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines column layout from per-line bounding boxes ({@code [x1, _, x2, _]}). Counts lines
|
||||
* whose X-midpoint sits clearly left of, or clearly right of, the page midpoint (with {@link
|
||||
* #MIDPOINT_SLACK_PT} slack each side). If both sides have at least {@link #MIN_SIDE_LINES}
|
||||
* lines, the page is treated as two-column with the gutter at the page midpoint. Otherwise it's
|
||||
* single-column.
|
||||
*
|
||||
* <p>Cross-column lines must already be split: callers should feed boxes from a line extractor
|
||||
* that splits same-Y glyphs at large X gaps (see {@code AllTextLineExtractor}). Without that
|
||||
* split, IEEE-style aligned-baseline 2-column PDFs produce one wide merged box per row and the
|
||||
* side tallies all end up classified as "spanning", falling to single-column.
|
||||
*/
|
||||
public static PageColumnLayout fromLineBoxes(List<float[]> lineBoxes, float pageWidth) {
|
||||
if (lineBoxes == null || lineBoxes.isEmpty()) {
|
||||
return new PageColumnLayout(List.of(new float[] {0f, pageWidth}), List.of());
|
||||
}
|
||||
float pageMid = pageWidth / 2f;
|
||||
int left = 0, right = 0;
|
||||
for (float[] lb : lineBoxes) {
|
||||
if (lb == null || lb.length < 3) continue;
|
||||
float width = lb[2] - lb[0];
|
||||
// Skip narrow lines — dates, page numbers, "Link" labels next to a heading should
|
||||
// not, on their own, make a single-column doc look two-column.
|
||||
if (width < MIN_COLUMN_LINE_WIDTH_PT) continue;
|
||||
float mid = (lb[0] + lb[2]) * 0.5f;
|
||||
if (mid < pageMid - MIDPOINT_SLACK_PT) left++;
|
||||
else if (mid > pageMid + MIDPOINT_SLACK_PT) right++;
|
||||
}
|
||||
if (left < MIN_SIDE_LINES || right < MIN_SIDE_LINES) {
|
||||
return new PageColumnLayout(List.of(new float[] {0f, pageWidth}), List.of());
|
||||
}
|
||||
float gutterL = pageMid - MIDPOINT_SLACK_PT;
|
||||
float gutterR = pageMid + MIDPOINT_SLACK_PT;
|
||||
return new PageColumnLayout(
|
||||
List.of(new float[] {0f, gutterL}, new float[] {gutterR, pageWidth}),
|
||||
List.of(new float[] {gutterL, gutterR}));
|
||||
}
|
||||
|
||||
/** All columns, left-to-right, as {@code [leftX, rightX]} pairs. Never empty. */
|
||||
public List<float[]> columns() {
|
||||
return columns;
|
||||
}
|
||||
|
||||
/** Gutters between columns, left-to-right, as {@code [leftX, rightX]} pairs. */
|
||||
public List<float[]> gutters() {
|
||||
return gutters;
|
||||
}
|
||||
|
||||
public int columnCount() {
|
||||
return columns.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the column index containing the X-midpoint of {@code [x1, x2]}, falling back to the
|
||||
* closest column if the midpoint sits inside a gutter.
|
||||
*/
|
||||
public int columnOf(float x1, float x2) {
|
||||
float mid = (x1 + x2) * 0.5f;
|
||||
int best = 0;
|
||||
float bestDist = Float.MAX_VALUE;
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
float[] c = columns.get(i);
|
||||
if (mid >= c[0] && mid <= c[1]) return i;
|
||||
float dist = mid < c[0] ? c[0] - mid : mid - c[1];
|
||||
if (dist < bestDist) {
|
||||
bestDist = dist;
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns every column index whose X-range overlaps {@code [x1, x2]} with at least {@link
|
||||
* #SPAN_SLACK_PT} of intrusion. A normal in-column line returns one index; a line crossing a
|
||||
* gutter returns two or more.
|
||||
*/
|
||||
public int[] columnsCrossing(float x1, float x2) {
|
||||
List<Integer> hits = new ArrayList<>();
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
float[] c = columns.get(i);
|
||||
float overlap = Math.min(x2, c[1]) - Math.max(x1, c[0]);
|
||||
if (overlap > SPAN_SLACK_PT) hits.add(i);
|
||||
}
|
||||
if (hits.isEmpty()) hits.add(columnOf(x1, x2));
|
||||
int[] out = new int[hits.size()];
|
||||
for (int i = 0; i < hits.size(); i++) out[i] = hits.get(i);
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.contentstream.PDFGraphicsStreamEngine;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImage;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
|
||||
/**
|
||||
* PDFGraphicsStreamEngine that intercepts {@code drawImage} calls and records each image's bounding
|
||||
* box in PDF user-space (origin bottom-left, Y up) by transforming the unit square through the
|
||||
* current transformation matrix (CTM).
|
||||
*
|
||||
* <p>Usage:
|
||||
*
|
||||
* <pre>{@code
|
||||
* PageImageLocator locator = new PageImageLocator(page, pageIndex);
|
||||
* locator.processPage(page);
|
||||
* List<ImageBox> boxes = locator.getImageBoxes();
|
||||
* }</pre>
|
||||
*
|
||||
* <p>Each {@link ImageBox} carries the 0-based page index and the axis-aligned bounding box {@code
|
||||
* (x1, y1, x2, y2)} in PDF user-space coordinates.
|
||||
*/
|
||||
public final class PageImageLocator extends PDFGraphicsStreamEngine {
|
||||
|
||||
/**
|
||||
* Bounding box of a raster or vector image found on a PDF page.
|
||||
*
|
||||
* @param pageIndex 0-based page index
|
||||
* @param x1 left edge in PDF user-space (origin bottom-left)
|
||||
* @param y1 bottom edge in PDF user-space
|
||||
* @param x2 right edge
|
||||
* @param y2 top edge
|
||||
*/
|
||||
public record ImageBox(int pageIndex, float x1, float y1, float x2, float y2) {}
|
||||
|
||||
private final int pageIndex;
|
||||
private final List<ImageBox> imageBoxes = new ArrayList<>();
|
||||
private final Point2D.Float currentPoint = new Point2D.Float();
|
||||
|
||||
/**
|
||||
* @param page the PDPage to process
|
||||
* @param pageIndex 0-based index of this page in the document (stored on each returned {@link
|
||||
* ImageBox})
|
||||
*/
|
||||
public PageImageLocator(PDPage page, int pageIndex) {
|
||||
super(page);
|
||||
this.pageIndex = pageIndex;
|
||||
}
|
||||
|
||||
/** Returns all image bounding boxes collected during {@link #processPage}. */
|
||||
public List<ImageBox> getImageBoxes() {
|
||||
return imageBoxes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void drawImage(PDImage pdImage) throws IOException {
|
||||
Matrix ctm = getGraphicsState().getCurrentTransformationMatrix();
|
||||
// An image occupies the unit square (0,0)→(1,1) in image space.
|
||||
// Transform all four corners through the CTM to get the page-space bounding box.
|
||||
float a = ctm.getScaleX();
|
||||
float b = ctm.getShearY();
|
||||
float c = ctm.getShearX();
|
||||
float d = ctm.getScaleY();
|
||||
float e = ctm.getTranslateX();
|
||||
float f = ctm.getTranslateY();
|
||||
float[] xs = {e, a + e, c + e, a + c + e};
|
||||
float[] ys = {f, b + f, d + f, b + d + f};
|
||||
float x1 = Float.MAX_VALUE, y1 = Float.MAX_VALUE;
|
||||
float x2 = -Float.MAX_VALUE, y2 = -Float.MAX_VALUE;
|
||||
for (float x : xs) {
|
||||
x1 = Math.min(x1, x);
|
||||
x2 = Math.max(x2, x);
|
||||
}
|
||||
for (float y : ys) {
|
||||
y1 = Math.min(y1, y);
|
||||
y2 = Math.max(y2, y);
|
||||
}
|
||||
imageBoxes.add(new ImageBox(pageIndex, x1, y1, x2, y2));
|
||||
}
|
||||
|
||||
// ---------- required abstract methods (no-op for path operations) ----------
|
||||
|
||||
@Override
|
||||
public void appendRectangle(Point2D p0, Point2D p1, Point2D p2, Point2D p3) {}
|
||||
|
||||
@Override
|
||||
public void clip(int windingRule) {}
|
||||
|
||||
@Override
|
||||
public void moveTo(float x, float y) {
|
||||
currentPoint.setLocation(x, y);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void lineTo(float x, float y) {
|
||||
currentPoint.setLocation(x, y);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void curveTo(float x1, float y1, float x2, float y2, float x3, float y3) {
|
||||
currentPoint.setLocation(x3, y3);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Point2D getCurrentPoint() {
|
||||
return currentPoint;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closePath() {}
|
||||
|
||||
@Override
|
||||
public void endPath() {}
|
||||
|
||||
@Override
|
||||
public void strokePath() {}
|
||||
|
||||
@Override
|
||||
public void fillPath(int windingRule) {}
|
||||
|
||||
@Override
|
||||
public void fillAndStrokePath(int windingRule) {}
|
||||
|
||||
@Override
|
||||
public void shadingFill(COSName shadingName) {}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Runs the per-page ingestion pipeline: {@link WordExtractingStripper} → {@link LineBuilder} →
|
||||
* {@link TableParser}, producing a {@link PdfModels.ParsedPage} per page. The caller owns the
|
||||
* {@link PDDocument} lifecycle.
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class PdfIngester {
|
||||
|
||||
private final LineBuilder lineBuilder;
|
||||
private final TableParser tableParser;
|
||||
|
||||
public List<ParsedPage> parse(PDDocument document) throws IOException {
|
||||
return parse(document, document.getNumberOfPages());
|
||||
}
|
||||
|
||||
public List<ParsedPage> parse(PDDocument document, int maxPages) throws IOException {
|
||||
int pageCount = Math.min(document.getNumberOfPages(), maxPages);
|
||||
List<ParsedPage> pages = new ArrayList<>(pageCount);
|
||||
long fragmentsMs = 0;
|
||||
long tablesMs = 0;
|
||||
long t0 = System.currentTimeMillis();
|
||||
|
||||
for (int p = 1; p <= pageCount; p++) {
|
||||
long ft = System.currentTimeMillis();
|
||||
List<TextFragment> fragments = extractFragments(document, p);
|
||||
fragmentsMs += System.currentTimeMillis() - ft;
|
||||
|
||||
PDPage page = document.getPage(p - 1);
|
||||
PDRectangle mediaBox = page.getMediaBox();
|
||||
List<RawLine> lines = lineBuilder.build(fragments, p);
|
||||
RawPage rawPage = new RawPage(p, mediaBox.getWidth(), mediaBox.getHeight(), lines);
|
||||
|
||||
long tt = System.currentTimeMillis();
|
||||
List<TableFragment> tables = tableParser.parse(document, rawPage);
|
||||
tablesMs += System.currentTimeMillis() - tt;
|
||||
|
||||
log.debug(
|
||||
"Page {}: {} fragments → {} lines, {} table(s)",
|
||||
p,
|
||||
fragments.size(),
|
||||
lines.size(),
|
||||
tables.size());
|
||||
pages.add(new ParsedPage(p, mediaBox.getWidth(), mediaBox.getHeight(), tables, lines));
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[timing] parse pages={} total={}ms fragments={}ms tables={}ms",
|
||||
pageCount,
|
||||
System.currentTimeMillis() - t0,
|
||||
fragmentsMs,
|
||||
tablesMs);
|
||||
return pages;
|
||||
}
|
||||
|
||||
private List<TextFragment> extractFragments(PDDocument document, int pageNumber)
|
||||
throws IOException {
|
||||
WordExtractingStripper stripper = new WordExtractingStripper(pageNumber);
|
||||
stripper.getText(document);
|
||||
return stripper.getFragments();
|
||||
}
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
/**
|
||||
* Extends {@link PDFTextStripper} to capture per-fragment geometry and font metadata.
|
||||
*
|
||||
* <p>Overrides {@link #writeString} to split each content-stream string into word-level {@link
|
||||
* TextFragment}s with bounding boxes, baseline, font name, and bold flag. Coordinates are in
|
||||
* PDFTextStripper space: (0,0) top-left, Y increases downward, {@code getY()} is the baseline.
|
||||
*/
|
||||
class WordExtractingStripper extends PDFTextStripper {
|
||||
|
||||
private final int targetPage;
|
||||
private final List<TextFragment> fragments = new ArrayList<>();
|
||||
private int fragmentIndex = 0;
|
||||
|
||||
WordExtractingStripper(int pageNumber) throws IOException {
|
||||
this.targetPage = pageNumber;
|
||||
setStartPage(pageNumber);
|
||||
setEndPage(pageNumber);
|
||||
setSortByPosition(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void startPage(PDPage page) throws IOException {
|
||||
super.startPage(page);
|
||||
fragments.clear();
|
||||
fragmentIndex = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeString(String text, List<TextPosition> textPositions) throws IOException {
|
||||
if (text == null || text.isBlank()) return;
|
||||
|
||||
// Fast path: no whitespace → emit one fragment (most financial PDFs have each
|
||||
// number as its own string operation, so this is the common case).
|
||||
if (text.indexOf(' ') < 0) {
|
||||
emitFragment(text, textPositions);
|
||||
return;
|
||||
}
|
||||
|
||||
// Per-word splitting requires 1:1 text-char to TextPosition correspondence.
|
||||
// Fall back to one fragment when sizes differ (ligatures, encoding edge cases).
|
||||
if (textPositions.size() != text.length()) {
|
||||
emitFragment(text, textPositions);
|
||||
return;
|
||||
}
|
||||
|
||||
// Emit one TextFragment per whitespace-delimited word with accurate per-word bounds.
|
||||
int start = 0;
|
||||
for (int i = 0; i <= text.length(); i++) {
|
||||
if (i == text.length() || text.charAt(i) == ' ') {
|
||||
if (start < i) {
|
||||
emitFragment(text.substring(start, i), textPositions.subList(start, i));
|
||||
}
|
||||
start = i + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void emitFragment(String text, List<TextPosition> positions) {
|
||||
if (positions.isEmpty()) return;
|
||||
|
||||
float minX = Float.MAX_VALUE;
|
||||
float minY = Float.MAX_VALUE;
|
||||
float maxRight = -Float.MAX_VALUE;
|
||||
float maxBaseline = -Float.MAX_VALUE;
|
||||
TextPosition first = null;
|
||||
|
||||
for (TextPosition tp : positions) {
|
||||
if (tp == null) continue;
|
||||
if (first == null) first = tp;
|
||||
|
||||
float x = tp.getX();
|
||||
// getY() is the baseline; top of character = getY() - getHeight().
|
||||
float top = tp.getY() - tp.getHeight();
|
||||
float right = x + tp.getWidth();
|
||||
float baseline = tp.getY();
|
||||
|
||||
minX = Math.min(minX, x);
|
||||
minY = Math.min(minY, top);
|
||||
maxRight = Math.max(maxRight, right);
|
||||
maxBaseline = Math.max(maxBaseline, baseline);
|
||||
}
|
||||
|
||||
if (first == null) return;
|
||||
|
||||
PDFont font = first.getFont();
|
||||
String fontName = font != null ? font.getName() : "";
|
||||
boolean bold = fontName != null && fontName.toLowerCase().contains("bold");
|
||||
// getHeight() gives the rendered glyph height, which is the most reliable visual size.
|
||||
float fontSize = first.getHeight();
|
||||
|
||||
Bounds bounds = new Bounds(minX, minY, maxRight - minX, maxBaseline - minY);
|
||||
String id = "tf-p" + targetPage + "-" + fragmentIndex++;
|
||||
fragments.add(new TextFragment(id, text, bounds, maxBaseline, fontSize, fontName, bold));
|
||||
}
|
||||
|
||||
List<TextFragment> getFragments() {
|
||||
return Collections.unmodifiableList(fragments);
|
||||
}
|
||||
}
|
||||
@@ -11,37 +11,21 @@ public interface FileStore {
|
||||
/** Stored file record. */
|
||||
record Stored(String fileId, long size) {}
|
||||
|
||||
/**
|
||||
* Store the given stream and return a generated file id and total bytes written. {@code owner}
|
||||
* may be null to indicate the file has no associated user (anonymous / desktop / async job with
|
||||
* no propagated security context); a non-null value is persisted alongside the data so {@link
|
||||
* #getOwner(String)} can return it later for authorization checks.
|
||||
*/
|
||||
Stored store(InputStream in, String originalName, String owner) throws IOException;
|
||||
|
||||
/** Store with no owner. Equivalent to {@link #store(InputStream, String, String)} with null. */
|
||||
default Stored store(InputStream in, String originalName) throws IOException {
|
||||
return store(in, originalName, null);
|
||||
}
|
||||
/** Store the given stream and return a generated file id and total bytes written. */
|
||||
Stored store(InputStream in, String originalName) throws IOException;
|
||||
|
||||
/**
|
||||
* Store the file at {@code source} and return a generated file id and total bytes written.
|
||||
*
|
||||
* <p>Default implementation opens {@code source} as a stream and delegates to {@link
|
||||
* #store(InputStream, String, String)}. Local-disk implementations should override to use a
|
||||
* direct file-to-file copy ({@code Files.copy(source, dest)} can use {@code sendfile(2)} on
|
||||
* Linux), which avoids the two-memory-copy hit of streaming a disk-backed upload through the
|
||||
* JVM heap.
|
||||
* #store(InputStream, String)}. Local-disk implementations should override to use a direct
|
||||
* file-to-file copy ({@code Files.copy(source, dest)} can use {@code sendfile(2)} on Linux),
|
||||
* which avoids the two-memory-copy hit of streaming a disk-backed upload through the JVM heap.
|
||||
*/
|
||||
default Stored store(Path source, String originalName, String owner) throws IOException {
|
||||
try (InputStream in = Files.newInputStream(source)) {
|
||||
return store(in, originalName, owner);
|
||||
}
|
||||
}
|
||||
|
||||
/** Store with no owner. Equivalent to {@link #store(Path, String, String)} with null. */
|
||||
default Stored store(Path source, String originalName) throws IOException {
|
||||
return store(source, originalName, null);
|
||||
try (InputStream in = Files.newInputStream(source)) {
|
||||
return store(in, originalName);
|
||||
}
|
||||
}
|
||||
|
||||
/** Open the stored file for streaming reads. Caller closes. */
|
||||
@@ -58,12 +42,4 @@ public interface FileStore {
|
||||
|
||||
/** Whether the file id exists in the store. */
|
||||
boolean exists(String fileId);
|
||||
|
||||
/**
|
||||
* Returns the owner identifier recorded at store time, or {@code null} if the file does not
|
||||
* exist or was stored without an owner. Implementations must not throw when the file is missing
|
||||
* or when the owner record is absent; they should return null so callers can treat "no owner"
|
||||
* as a non-authoritative case.
|
||||
*/
|
||||
String getOwner(String fileId) throws IOException;
|
||||
}
|
||||
|
||||
@@ -2,13 +2,7 @@ package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* Token-bucket rate limiting backed by the cluster backplane.
|
||||
*
|
||||
* <p>In-process implementations enforce a per-JVM limit; distributed implementations enforce a
|
||||
* single global limit across every node. Both use a Bucket4j greedy-refill token bucket so the
|
||||
* semantics match across single-node and cluster deployments.
|
||||
*/
|
||||
/** Token-bucket rate limiting backed by the cluster backplane. */
|
||||
public interface RateLimitStore {
|
||||
|
||||
/**
|
||||
|
||||
+23
-100
@@ -3,12 +3,9 @@ package stirling.software.common.cluster.inprocess;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@@ -18,47 +15,33 @@ import stirling.software.common.cluster.FileStore;
|
||||
@Slf4j
|
||||
public class LocalDiskFileStore implements FileStore {
|
||||
|
||||
private static final String OWNER_SUFFIX = ".owner";
|
||||
|
||||
// File ids are generated as random UUIDs; reject anything else so a tainted id can never reach
|
||||
// Files.* APIs (defence in depth on top of the resolve() prefix check, and silences CodeQL's
|
||||
// path-injection finding on the resolveOwner sidecar lookup).
|
||||
private static final Pattern UUID_PATTERN =
|
||||
Pattern.compile(
|
||||
"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
|
||||
|
||||
private final String baseDirPath;
|
||||
// Fixed-size lock stripes so concurrent store/delete on the same (or colliding) fileId
|
||||
// serialise the data-file + owner-sidecar pair as one critical section. Striped (not
|
||||
// per-id) so the map never has to be cleaned up; collisions across unrelated ids are
|
||||
// harmless contention.
|
||||
private static final int LOCK_STRIPES = 64;
|
||||
private final ReentrantLock[] stripes = new ReentrantLock[LOCK_STRIPES];
|
||||
|
||||
public LocalDiskFileStore(String baseDirPath) {
|
||||
this.baseDirPath = baseDirPath;
|
||||
for (int i = 0; i < LOCK_STRIPES; i++) {
|
||||
stripes[i] = new ReentrantLock();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stored store(InputStream in, String originalName, String owner) throws IOException {
|
||||
public Stored store(InputStream in, String originalName) throws IOException {
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
Path filePath = resolve(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
ReentrantLock lock = acquire(fileId);
|
||||
boolean success = false;
|
||||
try {
|
||||
long size = Files.copy(in, filePath);
|
||||
writeOwner(fileId, owner);
|
||||
success = true;
|
||||
return new Stored(fileId, size);
|
||||
} finally {
|
||||
if (!success) {
|
||||
cleanupAfterFailedStore(fileId, filePath);
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to clean up partial file {} after store failure",
|
||||
filePath,
|
||||
cleanupEx);
|
||||
}
|
||||
}
|
||||
release(fileId, lock);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,44 +52,27 @@ public class LocalDiskFileStore implements FileStore {
|
||||
* the source size before copying so the post-copy stat is unnecessary.
|
||||
*/
|
||||
@Override
|
||||
public Stored store(Path source, String originalName, String owner) throws IOException {
|
||||
public Stored store(Path source, String originalName) throws IOException {
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
Path filePath = resolve(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
long size = Files.size(source);
|
||||
ReentrantLock lock = acquire(fileId);
|
||||
boolean success = false;
|
||||
try {
|
||||
Files.copy(source, filePath);
|
||||
writeOwner(fileId, owner);
|
||||
success = true;
|
||||
return new Stored(fileId, size);
|
||||
} finally {
|
||||
if (!success) {
|
||||
cleanupAfterFailedStore(fileId, filePath);
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to clean up partial file {} after store failure",
|
||||
filePath,
|
||||
cleanupEx);
|
||||
}
|
||||
}
|
||||
release(fileId, lock);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeOwner(String fileId, String owner) throws IOException {
|
||||
if (owner == null || owner.isBlank()) {
|
||||
return;
|
||||
}
|
||||
Path ownerPath = resolveOwner(fileId);
|
||||
Files.write(ownerPath, owner.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
private void cleanupAfterFailedStore(String fileId, Path filePath) {
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn("Failed to clean up partial file {} after store failure", filePath, cleanupEx);
|
||||
}
|
||||
try {
|
||||
Files.deleteIfExists(resolveOwner(fileId));
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn("Failed to clean up owner sidecar for {} after store failure", fileId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,26 +101,11 @@ public class LocalDiskFileStore implements FileStore {
|
||||
|
||||
@Override
|
||||
public boolean delete(String fileId) {
|
||||
ReentrantLock lock = acquire(fileId);
|
||||
try {
|
||||
// Data first, owner second: a concurrent retrieve that observes the transient
|
||||
// (data-gone, owner-still-present) window simply fails with IOException; the inverse
|
||||
// order would briefly look like an unowned file and could grant cross-user access.
|
||||
boolean removed;
|
||||
try {
|
||||
removed = Files.deleteIfExists(resolve(fileId));
|
||||
} catch (IOException e) {
|
||||
log.error("Error deleting file with ID: {}", fileId, e);
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
Files.deleteIfExists(resolveOwner(fileId));
|
||||
} catch (IOException e) {
|
||||
log.warn("Error deleting owner sidecar for file ID: {}", fileId, e);
|
||||
}
|
||||
return removed;
|
||||
} finally {
|
||||
release(fileId, lock);
|
||||
return Files.deleteIfExists(resolve(fileId));
|
||||
} catch (IOException e) {
|
||||
log.error("Error deleting file with ID: {}", fileId, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,21 +114,8 @@ public class LocalDiskFileStore implements FileStore {
|
||||
return Files.exists(resolve(fileId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getOwner(String fileId) throws IOException {
|
||||
Path ownerPath = resolveOwner(fileId);
|
||||
if (!Files.exists(ownerPath)) {
|
||||
return null;
|
||||
}
|
||||
byte[] bytes = Files.readAllBytes(ownerPath);
|
||||
if (bytes.length == 0) {
|
||||
return null;
|
||||
}
|
||||
return new String(bytes, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public Path resolve(String fileId) {
|
||||
if (fileId == null || !UUID_PATTERN.matcher(fileId).matches()) {
|
||||
if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) {
|
||||
throw new IllegalArgumentException("Invalid file ID");
|
||||
}
|
||||
Path basePath = Path.of(baseDirPath).normalize().toAbsolutePath();
|
||||
@@ -187,19 +125,4 @@ public class LocalDiskFileStore implements FileStore {
|
||||
}
|
||||
return resolvedPath;
|
||||
}
|
||||
|
||||
private Path resolveOwner(String fileId) {
|
||||
Path data = resolve(fileId);
|
||||
return data.resolveSibling(data.getFileName().toString() + OWNER_SUFFIX);
|
||||
}
|
||||
|
||||
private ReentrantLock acquire(String fileId) {
|
||||
ReentrantLock lock = stripes[(fileId.hashCode() & Integer.MAX_VALUE) % LOCK_STRIPES];
|
||||
lock.lock();
|
||||
return lock;
|
||||
}
|
||||
|
||||
private void release(String fileId, ReentrantLock lock) {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package stirling.software.common.configuration;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Properties;
|
||||
@@ -121,18 +122,18 @@ public class AppConfig {
|
||||
|
||||
@Bean(name = "RunningInDocker")
|
||||
public boolean runningInDocker() {
|
||||
return Files.exists(Path.of("/.dockerenv"));
|
||||
return Files.exists(Paths.get("/.dockerenv"));
|
||||
}
|
||||
|
||||
@Bean(name = "configDirMounted")
|
||||
public boolean isRunningInDockerWithConfig() {
|
||||
Path dockerEnv = Path.of("/.dockerenv");
|
||||
Path dockerEnv = Paths.get("/.dockerenv");
|
||||
// default to true if not docker
|
||||
if (!Files.exists(dockerEnv)) {
|
||||
return true;
|
||||
}
|
||||
Path mountInfo = Path.of("/proc/1/mountinfo");
|
||||
// this should always exist, if not some unknown use case
|
||||
Path mountInfo = Paths.get("/proc/1/mountinfo");
|
||||
// this should always exist, if not some unknown usecase
|
||||
if (!Files.exists(mountInfo)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
+4
-3
@@ -7,6 +7,7 @@ import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.List;
|
||||
|
||||
@@ -26,7 +27,7 @@ public class ConfigInitializer {
|
||||
|
||||
public void ensureConfigExists() throws IOException, URISyntaxException {
|
||||
// 1) If settings file doesn't exist, create from template
|
||||
Path destPath = Path.of(InstallationPathConfig.getSettingsPath());
|
||||
Path destPath = Paths.get(InstallationPathConfig.getSettingsPath());
|
||||
|
||||
boolean settingsFileExists = Files.exists(destPath);
|
||||
|
||||
@@ -38,7 +39,7 @@ public class ConfigInitializer {
|
||||
if (settingsFileExists) {
|
||||
// move settings.yml to settings.yml.{timestamp}.bak
|
||||
Path backupPath =
|
||||
Path.of(
|
||||
Paths.get(
|
||||
InstallationPathConfig.getSettingsPath()
|
||||
+ "."
|
||||
+ System.currentTimeMillis()
|
||||
@@ -95,7 +96,7 @@ public class ConfigInitializer {
|
||||
}
|
||||
|
||||
// 3) Ensure custom settings file exists
|
||||
Path customSettingsPath = Path.of(InstallationPathConfig.getCustomSettingsPath());
|
||||
Path customSettingsPath = Paths.get(InstallationPathConfig.getCustomSettingsPath());
|
||||
if (Files.notExists(customSettingsPath)) {
|
||||
Files.createFile(customSettingsPath);
|
||||
log.info("Created custom_settings file: {}", customSettingsPath);
|
||||
|
||||
+6
-5
@@ -3,6 +3,7 @@ package stirling.software.common.configuration;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.InvalidPathException;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
@@ -200,7 +201,7 @@ public class RuntimePathConfig {
|
||||
|
||||
try {
|
||||
// Normalize to absolute path
|
||||
Path path = Path.of(pathStr.trim()).toAbsolutePath().normalize();
|
||||
Path path = Paths.get(pathStr.trim()).toAbsolutePath().normalize();
|
||||
String normalizedPath = path.toString();
|
||||
|
||||
// Check for duplicates
|
||||
@@ -223,9 +224,9 @@ public class RuntimePathConfig {
|
||||
|
||||
private void detectOverlappingPaths(List<String> paths) {
|
||||
for (int i = 0; i < paths.size(); i++) {
|
||||
Path path1 = Path.of(paths.get(i));
|
||||
Path path1 = Paths.get(paths.get(i));
|
||||
for (int j = i + 1; j < paths.size(); j++) {
|
||||
Path path2 = Path.of(paths.get(j));
|
||||
Path path2 = Paths.get(paths.get(j));
|
||||
|
||||
// Check if one path is a parent of the other
|
||||
if (path1.startsWith(path2)) {
|
||||
@@ -245,10 +246,10 @@ public class RuntimePathConfig {
|
||||
|
||||
private void validatePipelinePaths() {
|
||||
try {
|
||||
Path finishedPath = Path.of(pipelineFinishedFoldersPath).toAbsolutePath().normalize();
|
||||
Path finishedPath = Paths.get(pipelineFinishedFoldersPath).toAbsolutePath().normalize();
|
||||
|
||||
for (String watchedPathStr : pipelineWatchedFoldersPaths) {
|
||||
Path watchedPath = Path.of(watchedPathStr).toAbsolutePath().normalize();
|
||||
Path watchedPath = Paths.get(watchedPathStr).toAbsolutePath().normalize();
|
||||
|
||||
// Check if watched folder is same as finished folder
|
||||
if (watchedPath.equals(finishedPath)) {
|
||||
|
||||
+16
-181
@@ -77,10 +77,8 @@ public class ApplicationProperties {
|
||||
private ProcessExecutor processExecutor = new ProcessExecutor();
|
||||
private PdfEditor pdfEditor = new PdfEditor();
|
||||
private AiEngine aiEngine = new AiEngine();
|
||||
private Mcp mcp = new Mcp();
|
||||
private InternalApi internalApi = new InternalApi();
|
||||
private Cluster cluster = new Cluster();
|
||||
private Policies policies = new Policies();
|
||||
|
||||
@Bean
|
||||
public PropertySource<?> dynamicYamlPropertySource(ConfigurableEnvironment environment)
|
||||
@@ -204,50 +202,6 @@ public class ApplicationProperties {
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Policies {
|
||||
/**
|
||||
* Master switch for the policy + sources subsystem (the PAYG-metered automation surface).
|
||||
*/
|
||||
private boolean enabled = false;
|
||||
|
||||
/**
|
||||
* Absolute directories that policy folder input sources and output sinks may read from or
|
||||
* write to. Empty (the default) disables folder access entirely, so a policy can never be
|
||||
* pointed at an arbitrary server path. Stirling's own config directory is always
|
||||
* off-limits, and folder access is always disabled in SaaS mode regardless of this list.
|
||||
*/
|
||||
private List<String> allowedFolderRoots = new java.util.ArrayList<>();
|
||||
|
||||
/** How often (seconds) the schedule trigger checks for policies whose schedule is due. */
|
||||
private long scheduleSweepSeconds = 60;
|
||||
|
||||
/**
|
||||
* How often (seconds) the folder-watch trigger reconciles its watch registrations and
|
||||
* re-runs every folder-watch policy as a safety net for filesystem events that were missed
|
||||
* (NFS, bind mounts, inotify-queue overflow).
|
||||
*/
|
||||
private long watchReconcileSeconds = 300;
|
||||
|
||||
/**
|
||||
* How long (milliseconds) the folder-watch trigger keeps draining filesystem events after
|
||||
* the first, so a burst from a single file copy coalesces into one run instead of many.
|
||||
*/
|
||||
private long watchQuietPeriodMs = 500;
|
||||
|
||||
/**
|
||||
* SSE emitter timeout (milliseconds) for streamed runs; generous for long multi-step runs.
|
||||
*/
|
||||
private long streamTimeoutMs = 1800000;
|
||||
|
||||
/**
|
||||
* How long (minutes) a finished run's in-memory state is retained before eviction,
|
||||
* mirroring the job-result expiry so rich run state does not outlive the process. Active
|
||||
* and paused runs are kept regardless of age.
|
||||
*/
|
||||
private int runExpiryMinutes = 30;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class PdfEditor {
|
||||
private Cache cache = new Cache();
|
||||
@@ -302,103 +256,6 @@ public class ApplicationProperties {
|
||||
private int longRunningTimeoutSeconds = 600;
|
||||
}
|
||||
|
||||
/**
|
||||
* Model Context Protocol (MCP) server configuration. All keys live under the top-level {@code
|
||||
* mcp.*} prefix. {@link #enabled} defaults to {@code false}: when off, no MCP beans are wired,
|
||||
* no /mcp endpoint exists, and no protected-resource metadata is published.
|
||||
*/
|
||||
@Data
|
||||
public static class Mcp {
|
||||
|
||||
/** Master switch. When {@code false} (default), no MCP beans are wired. */
|
||||
private boolean enabled = false;
|
||||
|
||||
/**
|
||||
* When {@code true} (default), invocations require an OAuth scope: {@code mcp.tools.read}
|
||||
* for read-style operations and {@code mcp.tools.write} for write/destructive ones. When
|
||||
* {@code false}, scope checks are skipped (use only if your IdP issues a single coarse
|
||||
* scope).
|
||||
*/
|
||||
private boolean scopesEnabled = true;
|
||||
|
||||
/** How often to refresh the AI capabilities manifest from the engine. */
|
||||
private int engineCapabilityRefreshMinutes = 5;
|
||||
|
||||
/**
|
||||
* Tool allow-list (operation ids, e.g. {@code compress-pdf}). When non-empty, ONLY these
|
||||
* operations are exposed over MCP; everything else is hidden, undescribable, and
|
||||
* uninvocable - on top of the global endpoint enable/disable config. Empty = allow all.
|
||||
*/
|
||||
private List<String> allowedOperations = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* Tool deny-list (operation ids). Any operation listed here is removed from MCP even if it
|
||||
* would otherwise be allowed. Applied after {@link #allowedOperations}.
|
||||
*/
|
||||
private List<String> blockedOperations = new ArrayList<>();
|
||||
|
||||
/** Max MCP request body size in bytes; inline file uploads ride in the JSON-RPC body. */
|
||||
private long maxRequestBytes = 10L * 1024 * 1024;
|
||||
|
||||
/** Results up to this size return inline as base64; larger ones return a fileId only. */
|
||||
private long maxInlineResponseBytes = 10L * 1024 * 1024;
|
||||
|
||||
private Auth auth = new Auth();
|
||||
|
||||
@Data
|
||||
public static class Auth {
|
||||
/**
|
||||
* Authentication mode for the MCP endpoint. {@code oauth} (default) runs a full OAuth2
|
||||
* resource server (JWT, RFC 8707 audience, RFC 9728 metadata). {@code apikey} accepts a
|
||||
* Stirling per-user API key via the {@code X-API-KEY} header (or {@code Authorization:
|
||||
* Bearer <key>}) and binds the request to that user - the low-friction self-host path,
|
||||
* no external IdP required.
|
||||
*/
|
||||
private String mode = "oauth";
|
||||
|
||||
/** OAuth2 issuer URI, e.g. {@code http://localhost:9000}. Required when MCP is on. */
|
||||
private String issuerUri = "";
|
||||
|
||||
/**
|
||||
* JWKS URI. When blank, derived from the issuer's {@code
|
||||
* /.well-known/openid-configuration} document.
|
||||
*/
|
||||
private String jwksUri = "";
|
||||
|
||||
/**
|
||||
* RFC 8707 resource identifier of THIS MCP server, e.g. {@code
|
||||
* http://localhost:8080/mcp}. Tokens that do not list this id in their {@code aud}
|
||||
* claim are rejected with HTTP 401.
|
||||
*/
|
||||
private String resourceId = "";
|
||||
|
||||
/**
|
||||
* Additional JWT audiences accepted at the MCP endpoint, on top of {@link #resourceId}.
|
||||
* Empty (default) keeps strict RFC 8707 binding. Some IdPs cannot mint
|
||||
* resource-specific audiences - e.g. Supabase's OAuth server always issues {@code
|
||||
* aud=authenticated} - so operators list the audience their IdP actually emits here
|
||||
* (env: {@code MCP_AUTH_ACCEPTEDAUDIENCES}, comma-separated).
|
||||
*/
|
||||
private List<String> acceptedAudiences = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* JWT claim whose value is matched against a provisioned Stirling username. Defaults to
|
||||
* {@code sub}; set to {@code email} or {@code preferred_username} to match how your IdP
|
||||
* maps users to Stirling accounts.
|
||||
*/
|
||||
private String usernameClaim = "sub";
|
||||
|
||||
/**
|
||||
* When {@code true} (default), a validated token is accepted only if its {@link
|
||||
* #usernameClaim} value resolves to an existing, enabled Stirling user account. Tokens
|
||||
* whose subject has no Stirling account (or a disabled one) are rejected with HTTP 403.
|
||||
* Set to {@code false} only if you intentionally want any IdP-valid token to use MCP
|
||||
* without a local account.
|
||||
*/
|
||||
private boolean requireExistingAccount = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cluster backplane configuration. All keys live under the top-level {@code cluster.*} prefix
|
||||
* (e.g. env var {@code CLUSTER_ENABLED}). The master switch is {@link #enabled} and defaults to
|
||||
@@ -519,14 +376,6 @@ public class ApplicationProperties {
|
||||
private String accessibilityStatement;
|
||||
private String cookiePolicy;
|
||||
private String impressum;
|
||||
private LoginAgreement loginAgreement = new LoginAgreement();
|
||||
|
||||
@Data
|
||||
public static class LoginAgreement {
|
||||
private boolean enabled = false;
|
||||
private boolean showInAnonymousMode = true;
|
||||
private String fallbackText = "";
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
@@ -595,7 +444,7 @@ public class ApplicationProperties {
|
||||
public static class SAML2 {
|
||||
private String provider;
|
||||
private Boolean enabled = false;
|
||||
private Boolean autoCreateUser = true;
|
||||
private Boolean autoCreateUser = false;
|
||||
private Boolean blockRegistration = false;
|
||||
private String registrationId = "stirling";
|
||||
|
||||
@@ -672,7 +521,7 @@ public class ApplicationProperties {
|
||||
private String issuer;
|
||||
private String clientId;
|
||||
@ToString.Exclude private String clientSecret;
|
||||
private Boolean autoCreateUser = true;
|
||||
private Boolean autoCreateUser = false;
|
||||
private Boolean blockRegistration = false;
|
||||
private String useAsUsername;
|
||||
private Collection<String> scopes = new ArrayList<>();
|
||||
@@ -743,6 +592,7 @@ public class ApplicationProperties {
|
||||
@Data
|
||||
public static class Jwt {
|
||||
private boolean enableKeystore = true;
|
||||
private boolean enableKeyRotation = false;
|
||||
private boolean enableKeyCleanup = true;
|
||||
|
||||
/**
|
||||
@@ -846,8 +696,8 @@ public class ApplicationProperties {
|
||||
@Data
|
||||
public static class Trust {
|
||||
private boolean serverAsAnchor = true;
|
||||
private boolean useSystemTrust = true;
|
||||
private boolean useMozillaBundle = true;
|
||||
private boolean useSystemTrust = false;
|
||||
private boolean useMozillaBundle = false;
|
||||
private boolean useAATL = false;
|
||||
private boolean useEUTL = false;
|
||||
}
|
||||
@@ -881,8 +731,8 @@ public class ApplicationProperties {
|
||||
public static class System {
|
||||
private String defaultLocale;
|
||||
private boolean googlevisibility;
|
||||
private boolean showUpdate = true;
|
||||
private boolean showUpdateOnlyAdmin = true;
|
||||
private boolean showUpdate;
|
||||
private boolean showUpdateOnlyAdmin;
|
||||
private boolean showSettingsWhenNoLogin = true;
|
||||
private boolean customHTMLFiles;
|
||||
private String tessdataDir;
|
||||
@@ -890,10 +740,10 @@ public class ApplicationProperties {
|
||||
private Boolean enableAnalytics;
|
||||
private Boolean enablePosthog;
|
||||
private Boolean enableScarf;
|
||||
private Boolean enableDesktopInstallSlide = true;
|
||||
private Boolean enableDesktopInstallSlide;
|
||||
private Datasource datasource;
|
||||
private boolean disableSanitize;
|
||||
private int maxDPI = 500;
|
||||
private int maxDPI;
|
||||
private boolean enableUrlToPDF;
|
||||
private Html html = new Html();
|
||||
private CustomPaths customPaths = new CustomPaths();
|
||||
@@ -907,9 +757,8 @@ public class ApplicationProperties {
|
||||
private String frontendUrl; // Frontend URL for invite email links (e.g.
|
||||
|
||||
// 'https://app.example.com'). If not set, falls back to backendUrl.
|
||||
private boolean enableMobileScanner = true; // Enable mobile phone QR code upload feature
|
||||
private boolean enableMobileScanner = false; // Enable mobile phone QR code upload feature
|
||||
private MobileScannerSettings mobileScannerSettings = new MobileScannerSettings();
|
||||
private ServerCertificate serverCertificate = new ServerCertificate();
|
||||
|
||||
@Data
|
||||
public static class MobileScannerSettings {
|
||||
@@ -919,16 +768,6 @@ public class ApplicationProperties {
|
||||
private boolean stretchToFit = false; // Whether to stretch image to fill page
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class ServerCertificate {
|
||||
private boolean enabled =
|
||||
true; // Enable server-side "Sign with Stirling-PDF" certificate
|
||||
private String organizationName = "Stirling PDF Inc";
|
||||
private int validity = 365; // Certificate validity in days
|
||||
private boolean regenerateOnStartup =
|
||||
false; // Generate a new certificate on each startup
|
||||
}
|
||||
|
||||
public boolean isAnalyticsEnabled() {
|
||||
return this.enableAnalytics != null && this.enableAnalytics;
|
||||
}
|
||||
@@ -1013,7 +852,7 @@ public class ApplicationProperties {
|
||||
@Data
|
||||
public static class Sharing {
|
||||
private boolean enabled = false;
|
||||
private boolean linkEnabled = true;
|
||||
private boolean linkEnabled = false;
|
||||
private boolean emailEnabled = false;
|
||||
private int linkExpirationDays = 3;
|
||||
}
|
||||
@@ -1028,10 +867,6 @@ public class ApplicationProperties {
|
||||
@Data
|
||||
public static class Signing {
|
||||
private boolean enabled = false;
|
||||
|
||||
// Signing user-picker scope: 'org' (default) = whole instance, anything else =
|
||||
// caller's team only (fail-closed). The saas profile pins 'team'.
|
||||
private String userListScope = "org";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1187,7 +1022,7 @@ public class ApplicationProperties {
|
||||
|
||||
@Data
|
||||
public static class Metrics {
|
||||
private boolean enabled = true;
|
||||
private boolean enabled;
|
||||
}
|
||||
|
||||
@Data
|
||||
@@ -1239,7 +1074,7 @@ public class ApplicationProperties {
|
||||
private boolean enableInvites = false;
|
||||
private int inviteLinkExpiryHours = 72; // Default: 72 hours (3 days)
|
||||
private String host;
|
||||
private int port = 587;
|
||||
private int port;
|
||||
private String username;
|
||||
@ToString.Exclude private String password;
|
||||
private String from;
|
||||
@@ -1266,10 +1101,10 @@ public class ApplicationProperties {
|
||||
@ToString.Exclude private String botToken;
|
||||
private String botUsername;
|
||||
private String pipelineInboxFolder = "telegram";
|
||||
private Boolean customFolderSuffix = true;
|
||||
private Boolean enableAllowUserIDs = true;
|
||||
private Boolean customFolderSuffix = false;
|
||||
private Boolean enableAllowUserIDs = false;
|
||||
private List<Long> allowUserIDs = new ArrayList<>();
|
||||
private Boolean enableAllowChannelIDs = true;
|
||||
private Boolean enableAllowChannelIDs = false;
|
||||
private List<Long> allowChannelIDs = new ArrayList<>();
|
||||
private long processingTimeoutSeconds = 180;
|
||||
private long pollingIntervalMillis = 2000;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package stirling.software.common.model;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
@@ -23,7 +24,7 @@ public class FileInfo {
|
||||
|
||||
// Converts the file path string to a Path object.
|
||||
public Path getFilePathAsPath() {
|
||||
return Path.of(filePath);
|
||||
return Paths.get(filePath);
|
||||
}
|
||||
|
||||
// Formats the file size into a human-readable string.
|
||||
|
||||
@@ -1,191 +0,0 @@
|
||||
package stirling.software.common.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import stirling.software.jpdfium.text.PageText;
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
final class HeadingDetector {
|
||||
|
||||
private HeadingDetector() {}
|
||||
|
||||
/** A heading is at most this many words; longer lines are treated as body text. */
|
||||
private static final int MAX_HEADING_WORDS = 12;
|
||||
|
||||
/**
|
||||
* Returns the Markdown heading prefix for a line. The decision combines several signals, never
|
||||
* text matching, so a plain line that merely shares text with a heading is never promoted:
|
||||
*
|
||||
* <ul>
|
||||
* <li><b>Size</b> — dominant glyph font size vs. the document body median (primary signal).
|
||||
* Some PDFs encode visual size in the text matrix, so every glyph reports ~1.0; for those
|
||||
* the line height is used as the proxy instead.
|
||||
* <li><b>Brevity</b> — headings are short labels; a line over {@value #MAX_HEADING_WORDS}
|
||||
* words is body text regardless of size.
|
||||
* <li><b>Not a sentence</b> — a line ending in {@code . ! ?} reads as prose, not a heading.
|
||||
* </ul>
|
||||
*
|
||||
* <p>Boldness is deliberately <em>not</em> a heading signal — a bold-but-not-larger line is
|
||||
* emphasis, not a heading (see {@link #isBoldLabel}); promoting it to {@code #}/{@code ##} is
|
||||
* the main source of false-positive headings.
|
||||
*
|
||||
* <ul>
|
||||
* <li>size > baseline * 1.4 → {@code "# "}
|
||||
* <li>size > baseline * 1.2 → {@code "## "}
|
||||
* <li>otherwise → {@code ""}
|
||||
* </ul>
|
||||
*/
|
||||
static String headingPrefix(TextLine line, float medianBodySize, float medianBodyHeight) {
|
||||
String text = line.text().strip();
|
||||
if (text.isEmpty() || wordCount(text) > MAX_HEADING_WORDS || endsLikeSentence(text)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
float dominant = dominantFontSize(line);
|
||||
float value;
|
||||
float baseline;
|
||||
if (dominant > 2f && medianBodySize > 2f) {
|
||||
value = dominant;
|
||||
baseline = medianBodySize;
|
||||
} else {
|
||||
value = line.height();
|
||||
baseline = medianBodyHeight;
|
||||
}
|
||||
if (baseline <= 0f) {
|
||||
return "";
|
||||
}
|
||||
|
||||
float ratio = value / baseline;
|
||||
if (ratio > 1.4f) {
|
||||
return "# ";
|
||||
}
|
||||
if (ratio > 1.2f) {
|
||||
return "## ";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a line should be emphasised as bold (rendered {@code **like this**}) rather than
|
||||
* promoted to a heading: it is bold, short, and not a full sentence. Used for bold labels that
|
||||
* are not large enough to be headings.
|
||||
*/
|
||||
static boolean isBoldLabel(TextLine line) {
|
||||
String text = line.text().strip();
|
||||
if (text.isEmpty() || wordCount(text) > MAX_HEADING_WORDS || endsLikeSentence(text)) {
|
||||
return false;
|
||||
}
|
||||
return isBold(line);
|
||||
}
|
||||
|
||||
private static int wordCount(String text) {
|
||||
return text.split("\\s+").length;
|
||||
}
|
||||
|
||||
private static boolean endsLikeSentence(String text) {
|
||||
char last = text.charAt(text.length() - 1);
|
||||
return last == '.' || last == '!' || last == '?';
|
||||
}
|
||||
|
||||
/** True when the line's dominant font is bold, inferred from PostScript font names. */
|
||||
private static boolean isBold(TextLine line) {
|
||||
Map<String, Integer> counts = new HashMap<>();
|
||||
for (TextWord word : line.words()) {
|
||||
for (TextChar ch : word.chars()) {
|
||||
if (ch.isWhitespace() || ch.isNewline()) {
|
||||
continue;
|
||||
}
|
||||
String name = ch.fontName();
|
||||
if (name != null && !name.isBlank()) {
|
||||
counts.merge(name, 1, Integer::sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
String dominantFont = "";
|
||||
int max = -1;
|
||||
for (Map.Entry<String, Integer> e : counts.entrySet()) {
|
||||
if (e.getValue() > max) {
|
||||
max = e.getValue();
|
||||
dominantFont = e.getKey();
|
||||
}
|
||||
}
|
||||
String lower = dominantFont.toLowerCase(java.util.Locale.ROOT);
|
||||
return lower.contains("bold")
|
||||
|| lower.contains("black")
|
||||
|| lower.contains("heavy")
|
||||
|| lower.contains("semibold");
|
||||
}
|
||||
|
||||
/** Computes the median glyph font size across all pages. */
|
||||
static float medianFontSize(List<PageText> allPages) {
|
||||
List<Float> sizes = new ArrayList<>();
|
||||
for (PageText page : allPages) {
|
||||
for (TextChar ch : page.chars()) {
|
||||
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
|
||||
sizes.add(ch.fontSize());
|
||||
}
|
||||
}
|
||||
}
|
||||
return median(sizes, 12f);
|
||||
}
|
||||
|
||||
/** Computes the median TextLine height across all pages. Used when font size is degenerate. */
|
||||
static float medianLineHeight(List<PageText> allPages) {
|
||||
List<Float> heights = new ArrayList<>();
|
||||
for (PageText page : allPages) {
|
||||
for (TextLine line : page.lines()) {
|
||||
if (line.height() > 0f && !line.text().isBlank()) {
|
||||
heights.add(line.height());
|
||||
}
|
||||
}
|
||||
}
|
||||
return median(heights, 12f);
|
||||
}
|
||||
|
||||
private static float median(List<Float> values, float fallback) {
|
||||
if (values.isEmpty()) {
|
||||
return fallback;
|
||||
}
|
||||
Collections.sort(values);
|
||||
int mid = values.size() / 2;
|
||||
if (values.size() % 2 == 0) {
|
||||
return (values.get(mid - 1) + values.get(mid)) / 2f;
|
||||
}
|
||||
return values.get(mid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the font size that appears most often (by character count) in the given line. Ties
|
||||
* are broken in favour of the larger size.
|
||||
*/
|
||||
private static float dominantFontSize(TextLine line) {
|
||||
Map<Float, Integer> counts = new HashMap<>();
|
||||
for (TextWord word : line.words()) {
|
||||
for (TextChar ch : word.chars()) {
|
||||
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
|
||||
counts.merge(ch.fontSize(), 1, Integer::sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (counts.isEmpty()) {
|
||||
return 0f;
|
||||
}
|
||||
float dominant = 0f;
|
||||
int maxCount = -1;
|
||||
for (Map.Entry<Float, Integer> entry : counts.entrySet()) {
|
||||
int count = entry.getValue();
|
||||
float size = entry.getKey();
|
||||
if (count > maxCount || (count == maxCount && size > dominant)) {
|
||||
maxCount = count;
|
||||
dominant = size;
|
||||
}
|
||||
}
|
||||
return dominant;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,82 +0,0 @@
|
||||
package stirling.software.common.pdf;
|
||||
|
||||
import stirling.software.jpdfium.text.Table;
|
||||
|
||||
final class TableRenderer {
|
||||
private TableRenderer() {}
|
||||
|
||||
/** Renders a Table as a GitHub-Flavoured Markdown table string. */
|
||||
static String render(Table table) {
|
||||
if (table.rowCount() == 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
String[][] grid = table.asGrid();
|
||||
|
||||
if (table.rowCount() < 2) {
|
||||
// No separator row possible — return plain lines
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int c = 0; c < grid[0].length; c++) {
|
||||
if (c > 0) sb.append('\n');
|
||||
sb.append(escape(grid[0][c].trim()));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
int cols = grid[0].length;
|
||||
|
||||
// Compute column widths: max(3, max content length across all rows)
|
||||
int[] widths = new int[cols];
|
||||
for (int c = 0; c < cols; c++) {
|
||||
widths[c] = 3;
|
||||
}
|
||||
for (String[] row : grid) {
|
||||
for (int c = 0; c < cols; c++) {
|
||||
String cell = c < row.length ? row[c].trim() : "";
|
||||
widths[c] = Math.max(widths[c], escape(cell).length());
|
||||
}
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
// Header row
|
||||
sb.append(buildRow(grid[0], widths, cols));
|
||||
sb.append('\n');
|
||||
|
||||
// Separator row
|
||||
sb.append('|');
|
||||
for (int c = 0; c < cols; c++) {
|
||||
sb.append('-').append("-".repeat(widths[c])).append('-').append('|');
|
||||
}
|
||||
sb.append('\n');
|
||||
|
||||
// Data rows
|
||||
for (int r = 1; r < grid.length; r++) {
|
||||
sb.append(buildRow(grid[r], widths, cols));
|
||||
if (r < grid.length - 1) {
|
||||
sb.append('\n');
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static String buildRow(String[] row, int[] widths, int cols) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append('|');
|
||||
for (int c = 0; c < cols; c++) {
|
||||
String cell = c < row.length ? escape(row[c].trim()) : "";
|
||||
sb.append(' ').append(padRight(cell, widths[c])).append(' ').append('|');
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static String escape(String cell) {
|
||||
return cell.replace("|", "\\|");
|
||||
}
|
||||
|
||||
private static String padRight(String s, int width) {
|
||||
if (s.length() >= width) return s;
|
||||
return s + " ".repeat(width - s.length());
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.PipedInputStream;
|
||||
import java.io.PipedOutputStream;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
@@ -18,7 +17,6 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.FileStore;
|
||||
import stirling.software.common.util.JobContext;
|
||||
|
||||
/**
|
||||
* Service for storing and retrieving files with unique file IDs. Used by the AutoJobPostMapping
|
||||
@@ -34,10 +32,8 @@ public class FileStorage {
|
||||
|
||||
private final FileOrUploadService fileOrUploadService;
|
||||
private final FileStore fileStore;
|
||||
private final Optional<JobOwnershipService> jobOwnershipService;
|
||||
|
||||
public String storeFile(MultipartFile file) throws IOException {
|
||||
String owner = resolveOwner();
|
||||
// Fast path: when Spring buffered the multipart to disk (typical for large uploads), the
|
||||
// backing Resource exposes a real File. Hand the Path to the FileStore so it can do a
|
||||
// file-to-file copy (Linux sendfile, no copy through Java heap) rather than streaming
|
||||
@@ -52,7 +48,7 @@ public class FileStorage {
|
||||
if (res != null && res.isFile()) {
|
||||
try {
|
||||
FileStore.Stored stored =
|
||||
fileStore.store(res.getFile().toPath(), file.getOriginalFilename(), owner);
|
||||
fileStore.store(res.getFile().toPath(), file.getOriginalFilename());
|
||||
log.debug("Stored file with ID: {} (fast path)", stored.fileId());
|
||||
return stored.fileId();
|
||||
} catch (IOException ex) {
|
||||
@@ -61,45 +57,40 @@ public class FileStorage {
|
||||
}
|
||||
}
|
||||
try (InputStream in = file.getInputStream()) {
|
||||
FileStore.Stored stored = fileStore.store(in, file.getOriginalFilename(), owner);
|
||||
FileStore.Stored stored = fileStore.store(in, file.getOriginalFilename());
|
||||
log.debug("Stored file with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
}
|
||||
}
|
||||
|
||||
public String storeBytes(byte[] bytes, String originalName) throws IOException {
|
||||
FileStore.Stored stored =
|
||||
fileStore.store(new ByteArrayInputStream(bytes), originalName, resolveOwner());
|
||||
FileStore.Stored stored = fileStore.store(new ByteArrayInputStream(bytes), originalName);
|
||||
log.debug("Stored byte array with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
}
|
||||
|
||||
public MultipartFile retrieveFile(String fileId) throws IOException {
|
||||
enforceOwnership(fileId);
|
||||
byte[] fileData = fileStore.retrieveBytes(fileId);
|
||||
return fileOrUploadService.toMockMultipartFile(fileId, fileData);
|
||||
}
|
||||
|
||||
public byte[] retrieveBytes(String fileId) throws IOException {
|
||||
enforceOwnership(fileId);
|
||||
return fileStore.retrieveBytes(fileId);
|
||||
}
|
||||
|
||||
public InputStream retrieveInputStream(String fileId) throws IOException {
|
||||
enforceOwnership(fileId);
|
||||
return fileStore.retrieve(fileId);
|
||||
}
|
||||
|
||||
public StoredFile storeInputStream(InputStream inputStream, String originalName)
|
||||
throws IOException {
|
||||
FileStore.Stored stored = fileStore.store(inputStream, originalName, resolveOwner());
|
||||
FileStore.Stored stored = fileStore.store(inputStream, originalName);
|
||||
log.debug("Stored input stream with ID: {}", stored.fileId());
|
||||
return new StoredFile(stored.fileId(), stored.size());
|
||||
}
|
||||
|
||||
public String storeFromStreamingBody(StreamingResponseBody body, String originalName)
|
||||
throws IOException {
|
||||
String owner = resolveOwner();
|
||||
// Hold Throwable not IOException: an unchecked failure (NPE, IllegalState, OOM, etc.)
|
||||
// from the body writer would otherwise close the pipe with EOF and the consumer would
|
||||
// return a truncated file with no error surfaced to the caller.
|
||||
@@ -124,7 +115,7 @@ public class FileStorage {
|
||||
}
|
||||
}
|
||||
});
|
||||
FileStore.Stored stored = fileStore.store(in, originalName, owner);
|
||||
FileStore.Stored stored = fileStore.store(in, originalName);
|
||||
Throwable writerErr = bodyError.get();
|
||||
if (writerErr != null) {
|
||||
// Body failed mid-write: the FileStore persisted a truncated entry.
|
||||
@@ -168,62 +159,21 @@ public class FileStorage {
|
||||
|
||||
public String storeFromResource(Resource resource, String originalName) throws IOException {
|
||||
try (InputStream in = resource.getInputStream()) {
|
||||
FileStore.Stored stored = fileStore.store(in, originalName, resolveOwner());
|
||||
FileStore.Stored stored = fileStore.store(in, originalName);
|
||||
log.debug("Stored Resource with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
}
|
||||
}
|
||||
|
||||
public boolean deleteFile(String fileId) {
|
||||
enforceOwnership(fileId);
|
||||
return fileStore.delete(fileId);
|
||||
}
|
||||
|
||||
public boolean fileExists(String fileId) {
|
||||
enforceOwnership(fileId);
|
||||
return fileStore.exists(fileId);
|
||||
}
|
||||
|
||||
public long getFileSize(String fileId) throws IOException {
|
||||
enforceOwnership(fileId);
|
||||
return fileStore.size(fileId);
|
||||
}
|
||||
|
||||
private String resolveOwner() {
|
||||
String propagated = JobContext.getOwner();
|
||||
if (propagated != null) {
|
||||
return propagated;
|
||||
}
|
||||
return jobOwnershipService.flatMap(JobOwnershipService::getCurrentUserId).orElse(null);
|
||||
}
|
||||
|
||||
private void enforceOwnership(String fileId) {
|
||||
if (jobOwnershipService.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Optional<String> currentUser = jobOwnershipService.get().getCurrentUserId();
|
||||
if (currentUser.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String owner;
|
||||
try {
|
||||
owner = fileStore.getOwner(fileId);
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to read owner for file {}: {}", fileId, e.getMessage());
|
||||
throw new SecurityException(
|
||||
"Access denied: could not verify ownership of the requested file");
|
||||
}
|
||||
if (owner == null) {
|
||||
return;
|
||||
}
|
||||
if (!owner.equals(currentUser.get())) {
|
||||
log.warn(
|
||||
"Access denied: user {} attempted to access file {} owned by {}",
|
||||
currentUser.get(),
|
||||
fileId,
|
||||
owner);
|
||||
throw new SecurityException(
|
||||
"Access denied: you do not have permission to access this file");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,16 +50,6 @@ public class InternalApiClient {
|
||||
"^/api/v1/(general|misc|security|convert|filter)(/[A-Za-z0-9_-]+)+$"
|
||||
+ "|^/api/v1/ai/tools(/[A-Za-z0-9_-]+)+$");
|
||||
|
||||
/**
|
||||
* Marker propagated on every internal sub-step dispatch so the saas PAYG interceptor classifies
|
||||
* the call as {@code BillingCategory.AUTOMATION}. By construction every {@link
|
||||
* InternalApiClient#post} caller is an automation surface (pipeline executor, AI workflow,
|
||||
* policy runner) running a child tool inside a parent automation flow — see the saas {@code
|
||||
* PaygChargeInterceptor.determineCategory} precedence chain, where this header dominates any
|
||||
* per-tool {@code @RequiresFeature} annotation.
|
||||
*/
|
||||
public static final String AUTOMATION_HEADER = "X-Stirling-Automation";
|
||||
|
||||
private final ServletContext servletContext;
|
||||
private final UserServiceInterface userService;
|
||||
private final TempFileManager tempFileManager;
|
||||
@@ -106,23 +96,7 @@ public class InternalApiClient {
|
||||
if (apiKey != null && !apiKey.isEmpty()) {
|
||||
headers.add("X-API-KEY", apiKey);
|
||||
}
|
||||
// Tag the sub-step as automation so PAYG bills it under AUTOMATION regardless of which
|
||||
// tool-level @RequiresFeature annotation the dispatched controller carries (e.g. an AI-OCR
|
||||
// step inside a policy run must bill as AUTOMATION, not AI). Set unconditionally because
|
||||
// every caller of this dispatcher is an automation surface by design.
|
||||
headers.add(AUTOMATION_HEADER, "true");
|
||||
|
||||
// A no-file ai/tools call (e.g. create-pdf-from-html-agent) sends only string params, so
|
||||
// without this RestTemplate would use urlencoded instead of the multipart the controller
|
||||
// expects. File-bearing calls get the right multipart content-type from RestTemplate.
|
||||
boolean isAiTool = endpointPath.startsWith("/api/v1/ai/tools/");
|
||||
boolean hasFilePart =
|
||||
body.values().stream()
|
||||
.flatMap(java.util.List::stream)
|
||||
.anyMatch(v -> v instanceof Resource);
|
||||
if (isAiTool && !hasFilePart) {
|
||||
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
|
||||
}
|
||||
HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
|
||||
RequestCallback requestCallback = restTemplate.httpEntityCallback(entity, Resource.class);
|
||||
|
||||
|
||||
+132
-10
@@ -57,43 +57,85 @@ public class JobExecutorService {
|
||||
this.resourceMonitor = resourceMonitor;
|
||||
this.jobQueue = jobQueue;
|
||||
|
||||
// Parse session timeout and calculate effective timeout once during initialization
|
||||
long sessionTimeoutMs = parseSessionTimeout(sessionTimeout);
|
||||
this.effectiveTimeoutMs = Math.min(asyncRequestTimeoutMs, sessionTimeoutMs);
|
||||
log.debug(
|
||||
"Job executor configured with effective timeout of {} ms", this.effectiveTimeoutMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a job either asynchronously or synchronously
|
||||
*
|
||||
* @param async Whether to run the job asynchronously
|
||||
* @param work The work to be done
|
||||
* @return The response
|
||||
*/
|
||||
public ResponseEntity<?> runJobGeneric(boolean async, Supplier<Object> work) {
|
||||
return runJobGeneric(async, work, -1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a job either asynchronously or synchronously with a custom timeout
|
||||
*
|
||||
* @param async Whether to run the job asynchronously
|
||||
* @param work The work to be done
|
||||
* @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default
|
||||
* @return The response
|
||||
*/
|
||||
public ResponseEntity<?> runJobGeneric(
|
||||
boolean async, Supplier<Object> work, long customTimeoutMs) {
|
||||
return runJobGeneric(async, work, customTimeoutMs, false, 50);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a job either asynchronously or synchronously with custom parameters
|
||||
*
|
||||
* @param async Whether to run the job asynchronously
|
||||
* @param work The work to be done
|
||||
* @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default
|
||||
* @param queueable Whether this job can be queued when system resources are limited
|
||||
* @param resourceWeight The resource weight of this job (1-100)
|
||||
* @return The response
|
||||
*/
|
||||
public ResponseEntity<?> runJobGeneric(
|
||||
boolean async,
|
||||
Supplier<Object> work,
|
||||
long customTimeoutMs,
|
||||
boolean queueable,
|
||||
int resourceWeight) {
|
||||
// Generate base UUID
|
||||
String baseJobId = UUID.randomUUID().toString();
|
||||
|
||||
// Scope job to authenticated user if security is enabled
|
||||
String scopedJobKey = getScopedJobKey(baseJobId);
|
||||
|
||||
log.debug("Generated jobId: {} (base: {})", scopedJobKey, baseJobId);
|
||||
|
||||
// Store the scoped job ID in the request for potential use by other components
|
||||
if (request != null) {
|
||||
request.setAttribute("jobId", scopedJobKey);
|
||||
|
||||
// Also track this job ID in the user's session for authorization purposes
|
||||
// This ensures users can only cancel their own jobs
|
||||
if (request.getSession() != null) {
|
||||
@SuppressWarnings("unchecked")
|
||||
java.util.Set<String> userJobIds =
|
||||
(java.util.Set<String>) request.getSession().getAttribute("userJobIds");
|
||||
|
||||
if (userJobIds == null) {
|
||||
userJobIds = new java.util.concurrent.ConcurrentSkipListSet<>();
|
||||
request.getSession().setAttribute("userJobIds", userJobIds);
|
||||
}
|
||||
|
||||
userJobIds.add(scopedJobKey);
|
||||
log.debug("Added scoped job ID {} to user session", scopedJobKey);
|
||||
}
|
||||
}
|
||||
|
||||
String jobId = scopedJobKey;
|
||||
|
||||
final String jobOwner =
|
||||
jobOwnershipService != null
|
||||
? jobOwnershipService.getCurrentUserId().orElse(null)
|
||||
: null;
|
||||
|
||||
// Determine which timeout to use
|
||||
long timeoutToUse = customTimeoutMs > 0 ? customTimeoutMs : effectiveTimeoutMs;
|
||||
|
||||
log.debug(
|
||||
@@ -104,6 +146,7 @@ public class JobExecutorService {
|
||||
queueable,
|
||||
resourceWeight);
|
||||
|
||||
// Check if we need to queue this job based on resource availability
|
||||
boolean shouldQueue =
|
||||
queueable
|
||||
&& async
|
||||
@@ -111,6 +154,7 @@ public class JobExecutorService {
|
||||
resourceMonitor.shouldQueueJob(resourceWeight);
|
||||
|
||||
if (shouldQueue) {
|
||||
// Queue the job instead of executing immediately
|
||||
log.debug(
|
||||
"Queueing job {} due to resource constraints (weight: {})",
|
||||
jobId,
|
||||
@@ -118,13 +162,18 @@ public class JobExecutorService {
|
||||
|
||||
taskManager.createTask(jobId);
|
||||
|
||||
// Create a specialized wrapper that updates the TaskManager
|
||||
final String capturedJobIdForQueue = jobId;
|
||||
Supplier<Object> wrappedWork =
|
||||
() -> {
|
||||
try {
|
||||
// Set jobId in ThreadLocal context for the queued job
|
||||
stirling.software.common.util.JobContext.setJobId(
|
||||
capturedJobIdForQueue);
|
||||
stirling.software.common.util.JobContext.setOwner(jobOwner);
|
||||
log.debug(
|
||||
"Set jobId {} in JobContext for queued job execution",
|
||||
capturedJobIdForQueue);
|
||||
|
||||
Object result = work.get();
|
||||
processJobResult(capturedJobIdForQueue, result);
|
||||
return result;
|
||||
@@ -137,17 +186,21 @@ public class JobExecutorService {
|
||||
taskManager.setError(capturedJobIdForQueue, e.getMessage());
|
||||
throw e;
|
||||
} finally {
|
||||
// Clean up ThreadLocal to avoid memory leaks
|
||||
stirling.software.common.util.JobContext.clear();
|
||||
}
|
||||
};
|
||||
|
||||
// Queue the job and get the future
|
||||
CompletableFuture<ResponseEntity<?>> future =
|
||||
jobQueue.queueJob(jobId, resourceWeight, wrappedWork, timeoutToUse);
|
||||
|
||||
// Return immediately with job ID
|
||||
return ResponseEntity.ok().body(new JobResponse<>(true, jobId, null));
|
||||
} else if (async) {
|
||||
taskManager.createTask(jobId);
|
||||
|
||||
// Capture the jobId for the async thread
|
||||
final String capturedJobId = jobId;
|
||||
|
||||
executor.execute(
|
||||
@@ -158,8 +211,13 @@ public class JobExecutorService {
|
||||
capturedJobId,
|
||||
timeoutToUse);
|
||||
|
||||
// Set jobId in ThreadLocal context for the async thread
|
||||
stirling.software.common.util.JobContext.setJobId(capturedJobId);
|
||||
stirling.software.common.util.JobContext.setOwner(jobOwner);
|
||||
log.debug(
|
||||
"Set jobId {} in JobContext for async execution",
|
||||
capturedJobId);
|
||||
|
||||
// Execute with timeout
|
||||
Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
|
||||
processJobResult(capturedJobId, result);
|
||||
} catch (TimeoutException te) {
|
||||
@@ -169,6 +227,7 @@ public class JobExecutorService {
|
||||
log.error("Error executing job {}: {}", jobId, e.getMessage(), e);
|
||||
taskManager.setError(jobId, e.getMessage());
|
||||
} finally {
|
||||
// Clean up ThreadLocal to avoid memory leaks
|
||||
stirling.software.common.util.JobContext.clear();
|
||||
}
|
||||
});
|
||||
@@ -178,19 +237,27 @@ public class JobExecutorService {
|
||||
try {
|
||||
log.debug("Running sync job with timeout {} ms", timeoutToUse);
|
||||
|
||||
// Make jobId available to downstream components on the worker thread
|
||||
stirling.software.common.util.JobContext.setJobId(jobId);
|
||||
log.debug("Set jobId {} in JobContext for sync execution", jobId);
|
||||
|
||||
// Execute with timeout
|
||||
Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
|
||||
|
||||
// If the result is already a ResponseEntity, return it directly
|
||||
if (result instanceof ResponseEntity) {
|
||||
return (ResponseEntity<?>) result;
|
||||
}
|
||||
|
||||
// Process different result types
|
||||
return handleResultForSyncJob(result);
|
||||
} catch (TimeoutException te) {
|
||||
log.error("Synchronous job timed out after {} ms", timeoutToUse);
|
||||
return ResponseEntity.internalServerError()
|
||||
.body(Map.of("error", "Job timed out after " + timeoutToUse + " ms"));
|
||||
} catch (RuntimeException e) {
|
||||
// Check if this is a typed exception that should be handled by
|
||||
// GlobalExceptionHandler (either directly or wrapped)
|
||||
Throwable cause = e.getCause();
|
||||
if (e instanceof IllegalArgumentException
|
||||
|| cause
|
||||
@@ -200,13 +267,16 @@ public class JobExecutorService {
|
||||
instanceof
|
||||
stirling.software.common.util.ExceptionUtils
|
||||
.BaseValidationException) {
|
||||
// Rethrow so GlobalExceptionHandler can handle with proper HTTP status codes
|
||||
throw e;
|
||||
}
|
||||
// Handle other RuntimeExceptions as generic errors
|
||||
log.error("Error executing synchronous job: {}", e.getMessage(), e);
|
||||
return ResponseEntity.internalServerError()
|
||||
.body(Map.of("error", "Job failed: " + e.getMessage()));
|
||||
} catch (Exception e) {
|
||||
log.error("Error executing synchronous job: {}", e.getMessage(), e);
|
||||
// Construct a JSON error response
|
||||
return ResponseEntity.internalServerError()
|
||||
.body(Map.of("error", "Job failed: " + e.getMessage()));
|
||||
} finally {
|
||||
@@ -215,13 +285,23 @@ public class JobExecutorService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the result of an asynchronous job
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @param result The result
|
||||
*/
|
||||
private void processJobResult(String jobId, Object result) {
|
||||
try {
|
||||
if (result instanceof byte[]) {
|
||||
// Store byte array directly to disk to avoid double memory consumption
|
||||
String fileId = fileStorage.storeBytes((byte[]) result, "result.pdf");
|
||||
taskManager.setFileResult(
|
||||
jobId, fileId, "result.pdf", MediaType.APPLICATION_PDF_VALUE);
|
||||
log.debug("Stored byte[] result with fileId: {}", fileId);
|
||||
|
||||
// Let the byte array get collected naturally in the next GC cycle
|
||||
// We don't need to force System.gc() which can be harmful
|
||||
} else if (result instanceof ResponseEntity) {
|
||||
ResponseEntity<?> response = (ResponseEntity<?>) result;
|
||||
Object body = response.getBody();
|
||||
@@ -250,13 +330,16 @@ public class JobExecutorService {
|
||||
taskManager.setFileResult(jobId, fileId, filename, contentType);
|
||||
log.debug("Stored ResponseEntity<Resource> result with fileId: {}", fileId);
|
||||
} else {
|
||||
// Check if the response body contains a fileId
|
||||
if (body != null && body.toString().contains("fileId")) {
|
||||
try {
|
||||
// Try to extract fileId using reflection
|
||||
java.lang.reflect.Method getFileId =
|
||||
body.getClass().getMethod("getFileId");
|
||||
String fileId = (String) getFileId.invoke(body);
|
||||
|
||||
if (fileId != null && !fileId.isEmpty()) {
|
||||
// Try to get filename and content type
|
||||
String filename = "result.pdf";
|
||||
String contentType = MediaType.APPLICATION_PDF_VALUE;
|
||||
|
||||
@@ -296,6 +379,7 @@ public class JobExecutorService {
|
||||
}
|
||||
}
|
||||
|
||||
// Store generic result
|
||||
taskManager.setResult(jobId, body);
|
||||
}
|
||||
} else if (result instanceof MultipartFile file) {
|
||||
@@ -304,13 +388,16 @@ public class JobExecutorService {
|
||||
jobId, fileId, file.getOriginalFilename(), file.getContentType());
|
||||
log.debug("Stored MultipartFile result with fileId: {}", fileId);
|
||||
} else {
|
||||
// Check if result has a fileId field
|
||||
if (result != null) {
|
||||
try {
|
||||
// Try to extract fileId using reflection
|
||||
java.lang.reflect.Method getFileId =
|
||||
result.getClass().getMethod("getFileId");
|
||||
String fileId = (String) getFileId.invoke(result);
|
||||
|
||||
if (fileId != null && !fileId.isEmpty()) {
|
||||
// Try to get filename and content type
|
||||
String filename = "result.pdf";
|
||||
String contentType = MediaType.APPLICATION_PDF_VALUE;
|
||||
|
||||
@@ -348,6 +435,7 @@ public class JobExecutorService {
|
||||
}
|
||||
}
|
||||
|
||||
// Default case: store the result as is
|
||||
taskManager.setResult(jobId, result);
|
||||
}
|
||||
|
||||
@@ -358,8 +446,16 @@ public class JobExecutorService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle different result types for synchronous jobs
|
||||
*
|
||||
* @param result The result object
|
||||
* @return The appropriate ResponseEntity
|
||||
* @throws IOException If there is an error processing the result
|
||||
*/
|
||||
private ResponseEntity<?> handleResultForSyncJob(Object result) throws IOException {
|
||||
if (result instanceof byte[]) {
|
||||
// Return byte array as PDF
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.APPLICATION_PDF)
|
||||
.header(
|
||||
@@ -367,6 +463,7 @@ public class JobExecutorService {
|
||||
"form-data; name=\"attachment\"; filename=\"result.pdf\"")
|
||||
.body(result);
|
||||
} else if (result instanceof MultipartFile file) {
|
||||
// Return MultipartFile content
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.parseMediaType(file.getContentType()))
|
||||
.header(
|
||||
@@ -376,6 +473,7 @@ public class JobExecutorService {
|
||||
+ "\"")
|
||||
.body(file.getBytes());
|
||||
} else {
|
||||
// Default case: return as JSON
|
||||
return ResponseEntity.ok(result);
|
||||
}
|
||||
}
|
||||
@@ -395,9 +493,15 @@ public class JobExecutorService {
|
||||
return mediaType != null ? mediaType.toString() : MediaType.APPLICATION_PDF_VALUE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse session timeout string (e.g., "30m", "1h") to milliseconds
|
||||
*
|
||||
* @param timeout The timeout string
|
||||
* @return The timeout in milliseconds
|
||||
*/
|
||||
private long parseSessionTimeout(String timeout) {
|
||||
if (timeout == null || timeout.isEmpty()) {
|
||||
return 30 * 60 * 1000;
|
||||
return 30 * 60 * 1000; // Default: 30 minutes
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -419,16 +523,27 @@ public class JobExecutorService {
|
||||
case "m" -> (long) (numericValue * 60 * 1000);
|
||||
case "h" -> (long) (numericValue * 60 * 60 * 1000);
|
||||
case "d" -> (long) (numericValue * 24 * 60 * 60 * 1000);
|
||||
default -> (long) (numericValue * 60 * 1000);
|
||||
default -> (long) (numericValue * 60 * 1000); // Default to minutes
|
||||
};
|
||||
} catch (Exception e) {
|
||||
log.warn("Could not parse session timeout '{}', using default", timeout);
|
||||
return 30 * 60 * 1000;
|
||||
return 30 * 60 * 1000; // Default: 30 minutes
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a supplier with a timeout
|
||||
*
|
||||
* @param supplier The supplier to execute
|
||||
* @param timeoutMs The timeout in milliseconds
|
||||
* @return The result from the supplier
|
||||
* @throws TimeoutException If the execution times out
|
||||
* @throws Exception If the supplier throws an exception
|
||||
*/
|
||||
private <T> T executeWithTimeout(Supplier<T> supplier, long timeoutMs)
|
||||
throws TimeoutException, Exception {
|
||||
// Use the same executor as other async jobs for consistency
|
||||
// This ensures all operations run on the same thread pool
|
||||
String currentJobId = stirling.software.common.util.JobContext.getJobId();
|
||||
|
||||
java.util.concurrent.CompletableFuture<T> future =
|
||||
@@ -462,10 +577,17 @@ public class JobExecutorService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a scoped job key that includes user ownership when security is enabled.
|
||||
*
|
||||
* @param baseJobId the base job identifier
|
||||
* @return scoped job key, or just baseJobId if no ownership service available
|
||||
*/
|
||||
private String getScopedJobKey(String baseJobId) {
|
||||
if (jobOwnershipService != null) {
|
||||
return jobOwnershipService.createScopedJobKey(baseJobId);
|
||||
}
|
||||
// Security disabled, return unsecured job key
|
||||
return baseJobId;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,204 +0,0 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.AtomicMoveNotSupportedException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.LinkOption;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.configuration.InstallationPathConfig;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
|
||||
// Resolves login agreement text from customFiles/disclaimer/<locale>.md (read live);
|
||||
// enable/visibility come from the legal.loginAgreement settings.
|
||||
@Service
|
||||
@Slf4j
|
||||
public class LoginAgreementService {
|
||||
|
||||
// Locale codes only: rejects path separators and dots so the value can never escape the
|
||||
// disclaimer directory. Matches e.g. en, en-GB, fr-FR, zh-Hant, pt-BR.
|
||||
private static final Pattern LOCALE_PATTERN =
|
||||
Pattern.compile("^[A-Za-z]{2,3}([_-][A-Za-z0-9]{2,8})*$");
|
||||
|
||||
// BCP-47 tags are well under this; the cap also prevents the regex's repetition group
|
||||
// from recursing far enough to overflow the stack on a hostile over-length input.
|
||||
private static final int MAX_LOCALE_LENGTH = 35;
|
||||
|
||||
// Disclaimers are short markdown; cap the read so an oversized file can't be loaded
|
||||
// wholesale into heap on every public request.
|
||||
private static final long MAX_FILE_BYTES = 256 * 1024;
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
public LoginAgreementService(ApplicationProperties applicationProperties) {
|
||||
this.applicationProperties = applicationProperties;
|
||||
}
|
||||
|
||||
public boolean isEnabled() {
|
||||
return config().isEnabled();
|
||||
}
|
||||
|
||||
public boolean isShowInAnonymousMode() {
|
||||
return config().isShowInAnonymousMode();
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the markdown to show for the requested language, falling back through the base
|
||||
* language, the configured default locale (and its base), then the configured fallbackText.
|
||||
* Returns an empty string when nothing is configured.
|
||||
*/
|
||||
public String resolveContent(String requestedLang) {
|
||||
List<String> candidates = new ArrayList<>();
|
||||
addLocaleCandidates(candidates, requestedLang);
|
||||
addLocaleCandidates(candidates, applicationProperties.getSystem().getDefaultLocale());
|
||||
|
||||
for (String candidate : candidates) {
|
||||
String content = readFileIfExists(candidate);
|
||||
if (content != null && !content.isBlank()) {
|
||||
return content;
|
||||
}
|
||||
}
|
||||
|
||||
String fallback = config().getFallbackText();
|
||||
return fallback == null ? "" : fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Admin read of a single locale's raw file. Returns null for an invalid locale, "" if absent.
|
||||
*/
|
||||
public String readRawForLocale(String locale) {
|
||||
if (!isValidLocale(locale)) {
|
||||
return null;
|
||||
}
|
||||
String content = readFileIfExists(locale);
|
||||
return content == null ? "" : content;
|
||||
}
|
||||
|
||||
/** Admin write. Blank content deletes the file so it falls back cleanly. */
|
||||
public void writeForLocale(String locale, String content) throws IOException {
|
||||
Path file = resolveLocaleFile(locale);
|
||||
if (file == null) {
|
||||
throw new IllegalArgumentException("Invalid locale: " + locale);
|
||||
}
|
||||
if (content == null || content.isBlank()) {
|
||||
Files.deleteIfExists(file);
|
||||
return;
|
||||
}
|
||||
Files.createDirectories(file.getParent());
|
||||
// Write to a sibling temp file then atomically swap, so a concurrent reader (the public
|
||||
// /login-disclaimer fetch is lockless) never observes a truncated/partial file.
|
||||
Path tmp = Files.createTempFile(file.getParent(), "disclaimer", ".md.tmp");
|
||||
try {
|
||||
Files.writeString(tmp, content, StandardCharsets.UTF_8);
|
||||
try {
|
||||
Files.move(
|
||||
tmp,
|
||||
file,
|
||||
StandardCopyOption.ATOMIC_MOVE,
|
||||
StandardCopyOption.REPLACE_EXISTING);
|
||||
} catch (AtomicMoveNotSupportedException e) {
|
||||
Files.move(tmp, file, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
} finally {
|
||||
Files.deleteIfExists(tmp);
|
||||
}
|
||||
}
|
||||
|
||||
/** Locales that currently have a markdown file, for the admin editor. */
|
||||
public Set<String> listLocalesWithContent() {
|
||||
Set<String> result = new TreeSet<>();
|
||||
Path dir = disclaimerDir();
|
||||
if (!Files.isDirectory(dir)) {
|
||||
return result;
|
||||
}
|
||||
try (Stream<Path> files = Files.list(dir)) {
|
||||
files.filter(Files::isRegularFile)
|
||||
.map(path -> path.getFileName().toString())
|
||||
.filter(name -> name.endsWith(".md"))
|
||||
.map(name -> name.substring(0, name.length() - ".md".length()))
|
||||
.filter(this::isValidLocale)
|
||||
.forEach(result::add);
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed listing login agreement files", e);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private ApplicationProperties.Legal.LoginAgreement config() {
|
||||
return applicationProperties.getLegal().getLoginAgreement();
|
||||
}
|
||||
|
||||
private Path disclaimerDir() {
|
||||
return Path.of(InstallationPathConfig.getCustomFilesPath(), "disclaimer").normalize();
|
||||
}
|
||||
|
||||
private void addLocaleCandidates(List<String> out, String locale) {
|
||||
if (!isValidLocale(locale)) {
|
||||
return;
|
||||
}
|
||||
if (!out.contains(locale)) {
|
||||
out.add(locale);
|
||||
}
|
||||
String base = locale.split("[_-]", 2)[0];
|
||||
if (!base.equals(locale) && !out.contains(base)) {
|
||||
out.add(base);
|
||||
}
|
||||
}
|
||||
|
||||
private String readFileIfExists(String locale) {
|
||||
Path file = resolveLocaleFile(locale);
|
||||
if (file == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
// NOFOLLOW_LINKS: a symlinked entry is treated as non-regular and skipped, so a
|
||||
// planted symlink can't expose files outside the disclaimer dir via the public read.
|
||||
if (Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) {
|
||||
if (Files.size(file) > MAX_FILE_BYTES) {
|
||||
log.warn(
|
||||
"Login agreement file for locale {} exceeds {} bytes; ignoring",
|
||||
locale,
|
||||
MAX_FILE_BYTES);
|
||||
return null;
|
||||
}
|
||||
return Files.readString(file, StandardCharsets.UTF_8);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed reading login agreement file for locale {}", locale, e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private Path resolveLocaleFile(String locale) {
|
||||
if (!isValidLocale(locale)) {
|
||||
return null;
|
||||
}
|
||||
Path dir = disclaimerDir();
|
||||
Path file = dir.resolve(locale + ".md").normalize();
|
||||
// Defence in depth: the regex already blocks separators, but confirm containment.
|
||||
if (!file.startsWith(dir)) {
|
||||
return null;
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private boolean isValidLocale(String locale) {
|
||||
// Length check BEFORE the regex: LOCALE_PATTERN's repetition group recurses one stack
|
||||
// frame per repeat in java.util.regex, so an unbounded input could overflow the stack.
|
||||
return locale != null
|
||||
&& locale.length() <= MAX_LOCALE_LENGTH
|
||||
&& LOCALE_PATTERN.matcher(locale).matches();
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package stirling.software.common.service;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
@@ -34,7 +35,7 @@ public class MobileScannerService {
|
||||
public MobileScannerService() throws IOException {
|
||||
// Create temp directory for mobile scanner uploads
|
||||
this.tempDirectory =
|
||||
Path.of(System.getProperty("java.io.tmpdir"), "stirling-mobile-scanner");
|
||||
Paths.get(System.getProperty("java.io.tmpdir"), "stirling-mobile-scanner");
|
||||
Files.createDirectories(tempDirectory);
|
||||
log.info("Mobile scanner temp directory: {}", tempDirectory);
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import java.lang.management.OperatingSystemMXBean;
|
||||
import java.lang.management.RuntimeMXBean;
|
||||
import java.lang.management.ThreadMXBean;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.HashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
@@ -160,7 +160,7 @@ public class PostHogService {
|
||||
}
|
||||
|
||||
private boolean isRunningInDocker() {
|
||||
return Files.exists(Path.of("/.dockerenv"));
|
||||
return Files.exists(Paths.get("/.dockerenv"));
|
||||
}
|
||||
|
||||
private Map<String, Object> getDockerMetrics() {
|
||||
|
||||
@@ -1,21 +1,11 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** Provides metadata about tool endpoints for internal dispatch. */
|
||||
public interface ToolMetadataService {
|
||||
|
||||
/** Returns true if the given operation path accepts multiple input files. */
|
||||
boolean isMultiInput(String operationPath);
|
||||
|
||||
/**
|
||||
* Returns the file extensions (lowercase, no leading dot, e.g. {@code "pdf"}) that the
|
||||
* operation accepts as input ({@code output=false}) or produces as output ({@code
|
||||
* output=true}), derived from the endpoint's declared type. Returns {@code null} when the
|
||||
* endpoint declares no specific type, which callers should treat as "any type accepted".
|
||||
*/
|
||||
List<String> getExtensionTypes(boolean output, String operationPath);
|
||||
|
||||
/**
|
||||
* Returns true when the endpoint's ZIP response is a transport for multiple typed results and
|
||||
* should be unpacked: multi-output endpoints (Type:SIMO / Type:MIMO) and wrapper declarations
|
||||
|
||||
@@ -255,7 +255,7 @@ public class GeneralUtils {
|
||||
String pattern = locationPattern;
|
||||
if (pattern.startsWith("file:")) {
|
||||
String rawPath = pattern.substring(5).replace("\\*", "").replace("/*", "");
|
||||
Path normalizePath = Path.of(rawPath).normalize();
|
||||
Path normalizePath = Paths.get(rawPath).normalize();
|
||||
pattern = "file:" + normalizePath.toString().replace("\\", "/") + "/*";
|
||||
}
|
||||
return ResourcePatternUtils.getResourcePatternResolver(resourceLoader)
|
||||
@@ -837,7 +837,7 @@ public class GeneralUtils {
|
||||
}
|
||||
|
||||
public boolean createDir(String path) {
|
||||
Path folder = Path.of(path);
|
||||
Path folder = Paths.get(path);
|
||||
if (!Files.exists(folder)) {
|
||||
try {
|
||||
Files.createDirectories(folder);
|
||||
@@ -867,7 +867,7 @@ public class GeneralUtils {
|
||||
|
||||
public void saveKeyToSettings(String key, Object newValue) throws IOException {
|
||||
String[] keyArray = key.split("\\.");
|
||||
Path settingsPath = Path.of(InstallationPathConfig.getSettingsPath());
|
||||
Path settingsPath = Paths.get(InstallationPathConfig.getSettingsPath());
|
||||
YamlHelper settingsYaml = new YamlHelper(settingsPath);
|
||||
settingsYaml.updateValue(Arrays.asList(keyArray), newValue);
|
||||
settingsYaml.saveOverride(settingsPath);
|
||||
@@ -888,7 +888,7 @@ public class GeneralUtils {
|
||||
return;
|
||||
}
|
||||
|
||||
Path settingsPath = Path.of(InstallationPathConfig.getSettingsPath());
|
||||
Path settingsPath = Paths.get(InstallationPathConfig.getSettingsPath());
|
||||
YamlHelper settingsYaml = new YamlHelper(settingsPath);
|
||||
|
||||
// Apply all updates to the same YamlHelper instance
|
||||
@@ -974,11 +974,11 @@ public class GeneralUtils {
|
||||
*/
|
||||
public void extractPipeline() throws IOException {
|
||||
Path pipelineDir =
|
||||
Path.of(InstallationPathConfig.getPipelinePath(), DEFAULT_WEBUI_CONFIGS_DIR);
|
||||
Paths.get(InstallationPathConfig.getPipelinePath(), DEFAULT_WEBUI_CONFIGS_DIR);
|
||||
Files.createDirectories(pipelineDir);
|
||||
|
||||
for (String name : DEFAULT_VALID_PIPELINE) {
|
||||
if (!Path.of(name).getFileName().toString().equals(name)) {
|
||||
if (!Paths.get(name).getFileName().toString().equals(name)) {
|
||||
log.error("Invalid pipeline file name: {}", name);
|
||||
throw new IllegalArgumentException("Invalid pipeline file name: " + name);
|
||||
}
|
||||
@@ -1014,7 +1014,7 @@ public class GeneralUtils {
|
||||
throw new IllegalArgumentException(
|
||||
"scriptName must not contain path traversal characters");
|
||||
}
|
||||
if (!Path.of(scriptName).getFileName().toString().equals(scriptName)) {
|
||||
if (!Paths.get(scriptName).getFileName().toString().equals(scriptName)) {
|
||||
throw new IllegalArgumentException(
|
||||
"scriptName must not contain path traversal characters");
|
||||
}
|
||||
@@ -1024,7 +1024,7 @@ public class GeneralUtils {
|
||||
"scriptName must be either 'png_to_webp.py' or 'split_photos.py'");
|
||||
}
|
||||
|
||||
Path scriptsDir = Path.of(InstallationPathConfig.getScriptsPath(), PYTHON_SCRIPTS_DIR);
|
||||
Path scriptsDir = Paths.get(InstallationPathConfig.getScriptsPath(), PYTHON_SCRIPTS_DIR);
|
||||
Files.createDirectories(scriptsDir);
|
||||
|
||||
Path target = scriptsDir.resolve(scriptName);
|
||||
@@ -1185,181 +1185,23 @@ public class GeneralUtils {
|
||||
}
|
||||
|
||||
public String getLocalNetworkIp() {
|
||||
String routed = detectLocalIpViaDefaultRoute();
|
||||
if (routed != null) {
|
||||
return routed;
|
||||
}
|
||||
try {
|
||||
return selectBestSiteLocalIp(collectInterfaceInfo());
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to detect local network IP", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String detectLocalIpViaDefaultRoute() {
|
||||
try (DatagramSocket socket = new DatagramSocket()) {
|
||||
socket.connect(InetAddress.getByName("8.8.8.8"), 53);
|
||||
InetAddress local = socket.getLocalAddress();
|
||||
if (local instanceof Inet4Address
|
||||
&& !local.isAnyLocalAddress()
|
||||
&& !local.isLoopbackAddress()
|
||||
&& !local.isLinkLocalAddress()) {
|
||||
return local.getHostAddress();
|
||||
Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
|
||||
if (interfaces == null) return null;
|
||||
while (interfaces.hasMoreElements()) {
|
||||
NetworkInterface iface = interfaces.nextElement();
|
||||
if (!iface.isUp() || iface.isLoopback() || iface.isVirtual()) continue;
|
||||
Enumeration<InetAddress> addresses = iface.getInetAddresses();
|
||||
while (addresses.hasMoreElements()) {
|
||||
InetAddress addr = addresses.nextElement();
|
||||
if (addr instanceof Inet4Address && addr.isSiteLocalAddress()) {
|
||||
return addr.getHostAddress();
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Default-route IP detection failed; will scan interfaces", e);
|
||||
log.warn("Failed to detect local network IP", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private List<NetworkInterfaceInfo> collectInterfaceInfo() throws SocketException {
|
||||
List<NetworkInterfaceInfo> infos = new ArrayList<>();
|
||||
Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
|
||||
if (interfaces == null) {
|
||||
return infos;
|
||||
}
|
||||
while (interfaces.hasMoreElements()) {
|
||||
NetworkInterface iface = interfaces.nextElement();
|
||||
|
||||
List<String> siteLocalIpv4s = new ArrayList<>();
|
||||
Enumeration<InetAddress> addresses = iface.getInetAddresses();
|
||||
while (addresses.hasMoreElements()) {
|
||||
InetAddress addr = addresses.nextElement();
|
||||
if (addr instanceof Inet4Address && addr.isSiteLocalAddress()) {
|
||||
siteLocalIpv4s.add(addr.getHostAddress());
|
||||
}
|
||||
}
|
||||
if (siteLocalIpv4s.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
byte[] mac = iface.getHardwareAddress();
|
||||
infos.add(
|
||||
new NetworkInterfaceInfo(
|
||||
iface.getName(),
|
||||
iface.getDisplayName(),
|
||||
iface.getIndex(),
|
||||
iface.isUp(),
|
||||
iface.isLoopback(),
|
||||
iface.isPointToPoint(),
|
||||
iface.isVirtual(),
|
||||
mac != null && mac.length > 0,
|
||||
siteLocalIpv4s));
|
||||
} catch (SocketException e) {
|
||||
log.debug("Skipping interface {} while scanning for local IP", iface.getName(), e);
|
||||
}
|
||||
}
|
||||
return infos;
|
||||
}
|
||||
|
||||
static String selectBestSiteLocalIp(List<NetworkInterfaceInfo> interfaces) {
|
||||
return interfaces.stream()
|
||||
.filter(i -> i.up() && !i.loopback() && !i.pointToPoint() && !i.virtual())
|
||||
.filter(i -> !isLikelyVirtualInterface(i.name(), i.displayName()))
|
||||
.flatMap(
|
||||
i ->
|
||||
i.siteLocalIpv4s().stream()
|
||||
.map(
|
||||
ip ->
|
||||
new ScoredAddress(
|
||||
ip,
|
||||
scoreInterface(i, ip),
|
||||
i.index())))
|
||||
.max(
|
||||
Comparator.comparingInt(ScoredAddress::score)
|
||||
.thenComparing(
|
||||
Comparator.comparingInt(ScoredAddress::interfaceIndex)
|
||||
.reversed()))
|
||||
.map(ScoredAddress::ip)
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
private static int scoreInterface(NetworkInterfaceInfo iface, String ip) {
|
||||
int score = 0;
|
||||
if (isLikelyPhysicalInterface(iface.name(), iface.displayName())) {
|
||||
score += 100;
|
||||
}
|
||||
if (iface.hasHardwareAddress()) {
|
||||
score += 20;
|
||||
}
|
||||
if (ip.startsWith("192.168.")) {
|
||||
score += 30;
|
||||
} else if (ip.startsWith("10.")) {
|
||||
score += 20;
|
||||
} else {
|
||||
score += 5;
|
||||
}
|
||||
return score;
|
||||
}
|
||||
|
||||
static boolean isLikelyVirtualInterface(String name, String displayName) {
|
||||
String n = name == null ? "" : name.toLowerCase(Locale.ROOT);
|
||||
String d = displayName == null ? "" : displayName.toLowerCase(Locale.ROOT);
|
||||
String[] namePrefixes = {
|
||||
"tun", "tap", "utun", "veth", "virbr", "vmnet", "docker", "br-", "wg", "ppp", "awdl",
|
||||
"llw"
|
||||
};
|
||||
for (String prefix : namePrefixes) {
|
||||
if (n.startsWith(prefix)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
String[] displayMarkers = {
|
||||
"vmware",
|
||||
"virtualbox",
|
||||
"virtual box",
|
||||
"vbox",
|
||||
"hyper-v",
|
||||
"hyperv",
|
||||
"vethernet",
|
||||
"windows subsystem for linux",
|
||||
"wsl",
|
||||
"docker",
|
||||
"tap-windows",
|
||||
"tunnel",
|
||||
"vpn",
|
||||
"zerotier",
|
||||
"tailscale",
|
||||
"bluetooth",
|
||||
"teredo",
|
||||
"isatap",
|
||||
"loopback",
|
||||
"pseudo",
|
||||
"virtual"
|
||||
};
|
||||
for (String marker : displayMarkers) {
|
||||
if (d.contains(marker)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static boolean isLikelyPhysicalInterface(String name, String displayName) {
|
||||
String n = name == null ? "" : name.toLowerCase(Locale.ROOT);
|
||||
String d = displayName == null ? "" : displayName.toLowerCase(Locale.ROOT);
|
||||
return n.startsWith("eth")
|
||||
|| n.startsWith("en")
|
||||
|| n.startsWith("wl")
|
||||
|| n.startsWith("em")
|
||||
|| d.contains("ethernet")
|
||||
|| d.contains("wi-fi")
|
||||
|| d.contains("wifi")
|
||||
|| d.contains("wireless");
|
||||
}
|
||||
|
||||
record NetworkInterfaceInfo(
|
||||
String name,
|
||||
String displayName,
|
||||
int index,
|
||||
boolean up,
|
||||
boolean loopback,
|
||||
boolean pointToPoint,
|
||||
boolean virtual,
|
||||
boolean hasHardwareAddress,
|
||||
List<String> siteLocalIpv4s) {}
|
||||
|
||||
private record ScoredAddress(String ip, int score, int interfaceIndex) {}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import java.io.File;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@@ -19,7 +20,7 @@ public class JarPathUtil {
|
||||
public static Path currentJar() {
|
||||
try {
|
||||
Path jar =
|
||||
Path.of(
|
||||
Paths.get(
|
||||
JarPathUtil.class
|
||||
.getProtectionDomain()
|
||||
.getCodeSource()
|
||||
@@ -60,14 +61,14 @@ public class JarPathUtil {
|
||||
}
|
||||
|
||||
// Location 2: ./build/libs/ (development build)
|
||||
possibleLocations[1] = Path.of("build", "libs", "restart-helper.jar").toAbsolutePath();
|
||||
possibleLocations[1] = Paths.get("build", "libs", "restart-helper.jar").toAbsolutePath();
|
||||
|
||||
// Location 3: app/common/build/libs/ (multi-module build)
|
||||
possibleLocations[2] =
|
||||
Path.of("app", "common", "build", "libs", "restart-helper.jar").toAbsolutePath();
|
||||
Paths.get("app", "common", "build", "libs", "restart-helper.jar").toAbsolutePath();
|
||||
|
||||
// Location 4: Current working directory
|
||||
possibleLocations[3] = Path.of("restart-helper.jar").toAbsolutePath();
|
||||
possibleLocations[3] = Paths.get("restart-helper.jar").toAbsolutePath();
|
||||
|
||||
// Check each location
|
||||
for (Path location : possibleLocations) {
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
/** Thread-local context for passing job ID and owner across async boundaries */
|
||||
/** Thread-local context for passing job ID across async boundaries */
|
||||
public class JobContext {
|
||||
private static final ThreadLocal<String> CURRENT_JOB_ID = new ThreadLocal<>();
|
||||
private static final ThreadLocal<String> CURRENT_OWNER = new ThreadLocal<>();
|
||||
|
||||
public static void setJobId(String jobId) {
|
||||
CURRENT_JOB_ID.set(jobId);
|
||||
@@ -13,16 +12,7 @@ public class JobContext {
|
||||
return CURRENT_JOB_ID.get();
|
||||
}
|
||||
|
||||
public static void setOwner(String owner) {
|
||||
CURRENT_OWNER.set(owner);
|
||||
}
|
||||
|
||||
public static String getOwner() {
|
||||
return CURRENT_OWNER.get();
|
||||
}
|
||||
|
||||
public static void clear() {
|
||||
CURRENT_JOB_ID.remove();
|
||||
CURRENT_OWNER.remove();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,310 +0,0 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import javax.xml.XMLConstants;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.transform.OutputKeys;
|
||||
import javax.xml.transform.Transformer;
|
||||
import javax.xml.transform.TransformerException;
|
||||
import javax.xml.transform.TransformerFactory;
|
||||
import javax.xml.transform.dom.DOMSource;
|
||||
import javax.xml.transform.stream.StreamResult;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NamedNodeMap;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import io.github.pixee.security.ZipSecurity;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.SsrfProtectionService;
|
||||
|
||||
// Strips external refs from OOXML/ODF uploads so LibreOffice can't be made to fetch them.
|
||||
@Component
|
||||
@Slf4j
|
||||
public class OfficeDocumentSanitizer {
|
||||
|
||||
private static final Set<String> OOXML_EXTENSIONS =
|
||||
Set.of(
|
||||
"docx", "docm", "dotx", "dotm", "xlsx", "xlsm", "xltx", "xltm", "pptx", "pptm",
|
||||
"potx", "potm", "ppsx", "ppsm");
|
||||
|
||||
private static final Set<String> ODF_EXTENSIONS =
|
||||
Set.of(
|
||||
"odt", "ott", "ods", "ots", "odp", "otp", "odg", "otg", "odf", "odc", "odi",
|
||||
"odm");
|
||||
|
||||
private static final Set<String> ODF_XML_PARTS =
|
||||
Set.of("content.xml", "styles.xml", "meta.xml", "settings.xml");
|
||||
|
||||
private final SsrfProtectionService ssrfProtectionService;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
public OfficeDocumentSanitizer(
|
||||
SsrfProtectionService ssrfProtectionService,
|
||||
ApplicationProperties applicationProperties) {
|
||||
this.ssrfProtectionService = ssrfProtectionService;
|
||||
this.applicationProperties = applicationProperties;
|
||||
}
|
||||
|
||||
public boolean isSanitizableExtension(String extension) {
|
||||
if (extension == null) {
|
||||
return false;
|
||||
}
|
||||
String lower = extension.toLowerCase(Locale.ROOT);
|
||||
return OOXML_EXTENSIONS.contains(lower) || ODF_EXTENSIONS.contains(lower);
|
||||
}
|
||||
|
||||
public byte[] sanitize(byte[] documentBytes, String extension) throws IOException {
|
||||
if (documentBytes == null || documentBytes.length == 0) {
|
||||
throw new IOException("Office document input is empty or null");
|
||||
}
|
||||
if (applicationProperties.getSystem().isDisableSanitize()) {
|
||||
log.debug("Office document sanitization disabled by configuration");
|
||||
return documentBytes;
|
||||
}
|
||||
if (!isSanitizableExtension(extension)) {
|
||||
return documentBytes;
|
||||
}
|
||||
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream(documentBytes.length);
|
||||
try (ZipInputStream zipIn =
|
||||
ZipSecurity.createHardenedInputStream(
|
||||
new ByteArrayInputStream(documentBytes));
|
||||
ZipOutputStream zipOut = new ZipOutputStream(out)) {
|
||||
|
||||
ZipEntry entry;
|
||||
while ((entry = zipIn.getNextEntry()) != null) {
|
||||
String name = entry.getName();
|
||||
byte[] bytes = entry.isDirectory() ? new byte[0] : zipIn.readAllBytes();
|
||||
|
||||
if (!entry.isDirectory()) {
|
||||
bytes = sanitizeEntry(name, bytes);
|
||||
}
|
||||
|
||||
ZipEntry outEntry = new ZipEntry(name);
|
||||
if (entry.getComment() != null) {
|
||||
outEntry.setComment(entry.getComment());
|
||||
}
|
||||
if (entry.getExtra() != null) {
|
||||
outEntry.setExtra(entry.getExtra());
|
||||
}
|
||||
zipOut.putNextEntry(outEntry);
|
||||
if (!entry.isDirectory()) {
|
||||
zipOut.write(bytes);
|
||||
}
|
||||
zipOut.closeEntry();
|
||||
}
|
||||
}
|
||||
return out.toByteArray();
|
||||
}
|
||||
|
||||
private byte[] sanitizeEntry(String entryName, byte[] entryBytes) {
|
||||
String lower = entryName.toLowerCase(Locale.ROOT);
|
||||
try {
|
||||
if (lower.endsWith(".rels")) {
|
||||
return sanitizeOoxmlRels(entryBytes);
|
||||
}
|
||||
if (isOdfXmlPart(lower)) {
|
||||
return sanitizeOdfXml(entryBytes);
|
||||
}
|
||||
} catch (ParserConfigurationException
|
||||
| SAXException
|
||||
| IOException
|
||||
| TransformerException e) {
|
||||
log.warn(
|
||||
"Failed to parse XML part '{}' for sanitization, leaving as-is: {}",
|
||||
entryName,
|
||||
e.getMessage());
|
||||
}
|
||||
return entryBytes;
|
||||
}
|
||||
|
||||
private boolean isOdfXmlPart(String lowerName) {
|
||||
int slash = lowerName.lastIndexOf('/');
|
||||
String base = slash >= 0 ? lowerName.substring(slash + 1) : lowerName;
|
||||
return ODF_XML_PARTS.contains(base);
|
||||
}
|
||||
|
||||
private byte[] sanitizeOoxmlRels(byte[] xmlBytes)
|
||||
throws IOException, ParserConfigurationException, SAXException, TransformerException {
|
||||
Document doc = parseSecurely(xmlBytes);
|
||||
Element root = doc.getDocumentElement();
|
||||
if (root == null) {
|
||||
return xmlBytes;
|
||||
}
|
||||
NodeList relationships = root.getElementsByTagNameNS("*", "Relationship");
|
||||
List<Node> toRemove = new ArrayList<>();
|
||||
for (int i = 0; i < relationships.getLength(); i++) {
|
||||
Node node = relationships.item(i);
|
||||
NamedNodeMap attrs = node.getAttributes();
|
||||
if (attrs == null) {
|
||||
continue;
|
||||
}
|
||||
Node targetMode = attrs.getNamedItem("TargetMode");
|
||||
if (targetMode == null || !"external".equalsIgnoreCase(targetMode.getNodeValue())) {
|
||||
continue;
|
||||
}
|
||||
Node target = attrs.getNamedItem("Target");
|
||||
String targetValue = target == null ? "" : target.getNodeValue();
|
||||
if (isAdminAllowed(targetValue)) {
|
||||
continue;
|
||||
}
|
||||
log.warn(
|
||||
"Stripping OOXML external relationship target: {}",
|
||||
truncateForLog(targetValue));
|
||||
toRemove.add(node);
|
||||
}
|
||||
if (toRemove.isEmpty()) {
|
||||
return xmlBytes;
|
||||
}
|
||||
for (Node n : toRemove) {
|
||||
n.getParentNode().removeChild(n);
|
||||
}
|
||||
return serializeDocument(doc);
|
||||
}
|
||||
|
||||
private byte[] sanitizeOdfXml(byte[] xmlBytes)
|
||||
throws IOException, ParserConfigurationException, SAXException, TransformerException {
|
||||
Document doc = parseSecurely(xmlBytes);
|
||||
Element root = doc.getDocumentElement();
|
||||
if (root == null) {
|
||||
return xmlBytes;
|
||||
}
|
||||
boolean modified = stripExternalHrefs(root);
|
||||
if (!modified) {
|
||||
return xmlBytes;
|
||||
}
|
||||
return serializeDocument(doc);
|
||||
}
|
||||
|
||||
private boolean stripExternalHrefs(Node node) {
|
||||
boolean modified = false;
|
||||
if (node.getNodeType() == Node.ELEMENT_NODE) {
|
||||
NamedNodeMap attrs = node.getAttributes();
|
||||
List<String> hrefAttrsToRemove = new ArrayList<>();
|
||||
for (int i = 0; i < attrs.getLength(); i++) {
|
||||
Node attr = attrs.item(i);
|
||||
String name = attr.getNodeName();
|
||||
if (name == null) {
|
||||
continue;
|
||||
}
|
||||
String lower = name.toLowerCase(Locale.ROOT);
|
||||
if (!(lower.equals("xlink:href")
|
||||
|| lower.endsWith(":href")
|
||||
|| lower.equals("href"))) {
|
||||
continue;
|
||||
}
|
||||
String value = attr.getNodeValue();
|
||||
if (!isExternalUrl(value)) {
|
||||
continue;
|
||||
}
|
||||
if (isAdminAllowed(value)) {
|
||||
continue;
|
||||
}
|
||||
log.warn(
|
||||
"Stripping ODF external href attribute ({}): {}",
|
||||
name,
|
||||
truncateForLog(value));
|
||||
hrefAttrsToRemove.add(name);
|
||||
}
|
||||
Element element = (Element) node;
|
||||
for (String attrName : hrefAttrsToRemove) {
|
||||
element.removeAttribute(attrName);
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
NodeList children = node.getChildNodes();
|
||||
for (int i = 0; i < children.getLength(); i++) {
|
||||
if (stripExternalHrefs(children.item(i))) {
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
return modified;
|
||||
}
|
||||
|
||||
private boolean isExternalUrl(String url) {
|
||||
if (url == null) {
|
||||
return false;
|
||||
}
|
||||
String trimmed = url.trim().toLowerCase(Locale.ROOT);
|
||||
if (trimmed.isEmpty() || trimmed.startsWith("#") || trimmed.startsWith("../")) {
|
||||
return false;
|
||||
}
|
||||
return trimmed.startsWith("http://")
|
||||
|| trimmed.startsWith("https://")
|
||||
|| trimmed.startsWith("ftp://")
|
||||
|| trimmed.startsWith("ftps://")
|
||||
|| trimmed.startsWith("file:")
|
||||
|| trimmed.startsWith("smb:")
|
||||
|| trimmed.startsWith("\\\\")
|
||||
|| trimmed.startsWith("//");
|
||||
}
|
||||
|
||||
// Preserved only with an explicit allowedDomains entry; MEDIUM default would admit public URLs.
|
||||
private boolean isAdminAllowed(String url) {
|
||||
if (ssrfProtectionService == null || url == null || url.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
ApplicationProperties.Html.UrlSecurity config =
|
||||
applicationProperties.getSystem().getHtml().getUrlSecurity();
|
||||
if (config == null
|
||||
|| config.getAllowedDomains() == null
|
||||
|| config.getAllowedDomains().isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
return ssrfProtectionService.isUrlAllowed(url);
|
||||
}
|
||||
|
||||
private Document parseSecurely(byte[] xmlBytes)
|
||||
throws ParserConfigurationException, SAXException, IOException {
|
||||
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
||||
factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
|
||||
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
|
||||
factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
|
||||
factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
|
||||
factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
|
||||
factory.setXIncludeAware(false);
|
||||
factory.setExpandEntityReferences(false);
|
||||
factory.setNamespaceAware(true);
|
||||
DocumentBuilder builder = factory.newDocumentBuilder();
|
||||
return builder.parse(new ByteArrayInputStream(xmlBytes));
|
||||
}
|
||||
|
||||
private byte[] serializeDocument(Document doc) throws TransformerException {
|
||||
TransformerFactory tf = TransformerFactory.newInstance();
|
||||
tf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
|
||||
Transformer transformer = tf.newTransformer();
|
||||
transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
|
||||
transformer.setOutputProperty(OutputKeys.INDENT, "no");
|
||||
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "no");
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
transformer.transform(new DOMSource(doc), new StreamResult(baos));
|
||||
return baos.toByteArray();
|
||||
}
|
||||
|
||||
private String truncateForLog(String value) {
|
||||
if (value == null) {
|
||||
return "null";
|
||||
}
|
||||
return value.length() > 80 ? value.substring(0, 80) + "..." : value;
|
||||
}
|
||||
}
|
||||
@@ -244,7 +244,10 @@ public class SvgSanitizer {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
return normalized.startsWith("http://")
|
||||
|| normalized.startsWith("https://")
|
||||
|| normalized.startsWith("//")
|
||||
|| normalized.startsWith("file:");
|
||||
}
|
||||
|
||||
private boolean isUrlAllowed(String url) {
|
||||
|
||||
-48
@@ -1,48 +0,0 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Binds a multipart form value containing a JSON array into a typed {@code List<T>}. Used for
|
||||
* endpoints that accept structured list parameters via {@code @ModelAttribute} — the form field
|
||||
* carries the full JSON array as its value and the editor parses it once.
|
||||
*/
|
||||
@Slf4j
|
||||
public class JsonListPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER =
|
||||
JsonMapper.builder()
|
||||
.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
|
||||
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.build();
|
||||
|
||||
private final TypeReference<? extends List<T>> typeRef;
|
||||
|
||||
public JsonListPropertyEditor(TypeReference<? extends List<T>> typeRef) {
|
||||
this.typeRef = typeRef;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
setValue(new ArrayList<T>());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
setValue(OBJECT_MAPPER.readValue(text, typeRef));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to parse JSON list value", e);
|
||||
throw new IllegalArgumentException(
|
||||
"Expected a JSON array but could not parse: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Binds a multipart form value containing a JSON object into a typed {@code T}. Companion to {@link
|
||||
* JsonListPropertyEditor} for single-object nested fields on {@code @ModelAttribute} endpoints.
|
||||
*/
|
||||
@Slf4j
|
||||
public class JsonObjectPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER =
|
||||
JsonMapper.builder().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).build();
|
||||
|
||||
private final Class<T> type;
|
||||
|
||||
public JsonObjectPropertyEditor(Class<T> type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
setValue(null);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
setValue(OBJECT_MAPPER.readValue(text, type));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to parse JSON object value", e);
|
||||
throw new IllegalArgumentException(
|
||||
"Expected a JSON object but could not parse: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.JavaType;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Spring property editor that decodes a JSON string into a typed {@link ArrayList}. Used to bind
|
||||
* complex list parameters (e.g. {@code List<RedactionArea>}, {@code List<EditTextOperation>}) from
|
||||
* multipart form fields, where Spring's default binding cannot deserialize a JSON array.
|
||||
*/
|
||||
@Slf4j
|
||||
public class StringToArrayListPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private final ObjectMapper objectMapper =
|
||||
JsonMapper.builder()
|
||||
.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
|
||||
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.build();
|
||||
|
||||
private final Class<T> elementType;
|
||||
|
||||
public StringToArrayListPropertyEditor(Class<T> elementType) {
|
||||
this.elementType = elementType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
setValue(new ArrayList<>());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
JavaType listType =
|
||||
objectMapper
|
||||
.getTypeFactory()
|
||||
.constructCollectionType(ArrayList.class, elementType);
|
||||
List<T> list = objectMapper.readValue(text, listType);
|
||||
setValue(list);
|
||||
} catch (Exception e) {
|
||||
log.error("Exception while converting {}", e);
|
||||
throw new IllegalArgumentException(
|
||||
"Failed to convert java.lang.String to java.util.List");
|
||||
}
|
||||
}
|
||||
}
|
||||
-481
@@ -1,481 +0,0 @@
|
||||
package stirling.software.SPDF.config;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Nested;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.SPDF.config.EndpointConfiguration.DisableReason;
|
||||
import stirling.software.SPDF.config.EndpointConfiguration.EndpointAvailability;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link EndpointConfiguration}. The class wires up its endpoint/group registry in
|
||||
* {@code init()} during construction and then applies environment overrides. We build it with a
|
||||
* real {@link ApplicationProperties} (whose System/Endpoints sub-objects are non-null by default)
|
||||
* so the constructor runs cleanly without any mocking.
|
||||
*/
|
||||
class EndpointConfigurationGapTest {
|
||||
|
||||
private ApplicationProperties applicationProperties;
|
||||
|
||||
/**
|
||||
* Construct an EndpointConfiguration with the given pro flag and current applicationProperties.
|
||||
*/
|
||||
private EndpointConfiguration build(boolean runningProOrHigher) {
|
||||
return new EndpointConfiguration(applicationProperties, runningProOrHigher);
|
||||
}
|
||||
|
||||
/** Default config: not pro, no removals, url-to-pdf disabled (default System flag is false). */
|
||||
private EndpointConfiguration buildDefault() {
|
||||
return build(false);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
applicationProperties = new ApplicationProperties();
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("endpointKeyForUri (static)")
|
||||
class EndpointKeyForUriTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("returns null for null uri")
|
||||
void nullUri() {
|
||||
assertNull(EndpointConfiguration.endpointKeyForUri(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("returns null when uri does not contain /api/v1")
|
||||
void notApiPath() {
|
||||
assertNull(EndpointConfiguration.endpointKeyForUri("/foo/bar/baz"));
|
||||
assertNull(EndpointConfiguration.endpointKeyForUri("https://example.com/home"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("returns null when uri has too few path segments")
|
||||
void tooFewSegments() {
|
||||
// "/api/v1/general" splits to ["", "api", "v1", "general"] -> length 4, not > 4
|
||||
assertNull(EndpointConfiguration.endpointKeyForUri("/api/v1/general"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("extracts plain endpoint key from a standard /api/v1/<group>/<endpoint> uri")
|
||||
void plainEndpoint() {
|
||||
assertEquals(
|
||||
"remove-pages",
|
||||
EndpointConfiguration.endpointKeyForUri("/api/v1/general/remove-pages"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("builds a <from>-to-<to> key for convert endpoints")
|
||||
void convertEndpoint() {
|
||||
assertEquals(
|
||||
"pdf-to-img",
|
||||
EndpointConfiguration.endpointKeyForUri("/api/v1/convert/pdf/img"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("convert path without a target segment falls back to the segment after group")
|
||||
void convertWithoutTarget() {
|
||||
// "/api/v1/convert/pdf" -> length 5, the convert branch needs length > 5
|
||||
assertEquals("pdf", EndpointConfiguration.endpointKeyForUri("/api/v1/convert/pdf"));
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("enable / disable endpoint")
|
||||
class EnableDisableEndpointTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("a freshly registered endpoint is enabled by default")
|
||||
void enabledByDefault() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
assertTrue(config.isEndpointEnabled("merge-pdfs"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("disableEndpoint marks the endpoint disabled")
|
||||
void disableEndpoint() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableEndpoint("merge-pdfs");
|
||||
assertFalse(config.isEndpointEnabled("merge-pdfs"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("enableEndpoint re-enables a previously disabled endpoint")
|
||||
void reEnableEndpoint() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableEndpoint("merge-pdfs");
|
||||
assertFalse(config.isEndpointEnabled("merge-pdfs"));
|
||||
config.enableEndpoint("merge-pdfs");
|
||||
assertTrue(config.isEndpointEnabled("merge-pdfs"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("leading slash is normalized away on disable")
|
||||
void leadingSlashNormalizedOnDisable() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableEndpoint("/merge-pdfs");
|
||||
// both forms resolve to the same key
|
||||
assertFalse(config.isEndpointEnabled("merge-pdfs"));
|
||||
assertFalse(config.isEndpointEnabled("/merge-pdfs"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("isEndpointEnabled tolerates a leading slash on the query")
|
||||
void leadingSlashOnQuery() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
assertTrue(config.isEndpointEnabled("/merge-pdfs"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("disabling clears with enable, removing the disable reason")
|
||||
void enableClearsReason() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableEndpoint("split-pages", DisableReason.DEPENDENCY);
|
||||
assertEquals(
|
||||
DisableReason.DEPENDENCY,
|
||||
config.getEndpointAvailability("split-pages").getReason());
|
||||
config.enableEndpoint("split-pages");
|
||||
EndpointAvailability availability = config.getEndpointAvailability("split-pages");
|
||||
assertTrue(availability.isEnabled());
|
||||
assertNull(availability.getReason());
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("isEndpointEnabledForUri")
|
||||
class IsEndpointEnabledForUriTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("translates a /api/v1 uri to a key and reports its status")
|
||||
void translatesUri() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
assertTrue(config.isEndpointEnabledForUri("/api/v1/general/merge-pdfs"));
|
||||
config.disableEndpoint("merge-pdfs");
|
||||
assertFalse(config.isEndpointEnabledForUri("/api/v1/general/merge-pdfs"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("falls back to treating a non-api uri as a raw key")
|
||||
void fallsBackToRawKey() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableEndpoint("merge-pdfs");
|
||||
// non-api path: key resolution returns null, so the uri itself is used as the key
|
||||
assertFalse(config.isEndpointEnabledForUri("merge-pdfs"));
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("group enable / disable")
|
||||
class GroupTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("a functional group with all endpoints enabled reports enabled")
|
||||
void functionalGroupEnabled() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
assertTrue(config.isGroupEnabled("PageOps"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("disabling a functional group cascades to all its endpoints")
|
||||
void disableFunctionalGroupCascades() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableGroup("PageOps");
|
||||
assertFalse(config.isGroupEnabled("PageOps"));
|
||||
assertFalse(config.isEndpointEnabled("remove-pages"));
|
||||
assertFalse(config.isEndpointEnabled("split-pages"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("re-enabling a functional group re-enables its endpoints")
|
||||
void enableFunctionalGroupRestores() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableGroup("PageOps");
|
||||
assertFalse(config.isEndpointEnabled("remove-pages"));
|
||||
config.enableGroup("PageOps");
|
||||
assertTrue(config.isEndpointEnabled("remove-pages"));
|
||||
assertTrue(config.isGroupEnabled("PageOps"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a functional group with one disabled endpoint is not enabled")
|
||||
void functionalGroupWithDisabledEndpoint() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableEndpoint("remove-pages");
|
||||
assertFalse(config.isGroupEnabled("PageOps"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("disabledGroups reflects disabled groups and getDisabledGroups returns a copy")
|
||||
void getDisabledGroupsReturnsCopy() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableGroup("PageOps");
|
||||
Set<String> disabled = config.getDisabledGroups();
|
||||
assertTrue(disabled.contains("PageOps"));
|
||||
// mutating the returned set must not affect internal state
|
||||
disabled.clear();
|
||||
assertTrue(config.getDisabledGroups().contains("PageOps"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("an unknown group with no endpoints is not enabled")
|
||||
void unknownGroupNotEnabled() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
assertFalse(config.isGroupEnabled("NoSuchGroupXyz"));
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("tool group semantics")
|
||||
class ToolGroupTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("a tool group is enabled until explicitly disabled")
|
||||
void toolGroupEnabledUntilDisabled() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
assertTrue(config.isGroupEnabled("qpdf"));
|
||||
config.disableGroup("qpdf");
|
||||
assertFalse(config.isGroupEnabled("qpdf"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("disabling a tool group does NOT cascade to its endpoints directly")
|
||||
void toolGroupNoCascade() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
// repair has alternatives (qpdf, Ghostscript); disabling only qpdf keeps it enabled
|
||||
config.disableGroup("qpdf");
|
||||
assertTrue(config.isEndpointEnabled("repair"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("endpoint with alternatives is disabled only when all tool groups are gone")
|
||||
void allAlternativesDisabled() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableGroup("qpdf");
|
||||
config.disableGroup("Ghostscript");
|
||||
// repair's only alternatives are qpdf and Ghostscript
|
||||
assertFalse(config.isEndpointEnabled("repair"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("endpoint with a still-enabled alternative stays enabled")
|
||||
void oneAlternativeRemains() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
// compress-pdf alternatives: qpdf, Ghostscript, Java
|
||||
config.disableGroup("qpdf");
|
||||
config.disableGroup("Ghostscript");
|
||||
assertTrue(config.isEndpointEnabled("compress-pdf"));
|
||||
config.disableGroup("Java");
|
||||
assertFalse(config.isEndpointEnabled("compress-pdf"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("single-dependency endpoint (no alternatives) disabled when its tool group is")
|
||||
void singleDependencyDisabled() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
// pdf-to-epub depends on Calibre, no alternatives registered
|
||||
assertTrue(config.isEndpointEnabled("pdf-to-epub"));
|
||||
config.disableGroup("Calibre");
|
||||
assertFalse(config.isEndpointEnabled("pdf-to-epub"));
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("addEndpointToGroup / addEndpointAlternative")
|
||||
class RegistrationTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("addEndpointToGroup makes the endpoint part of the group")
|
||||
void addEndpointToGroup() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.addEndpointToGroup("CustomGroup", "custom-endpoint");
|
||||
Set<String> endpoints = config.getEndpointsForGroup("CustomGroup");
|
||||
assertTrue(endpoints.contains("custom-endpoint"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("disabling a custom functional group disables its added endpoint")
|
||||
void customFunctionalGroupCascades() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.addEndpointToGroup("CustomGroup", "custom-endpoint");
|
||||
assertTrue(config.isEndpointEnabled("custom-endpoint"));
|
||||
config.disableGroup("CustomGroup");
|
||||
assertFalse(config.isEndpointEnabled("custom-endpoint"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("getEndpointsForGroup returns an empty set for unknown groups")
|
||||
void unknownGroupEmptySet() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
Set<String> endpoints = config.getEndpointsForGroup("NoSuchGroupXyz");
|
||||
assertNotNull(endpoints);
|
||||
assertTrue(endpoints.isEmpty());
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("getEndpointAvailability / determineDisableReason")
|
||||
class AvailabilityTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("an enabled endpoint has a null disable reason")
|
||||
void enabledHasNullReason() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
EndpointAvailability availability = config.getEndpointAvailability("merge-pdfs");
|
||||
assertTrue(availability.isEnabled());
|
||||
assertNull(availability.getReason());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("explicit disable preserves the supplied reason")
|
||||
void explicitDisableReason() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableEndpoint("merge-pdfs", DisableReason.DEPENDENCY);
|
||||
EndpointAvailability availability = config.getEndpointAvailability("merge-pdfs");
|
||||
assertFalse(availability.isEnabled());
|
||||
assertEquals(DisableReason.DEPENDENCY, availability.getReason());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("default disableEndpoint reason is CONFIG")
|
||||
void defaultDisableReasonIsConfig() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableEndpoint("merge-pdfs");
|
||||
assertEquals(
|
||||
DisableReason.CONFIG, config.getEndpointAvailability("merge-pdfs").getReason());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("endpoint disabled via functional group reports the group's reason")
|
||||
void functionalGroupReason() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableGroup("PageOps", DisableReason.DEPENDENCY);
|
||||
EndpointAvailability availability = config.getEndpointAvailability("crop");
|
||||
assertFalse(availability.isEnabled());
|
||||
// crop is disabled both via group cascade and group membership; reason is DEPENDENCY
|
||||
assertEquals(DisableReason.DEPENDENCY, availability.getReason());
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("getAllEndpoints")
|
||||
class GetAllEndpointsTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("aggregates endpoints across all groups")
|
||||
void aggregatesAcrossGroups() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
Set<String> all = config.getAllEndpoints();
|
||||
assertTrue(all.contains("merge-pdfs"));
|
||||
assertTrue(all.contains("compress-pdf"));
|
||||
assertTrue(all.contains("ocr-pdf"));
|
||||
assertFalse(all.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("custom endpoints registered after init appear in getAllEndpoints")
|
||||
void includesCustomEndpoints() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.addEndpointToGroup("CustomGroup", "brand-new-endpoint");
|
||||
assertTrue(config.getAllEndpoints().contains("brand-new-endpoint"));
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("environment / constructor driven configuration")
|
||||
class EnvironmentConfigTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("url-to-pdf is disabled when enableUrlToPDF is false (default)")
|
||||
void urlToPdfDisabledByDefault() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
assertFalse(config.isEndpointEnabled("url-to-pdf"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("url-to-pdf stays enabled when enableUrlToPDF is true")
|
||||
void urlToPdfEnabledWhenFlagSet() {
|
||||
applicationProperties.getSystem().setEnableUrlToPDF(true);
|
||||
EndpointConfiguration config = build(false);
|
||||
assertTrue(config.isEndpointEnabled("url-to-pdf"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("endpoints.toRemove disables the listed endpoints at construction")
|
||||
void endpointsToRemove() {
|
||||
applicationProperties
|
||||
.getEndpoints()
|
||||
.setToRemove(List.of(" merge-pdfs ", "split-pages"));
|
||||
EndpointConfiguration config = build(false);
|
||||
// values are trimmed before disabling
|
||||
assertFalse(config.isEndpointEnabled("merge-pdfs"));
|
||||
assertFalse(config.isEndpointEnabled("split-pages"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("endpoints.groupsToRemove disables the listed groups at construction")
|
||||
void groupsToRemove() {
|
||||
applicationProperties.getEndpoints().setGroupsToRemove(List.of(" PageOps "));
|
||||
EndpointConfiguration config = build(false);
|
||||
assertTrue(config.getDisabledGroups().contains("PageOps"));
|
||||
assertFalse(config.isEndpointEnabled("remove-pages"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("non-pro build disables the enterprise group")
|
||||
void nonProDisablesEnterprise() {
|
||||
EndpointConfiguration config = build(false);
|
||||
assertTrue(config.getDisabledGroups().contains("enterprise"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("pro build does not disable the enterprise group")
|
||||
void proDoesNotDisableEnterprise() {
|
||||
EndpointConfiguration config = build(true);
|
||||
assertFalse(config.getDisabledGroups().contains("enterprise"));
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("getEndpointStatuses (Lombok getter) and logging summary")
|
||||
class MiscTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("getEndpointStatuses reflects explicit disable state")
|
||||
void endpointStatusesReflectDisable() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableEndpoint("merge-pdfs");
|
||||
assertEquals(Boolean.FALSE, config.getEndpointStatuses().get("merge-pdfs"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("logDisabledEndpointsSummary runs without throwing")
|
||||
void logSummaryDoesNotThrow() {
|
||||
EndpointConfiguration config = buildDefault();
|
||||
config.disableGroup("PageOps");
|
||||
config.disableGroup("qpdf");
|
||||
// purely a smoke test of the logging branch coverage
|
||||
config.logDisabledEndpointsSummary();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("logDisabledEndpointsSummary runs when nothing is disabled")
|
||||
void logSummaryNothingDisabled() {
|
||||
applicationProperties.getSystem().setEnableUrlToPDF(true);
|
||||
EndpointConfiguration config = build(true);
|
||||
config.logDisabledEndpointsSummary();
|
||||
}
|
||||
}
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link LineAlignmentTableParser}, focused on the coincident-line merge logic and
|
||||
* column-grid construction.
|
||||
*/
|
||||
class LineAlignmentTableParserTest {
|
||||
|
||||
private final LineAlignmentTableParser parser = new LineAlignmentTableParser();
|
||||
|
||||
// ── mergeCoincidentLines ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_singleLine_unchanged() {
|
||||
var lines = List.of(tokenized(rawLine(10f, 100f, "Revenue")));
|
||||
assertThat(parser.mergeCoincidentLines(lines)).hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_distinctYLines_unchanged() {
|
||||
// Two lines at different y positions — must NOT be merged.
|
||||
var lines =
|
||||
List.of(
|
||||
tokenized(rawLine(10f, 100f, "Revenue")),
|
||||
tokenized(rawLine(10f, 115f, "Cost")));
|
||||
assertThat(parser.mergeCoincidentLines(lines)).hasSize(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_sameY_merged() {
|
||||
// Simulates a financial-table row split by LineBuilder at the column gap:
|
||||
// label fragment at x=72 → "Revenue"
|
||||
// value fragment at x=350 → "1,234"
|
||||
// Both have y=100. After merge they should form one TokenizedLine.
|
||||
var label = rawLine(72f, 100f, "Revenue");
|
||||
var value = rawLine(350f, 100f, "1,234");
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
|
||||
|
||||
assertThat(merged).hasSize(1);
|
||||
// The merged line should contain tokens from both halves.
|
||||
var tokens = merged.get(0).all();
|
||||
assertThat(tokens.stream().map(t -> t.text()).toList())
|
||||
.containsExactlyInAnyOrder("Revenue", "1,234");
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_sameY_mergedLineHasCorrectBounds() {
|
||||
var label = rawLine(72f, 100f, "Revenue"); // 7 chars × 6pt = 42pt wide → right = 114
|
||||
var value = rawLine(350f, 100f, "1,234"); // 5 chars × 6pt = 30pt wide → right = 380
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
|
||||
|
||||
var bounds = merged.get(0).line().bounds();
|
||||
assertThat(bounds.x()).isEqualTo(72f);
|
||||
assertThat(bounds.right()).isEqualTo(380f);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_withinTolerance_merged() {
|
||||
// Lines 1.5pt apart (within ROW_MERGE_TOLERANCE_PT = 2pt) should merge.
|
||||
var a = rawLine(10f, 100.0f, "Alpha");
|
||||
var b = rawLine(200f, 101.5f, "99");
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
|
||||
assertThat(merged).hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_beyondTolerance_notMerged() {
|
||||
// Lines 3pt apart (beyond ROW_MERGE_TOLERANCE_PT = 2pt) should NOT merge.
|
||||
var a = rawLine(10f, 100.0f, "Alpha");
|
||||
var b = rawLine(200f, 103.0f, "99");
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
|
||||
assertThat(merged).hasSize(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_threeCoincident_allMerged() {
|
||||
// Three fragments at the same y (e.g. wide financial table with two value columns).
|
||||
var a = rawLine(72f, 100f, "Revenue");
|
||||
var b = rawLine(300f, 100f, "1,234");
|
||||
var c = rawLine(400f, 100f, "5,678");
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
|
||||
assertThat(merged).hasSize(1);
|
||||
assertThat(merged.get(0).all()).hasSize(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_coincidentPairFollowedByDistinctLine_twoGroups() {
|
||||
var a = rawLine(72f, 100f, "Revenue");
|
||||
var b = rawLine(350f, 100f, "1,234"); // same y as a → merges with a
|
||||
var c = rawLine(10f, 115f, "Expenses"); // different y → stays separate
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
|
||||
assertThat(merged).hasSize(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergeCoincidentLines_numericAnchorStatus_correctAfterMerge() {
|
||||
// After merging, the combined line should be an anchor (≥2 numeric tokens).
|
||||
// "Revenue" alone → not an anchor. "1,234 567" alone → anchor.
|
||||
// Merged → anchor with at least 2 numerics.
|
||||
var label = rawLine(72f, 100f, "Revenue");
|
||||
var values = rawLineMultiWord(350f, 100f, "1,234", 30f, "567", 30f);
|
||||
|
||||
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(values)));
|
||||
|
||||
assertThat(merged).hasSize(1);
|
||||
assertThat(merged.get(0).isAnchor()).isTrue();
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Creates a RawLine with a single TextFragment of the given text at the given position. */
|
||||
private static RawLine rawLine(float x, float y, String text) {
|
||||
float width = text.length() * 6f; // ~6pt per char — rough but consistent
|
||||
float height = 12f;
|
||||
Bounds bounds = new Bounds(x, y, width, height);
|
||||
TextFragment fragment =
|
||||
new TextFragment("tf-test", text, bounds, y + height, 11f, "Helvetica", false);
|
||||
return new RawLine("ln-test", List.of(fragment), bounds, 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a RawLine with two TextFragments representing two words separated by a small gap.
|
||||
* Used to simulate a values-only line with multiple numeric tokens.
|
||||
*/
|
||||
private static RawLine rawLineMultiWord(
|
||||
float x, float y, String word1, float w1, String word2, float w2) {
|
||||
float height = 12f;
|
||||
Bounds b1 = new Bounds(x, y, w1, height);
|
||||
Bounds b2 = new Bounds(x + w1 + 5f, y, w2, height);
|
||||
TextFragment f1 = new TextFragment("tf-1", word1, b1, y + height, 11f, "Helvetica", false);
|
||||
TextFragment f2 = new TextFragment("tf-2", word2, b2, y + height, 11f, "Helvetica", false);
|
||||
Bounds lineBounds = new Bounds(x, y, x + w1 + 5f + w2 - x, height);
|
||||
return new RawLine("ln-test", List.of(f1, f2), lineBounds, 1);
|
||||
}
|
||||
|
||||
/** Tokenises a RawLine via the parser's own tokenise logic (package-private access). */
|
||||
private LineAlignmentTableParser.TokenizedLine tokenized(RawLine line) {
|
||||
return parser.tokenize(line);
|
||||
}
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** Unit tests for {@link PageColumnLayout} gutter detection and column classification. */
|
||||
class PageColumnLayoutTest {
|
||||
|
||||
private static final float PAGE_WIDTH = 612f; // Letter portrait
|
||||
|
||||
// ── single-column ────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void singleColumn_oneColumnNoGutters() {
|
||||
List<float[]> lines = List.of(lineBox(72f, 396f));
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
assertThat(layout.gutters()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void singleColumn_classifyAnchor_returnsZero() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(List.of(lineBox(72f, 396f)), PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnOf(100f, 200f)).isEqualTo(0);
|
||||
}
|
||||
|
||||
// ── two-column ───────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void twoColumn_detectsGutter() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnCount()).isEqualTo(2);
|
||||
assertThat(layout.gutters()).hasSize(1);
|
||||
float[] gutter = layout.gutters().get(0);
|
||||
// Gutter is centered on pageWidth/2 with PageColumnLayout.MIDPOINT_SLACK_PT slack each
|
||||
// side.
|
||||
float pageMid = PAGE_WIDTH / 2f;
|
||||
assertThat(gutter[0]).isBetween(pageMid - 40f, pageMid - 20f);
|
||||
assertThat(gutter[1]).isBetween(pageMid + 20f, pageMid + 40f);
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumn_classifyLeftAndRightAnchors() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH);
|
||||
assertThat(layout.columnOf(100f, 200f)).isEqualTo(0);
|
||||
assertThat(layout.columnOf(380f, 460f)).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumn_columnsCrossing_leftLineOnlyHitsLeft() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH);
|
||||
assertThat(layout.columnsCrossing(72f, 280f)).containsExactly(0);
|
||||
assertThat(layout.columnsCrossing(320f, 540f)).containsExactly(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumn_spanningLine_returnsBothColumns() {
|
||||
List<float[]> lines = new ArrayList<>(buildTwoColumnLines(3));
|
||||
// Full-width header that crosses pageWidth/2.
|
||||
lines.add(lineBox(72f, 396f));
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnsCrossing(72f, 540f)).containsExactly(0, 1);
|
||||
assertThat(layout.columnsCrossing(72f, 280f)).containsExactly(0);
|
||||
}
|
||||
|
||||
private static List<float[]> buildTwoColumnLines(int rowsPerColumn) {
|
||||
List<float[]> lines = new ArrayList<>();
|
||||
for (int i = 0; i < rowsPerColumn; i++) {
|
||||
lines.add(lineBox(72f, 136f)); // left column body (72..208)
|
||||
lines.add(lineBox(320f, 220f)); // right column body (320..540)
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
// ── three-column ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void threeColumn_collapsesToLeftRightSplit() {
|
||||
// The midpoint-based detector splits the page at pageWidth/2 and treats anything else as
|
||||
// single-column or spanning. A genuine 3-column layout collapses to 2 columns; the middle
|
||||
// column's content ends up classified by midpoint as left or right of pageMid.
|
||||
List<float[]> lines = new ArrayList<>();
|
||||
for (int i = 0; i < 6; i++) {
|
||||
lines.add(lineBox(72f, 150f)); // 72..222
|
||||
lines.add(lineBox(252f, 150f)); // 252..402
|
||||
lines.add(lineBox(432f, 150f)); // 432..582
|
||||
}
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnCount()).isEqualTo(2);
|
||||
assertThat(layout.gutters()).hasSize(1);
|
||||
}
|
||||
|
||||
// ── empty page ───────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void emptyPage_singleColumnFallback() {
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(List.of(), PAGE_WIDTH);
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
assertThat(layout.gutters()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void onlyShortFragments_singleColumnFallback() {
|
||||
// Page numbers / decorations — too narrow to vote either side.
|
||||
List<float[]> lines = List.of(lineBox(300f, 6f));
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
}
|
||||
|
||||
// ── narrow gap should not be confused for a gutter ───────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void narrowInternalGap_doesNotProduceGutter() {
|
||||
// Both halves sit left of the page midpoint, so no line votes for a right column and
|
||||
// detection falls back to single-column.
|
||||
List<float[]> lines = new ArrayList<>();
|
||||
lines.add(lineBox(72f, 100f));
|
||||
lines.add(lineBox(180f, 100f));
|
||||
for (int i = 0; i < 5; i++) {
|
||||
lines.add(lineBox(72f, 208f));
|
||||
}
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Builds a line bounding box {@code [x1, 0, x1+width, 0]}; Y is unused by detection. */
|
||||
private static float[] lineBox(float x1, float width) {
|
||||
return new float[] {x1, 0f, x1 + width, 0f};
|
||||
}
|
||||
}
|
||||
@@ -1,191 +0,0 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.within;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.LosslessFactory;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Nested;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator.ImageBox;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link PageImageLocator}. PDFs are built in memory with PDFBox so each test is
|
||||
* deterministic and needs no fixtures or native libraries. The locator transforms the image unit
|
||||
* square through the CTM, so an image drawn at {@code (x, y)} with size {@code (w, h)} must yield
|
||||
* the box {@code (x, y, x+w, y+h)}.
|
||||
*/
|
||||
class PageImageLocatorTest {
|
||||
|
||||
/** A tiny opaque raster; pixel content is irrelevant, only its placement matters. */
|
||||
private static PDImageXObject tinyImage(PDDocument doc) throws Exception {
|
||||
BufferedImage img = new BufferedImage(4, 4, BufferedImage.TYPE_INT_RGB);
|
||||
return LosslessFactory.createFromImage(doc, img);
|
||||
}
|
||||
|
||||
/** Builds a one-page PDF that draws one image at the given placement. */
|
||||
private static byte[] pdfWithImageAt(float x, float y, float w, float h) throws Exception {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
PDImageXObject image = tinyImage(doc);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.drawImage(image, x, y, w, h);
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] save(PDDocument doc) throws Exception {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
doc.save(baos);
|
||||
return baos.toByteArray();
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("drawImage bounding boxes")
|
||||
class DrawImageBoxes {
|
||||
|
||||
@Test
|
||||
@DisplayName("a single image yields one box with the page index and CTM-derived bounds")
|
||||
void singleImageBox() throws Exception {
|
||||
byte[] pdf = pdfWithImageAt(100f, 200f, 50f, 80f);
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
PageImageLocator locator = new PageImageLocator(doc.getPage(0), 0);
|
||||
locator.processPage(doc.getPage(0));
|
||||
|
||||
List<ImageBox> boxes = locator.getImageBoxes();
|
||||
assertThat(boxes).hasSize(1);
|
||||
ImageBox box = boxes.get(0);
|
||||
assertThat(box.pageIndex()).isZero();
|
||||
assertThat(box.x1()).isCloseTo(100f, within(0.5f));
|
||||
assertThat(box.y1()).isCloseTo(200f, within(0.5f));
|
||||
assertThat(box.x2()).isCloseTo(150f, within(0.5f));
|
||||
assertThat(box.y2()).isCloseTo(280f, within(0.5f));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the supplied page index is stored on every box")
|
||||
void pageIndexStored() throws Exception {
|
||||
byte[] pdf = pdfWithImageAt(10f, 10f, 20f, 20f);
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
PageImageLocator locator = new PageImageLocator(doc.getPage(0), 7);
|
||||
locator.processPage(doc.getPage(0));
|
||||
assertThat(locator.getImageBoxes().get(0).pageIndex()).isEqualTo(7);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("two images on one page yield two boxes")
|
||||
void twoImages() throws Exception {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
PDImageXObject image = tinyImage(doc);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.drawImage(image, 50f, 50f, 30f, 30f);
|
||||
cs.drawImage(image, 200f, 400f, 60f, 40f);
|
||||
}
|
||||
byte[] pdf = save(doc);
|
||||
try (PDDocument reopened = Loader.loadPDF(pdf)) {
|
||||
PageImageLocator locator = new PageImageLocator(reopened.getPage(0), 0);
|
||||
locator.processPage(reopened.getPage(0));
|
||||
assertThat(locator.getImageBoxes()).hasSize(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a page with no images yields no boxes")
|
||||
void noImages() throws Exception {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
doc.addPage(new PDPage(PDRectangle.A4));
|
||||
byte[] pdf = save(doc);
|
||||
try (PDDocument reopened = Loader.loadPDF(pdf)) {
|
||||
PageImageLocator locator = new PageImageLocator(reopened.getPage(0), 0);
|
||||
locator.processPage(reopened.getPage(0));
|
||||
assertThat(locator.getImageBoxes()).isEmpty();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("getImageBoxes is empty before any page is processed")
|
||||
void emptyBeforeProcessing() throws Exception {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
doc.addPage(new PDPage(PDRectangle.A4));
|
||||
PageImageLocator locator = new PageImageLocator(doc.getPage(0), 0);
|
||||
assertThat(locator.getImageBoxes()).isEmpty();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("path operation no-ops")
|
||||
class PathNoOps {
|
||||
|
||||
private PageImageLocator newLocator() {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
return new PageImageLocator(page, 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("moveTo updates the current point")
|
||||
void moveToUpdatesPoint() {
|
||||
PageImageLocator locator = newLocator();
|
||||
locator.moveTo(12f, 34f);
|
||||
Point2D current = locator.getCurrentPoint();
|
||||
assertThat(current.getX()).isEqualTo(12d);
|
||||
assertThat(current.getY()).isEqualTo(34d);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("lineTo updates the current point")
|
||||
void lineToUpdatesPoint() {
|
||||
PageImageLocator locator = newLocator();
|
||||
locator.lineTo(5f, 6f);
|
||||
assertThat(locator.getCurrentPoint().getX()).isEqualTo(5d);
|
||||
assertThat(locator.getCurrentPoint().getY()).isEqualTo(6d);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("curveTo updates the current point to the final control point")
|
||||
void curveToUpdatesPoint() {
|
||||
PageImageLocator locator = newLocator();
|
||||
locator.curveTo(1f, 1f, 2f, 2f, 9f, 8f);
|
||||
assertThat(locator.getCurrentPoint().getX()).isEqualTo(9d);
|
||||
assertThat(locator.getCurrentPoint().getY()).isEqualTo(8d);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rectangle, clip, path and shading operations are no-ops that do not throw")
|
||||
void otherOpsDoNotThrow() {
|
||||
PageImageLocator locator = newLocator();
|
||||
Point2D p = new Point2D.Float(0f, 0f);
|
||||
// None of these record anything or alter state; they must simply not throw.
|
||||
locator.appendRectangle(p, p, p, p);
|
||||
locator.clip(0);
|
||||
locator.closePath();
|
||||
locator.endPath();
|
||||
locator.strokePath();
|
||||
locator.fillPath(0);
|
||||
locator.fillAndStrokePath(0);
|
||||
locator.shadingFill(COSName.getPDFName("Sh0"));
|
||||
assertThat(locator.getImageBoxes()).isEmpty();
|
||||
}
|
||||
}
|
||||
}
|
||||
-345
@@ -1,345 +0,0 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.RawPage;
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.TableCell;
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
|
||||
import static stirling.software.SPDF.pdf.parser.PdfModels.TableRow;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Nested;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link TabulaTableParser}. Tables are built in-memory with PDFBox so the tests are
|
||||
* deterministic and need no fixtures, network, or external processes.
|
||||
*/
|
||||
class TabulaTableParserGapTest {
|
||||
|
||||
private final TabulaTableParser parser = new TabulaTableParser();
|
||||
|
||||
// ── error / empty branches ───────────────────────────────────────────────
|
||||
|
||||
@Nested
|
||||
@DisplayName("Empty and error branches")
|
||||
class EmptyAndErrorBranches {
|
||||
|
||||
@Test
|
||||
@DisplayName("page number 0 is out of Tabula's 1-based range -> empty list, no throw")
|
||||
void pageNumberZeroReturnsEmpty() throws Exception {
|
||||
byte[] pdf = pdfWithText(new String[] {"hello"});
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
List<TableFragment> result = parser.parse(doc, 0);
|
||||
assertNotNull(result);
|
||||
assertTrue(result.isEmpty());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("page number beyond the document -> empty list, exception swallowed")
|
||||
void pageNumberOutOfRangeReturnsEmpty() throws Exception {
|
||||
byte[] pdf = pdfWithText(new String[] {"hello"});
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
List<TableFragment> result = parser.parse(doc, 99);
|
||||
assertNotNull(result);
|
||||
assertTrue(result.isEmpty());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("negative page number -> empty list")
|
||||
void negativePageNumberReturnsEmpty() throws Exception {
|
||||
byte[] pdf = pdfWithText(new String[] {"hello"});
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
assertTrue(parser.parse(doc, -5).isEmpty());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("lattice mode on a page with no ruled lines -> no tables")
|
||||
void latticeWithNoRulingsReturnsEmpty() throws Exception {
|
||||
byte[] pdf = pdfWithText(new String[] {"just some prose", "no table here"});
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
List<TableFragment> result = parser.parse(doc, new RawPage(1, 0f, 0f, List.of()));
|
||||
assertNotNull(result);
|
||||
assertTrue(
|
||||
result.isEmpty(), "borderless text must not be detected in lattice mode");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("blank page in lattice mode -> empty list")
|
||||
void blankPageLatticeReturnsEmpty() throws Exception {
|
||||
byte[] pdf = blankPdf();
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
assertTrue(parser.parse(doc, new RawPage(1, 0f, 0f, List.of())).isEmpty());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── stream mode (BasicExtractionAlgorithm) ───────────────────────────────
|
||||
|
||||
@Nested
|
||||
@DisplayName("Stream mode")
|
||||
class StreamMode {
|
||||
|
||||
@Test
|
||||
@DisplayName("page with text yields at least one well-formed fragment")
|
||||
void streamOnTextProducesFragment() throws Exception {
|
||||
byte[] pdf =
|
||||
pdfWithText(new String[] {"Name Age City", "Alice 30 Paris", "Bob 25 Rome"});
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
List<TableFragment> fragments =
|
||||
parser.parseStream(doc, new RawPage(1, 0f, 0f, List.of()));
|
||||
assertNotNull(fragments);
|
||||
assertFalse(fragments.isEmpty(), "stream mode always builds a table from text");
|
||||
assertFragmentWellFormed(fragments.get(0), 1, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("fragment ids encode page and index")
|
||||
void streamFragmentIdFormat() throws Exception {
|
||||
byte[] pdf = pdfWithText(new String[] {"col1 col2", "a b"});
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
List<TableFragment> fragments =
|
||||
parser.parseStream(doc, new RawPage(1, 0f, 0f, List.of()));
|
||||
assertFalse(fragments.isEmpty());
|
||||
assertEquals("tbl-p1-0", fragments.get(0).tableId());
|
||||
assertEquals(1, fragments.get(0).pageNumber());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rawRows and the parsed rows stay in lockstep")
|
||||
void streamRowsMatchRawRows() throws Exception {
|
||||
byte[] pdf = pdfWithText(new String[] {"x y", "1 2", "3 4"});
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
List<TableFragment> fragments =
|
||||
parser.parseStream(doc, new RawPage(1, 0f, 0f, List.of()));
|
||||
assertFalse(fragments.isEmpty());
|
||||
TableFragment f = fragments.get(0);
|
||||
assertEquals(f.rawRows().size(), f.rows().size());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── lattice mode with a real bordered grid ───────────────────────────────
|
||||
|
||||
@Nested
|
||||
@DisplayName("Lattice mode")
|
||||
class LatticeMode {
|
||||
|
||||
@Test
|
||||
@DisplayName("bordered grid is detected and produces well-formed fragments")
|
||||
void latticeDetectsBorderedTable() throws Exception {
|
||||
byte[] pdf = pdfWithGrid();
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
List<TableFragment> fragments =
|
||||
parser.parse(doc, new RawPage(1, 0f, 0f, List.of()));
|
||||
assertNotNull(fragments);
|
||||
assertFalse(
|
||||
fragments.isEmpty(), "a clean ruled grid must be detected in lattice mode");
|
||||
TableFragment f = fragments.get(0);
|
||||
assertFragmentWellFormed(f, 1, 0);
|
||||
assertTrue(f.columnCount() >= 1, "a detected grid must have at least one column");
|
||||
assertFalse(f.rawRows().isEmpty(), "a detected grid must have rows");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("convenience overload with page number routes to lattice mode")
|
||||
void parseByPageNumberDetectsGrid() throws Exception {
|
||||
byte[] pdf = pdfWithGrid();
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
List<TableFragment> fragments = parser.parse(doc, 1);
|
||||
assertNotNull(fragments);
|
||||
assertFalse(fragments.isEmpty());
|
||||
assertEquals(1, fragments.get(0).pageNumber());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("cell text is normalised (trimmed, newlines collapsed)")
|
||||
void latticeCellTextIsNormalised() throws Exception {
|
||||
byte[] pdf = pdfWithGrid();
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
List<TableFragment> fragments =
|
||||
parser.parse(doc, new RawPage(1, 0f, 0f, List.of()));
|
||||
assertFalse(fragments.isEmpty());
|
||||
for (List<String> row : fragments.get(0).rawRows()) {
|
||||
for (String cell : row) {
|
||||
assertNotNull(cell);
|
||||
assertFalse(cell.contains("\n"), "newlines must be collapsed");
|
||||
assertFalse(cell.contains("\r"), "carriage returns must be collapsed");
|
||||
assertEquals(cell.trim(), cell, "cell text must be trimmed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── contract invariants ──────────────────────────────────────────────────
|
||||
|
||||
@Nested
|
||||
@DisplayName("Contract invariants")
|
||||
class ContractInvariants {
|
||||
|
||||
@Test
|
||||
@DisplayName("parse never returns null")
|
||||
void parseNeverReturnsNull() throws Exception {
|
||||
byte[] pdf = pdfWithText(new String[] {"abc"});
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
assertNotNull(parser.parse(doc, new RawPage(1, 0f, 0f, List.of())));
|
||||
assertNotNull(parser.parse(doc, 1));
|
||||
assertNotNull(parser.parseStream(doc, new RawPage(1, 0f, 0f, List.of())));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the document is not closed by the parser")
|
||||
void documentRemainsOpenAfterParse() throws Exception {
|
||||
byte[] pdf = pdfWithText(new String[] {"keep me open"});
|
||||
try (PDDocument doc = Loader.loadPDF(pdf)) {
|
||||
parser.parse(doc, new RawPage(1, 0f, 0f, List.of()));
|
||||
parser.parseStream(doc, new RawPage(1, 0f, 0f, List.of()));
|
||||
// ObjectExtractor.close() would close the underlying COSDocument; the parser must
|
||||
// not.
|
||||
assertFalse(
|
||||
doc.getDocument().isClosed(),
|
||||
"parser must not close the caller's document");
|
||||
assertEquals(1, doc.getNumberOfPages());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
/** Asserts every field of a fragment satisfies the documented contract. */
|
||||
private static void assertFragmentWellFormed(
|
||||
TableFragment f, int expectedPage, int expectedIndex) {
|
||||
assertNotNull(f);
|
||||
assertEquals(expectedPage, f.pageNumber());
|
||||
assertEquals("tbl-p" + expectedPage + "-" + expectedIndex, f.tableId());
|
||||
assertNotNull(f.bounds());
|
||||
assertNotNull(f.headers());
|
||||
assertTrue(f.headers().isEmpty(), "headers are deferred to v2 and must be empty");
|
||||
assertNotNull(f.rows());
|
||||
assertNotNull(f.rawRows());
|
||||
assertNotNull(f.warnings());
|
||||
assertSame(null, f.continuedFromPage(), "continuedFromPage is deferred to v2");
|
||||
assertTrue(f.columnCount() >= 0);
|
||||
assertTrue(f.confidence() >= 0f && f.confidence() <= 1f, "confidence must be within [0,1]");
|
||||
assertEquals(f.rawRows().size(), f.rows().size());
|
||||
|
||||
for (TableRow row : f.rows()) {
|
||||
assertNotNull(row.cells());
|
||||
for (TableCell cell : row.cells()) {
|
||||
assertNotNull(cell.text());
|
||||
assertNotNull(cell.bounds());
|
||||
assertEquals(1, cell.colSpan(), "colSpan is always 1 in v1");
|
||||
assertEquals(1, cell.rowSpan(), "rowSpan is always 1 in v1");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] pdfWithText(String[] lines) throws Exception {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
cs.setNonStrokingColor(Color.BLACK);
|
||||
float y = 720f;
|
||||
for (String line : lines) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(72f, y);
|
||||
cs.showText(line);
|
||||
cs.endText();
|
||||
y -= 20f;
|
||||
}
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] blankPdf() throws Exception {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
doc.addPage(new PDPage(PDRectangle.A4));
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a small 3-row x 3-column ruled grid with text in each cell. The ruled lines make the
|
||||
* table detectable by lattice mode.
|
||||
*/
|
||||
private static byte[] pdfWithGrid() throws Exception {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
|
||||
float left = 100f;
|
||||
float right = 400f;
|
||||
float top = 700f;
|
||||
float bottom = 550f;
|
||||
int cols = 3;
|
||||
int rows = 3;
|
||||
float colStep = (right - left) / cols;
|
||||
float rowStep = (top - bottom) / rows;
|
||||
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.setStrokingColor(Color.BLACK);
|
||||
cs.setLineWidth(1f);
|
||||
|
||||
// vertical lines
|
||||
for (int c = 0; c <= cols; c++) {
|
||||
float x = left + c * colStep;
|
||||
cs.moveTo(x, bottom);
|
||||
cs.lineTo(x, top);
|
||||
}
|
||||
// horizontal lines
|
||||
for (int r = 0; r <= rows; r++) {
|
||||
float yLine = bottom + r * rowStep;
|
||||
cs.moveTo(left, yLine);
|
||||
cs.lineTo(right, yLine);
|
||||
}
|
||||
cs.stroke();
|
||||
|
||||
// cell text
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
|
||||
cs.setNonStrokingColor(Color.BLACK);
|
||||
for (int r = 0; r < rows; r++) {
|
||||
for (int c = 0; c < cols; c++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(left + c * colStep + 5f, top - (r + 1) * rowStep + 6f);
|
||||
cs.showText("R" + r + "C" + c);
|
||||
cs.endText();
|
||||
}
|
||||
}
|
||||
}
|
||||
return save(doc);
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] save(PDDocument doc) throws Exception {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
doc.save(baos);
|
||||
return baos.toByteArray();
|
||||
}
|
||||
}
|
||||
+1
-3
@@ -24,9 +24,7 @@ class InProcessDistributedLockTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void reentryFromSameThreadFails_parityWithValkey() {
|
||||
// The Valkey impl refuses reentry (SET NX semantics); the in-process impl must match,
|
||||
// otherwise code working in single-instance silently breaks in cluster mode.
|
||||
void reentryFromSameThreadFails() {
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
|
||||
Optional<DistributedLock.LockHandle> reentry = lock.tryAcquire("k", Duration.ofSeconds(30));
|
||||
|
||||
-44
@@ -3,13 +3,11 @@ package stirling.software.common.cluster.inprocess;
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -41,46 +39,4 @@ class LocalDiskFileStoreTest {
|
||||
assertThrows(IllegalArgumentException.class, () -> store.resolve("a/b"));
|
||||
assertThrows(IllegalArgumentException.class, () -> store.resolve("a\\b"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ownerSidecarCannotBeReadAsFileId(@TempDir Path dir) throws IOException {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
FileStore.Stored stored =
|
||||
store.store(new ByteArrayInputStream("hi".getBytes()), "f.bin", "alice");
|
||||
String sidecarId = stored.fileId() + ".owner";
|
||||
assertThrows(IllegalArgumentException.class, () -> store.resolve(sidecarId));
|
||||
assertThrows(IllegalArgumentException.class, () -> store.retrieveBytes(sidecarId));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ownerIsPersistedAndReturnedByGetOwner(@TempDir Path dir) throws IOException {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
FileStore.Stored stored =
|
||||
store.store(new ByteArrayInputStream("hi".getBytes()), "f.bin", "alice");
|
||||
assertEquals("alice", store.getOwner(stored.fileId()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void getOwnerReturnsNullWhenNoOwnerWasRecorded(@TempDir Path dir) throws IOException {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
FileStore.Stored stored =
|
||||
store.store(new ByteArrayInputStream("hi".getBytes()), "f.bin", null);
|
||||
assertNull(store.getOwner(stored.fileId()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void getOwnerReturnsNullForUnknownFileId(@TempDir Path dir) throws IOException {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
assertNull(store.getOwner("00000000-0000-0000-0000-000000000000"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteRemovesOwnerSidecar(@TempDir Path dir) throws IOException {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
FileStore.Stored stored =
|
||||
store.store(new ByteArrayInputStream("hi".getBytes()), "f.bin", "alice");
|
||||
assertTrue(store.delete(stored.fileId()));
|
||||
assertFalse(Files.exists(dir.resolve(stored.fileId() + ".owner")));
|
||||
assertNull(store.getOwner(stored.fileId()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,270 +0,0 @@
|
||||
package stirling.software.common.configuration;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Nested;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.mock.env.MockEnvironment;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
|
||||
class AppConfigTest {
|
||||
|
||||
private ApplicationProperties applicationProperties;
|
||||
private MockEnvironment env;
|
||||
private AppConfig appConfig;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
applicationProperties = new ApplicationProperties();
|
||||
env = new MockEnvironment();
|
||||
appConfig = new AppConfig(env, applicationProperties);
|
||||
ReflectionTestUtils.setField(appConfig, "contextPath", "/");
|
||||
ReflectionTestUtils.setField(appConfig, "serverPort", "8080");
|
||||
ReflectionTestUtils.setField(appConfig, "v2Enabled", true);
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("Value-backed getters and simple beans")
|
||||
class SimpleBeans {
|
||||
|
||||
@Test
|
||||
@DisplayName("getter fields reflect injected @Value values")
|
||||
void valueGetters() {
|
||||
assertThat(appConfig.getContextPath()).isEqualTo("/");
|
||||
assertThat(appConfig.getServerPort()).isEqualTo("8080");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("v2Enabled bean mirrors the field")
|
||||
void v2EnabledBean() {
|
||||
assertThat(appConfig.v2Enabled()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("constant beans return fixed values")
|
||||
void constants() {
|
||||
assertThat(appConfig.appName()).isEqualTo("Stirling PDF");
|
||||
assertThat(appConfig.homeText()).isEqualTo("null");
|
||||
assertThat(appConfig.contextPath("/ctx")).isEqualTo("/ctx");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("appVersion resolves from version.properties on classpath")
|
||||
void appVersion() {
|
||||
assertThat(appConfig.appVersion()).isNotBlank();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("StirlingPDFLabel embeds version")
|
||||
void stirlingLabel() {
|
||||
assertThat(appConfig.stirlingPDFLabel()).startsWith("Stirling-PDF v");
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("Beans backed by ApplicationProperties")
|
||||
class PropertyBackedBeans {
|
||||
|
||||
@Test
|
||||
@DisplayName("loginEnabled reflects security flag")
|
||||
void loginEnabled() {
|
||||
applicationProperties.getSecurity().setEnableLogin(true);
|
||||
assertThat(appConfig.loginEnabled()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("backendUrl falls back to localhost when unset")
|
||||
void backendUrlFallback() {
|
||||
assertThat(appConfig.getBackendUrl()).isEqualTo("http://localhost");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("backendUrl returns configured value when present")
|
||||
void backendUrlConfigured() {
|
||||
applicationProperties.getSystem().setBackendUrl("https://api.example.com");
|
||||
assertThat(appConfig.getBackendUrl()).isEqualTo("https://api.example.com");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("languages bean returns configured languages list")
|
||||
void languages() {
|
||||
applicationProperties.getUi().setLanguages(List.of("en", "de"));
|
||||
assertThat(appConfig.languages()).containsExactly("en", "de");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("navBarText falls back to Stirling PDF when unset")
|
||||
void navBarTextFallback() {
|
||||
assertThat(appConfig.navBarText()).isEqualTo("Stirling PDF");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("navBarText returns configured value")
|
||||
void navBarTextConfigured() {
|
||||
applicationProperties.getUi().setAppNameNavbar("My PDF");
|
||||
assertThat(appConfig.navBarText()).isEqualTo("My PDF");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("enableAlphaFunctionality reflects system flag")
|
||||
void alphaFunctionality() {
|
||||
applicationProperties.getSystem().setEnableAlphaFunctionality(true);
|
||||
assertThat(appConfig.enableAlphaFunctionality()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("legal text beans return configured values")
|
||||
void legalBeans() {
|
||||
var legal = applicationProperties.getLegal();
|
||||
legal.setTermsAndConditions("terms");
|
||||
legal.setPrivacyPolicy("privacy");
|
||||
legal.setCookiePolicy("cookie");
|
||||
legal.setImpressum("impressum");
|
||||
legal.setAccessibilityStatement("a11y");
|
||||
assertThat(appConfig.termsAndConditions()).isEqualTo("terms");
|
||||
assertThat(appConfig.privacyPolicy()).isEqualTo("privacy");
|
||||
assertThat(appConfig.cookiePolicy()).isEqualTo("cookie");
|
||||
assertThat(appConfig.impressum()).isEqualTo("impressum");
|
||||
assertThat(appConfig.accessibilityStatement()).isEqualTo("a11y");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("analyticsPrompt true when enableAnalytics null")
|
||||
void analyticsPrompt() {
|
||||
applicationProperties.getSystem().setEnableAnalytics(null);
|
||||
assertThat(appConfig.analyticsPrompt()).isTrue();
|
||||
applicationProperties.getSystem().setEnableAnalytics(Boolean.TRUE);
|
||||
assertThat(appConfig.analyticsPrompt()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("analyticsEnabled true when premium enabled regardless of system flag")
|
||||
void analyticsEnabledViaPremium() {
|
||||
applicationProperties.getPremium().setEnabled(true);
|
||||
assertThat(appConfig.analyticsEnabled()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("analyticsEnabled reflects system flag when premium disabled")
|
||||
void analyticsEnabledViaSystem() {
|
||||
applicationProperties.getPremium().setEnabled(false);
|
||||
applicationProperties.getSystem().setEnableAnalytics(Boolean.TRUE);
|
||||
assertThat(appConfig.analyticsEnabled()).isTrue();
|
||||
applicationProperties.getSystem().setEnableAnalytics(Boolean.FALSE);
|
||||
assertThat(appConfig.analyticsEnabled()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("scarf and posthog beans reflect derived flags")
|
||||
void scarfAndPosthog() {
|
||||
applicationProperties.getSystem().setEnableAnalytics(Boolean.TRUE);
|
||||
applicationProperties.getSystem().setEnableScarf(Boolean.TRUE);
|
||||
applicationProperties.getSystem().setEnablePosthog(Boolean.TRUE);
|
||||
assertThat(appConfig.scarfEnabled()).isTrue();
|
||||
assertThat(appConfig.posthogEnabled()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("uuid bean returns generated UUID")
|
||||
void uuidBean() {
|
||||
applicationProperties.getAutomaticallyGenerated().setUUID("abc-123");
|
||||
assertThat(appConfig.uuid()).isEqualTo("abc-123");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("typed config beans return live nested instances")
|
||||
void typedConfigBeans() {
|
||||
assertThat(appConfig.security()).isSameAs(applicationProperties.getSecurity());
|
||||
assertThat(appConfig.oAuth2())
|
||||
.isSameAs(applicationProperties.getSecurity().getOauth2());
|
||||
assertThat(appConfig.premium()).isSameAs(applicationProperties.getPremium());
|
||||
assertThat(appConfig.system()).isSameAs(applicationProperties.getSystem());
|
||||
assertThat(appConfig.datasource())
|
||||
.isSameAs(applicationProperties.getSystem().getDatasource());
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("Profile-default and environment beans")
|
||||
class ProfileAndEnvBeans {
|
||||
|
||||
@Test
|
||||
@DisplayName("default-profile license beans return community defaults")
|
||||
void licenseDefaults() {
|
||||
assertThat(appConfig.runningProOrHigher()).isFalse();
|
||||
assertThat(appConfig.runningEnterprise()).isFalse();
|
||||
assertThat(appConfig.licenseType()).isEqualTo("NORMAL");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("activeSecurity reflects classpath presence of SecurityConfiguration")
|
||||
void activeSecurity() {
|
||||
// Just exercise the branch; result depends on classpath, assert it does not throw.
|
||||
boolean present = appConfig.missingActiveSecurity();
|
||||
assertThat(present).isIn(true, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rateLimit parses system property")
|
||||
void rateLimitProperty() {
|
||||
String prev = System.getProperty("rateLimit");
|
||||
try {
|
||||
System.setProperty("rateLimit", "true");
|
||||
assertThat(appConfig.rateLimit()).isTrue();
|
||||
} finally {
|
||||
if (prev == null) {
|
||||
System.clearProperty("rateLimit");
|
||||
} else {
|
||||
System.setProperty("rateLimit", prev);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("runningInDocker false outside container")
|
||||
void runningInDocker() {
|
||||
// CI/test host is not a container with /.dockerenv.
|
||||
assertThat(appConfig.runningInDocker()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("configDirMounted defaults to true when not in docker")
|
||||
void configDirMounted() {
|
||||
assertThat(appConfig.isRunningInDockerWithConfig()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("directoryFilter accepts files and rejects processing dirs")
|
||||
void directoryFilter(@org.junit.jupiter.api.io.TempDir Path tempDir) throws Exception {
|
||||
Predicate<Path> filter = appConfig.processOnlyFiles();
|
||||
Path file = Files.createFile(tempDir.resolve("a.txt"));
|
||||
Path normalDir = Files.createDirectory(tempDir.resolve("normal"));
|
||||
Path processingDir = Files.createDirectory(tempDir.resolve("processing"));
|
||||
assertThat(filter.test(file)).isTrue();
|
||||
assertThat(filter.test(normalDir)).isTrue();
|
||||
assertThat(filter.test(processingDir)).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("machineType returns Server-jar in plain test environment")
|
||||
void machineTypeServerJar() {
|
||||
assertThat(appConfig.determineMachineType()).isEqualTo("Server-jar");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("machineType returns a Client-* variant when BROWSER_OPEN set")
|
||||
void machineTypeClient() {
|
||||
env.setProperty("BROWSER_OPEN", "true");
|
||||
assertThat(appConfig.determineMachineType()).startsWith("Client-");
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user