Compare commits

..
2 Commits
Author SHA1 Message Date
Anthony Stirling 4a09f30722 Delete .github/workflows directory 2025-09-22 14:00:15 +01:00
Anthony Stirling 2df943e110 init 2025-05-26 00:03:19 +01:00
391 changed files with 4652 additions and 19089 deletions
-21
View File
@@ -1,21 +0,0 @@
{
"permissions": {
"allow": [
"Bash(mkdir:*)",
"Bash(./gradlew:*)",
"Bash(grep:*)",
"Bash(find:*)",
"Bash(chmod:*)",
"Bash(rm:*)",
"Bash(cat:*)",
"Bash(if [ -d logs ])",
"Bash(then ls -la logs/)",
"Bash(fi)",
"Bash(ls:*)",
"Bash(./testing/cucumber/features/environment.py -v)",
"Bash(./testing/cucumber/features/autojob.feature -v)",
"Bash(git rev-parse:*)"
],
"deny": []
}
}
-33
View File
@@ -1,33 +0,0 @@
name: 'Setup GitHub App Bot'
description: 'Generates a GitHub App Token and configures Git for a bot'
inputs:
app-id:
description: 'GitHub App ID'
required: True
private-key:
description: 'GitHub App Private Key'
required: True
outputs:
token:
description: 'Generated GitHub App Token'
value: ${{ steps.generate-token.outputs.token }}
committer:
description: 'Committer string for Git'
value: "${{ steps.generate-token.outputs.app-slug }}[bot] <${{ steps.generate-token.outputs.app-slug }}[bot]@users.noreply.github.com>"
app-slug:
description: 'GitHub App slug'
value: ${{ steps.generate-token.outputs.app-slug }}
runs:
using: 'composite'
steps:
- name: Generate a GitHub App Token
id: generate-token
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
with:
app-id: ${{ inputs.app-id }}
private-key: ${{ inputs.private-key }}
- name: Configure Git
run: |
git config --global user.name "${{ steps.generate-token.outputs.app-slug }}[bot]"
git config --global user.email "${{ steps.generate-token.outputs.app-slug }}[bot]@users.noreply.github.com"
shell: bash
@@ -1,320 +0,0 @@
name: PR Deployment via Comment
on:
issue_comment:
types: [created]
permissions:
contents: read
issues: write # Required for adding reactions to comments
pull-requests: read # Required for reading PR information
jobs:
check-comment:
runs-on: ubuntu-latest
permissions:
issues: write
pull-requests: read
if: |
github.event.issue.pull_request &&
(
contains(github.event.comment.body, 'prdeploy') ||
contains(github.event.comment.body, 'deploypr')
)
&&
(
github.event.comment.user.login == 'frooodle' ||
github.event.comment.user.login == 'sf298' ||
github.event.comment.user.login == 'Ludy87' ||
github.event.comment.user.login == 'LaserKaspar' ||
github.event.comment.user.login == 'sbplat' ||
github.event.comment.user.login == 'reecebrowne' ||
github.event.comment.user.login == 'DarioGii' ||
github.event.comment.user.login == 'ConnorYoh'
)
outputs:
pr_number: ${{ steps.get-pr.outputs.pr_number }}
pr_repository: ${{ steps.get-pr-info.outputs.repository }}
pr_ref: ${{ steps.get-pr-info.outputs.ref }}
comment_id: ${{ github.event.comment.id }}
enable_security: ${{ steps.check-security-flag.outputs.enable_security }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
# Generate GitHub App token
- name: Generate GitHub App Token
id: generate-token
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Get PR data
id: get-pr
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
const prNumber = context.payload.issue.number;
console.log(`PR Number: ${prNumber}`);
core.setOutput('pr_number', prNumber);
- name: Get PR repository and ref
id: get-pr-info
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
const { owner, repo } = context.repo;
const prNumber = context.payload.issue.number;
const { data: pr } = await github.rest.pulls.get({
owner,
repo,
pull_number: prNumber,
});
// For forks, use the full repository name, for internal PRs use the current repo
const repository = pr.head.repo.fork ? pr.head.repo.full_name : `${owner}/${repo}`;
console.log(`PR Repository: ${repository}`);
console.log(`PR Branch: ${pr.head.ref}`);
core.setOutput('repository', repository);
core.setOutput('ref', pr.head.ref);
- name: Check for security/login flag
id: check-security-flag
env:
COMMENT_BODY: ${{ github.event.comment.body }}
run: |
if [[ "$COMMENT_BODY" == *"security"* ]] || [[ "$COMMENT_BODY" == *"login"* ]]; then
echo "Security flags detected in comment"
echo "enable_security=true" >> $GITHUB_OUTPUT
else
echo "No security flags detected in comment"
echo "enable_security=false" >> $GITHUB_OUTPUT
fi
- name: Add 'in_progress' reaction to comment
id: add-eyes-reaction
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
github-token: ${{ steps.generate-token.outputs.token }}
script: |
console.log(`Adding eyes reaction to comment ID: ${context.payload.comment.id}`);
try {
const { data: reaction } = await github.rest.reactions.createForIssueComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: context.payload.comment.id,
content: 'eyes'
});
console.log(`Added reaction with ID: ${reaction.id}`);
return { success: true, id: reaction.id };
} catch (error) {
console.error(`Failed to add reaction: ${error.message}`);
console.error(error);
return { success: false, error: error.message };
}
deploy-pr:
needs: check-comment
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Generate GitHub App Token
id: generate-token
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Checkout PR
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
repository: ${{ needs.check-comment.outputs.pr_repository }}
ref: ${{ needs.check-comment.outputs.pr_ref }}
token: ${{ secrets.GITHUB_TOKEN }}
- name: Set up JDK
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
with:
java-version: "17"
distribution: "temurin"
- name: Run Gradle Command
run: |
if [ "${{ needs.check-comment.outputs.enable_security }}" == "true" ]; then
export DOCKER_ENABLE_SECURITY=true
else
export DOCKER_ENABLE_SECURITY=false
fi
./gradlew clean build
env:
STIRLING_PDF_DESKTOP_UI: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
- name: Get version number
id: versionNumber
run: |
VERSION=$(grep "^version =" build.gradle | awk -F'"' '{print $2}')
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
- name: Login to Docker Hub
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_API }}
- name: Build and push PR-specific image
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
context: .
file: ./Dockerfile
push: true
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:pr-${{ needs.check-comment.outputs.pr_number }}
build-args: VERSION_TAG=alpha
platforms: linux/amd64
- name: Set up SSH
run: |
mkdir -p ~/.ssh/
echo "${{ secrets.VPS_SSH_KEY }}" > ../private.key
sudo chmod 600 ../private.key
- name: Deploy to VPS
id: deploy
run: |
# Set security settings based on flags
if [ "${{ needs.check-comment.outputs.enable_security }}" == "true" ]; then
DOCKER_SECURITY="true"
LOGIN_SECURITY="true"
SECURITY_STATUS="🔒 Security Enabled"
else
DOCKER_SECURITY="false"
LOGIN_SECURITY="false"
SECURITY_STATUS="Security Disabled"
fi
# First create the docker-compose content locally
cat > docker-compose.yml << EOF
version: '3.3'
services:
stirling-pdf:
container_name: stirling-pdf-pr-${{ needs.check-comment.outputs.pr_number }}
image: ${{ secrets.DOCKER_HUB_USERNAME }}/test:pr-${{ needs.check-comment.outputs.pr_number }}
ports:
- "${{ needs.check-comment.outputs.pr_number }}:8080"
volumes:
- /stirling/PR-${{ needs.check-comment.outputs.pr_number }}/data:/usr/share/tessdata:rw
- /stirling/PR-${{ needs.check-comment.outputs.pr_number }}/config:/configs:rw
- /stirling/PR-${{ needs.check-comment.outputs.pr_number }}/logs:/logs:rw
environment:
DOCKER_ENABLE_SECURITY: "${DOCKER_SECURITY}"
SECURITY_ENABLELOGIN: "${LOGIN_SECURITY}"
SYSTEM_DEFAULTLOCALE: en-GB
UI_APPNAME: "Stirling-PDF PR#${{ needs.check-comment.outputs.pr_number }}"
UI_HOMEDESCRIPTION: "PR#${{ needs.check-comment.outputs.pr_number }} for Stirling-PDF Latest"
UI_APPNAMENAVBAR: "PR#${{ needs.check-comment.outputs.pr_number }}"
SYSTEM_MAXFILESIZE: "100"
METRICS_ENABLED: "true"
SYSTEM_GOOGLEVISIBILITY: "false"
restart: on-failure:5
EOF
# Then copy the file and execute commands
scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null docker-compose.yml ${{ secrets.VPS_USERNAME }}@${{ secrets.VPS_HOST }}:/tmp/docker-compose.yml
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${{ secrets.VPS_USERNAME }}@${{ secrets.VPS_HOST }} << ENDSSH
# Create PR-specific directories
mkdir -p /stirling/PR-${{ needs.check-comment.outputs.pr_number }}/{data,config,logs}
# Move docker-compose file to correct location
mv /tmp/docker-compose.yml /stirling/PR-${{ needs.check-comment.outputs.pr_number }}/docker-compose.yml
# Start or restart the container
cd /stirling/PR-${{ needs.check-comment.outputs.pr_number }}
docker-compose pull
docker-compose up -d
ENDSSH
# Set output for use in PR comment
echo "security_status=${SECURITY_STATUS}" >> $GITHUB_ENV
- name: Add success reaction to comment
if: success()
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
github-token: ${{ steps.generate-token.outputs.token }}
script: |
console.log(`Adding rocket reaction to comment ID: ${{ needs.check-comment.outputs.comment_id }}`);
try {
const { data: reaction } = await github.rest.reactions.createForIssueComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: ${{ needs.check-comment.outputs.comment_id }},
content: 'rocket'
});
console.log(`Added rocket reaction with ID: ${reaction.id}`);
} catch (error) {
console.error(`Failed to add reaction: ${error.message}`);
console.error(error);
}
- name: Add failure reaction to comment
if: failure()
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
github-token: ${{ steps.generate-token.outputs.token }}
script: |
console.log(`Adding -1 reaction to comment ID: ${{ needs.check-comment.outputs.comment_id }}`);
try {
const { data: reaction } = await github.rest.reactions.createForIssueComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: ${{ needs.check-comment.outputs.comment_id }},
content: '-1'
});
console.log(`Added -1 reaction with ID: ${reaction.id}`);
} catch (error) {
console.error(`Failed to add reaction: ${error.message}`);
console.error(error);
}
- name: Post deployment URL to PR
if: success()
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
github-token: ${{ steps.generate-token.outputs.token }}
script: |
const { GITHUB_REPOSITORY } = process.env;
const [repoOwner, repoName] = GITHUB_REPOSITORY.split('/');
const prNumber = ${{ needs.check-comment.outputs.pr_number }};
const securityStatus = process.env.security_status || "Security Disabled";
const deploymentUrl = `http://${{ secrets.VPS_HOST }}:${prNumber}`;
const commentBody = `## 🚀 PR Test Deployment\n\n` +
`Your PR has been deployed for testing!\n\n` +
`🔗 **Test URL:** [${deploymentUrl}](${deploymentUrl})\n` +
`${securityStatus}\n\n` +
`This deployment will be automatically cleaned up when the PR is closed.\n\n`;
await github.rest.issues.createComment({
owner: repoOwner,
repo: repoName,
issue_number: prNumber,
body: commentBody
});
-59
View File
@@ -1,59 +0,0 @@
name: PR Deployment cleanup
on:
pull_request:
types: [opened, synchronize, reopened, closed]
permissions:
contents: read
env:
SERVER_IP: ${{ secrets.VPS_IP }} # Add this to your GitHub secrets
CLEANUP_PERFORMED: "false" # Add flag to track if cleanup occurred
jobs:
cleanup:
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
if: github.event.action == 'closed'
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Set up SSH
run: |
mkdir -p ~/.ssh/
echo "${{ secrets.VPS_SSH_KEY }}" > ../private.key
sudo chmod 600 ../private.key
- name: Cleanup PR deployment
id: cleanup
run: |
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${{ secrets.VPS_USERNAME }}@${{ secrets.VPS_HOST }} << 'ENDSSH'
if [ -d "/stirling/PR-${{ github.event.pull_request.number }}" ]; then
echo "Found PR directory, proceeding with cleanup..."
# Stop and remove containers
cd /stirling/PR-${{ github.event.pull_request.number }}
docker-compose down || true
# Go back to root before removal
cd /
# Remove PR-specific directories
rm -rf /stirling/PR-${{ github.event.pull_request.number }}
# Remove the Docker image
docker rmi --no-prune ${{ secrets.DOCKER_HUB_USERNAME }}/test:pr-${{ github.event.pull_request.number }} || true
echo "PERFORMED_CLEANUP"
else
echo "PR directory not found, nothing to clean up"
echo "NO_CLEANUP_NEEDED"
fi
ENDSSH
-27
View File
@@ -1,27 +0,0 @@
name: "Pull Request Labeler"
on:
pull_request_target:
types: [opened, synchronize]
permissions:
contents: read
jobs:
labeler:
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Apply Labels
uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
configuration-path: .github/labeler-config.yml
sync-labels: true
-148
View File
@@ -1,148 +0,0 @@
name: Build repo
on:
push:
branches: ["main"]
pull_request:
branches: ["main"]
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
permissions:
actions: read
security-events: write
strategy:
fail-fast: false
matrix:
jdk-version: [17, 21]
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up JDK ${{ matrix.jdk-version }}
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
with:
java-version: ${{ matrix.jdk-version }}
distribution: "temurin"
- name: Build with Gradle and no spring security
run: ./gradlew clean build
env:
DOCKER_ENABLE_SECURITY: false
- name: Build with Gradle and with spring security
run: ./gradlew clean build
env:
DOCKER_ENABLE_SECURITY: true
- name: Upload Test Reports
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: test-reports-jdk-${{ matrix.jdk-version }}
path: |
build/reports/tests/
build/test-results/
build/reports/problems/
common/build/reports/tests/
common/build/test-results/
common/build/reports/problems/
retention-days: 3
check-licence:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up JDK 17
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
with:
java-version: "17"
distribution: "adopt"
- name: check the licenses for compatibility
run: ./gradlew clean checkLicense
- name: FAILED - check the licenses for compatibility
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: dependencies-without-allowed-license.json
path: |
build/reports/dependency-license/dependencies-without-allowed-license.json
retention-days: 3
docker-compose-tests:
# if: github.event_name == 'push' && github.ref == 'refs/heads/main' ||
# (github.event_name == 'pull_request' &&
# contains(github.event.pull_request.labels.*.name, 'licenses') == false &&
# (
# contains(github.event.pull_request.labels.*.name, 'Front End') ||
# contains(github.event.pull_request.labels.*.name, 'Java') ||
# contains(github.event.pull_request.labels.*.name, 'Back End') ||
# contains(github.event.pull_request.labels.*.name, 'Security') ||
# contains(github.event.pull_request.labels.*.name, 'API') ||
# contains(github.event.pull_request.labels.*.name, 'Docker') ||
# contains(github.event.pull_request.labels.*.name, 'Test')
# )
# )
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Checkout Repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up Java 17
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
with:
java-version: "17"
distribution: "adopt"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
- name: Install Docker Compose
run: |
sudo curl -SL "https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
cache: 'pip' # caching pip dependencies
- name: Pip requirements
run: |
pip install --require-hashes -r ./testing/cucumber/requirements.txt
- name: Run Docker Compose Tests
run: |
chmod +x ./testing/test_webpages.sh
chmod +x ./testing/test.sh
chmod +x ./testing/test_disabledEndpoints.sh
./testing/test.sh
-255
View File
@@ -1,255 +0,0 @@
name: Check Properties Files on PR
on:
pull_request_target:
types: [opened, synchronize, reopened]
paths:
- "src/main/resources/messages_*.properties"
permissions:
contents: read # Allow read access to repository content
jobs:
check-files:
if: github.event_name == 'pull_request_target'
runs-on: ubuntu-latest
permissions:
issues: write # Allow posting comments on issues/PRs
pull-requests: write # Allow writing to pull requests
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Checkout main branch first
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup GitHub App Bot
id: setup-bot
uses: ./.github/actions/setup-bot
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Get PR data
id: get-pr-data
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
github-token: ${{ steps.setup-bot.outputs.token }}
script: |
const prNumber = context.payload.pull_request.number;
const repoOwner = context.payload.repository.owner.login;
const repoName = context.payload.repository.name;
const branch = context.payload.pull_request.head.ref;
console.log(`PR Number: ${prNumber}`);
console.log(`Repo Owner: ${repoOwner}`);
console.log(`Repo Name: ${repoName}`);
console.log(`Branch: ${branch}`);
core.setOutput("pr_number", prNumber);
core.setOutput("repo_owner", repoOwner);
core.setOutput("repo_name", repoName);
core.setOutput("branch", branch);
continue-on-error: true
- name: Fetch PR changed files
id: fetch-pr-changes
env:
GH_TOKEN: ${{ steps.setup-bot.outputs.token }}
run: |
echo "Fetching PR changed files..."
echo "Getting list of changed files from PR..."
gh pr view ${{ steps.get-pr-data.outputs.pr_number }} --json files -q ".files[].path" | grep -E '^src/main/resources/messages_[a-zA-Z_]{2}_[a-zA-Z_]{2,7}\.properties$' > changed_files.txt # Filter only matching property files
- name: Determine reference file test
id: determine-file
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
github-token: ${{ steps.setup-bot.outputs.token }}
script: |
const fs = require("fs");
const path = require("path");
const prNumber = ${{ steps.get-pr-data.outputs.pr_number }};
const repoOwner = "${{ steps.get-pr-data.outputs.repo_owner }}";
const repoName = "${{ steps.get-pr-data.outputs.repo_name }}";
const prRepoOwner = "${{ github.event.pull_request.head.repo.owner.login }}";
const prRepoName = "${{ github.event.pull_request.head.repo.name }}";
const branch = "${{ steps.get-pr-data.outputs.branch }}";
console.log(`Determining reference file for PR #${prNumber}`);
// Validate inputs
const validateInput = (input, regex, name) => {
if (!regex.test(input)) {
throw new Error(`Invalid ${name}: ${input}`);
}
};
validateInput(repoOwner, /^[a-zA-Z0-9_-]+$/, "repository owner");
validateInput(repoName, /^[a-zA-Z0-9._-]+$/, "repository name");
validateInput(branch, /^[a-zA-Z0-9._/-]+$/, "branch name");
// Get the list of changed files in the PR
const { data: files } = await github.rest.pulls.listFiles({
owner: repoOwner,
repo: repoName,
pull_number: prNumber,
});
// Filter for relevant files based on the PR changes
const changedFiles = files
.map(file => file.filename)
.filter(file => /^src\/main\/resources\/messages_[a-zA-Z_]{2}_[a-zA-Z_]{2,7}\.properties$/.test(file));
console.log("Changed files:", changedFiles);
// Create a temporary directory for PR files
const tempDir = "pr-branch";
if (!fs.existsSync(tempDir)) {
fs.mkdirSync(tempDir, { recursive: true });
}
// Download and save each changed file
for (const file of changedFiles) {
const { data: fileContent } = await github.rest.repos.getContent({
owner: prRepoOwner,
repo: prRepoName,
path: file,
ref: branch,
});
const content = Buffer.from(fileContent.content, "base64").toString("utf-8");
const filePath = path.join(tempDir, file);
const dirPath = path.dirname(filePath);
if (!fs.existsSync(dirPath)) {
fs.mkdirSync(dirPath, { recursive: true });
}
fs.writeFileSync(filePath, content);
console.log(`Saved file: ${filePath}`);
}
// Output the list of changed files for further processing
const fileList = changedFiles.join(" ");
core.exportVariable("FILES_LIST", fileList);
console.log("Files saved and listed in FILES_LIST.");
// Determine reference file
let referenceFilePath;
if (changedFiles.includes("src/main/resources/messages_en_GB.properties")) {
console.log("Using PR branch reference file.");
const { data: fileContent } = await github.rest.repos.getContent({
owner: prRepoOwner,
repo: prRepoName,
path: "src/main/resources/messages_en_GB.properties",
ref: branch,
});
referenceFilePath = "pr-branch-messages_en_GB.properties";
const content = Buffer.from(fileContent.content, "base64").toString("utf-8");
fs.writeFileSync(referenceFilePath, content);
} else {
console.log("Using main branch reference file.");
const { data: fileContent } = await github.rest.repos.getContent({
owner: repoOwner,
repo: repoName,
path: "src/main/resources/messages_en_GB.properties",
ref: "main",
});
referenceFilePath = "main-branch-messages_en_GB.properties";
const content = Buffer.from(fileContent.content, "base64").toString("utf-8");
fs.writeFileSync(referenceFilePath, content);
}
console.log(`Reference file path: ${referenceFilePath}`);
core.exportVariable("REFERENCE_FILE", referenceFilePath);
- name: Run Python script to check files
id: run-check
run: |
echo "Running Python script to check files..."
python .github/scripts/check_language_properties.py \
--actor ${{ github.event.pull_request.user.login }} \
--reference-file "${REFERENCE_FILE}" \
--branch "pr-branch" \
--files "${FILES_LIST[@]}" > result.txt
continue-on-error: true # Continue the job even if this step fails
- name: Capture output
id: capture-output
run: |
if [ -f result.txt ] && [ -s result.txt ]; then
echo "Test, capturing output..."
SCRIPT_OUTPUT=$(cat result.txt)
echo "SCRIPT_OUTPUT<<EOF" >> $GITHUB_ENV
echo "$SCRIPT_OUTPUT" >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
echo "${SCRIPT_OUTPUT}"
# Determine job failure based on script output
if [[ "$SCRIPT_OUTPUT" == *"❌"* ]]; then
echo "FAIL_JOB=true" >> $GITHUB_ENV
else
echo "FAIL_JOB=false" >> $GITHUB_ENV
fi
else
echo "No update found."
echo "SCRIPT_OUTPUT=" >> $GITHUB_ENV
echo "FAIL_JOB=false" >> $GITHUB_ENV
fi
- name: Post comment on PR
if: env.SCRIPT_OUTPUT != ''
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
github-token: ${{ steps.setup-bot.outputs.token }}
script: |
const { GITHUB_REPOSITORY, SCRIPT_OUTPUT } = process.env;
const [repoOwner, repoName] = GITHUB_REPOSITORY.split('/');
const issueNumber = context.issue.number;
// Find existing comment
const comments = await github.rest.issues.listComments({
owner: repoOwner,
repo: repoName,
issue_number: issueNumber
});
const comment = comments.data.find(c => c.body.includes("## 🚀 Translation Verification Summary"));
// Only update or create comments by the action user
const expectedActor = "${{ steps.setup-bot.outputs.app-slug }}[bot]";
if (comment && comment.user.login === expectedActor) {
// Update existing comment
await github.rest.issues.updateComment({
owner: repoOwner,
repo: repoName,
comment_id: comment.id,
body: `## 🚀 Translation Verification Summary\n\n\n${SCRIPT_OUTPUT}\n`
});
console.log("Updated existing comment.");
} else if (!comment) {
// Create new comment if no existing comment is found
await github.rest.issues.createComment({
owner: repoOwner,
repo: repoName,
issue_number: issueNumber,
body: `## 🚀 Translation Verification Summary\n\n\n${SCRIPT_OUTPUT}\n`
});
console.log("Created new comment.");
} else {
console.log("Comment update attempt denied. Actor does not match.");
}
- name: Fail job if errors found
if: env.FAIL_JOB == 'true'
run: |
echo "Failing the job because errors were detected."
exit 1
-79
View File
@@ -1,79 +0,0 @@
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"
#disable for now
#on:
# push:
# branches: ["main"]
# pull_request:
# The branches below must be a subset of the branches above
# branches: ["main"]
# schedule:
# - cron: "0 0 * * 1"
permissions:
contents: read
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: ["java"]
# CodeQL supports [ $supported-codeql-languages ]
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
steps:
- name: Harden Runner
uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
# ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
# If the Autobuild fails above, remove it and uncomment the following three lines.
# modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
# - run: |
# echo "Run, Build Application using script"
# ./location_of_script_within_repo/buildscript.sh
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
with:
category: "/language:${{matrix.language}}"
-27
View File
@@ -1,27 +0,0 @@
# Dependency Review Action
#
# This Action will scan dependency manifest files that change as part of a Pull Request,
# surfacing known-vulnerable versions of the packages declared or updated in the PR.
# Once installed, if the workflow run is marked as required,
# PRs introducing known-vulnerable packages will be blocked from merging.
#
# Source repository: https://github.com/actions/dependency-review-action
name: "Dependency Review"
on: [pull_request]
permissions:
contents: read
jobs:
dependency-review:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: "Checkout Repository"
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: "Dependency Review"
uses: actions/dependency-review-action@da24556b548a50705dd671f47852072ea4c105d9 # v4.7.1
-90
View File
@@ -1,90 +0,0 @@
name: License Report Workflow
on:
push:
branches:
- main
paths:
- "build.gradle"
permissions:
contents: read
jobs:
generate-license-report:
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
repository-projects: write # Required for enabling automerge
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Check out code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
- name: Setup GitHub App Bot
id: setup-bot
uses: ./.github/actions/setup-bot
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Set up JDK 17
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
with:
java-version: "17"
distribution: "adopt"
- name: Setup Gradle
uses: gradle/actions/setup-gradle@8379f6a1328ee0e06e2bb424dadb7b159856a326 # v4.4.0
- name: Check licenses for compatibility
run: ./gradlew clean checkLicense
- name: Upload artifact on failure
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: dependencies-without-allowed-license.json
path: build/reports/dependency-license/dependencies-without-allowed-license.json
retention-days: 3
- name: Move and rename license file
run: |
mv build/reports/dependency-license/index.json src/main/resources/static/3rdPartyLicenses.json
- name: Commit changes
run: |
git add src/main/resources/static/3rdPartyLicenses.json
git diff --staged --quiet || echo "CHANGES_DETECTED=true" >> $GITHUB_ENV
- name: Create Pull Request
id: cpr
if: env.CHANGES_DETECTED == 'true'
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
with:
token: ${{ steps.setup-bot.outputs.token }}
commit-message: "Update 3rd Party Licenses"
committer: ${{ steps.setup-bot.outputs.committer }}
author: ${{ steps.setup-bot.outputs.committer }}
signoff: true
branch: update-3rd-party-licenses
title: "Update 3rd Party Licenses"
body: |
Auto-generated by ${{ steps.setup-bot.outputs.app-slug }}[bot]
labels: licenses,github-actions
draft: false
delete-branch: true
sign-commits: true
- name: Enable Pull Request Automerge
if: steps.cpr.outputs.pull-request-operation == 'created'
run: gh pr merge --squash --auto "${{ steps.cpr.outputs.pull-request-number }}"
env:
GH_TOKEN: ${{ steps.setup-bot.outputs.token }}
-30
View File
@@ -1,30 +0,0 @@
name: Manage labels
on:
schedule:
- cron: "30 20 * * *"
permissions:
contents: read
jobs:
labeler:
name: Labeler
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Check out the repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Run Labeler
uses: crazy-max/ghaction-github-labeler@24d110aa46a59976b8a7f35518cb7f14f434c916 # v5.3.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
yaml-file: .github/labels.yml
skip-delete: true
-314
View File
@@ -1,314 +0,0 @@
name: Test Installers Build
on:
workflow_dispatch:
inputs:
test_mode:
description: "Run in test mode (skip release step)"
required: false
default: "false"
release:
types: [created]
permissions:
contents: read
jobs:
read_versions:
runs-on: ubuntu-latest
outputs:
version: ${{ steps.versionNumber.outputs.versionNumber }}
versionMac: ${{ steps.versionNumberMac.outputs.versionNumberMac }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
# Get version number
- name: Get version number
id: versionNumber
run: |
VERSION=$(grep "^version =" build.gradle | awk -F'"' '{print $2}')
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
- name: Get version number mac
id: versionNumberMac
run: |
VERSION=$(grep "^version =" build.gradle | awk -F'"' '{print $2}')
CURRENT_YEAR=$(date +'%Y')
IFS='.' read -r -a VERSION_PARTS <<< "$VERSION"
MAC_VERSION="$CURRENT_YEAR.${VERSION_PARTS[1]:-0}.${VERSION_PARTS[2]:-0}"
echo "versionNumberMac=$MAC_VERSION" >> $GITHUB_OUTPUT
build-portable:
needs: read_versions
runs-on: ubuntu-latest
strategy:
matrix:
enable_security: [true, false]
include:
- enable_security: true
file_suffix: "-with-login"
- enable_security: false
file_suffix: ""
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up JDK 21
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
with:
java-version: "21"
distribution: "temurin"
- uses: gradle/actions/setup-gradle@8379f6a1328ee0e06e2bb424dadb7b159856a326 # v4.4.0
with:
gradle-version: 8.14
- name: Generate jar (With Security=${{ matrix.enable_security }})
run: ./gradlew clean createExe
env:
DOCKER_ENABLE_SECURITY: ${{ matrix.enable_security }}
STIRLING_PDF_DESKTOP_UI: false
- name: Rename binaries
run: |
mkdir ./binaries
mv ./build/launch4j/Stirling-PDF.exe ./binaries/win-Stirling-PDF-portable-Server${{ matrix.file_suffix }}.exe
mv ./build/libs/Stirling-PDF-${{ needs.read_versions.outputs.version }}.jar ./binaries/Stirling-PDF${{ matrix.file_suffix }}.jar
- name: Upload build artifacts
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
retention-days: 1
if-no-files-found: error
name: stirling${{ matrix.file_suffix }}-binaries
path: |
./binaries/*
sign_verify-portable:
needs: [build-portable, read_versions]
runs-on: ubuntu-latest
strategy:
matrix:
enable_security: [true, false]
include:
- enable_security: true
file_suffix: "with-login-"
- enable_security: false
file_suffix: ""
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Download build artifacts
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: stirling-${{ matrix.file_suffix }}binaries
- name: Display structure of downloaded files
run: ls -R
- name: Upload signed artifacts
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
retention-days: 1
if-no-files-found: error
name: stirling-${{ matrix.file_suffix }}signed
path: |
./*
!cosign.*
build-installers:
needs: read_versions
strategy:
matrix:
include:
- os: windows-latest
platform: win-
- os: macos-latest
platform: mac-
# - os: ubuntu-latest
# platform: linux-
runs-on: ${{ matrix.os }}
permissions:
contents: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up JDK 21
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
with:
java-version: "21"
distribution: "temurin"
- uses: gradle/actions/setup-gradle@8379f6a1328ee0e06e2bb424dadb7b159856a326 # v4.4.0
with:
gradle-version: 8.14
# Install Windows dependencies
- name: Install WiX Toolset
if: matrix.os == 'windows-latest'
run: |
curl -L -o wix.exe https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314.exe
.\wix.exe /install /quiet
# Build installer
- name: Build Installer
run: ./gradlew build jpackage -x test --info
env:
DOCKER_ENABLE_SECURITY: false
STIRLING_PDF_DESKTOP_UI: true
BROWSER_OPEN: true
- name: Set up JDK (x86_64)
if: matrix.os == 'macos-latest'
run: |
curl -L -o jdk.tar.gz https://cdn.azul.com/zulu/bin/zulu17.56.15-ca-jdk17.0.14-macosx_x64.tar.gz
mkdir -p zulu17
tar -xzf jdk.tar.gz -C zulu17 --strip-components=1
echo "JAVA_HOME=$PWD/zulu17" >> $GITHUB_ENV
echo "$PWD/zulu17/bin" >> $GITHUB_PATH
- name: Verify JDK architecture
if: matrix.os == 'macos-latest'
run: file $JAVA_HOME/bin/java
- name: Build project and run jpackage (x86_64)
if: matrix.os == 'macos-latest'
run: arch -x86_64 ./gradlew jpackageMacX64
# Rename and collect artifacts based on OS
- name: Prepare artifacts
id: prepare
shell: bash
run: |
ls -lah ./build/jpackage/
mkdir ./binaries
if [ "${{ matrix.os }}" = "windows-latest" ]; then
mv "./build/jpackage/Stirling PDF-${{ needs.read_versions.outputs.version }}.exe" "./binaries/Stirling-PDF-win-installer.exe"
elif [ "${{ matrix.os }}" = "macos-latest" ]; then
mv "./build/jpackage/Stirling PDF-${{ needs.read_versions.outputs.versionMac }}.dmg" "./binaries/Stirling-PDF-mac-installer.dmg"
mv "./build/jpackage/x86_64/Stirling PDF (x86_64)-${{ needs.read_versions.outputs.versionMac }}.dmg" "./binaries/Stirling-PDF-mac-x86_64-installer.dmg"
else
mv "./build/jpackage/stirling-pdf_${{ needs.read_versions.outputs.version }}-1_amd64.deb" "./binaries/Stirling-PDF-linux-installer.deb"
fi
- name: Display structure of downloaded files
run: ls -R ./binaries
- name: Upload build artifacts
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
retention-days: 1
if-no-files-found: error
name: ${{ matrix.platform }}binaries
path: |
./binaries/*
sign_verify:
needs: [read_versions, build-installers]
strategy:
matrix:
include:
- os: windows-latest
platform: win-
- os: macos-latest
platform: mac-
# - os: ubuntu-latest
# platform: linux-
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Download build artifacts
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: ${{ matrix.platform }}binaries
- name: Display structure of downloaded files
run: ls -R
- name: Install Cosign
if: matrix.os == 'windows-latest'
uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
- name: Generate key pair
if: matrix.os == 'windows-latest'
run: cosign generate-key-pair
- name: Sign and generate attestations
if: matrix.os == 'windows-latest'
run: |
cosign sign-blob \
--key ./cosign.key \
--yes \
--output-signature ./Stirling-PDF-win-installer.exe.sig \
./Stirling-PDF-win-installer.exe
cosign attest-blob \
--predicate - \
--key ./cosign.key \
--yes \
--output-attestation ./Stirling-PDF-win-installer.exe.intoto.jsonl \
./Stirling-PDF-win-installer.exe
cosign verify-blob \
--key ./cosign.pub \
--signature ./Stirling-PDF-win-installer.exe.sig \
./Stirling-PDF-win-installer.exe
- name: Display structure of downloaded files
run: ls -R
- name: Upload signed artifacts
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
retention-days: 1
if-no-files-found: error
name: ${{ matrix.platform }}signed
path: |
./Stirling-PDF-${{ matrix.platform }}installer.*
./Stirling-PDF-${{ matrix.platform }}x86_64-installer.*
!cosign.*
create-release:
if: github.event_name != 'workflow_dispatch' || github.event.inputs.test_mode != 'true'
needs: [read_versions, sign_verify, sign_verify-portable]
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Download signed artifacts
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
- name: Display structure of downloaded files
run: ls -R
- name: Upload binaries, attestations and signatures to Release and create GitHub Release
uses: softprops/action-gh-release@01570a1f39cb168c169c802c3bceb9e93fb10974 # v2.1.0
with:
tag_name: v${{ needs.read_versions.outputs.version }}
generate_release_notes: true
files: |
./*signed/*
-71
View File
@@ -1,71 +0,0 @@
name: Pre-commit
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * 1"
permissions:
contents: read
jobs:
pre-commit:
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
- name: Setup GitHub App Bot
id: setup-bot
uses: ./.github/actions/setup-bot
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: 3.12
cache: 'pip' # caching pip dependencies
- name: Run Pre-Commit Hooks
run: |
pip install --require-hashes -r ./.github/scripts/requirements_pre_commit.txt
- run: pre-commit run --all-files -c .pre-commit-config.yaml
continue-on-error: true
- name: git add
run: |
git add .
git diff --staged --quiet || echo "CHANGES_DETECTED=true" >> $GITHUB_ENV
- name: Create Pull Request
if: env.CHANGES_DETECTED == 'true'
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
with:
token: ${{ steps.setup-bot.outputs.token }}
commit-message: ":file_folder: pre-commit"
committer: ${{ steps.setup-bot.outputs.committer }}
author: ${{ steps.setup-bot.outputs.committer }}
signoff: true
branch: pre-commit
title: "🤖 format everything with pre-commit by ${{ steps.setup-bot.outputs.app-slug }}"
body: |
Auto-generated by [create-pull-request][1] with **${{ steps.setup-bot.outputs.app-slug }}**
[1]: https://github.com/peter-evans/create-pull-request
draft: false
delete-branch: true
labels: github-actions
sign-commits: true
-195
View File
@@ -1,195 +0,0 @@
name: Push Docker Image with VersionNumber
on:
workflow_dispatch:
push:
branches:
- master
- main
permissions:
contents: read
jobs:
push:
runs-on: ubuntu-latest
permissions:
packages: write
id-token: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up JDK 17
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
with:
java-version: "17"
distribution: "temurin"
- uses: gradle/actions/setup-gradle@8379f6a1328ee0e06e2bb424dadb7b159856a326 # v4.4.0
with:
gradle-version: 8.14
- name: Run Gradle Command
run: ./gradlew clean build
env:
DOCKER_ENABLE_SECURITY: false
STIRLING_PDF_DESKTOP_UI: false
- name: Install cosign
if: github.ref == 'refs/heads/master'
uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
with:
cosign-release: "v2.4.1"
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
- name: Get version number
id: versionNumber
run: echo "versionNumber=$(./gradlew printVersion --quiet | tail -1)" >> $GITHUB_OUTPUT
- name: Login to Docker Hub
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_API }}
- name: Login to GitHub Container Registry
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ github.token }}
- name: Set up QEMU
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
- name: Convert repository owner to lowercase
id: repoowner
run: echo "lowercase=$(echo ${{ github.repository_owner }} | awk '{print tolower($0)}')" >> $GITHUB_OUTPUT
- name: Generate tags
id: meta
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
with:
images: |
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/s-pdf
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf
${{ secrets.DOCKER_HUB_ORG_USERNAME }}/stirling-pdf
tags: |
type=raw,value=${{ steps.versionNumber.outputs.versionNumber }},enable=${{ github.ref == 'refs/heads/master' }}
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
type=raw,value=alpha,enable=${{ github.ref == 'refs/heads/main' }}
- name: Build and push main Dockerfile
id: build-push-regular
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
file: ./Dockerfile
push: true
cache-from: type=gha
cache-to: type=gha,mode=max
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: VERSION_TAG=${{ steps.versionNumber.outputs.versionNumber }}
platforms: linux/amd64,linux/arm64/v8
provenance: true
sbom: true
- name: Sign regular images
if: github.ref == 'refs/heads/master'
env:
DIGEST: ${{ steps.build-push-regular.outputs.digest }}
TAGS: ${{ steps.meta.outputs.tags }}
COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
run: |
echo "$TAGS" | tr ',' '\n' | while read -r tag; do
cosign sign --yes \
--key env://COSIGN_PRIVATE_KEY \
"${tag}@${DIGEST}"
done
- name: Generate tags ultra-lite
id: meta2
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
if: github.ref != 'refs/heads/main'
with:
images: |
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/s-pdf
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf
${{ secrets.DOCKER_HUB_ORG_USERNAME }}/stirling-pdf
tags: |
type=raw,value=${{ steps.versionNumber.outputs.versionNumber }}-ultra-lite,enable=${{ github.ref == 'refs/heads/master' }}
type=raw,value=latest-ultra-lite,enable=${{ github.ref == 'refs/heads/master' }}
- name: Build and push Dockerfile-ultra-lite
id: build-push-lite
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
if: github.ref != 'refs/heads/main'
with:
context: .
file: ./Dockerfile.ultra-lite
push: true
cache-from: type=gha
cache-to: type=gha,mode=max
tags: ${{ steps.meta2.outputs.tags }}
labels: ${{ steps.meta2.outputs.labels }}
build-args: VERSION_TAG=${{ steps.versionNumber.outputs.versionNumber }}
platforms: linux/amd64,linux/arm64/v8
provenance: true
sbom: true
- name: Generate tags fat
id: meta3
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
if: github.ref != 'refs/heads/main'
with:
images: |
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/s-pdf
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf
${{ secrets.DOCKER_HUB_ORG_USERNAME }}/stirling-pdf
tags: |
type=raw,value=${{ steps.versionNumber.outputs.versionNumber }}-fat,enable=${{ github.ref == 'refs/heads/master' }}
type=raw,value=latest-fat,enable=${{ github.ref == 'refs/heads/master' }}
- name: Build and push main Dockerfile fat
id: build-push-fat
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
if: github.ref != 'refs/heads/main'
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
file: ./Dockerfile.fat
push: true
cache-from: type=gha
cache-to: type=gha,mode=max
tags: ${{ steps.meta3.outputs.tags }}
labels: ${{ steps.meta3.outputs.labels }}
build-args: VERSION_TAG=${{ steps.versionNumber.outputs.versionNumber }}
platforms: linux/amd64,linux/arm64/v8
provenance: true
sbom: true
- name: Sign fat images
if: github.ref == 'refs/heads/master'
env:
DIGEST: ${{ steps.build-push-fat.outputs.digest }}
TAGS: ${{ steps.meta3.outputs.tags }}
COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
run: |
echo "$TAGS" | tr ',' '\n' | while read -r tag; do
cosign sign --key env://COSIGN_PRIVATE_KEY --yes "${tag}@${DIGEST}"
done
-180
View File
@@ -1,180 +0,0 @@
name: Release Artifacts
on:
workflow_dispatch:
release:
types: [created]
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
enable_security: [true, false]
include:
- enable_security: true
file_suffix: "-with-login"
- enable_security: false
file_suffix: ""
outputs:
version: ${{ steps.versionNumber.outputs.versionNumber }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up JDK 17
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
with:
java-version: "17"
distribution: "temurin"
- uses: gradle/actions/setup-gradle@8379f6a1328ee0e06e2bb424dadb7b159856a326 # v4.4.0
with:
gradle-version: 8.14
- name: Generate jar (With Security=${{ matrix.enable_security }})
run: ./gradlew clean createExe
env:
DOCKER_ENABLE_SECURITY: ${{ matrix.enable_security }}
STIRLING_PDF_DESKTOP_UI: false
- name: Get version number
id: versionNumber
run: |
VERSION=$(grep "^version =" build.gradle | awk -F'"' '{print $2}')
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
- name: Rename binaries
run: |
mv ./build/launch4j/Stirling-PDF.exe ./build/launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe
mv ./build/libs/Stirling-PDF-${{ steps.versionNumber.outputs.versionNumber }}.jar ./build/libs/Stirling-PDF${{ matrix.file_suffix }}.jar
- name: Debug build artifacts
run: |
echo "Current Directory: $(pwd)"
ls -R ./build/libs
ls -R ./build/launch4j
- name: Upload build artifacts
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: binaries${{ matrix.file_suffix }}
path: |
./build/launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.*
./build/libs/Stirling-PDF${{ matrix.file_suffix }}.*
sign_verify:
needs: build
runs-on: ubuntu-latest
strategy:
matrix:
enable_security: [true, false]
include:
- enable_security: true
file_suffix: "-with-login"
- enable_security: false
file_suffix: ""
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Download build artifacts
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: binaries${{ matrix.file_suffix }}
- name: Display structure of downloaded files
run: ls -R
- name: Install Cosign
uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
- name: Generate key pair
run: cosign generate-key-pair
- name: Sign and generate attestations
run: |
cosign sign-blob \
--key ./cosign.key \
--yes \
--output-signature ./libs/Stirling-PDF${{ matrix.file_suffix }}.jar.sig \
./libs/Stirling-PDF${{ matrix.file_suffix }}.jar
cosign attest-blob \
--predicate - \
--key ./cosign.key \
--yes \
--output-attestation ./libs/Stirling-PDF${{ matrix.file_suffix }}.jar.intoto.jsonl \
./libs/Stirling-PDF${{ matrix.file_suffix }}.jar
cosign verify-blob \
--key ./cosign.pub \
--signature ./libs/Stirling-PDF${{ matrix.file_suffix }}.jar.sig \
./libs/Stirling-PDF${{ matrix.file_suffix }}.jar
cosign sign-blob \
--key ./cosign.key \
--yes \
--output-signature ./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe.sig \
./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe
cosign attest-blob \
--predicate - \
--key ./cosign.key \
--yes \
--output-attestation ./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe.intoto.jsonl \
./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe
cosign verify-blob \
--key ./cosign.pub \
--signature ./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe.sig \
./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe
- name: Upload signed artifacts
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: signed${{ matrix.file_suffix }}
path: |
./libs/Stirling-PDF${{ matrix.file_suffix }}.*
./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.*
release:
needs: [build, sign_verify]
runs-on: ubuntu-latest
permissions:
contents: write
strategy:
matrix:
enable_security: [true, false]
include:
- enable_security: true
file_suffix: "-with-login"
- enable_security: false
file_suffix: ""
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Download signed artifacts
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: signed${{ matrix.file_suffix }}
- name: Upload binaries, attestations and signatures to Release and create GitHub Release
uses: softprops/action-gh-release@01570a1f39cb168c169c802c3bceb9e93fb10974 # v2.1.0
with:
tag_name: v${{ needs.build.outputs.version }}
generate_release_notes: true
files: |
./libs/Stirling-PDF*
./launch4j/Stirling-PDF-Server*
-79
View File
@@ -1,79 +0,0 @@
# This workflow uses actions that are not certified by GitHub. They are provided
# by a third-party and are governed by separate terms of service, privacy
# policy, and support documentation.
name: Scorecard supply-chain security
on:
# For Branch-Protection check. Only the default branch is supported. See
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
branch_protection_rule:
# To guarantee Maintained check is occasionally updated. See
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
schedule:
- cron: "20 7 * * 2"
push:
branches: ["main"]
permissions: read-all
jobs:
analysis:
name: Scorecard analysis
runs-on: ubuntu-latest
permissions:
# Needed to upload the results to code-scanning dashboard.
security-events: write
# Needed to publish results and get a badge (see publish_results below).
id-token: write
contents: read
actions: read
# To allow GraphQL ListCommits to work
issues: read
pull-requests: read
# To detect SAST tools
checks: read
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: "Checkout code"
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false
- name: "Run analysis"
uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2
with:
results_file: results.sarif
results_format: sarif
# (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
# - you want to enable the Branch-Protection check on a *public* repository, or
# - you are installing Scorecards on a *private* repository
# To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
# repo_token: ${{ secrets.SCORECARD_TOKEN }}
# Public repositories:
# - Publish results to OpenSSF REST API for easy access by consumers
# - Allows the repository to include the Scorecard badge.
# - See https://github.com/ossf/scorecard-action#publishing-results.
# For private repositories:
# - `publish_results` will always be set to `false`, regardless
# of the value entered here.
publish_results: true
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
# format to the repository Actions tab.
- name: "Upload artifact"
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: SARIF file
path: results.sarif
retention-days: 5
# Upload the results to GitHub's code scanning dashboard.
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
with:
sarif_file: results.sarif
-63
View File
@@ -1,63 +0,0 @@
name: Run Sonarqube
on:
push:
branches:
- master
pull_request_target:
branches:
- main
workflow_dispatch:
permissions:
pull-requests: read
actions: read
jobs:
sonarqube:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
- name: Setup Gradle
uses: gradle/actions/setup-gradle@8379f6a1328ee0e06e2bb424dadb7b159856a326 # v4.4.0
- name: Build and analyze with Gradle
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
DOCKER_ENABLE_SECURITY: true
STIRLING_PDF_DESKTOP_UI: true
run: |
./gradlew clean build sonar \
-Dsonar.projectKey=Stirling-Tools_Stirling-PDF \
-Dsonar.organization=stirling-tools \
-Dsonar.host.url=https://sonarcloud.io \
-Dsonar.login=${SONAR_TOKEN} \
-Dsonar.log.level=DEBUG \
--info
- name: Upload Problems Report on Failure
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: gradle-problems-report
path: build/reports/problems/problems-report.html
retention-days: 7
- name: Upload Sonar Logs on Failure
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: sonar-logs
path: |
.scannerwork/report-task.txt
build/sonar/
retention-days: 7
-40
View File
@@ -1,40 +0,0 @@
name: Close stale issues
on:
schedule:
- cron: "30 0 * * *"
workflow_dispatch:
permissions:
contents: read
jobs:
stale:
runs-on: ubuntu-latest
permissions:
issues: write
pull-requests: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: 30 days stale issues
uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
days-before-stale: 30
days-before-close: 7
stale-issue-message: >
This issue has been automatically marked as stale because it has had no recent activity.
It will be closed if no further activity occurs. Thank you for your contributions.
close-issue-message: >
This issue has been automatically closed because it has had no recent activity after being marked as stale.
Please reopen if you need further assistance.
stale-issue-label: "Stale"
remove-stale-when-updated: true
only-issue-labels: "more-info-needed"
days-before-pr-stale: -1 # Prevents PRs from being marked as stale
days-before-pr-close: -1 # Prevents PRs from being closed
start-date: "2024-07-06T00:00:00Z" # ISO 8601 Format
-49
View File
@@ -1,49 +0,0 @@
name: Update Swagger
on:
workflow_dispatch:
push:
branches:
- master
permissions:
contents: read
jobs:
push:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up JDK 17
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
with:
java-version: "17"
distribution: "temurin"
- uses: gradle/actions/setup-gradle@8379f6a1328ee0e06e2bb424dadb7b159856a326 # v4.4.0
- name: Generate Swagger documentation
run: ./gradlew generateOpenApiDocs
- name: Upload Swagger Documentation to SwaggerHub
run: ./gradlew swaggerhubUpload
env:
SWAGGERHUB_API_KEY: ${{ secrets.SWAGGERHUB_API_KEY }}
SWAGGERHUB_USER: "Frooodle"
- name: Get version number
id: versionNumber
run: echo "versionNumber=$(./gradlew printVersion --quiet | tail -1)" >> $GITHUB_OUTPUT
- name: Set API version as published and default on SwaggerHub
run: |
curl -X PUT -H "Authorization: ${SWAGGERHUB_API_KEY}" "https://api.swaggerhub.com/apis/${SWAGGERHUB_USER}/Stirling-PDF/${{ steps.versionNumber.outputs.versionNumber }}/settings/lifecycle" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"published\":true,\"default\":true}"
env:
SWAGGERHUB_API_KEY: ${{ secrets.SWAGGERHUB_API_KEY }}
SWAGGERHUB_USER: "Frooodle"
-104
View File
@@ -1,104 +0,0 @@
name: Sync Files
on:
workflow_dispatch:
push:
branches:
- main
paths:
- "build.gradle"
- "README.md"
- "src/main/resources/messages_*.properties"
- "src/main/resources/static/3rdPartyLicenses.json"
- "scripts/ignore_translation.toml"
permissions:
contents: read
jobs:
sync-files:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup GitHub App Bot
id: setup-bot
uses: ./.github/actions/setup-bot
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
cache: "pip" # caching pip dependencies
- name: Sync translation property files
run: |
python .github/scripts/check_language_properties.py --reference-file "src/main/resources/messages_en_GB.properties" --branch main
- name: Commit translation files
run: |
git add src/main/resources/messages_*.properties
git diff --staged --quiet || git commit -m ":memo: Sync translation files" || echo "No changes detected"
- name: Install dependencies
run: pip install --require-hashes -r ./.github/scripts/requirements_sync_readme.txt
- name: Sync README.md
run: |
python scripts/counter_translation.py
- name: Run git add
run: |
git add README.md
git diff --staged --quiet || git commit -m ":memo: Sync README.md" || echo "No changes detected"
- name: Create Pull Request
if: always()
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
with:
token: ${{ steps.setup-bot.outputs.token }}
commit-message: Update files
committer: ${{ steps.setup-bot.outputs.committer }}
author: ${{ steps.setup-bot.outputs.committer }}
signoff: true
branch: sync_readme
title: ":globe_with_meridians: Sync Translations + Update README Progress Table"
body: |
### Description of Changes
This Pull Request was automatically generated to synchronize updates to translation files and documentation. Below are the details of the changes made:
#### **1. Synchronization of Translation Files**
- Updated translation files (`messages_*.properties`) to reflect changes in the reference file `messages_en_GB.properties`.
- Ensured consistency and synchronization across all supported language files.
- Highlighted any missing or incomplete translations.
#### **2. Update README.md**
- Generated the translation progress table in `README.md`.
- Added a summary of the current translation status for all supported languages.
- Included up-to-date statistics on translation coverage.
#### **Why these changes are necessary**
- Keeps translation files aligned with the latest reference updates.
- Ensures the documentation reflects the current translation progress.
---
Auto-generated by [create-pull-request][1].
[1]: https://github.com/peter-evans/create-pull-request
draft: false
delete-branch: true
labels: github-actions
sign-commits: true
add-paths: |
README.md
src/main/resources/messages_*.properties
-154
View File
@@ -1,154 +0,0 @@
name: UI test with TestDriverAI
on:
push:
branches: ["master", "UITest", "testdriver"]
permissions:
contents: read
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up JDK
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
with:
java-version: '17'
distribution: 'temurin'
- name: Build with Gradle
run: ./gradlew clean build
env:
DOCKER_ENABLE_SECURITY: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
- name: Get version number
id: versionNumber
run: |
VERSION=$(grep "^version =" build.gradle | awk -F'"' '{print $2}')
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
- name: Login to Docker Hub
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_API }}
- name: Build and push test image
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
context: .
file: ./Dockerfile
push: true
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:test-${{ github.sha }}
build-args: VERSION_TAG=${{ steps.versionNumber.outputs.versionNumber }}
platforms: linux/amd64
- name: Set up SSH
run: |
mkdir -p ~/.ssh/
echo "${{ secrets.VPS_SSH_KEY }}" > ../private.key
sudo chmod 600 ../private.key
- name: Deploy to VPS
run: |
cat > docker-compose.yml << EOF
version: '3.3'
services:
stirling-pdf:
container_name: stirling-pdf-test-${{ github.sha }}
image: ${{ secrets.DOCKER_HUB_USERNAME }}/test:test-${{ github.sha }}
ports:
- "1337:8080"
volumes:
- /stirling/test-${{ github.sha }}/data:/usr/share/tessdata:rw
- /stirling/test-${{ github.sha }}/config:/configs:rw
- /stirling/test-${{ github.sha }}/logs:/logs:rw
environment:
DOCKER_ENABLE_SECURITY: "false"
SECURITY_ENABLELOGIN: "false"
SYSTEM_DEFAULTLOCALE: en-GB
UI_APPNAME: "Stirling-PDF Test"
UI_HOMEDESCRIPTION: "Test Deployment"
UI_APPNAMENAVBAR: "Test"
SYSTEM_MAXFILESIZE: "100"
METRICS_ENABLED: "true"
SYSTEM_GOOGLEVISIBILITY: "false"
SYSTEM_ENABLEANALYTICS: "false"
restart: on-failure:5
EOF
scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null docker-compose.yml ${{ secrets.VPS_USERNAME }}@${{ secrets.VPS_HOST }}:/tmp/docker-compose.yml
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${{ secrets.VPS_USERNAME }}@${{ secrets.VPS_HOST }} << EOF
mkdir -p /stirling/test-${{ github.sha }}/{data,config,logs}
mv /tmp/docker-compose.yml /stirling/test-${{ github.sha }}/docker-compose.yml
cd /stirling/test-${{ github.sha }}
docker-compose pull
docker-compose up -d
EOF
test:
needs: deploy
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Run TestDriver.ai
uses: testdriverai/action@f0d0f45fdd684db628baa843fe9313f3ca3a8aa8 #1.1.3
with:
key: ${{secrets.TESTDRIVER_API_KEY}}
prerun: |
npm install
npm run build
npm install dashcam-chrome --save
Start-Process "C:/Program Files/Google/Chrome/Application/chrome.exe" -ArgumentList "--start-maximized", "--load-extension=$(pwd)/node_modules/dashcam-chrome/build", "http://${{ secrets.VPS_HOST }}:1337"
Start-Sleep -Seconds 20
prompt: |
1. /run testing/testdriver/test.yml
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
FORCE_COLOR: "3"
cleanup:
needs: [deploy, test]
runs-on: ubuntu-latest
if: always()
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Set up SSH
run: |
mkdir -p ~/.ssh/
echo "${{ secrets.VPS_SSH_KEY }}" > ../private.key
sudo chmod 600 ../private.key
- name: Cleanup deployment
run: |
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${{ secrets.VPS_USERNAME }}@${{ secrets.VPS_HOST }} << EOF
cd /stirling/test-${{ github.sha }}
docker-compose down
cd /stirling
rm -rf test-${{ github.sha }}
EOF
+2 -2
View File
@@ -1,6 +1,6 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.11.11
rev: v0.11.6
hooks:
- id: ruff
args:
@@ -22,7 +22,7 @@ repos:
files: \.(html|css|js|py|md)$
exclude: (.vscode|.devcontainer|src/main/resources|Dockerfile|.*/pdfjs.*|.*/thirdParty.*|bootstrap.*|.*\.min\..*|.*diff\.js)
- repo: https://github.com/gitleaks/gitleaks
rev: v8.26.0
rev: v8.24.3
hooks:
- id: gitleaks
- repo: https://github.com/pre-commit/pre-commit-hooks
+1 -4
View File
@@ -10,7 +10,7 @@
"java.configuration.updateBuildConfiguration": "interactive",
"java.format.enabled": true,
"java.format.settings.profile": "GoogleStyle",
"java.format.settings.google.version": "1.27.0",
"java.format.settings.google.version": "1.26.0",
"java.format.settings.google.extra": "--aosp --skip-sorting-imports --skip-javadoc-formatting",
// (DE) Aktiviert Kommentare im Java-Format.
// (EN) Enables comments in Java formatting.
@@ -49,9 +49,7 @@
".venv*/",
".vscode/",
"bin/",
"common/bin/",
"build/",
"common/build/",
"configs/",
"customFiles/",
"docs/",
@@ -65,7 +63,6 @@
".git-blame-ignore-revs",
".gitattributes",
".gitignore",
"common/.gitignore",
".pre-commit-config.yaml",
],
// Enables signature help in Java.
+3 -3
View File
@@ -1,5 +1,5 @@
# Main stage
FROM alpine:3.22.0@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
FROM alpine:3.21.3@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c
# Copy necessary files
COPY scripts /scripts
@@ -73,7 +73,7 @@ RUN echo "@main https://dl-cdn.alpinelinux.org/alpine/edge/main" | tee -a /etc/a
py3-pillow@testing \
py3-pdf2image@testing && \
python3 -m venv /opt/venv && \
/opt/venv/bin/pip install --upgrade pip setuptools && \
/opt/venv/bin/pip install --upgrade pip && \
/opt/venv/bin/pip install --no-cache-dir --upgrade unoserver weasyprint && \
ln -s /usr/lib/libreoffice/program/uno.py /opt/venv/lib/python3.12/site-packages/ && \
ln -s /usr/lib/libreoffice/program/unohelper.py /opt/venv/lib/python3.12/site-packages/ && \
@@ -92,4 +92,4 @@ EXPOSE 8080/tcp
# Set user and run command
ENTRYPOINT ["tini", "--", "/scripts/init.sh"]
CMD ["sh", "-c", "java -Dfile.encoding=UTF-8 -jar /app.jar & /opt/venv/bin/unoserver --port 2003 --interface 127.0.0.1"]
CMD ["java", "-Dfile.encoding=UTF-8", "-jar", "/app.jar"]
-1
View File
@@ -32,7 +32,6 @@ ENV SETUPTOOLS_USE_DISTUTILS=local
# Installation der benötigten Python-Pakete
RUN python3 -m venv --system-site-packages /opt/venv \
&& . /opt/venv/bin/activate \
&& pip install --upgrade pip setuptools \
&& pip install --no-cache-dir WeasyPrint pdf2image pillow unoserver opencv-python-headless pre-commit
# Füge den venv-Pfad zur globalen PATH-Variable hinzu, damit die Tools verfügbar sind
+3 -3
View File
@@ -19,7 +19,7 @@ RUN DOCKER_ENABLE_SECURITY=true \
./gradlew clean build -x spotlessApply -x spotlessCheck -x test -x sonarqube
# Main stage
FROM alpine:3.22.0@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
FROM alpine:3.21.3@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c
# Copy necessary files
COPY scripts /scripts
@@ -83,7 +83,7 @@ RUN echo "@main https://dl-cdn.alpinelinux.org/alpine/edge/main" | tee -a /etc/a
py3-pillow@testing \
py3-pdf2image@testing && \
python3 -m venv /opt/venv && \
/opt/venv/bin/pip install --upgrade pip setuptools && \
/opt/venv/bin/pip install --upgrade pip && \
/opt/venv/bin/pip install --no-cache-dir --upgrade unoserver weasyprint && \
ln -s /usr/lib/libreoffice/program/uno.py /opt/venv/lib/python3.12/site-packages/ && \
ln -s /usr/lib/libreoffice/program/unohelper.py /opt/venv/lib/python3.12/site-packages/ && \
@@ -101,4 +101,4 @@ RUN echo "@main https://dl-cdn.alpinelinux.org/alpine/edge/main" | tee -a /etc/a
EXPOSE 8080/tcp
# Set user and run command
ENTRYPOINT ["tini", "--", "/scripts/init.sh"]
CMD ["sh", "-c", "java -Dfile.encoding=UTF-8 -jar /app.jar & /opt/venv/bin/unoserver --port 2003 --interface 127.0.0.1"]
CMD ["sh", "-c", "java -Dfile.encoding=UTF-8 -jar /app.jar"]
+1 -1
View File
@@ -1,5 +1,5 @@
# use alpine
FROM alpine:3.22.0@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
FROM alpine:3.21.3@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c
ARG VERSION_TAG
+39 -39
View File
@@ -116,47 +116,47 @@ Stirling-PDF currently supports 40 languages!
| Language | Progress |
| -------------------------------------------- | -------------------------------------- |
| Arabic (العربية) (ar_AR) | ![73%](https://geps.dev/progress/73) |
| Azerbaijani (Azərbaycan Dili) (az_AZ) | ![74%](https://geps.dev/progress/74) |
| Basque (Euskara) (eu_ES) | ![43%](https://geps.dev/progress/43) |
| Bulgarian (Български) (bg_BG) | ![81%](https://geps.dev/progress/81) |
| Catalan (Català) (ca_CA) | ![81%](https://geps.dev/progress/81) |
| Croatian (Hrvatski) (hr_HR) | ![72%](https://geps.dev/progress/72) |
| Czech (Česky) (cs_CZ) | ![83%](https://geps.dev/progress/83) |
| Danish (Dansk) (da_DK) | ![73%](https://geps.dev/progress/73) |
| Dutch (Nederlands) (nl_NL) | ![71%](https://geps.dev/progress/71) |
| Arabic (العربية) (ar_AR) | ![83%](https://geps.dev/progress/83) |
| Azerbaijani (Azərbaycan Dili) (az_AZ) | ![82%](https://geps.dev/progress/82) |
| Basque (Euskara) (eu_ES) | ![48%](https://geps.dev/progress/48) |
| Bulgarian (Български) (bg_BG) | ![92%](https://geps.dev/progress/92) |
| Catalan (Català) (ca_CA) | ![89%](https://geps.dev/progress/89) |
| Croatian (Hrvatski) (hr_HR) | ![81%](https://geps.dev/progress/81) |
| Czech (Česky) (cs_CZ) | ![91%](https://geps.dev/progress/91) |
| Danish (Dansk) (da_DK) | ![80%](https://geps.dev/progress/80) |
| Dutch (Nederlands) (nl_NL) | ![79%](https://geps.dev/progress/79) |
| English (English) (en_GB) | ![100%](https://geps.dev/progress/100) |
| English (US) (en_US) | ![100%](https://geps.dev/progress/100) |
| French (Français) (fr_FR) | ![82%](https://geps.dev/progress/82) |
| German (Deutsch) (de_DE) | ![89%](https://geps.dev/progress/89) |
| Greek (Ελληνικά) (el_GR) | ![80%](https://geps.dev/progress/80) |
| Hindi (हिंदी) (hi_IN) | ![80%](https://geps.dev/progress/80) |
| Hungarian (Magyar) (hu_HU) | ![87%](https://geps.dev/progress/87) |
| Indonesian (Bahasa Indonesia) (id_ID) | ![74%](https://geps.dev/progress/74) |
| Irish (Gaeilge) (ga_IE) | ![81%](https://geps.dev/progress/81) |
| Italian (Italiano) (it_IT) | ![95%](https://geps.dev/progress/95) |
| Japanese (日本語) (ja_JP) | ![82%](https://geps.dev/progress/82) |
| Korean (한국어) (ko_KR) | ![80%](https://geps.dev/progress/80) |
| Norwegian (Norsk) (no_NB) | ![78%](https://geps.dev/progress/78) |
| Persian (فارسی) (fa_IR) | ![76%](https://geps.dev/progress/76) |
| Polish (Polski) (pl_PL) | ![86%](https://geps.dev/progress/86) |
| Portuguese (Português) (pt_PT) | ![82%](https://geps.dev/progress/82) |
| Portuguese Brazilian (Português) (pt_BR) | ![87%](https://geps.dev/progress/87) |
| Romanian (Română) (ro_RO) | ![68%](https://geps.dev/progress/68) |
| Russian (Русский) (ru_RU) | ![87%](https://geps.dev/progress/87) |
| Serbian Latin alphabet (Srpski) (sr_LATN_RS) | ![52%](https://geps.dev/progress/52) |
| Simplified Chinese (简体中文) (zh_CN) | ![86%](https://geps.dev/progress/86) |
| Slovakian (Slovensky) (sk_SK) | ![61%](https://geps.dev/progress/61) |
| Slovenian (Slovenščina) (sl_SI) | ![85%](https://geps.dev/progress/85) |
| Spanish (Español) (es_ES) | ![88%](https://geps.dev/progress/88) |
| Swedish (Svenska) (sv_SE) | ![78%](https://geps.dev/progress/78) |
| Thai (ไทย) (th_TH) | ![70%](https://geps.dev/progress/70) |
| Tibetan (བོད་ཡིག་) (bo_CN) | ![77%](https://geps.dev/progress/77) |
| Traditional Chinese (繁體中文) (zh_TW) | ![87%](https://geps.dev/progress/87) |
| Turkish (Türkçe) (tr_TR) | ![87%](https://geps.dev/progress/87) |
| Ukrainian (Українська) (uk_UA) | ![87%](https://geps.dev/progress/87) |
| Vietnamese (Tiếng Việt) (vi_VN) | ![68%](https://geps.dev/progress/68) |
| Malayalam (മലയാളം) (ml_IN) | ![87%](https://geps.dev/progress/87) |
| French (Français) (fr_FR) | ![92%](https://geps.dev/progress/92) |
| German (Deutsch) (de_DE) | ![99%](https://geps.dev/progress/99) |
| Greek (Ελληνικά) (el_GR) | ![91%](https://geps.dev/progress/91) |
| Hindi (हिंदी) (hi_IN) | ![91%](https://geps.dev/progress/91) |
| Hungarian (Magyar) (hu_HU) | ![99%](https://geps.dev/progress/99) |
| Indonesian (Bahasa Indonesia) (id_ID) | ![80%](https://geps.dev/progress/80) |
| Irish (Gaeilge) (ga_IE) | ![91%](https://geps.dev/progress/91) |
| Italian (Italiano) (it_IT) | ![99%](https://geps.dev/progress/99) |
| Japanese (日本語) (ja_JP) | ![93%](https://geps.dev/progress/93) |
| Korean (한국어) (ko_KR) | ![92%](https://geps.dev/progress/92) |
| Norwegian (Norsk) (no_NB) | ![86%](https://geps.dev/progress/86) |
| Persian (فارسی) (fa_IR) | ![87%](https://geps.dev/progress/87) |
| Polish (Polski) (pl_PL) | ![95%](https://geps.dev/progress/95) |
| Portuguese (Português) (pt_PT) | ![91%](https://geps.dev/progress/91) |
| Portuguese Brazilian (Português) (pt_BR) | ![97%](https://geps.dev/progress/97) |
| Romanian (Română) (ro_RO) | ![75%](https://geps.dev/progress/75) |
| Russian (Русский) (ru_RU) | ![93%](https://geps.dev/progress/93) |
| Serbian Latin alphabet (Srpski) (sr_LATN_RS) | ![60%](https://geps.dev/progress/60) |
| Simplified Chinese (简体中文) (zh_CN) | ![93%](https://geps.dev/progress/93) |
| Slovakian (Slovensky) (sk_SK) | ![69%](https://geps.dev/progress/69) |
| Slovenian (Slovenščina) (sl_SI) | ![94%](https://geps.dev/progress/94) |
| Spanish (Español) (es_ES) | ![99%](https://geps.dev/progress/99) |
| Swedish (Svenska) (sv_SE) | ![87%](https://geps.dev/progress/87) |
| Thai (ไทย) (th_TH) | ![80%](https://geps.dev/progress/80) |
| Tibetan (བོད་ཡིག་) (zh_BO) | ![88%](https://geps.dev/progress/88) |
| Traditional Chinese (繁體中文) (zh_TW) | ![99%](https://geps.dev/progress/99) |
| Turkish (Türkçe) (tr_TR) | ![97%](https://geps.dev/progress/97) |
| Ukrainian (Українська) (uk_UA) | ![96%](https://geps.dev/progress/96) |
| Vietnamese (Tiếng Việt) (vi_VN) | ![73%](https://geps.dev/progress/73) |
| Malayalam (മലയാളം) (ml_ML) | ![99%](https://geps.dev/progress/99) |
## Stirling PDF Enterprise
+16 -16
View File
@@ -1,12 +1,12 @@
plugins {
id "java"
id "jacoco"
id "org.springframework.boot" version "3.5.0"
id 'jacoco'
id "org.springframework.boot" version "3.4.5"
id "io.spring.dependency-management" version "1.1.7"
id "org.springdoc.openapi-gradle-plugin" version "1.9.0"
id "io.swagger.swaggerhub" version "1.3.2"
id "edu.sc.seis.launch4j" version "3.0.6"
id "com.diffplug.spotless" version "7.0.4"
id "com.diffplug.spotless" version "7.0.3"
id "com.github.jk1.dependency-license-report" version "2.9"
//id "nebula.lint" version "19.0.3"
id("org.panteleyev.jpackageplugin") version "1.6.1"
@@ -19,7 +19,7 @@ import java.nio.file.Files
import java.time.Year
ext {
springBootVersion = "3.5.0"
springBootVersion = "3.4.5"
pdfboxVersion = "3.0.5"
imageioVersion = "3.12.0"
lombokVersion = "1.18.38"
@@ -374,10 +374,9 @@ launch4j {
spotless {
java {
target sourceSets.main.allJava
target project(':common').sourceSets.main.allJava
target project.fileTree('src').include('**/*.java')
googleJavaFormat("1.27.0").aosp().reorderImports(false)
googleJavaFormat("1.26.0").aosp().reorderImports(false)
importOrder("java", "javax", "org", "com", "net", "io", "jakarta", "lombok", "me", "stirling")
toggleOffOn()
@@ -414,14 +413,13 @@ configurations.all {
// Exclude Tomcat
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
}
dependencies {
implementation project(':common')
//tmp for security bumps
implementation 'ch.qos.logback:logback-core:1.5.18'
implementation 'ch.qos.logback:logback-classic:1.5.18'
// Exclude vulnerable BouncyCastle version used in tableau
configurations.all {
exclude group: 'org.bouncycastle', module: 'bcpkix-jdk15on'
@@ -430,7 +428,7 @@ dependencies {
}
if (System.getenv("STIRLING_PDF_DESKTOP_UI") != "false") {
implementation "me.friwi:jcefmaven:135.0.20"
implementation "me.friwi:jcefmaven:132.3.1"
implementation "org.openjfx:javafx-controls:21"
implementation "org.openjfx:javafx-swing:21"
}
@@ -450,6 +448,8 @@ dependencies {
if (System.getenv("DOCKER_ENABLE_SECURITY") != "false") {
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'io.micrometer:micrometer-registry-prometheus'
implementation "org.springframework.boot:spring-boot-starter-security:$springBootVersion"
@@ -458,13 +458,13 @@ dependencies {
implementation "org.springframework.boot:spring-boot-starter-oauth2-client:$springBootVersion"
implementation "org.springframework.boot:spring-boot-starter-mail:$springBootVersion"
implementation "org.springframework.session:spring-session-core:3.5.0"
implementation "org.springframework.session:spring-session-core:3.4.3"
implementation "org.springframework:spring-jdbc:6.2.7"
implementation 'com.unboundid.product.scim2:scim2-sdk-client:2.3.5'
// Don't upgrade h2database
runtimeOnly "com.h2database:h2:2.3.232"
runtimeOnly "org.postgresql:postgresql:42.7.6"
runtimeOnly "org.postgresql:postgresql:42.7.5"
constraints {
implementation "org.opensaml:opensaml-core:$openSamlVersion"
implementation "org.opensaml:opensaml-saml-api:$openSamlVersion"
@@ -478,7 +478,6 @@ dependencies {
implementation 'org.snakeyaml:snakeyaml-engine:2.9'
testImplementation "org.springframework.boot:spring-boot-starter-test:$springBootVersion"
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
// Batik
implementation "org.apache.xmlgraphics:batik-all:1.19"
@@ -522,7 +521,6 @@ dependencies {
exclude group: "org.slf4j", module: "slf4j-simple"
exclude group: "org.bouncycastle", module: "bcprov-jdk15on"
exclude group: "com.google.code.gson", module: "gson"
exclude group: "commons-io", module: "commons-io"
}
implementation 'org.apache.pdfbox:jbig2-imageio:3.0.4'
@@ -545,8 +543,10 @@ dependencies {
compileOnly "org.projectlombok:lombok:$lombokVersion"
annotationProcessor "org.projectlombok:lombok:$lombokVersion"
// Mockito (core)
testImplementation 'org.mockito:mockito-core:5.18.0'
// Mockito (core)
testImplementation 'org.mockito:mockito-core:5.17.0'
testRuntimeOnly 'org.mockito:mockito-inline:5.2.0'
}
-196
View File
@@ -1,196 +0,0 @@
### Eclipse ###
.metadata
bin/
tmp/
*.tmp
*.bak
*.exe
*.swp
*~.nib
local.properties
.settings/
.loadpath
.recommenders
.classpath
.project
version.properties
#### Stirling-PDF Files ###
pipeline/watchedFolders/
pipeline/finishedFolders/
customFiles/
configs/
watchedFolders/
clientWebUI/
!cucumber/
!cucumber/exampleFiles/
!cucumber/exampleFiles/example_html.zip
exampleYmlFiles/stirling/
/testing/file_snapshots
SwaggerDoc.json
# Gradle
.gradle
.lock
# External tool builders
.externalToolBuilders/
# Locally stored "Eclipse launch configurations"
*.launch
# PyDev specific (Python IDE for Eclipse)
*.pydevproject
# CDT-specific (C/C++ Development Tooling)
.cproject
# CDT- autotools
.autotools
# Java annotation processor (APT)
.factorypath
# PDT-specific (PHP Development Tools)
.buildpath
# sbteclipse plugin
.target
# Tern plugin
.tern-project
# TeXlipse plugin
.texlipse
# STS (Spring Tool Suite)
.springBeans
# Code Recommenders
.recommenders/
# Annotation Processing
.apt_generated/
.apt_generated_test/
# Scala IDE specific (Scala & Java development for Eclipse)
.cache-main
.scala_dependencies
.worksheet
# Uncomment this line if you wish to ignore the project description file.
# Typically, this file would be tracked if it contains build/dependency configurations:
#.project
### Eclipse Patch ###
# Spring Boot Tooling
.sts4-cache/
### Git ###
# Created by git for backups. To disable backups in Git:
# $ git config --global mergetool.keepBackup false
*.orig
# Created by git when using merge tools for conflicts
*.BACKUP.*
*.BASE.*
*.LOCAL.*
*.REMOTE.*
*_BACKUP_*.txt
*_BASE_*.txt
*_LOCAL_*.txt
*_REMOTE_*.txt
### Java ###
# Compiled class file
*.class
# Log file
*.log
# BlueJ files
*.ctxt
# Mobile Tools for Java (J2ME)
.mtj.tmp/
# Package Files #
*.jar
*.war
*.nar
*.ear
*.zip
*.tar.gz
*.rar
*.db
/build
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*.pyo
# Virtual environments
.env*
.venv*
env*/
venv*/
ENV/
env.bak/
venv.bak/
# VS Code
/.vscode/**/*
!/.vscode/settings.json
!/.vscode/extensions.json
# IntelliJ IDEA
.idea/
*.iml
out/
# Ignore Mac DS_Store files
.DS_Store
**/.DS_Store
# cucumber
/cucumber/reports/**
# Certs and Security Files
*.p12
*.pk8
*.pem
*.crt
*.cer
*.cert
*.der
*.key
*.csr
*.kdbx
*.jks
*.asc
# SSH Keys
*.pub
*.priv
id_rsa
id_rsa.pub
id_ecdsa
id_ecdsa.pub
id_ed25519
id_ed25519.pub
.ssh/
*ssh
# cache
.cache
.ruff_cache
.mypy_cache
.pytest_cache
.ipynb_checkpoints
**/jcef-bundle/
# node_modules
node_modules/
*.mjs
-65
View File
@@ -1,65 +0,0 @@
plugins {
id 'java-library'
id 'io.spring.dependency-management' version '1.1.7'
}
group = 'stirling.software'
version = '0.46.2'
ext {
lombokVersion = "1.18.38"
}
java {
sourceCompatibility = JavaVersion.VERSION_17
}
repositories {
mavenCentral()
}
configurations.all {
exclude group: 'commons-logging', module: 'commons-logging'
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
}
dependencyManagement {
imports {
mavenBom 'org.springframework.boot:spring-boot-dependencies:3.5.0'
}
}
dependencies {
implementation 'org.springframework.boot:spring-boot-starter-web'
implementation 'org.springframework.boot:spring-boot-starter-thymeleaf'
implementation 'org.springframework.boot:spring-boot-starter-aop'
implementation 'org.springframework.boot:spring-boot-starter-websocket'
implementation 'org.springframework.boot:spring-boot-starter-test'
implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20240325.1'
implementation 'com.fathzer:javaluator:3.0.6'
implementation 'com.posthog.java:posthog:1.2.0'
implementation 'io.github.pixee:java-security-toolkit:1.2.1'
implementation 'org.apache.commons:commons-lang3:3.17.0'
implementation 'com.drewnoakes:metadata-extractor:2.19.0' // Image metadata extractor
implementation 'com.vladsch.flexmark:flexmark-html2md-converter:0.64.8'
implementation "org.apache.pdfbox:pdfbox:$pdfboxVersion"
implementation 'jakarta.servlet:jakarta.servlet-api:6.1.0'
implementation 'org.snakeyaml:snakeyaml-engine:2.9'
implementation "org.springdoc:springdoc-openapi-starter-webmvc-ui:2.8.8"
compileOnly "org.projectlombok:lombok:$lombokVersion"
annotationProcessor "org.projectlombok:lombok:$lombokVersion"
testImplementation "org.springframework.boot:spring-boot-starter-test"
testImplementation 'org.mockito:mockito-core:5.18.0'
testImplementation 'org.mockito:mockito-junit-jupiter:5.18.0'
testRuntimeOnly 'org.mockito:mockito-inline:5.2.0'
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}
@@ -1,47 +0,0 @@
package stirling.software.common.annotations;
import java.lang.annotation.*;
import org.springframework.core.annotation.AliasFor;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
@RequestMapping(method = RequestMethod.POST)
public @interface AutoJobPostMapping {
@AliasFor(annotation = RequestMapping.class, attribute = "value")
String[] value() default {};
@AliasFor(annotation = RequestMapping.class, attribute = "consumes")
String[] consumes() default {"multipart/form-data"};
/**
* Custom timeout in milliseconds for this specific job. If not specified, the default system
* timeout will be used.
*/
long timeout() default -1;
/** Maximum number of times to retry the job on failure. Default is 1 (no retries). */
int retryCount() default 1;
/**
* Whether to track and report progress for this job. If enabled, the job will send progress
* updates through WebSocket.
*/
boolean trackProgress() default true;
/**
* Whether this job can be queued when system resources are limited. If enabled, jobs will be
* queued instead of rejected when the system is under high load. The queue size is dynamically
* adjusted based on available memory and CPU resources.
*/
boolean queueable() default false;
/**
* Optional resource weight of this job (1-100). Higher values indicate more resource-intensive
* jobs that may need stricter queuing. Default is 50 (medium weight).
*/
int resourceWeight() default 50;
}
@@ -1,249 +0,0 @@
package stirling.software.common.aop;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.*;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.controller.WebSocketProgressController;
import stirling.software.common.model.api.PDFFile;
import stirling.software.common.model.job.JobProgress;
import stirling.software.common.service.FileOrUploadService;
import stirling.software.common.service.FileStorage;
import stirling.software.common.service.JobExecutorService;
@Aspect
@Component
@RequiredArgsConstructor
@Slf4j
public class AutoJobAspect {
private final JobExecutorService jobExecutorService;
private final HttpServletRequest request;
private final FileOrUploadService fileOrUploadService;
private final FileStorage fileStorage;
private final WebSocketProgressController webSocketSender;
@Around("@annotation(autoJobPostMapping)")
public Object wrapWithJobExecution(
ProceedingJoinPoint joinPoint, AutoJobPostMapping autoJobPostMapping) {
// Extract parameters from the request and annotation
boolean async = Boolean.parseBoolean(request.getParameter("async"));
long timeout = autoJobPostMapping.timeout();
int retryCount = autoJobPostMapping.retryCount();
boolean trackProgress = autoJobPostMapping.trackProgress();
log.debug(
"AutoJobPostMapping execution with async={}, timeout={}, retryCount={}, trackProgress={}",
async,
timeout > 0 ? timeout : "default",
retryCount,
trackProgress);
// Inspect and possibly mutate arguments
Object[] args = joinPoint.getArgs();
for (int i = 0; i < args.length; i++) {
Object arg = args[i];
if (arg instanceof PDFFile pdfFile) {
// Case 1: fileId is provided but no fileInput
if (pdfFile.getFileInput() == null && pdfFile.getFileId() != null) {
try {
log.debug("Using fileId {} to get file content", pdfFile.getFileId());
MultipartFile file = fileStorage.retrieveFile(pdfFile.getFileId());
pdfFile.setFileInput(file);
} catch (Exception e) {
throw new RuntimeException(
"Failed to resolve file by ID: " + pdfFile.getFileId(), e);
}
}
// Case 2: For async requests, we need to make a copy of the MultipartFile
else if (async && pdfFile.getFileInput() != null) {
try {
log.debug("Making persistent copy of uploaded file for async processing");
MultipartFile originalFile = pdfFile.getFileInput();
String fileId = fileStorage.storeFile(originalFile);
// Store the fileId for later reference
pdfFile.setFileId(fileId);
// Replace the original MultipartFile with our persistent copy
MultipartFile persistentFile = fileStorage.retrieveFile(fileId);
pdfFile.setFileInput(persistentFile);
log.debug("Created persistent file copy with fileId: {}", fileId);
} catch (IOException e) {
throw new RuntimeException(
"Failed to create persistent copy of uploaded file", e);
}
}
}
}
// Extract queueable and resourceWeight parameters
boolean queueable = autoJobPostMapping.queueable();
int resourceWeight = autoJobPostMapping.resourceWeight();
// Integrate with the enhanced JobExecutorService
if (retryCount <= 1) {
// No retries needed, simple execution
return jobExecutorService.runJobGeneric(
async,
() -> {
try {
if (trackProgress && async) {
String jobId = (String) request.getAttribute("jobId");
if (jobId != null) {
webSocketSender.sendProgress(
jobId,
new JobProgress(
jobId,
"Processing",
50,
"Executing operation"));
}
}
return joinPoint.proceed(args);
} catch (Throwable ex) {
log.error(
"AutoJobAspect caught exception during job execution: {}",
ex.getMessage(),
ex);
throw new RuntimeException(ex);
}
},
timeout,
queueable,
resourceWeight);
} else {
// Use retry logic
return executeWithRetries(
joinPoint,
args,
async,
timeout,
retryCount,
trackProgress,
queueable,
resourceWeight);
}
}
private Object executeWithRetries(
ProceedingJoinPoint joinPoint,
Object[] args,
boolean async,
long timeout,
int maxRetries,
boolean trackProgress,
boolean queueable,
int resourceWeight) {
AtomicInteger attempts = new AtomicInteger(0);
// Use AtomicReference to make the jobId effectively final for lambda usage
AtomicReference<String> jobIdRef = new AtomicReference<>();
return jobExecutorService.runJobGeneric(
async,
() -> {
int currentAttempt = attempts.incrementAndGet();
try {
if (trackProgress && async) {
// Only get the jobId once and store it in the AtomicReference
if (jobIdRef.get() == null) {
jobIdRef.set(getJobIdFromContext());
}
String jobId = jobIdRef.get();
if (jobId != null) {
webSocketSender.sendProgress(
jobId,
new JobProgress(
jobId,
"Started",
0,
"Executing (attempt "
+ currentAttempt
+ " of "
+ Math.max(1, maxRetries)
+ ")"));
}
}
return joinPoint.proceed(args);
} catch (Throwable ex) {
log.error(
"AutoJobAspect caught exception during job execution (attempt {}/{}): {}",
currentAttempt,
Math.max(1, maxRetries),
ex.getMessage(),
ex);
// Check if we should retry
if (currentAttempt < maxRetries) {
log.info(
"Retrying operation, attempt {}/{}",
currentAttempt + 1,
maxRetries);
String jobId = jobIdRef.get();
if (trackProgress && async && jobId != null) {
webSocketSender.sendProgress(
jobId,
new JobProgress(
jobId,
"Retrying",
(int) (currentAttempt * 100.0 / maxRetries),
"Retry attempt "
+ (currentAttempt + 1)
+ " of "
+ maxRetries));
}
try {
// Simple exponential backoff
Thread.sleep(100 * currentAttempt);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
}
// Recursive call to retry
return executeWithRetries(
joinPoint,
args,
async,
timeout,
maxRetries,
trackProgress,
queueable,
resourceWeight);
}
// No more retries, throw the exception
throw new RuntimeException("Job failed: " + ex.getMessage(), ex);
}
},
timeout,
queueable,
resourceWeight);
}
// Try to get the job ID from the context (if this is an async job)
private String getJobIdFromContext() {
try {
return (String) request.getAttribute("jobId");
} catch (Exception e) {
log.debug("Could not retrieve job ID from context: {}", e.getMessage());
return null;
}
}
}
@@ -1,23 +0,0 @@
package stirling.software.common.config;
import org.springframework.context.annotation.Configuration;
import org.springframework.messaging.simp.config.MessageBrokerRegistry;
import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker;
import org.springframework.web.socket.config.annotation.StompEndpointRegistry;
import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer;
@Configuration
@EnableWebSocketMessageBroker
public class WebSocketConfig implements WebSocketMessageBrokerConfigurer {
@Override
public void configureMessageBroker(MessageBrokerRegistry config) {
config.enableSimpleBroker("/topic");
config.setApplicationDestinationPrefixes("/app");
}
@Override
public void registerStompEndpoints(StompEndpointRegistry registry) {
registry.addEndpoint("/ws").setAllowedOriginPatterns("*").withSockJS();
}
}
@@ -1,7 +0,0 @@
package stirling.software.common.configuration.interfaces;
public interface ShowAdminInterface {
default boolean getShowUpdateOnlyAdmins() {
return true;
}
}
@@ -1,114 +0,0 @@
package stirling.software.common.controller;
import java.util.Map;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RestController;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.job.JobResult;
import stirling.software.common.model.job.JobStats;
import stirling.software.common.service.FileStorage;
import stirling.software.common.service.TaskManager;
/** REST controller for job-related endpoints */
@RestController
@RequiredArgsConstructor
@Slf4j
public class JobController {
private final TaskManager taskManager;
private final FileStorage fileStorage;
/**
* Get the status of a job
*
* @param jobId The job ID
* @return The job result
*/
@GetMapping("/api/v1/general/job/{jobId}")
public ResponseEntity<?> getJobStatus(@PathVariable("jobId") String jobId) {
JobResult result = taskManager.getJobResult(jobId);
if (result == null) {
return ResponseEntity.notFound().build();
}
return ResponseEntity.ok(result);
}
/**
* Get the result of a job
*
* @param jobId The job ID
* @return The job result
*/
@GetMapping("/api/v1/general/job/{jobId}/result")
public ResponseEntity<?> getJobResult(@PathVariable("jobId") String jobId) {
JobResult result = taskManager.getJobResult(jobId);
if (result == null) {
return ResponseEntity.notFound().build();
}
if (!result.isComplete()) {
return ResponseEntity.badRequest().body("Job is not complete yet");
}
if (result.getError() != null) {
return ResponseEntity.badRequest().body("Job failed: " + result.getError());
}
if (result.getFileId() != null) {
try {
byte[] fileContent = fileStorage.retrieveBytes(result.getFileId());
return ResponseEntity.ok()
.header("Content-Type", result.getContentType())
.header(
"Content-Disposition",
"form-data; name=\"attachment\"; filename=\""
+ result.getOriginalFileName()
+ "\"")
.body(fileContent);
} catch (Exception e) {
log.error("Error retrieving file for job {}: {}", jobId, e.getMessage(), e);
return ResponseEntity.internalServerError()
.body("Error retrieving file: " + e.getMessage());
}
}
return ResponseEntity.ok(result.getResult());
}
/**
* Get statistics about jobs in the system
*
* @return Job statistics
*/
@GetMapping("/api/v1/general/job/stats")
public ResponseEntity<JobStats> getJobStats() {
JobStats stats = taskManager.getJobStats();
return ResponseEntity.ok(stats);
}
/**
* Manually trigger cleanup of old jobs
*
* @return A response indicating how many jobs were cleaned up
*/
@PostMapping("/api/v1/general/job/cleanup")
public ResponseEntity<?> cleanupOldJobs() {
int beforeCount = taskManager.getJobStats().getTotalJobs();
taskManager.cleanupOldJobs();
int afterCount = taskManager.getJobStats().getTotalJobs();
int removedCount = beforeCount - afterCount;
return ResponseEntity.ok(
Map.of(
"message", "Cleanup complete",
"removedJobs", removedCount,
"remainingJobs", afterCount));
}
}
@@ -1,27 +0,0 @@
package stirling.software.common.controller;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.stereotype.Controller;
import lombok.NoArgsConstructor;
import stirling.software.common.model.job.JobProgress;
@Controller
@NoArgsConstructor
public class WebSocketProgressController {
private SimpMessagingTemplate messagingTemplate;
@Autowired(required = false)
public void setMessagingTemplate(SimpMessagingTemplate messagingTemplate) {
this.messagingTemplate = messagingTemplate;
}
public void sendProgress(String jobId, JobProgress progress) {
if (messagingTemplate != null) {
messagingTemplate.convertAndSend("/topic/progress/" + jobId, progress);
}
}
}
@@ -1,19 +0,0 @@
package stirling.software.common.model;
import java.util.Calendar;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
public class PdfMetadata {
private String author;
private String producer;
private String title;
private String creator;
private String subject;
private String keywords;
private Calendar creationDate;
private Calendar modificationDate;
}
@@ -1,19 +0,0 @@
package stirling.software.common.model.api;
import org.springframework.web.multipart.MultipartFile;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import lombok.EqualsAndHashCode;
@Data
@EqualsAndHashCode
public class GeneralFile {
@Schema(
description = "The input file",
requiredMode = Schema.RequiredMode.REQUIRED,
format = "binary")
private MultipartFile fileInput;
}
@@ -1,28 +0,0 @@
package stirling.software.common.model.api.security;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import lombok.EqualsAndHashCode;
@Data
@EqualsAndHashCode
public class RedactionArea {
@Schema(description = "The left edge point of the area to be redacted.")
private Double x;
@Schema(description = "The top edge point of the area to be redacted.")
private Double y;
@Schema(description = "The height of the area to be redacted.")
private Double height;
@Schema(description = "The width of the area to be redacted.")
private Double width;
@Schema(description = "The page on which the area should be redacted.")
private Integer page;
@Schema(description = "The color used to redact the specified area.")
private String color;
}
@@ -1,66 +0,0 @@
package stirling.software.common.model.enumeration;
import java.util.LinkedHashMap;
import java.util.Map;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
@Getter
@RequiredArgsConstructor
public enum Role {
// Unlimited access
ADMIN("ROLE_ADMIN", Integer.MAX_VALUE, Integer.MAX_VALUE, "adminUserSettings.admin"),
// Unlimited access
USER("ROLE_USER", Integer.MAX_VALUE, Integer.MAX_VALUE, "adminUserSettings.user"),
// 40 API calls Per Day, 40 web calls
LIMITED_API_USER("ROLE_LIMITED_API_USER", 40, 40, "adminUserSettings.apiUser"),
// 20 API calls Per Day, 20 web calls
EXTRA_LIMITED_API_USER("ROLE_EXTRA_LIMITED_API_USER", 20, 20, "adminUserSettings.extraApiUser"),
// 0 API calls per day and 20 web calls
WEB_ONLY_USER("ROLE_WEB_ONLY_USER", 0, 20, "adminUserSettings.webOnlyUser"),
INTERNAL_API_USER(
"STIRLING-PDF-BACKEND-API-USER",
Integer.MAX_VALUE,
Integer.MAX_VALUE,
"adminUserSettings.internalApiUser"),
DEMO_USER("ROLE_DEMO_USER", 100, 100, "adminUserSettings.demoUser");
private final String roleId;
private final int apiCallsPerDay;
private final int webCallsPerDay;
private final String roleName;
public static String getRoleNameByRoleId(String roleId) {
// Using the fromString method to get the Role enum based on the roleId
Role role = fromString(roleId);
// Return the roleName of the found Role enum
return role.getRoleName();
}
// Method to retrieve all role IDs and role names
public static Map<String, String> getAllRoleDetails() {
// Using LinkedHashMap to preserve order
Map<String, String> roleDetails = new LinkedHashMap<>();
for (Role role : Role.values()) {
roleDetails.put(role.getRoleId(), role.getRoleName());
}
return roleDetails;
}
public static Role fromString(String roleId) {
for (Role role : Role.values()) {
if (role.getRoleId().equalsIgnoreCase(roleId)) {
return role;
}
}
throw new IllegalArgumentException("No Role defined for id: " + roleId);
}
}
@@ -1,7 +0,0 @@
package stirling.software.common.model.exception;
public class UnsupportedClaimException extends RuntimeException {
public UnsupportedClaimException(String message) {
super(message);
}
}
@@ -1,15 +0,0 @@
package stirling.software.common.model.job;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class JobProgress {
private String jobId;
private String status;
private int percentComplete;
private String message;
}
@@ -1,14 +0,0 @@
package stirling.software.common.model.job;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class JobResponse<T> {
private boolean async;
private String jobId;
private T result;
}
@@ -1,94 +0,0 @@
package stirling.software.common.model.job;
import java.time.LocalDateTime;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/** Represents the result of a job execution. Used by the TaskManager to store job results. */
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class JobResult {
/** The job ID */
private String jobId;
/** Flag indicating if the job is complete */
private boolean complete;
/** Error message if the job failed */
private String error;
/** The file ID of the result file, if applicable */
private String fileId;
/** Original file name, if applicable */
private String originalFileName;
/** MIME type of the result, if applicable */
private String contentType;
/** Time when the job was created */
private LocalDateTime createdAt;
/** Time when the job was completed */
private LocalDateTime completedAt;
/** The actual result object, if not a file */
private Object result;
/**
* Create a new JobResult with the given job ID
*
* @param jobId The job ID
* @return A new JobResult
*/
public static JobResult createNew(String jobId) {
return JobResult.builder()
.jobId(jobId)
.complete(false)
.createdAt(LocalDateTime.now())
.build();
}
/**
* Mark this job as complete with a file result
*
* @param fileId The file ID of the result
* @param originalFileName The original file name
* @param contentType The content type of the file
*/
public void completeWithFile(String fileId, String originalFileName, String contentType) {
this.complete = true;
this.fileId = fileId;
this.originalFileName = originalFileName;
this.contentType = contentType;
this.completedAt = LocalDateTime.now();
}
/**
* Mark this job as complete with a general result
*
* @param result The result object
*/
public void completeWithResult(Object result) {
this.complete = true;
this.result = result;
this.completedAt = LocalDateTime.now();
}
/**
* Mark this job as failed with an error message
*
* @param error The error message
*/
public void failWithError(String error) {
this.complete = true;
this.error = error;
this.completedAt = LocalDateTime.now();
}
}
@@ -1,43 +0,0 @@
package stirling.software.common.model.job;
import java.time.LocalDateTime;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/** Represents statistics about jobs in the system */
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class JobStats {
/** Total number of jobs (active and completed) */
private int totalJobs;
/** Number of active (incomplete) jobs */
private int activeJobs;
/** Number of completed jobs */
private int completedJobs;
/** Number of failed jobs */
private int failedJobs;
/** Number of successful jobs */
private int successfulJobs;
/** Number of jobs with file results */
private int fileResultJobs;
/** The oldest active job's creation timestamp */
private LocalDateTime oldestActiveJobTime;
/** The newest active job's creation timestamp */
private LocalDateTime newestActiveJobTime;
/** The average processing time for completed jobs in milliseconds */
private long averageProcessingTimeMs;
}
@@ -1,78 +0,0 @@
package stirling.software.common.service;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.file.*;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class FileOrUploadService {
@Value("${stirling.tempDir:/tmp/stirling-files}")
private String tempDirPath;
public Path resolveFilePath(String fileId) {
return Path.of(tempDirPath).resolve(fileId);
}
public MultipartFile toMockMultipartFile(String name, byte[] data) throws IOException {
return new CustomMultipartFile(name, data);
}
// Custom implementation of MultipartFile
private static class CustomMultipartFile implements MultipartFile {
private final String name;
private final byte[] content;
public CustomMultipartFile(String name, byte[] content) {
this.name = name;
this.content = content;
}
@Override
public String getName() {
return name;
}
@Override
public String getOriginalFilename() {
return name;
}
@Override
public String getContentType() {
return "application/pdf";
}
@Override
public boolean isEmpty() {
return content == null || content.length == 0;
}
@Override
public long getSize() {
return content.length;
}
@Override
public byte[] getBytes() throws IOException {
return content;
}
@Override
public java.io.InputStream getInputStream() throws IOException {
return new ByteArrayInputStream(content);
}
@Override
public void transferTo(java.io.File dest) throws IOException, IllegalStateException {
Files.write(dest.toPath(), content);
}
}
}
@@ -1,152 +0,0 @@
package stirling.software.common.service;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.UUID;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Service for storing and retrieving files with unique file IDs. Used by the AutoJobPostMapping
* system to handle file references.
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class FileStorage {
@Value("${stirling.tempDir:/tmp/stirling-files}")
private String tempDirPath;
private final FileOrUploadService fileOrUploadService;
/**
* Store a file and return its unique ID
*
* @param file The file to store
* @return The unique ID assigned to the file
* @throws IOException If there is an error storing the file
*/
public String storeFile(MultipartFile file) throws IOException {
String fileId = generateFileId();
Path filePath = getFilePath(fileId);
// Ensure the directory exists
Files.createDirectories(filePath.getParent());
// Transfer the file to the storage location
file.transferTo(filePath.toFile());
log.debug("Stored file with ID: {}", fileId);
return fileId;
}
/**
* Store a byte array as a file and return its unique ID
*
* @param bytes The byte array to store
* @param originalName The original name of the file (for extension)
* @return The unique ID assigned to the file
* @throws IOException If there is an error storing the file
*/
public String storeBytes(byte[] bytes, String originalName) throws IOException {
String fileId = generateFileId();
Path filePath = getFilePath(fileId);
// Ensure the directory exists
Files.createDirectories(filePath.getParent());
// Write the bytes to the file
Files.write(filePath, bytes);
log.debug("Stored byte array with ID: {}", fileId);
return fileId;
}
/**
* Retrieve a file by its ID as a MultipartFile
*
* @param fileId The ID of the file to retrieve
* @return The file as a MultipartFile
* @throws IOException If the file doesn't exist or can't be read
*/
public MultipartFile retrieveFile(String fileId) throws IOException {
Path filePath = getFilePath(fileId);
if (!Files.exists(filePath)) {
throw new IOException("File not found with ID: " + fileId);
}
byte[] fileData = Files.readAllBytes(filePath);
return fileOrUploadService.toMockMultipartFile(fileId, fileData);
}
/**
* Retrieve a file by its ID as a byte array
*
* @param fileId The ID of the file to retrieve
* @return The file as a byte array
* @throws IOException If the file doesn't exist or can't be read
*/
public byte[] retrieveBytes(String fileId) throws IOException {
Path filePath = getFilePath(fileId);
if (!Files.exists(filePath)) {
throw new IOException("File not found with ID: " + fileId);
}
return Files.readAllBytes(filePath);
}
/**
* Delete a file by its ID
*
* @param fileId The ID of the file to delete
* @return true if the file was deleted, false otherwise
*/
public boolean deleteFile(String fileId) {
try {
Path filePath = getFilePath(fileId);
return Files.deleteIfExists(filePath);
} catch (IOException e) {
log.error("Error deleting file with ID: {}", fileId, e);
return false;
}
}
/**
* Check if a file exists by its ID
*
* @param fileId The ID of the file to check
* @return true if the file exists, false otherwise
*/
public boolean fileExists(String fileId) {
Path filePath = getFilePath(fileId);
return Files.exists(filePath);
}
/**
* Get the path for a file ID
*
* @param fileId The ID of the file
* @return The path to the file
*/
private Path getFilePath(String fileId) {
return Path.of(tempDirPath).resolve(fileId);
}
/**
* Generate a unique file ID
*
* @return A unique file ID
*/
private String generateFileId() {
return UUID.randomUUID().toString();
}
}
@@ -1,465 +0,0 @@
package stirling.software.common.service;
import java.io.IOException;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Supplier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import jakarta.servlet.http.HttpServletRequest;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.controller.WebSocketProgressController;
import stirling.software.common.model.job.JobProgress;
import stirling.software.common.model.job.JobResponse;
import stirling.software.common.util.ExecutorFactory;
/** Service for executing jobs asynchronously or synchronously */
@Service
@Slf4j
public class JobExecutorService {
private final TaskManager taskManager;
private final WebSocketProgressController webSocketSender;
private final FileStorage fileStorage;
private final HttpServletRequest request;
private final ResourceMonitor resourceMonitor;
private final JobQueue jobQueue;
private final ExecutorService executor = ExecutorFactory.newVirtualOrCachedThreadExecutor();
private final long effectiveTimeoutMs;
public JobExecutorService(
TaskManager taskManager,
WebSocketProgressController webSocketSender,
FileStorage fileStorage,
HttpServletRequest request,
ResourceMonitor resourceMonitor,
JobQueue jobQueue,
@Value("${spring.mvc.async.request-timeout:1200000}") long asyncRequestTimeoutMs,
@Value("${server.servlet.session.timeout:30m}") String sessionTimeout) {
this.taskManager = taskManager;
this.webSocketSender = webSocketSender;
this.fileStorage = fileStorage;
this.request = request;
this.resourceMonitor = resourceMonitor;
this.jobQueue = jobQueue;
// Parse session timeout and calculate effective timeout once during initialization
long sessionTimeoutMs = parseSessionTimeout(sessionTimeout);
this.effectiveTimeoutMs = Math.min(asyncRequestTimeoutMs, sessionTimeoutMs);
log.debug(
"Job executor configured with effective timeout of {} ms", this.effectiveTimeoutMs);
}
/**
* Run a job either asynchronously or synchronously
*
* @param async Whether to run the job asynchronously
* @param work The work to be done
* @return The response
*/
public ResponseEntity<?> runJobGeneric(boolean async, Supplier<Object> work) {
return runJobGeneric(async, work, -1);
}
/**
* Run a job either asynchronously or synchronously with a custom timeout
*
* @param async Whether to run the job asynchronously
* @param work The work to be done
* @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default
* @return The response
*/
public ResponseEntity<?> runJobGeneric(
boolean async, Supplier<Object> work, long customTimeoutMs) {
return runJobGeneric(async, work, customTimeoutMs, false, 50);
}
/**
* Run a job either asynchronously or synchronously with custom parameters
*
* @param async Whether to run the job asynchronously
* @param work The work to be done
* @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default
* @param queueable Whether this job can be queued when system resources are limited
* @param resourceWeight The resource weight of this job (1-100)
* @return The response
*/
public ResponseEntity<?> runJobGeneric(
boolean async,
Supplier<Object> work,
long customTimeoutMs,
boolean queueable,
int resourceWeight) {
String jobId = UUID.randomUUID().toString();
// Store the job ID in the request for potential use by other components
if (request != null) {
request.setAttribute("jobId", jobId);
}
// Determine which timeout to use
long timeoutToUse = customTimeoutMs > 0 ? customTimeoutMs : effectiveTimeoutMs;
log.debug(
"Running job with ID: {}, async: {}, timeout: {}ms, queueable: {}, weight: {}",
jobId,
async,
timeoutToUse,
queueable,
resourceWeight);
// Check if we need to queue this job based on resource availability
boolean shouldQueue =
queueable
&& async
&& // Only async jobs can be queued
resourceMonitor.shouldQueueJob(resourceWeight);
if (shouldQueue) {
// Queue the job instead of executing immediately
log.debug(
"Queueing job {} due to resource constraints (weight: {})",
jobId,
resourceWeight);
taskManager.createTask(jobId);
// Create a specialized wrapper that updates the TaskManager
Supplier<Object> wrappedWork =
() -> {
try {
Object result = work.get();
processJobResult(jobId, result);
return result;
} catch (Exception e) {
log.error(
"Error executing queued job {}: {}", jobId, e.getMessage(), e);
taskManager.setError(jobId, e.getMessage());
throw e;
}
};
// Queue the job and get the future
CompletableFuture<ResponseEntity<?>> future =
jobQueue.queueJob(jobId, resourceWeight, wrappedWork, timeoutToUse);
// Return immediately with job ID
return ResponseEntity.ok().body(new JobResponse<>(true, jobId, null));
} else if (async) {
taskManager.createTask(jobId);
webSocketSender.sendProgress(jobId, new JobProgress(jobId, "Started", 0, "Running"));
executor.execute(
() -> {
try {
log.debug(
"Running async job {} with timeout {} ms", jobId, timeoutToUse);
// Execute with timeout
Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
processJobResult(jobId, result);
webSocketSender.sendProgress(
jobId, new JobProgress(jobId, "Done", 100, "Complete"));
} catch (TimeoutException te) {
log.error("Job {} timed out after {} ms", jobId, timeoutToUse);
taskManager.setError(jobId, "Job timed out");
webSocketSender.sendProgress(
jobId, new JobProgress(jobId, "Error", 100, "Job timed out"));
} catch (Exception e) {
log.error("Error executing job {}: {}", jobId, e.getMessage(), e);
taskManager.setError(jobId, e.getMessage());
webSocketSender.sendProgress(
jobId, new JobProgress(jobId, "Error", 100, e.getMessage()));
}
});
return ResponseEntity.ok().body(new JobResponse<>(true, jobId, null));
} else {
try {
log.debug("Running sync job with timeout {} ms", timeoutToUse);
// Execute with timeout
Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
// If the result is already a ResponseEntity, return it directly
if (result instanceof ResponseEntity) {
return (ResponseEntity<?>) result;
}
// Process different result types
return handleResultForSyncJob(result);
} catch (TimeoutException te) {
log.error("Synchronous job timed out after {} ms", timeoutToUse);
return ResponseEntity.internalServerError()
.body(Map.of("error", "Job timed out after " + timeoutToUse + " ms"));
} catch (Exception e) {
log.error("Error executing synchronous job: {}", e.getMessage(), e);
// Construct a JSON error response
return ResponseEntity.internalServerError()
.body(Map.of("error", "Job failed: " + e.getMessage()));
}
}
}
/**
* Process the result of an asynchronous job
*
* @param jobId The job ID
* @param result The result
*/
private void processJobResult(String jobId, Object result) {
try {
if (result instanceof byte[]) {
// Store byte array as a file
String fileId = fileStorage.storeBytes((byte[]) result, "result.pdf");
taskManager.setFileResult(jobId, fileId, "result.pdf", "application/pdf");
log.debug("Stored byte[] result with fileId: {}", fileId);
} else if (result instanceof ResponseEntity) {
ResponseEntity<?> response = (ResponseEntity<?>) result;
Object body = response.getBody();
if (body instanceof byte[]) {
// Extract filename from content-disposition header if available
String filename = "result.pdf";
String contentType = "application/pdf";
if (response.getHeaders().getContentDisposition() != null) {
String disposition =
response.getHeaders().getContentDisposition().toString();
if (disposition.contains("filename=")) {
filename =
disposition.substring(
disposition.indexOf("filename=") + 9,
disposition.lastIndexOf("\""));
}
}
if (response.getHeaders().getContentType() != null) {
contentType = response.getHeaders().getContentType().toString();
}
String fileId = fileStorage.storeBytes((byte[]) body, filename);
taskManager.setFileResult(jobId, fileId, filename, contentType);
log.debug("Stored ResponseEntity<byte[]> result with fileId: {}", fileId);
} else {
// Check if the response body contains a fileId
if (body != null && body.toString().contains("fileId")) {
try {
// Try to extract fileId using reflection
java.lang.reflect.Method getFileId =
body.getClass().getMethod("getFileId");
String fileId = (String) getFileId.invoke(body);
if (fileId != null && !fileId.isEmpty()) {
// Try to get filename and content type
String filename = "result.pdf";
String contentType = "application/pdf";
try {
java.lang.reflect.Method getOriginalFileName =
body.getClass().getMethod("getOriginalFilename");
String origName = (String) getOriginalFileName.invoke(body);
if (origName != null && !origName.isEmpty()) {
filename = origName;
}
} catch (Exception e) {
log.debug(
"Could not get original filename: {}", e.getMessage());
}
try {
java.lang.reflect.Method getContentType =
body.getClass().getMethod("getContentType");
String ct = (String) getContentType.invoke(body);
if (ct != null && !ct.isEmpty()) {
contentType = ct;
}
} catch (Exception e) {
log.debug("Could not get content type: {}", e.getMessage());
}
taskManager.setFileResult(jobId, fileId, filename, contentType);
log.debug("Extracted fileId from response body: {}", fileId);
taskManager.setComplete(jobId);
return;
}
} catch (Exception e) {
log.debug(
"Failed to extract fileId from response body: {}",
e.getMessage());
}
}
// Store generic result
taskManager.setResult(jobId, body);
}
} else if (result instanceof MultipartFile) {
MultipartFile file = (MultipartFile) result;
String fileId = fileStorage.storeFile(file);
taskManager.setFileResult(
jobId, fileId, file.getOriginalFilename(), file.getContentType());
log.debug("Stored MultipartFile result with fileId: {}", fileId);
} else {
// Check if result has a fileId field
if (result != null) {
try {
// Try to extract fileId using reflection
java.lang.reflect.Method getFileId =
result.getClass().getMethod("getFileId");
String fileId = (String) getFileId.invoke(result);
if (fileId != null && !fileId.isEmpty()) {
// Try to get filename and content type
String filename = "result.pdf";
String contentType = "application/pdf";
try {
java.lang.reflect.Method getOriginalFileName =
result.getClass().getMethod("getOriginalFilename");
String origName = (String) getOriginalFileName.invoke(result);
if (origName != null && !origName.isEmpty()) {
filename = origName;
}
} catch (Exception e) {
log.debug("Could not get original filename: {}", e.getMessage());
}
try {
java.lang.reflect.Method getContentType =
result.getClass().getMethod("getContentType");
String ct = (String) getContentType.invoke(result);
if (ct != null && !ct.isEmpty()) {
contentType = ct;
}
} catch (Exception e) {
log.debug("Could not get content type: {}", e.getMessage());
}
taskManager.setFileResult(jobId, fileId, filename, contentType);
log.debug("Extracted fileId from result object: {}", fileId);
taskManager.setComplete(jobId);
return;
}
} catch (Exception e) {
log.debug(
"Failed to extract fileId from result object: {}", e.getMessage());
}
}
// Default case: store the result as is
taskManager.setResult(jobId, result);
}
taskManager.setComplete(jobId);
} catch (Exception e) {
log.error("Error processing job result: {}", e.getMessage(), e);
taskManager.setError(jobId, "Error processing result: " + e.getMessage());
}
}
/**
* Handle different result types for synchronous jobs
*
* @param result The result object
* @return The appropriate ResponseEntity
* @throws IOException If there is an error processing the result
*/
private ResponseEntity<?> handleResultForSyncJob(Object result) throws IOException {
if (result instanceof byte[]) {
// Return byte array as PDF
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_PDF)
.header(
HttpHeaders.CONTENT_DISPOSITION,
"form-data; name=\"attachment\"; filename=\"result.pdf\"")
.body(result);
} else if (result instanceof MultipartFile) {
// Return MultipartFile content
MultipartFile file = (MultipartFile) result;
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType(file.getContentType()))
.header(
HttpHeaders.CONTENT_DISPOSITION,
"form-data; name=\"attachment\"; filename=\""
+ file.getOriginalFilename()
+ "\"")
.body(file.getBytes());
} else {
// Default case: return as JSON
return ResponseEntity.ok(result);
}
}
/**
* Parse session timeout string (e.g., "30m", "1h") to milliseconds
*
* @param timeout The timeout string
* @return The timeout in milliseconds
*/
private long parseSessionTimeout(String timeout) {
if (timeout == null || timeout.isEmpty()) {
return 30 * 60 * 1000; // Default: 30 minutes
}
try {
String value = timeout.replaceAll("[^\\d.]", "");
String unit = timeout.replaceAll("[\\d.]", "");
double numericValue = Double.parseDouble(value);
return switch (unit.toLowerCase()) {
case "s" -> (long) (numericValue * 1000);
case "m" -> (long) (numericValue * 60 * 1000);
case "h" -> (long) (numericValue * 60 * 60 * 1000);
case "d" -> (long) (numericValue * 24 * 60 * 60 * 1000);
default -> (long) (numericValue * 60 * 1000); // Default to minutes
};
} catch (Exception e) {
log.warn("Could not parse session timeout '{}', using default", timeout);
return 30 * 60 * 1000; // Default: 30 minutes
}
}
/**
* Execute a supplier with a timeout
*
* @param supplier The supplier to execute
* @param timeoutMs The timeout in milliseconds
* @return The result from the supplier
* @throws TimeoutException If the execution times out
* @throws Exception If the supplier throws an exception
*/
private <T> T executeWithTimeout(Supplier<T> supplier, long timeoutMs)
throws TimeoutException, Exception {
java.util.concurrent.CompletableFuture<T> future =
java.util.concurrent.CompletableFuture.supplyAsync(supplier);
try {
return future.get(timeoutMs, TimeUnit.MILLISECONDS);
} catch (java.util.concurrent.TimeoutException e) {
future.cancel(true);
throw new TimeoutException("Execution timed out after " + timeoutMs + " ms");
} catch (java.util.concurrent.ExecutionException e) {
throw (Exception) e.getCause();
} catch (java.util.concurrent.CancellationException e) {
throw new Exception("Execution was cancelled", e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new Exception("Execution was interrupted", e);
}
}
}
@@ -1,428 +0,0 @@
package stirling.software.common.service;
import java.time.Instant;
import java.util.Map;
import java.util.concurrent.*;
import java.util.function.Supplier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.controller.WebSocketProgressController;
import stirling.software.common.model.job.JobProgress;
import stirling.software.common.util.ExecutorFactory;
/**
* Manages a queue of jobs with dynamic sizing based on system resources. Used when system resources
* are limited to prevent overloading.
*/
@Service
@Slf4j
public class JobQueue {
private final ResourceMonitor resourceMonitor;
private final WebSocketProgressController webSocketSender;
@Value("${stirling.job.queue.base-capacity:10}")
private int baseQueueCapacity = 10;
@Value("${stirling.job.queue.min-capacity:2}")
private int minQueueCapacity = 2;
@Value("${stirling.job.queue.check-interval-ms:1000}")
private long queueCheckIntervalMs = 1000;
@Value("${stirling.job.queue.max-wait-time-ms:600000}")
private long maxWaitTimeMs = 600000; // 10 minutes
private BlockingQueue<QueuedJob> jobQueue;
private final Map<String, QueuedJob> jobMap = new ConcurrentHashMap<>();
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
private final ExecutorService jobExecutor = ExecutorFactory.newVirtualOrCachedThreadExecutor();
private boolean shuttingDown = false;
@Getter private int rejectedJobs = 0;
@Getter private int totalQueuedJobs = 0;
@Getter private int currentQueueSize = 0;
/** Represents a job waiting in the queue. */
@Data
@AllArgsConstructor
private static class QueuedJob {
private final String jobId;
private final int resourceWeight;
private final Supplier<Object> work;
private final long timeoutMs;
private final Instant queuedAt;
private CompletableFuture<ResponseEntity<?>> future;
private volatile boolean cancelled = false;
}
public JobQueue(ResourceMonitor resourceMonitor, WebSocketProgressController webSocketSender) {
this.resourceMonitor = resourceMonitor;
this.webSocketSender = webSocketSender;
// Initialize with dynamic capacity
int capacity =
resourceMonitor.calculateDynamicQueueCapacity(baseQueueCapacity, minQueueCapacity);
this.jobQueue = new LinkedBlockingQueue<>(capacity);
}
@PostConstruct
public void initialize() {
log.debug(
"Starting job queue with base capacity {}, min capacity {}",
baseQueueCapacity,
minQueueCapacity);
// Periodically process the job queue
scheduler.scheduleWithFixedDelay(
this::processQueue, 0, queueCheckIntervalMs, TimeUnit.MILLISECONDS);
// Periodically update queue capacity based on resource usage
scheduler.scheduleWithFixedDelay(
this::updateQueueCapacity,
10000, // Initial delay
30000, // 30 second interval
TimeUnit.MILLISECONDS);
}
@PreDestroy
public void shutdown() {
log.info("Shutting down job queue");
shuttingDown = true;
// Complete any futures that are still waiting
jobMap.forEach(
(id, job) -> {
if (!job.future.isDone()) {
job.future.completeExceptionally(
new RuntimeException("Server shutting down, job cancelled"));
}
});
scheduler.shutdownNow();
jobExecutor.shutdownNow();
log.info(
"Job queue shutdown complete. Stats: total={}, rejected={}",
totalQueuedJobs,
rejectedJobs);
}
/**
* Queues a job for execution when resources permit.
*
* @param jobId The job ID
* @param resourceWeight The resource weight of the job (1-100)
* @param work The work to be done
* @param timeoutMs The timeout in milliseconds
* @return A CompletableFuture that will complete when the job is executed
*/
public CompletableFuture<ResponseEntity<?>> queueJob(
String jobId, int resourceWeight, Supplier<Object> work, long timeoutMs) {
// Create a CompletableFuture to track this job's completion
CompletableFuture<ResponseEntity<?>> future = new CompletableFuture<>();
// Create the queued job
QueuedJob job =
new QueuedJob(jobId, resourceWeight, work, timeoutMs, Instant.now(), future, false);
// Store in our map for lookup
jobMap.put(jobId, job);
// Update stats
totalQueuedJobs++;
currentQueueSize = jobQueue.size();
// Try to add to the queue
try {
boolean added = jobQueue.offer(job, 5, TimeUnit.SECONDS);
if (!added) {
log.warn("Queue full, rejecting job {}", jobId);
rejectedJobs++;
future.completeExceptionally(
new RuntimeException("Job queue full, please try again later"));
jobMap.remove(jobId);
return future;
}
log.debug(
"Job {} queued for execution (weight: {}, queue size: {})",
jobId,
resourceWeight,
jobQueue.size());
// Notify client via WebSocket that job is queued
webSocketSender.sendProgress(
jobId, new JobProgress(jobId, "Queued", 0, "Waiting in queue for resources"));
return future;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
future.completeExceptionally(new RuntimeException("Job queue interrupted"));
jobMap.remove(jobId);
return future;
}
}
/**
* Gets the current capacity of the job queue.
*
* @return The current capacity
*/
public int getQueueCapacity() {
return ((LinkedBlockingQueue<QueuedJob>) jobQueue).remainingCapacity() + jobQueue.size();
}
/** Updates the capacity of the job queue based on available system resources. */
private void updateQueueCapacity() {
try {
// Calculate new capacity
int newCapacity =
resourceMonitor.calculateDynamicQueueCapacity(
baseQueueCapacity, minQueueCapacity);
int currentCapacity = getQueueCapacity();
if (newCapacity != currentCapacity) {
log.debug(
"Updating job queue capacity from {} to {}", currentCapacity, newCapacity);
// Create new queue with updated capacity
BlockingQueue<QueuedJob> newQueue = new LinkedBlockingQueue<>(newCapacity);
// Transfer jobs from old queue to new queue
jobQueue.drainTo(newQueue);
jobQueue = newQueue;
currentQueueSize = jobQueue.size();
}
} catch (Exception e) {
log.error("Error updating queue capacity: {}", e.getMessage(), e);
}
}
/** Processes jobs in the queue, executing them when resources permit. */
private void processQueue() {
if (shuttingDown || jobQueue.isEmpty()) {
return;
}
try {
// Get current resource status
ResourceMonitor.ResourceStatus status = resourceMonitor.getCurrentStatus().get();
// Check if we should execute any jobs
boolean canExecuteJobs = (status != ResourceMonitor.ResourceStatus.CRITICAL);
if (!canExecuteJobs) {
// Under critical load, don't execute any jobs
log.debug("System under critical load, delaying job execution");
return;
}
// Get jobs from the queue, up to a limit based on resource availability
int jobsToProcess =
Math.max(
1,
switch (status) {
case OK -> 3;
case WARNING -> 1;
case CRITICAL -> 0;
});
for (int i = 0; i < jobsToProcess && !jobQueue.isEmpty(); i++) {
QueuedJob job = jobQueue.poll();
if (job == null) break;
// Check if it's been waiting too long
long waitTimeMs = Instant.now().toEpochMilli() - job.queuedAt.toEpochMilli();
if (waitTimeMs > maxWaitTimeMs) {
log.warn(
"Job {} exceeded maximum wait time ({} ms), executing anyway",
job.jobId,
waitTimeMs);
}
// Remove from our map
jobMap.remove(job.jobId);
currentQueueSize = jobQueue.size();
// Notify via WebSocket
webSocketSender.sendProgress(
job.jobId,
new JobProgress(job.jobId, "Processing", 10, "Starting job execution"));
// Execute the job
executeJob(job);
}
} catch (Exception e) {
log.error("Error processing job queue: {}", e.getMessage(), e);
}
}
/**
* Executes a job from the queue.
*
* @param job The job to execute
*/
private void executeJob(QueuedJob job) {
if (job.cancelled) {
log.debug("Job {} was cancelled, not executing", job.jobId);
return;
}
jobExecutor.execute(
() -> {
log.debug("Executing queued job {} (queued at {})", job.jobId, job.queuedAt);
try {
// Execute with timeout
Object result = executeWithTimeout(job.work, job.timeoutMs);
// Process the result
if (result instanceof ResponseEntity) {
job.future.complete((ResponseEntity<?>) result);
} else {
job.future.complete(ResponseEntity.ok(result));
}
// Update WebSocket
webSocketSender.sendProgress(
job.jobId,
new JobProgress(
job.jobId, "Complete", 100, "Job completed successfully"));
} catch (Exception e) {
log.error(
"Error executing queued job {}: {}", job.jobId, e.getMessage(), e);
job.future.completeExceptionally(e);
// Update WebSocket
webSocketSender.sendProgress(
job.jobId,
new JobProgress(
job.jobId, "Error", 100, "Job failed: " + e.getMessage()));
}
});
}
/**
* Execute a supplier with a timeout.
*
* @param supplier The supplier to execute
* @param timeoutMs The timeout in milliseconds
* @return The result from the supplier
* @throws Exception If there is an execution error
*/
private <T> T executeWithTimeout(Supplier<T> supplier, long timeoutMs) throws Exception {
CompletableFuture<T> future = CompletableFuture.supplyAsync(supplier);
try {
if (timeoutMs <= 0) {
// No timeout
return future.join();
} else {
// With timeout
return future.get(timeoutMs, TimeUnit.MILLISECONDS);
}
} catch (TimeoutException e) {
future.cancel(true);
throw new TimeoutException("Job timed out after " + timeoutMs + "ms");
} catch (ExecutionException e) {
throw (Exception) e.getCause();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new InterruptedException("Job was interrupted");
}
}
/**
* Checks if a job is queued.
*
* @param jobId The job ID
* @return true if the job is queued
*/
public boolean isJobQueued(String jobId) {
return jobMap.containsKey(jobId);
}
/**
* Gets the current position of a job in the queue.
*
* @param jobId The job ID
* @return The position (0-based) or -1 if not found
*/
public int getJobPosition(String jobId) {
if (!jobMap.containsKey(jobId)) {
return -1;
}
// Count positions
int position = 0;
for (QueuedJob job : jobQueue) {
if (job.jobId.equals(jobId)) {
return position;
}
position++;
}
// If we didn't find it in the queue but it's in the map,
// it might be executing already
return -1;
}
/**
* Cancels a queued job.
*
* @param jobId The job ID
* @return true if the job was cancelled, false if not found
*/
public boolean cancelJob(String jobId) {
QueuedJob job = jobMap.remove(jobId);
if (job != null) {
job.cancelled = true;
job.future.completeExceptionally(new RuntimeException("Job cancelled by user"));
// Try to remove from queue if it's still there
jobQueue.remove(job);
currentQueueSize = jobQueue.size();
log.debug("Job {} cancelled", jobId);
// Update WebSocket
webSocketSender.sendProgress(
jobId, new JobProgress(jobId, "Cancelled", 100, "Job cancelled by user"));
return true;
}
return false;
}
/**
* Get queue statistics.
*
* @return A map containing queue statistics
*/
public Map<String, Object> getQueueStats() {
return Map.of(
"queuedJobs", jobQueue.size(),
"queueCapacity", getQueueCapacity(),
"totalQueuedJobs", totalQueuedJobs,
"rejectedJobs", rejectedJobs,
"resourceStatus", resourceMonitor.getCurrentStatus().get().name());
}
}
@@ -1,277 +0,0 @@
package stirling.software.common.service;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.OperatingSystemMXBean;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
/**
* Monitors system resources (CPU, memory) to inform job scheduling decisions. Provides information
* about available resources to prevent overloading the system.
*/
@Service
@Slf4j
public class ResourceMonitor {
@Value("${stirling.resource.memory.critical-threshold:0.9}")
private double memoryCriticalThreshold = 0.9; // 90% usage is critical
@Value("${stirling.resource.memory.high-threshold:0.75}")
private double memoryHighThreshold = 0.75; // 75% usage is high
@Value("${stirling.resource.cpu.critical-threshold:0.9}")
private double cpuCriticalThreshold = 0.9; // 90% usage is critical
@Value("${stirling.resource.cpu.high-threshold:0.75}")
private double cpuHighThreshold = 0.75; // 75% usage is high
@Value("${stirling.resource.monitor.interval-ms:60000}")
private long monitorIntervalMs = 60000; // 60 seconds
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
private final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
private final OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean();
@Getter
private final AtomicReference<ResourceStatus> currentStatus =
new AtomicReference<>(ResourceStatus.OK);
@Getter
private final AtomicReference<ResourceMetrics> latestMetrics =
new AtomicReference<>(new ResourceMetrics());
/** Represents the current status of system resources. */
public enum ResourceStatus {
/** Resources are available, normal operations can proceed */
OK,
/** Resources are under strain, consider queueing high-resource operations */
WARNING,
/** Resources are critically low, queue all operations */
CRITICAL
}
/** Detailed metrics about system resources. */
@Getter
public static class ResourceMetrics {
private final double cpuUsage;
private final double memoryUsage;
private final long freeMemoryBytes;
private final long totalMemoryBytes;
private final long maxMemoryBytes;
private final Instant timestamp;
public ResourceMetrics() {
this(0, 0, 0, 0, 0, Instant.now());
}
public ResourceMetrics(
double cpuUsage,
double memoryUsage,
long freeMemoryBytes,
long totalMemoryBytes,
long maxMemoryBytes,
Instant timestamp) {
this.cpuUsage = cpuUsage;
this.memoryUsage = memoryUsage;
this.freeMemoryBytes = freeMemoryBytes;
this.totalMemoryBytes = totalMemoryBytes;
this.maxMemoryBytes = maxMemoryBytes;
this.timestamp = timestamp;
}
/**
* Gets the age of these metrics.
*
* @return Duration since these metrics were collected
*/
public Duration getAge() {
return Duration.between(timestamp, Instant.now());
}
/**
* Check if these metrics are stale (older than threshold).
*
* @param thresholdMs Staleness threshold in milliseconds
* @return true if metrics are stale
*/
public boolean isStale(long thresholdMs) {
return getAge().toMillis() > thresholdMs;
}
}
@PostConstruct
public void initialize() {
log.debug("Starting resource monitoring with interval of {}ms", monitorIntervalMs);
scheduler.scheduleAtFixedRate(
this::updateResourceMetrics, 0, monitorIntervalMs, TimeUnit.MILLISECONDS);
}
@PreDestroy
public void shutdown() {
log.info("Shutting down resource monitoring");
scheduler.shutdownNow();
}
/** Updates the resource metrics by sampling current system state. */
private void updateResourceMetrics() {
try {
// Get CPU usage
double cpuUsage = osMXBean.getSystemLoadAverage() / osMXBean.getAvailableProcessors();
if (cpuUsage < 0) cpuUsage = getAlternativeCpuLoad(); // Fallback if not available
// Get memory usage
long heapUsed = memoryMXBean.getHeapMemoryUsage().getUsed();
long nonHeapUsed = memoryMXBean.getNonHeapMemoryUsage().getUsed();
long totalUsed = heapUsed + nonHeapUsed;
long maxMemory = Runtime.getRuntime().maxMemory();
long totalMemory = Runtime.getRuntime().totalMemory();
long freeMemory = Runtime.getRuntime().freeMemory();
double memoryUsage = (double) totalUsed / maxMemory;
// Create new metrics
ResourceMetrics metrics =
new ResourceMetrics(
cpuUsage,
memoryUsage,
freeMemory,
totalMemory,
maxMemory,
Instant.now());
latestMetrics.set(metrics);
// Determine system status
ResourceStatus newStatus;
if (cpuUsage > cpuCriticalThreshold || memoryUsage > memoryCriticalThreshold) {
newStatus = ResourceStatus.CRITICAL;
} else if (cpuUsage > cpuHighThreshold || memoryUsage > memoryHighThreshold) {
newStatus = ResourceStatus.WARNING;
} else {
newStatus = ResourceStatus.OK;
}
// Update status if it changed
ResourceStatus oldStatus = currentStatus.getAndSet(newStatus);
if (oldStatus != newStatus) {
log.info("System resource status changed from {} to {}", oldStatus, newStatus);
log.info(
"Current metrics - CPU: {:.1f}%, Memory: {:.1f}%, Free Memory: {} MB",
cpuUsage * 100, memoryUsage * 100, freeMemory / (1024 * 1024));
}
} catch (Exception e) {
log.error("Error updating resource metrics: {}", e.getMessage(), e);
}
}
/**
* Alternative method to estimate CPU load if getSystemLoadAverage() is not available. This is a
* fallback and less accurate than the official JMX method.
*
* @return Estimated CPU load as a value between 0.0 and 1.0
*/
private double getAlternativeCpuLoad() {
try {
// Try to get CPU time if available through reflection
// This is a fallback since we can't directly cast to platform-specific classes
try {
java.lang.reflect.Method m =
osMXBean.getClass().getDeclaredMethod("getProcessCpuLoad");
m.setAccessible(true);
return (double) m.invoke(osMXBean);
} catch (Exception e) {
// Try the older method
try {
java.lang.reflect.Method m =
osMXBean.getClass().getDeclaredMethod("getSystemCpuLoad");
m.setAccessible(true);
return (double) m.invoke(osMXBean);
} catch (Exception e2) {
log.trace(
"Could not get CPU load through reflection, assuming moderate load (0.5)");
return 0.5;
}
}
} catch (Exception e) {
log.trace("Could not get CPU load, assuming moderate load (0.5)");
return 0.5; // Default to moderate load
}
}
/**
* Calculates the dynamic job queue capacity based on current resource usage.
*
* @param baseCapacity The base capacity when system is under minimal load
* @param minCapacity The minimum capacity to maintain even under high load
* @return The calculated job queue capacity
*/
public int calculateDynamicQueueCapacity(int baseCapacity, int minCapacity) {
ResourceMetrics metrics = latestMetrics.get();
ResourceStatus status = currentStatus.get();
// Simple linear reduction based on memory and CPU load
double capacityFactor =
switch (status) {
case OK -> 1.0;
case WARNING -> 0.6;
case CRITICAL -> 0.3;
};
// Apply additional reduction based on specific memory pressure
if (metrics.memoryUsage > 0.8) {
capacityFactor *= 0.5; // Further reduce capacity under memory pressure
}
// Calculate capacity with minimum safeguard
int capacity = (int) Math.max(minCapacity, Math.ceil(baseCapacity * capacityFactor));
log.debug(
"Dynamic queue capacity: {} (base: {}, factor: {:.2f}, status: {})",
capacity,
baseCapacity,
capacityFactor,
status);
return capacity;
}
/**
* Checks if a job with the given weight can be executed immediately or should be queued based
* on current resource availability.
*
* @param resourceWeight The resource weight of the job (1-100)
* @return true if the job should be queued, false if it can run immediately
*/
public boolean shouldQueueJob(int resourceWeight) {
ResourceStatus status = currentStatus.get();
// Always run lightweight jobs (weight < 20) unless critical
if (resourceWeight < 20 && status != ResourceStatus.CRITICAL) {
return false;
}
// Medium weight jobs run immediately if resources are OK
if (resourceWeight < 60 && status == ResourceStatus.OK) {
return false;
}
// Heavy jobs (weight >= 60) and any job during WARNING/CRITICAL should be queued
return true;
}
}
@@ -1,274 +0,0 @@
package stirling.software.common.service;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import jakarta.annotation.PreDestroy;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.job.JobResult;
import stirling.software.common.model.job.JobStats;
/** Manages async tasks and their results */
@Service
@Slf4j
public class TaskManager {
private final Map<String, JobResult> jobResults = new ConcurrentHashMap<>();
@Value("${stirling.jobResultExpiryMinutes:30}")
private int jobResultExpiryMinutes = 30;
private final FileStorage fileStorage;
private final ScheduledExecutorService cleanupExecutor =
Executors.newSingleThreadScheduledExecutor();
/** Initialize the task manager and start the cleanup scheduler */
public TaskManager(FileStorage fileStorage) {
this.fileStorage = fileStorage;
// Schedule periodic cleanup of old job results
cleanupExecutor.scheduleAtFixedRate(
this::cleanupOldJobs,
10, // Initial delay
10, // Interval
TimeUnit.MINUTES);
log.debug(
"Task manager initialized with job result expiry of {} minutes",
jobResultExpiryMinutes);
}
/**
* Create a new task with the given job ID
*
* @param jobId The job ID
*/
public void createTask(String jobId) {
jobResults.put(jobId, JobResult.createNew(jobId));
log.debug("Created task with job ID: {}", jobId);
}
/**
* Set the result of a task as a general object
*
* @param jobId The job ID
* @param result The result object
*/
public void setResult(String jobId, Object result) {
JobResult jobResult = getOrCreateJobResult(jobId);
jobResult.completeWithResult(result);
log.debug("Set result for job ID: {}", jobId);
}
/**
* Set the result of a task as a file
*
* @param jobId The job ID
* @param fileId The file ID
* @param originalFileName The original file name
* @param contentType The content type of the file
*/
public void setFileResult(
String jobId, String fileId, String originalFileName, String contentType) {
JobResult jobResult = getOrCreateJobResult(jobId);
jobResult.completeWithFile(fileId, originalFileName, contentType);
log.debug("Set file result for job ID: {} with file ID: {}", jobId, fileId);
}
/**
* Set an error for a task
*
* @param jobId The job ID
* @param error The error message
*/
public void setError(String jobId, String error) {
JobResult jobResult = getOrCreateJobResult(jobId);
jobResult.failWithError(error);
log.debug("Set error for job ID: {}: {}", jobId, error);
}
/**
* Mark a task as complete
*
* @param jobId The job ID
*/
public void setComplete(String jobId) {
JobResult jobResult = getOrCreateJobResult(jobId);
if (jobResult.getResult() == null
&& jobResult.getFileId() == null
&& jobResult.getError() == null) {
// If no result or error has been set, mark it as complete with an empty result
jobResult.completeWithResult("Task completed successfully");
}
log.debug("Marked job ID: {} as complete", jobId);
}
/**
* Check if a task is complete
*
* @param jobId The job ID
* @return true if the task is complete, false otherwise
*/
public boolean isComplete(String jobId) {
JobResult result = jobResults.get(jobId);
return result != null && result.isComplete();
}
/**
* Get the result of a task
*
* @param jobId The job ID
* @return The result object, or null if the task doesn't exist or is not complete
*/
public JobResult getJobResult(String jobId) {
return jobResults.get(jobId);
}
/**
* Get statistics about all jobs in the system
*
* @return Job statistics
*/
public JobStats getJobStats() {
int totalJobs = jobResults.size();
int activeJobs = 0;
int completedJobs = 0;
int failedJobs = 0;
int successfulJobs = 0;
int fileResultJobs = 0;
LocalDateTime oldestActiveJobTime = null;
LocalDateTime newestActiveJobTime = null;
long totalProcessingTimeMs = 0;
for (JobResult result : jobResults.values()) {
if (result.isComplete()) {
completedJobs++;
// Calculate processing time for completed jobs
if (result.getCreatedAt() != null && result.getCompletedAt() != null) {
long processingTimeMs =
java.time.Duration.between(
result.getCreatedAt(), result.getCompletedAt())
.toMillis();
totalProcessingTimeMs += processingTimeMs;
}
if (result.getError() != null) {
failedJobs++;
} else {
successfulJobs++;
if (result.getFileId() != null) {
fileResultJobs++;
}
}
} else {
activeJobs++;
// Track oldest and newest active jobs
if (result.getCreatedAt() != null) {
if (oldestActiveJobTime == null
|| result.getCreatedAt().isBefore(oldestActiveJobTime)) {
oldestActiveJobTime = result.getCreatedAt();
}
if (newestActiveJobTime == null
|| result.getCreatedAt().isAfter(newestActiveJobTime)) {
newestActiveJobTime = result.getCreatedAt();
}
}
}
}
// Calculate average processing time
long averageProcessingTimeMs =
completedJobs > 0 ? totalProcessingTimeMs / completedJobs : 0;
return JobStats.builder()
.totalJobs(totalJobs)
.activeJobs(activeJobs)
.completedJobs(completedJobs)
.failedJobs(failedJobs)
.successfulJobs(successfulJobs)
.fileResultJobs(fileResultJobs)
.oldestActiveJobTime(oldestActiveJobTime)
.newestActiveJobTime(newestActiveJobTime)
.averageProcessingTimeMs(averageProcessingTimeMs)
.build();
}
/**
* Get or create a job result
*
* @param jobId The job ID
* @return The job result
*/
private JobResult getOrCreateJobResult(String jobId) {
return jobResults.computeIfAbsent(jobId, JobResult::createNew);
}
/** Clean up old completed job results */
public void cleanupOldJobs() {
LocalDateTime expiryThreshold =
LocalDateTime.now().minus(jobResultExpiryMinutes, ChronoUnit.MINUTES);
int removedCount = 0;
try {
for (Map.Entry<String, JobResult> entry : jobResults.entrySet()) {
JobResult result = entry.getValue();
// Remove completed jobs that are older than the expiry threshold
if (result.isComplete()
&& result.getCompletedAt() != null
&& result.getCompletedAt().isBefore(expiryThreshold)) {
// If the job has a file result, delete the file
if (result.getFileId() != null) {
try {
fileStorage.deleteFile(result.getFileId());
} catch (Exception e) {
log.warn(
"Failed to delete file for job {}: {}",
entry.getKey(),
e.getMessage());
}
}
// Remove the job result
jobResults.remove(entry.getKey());
removedCount++;
}
}
if (removedCount > 0) {
log.info("Cleaned up {} expired job results", removedCount);
}
} catch (Exception e) {
log.error("Error during job cleanup: {}", e.getMessage(), e);
}
}
/** Shutdown the cleanup executor */
@PreDestroy
public void shutdown() {
try {
log.info("Shutting down job result cleanup executor");
cleanupExecutor.shutdown();
if (!cleanupExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
cleanupExecutor.shutdownNow();
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
cleanupExecutor.shutdownNow();
}
}
}
@@ -1,31 +0,0 @@
package stirling.software.common.util;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class ExecutorFactory {
/**
* Creates an ExecutorService using virtual threads if available (Java 21+), or falls back to a
* cached thread pool on older Java versions.
*/
public static ExecutorService newVirtualOrCachedThreadExecutor() {
try {
ExecutorService executor =
(ExecutorService)
Executors.class
.getMethod("newVirtualThreadPerTaskExecutor")
.invoke(null);
return executor;
} catch (NoSuchMethodException e) {
log.debug("Virtual threads not available; falling back to cached thread pool.");
} catch (Exception e) {
log.debug("Error initializing virtual thread executor: {}", e.getMessage(), e);
}
return Executors.newCachedThreadPool();
}
}
@@ -1,309 +0,0 @@
package stirling.software.common.util;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.InterruptedIOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import io.github.pixee.security.BoundedLineReader;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
@Slf4j
public class ProcessExecutor {
private static final Map<Processes, ProcessExecutor> instances = new ConcurrentHashMap<>();
private static ApplicationProperties applicationProperties = new ApplicationProperties();
private final Semaphore semaphore;
private final boolean liveUpdates;
private long timeoutDuration;
private ProcessExecutor(int semaphoreLimit, boolean liveUpdates, long timeout) {
this.semaphore = new Semaphore(semaphoreLimit);
this.liveUpdates = liveUpdates;
this.timeoutDuration = timeout;
}
public static ProcessExecutor getInstance(Processes processType) {
return getInstance(processType, true);
}
public static ProcessExecutor getInstance(Processes processType, boolean liveUpdates) {
return instances.computeIfAbsent(
processType,
key -> {
int semaphoreLimit =
switch (key) {
case LIBRE_OFFICE ->
applicationProperties
.getProcessExecutor()
.getSessionLimit()
.getLibreOfficeSessionLimit();
case PDFTOHTML ->
applicationProperties
.getProcessExecutor()
.getSessionLimit()
.getPdfToHtmlSessionLimit();
case PYTHON_OPENCV ->
applicationProperties
.getProcessExecutor()
.getSessionLimit()
.getPythonOpenCvSessionLimit();
case WEASYPRINT ->
applicationProperties
.getProcessExecutor()
.getSessionLimit()
.getWeasyPrintSessionLimit();
case INSTALL_APP ->
applicationProperties
.getProcessExecutor()
.getSessionLimit()
.getInstallAppSessionLimit();
case TESSERACT ->
applicationProperties
.getProcessExecutor()
.getSessionLimit()
.getTesseractSessionLimit();
case QPDF ->
applicationProperties
.getProcessExecutor()
.getSessionLimit()
.getQpdfSessionLimit();
case CALIBRE ->
applicationProperties
.getProcessExecutor()
.getSessionLimit()
.getCalibreSessionLimit();
};
long timeoutMinutes =
switch (key) {
case LIBRE_OFFICE ->
applicationProperties
.getProcessExecutor()
.getTimeoutMinutes()
.getLibreOfficeTimeoutMinutes();
case PDFTOHTML ->
applicationProperties
.getProcessExecutor()
.getTimeoutMinutes()
.getPdfToHtmlTimeoutMinutes();
case PYTHON_OPENCV ->
applicationProperties
.getProcessExecutor()
.getTimeoutMinutes()
.getPythonOpenCvTimeoutMinutes();
case WEASYPRINT ->
applicationProperties
.getProcessExecutor()
.getTimeoutMinutes()
.getWeasyPrintTimeoutMinutes();
case INSTALL_APP ->
applicationProperties
.getProcessExecutor()
.getTimeoutMinutes()
.getInstallAppTimeoutMinutes();
case TESSERACT ->
applicationProperties
.getProcessExecutor()
.getTimeoutMinutes()
.getTesseractTimeoutMinutes();
case QPDF ->
applicationProperties
.getProcessExecutor()
.getTimeoutMinutes()
.getQpdfTimeoutMinutes();
case CALIBRE ->
applicationProperties
.getProcessExecutor()
.getTimeoutMinutes()
.getCalibreTimeoutMinutes();
};
return new ProcessExecutor(semaphoreLimit, liveUpdates, timeoutMinutes);
});
}
public ProcessExecutorResult runCommandWithOutputHandling(List<String> command)
throws IOException, InterruptedException {
return runCommandWithOutputHandling(command, null);
}
public ProcessExecutorResult runCommandWithOutputHandling(
List<String> command, File workingDirectory) throws IOException, InterruptedException {
String messages = "";
int exitCode = 1;
semaphore.acquire();
try {
log.info("Running command: " + String.join(" ", command));
ProcessBuilder processBuilder = new ProcessBuilder(command);
// Use the working directory if it's set
if (workingDirectory != null) {
processBuilder.directory(workingDirectory);
}
Process process = processBuilder.start();
// Read the error stream and standard output stream concurrently
List<String> errorLines = new ArrayList<>();
List<String> outputLines = new ArrayList<>();
Thread errorReaderThread =
new Thread(
() -> {
try (BufferedReader errorReader =
new BufferedReader(
new InputStreamReader(
process.getErrorStream(),
StandardCharsets.UTF_8))) {
String line;
while ((line =
BoundedLineReader.readLine(
errorReader, 5_000_000))
!= null) {
errorLines.add(line);
if (liveUpdates) log.info(line);
}
} catch (InterruptedIOException e) {
log.warn("Error reader thread was interrupted due to timeout.");
} catch (IOException e) {
log.error("exception", e);
}
});
Thread outputReaderThread =
new Thread(
() -> {
try (BufferedReader outputReader =
new BufferedReader(
new InputStreamReader(
process.getInputStream(),
StandardCharsets.UTF_8))) {
String line;
while ((line =
BoundedLineReader.readLine(
outputReader, 5_000_000))
!= null) {
outputLines.add(line);
if (liveUpdates) log.info(line);
}
} catch (InterruptedIOException e) {
log.warn("Error reader thread was interrupted due to timeout.");
} catch (IOException e) {
log.error("exception", e);
}
});
errorReaderThread.start();
outputReaderThread.start();
// Wait for the conversion process to complete
boolean finished = process.waitFor(timeoutDuration, TimeUnit.MINUTES);
if (!finished) {
// Terminate the process
process.destroy();
// Interrupt the reader threads
errorReaderThread.interrupt();
outputReaderThread.interrupt();
throw new IOException("Process timeout exceeded.");
}
exitCode = process.exitValue();
// Wait for the reader threads to finish
errorReaderThread.join();
outputReaderThread.join();
boolean isQpdf =
command != null && !command.isEmpty() && command.get(0).contains("qpdf");
if (!outputLines.isEmpty()) {
String outputMessage = String.join("\n", outputLines);
messages += outputMessage;
if (!liveUpdates) {
log.info("Command output:\n" + outputMessage);
}
}
if (!errorLines.isEmpty()) {
String errorMessage = String.join("\n", errorLines);
messages += errorMessage;
if (!liveUpdates) {
log.warn("Command error output:\n" + errorMessage);
}
if (exitCode != 0) {
if (isQpdf && exitCode == 3) {
log.warn("qpdf succeeded with warnings: {}", messages);
} else {
throw new IOException(
"Command process failed with exit code "
+ exitCode
+ ". Error message: "
+ errorMessage);
}
}
}
if (exitCode != 0) {
if (isQpdf && exitCode == 3) {
log.warn("qpdf succeeded with warnings: {}", messages);
} else {
throw new IOException(
"Command process failed with exit code "
+ exitCode
+ "\nLogs: "
+ messages);
}
}
} finally {
semaphore.release();
}
return new ProcessExecutorResult(exitCode, messages);
}
public enum Processes {
LIBRE_OFFICE,
PDFTOHTML,
PYTHON_OPENCV,
WEASYPRINT,
INSTALL_APP,
CALIBRE,
TESSERACT,
QPDF
}
public class ProcessExecutorResult {
int rc;
String messages;
public ProcessExecutorResult(int rc, String messages) {
this.rc = rc;
this.messages = messages;
}
public int getRc() {
return rc;
}
public void setRc(int rc) {
this.rc = rc;
}
public String getMessages() {
return messages;
}
public void setMessages(String messages) {
this.messages = messages;
}
}
}
@@ -1,14 +0,0 @@
package stirling.software.common.util;
import java.util.Collection;
public class ValidationUtils {
public static boolean isStringEmpty(String input) {
return input == null || input.isBlank();
}
public static boolean isCollectionEmpty(Collection<String> input) {
return input == null || input.isEmpty();
}
}
@@ -1,214 +0,0 @@
package stirling.software.common.annotations;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import org.junit.jupiter.api.BeforeEach;
import java.util.Arrays;
import java.util.function.Supplier;
import org.aspectj.lang.ProceedingJoinPoint;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.ResponseEntity;
import org.springframework.web.multipart.MultipartFile;
import jakarta.servlet.http.HttpServletRequest;
import stirling.software.common.aop.AutoJobAspect;
import stirling.software.common.controller.WebSocketProgressController;
import stirling.software.common.model.api.PDFFile;
import stirling.software.common.service.FileOrUploadService;
import stirling.software.common.service.FileStorage;
import stirling.software.common.service.JobExecutorService;
import stirling.software.common.service.JobQueue;
import stirling.software.common.service.ResourceMonitor;
@ExtendWith(MockitoExtension.class)
class AutoJobPostMappingIntegrationTest {
private AutoJobAspect autoJobAspect;
@Mock
private JobExecutorService jobExecutorService;
@Mock
private HttpServletRequest request;
@Mock
private FileOrUploadService fileOrUploadService;
@Mock
private FileStorage fileStorage;
@Mock
private WebSocketProgressController webSocketSender;
@Mock
private ResourceMonitor resourceMonitor;
@Mock
private JobQueue jobQueue;
@BeforeEach
void setUp() {
autoJobAspect = new AutoJobAspect(
jobExecutorService,
request,
fileOrUploadService,
fileStorage,
webSocketSender
);
}
@Mock
private ProceedingJoinPoint joinPoint;
@Mock
private AutoJobPostMapping autoJobPostMapping;
@Captor
private ArgumentCaptor<Supplier<Object>> workCaptor;
@Captor
private ArgumentCaptor<Boolean> asyncCaptor;
@Captor
private ArgumentCaptor<Long> timeoutCaptor;
@Captor
private ArgumentCaptor<Boolean> queueableCaptor;
@Captor
private ArgumentCaptor<Integer> resourceWeightCaptor;
@Test
void shouldExecuteWithCustomParameters() throws Throwable {
// Given
PDFFile pdfFile = new PDFFile();
pdfFile.setFileId("test-file-id");
Object[] args = new Object[] { pdfFile };
when(joinPoint.getArgs()).thenReturn(args);
when(request.getParameter("async")).thenReturn("true");
when(autoJobPostMapping.timeout()).thenReturn(60000L);
when(autoJobPostMapping.retryCount()).thenReturn(3);
when(autoJobPostMapping.trackProgress()).thenReturn(true);
when(autoJobPostMapping.queueable()).thenReturn(true);
when(autoJobPostMapping.resourceWeight()).thenReturn(75);
MultipartFile mockFile = mock(MultipartFile.class);
when(fileStorage.retrieveFile("test-file-id")).thenReturn(mockFile);
when(jobExecutorService.runJobGeneric(
anyBoolean(), any(Supplier.class), anyLong(), anyBoolean(), anyInt()))
.thenReturn(ResponseEntity.ok("success"));
// When
Object result = autoJobAspect.wrapWithJobExecution(joinPoint, autoJobPostMapping);
// Then
assertEquals(ResponseEntity.ok("success"), result);
verify(jobExecutorService).runJobGeneric(
asyncCaptor.capture(),
workCaptor.capture(),
timeoutCaptor.capture(),
queueableCaptor.capture(),
resourceWeightCaptor.capture());
assertTrue(asyncCaptor.getValue(), "Async should be true");
assertEquals(60000L, timeoutCaptor.getValue(), "Timeout should be 60000ms");
assertTrue(queueableCaptor.getValue(), "Queueable should be true");
assertEquals(75, resourceWeightCaptor.getValue(), "Resource weight should be 75");
// Test that file was resolved
assertNotNull(pdfFile.getFileInput(), "File input should be set");
}
@Test
void shouldRetryOnError() throws Throwable {
// Given
when(joinPoint.getArgs()).thenReturn(new Object[0]);
when(request.getParameter("async")).thenReturn("false");
when(autoJobPostMapping.timeout()).thenReturn(-1L);
when(autoJobPostMapping.retryCount()).thenReturn(2);
when(autoJobPostMapping.trackProgress()).thenReturn(false);
when(autoJobPostMapping.queueable()).thenReturn(false);
when(autoJobPostMapping.resourceWeight()).thenReturn(50);
// First call throws exception, second succeeds
when(joinPoint.proceed(any()))
.thenThrow(new RuntimeException("First attempt failed"))
.thenReturn(ResponseEntity.ok("retry succeeded"));
// Mock jobExecutorService to execute the work immediately
when(jobExecutorService.runJobGeneric(
anyBoolean(), any(Supplier.class), anyLong(), anyBoolean(), anyInt()))
.thenAnswer(invocation -> {
Supplier<Object> work = invocation.getArgument(1);
return work.get();
});
// When
Object result = autoJobAspect.wrapWithJobExecution(joinPoint, autoJobPostMapping);
// Then
assertEquals(ResponseEntity.ok("retry succeeded"), result);
// Verify that proceed was called twice (initial attempt + 1 retry)
verify(joinPoint, times(2)).proceed(any());
}
@Test
void shouldHandlePDFFileWithAsyncRequests() throws Throwable {
// Given
PDFFile pdfFile = new PDFFile();
pdfFile.setFileInput(mock(MultipartFile.class));
Object[] args = new Object[] { pdfFile };
when(joinPoint.getArgs()).thenReturn(args);
when(request.getParameter("async")).thenReturn("true");
when(autoJobPostMapping.retryCount()).thenReturn(1);
when(fileStorage.storeFile(any(MultipartFile.class))).thenReturn("stored-file-id");
when(fileStorage.retrieveFile("stored-file-id")).thenReturn(mock(MultipartFile.class));
// Mock job executor to return a successful response
when(jobExecutorService.runJobGeneric(
anyBoolean(), any(Supplier.class), anyLong(), anyBoolean(), anyInt()))
.thenReturn(ResponseEntity.ok("success"));
// When
autoJobAspect.wrapWithJobExecution(joinPoint, autoJobPostMapping);
// Then
assertEquals("stored-file-id", pdfFile.getFileId(),
"FileId should be set to the stored file id");
assertNotNull(pdfFile.getFileInput(), "FileInput should be replaced with persistent file");
// Verify storage operations
verify(fileStorage).storeFile(any(MultipartFile.class));
verify(fileStorage).retrieveFile("stored-file-id");
}
}
// Move PDFFileTest to its own class file if needed
@@ -1,227 +0,0 @@
package stirling.software.common.controller;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import java.util.Map;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import stirling.software.common.model.job.JobResult;
import stirling.software.common.model.job.JobStats;
import stirling.software.common.service.FileStorage;
import stirling.software.common.service.TaskManager;
class JobControllerTest {
@Mock
private TaskManager taskManager;
@Mock
private FileStorage fileStorage;
@InjectMocks
private JobController controller;
@BeforeEach
void setUp() {
MockitoAnnotations.openMocks(this);
}
@Test
void testGetJobStatus_ExistingJob() {
// Arrange
String jobId = "test-job-id";
JobResult mockResult = new JobResult();
mockResult.setJobId(jobId);
when(taskManager.getJobResult(jobId)).thenReturn(mockResult);
// Act
ResponseEntity<?> response = controller.getJobStatus(jobId);
// Assert
assertEquals(HttpStatus.OK, response.getStatusCode());
assertEquals(mockResult, response.getBody());
}
@Test
void testGetJobStatus_NonExistentJob() {
// Arrange
String jobId = "non-existent-job";
when(taskManager.getJobResult(jobId)).thenReturn(null);
// Act
ResponseEntity<?> response = controller.getJobStatus(jobId);
// Assert
assertEquals(HttpStatus.NOT_FOUND, response.getStatusCode());
}
@Test
void testGetJobResult_CompletedSuccessfulWithObject() {
// Arrange
String jobId = "test-job-id";
JobResult mockResult = new JobResult();
mockResult.setJobId(jobId);
mockResult.setComplete(true);
String resultObject = "Test result";
mockResult.completeWithResult(resultObject);
when(taskManager.getJobResult(jobId)).thenReturn(mockResult);
// Act
ResponseEntity<?> response = controller.getJobResult(jobId);
// Assert
assertEquals(HttpStatus.OK, response.getStatusCode());
assertEquals(resultObject, response.getBody());
}
@Test
void testGetJobResult_CompletedSuccessfulWithFile() throws Exception {
// Arrange
String jobId = "test-job-id";
String fileId = "file-id";
String originalFileName = "test.pdf";
String contentType = "application/pdf";
byte[] fileContent = "Test file content".getBytes();
JobResult mockResult = new JobResult();
mockResult.setJobId(jobId);
mockResult.completeWithFile(fileId, originalFileName, contentType);
when(taskManager.getJobResult(jobId)).thenReturn(mockResult);
when(fileStorage.retrieveBytes(fileId)).thenReturn(fileContent);
// Act
ResponseEntity<?> response = controller.getJobResult(jobId);
// Assert
assertEquals(HttpStatus.OK, response.getStatusCode());
assertEquals(contentType, response.getHeaders().getFirst("Content-Type"));
assertTrue(response.getHeaders().getFirst("Content-Disposition").contains(originalFileName));
assertEquals(fileContent, response.getBody());
}
@Test
void testGetJobResult_CompletedWithError() {
// Arrange
String jobId = "test-job-id";
String errorMessage = "Test error";
JobResult mockResult = new JobResult();
mockResult.setJobId(jobId);
mockResult.failWithError(errorMessage);
when(taskManager.getJobResult(jobId)).thenReturn(mockResult);
// Act
ResponseEntity<?> response = controller.getJobResult(jobId);
// Assert
assertEquals(HttpStatus.BAD_REQUEST, response.getStatusCode());
assertTrue(response.getBody().toString().contains(errorMessage));
}
@Test
void testGetJobResult_IncompleteJob() {
// Arrange
String jobId = "test-job-id";
JobResult mockResult = new JobResult();
mockResult.setJobId(jobId);
mockResult.setComplete(false);
when(taskManager.getJobResult(jobId)).thenReturn(mockResult);
// Act
ResponseEntity<?> response = controller.getJobResult(jobId);
// Assert
assertEquals(HttpStatus.BAD_REQUEST, response.getStatusCode());
assertTrue(response.getBody().toString().contains("not complete"));
}
@Test
void testGetJobResult_NonExistentJob() {
// Arrange
String jobId = "non-existent-job";
when(taskManager.getJobResult(jobId)).thenReturn(null);
// Act
ResponseEntity<?> response = controller.getJobResult(jobId);
// Assert
assertEquals(HttpStatus.NOT_FOUND, response.getStatusCode());
}
@Test
void testGetJobResult_ErrorRetrievingFile() throws Exception {
// Arrange
String jobId = "test-job-id";
String fileId = "file-id";
String originalFileName = "test.pdf";
String contentType = "application/pdf";
JobResult mockResult = new JobResult();
mockResult.setJobId(jobId);
mockResult.completeWithFile(fileId, originalFileName, contentType);
when(taskManager.getJobResult(jobId)).thenReturn(mockResult);
when(fileStorage.retrieveBytes(fileId)).thenThrow(new RuntimeException("File not found"));
// Act
ResponseEntity<?> response = controller.getJobResult(jobId);
// Assert
assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, response.getStatusCode());
assertTrue(response.getBody().toString().contains("Error retrieving file"));
}
@Test
void testGetJobStats() {
// Arrange
JobStats mockStats = JobStats.builder()
.totalJobs(10)
.activeJobs(3)
.completedJobs(7)
.build();
when(taskManager.getJobStats()).thenReturn(mockStats);
// Act
ResponseEntity<JobStats> response = controller.getJobStats();
// Assert
assertEquals(HttpStatus.OK, response.getStatusCode());
assertEquals(mockStats, response.getBody());
}
@Test
void testCleanupOldJobs() {
// Arrange
when(taskManager.getJobStats())
.thenReturn(JobStats.builder().totalJobs(10).build())
.thenReturn(JobStats.builder().totalJobs(7).build());
// Act
ResponseEntity<?> response = controller.cleanupOldJobs();
// Assert
assertEquals(HttpStatus.OK, response.getStatusCode());
@SuppressWarnings("unchecked")
Map<String, Object> responseBody = (Map<String, Object>) response.getBody();
assertEquals("Cleanup complete", responseBody.get("message"));
assertEquals(3, responseBody.get("removedJobs"));
assertEquals(7, responseBody.get("remainingJobs"));
verify(taskManager).cleanupOldJobs();
}
}
@@ -1,69 +0,0 @@
package stirling.software.common.controller;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import stirling.software.common.model.job.JobProgress;
class WebSocketProgressControllerTest {
@Mock
private SimpMessagingTemplate messagingTemplate;
@InjectMocks
private WebSocketProgressController controller;
@BeforeEach
void setUp() {
MockitoAnnotations.openMocks(this);
}
@Test
void testSendProgress_WithMessagingTemplate() {
// Arrange
String jobId = "test-job-id";
JobProgress progress = new JobProgress(jobId, "In Progress", 50, "Processing");
// Act
controller.sendProgress(jobId, progress);
// Assert
verify(messagingTemplate).convertAndSend("/topic/progress/" + jobId, progress);
}
@Test
void testSendProgress_WithNullMessagingTemplate() {
// Arrange
WebSocketProgressController controllerWithNullTemplate = new WebSocketProgressController();
String jobId = "test-job-id";
JobProgress progress = new JobProgress(jobId, "In Progress", 50, "Processing");
// Act - should not throw exception even with null template
controllerWithNullTemplate.sendProgress(jobId, progress);
// No assertion needed - test passes if no exception is thrown
}
@Test
void testSetMessagingTemplate() {
// Arrange
WebSocketProgressController newController = new WebSocketProgressController();
SimpMessagingTemplate newTemplate = mock(SimpMessagingTemplate.class);
// Act
newController.setMessagingTemplate(newTemplate);
String jobId = "test-job-id";
JobProgress progress = new JobProgress(jobId, "In Progress", 50, "Processing");
newController.sendProgress(jobId, progress);
// Assert
verify(newTemplate).convertAndSend("/topic/progress/" + jobId, progress);
}
}
@@ -1,190 +0,0 @@
package stirling.software.common.service;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import static org.mockito.AdditionalAnswers.*;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.UUID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.springframework.test.util.ReflectionTestUtils;
import org.springframework.web.multipart.MultipartFile;
class FileStorageTest {
@TempDir
Path tempDir;
@Mock
private FileOrUploadService fileOrUploadService;
@InjectMocks
private FileStorage fileStorage;
private MultipartFile mockFile;
@BeforeEach
void setUp() {
MockitoAnnotations.openMocks(this);
ReflectionTestUtils.setField(fileStorage, "tempDirPath", tempDir.toString());
// Create a mock MultipartFile
mockFile = mock(MultipartFile.class);
when(mockFile.getOriginalFilename()).thenReturn("test.pdf");
when(mockFile.getContentType()).thenReturn("application/pdf");
}
@Test
void testStoreFile() throws IOException {
// Arrange
byte[] fileContent = "Test PDF content".getBytes();
when(mockFile.getBytes()).thenReturn(fileContent);
// Set up mock to handle transferTo by writing the file
doAnswer(invocation -> {
java.io.File file = invocation.getArgument(0);
Files.write(file.toPath(), fileContent);
return null;
}).when(mockFile).transferTo(any(java.io.File.class));
// Act
String fileId = fileStorage.storeFile(mockFile);
// Assert
assertNotNull(fileId);
assertTrue(Files.exists(tempDir.resolve(fileId)));
verify(mockFile).transferTo(any(java.io.File.class));
}
@Test
void testStoreBytes() throws IOException {
// Arrange
byte[] fileContent = "Test PDF content".getBytes();
String originalName = "test.pdf";
// Act
String fileId = fileStorage.storeBytes(fileContent, originalName);
// Assert
assertNotNull(fileId);
assertTrue(Files.exists(tempDir.resolve(fileId)));
assertArrayEquals(fileContent, Files.readAllBytes(tempDir.resolve(fileId)));
}
@Test
void testRetrieveFile() throws IOException {
// Arrange
byte[] fileContent = "Test PDF content".getBytes();
String fileId = UUID.randomUUID().toString();
Path filePath = tempDir.resolve(fileId);
Files.write(filePath, fileContent);
MultipartFile expectedFile = mock(MultipartFile.class);
when(fileOrUploadService.toMockMultipartFile(eq(fileId), eq(fileContent)))
.thenReturn(expectedFile);
// Act
MultipartFile result = fileStorage.retrieveFile(fileId);
// Assert
assertSame(expectedFile, result);
verify(fileOrUploadService).toMockMultipartFile(eq(fileId), eq(fileContent));
}
@Test
void testRetrieveBytes() throws IOException {
// Arrange
byte[] fileContent = "Test PDF content".getBytes();
String fileId = UUID.randomUUID().toString();
Path filePath = tempDir.resolve(fileId);
Files.write(filePath, fileContent);
// Act
byte[] result = fileStorage.retrieveBytes(fileId);
// Assert
assertArrayEquals(fileContent, result);
}
@Test
void testRetrieveFile_FileNotFound() {
// Arrange
String nonExistentFileId = "non-existent-file";
// Act & Assert
assertThrows(IOException.class, () -> fileStorage.retrieveFile(nonExistentFileId));
}
@Test
void testRetrieveBytes_FileNotFound() {
// Arrange
String nonExistentFileId = "non-existent-file";
// Act & Assert
assertThrows(IOException.class, () -> fileStorage.retrieveBytes(nonExistentFileId));
}
@Test
void testDeleteFile() throws IOException {
// Arrange
byte[] fileContent = "Test PDF content".getBytes();
String fileId = UUID.randomUUID().toString();
Path filePath = tempDir.resolve(fileId);
Files.write(filePath, fileContent);
// Act
boolean result = fileStorage.deleteFile(fileId);
// Assert
assertTrue(result);
assertFalse(Files.exists(filePath));
}
@Test
void testDeleteFile_FileNotFound() {
// Arrange
String nonExistentFileId = "non-existent-file";
// Act
boolean result = fileStorage.deleteFile(nonExistentFileId);
// Assert
assertFalse(result);
}
@Test
void testFileExists() throws IOException {
// Arrange
byte[] fileContent = "Test PDF content".getBytes();
String fileId = UUID.randomUUID().toString();
Path filePath = tempDir.resolve(fileId);
Files.write(filePath, fileContent);
// Act
boolean result = fileStorage.fileExists(fileId);
// Assert
assertTrue(result);
}
@Test
void testFileExists_FileNotFound() {
// Arrange
String nonExistentFileId = "non-existent-file";
// Act
boolean result = fileStorage.fileExists(nonExistentFileId);
// Assert
assertFalse(result);
}
}
@@ -1,208 +0,0 @@
package stirling.software.common.service;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeoutException;
import java.util.function.Supplier;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.test.util.ReflectionTestUtils;
import jakarta.servlet.http.HttpServletRequest;
import stirling.software.common.controller.WebSocketProgressController;
import stirling.software.common.model.job.JobProgress;
import stirling.software.common.model.job.JobResponse;
@ExtendWith(MockitoExtension.class)
class JobExecutorServiceTest {
private JobExecutorService jobExecutorService;
@Mock
private TaskManager taskManager;
@Mock
private WebSocketProgressController webSocketSender;
@Mock
private FileStorage fileStorage;
@Mock
private HttpServletRequest request;
@Mock
private ResourceMonitor resourceMonitor;
@Mock
private JobQueue jobQueue;
@Captor
private ArgumentCaptor<String> jobIdCaptor;
@BeforeEach
void setUp() {
// Initialize the service manually with all its dependencies
jobExecutorService = new JobExecutorService(
taskManager,
webSocketSender,
fileStorage,
request,
resourceMonitor,
jobQueue,
30000L, // asyncRequestTimeoutMs
"30m" // sessionTimeout
);
}
@Test
void shouldRunSyncJobSuccessfully() throws Exception {
// Given
Supplier<Object> work = () -> "test-result";
// When
ResponseEntity<?> response = jobExecutorService.runJobGeneric(false, work);
// Then
assertEquals(HttpStatus.OK, response.getStatusCode());
assertEquals("test-result", response.getBody());
// Verify request attribute was set with jobId
verify(request).setAttribute(eq("jobId"), anyString());
}
@Test
void shouldRunAsyncJobSuccessfully() throws Exception {
// Given
Supplier<Object> work = () -> "test-result";
// When
ResponseEntity<?> response = jobExecutorService.runJobGeneric(true, work);
// Then
assertEquals(HttpStatus.OK, response.getStatusCode());
assertTrue(response.getBody() instanceof JobResponse);
JobResponse<?> jobResponse = (JobResponse<?>) response.getBody();
assertTrue(jobResponse.isAsync());
assertNotNull(jobResponse.getJobId());
// Verify task manager was called
verify(taskManager).createTask(jobIdCaptor.capture());
verify(webSocketSender, Mockito.atLeastOnce()).sendProgress(eq(jobIdCaptor.getValue()), any(JobProgress.class));
}
@Test
void shouldHandleSyncJobError() {
// Given
Supplier<Object> work = () -> {
throw new RuntimeException("Test error");
};
// When
ResponseEntity<?> response = jobExecutorService.runJobGeneric(false, work);
// Then
assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, response.getStatusCode());
@SuppressWarnings("unchecked")
Map<String, String> errorMap = (Map<String, String>) response.getBody();
assertEquals("Job failed: Test error", errorMap.get("error"));
}
@Test
void shouldQueueJobWhenResourcesLimited() {
// Given
Supplier<Object> work = () -> "test-result";
CompletableFuture<ResponseEntity<?>> future = new CompletableFuture<>();
// Configure resourceMonitor to indicate job should be queued
when(resourceMonitor.shouldQueueJob(80)).thenReturn(true);
// Configure jobQueue to return our future
when(jobQueue.queueJob(anyString(), eq(80), any(), anyLong())).thenReturn(future);
// When
ResponseEntity<?> response = jobExecutorService.runJobGeneric(
true, work, 5000, true, 80);
// Then
assertEquals(HttpStatus.OK, response.getStatusCode());
assertTrue(response.getBody() instanceof JobResponse);
// Verify job was queued
verify(jobQueue).queueJob(anyString(), eq(80), any(), eq(5000L));
verify(taskManager).createTask(anyString());
}
@Test
void shouldUseCustomTimeoutWhenProvided() throws Exception {
// Given
Supplier<Object> work = () -> "test-result";
long customTimeout = 60000L;
// Use reflection to access the private executeWithTimeout method
java.lang.reflect.Method executeMethod = JobExecutorService.class
.getDeclaredMethod("executeWithTimeout", Supplier.class, long.class);
executeMethod.setAccessible(true);
// Create a spy on the JobExecutorService to verify method calls
JobExecutorService spy = Mockito.spy(jobExecutorService);
// When
spy.runJobGeneric(false, work, customTimeout);
// Then
verify(spy).runJobGeneric(eq(false), any(Supplier.class), eq(customTimeout));
}
@Test
void shouldHandleTimeout() throws Exception {
// Given
Supplier<Object> work = () -> {
try {
Thread.sleep(100); // Simulate long-running job
return "test-result";
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
};
// Use reflection to access the private executeWithTimeout method
java.lang.reflect.Method executeMethod = JobExecutorService.class
.getDeclaredMethod("executeWithTimeout", Supplier.class, long.class);
executeMethod.setAccessible(true);
// When/Then
try {
executeMethod.invoke(jobExecutorService, work, 1L); // Very short timeout
} catch (Exception e) {
assertTrue(e.getCause() instanceof TimeoutException);
}
}
}
@@ -1,107 +0,0 @@
package stirling.software.common.service;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.common.controller.WebSocketProgressController;
import stirling.software.common.model.job.JobProgress;
import stirling.software.common.service.ResourceMonitor.ResourceStatus;
@ExtendWith(MockitoExtension.class)
class JobQueueTest {
private JobQueue jobQueue;
@Mock
private ResourceMonitor resourceMonitor;
@Mock
private WebSocketProgressController webSocketSender;
private final AtomicReference<ResourceStatus> statusRef = new AtomicReference<>(ResourceStatus.OK);
@BeforeEach
void setUp() {
// Mark stubbing as lenient to avoid UnnecessaryStubbingException
lenient().when(resourceMonitor.calculateDynamicQueueCapacity(anyInt(), anyInt())).thenReturn(10);
lenient().when(resourceMonitor.getCurrentStatus()).thenReturn(statusRef);
jobQueue = new JobQueue(resourceMonitor, webSocketSender);
}
@Test
void shouldQueueJob() {
String jobId = "test-job-1";
int resourceWeight = 50;
Supplier<Object> work = () -> "test-result";
long timeoutMs = 1000;
jobQueue.queueJob(jobId, resourceWeight, work, timeoutMs);
verify(webSocketSender).sendProgress(
org.mockito.ArgumentMatchers.eq(jobId),
org.mockito.ArgumentMatchers.any(JobProgress.class));
assertTrue(jobQueue.isJobQueued(jobId));
assertEquals(1, jobQueue.getTotalQueuedJobs());
}
@Test
void shouldCancelJob() {
String jobId = "test-job-2";
Supplier<Object> work = () -> "test-result";
jobQueue.queueJob(jobId, 50, work, 1000);
boolean cancelled = jobQueue.cancelJob(jobId);
assertTrue(cancelled);
assertFalse(jobQueue.isJobQueued(jobId));
}
@Test
void shouldGetQueueStats() {
when(resourceMonitor.getCurrentStatus()).thenReturn(statusRef);
jobQueue.queueJob("job1", 50, () -> "ok", 1000);
jobQueue.queueJob("job2", 50, () -> "ok", 1000);
jobQueue.cancelJob("job2");
Map<String, Object> stats = jobQueue.getQueueStats();
assertEquals(2, stats.get("totalQueuedJobs"));
assertTrue(stats.containsKey("queuedJobs"));
assertTrue(stats.containsKey("resourceStatus"));
}
@Test
void shouldCalculateQueueCapacity() {
when(resourceMonitor.calculateDynamicQueueCapacity(5, 2)).thenReturn(8);
int capacity = resourceMonitor.calculateDynamicQueueCapacity(5, 2);
assertEquals(8, capacity);
}
@Test
void shouldCheckIfJobIsQueued() {
String jobId = "job-123";
Supplier<Object> work = () -> "hello";
jobQueue.queueJob(jobId, 40, work, 500);
assertTrue(jobQueue.isJobQueued(jobId));
assertFalse(jobQueue.isJobQueued("nonexistent"));
}
}
@@ -1,137 +0,0 @@
package stirling.software.common.service;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
import java.lang.management.OperatingSystemMXBean;
import java.time.Instant;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils;
import stirling.software.common.service.ResourceMonitor.ResourceMetrics;
import stirling.software.common.service.ResourceMonitor.ResourceStatus;
@ExtendWith(MockitoExtension.class)
class ResourceMonitorTest {
@InjectMocks
private ResourceMonitor resourceMonitor;
@Mock
private OperatingSystemMXBean osMXBean;
@Mock
private MemoryMXBean memoryMXBean;
@Spy
private AtomicReference<ResourceStatus> currentStatus = new AtomicReference<>(ResourceStatus.OK);
@Spy
private AtomicReference<ResourceMetrics> latestMetrics = new AtomicReference<>(new ResourceMetrics());
@BeforeEach
void setUp() {
// Set thresholds for testing
ReflectionTestUtils.setField(resourceMonitor, "memoryCriticalThreshold", 0.9);
ReflectionTestUtils.setField(resourceMonitor, "memoryHighThreshold", 0.75);
ReflectionTestUtils.setField(resourceMonitor, "cpuCriticalThreshold", 0.9);
ReflectionTestUtils.setField(resourceMonitor, "cpuHighThreshold", 0.75);
ReflectionTestUtils.setField(resourceMonitor, "osMXBean", osMXBean);
ReflectionTestUtils.setField(resourceMonitor, "memoryMXBean", memoryMXBean);
ReflectionTestUtils.setField(resourceMonitor, "currentStatus", currentStatus);
ReflectionTestUtils.setField(resourceMonitor, "latestMetrics", latestMetrics);
}
@Test
void shouldCalculateDynamicQueueCapacity() {
// Given
int baseCapacity = 10;
int minCapacity = 2;
// Mock current status as OK
currentStatus.set(ResourceStatus.OK);
// When
int capacity = resourceMonitor.calculateDynamicQueueCapacity(baseCapacity, minCapacity);
// Then
assertEquals(baseCapacity, capacity, "With OK status, capacity should equal base capacity");
// Given
currentStatus.set(ResourceStatus.WARNING);
// When
capacity = resourceMonitor.calculateDynamicQueueCapacity(baseCapacity, minCapacity);
// Then
assertEquals(6, capacity, "With WARNING status, capacity should be reduced to 60%");
// Given
currentStatus.set(ResourceStatus.CRITICAL);
// When
capacity = resourceMonitor.calculateDynamicQueueCapacity(baseCapacity, minCapacity);
// Then
assertEquals(3, capacity, "With CRITICAL status, capacity should be reduced to 30%");
// Test minimum capacity enforcement
assertEquals(minCapacity, resourceMonitor.calculateDynamicQueueCapacity(1, minCapacity),
"Should never go below minimum capacity");
}
@ParameterizedTest
@CsvSource({
"10, OK, false", // Light job, OK status
"10, WARNING, false", // Light job, WARNING status
"10, CRITICAL, true", // Light job, CRITICAL status
"30, OK, false", // Medium job, OK status
"30, WARNING, true", // Medium job, WARNING status
"30, CRITICAL, true", // Medium job, CRITICAL status
"80, OK, true", // Heavy job, OK status
"80, WARNING, true", // Heavy job, WARNING status
"80, CRITICAL, true" // Heavy job, CRITICAL status
})
void shouldQueueJobBasedOnWeightAndStatus(int weight, ResourceStatus status, boolean shouldQueue) {
// Given
currentStatus.set(status);
// When
boolean result = resourceMonitor.shouldQueueJob(weight);
// Then
assertEquals(shouldQueue, result,
String.format("For weight %d and status %s, shouldQueue should be %s",
weight, status, shouldQueue));
}
@Test
void resourceMetricsShouldDetectStaleState() {
// Given
Instant now = Instant.now();
Instant pastInstant = now.minusMillis(6000);
ResourceMetrics staleMetrics = new ResourceMetrics(0.5, 0.5, 1024, 2048, 4096, pastInstant);
ResourceMetrics freshMetrics = new ResourceMetrics(0.5, 0.5, 1024, 2048, 4096, now);
// When/Then
assertTrue(staleMetrics.isStale(5000), "Metrics from 6 seconds ago should be stale with 5s threshold");
assertFalse(freshMetrics.isStale(5000), "Fresh metrics should not be stale");
}
}
@@ -1,255 +0,0 @@
package stirling.software.common.service;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import java.time.LocalDateTime;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.springframework.test.util.ReflectionTestUtils;
import stirling.software.common.model.job.JobResult;
import stirling.software.common.model.job.JobStats;
class TaskManagerTest {
@Mock
private FileStorage fileStorage;
@InjectMocks
private TaskManager taskManager;
private AutoCloseable closeable;
@BeforeEach
void setUp() {
closeable = MockitoAnnotations.openMocks(this);
ReflectionTestUtils.setField(taskManager, "jobResultExpiryMinutes", 30);
}
@AfterEach
void tearDown() throws Exception {
closeable.close();
}
@Test
void testCreateTask() {
// Act
String jobId = UUID.randomUUID().toString();
taskManager.createTask(jobId);
// Assert
JobResult result = taskManager.getJobResult(jobId);
assertNotNull(result);
assertEquals(jobId, result.getJobId());
assertFalse(result.isComplete());
assertNotNull(result.getCreatedAt());
}
@Test
void testSetResult() {
// Arrange
String jobId = UUID.randomUUID().toString();
taskManager.createTask(jobId);
Object resultObject = "Test result";
// Act
taskManager.setResult(jobId, resultObject);
// Assert
JobResult result = taskManager.getJobResult(jobId);
assertNotNull(result);
assertTrue(result.isComplete());
assertEquals(resultObject, result.getResult());
assertNotNull(result.getCompletedAt());
}
@Test
void testSetFileResult() {
// Arrange
String jobId = UUID.randomUUID().toString();
taskManager.createTask(jobId);
String fileId = "file-id";
String originalFileName = "test.pdf";
String contentType = "application/pdf";
// Act
taskManager.setFileResult(jobId, fileId, originalFileName, contentType);
// Assert
JobResult result = taskManager.getJobResult(jobId);
assertNotNull(result);
assertTrue(result.isComplete());
assertEquals(fileId, result.getFileId());
assertEquals(originalFileName, result.getOriginalFileName());
assertEquals(contentType, result.getContentType());
assertNotNull(result.getCompletedAt());
}
@Test
void testSetError() {
// Arrange
String jobId = UUID.randomUUID().toString();
taskManager.createTask(jobId);
String errorMessage = "Test error";
// Act
taskManager.setError(jobId, errorMessage);
// Assert
JobResult result = taskManager.getJobResult(jobId);
assertNotNull(result);
assertTrue(result.isComplete());
assertEquals(errorMessage, result.getError());
assertNotNull(result.getCompletedAt());
}
@Test
void testSetComplete_WithExistingResult() {
// Arrange
String jobId = UUID.randomUUID().toString();
taskManager.createTask(jobId);
Object resultObject = "Test result";
taskManager.setResult(jobId, resultObject);
// Act
taskManager.setComplete(jobId);
// Assert
JobResult result = taskManager.getJobResult(jobId);
assertNotNull(result);
assertTrue(result.isComplete());
assertEquals(resultObject, result.getResult());
}
@Test
void testSetComplete_WithoutExistingResult() {
// Arrange
String jobId = UUID.randomUUID().toString();
taskManager.createTask(jobId);
// Act
taskManager.setComplete(jobId);
// Assert
JobResult result = taskManager.getJobResult(jobId);
assertNotNull(result);
assertTrue(result.isComplete());
assertEquals("Task completed successfully", result.getResult());
}
@Test
void testIsComplete() {
// Arrange
String jobId = UUID.randomUUID().toString();
taskManager.createTask(jobId);
// Assert - not complete initially
assertFalse(taskManager.isComplete(jobId));
// Act - mark as complete
taskManager.setComplete(jobId);
// Assert - now complete
assertTrue(taskManager.isComplete(jobId));
}
@Test
void testGetJobStats() {
// Arrange
// 1. Create active job
String activeJobId = "active-job";
taskManager.createTask(activeJobId);
// 2. Create completed successful job with file
String successFileJobId = "success-file-job";
taskManager.createTask(successFileJobId);
taskManager.setFileResult(successFileJobId, "file-id", "test.pdf", "application/pdf");
// 3. Create completed successful job without file
String successJobId = "success-job";
taskManager.createTask(successJobId);
taskManager.setResult(successJobId, "Result");
// 4. Create failed job
String failedJobId = "failed-job";
taskManager.createTask(failedJobId);
taskManager.setError(failedJobId, "Error message");
// Act
JobStats stats = taskManager.getJobStats();
// Assert
assertEquals(4, stats.getTotalJobs());
assertEquals(1, stats.getActiveJobs());
assertEquals(3, stats.getCompletedJobs());
assertEquals(1, stats.getFailedJobs());
assertEquals(2, stats.getSuccessfulJobs());
assertEquals(1, stats.getFileResultJobs());
assertNotNull(stats.getNewestActiveJobTime());
assertNotNull(stats.getOldestActiveJobTime());
assertTrue(stats.getAverageProcessingTimeMs() >= 0);
}
@Test
void testCleanupOldJobs() throws Exception {
// Arrange
// 1. Create a recent completed job
String recentJobId = "recent-job";
taskManager.createTask(recentJobId);
taskManager.setResult(recentJobId, "Result");
// 2. Create an old completed job with file result
String oldJobId = "old-job";
taskManager.createTask(oldJobId);
JobResult oldJob = taskManager.getJobResult(oldJobId);
// Manually set the completion time to be older than the expiry
LocalDateTime oldTime = LocalDateTime.now().minusHours(1);
ReflectionTestUtils.setField(oldJob, "completedAt", oldTime);
ReflectionTestUtils.setField(oldJob, "complete", true);
ReflectionTestUtils.setField(oldJob, "fileId", "file-id");
ReflectionTestUtils.setField(oldJob, "originalFileName", "test.pdf");
ReflectionTestUtils.setField(oldJob, "contentType", "application/pdf");
when(fileStorage.deleteFile("file-id")).thenReturn(true);
// Obtain access to the private jobResults map
Map<String, JobResult> jobResultsMap = (Map<String, JobResult>) ReflectionTestUtils.getField(taskManager, "jobResults");
// 3. Create an active job
String activeJobId = "active-job";
taskManager.createTask(activeJobId);
// Verify all jobs are in the map
assertTrue(jobResultsMap.containsKey(recentJobId));
assertTrue(jobResultsMap.containsKey(oldJobId));
assertTrue(jobResultsMap.containsKey(activeJobId));
// Act
taskManager.cleanupOldJobs();
// Assert - the old job should be removed
assertFalse(jobResultsMap.containsKey(oldJobId));
assertTrue(jobResultsMap.containsKey(recentJobId));
assertTrue(jobResultsMap.containsKey(activeJobId));
verify(fileStorage).deleteFile("file-id");
}
@Test
void testShutdown() throws Exception {
// This mainly tests that the shutdown method doesn't throw exceptions
taskManager.shutdown();
// Verify the executor service is shutdown
// This is difficult to test directly, but we can verify it doesn't throw exceptions
}
}
@@ -1,62 +0,0 @@
package stirling.software.common.util;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
public class ProcessExecutorTest {
private ProcessExecutor processExecutor;
@BeforeEach
public void setUp() {
// Initialize the ProcessExecutor instance
processExecutor = ProcessExecutor.getInstance(ProcessExecutor.Processes.LIBRE_OFFICE);
}
@Test
public void testRunCommandWithOutputHandling() throws IOException, InterruptedException {
// Mock the command to execute
List<String> command = new ArrayList<>();
command.add("java");
command.add("-version");
// Execute the command
ProcessExecutor.ProcessExecutorResult result =
processExecutor.runCommandWithOutputHandling(command);
// Check the exit code and output messages
assertEquals(0, result.getRc());
assertNotNull(result.getMessages()); // Check if messages are not null
}
@Test
public void testRunCommandWithOutputHandling_Error() {
// Mock the command to execute
List<String> command = new ArrayList<>();
command.add("nonexistent-command");
// Execute the command and expect an IOException
IOException thrown =
assertThrows(
IOException.class,
() -> {
processExecutor.runCommandWithOutputHandling(command);
});
// Check the exception message to ensure it indicates the command was not found
String errorMessage = thrown.getMessage();
assertTrue(
errorMessage.contains("error=2")
|| errorMessage.contains("No such file or directory"),
"Unexpected error message: " + errorMessage);
}
}
-168
View File
@@ -1,168 +0,0 @@
# AutoJobPostMapping Annotation
The `AutoJobPostMapping` annotation simplifies the creation of job-based REST endpoints in Stirling-PDF. It automatically handles job execution, file persistence, error handling, retries, and progress tracking.
## Features
- Wraps endpoint methods with job execution logic
- Supports both synchronous and asynchronous execution (via `?async=true` query parameter)
- Custom timeout configuration per endpoint
- Automatic retries with configurable retry count
- WebSocket-based progress tracking
- Consistent error handling and reporting
- Automatic persistence of uploaded files for async processing
## Usage
```java
@AutoJobPostMapping("/api/v1/security/remove-password")
public ResponseEntity<byte[]> removePassword(@ModelAttribute PDFPasswordRequest request)
throws IOException {
MultipartFile fileInput = request.getFileInput();
String password = request.getPassword();
PDDocument document = pdfDocumentFactory.load(fileInput, password);
document.setAllSecurityToBeRemoved(true);
return WebResponseUtils.pdfDocToWebResponse(
document,
Filenames.toSimpleFileName(fileInput.getOriginalFilename())
.replaceFirst("[.][^.]+$", "")
+ "_password_removed.pdf");
}
```
## Parameters
The `AutoJobPostMapping` annotation accepts the following parameters:
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `value` | String[] | `{}` | The path mapping URIs (e.g., "/api/v1/security/add-password") |
| `consumes` | String[] | `{"multipart/form-data"}` | Supported media types for requests |
| `timeout` | long | `-1` (use system default) | Custom timeout in milliseconds for this job |
| `retryCount` | int | `1` (no retries) | Maximum number of retry attempts on failure |
| `trackProgress` | boolean | `true` | Enable WebSocket progress tracking for async jobs |
| `queueable` | boolean | `false` | Whether this job can be queued when system resources are limited |
| `resourceWeight` | int | `50` | Resource weight of this job (1-100), higher values indicate more resource-intensive jobs |
## Examples
### Basic Usage
```java
@AutoJobPostMapping("/api/v1/security/remove-password")
public ResponseEntity<byte[]> removePassword(@ModelAttribute PDFPasswordRequest request) {
// Implementation
}
```
### With Custom Timeout
```java
// Set a 5-minute timeout for this operation
@AutoJobPostMapping(value = "/api/v1/misc/ocr-pdf", timeout = 300000)
public ResponseEntity<byte[]> ocrPdf(@ModelAttribute OCRRequest request) {
// OCR implementation
}
```
### With Retries
```java
// Allow up to 3 retry attempts for external API calls
@AutoJobPostMapping(value = "/api/v1/convert/url-to-pdf", retryCount = 3)
public ResponseEntity<byte[]> convertUrlToPdf(@ModelAttribute WebsiteToPDFRequest request) {
// Implementation
}
```
### Disable Progress Tracking
```java
// Simple, fast operation that doesn't need progress tracking
@AutoJobPostMapping(value = "/api/v1/misc/flatten", trackProgress = false)
public ResponseEntity<byte[]> flattenPdf(@ModelAttribute FlattenRequest request) {
// Implementation
}
```
### Enable Job Queueing for Resource-Intensive Operations
```java
// Resource-intensive operation that can be queued during high system load
@AutoJobPostMapping(
value = "/api/v1/misc/ocr-pdf",
queueable = true,
resourceWeight = 80, // High resource usage
timeout = 600000 // 10 minutes
)
public ResponseEntity<byte[]> ocrPdf(@ModelAttribute OCRRequest request) {
// OCR implementation
}
```
### Lightweight Operation
```java
// Very lightweight operation with low resource requirements
@AutoJobPostMapping(
value = "/api/v1/misc/get-page-count",
queueable = false,
resourceWeight = 10 // Very low resource usage
)
public ResponseEntity<Integer> getPageCount(@ModelAttribute PDFFile request) {
// Simple page count implementation
}
```
## Client-Side Integration
For asynchronous jobs, clients can:
1. Submit the job with `?async=true` parameter
2. Receive a job ID in the response
3. Connect to the WebSocket at `/ws/progress/{jobId}` to receive progress updates
4. Fetch the completed result from `/api/v1/general/job/{jobId}/result` when done
Example WebSocket message:
```json
{
"jobId": "b4c9a31d-4b7e-42b2-8ab9-3cbe99d5b94f",
"status": "Processing",
"progress": 65,
"message": "OCR processing page 13/20"
}
```
## Resource-Aware Job Queueing
The `queueable` parameter enables intelligent resource-aware job queueing for heavy operations. When enabled:
1. Jobs are automatically queued when system resources (CPU, memory) are constrained
2. Queue capacity dynamically adjusts based on available resources
3. Queue position and status updates are sent via WebSocket
4. Jobs with high `resourceWeight` values have stricter queueing conditions
5. Long-running jobs don't block the system from handling other requests
### Resource Weight Guidelines
When setting the `resourceWeight` parameter, use these guidelines:
| Weight Range | Appropriate For |
|--------------|----------------|
| 1-20 | Lightweight operations: metadata reads, simple transforms, etc. |
| 21-50 | Medium operations: basic PDF manipulation, simple image operations |
| 51-80 | Heavy operations: PDF merging, image conversions, medium OCR |
| 81-100 | Very intensive operations: large OCR jobs, complex transformations |
### Example Queue Status Messages
```json
{
"jobId": "b4c9a31d-4b7e-42b2-8ab9-3cbe99d5b94f",
"status": "Queued",
"progress": 0,
"message": "Waiting in queue for resources (position 3)"
}
```
```json
{
"jobId": "b4c9a31d-4b7e-42b2-8ab9-3cbe99d5b94f",
"status": "Starting",
"progress": 10,
"message": "Resources available, starting job execution"
}
```
+250
View File
@@ -0,0 +1,250 @@
# UnoServer Configuration Guide
## Overview
Stirling-PDF supports multiple UnoServer instances to improve concurrent document conversion performance. This document explains how to configure and use this feature.
The UnoServer component in Stirling-PDF is now conditional, meaning it will only be enabled if the required executables are available on your system. This allows Stirling-PDF to run in environments without LibreOffice/UnoServer while gracefully disabling office document conversion functionality.
## Configuration Options
### Multiple Local Instances
You can configure Stirling-PDF to start multiple UnoServer instances locally. Each instance will run on its own port starting from the base port.
In `settings.yml`:
```yaml
processExecutor:
sessionLimit:
libreOfficeSessionLimit: 4 # Set to desired number of instances
baseUnoconvPort: 2003 # Base port for UnoServer instances
useExternalUnoconvServers: false # Set to false to use local instances
```
### External UnoServer Instances
For more advanced setups or when running in a clustered environment, you can configure Stirling-PDF to use external UnoServer instances running on different hosts:
```yaml
processExecutor:
useExternalUnoconvServers: true
unoconvServers:
- "192.168.1.100:2003" # Format is host:port
- "192.168.1.101:2003"
- "unoserver-host:2003"
```
## Installation
### Docker
The easiest way to use UnoServer with Stirling-PDF is to use the "fat" Docker image, which includes all required dependencies:
```bash
docker pull frooodle/s-pdf:latest-fat
```
### Manual Installation
If you want to install UnoServer manually:
1. Install LibreOffice:
```bash
# Ubuntu/Debian
sudo apt-get update
sudo apt-get install -y libreoffice
# CentOS/RHEL
sudo yum install -y libreoffice
# macOS
brew install libreoffice
```
2. Install UnoServer using pip:
```bash
pip install unoserver
```
3. Verify installation:
```bash
unoserver --version
unoconvert --version
```
### Installation Location
Stirling-PDF will automatically detect UnoServer in these locations:
- In the same directory as the unoconvert executable
- At `/opt/venv/bin/unoserver` (Docker default)
- In standard system paths (`/usr/bin/unoserver`, `/usr/local/bin/unoserver`)
- In your PATH environment variable
## Advanced Features
### Health Checks
The system performs automatic health checks on all UnoServer instances every 60 seconds. These checks:
- Verify that each server is reachable and operational
- Automatically restart local instances that have failed
- Log the health status of all instances
- Update the circuit breaker status for each instance
Health checks are logged at INFO level and show the number of healthy and unhealthy instances.
### Circuit Breaker
For fault tolerance, each UnoServer instance implements a circuit breaker pattern that:
1. Tracks conversion failures for each instance
2. After 3 consecutive failures, marks the instance as unavailable (circuit open)
3. Waits for a cooldown period (30 seconds) before attempting to use the instance again
4. Automatically routes requests to healthy instances
The circuit breaker helps prevent cascading failures and provides automatic recovery.
### Performance Metrics
The UnoServerManager records and logs detailed metrics about UnoServer usage:
- Total number of conversions
- Success/failure rate
- Conversions per server instance
- Average conversion time per instance
These metrics are logged periodically and on application shutdown, helping you monitor and optimize performance.
Example metrics log:
```
UnoServer metrics - Total: 120, Failed: 5, Success Rate: 95.83%
Conversions per instance:
[0] 127.0.0.1:2003 - Count: 32 (26.67%), Avg Time: 1250.45ms
[1] 127.0.0.1:2004 - Count: 30 (25.00%), Avg Time: 1187.33ms
[2] 127.0.0.1:2005 - Count: 29 (24.17%), Avg Time: 1345.78ms
[3] 127.0.0.1:2006 - Count: 29 (24.17%), Avg Time: 1290.12ms
```
## Testing Multiple Instances
To test that multiple UnoServer instances are working correctly:
1. Set `libreOfficeSessionLimit` to a value greater than 1 (e.g., 4)
2. Start the application
3. Check logs for messages like:
```
Initializing UnoServerManager with maxInstances: 4, useExternal: false
Starting UnoServer on 127.0.0.1:2003
Starting UnoServer on 127.0.0.1:2004
Starting UnoServer on 127.0.0.1:2005
Starting UnoServer on 127.0.0.1:2006
```
4. Submit multiple file conversion requests simultaneously
5. Observe in logs that different server instances are being used in a round-robin manner
6. Check the metrics logs to verify the load distribution across instances
## Performance Considerations
- Each UnoServer instance requires additional memory (typically 100-200 MB)
- Set the `libreOfficeSessionLimit` according to your server's available resources
- For most use cases, a value between 2-4 provides a good balance
- Larger values may improve concurrency but increase memory usage
- The circuit breaker pattern helps maintain system stability under high load
## Queue Management
Stirling-PDF now includes a queue management system for office document conversions:
### Queue Status API
The system exposes REST endpoints for checking conversion queue status:
- `GET /api/v1/queue/status` - Get status of all process queues
- `GET /api/v1/queue/unoserver` - Get detailed information about UnoServer instances and active tasks
- `GET /api/v1/queue/task/{taskId}` - Get status of a specific task by ID
Example response from `/api/v1/queue/unoserver`:
```json
{
"instanceCount": 4,
"activeTaskCount": 2,
"instances": [
{
"id": 0,
"host": "127.0.0.1",
"port": 2003,
"managed": true,
"running": true,
"available": true,
"failureCount": 0,
"averageConversionTimeMs": 1523.45,
"lastConversionTimeMs": 1498
},
...
],
"activeTasks": [
{
"id": "office-123",
"name": "Convert document.docx to PDF",
"status": "RUNNING",
"queuePosition": 0,
"queueTimeMs": 0,
"processTimeMs": 542,
"totalTimeMs": 542,
"errorMessage": null
},
...
]
}
```
### UI Integration
The system includes a built-in UI for monitoring conversion status:
1. A status indicator appears when a document is being converted
2. The indicator shows the current status, position in queue, and estimated wait time
3. For pages that use office conversions, a "Check Office Conversion Status" button provides detailed information about server instances and active conversions
## Troubleshooting
If you encounter issues with UnoServer:
1. Check logs for any error messages related to UnoServer startup
2. Look for health check logs to identify problematic instances
3. Verify ports are not already in use by other applications
4. Ensure LibreOffice is correctly installed
5. Check that UnoServer is properly installed and in your PATH:
```
which unoserver
which unoconvert
```
6. Try running a single UnoServer instance manually to check if it works:
```
/opt/venv/bin/unoserver --port 2003 --interface 127.0.0.1
```
7. Use the queue status API to check the status of UnoServer instances:
```
curl http://localhost:8080/api/v1/queue/unoserver
```
8. For external servers, verify network connectivity from the Stirling-PDF server
### Common Error Messages
| Error Message | Possible Cause | Solution |
|---------------|----------------|----------|
| "UnoServer is not available" | UnoServer executable not found | Install UnoServer or use the fat Docker image |
| "Failed to start UnoServer instance" | Port in use or LibreOffice issues | Check ports, restart application, or verify LibreOffice installation |
| "Circuit breaker opened for UnoServer instance" | Multiple conversion failures | Check logs for specific errors, verify UnoServer is working correctly |
| "No UnoServer instances available" | All instances are down or in circuit-open state | Restart application or check for resource issues |
## Environment Variables
When using Docker, you can configure UnoServer instances using environment variables:
- `LIBREOFFICE_SESSION_LIMIT`: Number of UnoServer instances to start
- `BASE_UNOCONV_PORT`: Base port number for UnoServer instances
- `USE_EXTERNAL_UNOCONVSERVERS`: Set to "true" to use external servers
-56
View File
@@ -1,56 +0,0 @@
# Scan Upload Feature Improvements
## Code Improvements
1. **Modular JavaScript Architecture**
- Split the monolithic `scan-upload-direct.js` into logical modules:
- `logger.js`: Handles logging and status updates
- `peer-connection.js`: Manages WebRTC peer connections
- `camera.js`: Controls mobile camera functionality
- `scan-upload.js`: Main module that coordinates everything
2. **Separated CSS from HTML**
- Created dedicated CSS files:
- `scan-upload.css`: Styles for the desktop scan-upload page
- `mobile-scanner.css`: Styles for the mobile camera interface
3. **Improved Error Handling**
- Added better error handling throughout the codebase
- Improved user feedback for connection and camera issues
- Enhanced debug logging capabilities
4. **Backward Compatibility**
- Created a compatibility layer that maintains the old API
- Allows gradual migration to the new code structure
- Ensures existing integrations won't break
## UI Improvements
1. **Enhanced Responsive Design**
- Improved mobile layout with proper media queries
- Better handling of different screen sizes
2. **Better Visual Feedback**
- Clearer status messages for users
- Improved styling of the scan result display
- Enhanced debug information presentation
3. **Localization Support**
- Added proper Thymeleaf text references for all UI elements
- Uses the existing messages system for translations
## Security Improvements
1. **Enhanced WebRTC Implementation**
- Better handling of connection errors
- Improved security for peer connections
- Structured error handling for failed connections
## Next Steps
Potential future improvements:
1. Add more robust testing for the WebRTC functionality
2. Consider implementing a fallback method if WebRTC is not available
3. Add support for scanning multiple documents in one session
4. Implement better image quality control options
+1 -2
View File
@@ -182,8 +182,7 @@ def compare_files(
sort_ignore_translation[language]["ignore"].remove(
default_key.strip()
)
except ValueError as e:
print(f"Error processing line {line_num} in {file_path}: {e}")
except ValueError:
print(f"{line_default}|{line_file}")
exit(1)
except IndexError:
+9 -753
View File
@@ -1,190 +1,34 @@
[ar_AR]
ignore = [
'lang.div',
'lang.dzo',
'lang.que',
'language.direction',
]
[az_AZ]
ignore = [
'lang.afr',
'lang.bre',
'lang.div',
'lang.epo',
'lang.guj',
'lang.hin',
'lang.kan',
'lang.mal',
'lang.mar',
'lang.mlt',
'lang.mri',
'lang.msa',
'lang.nep',
'lang.ori',
'lang.pan',
'lang.san',
'lang.sin',
'lang.slk',
'lang.snd',
'lang.sun',
'lang.tam',
'lang.tat',
'lang.urd',
'lang.yor',
'language.direction',
]
[bg_BG]
ignore = [
'lang.div',
'lang.dzo',
'lang.iku',
'lang.que',
'language.direction',
]
[bo_CN]
ignore = [
'language.direction',
]
[ca_CA]
ignore = [
'PDFToText.tags',
'adminUserSettings.admin',
'lang.amh',
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.epo',
'lang.fao',
'lang.fry',
'lang.guj',
'lang.hin',
'lang.iku',
'lang.kan',
'lang.kaz',
'lang.lao',
'lang.mar',
'lang.mri',
'lang.ori',
'lang.pan',
'lang.pus',
'lang.que',
'lang.snd',
'lang.swa',
'lang.tam',
'lang.tat',
'lang.tel',
'lang.tgk',
'lang.tgl',
'lang.tir',
'lang.uzb',
'lang.uzb_cyrl',
'language.direction',
'watermark.type.1',
]
[cs_CZ]
ignore = [
'lang.amh',
'lang.asm',
'lang.bod',
'lang.bos',
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.frk',
'lang.gla',
'lang.guj',
'lang.iku',
'lang.jav',
'lang.kan',
'lang.kat',
'lang.khm',
'lang.kir',
'lang.lao',
'lang.mal',
'lang.mar',
'lang.mri',
'lang.msa',
'lang.nor',
'lang.oci',
'lang.ori',
'lang.pan',
'lang.pus',
'lang.que',
'lang.san',
'lang.sin',
'lang.snd',
'lang.sun',
'lang.tam',
'lang.tat',
'lang.tel',
'lang.tgl',
'lang.tha',
'lang.tir',
'lang.uig',
'lang.urd',
'lang.uzb',
'lang.uzb_cyrl',
'lang.yor',
'language.direction',
'text',
]
[da_DK]
ignore = [
'lang.afr',
'lang.amh',
'lang.ben',
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.epo',
'lang.frk',
'lang.guj',
'lang.hin',
'lang.iku',
'lang.jav',
'lang.kan',
'lang.khm',
'lang.lao',
'lang.lat',
'lang.ltz',
'lang.mal',
'lang.mar',
'lang.mri',
'lang.msa',
'lang.nep',
'lang.oci',
'lang.ori',
'lang.pan',
'lang.pus',
'lang.que',
'lang.san',
'lang.sin',
'lang.slk_frak',
'lang.snd',
'lang.sun',
'lang.swa',
'lang.tam',
'lang.tat',
'lang.tel',
'lang.tgk',
'lang.tgl',
'lang.tha',
'lang.tir',
'lang.ton',
'lang.uig',
'lang.urd',
'lang.uzb',
'lang.yor',
'language.direction',
]
@@ -197,36 +41,8 @@ ignore = [
'addPageNumbers.selectText.3',
'alphabet',
'certSign.name',
'cookieBanner.popUp.acceptAllBtn',
'endpointStatistics.top10',
'endpointStatistics.top20',
'fileChooser.dragAndDrop',
'home.pipeline.title',
'lang.afr',
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.epo',
'lang.guj',
'lang.hin',
'lang.iku',
'lang.kan',
'lang.mal',
'lang.mar',
'lang.mri',
'lang.nep',
'lang.ori',
'lang.pan',
'lang.que',
'lang.san',
'lang.snd',
'lang.tam',
'lang.tel',
'lang.tgl',
'lang.tir',
'lang.urd',
'lang.yor',
'language.direction',
'legal.impressum',
'licenses.version',
@@ -240,19 +56,13 @@ ignore = [
'validateSignature.cert.version',
'validateSignature.status',
'watermark.type.1',
'endpointStatistics.top10',
'endpointStatistics.top20',
'cookieBanner.popUp.acceptAllBtn',
]
[el_GR]
ignore = [
'lang.ceb',
'lang.dzo',
'lang.iku',
'lang.ori',
'lang.pan',
'lang.que',
'lang.sin',
'lang.uig',
'lang.uzb_cyrl',
'language.direction',
]
@@ -260,31 +70,6 @@ ignore = [
ignore = [
'adminUserSettings.roles',
'error',
'lang.asm',
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.fil',
'lang.frm',
'lang.guj',
'lang.iku',
'lang.kan',
'lang.lao',
'lang.mal',
'lang.mar',
'lang.oci',
'lang.ori',
'lang.pan',
'lang.san',
'lang.snd',
'lang.sun',
'lang.tam',
'lang.tel',
'lang.tir',
'lang.urd',
'lang.uzb',
'lang.yor',
'language.direction',
'no',
'showJS.tags',
@@ -292,23 +77,6 @@ ignore = [
[eu_ES]
ignore = [
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.guj',
'lang.iku',
'lang.kan',
'lang.mal',
'lang.pan',
'lang.que',
'lang.san',
'lang.slv',
'lang.snd',
'lang.sqi',
'lang.tat',
'lang.tir',
'lang.yor',
'language.direction',
]
@@ -328,31 +96,6 @@ ignore = [
'alphabet',
'compare.document.1',
'compare.document.2',
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.eus',
'lang.guj',
'lang.iku',
'lang.kan',
'lang.kaz',
'lang.khm',
'lang.lao',
'lang.ltz',
'lang.mal',
'lang.mar',
'lang.oci',
'lang.ori',
'lang.que',
'lang.san',
'lang.snd',
'lang.swa',
'lang.tel',
'lang.tgl',
'lang.tir',
'lang.yor',
'language.direction',
'licenses.license',
'licenses.module',
@@ -365,24 +108,6 @@ ignore = [
[ga_IE]
ignore = [
'lang.ceb',
'lang.cos',
'lang.div',
'lang.dzo',
'lang.epo',
'lang.guj',
'lang.hat',
'lang.iku',
'lang.lao',
'lang.oci',
'lang.ori',
'lang.pan',
'lang.sin',
'lang.snd',
'lang.sun',
'lang.tgk',
'lang.tir',
'lang.uig',
'language.direction',
]
@@ -395,126 +120,22 @@ ignore = [
ignore = [
'PDFToBook.selectText.1',
'home.pipeline.title',
'lang.bod',
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.dzo',
'lang.guj',
'lang.iku',
'lang.kan',
'lang.lao',
'lang.mri',
'lang.ori',
'lang.pan',
'lang.pus',
'lang.que',
'lang.san',
'lang.snd',
'lang.tam',
'lang.tel',
'lang.tgl',
'lang.tir',
'language.direction',
'showJS.tags',
]
[hu_HU]
ignore = [
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.fao',
'lang.iku',
'lang.kan',
'lang.lao',
'lang.mar',
'lang.mri',
'lang.ori',
'lang.que',
'lang.tel',
'lang.tgl',
'language.direction',
]
[id_ID]
ignore = [
'lang.aze',
'lang.aze_cyrl',
'lang.bre',
'lang.cat',
'lang.ceb',
'lang.chr',
'lang.cym',
'lang.div',
'lang.dzo',
'lang.epo',
'lang.eus',
'lang.fao',
'lang.frk',
'lang.guj',
'lang.hin',
'lang.iku',
'lang.kan',
'lang.kaz',
'lang.kir',
'lang.lao',
'lang.lat',
'lang.mal',
'lang.mar',
'lang.mri',
'lang.ori',
'lang.pan',
'lang.pus',
'lang.que',
'lang.slk_frak',
'lang.snd',
'lang.sun',
'lang.swa',
'lang.tam',
'lang.tat',
'lang.tel',
'lang.tgk',
'lang.tgl',
'lang.tha',
'lang.tir',
'lang.uig',
'lang.urd',
'lang.uzb',
'lang.uzb_cyrl',
'lang.yor',
'language.direction',
]
[it_IT]
ignore = [
'lang.asm',
'lang.aze',
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.fao',
'lang.guj',
'lang.iku',
'lang.kan',
'lang.lao',
'lang.mal',
'lang.mar',
'lang.mri',
'lang.ori',
'lang.pan',
'lang.pus',
'lang.que',
'lang.snd',
'lang.swa',
'lang.tam',
'lang.tel',
'lang.tgl',
'lang.urd',
'lang.yor',
'language.direction',
'no',
'password',
@@ -527,21 +148,11 @@ ignore = [
[ja_JP]
ignore = [
'lang.jav',
'language.direction',
]
[ko_KR]
ignore = [
'lang.fao',
'lang.pus',
'lang.sun',
'language.direction',
]
[ml_IN]
ignore = [
'lang.iku',
'language.direction',
]
@@ -549,37 +160,6 @@ ignore = [
ignore = [
'compare.document.1',
'compare.document.2',
'lang.afr',
'lang.asm',
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.dzo',
'lang.epo',
'lang.fao',
'lang.guj',
'lang.hin',
'lang.iku',
'lang.kan',
'lang.lao',
'lang.mal',
'lang.mar',
'lang.mri',
'lang.oci',
'lang.ori',
'lang.pan',
'lang.pus',
'lang.que',
'lang.sin',
'lang.snd',
'lang.sun',
'lang.swa',
'lang.tam',
'lang.tel',
'lang.tgl',
'lang.ton',
'lang.urd',
'lang.yor',
'language.direction',
'navbar.allTools',
'sponsor',
@@ -590,49 +170,6 @@ ignore = [
'PDFToBook.selectText.1',
'adminUserSettings.admin',
'info',
'lang.afr',
'lang.amh',
'lang.ben',
'lang.bos',
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.dan_frak',
'lang.div',
'lang.dzo',
'lang.epo',
'lang.guj',
'lang.hin',
'lang.iku',
'lang.kan',
'lang.khm',
'lang.lao',
'lang.lat',
'lang.ltz',
'lang.mal',
'lang.mar',
'lang.mri',
'lang.msa',
'lang.nep',
'lang.ori',
'lang.pan',
'lang.pus',
'lang.que',
'lang.san',
'lang.slk_frak',
'lang.snd',
'lang.swa',
'lang.tam',
'lang.tat',
'lang.tel',
'lang.tgk',
'lang.tgl',
'lang.tha',
'lang.tir',
'lang.ton',
'lang.uig',
'lang.urd',
'lang.yor',
'language.direction',
'oops',
'sponsor',
@@ -641,148 +178,27 @@ ignore = [
[pl_PL]
ignore = [
'PDFToBook.selectText.1',
'lang.afr',
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.cos',
'lang.div',
'lang.dzo',
'lang.fao',
'lang.frk',
'lang.guj',
'lang.hat',
'lang.iku',
'lang.kan',
'lang.khm',
'lang.lao',
'lang.mal',
'lang.mar',
'lang.mri',
'lang.nep',
'lang.oci',
'lang.ori',
'lang.pus',
'lang.que',
'lang.snd',
'lang.sun',
'lang.swa',
'lang.tam',
'lang.tat',
'lang.tel',
'lang.tgl',
'lang.tir',
'lang.uig',
'lang.urd',
'language.direction',
]
[pt_BR]
ignore = [
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.fao',
'lang.fil',
'lang.frk',
'lang.fry',
'lang.guj',
'lang.iku',
'lang.kan',
'lang.kir',
'lang.mar',
'lang.ori',
'lang.pan',
'lang.que',
'lang.snd',
'lang.tat',
'lang.tel',
'lang.tgl',
'lang.tir',
'lang.uig',
'lang.uzb',
'lang.yid',
'language.direction',
'pipelineOptions.pipelineHeader',
]
[pt_PT]
ignore = [
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.fao',
'lang.fil',
'lang.frk',
'lang.fry',
'lang.guj',
'lang.iku',
'lang.kan',
'lang.kir',
'lang.mar',
'lang.ori',
'lang.pan',
'lang.que',
'lang.snd',
'lang.tat',
'lang.tel',
'lang.tgl',
'lang.tir',
'lang.uig',
'lang.uzb',
'lang.yid',
'language.direction',
]
[ro_RO]
ignore = [
'lang.amh',
'lang.asm',
'lang.bod',
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.cos',
'lang.deu_frak',
'lang.div',
'lang.dzo',
'lang.est',
'lang.fao',
'lang.glg',
'lang.guj',
'lang.iku',
'lang.jav',
'lang.kan',
'lang.lao',
'lang.mal',
'lang.mar',
'lang.mri',
'lang.nep',
'lang.oci',
'lang.ori',
'lang.pan',
'lang.pus',
'lang.slk_frak',
'lang.snd',
'lang.sun',
'lang.swa',
'lang.tam',
'lang.tel',
'lang.tgl',
'lang.tir',
'lang.urd',
'lang.yor',
'language.direction',
]
[ru_RU]
ignore = [
'lang.iku',
'lang.pus',
'language.direction',
]
@@ -791,25 +207,6 @@ ignore = [
'adminUserSettings.admin',
'home.multiTool.title',
'info',
'lang.ceb',
'lang.chr',
'lang.dzo',
'lang.epo',
'lang.iku',
'lang.kaz',
'lang.mar',
'lang.ori',
'lang.pan',
'lang.que',
'lang.san',
'lang.sin',
'lang.snd',
'lang.tat',
'lang.tel',
'lang.tgl',
'lang.tir',
'lang.urd',
'lang.uzb',
'language.direction',
'navbar.sections.security',
'text',
@@ -818,37 +215,6 @@ ignore = [
[sl_SI]
ignore = [
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.epo',
'lang.eus',
'lang.fao',
'lang.frk',
'lang.guj',
'lang.iku',
'lang.kan',
'lang.lao',
'lang.mar',
'lang.mri',
'lang.oci',
'lang.ori',
'lang.pan',
'lang.que',
'lang.slk',
'lang.snd',
'lang.sun',
'lang.tam',
'lang.tat',
'lang.tel',
'lang.tgk',
'lang.tgl',
'lang.tir',
'lang.urd',
'lang.uzb',
'lang.yor',
'language.direction',
]
@@ -861,43 +227,11 @@ ignore = [
[sv_SE]
ignore = [
'lang.ben',
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.dzo',
'lang.epo',
'lang.guj',
'lang.hin',
'lang.kan',
'lang.lao',
'lang.lat',
'lang.mal',
'lang.mri',
'lang.ori',
'lang.pan',
'lang.que',
'lang.san',
'lang.slk_frak',
'lang.snd',
'lang.tam',
'lang.tat',
'lang.tel',
'lang.tir',
'lang.urd',
'lang.yor',
'language.direction',
]
[th_TH]
ignore = [
'lang.dzo',
'lang.kir',
'lang.pan',
'lang.sin',
'lang.slk_frak',
'lang.tir',
'lang.uzb_cyrl',
'language.direction',
'pipelineOptions.pipelineHeader',
'showJS.tags',
@@ -905,111 +239,33 @@ ignore = [
[tr_TR]
ignore = [
'lang.afr',
'lang.bre',
'lang.ceb',
'lang.chr',
'lang.div',
'lang.dzo',
'lang.epo',
'lang.fao',
'lang.guj',
'lang.kan',
'lang.lao',
'lang.mal',
'lang.mar',
'lang.mri',
'lang.msa',
'lang.ori',
'lang.pus',
'lang.que',
'lang.sin',
'lang.slk',
'lang.slk_frak',
'lang.snd',
'lang.tam',
'lang.tat',
'lang.tel',
'lang.tgk',
'lang.tgl',
'lang.tir',
'lang.urd',
'lang.yor',
'language.direction',
]
[uk_UA]
ignore = [
'lang.iku',
'language.direction',
]
[vi_VN]
ignore = [
'lang.amh',
'lang.asm',
'lang.aze',
'lang.aze_cyrl',
'lang.bos',
'lang.bre',
'lang.cat',
'lang.ceb',
'lang.chr',
'lang.cos',
'lang.div',
'lang.dzo',
'lang.epo',
'lang.eus',
'lang.fao',
'lang.glg',
'lang.guj',
'lang.iku',
'lang.kan',
'lang.kaz',
'lang.kir',
'lang.lat',
'lang.ltz',
'lang.mal',
'lang.mar',
'lang.mri',
'lang.msa',
'lang.ori',
'lang.pus',
'lang.que',
'lang.sin',
'lang.slk',
'lang.slk_frak',
'lang.snd',
'lang.swa',
'lang.syr',
'lang.tam',
'lang.tat',
'lang.tel',
'lang.tgk',
'lang.tir',
'lang.uig',
'lang.uzb',
'lang.uzb_cyrl',
'lang.yid',
'lang.yor',
'language.direction',
'pipeline.title',
'pipelineOptions.pipelineHeader',
'showJS.tags',
]
[zh_BO]
ignore = [
'language.direction',
]
[zh_CN]
ignore = [
'lang.dzo',
'lang.iku',
'lang.que',
'language.direction',
]
[zh_TW]
ignore = [
'lang.dzo',
'lang.iku',
'lang.que',
'language.direction',
]
-2
View File
@@ -3,5 +3,3 @@ plugins {
id 'org.gradle.toolchains.foojay-resolver-convention' version '1.0.0'
}
rootProject.name = 'Stirling-PDF'
include 'common'
@@ -8,10 +8,10 @@ import org.springframework.core.annotation.Order;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.EE.KeygenLicenseVerifier.License;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.ApplicationProperties.EnterpriseEdition;
import stirling.software.common.model.ApplicationProperties.Premium;
import stirling.software.common.model.ApplicationProperties.Premium.ProFeatures.GoogleDrive;
import stirling.software.SPDF.model.ApplicationProperties;
import stirling.software.SPDF.model.ApplicationProperties.EnterpriseEdition;
import stirling.software.SPDF.model.ApplicationProperties.Premium;
import stirling.software.SPDF.model.ApplicationProperties.Premium.ProFeatures.GoogleDrive;
@Configuration
@Order(Ordered.HIGHEST_PRECEDENCE)
@@ -19,8 +19,8 @@ import com.posthog.java.shaded.org.json.JSONObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.util.GeneralUtils;
import stirling.software.SPDF.model.ApplicationProperties;
import stirling.software.SPDF.utils.GeneralUtils;
@Service
@Slf4j
@@ -11,8 +11,8 @@ import org.springframework.stereotype.Component;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.EE.KeygenLicenseVerifier.License;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.util.GeneralUtils;
import stirling.software.SPDF.model.ApplicationProperties;
import stirling.software.SPDF.utils.GeneralUtils;
@Component
@Slf4j
@@ -3,11 +3,11 @@ package stirling.software.SPDF.Factories;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.common.model.api.misc.HighContrastColorCombination;
import stirling.software.common.model.api.misc.ReplaceAndInvert;
import stirling.software.common.util.misc.CustomColorReplaceStrategy;
import stirling.software.common.util.misc.InvertFullColorStrategy;
import stirling.software.common.util.misc.ReplaceAndInvertColorStrategy;
import stirling.software.SPDF.model.api.misc.HighContrastColorCombination;
import stirling.software.SPDF.model.api.misc.ReplaceAndInvert;
import stirling.software.SPDF.utils.misc.CustomColorReplaceStrategy;
import stirling.software.SPDF.utils.misc.InvertFullColorStrategy;
import stirling.software.SPDF.utils.misc.ReplaceAndInvertColorStrategy;
@Component
public class ReplaceAndInvertColorFactory {
@@ -14,8 +14,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;
import org.springframework.core.env.Environment;
import org.springframework.scheduling.annotation.EnableScheduling;
@@ -27,19 +25,14 @@ import jakarta.annotation.PreDestroy;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.UI.WebBrowser;
import stirling.software.common.configuration.ConfigInitializer;
import stirling.software.common.configuration.InstallationPathConfig;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.util.UrlUtils;
import stirling.software.SPDF.config.ConfigInitializer;
import stirling.software.SPDF.config.InstallationPathConfig;
import stirling.software.SPDF.model.ApplicationProperties;
import stirling.software.SPDF.utils.UrlUtils;
@Slf4j
@EnableScheduling
@SpringBootApplication(
scanBasePackages = {"stirling.software.common", "stirling.software.SPDF"},
exclude = {
DataSourceAutoConfiguration.class,
DataSourceTransactionManagerAutoConfiguration.class
})
@SpringBootApplication
public class SPDFApplication {
private static String serverPortStatic;
@@ -43,8 +43,8 @@ import me.friwi.jcefmaven.MavenCefAppHandlerAdapter;
import me.friwi.jcefmaven.impl.progress.ConsoleProgressHandler;
import stirling.software.SPDF.UI.WebBrowser;
import stirling.software.common.configuration.InstallationPathConfig;
import stirling.software.common.util.UIScaling;
import stirling.software.SPDF.config.InstallationPathConfig;
import stirling.software.SPDF.utils.UIScaling;
@Component
@Slf4j
@@ -15,7 +15,7 @@ import io.github.pixee.security.BoundedLineReader;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.util.UIScaling;
import stirling.software.SPDF.utils.UIScaling;
@Slf4j
public class LoadingWindow extends JDialog {
@@ -1,4 +1,4 @@
package stirling.software.common.configuration;
package stirling.software.SPDF.config;
import java.io.IOException;
import java.nio.file.Files;
@@ -22,33 +22,20 @@ import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.thymeleaf.spring6.SpringTemplateEngine;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.SPDF.model.ApplicationProperties;
@Configuration
@Lazy
@Slf4j
@Configuration
@RequiredArgsConstructor
public class AppConfig {
private final Environment env;
private final ApplicationProperties applicationProperties;
@Getter
@Value("${baseUrl:http://localhost}")
private String baseUrl;
@Getter
@Value("${server.servlet.context-path:/}")
private String contextPath;
@Getter
@Value("${server.port:8080}")
private String serverPort;
private final Environment env;
@Bean
@ConditionalOnProperty(name = "system.customHTMLFiles", havingValue = "true")
@@ -211,35 +198,15 @@ public class AppConfig {
return applicationProperties.getAutomaticallyGenerated().getUUID();
}
@Bean
public ApplicationProperties.Security security() {
return applicationProperties.getSecurity();
}
@Bean
public ApplicationProperties.Security.OAUTH2 oAuth2() {
return applicationProperties.getSecurity().getOauth2();
}
@Bean
public ApplicationProperties.Premium premium() {
return applicationProperties.getPremium();
}
@Bean
public ApplicationProperties.System system() {
return applicationProperties.getSystem();
}
@Bean
public ApplicationProperties.Datasource datasource() {
return applicationProperties.getSystem().getDatasource();
}
@Bean(name = "disablePixel")
public boolean disablePixel() {
return Boolean.getBoolean(env.getProperty("DISABLE_PIXEL"));
}
@Bean(name = "showQueueStatus")
public boolean showQueueStatus() {
return applicationProperties.getUi().isQueueStatusEnabled();
}
@Bean(name = "machineType")
public String determineMachineType() {
@@ -6,7 +6,7 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Scope;
import stirling.software.SPDF.config.interfaces.ShowAdminInterface;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.SPDF.model.ApplicationProperties;
@Configuration
class AppUpdateService {
@@ -23,10 +23,7 @@ public class CleanUrlInterceptor implements HandlerInterceptor {
"errorOAuth",
"file",
"messageType",
"infoMessage",
"async",
"session",
"t");
"infoMessage");
@Override
public boolean preHandle(
@@ -0,0 +1,19 @@
package stirling.software.SPDF.config;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.springframework.context.annotation.Conditional;
/**
* Annotation to conditionally enable components based on the availability of UnoServer. Components
* annotated with this will only be created if UnoServer is available on the system.
*/
@Target({ElementType.TYPE, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
@Documented
@Conditional(UnoServerAvailableCondition.class)
public @interface ConditionalOnUnoServerAvailable {}
@@ -1,4 +1,4 @@
package stirling.software.common.configuration;
package stirling.software.SPDF.config;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -13,8 +13,6 @@ import java.util.List;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.util.YamlHelper;
/**
* A naive, line-based approach to merging "settings.yml" with "settings.yml.template" while
* preserving exact whitespace, blank lines, and inline comments -- but we only rewrite the file if
@@ -78,7 +76,7 @@ public class ConfigInitializer {
Path customSettingsPath = Paths.get(InstallationPathConfig.getCustomSettingsPath());
if (Files.notExists(customSettingsPath)) {
Files.createFile(customSettingsPath);
log.info("Created custom_settings file: {}", customSettingsPath);
log.info("Created custom_settings file: {}", customSettingsPath.toString());
}
}
@@ -11,7 +11,7 @@ import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.SPDF.model.ApplicationProperties;
@Service
@Slf4j
@@ -173,7 +173,6 @@ public class EndpointConfiguration {
addEndpointToGroup("Other", "get-info-on-pdf");
addEndpointToGroup("Other", "show-javascript");
addEndpointToGroup("Other", "remove-image-pdf");
addEndpointToGroup("Other", "scan-upload");
// CLI
addEndpointToGroup("CLI", "compress-pdf");
@@ -12,8 +12,6 @@ import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.configuration.RuntimePathConfig;
@Configuration
@Slf4j
public class ExternalAppDepConfig {
@@ -1,4 +1,4 @@
package stirling.software.common.configuration;
package stirling.software.SPDF.config;
import java.io.IOException;
import java.io.InputStream;
@@ -13,7 +13,7 @@ import org.thymeleaf.templateresource.ITemplateResource;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.InputStreamTemplateResource;
import stirling.software.SPDF.model.InputStreamTemplateResource;
@Slf4j
public class FileFallbackTemplateResolver extends AbstractConfigurableTemplateResolver {
@@ -17,8 +17,8 @@ import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.util.GeneralUtils;
import stirling.software.SPDF.model.ApplicationProperties;
import stirling.software.SPDF.utils.GeneralUtils;
@Component
@Slf4j
@@ -1,4 +1,4 @@
package stirling.software.common.configuration;
package stirling.software.SPDF.config;
import java.io.File;
import java.nio.file.Paths;
@@ -48,21 +48,24 @@ public class InstallationPathConfig {
String os = System.getProperty("os.name").toLowerCase();
if (os.contains("win")) {
return Paths.get(
System.getenv("APPDATA"), // parent path
"Stirling-PDF")
System.getenv("APPDATA"), // parent path
"Stirling-PDF")
.toString()
+ File.separator;
} else if (os.contains("mac")) {
return Paths.get(
System.getProperty("user.home"),
"Library",
"Application Support",
"Stirling-PDF")
System.getProperty("user.home"),
"Library",
"Application Support",
"Stirling-PDF")
.toString()
+ File.separator;
} else {
return Paths.get(
System.getProperty("user.home"), // parent path
".config",
"Stirling-PDF")
System.getProperty("user.home"), // parent path
".config",
"Stirling-PDF")
.toString()
+ File.separator;
}
}
@@ -12,7 +12,7 @@ import org.springframework.web.servlet.i18n.SessionLocaleResolver;
import lombok.RequiredArgsConstructor;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.SPDF.model.ApplicationProperties;
@Configuration
@RequiredArgsConstructor
@@ -1,7 +1,5 @@
package stirling.software.SPDF.config;
import stirling.software.common.configuration.InstallationPathConfig;
import ch.qos.logback.core.PropertyDefinerBase;
public class LogbackPropertyLoader extends PropertyDefinerBase {
@@ -16,7 +16,7 @@ import jakarta.servlet.http.HttpSession;
import lombok.RequiredArgsConstructor;
import stirling.software.common.util.RequestUriUtils;
import stirling.software.SPDF.utils.RequestUriUtils;
@Component
@RequiredArgsConstructor
@@ -13,7 +13,7 @@ import io.swagger.v3.oas.models.security.SecurityScheme;
import lombok.RequiredArgsConstructor;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.SPDF.model.ApplicationProperties;
@Configuration
@RequiredArgsConstructor
@@ -1,4 +1,4 @@
package stirling.software.common.configuration;
package stirling.software.SPDF.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
@@ -1,4 +1,4 @@
package stirling.software.common.configuration;
package stirling.software.SPDF.config;
import org.springframework.stereotype.Component;
@@ -1,4 +1,4 @@
package stirling.software.common.configuration;
package stirling.software.SPDF.config;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -9,9 +9,9 @@ import org.springframework.context.annotation.Configuration;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.ApplicationProperties.CustomPaths.Operations;
import stirling.software.common.model.ApplicationProperties.CustomPaths.Pipeline;
import stirling.software.SPDF.model.ApplicationProperties;
import stirling.software.SPDF.model.ApplicationProperties.CustomPaths.Operations;
import stirling.software.SPDF.model.ApplicationProperties.CustomPaths.Pipeline;
@Slf4j
@Configuration
@@ -0,0 +1,193 @@
package stirling.software.SPDF.config;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import org.springframework.context.annotation.Condition;
import org.springframework.context.annotation.ConditionContext;
import org.springframework.core.type.AnnotatedTypeMetadata;
import lombok.extern.slf4j.Slf4j;
/**
* Condition that checks if UnoServer is available on the system. This condition will pass if: 1.
* The unoserver executable is found via RuntimePathConfig (from unoConvertPath) 2. The unoserver
* executable is found at /opt/venv/bin/unoserver (Docker path) 3. The unoserver executable is found
* in PATH 4. The unoserver executable is found in any common installation directories
*/
@Slf4j
public class UnoServerAvailableCondition implements Condition {
// Common installation paths to check
private static final String[] COMMON_UNOSERVER_PATHS = {
"/opt/venv/bin/unoserver", // Docker path
"/usr/bin/unoserver", // Linux system path
"/usr/local/bin/unoserver", // Linux local path
"/opt/homebrew/bin/unoserver", // Mac Homebrew path
"/opt/libreoffice/program/unoserver" // Custom LibreOffice path
};
@Override
public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
log.info("Checking if UnoServer is available on the system...");
// Collect all paths to check
List<String> pathsToCheck = new ArrayList<>();
// Check for Docker environment
boolean isDocker = Files.exists(Path.of("/.dockerenv"));
log.debug("Docker environment detected: {}", isDocker);
// Try to get unoserver path from RuntimePathConfig first (highest priority)
String unoserverFromRuntimeConfig = getUnoServerPathFromRuntimeConfig(context);
if (unoserverFromRuntimeConfig != null) {
pathsToCheck.add(unoserverFromRuntimeConfig);
}
// Add common installation paths
for (String path : COMMON_UNOSERVER_PATHS) {
pathsToCheck.add(path);
}
// Add "unoserver" to check in PATH
pathsToCheck.add("unoserver");
// Try all paths one by one
for (String path : pathsToCheck) {
log.debug("Checking for UnoServer at: {}", path);
if (isExecutableAvailable(path)) {
log.info("UnoServer found at: {}, enabling UnoServerManager", path);
return true;
}
}
// If we get here, we didn't find unoserver anywhere
log.warn(
"UnoServer not found in any of the expected locations. UnoServerManager will be disabled.");
log.info(
"To enable Office document conversions, please install UnoServer or use the 'fat' Docker image variant.");
return false;
}
/**
* Attempts to get the unoserver path from RuntimePathConfig by checking the parent directory of
* unoConvertPath.
*
* @param context The condition context
* @return The unoserver path if found, null otherwise
*/
private String getUnoServerPathFromRuntimeConfig(ConditionContext context) {
try {
RuntimePathConfig runtimePathConfig =
context.getBeanFactory().getBean(RuntimePathConfig.class);
if (runtimePathConfig != null) {
String unoConvertPath = runtimePathConfig.getUnoConvertPath();
log.debug("UnoConvert path from RuntimePathConfig: {}", unoConvertPath);
if (unoConvertPath != null && !unoConvertPath.isEmpty()) {
// First check if unoConvertPath itself exists
File unoConvertFile = new File(unoConvertPath);
if (!unoConvertFile.exists() || !unoConvertFile.canExecute()) {
log.info("UnoConvert not found at path: {}", unoConvertPath);
return null;
}
// If unoConvertPath exists, check for unoserver in the same directory
Path unoConvertDir = Paths.get(unoConvertPath).getParent();
if (unoConvertDir != null) {
Path potentialUnoServerPath = unoConvertDir.resolve("unoserver");
File unoServerFile = potentialUnoServerPath.toFile();
if (unoServerFile.exists() && unoServerFile.canExecute()) {
log.debug("UnoServer found at: {}", potentialUnoServerPath);
return potentialUnoServerPath.toString();
} else {
log.debug(
"UnoServer not found at expected path: {}",
potentialUnoServerPath);
// Continue checking other paths
}
}
}
}
} catch (Exception e) {
log.debug(
"RuntimePathConfig not available yet, falling back to default checks: {}",
e.getMessage());
}
return null;
}
/**
* Comprehensive check if an executable is available in the system
*
* @param executableName The name or path of the executable to check
* @return true if the executable is found and executable, false otherwise
*/
private boolean isExecutableAvailable(String executableName) {
// First, check if it's an absolute path and the file exists
if (executableName.startsWith("/") || executableName.contains(":\\")) {
File file = new File(executableName);
boolean exists = file.exists() && file.canExecute();
log.debug(
"Checking executable at absolute path {}: {}",
executableName,
exists ? "Found" : "Not found");
return exists;
}
// Next, try to execute the command with --version to verify it works
try {
ProcessBuilder pb = new ProcessBuilder(executableName, "--version");
pb.redirectError(ProcessBuilder.Redirect.DISCARD);
Process process = pb.start();
int exitCode = process.waitFor();
if (exitCode == 0) {
log.debug("Executable {} exists in PATH (--version returned 0)", executableName);
return true;
} else {
// Try with --help as a fallback
pb = new ProcessBuilder(executableName, "--help");
pb.redirectError(ProcessBuilder.Redirect.DISCARD);
process = pb.start();
exitCode = process.waitFor();
if (exitCode == 0) {
log.debug("Executable {} exists in PATH (--help returned 0)", executableName);
return true;
}
}
} catch (Exception e) {
log.debug("Error checking for executable {}: {}", executableName, e.getMessage());
}
// Finally, check each directory in PATH for the executable file
if (!executableName.contains("/") && !executableName.contains("\\")) {
String pathEnv = System.getenv("PATH");
if (pathEnv != null) {
String[] pathDirs = pathEnv.split(File.pathSeparator);
for (String pathDir : pathDirs) {
File file = new File(pathDir, executableName);
if (file.exists() && file.canExecute()) {
log.debug(
"Found executable {} in PATH directory {}",
executableName,
pathDir);
return true;
}
}
}
}
log.debug("Executable {} not found", executableName);
return false;
}
}
@@ -0,0 +1,887 @@
package stirling.software.SPDF.config;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import io.github.pixee.security.SystemCommand;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.model.ApplicationProperties;
import stirling.software.SPDF.utils.ConversionTask;
/**
* UnoServerManager is responsible for managing multiple instances of unoserver based on application
* configuration.
*
* <p>This component is only created if UnoServer is available on the system.
*/
@Slf4j
@Component
@ConditionalOnUnoServerAvailable
public class UnoServerManager {
private static final int INSTANCE_CHECK_TIMEOUT_MS = 1000;
private static final long INSTANCE_STARTUP_TIMEOUT_MS = 30000;
private static final long HEALTH_CHECK_INTERVAL_MS = 60000; // Health check every minute
@Autowired private ApplicationProperties properties;
@Autowired private RuntimePathConfig runtimePathConfig;
@Getter private List<ServerInstance> instances = new ArrayList<>();
private AtomicInteger currentInstanceIndex = new AtomicInteger(0);
private ScheduledExecutorService healthCheckExecutor;
// The path to the UnoServer executable that was found during initialization
private String detectedUnoServerPath;
// Circuit breaker settings for external servers
private static final int FAILURE_THRESHOLD = 3; // Number of failures before circuit opens
private static final long CIRCUIT_RESET_TIME_MS = 30000; // Time before retrying a failed server
// Performance metrics
private final AtomicInteger totalConversions = new AtomicInteger(0);
private final AtomicInteger failedConversions = new AtomicInteger(0);
private final Map<Integer, AtomicInteger> conversionsPerInstance = new ConcurrentHashMap<>();
// Queue tracking
private final ConcurrentHashMap<String, ConversionTask> activeTasks = new ConcurrentHashMap<>();
private final AtomicInteger taskIdCounter = new AtomicInteger(0);
private final Map<Integer, AtomicInteger> activeTasksPerInstance = new ConcurrentHashMap<>();
@PostConstruct
public void initialize() {
try {
int maxInstances =
properties.getProcessExecutor().getSessionLimit().getLibreOfficeSessionLimit();
boolean useExternal = properties.getProcessExecutor().isUseExternalUnoconvServers();
List<String> externalServers = properties.getProcessExecutor().getUnoconvServers();
int basePort = properties.getProcessExecutor().getBaseUnoconvPort();
boolean manageUnoServer = properties.getProcessExecutor().isManageUnoServer();
log.info(
"Initializing UnoServerManager with maxInstances: {}, useExternal: {}, externalServers: {}, unoConvertPath: {}, manageUnoServer: {}",
maxInstances,
useExternal,
externalServers,
runtimePathConfig.getUnoConvertPath(),
manageUnoServer);
// Get valid UnoServer executable path
String unoServerPath = findValidUnoServerPath();
if (unoServerPath == null) {
log.warn("UnoServer executable not found. Office conversions will be disabled.");
return;
}
log.info("Using UnoServer at: {}", unoServerPath);
// Store the path for use by server instances
this.detectedUnoServerPath = unoServerPath;
if (useExternal && !externalServers.isEmpty()) {
// Configure for external servers
for (String serverAddress : externalServers) {
String[] parts = serverAddress.split(":");
String host = parts[0];
int port = parts.length > 1 ? Integer.parseInt(parts[1]) : basePort;
ServerInstance instance = new ServerInstance(host, port, false);
instances.add(instance);
conversionsPerInstance.put(instances.size() - 1, new AtomicInteger(0));
activeTasksPerInstance.put(instances.size() - 1, new AtomicInteger(0));
}
log.info("Configured {} external UnoServer instances", instances.size());
} else if (manageUnoServer) {
// Configure for local instances only if manageUnoServer is true
boolean anyInstanceStarted = false;
for (int i = 0; i < maxInstances; i++) {
int port = basePort + i;
ServerInstance instance = new ServerInstance("127.0.0.1", port, true);
instances.add(instance);
conversionsPerInstance.put(i, new AtomicInteger(0));
activeTasksPerInstance.put(i, new AtomicInteger(0));
try {
instance.start();
anyInstanceStarted = true;
} catch (IOException e) {
log.warn(
"Failed to start UnoServer instance on port {}: {}",
port,
e.getMessage());
}
}
if (!anyInstanceStarted) {
log.warn(
"Failed to start any UnoServer instances. Office conversions may be affected.");
}
log.info("Started {} local UnoServer instances", instances.size());
} else {
log.info(
"Application is configured to not manage UnoServer instances. Assuming external management.");
}
// Start the health check scheduler
startHealthCheck();
// Log initial health status
logHealthStatus();
} catch (Exception e) {
log.warn("Failed to initialize UnoServerManager: {}", e.getMessage(), e);
}
}
/**
* Scans multiple locations to find a valid UnoServer executable
*
* @return Path to UnoServer if found, null otherwise
*/
private String findValidUnoServerPath() {
// Common paths to check for UnoServer
List<String> pathsToCheck = new ArrayList<>();
// Try to derive the path from unoConvertPath first (highest priority)
String unoConvertPath = runtimePathConfig.getUnoConvertPath();
if (unoConvertPath != null && !unoConvertPath.isEmpty()) {
File unoConvertFile = new File(unoConvertPath);
if (unoConvertFile.exists() && unoConvertFile.canExecute()) {
Path unoConvertDir = Paths.get(unoConvertPath).getParent();
if (unoConvertDir != null) {
Path potentialUnoServerPath = unoConvertDir.resolve("unoserver");
pathsToCheck.add(potentialUnoServerPath.toString());
}
} else {
log.warn("UnoConvert not found at configured path: {}", unoConvertPath);
}
}
// Add common installation paths
pathsToCheck.add("/opt/venv/bin/unoserver"); // Docker path
pathsToCheck.add("/usr/bin/unoserver"); // Linux system path
pathsToCheck.add("/usr/local/bin/unoserver"); // Linux local path
pathsToCheck.add("/opt/homebrew/bin/unoserver"); // Mac Homebrew path
pathsToCheck.add("/opt/libreoffice/program/unoserver"); // Custom LibreOffice path
// Check each path
for (String path : pathsToCheck) {
File file = new File(path);
if (file.exists() && file.canExecute()) {
log.info("Found valid UnoServer at: {}", path);
return path;
}
}
// If no absolute path works, try to find it in PATH
String pathEnv = System.getenv("PATH");
if (pathEnv != null) {
String[] pathDirs = pathEnv.split(File.pathSeparator);
for (String pathDir : pathDirs) {
File file = new File(pathDir, "unoserver");
if (file.exists() && file.canExecute()) {
log.info("Found UnoServer in PATH at: {}", file.getAbsolutePath());
return file.getAbsolutePath();
}
}
}
log.warn("UnoServer executable not found in any standard location");
return null;
}
/** Starts periodic health checks for all instances */
private void startHealthCheck() {
healthCheckExecutor = Executors.newSingleThreadScheduledExecutor();
healthCheckExecutor.scheduleAtFixedRate(
this::performHealthCheck,
HEALTH_CHECK_INTERVAL_MS,
HEALTH_CHECK_INTERVAL_MS,
TimeUnit.MILLISECONDS);
}
/** Perform health check on all instances */
private void performHealthCheck() {
log.debug("Running UnoServer health check for {} instances", instances.size());
int healthy = 0;
int unhealthy = 0;
for (int i = 0; i < instances.size(); i++) {
ServerInstance instance = instances.get(i);
boolean isRunning = instance.isRunning();
if (isRunning) {
healthy++;
instance.resetFailureCount(); // Reset failure count for healthy instance
} else {
unhealthy++;
log.warn(
"UnoServer instance {}:{} is not running",
instance.getHost(),
instance.getPort());
// For managed instances, try to restart if needed
if (instance.isManaged()) {
try {
instance.restartIfNeeded();
} catch (Exception e) {
log.error(
"Failed to restart UnoServer instance {}:{}",
instance.getHost(),
instance.getPort(),
e);
}
}
}
}
log.info("UnoServer health check: {} healthy, {} unhealthy instances", healthy, unhealthy);
// Log metrics periodically
logMetrics();
}
/** Logs the current health status of all instances */
private void logHealthStatus() {
StringBuilder status = new StringBuilder("UnoServer Instances Status:\n");
for (int i = 0; i < instances.size(); i++) {
ServerInstance instance = instances.get(i);
boolean isRunning = instance.isRunning();
int convCount = conversionsPerInstance.get(i).get();
status.append(
String.format(
" [%d] %s:%d - Status: %s, Managed: %s, Conversions: %d, Failures: %d\n",
i,
instance.getHost(),
instance.getPort(),
isRunning ? "RUNNING" : "DOWN",
instance.isManaged() ? "YES" : "NO",
convCount,
instance.getFailureCount()));
}
log.info(status.toString());
}
/** Logs performance metrics for UnoServer conversions */
private void logMetrics() {
int total = totalConversions.get();
int failed = failedConversions.get();
float successRate = total > 0 ? (float) (total - failed) / total * 100 : 0;
log.info(
"UnoServer metrics - Total: {}, Failed: {}, Success Rate: {:.2f}%",
total, failed, successRate);
// Log per-instance metrics
StringBuilder instanceMetrics = new StringBuilder("Conversions per instance:\n");
for (int i = 0; i < instances.size(); i++) {
ServerInstance instance = instances.get(i);
int count = conversionsPerInstance.get(i).get();
float percentage = total > 0 ? (float) count / total * 100 : 0;
instanceMetrics.append(
String.format(
" [%d] %s:%d - Count: %d (%.2f%%), Avg Time: %.2fms\n",
i,
instance.getHost(),
instance.getPort(),
count,
percentage,
instance.getAverageConversionTime()));
}
log.info(instanceMetrics.toString());
}
@PreDestroy
public void cleanup() {
log.info("Shutting down UnoServer instances and health check scheduler");
// Shutdown health check scheduler
if (healthCheckExecutor != null) {
healthCheckExecutor.shutdownNow();
}
// Shutdown all instances
for (ServerInstance instance : instances) {
instance.stop();
}
// Log final metrics
logMetrics();
}
/**
* Gets the next available server instance using load-balancing and circuit breaker pattern for
* fault tolerance
*
* @return The next UnoServer instance to use
*/
public ServerInstance getNextInstance() {
if (instances.isEmpty()) {
throw new IllegalStateException("No UnoServer instances available");
}
// First try to find a healthy instance with the least active tasks
int minActiveTasks = Integer.MAX_VALUE;
int selectedIndex = -1;
for (int i = 0; i < instances.size(); i++) {
ServerInstance instance = instances.get(i);
// Check if instance is available (not in circuit-open state)
if (instance.isAvailable() && instance.isRunning()) {
int activeTasks = activeTasksPerInstance.get(i).get();
// If this instance has fewer active tasks, select it
if (activeTasks < minActiveTasks) {
minActiveTasks = activeTasks;
selectedIndex = i;
// If we found an instance with no active tasks, use it immediately
if (minActiveTasks == 0) {
break;
}
}
}
}
// If we found a suitable instance, use it
if (selectedIndex >= 0) {
ServerInstance instance = instances.get(selectedIndex);
// Track this instance being selected
conversionsPerInstance.get(selectedIndex).incrementAndGet();
activeTasksPerInstance.get(selectedIndex).incrementAndGet();
totalConversions.incrementAndGet();
log.debug(
"Selected UnoServer instance {}:{} with {} active tasks",
instance.getHost(),
instance.getPort(),
minActiveTasks);
return instance;
}
// If all healthy instances are busy or no healthy instances found, use round-robin as
// fallback
log.warn(
"No available UnoServer instances found with good health. Using round-robin fallback.");
// Try to find any available instance using round-robin
for (int attempt = 0; attempt < instances.size(); attempt++) {
int index = currentInstanceIndex.getAndIncrement() % instances.size();
ServerInstance instance = instances.get(index);
// Check if the instance is available (circuit closed)
if (instance.isAvailable()) {
// Track this instance being selected
conversionsPerInstance.get(index).incrementAndGet();
activeTasksPerInstance.get(index).incrementAndGet();
totalConversions.incrementAndGet();
log.debug(
"Selected UnoServer instance {}:{} using round-robin fallback",
instance.getHost(),
instance.getPort());
return instance;
}
}
// Last resort - if all circuits are open, use the next instance anyway
int index = currentInstanceIndex.get() % instances.size();
ServerInstance instance = instances.get(index);
log.warn(
"All UnoServer instances are in circuit-open state. Using instance at {}:{} as fallback.",
instance.getHost(),
instance.getPort());
// Track metrics even for fallback case
conversionsPerInstance.get(index).incrementAndGet();
activeTasksPerInstance.get(index).incrementAndGet();
totalConversions.incrementAndGet();
return instance;
}
/**
* Creates a new task for tracking office conversions
*
* @param taskName A descriptive name for the task
* @param instance The server instance that will handle this task
* @return A unique task ID for tracking
*/
public String createTask(String taskName, ServerInstance instance) {
String taskId = "office-" + taskIdCounter.incrementAndGet();
ConversionTask task = new ConversionTask(taskName, taskId);
// Calculate queue position based on number of active tasks across all instances
int runningTasks = 0;
int availableInstances = 0;
for (int i = 0; i < instances.size(); i++) {
if (instances.get(i).isRunning() && instances.get(i).isAvailable()) {
availableInstances++;
runningTasks += activeTasksPerInstance.get(i).get();
}
}
// If all instances are busy, set a queue position
if (runningTasks >= availableInstances && availableInstances > 0) {
int queuePosition = runningTasks - availableInstances + 1;
task.setQueuePosition(queuePosition);
}
// Store the task in our tracking map
activeTasks.put(taskId, task);
// Find the instance index for updating metrics
for (int i = 0; i < instances.size(); i++) {
if (instances.get(i) == instance) {
activeTasksPerInstance.get(i).incrementAndGet();
break;
}
}
log.debug("Created task {} with ID {}", taskName, taskId);
return taskId;
}
/**
* Completes a task, updating metrics and removing it from active tasks
*
* @param taskId The task ID to complete
* @param instance The server instance that handled this task
* @param durationMs The time taken to complete the task in milliseconds
*/
public void completeTask(String taskId, ServerInstance instance, long durationMs) {
ConversionTask task = activeTasks.remove(taskId);
if (task != null) {
task.complete();
}
// Find the instance index for updating metrics
for (int i = 0; i < instances.size(); i++) {
if (instances.get(i) == instance) {
activeTasksPerInstance.get(i).decrementAndGet();
break;
}
}
// Record the success for circuit breaker and metrics
recordSuccess(instance, durationMs);
log.debug("Completed task with ID {}, duration: {}ms", taskId, durationMs);
}
/**
* Fails a task, updating metrics and removing it from active tasks
*
* @param taskId The task ID to fail
* @param instance The server instance that handled this task
* @param errorMessage The error message explaining the failure
*/
public void failTask(String taskId, ServerInstance instance, String errorMessage) {
ConversionTask task = activeTasks.remove(taskId);
if (task != null) {
task.fail(errorMessage);
}
// Find the instance index for updating metrics
for (int i = 0; i < instances.size(); i++) {
if (instances.get(i) == instance) {
activeTasksPerInstance.get(i).decrementAndGet();
break;
}
}
// Record the failure for circuit breaker and metrics
recordFailure(instance);
log.warn("Failed task with ID {}: {}", taskId, errorMessage);
}
/**
* Gets all active conversion tasks
*
* @return A map of task IDs to ConversionTask objects
*/
public Map<String, ConversionTask> getActiveTasks() {
return new HashMap<>(activeTasks);
}
/**
* Records a successful conversion for metrics
*
* @param instance The server instance that succeeded
* @param durationMs The time taken for the conversion in milliseconds
*/
public void recordSuccess(ServerInstance instance, long durationMs) {
instance.recordSuccess(durationMs);
}
/**
* Records a failed conversion for metrics and circuit breaker
*
* @param instance The server instance that failed
*/
public void recordFailure(ServerInstance instance) {
failedConversions.incrementAndGet();
instance.recordFailure();
}
/** Represents a single UnoServer instance with circuit breaker functionality */
public class ServerInstance {
@Getter private final String host;
@Getter private final int port;
@Getter private final boolean managed;
private ExecutorService executorService;
private Process process;
private boolean running = false;
// Circuit breaker state
private final AtomicInteger failureCount = new AtomicInteger(0);
private volatile Instant lastFailureTime = null;
private volatile boolean circuitOpen = false;
// Performance metrics
private final AtomicLong totalConversionTimeMs = new AtomicLong(0);
private final AtomicInteger conversionCount = new AtomicInteger(0);
private final AtomicLong lastConversionDuration = new AtomicLong(0);
public ServerInstance(String host, int port, boolean managed) {
this.host = host;
this.port = port;
this.managed = managed;
if (!managed) {
// For external servers, we assume they're running initially
this.running = true;
}
}
/** Gets the number of failures for circuit breaker */
public int getFailureCount() {
return failureCount.get();
}
/** Resets the failure count for circuit breaker */
public void resetFailureCount() {
failureCount.set(0);
circuitOpen = false;
}
/**
* Records a successful conversion
*
* @param durationMs The duration of the conversion in milliseconds
*/
public void recordSuccess(long durationMs) {
conversionCount.incrementAndGet();
totalConversionTimeMs.addAndGet(durationMs);
lastConversionDuration.set(durationMs);
// Reset failure count on success
resetFailureCount();
}
/** Records a conversion failure */
public void recordFailure() {
int currentFailures = failureCount.incrementAndGet();
lastFailureTime = Instant.now();
// Open circuit if threshold reached
if (currentFailures >= FAILURE_THRESHOLD) {
log.warn(
"Circuit breaker opened for UnoServer instance {}:{} after {} failures",
host,
port,
currentFailures);
circuitOpen = true;
}
}
/**
* Checks if this instance is available based on circuit breaker status
*
* @return true if available, false if circuit is open
*/
public boolean isAvailable() {
// If circuit is closed, instance is available
if (!circuitOpen) {
return true;
}
// If circuit is open but reset time has passed, try half-open state
if (lastFailureTime != null
&& Duration.between(lastFailureTime, Instant.now()).toMillis()
> CIRCUIT_RESET_TIME_MS) {
log.info("Circuit breaker half-open for UnoServer instance {}:{}", host, port);
circuitOpen = false;
return true;
}
// Circuit is open
return false;
}
/**
* Gets the average conversion time in milliseconds
*
* @return The average conversion time or 0 if no conversions yet
*/
public double getAverageConversionTime() {
int count = conversionCount.get();
return count > 0 ? (double) totalConversionTimeMs.get() / count : 0;
}
/**
* Gets the last conversion duration in milliseconds
*
* @return The last conversion duration
*/
public long getLastConversionDuration() {
return lastConversionDuration.get();
}
/**
* Checks if the UnoServer instance is running
*
* @return true if the server is accessible, false otherwise
*/
public boolean isRunning() {
try (Socket socket = new Socket()) {
socket.connect(new InetSocketAddress(host, port), INSTANCE_CHECK_TIMEOUT_MS);
return true;
} catch (Exception e) {
return false;
}
}
/**
* Starts the UnoServer if it's a managed instance
*
* @throws IOException if the server fails to start
*/
public synchronized void start() throws IOException {
if (!managed
|| (process != null && process.isAlive())
|| !properties.getProcessExecutor().isManageUnoServer()) {
return;
}
log.info("Starting UnoServer on {}:{}", host, port);
try {
// Use the detected UnoServer path from parent class
String unoServerPath = UnoServerManager.this.detectedUnoServerPath;
// If not available (shouldn't happen), try to determine it
if (unoServerPath == null || unoServerPath.isEmpty()) {
log.warn(
"detectedUnoServerPath is null, attempting to find unoserver executable");
unoServerPath = findUnoServerExecutable();
if (unoServerPath == null) {
throw new IOException(
"UnoServer executable not found. Cannot start server instance.");
}
}
log.debug("Using UnoServer executable: {}", unoServerPath);
// Create the command with the correct path and options
String command =
String.format("%s --port %d --interface %s", unoServerPath, port, host);
// Final verification that the executable exists and is executable
File executableFile = new File(unoServerPath);
if (!executableFile.exists() || !executableFile.canExecute()) {
throw new IOException(
"UnoServer executable not found or not executable at: "
+ executableFile.getAbsolutePath());
}
// Run the command
log.debug("Executing command: {}", command);
process = SystemCommand.runCommand(Runtime.getRuntime(), command);
// Start a background thread to monitor the process
executorService = Executors.newSingleThreadExecutor();
executorService.submit(
() -> {
try {
int exitCode = process.waitFor();
log.info(
"UnoServer process on port {} exited with code {}",
port,
exitCode);
running = false;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.warn("UnoServer monitoring thread was interrupted", e);
}
});
// Wait for the server to start up with timeout
long startTime = System.currentTimeMillis();
boolean startupSuccess = false;
while (System.currentTimeMillis() - startTime < INSTANCE_STARTUP_TIMEOUT_MS) {
if (isRunning()) {
running = true;
startupSuccess = true;
log.info("UnoServer started successfully on {}:{}", host, port);
break;
}
// Check if process is still alive
if (process == null || !process.isAlive()) {
int exitCode = process != null ? process.exitValue() : -1;
log.warn(
"UnoServer process terminated prematurely with exit code: {}, continuing without it",
exitCode);
return;
}
try {
Thread.sleep(1000); // Check every second
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.warn("Interrupted while waiting for UnoServer to start", e);
return;
}
}
if (!startupSuccess) {
// Timeout occurred, clean up and log warning
if (process != null && process.isAlive()) {
process.destroy();
}
log.warn(
"Failed to start UnoServer within timeout period of {} seconds, continuing without it",
(INSTANCE_STARTUP_TIMEOUT_MS / 1000));
}
} catch (IOException e) {
log.warn("Failed to start UnoServer: {}, continuing without it", e.getMessage());
// Don't rethrow - continue without the server
}
}
/**
* Helper method to find the UnoServer executable
*
* @return Path to UnoServer executable or null if not found
*/
private String findUnoServerExecutable() {
// Try to derive from unoConvertPath first
String unoConvertPath = UnoServerManager.this.runtimePathConfig.getUnoConvertPath();
if (unoConvertPath != null && !unoConvertPath.isEmpty()) {
Path unoConvertDir = Paths.get(unoConvertPath).getParent();
if (unoConvertDir != null) {
Path potentialUnoServerPath = unoConvertDir.resolve("unoserver");
File unoServerFile = potentialUnoServerPath.toFile();
if (unoServerFile.exists() && unoServerFile.canExecute()) {
return potentialUnoServerPath.toString();
}
}
}
// Check common paths
String[] commonPaths = {
"/opt/venv/bin/unoserver", "/usr/bin/unoserver", "/usr/local/bin/unoserver"
};
for (String path : commonPaths) {
File file = new File(path);
if (file.exists() && file.canExecute()) {
return path;
}
}
return null;
}
/** Stops the UnoServer if it's a managed instance */
public synchronized void stop() {
if (!managed) {
return;
}
// Stop the monitoring thread
if (executorService != null) {
executorService.shutdownNow();
}
// Stop the server process
if (process != null && process.isAlive()) {
log.info("Stopping UnoServer on port {}", port);
process.destroy();
}
running = false;
}
/**
* Restarts the UnoServer if it's a managed instance and not running
*
* @return true if restart succeeded or wasn't needed, false otherwise
*/
public synchronized boolean restartIfNeeded() {
if (!managed || running || !properties.getProcessExecutor().isManageUnoServer()) {
return true;
}
try {
log.info("Attempting to restart UnoServer on {}:{}", host, port);
start();
return true;
} catch (IOException e) {
log.warn("Failed to restart UnoServer on port {}, continuing without it", port, e);
return false;
}
}
/**
* Gets the connection string for this instance
*
* @return A connection string in the format host:port
*/
public String getConnectionString() {
return host + ":" + port;
}
}
}
@@ -0,0 +1,69 @@
package stirling.software.SPDF.config;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.utils.ConversionTask;
/**
* Fallback configuration for when UnoServerManager is not available. This will provide friendly
* error messages when users try to use LibreOffice conversion features without having UnoServer
* installed.
*/
@Configuration
@Slf4j
public class UnoServerManagerFallback {
/**
* Creates a bean that provides a friendly error message when LibreOffice conversion is
* attempted but UnoServer is not available.
*/
@Bean
@ConditionalOnMissingBean(UnoServerManager.class)
public UnoServerNotAvailableHandler unoServerNotAvailableHandler(
RuntimePathConfig runtimePathConfig) {
log.info("UnoServer is not available. Office document conversions will be disabled.");
log.info("If you need Office document conversions, please install UnoServer.");
log.info("For Docker users, use the 'fat' image variant which includes UnoServer.");
// Log the path where we would expect to find unoconvert
if (runtimePathConfig != null) {
log.info("Expected unoconvert path: {}", runtimePathConfig.getUnoConvertPath());
}
return new UnoServerNotAvailableHandler();
}
/**
* Handler that provides friendly error messages when LibreOffice conversion is attempted but
* UnoServer is not available.
*/
public static class UnoServerNotAvailableHandler {
/** Method that throws a friendly exception when office conversions are attempted. */
public void throwUnoServerNotAvailableException() {
throw new UnoServerNotAvailableException(
"UnoServer (LibreOffice) is not available. Office document conversions are disabled. "
+ "To enable this feature, please install UnoServer or use the 'fat' Docker image variant.");
}
/** Creates a failed conversion task with a friendly error message. */
public ConversionTask createFailedTask(String taskName) {
ConversionTask task = new ConversionTask(taskName, (String) null);
task.fail(
"UnoServer (LibreOffice) is not available. Office document conversions are disabled.");
return task;
}
}
/** Exception thrown when UnoServer features are used but UnoServer is not available. */
public static class UnoServerNotAvailableException extends RuntimeException {
private static final long serialVersionUID = 1L;
public UnoServerNotAvailableException(String message) {
super(message);
}
}
}
@@ -7,8 +7,6 @@ import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
import lombok.RequiredArgsConstructor;
import stirling.software.common.configuration.InstallationPathConfig;
@Configuration
@RequiredArgsConstructor
public class WebMvcConfig implements WebMvcConfigurer {
@@ -1,4 +1,4 @@
package stirling.software.common.util;
package stirling.software.SPDF.config;
import java.io.IOException;
import java.io.StringWriter;

Some files were not shown because too many files have changed in this diff Show More