Files
Stirling-PDF/.github/workflows/build-enterprise.yml
T
James Brunton f881828cd8 Fix intermittently failing Playwright tests (#6886)
# Description of Changes
Fixes intermittently failing tests (and replaces one that wasn't useful
in its previous state) and also adds a CI check to warn if there are any
Playwright tests which failed on their first go and succeeded on
retries, to hopefully help find intermittently failing tests more
quickly and avoid them being merged in the first place.
2026-07-06 21:37:21 +00:00

311 lines
15 KiB
YAML

name: Enterprise E2E (Playwright)
# Enterprise Playwright suite — exercises premium-key gated features (audit,
# teams, analytics) plus full OAuth + SAML logins via the Keycloak compose
# stacks under testing/compose. Slow and secret-gated, so it runs in four
# situations:
#
# - PRs that touch proprietary / premium / SSO compose / enterprise tests
# (driven by build.yml via workflow_call, path-filtered against
# .github/config/.files.yaml `proprietary`),
# - every push to main (post-merge safety net),
# - on a nightly cron schedule (catches Keycloak image drift, license
# expiry, upstream proprietary changes),
# - manual workflow_dispatch.
on:
workflow_call:
inputs:
depot_cores:
description: "Depot runner vCPU count (used in runs-on). Override for benchmarking."
required: false
type: string
default: "8"
push:
branches: ["main"]
schedule:
- cron: "0 4 * * *"
workflow_dispatch:
inputs:
depot_cores:
description: "Depot runner vCPU count (used in runs-on). Override for benchmarking."
required: false
type: string
default: "8"
# No `concurrency:` block here on purpose. When this workflow is called via
# workflow_call from build.yml, ${{ github.workflow }}/event_name/pr_number
# resolve to the *caller's* values, producing the same group key as build.yml
# and causing the workflow_call instantiation to self-cancel — the job
# silently fails to spawn while `${{ needs.X.result }}` still reports
# `failure`. Standalone runs (push-to-main, nightly cron, dispatch) don't
# overlap often enough to need explicit concurrency control.
permissions:
contents: read
jobs:
pick:
uses: ./.github/workflows/_runner-pick.yml
playwright-e2e-enterprise:
needs: pick
# Skip on fork PRs / untrusted authors: they have no PREMIUM_KEY_ENTERPRISE
# (nor DEPOT_TOKEN), so the suite can't boot premium and would fail. See the
# header comment. GitHub reports the skipped reusable workflow as success.
if: needs.pick.outputs.is_fork != 'true'
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || format('depot-ubuntu-24.04-{0}', inputs.depot_cores || '8') }}
timeout-minutes: 45
env:
PREMIUM_KEY: ${{ secrets.PREMIUM_KEY_ENTERPRISE }}
PREMIUM_ENABLED: "true"
SYSTEM_ENABLEANALYTICS: "false"
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up JDK 25
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
with:
java-version: "25"
distribution: "temurin"
- name: Set up Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: "22"
cache: "npm"
cache-dependency-path: frontend/package-lock.json
- name: Install Task
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
- name: Install Playwright (chromium only)
run: task e2e:install -- chromium
- name: Build frontend (needed for playwright's vite preview webServer)
# Enterprise tests target :8080 (Spring Boot's bundled frontend), but
# playwright's webServer config still launches `vite preview --port 5173`
# before any test run, which needs dist/ to exist. VITE_BUILD_FOR_PREVIEW
# forces absolute asset paths so vite preview can serve deep SPA routes.
env:
VITE_BUILD_FOR_PREVIEW: "1"
run: task frontend:build
- name: Resolve kubernetes.docker.internal to localhost
# The compose stacks set KC_HOSTNAME=kubernetes.docker.internal so
# Keycloak issues redirect URIs against that host. Docker Desktop
# auto-resolves it; GHA runners don't. Map it to 127.0.0.1 so the
# browser-driven OAuth flow lands back on Stirling-PDF correctly.
run: |
echo "127.0.0.1 kubernetes.docker.internal" | sudo tee -a /etc/hosts
# Helper function used by all phases — boots `:stirling-pdf:bootRun`
# with the React frontend baked in (-PbuildWithFrontend=true) so the
# SPA serves on :8080 and OAuth/SAML callbacks land on the same host
# that the browser is interacting with.
- name: Define helpers
run: |
{
echo 'wait_for_backend() {'
echo ' start=$SECONDS'
echo ' for i in $(seq 1 300); do'
echo ' if curl -fsS http://localhost:8080/api/v1/info/status >/dev/null 2>&1; then'
echo ' echo "Backend up after $((SECONDS - start))s"; return 0'
echo ' fi; sleep 2'
echo ' done'
echo ' tail -200 /tmp/backend.log || true; return 1'
echo '}'
echo 'stop_backend() {'
echo ' if [ -f /tmp/backend.pid ]; then'
echo ' kill "$(cat /tmp/backend.pid)" 2>/dev/null || true'
echo ' rm -f /tmp/backend.pid'
echo ' fi'
echo ' pkill -f "gradlew :stirling-pdf:bootRun" 2>/dev/null || true'
echo ' for i in $(seq 1 30); do'
echo ' curl -fsS http://localhost:8080/api/v1/info/status >/dev/null 2>&1 || return 0'
echo ' sleep 1'
echo ' done'
echo '}'
} > /tmp/helpers.sh
chmod +x /tmp/helpers.sh
# ───────── OAuth round-trip ─────────
- name: Bring up Keycloak (OAuth realm)
working-directory: testing/compose
run: docker compose -f docker-compose-keycloak-oauth.yml up -d --no-deps keycloak-oauth-db keycloak-oauth
- name: Wait for Keycloak (OAuth) ready
working-directory: testing/compose
run: |
for i in $(seq 1 60); do
bash validate-oauth-test.sh 2>/dev/null && exit 0 || true
# validate script also pings stirling on :8080 — accept just the
# keycloak realm as our gate here, stirling boots in the next step
curl -fsS http://localhost:9080/realms/stirling-oauth >/dev/null 2>&1 && exit 0
sleep 5
done
docker compose -f docker-compose-keycloak-oauth.yml logs --tail=200 keycloak-oauth
exit 1
- name: Boot Stirling-PDF (frontend baked in, OAuth env)
env:
SECURITY_ENABLELOGIN: "true"
SECURITY_LOGINMETHOD: "all"
SECURITY_OAUTH2_ENABLED: "true"
SECURITY_OAUTH2_AUTOCREATEUSER: "true"
# Keycloak issues redirect URIs against KC_HOSTNAME, which the
# compose default sets to kubernetes.docker.internal. Match here
# (resolves to localhost via /etc/hosts mapping above).
SECURITY_OAUTH2_CLIENT_KEYCLOAK_ISSUER: "http://kubernetes.docker.internal:9080/realms/stirling-oauth"
SECURITY_OAUTH2_CLIENT_KEYCLOAK_CLIENTID: "stirling-pdf-client"
SECURITY_OAUTH2_CLIENT_KEYCLOAK_CLIENTSECRET: "test-client-secret-change-in-production"
SECURITY_OAUTH2_CLIENT_KEYCLOAK_USEASUSERNAME: "email"
SECURITY_OAUTH2_CLIENT_KEYCLOAK_SCOPES: "openid,profile,email"
run: |
source /tmp/helpers.sh
nohup ./gradlew :stirling-pdf:bootRun -PbuildWithFrontend=true > /tmp/backend.log 2>&1 &
echo $! > /tmp/backend.pid
wait_for_backend
- name: Run enterprise OAuth Playwright tests
id: oauth-tests
env:
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results-oauth.json
run: task e2e:enterprise -- --grep "OAuth"
- name: Stop backend + tear down OAuth Keycloak
if: always()
run: |
source /tmp/helpers.sh
stop_backend
(cd testing/compose && docker compose -f docker-compose-keycloak-oauth.yml down -v)
# ───────── SAML round-trip ─────────
- name: Bring up Keycloak (SAML realm)
working-directory: testing/compose
run: docker compose -f docker-compose-keycloak-saml.yml up -d --no-deps keycloak-saml-db keycloak-saml
- name: Wait for Keycloak (SAML) ready
working-directory: testing/compose
run: |
for i in $(seq 1 60); do
curl -fsS http://localhost:9080/realms/stirling-saml >/dev/null 2>&1 && exit 0
sleep 5
done
docker compose -f docker-compose-keycloak-saml.yml logs --tail=200 keycloak-saml
exit 1
- name: Generate SAML SP certs + fetch Keycloak IdP cert
# The .pem/.crt/.key files are gitignored (test-only certs); the
# docker-based start-saml-test.sh generates them at runtime, so do
# the same in CI before bootRun reads them.
working-directory: testing/compose
run: |
openssl req -x509 -newkey rsa:2048 \
-keyout saml-private-key.key \
-out saml-public-cert.crt \
-days 3650 -nodes \
-subj "/CN=stirling-pdf-saml-sp" >/dev/null 2>&1
# Fetch Keycloak's SAML signing cert from the realm descriptor
CERT_BODY=$(curl -sf http://localhost:9080/realms/stirling-saml/protocol/saml/descriptor \
| awk 'BEGIN{RS="<[^>]*X509Certificate>|</[^>]*X509Certificate>"} NR==2{gsub(/[[:space:]]+/,""); print; exit}')
{
echo "-----BEGIN CERTIFICATE-----"
echo "$CERT_BODY"
echo "-----END CERTIFICATE-----"
} > keycloak-saml-cert.pem
test -s saml-private-key.key
test -s saml-public-cert.crt
test -s keycloak-saml-cert.pem
echo "✓ SAML certs prepared"
- name: Boot Stirling-PDF (frontend baked in, SAML env)
env:
SECURITY_ENABLELOGIN: "true"
SECURITY_LOGINMETHOD: "all"
SECURITY_SAML2_ENABLED: "true"
SECURITY_SAML2_AUTOCREATEUSER: "true"
SECURITY_SAML2_PROVIDER: "keycloak"
SECURITY_SAML2_REGISTRATIONID: "keycloak"
SECURITY_SAML2_IDP_ISSUER: "http://localhost:9080/realms/stirling-saml"
SECURITY_SAML2_IDP_ENTITYID: "http://localhost:9080/realms/stirling-saml"
SECURITY_SAML2_IDP_METADATAURI: "http://localhost:9080/realms/stirling-saml/protocol/saml/descriptor"
SECURITY_SAML2_IDPSINGLELOGINURL: "http://localhost:9080/realms/stirling-saml/protocol/saml"
SECURITY_SAML2_IDPSINGLELOGOUTURL: "http://localhost:9080/realms/stirling-saml/protocol/saml"
SECURITY_SAML2_IDP_CERT: "${{ github.workspace }}/testing/compose/keycloak-saml-cert.pem"
SECURITY_SAML2_PRIVATEKEY: "${{ github.workspace }}/testing/compose/saml-private-key.key"
SECURITY_SAML2_SP_CERT: "${{ github.workspace }}/testing/compose/saml-public-cert.crt"
# Realm registers the SP entity as the metadata URL — see
# keycloak-realm-saml.json `clientId`. Match it here so Keycloak
# accepts the AuthnRequest issuer.
SECURITY_SAML2_SP_ENTITYID: "http://localhost:8080/saml2/service-provider-metadata/keycloak"
SECURITY_SAML2_SP_ACS: "http://localhost:8080/login/saml2/sso/keycloak"
SECURITY_SAML2_SP_SLS: "http://localhost:8080/logout/saml2/slo"
run: |
source /tmp/helpers.sh
nohup ./gradlew :stirling-pdf:bootRun -PbuildWithFrontend=true > /tmp/backend.log 2>&1 &
echo $! > /tmp/backend.pid
wait_for_backend
- name: Run enterprise SAML Playwright tests
id: saml-tests
env:
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results-saml.json
run: task e2e:enterprise -- --grep "SAML"
- name: Stop backend + tear down SAML Keycloak
if: always()
run: |
source /tmp/helpers.sh
stop_backend
(cd testing/compose && docker compose -f docker-compose-keycloak-saml.yml down -v)
# ───────── License-gated feature tests (no IdP needed) ─────────
- name: Wipe DB so InitialSecuritySetup re-runs with admin/adminadmin
# Earlier phases (OAuth, SAML) create the default admin/stirling user.
# InitialSecuritySetup only honours SECURITY_INITIALLOGIN_* when the
# admin user doesn't already exist, so the persisted DB has to be
# cleared between phases for the feature env vars to take effect.
run: |
rm -f app/core/configs/stirling-pdf-DB*.mv.db
rm -rf app/core/configs/backup
- name: Boot Stirling-PDF (frontend baked in, premium only)
env:
SECURITY_INITIALLOGIN_USERNAME: admin
SECURITY_INITIALLOGIN_PASSWORD: adminadmin
SECURITY_ENABLELOGIN: "true"
SECURITY_LOGINMETHOD: "all"
run: |
source /tmp/helpers.sh
nohup ./gradlew :stirling-pdf:bootRun -PbuildWithFrontend=true > /tmp/backend.log 2>&1 &
echo $! > /tmp/backend.pid
wait_for_backend
- name: Run enterprise feature Playwright tests
id: feature-tests
env:
PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results-feature.json
run: task e2e:enterprise -- --grep "Enterprise license"
- name: Print backend log on failure
if: failure()
run: |
echo "::group::Enterprise backend log"
tail -500 /tmp/backend.log || true
echo "::endgroup::"
- name: Stop backend (final)
if: always()
run: |
source /tmp/helpers.sh
stop_backend
- name: Flag flaky tests
# Runs regardless of the test outcomes: a flaky test (passed on retry)
# leaves its step green, so this is the only place it surfaces. Merges
# all three phase reports (some may be absent if an earlier phase hard-
# failed and skipped the rest). Emits ::warning:: annotations + a job
# summary; never fails the job.
if: always()
working-directory: frontend
run: >
npx tsx editor/scripts/report-flaky-tests.mts
"${{ github.workspace }}/frontend/playwright-report/results-oauth.json"
"${{ github.workspace }}/frontend/playwright-report/results-saml.json"
"${{ github.workspace }}/frontend/playwright-report/results-feature.json"
- name: Upload Playwright report
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: playwright-report-enterprise-${{ github.run_id }}
path: frontend/playwright-report/
retention-days: 7