Files
Anthony Stirling 547dce4cf3 Add multi-node cluster regression suite (compose stack + behave e2e) (#7026)
# Description of Changes

- The multi-node compose stack + behave suite (11 features)
- The nightly multinode-e2e job in build-enterprise.yml


cuke features are

cluster_health - both nodes boot healthy and join the Valkey backplane
load_balancing - traffic spreads across nodes; no spurious 401 when
bounced
cross_node_auth - a token from one node validates on all nodes (shared
DB keys)
shared_state - teams/sources/org visible from every node
policy_management - create/rename/delete a policy on any node, reflected
everywhere
source_management - source CRUD cross-node; referenced source can't be
deleted anywhere
connections - S3 connection resolves (secret masked) and deletes
cluster-wide
processor_ledger - files processed exactly once even when both nodes
trigger together
policy_run_coordination - a run on one node is visible from every node
rate_limiting - rate-limit counters shared via Valkey, not per node
failover - LB keeps serving when a node dies; recovered node accepts
existing tokens


can now start a full node system with 
export PREMIUM_KEY=<your licence key> ./start-multinode-test.sh
starts a 40 person org DB install with multi node and database
(--no-seed to have without DB on startup)
4 teams
1 s3 connection
1 policy

---

## Checklist

### General

- [ ] I have read the [Contribution
Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md)
- [ ] I have read the [Stirling-PDF Developer
Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md)
(if applicable)
- [ ] I have read the [How to add new languages to
Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md)
(if applicable)
- [ ] I have performed a self-review of my own code
- [ ] My changes generate no new warnings

### Documentation

- [ ] I have updated relevant docs on [Stirling-PDF's doc
repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/)
(if functionality has heavily changed)
- [ ] I have read the section [Add New Translation
Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags)
(for new translation tags only)

### Translations (if applicable)

- [ ] I ran
[`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md)

### UI Changes (if applicable)

- [ ] Screenshots or videos demonstrating the UI changes are attached
(e.g., as comments or direct attachments in the PR)

### Testing (if applicable)

- [ ] I have run `task check` to verify linters, typechecks, and tests
pass
- [ ] I have tested my changes locally. Refer to the [Testing
Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing)
for more details.
2026-08-01 17:09:11 +01:00

141 lines
5.8 KiB
Bash

#!/bin/sh
# Seeds a running multi-node stack: 4 teams, ~40 users, an S3 connection, a scheduled S3 policy, and a webhook source if the build supports it.
# Auth uses the Bearer JWT from the login response body (not a cookie) since the global API key can't create teams.
# Idempotent-ish: re-running skips existing teams/users; each step is best-effort and logs failures without aborting.
set -u
BASE_URL="${BASE_URL:-http://localhost:8080}"
ADMIN_USER="${ADMIN_USER:-admin}"
ADMIN_PASS="${ADMIN_PASS:-stirling}"
USER_COUNT="${USER_COUNT:-40}"
USER_PASS="${USER_PASS:-Password123!}"
PGHOST="${PGHOST:-postgres}"
PGUSER="${PGUSER:-stirling}"
PGPASSWORD="${PGPASSWORD:-stirling}"
PGDATABASE="${PGDATABASE:-stirling}"
export PGPASSWORD
TEAMS="Engineering Finance Legal Operations"
log() { echo "[seed] $*"; }
psqlq() { psql -h "$PGHOST" -U "$PGUSER" -d "$PGDATABASE" -tAc "$1" 2>/dev/null | tr -d '[:space:]'; }
# --- wait for the load balancer to serve a healthy app -----------------------
log "waiting for $BASE_URL ..."
i=0
until curl -fsS "$BASE_URL/api/v1/info/status" 2>/dev/null | grep -q UP; do
i=$((i+1)); [ "$i" -gt 120 ] && { log "timed out waiting for API"; exit 1; }
sleep 3
done
log "API is up"
# --- admin login -> Bearer token ---------------------------------------------
code=$(curl -sS -o /tmp/login.json -w '%{http_code}' \
-X POST "$BASE_URL/api/v1/auth/login" \
-H 'Content-Type: application/json' \
-d "{\"username\":\"$ADMIN_USER\",\"password\":\"$ADMIN_PASS\"}")
log "admin login: HTTP $code"
[ "$code" = "200" ] || { log "login failed: $(cat /tmp/login.json)"; exit 1; }
TOKEN=$(jq -r '.session.access_token' </tmp/login.json)
[ -n "$TOKEN" ] && [ "$TOKEN" != "null" ] || { log "no access_token in login response"; exit 1; }
auth() { curl -sS -H "Authorization: Bearer $TOKEN" "$@"; }
# --- teams -------------------------------------------------------------------
for t in $TEAMS; do
code=$(auth -o /dev/null -w '%{http_code}' -X POST "$BASE_URL/api/v1/team/create" \
--data-urlencode "name=$t")
log "team '$t': HTTP $code"
done
# Resolve team ids from the DB (no admin list endpoint self-hosted).
seed_team_ids=""
for t in $TEAMS; do
id=$(psqlq "select team_id from teams where name='$t' limit 1")
[ -n "$id" ] && seed_team_ids="$seed_team_ids $id"
done
set -- $seed_team_ids
team_count=$#
log "seedable team ids:$seed_team_ids (count=$team_count)"
# --- users: spread across teams, first two are admins ------------------------
created=0; failed=0
n=1
while [ "$n" -le "$USER_COUNT" ]; do
uname=$(printf "user%02d@stirling.test" "$n")
role="ROLE_USER"; [ "$n" -le 2 ] && role="ROLE_ADMIN"
team_id=""
if [ "$team_count" -gt 0 ]; then
idx=$(( (n % team_count) + 1 )); team_id=$(eval echo "\${$idx}")
fi
code=$(auth -o /tmp/user.json -w '%{http_code}' -X POST "$BASE_URL/api/v1/user/admin/saveUser" \
--data-urlencode "username=$uname" \
--data-urlencode "password=$USER_PASS" \
--data-urlencode "role=$role" \
${team_id:+--data-urlencode "teamId=$team_id"} \
--data-urlencode "authType=WEB" \
--data-urlencode "forceChange=false")
case "$code" in
200|201) created=$((created+1));;
409) log "user $uname already exists";;
*) failed=$((failed+1)); [ "$failed" -le 3 ] && log "user $uname failed HTTP $code: $(cat /tmp/user.json)";;
esac
n=$((n+1))
done
log "users created: $created (failed: $failed, requested: $USER_COUNT)"
# --- S3 connection -> the in-cluster MinIO 'policy-data' bucket ---------------
conn_body=$(cat <<JSON
{"integrationType":"S3","name":"MinIO policy bucket","scope":"SERVER","enabled":true,"locked":false,"defaultAccess":"ORG_ALL",
"config":{"bucket":"policy-data","region":"us-east-1","endpoint":"http://minio:9000","accessKeyId":"minioadmin","secretAccessKey":"minioadmin","pathStyleAccess":true}}
JSON
)
conn_id=$(auth -X POST "$BASE_URL/api/v1/integrations" -H 'Content-Type: application/json' -d "$conn_body" \
| jq -r '.id // empty' 2>/dev/null)
log "S3 connection id: ${conn_id:-<none>}"
# --- a scheduled S3 -> compress -> S3 policy ---------------------------------
if [ -n "${conn_id:-}" ]; then
src_body=$(cat <<JSON
{"name":"Incoming S3","type":"s3","enabled":true,
"options":{"connectionId":$conn_id,"prefix":"incoming/","mode":"consume"}}
JSON
)
src_id=$(auth -X POST "$BASE_URL/api/v1/sources" -H 'Content-Type: application/json' -d "$src_body" \
| jq -r '.id // empty' 2>/dev/null)
log "S3 source id: ${src_id:-<none>}"
if [ -n "${src_id:-}" ]; then
pol_body=$(cat <<JSON
{"name":"Compress incoming PDFs","enabled":true,
"trigger":{"type":"schedule","options":{"schedule":{"type":"every","count":5,"unit":"MINUTES"}}},
"sourceIds":["$src_id"],
"steps":[{"operation":"/api/v1/misc/compress-pdf","parameters":{}}],
"output":{"type":"s3","options":{"connectionId":$conn_id,"prefix":"processed/"}}}
JSON
)
code=$(auth -o /tmp/pol.json -w '%{http_code}' -X POST "$BASE_URL/api/v1/policies" \
-H 'Content-Type: application/json' -d "$pol_body")
log "policy create: HTTP $code $( [ "$code" != 200 ] && head -c 160 /tmp/pol.json )"
fi
# --- webhook source + policy (only if this build has the webhook type) -----
wh_body=$(cat <<JSON
{"name":"Partner webhook","type":"webhook","enabled":true,
"options":{"connectionId":$conn_id,"mode":"consume"}}
JSON
)
wh=$(auth -o /tmp/wh.json -w '%{http_code}' -X POST "$BASE_URL/api/v1/sources" \
-H 'Content-Type: application/json' -d "$wh_body")
if [ "$wh" = "200" ] || [ "$wh" = "201" ]; then
wh_url=$(jq -r '.options.webhookId // empty' </tmp/wh.json 2>/dev/null)
log "webhook source created (deliver to /api/v1/webhooks/$wh_url)"
else
log "webhook source not created (HTTP $wh) - expected on builds without the webhook branch"
fi
fi
log "seed complete."
log " login: $ADMIN_USER / $ADMIN_PASS at $BASE_URL"
log " users: user01..$(printf '%02d' "$USER_COUNT")@stirling.test / $USER_PASS"