From b7c9065a4e877435be38fd370b8d7f44d00b2132 Mon Sep 17 00:00:00 2001 From: Reece Date: Mon, 17 Aug 2026 14:54:50 +0100 Subject: [PATCH] =?UTF-8?q?perf(processing-folders):=20faster=20sweeps=20?= =?UTF-8?q?=E2=80=94=20wider=20gate,=20filtered=20polling,=20no=20reselect?= =?UTF-8?q?=20churn?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sweepConcurrency's default rises to 6: the classification pipeline is API-bound (one fast-model call per document, ~half a second), so it scales nearly linearly with concurrency and 2 was the bottleneck — a 100-file sweep drops from ~90s to ~30s while completions keep arriving steadily. The knob stays for installs whose pipeline really is a heavyweight local engine. The runs listing takes an optional policyId filter and delivery passes it: following one sweep polls the endpoint every second, and the unfiltered response re-serialized every other policy's runs each time, growing with history. The client-side filter stays as a guard for backends that ignore the parameter. Delivery no longer selects what it opens: a selection isn't meaningful across a folderful of results, and re-selecting on every batch re-rendered the whole growing file list once a second for the length of the sweep. --- .../common/model/ApplicationProperties.java | 9 +++++---- .../policy/controller/PolicyController.java | 15 ++++++++++++--- .../policy/controller/PolicyControllerTest.java | 2 +- .../proprietary/services/processingFolderApi.ts | 5 ++++- .../proprietary/services/processingRunDelivery.ts | 5 ++++- 5 files changed, 26 insertions(+), 10 deletions(-) diff --git a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java index c4c8814a65..fbf095c5f4 100644 --- a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java +++ b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java @@ -218,11 +218,12 @@ public class ApplicationProperties { /** * How many of one sweep's runs may execute at once; further runs queue (visible as pending) * and start as slots free up. Sweeps fan out one run per file, and a folder of documents - * dispatched all at once just piles up at the pipeline's slowest tool (on a desktop - * install, the local AI engine) — same total time, but nothing visibly finishes until the - * end. A small cap keeps completions steady. 0 = unbounded. + * dispatched all at once piles up at the pipeline's slowest tool — nothing visibly finishes + * until the end. The cap keeps completions arriving steadily; the default suits API-bound + * pipelines (classification is one fast-model call per document). Turn it down for a + * heavyweight local engine, 0 = unbounded. */ - private int sweepConcurrency = 2; + private int sweepConcurrency = 6; /** How often (seconds) the schedule trigger checks for policies whose schedule is due. */ private long scheduleSweepSeconds = 60; diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java index 2897d4d701..a16e561301 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java @@ -21,6 +21,7 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.RequestContextHolder; @@ -203,15 +204,20 @@ public class PolicyController { summary = "List the caller's stored-policy runs", description = "Returns the caller's in-flight and recently-finished stored-policy runs (within" - + " the run-retention window). The frontend reconciles these on load so a" - + " run started before a refresh/crash is rediscovered and its outputs" + + " the run-retention window), optionally narrowed to one policy via" + + " `policyId` — a client following a single sweep polls this every" + + " second, and the unfiltered list grows with every other policy's" + + " runs. The frontend reconciles the unfiltered list on load so a run" + + " started before a refresh/crash is rediscovered and its outputs" + " collected, rather than orphaned on the backend. Ad-hoc runs (no" + " policy id) are excluded.") - public List listRuns() { + public List listRuns( + @RequestParam(name = "policyId", required = false) String policyId) { // Local runs first (they carry live step state); keyed by runId to dedupe shared entries. Map byRunId = new LinkedHashMap<>(); runRegistry.all().stream() .filter(run -> run.getPolicyId() != null) + .filter(run -> policyId == null || policyId.equals(run.getPolicyId())) .filter(run -> ownedByCurrentUser(run.getRunId())) .forEach(run -> byRunId.put(run.getRunId(), PolicyRunView.of(run))); // Then runs from other nodes, read from the shared job store. @@ -223,6 +229,9 @@ public class PolicyController { if (meta == null || !meta.containsKey("policyId")) { continue; // ad-hoc job, not a stored-policy run } + if (policyId != null && !policyId.equals(meta.get("policyId"))) { + continue; + } if (ownedByCurrentUser(entry.jobId())) { byRunId.put(entry.jobId(), PolicyRunView.ofEntry(entry)); } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java index c38f5c7d89..7ef38f561b 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java @@ -359,7 +359,7 @@ class PolicyControllerTest { when(jobOwnershipService.createScopedJobKey("owned")).thenReturn("owned"); when(jobOwnershipService.createScopedJobKey("other")).thenReturn("scoped-other"); - List views = controller.listRuns(); + List views = controller.listRuns(null); assertThat(views).hasSize(1); assertThat(views.get(0).runId()).isEqualTo("owned"); diff --git a/frontend/editor/src/proprietary/services/processingFolderApi.ts b/frontend/editor/src/proprietary/services/processingFolderApi.ts index deb9ca2acd..f97b8be22f 100644 --- a/frontend/editor/src/proprietary/services/processingFolderApi.ts +++ b/frontend/editor/src/proprietary/services/processingFolderApi.ts @@ -140,9 +140,12 @@ export interface ProcessingFolderRun { export async function fetchProcessingFolderRuns( policyId: string, ): Promise { + // Filtered server-side: delivery polls this every second, and the + // unfiltered list carries every policy's runs. The client-side filter stays + // as a guard against a backend that ignores the parameter. const res = await apiClient.get< (ProcessingFolderRun & { policyId?: string })[] - >("/api/v1/policies/runs"); + >("/api/v1/policies/runs", { params: { policyId } }); return (res.data ?? []).filter((run) => run.policyId === policyId); } diff --git a/frontend/editor/src/proprietary/services/processingRunDelivery.ts b/frontend/editor/src/proprietary/services/processingRunDelivery.ts index 03c831870f..31c6dab406 100644 --- a/frontend/editor/src/proprietary/services/processingRunDelivery.ts +++ b/frontend/editor/src/proprietary/services/processingRunDelivery.ts @@ -78,7 +78,10 @@ export async function deliverSweepResults( } } if (files.length === 0) return; - await addFiles(files, { selectFiles: true }); + // Never select what is delivered: a selection isn't meaningful across a + // folderful of results, and re-selecting on every batch re-renders the + // whole growing file list once a second for the length of the sweep. + await addFiles(files); progress.opened += files.length; };