mirror of
https://github.com/Stirling-Tools/Stirling-PDF.git
synced 2026-09-02 21:03:34 +03:00
Let a pipeline run on the editor, on upload or export (#7581)
Redesigns the policies system so that the backend has an understanding of policies running over the Editor. The Editor is not set up as a source for the backend because the backend can't actively get files from it, they come in via the frontend sending them to the backend, so instead pipelines have a specific editor key in them to encode whether the pipeline is triggered on file upload/export in the editor. Also make a big effort in the frontend code towards genericising policy running. Previously, there was specific support in the main policy executor for each policy that it had to run, which was not going to be appropriate long-term, especially when users can run any pipeline in the editor. There's more work needed here for me to really be happy with it but this PR is plenty large on its own and moves it in the right direction. All of the above was required to allow arbitrary user pipelines to run in the editor. This PR makes it so that the user can select Editor as a source in the pipeline creator, along with whether it should run on upload or export. <img width="1437" height="506" alt="image" src="https://github.com/user-attachments/assets/b2d176a1-185c-480b-9916-abdd1447d8e1" /> --------- Co-authored-by: James Brunton <james@stirlingpdf.com>
This commit is contained in:
co-authored by
James Brunton
parent
4ef2e3811c
commit
ceeec53df4
+11
-1
@@ -336,6 +336,15 @@ public class PolicyController {
|
||||
* nothing to check.
|
||||
*/
|
||||
private void requireAccessibleOutput(Policy policy) {
|
||||
// An editor policy hands its results back to the workspace the file came from. A stored
|
||||
// destination would send the run to a folder or bucket instead, leaving the editor's copy
|
||||
// untouched - and the editor's import would then have nothing to collect.
|
||||
if (policy.editor().allowed() && !policy.outputIds().isEmpty()) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST,
|
||||
"An editor policy delivers back to the editor and can't also have a"
|
||||
+ " destination");
|
||||
}
|
||||
for (String outputId : policy.outputIds()) {
|
||||
Source destination =
|
||||
sourceStore
|
||||
@@ -393,7 +402,8 @@ public class PolicyController {
|
||||
policy.steps(),
|
||||
policy.output(),
|
||||
policy.outputIds(),
|
||||
teamId);
|
||||
teamId,
|
||||
policy.editor());
|
||||
}
|
||||
|
||||
/** Output secrets never leave the server: reads return the redaction sentinel instead. */
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
package stirling.software.proprietary.policy.model;
|
||||
|
||||
/**
|
||||
* How a policy participates in the editor: it fires in the browser as each file passes through,
|
||||
* rather than being swept from a stored {@code Source} on a trigger.
|
||||
*
|
||||
* <p>An object rather than a bare flag so the moment it fires ({@code runOn}) travels with the
|
||||
* decision, and so later editor-only settings have somewhere to live.
|
||||
*
|
||||
* @param allowed whether the editor may run this policy at all
|
||||
* @param runOn which moment it fires on: {@code "upload"} or {@code "export"}
|
||||
*/
|
||||
public record EditorConfig(boolean allowed, String runOn) {
|
||||
|
||||
public static final String UPLOAD = "upload";
|
||||
public static final String EXPORT = "export";
|
||||
|
||||
public EditorConfig {
|
||||
runOn = EXPORT.equals(runOn) ? EXPORT : UPLOAD;
|
||||
}
|
||||
|
||||
/** Not an editor policy: swept server-side, or run only on demand. */
|
||||
public static EditorConfig disabled() {
|
||||
return new EditorConfig(false, UPLOAD);
|
||||
}
|
||||
|
||||
public static EditorConfig onUpload() {
|
||||
return new EditorConfig(true, UPLOAD);
|
||||
}
|
||||
|
||||
public static EditorConfig onExport() {
|
||||
return new EditorConfig(true, EXPORT);
|
||||
}
|
||||
}
|
||||
+32
-4
@@ -1,6 +1,7 @@
|
||||
package stirling.software.proprietary.policy.model;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* A stored automation: ordered tool steps, input bindings, and output destinations.
|
||||
@@ -24,13 +25,29 @@ public record Policy(
|
||||
List<PipelineStep> steps,
|
||||
OutputSpec output,
|
||||
List<String> outputIds,
|
||||
Long teamId) {
|
||||
Long teamId,
|
||||
EditorConfig editor) {
|
||||
|
||||
public Policy {
|
||||
inputs = inputs == null ? List.of() : List.copyOf(inputs);
|
||||
steps = steps == null ? List.of() : steps;
|
||||
output = output == null ? OutputSpec.inline() : output;
|
||||
outputIds = outputIds == null ? List.of() : List.copyOf(outputIds);
|
||||
editor = editor == null ? EditorConfig.disabled() : editor;
|
||||
}
|
||||
|
||||
/** Without editor participation: a swept or on-demand policy. */
|
||||
public Policy(
|
||||
String id,
|
||||
String name,
|
||||
String owner,
|
||||
boolean enabled,
|
||||
List<PipelineInput> inputs,
|
||||
List<PipelineStep> steps,
|
||||
OutputSpec output,
|
||||
List<String> outputIds,
|
||||
Long teamId) {
|
||||
this(id, name, owner, enabled, inputs, steps, output, outputIds, teamId, null);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -70,6 +87,14 @@ public record Policy(
|
||||
return inputs.stream().map(PipelineInput::sourceId).toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* The moment this policy fires in the editor ("upload" / "export"), or empty when the editor
|
||||
* does not run it. Legacy blobs are lifted onto {@link EditorConfig} when they are read.
|
||||
*/
|
||||
public Optional<String> editorRunOn() {
|
||||
return editor.allowed() ? Optional.of(editor.runOn()) : Optional.empty();
|
||||
}
|
||||
|
||||
/** The distinct trigger types configured across this policy's inputs (manual inputs aside). */
|
||||
public List<String> triggerTypes() {
|
||||
return inputs.stream()
|
||||
@@ -82,17 +107,20 @@ public record Policy(
|
||||
|
||||
/** A copy with the inline output replaced (e.g. resolved for the engine, or migrated). */
|
||||
public Policy withOutput(OutputSpec resolved) {
|
||||
return new Policy(id, name, owner, enabled, inputs, steps, resolved, outputIds, teamId);
|
||||
return new Policy(
|
||||
id, name, owner, enabled, inputs, steps, resolved, outputIds, teamId, editor);
|
||||
}
|
||||
|
||||
/** A copy under a different owner (e.g. moving a seed off a placeholder name). */
|
||||
public Policy withOwner(String newOwner) {
|
||||
return new Policy(id, name, newOwner, enabled, inputs, steps, output, outputIds, teamId);
|
||||
return new Policy(
|
||||
id, name, newOwner, enabled, inputs, steps, output, outputIds, teamId, editor);
|
||||
}
|
||||
|
||||
/** A copy referencing the given saved output destinations. */
|
||||
public Policy withOutputIds(List<String> newOutputIds) {
|
||||
return new Policy(id, name, owner, enabled, inputs, steps, output, newOutputIds, teamId);
|
||||
return new Policy(
|
||||
id, name, owner, enabled, inputs, steps, output, newOutputIds, teamId, editor);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+5
-1
@@ -114,10 +114,14 @@ public class PolicyOverviewService {
|
||||
/**
|
||||
* Summarise a policy's triggers for the overview row: "manual" when no input is triggered,
|
||||
* otherwise the distinct trigger types across its inputs (e.g. "folder-watch, schedule").
|
||||
*
|
||||
* <p>An editor policy has no wire input to trigger, but it is not manual either - it fires in
|
||||
* the editor on every upload or export, so it reports that rather than reading as on-demand.
|
||||
*/
|
||||
private static String triggerSummary(Policy policy) {
|
||||
List<String> types = policy.triggerTypes();
|
||||
return types.isEmpty() ? "manual" : String.join(", ", types);
|
||||
if (!types.isEmpty()) return String.join(", ", types);
|
||||
return policy.editorRunOn().map(runOn -> "editor-" + runOn).orElse("manual");
|
||||
}
|
||||
|
||||
private static String outputSummary(OutputSpec output) {
|
||||
|
||||
+6
-3
@@ -14,6 +14,7 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.model.TeamCreatedEvent;
|
||||
import stirling.software.proprietary.policy.model.EditorConfig;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
@@ -98,9 +99,8 @@ public class DefaultClassificationPolicySeeder {
|
||||
static Policy defaultPolicy(Long teamId) {
|
||||
Map<String, Object> options = new HashMap<>();
|
||||
options.put("categoryId", CATEGORY);
|
||||
options.put("runOn", "upload");
|
||||
options.put("mode", "new_version");
|
||||
options.put("sources", List.of("editor"));
|
||||
options.put("sources", List.of());
|
||||
options.put("scopeTypes", List.of());
|
||||
options.put("reviewerEmail", "");
|
||||
return new Policy(
|
||||
@@ -113,6 +113,9 @@ public class DefaultClassificationPolicySeeder {
|
||||
List.of(),
|
||||
List.of(new PipelineStep(CLASSIFY_ENDPOINT, Map.of())),
|
||||
new OutputSpec("inline", options),
|
||||
teamId);
|
||||
List.of(),
|
||||
teamId,
|
||||
// Classification runs in the editor on every upload.
|
||||
EditorConfig.onUpload());
|
||||
}
|
||||
}
|
||||
|
||||
+4
-6
@@ -107,14 +107,12 @@ public class SourceOverviewService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a policy runs from the editor. Editor membership is carried in the policy's output
|
||||
* metadata ({@code output.options.sources}) - a client-side list the editor writes when a
|
||||
* policy targets it - rather than as a persisted {@code sourceId}, because the editor is
|
||||
* virtual and has no stored source to reference.
|
||||
* Whether a policy runs from the editor. Read from the policy's first-class {@link
|
||||
* stirling.software.proprietary.policy.model.EditorConfig}, never inferred from a sources list
|
||||
* (the editor is not a real source).
|
||||
*/
|
||||
private static boolean runsFromEditor(Policy policy) {
|
||||
Object sources = policy.output().options().get("sources");
|
||||
return sources instanceof List<?> list && list.contains(EditorSource.ID);
|
||||
return policy.editor().allowed();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+2
-1
@@ -38,7 +38,8 @@ public class InProcessPolicyStore implements PolicyStore {
|
||||
policy.steps(),
|
||||
policy.output(),
|
||||
policy.outputIds(),
|
||||
policy.teamId());
|
||||
policy.teamId(),
|
||||
policy.editor());
|
||||
policies.put(id, stored);
|
||||
// Existing policy keeps its position; a new one appends to the end of its team's queue.
|
||||
sortOrders.computeIfAbsent(id, key -> nextSortOrder(stored.teamId()));
|
||||
|
||||
+65
-2
@@ -3,6 +3,7 @@ package stirling.software.proprietary.policy.store;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -11,8 +12,10 @@ import org.springframework.transaction.annotation.Transactional;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.policy.model.EditorConfig;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyBinding;
|
||||
import stirling.software.proprietary.policy.source.EditorSource;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
@@ -48,7 +51,8 @@ public class JpaPolicyStore implements PolicyStore {
|
||||
policy.steps(),
|
||||
policy.output(),
|
||||
policy.outputIds(),
|
||||
policy.teamId());
|
||||
policy.teamId(),
|
||||
policy.editor());
|
||||
|
||||
PolicyEntity entity = new PolicyEntity();
|
||||
entity.setId(id);
|
||||
@@ -148,7 +152,9 @@ public class JpaPolicyStore implements PolicyStore {
|
||||
// One unreadable row must never abort a bulk read or crash startup.
|
||||
private Optional<Policy> toPolicy(PolicyEntity entity) {
|
||||
try {
|
||||
JsonNode node = upgradeLegacyShape(objectMapper.readTree(entity.getPolicyJson()));
|
||||
JsonNode node =
|
||||
liftEditorConfig(
|
||||
upgradeLegacyShape(objectMapper.readTree(entity.getPolicyJson())));
|
||||
return Optional.of(objectMapper.treeToValue(node, Policy.class));
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
@@ -191,4 +197,61 @@ public class JpaPolicyStore implements PolicyStore {
|
||||
obj.remove("sourceIds");
|
||||
return obj;
|
||||
}
|
||||
|
||||
/** Categories whose editor moment defaulted to export before it was stored (see runOn.ts). */
|
||||
private static final Set<String> EXPORT_BY_DEFAULT = Set.of("security");
|
||||
|
||||
/**
|
||||
* Derive {@code editor} for a blob written before editor participation had its own field, from
|
||||
* its {@code output.options}: allowed when {@code sources} lists {@code "editor"}, or - for a
|
||||
* catalogue policy - when there is no {@code sources} list at all (an unnarrowed catalogue
|
||||
* policy runs in the editor).
|
||||
*
|
||||
* <p>Runs on every read, deliberately outside {@link #upgradeLegacyShape}'s early return: a
|
||||
* blob written after triggers moved onto {@code inputs} but before this field existed still
|
||||
* needs lifting, and that early return would skip exactly those rows.
|
||||
*/
|
||||
private JsonNode liftEditorConfig(JsonNode root) {
|
||||
if (!(root instanceof ObjectNode obj) || obj.hasNonNull("editor")) {
|
||||
return root;
|
||||
}
|
||||
JsonNode options = obj.path("output").path("options");
|
||||
String categoryId = text(options, "categoryId");
|
||||
JsonNode sources = options.get("sources");
|
||||
boolean listed = sources != null && sources.isArray() && !sources.isEmpty();
|
||||
boolean allowed;
|
||||
if (listed) {
|
||||
// An explicit scope list decides: only the editor's own id puts it on the editor.
|
||||
allowed = false;
|
||||
for (JsonNode source : sources) {
|
||||
if (source.isValueNode() && EditorSource.ID.equals(source.asString())) {
|
||||
allowed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No list: a catalogue policy ran in the editor by default, but a builder pipeline
|
||||
// (no category) could not reach the editor at all, so silence is not consent there.
|
||||
allowed = !categoryId.isBlank();
|
||||
}
|
||||
ObjectNode editor = objectMapper.createObjectNode();
|
||||
editor.put("allowed", allowed);
|
||||
editor.put("runOn", legacyRunOn(options, categoryId));
|
||||
obj.set("editor", editor);
|
||||
return obj;
|
||||
}
|
||||
|
||||
/** The stored moment, or the category default the client applied when none was stored. */
|
||||
private static String legacyRunOn(JsonNode options, String categoryId) {
|
||||
String stored = text(options, "runOn");
|
||||
if (EditorConfig.EXPORT.equals(stored) || EditorConfig.UPLOAD.equals(stored)) {
|
||||
return stored;
|
||||
}
|
||||
return EXPORT_BY_DEFAULT.contains(categoryId) ? EditorConfig.EXPORT : EditorConfig.UPLOAD;
|
||||
}
|
||||
|
||||
private static String text(JsonNode parent, String field) {
|
||||
JsonNode node = parent.path(field);
|
||||
return node.isValueNode() ? node.asString() : "";
|
||||
}
|
||||
}
|
||||
|
||||
+39
@@ -15,6 +15,7 @@ import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
import stirling.software.proprietary.policy.model.EditorConfig;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineInput;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
@@ -223,6 +224,44 @@ class PolicyOverviewServiceTest {
|
||||
teamId));
|
||||
}
|
||||
|
||||
@Test
|
||||
void editorPolicyReportsItsRunMomentRatherThanReadingAsManual() {
|
||||
policyStore.save(
|
||||
new Policy(
|
||||
null,
|
||||
"Editor flatten",
|
||||
"owner",
|
||||
true,
|
||||
List.of(),
|
||||
List.of(new PipelineStep("/api/v1/misc/flatten", Map.of())),
|
||||
OutputSpec.inline(),
|
||||
List.of(),
|
||||
1L,
|
||||
EditorConfig.onUpload()));
|
||||
|
||||
PolicyView view = find(service.overview(), "Editor flatten");
|
||||
|
||||
assertEquals("editor-upload", view.trigger());
|
||||
}
|
||||
|
||||
@Test
|
||||
void sweptPolicyWithNoTriggeredInputIsStillManual() {
|
||||
policyStore.save(
|
||||
new Policy(
|
||||
null,
|
||||
"Swept compress",
|
||||
"owner",
|
||||
true,
|
||||
List.of(),
|
||||
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
|
||||
OutputSpec.inline(),
|
||||
1L));
|
||||
|
||||
PolicyView view = find(service.overview(), "Swept compress");
|
||||
|
||||
assertEquals("manual", view.trigger());
|
||||
}
|
||||
|
||||
private static PolicyView find(PoliciesOverviewResponse response, String name) {
|
||||
return response.pipelines().stream()
|
||||
.filter(view -> view.name().equals(name))
|
||||
|
||||
+18
-2
@@ -64,14 +64,30 @@ class DefaultClassificationPolicySeederTest {
|
||||
assertThat(policy.teamId()).isEqualTo(7L);
|
||||
assertThat(policy.output().type()).isEqualTo("inline");
|
||||
assertThat(policy.output().options().get("categoryId")).isEqualTo("classification");
|
||||
assertThat(policy.output().options().get("runOn")).isEqualTo("upload");
|
||||
assertThat(policy.output().options().get("mode")).isEqualTo("new_version");
|
||||
assertThat(policy.output().options().get("sources")).isEqualTo(List.of("editor"));
|
||||
// Editor participation is the policy's own flag, not a marker in the output options.
|
||||
assertThat(policy.editor().allowed()).isTrue();
|
||||
assertThat(policy.editor().runOn()).isEqualTo("upload");
|
||||
assertThat(policy.steps()).hasSize(1);
|
||||
assertThat(policy.steps().get(0).operation())
|
||||
.isEqualTo("/api/v1/ai/tools/classify-and-label");
|
||||
}
|
||||
|
||||
@Test
|
||||
void marksEditorParticipationOnEditorConfigAndSeedsNoSources() {
|
||||
when(policyStore.findByTeam(7L)).thenReturn(List.of());
|
||||
|
||||
seeder().onTeamCreated(new TeamCreatedEvent(7L, "Acme"));
|
||||
|
||||
ArgumentCaptor<Policy> saved = ArgumentCaptor.forClass(Policy.class);
|
||||
verify(policyStore).save(saved.capture());
|
||||
Policy policy = saved.getValue();
|
||||
// Editor participation is on EditorConfig, not the sources list; the seed carries no
|
||||
// sources.
|
||||
assertThat(policy.editor().allowed()).isTrue();
|
||||
assertThat(policy.output().options().get("sources")).isEqualTo(List.of());
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotSeedWhenAClassificationPolicyAlreadyExists() {
|
||||
when(policyStore.findByTeam(7L)).thenReturn(List.of(classificationPolicy(7L)));
|
||||
|
||||
+6
-4
@@ -15,6 +15,7 @@ import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
import stirling.software.proprietary.policy.model.EditorConfig;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineInput;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
@@ -222,9 +223,7 @@ class SourceOverviewServiceTest {
|
||||
OutputSpec.inline()));
|
||||
}
|
||||
|
||||
/**
|
||||
* A policy that targets the editor: membership rides in its output metadata, not a sourceId.
|
||||
*/
|
||||
/** A policy that targets the editor: membership on its {@link EditorConfig}, not a sourceId. */
|
||||
private void editorPolicy(String name) {
|
||||
policyStore.save(
|
||||
new Policy(
|
||||
@@ -234,7 +233,10 @@ class SourceOverviewServiceTest {
|
||||
true,
|
||||
List.of(),
|
||||
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
|
||||
new OutputSpec("inline", Map.of("sources", List.of("editor")))));
|
||||
OutputSpec.inline(),
|
||||
List.of(),
|
||||
null,
|
||||
EditorConfig.onUpload()));
|
||||
}
|
||||
|
||||
private void teamPolicy(String name, Long teamId, String... sourceIds) {
|
||||
|
||||
+124
@@ -18,6 +18,7 @@ import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import stirling.software.proprietary.policy.model.EditorConfig;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineInput;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
@@ -113,6 +114,129 @@ class JpaPolicyStoreTest {
|
||||
upgraded.inputs());
|
||||
}
|
||||
|
||||
/**
|
||||
* The regression this guards: before the editor lift, a blob written by the pre-{@code editor}
|
||||
* seeder deserialized straight onto {@link EditorConfig#disabled()}, silently taking every
|
||||
* upgraded install's Classification policy off the editor.
|
||||
*
|
||||
* <p>The {@code inputs} variant is the important one - {@link
|
||||
* JpaPolicyStore#upgradeLegacyShape} returns early on it, so a lift living inside that method
|
||||
* would miss exactly the rows written between the trigger migration and this field.
|
||||
*/
|
||||
@Test
|
||||
void getLiftsALegacyEditorSourceOntoEditorConfigWhenInputsArePresent() {
|
||||
Policy lifted = readLegacy(legacyJson("\"inputs\":[],", "\"sources\":[\"editor\"],"));
|
||||
|
||||
assertEquals(EditorConfig.onUpload(), lifted.editor());
|
||||
assertEquals(Optional.of("upload"), lifted.editorRunOn());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getLiftsALegacyEditorSourceOnThePreInputsShapeToo() {
|
||||
// Oldest shape: policy-level trigger + sourceIds, so both migrations have to compose.
|
||||
Policy lifted =
|
||||
readLegacy(
|
||||
legacyJson(
|
||||
"\"trigger\":{\"type\":\"schedule\",\"options\":{}},"
|
||||
+ "\"sourceIds\":[\"s1\"],",
|
||||
"\"sources\":[\"editor\"],"));
|
||||
|
||||
assertEquals(EditorConfig.onUpload(), lifted.editor());
|
||||
assertEquals(
|
||||
List.of(new PipelineInput("s1", new TriggerConfig("schedule", Map.of()))),
|
||||
lifted.inputs());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getTreatsAnUnnarrowedCataloguePolicyAsEditorRun() {
|
||||
// Empty and absent both meant "nobody narrowed it", which the editor read as its own.
|
||||
assertTrue(readLegacy(legacyJson("\"inputs\":[],", "\"sources\":[],")).editor().allowed());
|
||||
assertTrue(readLegacy(legacyJson("\"inputs\":[],", "")).editor().allowed());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getLeavesACataloguePolicyScopedElsewhereOffTheEditor() {
|
||||
Policy lifted = readLegacy(legacyJson("\"inputs\":[],", "\"sources\":[\"sharepoint\"],"));
|
||||
|
||||
assertFalse(lifted.editor().allowed());
|
||||
assertEquals(Optional.empty(), lifted.editorRunOn());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getLeavesASourcelessBuilderPipelineOffTheEditor() {
|
||||
// No categoryId: a pipeline built on the Pipelines page, which never reached the editor.
|
||||
String json =
|
||||
"{\"id\":\"p1\",\"name\":\"legacy\",\"enabled\":true,\"inputs\":[],"
|
||||
+ "\"steps\":[],\"output\":{\"type\":\"inline\",\"options\":{}}}";
|
||||
|
||||
assertFalse(readLegacy(json).editor().allowed());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getKeepsTheCategoryDefaultMomentWhenNoRunOnWasStored() {
|
||||
// Security enforced on export before runOn was persisted (frontend runOn.ts
|
||||
// DEFAULT_RUN_ON).
|
||||
String json =
|
||||
"{\"id\":\"p1\",\"name\":\"legacy\",\"enabled\":true,\"inputs\":[],"
|
||||
+ "\"steps\":[],\"output\":{\"type\":\"inline\",\"options\":{"
|
||||
+ "\"categoryId\":\"security\",\"sources\":[\"editor\"]}}}";
|
||||
|
||||
assertEquals(EditorConfig.onExport(), readLegacy(json).editor());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getNeverOverridesAnExplicitlyStoredEditorBlock() {
|
||||
// A deliberate opt-out survives, so the lift stays safe to leave in permanently.
|
||||
String json =
|
||||
"{\"id\":\"p1\",\"name\":\"legacy\",\"enabled\":true,\"inputs\":[],"
|
||||
+ "\"steps\":[],\"editor\":{\"allowed\":false,\"runOn\":\"upload\"},"
|
||||
+ "\"output\":{\"type\":\"inline\",\"options\":{"
|
||||
+ "\"categoryId\":\"classification\",\"sources\":[\"editor\"]}}}";
|
||||
|
||||
assertFalse(readLegacy(json).editor().allowed());
|
||||
}
|
||||
|
||||
/**
|
||||
* Pins the wire shape the stubbed Playwright spec hardcodes: the derived block is additive, so
|
||||
* a real response carries it alongside the untouched legacy options bag.
|
||||
*/
|
||||
@Test
|
||||
void getLeavesTheLegacyOptionsBagIntactSoTheResponseCarriesBoth() {
|
||||
Policy lifted = readLegacy(legacyJson("\"inputs\":[],", "\"sources\":[\"editor\"],"));
|
||||
|
||||
assertEquals(List.of("editor"), lifted.output().options().get("sources"));
|
||||
String wire = objectMapper.writeValueAsString(lifted);
|
||||
assertTrue(
|
||||
wire.contains("\"editor\":{\"allowed\":true,\"runOn\":\"upload\"}"),
|
||||
"expected the derived editor block on the wire, got: " + wire);
|
||||
}
|
||||
|
||||
/**
|
||||
* The blob main's DefaultClassificationPolicySeeder wrote, with the shape bits parameterised.
|
||||
*/
|
||||
private static String legacyJson(String shapeFields, String sourcesField) {
|
||||
return "{\"id\":\"p1\",\"name\":\"Classification Policy\",\"owner\":\"system\","
|
||||
+ "\"enabled\":true,"
|
||||
+ shapeFields
|
||||
+ "\"steps\":[{\"operation\":\"/api/v1/ai/tools/classify-and-label\","
|
||||
+ "\"parameters\":{}}],"
|
||||
+ "\"output\":{\"type\":\"inline\",\"options\":{"
|
||||
+ "\"categoryId\":\"classification\",\"runOn\":\"upload\","
|
||||
+ "\"mode\":\"new_version\","
|
||||
+ sourcesField
|
||||
+ "\"scopeTypes\":[],\"reviewerEmail\":\"\"}},\"teamId\":1}";
|
||||
}
|
||||
|
||||
private Policy readLegacy(String policyJson) {
|
||||
PolicyEntity entity = new PolicyEntity();
|
||||
entity.setId("p1");
|
||||
entity.setName("legacy");
|
||||
entity.setEnabled(true);
|
||||
entity.setPolicyJson(policyJson);
|
||||
when(repository.findById("p1")).thenReturn(Optional.of(entity));
|
||||
return store.get("p1").orElseThrow();
|
||||
}
|
||||
|
||||
@Test
|
||||
void saveDenormalizesTeamIdForScopedQueries() {
|
||||
store.save(
|
||||
|
||||
@@ -8220,6 +8220,9 @@ chooseDestination = "Choose a destination"
|
||||
chooseOperation = "Choose what this step does"
|
||||
chooseSource = "Choose a source"
|
||||
discard = "Discard changes"
|
||||
editorDestination = "Editor"
|
||||
editorDestinationDetail = "Replaces the file you ran it on"
|
||||
editorDestinationHelp = "This pipeline runs on the files in your workspace, and its results replace the file it ran on. There is nowhere else to send them."
|
||||
inputs = "Input"
|
||||
inputSource = "Input source"
|
||||
inputTrigger = "Trigger"
|
||||
@@ -8231,6 +8234,10 @@ needsSource = "No source chosen"
|
||||
noToolMatches = "No tools match your search."
|
||||
pause = "Pause"
|
||||
rename = "Rename pipeline"
|
||||
runOn = "Runs on"
|
||||
runOnExport = "Every export"
|
||||
runOnTooltip = "Choose when this pipeline runs on your files: when you add them, or when you export them."
|
||||
runOnUpload = "Every upload"
|
||||
searchTools = "Search tools"
|
||||
sendToSystem = "Send to another system"
|
||||
stepsIncompatible = "These steps can't run on what their prior step produces: {{tools}}."
|
||||
@@ -8371,6 +8378,8 @@ steps = "Steps"
|
||||
trigger = "Trigger"
|
||||
|
||||
[portal.pipelines.trigger]
|
||||
editor-export = "Every export"
|
||||
editor-upload = "Every upload"
|
||||
folder-watch = "Folder watch"
|
||||
manual = "Manual"
|
||||
schedule = "Scheduled"
|
||||
|
||||
@@ -7,6 +7,7 @@ import ChevronLeftIcon from "@mui/icons-material/ChevronLeft";
|
||||
import ChevronRightIcon from "@mui/icons-material/ChevronRight";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { getFileSize } from "@app/utils/fileUtils";
|
||||
import { toolOperationLabel } from "@app/utils/toolOperationLabel";
|
||||
import { StirlingFileStub } from "@app/types/fileContext";
|
||||
import { PrivateContent } from "@app/components/shared/PrivateContent";
|
||||
|
||||
@@ -115,7 +116,7 @@ const CompactFileDetails: React.FC<CompactFileDetailsProps> = ({
|
||||
{currentFile?.toolHistory && currentFile.toolHistory.length > 0 && (
|
||||
<Text size="xs" c="dimmed">
|
||||
{currentFile.toolHistory
|
||||
.map((tool) => t(`home.${tool.toolId}.title`, tool.toolId))
|
||||
.map((tool) => toolOperationLabel(tool, t))
|
||||
.join(" → ")}
|
||||
</Text>
|
||||
)}
|
||||
|
||||
@@ -10,7 +10,7 @@ import HistoryIcon from "@mui/icons-material/History";
|
||||
import MoreVertIcon from "@mui/icons-material/MoreVert";
|
||||
|
||||
import { FileId, ToolOperation } from "@app/types/file";
|
||||
import { ToolId } from "@app/types/toolId";
|
||||
import { toolOperationLabel } from "@app/utils/toolOperationLabel";
|
||||
import { StirlingFileStub } from "@app/types/fileContext";
|
||||
import { formatFileSize, getFileDate } from "@app/utils/fileUtils";
|
||||
import { downloadFileFromStorage } from "@app/utils/downloadUtils";
|
||||
@@ -64,10 +64,10 @@ function deltaToolFor(
|
||||
return curr[priorLen] ?? null;
|
||||
}
|
||||
|
||||
/** Translated tool name via `home.{toolId}.title`. */
|
||||
function ToolLabel({ toolId }: { toolId: ToolId }) {
|
||||
/** The operation's own label when it has one, else its translated tool name. */
|
||||
function ToolLabel({ operation }: { operation: ToolOperation }) {
|
||||
const { t } = useTranslation();
|
||||
return <span>{t(`home.${toolId}.title`, toolId)}</span>;
|
||||
return <span>{toolOperationLabel(operation, t)}</span>;
|
||||
}
|
||||
|
||||
export interface VersionTimelineProps {
|
||||
@@ -242,7 +242,7 @@ export function VersionTimeline({
|
||||
style={{ color: "var(--c-text)" }}
|
||||
>
|
||||
{delta ? (
|
||||
<ToolLabel toolId={delta.toolId} />
|
||||
<ToolLabel operation={delta} />
|
||||
) : (
|
||||
t("filesPage.versionOrigin", "Original upload")
|
||||
)}
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
import React from "react";
|
||||
import { Text, Tooltip, Badge, Group } from "@mantine/core";
|
||||
import { ToolOperation } from "@app/types/file";
|
||||
import { toolOperationLabel } from "@app/utils/toolOperationLabel";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { ToolId } from "@app/types/toolId";
|
||||
|
||||
interface ToolChainProps {
|
||||
toolChain: ToolOperation[];
|
||||
@@ -29,11 +29,7 @@ const ToolChain: React.FC<ToolChainProps> = ({
|
||||
const { t } = useTranslation();
|
||||
if (!toolChain || toolChain.length === 0) return null;
|
||||
|
||||
const toolIds = toolChain.map((tool) => tool.toolId);
|
||||
|
||||
const getToolName = (toolId: ToolId) => {
|
||||
return t(`home.${toolId}.title`, toolId);
|
||||
};
|
||||
const getToolName = (tool: ToolOperation) => toolOperationLabel(tool, t);
|
||||
|
||||
// Create full tool chain for tooltip
|
||||
const fullChainDisplay =
|
||||
@@ -42,7 +38,7 @@ const ToolChain: React.FC<ToolChainProps> = ({
|
||||
{toolChain.map((tool, index) => (
|
||||
<React.Fragment key={`${tool.toolId}-${index}`}>
|
||||
<Badge size="sm" variant="light" color="blue">
|
||||
{getToolName(tool.toolId)}
|
||||
{getToolName(tool)}
|
||||
</Badge>
|
||||
{index < toolChain.length - 1 && (
|
||||
<Text size="sm" c="dimmed">
|
||||
@@ -53,18 +49,21 @@ const ToolChain: React.FC<ToolChainProps> = ({
|
||||
))}
|
||||
</Group>
|
||||
) : (
|
||||
<Text size="sm">{toolIds.map(getToolName).join(" → ")}</Text>
|
||||
<Text size="sm">{toolChain.map(getToolName).join(" → ")}</Text>
|
||||
);
|
||||
|
||||
// Create truncated display based on available space
|
||||
const getTruncatedDisplay = () => {
|
||||
if (toolIds.length <= 2) {
|
||||
if (toolChain.length <= 2) {
|
||||
// Show all tools if 2 or fewer
|
||||
return { text: toolIds.map(getToolName).join(" → "), isTruncated: false };
|
||||
return {
|
||||
text: toolChain.map(getToolName).join(" → "),
|
||||
isTruncated: false,
|
||||
};
|
||||
} else {
|
||||
// Show first tool ... last tool for longer chains
|
||||
return {
|
||||
text: `${getToolName(toolIds[0])} → +${toolIds.length - 2} → ${getToolName(toolIds[toolIds.length - 1])}`,
|
||||
text: `${getToolName(toolChain[0])} → +${toolChain.length - 2} → ${getToolName(toolChain[toolChain.length - 1])}`,
|
||||
isTruncated: true,
|
||||
};
|
||||
}
|
||||
@@ -75,10 +74,10 @@ const ToolChain: React.FC<ToolChainProps> = ({
|
||||
// Compact style for very small spaces
|
||||
if (displayStyle === "compact") {
|
||||
const compactText =
|
||||
toolIds.length === 1
|
||||
? getToolName(toolIds[0])
|
||||
: `${toolIds.length} tools`;
|
||||
const isCompactTruncated = toolIds.length > 1;
|
||||
toolChain.length === 1
|
||||
? getToolName(toolChain[0])
|
||||
: `${toolChain.length} tools`;
|
||||
const isCompactTruncated = toolChain.length > 1;
|
||||
|
||||
const compactElement = (
|
||||
<Text
|
||||
@@ -116,7 +115,7 @@ const ToolChain: React.FC<ToolChainProps> = ({
|
||||
{toolChain.slice(0, 3).map((tool, index) => (
|
||||
<React.Fragment key={`${tool.toolId}-${index}`}>
|
||||
<Badge size={size} variant="light" color="blue">
|
||||
{getToolName(tool.toolId)}
|
||||
{getToolName(tool)}
|
||||
</Badge>
|
||||
{index < Math.min(toolChain.length - 1, 2) && (
|
||||
<Text size="xs" c="dimmed">
|
||||
@@ -131,7 +130,7 @@ const ToolChain: React.FC<ToolChainProps> = ({
|
||||
...
|
||||
</Text>
|
||||
<Badge size={size} variant="light" color="blue">
|
||||
{getToolName(toolChain[toolChain.length - 1].toolId)}
|
||||
{getToolName(toolChain[toolChain.length - 1])}
|
||||
</Badge>
|
||||
</>
|
||||
)}
|
||||
@@ -140,7 +139,7 @@ const ToolChain: React.FC<ToolChainProps> = ({
|
||||
);
|
||||
|
||||
return isBadgesTruncated ? (
|
||||
<Tooltip label={`${toolIds.map(getToolName).join(" → ")}`} withinPortal>
|
||||
<Tooltip label={`${toolChain.map(getToolName).join(" → ")}`} withinPortal>
|
||||
{badgesElement}
|
||||
</Tooltip>
|
||||
) : (
|
||||
|
||||
@@ -192,6 +192,39 @@ describe("fileContextReducer — silent CONSUME_FILES (background enforcement)",
|
||||
]);
|
||||
});
|
||||
|
||||
it("carries a no-label [] verdict forward (classified, not unclassified)", () => {
|
||||
// "a" was classified and found nothing ([]) - distinct from null (never classified). The output
|
||||
// must inherit [] so the local pass treats it as already-classified and never re-classifies (or
|
||||
// re-bills) it.
|
||||
const start = stateWith([stub("a", { classificationLabels: [] })]);
|
||||
const next = fileContextReducer(start, {
|
||||
type: "CONSUME_FILES",
|
||||
payload: {
|
||||
inputFileIds: ["a" as FileId],
|
||||
outputStirlingFileStubs: [stub("b")],
|
||||
},
|
||||
});
|
||||
expect(next.files.byId["b" as FileId].classificationLabels).toEqual([]);
|
||||
});
|
||||
|
||||
it("prefers a real label over a merge input's no-label [] verdict", () => {
|
||||
// Merge of a labelled file and a no-label one: the output should keep the real label.
|
||||
const start = stateWith([
|
||||
stub("a", { classificationLabels: [] }),
|
||||
stub("b", { classificationLabels: ["Invoice"] }),
|
||||
]);
|
||||
const next = fileContextReducer(start, {
|
||||
type: "CONSUME_FILES",
|
||||
payload: {
|
||||
inputFileIds: ["a" as FileId, "b" as FileId],
|
||||
outputStirlingFileStubs: [stub("c")],
|
||||
},
|
||||
});
|
||||
expect(next.files.byId["c" as FileId].classificationLabels).toEqual([
|
||||
"Invoice",
|
||||
]);
|
||||
});
|
||||
|
||||
it("an output's own classificationLabels win over the input's", () => {
|
||||
// A re-classify produces an output that already carries (fresher) labels.
|
||||
const start = stateWith([stub("a", { classificationLabels: ["Invoice"] })]);
|
||||
@@ -211,7 +244,7 @@ describe("fileContextReducer — silent CONSUME_FILES (background enforcement)",
|
||||
|
||||
it("carries classificationConfidence forward with the labels", () => {
|
||||
// The confidence is part of the verdict: without it the escalation decision
|
||||
// (shouldDispatchToAi) dies at the version boundary and a chained
|
||||
// (localVerdictNeedsEscalation) dies at the version boundary and a chained
|
||||
// classification never runs.
|
||||
const start = stateWith([
|
||||
stub("a", {
|
||||
|
||||
@@ -389,16 +389,16 @@ export function fileContextReducer(
|
||||
// Carry the document's classification verdict forward across the edit: any
|
||||
// tool that versions/derives a classified file keeps it in its label
|
||||
// groups instead of dropping to "Other" and waiting on a PDF re-read.
|
||||
// Inherited from the first input that has labels, together with that
|
||||
// verdict's confidence - the escalation decision (shouldDispatchToAi) is
|
||||
// about the document, not about which step produced the current bytes, so
|
||||
// it must survive the version boundary. An output that already carries its
|
||||
// own verdict (e.g. a fresh classify result) keeps it.
|
||||
const verdictDonor = inputFileIds
|
||||
.map((id) => state.files.byId[id])
|
||||
.find(
|
||||
// Inherited together with that verdict's confidence - the escalation
|
||||
// decision (localVerdictNeedsEscalation) is about the document, not about
|
||||
// which step produced the current bytes, so it must survive the version
|
||||
// boundary. An output that already carries its own verdict (e.g. a fresh
|
||||
// classify result) keeps it.
|
||||
const inputStubs = inputFileIds.map((id) => state.files.byId[id]);
|
||||
const verdictDonor =
|
||||
inputStubs.find(
|
||||
(s) => s?.classificationLabels && s.classificationLabels.length > 0,
|
||||
);
|
||||
) ?? inputStubs.find((s) => s?.classificationLabels !== undefined);
|
||||
|
||||
// Mark every consume output as tool-produced (the single chokepoint for
|
||||
// both versioned edits and independent artifacts like convert/split/merge)
|
||||
@@ -409,7 +409,7 @@ export function fileContextReducer(
|
||||
...stub,
|
||||
derivedFromTool: true,
|
||||
sourceFileIds,
|
||||
...(stub.classificationLabels == null && verdictDonor
|
||||
...(stub.classificationLabels === undefined && verdictDonor
|
||||
? {
|
||||
classificationLabels: verdictDonor.classificationLabels,
|
||||
classificationConfidence:
|
||||
|
||||
@@ -14,6 +14,8 @@ export async function createStirlingFilesAndStubs(
|
||||
files: File[],
|
||||
parentStub: StirlingFileStub,
|
||||
toolId: ToolId,
|
||||
/** Shown instead of the tool's name in version history (a policy passes its pipeline name). */
|
||||
label?: string,
|
||||
): Promise<{ stirlingFiles: StirlingFile[]; stubs: StirlingFileStub[] }> {
|
||||
const stirlingFiles: StirlingFile[] = [];
|
||||
const stubs: StirlingFileStub[] = [];
|
||||
@@ -22,7 +24,7 @@ export async function createStirlingFilesAndStubs(
|
||||
const processedFileMetadata = await generateProcessedFileMetadata(file);
|
||||
const childStub = createChildStub(
|
||||
parentStub,
|
||||
{ toolId, timestamp: Date.now() },
|
||||
{ toolId, timestamp: Date.now(), ...(label ? { label } : {}) },
|
||||
file,
|
||||
processedFileMetadata?.thumbnailUrl,
|
||||
processedFileMetadata,
|
||||
|
||||
@@ -12,14 +12,20 @@ const FIXTURES = path.join(
|
||||
"../test-fixtures/classification/unlabelled",
|
||||
);
|
||||
|
||||
/** The stored policy DefaultClassificationPolicySeeder writes for a new team. */
|
||||
/**
|
||||
* What GET /api/v1/policies returns for the row an older
|
||||
* DefaultClassificationPolicySeeder wrote - i.e. one stored before editor
|
||||
* participation had its own field. `JpaPolicyStore.liftEditorConfig` derives the
|
||||
* `editor` block from the legacy `output.options` on read; the lift is additive,
|
||||
* so a real response carries both. Migration of the stored shape itself is
|
||||
* covered by JpaPolicyStoreTest, which exercises the Java the stub stands in for.
|
||||
*/
|
||||
const SEEDED_POLICY = {
|
||||
id: "seeded-classification",
|
||||
name: "Classification Policy",
|
||||
owner: "system",
|
||||
enabled: true,
|
||||
trigger: null,
|
||||
sourceIds: [],
|
||||
inputs: [],
|
||||
steps: [{ operation: "/api/v1/ai/tools/classify-and-label", parameters: {} }],
|
||||
output: {
|
||||
type: "inline",
|
||||
@@ -32,7 +38,9 @@ const SEEDED_POLICY = {
|
||||
reviewerEmail: "",
|
||||
},
|
||||
},
|
||||
outputIds: [],
|
||||
teamId: 1,
|
||||
editor: { allowed: true, runOn: "upload" },
|
||||
};
|
||||
|
||||
test("a 10-file upload wave classifies every file into its group", async ({
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import path from "path";
|
||||
import { test, expect } from "@app/tests/helpers/stub-test-base";
|
||||
import { uploadFiles } from "@app/tests/helpers/ui-helpers";
|
||||
|
||||
// A pipeline reaches the editor auto-run through its own editor flag; a swept one must not.
|
||||
|
||||
test.use({ autoGoto: false });
|
||||
|
||||
const SAMPLE = path.join(
|
||||
import.meta.dirname,
|
||||
"../test-fixtures/classification/unlabelled/invoice_acme.pdf",
|
||||
);
|
||||
|
||||
/** A builder-made pipeline: no categoryId, one harmless step. */
|
||||
function builderPipeline(editor: { allowed: boolean; runOn: string }) {
|
||||
return {
|
||||
id: "builder-pipeline-1",
|
||||
name: "Flatten everything",
|
||||
owner: "system",
|
||||
enabled: true,
|
||||
trigger: null,
|
||||
sourceIds: [],
|
||||
steps: [{ operation: "/api/v1/misc/flatten", parameters: {} }],
|
||||
output: { type: "inline", options: { mode: "new_version" } },
|
||||
editor,
|
||||
teamId: 1,
|
||||
};
|
||||
}
|
||||
|
||||
/** Install the policy list + capture every stored-policy run dispatch. */
|
||||
async function armed(page: import("@playwright/test").Page, policy: unknown) {
|
||||
const dispatched: string[] = [];
|
||||
await page.route("**/api/v1/policies", (route) =>
|
||||
route.fulfill({ json: [policy] }),
|
||||
);
|
||||
await page.route("**/api/v1/policies/*/run", (route) => {
|
||||
dispatched.push(new URL(route.request().url()).pathname);
|
||||
return route.fulfill({ json: { jobId: "job-1" } });
|
||||
});
|
||||
return dispatched;
|
||||
}
|
||||
|
||||
test("an editor pipeline set to run on upload dispatches when a file is added", async ({
|
||||
page,
|
||||
}) => {
|
||||
const dispatched = await armed(
|
||||
page,
|
||||
builderPipeline({ allowed: true, runOn: "upload" }),
|
||||
);
|
||||
|
||||
await page.goto("/editor", { waitUntil: "domcontentloaded" });
|
||||
await uploadFiles(page, SAMPLE);
|
||||
|
||||
await expect
|
||||
.poll(() => dispatched, { timeout: 15_000 })
|
||||
.toContain("/api/v1/policies/builder-pipeline-1/run");
|
||||
});
|
||||
|
||||
test("a swept pipeline never runs on editor upload", async ({ page }) => {
|
||||
const dispatched = await armed(
|
||||
page,
|
||||
builderPipeline({ allowed: false, runOn: "upload" }),
|
||||
);
|
||||
|
||||
await page.goto("/editor", { waitUntil: "domcontentloaded" });
|
||||
await uploadFiles(page, SAMPLE);
|
||||
|
||||
await page.waitForTimeout(5_000);
|
||||
expect(dispatched).toEqual([]);
|
||||
});
|
||||
|
||||
test("an editor pipeline set to run on export does not fire on upload", async ({
|
||||
page,
|
||||
}) => {
|
||||
const dispatched = await armed(
|
||||
page,
|
||||
builderPipeline({ allowed: true, runOn: "export" }),
|
||||
);
|
||||
|
||||
await page.goto("/editor", { waitUntil: "domcontentloaded" });
|
||||
await uploadFiles(page, SAMPLE);
|
||||
|
||||
await page.waitForTimeout(5_000);
|
||||
expect(dispatched).toEqual([]);
|
||||
});
|
||||
@@ -16,6 +16,9 @@ export type FileId = string & { readonly [tag]: "FileId" };
|
||||
export interface ToolOperation {
|
||||
toolId: ToolId;
|
||||
timestamp: number;
|
||||
/** Overrides the tool's own name in history. Set by a policy run to its pipeline's name, since
|
||||
* every policy records the same "automate" toolId. */
|
||||
label?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import type { TFunction } from "i18next";
|
||||
import { toolOperationLabel } from "@app/utils/toolOperationLabel";
|
||||
import type { ToolOperation } from "@app/types/file";
|
||||
|
||||
// Stands in for i18next: echoes the key so the assertions show which lookup ran.
|
||||
const t = ((key: string, fallback?: string) =>
|
||||
key === "home.automate.title" ? "Automate" : (fallback ?? key)) as TFunction;
|
||||
|
||||
const op = (over: Partial<ToolOperation>): ToolOperation =>
|
||||
({ toolId: "automate", timestamp: 0, ...over }) as ToolOperation;
|
||||
|
||||
describe("toolOperationLabel", () => {
|
||||
it("prefers the operation's own label", () => {
|
||||
expect(toolOperationLabel(op({ label: "add-page-numbers" }), t)).toBe(
|
||||
"add-page-numbers",
|
||||
);
|
||||
});
|
||||
|
||||
// Every policy records the same "automate" toolId, so without a label each automated version
|
||||
// reads identically no matter which pipeline produced it.
|
||||
it("falls back to the tool's name when unlabelled", () => {
|
||||
expect(toolOperationLabel(op({}), t)).toBe("Automate");
|
||||
});
|
||||
|
||||
it("keeps the fallback for an empty label rather than rendering a blank", () => {
|
||||
expect(toolOperationLabel(op({ label: "" }), t)).toBe("Automate");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
import type { TFunction } from "i18next";
|
||||
import type { ToolOperation } from "@app/types/file";
|
||||
|
||||
/**
|
||||
* What produced a version, for the history surfaces. A policy run carries its own label (the
|
||||
* pipeline's name) because every policy records the same "automate" toolId, which would otherwise
|
||||
* render every automated version identically.
|
||||
*/
|
||||
export function toolOperationLabel(
|
||||
operation: ToolOperation,
|
||||
t: TFunction,
|
||||
): string {
|
||||
// Truthiness, not nullish: a blank label would otherwise render as an empty history entry.
|
||||
return (
|
||||
operation.label || t(`home.${operation.toolId}.title`, operation.toolId)
|
||||
);
|
||||
}
|
||||
@@ -70,6 +70,8 @@ export interface Policy {
|
||||
* output} is used.
|
||||
*/
|
||||
outputIds: string[];
|
||||
/** Whether the editor runs this policy per file, and on which moment. */
|
||||
editor?: { allowed: boolean; runOn: "upload" | "export" };
|
||||
teamId?: number | null;
|
||||
}
|
||||
|
||||
|
||||
@@ -76,6 +76,8 @@ export interface PolicyState {
|
||||
configured: boolean;
|
||||
status: PolicyStatus;
|
||||
sources: string[];
|
||||
/** Whether the editor runs this policy per file; stored, not derived from `sources`. */
|
||||
runsOnEditor?: boolean;
|
||||
scopeTypes: string[];
|
||||
reviewerEmail: string;
|
||||
fieldValues: Record<string, boolean | string | string[]>;
|
||||
@@ -92,6 +94,7 @@ export interface PolicyState {
|
||||
export interface PolicySetupResult {
|
||||
fieldValues: Record<string, boolean | string | string[]>;
|
||||
sources: string[];
|
||||
runsOnEditor: boolean;
|
||||
scopeTypes: string[];
|
||||
reviewerEmail: string;
|
||||
outputMode: "new_file" | "new_version";
|
||||
@@ -433,6 +436,7 @@ function decoratePolicy(
|
||||
configured: true,
|
||||
status,
|
||||
sources: decoded.sources,
|
||||
runsOnEditor: decoded.runsOnEditor,
|
||||
scopeTypes: decoded.scopeTypes,
|
||||
reviewerEmail: decoded.reviewerEmail,
|
||||
fieldValues: decoded.fieldValues,
|
||||
@@ -595,6 +599,7 @@ export function buildWireFromSetup(
|
||||
enabled,
|
||||
categoryId: entry.category.id,
|
||||
sources: result.sources,
|
||||
runsOnEditor: result.runsOnEditor,
|
||||
scopeTypes: result.scopeTypes,
|
||||
reviewerEmail: result.reviewerEmail,
|
||||
fieldValues: result.fieldValues,
|
||||
@@ -625,6 +630,8 @@ export function buildWireFromState(
|
||||
enabled,
|
||||
categoryId: entry.category.id,
|
||||
sources: s.sources,
|
||||
// Carry the stored value through: pause/resume must not re-derive it.
|
||||
runsOnEditor: s.runsOnEditor === true,
|
||||
scopeTypes: s.scopeTypes,
|
||||
reviewerEmail: s.reviewerEmail,
|
||||
fieldValues: s.fieldValues,
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
// Swept sources are scheduled or triggered server-side; the editor runs client-side as each file
|
||||
// passes through, so the two get different controls.
|
||||
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Tooltip } from "@mantine/core";
|
||||
import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined";
|
||||
import { FormField, Input, Select } from "@app/ui";
|
||||
|
||||
export type ScheduleUnit = "MINUTES" | "HOURS" | "DAYS";
|
||||
export type EditorRunOn = "upload" | "export";
|
||||
|
||||
const SCHEDULE_UNITS: ScheduleUnit[] = ["MINUTES", "HOURS", "DAYS"];
|
||||
|
||||
/** Empty trigger type = manual-only (no automatic trigger). */
|
||||
export const MANUAL = "";
|
||||
/** Sentinel for manual: Mantine's Select reads "" as no selection. Maps to {@link MANUAL}. */
|
||||
export const MANUAL_OPTION = "manual";
|
||||
|
||||
/** One input row in the builder: a source paired with its own trigger config. */
|
||||
export interface WorkingInput {
|
||||
sourceId: string;
|
||||
triggerType: string;
|
||||
scheduleCount: string;
|
||||
scheduleUnit: ScheduleUnit;
|
||||
}
|
||||
|
||||
export interface PipelineInputTriggerProps {
|
||||
input: WorkingInput;
|
||||
onInputChange: (patch: Partial<WorkingInput>) => void;
|
||||
/** Trigger types offered for this row's source (manual first). */
|
||||
triggerOptions: { value: string; label: string }[];
|
||||
/** The chosen source is the editor, so the pipeline runs in the browser. */
|
||||
isEditorInput: boolean;
|
||||
runOn: EditorRunOn;
|
||||
onRunOnChange: (runOn: EditorRunOn) => void;
|
||||
}
|
||||
|
||||
export function PipelineInputTrigger({
|
||||
input,
|
||||
onInputChange,
|
||||
triggerOptions,
|
||||
isEditorInput,
|
||||
runOn,
|
||||
onRunOnChange,
|
||||
}: PipelineInputTriggerProps) {
|
||||
const { t } = useTranslation();
|
||||
|
||||
if (isEditorInput) {
|
||||
const label = t("portal.pipelines.builder.runOn", "Runs on");
|
||||
return (
|
||||
<FormField
|
||||
label={
|
||||
<Tooltip
|
||||
label={t(
|
||||
"portal.pipelines.builder.runOnTooltip",
|
||||
"Choose when this pipeline runs on your files: when you add them, or when you export them.",
|
||||
)}
|
||||
position="right"
|
||||
withinPortal
|
||||
multiline
|
||||
w={260}
|
||||
>
|
||||
<span className="portal-builder__label-hint">
|
||||
{label}
|
||||
<InfoOutlinedIcon style={{ fontSize: "0.875rem" }} />
|
||||
</span>
|
||||
</Tooltip>
|
||||
}
|
||||
>
|
||||
<Select
|
||||
inputSize="sm"
|
||||
aria-label={label}
|
||||
value={runOn}
|
||||
onChange={(value) =>
|
||||
onRunOnChange(value === "export" ? "export" : "upload")
|
||||
}
|
||||
options={[
|
||||
{
|
||||
value: "upload",
|
||||
label: t("portal.pipelines.builder.runOnUpload", "Every upload"),
|
||||
},
|
||||
{
|
||||
value: "export",
|
||||
label: t("portal.pipelines.builder.runOnExport", "Every export"),
|
||||
},
|
||||
]}
|
||||
/>
|
||||
</FormField>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<FormField label={t("portal.pipelines.builder.inputTrigger")}>
|
||||
<Select
|
||||
inputSize="sm"
|
||||
aria-label={t("portal.pipelines.builder.inputTrigger")}
|
||||
value={
|
||||
input.triggerType === MANUAL ? MANUAL_OPTION : input.triggerType
|
||||
}
|
||||
disabled={input.sourceId === ""}
|
||||
onChange={(value) =>
|
||||
onInputChange({
|
||||
triggerType: value && value !== MANUAL_OPTION ? value : MANUAL,
|
||||
})
|
||||
}
|
||||
options={triggerOptions}
|
||||
/>
|
||||
</FormField>
|
||||
|
||||
{input.triggerType === "schedule" && (
|
||||
<div className="portal-builder__schedule">
|
||||
<span className="portal-builder__muted">
|
||||
{t("portal.pipelines.composer.scheduleEvery")}
|
||||
</span>
|
||||
<Input
|
||||
inputSize="sm"
|
||||
type="number"
|
||||
min={1}
|
||||
value={input.scheduleCount}
|
||||
invalid={Number(input.scheduleCount) <= 0}
|
||||
onChange={(e) => onInputChange({ scheduleCount: e.target.value })}
|
||||
className="portal-builder__schedule-count"
|
||||
/>
|
||||
<Select
|
||||
inputSize="sm"
|
||||
value={input.scheduleUnit}
|
||||
onChange={(value) =>
|
||||
value && onInputChange({ scheduleUnit: value as ScheduleUnit })
|
||||
}
|
||||
options={SCHEDULE_UNITS.map((unit) => ({
|
||||
value: unit,
|
||||
label: t(`portal.pipelines.composer.unit.${unit.toLowerCase()}`),
|
||||
}))}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -51,6 +51,8 @@ export interface GraphNodeContent {
|
||||
warning?: string;
|
||||
/** Why the input will not be much use. */
|
||||
inputWarning?: ChainWarning;
|
||||
/** An end the pipeline decides for itself, so it carries no remove control. */
|
||||
fixed?: boolean;
|
||||
}
|
||||
|
||||
export interface GraphStepContent extends GraphNodeContent {
|
||||
@@ -294,7 +296,9 @@ export function PipelineGraph({
|
||||
warning={content.warning}
|
||||
selected={selected === kind}
|
||||
onSelect={() => onSelect(kind)}
|
||||
onRemove={() => onRemoveEnd(kind)}
|
||||
onRemove={
|
||||
content.fixed ? undefined : () => onRemoveEnd(kind)
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -116,12 +116,15 @@ export function PolicyDetailPanel({
|
||||
const { category, config, state, steps, stats, activity } = policy;
|
||||
const isPaused = state.status === "paused";
|
||||
const canDelete = state.isDefault !== true;
|
||||
// Editor participation is its own flag (runsOnEditor), not a source. A legacy policy still carries
|
||||
// "editor" in its stored sources until re-saved, so drop it here to count only real watched sources.
|
||||
const realSources = state.sources.filter((s) => s !== "editor");
|
||||
// Processed history only exists for watched sources; editor uploads are never ledgered.
|
||||
const canClearHistory =
|
||||
onClearHistory !== undefined && state.sources.some((s) => s !== "editor");
|
||||
onClearHistory !== undefined && realSources.length > 0;
|
||||
|
||||
const enforceItems = steps.length > 0 ? steps.map((s) => s.operation) : null;
|
||||
const hasEditorSource = state.sources.includes("editor");
|
||||
const hasEditorSource = state.runsOnEditor === true;
|
||||
const trigger =
|
||||
state.runOn === "export"
|
||||
? t("portal.policies.detail.onEveryExport")
|
||||
@@ -131,11 +134,6 @@ export function PolicyDetailPanel({
|
||||
? t("portal.policies.detail.outputAsNewFile")
|
||||
: t("portal.policies.detail.outputAsNewVersion");
|
||||
|
||||
function sourceLabel(id: string) {
|
||||
if (id === "editor") return t("portal.sources.types.editor.label");
|
||||
return id;
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<Modal
|
||||
@@ -243,13 +241,13 @@ export function PolicyDetailPanel({
|
||||
</div>
|
||||
|
||||
{/* Sources */}
|
||||
{state.sources.length > 0 && (
|
||||
{realSources.length > 0 && (
|
||||
<div className="portal-policies__detail-inline">
|
||||
<span className="portal-policies__detail-inline-label">
|
||||
{t("portal.policies.detail.sources")}
|
||||
</span>
|
||||
<span className="portal-policies__detail-inline-value">
|
||||
{state.sources.map(sourceLabel).join(" · ")}
|
||||
{realSources.join(" · ")}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -268,8 +268,14 @@ function PolicySetupWizardBody({
|
||||
const [fieldValues, setFieldValues] = useState(() =>
|
||||
resolveFieldValues(entry),
|
||||
);
|
||||
const [sources, setSources] = useState<string[]>(
|
||||
policy?.state.sources ?? ["editor"],
|
||||
// Real sources only; editor participation is its own flag, not an entry here.
|
||||
const [sources, setSources] = useState<string[]>(() =>
|
||||
(policy?.state.sources ?? []).filter((s) => s !== "editor"),
|
||||
);
|
||||
// Whether the policy runs in the editor. Defaults on for a new policy (the common case);
|
||||
// on edit it comes straight from the stored flag, never re-derived from the sources list.
|
||||
const [runsOnEditor, setRunsOnEditor] = useState<boolean>(
|
||||
policy?.state.runsOnEditor ?? true,
|
||||
);
|
||||
|
||||
const sourcesAsync = useSources();
|
||||
@@ -361,6 +367,12 @@ function PolicySetupWizardBody({
|
||||
}
|
||||
|
||||
function toggleSource(id: string) {
|
||||
// The editor is not a real source: its tile toggles the runsOnEditor flag instead of
|
||||
// adding "editor" to the sources list.
|
||||
if (id === "editor") {
|
||||
setRunsOnEditor((on) => !on);
|
||||
return;
|
||||
}
|
||||
setSources((prev) =>
|
||||
prev.includes(id) ? prev.filter((s) => s !== id) : [...prev, id],
|
||||
);
|
||||
@@ -382,6 +394,7 @@ function PolicySetupWizardBody({
|
||||
await onSubmit(entry, {
|
||||
fieldValues,
|
||||
sources,
|
||||
runsOnEditor,
|
||||
scopeTypes,
|
||||
reviewerEmail,
|
||||
outputMode,
|
||||
@@ -598,7 +611,8 @@ function PolicySetupWizardBody({
|
||||
// loaded list is never empty - no "no sources" state exists.
|
||||
<div className="portal-policies__sources">
|
||||
{availableSources.map((src) => {
|
||||
const on = sources.includes(src.id);
|
||||
const on =
|
||||
src.id === "editor" ? runsOnEditor : sources.includes(src.id);
|
||||
return (
|
||||
<Button
|
||||
key={src.id}
|
||||
@@ -636,7 +650,7 @@ function PolicySetupWizardBody({
|
||||
{t("portal.policies.wizard.output.heading")}
|
||||
</h3>
|
||||
<div className="portal-policies__fields">
|
||||
{sources.includes("editor") && (
|
||||
{runsOnEditor && (
|
||||
<>
|
||||
<FormField
|
||||
label={t("portal.policies.wizard.output.runOn.label")}
|
||||
|
||||
@@ -194,6 +194,18 @@
|
||||
color: var(--c-text-subtle);
|
||||
}
|
||||
|
||||
/* A field label that carries an info icon explaining the choice. */
|
||||
.portal-builder__label-hint {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.25rem;
|
||||
cursor: help;
|
||||
}
|
||||
|
||||
.portal-builder__label-hint svg {
|
||||
color: var(--c-text-subtle);
|
||||
}
|
||||
|
||||
.portal-builder__input-row .portal-builder__source-edit:hover:not(:disabled) {
|
||||
color: var(--c-accent-fg, var(--c-primary));
|
||||
}
|
||||
|
||||
@@ -270,6 +270,20 @@ const POLICY: Policy = {
|
||||
outputIds: [],
|
||||
};
|
||||
|
||||
/** The built-in editor source, offered as an input so a pipeline can run in the browser. */
|
||||
const EDITOR_SOURCE: SourceView = {
|
||||
id: "src-editor",
|
||||
name: "Editor",
|
||||
type: "editor",
|
||||
status: "active",
|
||||
referenceCount: 0,
|
||||
referencingPolicies: [],
|
||||
config: [],
|
||||
docsTotal: 0,
|
||||
docs24h: 0,
|
||||
docs30d: 0,
|
||||
};
|
||||
|
||||
const SOURCE: SourceView = {
|
||||
id: "src-in",
|
||||
name: "Claims intake",
|
||||
@@ -799,6 +813,31 @@ describe("PipelineBuilder", () => {
|
||||
expect(screen.getByText("source-modal:src-1")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("saves an editor pipeline as its own flag, not as a wire input", async () => {
|
||||
fetchSources.mockResolvedValue({
|
||||
kpis: [],
|
||||
sources: [SOURCE, EDITOR_SOURCE],
|
||||
});
|
||||
renderBuilder("/processor/pipelines/new");
|
||||
fireEvent.change(
|
||||
await screen.findByLabelText("portal.pipelines.composer.name"),
|
||||
{ target: { value: "Label on upload" } },
|
||||
);
|
||||
await addTool("Compress");
|
||||
await pickInputSource("Editor");
|
||||
|
||||
fireEvent.click(screen.getByText("portal.pipelines.composer.create"));
|
||||
|
||||
await waitFor(() => expect(savePipeline).toHaveBeenCalledTimes(1));
|
||||
const body = savePipeline.mock.calls[0][0];
|
||||
// The editor is virtual: nothing sweeps it server-side, so it is recorded as the policy's own
|
||||
// editor flag rather than as an input the backend would try to pull from.
|
||||
expect(body.inputs).toEqual([]);
|
||||
expect(body.editor).toEqual({ allowed: true, runOn: "upload" });
|
||||
// And it needs no destination - results land back in the workspace the file came from.
|
||||
expect(body.outputIds).toEqual([]);
|
||||
});
|
||||
|
||||
it("runs an existing pipeline and reports success", async () => {
|
||||
renderBuilder("/processor/pipelines/plc-1");
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ import {
|
||||
Banner,
|
||||
Button,
|
||||
FormField,
|
||||
Input,
|
||||
Modal,
|
||||
Select,
|
||||
Spinner,
|
||||
@@ -103,20 +102,16 @@ import {
|
||||
newIntegrationStep,
|
||||
stepOperation,
|
||||
} from "@portal/components/pipelines/integrationStep";
|
||||
import {
|
||||
MANUAL,
|
||||
MANUAL_OPTION,
|
||||
PipelineInputTrigger,
|
||||
type EditorRunOn,
|
||||
type ScheduleUnit,
|
||||
type WorkingInput,
|
||||
} from "@portal/components/pipelines/PipelineInputTrigger";
|
||||
import "@portal/views/PipelineBuilder.css";
|
||||
|
||||
type ScheduleUnit = "MINUTES" | "HOURS" | "DAYS";
|
||||
|
||||
const SCHEDULE_UNITS: ScheduleUnit[] = ["MINUTES", "HOURS", "DAYS"];
|
||||
/** Empty trigger type = manual-only (no automatic trigger). */
|
||||
const MANUAL = "";
|
||||
/**
|
||||
* Sentinel value for the manual choice in the trigger dropdown. Mantine's Select treats an empty
|
||||
* string as "no selection" (it shows the placeholder, not the option), so the manual option needs a
|
||||
* real value; it maps to/from the empty {@link MANUAL} trigger type at the edges.
|
||||
*/
|
||||
const MANUAL_OPTION = "manual";
|
||||
|
||||
const TERMINAL_STATUSES = new Set(["COMPLETED", "FAILED", "CANCELLED"]);
|
||||
const POLL_INTERVAL_MS = 1500;
|
||||
const POLL_ATTEMPTS = 60;
|
||||
@@ -149,14 +144,6 @@ function parseTrigger(trigger: TriggerConfig | null): {
|
||||
return { triggerType: trigger.type, count: "1", unit: "HOURS" };
|
||||
}
|
||||
|
||||
/** One input row in the builder: a source paired with its own trigger config. */
|
||||
interface WorkingInput {
|
||||
sourceId: string;
|
||||
triggerType: string;
|
||||
scheduleCount: string;
|
||||
scheduleUnit: ScheduleUnit;
|
||||
}
|
||||
|
||||
/** The input row with nothing chosen yet: no source, manual trigger. */
|
||||
function blankInput(): WorkingInput {
|
||||
return {
|
||||
@@ -237,13 +224,10 @@ export function PipelineBuilder() {
|
||||
async () => await fetchTriggers(),
|
||||
[],
|
||||
);
|
||||
// The editor is a built-in, client-driven source (it runs on editor upload,
|
||||
// not as a pipeline input), so it's excluded from a pipeline's inputs.
|
||||
// Includes the virtual editor source: a valid input, but never a wire input (see save) and not
|
||||
// writable, so isWritableSource keeps it out of the destinations below.
|
||||
const availableSources = useMemo<SourceView[]>(
|
||||
() =>
|
||||
(sourcesState.data?.sources ?? []).filter(
|
||||
(source) => source.type !== EDITOR_SOURCE_TYPE,
|
||||
),
|
||||
() => sourcesState.data?.sources ?? [],
|
||||
[sourcesState.data],
|
||||
);
|
||||
// A destination is a source used as a write target: only writable types (folder/S3, filtered per
|
||||
@@ -262,6 +246,17 @@ export function PipelineBuilder() {
|
||||
// Exactly one input: the row is always present, so the working state is a single object; the
|
||||
// wire shape stays a list (see save()).
|
||||
const [input, setInput] = useState<WorkingInput>(blankInput);
|
||||
// When the editor is the source, the pipeline fires client-side on each file: on upload as it
|
||||
// arrives, or on export as it leaves. Meaningless for a swept source, which has no such moment.
|
||||
const [runOn, setRunOn] = useState<EditorRunOn>("upload");
|
||||
const isEditorInput = useMemo(
|
||||
() =>
|
||||
availableSources.some(
|
||||
(source) =>
|
||||
source.id === input.sourceId && source.type === EDITOR_SOURCE_TYPE,
|
||||
),
|
||||
[availableSources, input.sourceId],
|
||||
);
|
||||
const [steps, setSteps] = useState<WorkingToolStep[]>([]);
|
||||
/** Which node the inspector is editing: an end of the chain, a step, or nothing. */
|
||||
const [selected, setSelected] = useState<GraphSelection>(null);
|
||||
@@ -345,13 +340,27 @@ export function PipelineBuilder() {
|
||||
if (seeded) return;
|
||||
if (isEdit && !policyState.data) return;
|
||||
const policy = policyState.data ?? undefined;
|
||||
// An editor pipeline is recognised by the editor source id, which arrives with the sources
|
||||
// fetch. If the policy loads first, seeding now would latch a blank input and re-save the
|
||||
// pipeline off the editor (editor.allowed:false), so wait for that fetch to settle.
|
||||
if (policy?.editor?.allowed && !sourcesState.data && !sourcesState.error) {
|
||||
return;
|
||||
}
|
||||
setName(policy?.name ?? "");
|
||||
setEnabled(policy?.enabled ?? true);
|
||||
// The one input row is always present: blank for a new pipeline (or a legacy policy saved
|
||||
// without inputs), the stored input for an edit. A legacy multi-input policy shows only its
|
||||
// first input; saving persists just that one (the backend rejects more anyway).
|
||||
// An editor pipeline has no wire input; it is recognised by its recorded sources.
|
||||
const editorSourceId = (sourcesState.data?.sources ?? []).find(
|
||||
(source) => source.type === EDITOR_SOURCE_TYPE,
|
||||
)?.id;
|
||||
setRunOn(policy?.editor?.runOn === "export" ? "export" : "upload");
|
||||
const stored = policy?.inputs[0];
|
||||
if (stored) {
|
||||
const seedsEditor = Boolean(policy?.editor?.allowed && editorSourceId);
|
||||
if (seedsEditor && editorSourceId) {
|
||||
setInput({ ...blankInput(), sourceId: editorSourceId });
|
||||
} else if (stored) {
|
||||
const trigger = parseTrigger(stored.trigger);
|
||||
setInput({
|
||||
sourceId: stored.sourceId,
|
||||
@@ -365,9 +374,16 @@ export function PipelineBuilder() {
|
||||
setSteps(
|
||||
(policy?.steps ?? []).map((step) => deserializeToolStep(step, allTools)),
|
||||
);
|
||||
setOutputIds(policy?.outputIds ?? []);
|
||||
setOutputIds(seedsEditor ? [] : (policy?.outputIds ?? []));
|
||||
setSeeded(true);
|
||||
}, [isEdit, policyState.data, allTools, seeded]);
|
||||
}, [
|
||||
isEdit,
|
||||
policyState.data,
|
||||
allTools,
|
||||
seeded,
|
||||
sourcesState.data,
|
||||
sourcesState.error,
|
||||
]);
|
||||
|
||||
const sourceType = (sourceId: string) =>
|
||||
availableSources.find((s) => s.id === sourceId)?.type;
|
||||
@@ -413,8 +429,8 @@ export function PipelineBuilder() {
|
||||
// Changing the source may make the current trigger incompatible (folder-watch on a non-folder);
|
||||
// drop it back to manual when that happens so the row can't hold an invalid pairing.
|
||||
function changeInputSource(sourceId: string) {
|
||||
setInput((current) => {
|
||||
const type = sourceType(sourceId);
|
||||
setInput((current) => {
|
||||
const trigger = triggers.find((tr) => tr.type === current.triggerType);
|
||||
const keepTrigger =
|
||||
current.triggerType === MANUAL ||
|
||||
@@ -425,6 +441,11 @@ export function PipelineBuilder() {
|
||||
triggerType: keepTrigger ? current.triggerType : MANUAL,
|
||||
};
|
||||
});
|
||||
// The editor hands results back to the workspace, so it has no destination to choose.
|
||||
if (type === EDITOR_SOURCE_TYPE) {
|
||||
setOutputIds([]);
|
||||
setOutputAsked(false);
|
||||
}
|
||||
}
|
||||
|
||||
/** Put an end on the chain and open it, so the click that asks for it also offers the choice. */
|
||||
@@ -667,10 +688,14 @@ export function PipelineBuilder() {
|
||||
// Each validity condition is defined exactly once here, then consumed both by the graph (which
|
||||
// flags each end) and by the blocker list below.
|
||||
const sourceChosen = input.sourceId !== "";
|
||||
// An editor pipeline has no trigger to schedule: it fires as each file passes through.
|
||||
const scheduleValid =
|
||||
input.triggerType !== "schedule" || Number(input.scheduleCount) > 0;
|
||||
isEditorInput ||
|
||||
input.triggerType !== "schedule" ||
|
||||
Number(input.scheduleCount) > 0;
|
||||
const inputValid = sourceChosen && scheduleValid;
|
||||
const outputValid = outputIds.length === 1;
|
||||
// Nor a destination: an editor pipeline's results land back in the workspace the file came from.
|
||||
const outputValid = isEditorInput || outputIds.length === 1;
|
||||
|
||||
// The single source of truth for "can this be committed": every reason it can't be, in the order
|
||||
// they appear down the form, so a disabled Create / Save button can say exactly what is still owed.
|
||||
@@ -781,13 +806,19 @@ export function PipelineBuilder() {
|
||||
id: policyState.data?.id ?? undefined,
|
||||
name: name.trim(),
|
||||
enabled: enabledOverride ?? enabled,
|
||||
// The wire shape stays a list; canSave guarantees the one input has a source.
|
||||
inputs: [{ sourceId: input.sourceId, trigger: buildTriggerFor(input) }],
|
||||
// The editor is virtual - there is no stored Source to pull from, and nothing server-side
|
||||
// sweeps it - so it is never a wire input; its participation is recorded on `editor` below.
|
||||
inputs: isEditorInput
|
||||
? []
|
||||
: [{ sourceId: input.sourceId, trigger: buildTriggerFor(input) }],
|
||||
steps: await serializeStepsForSave(),
|
||||
// Destinations are the referenced saved sources; the inline output field is
|
||||
// preserved as-is (e.g. an editor policy's membership metadata) or defaults to inline.
|
||||
// Destinations are the referenced saved sources; the inline output is preserved as-is
|
||||
// or defaults to inline.
|
||||
output: policyState.data?.output ?? { type: "inline", options: {} },
|
||||
outputIds,
|
||||
editor: { allowed: isEditorInput, runOn },
|
||||
// An editor pipeline delivers back into the workspace. A stored destination would send the
|
||||
// run to a folder or bucket instead, leaving the editor's copy untouched.
|
||||
outputIds: isEditorInput ? [] : outputIds,
|
||||
};
|
||||
await savePipeline(policy);
|
||||
await invalidatePipelines();
|
||||
@@ -1046,6 +1077,11 @@ export function PipelineBuilder() {
|
||||
|
||||
/** How this input fires, in a few words, for the input node's summary line. */
|
||||
function triggerSummary(): string {
|
||||
// The editor has no trigger to schedule; it fires as each file passes through.
|
||||
if (isEditorInput)
|
||||
return runOn === "export"
|
||||
? t("portal.pipelines.builder.runOnExport", "Every export")
|
||||
: t("portal.pipelines.builder.runOnUpload", "Every upload");
|
||||
if (input.triggerType === MANUAL)
|
||||
return t("portal.pipelines.composer.triggerManual");
|
||||
if (input.triggerType === "schedule")
|
||||
@@ -1158,7 +1194,7 @@ export function PipelineBuilder() {
|
||||
variant="tertiary"
|
||||
className="portal-builder__source-edit"
|
||||
aria-label={t("portal.pipelines.composer.editSource")}
|
||||
disabled={input.sourceId === ""}
|
||||
disabled={input.sourceId === "" || isEditorInput}
|
||||
onClick={() =>
|
||||
setSourceModal({ open: true, sourceId: input.sourceId })
|
||||
}
|
||||
@@ -1168,58 +1204,14 @@ export function PipelineBuilder() {
|
||||
</div>
|
||||
</FormField>
|
||||
|
||||
<FormField label={t("portal.pipelines.builder.inputTrigger")}>
|
||||
<Select
|
||||
inputSize="sm"
|
||||
aria-label={t("portal.pipelines.builder.inputTrigger")}
|
||||
value={
|
||||
input.triggerType === MANUAL
|
||||
? MANUAL_OPTION
|
||||
: input.triggerType
|
||||
}
|
||||
disabled={input.sourceId === ""}
|
||||
onChange={(value) =>
|
||||
updateInput({
|
||||
triggerType:
|
||||
value && value !== MANUAL_OPTION ? value : MANUAL,
|
||||
})
|
||||
}
|
||||
options={triggerOptionsFor(input.sourceId)}
|
||||
<PipelineInputTrigger
|
||||
input={input}
|
||||
onInputChange={updateInput}
|
||||
triggerOptions={triggerOptionsFor(input.sourceId)}
|
||||
isEditorInput={isEditorInput}
|
||||
runOn={runOn}
|
||||
onRunOnChange={setRunOn}
|
||||
/>
|
||||
</FormField>
|
||||
|
||||
{input.triggerType === "schedule" && (
|
||||
<div className="portal-builder__schedule">
|
||||
<span className="portal-builder__muted">
|
||||
{t("portal.pipelines.composer.scheduleEvery")}
|
||||
</span>
|
||||
<Input
|
||||
inputSize="sm"
|
||||
type="number"
|
||||
min={1}
|
||||
value={input.scheduleCount}
|
||||
invalid={Number(input.scheduleCount) <= 0}
|
||||
onChange={(e) =>
|
||||
updateInput({ scheduleCount: e.target.value })
|
||||
}
|
||||
className="portal-builder__schedule-count"
|
||||
/>
|
||||
<Select
|
||||
inputSize="sm"
|
||||
value={input.scheduleUnit}
|
||||
onChange={(value) =>
|
||||
value &&
|
||||
updateInput({ scheduleUnit: value as ScheduleUnit })
|
||||
}
|
||||
options={SCHEDULE_UNITS.map((unit) => ({
|
||||
value: unit,
|
||||
label: t(
|
||||
`portal.pipelines.composer.unit.${unit.toLowerCase()}`,
|
||||
),
|
||||
}))}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
@@ -1235,6 +1227,17 @@ export function PipelineBuilder() {
|
||||
);
|
||||
}
|
||||
|
||||
if (selected === "output" && isEditorInput) {
|
||||
return (
|
||||
<p className="portal-builder__muted">
|
||||
{t(
|
||||
"portal.pipelines.builder.editorDestinationHelp",
|
||||
"This pipeline runs on the files in your workspace, and its results replace the file it ran on. There is nowhere else to send them.",
|
||||
)}
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
||||
if (selected === "output") {
|
||||
return (
|
||||
<DestinationPicker
|
||||
@@ -1344,7 +1347,19 @@ export function PipelineBuilder() {
|
||||
: null
|
||||
}
|
||||
output={
|
||||
outputAsked || outputValid
|
||||
isEditorInput
|
||||
? {
|
||||
label: t(
|
||||
"portal.pipelines.builder.editorDestination",
|
||||
"Editor",
|
||||
),
|
||||
detail: t(
|
||||
"portal.pipelines.builder.editorDestinationDetail",
|
||||
"Replaces the file you ran it on",
|
||||
),
|
||||
fixed: true,
|
||||
}
|
||||
: outputAsked || outputValid
|
||||
? {
|
||||
label:
|
||||
chosenDestination?.name ??
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { usePolicyAutoRun } from "@app/components/policies/usePolicyAutoRun";
|
||||
import { useClientSideClassification } from "@app/components/policies/useClientSideClassification";
|
||||
import { usePolicyLocalPasses } from "@app/components/policies/usePolicyLocalPasses";
|
||||
|
||||
/**
|
||||
* Headless controller that drives policy auto-run (enforce every enabled policy
|
||||
@@ -7,8 +7,9 @@ import { useClientSideClassification } from "@app/components/policies/useClientS
|
||||
* regardless of whether the policy panel is visible. Renders nothing.
|
||||
*/
|
||||
export function PolicyAutoRunController() {
|
||||
// Server-dispatched, file-producing policies and their chain.
|
||||
usePolicyAutoRun();
|
||||
// Non-AI systems classify uploads in the browser; inert when the AI engine is on.
|
||||
useClientSideClassification();
|
||||
// Policies with a browser-side fast path (e.g. classification's heuristic), run generically.
|
||||
usePolicyLocalPasses();
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* The Classification policy's browser-side fast path, as a {@link LocalPass} the generic local-pass
|
||||
* engine runs. Everything classification-specific lives here: the heuristic, the label/confidence it
|
||||
* writes, metering, and the browser-local run it records. The engine only sees the generic result
|
||||
* (fields to write + whether the AI server run is still needed).
|
||||
*/
|
||||
|
||||
import { fileStorage } from "@app/services/fileStorage";
|
||||
import { classifyFileHeuristically } from "@app/services/heuristic/heuristicClassification";
|
||||
import { meterClassificationRun } from "@app/services/classificationMeter";
|
||||
import {
|
||||
isDispatched,
|
||||
markDispatched,
|
||||
recordRunStart,
|
||||
updateRun,
|
||||
} from "@app/components/policies/policyRunStore";
|
||||
import type { FileId } from "@app/types/file";
|
||||
import type { StirlingFile } from "@app/types/fileContext";
|
||||
import type { HeuristicConfidence } from "@app/services/heuristic/types";
|
||||
import {
|
||||
CLASSIFICATION_CATEGORY_ID,
|
||||
localVerdictNeedsEscalation,
|
||||
} from "@app/data/classificationPolicy";
|
||||
import type { LocalPass } from "@app/components/policies/policyLocalPass";
|
||||
|
||||
/** How long to wait for an upload's bytes to land in IndexedDB (20 × 250ms ≈ 5s).
|
||||
* The stub can surface in the file list a beat before its bytes are committed. */
|
||||
const FILE_WAIT_TRIES = 20;
|
||||
const FILE_WAIT_MS = 250;
|
||||
|
||||
/** localStorage flag: set to "true" for a full per-file scoring breakdown in the console. */
|
||||
const DEBUG_FLAG = "stirling-classification-debug";
|
||||
|
||||
function isClassificationDebug(): boolean {
|
||||
try {
|
||||
return localStorage.getItem(DEBUG_FLAG) === "true";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
export const classificationLocalPass: LocalPass = {
|
||||
// Any document without a verdict yet: an upload, or a policy's new-file output that had no parent
|
||||
// to inherit one from.
|
||||
// A file that did inherit (or was classified) carries a label array and is skipped.
|
||||
eligible: (stub) => stub.classificationLabels === undefined,
|
||||
run: async (fileId, stub) => {
|
||||
const verdict = await classifyStub(fileId, stub.name, stub.size ?? 0);
|
||||
// Bytes never landed (file removed mid-wait): leave unclassified so a reload retries.
|
||||
if (verdict == null) return null;
|
||||
return {
|
||||
stubUpdates: {
|
||||
classificationLabels: verdict.labels,
|
||||
classificationConfidence: verdict.confidence,
|
||||
},
|
||||
// A confident local verdict stands; anything less asks the AI engine, which overwrites it.
|
||||
needsServerRun: localVerdictNeedsEscalation(verdict.confidence),
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
/** Classify one file, metering exactly once; null = no verdict, retried later. */
|
||||
async function classifyStub(
|
||||
fileId: FileId,
|
||||
fileName: string,
|
||||
fileSize: number,
|
||||
): Promise<{ labels: string[]; confidence: HeuristicConfidence } | null> {
|
||||
let file: StirlingFile | null = null;
|
||||
for (let i = 0; i < FILE_WAIT_TRIES; i++) {
|
||||
file = await fileStorage.getStirlingFile(fileId).catch(() => null);
|
||||
if (file) break;
|
||||
await delay(FILE_WAIT_MS);
|
||||
}
|
||||
if (!file) {
|
||||
console.warn(
|
||||
`[Classify] ${fileName}: bytes never arrived in storage; will retry on next load`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const debug = isClassificationDebug();
|
||||
const startedAt = performance.now();
|
||||
// A local run is still a billable policy run, so it belongs in the activity feed; recorded only
|
||||
// once the bytes are in hand, so a file whose bytes never land leaves no phantom row.
|
||||
|
||||
// Read before recordRunStart, which takes the dispatch key itself and would otherwise always
|
||||
// answer "already dispatched", silently stopping metering.
|
||||
const alreadyMetered = isDispatched(CLASSIFICATION_CATEGORY_ID, fileId);
|
||||
const runId = `local-${CLASSIFICATION_CATEGORY_ID}-${fileId}-${Date.now()}`;
|
||||
recordRunStart({
|
||||
runId,
|
||||
categoryId: CLASSIFICATION_CATEGORY_ID,
|
||||
fileId: fileId as string,
|
||||
fileName,
|
||||
fileSize,
|
||||
target: "local",
|
||||
// The heuristic ran in the browser - there is no server run to poll (see the poll effect).
|
||||
browserLocal: true,
|
||||
status: "RUNNING",
|
||||
outputs: [],
|
||||
error: null,
|
||||
startedAt: Date.now(),
|
||||
});
|
||||
try {
|
||||
const result = await classifyFileHeuristically(file, { explain: debug });
|
||||
const { labels } = result;
|
||||
const ms = Math.round(performance.now() - startedAt);
|
||||
const verdict =
|
||||
labels.length > 0
|
||||
? labels.join(", ")
|
||||
: result.isEnglish
|
||||
? "no label"
|
||||
: "no label (not English)";
|
||||
console.debug(
|
||||
`[Classify] ${fileName} -> ${verdict} (${result.confidence}, score ${result.score}, ${ms}ms)` +
|
||||
(alreadyMetered ? " [heal: not re-metered]" : ""),
|
||||
);
|
||||
if (debug && result.explain) logExplanation(fileName, result);
|
||||
// Meter on the first classification only; a healing re-run of an undelivered
|
||||
// result (already dispatched) is not a new billable run.
|
||||
if (!alreadyMetered) {
|
||||
meterClassificationRun({
|
||||
policyName: "Classification",
|
||||
documentCount: 1,
|
||||
labels,
|
||||
});
|
||||
}
|
||||
markDispatched(CLASSIFICATION_CATEGORY_ID, fileId);
|
||||
// Labels, no output file - the same settle shape the server-run classification uses.
|
||||
updateRun(runId, {
|
||||
status: "COMPLETED",
|
||||
imported: true,
|
||||
outputFileIds: [fileId as string],
|
||||
});
|
||||
return { labels, confidence: result.confidence };
|
||||
} catch (err) {
|
||||
// Never persist a verdict for an unreadable file - the failure may be
|
||||
// environmental, so it must stay eligible to retry (and meter) later.
|
||||
console.warn(`[Classify] ${fileName}: could not be read, will retry`, err);
|
||||
updateRun(runId, {
|
||||
status: "FAILED",
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Full scoring breakdown, one collapsed console group per file (debug flag only). */
|
||||
function logExplanation(
|
||||
fileName: string,
|
||||
result: Awaited<ReturnType<typeof classifyFileHeuristically>>,
|
||||
): void {
|
||||
const ex = result.explain;
|
||||
if (!ex) return;
|
||||
console.groupCollapsed(
|
||||
`[Classify] ${fileName} scoring (english=${ex.isEnglish}, lowText=${ex.lowText})`,
|
||||
);
|
||||
if (ex.candidates.length === 0) {
|
||||
console.log("no label scored above zero");
|
||||
}
|
||||
for (const c of ex.candidates) {
|
||||
console.log(
|
||||
`${c.id}${c.emit ? "" : " (suppressed)"}: score ${c.score}, ${c.distinct} distinct signals`,
|
||||
);
|
||||
for (const s of c.signals) console.log(` ${s}`);
|
||||
}
|
||||
console.groupEnd();
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* The generic "browser-side fast path" seam. A policy may declare a {@link LocalPass}: cheap local
|
||||
* work that runs before any server dispatch and can settle a file on its own, or decide the server
|
||||
* run is still needed. The local-pass engine ({@link ../../hooks/usePolicyLocalPasses}) runs it
|
||||
* without knowing what it computes; the policy-specific logic lives entirely inside the pass.
|
||||
*/
|
||||
|
||||
import type { FileId } from "@app/types/file";
|
||||
import type { StirlingFileStub } from "@app/types/fileContext";
|
||||
import { CLASSIFICATION_CATEGORY_ID } from "@app/data/classificationPolicy";
|
||||
import { classificationLocalPass } from "@app/components/policies/classificationLocalPass";
|
||||
|
||||
export interface LocalPassResult {
|
||||
/** Fields to merge onto the file's stub and stored metadata. Opaque to the engine. */
|
||||
stubUpdates: Partial<StirlingFileStub>;
|
||||
/** Whether the policy's server run should still be dispatched after this pass. */
|
||||
needsServerRun: boolean;
|
||||
}
|
||||
|
||||
export interface LocalPass {
|
||||
/** Files this pass should run on (e.g. new documents it has not processed yet). */
|
||||
eligible(stub: StirlingFileStub): boolean;
|
||||
/**
|
||||
* Do the local work for one file. Returns the stub fields to write and whether the server run is
|
||||
* still needed, or null if the work could not be done and should be retried later.
|
||||
*/
|
||||
run(fileId: FileId, stub: StirlingFileStub): Promise<LocalPassResult | null>;
|
||||
}
|
||||
|
||||
/** The local fast path a policy declares, if any. The default (most policies) is none. */
|
||||
export function localPassFor(categoryId: string): LocalPass | undefined {
|
||||
if (categoryId === CLASSIFICATION_CATEGORY_ID) return classificationLocalPass;
|
||||
return undefined;
|
||||
}
|
||||
@@ -83,29 +83,6 @@ describe("policyRunStore", () => {
|
||||
expect(isDispatched("security", "f1")).toBe(true);
|
||||
});
|
||||
|
||||
it("a browser-local run does not claim the (policy, file) dispatch key", () => {
|
||||
// The local classification heuristic records a run for the same (classification, file) pair
|
||||
// the server escalation is keyed on. If that claimed the key, the auto-run would read
|
||||
// "already dispatched" and never ask the AI - which killed escalation entirely.
|
||||
recordRunStart(
|
||||
rec({
|
||||
runId: "local-classification-f1-1",
|
||||
categoryId: "classification",
|
||||
fileId: "f1",
|
||||
target: "local",
|
||||
browserLocal: true,
|
||||
status: "RUNNING",
|
||||
}),
|
||||
);
|
||||
expect(getRun("local-classification-f1-1")).toBeDefined();
|
||||
expect(isDispatched("classification", "f1")).toBe(false);
|
||||
});
|
||||
|
||||
it("a real backend run still claims the dispatch key", () => {
|
||||
recordRunStart(rec({ runId: "srv-1", categoryId: "classification" }));
|
||||
expect(isDispatched("classification", "f1")).toBe(true);
|
||||
});
|
||||
|
||||
it("never evicts in-flight runs, even past the soft cap", () => {
|
||||
// A large upload batch can exceed the cap while still processing. Dropping a
|
||||
// live run would orphan its polling/import and undercount progress, so every
|
||||
|
||||
@@ -45,16 +45,12 @@ export interface PolicyRunRecord {
|
||||
/** Set while an auto-retry is pending after a transient (queue-full) rejection, so the activity
|
||||
* feed shows a soft "busy" row instead of a hard failure during the backoff window. */
|
||||
retrying?: boolean;
|
||||
/** A run computed entirely in the browser - it has no server run behind it,
|
||||
* so it must never be polled for status (a status poll 404s and would flip a
|
||||
* succeeded run to FAILED) or reconciled against the server. */
|
||||
browserLocal?: boolean;
|
||||
/** Epoch ms when the run was dispatched. */
|
||||
startedAt: number;
|
||||
/**
|
||||
* Ran in the browser (the local classification heuristic), not on a backend. Such a run has no
|
||||
* server-side status to poll, and - crucially - must NOT claim the (policy, file) dispatch key:
|
||||
* it is the first pass, not the policy's run, so claiming it would suppress the server run the
|
||||
* verdict may still need to escalate to. Distinct from {@link target}, which says which BACKEND
|
||||
* holds a real run's outputs.
|
||||
*/
|
||||
browserLocal?: boolean;
|
||||
}
|
||||
|
||||
/** Statuses of a run that is still executing (not yet settled). */
|
||||
@@ -224,13 +220,9 @@ export function recordRunStart(record: PolicyRunRecord) {
|
||||
const waveStartedAt = state.runs.some(isRunInFlight)
|
||||
? state.waveStartedAt
|
||||
: record.startedAt;
|
||||
// A browser-local run is the first pass, not the policy's run: claiming the dispatch key here
|
||||
// would permanently suppress the server run its verdict may still need to escalate to.
|
||||
const claimsDispatch = !record.browserLocal;
|
||||
state = {
|
||||
runs: capRuns([record, ...state.runs]),
|
||||
dispatched:
|
||||
!claimsDispatch || state.dispatched.includes(key)
|
||||
dispatched: state.dispatched.includes(key)
|
||||
? state.dispatched
|
||||
: [...state.dispatched, key],
|
||||
waveStartedAt,
|
||||
|
||||
@@ -1,251 +0,0 @@
|
||||
// The Classification policy's first pass: every upload is labelled locally before the AI is asked.
|
||||
// The confidence reported here decides whether the AI is asked at all - see usePolicyAutoRun.
|
||||
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { useAllFiles, useFileManagement } from "@app/contexts/FileContext";
|
||||
import { useAppConfig } from "@app/contexts/AppConfigContext";
|
||||
import { useIndexedDB } from "@app/contexts/IndexedDBContext";
|
||||
import { fileStorage } from "@app/services/fileStorage";
|
||||
import { useClassificationEnabled } from "@app/hooks/useClassificationEnabled";
|
||||
import { scheduleIdle } from "@app/utils/scheduleIdle";
|
||||
import { usePolicies } from "@app/hooks/usePolicies";
|
||||
import { classifyFileHeuristically } from "@app/services/heuristic/heuristicClassification";
|
||||
import { meterClassificationRun } from "@app/services/classificationMeter";
|
||||
import {
|
||||
isDispatched,
|
||||
markDispatched,
|
||||
recordRunStart,
|
||||
updateRun,
|
||||
} from "@app/components/policies/policyRunStore";
|
||||
import type { FileId } from "@app/types/file";
|
||||
import type { StirlingFile, StirlingFileStub } from "@app/types/fileContext";
|
||||
import type { HeuristicConfidence } from "@app/services/heuristic/types";
|
||||
import { CLASSIFICATION_CATEGORY_ID } from "@app/data/classificationPolicy";
|
||||
|
||||
/**
|
||||
* Dispatch-store key namespace for "this file's local pass has been metered". Deliberately NOT the
|
||||
* Classification category id: that key is the server escalation's own guard, so metering under it
|
||||
* would tell the auto-run the policy had already run and kill the escalation entirely.
|
||||
*/
|
||||
export const LOCAL_METER_CATEGORY = `${CLASSIFICATION_CATEGORY_ID}:local-meter`;
|
||||
/** Files classified per idle pass, so a large library drains over several ticks. */
|
||||
const CLASSIFY_BATCH = 3;
|
||||
/** How long to wait for an upload's bytes to land in IndexedDB (20 × 250ms ≈ 5s).
|
||||
* The stub can surface in the file list a beat before its bytes are committed. */
|
||||
const FILE_WAIT_TRIES = 20;
|
||||
const FILE_WAIT_MS = 250;
|
||||
|
||||
/** localStorage flag: set to "true" for a full per-file scoring breakdown in the console. */
|
||||
const DEBUG_FLAG = "stirling-classification-debug";
|
||||
|
||||
function isClassificationDebug(): boolean {
|
||||
try {
|
||||
return localStorage.getItem(DEBUG_FLAG) === "true";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
export function useClientSideClassification(): void {
|
||||
const { fileStubs } = useAllFiles();
|
||||
const { updateStirlingFileStub } = useFileManagement();
|
||||
const { bumpRevision } = useIndexedDB();
|
||||
const { policies } = usePolicies();
|
||||
const classificationEnabled = useClassificationEnabled();
|
||||
// Still waited on: a verdict written before app-config lands would be acted on by the
|
||||
// escalation decision before it knows whether the AI engine is even available.
|
||||
const { loading: configLoading } = useAppConfig();
|
||||
// Files claimed this session, keyed id+lastModified so a new version is retried once. A claim is
|
||||
// taken synchronously right before classifying, so overlapping batches never double-classify.
|
||||
const claimed = useRef<Set<string>>(new Set());
|
||||
// Bumped after each batch to drain the next one.
|
||||
const [tick, setTick] = useState(0);
|
||||
|
||||
// TODO: keyed on the Classification CATEGORY, so a pipeline that merely contains a classify
|
||||
// step gets no local pass - suppressing one step of a chain is not expressible today.
|
||||
const policy = policies[CLASSIFICATION_CATEGORY_ID];
|
||||
// Only when the admin has an active Classification policy - the same gate the AI path uses.
|
||||
const active = Boolean(
|
||||
policy?.configured &&
|
||||
policy.status === "active" &&
|
||||
policy.backendId &&
|
||||
(!policy.sources ||
|
||||
policy.sources.length === 0 ||
|
||||
policy.sources.includes("editor")),
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
// Runs whether or not the AI engine is on: it is the first pass either way, not a fallback.
|
||||
if (configLoading || !classificationEnabled || !active) {
|
||||
return;
|
||||
}
|
||||
const claimKey = (s: StirlingFileStub) =>
|
||||
`${s.id as string}:${s.lastModified ?? 0}`;
|
||||
// null labels = never delivered, retried here; [] = definitive no-label verdict.
|
||||
const pending = fileStubs
|
||||
.filter(
|
||||
(s) =>
|
||||
!s.derivedFromTool &&
|
||||
s.classificationLabels == null &&
|
||||
!claimed.current.has(claimKey(s)),
|
||||
)
|
||||
.slice(0, CLASSIFY_BATCH);
|
||||
if (pending.length === 0) return;
|
||||
let cancelled = false;
|
||||
const cancelIdle = scheduleIdle(() => {
|
||||
// Superseded before starting: the newer effect instance owns the queue.
|
||||
if (cancelled) return;
|
||||
void (async () => {
|
||||
let wrote = false;
|
||||
for (const stub of pending) {
|
||||
const key = claimKey(stub);
|
||||
// Re-validate at execution time - another batch may have claimed it since.
|
||||
if (claimed.current.has(key)) continue;
|
||||
claimed.current.add(key);
|
||||
const verdict = await classifyStub(
|
||||
stub.id,
|
||||
stub.name,
|
||||
stub.size ?? 0,
|
||||
);
|
||||
// Bytes never landed (file removed mid-wait): leave undelivered so a
|
||||
// reload (or new version) retries; the claim stops churn this session.
|
||||
if (verdict == null) continue;
|
||||
// Deliver unconditionally - a re-render must never discard a computed
|
||||
// (and already metered) result. Writes are idempotent.
|
||||
updateStirlingFileStub(stub.id, {
|
||||
classificationLabels: verdict.labels,
|
||||
classificationConfidence: verdict.confidence,
|
||||
});
|
||||
const ok = await fileStorage.updateFileMetadata(stub.id, {
|
||||
classificationLabels: verdict.labels,
|
||||
classificationConfidence: verdict.confidence,
|
||||
});
|
||||
if (ok) wrote = true;
|
||||
}
|
||||
if (wrote) bumpRevision();
|
||||
// Drain the next batch; the terminal pass finds nothing pending and stops.
|
||||
setTick((n) => n + 1);
|
||||
})();
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
cancelIdle();
|
||||
};
|
||||
}, [
|
||||
fileStubs,
|
||||
active,
|
||||
classificationEnabled,
|
||||
configLoading,
|
||||
updateStirlingFileStub,
|
||||
bumpRevision,
|
||||
tick,
|
||||
]);
|
||||
}
|
||||
|
||||
/** Classify one file, metering exactly once; null = no verdict, retried later. */
|
||||
async function classifyStub(
|
||||
fileId: FileId,
|
||||
fileName: string,
|
||||
fileSize: number,
|
||||
): Promise<{ labels: string[]; confidence: HeuristicConfidence } | null> {
|
||||
let file: StirlingFile | null = null;
|
||||
for (let i = 0; i < FILE_WAIT_TRIES; i++) {
|
||||
file = await fileStorage.getStirlingFile(fileId).catch(() => null);
|
||||
if (file) break;
|
||||
await delay(FILE_WAIT_MS);
|
||||
}
|
||||
if (!file) {
|
||||
console.warn(
|
||||
`[Classify] ${fileName}: bytes never arrived in storage; will retry on next load`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const debug = isClassificationDebug();
|
||||
const startedAt = performance.now();
|
||||
// A local run is still a billable policy run, so it belongs in the activity feed; recorded only
|
||||
// once the bytes are in hand, so a file whose bytes never land leaves no phantom row.
|
||||
|
||||
const alreadyMetered = isDispatched(LOCAL_METER_CATEGORY, fileId);
|
||||
const runId = `local-${CLASSIFICATION_CATEGORY_ID}-${fileId}-${Date.now()}`;
|
||||
recordRunStart({
|
||||
runId,
|
||||
categoryId: CLASSIFICATION_CATEGORY_ID,
|
||||
fileId: fileId as string,
|
||||
fileName,
|
||||
fileSize,
|
||||
target: "local",
|
||||
// Ran here, not on a backend: nothing to poll, and it must not claim the classification
|
||||
// dispatch key - that key is what the server escalation checks before running.
|
||||
browserLocal: true,
|
||||
status: "RUNNING",
|
||||
outputs: [],
|
||||
error: null,
|
||||
startedAt: Date.now(),
|
||||
});
|
||||
try {
|
||||
const result = await classifyFileHeuristically(file, { explain: debug });
|
||||
const { labels } = result;
|
||||
const ms = Math.round(performance.now() - startedAt);
|
||||
const verdict =
|
||||
labels.length > 0
|
||||
? labels.join(", ")
|
||||
: result.isEnglish
|
||||
? "no label"
|
||||
: "no label (not English)";
|
||||
console.debug(
|
||||
`[Classify] ${fileName} -> ${verdict} (${result.confidence}, score ${result.score}, ${ms}ms)` +
|
||||
(alreadyMetered ? " [heal: not re-metered]" : ""),
|
||||
);
|
||||
if (debug && result.explain) logExplanation(fileName, result);
|
||||
// Meter on the first classification only; a healing re-run of an undelivered
|
||||
// result (already dispatched) is not a new billable run.
|
||||
if (!alreadyMetered) {
|
||||
meterClassificationRun({
|
||||
policyName: "Classification",
|
||||
documentCount: 1,
|
||||
labels,
|
||||
});
|
||||
}
|
||||
markDispatched(LOCAL_METER_CATEGORY, fileId);
|
||||
// Labels, no output file - the same settle shape the server-run classification uses.
|
||||
updateRun(runId, {
|
||||
status: "COMPLETED",
|
||||
imported: true,
|
||||
outputFileIds: [fileId as string],
|
||||
});
|
||||
return { labels, confidence: result.confidence };
|
||||
} catch (err) {
|
||||
// Never persist a verdict for an unreadable file - the failure may be
|
||||
// environmental, so it must stay eligible to retry (and meter) later.
|
||||
console.warn(`[Classify] ${fileName}: could not be read, will retry`, err);
|
||||
updateRun(runId, {
|
||||
status: "FAILED",
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Full scoring breakdown, one collapsed console group per file (debug flag only). */
|
||||
function logExplanation(
|
||||
fileName: string,
|
||||
result: Awaited<ReturnType<typeof classifyFileHeuristically>>,
|
||||
): void {
|
||||
const ex = result.explain;
|
||||
if (!ex) return;
|
||||
console.groupCollapsed(
|
||||
`[Classify] ${fileName} scoring (english=${ex.isEnglish}, lowText=${ex.lowText})`,
|
||||
);
|
||||
if (ex.candidates.length === 0) {
|
||||
console.log("no label scored above zero");
|
||||
}
|
||||
for (const c of ex.candidates) {
|
||||
console.log(
|
||||
`${c.id}${c.emit ? "" : " (suppressed)"}: score ${c.score}, ${c.distinct} distinct signals`,
|
||||
);
|
||||
for (const s of c.signals) console.log(` ${s}`);
|
||||
}
|
||||
console.groupEnd();
|
||||
}
|
||||
+100
-29
@@ -3,8 +3,10 @@ import { renderHook, act } from "@testing-library/react";
|
||||
import type { ClassificationConfidence } from "@app/types/fileContext";
|
||||
|
||||
/**
|
||||
* Batch integration test (61 files, two chained upload policies) driving the real
|
||||
* store + hook effects, IO mocked. Classification is forced last (see the sort).
|
||||
* Batch integration test (61 files, Security then Classification) driving the real store + both
|
||||
* hooks, IO mocked. The auto-run engine dispatches the file-producing Security policy and versions
|
||||
* its output in place; the Classification policy runs itself (useClassificationPolicy) on each settled
|
||||
* output - here a confident local verdict, so it stamps labels without escalating to the AI.
|
||||
*/
|
||||
|
||||
const FILE_COUNT = 61;
|
||||
@@ -15,6 +17,9 @@ const FILE_COUNT = 61;
|
||||
const mocks = vi.hoisted(() => ({
|
||||
workspace: [] as Array<{
|
||||
id: string;
|
||||
name?: string;
|
||||
size?: number;
|
||||
derivedFromTool?: boolean;
|
||||
classificationLabels?: string[];
|
||||
classificationConfidence?: ClassificationConfidence;
|
||||
}>,
|
||||
@@ -39,13 +44,25 @@ const mocks = vi.hoisted(() => ({
|
||||
addFiles: vi.fn(),
|
||||
updateStirlingFileStub: vi.fn(),
|
||||
consumeFiles: vi.fn(),
|
||||
classify: vi.fn(),
|
||||
meter: vi.fn(),
|
||||
}));
|
||||
|
||||
// The second (file-producing) policy's timing, flippable per test to prove classification's local
|
||||
// pass is independent of when the rewriter runs.
|
||||
const securityRunOn = vi.hoisted(() => ({
|
||||
value: "upload" as "upload" | "export",
|
||||
}));
|
||||
|
||||
// Classification chains server-side only when the AI engine is on (else it runs
|
||||
// client-side); this batch exercises the server chain, so force the engine on.
|
||||
vi.mock("@app/hooks/useAiEngineEnabled", () => ({
|
||||
useAiEngineEnabled: () => true,
|
||||
}));
|
||||
vi.mock("@app/hooks/useClassificationEnabled", () => ({
|
||||
useClassificationEnabled: () => true,
|
||||
}));
|
||||
vi.mock("@app/contexts/AppConfigContext", () => ({
|
||||
useAppConfig: () => ({ config: {}, loading: false }),
|
||||
}));
|
||||
vi.mock("@app/contexts/FileContext", () => ({
|
||||
useAllFiles: () => ({ fileStubs: mocks.workspace }),
|
||||
useFileManagement: () => ({
|
||||
@@ -60,10 +77,9 @@ vi.mock("@app/contexts/IndexedDBContext", () => ({
|
||||
vi.mock("@app/hooks/usePolicies", () => ({
|
||||
usePolicies: () => ({
|
||||
policies: {
|
||||
// Classification is configured first (order 0) but is FORCED to run last
|
||||
// by the orchestrator; Security (order 1) therefore runs first.
|
||||
classification: {
|
||||
configured: true,
|
||||
runsOnEditor: true,
|
||||
status: "active",
|
||||
backendId: "backend-classification",
|
||||
runOn: "upload",
|
||||
@@ -73,9 +89,10 @@ vi.mock("@app/hooks/usePolicies", () => ({
|
||||
},
|
||||
security: {
|
||||
configured: true,
|
||||
runsOnEditor: true,
|
||||
status: "active",
|
||||
backendId: "backend-security",
|
||||
runOn: "upload",
|
||||
runOn: securityRunOn.value,
|
||||
order: 1,
|
||||
outputMode: "new_version",
|
||||
outputName: "",
|
||||
@@ -102,39 +119,54 @@ vi.mock("@app/services/fileStubHelpers", () => ({
|
||||
createStirlingFilesAndStubs: mocks.createStirlingFilesAndStubs,
|
||||
}));
|
||||
vi.mock("@app/services/fileClassification", () => ({
|
||||
// Classification always resolves labels here, so the metadata-only import path
|
||||
// stamps them onto the stub.
|
||||
// The AI import path (if ever reached) resolves labels from the output PDF.
|
||||
readClassificationLabelsFromFile: vi.fn().mockResolvedValue(["Invoice"]),
|
||||
}));
|
||||
vi.mock("@app/services/heuristic/heuristicClassification", () => ({
|
||||
classifyFileHeuristically: (file: File) => mocks.classify(file),
|
||||
}));
|
||||
vi.mock("@app/services/classificationMeter", () => ({
|
||||
meterClassificationRun: (payload: unknown) => mocks.meter(payload),
|
||||
}));
|
||||
|
||||
import { usePolicyAutoRun } from "@app/components/policies/usePolicyAutoRun";
|
||||
import { usePolicyLocalPasses } from "@app/components/policies/usePolicyLocalPasses";
|
||||
import {
|
||||
usePolicyRuns,
|
||||
resetPolicyRuns,
|
||||
} from "@app/components/policies/policyRunStore";
|
||||
import type { PolicyRunRecord } from "@app/components/policies/policyRunStore";
|
||||
|
||||
// Run idle callbacks immediately so the local-pass engine's batches start without timer waits.
|
||||
vi.stubGlobal("requestIdleCallback", (cb: () => void) => {
|
||||
cb();
|
||||
return 1;
|
||||
});
|
||||
vi.stubGlobal("cancelIdleCallback", () => {});
|
||||
|
||||
/** A stable snapshot of the store, read after the flow settles. */
|
||||
let latestRuns: PolicyRunRecord[] = [];
|
||||
function Harness() {
|
||||
usePolicyAutoRun();
|
||||
usePolicyLocalPasses();
|
||||
latestRuns = usePolicyRuns();
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The heuristic verdict that escalates to the AI classifier; only "high" stands alone. */
|
||||
const LOW = "low" as const;
|
||||
|
||||
function replaceInWorkspace(inputIds: string[], outputIds: string[]) {
|
||||
// A versioned output carries its input's heuristic verdict; the escalation decision is about the
|
||||
// document, not about which step produced the current bytes.
|
||||
const inherited =
|
||||
mocks.workspace.find((s) => inputIds.includes(s.id))
|
||||
?.classificationConfidence ?? LOW;
|
||||
// A versioned output inherits its input's classification verdict, exactly as the real CONSUME_FILES
|
||||
// reducer does - so a label put on the upload rides forward without re-classifying the output.
|
||||
const donor = mocks.workspace.find((s) => inputIds.includes(s.id));
|
||||
mocks.workspace = mocks.workspace
|
||||
.filter((s) => !inputIds.includes(s.id))
|
||||
.concat(
|
||||
outputIds.map((id) => ({ id, classificationConfidence: inherited })),
|
||||
outputIds.map((id) => ({
|
||||
id,
|
||||
name: "doc.pdf",
|
||||
derivedFromTool: true,
|
||||
classificationLabels: donor?.classificationLabels,
|
||||
classificationConfidence: donor?.classificationConfidence,
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -150,10 +182,11 @@ beforeEach(() => {
|
||||
mocks.backendOutCounter = 0;
|
||||
mocks.dispatchInFlight = 0;
|
||||
mocks.maxDispatchInFlight = 0;
|
||||
securityRunOn.value = "upload";
|
||||
|
||||
mocks.workspace = Array.from({ length: FILE_COUNT }, (_, i) => ({
|
||||
id: `file-${i}`,
|
||||
classificationConfidence: LOW,
|
||||
name: `doc-${i}.pdf`,
|
||||
}));
|
||||
|
||||
mocks.listPolicyRuns.mockResolvedValue([]);
|
||||
@@ -169,8 +202,14 @@ beforeEach(() => {
|
||||
mocks.downloadPolicyOutput.mockResolvedValue(
|
||||
new Blob(["x"], { type: "application/pdf" }),
|
||||
);
|
||||
// Apply stub updates to the shared workspace, as the real reducer does — the
|
||||
// label stamp's second pass reads them back to stay idempotent.
|
||||
// A confident local verdict: classification stamps labels and does NOT escalate to the AI.
|
||||
mocks.classify.mockResolvedValue({
|
||||
labels: ["Invoice"],
|
||||
confidence: "high",
|
||||
isEnglish: true,
|
||||
score: 5,
|
||||
});
|
||||
// Apply stub updates to the shared workspace, as the real reducer does.
|
||||
mocks.updateStirlingFileStub.mockImplementation(
|
||||
(id: string, updates: Record<string, unknown>) => {
|
||||
const stub = mocks.workspace.find((s) => s.id === id);
|
||||
@@ -205,7 +244,7 @@ beforeEach(() => {
|
||||
],
|
||||
}));
|
||||
// Deliver a unique workspace child stub per output, derived from the parent so
|
||||
// the chain's second policy can find + version it.
|
||||
// classification can find + tag it.
|
||||
mocks.createStirlingFilesAndStubs.mockImplementation(
|
||||
async (files: File[], parentStub: { id: string }) => {
|
||||
const stubs = files.map(() => ({
|
||||
@@ -237,7 +276,7 @@ beforeEach(() => {
|
||||
);
|
||||
});
|
||||
|
||||
/** Drive the hook until the store shows the expected number of imported runs. */
|
||||
/** Drive the hooks until the store shows the expected number of imported runs. */
|
||||
async function runUntilSettled(expectedRuns: number) {
|
||||
renderHook(() => Harness());
|
||||
await act(async () => {
|
||||
@@ -263,6 +302,8 @@ describe("policy auto-run — 61-file batch through a Security → Classificatio
|
||||
expect(classification).toHaveLength(FILE_COUNT);
|
||||
expect(security).toHaveLength(FILE_COUNT);
|
||||
expect(latestRuns).toHaveLength(FILE_COUNT * 2);
|
||||
// A confident local verdict stands on its own: no AI dispatch for classification.
|
||||
expect(classification.every((r) => r.target === "local")).toBe(true);
|
||||
});
|
||||
|
||||
it("bounds concurrent dispatch uploads so polls/downloads keep connections", async () => {
|
||||
@@ -280,7 +321,10 @@ describe("policy auto-run — 61-file batch through a Security → Classificatio
|
||||
// Classification never forks a version — it only stamps labels onto the stub.
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledTimes(FILE_COUNT);
|
||||
for (const call of mocks.updateStirlingFileStub.mock.calls) {
|
||||
expect(call[1]).toEqual({ classificationLabels: ["Invoice"] });
|
||||
expect(call[1]).toEqual({
|
||||
classificationLabels: ["Invoice"],
|
||||
classificationConfidence: "high",
|
||||
});
|
||||
}
|
||||
// Never added as brand-new files either.
|
||||
expect(mocks.addFilesCalls).toBe(0);
|
||||
@@ -303,18 +347,45 @@ describe("policy auto-run — 61-file batch through a Security → Classificatio
|
||||
await act(async () => {
|
||||
await vi.waitFor(
|
||||
() => {
|
||||
const imported = latestRuns.filter((r) => r.imported).length;
|
||||
expect(imported).toBe(FILE_COUNT * 2);
|
||||
const security = latestRuns.filter(
|
||||
(r) => r.categoryId === "security" && r.imported,
|
||||
);
|
||||
expect(security).toHaveLength(FILE_COUNT);
|
||||
},
|
||||
{ timeout: 8000, interval: 20 },
|
||||
);
|
||||
});
|
||||
|
||||
// Still fully processed (chain intact), but Security's versions went to
|
||||
// STORAGE, never re-added to the workbench — the workspace stays empty.
|
||||
expect(latestRuns).toHaveLength(FILE_COUNT * 2);
|
||||
// Security's versions went to STORAGE, never re-added to the workbench, so the
|
||||
// workspace stays empty. (Classification may have tagged the few files still open
|
||||
// when the workbench was cleared; the point here is the runner does not re-open them.)
|
||||
expect(mocks.workspace).toHaveLength(0);
|
||||
expect(mocks.consumeSilentCalls).toBe(0);
|
||||
expect(mocks.persistCalls).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("classifies uploads even when the other editor policy runs on export, not upload", async () => {
|
||||
// Repro: with a file-producing policy set to export, the auto-run engine dispatches nothing on
|
||||
// upload - but classification's local pass is independent and must still run on every upload.
|
||||
securityRunOn.value = "export";
|
||||
|
||||
renderHook(() => Harness());
|
||||
await act(async () => {
|
||||
await vi.waitFor(
|
||||
() => {
|
||||
const classification = latestRuns.filter(
|
||||
(r) => r.categoryId === "classification" && r.imported,
|
||||
);
|
||||
expect(classification).toHaveLength(FILE_COUNT);
|
||||
},
|
||||
{ timeout: 8000, interval: 20 },
|
||||
);
|
||||
});
|
||||
|
||||
// The export policy did not run on upload; nothing versioned in place.
|
||||
expect(latestRuns.filter((r) => r.categoryId === "security")).toHaveLength(
|
||||
0,
|
||||
);
|
||||
expect(mocks.consumeSilentCalls).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
+89
-208
@@ -1,22 +1,12 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { renderHook, act } from "@testing-library/react";
|
||||
|
||||
// Two active upload policies, so the auto-run should CHAIN them: fire the first on
|
||||
// the upload, then the second on the first's output. Stub the contexts + network so
|
||||
// we can drive the dispatch against the REAL run store.
|
||||
// Controllable AI-engine flag: on by default so classification chains server-side; one
|
||||
// test flips it off to assert classification is kept OUT of the server chain.
|
||||
const aiEnabled = vi.hoisted(() => ({ value: true }));
|
||||
vi.mock("@app/hooks/useAiEngineEnabled", () => ({
|
||||
useAiEngineEnabled: () => aiEnabled.value,
|
||||
}));
|
||||
const fileStubs: {
|
||||
id: string;
|
||||
name: string;
|
||||
derivedFromTool?: boolean;
|
||||
classificationLabels?: string[];
|
||||
classificationConfidence?: "none" | "low" | "medium" | "high";
|
||||
}[] = [];
|
||||
// Two active file-producing upload policies, so the auto-run should CHAIN them: fire the first on
|
||||
// the upload, then the second on the first's output. A classification policy is also present to
|
||||
// assert the engine leaves it alone - annotating policies run themselves (see useClassificationPolicy),
|
||||
// so they are never in this server chain. Stub the contexts + network to drive dispatch against the
|
||||
// REAL run store.
|
||||
const fileStubs: { id: string; name: string; derivedFromTool?: boolean }[] = [];
|
||||
vi.mock("@app/contexts/FileContext", () => ({
|
||||
useAllFiles: () => ({ fileStubs }),
|
||||
useFileManagement: () => ({ addFiles: vi.fn() }),
|
||||
@@ -27,17 +17,27 @@ vi.mock("@app/hooks/usePolicies", () => ({
|
||||
policies: {
|
||||
security: {
|
||||
configured: true,
|
||||
runsOnEditor: true,
|
||||
status: "active",
|
||||
backendId: "backend-sec",
|
||||
runOn: "upload",
|
||||
order: 0,
|
||||
},
|
||||
compliance: {
|
||||
configured: true,
|
||||
runsOnEditor: true,
|
||||
status: "active",
|
||||
backendId: "backend-comp",
|
||||
runOn: "upload",
|
||||
order: 1,
|
||||
},
|
||||
classification: {
|
||||
configured: true,
|
||||
runsOnEditor: true,
|
||||
status: "active",
|
||||
backendId: "backend-cls",
|
||||
runOn: "upload",
|
||||
order: 1,
|
||||
order: 2,
|
||||
},
|
||||
},
|
||||
}),
|
||||
@@ -59,13 +59,14 @@ import { usePolicyAutoRun } from "@app/components/policies/usePolicyAutoRun";
|
||||
import {
|
||||
recordRunStart,
|
||||
updateRun,
|
||||
getRun,
|
||||
resetPolicyRuns,
|
||||
} from "@app/components/policies/policyRunStore";
|
||||
import { runStoredPolicy, getPolicyRun } from "@app/services/policyApi";
|
||||
import { fileStorage } from "@app/services/fileStorage";
|
||||
|
||||
const runStored = vi.mocked(runStoredPolicy);
|
||||
const getPolicyRunMock = vi.mocked(getPolicyRun);
|
||||
const getRunStatus = vi.mocked(getPolicyRun);
|
||||
const getFile = vi.mocked(fileStorage.getStirlingFile);
|
||||
|
||||
/** Reset the shared file list between tests without swapping the array identity. */
|
||||
@@ -74,12 +75,16 @@ function setFileStubs(next: typeof fileStubs) {
|
||||
fileStubs.push(...next);
|
||||
}
|
||||
|
||||
/** A completed security run whose imported output is file-1-v2, ready to chain from. */
|
||||
function seedCompletedSecurityRun() {
|
||||
function completeRun(
|
||||
runId: string,
|
||||
categoryId: string,
|
||||
fileId: string,
|
||||
outputFileIds: string[],
|
||||
) {
|
||||
recordRunStart({
|
||||
runId: "run-sec",
|
||||
categoryId: "security",
|
||||
fileId: "file-1",
|
||||
runId,
|
||||
categoryId,
|
||||
fileId,
|
||||
fileName: "doc.pdf",
|
||||
fileSize: 100,
|
||||
target: "saas",
|
||||
@@ -88,11 +93,7 @@ function seedCompletedSecurityRun() {
|
||||
error: null,
|
||||
startedAt: 0,
|
||||
});
|
||||
updateRun("run-sec", {
|
||||
status: "COMPLETED",
|
||||
imported: true,
|
||||
outputFileIds: ["file-1-v2"],
|
||||
});
|
||||
updateRun(runId, { status: "COMPLETED", imported: true, outputFileIds });
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -100,7 +101,6 @@ beforeEach(() => {
|
||||
localStorage.clear();
|
||||
resetPolicyRuns();
|
||||
setFileStubs([]);
|
||||
aiEnabled.value = true;
|
||||
runStored.mockReset();
|
||||
getFile.mockReset();
|
||||
getFile.mockResolvedValue({ size: 100 } as never);
|
||||
@@ -129,24 +129,8 @@ describe("auto-run ordered chaining", () => {
|
||||
|
||||
it("chains the next policy onto a completed run's output", async () => {
|
||||
// A first-policy run that has completed and imported its output as file-1-v2.
|
||||
recordRunStart({
|
||||
runId: "run-sec",
|
||||
categoryId: "security",
|
||||
fileId: "file-1",
|
||||
fileName: "doc.pdf",
|
||||
fileSize: 100,
|
||||
target: "saas",
|
||||
status: "PENDING",
|
||||
outputs: [],
|
||||
error: null,
|
||||
startedAt: 0,
|
||||
});
|
||||
updateRun("run-sec", {
|
||||
status: "COMPLETED",
|
||||
imported: true,
|
||||
outputFileIds: ["file-1-v2"],
|
||||
});
|
||||
runStored.mockResolvedValue("run-cls");
|
||||
completeRun("run-sec", "security", "file-1", ["file-1-v2"]);
|
||||
runStored.mockResolvedValue("run-comp");
|
||||
|
||||
renderHook(() => usePolicyAutoRun());
|
||||
await act(async () => {
|
||||
@@ -155,182 +139,30 @@ describe("auto-run ordered chaining", () => {
|
||||
|
||||
// Fires on the first policy's output and reports that output's own id, not the original's.
|
||||
expect(runStored).toHaveBeenCalledWith(
|
||||
"backend-cls",
|
||||
"backend-comp",
|
||||
[{ size: 100 }],
|
||||
"file-1-v2",
|
||||
);
|
||||
});
|
||||
|
||||
it("escalates a chained output that carries no verdict", async () => {
|
||||
// The output stub is in the workspace shaped as a new_file-mode delivery (or a
|
||||
// version made before the upload's verdict landed) produces it: tool-derived,
|
||||
// labels inherited, NO classificationConfidence. No local pass ever runs on a
|
||||
// derived file, so waiting for a verdict would skip classification forever —
|
||||
// it must dispatch to the engine instead.
|
||||
seedCompletedSecurityRun();
|
||||
setFileStubs([
|
||||
{
|
||||
id: "file-1-v2",
|
||||
name: "doc.pdf",
|
||||
derivedFromTool: true,
|
||||
classificationLabels: ["invoice"],
|
||||
},
|
||||
]);
|
||||
runStored.mockResolvedValue("run-cls");
|
||||
it("never chains an annotating (classification) policy - it runs itself", async () => {
|
||||
// The last file-producing policy has completed; the engine's chain ends there. Classification
|
||||
// is not a wire link in the chain, so nothing dispatches its backend here.
|
||||
completeRun("run-comp", "compliance", "file-1", ["file-1-v2"]);
|
||||
runStored.mockResolvedValue("run-x");
|
||||
|
||||
renderHook(() => usePolicyAutoRun());
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
});
|
||||
|
||||
expect(runStored).toHaveBeenCalledWith(
|
||||
expect(runStored).not.toHaveBeenCalledWith(
|
||||
"backend-cls",
|
||||
[{ size: 100 }],
|
||||
"file-1-v2",
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it("chains classification onto an output that inherited an unsure verdict", async () => {
|
||||
// The default (new_version) delivery: createChildStub copies the parent's
|
||||
// verdict onto the output, so a low confidence rides through and escalates.
|
||||
seedCompletedSecurityRun();
|
||||
setFileStubs([
|
||||
{
|
||||
id: "file-1-v2",
|
||||
name: "doc.pdf",
|
||||
derivedFromTool: true,
|
||||
classificationLabels: ["invoice"],
|
||||
classificationConfidence: "low",
|
||||
},
|
||||
]);
|
||||
runStored.mockResolvedValue("run-cls");
|
||||
|
||||
renderHook(() => usePolicyAutoRun());
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
});
|
||||
|
||||
expect(runStored).toHaveBeenCalledWith(
|
||||
"backend-cls",
|
||||
[{ size: 100 }],
|
||||
"file-1-v2",
|
||||
);
|
||||
});
|
||||
|
||||
it("lets an inherited confident verdict stand — no engine call for the chained output", async () => {
|
||||
seedCompletedSecurityRun();
|
||||
setFileStubs([
|
||||
{
|
||||
id: "file-1-v2",
|
||||
name: "doc.pdf",
|
||||
derivedFromTool: true,
|
||||
classificationLabels: ["invoice"],
|
||||
classificationConfidence: "high",
|
||||
},
|
||||
]);
|
||||
runStored.mockResolvedValue("run-cls");
|
||||
|
||||
renderHook(() => usePolicyAutoRun());
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
});
|
||||
|
||||
expect(
|
||||
runStored.mock.calls.some(([backendId]) => backendId === "backend-cls"),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("still escalates after the local pass has recorded its own run for the file", async () => {
|
||||
// The regression that made the whole escalation dead in practice: the local heuristic records
|
||||
// a run for the SAME (classification, file) pair, and recordRunStart claims the dispatch key.
|
||||
// The auto-run then reads "already dispatched" and skips the server run forever. A
|
||||
// browser-local run must not claim that key - it is the first pass, not the policy's run.
|
||||
seedCompletedSecurityRun();
|
||||
// The local pass ran on the chained output and recorded its own run for it.
|
||||
recordRunStart({
|
||||
runId: "local-classification-file-1-v2-123",
|
||||
categoryId: "classification",
|
||||
fileId: "file-1-v2",
|
||||
fileName: "doc.pdf",
|
||||
fileSize: 100,
|
||||
target: "local",
|
||||
browserLocal: true,
|
||||
status: "COMPLETED",
|
||||
outputs: [],
|
||||
error: null,
|
||||
startedAt: 0,
|
||||
});
|
||||
// Its verdict was unsure, so the AI must still be asked.
|
||||
setFileStubs([
|
||||
{
|
||||
id: "file-1-v2",
|
||||
name: "doc.pdf",
|
||||
derivedFromTool: true,
|
||||
classificationConfidence: "low",
|
||||
},
|
||||
]);
|
||||
runStored.mockResolvedValue("run-cls");
|
||||
|
||||
renderHook(() => usePolicyAutoRun());
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
});
|
||||
|
||||
expect(runStored).toHaveBeenCalledWith(
|
||||
"backend-cls",
|
||||
[{ size: 100 }],
|
||||
"file-1-v2",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not poll a browser-local run against the server", async () => {
|
||||
// There is no server-side run to ask about: polling 404s, and MAX_NOT_FOUND consecutive
|
||||
// misses would mark a local run that actually succeeded as FAILED.
|
||||
recordRunStart({
|
||||
runId: "local-classification-file-9-456",
|
||||
categoryId: "classification",
|
||||
fileId: "file-9",
|
||||
fileName: "doc.pdf",
|
||||
fileSize: 100,
|
||||
target: "local",
|
||||
browserLocal: true,
|
||||
status: "RUNNING",
|
||||
outputs: [],
|
||||
error: null,
|
||||
startedAt: 0,
|
||||
});
|
||||
setFileStubs([]);
|
||||
|
||||
renderHook(() => usePolicyAutoRun());
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(3000);
|
||||
});
|
||||
|
||||
expect(getPolicyRunMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps classification out of the server chain when the AI engine is off", async () => {
|
||||
// AI off: classification runs client-side (useClientSideClassification), so the
|
||||
// server chain must skip it - only the normal (security) policy dispatches.
|
||||
aiEnabled.value = false;
|
||||
setFileStubs([{ id: "file-1", name: "doc.pdf" }]);
|
||||
runStored.mockResolvedValue("run-sec");
|
||||
|
||||
renderHook(() => usePolicyAutoRun());
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
});
|
||||
|
||||
expect(runStored).toHaveBeenCalledWith(
|
||||
"backend-sec",
|
||||
[{ size: 100 }],
|
||||
"file-1",
|
||||
);
|
||||
expect(
|
||||
runStored.mock.calls.some(([backendId]) => backendId === "backend-cls"),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("never dispatches on a file marked derivedFromTool", async () => {
|
||||
// A policy run is billed, so this gate is what stops `importOutputs` re-enforcing a policy on
|
||||
// its own output forever. If this fails, fix the gate rather than the test.
|
||||
@@ -346,4 +178,53 @@ describe("auto-run ordered chaining", () => {
|
||||
|
||||
expect(runStored).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("never polls a browser-local run (no server behind it) so its success can't 404 to FAILED", async () => {
|
||||
getRunStatus.mockResolvedValue({
|
||||
runId: "srv-1",
|
||||
policyId: null,
|
||||
status: "COMPLETED",
|
||||
currentStep: 1,
|
||||
stepCount: 1,
|
||||
error: null,
|
||||
outputs: [],
|
||||
} as never);
|
||||
// A browser-local heuristic run and a real server run, both left in flight.
|
||||
recordRunStart({
|
||||
runId: "local-1",
|
||||
categoryId: "classification",
|
||||
fileId: "f1",
|
||||
fileName: "d.pdf",
|
||||
fileSize: 1,
|
||||
target: "local",
|
||||
browserLocal: true,
|
||||
status: "RUNNING",
|
||||
outputs: [],
|
||||
error: null,
|
||||
startedAt: 0,
|
||||
});
|
||||
recordRunStart({
|
||||
runId: "srv-1",
|
||||
categoryId: "security",
|
||||
fileId: "f2",
|
||||
fileName: "d.pdf",
|
||||
fileSize: 1,
|
||||
target: "saas",
|
||||
status: "RUNNING",
|
||||
outputs: [],
|
||||
error: null,
|
||||
startedAt: 0,
|
||||
});
|
||||
|
||||
renderHook(() => usePolicyAutoRun());
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(700); // past the first poll (500ms)
|
||||
});
|
||||
|
||||
const polled = getRunStatus.mock.calls.map((c) => c[0]);
|
||||
expect(polled).toContain("srv-1"); // the server run is polled…
|
||||
expect(polled).not.toContain("local-1"); // …the browser-local one never is
|
||||
// And its success is left intact, not flipped to FAILED by a 404 streak.
|
||||
expect(getRun("local-1")?.status).toBe("RUNNING");
|
||||
});
|
||||
});
|
||||
|
||||
-155
@@ -1,155 +0,0 @@
|
||||
/**
|
||||
* The default shipped setup: Classification is the ONLY upload policy, so it dispatches directly
|
||||
* on the upload rather than through the chain. This is the configuration the escalation was built
|
||||
* for, and the one where it was completely dead: the browser-side first pass records its own run
|
||||
* for the same (classification, file) pair, and recordRunStart claims the dispatch key, so the
|
||||
* auto-run read "already dispatched" and never asked the AI - whatever the verdict said.
|
||||
*
|
||||
* Driven against the REAL run store; mocking the store is what let the regression through.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { renderHook, act } from "@testing-library/react";
|
||||
|
||||
vi.mock("@app/hooks/useAiEngineEnabled", () => ({
|
||||
useAiEngineEnabled: () => true,
|
||||
}));
|
||||
const fileStubs: {
|
||||
id: string;
|
||||
name: string;
|
||||
derivedFromTool?: boolean;
|
||||
classificationLabels?: string[];
|
||||
classificationConfidence?: "none" | "low" | "medium" | "high";
|
||||
}[] = [];
|
||||
vi.mock("@app/contexts/FileContext", () => ({
|
||||
useAllFiles: () => ({ fileStubs }),
|
||||
useFileManagement: () => ({ addFiles: vi.fn() }),
|
||||
useFileContext: () => ({ consumeFiles: vi.fn() }),
|
||||
}));
|
||||
vi.mock("@app/hooks/usePolicies", () => ({
|
||||
usePolicies: () => ({
|
||||
policies: {
|
||||
classification: {
|
||||
configured: true,
|
||||
status: "active",
|
||||
backendId: "backend-cls",
|
||||
runOn: "upload",
|
||||
order: 0,
|
||||
},
|
||||
},
|
||||
}),
|
||||
}));
|
||||
vi.mock("@app/services/policyApi", () => ({
|
||||
runStoredPolicy: vi.fn(),
|
||||
getPolicyRun: vi.fn(),
|
||||
downloadPolicyOutput: vi.fn(),
|
||||
resolvePolicyRunTarget: () => "saas",
|
||||
}));
|
||||
vi.mock("@app/services/fileStorage", () => ({
|
||||
fileStorage: { getStirlingFile: vi.fn(), getStirlingFileStub: vi.fn() },
|
||||
}));
|
||||
vi.mock("@app/contexts/IndexedDBContext", () => ({
|
||||
useIndexedDB: () => ({ bumpRevision: vi.fn() }),
|
||||
}));
|
||||
|
||||
import { usePolicyAutoRun } from "@app/components/policies/usePolicyAutoRun";
|
||||
import {
|
||||
recordRunStart,
|
||||
resetPolicyRuns,
|
||||
} from "@app/components/policies/policyRunStore";
|
||||
import { runStoredPolicy } from "@app/services/policyApi";
|
||||
import { fileStorage } from "@app/services/fileStorage";
|
||||
|
||||
const runStored = vi.mocked(runStoredPolicy);
|
||||
const getFile = vi.mocked(fileStorage.getStirlingFile);
|
||||
|
||||
function setFileStubs(next: typeof fileStubs) {
|
||||
fileStubs.length = 0;
|
||||
fileStubs.push(...next);
|
||||
}
|
||||
|
||||
/**
|
||||
* Exactly what useClientSideClassification does when its heuristic pass finishes: a run row for
|
||||
* the activity feed, categorised as classification, for the file it just read.
|
||||
*/
|
||||
function recordLocalPassFor(fileId: string) {
|
||||
recordRunStart({
|
||||
runId: `local-classification-${fileId}-1`,
|
||||
categoryId: "classification",
|
||||
fileId,
|
||||
fileName: "low-confidence-classification-test.pdf",
|
||||
fileSize: 1460,
|
||||
target: "local",
|
||||
browserLocal: true,
|
||||
status: "COMPLETED",
|
||||
outputs: [],
|
||||
error: null,
|
||||
startedAt: 0,
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
resetPolicyRuns();
|
||||
runStored.mockReset();
|
||||
runStored.mockResolvedValue("run-cls");
|
||||
getFile.mockReset();
|
||||
getFile.mockResolvedValue({ size: 1460 } as never);
|
||||
setFileStubs([]);
|
||||
});
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
async function render() {
|
||||
renderHook(() => usePolicyAutoRun());
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
});
|
||||
}
|
||||
|
||||
describe("classification escalation (single-policy setup)", () => {
|
||||
it("asks the AI about an unsure verdict even though the local pass already ran", async () => {
|
||||
// low-confidence-classification-test.pdf: the heuristic emits labels but only at "low".
|
||||
recordLocalPassFor("file-1");
|
||||
setFileStubs([
|
||||
{
|
||||
id: "file-1",
|
||||
name: "low-confidence-classification-test.pdf",
|
||||
classificationLabels: ["contract", "invoice"],
|
||||
classificationConfidence: "low",
|
||||
},
|
||||
]);
|
||||
|
||||
await render();
|
||||
|
||||
expect(runStored).toHaveBeenCalledWith(
|
||||
"backend-cls",
|
||||
[{ size: 1460 }],
|
||||
"file-1",
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves a confident local verdict alone (no engine call, no charge)", async () => {
|
||||
recordLocalPassFor("file-2");
|
||||
setFileStubs([
|
||||
{
|
||||
id: "file-2",
|
||||
name: "invoice.pdf",
|
||||
classificationLabels: ["invoice"],
|
||||
classificationConfidence: "high",
|
||||
},
|
||||
]);
|
||||
|
||||
await render();
|
||||
|
||||
expect(runStored).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("waits for the verdict rather than racing the local pass", async () => {
|
||||
// No verdict yet on a plain upload: dispatching now would pay for an answer the free
|
||||
// first pass is about to produce. The effect re-runs when the verdict lands.
|
||||
setFileStubs([{ id: "file-3", name: "unknown.pdf" }]);
|
||||
|
||||
await render();
|
||||
|
||||
expect(runStored).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -37,6 +37,7 @@ vi.mock("@app/hooks/usePolicies", () => ({
|
||||
policies: {
|
||||
security: {
|
||||
configured: true,
|
||||
runsOnEditor: true,
|
||||
status: "active",
|
||||
backendId: "backend-1",
|
||||
runOn: "upload",
|
||||
|
||||
@@ -36,11 +36,6 @@ const mocks = vi.hoisted(() => ({
|
||||
bumpRevision: vi.fn(),
|
||||
}));
|
||||
|
||||
// Classification chains server-side only when the AI engine is on (else it runs
|
||||
// client-side); this race is in the server import path, so force the engine on.
|
||||
vi.mock("@app/hooks/useAiEngineEnabled", () => ({
|
||||
useAiEngineEnabled: () => true,
|
||||
}));
|
||||
vi.mock("@app/contexts/FileContext", () => ({
|
||||
useAllFiles: () => ({ fileStubs: mocks.workspace }),
|
||||
useFileManagement: () => ({
|
||||
@@ -57,6 +52,7 @@ vi.mock("@app/hooks/usePolicies", () => ({
|
||||
policies: {
|
||||
classification: {
|
||||
configured: true,
|
||||
runsOnEditor: true,
|
||||
status: "active",
|
||||
backendId: "backend-classification",
|
||||
runOn: "upload",
|
||||
@@ -93,6 +89,8 @@ import { usePolicyAutoRun } from "@app/components/policies/usePolicyAutoRun";
|
||||
import {
|
||||
usePolicyRuns,
|
||||
resetPolicyRuns,
|
||||
recordRunStart,
|
||||
updateRun,
|
||||
} from "@app/components/policies/policyRunStore";
|
||||
import type { PolicyRunRecord } from "@app/components/policies/policyRunStore";
|
||||
|
||||
@@ -142,6 +140,26 @@ beforeEach(() => {
|
||||
error: null,
|
||||
outputs: [{ fileId: "backend-out-0", fileName: "doc.pdf" }],
|
||||
});
|
||||
|
||||
// Classification now dispatches its own AI run (see usePolicyLocalPasses); this suite exercises
|
||||
// the auto-run engine's generic import/label-stamping path, so seed a completed classification run
|
||||
// for it to pick up rather than driving a dispatch.
|
||||
recordRunStart({
|
||||
runId: "run-0",
|
||||
categoryId: "classification",
|
||||
fileId: "file-0",
|
||||
fileName: "doc.pdf",
|
||||
fileSize: 100,
|
||||
target: "saas",
|
||||
status: "PENDING",
|
||||
outputs: [],
|
||||
error: null,
|
||||
startedAt: 0,
|
||||
});
|
||||
updateRun("run-0", {
|
||||
status: "COMPLETED",
|
||||
outputs: [{ fileId: "backend-out-0", fileName: "doc.pdf" }],
|
||||
});
|
||||
});
|
||||
|
||||
async function settleImport(timeout = 8000) {
|
||||
|
||||
@@ -31,6 +31,7 @@ vi.mock("@app/hooks/usePolicies", () => ({
|
||||
policies: {
|
||||
security: {
|
||||
configured: true,
|
||||
runsOnEditor: true,
|
||||
status: "active",
|
||||
backendId: "backend-security",
|
||||
runOn: "upload",
|
||||
|
||||
@@ -13,6 +13,7 @@ vi.mock("@app/hooks/usePolicies", () => ({
|
||||
policies: {
|
||||
security: {
|
||||
configured: true,
|
||||
runsOnEditor: true,
|
||||
status: "active",
|
||||
backendId: "backend-1",
|
||||
runOn: "upload",
|
||||
|
||||
@@ -14,11 +14,9 @@ import { refreshNotificationsNow } from "@app/hooks/useNotifications";
|
||||
import { useIndexedDB } from "@app/contexts/IndexedDBContext";
|
||||
import i18n from "@app/i18n";
|
||||
import {
|
||||
runStoredPolicy,
|
||||
getPolicyRun,
|
||||
listPolicyRuns,
|
||||
downloadPolicyOutput,
|
||||
resolvePolicyRunTarget,
|
||||
} from "@app/services/policyApi";
|
||||
import type {
|
||||
PolicyRunStatus,
|
||||
@@ -29,26 +27,19 @@ import type { FileId } from "@app/types/file";
|
||||
import { createStirlingFilesAndStubs } from "@app/services/fileStubHelpers";
|
||||
import { readClassificationLabelsFromFile } from "@app/services/fileClassification";
|
||||
import {
|
||||
orderedRewritingCategories,
|
||||
policyDeliversOutputFiles,
|
||||
policyRequiresAiEngine,
|
||||
policyRewritesDocument,
|
||||
shouldDispatchToAi,
|
||||
} from "@app/data/classificationPolicy";
|
||||
import {
|
||||
acquireDispatchSlot,
|
||||
releaseDispatchSlot,
|
||||
} from "@app/components/policies/dispatchSemaphore";
|
||||
import { runPolicyOnFile } from "@app/services/policyDispatch";
|
||||
import type { StirlingFile, StirlingFileStub } from "@app/types/fileContext";
|
||||
import type { PoliciesByCategory } from "@app/types/policies";
|
||||
import { usePolicies } from "@app/hooks/usePolicies";
|
||||
import { useAiEngineEnabled } from "@app/hooks/useAiEngineEnabled";
|
||||
import {
|
||||
addReconciledRun,
|
||||
dispatchKey,
|
||||
getRun,
|
||||
isDispatched,
|
||||
markDispatched,
|
||||
recordRunStart,
|
||||
removeRun,
|
||||
updateRun,
|
||||
usePolicyRuns,
|
||||
@@ -107,11 +98,6 @@ function failRun(runId: string, message: string): void {
|
||||
updateRun(runId, { status: "FAILED", error: message, errorCode: null });
|
||||
}
|
||||
|
||||
/** Wait for an upload's bytes to land in IndexedDB (~5s): the stub surfaces in the
|
||||
* file list before its bytes are committed, so an eager fetch would miss the file. */
|
||||
const FILE_WAIT_TRIES = 20;
|
||||
const FILE_WAIT_MS = 250;
|
||||
|
||||
/** A policy that changed nothing completes with no output; left unimported its badge
|
||||
* and blocking overlay spin forever. */
|
||||
export function finishedWithNothingToDeliver(run: PolicyRunRecord): boolean {
|
||||
@@ -138,7 +124,6 @@ export function usePolicyAutoRun(): void {
|
||||
const { consumeFiles } = useFileContext();
|
||||
const { bumpRevision } = useIndexedDB();
|
||||
const { policies } = usePolicies();
|
||||
const aiEnabled = useAiEngineEnabled();
|
||||
const runs = usePolicyRuns();
|
||||
// Read in the import effect via ref, not as a dependency: delivery mutates fileStubs,
|
||||
// so depending on them would re-fire the effect on its own delivery (infinite cascade).
|
||||
@@ -155,33 +140,13 @@ export function usePolicyAutoRun(): void {
|
||||
// sentinel for a saas listener to open the modal. Deduped per run.
|
||||
const firedLimitModal = useRef<Set<string>>(new Set());
|
||||
|
||||
// Active upload policies in chain order, so effects accumulate instead of racing to fork
|
||||
// the same version. Mirrors the dispatch filter so the chain honours the same eligibility.
|
||||
// The file-producing upload policies this engine dispatches and chains, in run order, so effects
|
||||
// accumulate instead of racing to fork the same version. Annotating policies (classification) are
|
||||
// absent by design: they run themselves (local pass, then AI escalation), so the engine never sees
|
||||
// their two ways to run.
|
||||
const orderedUploadCategories = useMemo(
|
||||
() =>
|
||||
Object.entries(policies)
|
||||
.filter(
|
||||
([id, s]) =>
|
||||
s.configured &&
|
||||
s.status === "active" &&
|
||||
s.backendId &&
|
||||
(!s.sources ||
|
||||
s.sources.length === 0 ||
|
||||
s.sources.includes("editor")) &&
|
||||
(s.runOn ?? "upload") === "upload" &&
|
||||
// An escalation-only policy has nothing to do with no engine to escalate to.
|
||||
!(policyRequiresAiEngine(id) && !aiEnabled),
|
||||
)
|
||||
// Annotating policies run last: a rewriting one after them would fork a new
|
||||
// version from the pre-annotation state and drop their labels.
|
||||
.sort(([idA, a], [idB, b]) => {
|
||||
const ra = policyRewritesDocument(idA) ? 0 : 1;
|
||||
const rb = policyRewritesDocument(idB) ? 0 : 1;
|
||||
if (ra !== rb) return ra - rb;
|
||||
return (a.order ?? 0) - (b.order ?? 0);
|
||||
})
|
||||
.map(([id]) => id),
|
||||
[policies, aiEnabled],
|
||||
() => orderedRewritingCategories(policies),
|
||||
[policies],
|
||||
);
|
||||
|
||||
// Chain-continuations handled this session, so the next policy fires once per run.
|
||||
@@ -190,9 +155,6 @@ export function usePolicyAutoRun(): void {
|
||||
// Latest policies, read from inside the stable retry callback (which has no deps).
|
||||
const policiesRef = useRef(policies);
|
||||
policiesRef.current = policies;
|
||||
// Latest stubs for the chaining effect, which keys off runs and must not depend on stubs.
|
||||
const stubsRef = useRef(fileStubs);
|
||||
stubsRef.current = fileStubs;
|
||||
// Per-file (dispatchKey) count of consecutive queue-rejection retries, so backoff escalates and
|
||||
// eventually gives up. Survives the run-id changing on each retry; reset on any real outcome.
|
||||
const queueRetries = useRef<Map<string, number>>(new Map());
|
||||
@@ -272,8 +234,6 @@ export function usePolicyAutoRun(): void {
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
// A confident local verdict stands; only an unsure one is escalated to the engine.
|
||||
if (!shouldDispatchToAi(firstCategory, stub)) continue;
|
||||
dispatching.current.add(key);
|
||||
void runPolicyOnFile(firstCategory, backendId, stub.id, stub.name)
|
||||
.catch(() => {
|
||||
@@ -307,13 +267,6 @@ export function usePolicyAutoRun(): void {
|
||||
// would otherwise silently skip the next policy on outputs 2..N.
|
||||
for (const outputId of outputIds) {
|
||||
if (isDispatched(nextCategory, outputId as FileId)) continue;
|
||||
const outputStub = stubsRef.current.find((s) => s.id === outputId);
|
||||
// The output's inherited verdict decides here and now (no local pass ever runs
|
||||
// on a derived file, so there is nothing to defer to): a confident one stands,
|
||||
// anything else - including no verdict at all, e.g. a new_file-mode delivery -
|
||||
// escalates. A stub not yet in the snapshot falls through to dispatch too.
|
||||
if (outputStub && !shouldDispatchToAi(nextCategory, outputStub))
|
||||
continue;
|
||||
void runPolicyOnFile(
|
||||
nextCategory,
|
||||
backendId,
|
||||
@@ -323,15 +276,18 @@ export function usePolicyAutoRun(): void {
|
||||
).catch(() => {});
|
||||
}
|
||||
}
|
||||
}, [runs, policies, orderedUploadCategories, fileStubs]);
|
||||
}, [runs, policies, orderedUploadCategories]);
|
||||
|
||||
// Poll each in-flight run to a terminal state.
|
||||
// Poll each in-flight run to a terminal state. A browser-local run (the classification heuristic's
|
||||
// first pass) has no server run behind it, so polling it 404s and would flip its success to FAILED.
|
||||
useEffect(() => {
|
||||
for (const run of runs) {
|
||||
// A browser-local run has no server-side status: polling it 404s (and after MAX_NOT_FOUND
|
||||
// marks a run that actually succeeded as failed). Its own pass settles it.
|
||||
if (run.browserLocal) continue;
|
||||
if (isTerminal(run.status) || polling.current.has(run.runId)) continue;
|
||||
if (
|
||||
run.browserLocal ||
|
||||
isTerminal(run.status) ||
|
||||
polling.current.has(run.runId)
|
||||
)
|
||||
continue;
|
||||
polling.current.add(run.runId);
|
||||
void poll(run.runId, onRunFinished).finally(() =>
|
||||
polling.current.delete(run.runId),
|
||||
@@ -377,6 +333,7 @@ export function usePolicyAutoRun(): void {
|
||||
void importOutputs(run, {
|
||||
addFiles,
|
||||
consumeFiles,
|
||||
policyName: policies[run.categoryId]?.name,
|
||||
updateStirlingFileStub,
|
||||
bumpRevision,
|
||||
outputMode,
|
||||
@@ -427,6 +384,8 @@ interface ImportContext {
|
||||
bumpRevision: () => void;
|
||||
/** "new_file" adds the output as a separate file; "new_version" versions the input. */
|
||||
outputMode: "new_file" | "new_version";
|
||||
/** The policy's name, shown in version history instead of the generic "automate" tool. */
|
||||
policyName?: string;
|
||||
/** Rename rule. Empty → keep the input's filename. */
|
||||
outputName: string;
|
||||
/** Rename position around the base filename; defaults to "suffix" when absent. */
|
||||
@@ -732,7 +691,8 @@ async function importOutputs(
|
||||
// origin) — so a 60-file batch doesn't re-read every downstream output.
|
||||
const parentLabels = parentStub?.classificationLabels;
|
||||
const resolveLabels = async (file: File) =>
|
||||
(parentLabels && parentLabels.length > 0 ? parentLabels : undefined) ??
|
||||
// Inherit the parent's verdict
|
||||
(Array.isArray(parentLabels) ? parentLabels : undefined) ??
|
||||
(await readClassificationLabelsFromFile(file)) ??
|
||||
undefined;
|
||||
|
||||
@@ -744,6 +704,7 @@ async function importOutputs(
|
||||
files,
|
||||
parentStub,
|
||||
"automate",
|
||||
ctx.policyName,
|
||||
);
|
||||
// Transitive provenance for the PERSISTED record, mirroring what the
|
||||
// CONSUME_FILES reducer computes for workspace state: the output derives
|
||||
@@ -773,7 +734,7 @@ async function importOutputs(
|
||||
// Mark the outputs handled BEFORE adding them (belt-and-suspenders session
|
||||
// guard on top of derivedFromTool) so the auto-run never enforces the policy
|
||||
// on its own output — that would version endlessly in a loop.
|
||||
for (const s of categorized) markHandled(s.id);
|
||||
for (const s of categorized) markHandled(s.id as string);
|
||||
deliveredIds = categorized.map((s) => s.id as string);
|
||||
if (ctx.parentStub) {
|
||||
// Input is in the active workspace: version it in place, silently — the
|
||||
@@ -804,7 +765,7 @@ async function importOutputs(
|
||||
derivedFromTool: true,
|
||||
});
|
||||
// Belt-and-suspenders session guard on top of derivedFromTool.
|
||||
for (const f of added) markHandled(f.fileId);
|
||||
for (const f of added) markHandled(f.fileId as string);
|
||||
deliveredIds = added.map((f) => f.fileId as string);
|
||||
// Mark each new-file output as tool-derived (the versioned path gets this from the
|
||||
// CONSUME_FILES reducer; the addFiles path doesn't). This is the real loop guard: the dispatch
|
||||
@@ -850,75 +811,6 @@ async function importOutputs(
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve the file's bytes, fire a backend run, and record it. */
|
||||
async function runPolicyOnFile(
|
||||
categoryId: string,
|
||||
backendId: string,
|
||||
fileId: FileId,
|
||||
fileName: string,
|
||||
// Chained (downstream) dispatch — jumps the dispatch queue so a file mid-chain
|
||||
// finishes its flow before new files start (see acquireDispatchSlot).
|
||||
priority = false,
|
||||
): Promise<void> {
|
||||
// A freshly-uploaded file's bytes are written to IndexedDB asynchronously, so
|
||||
// its stub can appear in the file list a beat before getStirlingFile resolves
|
||||
// it. Wait briefly rather than bail — and DON'T mark dispatched until we hold
|
||||
// the file, or a too-early miss would skip enforcement on that file forever.
|
||||
// (The caller's in-flight guard prevents double-dispatch during this wait.)
|
||||
// A transient IndexedDB error is treated as a miss (not a throw), so it retries
|
||||
// and then marks dispatched rather than rejecting into a hot re-dispatch loop.
|
||||
const tryGetFile = async (): Promise<StirlingFile | null> => {
|
||||
try {
|
||||
return await fileStorage.getStirlingFile(fileId);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
let file = await tryGetFile();
|
||||
for (let i = 0; i < FILE_WAIT_TRIES && !file; i++) {
|
||||
await delay(FILE_WAIT_MS);
|
||||
file = await tryGetFile();
|
||||
}
|
||||
if (!file) {
|
||||
// File genuinely gone (removed before it could run) — mark so we don't loop.
|
||||
markDispatched(categoryId, fileId);
|
||||
return;
|
||||
}
|
||||
// Bounded upload window — see MAX_CONCURRENT_DISPATCHES. Only the POST is
|
||||
// gated; the IDB wait above never holds a slot.
|
||||
await acquireDispatchSlot(priority);
|
||||
try {
|
||||
const target = resolvePolicyRunTarget();
|
||||
// Recorded against a document this browser can resolve. One file per run, which is the only
|
||||
// shape the server keeps a reference for.
|
||||
const runId = await runStoredPolicy(backendId, [file], fileId);
|
||||
// recordRunStart marks this (policy, file) dispatched as it records the run.
|
||||
recordRunStart({
|
||||
runId,
|
||||
categoryId,
|
||||
fileId,
|
||||
fileName,
|
||||
fileSize: file.size,
|
||||
target,
|
||||
status: "PENDING",
|
||||
outputs: [],
|
||||
error: null,
|
||||
startedAt: Date.now(),
|
||||
});
|
||||
} catch (err) {
|
||||
// Dispatch failed (e.g. policy deleted/404 or backend offline). Mark dispatched so we don't hammer;
|
||||
// the absent run simply won't appear in the activity feed. If the backend did
|
||||
// start a run we never recorded, reconcileServerRuns rediscovers it.
|
||||
console.debug(
|
||||
`[PolicyAutoRun] Failed to dispatch policy ${categoryId} (${backendId}):`,
|
||||
err,
|
||||
);
|
||||
markDispatched(categoryId, fileId);
|
||||
} finally {
|
||||
releaseDispatchSlot();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll a run's status until it reaches a terminal state (or the budget). Calls {@code onTerminal} once
|
||||
* with the final view when it terminates — the caller uses that to pop the usage-limit modal when a
|
||||
|
||||
+130
-20
@@ -1,5 +1,6 @@
|
||||
// Delivery guarantees of the client-side classification hook, driving the real
|
||||
// policyRunStore and mocking only IO (storage, the heuristic engine, the meter).
|
||||
// The Classification policy hook: classifies each settled document locally and escalates an unsure
|
||||
// verdict to the AI engine itself. Drives the real policyRunStore and mocks only IO (storage, the
|
||||
// heuristic engine, the meter, the shared dispatch primitive).
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { renderHook, waitFor } from "@testing-library/react";
|
||||
@@ -16,6 +17,9 @@ interface TestStub {
|
||||
classificationLabels?: string[];
|
||||
}
|
||||
|
||||
const aiEnabled = vi.hoisted(() => ({ value: false }));
|
||||
const runOn = vi.hoisted(() => ({ value: "upload" as "upload" | "export" }));
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
workspace: [] as Array<{
|
||||
id: string;
|
||||
@@ -31,6 +35,7 @@ const mocks = vi.hoisted(() => ({
|
||||
updateFileMetadata: vi.fn(async (_id: string, _updates: unknown) => true),
|
||||
classify: vi.fn(),
|
||||
meter: vi.fn(),
|
||||
runPolicyOnFile: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@app/contexts/AppConfigContext", () => ({
|
||||
@@ -40,15 +45,20 @@ vi.mock("@app/hooks/useClassificationEnabled", () => ({
|
||||
useClassificationEnabled: () => true,
|
||||
}));
|
||||
vi.mock("@app/hooks/useAiEngineEnabled", () => ({
|
||||
useAiEngineEnabled: () => false,
|
||||
useAiEngineEnabled: () => aiEnabled.value,
|
||||
}));
|
||||
vi.mock("@app/services/policyDispatch", () => ({
|
||||
runPolicyOnFile: (...args: unknown[]) => mocks.runPolicyOnFile(...args),
|
||||
}));
|
||||
vi.mock("@app/hooks/usePolicies", () => ({
|
||||
usePolicies: () => ({
|
||||
policies: {
|
||||
classification: {
|
||||
configured: true,
|
||||
runsOnEditor: true,
|
||||
status: "active",
|
||||
backendId: "backend-classification",
|
||||
runOn: runOn.value,
|
||||
sources: ["editor"],
|
||||
},
|
||||
},
|
||||
@@ -77,10 +87,7 @@ vi.mock("@app/services/classificationMeter", () => ({
|
||||
meterClassificationRun: (payload: unknown) => mocks.meter(payload),
|
||||
}));
|
||||
|
||||
import {
|
||||
useClientSideClassification,
|
||||
LOCAL_METER_CATEGORY,
|
||||
} from "@app/components/policies/useClientSideClassification";
|
||||
import { usePolicyLocalPasses } from "@app/components/policies/usePolicyLocalPasses";
|
||||
|
||||
// Run idle callbacks immediately so batches start without timer waits.
|
||||
vi.stubGlobal("requestIdleCallback", (cb: () => void) => {
|
||||
@@ -98,7 +105,7 @@ const stub = (id: string, extra: Partial<TestStub> = {}): TestStub => ({
|
||||
|
||||
const fakeFile = (id: string) => new File([id], `${id}.pdf`);
|
||||
|
||||
describe("useClientSideClassification delivery", () => {
|
||||
describe("usePolicyLocalPasses delivery", () => {
|
||||
beforeEach(() => {
|
||||
localStorage.clear();
|
||||
resetPolicyRuns();
|
||||
@@ -113,6 +120,10 @@ describe("useClientSideClassification delivery", () => {
|
||||
fakeFile(id),
|
||||
);
|
||||
mocks.classify.mockReset();
|
||||
aiEnabled.value = false;
|
||||
runOn.value = "upload";
|
||||
mocks.runPolicyOnFile.mockReset();
|
||||
mocks.runPolicyOnFile.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
it("classifies pending uploads, writes labels, and meters once per file", async () => {
|
||||
@@ -121,7 +132,7 @@ describe("useClientSideClassification delivery", () => {
|
||||
labels: [file.name.startsWith("a") ? "invoice" : "resume"],
|
||||
}));
|
||||
|
||||
renderHook(() => useClientSideClassification());
|
||||
renderHook(() => usePolicyLocalPasses());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledTimes(2),
|
||||
@@ -144,7 +155,7 @@ describe("useClientSideClassification delivery", () => {
|
||||
);
|
||||
mocks.workspace = [stub("a")];
|
||||
|
||||
const { rerender } = renderHook(() => useClientSideClassification());
|
||||
const { rerender } = renderHook(() => usePolicyLocalPasses());
|
||||
await waitFor(() => expect(mocks.classify).toHaveBeenCalledTimes(1));
|
||||
|
||||
// A new upload mid-classify re-fires the effect and cancels the in-flight
|
||||
@@ -172,7 +183,7 @@ describe("useClientSideClassification delivery", () => {
|
||||
mocks.workspace = [stub("plain")];
|
||||
mocks.classify.mockResolvedValue({ labels: [] });
|
||||
|
||||
renderHook(() => useClientSideClassification());
|
||||
renderHook(() => usePolicyLocalPasses());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("plain", {
|
||||
@@ -184,13 +195,12 @@ describe("useClientSideClassification delivery", () => {
|
||||
});
|
||||
|
||||
it("heals a previously-dispatched file whose result was lost, without re-metering", async () => {
|
||||
// A past session classified + metered this file but the delivery was lost. The marker is the
|
||||
// local-meter key, NOT the classification dispatch key - that one belongs to the server run.
|
||||
markDispatched(LOCAL_METER_CATEGORY, "lost");
|
||||
// A past session classified + metered this file but the delivery was lost.
|
||||
markDispatched("classification", "lost");
|
||||
mocks.workspace = [stub("lost")];
|
||||
mocks.classify.mockResolvedValue({ labels: ["bank-statement"] });
|
||||
|
||||
renderHook(() => useClientSideClassification());
|
||||
renderHook(() => usePolicyLocalPasses());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("lost", {
|
||||
@@ -207,7 +217,7 @@ describe("useClientSideClassification delivery", () => {
|
||||
mocks.workspace = [stub("corrupt")];
|
||||
mocks.classify.mockRejectedValue(new Error("bad pdf"));
|
||||
|
||||
renderHook(() => useClientSideClassification());
|
||||
renderHook(() => usePolicyLocalPasses());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
@@ -229,7 +239,7 @@ describe("useClientSideClassification delivery", () => {
|
||||
mocks.workspace = [stub("early")];
|
||||
mocks.classify.mockResolvedValue({ labels: ["invoice"] });
|
||||
|
||||
const { rerender } = renderHook(() => useClientSideClassification());
|
||||
const { rerender } = renderHook(() => usePolicyLocalPasses());
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
expect(mocks.classify).not.toHaveBeenCalled();
|
||||
|
||||
@@ -243,17 +253,117 @@ describe("useClientSideClassification delivery", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("skips tool outputs and already-labelled files", async () => {
|
||||
it("skips already-classified files (a real label, or the no-label [] verdict)", async () => {
|
||||
mocks.workspace = [
|
||||
stub("derived", { derivedFromTool: true }),
|
||||
stub("done", { classificationLabels: ["invoice"] }),
|
||||
// [] means classified-nothing-found: it must NOT be re-classified (or re-billed).
|
||||
stub("verdict", { classificationLabels: [] }),
|
||||
];
|
||||
|
||||
renderHook(() => useClientSideClassification());
|
||||
renderHook(() => usePolicyLocalPasses());
|
||||
|
||||
// Nothing to classify; give the (immediate) idle path a beat to prove it.
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
expect(mocks.classify).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("classifies a policy new-file output that has no verdict to inherit", async () => {
|
||||
// A new-file policy output is derivedFromTool but carries no label - there is no parent to
|
||||
// inherit from. Main classified these through the chain, so the local pass must pick them up.
|
||||
mocks.classify.mockResolvedValue({ labels: ["invoice"] });
|
||||
mocks.workspace = [stub("newfile", { derivedFromTool: true })];
|
||||
|
||||
renderHook(() => usePolicyLocalPasses());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("newfile", {
|
||||
classificationLabels: ["invoice"],
|
||||
}),
|
||||
);
|
||||
expect(mocks.meter).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("escalates an unsure local verdict to the AI engine itself", async () => {
|
||||
aiEnabled.value = true;
|
||||
mocks.workspace = [stub("a")];
|
||||
mocks.classify.mockResolvedValue({
|
||||
labels: ["invoice"],
|
||||
confidence: "low",
|
||||
});
|
||||
|
||||
renderHook(() => usePolicyLocalPasses());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mocks.runPolicyOnFile).toHaveBeenCalledWith(
|
||||
"classification",
|
||||
"backend-classification",
|
||||
"a",
|
||||
"a.pdf",
|
||||
),
|
||||
);
|
||||
// The local verdict is still delivered before escalation.
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("a", {
|
||||
classificationLabels: ["invoice"],
|
||||
classificationConfidence: "low",
|
||||
});
|
||||
});
|
||||
|
||||
it("lets a confident local verdict stand without asking the AI", async () => {
|
||||
aiEnabled.value = true;
|
||||
mocks.workspace = [stub("a")];
|
||||
mocks.classify.mockResolvedValue({
|
||||
labels: ["invoice"],
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
renderHook(() => usePolicyLocalPasses());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("a", {
|
||||
classificationLabels: ["invoice"],
|
||||
classificationConfidence: "high",
|
||||
}),
|
||||
);
|
||||
expect(mocks.runPolicyOnFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("never escalates when the AI engine is off, however unsure the verdict", async () => {
|
||||
aiEnabled.value = false;
|
||||
mocks.workspace = [stub("a")];
|
||||
mocks.classify.mockResolvedValue({
|
||||
labels: ["invoice"],
|
||||
confidence: "none",
|
||||
});
|
||||
|
||||
renderHook(() => usePolicyLocalPasses());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("a", {
|
||||
classificationLabels: ["invoice"],
|
||||
classificationConfidence: "none",
|
||||
}),
|
||||
);
|
||||
expect(mocks.runPolicyOnFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not run on upload when the policy is set to run on export", async () => {
|
||||
// An export-time policy is enforced by the export path, not this upload engine, so an upload
|
||||
// must not classify, meter, or escalate.
|
||||
runOn.value = "export";
|
||||
aiEnabled.value = true;
|
||||
mocks.workspace = [stub("a")];
|
||||
mocks.classify.mockResolvedValue({
|
||||
labels: ["invoice"],
|
||||
confidence: "low",
|
||||
});
|
||||
|
||||
renderHook(() => usePolicyLocalPasses());
|
||||
|
||||
// Give the (immediate) idle path a beat to prove it stays silent.
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
expect(mocks.classify).not.toHaveBeenCalled();
|
||||
expect(mocks.updateStirlingFileStub).not.toHaveBeenCalled();
|
||||
expect(mocks.meter).not.toHaveBeenCalled();
|
||||
expect(mocks.runPolicyOnFile).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,144 @@
|
||||
/**
|
||||
* Generic engine for policies' browser-side fast paths. Any active editor policy that declares a
|
||||
* {@link LocalPass} has it run here: eligible files are classified/processed locally, the returned
|
||||
* fields are written to the stub, and the policy's server run is dispatched only if the pass says it
|
||||
* is still needed (and the AI engine, if the policy needs it, is on).
|
||||
*/
|
||||
|
||||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import { useAllFiles, useFileManagement } from "@app/contexts/FileContext";
|
||||
import { useAppConfig } from "@app/contexts/AppConfigContext";
|
||||
import { useIndexedDB } from "@app/contexts/IndexedDBContext";
|
||||
import { fileStorage } from "@app/services/fileStorage";
|
||||
import { useAiEngineEnabled } from "@app/hooks/useAiEngineEnabled";
|
||||
import { scheduleIdle } from "@app/utils/scheduleIdle";
|
||||
import { usePolicies } from "@app/hooks/usePolicies";
|
||||
import { runPolicyOnFile } from "@app/services/policyDispatch";
|
||||
import {
|
||||
localPassFor,
|
||||
type LocalPass,
|
||||
} from "@app/components/policies/policyLocalPass";
|
||||
import { policyRequiresAiEngine } from "@app/data/classificationPolicy";
|
||||
import type { StirlingFileStub } from "@app/types/fileContext";
|
||||
|
||||
/** Files processed per idle pass, so a large upload drains over several ticks instead of janking. */
|
||||
const LOCAL_PASS_BATCH = 3;
|
||||
|
||||
interface ActivePass {
|
||||
categoryId: string;
|
||||
backendId: string;
|
||||
pass: LocalPass;
|
||||
/** When true, the server run is skipped while the AI engine is off (nothing to escalate to). */
|
||||
requiresAiEngine: boolean;
|
||||
}
|
||||
|
||||
export function usePolicyLocalPasses(): void {
|
||||
const { fileStubs } = useAllFiles();
|
||||
const { updateStirlingFileStub } = useFileManagement();
|
||||
const { bumpRevision } = useIndexedDB();
|
||||
const { policies } = usePolicies();
|
||||
const aiEnabled = useAiEngineEnabled();
|
||||
// Waited on so a verdict is not written and escalated before it is known whether the AI engine
|
||||
// (which the server run may need) is even available.
|
||||
const { loading: configLoading } = useAppConfig();
|
||||
// Files claimed this session, keyed policy+id+lastModified so a new version is retried once. Claimed
|
||||
// synchronously right before running, so overlapping batches never double-process.
|
||||
const claimed = useRef<Set<string>>(new Set());
|
||||
// Bumped after each batch to drain the next one.
|
||||
const [tick, setTick] = useState(0);
|
||||
|
||||
// Active editor upload policies that declare a local fast path.
|
||||
const passes = useMemo<ActivePass[]>(() => {
|
||||
const out: ActivePass[] = [];
|
||||
for (const [categoryId, s] of Object.entries(policies)) {
|
||||
const active =
|
||||
s.configured &&
|
||||
s.status === "active" &&
|
||||
s.backendId &&
|
||||
s.runsOnEditor &&
|
||||
(s.runOn ?? "upload") === "upload";
|
||||
if (!active) continue;
|
||||
const pass = localPassFor(categoryId);
|
||||
if (!pass) continue;
|
||||
out.push({
|
||||
categoryId,
|
||||
backendId: s.backendId as string,
|
||||
pass,
|
||||
requiresAiEngine: policyRequiresAiEngine(categoryId),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}, [policies]);
|
||||
|
||||
useEffect(() => {
|
||||
if (configLoading || passes.length === 0) return;
|
||||
const claimKey = (categoryId: string, s: StirlingFileStub) =>
|
||||
`${categoryId}:${s.id as string}:${s.lastModified ?? 0}`;
|
||||
// Collect one idle batch of pending (pass, file) work across all passes.
|
||||
const batch: { active: ActivePass; stub: StirlingFileStub }[] = [];
|
||||
outer: for (const active of passes) {
|
||||
for (const stub of fileStubs) {
|
||||
if (batch.length >= LOCAL_PASS_BATCH) break outer;
|
||||
if (!active.pass.eligible(stub)) continue;
|
||||
if (claimed.current.has(claimKey(active.categoryId, stub))) continue;
|
||||
batch.push({ active, stub });
|
||||
}
|
||||
}
|
||||
if (batch.length === 0) return;
|
||||
let cancelled = false;
|
||||
const cancelIdle = scheduleIdle(() => {
|
||||
// Superseded before starting: the newer effect instance owns the queue.
|
||||
if (cancelled) return;
|
||||
void (async () => {
|
||||
let wrote = false;
|
||||
for (const { active, stub } of batch) {
|
||||
const key = claimKey(active.categoryId, stub);
|
||||
// Re-validate at execution time - another batch may have claimed it since.
|
||||
if (claimed.current.has(key)) continue;
|
||||
claimed.current.add(key);
|
||||
const result = await active.pass.run(stub.id, stub);
|
||||
// Could not run (e.g. bytes not in storage yet): leave unprocessed so a reload retries.
|
||||
if (result == null) continue;
|
||||
// Deliver unconditionally - a re-render must never discard a computed result. Writes are
|
||||
// idempotent. The engine applies the fields the pass returned without reading them.
|
||||
updateStirlingFileStub(stub.id, result.stubUpdates);
|
||||
const ok = await fileStorage.updateFileMetadata(
|
||||
stub.id,
|
||||
result.stubUpdates,
|
||||
);
|
||||
if (ok) wrote = true;
|
||||
// Dispatch the server run only if the pass still wants it, and skip it while an
|
||||
// AI-engine-dependent policy has no engine to reach.
|
||||
if (
|
||||
result.needsServerRun &&
|
||||
!(active.requiresAiEngine && !aiEnabled)
|
||||
) {
|
||||
void runPolicyOnFile(
|
||||
active.categoryId,
|
||||
active.backendId,
|
||||
stub.id,
|
||||
stub.name,
|
||||
).catch(() => {
|
||||
// Backstop: runPolicyOnFile handles its own failures.
|
||||
});
|
||||
}
|
||||
}
|
||||
if (wrote) bumpRevision();
|
||||
// Drain the next batch; the terminal pass finds nothing pending and stops.
|
||||
setTick((n) => n + 1);
|
||||
})();
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
cancelIdle();
|
||||
};
|
||||
}, [
|
||||
fileStubs,
|
||||
passes,
|
||||
aiEnabled,
|
||||
configLoading,
|
||||
updateStirlingFileStub,
|
||||
bumpRevision,
|
||||
tick,
|
||||
]);
|
||||
}
|
||||
@@ -1,26 +1,23 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
isClassificationCategory,
|
||||
localVerdictNeedsEscalation,
|
||||
orderRewritesFirst,
|
||||
orderedRewritingCategories,
|
||||
policyDeliversOutputFiles,
|
||||
policyRequiresAiEngine,
|
||||
policyRewritesDocument,
|
||||
shouldDispatchToAi,
|
||||
} from "@app/data/classificationPolicy";
|
||||
import type { StirlingFileStub } from "@app/types/fileContext";
|
||||
import type { PoliciesByCategory } from "@app/types/policies";
|
||||
|
||||
const stub = (
|
||||
confidence?: StirlingFileStub["classificationConfidence"],
|
||||
): StirlingFileStub =>
|
||||
({ classificationConfidence: confidence }) as StirlingFileStub;
|
||||
|
||||
const derivedStub = (
|
||||
confidence?: StirlingFileStub["classificationConfidence"],
|
||||
): StirlingFileStub =>
|
||||
const rewriter = (order: number) =>
|
||||
({
|
||||
derivedFromTool: true,
|
||||
classificationConfidence: confidence,
|
||||
}) as StirlingFileStub;
|
||||
configured: true,
|
||||
status: "active",
|
||||
backendId: `backend-${order}`,
|
||||
runsOnEditor: true,
|
||||
runOn: "upload",
|
||||
order,
|
||||
}) as unknown as PoliciesByCategory[string];
|
||||
|
||||
describe("isClassificationCategory", () => {
|
||||
it("recognises the classification category and nothing else", () => {
|
||||
@@ -42,11 +39,6 @@ describe("policy capabilities", () => {
|
||||
expect(policyDeliversOutputFiles("security")).toBe(true);
|
||||
expect(policyDeliversOutputFiles("classification")).toBe(false);
|
||||
});
|
||||
|
||||
it("marks classification as the AI-escalation policy", () => {
|
||||
expect(policyRequiresAiEngine("classification")).toBe(true);
|
||||
expect(policyRequiresAiEngine("security")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("orderRewritesFirst", () => {
|
||||
@@ -75,40 +67,52 @@ describe("orderRewritesFirst", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldDispatchToAi", () => {
|
||||
it("always dispatches a policy that is not classification", () => {
|
||||
expect(shouldDispatchToAi("security", stub())).toBe(true);
|
||||
expect(shouldDispatchToAi("security", stub("high"))).toBe(true);
|
||||
describe("orderedRewritingCategories", () => {
|
||||
it("lists only file-producing policies, ordered by order, excluding classification", () => {
|
||||
const policies = {
|
||||
classification: rewriter(0), // annotating: excluded despite being active
|
||||
security: rewriter(2),
|
||||
compliance: rewriter(1),
|
||||
} as unknown as PoliciesByCategory;
|
||||
// classification is filtered by policyDeliversOutputFiles, not by the shape above.
|
||||
expect(orderedRewritingCategories(policies)).toEqual([
|
||||
"compliance",
|
||||
"security",
|
||||
]);
|
||||
});
|
||||
|
||||
it("holds back until the local heuristic has reported", () => {
|
||||
// Not a skip: dispatching now races the local pass and pays for a free answer;
|
||||
// the caller re-evaluates once the verdict lands.
|
||||
expect(shouldDispatchToAi("classification", stub())).toBe(false);
|
||||
it("excludes inactive, non-editor, export-triggered, and unconfigured policies", () => {
|
||||
const mixed = {
|
||||
security: rewriter(0),
|
||||
inactive: { ...rewriter(1), status: "paused" },
|
||||
notEditor: { ...rewriter(2), runsOnEditor: false },
|
||||
onExport: { ...rewriter(3), runOn: "export" },
|
||||
unconfigured: { ...rewriter(4), configured: false },
|
||||
noBackend: { ...rewriter(5), backendId: undefined },
|
||||
} as unknown as PoliciesByCategory;
|
||||
expect(orderedRewritingCategories(mixed)).toEqual(["security"]);
|
||||
});
|
||||
|
||||
it("is empty when classification is the only policy", () => {
|
||||
const only = {
|
||||
classification: rewriter(0),
|
||||
} as unknown as PoliciesByCategory;
|
||||
expect(orderedRewritingCategories(only)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("localVerdictNeedsEscalation", () => {
|
||||
it("lets a confident local verdict stand", () => {
|
||||
expect(shouldDispatchToAi("classification", stub("high"))).toBe(false);
|
||||
expect(localVerdictNeedsEscalation("high")).toBe(false);
|
||||
});
|
||||
|
||||
it("does not escalate when no verdict has been recorded yet", () => {
|
||||
expect(localVerdictNeedsEscalation(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it("escalates anything less than confident", () => {
|
||||
expect(shouldDispatchToAi("classification", stub("medium"))).toBe(true);
|
||||
expect(shouldDispatchToAi("classification", stub("low"))).toBe(true);
|
||||
expect(shouldDispatchToAi("classification", stub("none"))).toBe(true);
|
||||
});
|
||||
|
||||
it("escalates a tool-derived file with no verdict at all", () => {
|
||||
// A derived file gets no local pass (useClientSideClassification skips it), so
|
||||
// there is no verdict to wait for: holding back would skip it forever. This is
|
||||
// the chained case for a new_file-mode output, or a version made before the
|
||||
// upload's verdict landed.
|
||||
expect(shouldDispatchToAi("classification", derivedStub())).toBe(true);
|
||||
});
|
||||
|
||||
it("lets a derived file's inherited verdict decide like an upload's own", () => {
|
||||
expect(shouldDispatchToAi("classification", derivedStub("high"))).toBe(
|
||||
false,
|
||||
);
|
||||
expect(shouldDispatchToAi("classification", derivedStub("low"))).toBe(true);
|
||||
expect(localVerdictNeedsEscalation("medium")).toBe(true);
|
||||
expect(localVerdictNeedsEscalation("low")).toBe(true);
|
||||
expect(localVerdictNeedsEscalation("none")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
/**
|
||||
* Everything specific to the built-in Classification policy, in one module. The generic policy
|
||||
* runner asks the capability questions below instead of naming classification itself, so a second
|
||||
* annotating policy needs a change here rather than in the runner.
|
||||
* Everything specific to the built-in Classification policy, in one module. The generic policy runner
|
||||
* dispatches and chains only file-producing policies (see {@link orderedRewritingCategories}), and the
|
||||
* generic local-pass engine runs whatever browser-side fast path a policy declares. Classification's
|
||||
* fast path (its heuristic) lives in classificationLocalPass; the capability answers below let the
|
||||
* generic engines treat it without naming it. A second annotating policy is a change here, not there.
|
||||
*
|
||||
* These are still keyed on the category id rather than a property each policy declares. That is
|
||||
* deliberate for now: policies are becoming pipelines with labels behind a separate enforcement
|
||||
@@ -11,10 +13,8 @@
|
||||
* mode, and a run result that can carry findings as well as files), not in a flag added here first.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ClassificationConfidence,
|
||||
StirlingFileStub,
|
||||
} from "@app/types/fileContext";
|
||||
import type { ClassificationConfidence } from "@app/types/fileContext";
|
||||
import type { PoliciesByCategory } from "@app/types/policies";
|
||||
|
||||
/** Catalogue category id of the built-in Classification policy. */
|
||||
export const CLASSIFICATION_CATEGORY_ID = "classification";
|
||||
@@ -36,7 +36,10 @@ export function policyDeliversOutputFiles(categoryId: string): boolean {
|
||||
return policyRewritesDocument(categoryId);
|
||||
}
|
||||
|
||||
/** Whether the policy's server-side run exists only to escalate to the AI engine. */
|
||||
/**
|
||||
* Whether the policy's server run needs the AI engine. The local-pass engine skips dispatching such
|
||||
* a run when the engine is off - there is nothing to escalate to, and the local verdict stands.
|
||||
*/
|
||||
export function policyRequiresAiEngine(categoryId: string): boolean {
|
||||
return isClassificationCategory(categoryId);
|
||||
}
|
||||
@@ -49,6 +52,29 @@ export function orderRewritesFirst(categoryIds: string[]): string[] {
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* The active editor upload policies the generic runner dispatches and chains, in run order. Only
|
||||
* file-producing policies: an annotating policy (classification) has no output to chain onto and
|
||||
* runs itself, so it is intentionally absent here. Both the runner and the classification policy
|
||||
* read this - the runner to sequence the chain, classification to know when that chain is done.
|
||||
*/
|
||||
export function orderedRewritingCategories(
|
||||
policies: PoliciesByCategory,
|
||||
): string[] {
|
||||
return Object.entries(policies)
|
||||
.filter(
|
||||
([id, s]) =>
|
||||
s.configured &&
|
||||
s.status === "active" &&
|
||||
Boolean(s.backendId) &&
|
||||
s.runsOnEditor &&
|
||||
(s.runOn ?? "upload") === "upload" &&
|
||||
policyDeliversOutputFiles(id),
|
||||
)
|
||||
.sort(([, a], [, b]) => (a.order ?? 0) - (b.order ?? 0))
|
||||
.map(([id]) => id);
|
||||
}
|
||||
|
||||
/**
|
||||
* The one heuristic verdict trusted to stand on its own; anything less escalates to the AI, which
|
||||
* overwrites it. Deliberately strict - a wrong label costs more than an engine call.
|
||||
@@ -56,18 +82,12 @@ export function orderRewritesFirst(categoryIds: string[]): string[] {
|
||||
const TRUSTED_CONFIDENCE: ClassificationConfidence = "high";
|
||||
|
||||
/**
|
||||
* Whether the AI classifier should be asked about this file. For an upload, only once the
|
||||
* heuristic has reported: dispatching before then races the first pass and bills for an answer it
|
||||
* was about to produce. A tool-derived file gets no local pass (useClientSideClassification skips
|
||||
* it) and only ever carries an inherited verdict, so an absent verdict there is permanent -
|
||||
* escalate rather than wait for a report that will never come.
|
||||
* Whether a local classification verdict must be escalated to the AI engine. A confident verdict
|
||||
* stands on its own; anything less is escalated and overwritten. Owned here, alongside the local
|
||||
* pass that produces the verdict - the runner is not involved.
|
||||
*/
|
||||
export function shouldDispatchToAi(
|
||||
categoryId: string,
|
||||
stub: StirlingFileStub,
|
||||
export function localVerdictNeedsEscalation(
|
||||
confidence: ClassificationConfidence | undefined,
|
||||
): boolean {
|
||||
if (!isClassificationCategory(categoryId)) return true;
|
||||
const confidence = stub.classificationConfidence;
|
||||
if (confidence == null) return Boolean(stub.derivedFromTool);
|
||||
return confidence !== TRUSTED_CONFIDENCE;
|
||||
return confidence != null && confidence !== TRUSTED_CONFIDENCE;
|
||||
}
|
||||
|
||||
@@ -48,7 +48,8 @@ const wizardResult = {
|
||||
updatedAt: "",
|
||||
},
|
||||
fieldValues: {},
|
||||
sources: ["editor"],
|
||||
sources: [],
|
||||
runsOnEditor: true,
|
||||
scopeTypes: [],
|
||||
reviewerEmail: "reviewer@x.com",
|
||||
folder: {
|
||||
@@ -153,4 +154,76 @@ describe("usePolicies", () => {
|
||||
});
|
||||
expect(result.current.policies.ingestion.folderId).toBeTruthy();
|
||||
});
|
||||
|
||||
// A builder pipeline has no category tile, so the reconcile must key it by id to reach the map
|
||||
// the auto-run iterates.
|
||||
it("reconciles a builder pipeline that has no category", async () => {
|
||||
api.store.set("be-pipeline", {
|
||||
id: "be-pipeline",
|
||||
name: "My pipeline",
|
||||
enabled: true,
|
||||
inputs: [],
|
||||
steps: [{ operation: "/api/v1/misc/compress-pdf", parameters: {} }],
|
||||
output: { type: "inline", options: {} },
|
||||
outputIds: [],
|
||||
editor: { allowed: true, runOn: "upload" },
|
||||
} as unknown as { id: string });
|
||||
|
||||
const { result } = renderHook(() => usePolicies());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(result.current.policies["be-pipeline"]?.configured).toBe(true),
|
||||
);
|
||||
const pipeline = result.current.policies["be-pipeline"];
|
||||
expect(pipeline.runsOnEditor).toBe(true);
|
||||
// Not a catalogue tile, so it is deletable rather than a built-in default.
|
||||
expect(pipeline.isDefault).toBe(false);
|
||||
});
|
||||
|
||||
it("does not put a builder pipeline on the editor unless it opts in", async () => {
|
||||
api.store.set("be-s3", {
|
||||
id: "be-s3",
|
||||
name: "S3 sweep",
|
||||
enabled: true,
|
||||
inputs: [],
|
||||
steps: [{ operation: "/api/v1/misc/compress-pdf", parameters: {} }],
|
||||
output: { type: "inline", options: {} },
|
||||
outputIds: [],
|
||||
} as unknown as { id: string });
|
||||
|
||||
const { result } = renderHook(() => usePolicies());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(result.current.policies["be-s3"]?.configured).toBe(true),
|
||||
);
|
||||
expect(result.current.policies["be-s3"].runsOnEditor).toBe(false);
|
||||
});
|
||||
|
||||
// Deleting a pipeline on the Pipelines page leaves its cached entry behind. It still satisfies
|
||||
// every auto-run condition but its backendId is dead, so the dispatch fails, the run never
|
||||
// completes, and every policy behind it in the chain is skipped on every upload.
|
||||
it("forgets a builder pipeline the backend no longer has", async () => {
|
||||
localStorage.setItem(
|
||||
"stirling-policies-state",
|
||||
JSON.stringify({
|
||||
"be-deleted": {
|
||||
configured: true,
|
||||
status: "active",
|
||||
backendId: "be-deleted",
|
||||
sources: ["editor"],
|
||||
runsOnEditor: true,
|
||||
runOn: "upload",
|
||||
isDefault: false,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const { result } = renderHook(() => usePolicies());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(result.current.policies["be-deleted"]).toBeUndefined(),
|
||||
);
|
||||
// A catalogue tile is never forgotten: it reseeds from the catalogue.
|
||||
expect(result.current.policies.security).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
onPoliciesChange,
|
||||
updatePolicy,
|
||||
resetPolicy,
|
||||
forgetPolicies,
|
||||
reorderPolicies as persistPolicyOrder,
|
||||
} from "@app/services/policyStorage";
|
||||
import { loadPolicyCatalog } from "@app/services/policyCatalog";
|
||||
@@ -36,7 +37,7 @@ import {
|
||||
} from "@app/services/policyBackend";
|
||||
import { reorderPolicies as reorderBackendPolicies } from "@app/services/policyApi";
|
||||
import { orderRewritesFirst } from "@app/data/classificationPolicy";
|
||||
import type { PolicyToStore } from "@app/services/policyPipeline";
|
||||
import { type PolicyToStore } from "@app/services/policyPipeline";
|
||||
import type {
|
||||
PoliciesByCategory,
|
||||
PolicyConfigResult,
|
||||
@@ -65,6 +66,7 @@ function toStoreRequest(
|
||||
automation: result.automation,
|
||||
pipelineSteps: result.pipelineSteps,
|
||||
sources: result.sources,
|
||||
runsOnEditor: result.runsOnEditor,
|
||||
scopeTypes: result.scopeTypes,
|
||||
reviewerEmail: result.reviewerEmail,
|
||||
fieldValues: result.fieldValues,
|
||||
@@ -114,6 +116,20 @@ export function usePolicies() {
|
||||
backendId: undefined,
|
||||
};
|
||||
}
|
||||
// Builder-made pipelines have no category, so the built-in loop above skips them. They are
|
||||
// still policies: one set to run on the editor has to reach the auto-run.
|
||||
for (const [key, decoded] of byCategory) {
|
||||
if (reconciled[key]) continue;
|
||||
reconciled[key] = decodedToState(decoded, local[key]?.folderId);
|
||||
}
|
||||
// A builder pipeline the backend no longer has was deleted on the Pipelines page. Its cached
|
||||
// entry keeps a dead backendId that still satisfies the auto-run filter, so the dispatch
|
||||
// fails, the run never completes, and the chain behind it never advances.
|
||||
forgetPolicies(
|
||||
Object.keys(local).filter(
|
||||
(id) => !reconciled[id] && !byCategory.has(id),
|
||||
),
|
||||
);
|
||||
for (const [id, state] of Object.entries(reconciled)) {
|
||||
updatePolicy(id, state);
|
||||
}
|
||||
@@ -157,6 +173,7 @@ export function usePolicies() {
|
||||
backendId,
|
||||
fieldValues: result.fieldValues,
|
||||
sources: result.sources,
|
||||
runsOnEditor: result.runsOnEditor,
|
||||
scopeTypes: result.scopeTypes,
|
||||
reviewerEmail: result.reviewerEmail,
|
||||
outputMode: result.folder.outputMode,
|
||||
@@ -194,6 +211,7 @@ export function usePolicies() {
|
||||
backendId,
|
||||
fieldValues: result.fieldValues,
|
||||
sources: result.sources,
|
||||
runsOnEditor: result.runsOnEditor,
|
||||
scopeTypes: result.scopeTypes,
|
||||
reviewerEmail: result.reviewerEmail,
|
||||
outputMode: result.folder.outputMode,
|
||||
@@ -246,6 +264,7 @@ export function usePolicies() {
|
||||
},
|
||||
pipelineSteps: result.pipelineSteps,
|
||||
sources: result.sources,
|
||||
runsOnEditor: result.runsOnEditor,
|
||||
scopeTypes: result.scopeTypes,
|
||||
reviewerEmail: result.reviewerEmail,
|
||||
fieldValues: result.fieldValues,
|
||||
@@ -259,6 +278,7 @@ export function usePolicies() {
|
||||
backendId,
|
||||
fieldValues: result.fieldValues,
|
||||
sources: result.sources,
|
||||
runsOnEditor: result.runsOnEditor,
|
||||
scopeTypes: result.scopeTypes,
|
||||
reviewerEmail: result.reviewerEmail,
|
||||
outputMode: result.folder.outputMode,
|
||||
|
||||
@@ -8,6 +8,7 @@ const FULL_STATE: PolicyDecodedState = {
|
||||
enabled: true,
|
||||
categoryId: "security",
|
||||
sources: ["editor", "gdrive"],
|
||||
runsOnEditor: true,
|
||||
scopeTypes: ["Contracts", "Invoices"],
|
||||
reviewerEmail: "admin@example.com",
|
||||
fieldValues: { auditTrail: true, frameworks: ["HIPAA"] },
|
||||
@@ -44,6 +45,26 @@ describe("toWirePolicy", () => {
|
||||
expect(opts.position).toBe("prefix");
|
||||
});
|
||||
|
||||
it("sends the editor block so a save never drops editor participation", () => {
|
||||
expect(toWirePolicy(FULL_STATE).editor).toEqual({
|
||||
allowed: true,
|
||||
runOn: "upload",
|
||||
});
|
||||
expect(toWirePolicy({ ...FULL_STATE, runsOnEditor: false }).editor).toEqual(
|
||||
{ allowed: false, runOn: "upload" },
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps editor participation that empty sources would have re-derived away", () => {
|
||||
// The seeded Classification policy: editor-run, no sources.
|
||||
const wire = toWirePolicy({
|
||||
...FULL_STATE,
|
||||
sources: [],
|
||||
runsOnEditor: true,
|
||||
});
|
||||
expect(wire.editor?.allowed).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves steps at the top level", () => {
|
||||
const wire = toWirePolicy(FULL_STATE);
|
||||
expect(wire.steps).toEqual(FULL_STATE.steps);
|
||||
@@ -69,15 +90,24 @@ describe("fromWirePolicy → round-trip", () => {
|
||||
expect(decoded.steps).toEqual(FULL_STATE.steps);
|
||||
});
|
||||
|
||||
it("defaults a missing runOn to the category default (security → export)", () => {
|
||||
const wire = toWirePolicy(FULL_STATE);
|
||||
// The moment has two possible homes now (the `editor` block, and the legacy
|
||||
// options bag), so "nothing stored" means clearing both.
|
||||
const withNoStoredRunOn = (state: PolicyDecodedState) => {
|
||||
const wire = toWirePolicy(state);
|
||||
delete (wire.output.options as Record<string, unknown>).runOn;
|
||||
expect(fromWirePolicy(wire).runOn).toBe("export");
|
||||
delete wire.editor;
|
||||
return wire;
|
||||
};
|
||||
|
||||
it("defaults a missing runOn to the category default (security → export)", () => {
|
||||
expect(fromWirePolicy(withNoStoredRunOn(FULL_STATE)).runOn).toBe("export");
|
||||
});
|
||||
|
||||
it("defaults a missing runOn to upload for other categories", () => {
|
||||
const wire = toWirePolicy({ ...FULL_STATE, categoryId: "classification" });
|
||||
delete (wire.output.options as Record<string, unknown>).runOn;
|
||||
const wire = withNoStoredRunOn({
|
||||
...FULL_STATE,
|
||||
categoryId: "classification",
|
||||
});
|
||||
expect(fromWirePolicy(wire).runOn).toBe("upload");
|
||||
});
|
||||
|
||||
@@ -109,6 +139,28 @@ describe("fromWirePolicy → round-trip", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("reads editor participation off the editor block, not sources", () => {
|
||||
const wire = toWirePolicy(FULL_STATE);
|
||||
expect(fromWirePolicy(wire).runsOnEditor).toBe(true);
|
||||
expect(
|
||||
fromWirePolicy({ ...wire, editor: { allowed: false, runOn: "upload" } })
|
||||
.runsOnEditor,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("prefers the editor block's moment over the legacy options bag", () => {
|
||||
const wire = toWirePolicy(FULL_STATE);
|
||||
wire.output.options.runOn = "upload";
|
||||
wire.editor = { allowed: true, runOn: "export" };
|
||||
expect(fromWirePolicy(wire).runOn).toBe("export");
|
||||
});
|
||||
|
||||
it("falls back to the stored moment when the editor does not run it", () => {
|
||||
const wire = toWirePolicy({ ...FULL_STATE, runOn: "export" });
|
||||
wire.editor = { allowed: false, runOn: "upload" };
|
||||
expect(fromWirePolicy(wire).runOn).toBe("export");
|
||||
});
|
||||
|
||||
it("handles empty options gracefully", () => {
|
||||
const decoded = fromWirePolicy({
|
||||
id: "x",
|
||||
@@ -120,6 +172,7 @@ describe("fromWirePolicy → round-trip", () => {
|
||||
});
|
||||
expect(decoded.categoryId).toBe("");
|
||||
expect(decoded.sources).toEqual([]);
|
||||
expect(decoded.runsOnEditor).toBe(false);
|
||||
expect(decoded.runOn).toBe("upload");
|
||||
expect(decoded.outputMode).toBe("new_version");
|
||||
});
|
||||
|
||||
@@ -41,6 +41,9 @@ export function toWirePolicy(state: PolicyDecodedState): WirePolicy {
|
||||
trigger: null,
|
||||
steps: state.steps,
|
||||
output: { type: "inline", options },
|
||||
// Omitting this makes the backend stamp EditorConfig.disabled(), so a pause or a
|
||||
// wizard save would quietly take the policy off the editor.
|
||||
editor: { allowed: state.runsOnEditor, runOn: state.runOn },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -63,10 +66,17 @@ export function fromWirePolicy(policy: WirePolicy): PolicyDecodedState {
|
||||
enabled: policy.enabled,
|
||||
categoryId,
|
||||
sources: Array.isArray(raw.sources) ? raw.sources : [],
|
||||
runsOnEditor: policy.editor?.allowed === true,
|
||||
scopeTypes: Array.isArray(raw.scopeTypes) ? raw.scopeTypes : [],
|
||||
reviewerEmail: str(raw.reviewerEmail),
|
||||
fieldValues: raw.fieldValues ?? {},
|
||||
runOn: resolveRunOn(raw.runOn, categoryId),
|
||||
// The moment lives on `editor` now, but only carries meaning while the editor
|
||||
// runs it (EditorConfig coerces a disabled policy's runOn to "upload"); fall back
|
||||
// to the legacy options bag otherwise so the wizard still shows what was chosen.
|
||||
runOn: resolveRunOn(
|
||||
policy.editor?.allowed ? policy.editor.runOn : raw.runOn,
|
||||
categoryId,
|
||||
),
|
||||
outputMode: raw.mode === "new_file" ? "new_file" : "new_version",
|
||||
outputName: str(raw.name),
|
||||
outputNamePosition: position,
|
||||
|
||||
@@ -36,6 +36,16 @@ export interface WireOutputSpec {
|
||||
options: Partial<WireOutputOptions>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirrors `EditorConfig.java`. Absent only on records that never went through the
|
||||
* backend (hand-built fixtures); a stored policy always carries it, derived from
|
||||
* the legacy `output.options` bag when it predates the field.
|
||||
*/
|
||||
export interface WireEditorConfig {
|
||||
allowed: boolean;
|
||||
runOn: "upload" | "export";
|
||||
}
|
||||
|
||||
export interface WirePolicy {
|
||||
id: string;
|
||||
name: string;
|
||||
@@ -44,6 +54,7 @@ export interface WirePolicy {
|
||||
trigger: null;
|
||||
steps: WirePipelineStep[];
|
||||
output: WireOutputSpec;
|
||||
editor?: WireEditorConfig;
|
||||
teamId?: string;
|
||||
}
|
||||
|
||||
@@ -80,6 +91,12 @@ export interface PolicyDecodedState {
|
||||
enabled: boolean;
|
||||
categoryId: string;
|
||||
sources: string[];
|
||||
/**
|
||||
* Whether the editor runs this policy per file. Its own field, not derived from
|
||||
* `sources`: the seeded Classification policy is editor-run with empty sources,
|
||||
* so re-deriving on write would silently take it off the editor.
|
||||
*/
|
||||
runsOnEditor: boolean;
|
||||
scopeTypes: string[];
|
||||
reviewerEmail: string;
|
||||
fieldValues: Record<string, boolean | string | string[]>;
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import {
|
||||
decodedToState,
|
||||
fetchPoliciesByCategory,
|
||||
} from "@app/services/policyBackend";
|
||||
|
||||
const listPolicies = vi.fn();
|
||||
vi.mock("@app/services/policyApi", () => ({
|
||||
listPolicies: () => listPolicies(),
|
||||
}));
|
||||
|
||||
/** A stored policy in the shape the backend returns. */
|
||||
const policy = (
|
||||
id: string,
|
||||
categoryId?: string,
|
||||
editor?: { allowed: boolean; runOn?: "upload" | "export" },
|
||||
) => ({
|
||||
id,
|
||||
name: id,
|
||||
enabled: true,
|
||||
inputs: [],
|
||||
steps: [{ operation: "/api/v1/misc/compress-pdf", parameters: {} }],
|
||||
output: {
|
||||
type: "inline",
|
||||
options: { ...(categoryId ? { categoryId } : {}) },
|
||||
},
|
||||
outputIds: [],
|
||||
editor: {
|
||||
allowed: editor?.allowed ?? false,
|
||||
runOn: editor?.runOn ?? ("upload" as const),
|
||||
},
|
||||
});
|
||||
|
||||
/** Decode one stored policy and project it onto the state the editor reads. */
|
||||
async function stateOf(wire: ReturnType<typeof policy>, key: string) {
|
||||
listPolicies.mockResolvedValue([wire]);
|
||||
const decoded = (await fetchPoliciesByCategory()).get(key);
|
||||
if (!decoded) throw new Error(`no decoded policy for ${key}`);
|
||||
return decodedToState(decoded, undefined);
|
||||
}
|
||||
|
||||
describe("fetchPoliciesByCategory", () => {
|
||||
beforeEach(() => listPolicies.mockReset());
|
||||
|
||||
it("keys a catalogue policy by its category", async () => {
|
||||
listPolicies.mockResolvedValue([
|
||||
policy("pol-1", "classification", { allowed: true }),
|
||||
]);
|
||||
|
||||
const map = await fetchPoliciesByCategory();
|
||||
|
||||
expect(map.get("classification")?.id).toBe("pol-1");
|
||||
});
|
||||
|
||||
it("keeps a pipeline that has no category, keyed by its id", async () => {
|
||||
// Built on the Pipelines page, so no category tile stamped it. It is still a policy: one set
|
||||
// to run on editor uploads has to reach the editor's auto-run, which iterates this map.
|
||||
listPolicies.mockResolvedValue([policy("pol-adhoc")]);
|
||||
|
||||
const map = await fetchPoliciesByCategory();
|
||||
|
||||
expect(map.has("pol-adhoc")).toBe(true);
|
||||
expect(map.get("pol-adhoc")?.id).toBe("pol-adhoc");
|
||||
});
|
||||
|
||||
it("carries both kinds at once without either displacing the other", async () => {
|
||||
listPolicies.mockResolvedValue([
|
||||
policy("pol-1", "classification", { allowed: true }),
|
||||
policy("pol-adhoc"),
|
||||
]);
|
||||
|
||||
const map = await fetchPoliciesByCategory();
|
||||
|
||||
expect([...map.keys()].sort()).toEqual(["classification", "pol-adhoc"]);
|
||||
});
|
||||
|
||||
it("records run order from the list, which is the team's order", async () => {
|
||||
listPolicies.mockResolvedValue([
|
||||
policy("pol-1", "security", { allowed: true }),
|
||||
policy("pol-adhoc"),
|
||||
]);
|
||||
|
||||
const map = await fetchPoliciesByCategory();
|
||||
|
||||
expect(map.get("security")?.order).toBe(0);
|
||||
expect(map.get("pol-adhoc")?.order).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("decodedToState — runsOnEditor", () => {
|
||||
beforeEach(() => listPolicies.mockReset());
|
||||
|
||||
it("runs a catalogue tile that opted into the editor", async () => {
|
||||
const state = await stateOf(
|
||||
policy("pol-1", "security", { allowed: true }),
|
||||
"security",
|
||||
);
|
||||
|
||||
expect(state.runsOnEditor).toBe(true);
|
||||
});
|
||||
|
||||
// Participation is the policy's own flag now, so a tile that never opted in does not run in the
|
||||
// editor just because nobody narrowed its scope.
|
||||
it("does not run a catalogue tile that never opted in", async () => {
|
||||
const state = await stateOf(policy("pol-1", "security"), "security");
|
||||
|
||||
expect(state.runsOnEditor).toBe(false);
|
||||
});
|
||||
|
||||
it("does not run a builder pipeline that never named the editor", async () => {
|
||||
// Blank here means nothing stamped it - the tile default would fire an S3 or folder
|
||||
// pipeline on every editor upload.
|
||||
const state = await stateOf(policy("pol-adhoc"), "pol-adhoc");
|
||||
|
||||
expect(state.runsOnEditor).toBe(false);
|
||||
});
|
||||
|
||||
it("runs a builder pipeline that names the editor outright", async () => {
|
||||
const state = await stateOf(
|
||||
policy("pol-adhoc", undefined, { allowed: true }),
|
||||
"pol-adhoc",
|
||||
);
|
||||
|
||||
expect(state.runsOnEditor).toBe(true);
|
||||
});
|
||||
|
||||
it("marks only a catalogue tile as a built-in default", async () => {
|
||||
expect(
|
||||
(await stateOf(policy("pol-1", "security"), "security")).isDefault,
|
||||
).toBe(true);
|
||||
expect((await stateOf(policy("pol-adhoc"), "pol-adhoc")).isDefault).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -32,8 +32,10 @@ export async function fetchPoliciesByCategory(): Promise<
|
||||
const byCategory = new Map<string, DecodedPolicy>();
|
||||
stored.forEach((policy, index) => {
|
||||
const decoded = fromBackendPolicy(policy);
|
||||
if (decoded.categoryId)
|
||||
byCategory.set(decoded.categoryId, { ...decoded, order: index });
|
||||
// A pipeline built on the Pipelines page has no category tile, so it keys by its own id
|
||||
// rather than being dropped - one set to run on the editor still has to reach the auto-run.
|
||||
const key = decoded.categoryId || decoded.id;
|
||||
if (key) byCategory.set(key, { ...decoded, order: index });
|
||||
});
|
||||
return byCategory;
|
||||
}
|
||||
@@ -50,7 +52,9 @@ export function decodedToState(
|
||||
return {
|
||||
configured: true,
|
||||
status: decoded.enabled ? "active" : "paused",
|
||||
name: decoded.name,
|
||||
sources: decoded.sources,
|
||||
runsOnEditor: decoded.runsOnEditor,
|
||||
scopeTypes: decoded.scopeTypes,
|
||||
reviewerEmail: decoded.reviewerEmail,
|
||||
fieldValues: decoded.fieldValues,
|
||||
@@ -62,8 +66,8 @@ export function decodedToState(
|
||||
backendId: decoded.id,
|
||||
// Server-side run-order position (team-wide); drives the settings reorder list.
|
||||
order: decoded.order,
|
||||
// Catalog-category policies are built-in defaults (not deletable).
|
||||
isDefault: true,
|
||||
// Catalog-category policies are built-in defaults (not deletable); a builder pipeline is not.
|
||||
isDefault: Boolean(decoded.categoryId),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* Fire a single backend policy run for one file and record it. Shared by the auto-run engine (which
|
||||
* dispatches file-producing policies and their chain) and the classification policy (which dispatches
|
||||
* its own AI escalation), so both take one bounded dispatch slot and record runs the same way.
|
||||
*/
|
||||
|
||||
import { fileStorage } from "@app/services/fileStorage";
|
||||
import {
|
||||
runStoredPolicy,
|
||||
resolvePolicyRunTarget,
|
||||
} from "@app/services/policyApi";
|
||||
import {
|
||||
acquireDispatchSlot,
|
||||
releaseDispatchSlot,
|
||||
} from "@app/components/policies/dispatchSemaphore";
|
||||
import {
|
||||
markDispatched,
|
||||
recordRunStart,
|
||||
} from "@app/components/policies/policyRunStore";
|
||||
import type { FileId } from "@app/types/file";
|
||||
import type { StirlingFile } from "@app/types/fileContext";
|
||||
|
||||
/** Wait for an upload's bytes to land in IndexedDB (~5s): the stub surfaces in the
|
||||
* file list before its bytes are committed, so an eager fetch would miss the file. */
|
||||
const FILE_WAIT_TRIES = 20;
|
||||
const FILE_WAIT_MS = 250;
|
||||
|
||||
const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/** Resolve the file's bytes, fire a backend run, and record it. */
|
||||
export async function runPolicyOnFile(
|
||||
categoryId: string,
|
||||
backendId: string,
|
||||
fileId: FileId,
|
||||
fileName: string,
|
||||
// Chained (downstream) dispatch — jumps the dispatch queue so a file mid-chain
|
||||
// finishes its flow before new files start (see acquireDispatchSlot).
|
||||
priority = false,
|
||||
): Promise<void> {
|
||||
// A freshly-uploaded file's bytes are written to IndexedDB asynchronously, so
|
||||
// its stub can appear in the file list a beat before getStirlingFile resolves
|
||||
// it. Wait briefly rather than bail — and DON'T mark dispatched until we hold
|
||||
// the file, or a too-early miss would skip enforcement on that file forever.
|
||||
// (The caller's in-flight guard prevents double-dispatch during this wait.)
|
||||
// A transient IndexedDB error is treated as a miss (not a throw), so it retries
|
||||
// and then marks dispatched rather than rejecting into a hot re-dispatch loop.
|
||||
const tryGetFile = async (): Promise<StirlingFile | null> => {
|
||||
try {
|
||||
return await fileStorage.getStirlingFile(fileId);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
let file = await tryGetFile();
|
||||
for (let i = 0; i < FILE_WAIT_TRIES && !file; i++) {
|
||||
await delay(FILE_WAIT_MS);
|
||||
file = await tryGetFile();
|
||||
}
|
||||
if (!file) {
|
||||
// File genuinely gone (removed before it could run) — mark so we don't loop.
|
||||
markDispatched(categoryId, fileId);
|
||||
return;
|
||||
}
|
||||
// Bounded upload window — see MAX_CONCURRENT_DISPATCHES. Only the POST is
|
||||
// gated; the IDB wait above never holds a slot.
|
||||
await acquireDispatchSlot(priority);
|
||||
try {
|
||||
const target = resolvePolicyRunTarget();
|
||||
// Recorded against a document this browser can resolve. One file per run, which is the only
|
||||
// shape the server keeps a reference for.
|
||||
const runId = await runStoredPolicy(backendId, [file], fileId);
|
||||
// recordRunStart marks this (policy, file) dispatched as it records the run.
|
||||
recordRunStart({
|
||||
runId,
|
||||
categoryId,
|
||||
fileId,
|
||||
fileName,
|
||||
fileSize: file.size,
|
||||
target,
|
||||
status: "PENDING",
|
||||
outputs: [],
|
||||
error: null,
|
||||
startedAt: Date.now(),
|
||||
});
|
||||
} catch (err) {
|
||||
// Dispatch failed (e.g. policy deleted/404 or backend offline). Mark dispatched so we don't hammer;
|
||||
// the absent run simply won't appear in the activity feed. If the backend did
|
||||
// start a run we never recorded, reconcileServerRuns rediscovers it.
|
||||
console.debug(
|
||||
`[PolicyAutoRun] Failed to dispatch policy ${categoryId} (${backendId}):`,
|
||||
err,
|
||||
);
|
||||
markDispatched(categoryId, fileId);
|
||||
} finally {
|
||||
releaseDispatchSlot();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import type { PoliciesByCategory, PolicyState } from "@app/types/policies";
|
||||
|
||||
// Which policies export-time enforcement picks up: the policy's own editor flag, not its scope.
|
||||
|
||||
const loadPolicies = vi.fn<() => PoliciesByCategory>();
|
||||
vi.mock("@app/services/policyStorage", () => ({
|
||||
loadPolicies: () => loadPolicies(),
|
||||
}));
|
||||
|
||||
const runStoredPolicy = vi.fn(async (_id: string) => "run-1");
|
||||
vi.mock("@app/services/policyApi", () => ({
|
||||
runStoredPolicy: (id: string) => runStoredPolicy(id),
|
||||
// One output, so a run completes rather than throwing "produced no output" - which would abort
|
||||
// the per-file policy loop after the first policy and hide the order under test.
|
||||
getPolicyRun: async () => ({
|
||||
status: "COMPLETED",
|
||||
outputs: [{ fileId: "out-1", fileName: "doc.pdf" }],
|
||||
}),
|
||||
downloadPolicyOutput: async () => new Blob(),
|
||||
resolvePolicyRunTarget: () => "local",
|
||||
}));
|
||||
|
||||
vi.mock("@app/components/policies/policyRunStore", () => ({
|
||||
recordRunStart: vi.fn(),
|
||||
isDispatched: () => false,
|
||||
}));
|
||||
// Run the queued task inline: the queue's own behaviour is not under test here.
|
||||
vi.mock("@app/components/policies/enforcementQueue", () => ({
|
||||
runQueued: <T>(_meta: unknown, task: () => Promise<T>) => task(),
|
||||
}));
|
||||
vi.mock("@app/components/toast", () => ({
|
||||
alert: () => "toast-1",
|
||||
updateToast: vi.fn(),
|
||||
dismissToast: vi.fn(),
|
||||
}));
|
||||
vi.mock("@app/i18n", () => ({ default: { t: (key: string) => key } }));
|
||||
|
||||
const { enforceExportPolicies } = await import("@app/services/policyExport");
|
||||
|
||||
/** An active export-time policy as the local store holds it. */
|
||||
const exportPolicy = (over: Partial<PolicyState>): PolicyState =>
|
||||
({
|
||||
configured: true,
|
||||
status: "active",
|
||||
backendId: "backend-1",
|
||||
sources: [],
|
||||
runsOnEditor: false,
|
||||
scopeTypes: [],
|
||||
reviewerEmail: "",
|
||||
fieldValues: {},
|
||||
outputMode: "new_version",
|
||||
outputName: "",
|
||||
runOn: "export",
|
||||
isDefault: false,
|
||||
...over,
|
||||
}) as PolicyState;
|
||||
|
||||
const pdf = () =>
|
||||
new File(["%PDF-1.4"], "doc.pdf", { type: "application/pdf" });
|
||||
|
||||
describe("export-time policy selection", () => {
|
||||
beforeEach(() => runStoredPolicy.mockClear());
|
||||
|
||||
it("enforces an editor pipeline set to run on export", async () => {
|
||||
loadPolicies.mockReturnValue({
|
||||
"builder-1": exportPolicy({
|
||||
sources: ["editor"],
|
||||
runsOnEditor: true,
|
||||
backendId: "backend-editor",
|
||||
}),
|
||||
} as unknown as PoliciesByCategory);
|
||||
|
||||
await enforceExportPolicies([pdf()], ["file-1"]);
|
||||
|
||||
expect(runStoredPolicy).toHaveBeenCalledWith("backend-editor");
|
||||
});
|
||||
|
||||
it("leaves a swept pipeline alone, even though its source list is blank", async () => {
|
||||
loadPolicies.mockReturnValue({
|
||||
"builder-2": exportPolicy({
|
||||
sources: [],
|
||||
runsOnEditor: false,
|
||||
backendId: "backend-swept",
|
||||
}),
|
||||
} as unknown as PoliciesByCategory);
|
||||
|
||||
await enforceExportPolicies([pdf()], ["file-1"]);
|
||||
|
||||
expect(runStoredPolicy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("still enforces a catalogue tile that nobody has narrowed", async () => {
|
||||
loadPolicies.mockReturnValue({
|
||||
security: exportPolicy({
|
||||
sources: [],
|
||||
// A tile is blank because it was never narrowed, so it does run on the editor.
|
||||
runsOnEditor: true,
|
||||
backendId: "backend-security",
|
||||
}),
|
||||
} as unknown as PoliciesByCategory);
|
||||
|
||||
await enforceExportPolicies([pdf()], ["file-1"]);
|
||||
|
||||
expect(runStoredPolicy).toHaveBeenCalledWith("backend-security");
|
||||
});
|
||||
|
||||
it("enforces in the team's run order, not object order", async () => {
|
||||
loadPolicies.mockReturnValue({
|
||||
second: exportPolicy({
|
||||
runsOnEditor: true,
|
||||
backendId: "backend-second",
|
||||
order: 1,
|
||||
}),
|
||||
first: exportPolicy({
|
||||
runsOnEditor: true,
|
||||
backendId: "backend-first",
|
||||
order: 0,
|
||||
}),
|
||||
} as unknown as PoliciesByCategory);
|
||||
|
||||
await enforceExportPolicies([pdf()], ["file-1"]);
|
||||
|
||||
expect(runStoredPolicy.mock.calls.map(([id]) => id)).toEqual([
|
||||
"backend-first",
|
||||
"backend-second",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -62,22 +62,28 @@ function activeExportPolicies(): ExportPolicy[] {
|
||||
const labels = new Map(
|
||||
loadPolicyCatalog().categories.map((c) => [c.id, c.label]),
|
||||
);
|
||||
return Object.entries(loadPolicies())
|
||||
return (
|
||||
Object.entries(loadPolicies())
|
||||
.filter(
|
||||
([, s]) =>
|
||||
s.configured &&
|
||||
s.status === "active" &&
|
||||
s.backendId &&
|
||||
(s.sources.length === 0 || s.sources.includes("editor")) &&
|
||||
s.runsOnEditor &&
|
||||
s.runOn === "export",
|
||||
)
|
||||
// Same team-wide run order the upload path uses: enforcement is not commutative (a watermark
|
||||
// then a flatten is not a flatten then a watermark), so both paths must agree on the sequence.
|
||||
.sort(([, a], [, b]) => (a.order ?? 0) - (b.order ?? 0))
|
||||
.map(([id, s]) => ({
|
||||
categoryId: id,
|
||||
backendId: s.backendId as string,
|
||||
label: labels.get(id) ?? "Policy",
|
||||
// A builder pipeline has no built-in category, so it labels by its own name.
|
||||
label: labels.get(id) ?? s.name ?? "Policy",
|
||||
outputMode: s.outputMode === "new_file" ? "new_file" : "new_version",
|
||||
accent: `var(--color-${ROW_ACCENT[id] ?? "blue"})`,
|
||||
}));
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/** Run one policy on a file and resolve the enforced bytes + run info (throws on
|
||||
|
||||
@@ -111,7 +111,8 @@ const samplePolicy = {
|
||||
updatedAt: "",
|
||||
},
|
||||
pipelineSteps: [{ operation: "/api/v1/misc/compress-pdf", parameters: {} }],
|
||||
sources: ["editor"],
|
||||
sources: [],
|
||||
runsOnEditor: true,
|
||||
scopeTypes: ["Contracts"],
|
||||
reviewerEmail: "me@x.com",
|
||||
fieldValues: { minConfidence: "80%" },
|
||||
@@ -147,7 +148,8 @@ describe("buildBackendPolicy", () => {
|
||||
expect(decoded.id).toBe("p1");
|
||||
expect(decoded.categoryId).toBe("security");
|
||||
expect(decoded.enabled).toBe(true);
|
||||
expect(decoded.sources).toEqual(["editor"]);
|
||||
expect(decoded.sources).toEqual([]);
|
||||
expect(decoded.runsOnEditor).toBe(true);
|
||||
expect(decoded.scopeTypes).toEqual(["Contracts"]);
|
||||
expect(decoded.reviewerEmail).toBe("me@x.com");
|
||||
expect(decoded.fieldValues).toEqual({ minConfidence: "80%" });
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
* using that registry.
|
||||
*/
|
||||
|
||||
import { resolveRunOn } from "@app/policies/runOn";
|
||||
import { resolveRunOn, type PolicyRunOn } from "@app/policies/runOn";
|
||||
import type { AutomationConfig } from "@app/types/automation";
|
||||
import type { ToolRegistry } from "@app/data/toolsTaxonomy";
|
||||
import type { PolicyFolderSettings } from "@app/types/policies";
|
||||
@@ -56,6 +56,14 @@ export interface BackendPolicy {
|
||||
trigger: BackendTriggerConfig | null;
|
||||
steps: BackendPipelineStep[];
|
||||
output: BackendOutputSpec;
|
||||
/** Whether the editor runs this policy per file, and on which moment. */
|
||||
editor?: BackendEditorConfig;
|
||||
}
|
||||
|
||||
/** Mirrors the backend `EditorConfig`. */
|
||||
export interface BackendEditorConfig {
|
||||
allowed: boolean;
|
||||
runOn: PolicyRunOn;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -217,6 +225,7 @@ export interface PolicyToStore {
|
||||
*/
|
||||
pipelineSteps: BackendPipelineStep[];
|
||||
sources: string[];
|
||||
runsOnEditor: boolean;
|
||||
scopeTypes: string[];
|
||||
reviewerEmail: string;
|
||||
fieldValues: Record<string, boolean | string | string[]>;
|
||||
@@ -233,6 +242,8 @@ export interface DecodedPolicy {
|
||||
/** Null if the stored policy carried no automation blob. */
|
||||
automation: AutomationConfig | null;
|
||||
sources: string[];
|
||||
/** Whether the editor runs this policy per file, straight from the policy's own flag. */
|
||||
runsOnEditor: boolean;
|
||||
scopeTypes: string[];
|
||||
reviewerEmail: string;
|
||||
fieldValues: Record<string, boolean | string | string[]>;
|
||||
@@ -280,7 +291,6 @@ export function buildBackendPolicy(input: PolicyToStore): BackendPolicy {
|
||||
maxRetries: input.folder.maxRetries,
|
||||
retryDelayMinutes: input.folder.retryDelayMinutes,
|
||||
automation: input.automation,
|
||||
runOn: input.folder.runOn,
|
||||
// Policy-level metadata (no trigger bag to hold it any more).
|
||||
categoryId: input.categoryId,
|
||||
sources: input.sources,
|
||||
@@ -289,6 +299,10 @@ export function buildBackendPolicy(input: PolicyToStore): BackendPolicy {
|
||||
fieldValues: input.fieldValues,
|
||||
},
|
||||
},
|
||||
editor: {
|
||||
allowed: input.runsOnEditor,
|
||||
runOn: input.folder.runOn,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -298,6 +312,7 @@ export function fromBackendPolicy(policy: BackendPolicy): DecodedPolicy {
|
||||
// Metadata lives in output.options; legacy records kept it in trigger.options,
|
||||
// so merge both (output wins) to decode either shape.
|
||||
const meta = { ...(policy.trigger?.options ?? {}), ...output };
|
||||
const editor = policy.editor;
|
||||
const str = (v: unknown, fallback = "") =>
|
||||
typeof v === "string" ? v : fallback;
|
||||
const num = (v: unknown, fallback: number) =>
|
||||
@@ -316,8 +331,9 @@ export function fromBackendPolicy(policy: BackendPolicy): DecodedPolicy {
|
||||
reviewerEmail: str(meta.reviewerEmail),
|
||||
fieldValues:
|
||||
(meta.fieldValues as DecodedPolicy["fieldValues"] | undefined) ?? {},
|
||||
runsOnEditor: editor?.allowed === true,
|
||||
folder: {
|
||||
runOn: resolveRunOn(meta.runOn, categoryId),
|
||||
runOn: resolveRunOn(editor?.runOn, categoryId),
|
||||
// Legacy/missing output.mode defaults to new_version, not new_file.
|
||||
outputMode: output.mode === "new_file" ? "new_file" : "new_version",
|
||||
outputName: str(output.name),
|
||||
|
||||
@@ -60,6 +60,43 @@ describe("policyStorage", () => {
|
||||
expect(p.routing.configured).toBe(false);
|
||||
});
|
||||
|
||||
it("migrates a pre-runsOnEditor row narrowed to non-editor sources off the editor", () => {
|
||||
// Stored before runsOnEditor existed: no such field, sources exclude the editor.
|
||||
localStorage.setItem(
|
||||
"stirling-policies-state",
|
||||
JSON.stringify({
|
||||
security: { configured: true, status: "active", sources: ["s3"] },
|
||||
}),
|
||||
);
|
||||
// Without the migration the default (true) would wrongly win.
|
||||
expect(loadPolicies().security.runsOnEditor).toBe(false);
|
||||
});
|
||||
|
||||
it("migrates a pre-runsOnEditor row listing the editor onto the editor", () => {
|
||||
localStorage.setItem(
|
||||
"stirling-policies-state",
|
||||
JSON.stringify({
|
||||
security: { configured: true, status: "active", sources: ["editor"] },
|
||||
}),
|
||||
);
|
||||
expect(loadPolicies().security.runsOnEditor).toBe(true);
|
||||
});
|
||||
|
||||
it("leaves an explicit runsOnEditor untouched", () => {
|
||||
localStorage.setItem(
|
||||
"stirling-policies-state",
|
||||
JSON.stringify({
|
||||
security: {
|
||||
configured: true,
|
||||
status: "active",
|
||||
sources: ["editor"],
|
||||
runsOnEditor: false,
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(loadPolicies().security.runsOnEditor).toBe(false);
|
||||
});
|
||||
|
||||
it("fires a change event on update", () => {
|
||||
const cb = vi.fn();
|
||||
const off = onPoliciesChange(cb);
|
||||
|
||||
@@ -19,6 +19,7 @@ function defaultState(categoryId: string): PolicyState {
|
||||
configured: false,
|
||||
status: "default",
|
||||
sources: ["editor"],
|
||||
runsOnEditor: true,
|
||||
scopeTypes: [],
|
||||
// Empty by default; the wizard defaults the reviewer to the signed-in user.
|
||||
reviewerEmail: "",
|
||||
@@ -54,7 +55,14 @@ export function loadPolicies(): PoliciesByCategory {
|
||||
// category gets a default rather than being undefined.
|
||||
const out: PoliciesByCategory = {};
|
||||
loadPolicyCatalog().categories.forEach((cat, index) => {
|
||||
const merged = { ...defaultState(cat.id), ...(parsed[cat.id] ?? {}) };
|
||||
const stored = parsed[cat.id];
|
||||
const merged = { ...defaultState(cat.id), ...(stored ?? {}) };
|
||||
// Migration: a row stored before runsOnEditor existed has no such field, so the default (true)
|
||||
// would put a tile narrowed to non-editor sources on the editor until the first reconcile lands.
|
||||
// Derive it from the legacy signal (the editor in its sources), mirroring the decode rule.
|
||||
if (stored && stored.runsOnEditor === undefined) {
|
||||
merged.runsOnEditor = (stored.sources ?? []).includes("editor");
|
||||
}
|
||||
// Migration: clear the obsolete persisted reviewer email so it re-defaults
|
||||
// to the real signed-in user.
|
||||
if (merged.reviewerEmail === STALE_REVIEWER_EMAIL)
|
||||
@@ -64,6 +72,11 @@ export function loadPolicies(): PoliciesByCategory {
|
||||
if (merged.order == null) merged.order = index;
|
||||
out[cat.id] = merged;
|
||||
});
|
||||
// Builder pipelines key by their own id, so the walk above misses them. Carried through as
|
||||
// stored: a tile's defaults would mark them built-in and put them on the editor uninvited.
|
||||
for (const [key, state] of Object.entries(parsed)) {
|
||||
if (!out[key] && state) out[key] = state as PolicyState;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -118,6 +131,26 @@ export function reorderPolicies(
|
||||
return next;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop cached entries entirely (no default seeded back). For builder pipelines the backend has
|
||||
* deleted: keyed by their own id, they have no built-in category to fall back to, so a left-behind
|
||||
* entry keeps a dead backendId that the auto-run still tries to dispatch. Built-in categories are
|
||||
* never forgotten - they reseed on the next read anyway.
|
||||
*/
|
||||
export function forgetPolicies(ids: string[]): PoliciesByCategory {
|
||||
const current = loadPolicies();
|
||||
const catalogIds = new Set(loadPolicyCatalog().categories.map((c) => c.id));
|
||||
const next: PoliciesByCategory = { ...current };
|
||||
let removed = false;
|
||||
for (const id of ids) {
|
||||
if (catalogIds.has(id) || !(id in next)) continue;
|
||||
delete next[id];
|
||||
removed = true;
|
||||
}
|
||||
if (removed) persist(next);
|
||||
return next;
|
||||
}
|
||||
|
||||
/** Reset a category to its unconfigured default (the "Delete policy" action). */
|
||||
export function resetPolicy(categoryId: string): PoliciesByCategory {
|
||||
return updatePolicy(categoryId, {
|
||||
|
||||
@@ -120,6 +120,10 @@ export interface PolicyState {
|
||||
status: PolicyStatus;
|
||||
/** Selected sources (ids from POLICY_SOURCES). */
|
||||
sources: string[];
|
||||
/** The policy's own name. Set for builder pipelines, which have no built-in category label. */
|
||||
name?: string;
|
||||
/** Whether the policy runs in the editor as each file passes through (resolved at decode). */
|
||||
runsOnEditor?: boolean;
|
||||
/** When non-empty, narrows the policy to these document types. */
|
||||
scopeTypes: string[];
|
||||
/** Email that low-confidence enforcements are routed to. */
|
||||
@@ -188,6 +192,7 @@ export interface PolicyWizardResult {
|
||||
automation: AutomationConfig;
|
||||
fieldValues: Record<string, boolean | string | string[]>;
|
||||
sources: string[];
|
||||
runsOnEditor: boolean;
|
||||
scopeTypes: string[];
|
||||
reviewerEmail: string;
|
||||
/** Output + retry settings for the backing folder. */
|
||||
@@ -224,6 +229,7 @@ export interface PolicyConfigResult {
|
||||
unresolvedOps: string[];
|
||||
fieldValues: Record<string, boolean | string | string[]>;
|
||||
sources: string[];
|
||||
runsOnEditor: boolean;
|
||||
scopeTypes: string[];
|
||||
reviewerEmail: string;
|
||||
folder: PolicyFolderSettings;
|
||||
|
||||
Reference in New Issue
Block a user