mirror of
https://github.com/Stirling-Tools/Stirling-PDF.git
synced 2026-09-02 21:03:34 +03:00
Add heuristic classification (#7050)
# Description of Changes - Adds a non-AI heuristic classification engine that classifies documents client-side in the browser when AI is disabled - Classification is billed as a policy run via a fast, non-blocking meter endpoint; a default Classification policy is seeded per team - Enables the policy engine by default --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details.
This commit is contained in:
Anthony Stirling
@@ -139,7 +139,6 @@ jobs:
|
||||
tags: |
|
||||
type=raw,value=${{ steps.versionNumber.outputs.versionNumber }},enable=${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master' }}
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master' }}
|
||||
type=raw,value=alpha,enable=${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/testMain' }}
|
||||
|
||||
- name: Build and push Unified Dockerfile (latest variant)
|
||||
id: build-push-latest
|
||||
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
package stirling.software.proprietary.classification;
|
||||
|
||||
/** Meters a client-side classification run; SaaS charges PAYG, other flavors have no bean. */
|
||||
public interface ClassificationRunBiller {
|
||||
|
||||
/** Charge one classification policy run covering {@code documentCount} documents. */
|
||||
void recordClassificationRun(int documentCount);
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import stirling.software.proprietary.security.model.User;
|
||||
|
||||
@Entity
|
||||
@Table(name = "teams")
|
||||
@EntityListeners(TeamEntityListener.class)
|
||||
@NoArgsConstructor
|
||||
@Getter
|
||||
@Setter
|
||||
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package stirling.software.proprietary.model;
|
||||
|
||||
/** Published once a new {@link Team} row is inserted, so listeners can seed per-team defaults. */
|
||||
public record TeamCreatedEvent(Long teamId, String teamName) {}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package stirling.software.proprietary.model;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import jakarta.persistence.PostPersist;
|
||||
|
||||
/** Publishes {@link TeamCreatedEvent} on insert; Spring bridges the publisher via a static. */
|
||||
@Component
|
||||
public class TeamEntityListener {
|
||||
|
||||
private static ApplicationEventPublisher publisher;
|
||||
|
||||
@Autowired
|
||||
void setPublisher(ApplicationEventPublisher applicationEventPublisher) {
|
||||
TeamEntityListener.publisher = applicationEventPublisher;
|
||||
}
|
||||
|
||||
@PostPersist
|
||||
public void onCreate(Team team) {
|
||||
if (publisher != null) {
|
||||
publisher.publishEvent(new TeamCreatedEvent(team.getId(), team.getName()));
|
||||
}
|
||||
}
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
package stirling.software.proprietary.policy.controller;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.audit.AuditContext;
|
||||
import stirling.software.proprietary.classification.ClassificationRunBiller;
|
||||
|
||||
/**
|
||||
* Meters + audits a client-side (non-AI) classification run so both classify paths bill
|
||||
* identically. Side-effect only; does no classification itself.
|
||||
*/
|
||||
@Slf4j
|
||||
@Hidden
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/policies")
|
||||
public class ClassificationMeterController {
|
||||
|
||||
/** Audit step label mirrors the AI classify tool so both paths read alike in the trail. */
|
||||
private static final String CLASSIFY_STEP = "/api/v1/ai/tools/classify-and-label";
|
||||
|
||||
/** Client-supplied count cap: the frontend meters one document per call. */
|
||||
private static final int MAX_DOCUMENTS = 10_000;
|
||||
|
||||
private final ObjectProvider<ClassificationRunBiller> biller;
|
||||
|
||||
public ClassificationMeterController(ObjectProvider<ClassificationRunBiller> biller) {
|
||||
this.biller = biller;
|
||||
}
|
||||
|
||||
@PostMapping("/classify/meter")
|
||||
@Operation(
|
||||
summary = "Meter a client-side classification run",
|
||||
description =
|
||||
"Records billing + audit for a non-AI classification performed in the browser."
|
||||
+ " Does no classification itself. Dispatched by the frontend, not for"
|
||||
+ " direct use.")
|
||||
public ResponseEntity<Void> meterClassification(
|
||||
@RequestBody(required = false) ClassifyMeterRequest body, HttpServletRequest request) {
|
||||
int documents = body != null && body.documentCount() != null ? body.documentCount() : 1;
|
||||
if (documents < 1) documents = 1;
|
||||
if (documents > MAX_DOCUMENTS) documents = MAX_DOCUMENTS;
|
||||
String policyName =
|
||||
body != null && body.policyName() != null && !body.policyName().isBlank()
|
||||
? body.policyName()
|
||||
: "Classification";
|
||||
|
||||
// Stamp the run so ControllerAuditAspect records it as a policy run, like the AI path.
|
||||
request.setAttribute(AuditContext.REQ_ATTR_POLICY_NAME, policyName);
|
||||
request.setAttribute(AuditContext.REQ_ATTR_POLICY_STEPS, List.of(CLASSIFY_STEP));
|
||||
|
||||
ClassificationRunBiller runBiller = biller.getIfAvailable();
|
||||
if (runBiller != null) {
|
||||
try {
|
||||
runBiller.recordClassificationRun(documents);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn(
|
||||
"[classify meter] billing failed; classification proceeds unbilled: {}",
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
return ResponseEntity.accepted().build();
|
||||
}
|
||||
|
||||
/** Frontend payload: documents classified, plus the policy name for the audit label. */
|
||||
public record ClassifyMeterRequest(
|
||||
String policyName, Integer documentCount, List<String> labels) {}
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
package stirling.software.proprietary.policy.seed;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.event.TransactionPhase;
|
||||
import org.springframework.transaction.event.TransactionalEventListener;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.model.TeamCreatedEvent;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
import stirling.software.proprietary.security.repository.TeamRepository;
|
||||
import stirling.software.proprietary.security.service.TeamService;
|
||||
|
||||
/**
|
||||
* Seeds an enabled Classification policy per team so classification is on by default. Idempotent;
|
||||
* skips the internal team.
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class DefaultClassificationPolicySeeder {
|
||||
|
||||
static final String CATEGORY = "classification";
|
||||
private static final String CLASSIFY_ENDPOINT = "/api/v1/ai/tools/classify-and-label";
|
||||
private static final String POLICY_NAME = "Classification Policy";
|
||||
|
||||
private final PolicyStore policyStore;
|
||||
private final TeamRepository teamRepository;
|
||||
|
||||
// The default team is created during startup, before the entity event listener is guaranteed
|
||||
// wired, so ensure it once the context is fully ready (self-hosted first boot).
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void seedDefaultTeamOnStartup() {
|
||||
teamRepository
|
||||
.findByName(TeamService.DEFAULT_TEAM_NAME)
|
||||
.ifPresent(team -> seedIfMissing(team.getId(), team.getName()));
|
||||
}
|
||||
|
||||
// Any team created at runtime (admin-created, SaaS sign-ups); after the team's commit so a
|
||||
// rolled-back team never leaves a policy behind.
|
||||
@TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT)
|
||||
public void onTeamCreated(TeamCreatedEvent event) {
|
||||
seedIfMissing(event.teamId(), event.teamName());
|
||||
}
|
||||
|
||||
private void seedIfMissing(Long teamId, String teamName) {
|
||||
if (teamId == null || TeamService.INTERNAL_TEAM_NAME.equals(teamName)) {
|
||||
return;
|
||||
}
|
||||
boolean alreadySeeded =
|
||||
policyStore.findByTeam(teamId).stream()
|
||||
.anyMatch(DefaultClassificationPolicySeeder::isClassification);
|
||||
if (alreadySeeded) {
|
||||
return;
|
||||
}
|
||||
policyStore.save(defaultPolicy(teamId));
|
||||
log.info("Seeded default Classification policy for team {}", teamId);
|
||||
}
|
||||
|
||||
private static boolean isClassification(Policy policy) {
|
||||
return policy.output() != null
|
||||
&& CATEGORY.equals(policy.output().options().get("categoryId"));
|
||||
}
|
||||
|
||||
/** The default Classification policy: classify each upload, versioning the file in place. */
|
||||
static Policy defaultPolicy(Long teamId) {
|
||||
Map<String, Object> options = new HashMap<>();
|
||||
options.put("categoryId", CATEGORY);
|
||||
options.put("runOn", "upload");
|
||||
options.put("mode", "new_version");
|
||||
options.put("sources", List.of("editor"));
|
||||
options.put("scopeTypes", List.of());
|
||||
options.put("reviewerEmail", "");
|
||||
return new Policy(
|
||||
null,
|
||||
POLICY_NAME,
|
||||
"system",
|
||||
true,
|
||||
null,
|
||||
List.of(),
|
||||
List.of(new PipelineStep(CLASSIFY_ENDPOINT, Map.of())),
|
||||
new OutputSpec("inline", options),
|
||||
teamId);
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package stirling.software.proprietary.model;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class TeamEntityListenerTest {
|
||||
|
||||
@Mock private ApplicationEventPublisher publisher;
|
||||
|
||||
private static Team team(Long id, String name) {
|
||||
Team team = new Team();
|
||||
team.setId(id);
|
||||
team.setName(name);
|
||||
return team;
|
||||
}
|
||||
|
||||
@Test
|
||||
void publishesTeamCreatedEventOnPersist() {
|
||||
TeamEntityListener listener = new TeamEntityListener();
|
||||
listener.setPublisher(publisher);
|
||||
|
||||
listener.onCreate(team(5L, "Acme"));
|
||||
|
||||
ArgumentCaptor<TeamCreatedEvent> event = ArgumentCaptor.forClass(TeamCreatedEvent.class);
|
||||
org.mockito.Mockito.verify(publisher).publishEvent(event.capture());
|
||||
assertThat(event.getValue().teamId()).isEqualTo(5L);
|
||||
assertThat(event.getValue().teamName()).isEqualTo("Acme");
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotThrowWhenNoPublisherIsSet() {
|
||||
// JPA can build the listener before Spring wires the publisher; must be a safe no-op.
|
||||
TeamEntityListener listener = new TeamEntityListener();
|
||||
listener.setPublisher(null);
|
||||
|
||||
assertThatCode(() -> listener.onCreate(team(1L, "X"))).doesNotThrowAnyException();
|
||||
}
|
||||
}
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
package stirling.software.proprietary.policy.seed;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import stirling.software.proprietary.model.Team;
|
||||
import stirling.software.proprietary.model.TeamCreatedEvent;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
import stirling.software.proprietary.security.repository.TeamRepository;
|
||||
import stirling.software.proprietary.security.service.TeamService;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class DefaultClassificationPolicySeederTest {
|
||||
|
||||
@Mock private PolicyStore policyStore;
|
||||
@Mock private TeamRepository teamRepository;
|
||||
|
||||
private DefaultClassificationPolicySeeder seeder() {
|
||||
return new DefaultClassificationPolicySeeder(policyStore, teamRepository);
|
||||
}
|
||||
|
||||
private static Policy classificationPolicy(Long teamId) {
|
||||
return new Policy(
|
||||
"p1",
|
||||
"Classification Policy",
|
||||
"system",
|
||||
true,
|
||||
null,
|
||||
List.of(),
|
||||
List.of(),
|
||||
new OutputSpec("inline", Map.of("categoryId", "classification")),
|
||||
teamId);
|
||||
}
|
||||
|
||||
@Test
|
||||
void seedsAnEnabledClassificationPolicyWhenTheTeamHasNone() {
|
||||
when(policyStore.findByTeam(7L)).thenReturn(List.of());
|
||||
|
||||
seeder().onTeamCreated(new TeamCreatedEvent(7L, "Acme"));
|
||||
|
||||
ArgumentCaptor<Policy> saved = ArgumentCaptor.forClass(Policy.class);
|
||||
verify(policyStore).save(saved.capture());
|
||||
Policy policy = saved.getValue();
|
||||
assertThat(policy.enabled()).isTrue();
|
||||
assertThat(policy.teamId()).isEqualTo(7L);
|
||||
assertThat(policy.output().type()).isEqualTo("inline");
|
||||
assertThat(policy.output().options().get("categoryId")).isEqualTo("classification");
|
||||
assertThat(policy.output().options().get("runOn")).isEqualTo("upload");
|
||||
assertThat(policy.output().options().get("mode")).isEqualTo("new_version");
|
||||
assertThat(policy.output().options().get("sources")).isEqualTo(List.of("editor"));
|
||||
assertThat(policy.steps()).hasSize(1);
|
||||
assertThat(policy.steps().get(0).operation())
|
||||
.isEqualTo("/api/v1/ai/tools/classify-and-label");
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotSeedWhenAClassificationPolicyAlreadyExists() {
|
||||
when(policyStore.findByTeam(7L)).thenReturn(List.of(classificationPolicy(7L)));
|
||||
|
||||
seeder().onTeamCreated(new TeamCreatedEvent(7L, "Acme"));
|
||||
|
||||
verify(policyStore, never()).save(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotSeedForTheInternalTeam() {
|
||||
seeder().onTeamCreated(new TeamCreatedEvent(2L, "Internal"));
|
||||
|
||||
verify(policyStore, never()).findByTeam(anyLong());
|
||||
verify(policyStore, never()).save(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotSeedWhenTeamIdIsNull() {
|
||||
seeder().onTeamCreated(new TeamCreatedEvent(null, "Acme"));
|
||||
|
||||
verify(policyStore, never()).save(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void seedsTheDefaultTeamOnStartupWhenItExistsAndHasNoPolicy() {
|
||||
Team defaultTeam = new Team();
|
||||
defaultTeam.setId(1L);
|
||||
defaultTeam.setName(TeamService.DEFAULT_TEAM_NAME);
|
||||
when(teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME))
|
||||
.thenReturn(Optional.of(defaultTeam));
|
||||
when(policyStore.findByTeam(1L)).thenReturn(List.of());
|
||||
|
||||
seeder().seedDefaultTeamOnStartup();
|
||||
|
||||
verify(policyStore).save(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotSeedOnStartupWhenThereIsNoDefaultTeam() {
|
||||
when(teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME)).thenReturn(Optional.empty());
|
||||
|
||||
seeder().seedDefaultTeamOnStartup();
|
||||
|
||||
verify(policyStore, never()).save(any());
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package stirling.software.saas.payg.charge;
|
||||
|
||||
import org.springframework.context.annotation.Profile;
|
||||
import org.springframework.security.core.Authentication;
|
||||
import org.springframework.security.core.context.SecurityContextHolder;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.proprietary.classification.ClassificationRunBiller;
|
||||
import stirling.software.proprietary.security.database.repository.UserRepository;
|
||||
import stirling.software.proprietary.security.model.ApiKeyAuthenticationToken;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.saas.payg.model.BillingCategory;
|
||||
import stirling.software.saas.payg.model.JobSource;
|
||||
import stirling.software.saas.payg.model.ProcessType;
|
||||
import stirling.software.saas.util.AuthenticationUtils;
|
||||
|
||||
/**
|
||||
* Charges one PAYG unit per document as an AUTOMATION job, matching what a server-side
|
||||
* classify policy step bills.
|
||||
*/
|
||||
@Component
|
||||
@Profile("saas")
|
||||
@RequiredArgsConstructor
|
||||
public class SaasClassificationRunBiller implements ClassificationRunBiller {
|
||||
|
||||
private final UserRepository userRepository;
|
||||
private final JobChargeService jobChargeService;
|
||||
|
||||
@Override
|
||||
public void recordClassificationRun(int documentCount) {
|
||||
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
|
||||
User user = AuthenticationUtils.getCurrentUser(auth, userRepository);
|
||||
if (user == null || user.getTeam() == null) {
|
||||
return;
|
||||
}
|
||||
JobSource source =
|
||||
auth instanceof ApiKeyAuthenticationToken ? JobSource.API : JobSource.WEB;
|
||||
ChargeContext ctx =
|
||||
new ChargeContext(
|
||||
user.getId(),
|
||||
user.getTeam().getId(),
|
||||
source,
|
||||
ProcessType.AUTOMATION,
|
||||
BillingCategory.AUTOMATION);
|
||||
jobChargeService.chargeStandalone(ctx, Math.max(1, documentCount));
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,5 @@
|
||||
// Whether document classification (and everything it drives in the UI: the
|
||||
// Files-sidebar category grouping, per-file label chips, and the file-details
|
||||
// Classification section) is active in this build. Classification is a
|
||||
// SaaS-only feature gated on the AI engine, so core — and every build that
|
||||
// doesn't override this seam (proprietary, desktop, cloud) — returns false, and
|
||||
// none of that UI ever renders. The saas layer overrides it to track the AI
|
||||
// engine's enabled flag, so the feature shows up only on SaaS when AI is on.
|
||||
// Whether classification (sidebar grouping, label chips, file-details section)
|
||||
// is active in this build. Core has no classifier; proprietary overrides to true.
|
||||
|
||||
export function useClassificationEnabled(): boolean {
|
||||
return false;
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import path from "path";
|
||||
import { test, expect } from "@app/tests/helpers/stub-test-base";
|
||||
import { uploadFiles } from "@app/tests/helpers/ui-helpers";
|
||||
|
||||
// The sidebar groups files by the category in their `StirlingPDFClassification`
|
||||
// metadata; these specs exercise that seam with pre-labelled fixtures.
|
||||
|
||||
const FIXTURES = path.join(
|
||||
import.meta.dirname,
|
||||
"../test-fixtures/classification",
|
||||
);
|
||||
|
||||
const categoryHeaders = (page: import("@playwright/test").Page) =>
|
||||
page.locator(".file-sidebar-group .file-sidebar-group-header");
|
||||
|
||||
test("classified files group by category family in the sidebar", async ({
|
||||
page,
|
||||
}) => {
|
||||
await uploadFiles(page, [
|
||||
path.join(FIXTURES, "classified_invoice.pdf"), // -> Financial
|
||||
path.join(FIXTURES, "classified_nda.pdf"), // -> Legal
|
||||
path.join(FIXTURES, "classified_resume.pdf"), // -> HR
|
||||
]);
|
||||
|
||||
// The backfill reads each file's classification metadata on idle and regroups;
|
||||
// the category headers appear once it resolves (Playwright auto-retries).
|
||||
const headers = categoryHeaders(page);
|
||||
await expect(headers.filter({ hasText: "Financial" })).toBeVisible({
|
||||
timeout: 15_000,
|
||||
});
|
||||
await expect(headers.filter({ hasText: "Legal" })).toBeVisible();
|
||||
await expect(headers.filter({ hasText: "HR" })).toBeVisible();
|
||||
});
|
||||
|
||||
test("an unclassified file is not placed in a category group", async ({
|
||||
page,
|
||||
}) => {
|
||||
// sample.pdf carries no StirlingPDFClassification metadata, so it must not
|
||||
// create or join any category family group - it falls into the catch-all.
|
||||
await uploadFiles(page, path.join(FIXTURES, "../sample.pdf"));
|
||||
|
||||
// Give the idle backfill a chance to run and (find nothing to) regroup.
|
||||
await expect(page.locator(".file-sidebar-file-item")).toHaveCount(1);
|
||||
await expect(
|
||||
categoryHeaders(page).filter({ hasText: "Financial" }),
|
||||
).toHaveCount(0);
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
import path from "path";
|
||||
import { test, expect } from "@app/tests/helpers/stub-test-base";
|
||||
import { uploadFiles } from "@app/tests/helpers/ui-helpers";
|
||||
|
||||
// A bulk upload must classify every file in the browser and group it - no file
|
||||
// may be stranded in "Other" by races between the upload wave and delivery.
|
||||
|
||||
test.use({ autoGoto: false });
|
||||
|
||||
const FIXTURES = path.join(
|
||||
import.meta.dirname,
|
||||
"../test-fixtures/classification/unlabelled",
|
||||
);
|
||||
|
||||
/** The stored policy DefaultClassificationPolicySeeder writes for a new team. */
|
||||
const SEEDED_POLICY = {
|
||||
id: "seeded-classification",
|
||||
name: "Classification Policy",
|
||||
owner: "system",
|
||||
enabled: true,
|
||||
trigger: null,
|
||||
sourceIds: [],
|
||||
steps: [{ operation: "/api/v1/ai/tools/classify-and-label", parameters: {} }],
|
||||
output: {
|
||||
type: "inline",
|
||||
options: {
|
||||
categoryId: "classification",
|
||||
runOn: "upload",
|
||||
mode: "new_version",
|
||||
sources: ["editor"],
|
||||
scopeTypes: [],
|
||||
reviewerEmail: "",
|
||||
},
|
||||
},
|
||||
teamId: 1,
|
||||
};
|
||||
|
||||
test("a 10-file upload wave classifies every file into its group", async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(180_000);
|
||||
|
||||
await page.route("**/api/v1/policies", (route) =>
|
||||
route.fulfill({ json: [SEEDED_POLICY] }),
|
||||
);
|
||||
await page.route("**/api/v1/policies/classify/meter", (route) =>
|
||||
route.fulfill({ status: 202, body: "" }),
|
||||
);
|
||||
await page.goto("/", { waitUntil: "domcontentloaded", timeout: 120_000 });
|
||||
|
||||
await uploadFiles(
|
||||
page,
|
||||
[
|
||||
"invoice_acme.pdf",
|
||||
"bank_statement.pdf",
|
||||
"purchase_order.pdf",
|
||||
"nda_mutual.pdf",
|
||||
"service_agreement.pdf",
|
||||
"resume_jane_doe.pdf",
|
||||
"cover_letter.pdf",
|
||||
"offer_letter.pdf",
|
||||
"generic_notes.pdf",
|
||||
"spanish_contrato.pdf",
|
||||
].map((f) => path.join(FIXTURES, f)),
|
||||
);
|
||||
|
||||
// Each group header is a collapsible button whose name carries the member count.
|
||||
// Classification runs a few files per idle pass; wait for the full drain.
|
||||
const header = (name: string, count: number) =>
|
||||
page.getByRole("button", { name: `${name} ${count}`, exact: true });
|
||||
await expect(header("Financial", 3)).toBeVisible({ timeout: 90_000 });
|
||||
await expect(header("HR", 3)).toBeVisible({ timeout: 30_000 });
|
||||
await expect(header("Legal", 2)).toBeVisible({ timeout: 30_000 });
|
||||
|
||||
// The regression: nothing classifiable may be stranded in Other - only the
|
||||
// genuinely unlabellable pair (generic prose + non-English) belongs there.
|
||||
await expect(header("Other", 2)).toBeVisible({ timeout: 30_000 });
|
||||
// The filename can render in several places (Recent, group, viewer); any hit proves presence.
|
||||
await expect(page.getByText("generic_notes.pdf").first()).toBeVisible();
|
||||
await expect(page.getByText("spanish_contrato.pdf").first()).toBeVisible();
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
%PDF-1.4
|
||||
1 0 obj
|
||||
<< /Type /Catalog /Pages 2 0 R >>
|
||||
endobj
|
||||
2 0 obj
|
||||
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
|
||||
endobj
|
||||
3 0 obj
|
||||
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>
|
||||
endobj
|
||||
4 0 obj
|
||||
<< /Length 38 >>
|
||||
stream
|
||||
BT /F1 24 Tf 72 720 Td (INVOICE) Tj ET
|
||||
endstream
|
||||
endobj
|
||||
5 0 obj
|
||||
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
|
||||
endobj
|
||||
6 0 obj
|
||||
<< /Title (Invoice) /StirlingPDFClassification ({"labels":["invoice"]}) >>
|
||||
endobj
|
||||
xref
|
||||
0 7
|
||||
0000000000 65535 f
|
||||
0000000009 00000 n
|
||||
0000000058 00000 n
|
||||
0000000115 00000 n
|
||||
0000000241 00000 n
|
||||
0000000329 00000 n
|
||||
0000000426 00000 n
|
||||
trailer
|
||||
<< /Size 7 /Root 1 0 R /Info 6 0 R >>
|
||||
startxref
|
||||
516
|
||||
%%EOF
|
||||
@@ -0,0 +1,36 @@
|
||||
%PDF-1.4
|
||||
1 0 obj
|
||||
<< /Type /Catalog /Pages 2 0 R >>
|
||||
endobj
|
||||
2 0 obj
|
||||
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
|
||||
endobj
|
||||
3 0 obj
|
||||
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>
|
||||
endobj
|
||||
4 0 obj
|
||||
<< /Length 55 >>
|
||||
stream
|
||||
BT /F1 24 Tf 72 720 Td (NON-DISCLOSURE AGREEMENT) Tj ET
|
||||
endstream
|
||||
endobj
|
||||
5 0 obj
|
||||
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
|
||||
endobj
|
||||
6 0 obj
|
||||
<< /Title (Non-Disclosure Agreement) /StirlingPDFClassification ({"labels":["nda"]}) >>
|
||||
endobj
|
||||
xref
|
||||
0 7
|
||||
0000000000 65535 f
|
||||
0000000009 00000 n
|
||||
0000000058 00000 n
|
||||
0000000115 00000 n
|
||||
0000000241 00000 n
|
||||
0000000346 00000 n
|
||||
0000000443 00000 n
|
||||
trailer
|
||||
<< /Size 7 /Root 1 0 R /Info 6 0 R >>
|
||||
startxref
|
||||
546
|
||||
%%EOF
|
||||
@@ -0,0 +1,36 @@
|
||||
%PDF-1.4
|
||||
1 0 obj
|
||||
<< /Type /Catalog /Pages 2 0 R >>
|
||||
endobj
|
||||
2 0 obj
|
||||
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
|
||||
endobj
|
||||
3 0 obj
|
||||
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>
|
||||
endobj
|
||||
4 0 obj
|
||||
<< /Length 47 >>
|
||||
stream
|
||||
BT /F1 24 Tf 72 720 Td (CURRICULUM VITAE) Tj ET
|
||||
endstream
|
||||
endobj
|
||||
5 0 obj
|
||||
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
|
||||
endobj
|
||||
6 0 obj
|
||||
<< /Title (Curriculum Vitae) /StirlingPDFClassification ({"labels":["resume"]}) >>
|
||||
endobj
|
||||
xref
|
||||
0 7
|
||||
0000000000 65535 f
|
||||
0000000009 00000 n
|
||||
0000000058 00000 n
|
||||
0000000115 00000 n
|
||||
0000000241 00000 n
|
||||
0000000338 00000 n
|
||||
0000000435 00000 n
|
||||
trailer
|
||||
<< /Size 7 /Root 1 0 R /Info 6 0 R >>
|
||||
startxref
|
||||
533
|
||||
%%EOF
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
+90
@@ -0,0 +1,90 @@
|
||||
%PDF-1.3
|
||||
%éëñ¿
|
||||
1 0 obj
|
||||
<<
|
||||
/Count 1
|
||||
/Kids [3 0 R]
|
||||
/MediaBox [0 0 595.28 841.89]
|
||||
/Type /Pages
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/OpenAction [3 0 R /FitH null]
|
||||
/PageLayout /OneColumn
|
||||
/Pages 1 0 R
|
||||
/Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/Contents 4 0 R
|
||||
/Parent 1 0 R
|
||||
/Resources 7 0 R
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Filter /FlateDecode
|
||||
/Length 685
|
||||
>>
|
||||
stream
|
||||
xœm“OsÚ0Åïý{$ÓT±�±1§&”Ì$%ýLO¹{ J,É‘dˆ¿}W‚�t
|
||||
3 áÝ}¿÷V Ü~ŠØ(‡Ý§«.®cHREP®aVú£aÌâ1äã‚␍S(kL—÷÷7Óå|ynÊËÙ”O‡w/®ˆãS壌␍‹P~ËÂ7�@Ÿ'zfµÆ¯øÊeÛ «´ôçŸÓòœÚQ'C4תÖê –J8¬á»P�µ–o³_à}ÔpÈòØ�JY’␍¡ÜÁà—Ñk´VhÅXtRrÓOàR¾¶hªŠZZ½v;nP=
|
||||
…h`'ܤ¦#·á
|
||||
*è‘ z␍퇖ðûpœ±<
|
||||
Àï3`Õ‰¦&å`°|Õ ìp¼mQqG½,pUCƒ<¼e%oË¥e'H³‚’JüŒˆ%É8�N €”ÿ\=aåÄ'PjpüA+à`Q mŽ„AŠö26H”7P¦—ÓƒìÁ>‹¦± ÔÂ:#V�wÞöÖ¡ BOpgy~Ð4�¨ ÍÒŽz 'Z¯ƒNû°:‡òQú¶fÿd6;z89eýd™¯ŠÙ8‹ƒ‹=éì0÷ ~hã6;AöÎ]␍ƒ$Š pš‚DrÅ=œ1˜ž×&Å£ qø ýÁŠŒð^µ␍wkmdHIHÚ�-žt!NY6
|
||||
û¤E#\¼2Ú’sê¡qå—Ë¢ÙŠ
|
||||
Of<*bVŒÑ ímQßᦚ¼¶ÓT{´Ô£yDÏuEkç‚fÉ…rô%Ъ³NK4_Ö¼òlkä®#/ ³ô懲‚m”§¬È[¸œì"wȬ5ÝaÀ-š jÞÿ ïh”E¨�ÕÝ~å'pµ¨@¶´^Uˆ9\tjd½odÖ WÕió̉®IÁ²pã÷¸¦E¦zryËEîX×’8ƒ/ Õ «ÿ'˜„l
|
||||
endstream
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold
|
||||
/Encoding /WinAnsiEncoding
|
||||
/Subtype /Type1
|
||||
/Type /Font
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica
|
||||
/Encoding /WinAnsiEncoding
|
||||
/Subtype /Type1
|
||||
/Type /Font
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Font <</F1 5 0 R
|
||||
/F2 6 0 R>>
|
||||
/ProcSet [/PDF /Text /ImageB /ImageC /ImageI]
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/CreationDate (D:20260716092355Z)
|
||||
>>
|
||||
endobj
|
||||
xref
|
||||
0 9
|
||||
0000000000 65535 f
|
||||
0000000015 00000 n
|
||||
0000000102 00000 n
|
||||
0000000205 00000 n
|
||||
0000000285 00000 n
|
||||
0000001042 00000 n
|
||||
0000001144 00000 n
|
||||
0000001241 00000 n
|
||||
0000001338 00000 n
|
||||
trailer
|
||||
<<
|
||||
/Size 9
|
||||
/Root 2 0 R
|
||||
/Info 8 0 R
|
||||
/ID [<7F4FFDD51531EF25A5301EAFCA4542A0><7F4FFDD51531EF25A5301EAFCA4542A0>]
|
||||
>>
|
||||
startxref
|
||||
1393
|
||||
%%EOF
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -194,7 +194,6 @@ export const POLICY_CATEGORIES: PolicyCategory[] = [
|
||||
tone: "blue",
|
||||
desc: "portal.policies.categories.classification.desc",
|
||||
providesClassification: true,
|
||||
requiresAiEngine: true,
|
||||
},
|
||||
{
|
||||
id: "compliance",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { usePolicyAutoRun } from "@app/components/policies/usePolicyAutoRun";
|
||||
import { useClientSideClassification } from "@app/components/policies/useClientSideClassification";
|
||||
|
||||
/**
|
||||
* Headless controller that drives policy auto-run (enforce every enabled policy
|
||||
@@ -7,5 +8,7 @@ import { usePolicyAutoRun } from "@app/components/policies/usePolicyAutoRun";
|
||||
*/
|
||||
export function PolicyAutoRunController() {
|
||||
usePolicyAutoRun();
|
||||
// Non-AI systems classify uploads in the browser; inert when the AI engine is on.
|
||||
useClientSideClassification();
|
||||
return null;
|
||||
}
|
||||
|
||||
+255
@@ -0,0 +1,255 @@
|
||||
// Delivery guarantees of the client-side classification hook, driving the real
|
||||
// policyRunStore and mocking only IO (storage, the heuristic engine, the meter).
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { renderHook, waitFor } from "@testing-library/react";
|
||||
import {
|
||||
markDispatched,
|
||||
resetPolicyRuns,
|
||||
} from "@app/components/policies/policyRunStore";
|
||||
|
||||
interface TestStub {
|
||||
id: string;
|
||||
name: string;
|
||||
lastModified: number;
|
||||
derivedFromTool?: boolean;
|
||||
classificationLabels?: string[];
|
||||
}
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
workspace: [] as Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
lastModified: number;
|
||||
derivedFromTool?: boolean;
|
||||
classificationLabels?: string[];
|
||||
}>,
|
||||
configLoading: false,
|
||||
updateStirlingFileStub: vi.fn(),
|
||||
bumpRevision: vi.fn(),
|
||||
getStirlingFile: vi.fn(),
|
||||
updateFileMetadata: vi.fn(async (_id: string, _updates: unknown) => true),
|
||||
classify: vi.fn(),
|
||||
meter: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@app/contexts/AppConfigContext", () => ({
|
||||
useAppConfig: () => ({ config: {}, loading: mocks.configLoading }),
|
||||
}));
|
||||
vi.mock("@app/hooks/useClassificationEnabled", () => ({
|
||||
useClassificationEnabled: () => true,
|
||||
}));
|
||||
vi.mock("@app/hooks/useAiEngineEnabled", () => ({
|
||||
useAiEngineEnabled: () => false,
|
||||
}));
|
||||
vi.mock("@app/hooks/usePolicies", () => ({
|
||||
usePolicies: () => ({
|
||||
policies: {
|
||||
classification: {
|
||||
configured: true,
|
||||
status: "active",
|
||||
backendId: "backend-classification",
|
||||
sources: ["editor"],
|
||||
},
|
||||
},
|
||||
}),
|
||||
}));
|
||||
vi.mock("@app/contexts/FileContext", () => ({
|
||||
useAllFiles: () => ({ fileStubs: mocks.workspace }),
|
||||
useFileManagement: () => ({
|
||||
updateStirlingFileStub: mocks.updateStirlingFileStub,
|
||||
}),
|
||||
}));
|
||||
vi.mock("@app/contexts/IndexedDBContext", () => ({
|
||||
useIndexedDB: () => ({ bumpRevision: mocks.bumpRevision }),
|
||||
}));
|
||||
vi.mock("@app/services/fileStorage", () => ({
|
||||
fileStorage: {
|
||||
getStirlingFile: (id: string) => mocks.getStirlingFile(id),
|
||||
updateFileMetadata: (id: string, updates: unknown) =>
|
||||
mocks.updateFileMetadata(id, updates),
|
||||
},
|
||||
}));
|
||||
vi.mock("@app/services/heuristic/heuristicClassification", () => ({
|
||||
classifyFileHeuristically: (file: File) => mocks.classify(file),
|
||||
}));
|
||||
vi.mock("@app/services/classificationMeter", () => ({
|
||||
meterClassificationRun: (payload: unknown) => mocks.meter(payload),
|
||||
}));
|
||||
|
||||
import { useClientSideClassification } from "@app/components/policies/useClientSideClassification";
|
||||
|
||||
// Run idle callbacks immediately so batches start without timer waits.
|
||||
vi.stubGlobal("requestIdleCallback", (cb: () => void) => {
|
||||
cb();
|
||||
return 1;
|
||||
});
|
||||
vi.stubGlobal("cancelIdleCallback", () => {});
|
||||
|
||||
const stub = (id: string, extra: Partial<TestStub> = {}): TestStub => ({
|
||||
id,
|
||||
name: `${id}.pdf`,
|
||||
lastModified: 1,
|
||||
...extra,
|
||||
});
|
||||
|
||||
const fakeFile = (id: string) => new File([id], `${id}.pdf`);
|
||||
|
||||
describe("useClientSideClassification delivery", () => {
|
||||
beforeEach(() => {
|
||||
localStorage.clear();
|
||||
resetPolicyRuns();
|
||||
mocks.workspace = [];
|
||||
mocks.configLoading = false;
|
||||
mocks.updateStirlingFileStub.mockClear();
|
||||
mocks.bumpRevision.mockClear();
|
||||
mocks.updateFileMetadata.mockClear();
|
||||
mocks.meter.mockClear();
|
||||
mocks.getStirlingFile.mockReset();
|
||||
mocks.getStirlingFile.mockImplementation(async (id: string) =>
|
||||
fakeFile(id),
|
||||
);
|
||||
mocks.classify.mockReset();
|
||||
});
|
||||
|
||||
it("classifies pending uploads, writes labels, and meters once per file", async () => {
|
||||
mocks.workspace = [stub("a"), stub("b")];
|
||||
mocks.classify.mockImplementation(async (file: File) => ({
|
||||
labels: [file.name.startsWith("a") ? "invoice" : "resume"],
|
||||
}));
|
||||
|
||||
renderHook(() => useClientSideClassification());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledTimes(2),
|
||||
);
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("a", {
|
||||
classificationLabels: ["invoice"],
|
||||
});
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("b", {
|
||||
classificationLabels: ["resume"],
|
||||
});
|
||||
expect(mocks.meter).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.bumpRevision).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("delivers a result computed while the effect re-fired mid-batch (upload-wave race)", async () => {
|
||||
let resolveA!: (v: { labels: string[] }) => void;
|
||||
const gateA = new Promise<{ labels: string[] }>((r) => (resolveA = r));
|
||||
mocks.classify.mockImplementation((file: File) =>
|
||||
file.name.startsWith("a") ? gateA : Promise.resolve({ labels: ["nda"] }),
|
||||
);
|
||||
mocks.workspace = [stub("a")];
|
||||
|
||||
const { rerender } = renderHook(() => useClientSideClassification());
|
||||
await waitFor(() => expect(mocks.classify).toHaveBeenCalledTimes(1));
|
||||
|
||||
// A new upload mid-classify re-fires the effect and cancels the in-flight
|
||||
// batch; a's already-computed result must still be delivered.
|
||||
mocks.workspace = [stub("a"), stub("b")];
|
||||
rerender();
|
||||
|
||||
resolveA({ labels: ["purchase-order"] });
|
||||
await waitFor(() =>
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("a", {
|
||||
classificationLabels: ["purchase-order"],
|
||||
}),
|
||||
);
|
||||
// The newly-arrived file classifies too, and neither is double-classified.
|
||||
await waitFor(() =>
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("b", {
|
||||
classificationLabels: ["nda"],
|
||||
}),
|
||||
);
|
||||
expect(mocks.classify).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.meter).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("persists a definitive [] verdict for an unlabelled file and does not retry it", async () => {
|
||||
mocks.workspace = [stub("plain")];
|
||||
mocks.classify.mockResolvedValue({ labels: [] });
|
||||
|
||||
renderHook(() => useClientSideClassification());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("plain", {
|
||||
classificationLabels: [],
|
||||
}),
|
||||
);
|
||||
expect(mocks.classify).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.meter).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("heals a previously-dispatched file whose result was lost, without re-metering", async () => {
|
||||
// A past session classified + metered this file but the delivery was lost.
|
||||
markDispatched("classification", "lost");
|
||||
mocks.workspace = [stub("lost")];
|
||||
mocks.classify.mockResolvedValue({ labels: ["bank-statement"] });
|
||||
|
||||
renderHook(() => useClientSideClassification());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("lost", {
|
||||
classificationLabels: ["bank-statement"],
|
||||
}),
|
||||
);
|
||||
expect(mocks.meter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves an unreadable file undelivered (no verdict, no meter) so it can retry", async () => {
|
||||
// An extraction failure may be environmental, so it must never poison the
|
||||
// file with a persisted verdict. The read path deliberately warns.
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
mocks.workspace = [stub("corrupt")];
|
||||
mocks.classify.mockRejectedValue(new Error("bad pdf"));
|
||||
|
||||
renderHook(() => useClientSideClassification());
|
||||
|
||||
await waitFor(() =>
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining("corrupt.pdf: could not be read"),
|
||||
expect.any(Error),
|
||||
),
|
||||
);
|
||||
expect(mocks.classify).toHaveBeenCalledTimes(1); // claimed: once per session
|
||||
expect(mocks.updateStirlingFileStub).not.toHaveBeenCalled();
|
||||
expect(mocks.updateFileMetadata).not.toHaveBeenCalled();
|
||||
expect(mocks.meter).not.toHaveBeenCalled();
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it("waits for app-config before classifying (AI flag unknown = possible double-run)", async () => {
|
||||
// While the config loads, aiEnabled reads false even on an AI-on tenant; classifying
|
||||
// then would race the server-side classify policy and double-bill the same files.
|
||||
mocks.configLoading = true;
|
||||
mocks.workspace = [stub("early")];
|
||||
mocks.classify.mockResolvedValue({ labels: ["invoice"] });
|
||||
|
||||
const { rerender } = renderHook(() => useClientSideClassification());
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
expect(mocks.classify).not.toHaveBeenCalled();
|
||||
|
||||
// Config resolves (AI stays off): the pending file classifies normally.
|
||||
mocks.configLoading = false;
|
||||
rerender();
|
||||
await waitFor(() =>
|
||||
expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("early", {
|
||||
classificationLabels: ["invoice"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("skips tool outputs and already-labelled files", async () => {
|
||||
mocks.workspace = [
|
||||
stub("derived", { derivedFromTool: true }),
|
||||
stub("done", { classificationLabels: ["invoice"] }),
|
||||
stub("verdict", { classificationLabels: [] }),
|
||||
];
|
||||
|
||||
renderHook(() => useClientSideClassification());
|
||||
|
||||
// Nothing to classify; give the (immediate) idle path a beat to prove it.
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
expect(mocks.classify).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,207 @@
|
||||
// With the AI engine off, the Classification policy runs here in the browser:
|
||||
// each upload is labelled by the heuristic engine and metered for billing parity.
|
||||
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { useAllFiles, useFileManagement } from "@app/contexts/FileContext";
|
||||
import { useAppConfig } from "@app/contexts/AppConfigContext";
|
||||
import { useIndexedDB } from "@app/contexts/IndexedDBContext";
|
||||
import { fileStorage } from "@app/services/fileStorage";
|
||||
import { useClassificationEnabled } from "@app/hooks/useClassificationEnabled";
|
||||
import { useAiEngineEnabled } from "@app/hooks/useAiEngineEnabled";
|
||||
import { scheduleIdle } from "@app/utils/scheduleIdle";
|
||||
import { usePolicies } from "@app/hooks/usePolicies";
|
||||
import { classifyFileHeuristically } from "@app/services/heuristic/heuristicClassification";
|
||||
import { meterClassificationRun } from "@app/services/classificationMeter";
|
||||
import {
|
||||
isDispatched,
|
||||
markDispatched,
|
||||
} from "@app/components/policies/policyRunStore";
|
||||
import type { FileId } from "@app/types/file";
|
||||
import type { StirlingFile, StirlingFileStub } from "@app/types/fileContext";
|
||||
|
||||
/** The category id of the Classification policy (see policyDefinitions). */
|
||||
const CLASSIFICATION_CATEGORY = "classification";
|
||||
/** Files classified per idle pass, so a large library drains over several ticks. */
|
||||
const CLASSIFY_BATCH = 3;
|
||||
/** How long to wait for an upload's bytes to land in IndexedDB (20 × 250ms ≈ 5s).
|
||||
* The stub can surface in the file list a beat before its bytes are committed. */
|
||||
const FILE_WAIT_TRIES = 20;
|
||||
const FILE_WAIT_MS = 250;
|
||||
|
||||
/** localStorage flag: set to "true" for a full per-file scoring breakdown in the console. */
|
||||
const DEBUG_FLAG = "stirling-classification-debug";
|
||||
|
||||
function isClassificationDebug(): boolean {
|
||||
try {
|
||||
return localStorage.getItem(DEBUG_FLAG) === "true";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
export function useClientSideClassification(): void {
|
||||
const { fileStubs } = useAllFiles();
|
||||
const { updateStirlingFileStub } = useFileManagement();
|
||||
const { bumpRevision } = useIndexedDB();
|
||||
const { policies } = usePolicies();
|
||||
const classificationEnabled = useClassificationEnabled();
|
||||
const aiEnabled = useAiEngineEnabled();
|
||||
// While app-config loads, aiEnabled reads false even on AI-on tenants; classifying
|
||||
// in that window would double-run (and double-bill) files the server also labels.
|
||||
const { loading: configLoading } = useAppConfig();
|
||||
// Files claimed this session, keyed id+lastModified so a new version is retried once. A claim is
|
||||
// taken synchronously right before classifying, so overlapping batches never double-classify.
|
||||
const claimed = useRef<Set<string>>(new Set());
|
||||
// Bumped after each batch to drain the next one.
|
||||
const [tick, setTick] = useState(0);
|
||||
|
||||
const policy = policies[CLASSIFICATION_CATEGORY];
|
||||
// Only when the admin has an active Classification policy - the same gate the AI path uses.
|
||||
const active = Boolean(
|
||||
policy?.configured &&
|
||||
policy.status === "active" &&
|
||||
policy.backendId &&
|
||||
(!policy.sources ||
|
||||
policy.sources.length === 0 ||
|
||||
policy.sources.includes("editor")),
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (configLoading || !classificationEnabled || aiEnabled || !active) {
|
||||
return;
|
||||
}
|
||||
const claimKey = (s: StirlingFileStub) =>
|
||||
`${s.id as string}:${s.lastModified ?? 0}`;
|
||||
// null labels = never delivered, retried here; [] = definitive no-label verdict.
|
||||
const pending = fileStubs
|
||||
.filter(
|
||||
(s) =>
|
||||
!s.derivedFromTool &&
|
||||
s.classificationLabels == null &&
|
||||
!claimed.current.has(claimKey(s)),
|
||||
)
|
||||
.slice(0, CLASSIFY_BATCH);
|
||||
if (pending.length === 0) return;
|
||||
let cancelled = false;
|
||||
const cancelIdle = scheduleIdle(() => {
|
||||
// Superseded before starting: the newer effect instance owns the queue.
|
||||
if (cancelled) return;
|
||||
void (async () => {
|
||||
let wrote = false;
|
||||
for (const stub of pending) {
|
||||
const key = claimKey(stub);
|
||||
// Re-validate at execution time - another batch may have claimed it since.
|
||||
if (claimed.current.has(key)) continue;
|
||||
claimed.current.add(key);
|
||||
const labels = await classifyStub(stub.id as FileId, stub.name);
|
||||
// Bytes never landed (file removed mid-wait): leave undelivered so a
|
||||
// reload (or new version) retries; the claim stops churn this session.
|
||||
if (labels == null) continue;
|
||||
// Deliver unconditionally - a re-render must never discard a computed
|
||||
// (and already metered) result. Writes are idempotent.
|
||||
updateStirlingFileStub(stub.id as FileId, {
|
||||
classificationLabels: labels,
|
||||
});
|
||||
const ok = await fileStorage.updateFileMetadata(stub.id as FileId, {
|
||||
classificationLabels: labels,
|
||||
});
|
||||
if (ok) wrote = true;
|
||||
}
|
||||
if (wrote) bumpRevision();
|
||||
// Drain the next batch; the terminal pass finds nothing pending and stops.
|
||||
setTick((n) => n + 1);
|
||||
})();
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
cancelIdle();
|
||||
};
|
||||
}, [
|
||||
fileStubs,
|
||||
active,
|
||||
classificationEnabled,
|
||||
aiEnabled,
|
||||
configLoading,
|
||||
updateStirlingFileStub,
|
||||
bumpRevision,
|
||||
tick,
|
||||
]);
|
||||
}
|
||||
|
||||
/** Classify one file, metering exactly once; null = no verdict, retried later. */
|
||||
async function classifyStub(
|
||||
fileId: FileId,
|
||||
fileName: string,
|
||||
): Promise<string[] | null> {
|
||||
let file: StirlingFile | null = null;
|
||||
for (let i = 0; i < FILE_WAIT_TRIES; i++) {
|
||||
file = await fileStorage.getStirlingFile(fileId).catch(() => null);
|
||||
if (file) break;
|
||||
await delay(FILE_WAIT_MS);
|
||||
}
|
||||
if (!file) {
|
||||
console.warn(
|
||||
`[Classify] ${fileName}: bytes never arrived in storage; will retry on next load`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const debug = isClassificationDebug();
|
||||
const startedAt = performance.now();
|
||||
try {
|
||||
const result = await classifyFileHeuristically(file, { explain: debug });
|
||||
const { labels } = result;
|
||||
const alreadyMetered = isDispatched(CLASSIFICATION_CATEGORY, fileId);
|
||||
const ms = Math.round(performance.now() - startedAt);
|
||||
const verdict =
|
||||
labels.length > 0
|
||||
? labels.join(", ")
|
||||
: result.isEnglish
|
||||
? "no label"
|
||||
: "no label (not English)";
|
||||
console.debug(
|
||||
`[Classify] ${fileName} -> ${verdict} (${result.confidence}, score ${result.score}, ${ms}ms)` +
|
||||
(alreadyMetered ? " [heal: not re-metered]" : ""),
|
||||
);
|
||||
if (debug && result.explain) logExplanation(fileName, result);
|
||||
// Meter on the first classification only; a healing re-run of an undelivered
|
||||
// result (already dispatched) is not a new billable run.
|
||||
if (!alreadyMetered) {
|
||||
meterClassificationRun({
|
||||
policyName: "Classification",
|
||||
documentCount: 1,
|
||||
labels,
|
||||
});
|
||||
}
|
||||
markDispatched(CLASSIFICATION_CATEGORY, fileId);
|
||||
return labels;
|
||||
} catch (err) {
|
||||
// Never persist a verdict for an unreadable file - the failure may be
|
||||
// environmental, so it must stay eligible to retry (and meter) later.
|
||||
console.warn(`[Classify] ${fileName}: could not be read, will retry`, err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Full scoring breakdown, one collapsed console group per file (debug flag only). */
|
||||
function logExplanation(
|
||||
fileName: string,
|
||||
result: Awaited<ReturnType<typeof classifyFileHeuristically>>,
|
||||
): void {
|
||||
const ex = result.explain;
|
||||
if (!ex) return;
|
||||
console.groupCollapsed(
|
||||
`[Classify] ${fileName} scoring (english=${ex.isEnglish}, lowText=${ex.lowText})`,
|
||||
);
|
||||
if (ex.candidates.length === 0) {
|
||||
console.log("no label scored above zero");
|
||||
}
|
||||
for (const c of ex.candidates) {
|
||||
console.log(
|
||||
`${c.id}${c.emit ? "" : " (suppressed)"}: score ${c.score}, ${c.distinct} distinct signals`,
|
||||
);
|
||||
for (const s of c.signals) console.log(` ${s}`);
|
||||
}
|
||||
console.groupEnd();
|
||||
}
|
||||
@@ -47,6 +47,11 @@ const mocks = vi.hoisted(() => ({
|
||||
consumeFiles: vi.fn(),
|
||||
}));
|
||||
|
||||
// Classification chains server-side only when the AI engine is on (else it runs
|
||||
// client-side); this batch exercises the server chain, so force the engine on.
|
||||
vi.mock("@app/hooks/useAiEngineEnabled", () => ({
|
||||
useAiEngineEnabled: () => true,
|
||||
}));
|
||||
vi.mock("@app/contexts/FileContext", () => ({
|
||||
useAllFiles: () => ({ fileStubs: mocks.workspace }),
|
||||
useFileManagement: () => ({
|
||||
|
||||
@@ -4,6 +4,12 @@ import { renderHook, act } from "@testing-library/react";
|
||||
// Two active upload policies, so the auto-run should CHAIN them: fire the first on
|
||||
// the upload, then the second on the first's output. Stub the contexts + network so
|
||||
// we can drive the dispatch against the REAL run store.
|
||||
// Controllable AI-engine flag: on by default so classification chains server-side; one
|
||||
// test flips it off to assert classification is kept OUT of the server chain.
|
||||
const aiEnabled = vi.hoisted(() => ({ value: true }));
|
||||
vi.mock("@app/hooks/useAiEngineEnabled", () => ({
|
||||
useAiEngineEnabled: () => aiEnabled.value,
|
||||
}));
|
||||
const fileStubs: { id: string; name: string; derivedFromTool?: boolean }[] = [];
|
||||
vi.mock("@app/contexts/FileContext", () => ({
|
||||
useAllFiles: () => ({ fileStubs }),
|
||||
@@ -66,6 +72,7 @@ beforeEach(() => {
|
||||
localStorage.clear();
|
||||
resetPolicyRuns();
|
||||
setFileStubs([]);
|
||||
aiEnabled.value = true;
|
||||
runStored.mockReset();
|
||||
getFile.mockReset();
|
||||
getFile.mockResolvedValue({ size: 100 } as never);
|
||||
@@ -116,4 +123,23 @@ describe("auto-run ordered chaining", () => {
|
||||
// The next policy (order 1) fires on the first policy's output, not the original.
|
||||
expect(runStored).toHaveBeenCalledWith("backend-cls", [{ size: 100 }]);
|
||||
});
|
||||
|
||||
it("keeps classification out of the server chain when the AI engine is off", async () => {
|
||||
// AI off: classification runs client-side (useClientSideClassification), so the
|
||||
// server chain must skip it - only the normal (security) policy dispatches.
|
||||
aiEnabled.value = false;
|
||||
setFileStubs([{ id: "file-1", name: "doc.pdf" }]);
|
||||
runStored.mockResolvedValue("run-sec");
|
||||
|
||||
renderHook(() => usePolicyAutoRun());
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
});
|
||||
|
||||
expect(runStored).toHaveBeenCalledWith("backend-sec", [{ size: 100 }]);
|
||||
expect(runStored).not.toHaveBeenCalledWith(
|
||||
"backend-cls",
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -42,6 +42,7 @@ import { readClassificationLabelsFromFile } from "@app/services/fileClassificati
|
||||
import type { StirlingFile, StirlingFileStub } from "@app/types/fileContext";
|
||||
import type { PoliciesByCategory } from "@app/types/policies";
|
||||
import { usePolicies } from "@app/hooks/usePolicies";
|
||||
import { useAiEngineEnabled } from "@app/hooks/useAiEngineEnabled";
|
||||
import {
|
||||
addReconciledRun,
|
||||
dispatchKey,
|
||||
@@ -128,6 +129,7 @@ export function usePolicyAutoRun(): void {
|
||||
const { consumeFiles } = useFileContext();
|
||||
const { bumpRevision } = useIndexedDB();
|
||||
const { policies } = usePolicies();
|
||||
const aiEnabled = useAiEngineEnabled();
|
||||
const runs = usePolicyRuns();
|
||||
// Live view of the workspace files, read inside the import effect WITHOUT making
|
||||
// it a dependency. The silent consume that delivers an output mutates fileStubs,
|
||||
@@ -161,18 +163,21 @@ export function usePolicyAutoRun(): void {
|
||||
() =>
|
||||
Object.entries(policies)
|
||||
.filter(
|
||||
([, s]) =>
|
||||
([id, s]) =>
|
||||
s.configured &&
|
||||
s.status === "active" &&
|
||||
s.backendId &&
|
||||
(!s.sources ||
|
||||
s.sources.length === 0 ||
|
||||
s.sources.includes("editor")) &&
|
||||
(s.runOn ?? "upload") === "upload",
|
||||
(s.runOn ?? "upload") === "upload" &&
|
||||
// Non-AI systems classify in the browser (useClientSideClassification), so keep the
|
||||
// Classification policy out of the server chain when the AI engine is off.
|
||||
!(id === "classification" && !aiEnabled),
|
||||
)
|
||||
.sort(([, a], [, b]) => (a.order ?? 0) - (b.order ?? 0))
|
||||
.map(([id]) => id),
|
||||
[policies],
|
||||
[policies, aiEnabled],
|
||||
);
|
||||
|
||||
// Runs whose chain-continuation we've already handled this session, so the next
|
||||
|
||||
+10
-18
@@ -1,4 +1,5 @@
|
||||
// Classification override of the Files-sidebar grouping seam: Recent, one group per VISIBLE category (the fixed, shared label families; each can be hidden device-local), then Other for files in none of those. Labels are cached on the stub via a lazy metadata backfill so grouping stays cheap.
|
||||
// Classification override of the Files-sidebar grouping seam: Recent, one group
|
||||
// per visible category, then Other. Labels cache onto stubs via a lazy backfill.
|
||||
|
||||
import {
|
||||
useEffect,
|
||||
@@ -18,6 +19,7 @@ import {
|
||||
subscribeSidebarCategories,
|
||||
} from "@app/services/fileSidebarCategories";
|
||||
import { buildLabelGroups } from "@app/components/shared/fileSidebarGroupingLogic";
|
||||
import { scheduleIdle } from "@app/utils/scheduleIdle";
|
||||
import type { FileId } from "@app/types/file";
|
||||
import type { StirlingFileStub } from "@app/types/fileContext";
|
||||
import type { FileSidebarGroup } from "@core/components/shared/fileSidebarGrouping";
|
||||
@@ -36,33 +38,22 @@ const BACKFILL_BATCH = 3;
|
||||
/** Recheck delay when the backfill yields to an active policy wave. */
|
||||
const BACKFILL_BUSY_RETRY_MS = 4000;
|
||||
|
||||
/** Schedule work for the browser's idle time (or soon after, as a fallback). */
|
||||
function scheduleIdle(task: () => void): () => void {
|
||||
if (typeof requestIdleCallback === "function") {
|
||||
const handle = requestIdleCallback(task, { timeout: 2000 });
|
||||
return () => cancelIdleCallback(handle);
|
||||
}
|
||||
const timer = window.setTimeout(task, 200);
|
||||
return () => window.clearTimeout(timer);
|
||||
}
|
||||
|
||||
export function useFileSidebarGroups(
|
||||
stubs: StirlingFileStub[],
|
||||
): FileSidebarGroup[] | null {
|
||||
const { t } = useTranslation();
|
||||
// Classification off (AI disabled) → no grouping at all: return the flat list
|
||||
// like core, and don't fetch team labels or backfill from metadata. Gates the
|
||||
// whole feature so an AI-off SaaS tenant sees no Recent/Other/category chrome.
|
||||
// Classification off (core): flat list, no category fetch or backfill.
|
||||
const enabled = useClassificationEnabled();
|
||||
const { bumpRevision } = useIndexedDB();
|
||||
// Attempted reads keyed by id+lastModified: a re-classified file (new version bumps lastModified) is re-read and leaves "Other" on its own, while a truly-unlabelled file keeps a stable key and is read once.
|
||||
// Reads keyed by id+lastModified, so a new file version is re-read exactly once.
|
||||
const attempted = useRef<Set<string>>(new Set());
|
||||
const attemptKey = (s: StirlingFileStub) =>
|
||||
`${s.id as string}:${s.lastModified ?? 0}`;
|
||||
// Bumped to re-attempt a backfill pass that yielded to an active policy wave.
|
||||
const [retryTick, setRetryTick] = useState(0);
|
||||
|
||||
// Fallback for files that arrive with labels already in metadata but no policy delivery (imports/shares): read+cache a few per idle pass, yielding while a policy wave is in flight since those stubs get stamped on delivery anyway.
|
||||
// Backfill labels from file metadata onto stubs, a few per idle pass; yields
|
||||
// while a policy wave is in flight. The heuristic path stamps stubs directly.
|
||||
useEffect(() => {
|
||||
if (!enabled) return;
|
||||
const pending = stubs
|
||||
@@ -75,7 +66,8 @@ export function useFileSidebarGroups(
|
||||
let retryTimer: number | undefined;
|
||||
const cancelIdle = scheduleIdle(() => {
|
||||
if (cancelled) return;
|
||||
// Deliveries stamp labels during a wave, so reading now is wasted parsing; recheck after it (a timer self-heals when a wave ends without a stubs change).
|
||||
// Reading during a wave is wasted parsing; recheck after it. The timer
|
||||
// self-heals when a wave ends without a stubs change.
|
||||
if (hasInFlightPolicyRuns()) {
|
||||
retryTimer = window.setTimeout(() => {
|
||||
if (!cancelled) setRetryTick((n) => n + 1);
|
||||
@@ -85,9 +77,9 @@ export function useFileSidebarGroups(
|
||||
void (async () => {
|
||||
let wrote = false;
|
||||
for (const stub of pending) {
|
||||
attempted.current.add(attemptKey(stub));
|
||||
const labels = await readStubClassificationLabels(stub);
|
||||
if (cancelled) return;
|
||||
attempted.current.add(attemptKey(stub));
|
||||
if (labels) {
|
||||
const ok = await fileStorage.updateFileMetadata(stub.id as FileId, {
|
||||
classificationLabels: labels,
|
||||
@@ -43,8 +43,6 @@ export const POLICY_CATEGORIES: PolicyCategory[] = [
|
||||
label: "Classification",
|
||||
icon: policyCategoryIcon("classification", ICON_SX),
|
||||
desc: "Identify each document's type on upload and tag its metadata for filing and search.",
|
||||
// Needs the AI engine to classify; hidden from the policy list when it's off.
|
||||
requiresAiEngine: true,
|
||||
},
|
||||
{
|
||||
id: "compliance",
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
// Classification is available on every proprietary-based build: the classify
|
||||
// policy labels server-side with AI on, the in-browser heuristic labels with AI off.
|
||||
|
||||
export function useClassificationEnabled(): boolean {
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
// Meters an in-browser (non-AI) classification run for billing/audit parity with
|
||||
// the server-side classify path. Fire-and-forget; failures never block the user.
|
||||
|
||||
import apiClient from "@app/services/apiClient";
|
||||
import { getPolicyOutputBaseUrl } from "@app/services/policyOutputBaseUrl";
|
||||
import { resolvePolicyRunTarget } from "@app/services/policyApi";
|
||||
|
||||
interface ClassifyMeterPayload {
|
||||
/** Policy name for the audit-trail label; defaults to "Classification" server-side. */
|
||||
policyName?: string;
|
||||
/** Documents covered by this meter call (defaults to 1 server-side). */
|
||||
documentCount?: number;
|
||||
/** Resolved labels, carried for the audit record. */
|
||||
labels?: string[];
|
||||
}
|
||||
|
||||
/** Meter a completed client-side classification. Does not throw and is not awaited by callers. */
|
||||
export function meterClassificationRun(payload: ClassifyMeterPayload): void {
|
||||
const base = getPolicyOutputBaseUrl(resolvePolicyRunTarget());
|
||||
void apiClient
|
||||
.post(`${base}/api/v1/policies/classify/meter`, payload, {
|
||||
suppressErrorToast: true,
|
||||
})
|
||||
.catch(() => {
|
||||
// Best-effort billing; the classification already succeeded in the browser.
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// Client-side classification entry point: load rules, extract the PDF, classify.
|
||||
|
||||
import {
|
||||
ensureRulesLoaded,
|
||||
classifyHeuristic,
|
||||
} from "@app/services/heuristic/heuristicEngine";
|
||||
import { extractHeuristicDoc } from "@app/services/heuristic/heuristicExtractor";
|
||||
import type { HeuristicResult } from "@app/services/heuristic/types";
|
||||
|
||||
/** Classify a file in the browser. Throws if extraction fails (unreadable / non-PDF). */
|
||||
export async function classifyFileHeuristically(
|
||||
file: File,
|
||||
opts?: { explain?: boolean },
|
||||
): Promise<HeuristicResult> {
|
||||
await ensureRulesLoaded();
|
||||
const doc = await extractHeuristicDoc(file, file.name);
|
||||
return classifyHeuristic(doc, opts);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,118 @@
|
||||
// Regression cases: each feeds the real engine a typical specimen's actual text.
|
||||
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
classifyHeuristic,
|
||||
ensureRulesLoaded,
|
||||
} from "@app/services/heuristic/heuristicEngine";
|
||||
import type { HeuristicDoc } from "@app/services/heuristic/types";
|
||||
|
||||
beforeAll(async () => {
|
||||
await ensureRulesLoaded();
|
||||
});
|
||||
|
||||
function classify(
|
||||
title: string,
|
||||
body: string,
|
||||
fileName = "doc.pdf",
|
||||
pageCount = 1,
|
||||
) {
|
||||
const doc: HeuristicDoc = {
|
||||
fileName,
|
||||
pageCount,
|
||||
meta: {},
|
||||
titleZone: title,
|
||||
firstZone: body,
|
||||
allZone: body,
|
||||
};
|
||||
return classifyHeuristic(doc);
|
||||
}
|
||||
|
||||
describe("scoring explanations", () => {
|
||||
const doc: HeuristicDoc = {
|
||||
fileName: "invoice_acme.pdf",
|
||||
pageCount: 1,
|
||||
meta: {},
|
||||
titleZone: "TAX INVOICE",
|
||||
firstZone: "Invoice Number: INV-9 Invoice Total: 950.00",
|
||||
allZone: "Invoice Number: INV-9 Invoice Total: 950.00",
|
||||
};
|
||||
|
||||
it("returns candidates with per-rule signals when requested", () => {
|
||||
const r = classifyHeuristic(doc, { explain: true });
|
||||
expect(r.labels[0]).toBe("invoice");
|
||||
const top = r.explain?.candidates[0];
|
||||
expect(top?.id).toBe("invoice");
|
||||
expect(top?.score).toBeGreaterThan(0);
|
||||
expect(top?.signals.some((s) => s.includes('phrase "tax invoice"'))).toBe(
|
||||
true,
|
||||
);
|
||||
expect(top?.signals.some((s) => s.includes("filename"))).toBe(true);
|
||||
});
|
||||
|
||||
it("omits the explanation by default", () => {
|
||||
expect(classifyHeuristic(doc).explain).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("documents observed lost on upload (engine must label them)", () => {
|
||||
it("labels a resume", () => {
|
||||
const body = [
|
||||
"CURRICULUM VITAE",
|
||||
"Jane Doe jane.doe@example.com +44 7700 900123 London, United Kingdom",
|
||||
"Professional Summary: An experienced software engineer with more than ten years of",
|
||||
"professional experience building reliable web applications and leading small teams.",
|
||||
"Career Objective: To take on a senior engineering role where I can apply my skills in",
|
||||
"distributed systems and mentor other engineers on the team.",
|
||||
"Professional Experience:",
|
||||
"Senior Engineer, Northwind Ltd (2019 to present). Led the migration of the billing platform",
|
||||
"and improved reliability across all of the core services.",
|
||||
"Software Engineer, Contoso plc (2014 to 2019). Built and maintained customer-facing features",
|
||||
"used by more than a million people every day.",
|
||||
"Education: BSc Computer Science, University of Manchester.",
|
||||
"References available upon request.",
|
||||
].join("\n");
|
||||
const r = classify("CURRICULUM VITAE", body, "resume_jane_doe.pdf");
|
||||
expect(r.labels[0]).toBe("resume");
|
||||
});
|
||||
|
||||
it("labels a purchase order", () => {
|
||||
const body = [
|
||||
"PURCHASE ORDER",
|
||||
"Purchase Order Number: PO-55231 Requisition Number: REQ-9910 Date: 2 April 2024",
|
||||
"To: Global Office Supplies Ltd. Please supply the following goods to our warehouse at the",
|
||||
"address shown below and confirm the expected delivery date by return.",
|
||||
"Qty Ordered: 20 Item: Ergonomic office chair Unit Price: $180.00",
|
||||
"Qty Ordered: 15 Item: Height-adjustable desk Unit Price: $420.00",
|
||||
"Qty Ordered: 50 Item: LED desk lamp Unit Price: $35.00",
|
||||
"This is an official order. All goods supplied against this purchase order must reference the",
|
||||
"requisition number on the delivery note and on your invoice.",
|
||||
"Authorised by: Procurement Department, Northwind Ltd.",
|
||||
].join("\n");
|
||||
const r = classify("PURCHASE ORDER", body, "purchase_order.pdf");
|
||||
expect(r.labels[0]).toBe("purchase-order");
|
||||
});
|
||||
|
||||
it("labels a master services agreement", () => {
|
||||
const body = [
|
||||
"MASTER SERVICES AGREEMENT",
|
||||
"This Master Services Agreement is made between the Client and the Service Provider and sets",
|
||||
"out the terms on which the Service Provider will provide services to the Client.",
|
||||
"1. Engagement. The Client engages the Service Provider to perform the services described in",
|
||||
"each Statement of Work agreed between the parties from time to time.",
|
||||
"2. Fees. The Client shall pay the fees set out in the applicable Statement of Work within",
|
||||
"thirty days of the date of each invoice.",
|
||||
"3. Term and Termination. This agreement shall continue until terminated by either party on",
|
||||
"sixty days written notice to the other party.",
|
||||
"4. Confidentiality. Each party shall keep confidential the confidential information of the",
|
||||
"other party that it receives under this agreement.",
|
||||
"We are pleased to act for you and look forward to a productive working relationship.",
|
||||
].join("\n");
|
||||
const r = classify(
|
||||
"MASTER SERVICES AGREEMENT",
|
||||
body,
|
||||
"service_agreement.pdf",
|
||||
);
|
||||
expect(r.labels[0]).toBe("service-agreement");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,125 @@
|
||||
// Core fidelity cases for the heuristic engine against the real rules pack.
|
||||
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
classifyHeuristic,
|
||||
detectEnglish,
|
||||
ensureRulesLoaded,
|
||||
} from "@app/services/heuristic/heuristicEngine";
|
||||
import type { HeuristicDoc } from "@app/services/heuristic/types";
|
||||
|
||||
beforeAll(async () => {
|
||||
await ensureRulesLoaded();
|
||||
});
|
||||
|
||||
function classify(title: string, body: string) {
|
||||
const doc: HeuristicDoc = {
|
||||
fileName: "doc.pdf",
|
||||
pageCount: 1,
|
||||
meta: {},
|
||||
titleZone: title,
|
||||
firstZone: body,
|
||||
allZone: body,
|
||||
};
|
||||
return classifyHeuristic(doc);
|
||||
}
|
||||
|
||||
describe("heuristic engine port fidelity", () => {
|
||||
it("classifies an invoice as invoice", () => {
|
||||
const body = [
|
||||
"INVOICE",
|
||||
"Acme Web Services Ltd",
|
||||
"123 High Street, London, EC1A 4JQ",
|
||||
"Invoice Number: INV-2024-0117",
|
||||
"Invoice Date: 14 March 2024",
|
||||
"Due Date: 13 April 2024",
|
||||
"Bill To: Northwind Trading Company",
|
||||
"Description Qty Unit Price Amount",
|
||||
"Website hosting (annual) 1 480.00 480.00",
|
||||
"Subtotal: 930.00",
|
||||
"VAT (20%): 186.00",
|
||||
"Total Due: 1,116.00",
|
||||
"Payment Terms: Net 30. Please quote the invoice number with payment.",
|
||||
].join("\n");
|
||||
const r = classify("INVOICE", body);
|
||||
expect(r.labels.length).toBeGreaterThan(0);
|
||||
expect(r.labels[0]).toBe("invoice");
|
||||
});
|
||||
|
||||
it("classifies a curriculum vitae as resume", () => {
|
||||
const body = [
|
||||
"CURRICULUM VITAE",
|
||||
"Jordan Ellis",
|
||||
"Bristol, UK | jordan.ellis@example.com | 07700 900123",
|
||||
"Professional Summary",
|
||||
"Experienced software engineer with 8 years building web platforms.",
|
||||
"Work Experience",
|
||||
"Senior Engineer, Northwind Ltd (2020-present)",
|
||||
"Education",
|
||||
"BSc Computer Science, University of Bristol, 2016",
|
||||
"Skills",
|
||||
"TypeScript, Java, React, cloud architecture, mentoring",
|
||||
"References available on request.",
|
||||
].join("\n");
|
||||
const r = classify("CURRICULUM VITAE", body);
|
||||
expect(r.labels.length).toBeGreaterThan(0);
|
||||
expect(r.labels[0]).toBe("resume");
|
||||
});
|
||||
|
||||
it("classifies a boarding pass as ticket", () => {
|
||||
const body = [
|
||||
"BOARDING PASS",
|
||||
"British Airways",
|
||||
"Passenger: SMITH/JANE MS",
|
||||
"Flight: BA 117 Date: 22 APR 2024",
|
||||
"From: LONDON HEATHROW (LHR) Terminal 5",
|
||||
"To: NEW YORK JFK (JFK)",
|
||||
"Departure: 11:20 Boarding Time: 10:35 Gate: B44",
|
||||
"Seat: 34K Group: 3 Class: Economy",
|
||||
"Booking Reference: XK9PLQ",
|
||||
"Please be at the gate 45 minutes before departure.",
|
||||
].join("\n");
|
||||
const r = classify("BOARDING PASS", body);
|
||||
expect(r.labels.length).toBeGreaterThan(0);
|
||||
expect(r.labels[0]).toBe("ticket");
|
||||
});
|
||||
|
||||
it("classifies an NDA as nda", () => {
|
||||
const body = [
|
||||
"NON-DISCLOSURE AGREEMENT",
|
||||
"This Mutual Non-Disclosure Agreement (the Agreement) is entered into",
|
||||
"by and between Stirling Systems Ltd and the Receiving Party.",
|
||||
"1. Confidential Information means any proprietary data disclosed by a party.",
|
||||
"2. Obligations: The Receiving Party shall hold all Confidential Information",
|
||||
"in strict confidence and not disclose it to any third party.",
|
||||
"3. Term: The obligations survive for a period of five (5) years.",
|
||||
"4. Governing Law: This Agreement is governed by the laws of England and Wales.",
|
||||
"Accepted and agreed by the authorised representatives of the parties.",
|
||||
].join("\n");
|
||||
const r = classify("NON-DISCLOSURE AGREEMENT", body);
|
||||
expect(r.labels.length).toBeGreaterThan(0);
|
||||
expect(r.labels[0]).toBe("nda");
|
||||
});
|
||||
|
||||
it("does not classify a non-English (Spanish) document", () => {
|
||||
const body = [
|
||||
"CONTRATO DE ARRENDAMIENTO DE VIVIENDA",
|
||||
"Este contrato de arrendamiento se celebra entre el arrendador y el",
|
||||
"arrendatario para la vivienda situada en la ciudad.",
|
||||
"El arrendatario pagara una renta mensual de 1150 euros segun las",
|
||||
"condiciones que las partes acuerdan por el plazo de doce meses.",
|
||||
"Ambas partes firman este documento segun la ley aplicable.",
|
||||
].join("\n");
|
||||
const r = classify("CONTRATO DE ARRENDAMIENTO", body);
|
||||
expect(r.isEnglish).toBe(false);
|
||||
expect(r.labels).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("detects English prose", () => {
|
||||
const english =
|
||||
"This agreement is made between the parties and shall be governed by the laws" +
|
||||
" of England. The tenant agrees to pay the rent that is due under this" +
|
||||
" contract for the property.";
|
||||
expect(detectEnglish(english).isEnglish).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,976 @@
|
||||
// Heuristic (non-AI) document classifier: string/regex/structural scoring over
|
||||
// extracted text, filename and metadata. Rules lazy-load as a separate chunk.
|
||||
|
||||
import type {
|
||||
HeuristicConfidence,
|
||||
HeuristicDoc,
|
||||
HeuristicExplanation,
|
||||
HeuristicResult,
|
||||
} from "@app/services/heuristic/types";
|
||||
|
||||
export type {
|
||||
HeuristicConfidence,
|
||||
HeuristicDoc,
|
||||
HeuristicExplanation,
|
||||
HeuristicResult,
|
||||
};
|
||||
|
||||
// --- scoring constants ---
|
||||
const ZONE_MULT: Record<string, number> = { title: 2.0, first: 1.35, any: 1.0 };
|
||||
const FLOOR = 18;
|
||||
const HIGH_SCORE = 45;
|
||||
const HIGH_MARGIN = 15;
|
||||
const HIGH_SIGNALS = 3;
|
||||
const MED_SCORE = 28;
|
||||
const MED_MARGIN = 8;
|
||||
const SEC_FLOOR = 28;
|
||||
const SEC_FRAC = 0.5;
|
||||
const SEC_SIGNALS = 2;
|
||||
const SEC_MAX = 4;
|
||||
|
||||
const STOPWORDS = new Set<string>([
|
||||
"the",
|
||||
"and",
|
||||
"of",
|
||||
"to",
|
||||
"in",
|
||||
"is",
|
||||
"that",
|
||||
"for",
|
||||
"on",
|
||||
"with",
|
||||
"as",
|
||||
"are",
|
||||
"this",
|
||||
"be",
|
||||
"by",
|
||||
"at",
|
||||
"from",
|
||||
"or",
|
||||
"an",
|
||||
"not",
|
||||
"your",
|
||||
"you",
|
||||
"we",
|
||||
"has",
|
||||
"have",
|
||||
"will",
|
||||
"was",
|
||||
"were",
|
||||
"been",
|
||||
"their",
|
||||
"they",
|
||||
"which",
|
||||
"any",
|
||||
"all",
|
||||
"may",
|
||||
"shall",
|
||||
"if",
|
||||
"can",
|
||||
"our",
|
||||
"its",
|
||||
"it",
|
||||
"no",
|
||||
"but",
|
||||
"other",
|
||||
"than",
|
||||
"these",
|
||||
"such",
|
||||
"must",
|
||||
"each",
|
||||
"per",
|
||||
"under",
|
||||
"more",
|
||||
"when",
|
||||
"also",
|
||||
"into",
|
||||
"only",
|
||||
"should",
|
||||
"would",
|
||||
]);
|
||||
|
||||
// Non-Latin scripts end English classification outright when they dominate.
|
||||
const SCRIPT_RANGES: RegExp[] = [
|
||||
/[一-鿿-ヿ]/g, // CJK + Kana
|
||||
/[가-ᄀ-ᇿ]/g, // Hangul
|
||||
/[Ѐ-ӿ]/g, // Cyrillic
|
||||
/[-ۿݐ-ݿ]/g, // Arabic
|
||||
/[Ͱ-Ϳ]/g, // Greek
|
||||
/[ऀ-ॿ]/g, // Devanagari
|
||||
/[-]/g, // Hebrew
|
||||
/[-]/g, // Thai
|
||||
];
|
||||
|
||||
interface LatinProfile {
|
||||
words: Set<string>;
|
||||
dia: RegExp | null;
|
||||
}
|
||||
|
||||
// Function-word and diacritic profiles for common Latin-script languages.
|
||||
const LATIN_PROFILES: LatinProfile[] = [
|
||||
{
|
||||
words: new Set([
|
||||
"el",
|
||||
"los",
|
||||
"las",
|
||||
"que",
|
||||
"para",
|
||||
"una",
|
||||
"por",
|
||||
"según",
|
||||
"más",
|
||||
]),
|
||||
dia: /[áéíóúñ¿¡]/g,
|
||||
},
|
||||
{
|
||||
words: new Set([
|
||||
"le",
|
||||
"les",
|
||||
"des",
|
||||
"une",
|
||||
"est",
|
||||
"pour",
|
||||
"avec",
|
||||
"dans",
|
||||
"vous",
|
||||
"votre",
|
||||
"être",
|
||||
"nous",
|
||||
"cette",
|
||||
"sont",
|
||||
"été",
|
||||
]),
|
||||
dia: /[àâçèéêëîïôùûœ]/g,
|
||||
},
|
||||
{
|
||||
words: new Set([
|
||||
"der",
|
||||
"die",
|
||||
"das",
|
||||
"und",
|
||||
"ist",
|
||||
"für",
|
||||
"mit",
|
||||
"von",
|
||||
"nicht",
|
||||
"ein",
|
||||
"eine",
|
||||
"werden",
|
||||
"wird",
|
||||
"bei",
|
||||
"sind",
|
||||
"dem",
|
||||
]),
|
||||
dia: /[äöüß]/g,
|
||||
},
|
||||
{
|
||||
words: new Set([
|
||||
"il",
|
||||
"di",
|
||||
"che",
|
||||
"per",
|
||||
"con",
|
||||
"una",
|
||||
"del",
|
||||
"della",
|
||||
"sono",
|
||||
"questo",
|
||||
"essere",
|
||||
"più",
|
||||
"nel",
|
||||
"anche",
|
||||
"gli",
|
||||
]),
|
||||
dia: /[àèéìòù]/g,
|
||||
},
|
||||
{
|
||||
words: new Set([
|
||||
"os",
|
||||
"as",
|
||||
"que",
|
||||
"para",
|
||||
"com",
|
||||
"uma",
|
||||
"por",
|
||||
"são",
|
||||
"não",
|
||||
"você",
|
||||
"está",
|
||||
"mais",
|
||||
]),
|
||||
dia: /[ãõçáéíóúâêô]/g,
|
||||
},
|
||||
{
|
||||
words: new Set([
|
||||
"het",
|
||||
"een",
|
||||
"van",
|
||||
"voor",
|
||||
"met",
|
||||
"aan",
|
||||
"niet",
|
||||
"zijn",
|
||||
"wordt",
|
||||
"deze",
|
||||
"als",
|
||||
"bij",
|
||||
"ook",
|
||||
"naar",
|
||||
]),
|
||||
dia: null,
|
||||
},
|
||||
{
|
||||
words: new Set([
|
||||
"och",
|
||||
"att",
|
||||
"det",
|
||||
"som",
|
||||
"på",
|
||||
"är",
|
||||
"av",
|
||||
"för",
|
||||
"med",
|
||||
"den",
|
||||
"till",
|
||||
"inte",
|
||||
"har",
|
||||
"ett",
|
||||
"du",
|
||||
]),
|
||||
dia: /[åäö]/g,
|
||||
},
|
||||
{
|
||||
words: new Set([
|
||||
"nie",
|
||||
"jest",
|
||||
"się",
|
||||
"że",
|
||||
"oraz",
|
||||
"dla",
|
||||
"przez",
|
||||
"lub",
|
||||
"być",
|
||||
"może",
|
||||
"przy",
|
||||
"jak",
|
||||
]),
|
||||
dia: /[ąćęłńśźż]/g,
|
||||
},
|
||||
{
|
||||
words: new Set([
|
||||
"ve",
|
||||
"bir",
|
||||
"bu",
|
||||
"için",
|
||||
"ile",
|
||||
"olarak",
|
||||
"olan",
|
||||
"gibi",
|
||||
"daha",
|
||||
"çok",
|
||||
"her",
|
||||
"kadar",
|
||||
"sonra",
|
||||
]),
|
||||
dia: /[çğışöü]/g,
|
||||
},
|
||||
];
|
||||
|
||||
// detectEnglish helper patterns (global for counting; \p{L} needs the u flag).
|
||||
const LETTERS = /\p{L}/gu;
|
||||
const LATIN_LETTER = /[a-z]/gi;
|
||||
const WORD = /[\p{L}']+/gu;
|
||||
|
||||
// ASCII whitespace plus the no-break spaces pdf.js extraction commonly emits.
|
||||
// eslint-disable-next-line no-control-regex -- vertical tab is intentional ASCII whitespace
|
||||
const WHITESPACE = /[\t\n\x0B\f\r \u00A0\u2007\u202F]+/g;
|
||||
|
||||
// Structural signal patterns. Boolean-presence ones stay non-global (safe .test()),
|
||||
// counting ones are global (used via countAll). Currency symbols are \u-escaped.
|
||||
const CURRENCY = new RegExp(
|
||||
"[$£€]\\s?\\d[\\d,.]*|\\d[\\d,.]*\\s?(usd|gbp|eur)\\b",
|
||||
"gi",
|
||||
);
|
||||
const NUMERIC_TOKEN = new RegExp("^[\\d$£€.,%-]+$");
|
||||
const DIGIT = /\d/;
|
||||
const FORM_LABEL = /^[A-Za-z][A-Za-z /()&']{2,30}:\s*$/;
|
||||
const UNDERSCORE4 = /_{4,}/;
|
||||
const CHECKBOX = /[☐☑□■]\s/;
|
||||
const DOT_LEADER = /\.{5,}\s*\d+\s*$/;
|
||||
const BULLET = /^[•▪◦*-]\s+\S/;
|
||||
const URL = /https?:\/\/|www\./gi;
|
||||
const TOC = /table of contents/i;
|
||||
const SIG1 = /\b(signature|signed by|authorized signature|\/s\/)\b/i;
|
||||
const SIG2 = /_{6,}\s*\n\s*(date|name|sign)/i;
|
||||
const REF1 = /\b(references|bibliography)\b/i;
|
||||
const REF2 = /\[\d{1,3}\]|\(\d{4}\)/;
|
||||
const EMAIL_FROM = /\bfrom:\s.+\n(.*\n){0,3}?\s*(to|sent|date):\s/i;
|
||||
const EMAIL_SUBJ = /subject:\s/i;
|
||||
const ADDRESS = /\b\d{5}(-\d{4})?\b|\b[A-Z]{1,2}\d{1,2}[A-Z]?\s?\d[A-Z]{2}\b/g;
|
||||
|
||||
// --- prepared rule model ---
|
||||
interface Phrase {
|
||||
text: string;
|
||||
weight: number;
|
||||
where: string;
|
||||
}
|
||||
interface Rx {
|
||||
re: RegExp;
|
||||
weight: number;
|
||||
where: string;
|
||||
}
|
||||
interface FileRx {
|
||||
re: RegExp;
|
||||
weight: number;
|
||||
}
|
||||
interface MetaRx {
|
||||
field: string;
|
||||
re: RegExp;
|
||||
weight: number;
|
||||
}
|
||||
interface Negative {
|
||||
text: string | null;
|
||||
re: RegExp | null;
|
||||
weight: number;
|
||||
}
|
||||
interface Structural {
|
||||
signal: string;
|
||||
weight: number;
|
||||
}
|
||||
interface PreparedLabel {
|
||||
id: string;
|
||||
emit: boolean;
|
||||
phrases: Phrase[];
|
||||
regexes: Rx[];
|
||||
filenames: FileRx[];
|
||||
metadata: MetaRx[];
|
||||
negatives: Negative[];
|
||||
structural: Structural[];
|
||||
}
|
||||
interface Prior {
|
||||
min: number;
|
||||
max: number | null;
|
||||
}
|
||||
|
||||
// Raw JSON shapes (loose - the pack is authored by hand).
|
||||
interface RawRule {
|
||||
text?: unknown;
|
||||
pattern?: unknown;
|
||||
weight?: unknown;
|
||||
where?: unknown;
|
||||
flags?: unknown;
|
||||
field?: unknown;
|
||||
signal?: unknown;
|
||||
}
|
||||
interface RawLabel {
|
||||
id?: unknown;
|
||||
emit?: unknown;
|
||||
phrases?: RawRule[];
|
||||
regexes?: RawRule[];
|
||||
filenames?: RawRule[];
|
||||
metadata?: RawRule[];
|
||||
negatives?: RawRule[];
|
||||
structural?: RawRule[];
|
||||
}
|
||||
interface RulesFile {
|
||||
labels?: RawLabel[];
|
||||
priors?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
let PREPARED: PreparedLabel[] | null = null;
|
||||
let PRIORS: Map<string, Prior> | null = null;
|
||||
let loadPromise: Promise<void> | null = null;
|
||||
|
||||
/** Load and prepare the rules pack once. Must resolve before classifyHeuristic. */
|
||||
export async function ensureRulesLoaded(): Promise<void> {
|
||||
if (PREPARED && PRIORS) return;
|
||||
if (!loadPromise) {
|
||||
loadPromise = import("@app/services/heuristic/heuristicRules.json").then(
|
||||
(mod) => {
|
||||
const root = ((mod as { default?: RulesFile }).default ??
|
||||
(mod as RulesFile)) as RulesFile;
|
||||
PREPARED = prepare(root.labels ?? []);
|
||||
PRIORS = loadPriors(root.priors ?? {});
|
||||
},
|
||||
(err) => {
|
||||
// A failed chunk load (flaky network) must not poison later attempts.
|
||||
loadPromise = null;
|
||||
throw err;
|
||||
},
|
||||
);
|
||||
}
|
||||
await loadPromise;
|
||||
}
|
||||
|
||||
// --- Preparation ---
|
||||
|
||||
function prepare(labels: RawLabel[]): PreparedLabel[] {
|
||||
const out: PreparedLabel[] = [];
|
||||
for (const label of labels) {
|
||||
const id = typeof label.id === "string" ? label.id : "";
|
||||
const emit = typeof label.emit !== "boolean" ? true : label.emit;
|
||||
|
||||
const phrases: Phrase[] = [];
|
||||
for (const p of label.phrases ?? []) {
|
||||
const text = typeof p.text === "string" ? p.text : "";
|
||||
const w = num(p.weight);
|
||||
if (text.length === 0 || w <= 0) continue;
|
||||
phrases.push({
|
||||
text: normalize(text),
|
||||
weight: Math.min(w, 40),
|
||||
where: where(p),
|
||||
});
|
||||
}
|
||||
|
||||
const regexes: Rx[] = [];
|
||||
for (const r of label.regexes ?? []) {
|
||||
const re = compileRegex(str(r.pattern), flags(r));
|
||||
if (re == null) continue;
|
||||
regexes.push({
|
||||
re,
|
||||
weight: Math.min(num(r.weight), 30),
|
||||
where: where(r),
|
||||
});
|
||||
}
|
||||
|
||||
const filenames: FileRx[] = [];
|
||||
for (const r of label.filenames ?? []) {
|
||||
const re = compileRegex(str(r.pattern), flags(r));
|
||||
if (re == null) continue;
|
||||
filenames.push({ re, weight: Math.min(num(r.weight), 30) });
|
||||
}
|
||||
|
||||
const metadata: MetaRx[] = [];
|
||||
for (const r of label.metadata ?? []) {
|
||||
const re = compileRegex(str(r.pattern), flags(r));
|
||||
if (re == null) continue;
|
||||
const field = typeof r.field === "string" && r.field ? r.field : "any";
|
||||
metadata.push({ field, re, weight: Math.min(num(r.weight), 20) });
|
||||
}
|
||||
|
||||
const negatives: Negative[] = [];
|
||||
for (const n of label.negatives ?? []) {
|
||||
const text = n.text != null ? normalize(String(n.text)) : null;
|
||||
const re =
|
||||
n.pattern != null ? compileRegex(String(n.pattern), flags(n)) : null;
|
||||
if (text == null && re == null) continue;
|
||||
negatives.push({
|
||||
text,
|
||||
re,
|
||||
weight: Math.min(Math.abs(num(n.weight)), 30),
|
||||
});
|
||||
}
|
||||
|
||||
const structural: Structural[] = [];
|
||||
for (const s of label.structural ?? []) {
|
||||
const signal = typeof s.signal === "string" ? s.signal : "";
|
||||
const w = num(s.weight);
|
||||
if (signal.length === 0 || w <= 0) continue;
|
||||
structural.push({ signal, weight: Math.min(w, 12) });
|
||||
}
|
||||
|
||||
out.push({
|
||||
id,
|
||||
emit,
|
||||
phrases,
|
||||
regexes,
|
||||
filenames,
|
||||
metadata,
|
||||
negatives,
|
||||
structural,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function loadPriors(priorsNode: Record<string, unknown>): Map<string, Prior> {
|
||||
const out = new Map<string, Prior>();
|
||||
for (const [key, val] of Object.entries(priorsNode)) {
|
||||
if (!Array.isArray(val) || val.length === 0) continue;
|
||||
const min = Math.trunc(num(val[0]));
|
||||
const max =
|
||||
val.length > 1 && val[1] != null ? Math.trunc(num(val[1])) : null;
|
||||
out.set(key, { min, max });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function where(node: RawRule): string {
|
||||
const w = typeof node.where === "string" ? node.where : "";
|
||||
return w.length === 0 ? "any" : w;
|
||||
}
|
||||
|
||||
function flags(node: RawRule): string {
|
||||
return typeof node.flags === "string" ? node.flags : "";
|
||||
}
|
||||
|
||||
/** Compile a rule regex to a RegExp, or null when it won't compile. */
|
||||
export function compileRegex(
|
||||
pattern: string | null,
|
||||
flagStr: string,
|
||||
): RegExp | null {
|
||||
if (pattern == null) return null;
|
||||
try {
|
||||
const fl = flagStr.length === 0 ? "gi" : flagStr;
|
||||
let f = "g"; // always global for iterative counting
|
||||
if (fl.indexOf("i") >= 0) f += "i";
|
||||
if (fl.indexOf("m") >= 0) f += "m";
|
||||
if (fl.indexOf("s") >= 0) f += "s";
|
||||
return new RegExp(pattern, f);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Public API ---
|
||||
|
||||
interface ScoredLabel {
|
||||
label: PreparedLabel;
|
||||
score: number;
|
||||
distinct: number;
|
||||
/** Rule-hit descriptions, collected only when explain is requested. */
|
||||
signals: string[] | null;
|
||||
}
|
||||
|
||||
/** Max candidates and per-candidate signals included in an explanation. */
|
||||
const EXPLAIN_CANDIDATES = 6;
|
||||
const EXPLAIN_SIGNALS = 12;
|
||||
|
||||
const fmt = (n: number) => Math.round(n * 10) / 10;
|
||||
|
||||
function toExplanation(
|
||||
en: { isEnglish: boolean; lowText: boolean },
|
||||
scored: ScoredLabel[],
|
||||
): HeuristicExplanation {
|
||||
return {
|
||||
isEnglish: en.isEnglish,
|
||||
lowText: en.lowText,
|
||||
candidates: scored.slice(0, EXPLAIN_CANDIDATES).map((s) => ({
|
||||
id: s.label.id,
|
||||
emit: s.label.emit,
|
||||
score: fmt(s.score),
|
||||
distinct: s.distinct,
|
||||
signals: (s.signals ?? []).slice(0, EXPLAIN_SIGNALS),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
/** Classify a document; returns emitted label ids (primary + secondaries, capped at 5). */
|
||||
export function classifyHeuristic(
|
||||
doc: HeuristicDoc,
|
||||
opts?: { explain?: boolean },
|
||||
): HeuristicResult {
|
||||
if (!PREPARED || !PRIORS) {
|
||||
throw new Error(
|
||||
"Heuristic rules not loaded; await ensureRulesLoaded() before classifyHeuristic().",
|
||||
);
|
||||
}
|
||||
const explain = opts?.explain === true;
|
||||
|
||||
const en = detectEnglish(doc.allZone);
|
||||
// Non-English with real text: honestly out of scope for the English heuristics.
|
||||
if (!en.isEnglish && !en.lowText) {
|
||||
return {
|
||||
labels: [],
|
||||
confidence: "none",
|
||||
score: 0,
|
||||
isEnglish: false,
|
||||
...(explain ? { explain: toExplanation(en, []) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
const titleRaw = nz(doc.titleZone);
|
||||
const firstRaw = nz(doc.firstZone);
|
||||
const anyRaw = nz(doc.allZone);
|
||||
const titleNorm = normalize(titleRaw);
|
||||
const firstNorm = normalize(firstRaw);
|
||||
const anyNorm = normalize(anyRaw);
|
||||
const fileNameLower = nz(doc.fileName).toLowerCase();
|
||||
const meta = doc.meta ?? {};
|
||||
const metaAll = Object.values(meta).join(" \n ");
|
||||
const struct = computeStructural(doc);
|
||||
|
||||
const scored: ScoredLabel[] = [];
|
||||
for (const label of PREPARED) {
|
||||
let score = 0;
|
||||
let distinct = 0;
|
||||
const sig: string[] | null = explain ? [] : null;
|
||||
|
||||
for (const phrase of label.phrases) {
|
||||
let best = 0;
|
||||
let bestZone = "";
|
||||
for (const zone of ["title", "first", "any"] as const) {
|
||||
const hay =
|
||||
zone === "title" ? titleNorm : zone === "first" ? firstNorm : anyNorm;
|
||||
const count = countOccurrences(hay, phrase.text);
|
||||
if (count === 0) continue;
|
||||
const zf = phrase.where === "any" || phrase.where === zone ? 1 : 0.75;
|
||||
const value = phrase.weight * ZONE_MULT[zone] * zf * damp(count);
|
||||
if (value > best) {
|
||||
best = value;
|
||||
bestZone = zone;
|
||||
}
|
||||
}
|
||||
if (best > 0) {
|
||||
score += best;
|
||||
distinct++;
|
||||
sig?.push(`phrase "${phrase.text}" +${fmt(best)} (${bestZone})`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const rx of label.regexes) {
|
||||
let best = 0;
|
||||
let bestZone = "";
|
||||
for (const zone of ["title", "first", "any"] as const) {
|
||||
const hay =
|
||||
zone === "title" ? titleRaw : zone === "first" ? firstRaw : anyRaw;
|
||||
const count = countRegex(rx.re, hay);
|
||||
if (count === 0) continue;
|
||||
const zf = rx.where === "any" || rx.where === zone ? 1 : 0.75;
|
||||
const value = rx.weight * ZONE_MULT[zone] * zf * damp(count);
|
||||
if (value > best) {
|
||||
best = value;
|
||||
bestZone = zone;
|
||||
}
|
||||
}
|
||||
if (best > 0) {
|
||||
score += best;
|
||||
distinct++;
|
||||
sig?.push(`regex ${rx.re.source} +${fmt(best)} (${bestZone})`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const fn of label.filenames) {
|
||||
if (countRegex(fn.re, fileNameLower) > 0) {
|
||||
score += fn.weight;
|
||||
distinct++;
|
||||
sig?.push(`filename ${fn.re.source} +${fn.weight}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const md of label.metadata) {
|
||||
const value = md.field === "any" ? metaAll : (meta[md.field] ?? "");
|
||||
if (countRegex(md.re, value) > 0) {
|
||||
score += md.weight;
|
||||
distinct++;
|
||||
sig?.push(`metadata(${md.field}) ${md.re.source} +${md.weight}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const st of label.structural) {
|
||||
const value = struct[st.signal] ?? 0;
|
||||
if (value > 0) {
|
||||
score += st.weight * value;
|
||||
sig?.push(`structural ${st.signal} +${fmt(st.weight * value)}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const neg of label.negatives) {
|
||||
const count =
|
||||
neg.text != null
|
||||
? countOccurrences(anyNorm, neg.text)
|
||||
: countRegex(neg.re, anyRaw);
|
||||
if (count > 0) {
|
||||
const value = neg.weight * damp(Math.min(count, 3));
|
||||
score -= value;
|
||||
sig?.push(
|
||||
`negative ${neg.text != null ? `"${neg.text}"` : (neg.re?.source ?? "")} -${fmt(value)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (score > 0) {
|
||||
const prior = pagePriorMultiplier(label.id, doc.pageCount);
|
||||
if (prior !== 1) sig?.push(`page-prior x${fmt(prior)}`);
|
||||
score *= prior;
|
||||
scored.push({ label, score, distinct, signals: sig });
|
||||
}
|
||||
}
|
||||
|
||||
// Stable sort by score descending.
|
||||
scored.sort((a, b) => b.score - a.score);
|
||||
|
||||
const top = scored.length === 0 ? null : scored[0];
|
||||
const s1 = top != null ? top.score : 0;
|
||||
const s2 = scored.length > 1 ? scored[1].score : 0;
|
||||
const margin = s1 - s2;
|
||||
|
||||
let confidence: HeuristicConfidence = "none";
|
||||
if (top != null && s1 >= FLOOR) {
|
||||
if (
|
||||
s1 >= HIGH_SCORE &&
|
||||
margin >= HIGH_MARGIN &&
|
||||
top.distinct >= HIGH_SIGNALS &&
|
||||
s2 <= s1 * 0.65
|
||||
) {
|
||||
confidence = "high";
|
||||
} else if (s1 >= MED_SCORE && margin >= MED_MARGIN) {
|
||||
confidence = "medium";
|
||||
} else {
|
||||
confidence = "low";
|
||||
}
|
||||
}
|
||||
|
||||
const roundedScore = Math.round(s1);
|
||||
const explanation = explain ? { explain: toExplanation(en, scored) } : {};
|
||||
if (top == null || confidence === "none") {
|
||||
return {
|
||||
labels: [],
|
||||
confidence: "none",
|
||||
score: roundedScore,
|
||||
isEnglish: en.isEnglish,
|
||||
...explanation,
|
||||
};
|
||||
}
|
||||
// Internal-only winner (book, menu...): suppress output rather than mislabel.
|
||||
if (!top.label.emit) {
|
||||
return {
|
||||
labels: [],
|
||||
confidence,
|
||||
score: roundedScore,
|
||||
isEnglish: en.isEnglish,
|
||||
...explanation,
|
||||
};
|
||||
}
|
||||
|
||||
const labels: string[] = [top.label.id];
|
||||
for (let i = 1; i < scored.length && labels.length < 5; i++) {
|
||||
const s = scored[i];
|
||||
if (labels.length - 1 >= SEC_MAX) break;
|
||||
if (
|
||||
s.label.emit &&
|
||||
s.score >= SEC_FLOOR &&
|
||||
s.score >= s1 * SEC_FRAC &&
|
||||
s.distinct >= SEC_SIGNALS
|
||||
) {
|
||||
labels.push(s.label.id);
|
||||
}
|
||||
}
|
||||
return {
|
||||
labels,
|
||||
confidence,
|
||||
score: roundedScore,
|
||||
isEnglish: en.isEnglish,
|
||||
...explanation,
|
||||
};
|
||||
}
|
||||
|
||||
/** True when the top match cleared the high-confidence bar. */
|
||||
export function isHighConfidence(r: HeuristicResult): boolean {
|
||||
return r.confidence === "high";
|
||||
}
|
||||
|
||||
/** High confidence AND an emitted label - trustworthy enough to skip the AI engine. */
|
||||
export function isDefinitive(r: HeuristicResult): boolean {
|
||||
return isHighConfidence(r) && r.labels.length > 0;
|
||||
}
|
||||
|
||||
// --- English detection ---
|
||||
|
||||
interface EnglishResult {
|
||||
isEnglish: boolean;
|
||||
lowText: boolean;
|
||||
}
|
||||
|
||||
export function detectEnglish(text: string): EnglishResult {
|
||||
const raw = nz(text);
|
||||
const letters = countAll(LETTERS, raw);
|
||||
if (letters < 25) return { isEnglish: false, lowText: true };
|
||||
|
||||
for (const re of SCRIPT_RANGES) {
|
||||
const hits = countAll(re, raw);
|
||||
if (hits / letters > 0.25) return { isEnglish: false, lowText: false };
|
||||
}
|
||||
|
||||
const latinRatio = countAll(LATIN_LETTER, raw) / letters;
|
||||
const words = allMatches(WORD, normalize(raw));
|
||||
const totalWords = Math.max(words.length, 1);
|
||||
let enHits = 0;
|
||||
for (const w of words) if (STOPWORDS.has(w)) enHits++;
|
||||
const stopRatio = enHits / totalWords;
|
||||
|
||||
let bestScore = 0;
|
||||
let bestRatio = 0;
|
||||
let bestDistinct = 0;
|
||||
let bestDia = 0;
|
||||
for (const profile of LATIN_PROFILES) {
|
||||
let hits = 0;
|
||||
const distinct = new Set<string>();
|
||||
for (const w of words) {
|
||||
if (profile.words.has(w)) {
|
||||
hits++;
|
||||
distinct.add(w);
|
||||
}
|
||||
}
|
||||
const diaCount = profile.dia == null ? 0 : countAll(profile.dia, raw);
|
||||
const ratio = hits / totalWords;
|
||||
const score = ratio + Math.min(diaCount / totalWords, 0.15) * 6;
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
bestRatio = ratio;
|
||||
bestDistinct = distinct.size;
|
||||
bestDia = diaCount;
|
||||
}
|
||||
}
|
||||
|
||||
const lowText = totalWords < 30;
|
||||
const nonEnglish =
|
||||
latinRatio >= 0.7 &&
|
||||
totalWords >= 12 &&
|
||||
(bestDistinct >= 3 || bestDia >= 6) &&
|
||||
(bestDia >= 3 || bestRatio >= 0.1) &&
|
||||
bestScore > stopRatio * 1.2 &&
|
||||
(stopRatio < 0.04 || bestRatio > stopRatio * 1.5);
|
||||
if (nonEnglish) return { isEnglish: false, lowText };
|
||||
|
||||
const bar = lowText ? 0.03 : 0.045;
|
||||
// Data-dense docs (tickets, itineraries, prescriptions) are mostly names and numbers with few
|
||||
// function words in ANY language; reject stop-poor text only on affirmative foreign evidence.
|
||||
const foreignEvidence = bestDistinct >= 3 || bestDia >= 6;
|
||||
return {
|
||||
isEnglish: latinRatio >= 0.75 && (stopRatio >= bar || !foreignEvidence),
|
||||
lowText,
|
||||
};
|
||||
}
|
||||
|
||||
// --- Structural signals ---
|
||||
|
||||
function computeStructural(doc: HeuristicDoc): Record<string, number> {
|
||||
const all = nz(doc.allZone);
|
||||
const lines: string[] = [];
|
||||
for (const l of all.split("\n")) {
|
||||
const t = l.trim();
|
||||
if (t.length > 0) lines.push(t);
|
||||
}
|
||||
const tokens: string[] = [];
|
||||
for (const t of all.split(WHITESPACE)) {
|
||||
if (t.length > 0) tokens.push(t);
|
||||
}
|
||||
const totalTokens = Math.max(tokens.length, 1);
|
||||
|
||||
const currency = countAll(CURRENCY, all);
|
||||
let numericTokens = 0;
|
||||
for (const t of tokens) {
|
||||
if (NUMERIC_TOKEN.test(t) && DIGIT.test(t)) numericTokens++;
|
||||
}
|
||||
let formLines = 0;
|
||||
for (const l of lines) {
|
||||
if (FORM_LABEL.test(l) || UNDERSCORE4.test(l) || CHECKBOX.test(l))
|
||||
formLines++;
|
||||
}
|
||||
let dotLeaders = 0;
|
||||
for (const l of lines) if (DOT_LEADER.test(l)) dotLeaders++;
|
||||
let bullets = 0;
|
||||
for (const l of lines) if (BULLET.test(l)) bullets++;
|
||||
const urls = countAll(URL, all);
|
||||
const tail = all.length > 2500 ? all.slice(all.length - 2500) : all;
|
||||
const last4000 = all.length > 4000 ? all.slice(all.length - 4000) : all;
|
||||
|
||||
const s: Record<string, number> = {};
|
||||
s["currency_heavy"] = currency >= 8 ? 1.0 : Math.min(currency / 8.0, 1.0);
|
||||
s["number_table"] = numericTokens / totalTokens >= 0.22 ? 1.0 : 0.0;
|
||||
s["form_like"] = formLines >= 6 ? 1.0 : formLines >= 3 ? 0.5 : 0.0;
|
||||
s["toc"] = TOC.test(all) || dotLeaders >= 5 ? 1.0 : 0.0;
|
||||
s["signature_block"] = SIG1.test(tail) || SIG2.test(tail) ? 1.0 : 0.0;
|
||||
s["references_section"] =
|
||||
REF1.test(last4000) && REF2.test(last4000) ? 1.0 : 0.0;
|
||||
s["short_doc"] = doc.pageCount > 0 && doc.pageCount <= 2 ? 1.0 : 0.0;
|
||||
s["long_doc"] = doc.pageCount >= 40 ? 1.0 : 0.0;
|
||||
s["bullet_heavy"] = bullets >= 12 ? 1.0 : bullets >= 6 ? 0.5 : 0.0;
|
||||
s["email_headers"] = EMAIL_FROM.test(all) && EMAIL_SUBJ.test(all) ? 1.0 : 0.0;
|
||||
s["url_heavy"] = urls >= 6 ? 1.0 : 0.0;
|
||||
s["address_block"] = countAll(ADDRESS, all) >= 2 ? 1.0 : 0.0;
|
||||
return s;
|
||||
}
|
||||
|
||||
function pagePriorMultiplier(labelId: string, pageCount: number): number {
|
||||
const prior = PRIORS!.get(labelId);
|
||||
if (prior == null || pageCount < 1) return 1;
|
||||
if (prior.max != null && pageCount > prior.max) {
|
||||
return Math.max(0.3, prior.max / pageCount);
|
||||
}
|
||||
if (pageCount < prior.min) return Math.max(0.3, pageCount / prior.min);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
function nz(s: string | null | undefined): string {
|
||||
return s == null ? "" : s;
|
||||
}
|
||||
|
||||
function num(v: unknown): number {
|
||||
return typeof v === "number" && Number.isFinite(v) ? v : 0;
|
||||
}
|
||||
|
||||
function str(v: unknown): string | null {
|
||||
return typeof v === "string" ? v : null;
|
||||
}
|
||||
|
||||
// Curly apostrophes and fi/fl ligatures survive pdf.js extraction in many PDFs;
|
||||
// fold them to ASCII so rule phrases authored with ' / fi / fl still match.
|
||||
const CURLY_APOSTROPHE = /[\u2018\u2019]/g;
|
||||
const LIGATURE_FI = /\uFB01/g;
|
||||
const LIGATURE_FL = /\uFB02/g;
|
||||
|
||||
function normalize(text: string | null | undefined): string {
|
||||
return nz(text)
|
||||
.toLowerCase()
|
||||
.replace(CURLY_APOSTROPHE, "'")
|
||||
.replace(LIGATURE_FI, "fi")
|
||||
.replace(LIGATURE_FL, "fl")
|
||||
.replace(WHITESPACE, " ");
|
||||
}
|
||||
|
||||
function damp(count: number): number {
|
||||
if (count <= 0) return 0;
|
||||
return 1 + 0.35 * (Math.log(Math.min(count, 12)) / Math.log(2));
|
||||
}
|
||||
|
||||
function countOccurrences(haystack: string, needle: string | null): number {
|
||||
if (needle == null || needle.length === 0) return 0;
|
||||
let count = 0;
|
||||
let idx = haystack.indexOf(needle);
|
||||
while (idx !== -1 && count < 12) {
|
||||
count++;
|
||||
idx = haystack.indexOf(needle, idx + needle.length);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// Non-overlapping matches capped at 12.
|
||||
function countRegex(re: RegExp | null, text: string | null): number {
|
||||
if (re == null || text == null || text.length === 0) return 0;
|
||||
re.lastIndex = 0;
|
||||
let count = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
while (count < 12 && (m = re.exec(text)) !== null) {
|
||||
count++;
|
||||
if (m.index === re.lastIndex) re.lastIndex++; // advance past zero-width match
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function countAll(re: RegExp, text: string | null): number {
|
||||
if (text == null || text.length === 0) return 0;
|
||||
re.lastIndex = 0;
|
||||
let count = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text)) !== null) {
|
||||
count++;
|
||||
if (m.index === re.lastIndex) re.lastIndex++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function allMatches(re: RegExp, text: string | null): string[] {
|
||||
const out: string[] = [];
|
||||
if (text == null || text.length === 0) return out;
|
||||
re.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text)) !== null) {
|
||||
out.push(m[0]);
|
||||
if (m.index === re.lastIndex) re.lastIndex++;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
// pdf.js extraction feeding the engine: page-1 text, a first-5 + last-2 page
|
||||
// window, Info-dict metadata, and a large-font page-1 "title" zone.
|
||||
|
||||
import { pdfWorkerManager } from "@app/services/pdfWorkerManager";
|
||||
import type {
|
||||
PDFDocumentProxy,
|
||||
TextItem,
|
||||
} from "pdfjs-dist/types/src/display/api";
|
||||
import type { HeuristicDoc } from "@app/services/heuristic/types";
|
||||
|
||||
const WINDOW_FIRST = 5;
|
||||
const WINDOW_LAST = 2;
|
||||
const PAGE_CHAR_CAP = 8000;
|
||||
const TITLE_CAP = 400;
|
||||
|
||||
/** One rebuilt text line: baseline y (bottom-origin), its largest font size, and the text. */
|
||||
interface Line {
|
||||
text: string;
|
||||
size: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
/** Build the engine's input document from a PDF blob. Throws if the PDF can't be read. */
|
||||
export async function extractHeuristicDoc(
|
||||
file: Blob,
|
||||
fileName: string,
|
||||
): Promise<HeuristicDoc> {
|
||||
const arrayBuffer = await file.arrayBuffer();
|
||||
let pdfDoc: PDFDocumentProxy | null = null;
|
||||
try {
|
||||
pdfDoc = await pdfWorkerManager.createDocument(arrayBuffer, {
|
||||
disableAutoFetch: true,
|
||||
disableStream: true,
|
||||
});
|
||||
const pageCount = pdfDoc.numPages;
|
||||
let firstZone = "";
|
||||
let titleZone = "";
|
||||
if (pageCount >= 1) {
|
||||
// Page 1 feeds three zones (first, title, window); pump its items once.
|
||||
const page1 = await pdfDoc.getPage(1);
|
||||
const items = await pageTextItems(page1);
|
||||
firstZone = textFromItems(items);
|
||||
titleZone = titleFromLines(
|
||||
buildLines(items),
|
||||
page1.getViewport({ scale: 1 }).height,
|
||||
);
|
||||
}
|
||||
const parts: string[] = [];
|
||||
for (const pageNo of windowPages(pageCount)) {
|
||||
const text = pageNo === 1 ? firstZone : await pageText(pdfDoc, pageNo);
|
||||
if (text.length > 0) parts.push(text);
|
||||
}
|
||||
const meta = await metadata(pdfDoc);
|
||||
return {
|
||||
fileName,
|
||||
pageCount,
|
||||
meta,
|
||||
titleZone,
|
||||
firstZone,
|
||||
allZone: parts.join("\n"),
|
||||
};
|
||||
} finally {
|
||||
if (pdfDoc) {
|
||||
try {
|
||||
pdfWorkerManager.destroyDocument(pdfDoc);
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** First WINDOW_FIRST + last WINDOW_LAST page numbers, deduped, in order. */
|
||||
function windowPages(pageCount: number): number[] {
|
||||
const pages = new Set<number>();
|
||||
for (let p = 1; p <= Math.min(WINDOW_FIRST, pageCount); p++) pages.add(p);
|
||||
for (let p = Math.max(1, pageCount - WINDOW_LAST + 1); p <= pageCount; p++) {
|
||||
pages.add(p);
|
||||
}
|
||||
return [...pages].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
function isTextItem(item: unknown): item is TextItem {
|
||||
return typeof (item as TextItem).str === "string";
|
||||
}
|
||||
|
||||
/**
|
||||
* Pump text items with a plain reader loop: Safari/WebKit cannot async-iterate
|
||||
* the ReadableStream behind pdf.js getTextContent.
|
||||
*/
|
||||
async function pageTextItems(
|
||||
page: Awaited<ReturnType<PDFDocumentProxy["getPage"]>>,
|
||||
): Promise<unknown[]> {
|
||||
const reader = page.streamTextContent().getReader();
|
||||
const items: unknown[] = [];
|
||||
for (;;) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
if (Array.isArray(value?.items)) items.push(...value.items);
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
/** A page's text (items joined, newline on hasEOL), trimmed and capped. */
|
||||
async function pageText(
|
||||
pdfDoc: PDFDocumentProxy,
|
||||
pageNo: number,
|
||||
): Promise<string> {
|
||||
if (pageNo < 1 || pageNo > pdfDoc.numPages) return "";
|
||||
const page = await pdfDoc.getPage(pageNo);
|
||||
return textFromItems(await pageTextItems(page));
|
||||
}
|
||||
|
||||
function textFromItems(items: readonly unknown[]): string {
|
||||
let text = "";
|
||||
for (const item of items) {
|
||||
if (!isTextItem(item)) continue;
|
||||
text += item.str;
|
||||
text += item.hasEOL ? "\n" : " ";
|
||||
}
|
||||
const trimmed = text.trim();
|
||||
return trimmed.length > PAGE_CHAR_CAP
|
||||
? trimmed.slice(0, PAGE_CHAR_CAP)
|
||||
: trimmed;
|
||||
}
|
||||
|
||||
/** Group items into lines (break on hasEOL), tracking each line's max font size + baseline y. */
|
||||
function buildLines(items: readonly unknown[]): Line[] {
|
||||
const lines: Line[] = [];
|
||||
let current = "";
|
||||
let size = 0;
|
||||
let y = -1;
|
||||
const flush = () => {
|
||||
const text = current.trim();
|
||||
if (text.length > 0) lines.push({ text, size, y });
|
||||
current = "";
|
||||
size = 0;
|
||||
y = -1;
|
||||
};
|
||||
for (const item of items) {
|
||||
if (!isTextItem(item)) continue;
|
||||
const itemSize = Math.hypot(item.transform[0], item.transform[1]);
|
||||
if (itemSize > size) size = itemSize;
|
||||
if (y < 0) y = item.transform[5];
|
||||
current += item.str;
|
||||
if (item.hasEOL) flush();
|
||||
}
|
||||
flush();
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** Large-font lines near the top of page 1 approximate the title. */
|
||||
function titleFromLines(lines: Line[], pageHeight: number): string {
|
||||
if (lines.length === 0) return "";
|
||||
// pdf.js y is bottom-origin: the top 45% of the page is y > 0.55 * height.
|
||||
const top = lines.filter((l) => l.y > pageHeight * 0.55);
|
||||
const pool = top.length > 0 ? top : lines.slice(0, Math.min(8, lines.length));
|
||||
let maxSize = 0;
|
||||
for (const l of pool) maxSize = Math.max(maxSize, l.size);
|
||||
|
||||
const parts: string[] = [];
|
||||
if (maxSize === 0) {
|
||||
for (let i = 0; i < Math.min(3, pool.length); i++) parts.push(pool[i].text);
|
||||
return parts.join("\n");
|
||||
}
|
||||
let taken = 0;
|
||||
for (const l of pool) {
|
||||
if (taken >= 6) break;
|
||||
if (l.size >= maxSize * 0.72) {
|
||||
parts.push(l.text);
|
||||
taken++;
|
||||
}
|
||||
}
|
||||
const result = parts.join("\n");
|
||||
return result.length > TITLE_CAP ? result.slice(0, TITLE_CAP) : result;
|
||||
}
|
||||
|
||||
/** Info-dict fields keyed lowercase to match the engine's metadata rules. */
|
||||
async function metadata(
|
||||
pdfDoc: PDFDocumentProxy,
|
||||
): Promise<Record<string, string>> {
|
||||
let info: Record<string, unknown> = {};
|
||||
try {
|
||||
const md = await pdfDoc.getMetadata();
|
||||
info = (md.info ?? {}) as Record<string, unknown>;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
const get = (k: string) =>
|
||||
typeof info[k] === "string" ? (info[k] as string) : "";
|
||||
return {
|
||||
title: get("Title"),
|
||||
author: get("Author"),
|
||||
subject: get("Subject"),
|
||||
keywords: get("Keywords"),
|
||||
creator: get("Creator"),
|
||||
producer: get("Producer"),
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,173 @@
|
||||
// Structural lint for the rules pack: the engine silently drops or clamps
|
||||
// malformed rules, so authoring mistakes must fail here instead.
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { compileRegex } from "@app/services/heuristic/heuristicEngine";
|
||||
import rules from "@app/services/heuristic/heuristicRules.json";
|
||||
|
||||
interface RawRule {
|
||||
text?: string;
|
||||
pattern?: string;
|
||||
weight?: number;
|
||||
where?: string;
|
||||
flags?: string;
|
||||
field?: string;
|
||||
signal?: string;
|
||||
}
|
||||
interface RawLabel {
|
||||
id: string;
|
||||
emit?: boolean;
|
||||
phrases?: RawRule[];
|
||||
regexes?: RawRule[];
|
||||
filenames?: RawRule[];
|
||||
metadata?: RawRule[];
|
||||
negatives?: RawRule[];
|
||||
structural?: RawRule[];
|
||||
}
|
||||
const labels = (rules as { labels: RawLabel[] }).labels;
|
||||
const priors = (rules as { priors: Record<string, unknown> }).priors;
|
||||
|
||||
// Mirrors computeStructural's emitted keys; extend together with the engine.
|
||||
const SIGNALS = new Set([
|
||||
"currency_heavy",
|
||||
"number_table",
|
||||
"form_like",
|
||||
"toc",
|
||||
"signature_block",
|
||||
"references_section",
|
||||
"short_doc",
|
||||
"long_doc",
|
||||
"bullet_heavy",
|
||||
"email_headers",
|
||||
"url_heavy",
|
||||
"address_block",
|
||||
]);
|
||||
const ZONES = new Set(["title", "first", "any"]);
|
||||
const META_FIELDS = new Set([
|
||||
"title",
|
||||
"author",
|
||||
"subject",
|
||||
"keywords",
|
||||
"creator",
|
||||
"producer",
|
||||
"any",
|
||||
]);
|
||||
// The engine clamps at these; authoring past them is a hidden no-op, so fail instead.
|
||||
const WEIGHT_CAPS = {
|
||||
phrases: 40,
|
||||
regexes: 30,
|
||||
filenames: 30,
|
||||
metadata: 20,
|
||||
negatives: 30,
|
||||
structural: 12,
|
||||
} as const;
|
||||
|
||||
describe("heuristicRules.json pack lint", () => {
|
||||
it("has unique, non-empty label ids", () => {
|
||||
const ids = labels.map((l) => l.id);
|
||||
expect(ids.every((id) => typeof id === "string" && id.length > 0)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(new Set(ids).size).toBe(ids.length);
|
||||
});
|
||||
|
||||
it("keys every page prior to an existing label id", () => {
|
||||
const ids = new Set(labels.map((l) => l.id));
|
||||
const orphans = Object.keys(priors).filter((k) => !ids.has(k));
|
||||
expect(orphans, "priors keyed to no label never apply").toEqual([]);
|
||||
});
|
||||
|
||||
it("compiles every regex/filename/metadata/negative pattern", () => {
|
||||
const broken: string[] = [];
|
||||
for (const label of labels) {
|
||||
for (const kind of ["regexes", "filenames", "metadata"] as const) {
|
||||
for (const r of label[kind] ?? []) {
|
||||
if (compileRegex(r.pattern ?? null, r.flags ?? "") == null) {
|
||||
broken.push(`${label.id} ${kind}: ${r.pattern}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const n of label.negatives ?? []) {
|
||||
if (
|
||||
n.pattern != null &&
|
||||
compileRegex(n.pattern, n.flags ?? "") == null
|
||||
) {
|
||||
broken.push(`${label.id} negative: ${n.pattern}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(broken, "non-compiling patterns are silently dropped").toEqual([]);
|
||||
});
|
||||
|
||||
it("uses only structural signals the engine computes", () => {
|
||||
const unknown: string[] = [];
|
||||
for (const label of labels) {
|
||||
for (const s of label.structural ?? []) {
|
||||
if (!SIGNALS.has(s.signal ?? "")) {
|
||||
unknown.push(`${label.id}: ${s.signal}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(unknown, "unknown signals always score 0").toEqual([]);
|
||||
});
|
||||
|
||||
it("uses only zones and metadata fields the engine reads", () => {
|
||||
const bad: string[] = [];
|
||||
for (const label of labels) {
|
||||
for (const kind of ["phrases", "regexes"] as const) {
|
||||
for (const r of label[kind] ?? []) {
|
||||
if (r.where != null && !ZONES.has(r.where)) {
|
||||
bad.push(`${label.id} ${kind} where: ${r.where}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const m of label.metadata ?? []) {
|
||||
if (m.field != null && !META_FIELDS.has(m.field)) {
|
||||
bad.push(`${label.id} metadata field: ${m.field}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(bad).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps every weight positive and within the engine's clamp", () => {
|
||||
const bad: string[] = [];
|
||||
for (const label of labels) {
|
||||
for (const kind of [
|
||||
"phrases",
|
||||
"regexes",
|
||||
"filenames",
|
||||
"metadata",
|
||||
"negatives",
|
||||
"structural",
|
||||
] as const) {
|
||||
for (const r of label[kind] ?? []) {
|
||||
const w = r.weight;
|
||||
if (typeof w !== "number" || !Number.isFinite(w) || w <= 0) {
|
||||
bad.push(`${label.id} ${kind}: weight ${String(w)}`);
|
||||
} else if (Math.abs(w) > WEIGHT_CAPS[kind]) {
|
||||
bad.push(`${label.id} ${kind}: weight ${w} over cap`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(bad).toEqual([]);
|
||||
});
|
||||
|
||||
it("gives every phrase a non-empty text and every negative a matcher", () => {
|
||||
const bad: string[] = [];
|
||||
for (const label of labels) {
|
||||
for (const p of label.phrases ?? []) {
|
||||
if (typeof p.text !== "string" || p.text.trim().length === 0) {
|
||||
bad.push(`${label.id} phrase with empty text`);
|
||||
}
|
||||
}
|
||||
for (const n of label.negatives ?? []) {
|
||||
if (n.text == null && n.pattern == null) {
|
||||
bad.push(`${label.id} negative with neither text nor pattern`);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(bad).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,42 @@
|
||||
// Shared types for the client-side heuristic (non-AI) document classifier.
|
||||
|
||||
/** Input document for the heuristic engine. */
|
||||
export interface HeuristicDoc {
|
||||
fileName: string;
|
||||
pageCount: number;
|
||||
meta: Record<string, string>;
|
||||
titleZone: string;
|
||||
firstZone: string;
|
||||
allZone: string;
|
||||
}
|
||||
|
||||
// "none" = no match or non-English; a real runtime value, not just a type state.
|
||||
export type HeuristicConfidence = "none" | "low" | "medium" | "high";
|
||||
|
||||
/** One scored candidate label with the rule hits that produced its score (debug only). */
|
||||
export interface LabelScoreExplanation {
|
||||
id: string;
|
||||
emit: boolean;
|
||||
score: number;
|
||||
distinct: number;
|
||||
/** Human-readable contributions, e.g. `phrase "tax invoice" +60 (title)`. */
|
||||
signals: string[];
|
||||
}
|
||||
|
||||
/** Why a document scored the way it did; produced only when explain is requested. */
|
||||
export interface HeuristicExplanation {
|
||||
isEnglish: boolean;
|
||||
lowText: boolean;
|
||||
/** Top candidates by score, best first. Empty when rejected as non-English. */
|
||||
candidates: LabelScoreExplanation[];
|
||||
}
|
||||
|
||||
/** Classification outcome: emitted vocabulary label ids (primary first, capped at 5). */
|
||||
export interface HeuristicResult {
|
||||
labels: string[];
|
||||
confidence: HeuristicConfidence;
|
||||
score: number;
|
||||
isEnglish: boolean;
|
||||
/** Present only when classify was called with `{ explain: true }`. */
|
||||
explain?: HeuristicExplanation;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Idle-time scheduling shared by the classification/backfill passes.
|
||||
|
||||
/** Schedule work for the browser's idle time (or soon after, as a fallback). */
|
||||
export function scheduleIdle(task: () => void): () => void {
|
||||
if (typeof requestIdleCallback === "function") {
|
||||
const handle = requestIdleCallback(task, { timeout: 2000 });
|
||||
return () => cancelIdleCallback(handle);
|
||||
}
|
||||
const timer = window.setTimeout(task, 200);
|
||||
return () => window.clearTimeout(timer);
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
// SaaS override of the classification-enabled seam: classification is available
|
||||
// exactly when the AI engine is on for this tenant. Off → the sidebar grouping,
|
||||
// group-picker, per-file label chips and the file-details Classification section
|
||||
// all stay hidden, so an AI-disabled SaaS tenant sees the plain flat file list
|
||||
// with no hint the feature exists.
|
||||
|
||||
import { useAiEngineEnabled } from "@app/hooks/useAiEngineEnabled";
|
||||
|
||||
export function useClassificationEnabled(): boolean {
|
||||
return useAiEngineEnabled();
|
||||
}
|
||||
Reference in New Issue
Block a user