From b3875d3149c9bf428b22c155efdd57b33474576f Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Thu, 23 Jul 2026 08:28:26 +0000 Subject: [PATCH] Add heuristic classification (#7050) # Description of Changes - Adds a non-AI heuristic classification engine that classifies documents client-side in the browser when AI is disabled - Classification is billed as a policy run via a fast, non-blocking meter endpoint; a default Classification policy is seeded per team - Enables the policy engine by default --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details. --- .github/workflows/push-docker.yml | 1 - .../ClassificationRunBiller.java | 8 + .../software/proprietary/model/Team.java | 1 + .../proprietary/model/TeamCreatedEvent.java | 4 + .../proprietary/model/TeamEntityListener.java | 26 + .../ClassificationMeterController.java | 81 + .../DefaultClassificationPolicySeeder.java | 95 + .../model/TeamEntityListenerTest.java | 46 + ...DefaultClassificationPolicySeederTest.java | 118 + .../charge/SaasClassificationRunBiller.java | 49 + .../core/hooks/useClassificationEnabled.ts | 9 +- .../stubbed/classification-grouping.spec.ts | 47 + .../classification-heuristic-upload.spec.ts | 81 + .../classification/classified_invoice.pdf | 36 + .../classification/classified_nda.pdf | 36 + .../classification/classified_resume.pdf | 36 + .../unlabelled/bank_statement.pdf | Bin 0 -> 1535 bytes .../unlabelled/cover_letter.pdf | Bin 0 -> 1635 bytes .../unlabelled/generic_notes.pdf | Bin 0 -> 1451 bytes .../unlabelled/invoice_acme.pdf | Bin 0 -> 1630 bytes .../classification/unlabelled/nda_mutual.pdf | Bin 0 -> 1698 bytes .../unlabelled/offer_letter.pdf | Bin 0 -> 1618 bytes .../unlabelled/purchase_order.pdf | Bin 0 -> 1591 bytes .../unlabelled/resume_jane_doe.pdf | 90 + .../unlabelled/service_agreement.pdf | Bin 0 -> 1659 bytes .../unlabelled/spanish_contrato.pdf | Bin 0 -> 1505 bytes frontend/editor/src/portal/api/policies.ts | 1 - .../policies/PolicyAutoRunController.tsx | 3 + .../useClientSideClassification.test.tsx | 255 + .../policies/useClientSideClassification.ts | 207 + .../policies/usePolicyAutoRun.batch.test.tsx | 5 + .../policies/usePolicyAutoRun.chain.test.tsx | 26 + .../components/policies/usePolicyAutoRun.ts | 11 +- .../shared/FileSidebarGroupControls.css | 0 .../shared/FileSidebarGroupControls.tsx | 0 .../shared/fileSidebarGrouping.test.ts | 0 .../components/shared/fileSidebarGrouping.tsx | 28 +- .../shared/fileSidebarGroupingLogic.ts | 0 .../proprietary/data/policyDefinitions.tsx | 2 - .../hooks/useClassificationEnabled.ts | 6 + .../services/classificationMeter.ts | 27 + .../heuristic/heuristicClassification.ts | 18 + .../heuristic/heuristicEngine.corpus.test.ts | 2423 ++ .../heuristic/heuristicEngine.docs.test.ts | 118 + .../heuristic/heuristicEngine.test.ts | 125 + .../services/heuristic/heuristicEngine.ts | 976 + .../services/heuristic/heuristicExtractor.ts | 199 + .../services/heuristic/heuristicRules.json | 22352 ++++++++++++++++ .../heuristic/heuristicRules.lint.test.ts | 173 + .../proprietary/services/heuristic/types.ts | 42 + .../src/proprietary/utils/scheduleIdle.ts | 11 + .../saas/hooks/useClassificationEnabled.ts | 11 - 52 files changed, 27740 insertions(+), 43 deletions(-) create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/classification/ClassificationRunBiller.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/model/TeamCreatedEvent.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/model/TeamEntityListener.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/ClassificationMeterController.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/seed/DefaultClassificationPolicySeeder.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/model/TeamEntityListenerTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/seed/DefaultClassificationPolicySeederTest.java create mode 100644 app/saas/src/main/java/stirling/software/saas/payg/charge/SaasClassificationRunBiller.java create mode 100644 frontend/editor/src/core/tests/stubbed/classification-grouping.spec.ts create mode 100644 frontend/editor/src/core/tests/stubbed/classification-heuristic-upload.spec.ts create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/classified_invoice.pdf create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/classified_nda.pdf create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/classified_resume.pdf create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/bank_statement.pdf create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/cover_letter.pdf create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/generic_notes.pdf create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/invoice_acme.pdf create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/nda_mutual.pdf create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/offer_letter.pdf create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/purchase_order.pdf create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/resume_jane_doe.pdf create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/service_agreement.pdf create mode 100644 frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/spanish_contrato.pdf create mode 100644 frontend/editor/src/proprietary/components/policies/useClientSideClassification.test.tsx create mode 100644 frontend/editor/src/proprietary/components/policies/useClientSideClassification.ts rename frontend/editor/src/{saas => proprietary}/components/shared/FileSidebarGroupControls.css (100%) rename frontend/editor/src/{saas => proprietary}/components/shared/FileSidebarGroupControls.tsx (100%) rename frontend/editor/src/{saas => proprietary}/components/shared/fileSidebarGrouping.test.ts (100%) rename frontend/editor/src/{saas => proprietary}/components/shared/fileSidebarGrouping.tsx (71%) rename frontend/editor/src/{saas => proprietary}/components/shared/fileSidebarGroupingLogic.ts (100%) create mode 100644 frontend/editor/src/proprietary/hooks/useClassificationEnabled.ts create mode 100644 frontend/editor/src/proprietary/services/classificationMeter.ts create mode 100644 frontend/editor/src/proprietary/services/heuristic/heuristicClassification.ts create mode 100644 frontend/editor/src/proprietary/services/heuristic/heuristicEngine.corpus.test.ts create mode 100644 frontend/editor/src/proprietary/services/heuristic/heuristicEngine.docs.test.ts create mode 100644 frontend/editor/src/proprietary/services/heuristic/heuristicEngine.test.ts create mode 100644 frontend/editor/src/proprietary/services/heuristic/heuristicEngine.ts create mode 100644 frontend/editor/src/proprietary/services/heuristic/heuristicExtractor.ts create mode 100644 frontend/editor/src/proprietary/services/heuristic/heuristicRules.json create mode 100644 frontend/editor/src/proprietary/services/heuristic/heuristicRules.lint.test.ts create mode 100644 frontend/editor/src/proprietary/services/heuristic/types.ts create mode 100644 frontend/editor/src/proprietary/utils/scheduleIdle.ts delete mode 100644 frontend/editor/src/saas/hooks/useClassificationEnabled.ts diff --git a/.github/workflows/push-docker.yml b/.github/workflows/push-docker.yml index a613cfdebc..a87f6d68b5 100644 --- a/.github/workflows/push-docker.yml +++ b/.github/workflows/push-docker.yml @@ -139,7 +139,6 @@ jobs: tags: | type=raw,value=${{ steps.versionNumber.outputs.versionNumber }},enable=${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master' }} type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/V2-master' }} - type=raw,value=alpha,enable=${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/testMain' }} - name: Build and push Unified Dockerfile (latest variant) id: build-push-latest diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/classification/ClassificationRunBiller.java b/app/proprietary/src/main/java/stirling/software/proprietary/classification/ClassificationRunBiller.java new file mode 100644 index 0000000000..dd0467c9ca --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/classification/ClassificationRunBiller.java @@ -0,0 +1,8 @@ +package stirling.software.proprietary.classification; + +/** Meters a client-side classification run; SaaS charges PAYG, other flavors have no bean. */ +public interface ClassificationRunBiller { + + /** Charge one classification policy run covering {@code documentCount} documents. */ + void recordClassificationRun(int documentCount); +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/Team.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/Team.java index 119c909557..0009ee386e 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/model/Team.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/Team.java @@ -14,6 +14,7 @@ import stirling.software.proprietary.security.model.User; @Entity @Table(name = "teams") +@EntityListeners(TeamEntityListener.class) @NoArgsConstructor @Getter @Setter diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/TeamCreatedEvent.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/TeamCreatedEvent.java new file mode 100644 index 0000000000..3f632fe1d8 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/TeamCreatedEvent.java @@ -0,0 +1,4 @@ +package stirling.software.proprietary.model; + +/** Published once a new {@link Team} row is inserted, so listeners can seed per-team defaults. */ +public record TeamCreatedEvent(Long teamId, String teamName) {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/TeamEntityListener.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/TeamEntityListener.java new file mode 100644 index 0000000000..6a4725a142 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/TeamEntityListener.java @@ -0,0 +1,26 @@ +package stirling.software.proprietary.model; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Component; + +import jakarta.persistence.PostPersist; + +/** Publishes {@link TeamCreatedEvent} on insert; Spring bridges the publisher via a static. */ +@Component +public class TeamEntityListener { + + private static ApplicationEventPublisher publisher; + + @Autowired + void setPublisher(ApplicationEventPublisher applicationEventPublisher) { + TeamEntityListener.publisher = applicationEventPublisher; + } + + @PostPersist + public void onCreate(Team team) { + if (publisher != null) { + publisher.publishEvent(new TeamCreatedEvent(team.getId(), team.getName())); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/ClassificationMeterController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/ClassificationMeterController.java new file mode 100644 index 0000000000..dc978648b6 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/ClassificationMeterController.java @@ -0,0 +1,81 @@ +package stirling.software.proprietary.policy.controller; + +import java.util.List; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.Operation; + +import jakarta.servlet.http.HttpServletRequest; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.proprietary.audit.AuditContext; +import stirling.software.proprietary.classification.ClassificationRunBiller; + +/** + * Meters + audits a client-side (non-AI) classification run so both classify paths bill + * identically. Side-effect only; does no classification itself. + */ +@Slf4j +@Hidden +@RestController +@RequestMapping("/api/v1/policies") +public class ClassificationMeterController { + + /** Audit step label mirrors the AI classify tool so both paths read alike in the trail. */ + private static final String CLASSIFY_STEP = "/api/v1/ai/tools/classify-and-label"; + + /** Client-supplied count cap: the frontend meters one document per call. */ + private static final int MAX_DOCUMENTS = 10_000; + + private final ObjectProvider biller; + + public ClassificationMeterController(ObjectProvider biller) { + this.biller = biller; + } + + @PostMapping("/classify/meter") + @Operation( + summary = "Meter a client-side classification run", + description = + "Records billing + audit for a non-AI classification performed in the browser." + + " Does no classification itself. Dispatched by the frontend, not for" + + " direct use.") + public ResponseEntity meterClassification( + @RequestBody(required = false) ClassifyMeterRequest body, HttpServletRequest request) { + int documents = body != null && body.documentCount() != null ? body.documentCount() : 1; + if (documents < 1) documents = 1; + if (documents > MAX_DOCUMENTS) documents = MAX_DOCUMENTS; + String policyName = + body != null && body.policyName() != null && !body.policyName().isBlank() + ? body.policyName() + : "Classification"; + + // Stamp the run so ControllerAuditAspect records it as a policy run, like the AI path. + request.setAttribute(AuditContext.REQ_ATTR_POLICY_NAME, policyName); + request.setAttribute(AuditContext.REQ_ATTR_POLICY_STEPS, List.of(CLASSIFY_STEP)); + + ClassificationRunBiller runBiller = biller.getIfAvailable(); + if (runBiller != null) { + try { + runBiller.recordClassificationRun(documents); + } catch (RuntimeException e) { + log.warn( + "[classify meter] billing failed; classification proceeds unbilled: {}", + e.getMessage()); + } + } + return ResponseEntity.accepted().build(); + } + + /** Frontend payload: documents classified, plus the policy name for the audit label. */ + public record ClassifyMeterRequest( + String policyName, Integer documentCount, List labels) {} +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/seed/DefaultClassificationPolicySeeder.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/seed/DefaultClassificationPolicySeeder.java new file mode 100644 index 0000000000..aca6ae0e84 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/seed/DefaultClassificationPolicySeeder.java @@ -0,0 +1,95 @@ +package stirling.software.proprietary.policy.seed; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import org.springframework.transaction.event.TransactionPhase; +import org.springframework.transaction.event.TransactionalEventListener; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.proprietary.model.TeamCreatedEvent; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.store.PolicyStore; +import stirling.software.proprietary.security.repository.TeamRepository; +import stirling.software.proprietary.security.service.TeamService; + +/** + * Seeds an enabled Classification policy per team so classification is on by default. Idempotent; + * skips the internal team. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class DefaultClassificationPolicySeeder { + + static final String CATEGORY = "classification"; + private static final String CLASSIFY_ENDPOINT = "/api/v1/ai/tools/classify-and-label"; + private static final String POLICY_NAME = "Classification Policy"; + + private final PolicyStore policyStore; + private final TeamRepository teamRepository; + + // The default team is created during startup, before the entity event listener is guaranteed + // wired, so ensure it once the context is fully ready (self-hosted first boot). + @EventListener(ApplicationReadyEvent.class) + public void seedDefaultTeamOnStartup() { + teamRepository + .findByName(TeamService.DEFAULT_TEAM_NAME) + .ifPresent(team -> seedIfMissing(team.getId(), team.getName())); + } + + // Any team created at runtime (admin-created, SaaS sign-ups); after the team's commit so a + // rolled-back team never leaves a policy behind. + @TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT) + public void onTeamCreated(TeamCreatedEvent event) { + seedIfMissing(event.teamId(), event.teamName()); + } + + private void seedIfMissing(Long teamId, String teamName) { + if (teamId == null || TeamService.INTERNAL_TEAM_NAME.equals(teamName)) { + return; + } + boolean alreadySeeded = + policyStore.findByTeam(teamId).stream() + .anyMatch(DefaultClassificationPolicySeeder::isClassification); + if (alreadySeeded) { + return; + } + policyStore.save(defaultPolicy(teamId)); + log.info("Seeded default Classification policy for team {}", teamId); + } + + private static boolean isClassification(Policy policy) { + return policy.output() != null + && CATEGORY.equals(policy.output().options().get("categoryId")); + } + + /** The default Classification policy: classify each upload, versioning the file in place. */ + static Policy defaultPolicy(Long teamId) { + Map options = new HashMap<>(); + options.put("categoryId", CATEGORY); + options.put("runOn", "upload"); + options.put("mode", "new_version"); + options.put("sources", List.of("editor")); + options.put("scopeTypes", List.of()); + options.put("reviewerEmail", ""); + return new Policy( + null, + POLICY_NAME, + "system", + true, + null, + List.of(), + List.of(new PipelineStep(CLASSIFY_ENDPOINT, Map.of())), + new OutputSpec("inline", options), + teamId); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/model/TeamEntityListenerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/model/TeamEntityListenerTest.java new file mode 100644 index 0000000000..6807aa8687 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/model/TeamEntityListenerTest.java @@ -0,0 +1,46 @@ +package stirling.software.proprietary.model; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.ApplicationEventPublisher; + +@ExtendWith(MockitoExtension.class) +class TeamEntityListenerTest { + + @Mock private ApplicationEventPublisher publisher; + + private static Team team(Long id, String name) { + Team team = new Team(); + team.setId(id); + team.setName(name); + return team; + } + + @Test + void publishesTeamCreatedEventOnPersist() { + TeamEntityListener listener = new TeamEntityListener(); + listener.setPublisher(publisher); + + listener.onCreate(team(5L, "Acme")); + + ArgumentCaptor event = ArgumentCaptor.forClass(TeamCreatedEvent.class); + org.mockito.Mockito.verify(publisher).publishEvent(event.capture()); + assertThat(event.getValue().teamId()).isEqualTo(5L); + assertThat(event.getValue().teamName()).isEqualTo("Acme"); + } + + @Test + void doesNotThrowWhenNoPublisherIsSet() { + // JPA can build the listener before Spring wires the publisher; must be a safe no-op. + TeamEntityListener listener = new TeamEntityListener(); + listener.setPublisher(null); + + assertThatCode(() -> listener.onCreate(team(1L, "X"))).doesNotThrowAnyException(); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/seed/DefaultClassificationPolicySeederTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/seed/DefaultClassificationPolicySeederTest.java new file mode 100644 index 0000000000..2f270afddb --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/seed/DefaultClassificationPolicySeederTest.java @@ -0,0 +1,118 @@ +package stirling.software.proprietary.policy.seed; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.proprietary.model.Team; +import stirling.software.proprietary.model.TeamCreatedEvent; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.store.PolicyStore; +import stirling.software.proprietary.security.repository.TeamRepository; +import stirling.software.proprietary.security.service.TeamService; + +@ExtendWith(MockitoExtension.class) +class DefaultClassificationPolicySeederTest { + + @Mock private PolicyStore policyStore; + @Mock private TeamRepository teamRepository; + + private DefaultClassificationPolicySeeder seeder() { + return new DefaultClassificationPolicySeeder(policyStore, teamRepository); + } + + private static Policy classificationPolicy(Long teamId) { + return new Policy( + "p1", + "Classification Policy", + "system", + true, + null, + List.of(), + List.of(), + new OutputSpec("inline", Map.of("categoryId", "classification")), + teamId); + } + + @Test + void seedsAnEnabledClassificationPolicyWhenTheTeamHasNone() { + when(policyStore.findByTeam(7L)).thenReturn(List.of()); + + seeder().onTeamCreated(new TeamCreatedEvent(7L, "Acme")); + + ArgumentCaptor saved = ArgumentCaptor.forClass(Policy.class); + verify(policyStore).save(saved.capture()); + Policy policy = saved.getValue(); + assertThat(policy.enabled()).isTrue(); + assertThat(policy.teamId()).isEqualTo(7L); + assertThat(policy.output().type()).isEqualTo("inline"); + assertThat(policy.output().options().get("categoryId")).isEqualTo("classification"); + assertThat(policy.output().options().get("runOn")).isEqualTo("upload"); + assertThat(policy.output().options().get("mode")).isEqualTo("new_version"); + assertThat(policy.output().options().get("sources")).isEqualTo(List.of("editor")); + assertThat(policy.steps()).hasSize(1); + assertThat(policy.steps().get(0).operation()) + .isEqualTo("/api/v1/ai/tools/classify-and-label"); + } + + @Test + void doesNotSeedWhenAClassificationPolicyAlreadyExists() { + when(policyStore.findByTeam(7L)).thenReturn(List.of(classificationPolicy(7L))); + + seeder().onTeamCreated(new TeamCreatedEvent(7L, "Acme")); + + verify(policyStore, never()).save(any()); + } + + @Test + void doesNotSeedForTheInternalTeam() { + seeder().onTeamCreated(new TeamCreatedEvent(2L, "Internal")); + + verify(policyStore, never()).findByTeam(anyLong()); + verify(policyStore, never()).save(any()); + } + + @Test + void doesNotSeedWhenTeamIdIsNull() { + seeder().onTeamCreated(new TeamCreatedEvent(null, "Acme")); + + verify(policyStore, never()).save(any()); + } + + @Test + void seedsTheDefaultTeamOnStartupWhenItExistsAndHasNoPolicy() { + Team defaultTeam = new Team(); + defaultTeam.setId(1L); + defaultTeam.setName(TeamService.DEFAULT_TEAM_NAME); + when(teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME)) + .thenReturn(Optional.of(defaultTeam)); + when(policyStore.findByTeam(1L)).thenReturn(List.of()); + + seeder().seedDefaultTeamOnStartup(); + + verify(policyStore).save(any()); + } + + @Test + void doesNotSeedOnStartupWhenThereIsNoDefaultTeam() { + when(teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME)).thenReturn(Optional.empty()); + + seeder().seedDefaultTeamOnStartup(); + + verify(policyStore, never()).save(any()); + } +} diff --git a/app/saas/src/main/java/stirling/software/saas/payg/charge/SaasClassificationRunBiller.java b/app/saas/src/main/java/stirling/software/saas/payg/charge/SaasClassificationRunBiller.java new file mode 100644 index 0000000000..79533fb102 --- /dev/null +++ b/app/saas/src/main/java/stirling/software/saas/payg/charge/SaasClassificationRunBiller.java @@ -0,0 +1,49 @@ +package stirling.software.saas.payg.charge; + +import org.springframework.context.annotation.Profile; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; + +import stirling.software.proprietary.classification.ClassificationRunBiller; +import stirling.software.proprietary.security.database.repository.UserRepository; +import stirling.software.proprietary.security.model.ApiKeyAuthenticationToken; +import stirling.software.proprietary.security.model.User; +import stirling.software.saas.payg.model.BillingCategory; +import stirling.software.saas.payg.model.JobSource; +import stirling.software.saas.payg.model.ProcessType; +import stirling.software.saas.util.AuthenticationUtils; + +/** + * Charges one PAYG unit per document as an AUTOMATION job, matching what a server-side + * classify policy step bills. + */ +@Component +@Profile("saas") +@RequiredArgsConstructor +public class SaasClassificationRunBiller implements ClassificationRunBiller { + + private final UserRepository userRepository; + private final JobChargeService jobChargeService; + + @Override + public void recordClassificationRun(int documentCount) { + Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + User user = AuthenticationUtils.getCurrentUser(auth, userRepository); + if (user == null || user.getTeam() == null) { + return; + } + JobSource source = + auth instanceof ApiKeyAuthenticationToken ? JobSource.API : JobSource.WEB; + ChargeContext ctx = + new ChargeContext( + user.getId(), + user.getTeam().getId(), + source, + ProcessType.AUTOMATION, + BillingCategory.AUTOMATION); + jobChargeService.chargeStandalone(ctx, Math.max(1, documentCount)); + } +} diff --git a/frontend/editor/src/core/hooks/useClassificationEnabled.ts b/frontend/editor/src/core/hooks/useClassificationEnabled.ts index e5a2b48ff6..441b81cb45 100644 --- a/frontend/editor/src/core/hooks/useClassificationEnabled.ts +++ b/frontend/editor/src/core/hooks/useClassificationEnabled.ts @@ -1,10 +1,5 @@ -// Whether document classification (and everything it drives in the UI: the -// Files-sidebar category grouping, per-file label chips, and the file-details -// Classification section) is active in this build. Classification is a -// SaaS-only feature gated on the AI engine, so core — and every build that -// doesn't override this seam (proprietary, desktop, cloud) — returns false, and -// none of that UI ever renders. The saas layer overrides it to track the AI -// engine's enabled flag, so the feature shows up only on SaaS when AI is on. +// Whether classification (sidebar grouping, label chips, file-details section) +// is active in this build. Core has no classifier; proprietary overrides to true. export function useClassificationEnabled(): boolean { return false; diff --git a/frontend/editor/src/core/tests/stubbed/classification-grouping.spec.ts b/frontend/editor/src/core/tests/stubbed/classification-grouping.spec.ts new file mode 100644 index 0000000000..682d591804 --- /dev/null +++ b/frontend/editor/src/core/tests/stubbed/classification-grouping.spec.ts @@ -0,0 +1,47 @@ +import path from "path"; +import { test, expect } from "@app/tests/helpers/stub-test-base"; +import { uploadFiles } from "@app/tests/helpers/ui-helpers"; + +// The sidebar groups files by the category in their `StirlingPDFClassification` +// metadata; these specs exercise that seam with pre-labelled fixtures. + +const FIXTURES = path.join( + import.meta.dirname, + "../test-fixtures/classification", +); + +const categoryHeaders = (page: import("@playwright/test").Page) => + page.locator(".file-sidebar-group .file-sidebar-group-header"); + +test("classified files group by category family in the sidebar", async ({ + page, +}) => { + await uploadFiles(page, [ + path.join(FIXTURES, "classified_invoice.pdf"), // -> Financial + path.join(FIXTURES, "classified_nda.pdf"), // -> Legal + path.join(FIXTURES, "classified_resume.pdf"), // -> HR + ]); + + // The backfill reads each file's classification metadata on idle and regroups; + // the category headers appear once it resolves (Playwright auto-retries). + const headers = categoryHeaders(page); + await expect(headers.filter({ hasText: "Financial" })).toBeVisible({ + timeout: 15_000, + }); + await expect(headers.filter({ hasText: "Legal" })).toBeVisible(); + await expect(headers.filter({ hasText: "HR" })).toBeVisible(); +}); + +test("an unclassified file is not placed in a category group", async ({ + page, +}) => { + // sample.pdf carries no StirlingPDFClassification metadata, so it must not + // create or join any category family group - it falls into the catch-all. + await uploadFiles(page, path.join(FIXTURES, "../sample.pdf")); + + // Give the idle backfill a chance to run and (find nothing to) regroup. + await expect(page.locator(".file-sidebar-file-item")).toHaveCount(1); + await expect( + categoryHeaders(page).filter({ hasText: "Financial" }), + ).toHaveCount(0); +}); diff --git a/frontend/editor/src/core/tests/stubbed/classification-heuristic-upload.spec.ts b/frontend/editor/src/core/tests/stubbed/classification-heuristic-upload.spec.ts new file mode 100644 index 0000000000..223a2eae51 --- /dev/null +++ b/frontend/editor/src/core/tests/stubbed/classification-heuristic-upload.spec.ts @@ -0,0 +1,81 @@ +import path from "path"; +import { test, expect } from "@app/tests/helpers/stub-test-base"; +import { uploadFiles } from "@app/tests/helpers/ui-helpers"; + +// A bulk upload must classify every file in the browser and group it - no file +// may be stranded in "Other" by races between the upload wave and delivery. + +test.use({ autoGoto: false }); + +const FIXTURES = path.join( + import.meta.dirname, + "../test-fixtures/classification/unlabelled", +); + +/** The stored policy DefaultClassificationPolicySeeder writes for a new team. */ +const SEEDED_POLICY = { + id: "seeded-classification", + name: "Classification Policy", + owner: "system", + enabled: true, + trigger: null, + sourceIds: [], + steps: [{ operation: "/api/v1/ai/tools/classify-and-label", parameters: {} }], + output: { + type: "inline", + options: { + categoryId: "classification", + runOn: "upload", + mode: "new_version", + sources: ["editor"], + scopeTypes: [], + reviewerEmail: "", + }, + }, + teamId: 1, +}; + +test("a 10-file upload wave classifies every file into its group", async ({ + page, +}) => { + test.setTimeout(180_000); + + await page.route("**/api/v1/policies", (route) => + route.fulfill({ json: [SEEDED_POLICY] }), + ); + await page.route("**/api/v1/policies/classify/meter", (route) => + route.fulfill({ status: 202, body: "" }), + ); + await page.goto("/", { waitUntil: "domcontentloaded", timeout: 120_000 }); + + await uploadFiles( + page, + [ + "invoice_acme.pdf", + "bank_statement.pdf", + "purchase_order.pdf", + "nda_mutual.pdf", + "service_agreement.pdf", + "resume_jane_doe.pdf", + "cover_letter.pdf", + "offer_letter.pdf", + "generic_notes.pdf", + "spanish_contrato.pdf", + ].map((f) => path.join(FIXTURES, f)), + ); + + // Each group header is a collapsible button whose name carries the member count. + // Classification runs a few files per idle pass; wait for the full drain. + const header = (name: string, count: number) => + page.getByRole("button", { name: `${name} ${count}`, exact: true }); + await expect(header("Financial", 3)).toBeVisible({ timeout: 90_000 }); + await expect(header("HR", 3)).toBeVisible({ timeout: 30_000 }); + await expect(header("Legal", 2)).toBeVisible({ timeout: 30_000 }); + + // The regression: nothing classifiable may be stranded in Other - only the + // genuinely unlabellable pair (generic prose + non-English) belongs there. + await expect(header("Other", 2)).toBeVisible({ timeout: 30_000 }); + // The filename can render in several places (Recent, group, viewer); any hit proves presence. + await expect(page.getByText("generic_notes.pdf").first()).toBeVisible(); + await expect(page.getByText("spanish_contrato.pdf").first()).toBeVisible(); +}); diff --git a/frontend/editor/src/core/tests/test-fixtures/classification/classified_invoice.pdf b/frontend/editor/src/core/tests/test-fixtures/classification/classified_invoice.pdf new file mode 100644 index 0000000000..185fb4cc81 --- /dev/null +++ b/frontend/editor/src/core/tests/test-fixtures/classification/classified_invoice.pdf @@ -0,0 +1,36 @@ +%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj +2 0 obj +<< /Type /Pages /Kids [3 0 R] /Count 1 >> +endobj +3 0 obj +<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >> +endobj +4 0 obj +<< /Length 38 >> +stream +BT /F1 24 Tf 72 720 Td (INVOICE) Tj ET +endstream +endobj +5 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >> +endobj +6 0 obj +<< /Title (Invoice) /StirlingPDFClassification ({"labels":["invoice"]}) >> +endobj +xref +0 7 +0000000000 65535 f +0000000009 00000 n +0000000058 00000 n +0000000115 00000 n +0000000241 00000 n +0000000329 00000 n +0000000426 00000 n +trailer +<< /Size 7 /Root 1 0 R /Info 6 0 R >> +startxref +516 +%%EOF \ No newline at end of file diff --git a/frontend/editor/src/core/tests/test-fixtures/classification/classified_nda.pdf b/frontend/editor/src/core/tests/test-fixtures/classification/classified_nda.pdf new file mode 100644 index 0000000000..ec1eddb1ee --- /dev/null +++ b/frontend/editor/src/core/tests/test-fixtures/classification/classified_nda.pdf @@ -0,0 +1,36 @@ +%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj +2 0 obj +<< /Type /Pages /Kids [3 0 R] /Count 1 >> +endobj +3 0 obj +<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >> +endobj +4 0 obj +<< /Length 55 >> +stream +BT /F1 24 Tf 72 720 Td (NON-DISCLOSURE AGREEMENT) Tj ET +endstream +endobj +5 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >> +endobj +6 0 obj +<< /Title (Non-Disclosure Agreement) /StirlingPDFClassification ({"labels":["nda"]}) >> +endobj +xref +0 7 +0000000000 65535 f +0000000009 00000 n +0000000058 00000 n +0000000115 00000 n +0000000241 00000 n +0000000346 00000 n +0000000443 00000 n +trailer +<< /Size 7 /Root 1 0 R /Info 6 0 R >> +startxref +546 +%%EOF \ No newline at end of file diff --git a/frontend/editor/src/core/tests/test-fixtures/classification/classified_resume.pdf b/frontend/editor/src/core/tests/test-fixtures/classification/classified_resume.pdf new file mode 100644 index 0000000000..79a17e901b --- /dev/null +++ b/frontend/editor/src/core/tests/test-fixtures/classification/classified_resume.pdf @@ -0,0 +1,36 @@ +%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj +2 0 obj +<< /Type /Pages /Kids [3 0 R] /Count 1 >> +endobj +3 0 obj +<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >> +endobj +4 0 obj +<< /Length 47 >> +stream +BT /F1 24 Tf 72 720 Td (CURRICULUM VITAE) Tj ET +endstream +endobj +5 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >> +endobj +6 0 obj +<< /Title (Curriculum Vitae) /StirlingPDFClassification ({"labels":["resume"]}) >> +endobj +xref +0 7 +0000000000 65535 f +0000000009 00000 n +0000000058 00000 n +0000000115 00000 n +0000000241 00000 n +0000000338 00000 n +0000000435 00000 n +trailer +<< /Size 7 /Root 1 0 R /Info 6 0 R >> +startxref +533 +%%EOF \ No newline at end of file diff --git a/frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/bank_statement.pdf b/frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/bank_statement.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ad4ef7ab7b0d8579827dad75aaac5d67b44a63ef GIT binary patch literal 1535 zcmY!laBn<8=LNwm zaZW5r%*jtjvIgWtgzNK4Qu9iR6-*#{0uqaWT$l+#sm1xFMaikf3g*a0f-FZe52O`l zo?B*4Noo<$tvQJ$sV=F>`6;Pf`aY?7=_MHoCYEL(UBx9usfoE<6?1ZfPv+e=5ZL=% z``Lr$Xo((=uBe*gnB@+Q7M^Ptq`NK3+Gg}JD1_&K?eVBAQKNjB54xYzPCU3B>)^t( ze0Mv~J9*6w6Id8iRC6|YcLqBNcTAr0L0}Wpv(;aJ{mh+xP%k=l_VTl4zgcds|M&5) zGP76yLsyrITaU`)s!9Wn9qul=CFZ^0gG%f^cCiftdt6WKXh@&>e4FEeZI*kqx3`?H za1l{gej3+SEOa^S+0PX{X7&#q!!A9^;@n(tP-C0cvK8H__ZID2v3Tv)^^UWc*4)=- z-x?)u{4vl(fayLav zH%^Jgsc!kel&9~Jnp2iql9`;S>y)39!lmz;2TVkndFcxJ;hA}kdBvG90WSUE(xeh_ zMgnCgU=D=jDv&WqsnZOjR4u1^tlJ3ZQ>IbAh?Z2}(OdY0p?BA6Ot+;0(+=phCd~n7tGR!O%2V=&7EA0-JH#woJ}koEiFyV9nB1!44o|HVo?b==nRbvOt@54UH#p-03%lg|0k4>;KYIUp_x-)^^Lsw_ z`8~3DgGr7@5s0kw_U|e2&(5DoB(hHjb+m8Ae9iWEK3k{0Ze0P9q$CG zn23;SGD@KWDkUDJQj52XvQ|TY#?u+lj>N-}s zI7=M)+T3C+Xv@96nKugOeH}e#Z2Nic=!zM4OJ`JgT-FcU8qRwt`a#d+i5Ko{TAEi& zG~ZiClzH#dKX#31*qR%ni3!j9?N@G0qTlBSyb-ZUw#^+$)ofs*e{taYZ_3)1hWYQP z@}9gKa@_NxQ_=9V#m?WEUA{p+BDXnW+L@yC4a<^4$=9ZQ?=@c7*L9Mt*>~yDXMgzj zd=hCjj+?cHUA(hxZ1D@;0MksDAOC*-@rbZ)xsET%S-9XkF4(zVe|!I_uyR(BJ*rOr zg`*`uaCD;kJ~?UR+4|u3)@6S>DXD7vgp--!D|04X3O*QVt_msAo6TR9;#Tv9W9Qlq za- zZibaSzw_GcO4%4Ky>by3Tj((@8oUo=u`swAnO51~OoXeHx91NDLrq!f- z+h?9=C3iLjmF<;#Zk_3-#_d_A%&wpfMpu32)YEgW8aFS@#c2fenABPz}e_q&UF-Zi{2&uy;644)cL4l{iM*22@R86#+QYk4VMX4w)rZwpdlu>IS2(8ITU^=bQ b6!YIN`fF|%Xq(VqI-J0ih)iaTGa-KgSUE%G literal 0 HcmV?d00001 diff --git a/frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/generic_notes.pdf b/frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/generic_notes.pdf new file mode 100644 index 0000000000000000000000000000000000000000..15dce23df49c255573ee2888e16f71373f345e1c GIT binary patch literal 1451 zcmbtUTTC2P7#2dnjF(y@#5CHTP*Kq=GqXEa-E3tqlijpjT%eU*t4?>%Zinv9l$}|+ ztF2aQV=Jtr+f5;<5%v3|}h+Rp7UvBkx8GFNHx# zv?~eVClEsObg`KO97Ps$JUXpPZiN7^5W`T#Y&HzaL7mQQpe^4D<&uCVDRP<<;DV&N zfgF!S5P@#GPE0Cs4dCT6bSRN{RGx=bpoiyOaN-a(F`|SGZuAiUj$YQFtf_#Sx2X_g z=y!p^3sof^3qTdHMoQhefloJG;1i^X24g7Jh^Rp)3@AZ};p?Cr);0r*H0xHX7K36G zOB{;sBb8y|mfce`&$Fi<>~r_79@?@qc=24po}HimcsWRRdYZS!d|g0f1h%Hva)7Z z&$b*l+i>ND<+(ZJm0xRWcP&f&cDC=v7FVq0 zy^vXOQ`z;~Gh9^HiM+S!io=2P59itb=xhA_<@OoxsfNLy>$^?l@lgJe)~2^_%lzn} z(^s?Bl8w0S(RL|SR$bb5Z1ApC&kFU{6(1jtzU0o~hInEB7k7*gfA@7qukFlqkUMZ+ zUHQI8ayyD%9C$S=_gh$ZXEU~+F7lX?jiW!Et!{hATfL*^^RCAl&g9+QyQizC>Z|@g zsi!lJy1pFK1(|-Pv@~hM8{0({3JOvQ?uL(#P1>C3b?KvSnePCd#-JqT-@W{Q2fB2xg#1;6)3mShOtY@umi zk-;nxgCUFnJVq=oz|gdX1|hJBktj@k0eKN42{bdV!OX_^l3_Vh9IM2^>N5 ptU$3Pf`cJ>ils|fj<686|GqI-b5#>#+Fa>K(oA55g|2b|`wLP@PPlIPM5)gnZD8y7M45v9a%L5X_->_L+AWw{bnes6_9a?ORmh*r- z0wGeRG?7vOg@jC0DAC^)r5hl?vluIMVQMu7S&KkN`Os!=fUL&M+c;gi0rimmp!!vfy>XSw&@$c$c9edNc%5h;s2x$T%>M z*YQpVdxr>09U896J+|5J^6b>d@TvEeJqy!s=lGwxysh@(`&)HMA)>a5AL`?l%&}|2 zjpFS?bMUZ{*1D*S`m?Q9`yQ*OhXxxX*PPGqPZ``(8TQew-2?v5<}Mn(_mWC=MxAo^ zZoc-9lEq(~vIcE?kaWLt;MTR_x+M)*K&Io&-s1k)Y<;ezQ8Fw0E9;t#@m?=IuPu|= z75M9chBJENK~&((f#XeampdcgS`qDNFb14_xv}f%z{B=w3unj=e%hG&b60L`{zh-0DYjSG7>dt<{yLr23{A-<0Ki8NOxh z+oI}Y3r^YU-a8WfVP&&)K~;QP*v$_@J68G~zs}O&(kkigl8ZYFZ%jX%k`OIch0Ag- zUTk^yTD)a_Z;0{elC+yHW5jEc@b03Z_-=!f`Z_8wpl0U1W;=H1^Ng_MOB){@Z1(Q& zZM$$M>cA1Q+0m2uJCjh@T`1qq8D5XQ(2&x&G|bSI{ae@JeEe`F);53Z>Q5lvgx?_6 z4wlC4x?(RjmVei+FFpUHxajbfjJ{{HsP?OVvC$(HduyK*&QphetZ6Sl8GSm@dF7L{jQvlC?s!Q5$tRFJ~I8!X(uqnFM@`~H)~u?QlGWdGA^j+&|>3h&|VCA zo0&<_a&`-br?O~AZLAgGZ`xQ5>#|J%7@qAe;74^Ts1*r(Otb=x&+^OuJhmao&rJzCZsDo%gLFuArJC!M*0X?W+g9C|GJN~4{pN4#&4ir8K zXe3R+b96|(AV!x&5tNLOlQKd{iKWuju|8oQCtQaSK#38P2gsySu@tNWlNgE06b@h~ zF_J(wHwB}J7w=Ju7cr#-75mRvl8_>Uoq~}PG_@%hB|$zlf$>hpW{1vE1JAaVLDU9r z;y4}%7L3MfVApX#7>w|)19>VPsZAwmxrQW_Vp1WdWMa97P|y-uu29P4lwAGaH%4df U;u$ADIy#b4Dlw5LHB*oM1?IpeUH||9 literal 0 HcmV?d00001 diff --git a/frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/nda_mutual.pdf b/frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/nda_mutual.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ceece9ecdf87dbbb8c06760a1feaa6108aa41930 GIT binary patch literal 1698 zcmbtVYfu$s7*0`#HlPs+2PpZxpd!ki-91-^taJ5-oWo_zAkpRU9a!A6i+lFaf$)}s zjzwO=%M^;D3??#)kq%|3+;l8SKvEIKOUQ6JXlQsTr*;wOq%-xS{qyZU-}gSx`@Zu% z?*^H4hFMrBfdqAS|9T3+fCR9KsYrM@f@)bi!vhRK7f?wy5KjnO(Mu3?Axxr34eJ2$ z5&@B^WTChcD5Y4aQYGxhW~4!Yn#g2mLuSuLAd@80;V!hsG{~sUJjF7DoB%aY{9M4; zX<8r5<-X8}EGHvQ*ibOYDf;olrA1YGnHa1Dx#9I-|l@ zHb2A>(i6~1OKw8V;^r#-=BW4Xsy9HBBS};j%Q!yoS3yZw>7h}Yi8)C z?EA-Wyl!nASLLRZd-jPx#=FwJ7}n&OP| zxVYE9`P;2)%i?1DQz|BXkoY9;bgpSew~yDW6L(dn=4?5(QC~LaDe9)5#MT7yE6=eg zz5nT!CB>t^A&+Dim4yY?jW%inQ!g#^r|KIoy%iASA5&oPZZwrFqBTT*;&1z!$8>^Y z6I=Hl^o(71#&`XSrrV#@{j#!Q^cBSK@uke^pG-;oDzM?)+p7~yZL1Dp-VP^Sy1OfY zv*z5dYs!b_;=E%@FGuh?KYLAWU+=TY*XBG5X^Zw>WmN18xRMfDon3gfys&Q}FvrpU z?>KM$uG*uU`5L>|7&5M9LG#XDkmZzKZ|L_?I19J*^t@reIH807BEz}C?__x1*NGMD zcLvrL?rK}@3HN!sciF0g@vPcICl+sbCGvFZ$;-ueeK$A9ktO|SWBwdbsQKPY^x@#` z9CW12CK;gbQH+|gQ9}TN#@G}20k4aW z7ly=hefsW^mOGuJe|-RG@Sb&G0ARW<442??i2{>LR5&4%#Ra>BIXGxR zBtV5oh96-{$Osv*fME<1(i9)S3}cu?7?m$zxa8$?xbh_olVC!z|BS^jOeo+541ZY{ zBQO_To+Bw5asxks7%CGAWuVb4%L8$Y3vHdpSXdx7Mts+S_;6fnkP^67qr@Z{g;b@N o>a|Llnouby#l1h zuSl~faK;a?lt2VC1(fmw(IUktZNL~x9ZZy&9HAQsFVB%^m#;3IsXu!Ed3S&B`}}^- z^Um{oLW~;i3N%6th1|P8bRI$h0+{q1NTGmWHRGT;fI@IAmB9kDm_IV5LU25pK@lpZ z0GJUT!R2@aCId1F8X=SOr-_AmB!G=X7Rf?UQ4mRI2y~bSts#%3l@^X-=t)ihYbj0# zXouCx69}PkL?Pqg0BoR1HDh(y=n0qwLVBht5jDXPRwm0sMo97R>}igqITlDJVvK~H z|D9qmku2k|TSyj2Jt+lo4?ZDuicd>fInvJOY9%;QLt2;&5`yDMI*ZE&xKu7gv7DVG zY*4{|TSDxImB`C2&f|O4&)54jDrPNj|Gc8^h^u>k{q<_EWv6nH=J@cj)&q;Xw7$}Y z_B;FBm(ZwN+mscSGj3y_>xH0RZ(@7XSFK#E-beH1W#@a~=c=QPg;D;lw4ve)e-8EE zc0b4;OKgpCTQ=VcNY`{OQdfBEZ!e6p*XF}+=V1RYhR1Co`j2AfF6jCI?Rcir=^w|= zDHpfpCVO{F)?1Q-t+9R`vqg!9?;c*-SS5M=B=o5*;dL9ml zMDv{YyFZrhrgm1OlF&XUcskNY;9}}+_#M=zvqUBQ_JW!X^tvIS^_dC2a+ckex{%17@z4MZq_wFd@CeD~w zs&?&kwx|P#r% zmOE+^U5c~E!cSW@gK^h)=SX3%SKE7MZ%T_R9SB=>H7DQyE%w{}xlR2?f1edPxBf@J zMTbIbM~%xOMBQuN76n{el6KenxLS7N#o`U4{_D^E;Qfhn%aT9xdX!$lVMSN^TP}uP z&DnGL*|k3RT<5dcsz+YUW6|+~Zzj8G(m}XKE2;>V)G~ZC!#dLX2FXztVugyaW z&G#imX90LEMJs8RngSp=!I935dsA>IZ|Vu73N#)KANlVp|6f3G+q9#E5)=wpivpoK z-q%_T2pO1oFTqATV@V)6V21e#1>i)ofUj3?;|;2s+|`r2KGmZFnTLRyx1%s;G`wR$ zm}Vu0V3CLvjYQ;_7{}LzdxRC($xH|VatN7TAQH#LILHLk7|N$9EI?0VD8i4n85o8< zc@L94iOI!$vHy%k5iwuD3=HM7n2BLx4_(eqP*&1DKC~04B9boyHZcqbgt5y%tDeqe zfbcNFw+5IMm=aSeQ3O?Kl@g6yB~@dQxD>@D5~UcAR^e#Wf8Q8?bCx6Q-1yU>m`n9mW(23=d~4GzUlwf5Di{f_O1{(j{Pc6ihZ!N+t`$ z69|#Yr2!%tkV(h@nH;_D%}$2^kERTe#X>_dNGJ1jB0E}bI;6w&oROiIIRUOQaytQS zF_{nnZ@QbxW-J`QwKNQ8OqMiy31$Hwo^HiSIK@#W#$e}$5AmPqX%5mH3nWW6(Ucio zR~U4VWh`brWI>QUCGXshk2hW6(-=)0G^1Ee6bDsM&m=<(-wkO4mkOj(5pTtEW=N%B zS!HQ4QS1kV;M{L<;}wpXReK&6u5W8;7^*LM(>-r~-Id}3U17fGi0xX$>&LRXDH)NY z{d#I5fLwS<<=LO!Ja=H^_WRpBN_>DQd`jlFO|Qm`4j_jwoVUBl_Dr)@bI zCk_ZC*o>S_dP`XM<9Ay#1gEbVvbw!tap9>AyLOHEL}Z^o`$Nq97c*~M-amQ7?DzY{ zhBW`&u9_PT)`KT|O)|wd`yLKS7`B1S9!G0ha)&OKcf97dXLMjzG(1ZV6x42M9CVxs zd~8zd@X_XCb?dsPu5aoP^u$`rB~dP5p($z1!O^wh*ruCKlTF{=blQH%AdnYjzgi~v zrK&maTxL($iyf^2t(p^Msa5;Ww8nl_6n~3K#9Jj53uh}k`{wF=+b${os45pHmwJxM z*GKJeqfcIqn=HOlJrLaZS#IL_g%G{4CsBNg74;6d4VJjmuG0_Ir9>UwP*&;P?ERN^ zq*QQxWZM%%rrP%6Eer5)VYuz-ThB)l*0G?xi9)B1jRxJ7m%nTuY^!dZin#IaO!xSF zeV=u!v&6e%-|^J;{0`5)$fDW1w#M{{PWsc6E7;2IL}*&*?2M1YS#hXjL1n;Cr!%q^CVM3u$7ywq9^_BRf>E~o$i literal 0 HcmV?d00001 diff --git a/frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/resume_jane_doe.pdf b/frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/resume_jane_doe.pdf new file mode 100644 index 0000000000..be462dcc6c --- /dev/null +++ b/frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/resume_jane_doe.pdf @@ -0,0 +1,90 @@ +%PDF-1.3 +% +1 0 obj +<< +/Count 1 +/Kids [3 0 R] +/MediaBox [0 0 595.28 841.89] +/Type /Pages +>> +endobj +2 0 obj +<< +/OpenAction [3 0 R /FitH null] +/PageLayout /OneColumn +/Pages 1 0 R +/Type /Catalog +>> +endobj +3 0 obj +<< +/Contents 4 0 R +/Parent 1 0 R +/Resources 7 0 R +/Type /Page +>> +endobj +4 0 obj +<< +/Filter /FlateDecode +/Length 685 +>> +stream +xmOs0{$T1&$%LO{ J,ɑd}Wt +3 }V ~(ݧ.cHREPaVa1 S(kL7|ynOw/S壌 P~7@'zfƯe Q'C4תJ8Po_}p؏JY kVhXtRrORhZZv;nP= +h`'# +* z 퇖p< +3`Չ&`| pmQqG,pUC :#Vw֡ BOpgy~4Ҏz'ZN:Qfd6;z89ed8=0~h6;A] $ pDr=1&ţ q^ wkmdHIHځ-t!NY6 +E#\2ڒsqˢي +Of<*bVmQᦚT{ԣyDuEkfɅr%ЪNK4_ּlk#/m["wȬ5a-jhE~'p@^U9\tjdodWỉIpEzryEXג8/'l +endstream +endobj +5 0 obj +<< +/BaseFont /Helvetica-Bold +/Encoding /WinAnsiEncoding +/Subtype /Type1 +/Type /Font +>> +endobj +6 0 obj +<< +/BaseFont /Helvetica +/Encoding /WinAnsiEncoding +/Subtype /Type1 +/Type /Font +>> +endobj +7 0 obj +<< +/Font <> +/ProcSet [/PDF /Text /ImageB /ImageC /ImageI] +>> +endobj +8 0 obj +<< +/CreationDate (D:20260716092355Z) +>> +endobj +xref +0 9 +0000000000 65535 f +0000000015 00000 n +0000000102 00000 n +0000000205 00000 n +0000000285 00000 n +0000001042 00000 n +0000001144 00000 n +0000001241 00000 n +0000001338 00000 n +trailer +<< +/Size 9 +/Root 2 0 R +/Info 8 0 R +/ID [<7F4FFDD51531EF25A5301EAFCA4542A0><7F4FFDD51531EF25A5301EAFCA4542A0>] +>> +startxref +1393 +%%EOF diff --git a/frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/service_agreement.pdf b/frontend/editor/src/core/tests/test-fixtures/classification/unlabelled/service_agreement.pdf new file mode 100644 index 0000000000000000000000000000000000000000..68c7db83529142c89892efce2743eeb231ec4607 GIT binary patch literal 1659 zcmbtUdr%a094|1N*^mM=GadSYAoyY*cY6;7&bz(4kVhPMh&)Dd;g)0Z*u~pD1Q{j` zLqKJUcsTNk6cl`Z!6_B7@(%mFby<5UoWaUPcb|Pq>p_Jq)qsp zVlYx1Yqup)9FRLwioqTDMAs=kJ#FPFn~T8?f&?9 z40G|)nb6pv@_|DUh6mNfztyl>? zA3fHTpZ%kU--+^BU(O$YHZqrW^H|y8*-&w;^5cs;@HhGl^IR3TU3=Si%#O+)sdu|{ zZBA^6`NpbbsmqbZpY9gsUSBcTHK(+?{`S?>1-CrgakjD`;PQCG*O&c9JDoyW-#@k0 z_k)(v4gDEovkf1444>=I$*-}beAgGN_2sp(K%X=^D_T2cQ)UC4D^JkN#cO~m`wrJDC)Cyv_!PCX;UAK9|@fT(s zUGw1NfXcY7_@*eZ$+4T18tyQ&8j_>*tT6R(FlEi4cshw(u3@ca2-Y#e0Mm>Gz-wtn z&2aP-0Kq1EB0nKwQObh)A1hjn<8=LNwm zaZW5r%*jtjvIgWtgzNK4Qu9iR6-*#{0uqaWT$l+#sm1xFMaikf3g*a0f-FZe52O`l zo?B*4Noo<$tvQJ$sV=F>`6;Pf`aY?7=_MHoCT3@aR^8LzkJRWybt{&XHwvt~Kz9)Tce4eb@4D#fc-u zp6~hNv#V;Rn$!zLx_#|iWtICh=mhK3?xiQ@uefINdt2o7+o5|GPVWl4ox8Oz^6#Ix zwXe^Lue!g?y?=f2q)*PzJ>7S{9uLEiW1S;-z+3fD|$wm-dqxIUSjz|O!@A~ zIX*!PGJZL}4B^W6n0+8OyT*OlF?ey%~iRi7U*)p=`j2c~%jD#cVT{F9>pv7(&g zp5YGBH}66{jutF$w<|9U5L?D*eJruo#qU_*fkmdV-LcH#7FO1eD%jUW=(!rQ*>&%c z7xMGbWM;OJd1+Db>CndT-lt**UUq+2Qfs|q%3s?fi*#Jg?9GFwb=M07yC2hg8!XIF z$}9HhY{9yQ=<-|B9zC19U7auZ&%`;M?>4`!UgD}|WI0J```njPOzQ(w`ZATPU{xl=$HZ9OMNX8+|uJ z1(5H6+04yI0px-pVA9qPD9TR`PAyT0)&~|C3i=_b6+r)b<^uDJ6O?v_(w?zMKCnQt zz!{i3K-t~}n1K{DT&#=?jLZzo4b2QJjf_oAqcoAURTQPBaTzFBav8vZf|;qQv8h6u z0$j`x=u8k$$b*X+8URZgG%+IsOm#*Um|_-2z~TcHXgYxrB6hNgQFjzhF(()BR0Rv*YC`8*hIXjyh85p>kSsJ()n46gx sxw<(zyE&PdxjC5|n3+1;5mo^X^Wu`kq7rb>85)|KaH*=g`nz!f00k5J!vFvP literal 0 HcmV?d00001 diff --git a/frontend/editor/src/portal/api/policies.ts b/frontend/editor/src/portal/api/policies.ts index 1e2087537c..a26556f5a2 100644 --- a/frontend/editor/src/portal/api/policies.ts +++ b/frontend/editor/src/portal/api/policies.ts @@ -194,7 +194,6 @@ export const POLICY_CATEGORIES: PolicyCategory[] = [ tone: "blue", desc: "portal.policies.categories.classification.desc", providesClassification: true, - requiresAiEngine: true, }, { id: "compliance", diff --git a/frontend/editor/src/proprietary/components/policies/PolicyAutoRunController.tsx b/frontend/editor/src/proprietary/components/policies/PolicyAutoRunController.tsx index 7c4e921943..7dee491d71 100644 --- a/frontend/editor/src/proprietary/components/policies/PolicyAutoRunController.tsx +++ b/frontend/editor/src/proprietary/components/policies/PolicyAutoRunController.tsx @@ -1,4 +1,5 @@ import { usePolicyAutoRun } from "@app/components/policies/usePolicyAutoRun"; +import { useClientSideClassification } from "@app/components/policies/useClientSideClassification"; /** * Headless controller that drives policy auto-run (enforce every enabled policy @@ -7,5 +8,7 @@ import { usePolicyAutoRun } from "@app/components/policies/usePolicyAutoRun"; */ export function PolicyAutoRunController() { usePolicyAutoRun(); + // Non-AI systems classify uploads in the browser; inert when the AI engine is on. + useClientSideClassification(); return null; } diff --git a/frontend/editor/src/proprietary/components/policies/useClientSideClassification.test.tsx b/frontend/editor/src/proprietary/components/policies/useClientSideClassification.test.tsx new file mode 100644 index 0000000000..d399b09553 --- /dev/null +++ b/frontend/editor/src/proprietary/components/policies/useClientSideClassification.test.tsx @@ -0,0 +1,255 @@ +// Delivery guarantees of the client-side classification hook, driving the real +// policyRunStore and mocking only IO (storage, the heuristic engine, the meter). + +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { renderHook, waitFor } from "@testing-library/react"; +import { + markDispatched, + resetPolicyRuns, +} from "@app/components/policies/policyRunStore"; + +interface TestStub { + id: string; + name: string; + lastModified: number; + derivedFromTool?: boolean; + classificationLabels?: string[]; +} + +const mocks = vi.hoisted(() => ({ + workspace: [] as Array<{ + id: string; + name: string; + lastModified: number; + derivedFromTool?: boolean; + classificationLabels?: string[]; + }>, + configLoading: false, + updateStirlingFileStub: vi.fn(), + bumpRevision: vi.fn(), + getStirlingFile: vi.fn(), + updateFileMetadata: vi.fn(async (_id: string, _updates: unknown) => true), + classify: vi.fn(), + meter: vi.fn(), +})); + +vi.mock("@app/contexts/AppConfigContext", () => ({ + useAppConfig: () => ({ config: {}, loading: mocks.configLoading }), +})); +vi.mock("@app/hooks/useClassificationEnabled", () => ({ + useClassificationEnabled: () => true, +})); +vi.mock("@app/hooks/useAiEngineEnabled", () => ({ + useAiEngineEnabled: () => false, +})); +vi.mock("@app/hooks/usePolicies", () => ({ + usePolicies: () => ({ + policies: { + classification: { + configured: true, + status: "active", + backendId: "backend-classification", + sources: ["editor"], + }, + }, + }), +})); +vi.mock("@app/contexts/FileContext", () => ({ + useAllFiles: () => ({ fileStubs: mocks.workspace }), + useFileManagement: () => ({ + updateStirlingFileStub: mocks.updateStirlingFileStub, + }), +})); +vi.mock("@app/contexts/IndexedDBContext", () => ({ + useIndexedDB: () => ({ bumpRevision: mocks.bumpRevision }), +})); +vi.mock("@app/services/fileStorage", () => ({ + fileStorage: { + getStirlingFile: (id: string) => mocks.getStirlingFile(id), + updateFileMetadata: (id: string, updates: unknown) => + mocks.updateFileMetadata(id, updates), + }, +})); +vi.mock("@app/services/heuristic/heuristicClassification", () => ({ + classifyFileHeuristically: (file: File) => mocks.classify(file), +})); +vi.mock("@app/services/classificationMeter", () => ({ + meterClassificationRun: (payload: unknown) => mocks.meter(payload), +})); + +import { useClientSideClassification } from "@app/components/policies/useClientSideClassification"; + +// Run idle callbacks immediately so batches start without timer waits. +vi.stubGlobal("requestIdleCallback", (cb: () => void) => { + cb(); + return 1; +}); +vi.stubGlobal("cancelIdleCallback", () => {}); + +const stub = (id: string, extra: Partial = {}): TestStub => ({ + id, + name: `${id}.pdf`, + lastModified: 1, + ...extra, +}); + +const fakeFile = (id: string) => new File([id], `${id}.pdf`); + +describe("useClientSideClassification delivery", () => { + beforeEach(() => { + localStorage.clear(); + resetPolicyRuns(); + mocks.workspace = []; + mocks.configLoading = false; + mocks.updateStirlingFileStub.mockClear(); + mocks.bumpRevision.mockClear(); + mocks.updateFileMetadata.mockClear(); + mocks.meter.mockClear(); + mocks.getStirlingFile.mockReset(); + mocks.getStirlingFile.mockImplementation(async (id: string) => + fakeFile(id), + ); + mocks.classify.mockReset(); + }); + + it("classifies pending uploads, writes labels, and meters once per file", async () => { + mocks.workspace = [stub("a"), stub("b")]; + mocks.classify.mockImplementation(async (file: File) => ({ + labels: [file.name.startsWith("a") ? "invoice" : "resume"], + })); + + renderHook(() => useClientSideClassification()); + + await waitFor(() => + expect(mocks.updateStirlingFileStub).toHaveBeenCalledTimes(2), + ); + expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("a", { + classificationLabels: ["invoice"], + }); + expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("b", { + classificationLabels: ["resume"], + }); + expect(mocks.meter).toHaveBeenCalledTimes(2); + expect(mocks.bumpRevision).toHaveBeenCalled(); + }); + + it("delivers a result computed while the effect re-fired mid-batch (upload-wave race)", async () => { + let resolveA!: (v: { labels: string[] }) => void; + const gateA = new Promise<{ labels: string[] }>((r) => (resolveA = r)); + mocks.classify.mockImplementation((file: File) => + file.name.startsWith("a") ? gateA : Promise.resolve({ labels: ["nda"] }), + ); + mocks.workspace = [stub("a")]; + + const { rerender } = renderHook(() => useClientSideClassification()); + await waitFor(() => expect(mocks.classify).toHaveBeenCalledTimes(1)); + + // A new upload mid-classify re-fires the effect and cancels the in-flight + // batch; a's already-computed result must still be delivered. + mocks.workspace = [stub("a"), stub("b")]; + rerender(); + + resolveA({ labels: ["purchase-order"] }); + await waitFor(() => + expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("a", { + classificationLabels: ["purchase-order"], + }), + ); + // The newly-arrived file classifies too, and neither is double-classified. + await waitFor(() => + expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("b", { + classificationLabels: ["nda"], + }), + ); + expect(mocks.classify).toHaveBeenCalledTimes(2); + expect(mocks.meter).toHaveBeenCalledTimes(2); + }); + + it("persists a definitive [] verdict for an unlabelled file and does not retry it", async () => { + mocks.workspace = [stub("plain")]; + mocks.classify.mockResolvedValue({ labels: [] }); + + renderHook(() => useClientSideClassification()); + + await waitFor(() => + expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("plain", { + classificationLabels: [], + }), + ); + expect(mocks.classify).toHaveBeenCalledTimes(1); + expect(mocks.meter).toHaveBeenCalledTimes(1); + }); + + it("heals a previously-dispatched file whose result was lost, without re-metering", async () => { + // A past session classified + metered this file but the delivery was lost. + markDispatched("classification", "lost"); + mocks.workspace = [stub("lost")]; + mocks.classify.mockResolvedValue({ labels: ["bank-statement"] }); + + renderHook(() => useClientSideClassification()); + + await waitFor(() => + expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("lost", { + classificationLabels: ["bank-statement"], + }), + ); + expect(mocks.meter).not.toHaveBeenCalled(); + }); + + it("leaves an unreadable file undelivered (no verdict, no meter) so it can retry", async () => { + // An extraction failure may be environmental, so it must never poison the + // file with a persisted verdict. The read path deliberately warns. + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + mocks.workspace = [stub("corrupt")]; + mocks.classify.mockRejectedValue(new Error("bad pdf")); + + renderHook(() => useClientSideClassification()); + + await waitFor(() => + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("corrupt.pdf: could not be read"), + expect.any(Error), + ), + ); + expect(mocks.classify).toHaveBeenCalledTimes(1); // claimed: once per session + expect(mocks.updateStirlingFileStub).not.toHaveBeenCalled(); + expect(mocks.updateFileMetadata).not.toHaveBeenCalled(); + expect(mocks.meter).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it("waits for app-config before classifying (AI flag unknown = possible double-run)", async () => { + // While the config loads, aiEnabled reads false even on an AI-on tenant; classifying + // then would race the server-side classify policy and double-bill the same files. + mocks.configLoading = true; + mocks.workspace = [stub("early")]; + mocks.classify.mockResolvedValue({ labels: ["invoice"] }); + + const { rerender } = renderHook(() => useClientSideClassification()); + await new Promise((r) => setTimeout(r, 50)); + expect(mocks.classify).not.toHaveBeenCalled(); + + // Config resolves (AI stays off): the pending file classifies normally. + mocks.configLoading = false; + rerender(); + await waitFor(() => + expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("early", { + classificationLabels: ["invoice"], + }), + ); + }); + + it("skips tool outputs and already-labelled files", async () => { + mocks.workspace = [ + stub("derived", { derivedFromTool: true }), + stub("done", { classificationLabels: ["invoice"] }), + stub("verdict", { classificationLabels: [] }), + ]; + + renderHook(() => useClientSideClassification()); + + // Nothing to classify; give the (immediate) idle path a beat to prove it. + await new Promise((r) => setTimeout(r, 50)); + expect(mocks.classify).not.toHaveBeenCalled(); + }); +}); diff --git a/frontend/editor/src/proprietary/components/policies/useClientSideClassification.ts b/frontend/editor/src/proprietary/components/policies/useClientSideClassification.ts new file mode 100644 index 0000000000..aac4b7e3da --- /dev/null +++ b/frontend/editor/src/proprietary/components/policies/useClientSideClassification.ts @@ -0,0 +1,207 @@ +// With the AI engine off, the Classification policy runs here in the browser: +// each upload is labelled by the heuristic engine and metered for billing parity. + +import { useEffect, useRef, useState } from "react"; +import { useAllFiles, useFileManagement } from "@app/contexts/FileContext"; +import { useAppConfig } from "@app/contexts/AppConfigContext"; +import { useIndexedDB } from "@app/contexts/IndexedDBContext"; +import { fileStorage } from "@app/services/fileStorage"; +import { useClassificationEnabled } from "@app/hooks/useClassificationEnabled"; +import { useAiEngineEnabled } from "@app/hooks/useAiEngineEnabled"; +import { scheduleIdle } from "@app/utils/scheduleIdle"; +import { usePolicies } from "@app/hooks/usePolicies"; +import { classifyFileHeuristically } from "@app/services/heuristic/heuristicClassification"; +import { meterClassificationRun } from "@app/services/classificationMeter"; +import { + isDispatched, + markDispatched, +} from "@app/components/policies/policyRunStore"; +import type { FileId } from "@app/types/file"; +import type { StirlingFile, StirlingFileStub } from "@app/types/fileContext"; + +/** The category id of the Classification policy (see policyDefinitions). */ +const CLASSIFICATION_CATEGORY = "classification"; +/** Files classified per idle pass, so a large library drains over several ticks. */ +const CLASSIFY_BATCH = 3; +/** How long to wait for an upload's bytes to land in IndexedDB (20 × 250ms ≈ 5s). + * The stub can surface in the file list a beat before its bytes are committed. */ +const FILE_WAIT_TRIES = 20; +const FILE_WAIT_MS = 250; + +/** localStorage flag: set to "true" for a full per-file scoring breakdown in the console. */ +const DEBUG_FLAG = "stirling-classification-debug"; + +function isClassificationDebug(): boolean { + try { + return localStorage.getItem(DEBUG_FLAG) === "true"; + } catch { + return false; + } +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export function useClientSideClassification(): void { + const { fileStubs } = useAllFiles(); + const { updateStirlingFileStub } = useFileManagement(); + const { bumpRevision } = useIndexedDB(); + const { policies } = usePolicies(); + const classificationEnabled = useClassificationEnabled(); + const aiEnabled = useAiEngineEnabled(); + // While app-config loads, aiEnabled reads false even on AI-on tenants; classifying + // in that window would double-run (and double-bill) files the server also labels. + const { loading: configLoading } = useAppConfig(); + // Files claimed this session, keyed id+lastModified so a new version is retried once. A claim is + // taken synchronously right before classifying, so overlapping batches never double-classify. + const claimed = useRef>(new Set()); + // Bumped after each batch to drain the next one. + const [tick, setTick] = useState(0); + + const policy = policies[CLASSIFICATION_CATEGORY]; + // Only when the admin has an active Classification policy - the same gate the AI path uses. + const active = Boolean( + policy?.configured && + policy.status === "active" && + policy.backendId && + (!policy.sources || + policy.sources.length === 0 || + policy.sources.includes("editor")), + ); + + useEffect(() => { + if (configLoading || !classificationEnabled || aiEnabled || !active) { + return; + } + const claimKey = (s: StirlingFileStub) => + `${s.id as string}:${s.lastModified ?? 0}`; + // null labels = never delivered, retried here; [] = definitive no-label verdict. + const pending = fileStubs + .filter( + (s) => + !s.derivedFromTool && + s.classificationLabels == null && + !claimed.current.has(claimKey(s)), + ) + .slice(0, CLASSIFY_BATCH); + if (pending.length === 0) return; + let cancelled = false; + const cancelIdle = scheduleIdle(() => { + // Superseded before starting: the newer effect instance owns the queue. + if (cancelled) return; + void (async () => { + let wrote = false; + for (const stub of pending) { + const key = claimKey(stub); + // Re-validate at execution time - another batch may have claimed it since. + if (claimed.current.has(key)) continue; + claimed.current.add(key); + const labels = await classifyStub(stub.id as FileId, stub.name); + // Bytes never landed (file removed mid-wait): leave undelivered so a + // reload (or new version) retries; the claim stops churn this session. + if (labels == null) continue; + // Deliver unconditionally - a re-render must never discard a computed + // (and already metered) result. Writes are idempotent. + updateStirlingFileStub(stub.id as FileId, { + classificationLabels: labels, + }); + const ok = await fileStorage.updateFileMetadata(stub.id as FileId, { + classificationLabels: labels, + }); + if (ok) wrote = true; + } + if (wrote) bumpRevision(); + // Drain the next batch; the terminal pass finds nothing pending and stops. + setTick((n) => n + 1); + })(); + }); + return () => { + cancelled = true; + cancelIdle(); + }; + }, [ + fileStubs, + active, + classificationEnabled, + aiEnabled, + configLoading, + updateStirlingFileStub, + bumpRevision, + tick, + ]); +} + +/** Classify one file, metering exactly once; null = no verdict, retried later. */ +async function classifyStub( + fileId: FileId, + fileName: string, +): Promise { + let file: StirlingFile | null = null; + for (let i = 0; i < FILE_WAIT_TRIES; i++) { + file = await fileStorage.getStirlingFile(fileId).catch(() => null); + if (file) break; + await delay(FILE_WAIT_MS); + } + if (!file) { + console.warn( + `[Classify] ${fileName}: bytes never arrived in storage; will retry on next load`, + ); + return null; + } + const debug = isClassificationDebug(); + const startedAt = performance.now(); + try { + const result = await classifyFileHeuristically(file, { explain: debug }); + const { labels } = result; + const alreadyMetered = isDispatched(CLASSIFICATION_CATEGORY, fileId); + const ms = Math.round(performance.now() - startedAt); + const verdict = + labels.length > 0 + ? labels.join(", ") + : result.isEnglish + ? "no label" + : "no label (not English)"; + console.debug( + `[Classify] ${fileName} -> ${verdict} (${result.confidence}, score ${result.score}, ${ms}ms)` + + (alreadyMetered ? " [heal: not re-metered]" : ""), + ); + if (debug && result.explain) logExplanation(fileName, result); + // Meter on the first classification only; a healing re-run of an undelivered + // result (already dispatched) is not a new billable run. + if (!alreadyMetered) { + meterClassificationRun({ + policyName: "Classification", + documentCount: 1, + labels, + }); + } + markDispatched(CLASSIFICATION_CATEGORY, fileId); + return labels; + } catch (err) { + // Never persist a verdict for an unreadable file - the failure may be + // environmental, so it must stay eligible to retry (and meter) later. + console.warn(`[Classify] ${fileName}: could not be read, will retry`, err); + return null; + } +} + +/** Full scoring breakdown, one collapsed console group per file (debug flag only). */ +function logExplanation( + fileName: string, + result: Awaited>, +): void { + const ex = result.explain; + if (!ex) return; + console.groupCollapsed( + `[Classify] ${fileName} scoring (english=${ex.isEnglish}, lowText=${ex.lowText})`, + ); + if (ex.candidates.length === 0) { + console.log("no label scored above zero"); + } + for (const c of ex.candidates) { + console.log( + `${c.id}${c.emit ? "" : " (suppressed)"}: score ${c.score}, ${c.distinct} distinct signals`, + ); + for (const s of c.signals) console.log(` ${s}`); + } + console.groupEnd(); +} diff --git a/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.batch.test.tsx b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.batch.test.tsx index e936d97d00..5837bfb285 100644 --- a/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.batch.test.tsx +++ b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.batch.test.tsx @@ -47,6 +47,11 @@ const mocks = vi.hoisted(() => ({ consumeFiles: vi.fn(), })); +// Classification chains server-side only when the AI engine is on (else it runs +// client-side); this batch exercises the server chain, so force the engine on. +vi.mock("@app/hooks/useAiEngineEnabled", () => ({ + useAiEngineEnabled: () => true, +})); vi.mock("@app/contexts/FileContext", () => ({ useAllFiles: () => ({ fileStubs: mocks.workspace }), useFileManagement: () => ({ diff --git a/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.chain.test.tsx b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.chain.test.tsx index b2c2d6f5dc..b87cbda445 100644 --- a/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.chain.test.tsx +++ b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.chain.test.tsx @@ -4,6 +4,12 @@ import { renderHook, act } from "@testing-library/react"; // Two active upload policies, so the auto-run should CHAIN them: fire the first on // the upload, then the second on the first's output. Stub the contexts + network so // we can drive the dispatch against the REAL run store. +// Controllable AI-engine flag: on by default so classification chains server-side; one +// test flips it off to assert classification is kept OUT of the server chain. +const aiEnabled = vi.hoisted(() => ({ value: true })); +vi.mock("@app/hooks/useAiEngineEnabled", () => ({ + useAiEngineEnabled: () => aiEnabled.value, +})); const fileStubs: { id: string; name: string; derivedFromTool?: boolean }[] = []; vi.mock("@app/contexts/FileContext", () => ({ useAllFiles: () => ({ fileStubs }), @@ -66,6 +72,7 @@ beforeEach(() => { localStorage.clear(); resetPolicyRuns(); setFileStubs([]); + aiEnabled.value = true; runStored.mockReset(); getFile.mockReset(); getFile.mockResolvedValue({ size: 100 } as never); @@ -116,4 +123,23 @@ describe("auto-run ordered chaining", () => { // The next policy (order 1) fires on the first policy's output, not the original. expect(runStored).toHaveBeenCalledWith("backend-cls", [{ size: 100 }]); }); + + it("keeps classification out of the server chain when the AI engine is off", async () => { + // AI off: classification runs client-side (useClientSideClassification), so the + // server chain must skip it - only the normal (security) policy dispatches. + aiEnabled.value = false; + setFileStubs([{ id: "file-1", name: "doc.pdf" }]); + runStored.mockResolvedValue("run-sec"); + + renderHook(() => usePolicyAutoRun()); + await act(async () => { + await vi.advanceTimersByTimeAsync(1); + }); + + expect(runStored).toHaveBeenCalledWith("backend-sec", [{ size: 100 }]); + expect(runStored).not.toHaveBeenCalledWith( + "backend-cls", + expect.anything(), + ); + }); }); diff --git a/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.ts b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.ts index a876a785ff..ca096b6792 100644 --- a/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.ts +++ b/frontend/editor/src/proprietary/components/policies/usePolicyAutoRun.ts @@ -42,6 +42,7 @@ import { readClassificationLabelsFromFile } from "@app/services/fileClassificati import type { StirlingFile, StirlingFileStub } from "@app/types/fileContext"; import type { PoliciesByCategory } from "@app/types/policies"; import { usePolicies } from "@app/hooks/usePolicies"; +import { useAiEngineEnabled } from "@app/hooks/useAiEngineEnabled"; import { addReconciledRun, dispatchKey, @@ -128,6 +129,7 @@ export function usePolicyAutoRun(): void { const { consumeFiles } = useFileContext(); const { bumpRevision } = useIndexedDB(); const { policies } = usePolicies(); + const aiEnabled = useAiEngineEnabled(); const runs = usePolicyRuns(); // Live view of the workspace files, read inside the import effect WITHOUT making // it a dependency. The silent consume that delivers an output mutates fileStubs, @@ -161,18 +163,21 @@ export function usePolicyAutoRun(): void { () => Object.entries(policies) .filter( - ([, s]) => + ([id, s]) => s.configured && s.status === "active" && s.backendId && (!s.sources || s.sources.length === 0 || s.sources.includes("editor")) && - (s.runOn ?? "upload") === "upload", + (s.runOn ?? "upload") === "upload" && + // Non-AI systems classify in the browser (useClientSideClassification), so keep the + // Classification policy out of the server chain when the AI engine is off. + !(id === "classification" && !aiEnabled), ) .sort(([, a], [, b]) => (a.order ?? 0) - (b.order ?? 0)) .map(([id]) => id), - [policies], + [policies, aiEnabled], ); // Runs whose chain-continuation we've already handled this session, so the next diff --git a/frontend/editor/src/saas/components/shared/FileSidebarGroupControls.css b/frontend/editor/src/proprietary/components/shared/FileSidebarGroupControls.css similarity index 100% rename from frontend/editor/src/saas/components/shared/FileSidebarGroupControls.css rename to frontend/editor/src/proprietary/components/shared/FileSidebarGroupControls.css diff --git a/frontend/editor/src/saas/components/shared/FileSidebarGroupControls.tsx b/frontend/editor/src/proprietary/components/shared/FileSidebarGroupControls.tsx similarity index 100% rename from frontend/editor/src/saas/components/shared/FileSidebarGroupControls.tsx rename to frontend/editor/src/proprietary/components/shared/FileSidebarGroupControls.tsx diff --git a/frontend/editor/src/saas/components/shared/fileSidebarGrouping.test.ts b/frontend/editor/src/proprietary/components/shared/fileSidebarGrouping.test.ts similarity index 100% rename from frontend/editor/src/saas/components/shared/fileSidebarGrouping.test.ts rename to frontend/editor/src/proprietary/components/shared/fileSidebarGrouping.test.ts diff --git a/frontend/editor/src/saas/components/shared/fileSidebarGrouping.tsx b/frontend/editor/src/proprietary/components/shared/fileSidebarGrouping.tsx similarity index 71% rename from frontend/editor/src/saas/components/shared/fileSidebarGrouping.tsx rename to frontend/editor/src/proprietary/components/shared/fileSidebarGrouping.tsx index 659efbdf31..baf3d812cb 100644 --- a/frontend/editor/src/saas/components/shared/fileSidebarGrouping.tsx +++ b/frontend/editor/src/proprietary/components/shared/fileSidebarGrouping.tsx @@ -1,4 +1,5 @@ -// Classification override of the Files-sidebar grouping seam: Recent, one group per VISIBLE category (the fixed, shared label families; each can be hidden device-local), then Other for files in none of those. Labels are cached on the stub via a lazy metadata backfill so grouping stays cheap. +// Classification override of the Files-sidebar grouping seam: Recent, one group +// per visible category, then Other. Labels cache onto stubs via a lazy backfill. import { useEffect, @@ -18,6 +19,7 @@ import { subscribeSidebarCategories, } from "@app/services/fileSidebarCategories"; import { buildLabelGroups } from "@app/components/shared/fileSidebarGroupingLogic"; +import { scheduleIdle } from "@app/utils/scheduleIdle"; import type { FileId } from "@app/types/file"; import type { StirlingFileStub } from "@app/types/fileContext"; import type { FileSidebarGroup } from "@core/components/shared/fileSidebarGrouping"; @@ -36,33 +38,22 @@ const BACKFILL_BATCH = 3; /** Recheck delay when the backfill yields to an active policy wave. */ const BACKFILL_BUSY_RETRY_MS = 4000; -/** Schedule work for the browser's idle time (or soon after, as a fallback). */ -function scheduleIdle(task: () => void): () => void { - if (typeof requestIdleCallback === "function") { - const handle = requestIdleCallback(task, { timeout: 2000 }); - return () => cancelIdleCallback(handle); - } - const timer = window.setTimeout(task, 200); - return () => window.clearTimeout(timer); -} - export function useFileSidebarGroups( stubs: StirlingFileStub[], ): FileSidebarGroup[] | null { const { t } = useTranslation(); - // Classification off (AI disabled) → no grouping at all: return the flat list - // like core, and don't fetch team labels or backfill from metadata. Gates the - // whole feature so an AI-off SaaS tenant sees no Recent/Other/category chrome. + // Classification off (core): flat list, no category fetch or backfill. const enabled = useClassificationEnabled(); const { bumpRevision } = useIndexedDB(); - // Attempted reads keyed by id+lastModified: a re-classified file (new version bumps lastModified) is re-read and leaves "Other" on its own, while a truly-unlabelled file keeps a stable key and is read once. + // Reads keyed by id+lastModified, so a new file version is re-read exactly once. const attempted = useRef>(new Set()); const attemptKey = (s: StirlingFileStub) => `${s.id as string}:${s.lastModified ?? 0}`; // Bumped to re-attempt a backfill pass that yielded to an active policy wave. const [retryTick, setRetryTick] = useState(0); - // Fallback for files that arrive with labels already in metadata but no policy delivery (imports/shares): read+cache a few per idle pass, yielding while a policy wave is in flight since those stubs get stamped on delivery anyway. + // Backfill labels from file metadata onto stubs, a few per idle pass; yields + // while a policy wave is in flight. The heuristic path stamps stubs directly. useEffect(() => { if (!enabled) return; const pending = stubs @@ -75,7 +66,8 @@ export function useFileSidebarGroups( let retryTimer: number | undefined; const cancelIdle = scheduleIdle(() => { if (cancelled) return; - // Deliveries stamp labels during a wave, so reading now is wasted parsing; recheck after it (a timer self-heals when a wave ends without a stubs change). + // Reading during a wave is wasted parsing; recheck after it. The timer + // self-heals when a wave ends without a stubs change. if (hasInFlightPolicyRuns()) { retryTimer = window.setTimeout(() => { if (!cancelled) setRetryTick((n) => n + 1); @@ -85,9 +77,9 @@ export function useFileSidebarGroups( void (async () => { let wrote = false; for (const stub of pending) { - attempted.current.add(attemptKey(stub)); const labels = await readStubClassificationLabels(stub); if (cancelled) return; + attempted.current.add(attemptKey(stub)); if (labels) { const ok = await fileStorage.updateFileMetadata(stub.id as FileId, { classificationLabels: labels, diff --git a/frontend/editor/src/saas/components/shared/fileSidebarGroupingLogic.ts b/frontend/editor/src/proprietary/components/shared/fileSidebarGroupingLogic.ts similarity index 100% rename from frontend/editor/src/saas/components/shared/fileSidebarGroupingLogic.ts rename to frontend/editor/src/proprietary/components/shared/fileSidebarGroupingLogic.ts diff --git a/frontend/editor/src/proprietary/data/policyDefinitions.tsx b/frontend/editor/src/proprietary/data/policyDefinitions.tsx index f3377c028e..e725dff68b 100644 --- a/frontend/editor/src/proprietary/data/policyDefinitions.tsx +++ b/frontend/editor/src/proprietary/data/policyDefinitions.tsx @@ -43,8 +43,6 @@ export const POLICY_CATEGORIES: PolicyCategory[] = [ label: "Classification", icon: policyCategoryIcon("classification", ICON_SX), desc: "Identify each document's type on upload and tag its metadata for filing and search.", - // Needs the AI engine to classify; hidden from the policy list when it's off. - requiresAiEngine: true, }, { id: "compliance", diff --git a/frontend/editor/src/proprietary/hooks/useClassificationEnabled.ts b/frontend/editor/src/proprietary/hooks/useClassificationEnabled.ts new file mode 100644 index 0000000000..bf37cf40da --- /dev/null +++ b/frontend/editor/src/proprietary/hooks/useClassificationEnabled.ts @@ -0,0 +1,6 @@ +// Classification is available on every proprietary-based build: the classify +// policy labels server-side with AI on, the in-browser heuristic labels with AI off. + +export function useClassificationEnabled(): boolean { + return true; +} diff --git a/frontend/editor/src/proprietary/services/classificationMeter.ts b/frontend/editor/src/proprietary/services/classificationMeter.ts new file mode 100644 index 0000000000..036b7ef242 --- /dev/null +++ b/frontend/editor/src/proprietary/services/classificationMeter.ts @@ -0,0 +1,27 @@ +// Meters an in-browser (non-AI) classification run for billing/audit parity with +// the server-side classify path. Fire-and-forget; failures never block the user. + +import apiClient from "@app/services/apiClient"; +import { getPolicyOutputBaseUrl } from "@app/services/policyOutputBaseUrl"; +import { resolvePolicyRunTarget } from "@app/services/policyApi"; + +interface ClassifyMeterPayload { + /** Policy name for the audit-trail label; defaults to "Classification" server-side. */ + policyName?: string; + /** Documents covered by this meter call (defaults to 1 server-side). */ + documentCount?: number; + /** Resolved labels, carried for the audit record. */ + labels?: string[]; +} + +/** Meter a completed client-side classification. Does not throw and is not awaited by callers. */ +export function meterClassificationRun(payload: ClassifyMeterPayload): void { + const base = getPolicyOutputBaseUrl(resolvePolicyRunTarget()); + void apiClient + .post(`${base}/api/v1/policies/classify/meter`, payload, { + suppressErrorToast: true, + }) + .catch(() => { + // Best-effort billing; the classification already succeeded in the browser. + }); +} diff --git a/frontend/editor/src/proprietary/services/heuristic/heuristicClassification.ts b/frontend/editor/src/proprietary/services/heuristic/heuristicClassification.ts new file mode 100644 index 0000000000..180d7ea409 --- /dev/null +++ b/frontend/editor/src/proprietary/services/heuristic/heuristicClassification.ts @@ -0,0 +1,18 @@ +// Client-side classification entry point: load rules, extract the PDF, classify. + +import { + ensureRulesLoaded, + classifyHeuristic, +} from "@app/services/heuristic/heuristicEngine"; +import { extractHeuristicDoc } from "@app/services/heuristic/heuristicExtractor"; +import type { HeuristicResult } from "@app/services/heuristic/types"; + +/** Classify a file in the browser. Throws if extraction fails (unreadable / non-PDF). */ +export async function classifyFileHeuristically( + file: File, + opts?: { explain?: boolean }, +): Promise { + await ensureRulesLoaded(); + const doc = await extractHeuristicDoc(file, file.name); + return classifyHeuristic(doc, opts); +} diff --git a/frontend/editor/src/proprietary/services/heuristic/heuristicEngine.corpus.test.ts b/frontend/editor/src/proprietary/services/heuristic/heuristicEngine.corpus.test.ts new file mode 100644 index 0000000000..db63823212 --- /dev/null +++ b/frontend/editor/src/proprietary/services/heuristic/heuristicEngine.corpus.test.ts @@ -0,0 +1,2423 @@ +// Broad-corpus accuracy gate: one realistic, natural-prose specimen per document +// type, asserted on the real engine + rules pack. + +import { beforeAll, describe, expect, it } from "vitest"; +import { + classifyHeuristic, + ensureRulesLoaded, +} from "@app/services/heuristic/heuristicEngine"; +import type { HeuristicDoc } from "@app/services/heuristic/types"; + +beforeAll(async () => { + await ensureRulesLoaded(); +}); + +interface Case { + /** Expected top label; "" asserts the document stays unlabelled. */ + expect: string; + file: string; + pages?: number; + /** Replace every space with U+00A0, as some pdf.js text layers do. */ + nbsp?: boolean; + /** Info-dict metadata (lowercase keys), e.g. a generator's producer string. */ + meta?: Record; + title: string; + body: string[]; +} + +const run = (c: Case) => { + let body = c.body.join("\n"); + let title = c.title; + if (c.nbsp) { + body = body.replace(/ /g, "\u00A0"); + title = title.replace(/ /g, "\u00A0"); + } + const doc: HeuristicDoc = { + fileName: c.file, + pageCount: c.pages ?? 1, + meta: c.meta ?? {}, + titleZone: title, + firstZone: body, + allZone: body, + }; + return classifyHeuristic(doc); +}; + +const CASES: Case[] = [ + { + expect: "invoice", + file: "2024-014.pdf", + title: "Invoice", + body: [ + "Invoice", + "Jane Mills Design, 14 Fern Road, Bristol BS1 5TR", + "Invoice #: 2024-014 Date: 3 May 2024", + "Bill To: Harbour Cafe Ltd", + "Logo refresh and brand guidelines ... $950.00", + "Menu redesign (2 rounds) ... $250.00", + "Total: $1,200.00", + "Payment terms: Net 30. Bank transfer to sort code 20-00-00, account 55667788.", + ], + }, + { + expect: "receipt", + file: "20240506_183212.pdf", + title: "Greenway Grocers", + body: [ + "Greenway Grocers", + "112 Station Road 06/05/2024 18:32", + "Oat milk 1L 2.20", + "Sourdough loaf 3.80", + "Free range eggs x12 4.10", + "TOTAL 10.10", + "VISA ****4421 Contactless", + "Approval code: 08841", + "Thank you for shopping with us", + ], + }, + { + expect: "bank-statement", + file: "statement_march.pdf", + pages: 3, + title: "Your account statement", + body: [ + "Your account statement", + "Statement period: 1 March 2024 - 31 March 2024", + "Sort code: 40-12-31 Account number: 71002233", + "Opening balance 2,140.55", + "2 Mar Faster payment - RENT -950.00", + "5 Mar Salary ACME LTD +2,850.00", + "12 Mar Direct debit - CITY POWER -84.20", + "19 Mar Card purchase - GREENWAY -10.10", + "Closing balance 3,946.25", + "Money in: 2,850.00 Money out: 1,044.30", + ], + }, + { + expect: "statement-of-account", + file: "april_card.pdf", + pages: 2, + title: "Monthly statement", + body: [ + "Monthly statement", + "Account ending in 8891", + "Previous balance: $412.90 New balance: $388.15", + "Minimum payment due: $25.00 Payment due date: 28 April 2024", + "Credit limit: $4,500 Available credit: $4,111.85", + "Annual percentage rate (APR): 22.9% variable", + "Late payment warning: if we do not receive your minimum payment by the date listed above,", + "you may have to pay a late fee of up to $39.", + "Interest charge calculation: your APR is applied to the daily balance.", + ], + }, + { + expect: "purchase-order", + file: "PO-88412.pdf", + title: "Purchase Order", + body: [ + "Purchase Order", + "P.O. Number: PO-88412 Date: 11 June 2024", + "Vendor: Brightline Packaging Ltd", + "Ship To: Unit 4, Riverside Industrial Estate, Leeds LS10 1AB", + "Qty Ordered Description Unit Price", + "500 Corrugated boxes 40x30cm 0.85", + "200 Kraft void fill rolls 3.20", + "Delivery date: on or before 25 June 2024.", + "All invoices must quote the PO number. No PO, no pay.", + ], + }, + { + expect: "quote", + file: "landscaping_quote.pdf", + title: "Quotation", + body: [ + "Quotation", + "Quotation number: Q-2231 Date: 8 April 2024", + "Prepared for: Mr and Mrs Aldridge", + "We are pleased to quote for the following garden works:", + "Re-turf rear lawn (85 sqm) ... 1,540.00", + "Patio pressure wash and re-point ... 380.00", + "Total (inc VAT): 1,920.00", + "This quote is valid for 30 days from the date above.", + ], + }, + { + expect: "expense-report", + file: "march_expenses.pdf", + title: "Expense report", + body: [ + "Expense report", + "Employee: D. Okafor Department: Field Sales", + "Period: 1-31 March 2024", + "04 Mar Client lunch, Manchester 38.50", + "11 Mar Rail ticket, London return 86.00", + "18 Mar Hotel, 1 night (conference) 129.00", + "Per diem (2 days) 90.00", + "Total reimbursable: 343.50", + "Receipts attached. Approved by: S. Hughes", + ], + }, + { + expect: "payslip", + file: "payslip_2024_05.pdf", + title: "Payslip", + body: [ + "Payslip", + "Employee: R. Patel Employee no: 00482 Pay date: 31 May 2024", + "Payments Deductions", + "Basic pay 2,916.67 PAYE tax 382.40", + " National Insurance 231.15", + " Pension (5%) 145.83", + "Gross pay: 2,916.67 Net pay: 2,157.29", + "Tax code: 1257L NI category: A", + "Year to date: gross 14,583.35, tax 1,912.00", + ], + }, + { + expect: "tax-form", + file: "W2_2023.pdf", + title: "Wage and Tax Statement", + body: [ + "Form W-2 Wage and Tax Statement 2023", + "Employer: Lakeshore Analytics Inc, EIN 84-2210043", + "Employee: M. Torres SSN XXX-XX-4821", + "1 Wages, tips, other compensation 72,450.00", + "2 Federal income tax withheld 9,812.00", + "3 Social security wages 72,450.00", + "4 Social security tax withheld 4,491.90", + "5 Medicare wages and tips 72,450.00", + "Copy B - to be filed with employee's federal tax return.", + ], + }, + { + expect: "nda", + file: "acme_nda_signed.pdf", + pages: 4, + title: "Non-Disclosure Agreement", + body: [ + "Non-Disclosure Agreement", + "This Agreement is made between Acme Ltd (the Disclosing Party) and the undersigned", + "recipient (the Receiving Party).", + "1. The Receiving Party shall hold all Confidential Information in strict confidence.", + "2. Confidential Information excludes information that is or becomes publicly available.", + "3. This Agreement shall remain in force for five years from the date of signature.", + "Signed for and on behalf of the parties:", + ], + }, + { + expect: "contract", + file: "signed_agreement_final_v3.pdf", + pages: 9, + title: "Agreement", + body: [ + "Agreement", + "This Agreement is entered into as of 1 July 2024 between Corven Holdings Ltd and", + "Meridian Supplies Ltd.", + "WITNESSETH: in consideration of the mutual covenants contained herein, the parties", + "agree as follows:", + "1. Definitions. Capitalised terms have the meanings set out in Schedule 1.", + "5. Limitation of liability. Neither party shall be liable for indirect losses.", + "9. Entire agreement. This Agreement constitutes the entire agreement between the parties.", + "IN WITNESS WHEREOF the parties have executed this Agreement.", + ], + }, + { + expect: "employment-contract", + file: "contract_r_patel.pdf", + pages: 7, + title: "Contract of Employment", + body: [ + "Contract of Employment", + "This contract of employment is made between Northgate Logistics Ltd (the Employer)", + "and R. Patel (the Employee).", + "1. Commencement. Your employment begins on 2 September 2024.", + "2. Probationary period. The first six months are a probationary period.", + "3. Hours. The Employee shall work 37.5 hours per week.", + "4. Holiday. 25 days per annum plus public holidays.", + "5. Notice. Either party may terminate on eight weeks written notice.", + ], + }, + { + expect: "lease-agreement", + file: "flat2_tenancy.pdf", + pages: 12, + title: "Assured Shorthold Tenancy Agreement", + body: [ + "Assured Shorthold Tenancy Agreement", + "Property: Flat 2, 18 Camden Row, London NW1 8QP", + "Landlord: H. Berger Tenant: L. Novak", + "Term: 12 months commencing 1 August 2024", + "Rent: 1,650 per calendar month, payable in advance", + "Security deposit: 1,903.85, protected with the Deposit Protection Service", + "The tenant shall not sublet the property without written consent.", + ], + }, + { + expect: "terms-and-conditions", + file: "tos.pdf", + pages: 14, + title: "Terms of Service", + body: [ + "Terms of Service", + "Last updated: 12 February 2024", + "By accessing or using the Kestrel platform you agree to be bound by these terms.", + "1. Your account. You are responsible for safeguarding your credentials.", + "2. Acceptable use. You must not misuse the services or interfere with their operation.", + "3. Termination. We may suspend or terminate access for breach of these terms.", + "4. Disclaimers. The services are provided on an as-is basis.", + ], + }, + { + expect: "privacy-policy", + file: "privacy.pdf", + pages: 8, + title: "Privacy Notice", + body: [ + "Privacy Notice", + "This notice explains how Kestrel Ltd, as data controller, handles your information.", + "Personal data we collect: account details, usage data, and support correspondence.", + "How we use your information: to provide the service, for billing, and for security.", + "Your rights include access, rectification, and the right to erasure.", + "We retain personal data only as long as necessary for the purposes described.", + "Contact our data protection officer at privacy@kestrel.example.", + ], + }, + { + expect: "will", + file: "last_will.pdf", + pages: 5, + title: "Last Will and Testament", + body: [ + "Last Will and Testament", + "I, Margaret Ellen Shaw, being of sound mind, declare this to be my last will and", + "testament, revoking all former wills.", + "1. I appoint my daughter, C. Shaw, as executor of this will.", + "2. I give, devise and bequeath my residuary estate to my children in equal shares,", + "per stirpes.", + "Signed by the testator in our presence, who in her presence signed as witnesses.", + ], + }, + { + expect: "power-of-attorney", + file: "lpa_finance.pdf", + pages: 10, + title: "Lasting Power of Attorney", + body: [ + "Lasting Power of Attorney", + "Property and financial affairs", + "Donor: A. Whitfield Attorney: J. Whitfield", + "I appoint my attorney to make decisions about my property and financial affairs.", + "This lasting power of attorney is registered with the Office of the Public Guardian.", + "Certificate provider statement: the donor understands the purpose of this LPA.", + ], + }, + { + expect: "resume", + file: "john_smith.pdf", + pages: 2, + title: "JOHN SMITH", + body: [ + "JOHN SMITH", + "Software Engineer | San Francisco, CA | john.smith@example.com | linkedin.com/in/jsmith", + "EXPERIENCE", + "Senior Software Engineer, Meridian Labs 2021 - Present", + "- Led a team of four building a real-time analytics pipeline handling 2B events/day", + "- Cut infrastructure spend 30% by consolidating streaming services", + "Software Engineer, Halcyon Systems 2017 - 2021", + "- Shipped the customer-facing dashboard used by 40k monthly users", + "EDUCATION", + "B.S. Computer Science, University of Washington, 2017", + "SKILLS", + "Go, TypeScript, Kubernetes, PostgreSQL, Kafka", + ], + }, + { + expect: "cover-letter", + file: "application_kestrel.pdf", + title: "Application for Product Designer", + body: [ + "Dear Hiring Manager,", + "I am writing to apply for the Product Designer role advertised on your careers page.", + "In my current position at Fieldnote I redesigned the onboarding flow, lifting", + "activation by 22%, and I run our fortnightly usability testing programme.", + "Kestrel's focus on accessible tools is why I am excited to apply; my portfolio", + "includes two WCAG AA redesigns shipped end to end.", + "Thank you for considering my application. I look forward to the opportunity to", + "discuss how I can contribute.", + "Yours sincerely, Amara Diallo", + ], + }, + { + expect: "offer-letter", + file: "kestrel_offer.pdf", + title: "Offer of Employment", + body: [ + "Offer of Employment", + "Dear Amara,", + "We are pleased to offer you the position of Product Designer at Kestrel Ltd.", + "Your start date will be 7 October 2024, and your annual base salary will be 68,000.", + "This offer is contingent on satisfactory references and right-to-work checks.", + "To accept this offer, please sign and return a copy of this letter by 20 September.", + "We look forward to welcoming you to the team.", + ], + }, + { + expect: "reference-letter", + file: "reference_adiallo.pdf", + title: "Letter of Recommendation", + body: [ + "Letter of Recommendation", + "To whom it may concern,", + "It is my pleasure to recommend Amara Diallo, who reported to me for three years", + "at Fieldnote as a product designer.", + "Amara combines rigorous research habits with fast, pragmatic delivery; her", + "onboarding redesign became the template for the rest of the product.", + "I highly recommend her without reservation. Please contact me with any questions.", + "Daniel Reyes, Head of Design, Fieldnote", + ], + }, + { + expect: "job-description", + file: "jd_platform_eng.pdf", + title: "Senior Platform Engineer", + body: [ + "Senior Platform Engineer", + "About the role", + "We are seeking a senior engineer to own our deployment platform and developer", + "tooling, reporting to the Head of Infrastructure.", + "Responsibilities", + "- Operate and evolve the Kubernetes-based delivery pipeline", + "Required qualifications", + "- 5+ years running production cloud infrastructure", + "Preferred qualifications", + "- Experience with progressive delivery and service meshes", + ], + }, + { + expect: "performance-review", + file: "h2_review_jsmith.pdf", + title: "Performance Review", + body: [ + "Performance Review - H2 2024", + "Employee: J. Smith Reviewer: T. Nguyen Role: Senior Engineer", + "Overall rating: exceeds expectations", + "Strengths: consistently unblocks the team; the analytics migration landed a", + "quarter early with zero data loss.", + "Areas for improvement: delegate more of the operational load; invest in", + "documenting decisions for newer engineers.", + "Goals for next period: lead the multi-region rollout; mentor two engineers.", + ], + }, + { + expect: "timesheet", + file: "week_22_hours.pdf", + title: "Timesheet", + body: [ + "Timesheet", + "Employee: K. Osei Week ending: 31 May 2024", + "Mon 7.50 Tue 7.50 Wed 8.00 Thu 7.50 Fri 6.00", + "Regular hours: 36.50", + "Overtime hours: 0.00", + "Total hours: 36.50", + "Employee signature: ____________ Approved by: L. Grant", + ], + }, + { + expect: "meeting-minutes", + file: "board_minutes_may.pdf", + pages: 3, + title: "Minutes of the Meeting", + body: [ + "Minutes of the Meeting of the Facilities Committee", + "Held 14 May 2024, Committee Room B. Present: R. Holt (chair), five members.", + "Apologies for absence: D. Cole.", + "The meeting was called to order at 18:05. A quorum was present.", + "Matters arising: the roofing tender shortlist was approved. Motion carried.", + "Any other business: none.", + "Date of next meeting: 11 June 2024. The meeting adjourned at 19:20.", + ], + }, + { + expect: "meeting-agenda", + file: "agenda_11june.pdf", + title: "Meeting Agenda", + body: [ + "Meeting Agenda", + "Facilities Committee - 11 June 2024, 18:00, Committee Room B", + "1. Call to order and apologies for absence", + "2. Adoption of the agenda", + "3. Approval of minutes of the previous meeting", + "4. Agenda item: roofing contract award", + "5. Agenda item: car park resurfacing budget", + "6. Any other business", + "7. Date of next meeting", + ], + }, + { + expect: "memo", + file: "memo_hybrid_policy.pdf", + title: "Memorandum", + body: [ + "Memorandum", + "To: All staff From: Operations Date: 3 June 2024", + "Re: Updated hybrid working arrangements", + "Effective 1 July, core in-office days move to Tuesday through Thursday.", + "Team leads may approve exceptions for caregiving or medical needs.", + "Desk booking opens each Thursday for the following week.", + "Questions to operations@corven.example.", + ], + }, + { + expect: "press-release", + file: "kestrel_series_b.pdf", + title: "FOR IMMEDIATE RELEASE", + body: [ + "FOR IMMEDIATE RELEASE", + "Kestrel raises $40M Series B to expand accessible design tooling", + "LONDON, 4 June 2024 - Kestrel Ltd today announced a $40 million Series B round", + "led by Meridian Growth, with participation from existing investors.", + "The funding will accelerate hiring and the launch of Kestrel's audit platform.", + '"Accessibility is still an afterthought in most tooling," said CEO N. Okafor.', + "Media contact: press@kestrel.example Notes to editors: photography available.", + ], + }, + { + expect: "newsletter", + file: "parish_news_june.pdf", + pages: 4, + title: "The Elmswell Bulletin", + body: [ + "The Elmswell Bulletin - June edition", + "In this issue: the summer fete returns, allotment news, and a new cafe on the green.", + "From the editor: thank you to everyone who volunteered at the spring clean-up.", + "Dates for your diary: 15 June - summer fete; 22 June - open gardens.", + "Allotment corner: plots 12 and 14 are available; contact the parish office.", + "Next month we profile the new head teacher at Elmswell Primary.", + ], + }, + { + expect: "white-paper", + file: "zero_trust_wp.pdf", + pages: 16, + title: "Zero Trust in Practice", + body: [ + "Zero Trust in Practice", + "A white paper on migrating enterprise networks beyond the perimeter model.", + "Executive summary", + "This white paper examines the operational realities of zero-trust rollouts across", + "three sectors, drawing on 40 practitioner interviews.", + "Key findings: phased device-trust rollouts outperform big-bang migrations; identity", + "debt is the leading cause of stalled programmes.", + "Conclusions and recommendations follow in section 6.", + ], + }, + { + expect: "case-study", + file: "meridian_case_study.pdf", + pages: 3, + title: "Customer Case Study", + body: [ + "Customer Case Study", + "How Meridian Logistics cut fleet downtime by 34% with Kestrel Audit", + "About the client: Meridian operates 1,200 vehicles across the UK and Ireland.", + "The challenge: paper-based inspections meant defects surfaced days late.", + "The solution: digital inspections with automated triage and parts ordering.", + "The results: downtime fell 34% in six months; audit prep time fell from three", + "weeks to two days.", + '"It paid for itself in a quarter," said the fleet director.', + ], + }, + { + expect: "financial-statement", + file: "annual_accounts_2023.pdf", + pages: 42, + title: "Consolidated Financial Statements", + body: [ + "Consolidated Financial Statements", + "For the year ended 31 December 2023", + "Independent auditor's report to the members of Corven Holdings plc", + "Consolidated statement of financial position", + "Consolidated statement of cash flows", + "Notes to the financial statements", + "1. Basis of preparation. These statements are prepared under IFRS.", + "Revenue for the year was 84.2m (2022: 71.9m).", + ], + }, + { + expect: "budget", + file: "fy25_operating_budget.pdf", + pages: 6, + title: "FY25 Operating Budget", + body: [ + "FY25 Operating Budget", + "Department: Product Engineering", + "Budget vs actual, FY24, and proposed FY25 allocations", + "Headcount FY24 actual 3.2m FY25 budget 3.6m", + "Cloud infrastructure FY24 actual 1.1m FY25 budget 1.0m", + "Tooling and licences FY24 actual 0.3m FY25 budget 0.4m", + "The rolling forecast is updated quarterly; variances above 10% require sign-off.", + ], + }, + { + expect: "insurance-policy", + file: "home_policy_renewal.pdf", + pages: 18, + title: "Your Home Insurance Policy", + body: [ + "Your Home Insurance Policy", + "Policy number: HP-2214-8871 Period of insurance: 1 Aug 2024 - 31 Jul 2025", + "Your policy is due for renewal. Renewal premium: 341.20 including insurance", + "premium tax.", + "This policy wording, together with your schedule, forms your contract of insurance.", + "Section 1 - Buildings. We insure the buildings against loss or damage.", + "Insuring agreement: we will indemnify you subject to the terms and exclusions.", + ], + }, + { + expect: "insurance-claim", + file: "claim_ref_44821.pdf", + title: "Claim acknowledgement", + body: [ + "Claim acknowledgement", + "Claim reference: 44821 Policy number: HP-2214-8871", + "We have received your claim for water damage reported on 12 May 2024.", + "Date of loss: 10 May 2024", + "A claims adjuster will contact you within two working days to arrange an", + "inspection and to explain the proof of loss requirements.", + "Please retain damaged items until the adjuster has visited.", + ], + }, + { + expect: "lab-report", + file: "bloods_2024_05.pdf", + title: "Laboratory Report", + body: [ + "Laboratory Report", + "Patient: R. Patel DOB: 14/02/1988 Collected: 21 May 2024", + "Full blood count", + "Haemoglobin 142 g/L Reference range 130-170", + "White cell count 6.1 x10^9/L Reference range 4.0-11.0", + "Platelets 260 x10^9/L Reference range 150-400", + "Comprehensive metabolic panel: sodium 139 mmol/L (135-145), potassium 4.2 (3.5-5.3)", + "Reported by: City Pathology Services", + ], + }, + { + expect: "prescription", + file: "rx_amoxicillin.pdf", + title: "Prescription", + body: [ + "Prescription", + "Patient: L. Novak DOB: 02/11/1979", + "Amoxicillin 500mg capsules", + "Take one capsule three times daily for 7 days. Qty dispensed: 21", + "Refills remaining: 0 Days supply: 7", + "Prescriber: Dr H. Adeyemi, Riverside Practice", + "Pharmacy stamp: Elmswell Pharmacy, 4 High Street", + ], + }, + { + expect: "utility-bill", + file: "city_power_june.pdf", + pages: 3, + title: "Your energy statement", + body: [ + "Your energy statement", + "Supply address: Flat 2, 18 Camden Row, London NW1 8QP", + "Billing period: 1 May - 31 May 2024", + "Electricity used: 212 kWh at 27.4p per kWh ... 58.09", + "Standing charge: 31 days at 53.8p ... 16.68", + "Meter number: E19D 44210 Previous reading: 30412 (estimated reading)", + "Present reading: 30624", + "Total charges this period: 74.77", + ], + }, + { + expect: "booking-confirmation", + file: "hotel_confirmation.pdf", + title: "Booking Confirmation", + body: [ + "Booking Confirmation", + "Confirmation number: HTL-99281", + "Your booking is confirmed at the Harbourview Hotel, Edinburgh.", + "Check-in date: 12 July 2024 from 15:00", + "Check-out date: 14 July 2024 by 11:00", + "Room: Double, city view, breakfast included. 2 nights, 2 guests.", + "Total: 286.00, payable at the property. Free cancellation until 10 July.", + ], + }, + { + expect: "itinerary", + file: "trip_edinburgh.pdf", + title: "Travel Itinerary", + body: [ + "Travel Itinerary", + "Traveller: A. Diallo Booking reference: XK9PLQ", + "Outbound - 12 July 2024", + "LHR London Heathrow 09:15 -> EDI Edinburgh 10:40, BA 1442, seat 14C", + "Return - 14 July 2024", + "EDI Edinburgh 18:05 -> LHR London Heathrow 19:35, BA 1447, seat 9A", + "Hotel: Harbourview Hotel, 2 nights. Trip summary and e-tickets attached.", + ], + }, + { + expect: "order-confirmation", + file: "order_10422.pdf", + title: "Order Confirmation", + body: [ + "Order Confirmation", + "Thank you for your order, Amara.", + "Order number: 10422 Placed: 2 June 2024", + "1x Ergonomic split keyboard ... 129.00", + "1x Palm rest, walnut ... 39.00", + "Estimated delivery: 6-8 June to 14 Fern Road, Bristol.", + "We will email you when your order ships.", + ], + }, + { + expect: "packing-slip", + file: "packing_slip_10422.pdf", + title: "Packing Slip", + body: [ + "Packing Slip", + "Order number: 10422 Shipped: 5 June 2024", + "Ship to: A. Diallo, 14 Fern Road, Bristol BS1 5TR", + "Qty shipped Item", + "1 Ergonomic split keyboard", + "1 Palm rest, walnut", + "Items in this shipment: 2 of 2. This is not an invoice.", + ], + }, + { + expect: "delivery-note", + file: "delivery_note_8871.pdf", + title: "Delivery Note", + body: [ + "Delivery Note", + "Delivery note no: DN-8871 Date: 25 June 2024", + "Deliver to: Unit 4, Riverside Industrial Estate, Leeds LS10 1AB", + "Against order: PO-88412", + "500 x Corrugated boxes 40x30cm", + "200 x Kraft void fill rolls", + "Received in good condition. Signed for by: ____________ Time: ______", + ], + }, + { + expect: "user-guide", + file: "kettle_manual.pdf", + pages: 24, + title: "Instruction Manual", + body: [ + "Instruction Manual", + "Rapid-boil kettle, model KB-170", + "Important safety instructions - read before first use.", + "This appliance can be used by children aged from 8 years and above if they have", + "been given supervision or instruction concerning use of the appliance.", + "Operating instructions: fill between MIN and MAX, close the lid, press the switch.", + "Cleaning and descaling: unplug and allow to cool before cleaning.", + "Troubleshooting and warranty information on page 20.", + ], + }, + { + expect: "safety-data-sheet", + file: "sds_isopropanol.pdf", + pages: 11, + title: "Safety Data Sheet", + body: [ + "Safety Data Sheet", + "Section 1: Identification. Product: Isopropyl alcohol 99.9%", + "Section 2: Hazards identification. Highly flammable liquid and vapour.", + "Section 4: First aid measures. IF IN EYES: rinse cautiously with water.", + "Section 6: Accidental release measures. Eliminate all ignition sources.", + "Section 11: Toxicological information. May cause drowsiness or dizziness.", + "According to Regulation (EC) No 1907/2006 (REACH).", + ], + }, + { + expect: "certificate", + file: "first_aid_cert.pdf", + title: "Certificate of Completion", + body: [ + "Certificate of Completion", + "This is to certify that", + "K. Osei", + "has successfully completed the Emergency First Aid at Work course", + "held on 20 May 2024 at the Elmswell Training Centre.", + "Valid for three years from the date of issue.", + "Instructor: P. Marsh Certificate no: EFAW-20240520-081", + ], + }, + { + expect: "transcript", + file: "uw_transcript.pdf", + pages: 2, + title: "Official Transcript", + body: [ + "Official Transcript", + "Office of the Registrar, University of Washington", + "Student: John Smith Student ID: 1735522", + "Program: B.S. Computer Science", + "CSE 142 Programming I 4.0 credits grade 3.8", + "CSE 143 Programming II 4.0 credits grade 3.9", + "MATH 126 Calculus III 5.0 credits grade 3.6", + "Cumulative GPA: 3.72 Degree conferred: 10 June 2017", + ], + }, + { + expect: "research-paper", + file: "streaming_paper.pdf", + pages: 12, + title: "Adaptive Backpressure in Distributed Stream Processors", + body: [ + "Adaptive Backpressure in Distributed Stream Processors", + "J. Smith, L. Chen - Meridian Labs", + "Abstract - We present an adaptive backpressure protocol that reduces tail", + "latency by 41% under skewed load. In this paper we formalise the problem,", + "prove stability bounds, and evaluate on three production traces.", + "1. Introduction. Stream processors must balance throughput against latency.", + "Corresponding author: j.smith@meridianlabs.example", + "References [1] Chen et al. (2021) [2] Okafor and Reyes (2019)", + ], + }, + { + expect: "statement-of-work", + file: "sow_kestrel_phase2.pdf", + pages: 6, + title: "Statement of Work", + body: [ + "Statement of Work", + "This Statement of Work is entered into under the Master Services Agreement", + "dated 1 March 2024 between Kestrel Ltd and Corven Holdings.", + "Scope of work: phase 2 accessibility audit of the claims portal.", + "Deliverables: audit report, remediation backlog, retest of priority issues.", + "Period of performance: 1 July - 30 September 2024.", + "Acceptance criteria: all priority-1 issues verified fixed on retest.", + "Fees: time and materials, capped at 48,000.", + ], + }, + { + expect: "proposal", + file: "proposal_fleet_audit.pdf", + pages: 8, + title: "Proposal", + body: [ + "Proposal for Meridian Logistics", + "Fleet inspection digitisation - our proposed approach", + "Scope of work: replace paper inspection sheets across 14 depots with digital", + "checklists, defect triage, and parts integration.", + "Timeline: 12 weeks from kick-off, phased by region.", + "Investment: 86,000 fixed fee, including training.", + "Acceptance of this proposal: sign below to authorise the work.", + "This proposal is valid for 60 days.", + ], + }, + { + expect: "form", + file: "trip_permission.pdf", + title: "School Trip Permission Slip", + body: [ + "School Trip Permission Slip", + "Elmswell Primary School - Year 4 visit to the Natural History Museum", + "Date of trip: 9 July 2024. Cost: 12.50, payable via the school office.", + "Please print clearly and return by 28 June.", + "Child's name: ______________________ Class: ________", + "I give permission for my child to attend the trip.", + "Parent/guardian signature: ______________________", + "Emergency contact number: ______________________", + "For office use only: payment received [ ] consent received [ ]", + ], + }, + { + expect: "invitation", + file: "wedding_invite.pdf", + title: "You are cordially invited", + body: [ + "You are cordially invited", + "to celebrate the marriage of", + "Amara Diallo and Daniel Reyes", + "Saturday the fourteenth of September, two thousand twenty-four", + "at three o'clock in the afternoon", + "The Orangery, Holland Park, London", + "Reception to follow. Kindly respond by the first of August.", + ], + }, + { + expect: "email-thread", + file: "re_re_roofing_tender.pdf", + pages: 3, + title: "RE: RE: Roofing tender", + body: [ + "From: R. Holt ", + "Sent: Tuesday 14 May 2024 09:12", + "To: D. Cole ", + "Subject: RE: RE: Roofing tender", + "Agreed - let's shortlist the two lowest compliant bids.", + "-----Original Message-----", + "From: D. Cole Sent: Monday 13 May 2024 16:40", + "On Mon, 13 May 2024, R. Holt wrote:", + "> The committee meets Tuesday; can you circulate the tender summary?", + "Sent from my iPhone", + ], + }, + { + expect: "warranty-document", + file: "kb170_warranty.pdf", + title: "Warranty Certificate", + body: [ + "Warranty Certificate", + "Rapid-boil kettle, model KB-170", + "This limited warranty covers defects in materials and workmanship for a", + "warranty period of two years from the date of purchase.", + "This warranty does not cover damage from misuse, descaling neglect, or", + "commercial use.", + "To make a claim, contact support with your receipt and serial number.", + ], + }, + // --- scanned documents: no text layer, the filename is the only signal --- + { + expect: "invoice", + file: "invoice_4412_scan.pdf", + title: "", + body: [], + }, + { + expect: "bank-statement", + file: "hsbc_bank_statement_jan.pdf", + title: "", + body: [], + }, + // --- pdf.js extraction artifacts: NBSP separators, curly quotes, fi/fl ligatures --- + { + expect: "invoice", + file: "doc.pdf", + nbsp: true, + title: "Tax Invoice", + body: [ + "Tax Invoice", + "Invoice Number: INV-2024-3319 Invoice Date: 12 June 2024", + "Bill To: Harbour Cafe Ltd", + "Consulting services — May ... 1,400.00", + "Invoice Total: 1,400.00. Remit payment to the account below.", + ], + }, + { + expect: "nda", + file: "doc.pdf", + pages: 3, + title: "Confidentiality Agreement", + body: [ + "Confidentiality Agreement", + "This confidentiality agreement is made between the Disclosing Party and the", + "Receiving Party. Each party’s confidential information shall be held in", + "strict confidence and used only to evaluate the proposed relationship.", + "The obligations survive for five years from the date of disclosure.", + ], + }, + // --- additional common types --- + { + expect: "tax-return", + file: "1040_2023.pdf", + pages: 4, + title: "U.S. Individual Income Tax Return", + body: [ + "Form 1040 - U.S. Individual Income Tax Return 2023", + "Filing status: married filing jointly", + "1a Total amount from Form(s) W-2, box 1 ... 72,450", + "11 Adjusted gross income ... 71,200", + "16 Tax ... 8,102", + "25 Federal income tax withheld ... 9,812", + "34 Refund: amount you overpaid ... 1,710", + ], + }, + { + expect: "subscription-confirmation", + file: "apple_receipt.pdf", + title: "Your receipt from Apple", + body: [ + "Your receipt from Apple", + "Subscription: CloudSync Pro, monthly", + "Renewal price: 2.99. Next billing date: 12 July 2024.", + "Your subscription will automatically renew each month until cancelled.", + "Manage your subscription in Settings.", + ], + }, + { + expect: "donation-receipt", + file: "gift_aid_receipt.pdf", + title: "Donation Receipt", + body: [ + "Donation Receipt", + "Riverside Foodbank, registered charity 1122334", + "Thank you for your donation of 50.00 received on 3 June 2024.", + "No goods or services were provided in exchange for this contribution.", + "Gift Aid: as a UK taxpayer, your donation is worth 25% more at no cost to you.", + ], + }, + { + expect: "waybill", + file: "shipping_label_1Z999.pdf", + title: "Shipping label", + body: [ + "Shipping label", + "UPS Ground Tracking: 1Z 999 AA1 01 2345 6784", + "Ship from: Brightline Packaging, Leeds LS10 1AB", + "Ship to: A. Diallo, 14 Fern Road, Bristol BS1 5TR", + "Weight: 2.4 kg Billing: P/P", + "Package 1 of 1", + ], + }, + { + expect: "customs-declaration", + file: "cn22.pdf", + title: "Customs Declaration CN22", + body: [ + "Customs Declaration CN22", + "May be opened officially", + "Description of contents: cotton t-shirts (2)", + "HS tariff number: 610910 Country of origin: Portugal", + "Declared value: 38.00 Reason for export: sale of goods", + "I certify that the particulars given are correct.", + ], + }, + { + expect: "medical-report", + file: "clinic_letter_rp.pdf", + pages: 2, + title: "Clinical Assessment", + body: [ + "Clinical Assessment", + "Patient: R. Patel NHS no: 943 476 5919 Seen: 21 May 2024", + "Presenting complaint: six weeks of intermittent chest tightness on exertion.", + "History of present illness: symptoms began after a respiratory infection;", + "no orthopnoea; exercise tolerance reduced to one flight of stairs.", + "Past medical history: hypertension, well controlled.", + "On examination: chest clear, heart sounds normal, BP 132/84.", + "Impression and plan: likely post-viral; arrange ECG and review in 4 weeks.", + ], + }, + { + expect: "referral-letter", + file: "referral_cardiology.pdf", + title: "Referral Letter", + body: [ + "Referral Letter", + "Dear colleague,", + "Re: R. Patel, DOB 14/02/1988", + "Reason for referral: exertional chest tightness with an abnormal baseline ECG.", + "I am referring this patient to the rapid access chest pain clinic.", + "Please see this patient within two weeks given the symptom pattern.", + "Referring GP: Dr H. Adeyemi, Riverside Practice", + ], + }, + { + expect: "consent-form", + file: "consent_gastroscopy.pdf", + title: "Consent Form", + body: [ + "Consent Form", + "Procedure: diagnostic gastroscopy", + "I confirm the procedure, its benefits, and its risks have been explained to me.", + "I have had the opportunity to ask questions and all my questions have been", + "answered to my satisfaction.", + "I consent to the procedure described above.", + "Patient signature: ____________ Person taking consent: ____________", + ], + }, + { + expect: "benefits-summary", + file: "pension_statement_2024.pdf", + pages: 3, + title: "Annual Benefit Statement", + body: [ + "Annual Benefit Statement", + "Scheme: Northgate Group Personal Pension", + "Statement date: 5 April 2024", + "Your pension pot on 5 April 2024: 48,210", + "Contributions received this year: 4,860 (you 2,430, employer 2,430)", + "Projected pot at age 67: 214,000 in today's money.", + "This statement is provided annually; it is not a guarantee.", + ], + }, + { + expect: "grade-report", + file: "y4_summer_report.pdf", + title: "End of Year Report", + body: [ + "End of Year Report - Year 4", + "Pupil: T. Diallo Class: 4B Attendance: 96.8%", + "Reading: working at greater depth. Thoughtful, fluent, and adventurous choices.", + "Writing: working towards the expected standard; handwriting is improving.", + "Mathematics: expected standard. Confident with fractions and perimeter.", + "Effort grade: excellent across all subjects.", + "Teacher: Ms P. Whitlow Headteacher: Mr J. Okafor", + ], + }, + { + expect: "course-syllabus", + file: "cse143_syllabus.pdf", + pages: 5, + title: "Course Syllabus", + body: [ + "Course Syllabus", + "CSE 143: Computer Programming II, Autumn 2024", + "Learning objectives: implement and analyse core data structures; write", + "well-tested object-oriented programs.", + "Required textbook: Building Java Programs, 5th edition.", + "Grading: homework 40%, midterm 25%, final 35%.", + "Week 1: recursion. Week 2: linked lists. Week 3: stacks and queues.", + "Office hours: Tuesdays 2-4pm, Allen Center 303.", + ], + }, + { + expect: "datasheet", + file: "lm317_datasheet.pdf", + pages: 22, + title: "LM317 Adjustable Regulator", + body: [ + "LM317 3-Terminal Adjustable Regulator", + "1 Features: output voltage range adjustable from 1.25V to 37V.", + "6 Specifications", + "6.1 Absolute maximum ratings over operating temperature range", + "6.3 Recommended operating conditions", + "6.5 Electrical characteristics", + "7.2 Typical application circuit: adjustable voltage regulator with protection diodes.", + "8 Pin configuration and functions: ADJ, OUT, IN.", + ], + }, + { + expect: "incident-report", + file: "ir_2024_007.pdf", + pages: 6, + title: "Security Incident Report", + body: [ + "Security Incident Report IR-2024-007", + "Severity: SEV-2 Status: closed", + "Summary: credential stuffing against the customer login endpoint.", + "Timeline: detected 02:14 UTC by rate-limit alerting; contained 03:05 UTC.", + "Containment and eradication: blocked source ASNs, forced resets for 112 accounts.", + "Indicators of compromise: IP list attached; no data exfiltration observed.", + "Follow-ups: roll out WebAuthn, tighten per-IP limits.", + ], + }, + { + expect: "audit-report", + file: "soc2_type2_2024.pdf", + pages: 48, + title: "SOC 2 Type II Report", + body: [ + "SOC 2 Type II Report", + "System and Organization Controls report on Kestrel Ltd's audit platform.", + "Independent service auditor's report for the period 1 April 2023 to 31 March 2024.", + "Applicable trust services criteria: security, availability, confidentiality.", + "Section 3: description of the system. Section 4: tests of controls and results.", + "In our opinion, controls were suitably designed and operated effectively.", + ], + }, + // A US-style resume with bare section headings, no CV title, and no LinkedIn URL. + { + expect: "resume", + file: "jordan_avery_2024.pdf", + title: "JORDAN AVERY", + body: [ + "JORDAN AVERY", + "Portland, OR | (503) 555-0142 | j.avery@example.com", + "EXPERIENCE", + "Operations Manager, Cascade Outfitters 2019 - Present", + "- Run daily operations for a 40-person retail and rental business", + "- Cut inventory shrinkage 18% by introducing weekly cycle counts", + "Assistant Manager, Trailhead Sports 2015 - 2019", + "EDUCATION", + "B.A. Business Administration, University of Oregon, 2015", + "SKILLS", + "Scheduling, budgeting, vendor negotiation, POS systems", + ], + }, + // --- negative controls: must stay unlabelled --- + { + expect: "", + file: "weekend_notes.pdf", + title: "Weekend Notes", + body: [ + "The weather was clear on Saturday so we walked along the river and stopped", + "for coffee near the old bridge before the rain started in the afternoon.", + "On Sunday we tidied the garden, read for a while and then cooked a simple", + "dinner with the vegetables that were still fresh from last week.", + "Nothing else of note happened, and it was a quiet couple of days overall.", + ], + }, + { + expect: "", + file: "contrato.pdf", + title: "CONTRATO DE ARRENDAMIENTO", + body: [ + "El presente contrato de arrendamiento se celebra entre el arrendador y el", + "arrendatario para el alquiler de la vivienda situada en la direccion indicada.", + "El arrendatario se compromete a pagar la renta mensual dentro de los primeros", + "cinco dias de cada mes segun las condiciones que se establecen en este documento.", + "Ambas partes firman este contrato en senal de conformidad con sus clausulas.", + ], + }, + // --- extended coverage: every remaining emitted label + new-rule checks --- + { + expect: "remittance-advice", + file: "remit_20240418.pdf", + title: "Remittance Advice", + body: [ + "Remittance Advice", + "From: Harlow Manufacturing Ltd To: Brightline Packaging Ltd", + "Payment date: 18 April 2024 Payment reference: HM-2024-0418", + "BACS payment to sort code 20-00-00, account ending 4471.", + "The following invoices are covered by this payment:", + "INV-2201 12 March 2024 1,480.00", + "INV-2214 28 March 2024 960.00", + "Amount remitted: 2,440.00", + "Your account has been credited accordingly. No action is required.", + ], + }, + { + expect: "credit-note", + file: "CN-0092.pdf", + title: "Credit Note", + body: [ + "Credit Note", + "Credit note number: CN-0092 Credit note date: 6 June 2024", + "Customer: Harbour Cafe Ltd Original invoice: INV-2024-014", + "Reason for credit: goods returned - two chairs received damaged.", + "Item: Bistro chair (oak) Qty: 2 Unit price: 85.00", + "Total credit: 170.00", + "Your account has been credited with the amount shown above.", + ], + }, + { + expect: "loan-agreement", + file: "loan_agreement_final.pdf", + pages: 9, + title: "Loan Agreement", + body: [ + "Loan Agreement", + "This Loan Agreement is made between Fairview Capital LLC (the Lender)", + "and Meadowbrook Joinery Ltd (the Borrower).", + "1. Principal amount. The Lender agrees to advance the principal amount of", + "$150,000 to the Borrower on the terms set out below.", + "2. Interest. Interest shall accrue on the outstanding balance at a rate of", + "7.5% per annum, calculated daily.", + "3. Repayment. The Borrower shall repay the loan in accordance with the", + "repayment schedule in Schedule 1, over sixty equal monthly instalments.", + "4. Default. Each of the following is an event of default: failure to pay any", + "sum within fourteen days of its due date, or breach of any obligation herein.", + "Signed for and on behalf of the Borrower and the Lender.", + ], + }, + { + expect: "letter", + file: "important_account_information.pdf", + title: "Important information about your account", + body: [ + "Important information about your account", + "Dear account holder,", + "We are writing to inform you about changes to your account that take effect", + "from 1 September 2024. This letter confirms that your everyday banking", + "will continue as normal and there is nothing you need to do.", + "The changes to your interest rate are set out in the enclosed leaflet, and", + "your account with us will keep the same sort code and account number.", + "Thank you for banking with us.", + "Yours faithfully,", + "Customer Services Team", + ], + }, + { + expect: "board-report", + file: "board_pack_q2.pdf", + pages: 18, + title: "Board Pack", + body: [ + "Board Pack", + "Quarterly board meeting of Northwind Trading Ltd, 20 June 2024.", + "These confidential board materials have been prepared for the board and", + "distributed to the directors one week ahead of the meeting.", + "Contents: CEO overview, KPI review, finance update, people update, and", + "items for board discussion including the proposed warehouse lease.", + "Strategic priorities remain unchanged from the March meeting.", + "Appendix B sets out the management accounts for the quarter.", + ], + }, + { + expect: "pitch-deck", + file: "acme_seed_deck.pdf", + pages: 14, + title: "Acme Robotics", + body: [ + "Acme Robotics", + "The problem we solve: warehouse picking is slow, error-prone and expensive.", + "Our solution: a modular picking robot that installs in a weekend.", + "Total addressable market: $8.4B across e-commerce fulfilment.", + "Traction to date: 11 paying customers, 240% net revenue retention.", + "Why now: labour shortages and cheap depth sensors change the economics.", + "Meet the team: ex-Amazon robotics and two-time founders.", + "The ask: we are raising a seed round of $2.5M.", + "Use of funds: 60% engineering, 25% go-to-market, 15% operations.", + ], + }, + { + expect: "investment-summary", + file: "investor_update_march.pdf", + title: "Investor Update", + body: [ + "Investor Update - March 2024", + "Dear investors,", + "Highlights and lowlights: we shipped the v2 dashboard and closed our two", + "largest deals to date; hiring for the platform team slipped a month.", + "Key metrics this month: monthly recurring revenue of $86k, up 9% on", + "February; net new MRR of $7.1k; burn rate steady at $110k.", + "That leaves fourteen months of runway at the current plan.", + "How you can help: intros to heads of operations at mid-market 3PLs.", + "Thank you, as ever, for your support.", + ], + }, + { + expect: "letter-of-intent", + file: "loi_project_falcon.pdf", + pages: 4, + title: "Letter of Intent", + body: [ + "Letter of Intent", + "Project Falcon - strictly private and confidential", + "This letter of intent sets out the indicative terms on which Redwood", + "Industrial Group would be willing to pursue the acquisition of the", + "business and assets of Meadowbrook Joinery Ltd.", + "This letter is non-binding and does not create any obligation on either", + "party, and any transaction remains subject to definitive agreement.", + "During the exclusivity period of sixty days the Sellers shall not solicit", + "offers from any other person, and both parties shall proceed in good", + "faith negotiations toward completion.", + ], + }, + { + expect: "quarterly-report", + file: "q3_earnings_release.pdf", + pages: 8, + title: "Quarterly Earnings Release", + body: [ + "Quarterly Earnings Release", + "Northwind Group reports third quarter results and raises full-year guidance.", + "Revenue of $412M, up 14% year-over-year; operating margin of 18.2%.", + "Diluted earnings per share of $0.94, ahead of consensus estimates.", + "The company beat analyst estimates on both revenue and earnings.", + "Management will host a conference call to discuss these results at 5:00pm", + "Eastern Time today; a replay of the earnings call will be available.", + "Guidance for the full year: revenue of $1.63B to $1.66B.", + ], + }, + { + expect: "report", + file: "report_to_cabinet_leisure.pdf", + pages: 12, + title: "Report to Cabinet", + body: [ + "Report to Cabinet", + "Report of the Director of Communities: future of the Riverside Leisure Centre.", + "Portfolio holder: Councillor J. Ellis Wards affected: Castle, Riverside.", + "Key decision: yes.", + "The committee is recommended to approve option two, refurbishment in", + "phases, and it is recommended that officers procure a design partner.", + "Financial implications: capital cost of 4.2m over two years.", + "Legal implications: none beyond standard procurement obligations.", + "Reason for the decision: the centre no longer meets accessibility standards.", + ], + }, + { + expect: "regulatory-filing", + file: "uksi_2024_0871.pdf", + pages: 16, + title: "The Data Protection (Amendment) Regulations 2024", + body: [ + "Statutory Instruments", + "2024 No. 871", + "The Data Protection (Amendment) Regulations 2024", + "Made 4 June 2024. Laid before Parliament 6 June 2024.", + "The Secretary of State makes these Regulations in exercise of the powers", + "conferred by sections 16 and 211 of the Data Protection Act 2018.", + "1. These Regulations may be cited as the Data Protection (Amendment)", + "Regulations 2024 and come into force on 1 August 2024.", + "2. Regulation 4 of the principal Regulations is amended as follows.", + "Explanatory note: this note is not part of the Regulations.", + ], + }, + { + expect: "appraisal-report", + file: "409a_valuation_2024.pdf", + pages: 34, + title: "409A Valuation Report", + body: [ + "409A Valuation Report", + "Prepared for the board of Acme Robotics, Inc.", + "Valuation date: 31 March 2024.", + "Purpose: to estimate the fair market value of the company's common stock.", + "We considered the market approach and the income approach. Under the", + "income approach we applied a discounted cash flow analysis using a", + "weighted average cost of capital of 24%.", + "Under the market approach we performed a comparable company analysis", + "using the guideline public company method across eight peers.", + "Concluded fair market value: $2.84 per common share.", + ], + }, + { + expect: "tax-statement", + file: "cp14_notice.pdf", + title: "Notice of Assessment", + body: [ + "Notice of Assessment", + "Notice date: 12 May 2024 Taxpayer ID ending: 4417", + "You have unpaid taxes for the 2023 tax year.", + "Proposed amount due: $1,240.00, which includes a late payment penalty", + "of $62.00 and interest calculated to the date of this notice.", + "Amount due immediately: $1,240.00.", + "If you do not pay by 5 June 2024, additional penalties will apply.", + "Payment options are listed on the back of this notice.", + ], + }, + { + expect: "court-filing", + file: "motion_to_dismiss.pdf", + pages: 11, + title: "Motion to Dismiss", + body: [ + "United States District Court for the Northern District of Ohio", + "Civil Action No. 5:24-cv-00311", + "Meridian Supply Co., Plaintiff, v. Lakeshore Logistics LLC, Defendant.", + "Defendant's Motion to Dismiss", + "Comes now Defendant Lakeshore Logistics LLC, by and through undersigned", + "counsel, and moves this Court to dismiss the complaint for failure to", + "state a cause of action, as set out in the memorandum in support filed", + "herewith. The complaint's prayer for relief seeks damages unavailable", + "as a matter of law.", + "Respectfully submitted, /s/ Dana Whitfield, Counsel for Defendant.", + "Certificate of service: a copy was served on all counsel of record.", + ], + }, + { + expect: "affidavit", + file: "affidavit_of_residence.pdf", + title: "Affidavit of Residence", + body: [ + "Affidavit of Residence", + "State of Texas, County of Travis.", + "I, Morgan Reyes, being duly sworn, deposes and says:", + "1. I am over eighteen years of age and competent to make this affidavit.", + "2. I have resided at 118 Fern Road, Austin, Texas since March 2019.", + "3. The facts stated here are true and correct to the best of my knowledge.", + "Affiant: Morgan Reyes", + "Subscribed and sworn to before me this 9th day of May 2024.", + "Notary Public, State of Texas. My commission expires 01/31/2026.", + ], + }, + { + expect: "legal-notice", + file: "letter_before_action.pdf", + title: "Letter Before Action", + body: [ + "Letter Before Action", + "Dear Mr Hale,", + "We act for Brightline Packaging Ltd. This is a letter before claim under", + "the pre-action protocol for debt claims.", + "Our client is owed 4,860.00 under invoices that remain unpaid despite", + "repeated reminders. This is our client's final demand for payment.", + "Unless payment is received within fourteen days we are instructed to", + "issue proceedings without further notice, and further legal action may", + "include a claim for interest and costs.", + "Govern yourself accordingly.", + ], + }, + { + expect: "purchase-agreement", + file: "asset_purchase_agreement.pdf", + pages: 64, + title: "Asset Purchase Agreement", + body: [ + "Asset Purchase Agreement", + "by and among Redwood Industrial Group, Inc., as Buyer, and Meadowbrook", + "Joinery Ltd, as Seller, dated as of 12 July 2024.", + "Article III sets out the representations and warranties of the Seller,", + "qualified by the disclosure schedule delivered at signing.", + "Article II provides for a purchase price adjustment based on closing", + "working capital, with an escrow amount of $1,500,000 to be held by the", + "escrow agent for eighteen months.", + "Article VII sets out the closing conditions, including the absence of any", + "material adverse effect on the business of the target company.", + ], + }, + { + expect: "shareholder-agreement", + file: "shareholders_agreement_2024.pdf", + pages: 28, + title: "Shareholders' Agreement", + body: [ + "Shareholders' Agreement", + "relating to Acme Robotics Ltd, entered into by the persons listed in", + "Schedule 1 as holders of shares in the company.", + "5. Transfer of shares. No shareholder may transfer shares except as", + "permitted by this clause, subject to the pre-emption rights in clause 6.", + "7. Drag-along rights. If holders of 75% of the shares accept an offer,", + "they may require every other shareholder to sell on the same terms.", + "8. Tag-along rights. No transfer may complete unless each minority", + "shareholder is offered the same price per share.", + "10. Reserved matters. The consent of investor directors is required for", + "the matters in Schedule 3, including changes to board composition.", + ], + }, + { + expect: "legal-opinion", + file: "approved_judgment_hale.pdf", + pages: 22, + title: "Approved Judgment", + body: [ + "Approved Judgment", + "In the matter of Meridian Supply Co v Lakeshore Logistics.", + "Before the Honourable Mrs Justice Carey, handed down 14 June 2024.", + "This is the judgment of the court on the defendant's application.", + "For the reasons given below, the appeal is dismissed.", + "The claimant's construction of clause 9 is to be preferred; the", + "commercial context points firmly in the same direction.", + "Costs follow the event. Permission to appeal is refused.", + ], + }, + { + expect: "subpoena", + file: "subpoena_duces_tecum.pdf", + title: "Subpoena to Produce Documents", + body: [ + "Subpoena to Produce Documents, Information, or Objects", + "To: Custodian of Records, Lakeshore Logistics LLC.", + "You are commanded to produce at the time, date and place set forth below", + "the documents described in Attachment A.", + "This subpoena duces tecum is issued in the matter of Meridian Supply Co", + "v Lakeshore Logistics, pending in the district court.", + "Place of compliance: 400 Main Street, Suite 210, Cleveland, Ohio.", + "Return date: 2 August 2024 at 10:00 a.m.", + "Issued by the clerk of court on application of counsel for the plaintiff.", + ], + }, + { + expect: "settlement-agreement", + file: "settlement_agreement_reyes.pdf", + pages: 7, + title: "Settlement Agreement", + body: [ + "Settlement Agreement", + "between Morgan Reyes (the Claimant) and Northwind Trading Ltd", + "(the Respondent), each a party and together the parties.", + "The parties desire to resolve all matters between them arising out of", + "the Claimant's employment, without any admission of liability.", + "1. The Respondent shall pay the Claimant a settlement sum of 24,000", + "within 21 days, in full and final settlement of all claims.", + "2. The Claimant agrees to withdraw the claim before the employment", + "tribunal and enters into the release of claims in Schedule 1.", + "3. This agreement was reached following discussions through a", + "conciliation officer.", + ], + }, + { + expect: "employee-handbook", + file: "staff_handbook_2024.pdf", + pages: 48, + title: "Employee Handbook", + body: [ + "Employee Handbook", + "Welcome to the company. This handbook explains what you can expect from", + "us and what we expect from you.", + "Code of conduct: we treat colleagues and customers with respect.", + "Dress code: smart casual, with site rules taking precedence in the yard.", + "Annual leave: 25 days plus public holidays; sick leave is set out in", + "section 6 together with reporting requirements.", + "Your first three months are a probationary period.", + "The disciplinary procedure and grievance procedure are in section 9,", + "and our equal opportunity commitments are in section 10.", + "Working hours are 9:00 to 17:30, Monday to Friday.", + ], + }, + { + expect: "risk-assessment", + file: "enterprise_risk_register.pdf", + pages: 12, + title: "Risk Assessment Report", + body: [ + "Risk Assessment Report", + "Enterprise risk register for Northwind Trading Ltd, reviewed quarterly.", + "Methodology: each risk is scored for likelihood and impact on a five", + "point scale, before and after controls, and assigned a risk owner.", + "The risk heat map on page 3 summarises the top twelve risks.", + "R-04 Supply concentration: inherent risk high; mitigation plan agreed", + "with procurement; residual risk medium, within our risk appetite.", + "R-07 Warehouse fire: risk rating severe; sprinkler upgrade underway.", + "Control effectiveness is tested by internal audit on rotation.", + ], + }, + { + expect: "compliance-document", + file: "information_security_policy.pdf", + pages: 18, + title: "Information Security Policy", + body: [ + "Information Security Policy", + "Policy owner: Head of IT Security. Effective date: 1 May 2024.", + "Scope: this policy applies to all staff, contractors and systems and", + "forms part of our information security management system.", + "Access is granted on the principle of least privilege and reviewed", + "quarterly, as set out in the access control policy.", + "Information is handled according to the data classification policy;", + "the acceptable use policy governs personal use of company systems.", + "The password policy requires a unique passphrase per system.", + "This document follows a twelve month policy review cycle.", + ], + }, + { + expect: "questionnaire", + file: "vendor_security_questionnaire.pdf", + pages: 9, + title: "Vendor Security Questionnaire", + body: [ + "Vendor Security Questionnaire", + "To be completed by the supplier as part of our vendor risk management", + "and due diligence questionnaire process.", + "Please answer yes or no in the response column, adding detail where", + "relevant to your data handling practices.", + "Section 3: do you maintain a list of subprocessors, and are they bound", + "by equivalent contractual obligations?", + "Section 5: describe any compensating control where a requirement is", + "not met in full.", + "This questionnaire is based on the standardized information gathering", + "format and should take about an hour to complete.", + ], + }, + { + expect: "immunization-record", + file: "immunization_record_reyes.pdf", + title: "Immunization Record", + body: [ + "Immunization Record", + "Patient name: Alex Reyes Date of birth: 04/12/2016", + "Vaccine Date administered Administered by Lot number", + "DTaP (dose 5) 03/22/2021 Dr. N. Okafor AC41B", + "MMR (dose 2) 03/22/2021 Dr. N. Okafor KL98D", + "Varicella (dose 2) 03/22/2021 Dr. N. Okafor MM31A", + "Hepatitis B (dose 3) 11/02/2017 Dr. N. Okafor KT77C", + "Tdap booster due: 2028.", + "This record was printed from the state immunisation history registry.", + ], + }, + { + expect: "medical-invoice", + file: "eob_march_2024.pdf", + title: "Explanation of Benefits", + body: [ + "Explanation of Benefits", + "This is not a bill. Keep this statement for your records.", + "Member: Morgan Reyes Claim number: 88-4412-07", + "Provider: Lakeside Family Medicine Date of service: 03/14/2024", + "Amount billed: $240.00 Allowed amount: $132.00", + "Plan paid: $105.60 Coinsurance: $26.40 Copay: $0.00", + "Patient responsibility: $26.40", + "Applied to deductible: $0.00 Out-of-pocket maximum met: no.", + "Your provider may bill you for the patient responsibility shown above.", + ], + }, + { + expect: "discharge-summary", + file: "discharge_summary_hale.pdf", + pages: 3, + title: "Discharge Summary", + body: [ + "Discharge Summary", + "Patient: Thomas Hale NHS number: 943 476 5919", + "Date of admission: 2 June 2024 Date of discharge: 6 June 2024", + "Discharge diagnosis: community acquired pneumonia.", + "Hospital course: the patient responded well to intravenous antibiotics", + "and was stepped down to oral therapy on day two.", + "Medications on discharge: amoxicillin 500mg three times daily for five", + "days; continue regular inhalers.", + "Condition at discharge: stable, mobilising independently; discharged home.", + "Follow-up arrangements: chest x-ray in six weeks. Copy to GP.", + ], + }, + { + expect: "insurance-certificate", + file: "acord_25_certificate.pdf", + title: "Certificate of Liability Insurance", + body: [ + "Certificate of Liability Insurance", + "This certificate is issued as a matter of information only and confers", + "no rights upon the certificate holder.", + "Insured: Lakeshore Logistics LLC, 400 Main Street, Cleveland, OH.", + "Commercial general liability: each occurrence $1,000,000; general", + "aggregate $2,000,000. Policy number GL-8841-22, ACORD 25 form.", + "Certificate holder: Meridian Supply Co is named as additional insured", + "with respect to work performed under contract 2024-118.", + "Should any of the above described policies be cancelled before the", + "expiration date thereof, notice will be delivered in accordance with", + "the policy provisions.", + ], + }, + { + expect: "thesis", + file: "reyes_phd_thesis.pdf", + pages: 186, + title: "Consensus Protocols for Unreliable Networks", + body: [ + "Consensus Protocols for Unreliable Networks", + "A thesis submitted in partial fulfilment of the requirements for the", + "degree of Doctor of Philosophy.", + "School of Computing, University of Leeds, September 2024.", + "Thesis supervisor: Professor A. Whitmore.", + "I hereby declare that this thesis is my own work and has not been", + "submitted for any other degree; see the declaration of authorship.", + "Acknowledgements: I thank my supervisor and the systems group.", + "Abstract: we study agreement under message loss and partition.", + ], + }, + { + expect: "assignment-brief", + file: "problem_set_4.pdf", + title: "Problem Set 4", + body: [ + "Problem Set 4", + "MATH 2210 Linear Algebra Due date: Friday 24 May at 5pm", + "Answer all questions. Show your work for full credit; unsupported", + "answers receive no marks.", + "Total marks: 40.", + "Question 1 (8 marks): find the eigenvalues of the matrix A below.", + "Question 2 (12 marks): prove that similar matrices share a determinant.", + "Submit your solutions as a single PDF through the course portal.", + "Late submissions will lose 10% per day up to three days.", + ], + }, + { + expect: "registration-form", + file: "student_enrollment_form.pdf", + title: "Student Enrollment Form", + body: [ + "Student Enrollment Form", + "Section A - pupil details: full name, date of birth, year group applied", + "for, and current or previous school attended.", + "Section B - parent/guardian details and emergency contacts, including", + "at least two adults we may call during the school day.", + "Section C - home language survey: which language is spoken most at home?", + "Section D - proof of residency: please attach a recent council tax bill", + "or tenancy agreement showing your home address.", + "Return the completed admission form to the school office by 15 June.", + ], + }, + { + expect: "lesson-plan", + file: "lesson_plan_fractions.pdf", + title: "Lesson Plan", + body: [ + "Lesson Plan - Year 5 Mathematics - Comparing Fractions", + "Lesson objective: pupils can compare fractions with unlike denominators.", + "Success criteria: I can find a common denominator; I can order three", + "fractions; I can explain my reasoning to a partner.", + "Key vocabulary: numerator, denominator, equivalent, common multiple.", + "Starter activity (5 min): match equivalent fraction cards in pairs.", + "Main activity (30 min): guided practice, then independent questions.", + "Differentiation: number lines for support; extension into improper", + "fractions for rapid graspers. Assessment for learning: exit ticket", + "with three ordering problems collected as the plenary.", + ], + }, + { + expect: "mortgage-document", + file: "closing_disclosure.pdf", + pages: 5, + title: "Closing Disclosure", + body: [ + "Closing Disclosure", + "This form is a statement of final loan terms and closing costs.", + "Borrower: Morgan Reyes Property: 118 Fern Road, Austin, TX", + "Loan amount: $312,000 Loan term: 30 years Fixed rate: 6.25%", + "Loan-to-value: 80%. Your loan has no prepayment penalty.", + "Projected payments include amounts held in your escrow account for", + "property taxes and homeowner's insurance.", + "The amortization schedule in the appendix shows principal and interest", + "over the life of the loan; this note is secured by a deed of trust.", + "Signed at closing alongside the promissory note.", + ], + }, + { + expect: "deed", + file: "warranty_deed_recorded.pdf", + title: "Warranty Deed", + body: [ + "Warranty Deed", + "Recorded at the request of Travis County Title Co.", + "The grantor, Evelyn Marsh, a single person, for good and valuable", + "consideration, grants and conveys to the grantee, Morgan Reyes, the", + "following described real property in fee simple:", + "Legal description: Lot 14, Block C, Fernwood Addition, according to the", + "plat recorded in Volume 88, Page 12, plat records of Travis County.", + "Filed for record with the county recorder on 21 June 2024.", + ], + }, + { + expect: "property-listing", + file: "14_fern_road_brochure.pdf", + title: "14 Fern Road, Bristol", + body: [ + "14 Fern Road, Bristol", + "Guide price 425,000. Offers in excess of the guide will be considered.", + "A well presented three bedroom semi with a south facing garden, fitted", + "kitchen, and off-street parking for two cars.", + "Approximately 1,180 sq ft of accommodation over two floors.", + "Council tax band D. EPC rating C.", + "Viewing strictly by appointment through the vendor's estate agent.", + "Open house Saturday 10:00 to 12:00.", + ], + }, + { + expect: "inspection-report", + file: "home_inspection_fern_rd.pdf", + pages: 32, + title: "Home Inspection Report", + body: [ + "Home Inspection Report", + "Property: 118 Fern Road, Austin, TX Inspected: 14 June 2024", + "Each system is given a condition rating: satisfactory, marginal, or", + "defective, with photographs in the appendix.", + "Roof covering: architectural shingle, marginal; several lifted tabs.", + "Electrical: two bathroom receptacles lack GFCI protection; recommend", + "evaluation by a licensed electrician.", + "Crawl space: minor moisture staining at the north wall; downspouts", + "discharge against the foundation and should be extended.", + "A summary of deficiencies appears on the final page.", + ], + }, + { + expect: "hoa-document", + file: "hoa_annual_assessment.pdf", + title: "Fernwood Homeowners Association", + body: [ + "Fernwood Homeowners Association", + "Annual assessment notice for 2024-2025.", + "Dear owner, the board has approved the budget and the annual assessment", + "of $640, payable in two instalments; HOA dues fund insurance, common", + "areas maintenance and the reserve fund.", + "A special assessment of $150 was approved for pool resurfacing.", + "Reminder: exterior changes require architectural review before work", + "begins, per the CC&Rs.", + "Questions may be directed to the managing agent, Brookside Community", + "Management.", + ], + }, + { + expect: "application-form", + file: "planning_application_2024.pdf", + title: "Planning Application", + body: [ + "Planning Application", + "Application for planning permission under the Town and Country Planning", + "Act 1990.", + "Proposal: erection of a single storey rear extension and detached", + "garden studio at 14 Fern Road.", + "The proposed development falls within the Fernwood conservation area;", + "a heritage statement accompanies this application.", + "Please include a site plan at 1:1250 showing the boundary in red.", + "Case officer use only: application reference and date received.", + "The planning officer will contact you if further information is needed.", + ], + }, + { + expect: "government-notice", + file: "foi_response_2024_0441.pdf", + title: "Freedom of Information Request", + body: [ + "Freedom of Information Request - Response", + "Our reference: FOI 2024/0441", + "Thank you for your request for information about road maintenance", + "spending. Your request has been handled under the Freedom of", + "Information Act 2000.", + "We have located records responsive to your request; extracts are", + "enclosed. Some material is withheld under section 43, and this", + "exemption is subject to a public interest test, which we concluded", + "favours withholding.", + "You have the right to an internal review, and thereafter may complain", + "to the Information Commissioner's Office.", + ], + }, + { + expect: "visa-document", + file: "i797_approval_notice.pdf", + title: "U.S. Citizenship and Immigration Services", + body: [ + "U.S. Citizenship and Immigration Services", + "I-797A, Notice of Action", + "Case type: I-129, petition for a nonimmigrant worker.", + "Receipt number: WAC-24-118-50441 Beneficiary: Reyes, Morgan", + "The petition has been approved. The beneficiary's alien registration", + "number and the attached I-94 record the new period of admission.", + "Immigration status: H-1B, valid to 30 September 2027.", + "Please contact USCIS if any information on this notice is incorrect.", + ], + }, + { + expect: "license", + file: "premises_licence_ph0441.pdf", + title: "Premises Licence", + body: [ + "Premises Licence", + "Granted under the Licensing Act 2003 by Bristol City Council as the", + "licensing authority.", + "Licence number: PL-0441 Premises: The Harbour Cafe, 112 Station Road.", + "The permit holder is hereby licensed to sell alcohol for consumption on", + "the premises between 11:00 and 23:00 daily.", + "The licence is granted subject to the following conditions: CCTV shall", + "be maintained, and a refusals log kept.", + "This permit must be displayed prominently at the premises.", + ], + }, + { + expect: "public-notice", + file: "notice_public_hearing_zoning.pdf", + title: "Notice of Public Hearing", + body: [ + "Notice of Public Hearing", + "Notice is hereby given that the Fernwood Town Council will hold a", + "public hearing on Tuesday 16 July 2024 at 7:00 pm in the council", + "chamber, 1 Civic Square.", + "The purpose of the hearing is to receive comment on the proposed", + "amendment to the zoning bylaw for the Station Road corridor.", + "All interested persons may appear and be heard, and written comments", + "may be filed with the clerk before the hearing.", + "Published by authority of the town clerk.", + ], + }, + { + expect: "analytics-report", + file: "labour_market_bulletin.pdf", + pages: 18, + title: "Statistical Bulletin", + body: [ + "Statistical Bulletin", + "Labour market overview, UK: July 2024.", + "Source: Office for National Statistics.", + "The employment rate was 74.6%, up 0.2 percentage points quarter on", + "quarter and broadly flat year on year, seasonally adjusted.", + "Estimates are subject to sampling error; confidence intervals are", + "shown in the accompanying data tables.", + "Next publication date: 13 August 2024.", + "These figures are designated national statistics.", + ], + }, + { + expect: "grant-agreement", + file: "notice_of_award_r01.pdf", + title: "Notice of Award", + body: [ + "Notice of Award", + "Federal award identification number: R01-HL-158812.", + "Recipient: University of Leeds Award amount: $412,000.", + "Assistance listing number: 93.837.", + "Period of performance: 1 September 2024 to 31 August 2027.", + "This award is subject to the terms and conditions of the grant,", + "including the reporting requirements in Section IV.", + "Grant number and document number must be quoted on all drawdown", + "requests. Authorized organizational representative: Dr P. Shah.", + ], + }, + { + expect: "tender-document", + file: "itt_fleet_services.pdf", + pages: 42, + title: "Invitation to Tender", + body: [ + "Invitation to Tender", + "Provision of fleet maintenance services, reference ITT-2024-081.", + "Section 2, instructions to tenderers: tenders must be submitted through", + "the e-procurement portal; the submission deadline is 12:00 noon on", + "30 August 2024. Clarification questions close ten days earlier.", + "Section 3 sets out the scope of requirements, including response times", + "and workshop standards.", + "Award will be to the most economically advantageous tender against the", + "evaluation criteria: 60% quality, 40% price.", + "A pre-qualification questionnaire must accompany each submission.", + ], + }, + { + expect: "sales-proposal", + file: "tender_response_fleet.pdf", + pages: 36, + title: "Tender Response", + body: [ + "Tender Response", + "Provision of fleet maintenance services, reference ITT-2024-081.", + "We are pleased to submit our tender in response to your invitation.", + "Our technical response demonstrates full coverage of the scope, and the", + "compliance matrix at Appendix A confirms we are fully compliant with", + "every mandatory requirement.", + "The pricing schedule at Appendix B sets out fixed rates for the first", + "two years.", + "Our proposed approach pairs a dedicated account manager with a mobile", + "technician team based within eight miles of your depot.", + ], + }, + { + expect: "supply-order", + file: "purchase_requisition_2214.pdf", + title: "Purchase Requisition", + body: [ + "Purchase Requisition", + "Requisition number: REQ-2214 Requisition date: 3 July 2024", + "Requested by: Facilities Budget holder: J. Okafor", + "Cost centre: FAC-110 Budget code: 5400-EQ", + "Justification for purchase: replacement of the workshop compressor,", + "which failed its service inspection.", + "Preferred supplier: Brightline Industrial Supplies.", + "Item: 90L belt-drive compressor Qty: 1 Estimated cost: 1,140.00", + "Approval routing: budget holder, then procurement.", + ], + }, + { + expect: "grant-application", + file: "community_fund_application.pdf", + pages: 9, + title: "Grant Application", + body: [ + "Grant Application", + "Funding opportunity: Community Facilities Fund 2024.", + "Applicant organisation: Fernwood Sports Association, registered", + "charity 1148812.", + "Project summary: refurbish the pavilion kitchen and accessible toilet.", + "Amount requested: 24,500. We are applying for 70% of project costs,", + "with match funding of 10,500 secured from club reserves.", + "Outcomes and milestones: works complete by March 2025; open kitchen", + "sessions for four community groups per week from April.", + "Declaration signed by two trustees.", + ], + }, + { + expect: "technical-drawing", + file: "GA-1104_rev_C.pdf", + title: "General Arrangement", + body: [ + "General Arrangement - Conveyor Frame Assembly", + "Drawing number GA-1104 Revision C Scale 1:10 Sheet 1 of 2", + "Third angle projection. Do not scale. Work to figured dimensions only.", + "All dimensions in millimetres unless noted otherwise.", + "Break all sharp edges 0.5 x 45 degrees.", + "Material: S275 structural steel, hot dip galvanised after fabrication.", + "Drawn by: MR Checked by: TH Approved: 21/06/2024", + "Issued for construction.", + ], + }, + { + expect: "bill-of-materials", + file: "bom_ctrl_board_a3.pdf", + title: "Bill of Material", + body: [ + "Bill of Material - Controller Board, Assembly CTRL-0301 Rev A3", + "Top level assembly: CTRL-0301 Qty per assembly shown per unit.", + "Ref designator Manufacturer part number Description Qty", + "C1-C8 GRM188R71C104KA01 100nF 0402 X7R 8", + "R12 ERJ-3EKF1002V 10k 0603 1% 1", + "U4 STM32G071KBU6 MCU, 64k flash 1", + "J2 Do not populate - programming header, test builds only.", + "Reference designator order follows the assembly drawing.", + ], + }, + { + expect: "safety-procedure", + file: "sop_014_lockout.pdf", + title: "Standard Operating Procedure", + body: [ + "Standard Operating Procedure SOP-014: Conveyor Lockout and Cleaning", + "Purpose: safe isolation of the packing conveyor for scheduled cleaning.", + "A permit to work is required before starting; the shift supervisor", + "delivers a toolbox talk covering this procedure at the start of shift.", + "Personal protective equipment: cut-resistant gloves and safety glasses.", + "Sequence of work: isolate at the local breaker, apply personal lock and", + "tag, verify zero energy, then clean per the schedule.", + "Control measures: an exclusion zone is marked while guards are off.", + "First aid arrangements are posted at the workshop entrance.", + ], + }, + { + expect: "test-report", + file: "mill_cert_heat_88412.pdf", + title: "Mill Test Certificate", + body: [ + "Mill Test Certificate", + "Certificate number: MTC-2024-1180 Standard: EN 10204 3.1", + "Product: S355J2 structural plate, 12mm. Heat number: 88412.", + "Cast number: 88412-2.", + "Chemical composition (%): C 0.16, Si 0.35, Mn 1.42, P 0.012, S 0.008.", + "Mechanical properties: yield strength 372 MPa, tensile strength", + "528 MPa, elongation 24%.", + "Charpy impact at -20C: 41J average, batch tested.", + "We certify the material supplied conforms to the order requirements.", + ], + }, + { + expect: "quality-report", + file: "fai_report_ctrl0301.pdf", + title: "First Article Inspection Report", + body: [ + "First Article Inspection Report", + "Part: mounting bracket BRK-2210 Rev B Inspection lot: 24-0611", + "Purpose: dimensional inspection of the first production article against", + "the drawing, using the sampling plan in QP-07.", + "Characteristic 4: hole spacing 42.00 +/- 0.10, measured 42.06, pass.", + "Characteristic 9: flatness 0.20 max, measured 0.31, out of tolerance.", + "Disposition: non-conformance NC-118 raised; use-as-is rejected; rework", + "and re-present for final inspection.", + "Inspected by: T. Hale Acceptance criteria: drawing plus QP-07.", + ], + }, + { + expect: "return-authorization", + file: "rma_return_label.pdf", + title: "Return Merchandise Authorization", + body: [ + "Return Merchandise Authorization", + "RMA number: RMA-88412 Order: 2024-1180", + "Return reason: wrong size.", + "Return instructions: pack the item in its original packaging, include", + "this sheet, and affix this label to the outside of the box.", + "The enclosed prepaid return label covers shipping; drop the parcel at", + "any service point within 30 days.", + "Refunds are processed by the returns department within five working", + "days of receipt.", + ], + }, + { + expect: "presentation", + file: "qbr_slides_q2.pdf", + pages: 22, + meta: { producer: "Microsoft PowerPoint for Microsoft 365" }, + title: "Quarterly Business Review", + body: [ + "Quarterly Business Review", + "Agenda: performance recap, customer health, roadmap, open actions.", + "In this presentation we cover Q2 results and the outlook for Q3.", + "Key takeaways: renewals held at 96%, onboarding time halved, and the", + "new reporting module is the most requested add-on.", + "Customer health: two accounts at risk, recovery plans in flight.", + "Q&A", + "Thank you for your attention.", + ], + }, + { + expect: "brochure", + file: "spring_offers_flyer.pdf", + title: "Fernwood Garden Services", + body: [ + "Fernwood Garden Services", + "Why choose us? Fully insured, local, and rated 4.9 across 300 reviews.", + "Spring tidy-ups, hedge cutting, lawn treatment plans and patio cleaning.", + "Features and benefits: fixed prices, no contract, satisfaction", + "guaranteed on every visit.", + "Limited time offer: 20% off your first lawn treatment.", + "Book your free consultation today - call us today on 0117 946 0000 or", + "visit our website for a free quote.", + "Bring this flyer to receive the discount. No obligation.", + ], + }, + { + expect: "business-plan", + file: "business_plan_2024.pdf", + pages: 26, + title: "Business Plan", + body: [ + "Business Plan", + "Harbour Roastery Ltd, 2024 to 2027.", + "Executive summary: a speciality coffee roastery supplying cafes across", + "the south west, expanding into direct subscriptions.", + "Company description: founded 2021; roastery and training room in Bristol.", + "Market analysis: the regional speciality segment is growing 11% a year;", + "our target market is independent cafes within 90 minutes.", + "SWOT analysis and competitive landscape are set out in section 4.", + "Financial projections: revenue of 640k in year one rising to 1.1m in", + "year three, with break-even analysis at month 14.", + "Funding requirements: 180k for a second roaster and packing line.", + "Revenue model: wholesale contracts plus consumer subscriptions.", + ], + }, + { + expect: "organization-chart", + file: "org_chart_july.pdf", + title: "Organisation Chart", + body: [ + "Organisation Chart - Northwind Trading Ltd, July 2024", + "This chart shows reporting lines and department structure after the", + "June reorganisation; headcount is shown in brackets for each team.", + "Chief Executive", + "Operations Director - direct reports: warehouse manager, fleet", + "manager, facilities lead. The warehouse manager reports to the", + "Operations Director with a team of 24.", + "Finance Director - direct reports: financial controller, payroll lead.", + "The reporting structure for the interim data team is under review.", + ], + }, + { + expect: "technical-specification", + file: "srs_dispatch_v2.pdf", + pages: 34, + title: "Software Requirements Specification", + body: [ + "Software Requirements Specification", + "Dispatch Planning System, version 2.0. This document specifies the", + "functional requirements and non-functional requirements for the", + "dispatch planner used by the operations team.", + "Intended audience: developers, testers and the product owner.", + "Normative references: RFC 2119 key words; see terms and definitions.", + "FR-12: the system shall comply with the routing rules in Appendix C.", + "NFR-3: page loads shall complete within two seconds at the 95th", + "percentile.", + "Revision history and acceptance criteria appear at the end of this", + "document.", + ], + }, + { + expect: "patent", + file: "US10884412.pdf", + pages: 24, + title: "United States Patent", + body: [ + "United States Patent", + "Patent number: US 10,884,412 B2 Date of patent: Jan 5, 2021", + "Field of the invention: the present invention relates to modular", + "conveyor systems for warehouse automation.", + "Background of the invention: existing pickers require fixed rails.", + "Summary of the invention: a drive unit with a detachable guide.", + "Brief description of the drawings: FIG. 1 shows the drive unit;", + "FIG. 2 shows the guide in a released position.", + "In one embodiment, the preferred embodiment uses a magnetic coupling", + "distinguishing over the prior art. References cited appear on the", + "cover page.", + "What is claimed is: 1. A conveyor drive unit comprising a housing...", + ], + }, + { + expect: "api-documentation", + file: "dispatch_api_reference.pdf", + pages: 58, + title: "API Reference", + body: [ + "API Reference", + "Dispatch Planning API, version 2.3. See the changelog and release", + "notes for differences from 2.2, including known issues.", + "Authentication: pass your API key in the Authorization header. The", + "sandbox base URL is http://localhost:8080 when using the emulator.", + "POST /routes - create a route. Query parameters: dryRun (boolean).", + "The request body accepts a JSON route plan; the response body returns", + "the planned stops with ETAs.", + "Configuration: set the DISPATCH_ENV environment variable, or use the", + "configuration file described in the developer guide. A command line", + "client is available for scripting.", + ], + }, + { + expect: "license-agreement", + file: "license_certificate_pro.pdf", + title: "License Certificate", + body: [ + "License Certificate", + "Product: FlowChart Studio Professional License type: perpetual", + "license with twelve months of updates.", + "Licensed to: Northwind Trading Ltd Number of seats: 25", + "License key: XXXXX-XXXXX-XXXXX-88412 Serial number: 118812", + "Maintenance expires: 31 July 2025.", + "How to activate: open Help, choose Enter activation code, and paste", + "the license key above. Activation instructions for offline machines", + "are on the reverse.", + "Keep this certificate for your records.", + ], + }, + { + expect: "gift-certificate", + file: "gift_voucher_50.pdf", + title: "Gift Voucher", + body: [ + "Gift Voucher", + "The Harbour Cafe", + "This voucher entitles the bearer to fifty pounds toward food and drink.", + "Voucher code: HC-4412-GIFT Value: 50.00", + "To redeem, present this voucher at the till; redeemable at our Station", + "Road cafe only.", + "Valid until 30 June 2025. Not redeemable for cash; no cash value if", + "lost or expired.", + "From: Aunt Priya To: Sam - happy graduation!", + ], + }, + { + expect: "confirmation-letter", + file: "appointment_confirmation_dental.pdf", + title: "Appointment Confirmation", + body: [ + "Appointment Confirmation", + "Dear Morgan Reyes,", + "Your appointment has been confirmed with Dr Okafor at Fernwood Dental.", + "Appointment date: Tuesday 23 July 2024 Appointment time: 09:40", + "Please arrive ten minutes early with a list of current medications.", + "If you are unable to attend, you can cancel or reschedule up to 24", + "hours in advance without charge; to reschedule call 0117 946 0001.", + "Failure to attend without notice may incur a fee.", + "We look forward to seeing you.", + ], + }, + { + expect: "waybill", + file: "bill_of_lading_88412.pdf", + title: "Bill of Lading", + body: [ + "Straight Bill of Lading", + "Carrier: Lakeshore Logistics LLC Pro number: 8841-2214", + "Shipper: Brightline Packaging Ltd, Unit 4, Riverside Estate, Leeds.", + "Consignee: Northwind Trading Ltd, 12 Dock Street, Hull.", + "6 pallets, corrugated cartons, 1,420 kg, freight charges collect.", + "Received in apparent good order for carriage subject to the conditions", + "on the reverse.", + "Shipper signature: J. Mills Driver: T. Hale Trailer: 4471", + ], + }, + { + expect: "incident-report", + file: "accident_report_yard.pdf", + title: "Accident Report", + body: [ + "Accident Report", + "Reference: AR-2024-0611 Location: Northwind Trading, dispatch yard.", + "Date and time: 11 June 2024, 14:20.", + "Description: a reversing forklift struck a pallet stack; a case fell", + "and bruised an operative's shoulder. First aid was given on site and", + "the operative attended hospital as a precaution.", + "Reporting officer: S. Whitfield, shift supervisor.", + "A copy of the police report was requested for the insurer, and the", + "root cause analysis is scheduled for Friday's safety meeting.", + "Witnesses: two, statements attached. Severity level: minor.", + ], + }, + { + expect: "certificate", + file: "birth_certificate_reyes.pdf", + title: "Certificate of Live Birth", + body: [ + "Certificate of Live Birth", + "Registration district: Travis County, Texas.", + "This is to certify that the following particulars are recorded in the", + "register of births.", + "Child: Alex Jordan Reyes Date of birth: 4 December 2016", + "Place of birth: Austin, Texas.", + "Parents: Morgan Reyes and Casey Reyes.", + "Registered: 18 December 2016 Registrar: E. Marsh", + "Certified to be a true copy of an entry in the register.", + ], + }, + { + expect: "regulatory-filing", + file: "articles_of_incorporation_acme.pdf", + pages: 4, + title: "Articles of Incorporation", + body: [ + "Articles of Incorporation", + "of Acme Robotics, Inc., filed with the Secretary of State of Delaware.", + "First: the name of the corporation is Acme Robotics, Inc.", + "Second: the registered office is 1209 Orange Street, Wilmington.", + "Third: the corporation is authorized to issue 10,000,000 shares of", + "common stock with a par value of $0.0001 per share.", + "Fourth: the name and mailing address of the incorporator are stated", + "below. Upon filing, the certificate of incorporation becomes effective.", + "In witness whereof, the incorporator has executed these articles on", + "12 July 2024.", + ], + }, + { + expect: "invoice", + file: "qb_invoice_1181.pdf", + meta: { producer: "Intuit QuickBooks Online" }, + title: "Invoice", + body: [ + "Invoice", + "Harbour Roastery Ltd", + "Invoice number: 1181 Invoice date: 2 August 2024", + "Bill to: The Harbour Cafe, 112 Station Road.", + "House espresso 6kg ... 96.00", + "Filter subscription (August) ... 48.00", + "Total: 144.00. Payment terms: net 30.", + ], + }, + { + expect: "payslip", + file: "adp_earnings_statement.pdf", + meta: { producer: "ADP, LLC" }, + title: "Earnings Statement", + body: [ + "Earnings Statement", + "Northwind Trading Ltd Pay period: 01/07/2024 to 31/07/2024", + "Pay date: 31 July 2024 Employee: T. Hale", + "Gross pay: 2,860.00", + "Total deductions: 642.10, including income tax and pension.", + "Net pay: 2,217.90", + "Year to date gross: 20,020.00 Tax code: 1257L", + ], + }, + { + expect: "tax-return", + file: "turbotax_2023_return.pdf", + pages: 14, + meta: { producer: "Intuit TurboTax 2023" }, + title: "U.S. Individual Income Tax Return", + body: [ + "Form 1040: U.S. Individual Income Tax Return, tax year 2023.", + "Filing status: married filing jointly.", + "Wages from W-2 box 1: 96,400. Interest income: 412.", + "Adjusted gross income: 96,812.", + "Standard deduction: 27,700. Taxable income: 69,112.", + "Total tax: 7,864. Federal income tax withheld: 8,120.", + "Refund: 256. Routing and account numbers as provided.", + ], + }, + { + expect: "expense-report", + file: "concur_expense_july.pdf", + meta: { producer: "SAP Concur" }, + title: "Expense Report", + body: [ + "Expense Report", + "Employee: S. Whitfield Report: July client visits", + "Business purpose: quarterly reviews with three northern accounts.", + "12 July - rail, Leeds to Hull, 34.50, receipts attached.", + "12 July - lunch with client, 28.20.", + "18 July - mileage claimed: 96 miles at 0.45 per mile, 43.20.", + "Per diem (two days): 50.00.", + "Total reimbursable: 155.90.", + "Approved by cost center FAC-110 manager.", + ], + }, + { + expect: "", + file: "angebot_gartenpflege.pdf", + title: "Angebot Gartenpflege", + body: [ + "Angebot fur die Gartenpflege", + "Sehr geehrte Frau Weber, vielen Dank fur Ihre Anfrage.", + "Gerne unterbreiten wir Ihnen das folgende Angebot fur die regelmassige", + "Pflege Ihres Gartens mit einem monatlichen Besuch unseres Teams.", + "Der Preis betragt 120 Euro pro Monat und die Rechnung wird jeweils am", + "Anfang des Monats gestellt. Das Angebot ist bis Ende August gultig und", + "wir freuen uns sehr auf Ihre Ruckmeldung.", + "Mit freundlichen Grussen, Fernwood Gartenservice GmbH", + ], + }, +]; + +describe("heuristic engine broad corpus", () => { + it.each(CASES.map((c) => [c.expect || `nothing (${c.file})`, c] as const))( + "labels a %s correctly", + (_label, c) => { + const r = run(c); + const detail = `expected ${c.expect || "no label"}, got [${r.labels.join(", ")}] (confidence ${r.confidence}, score ${r.score})`; + if (c.expect === "") { + expect(r.labels, detail).toEqual([]); + } else { + expect(r.labels[0], detail).toBe(c.expect); + } + }, + ); +}); diff --git a/frontend/editor/src/proprietary/services/heuristic/heuristicEngine.docs.test.ts b/frontend/editor/src/proprietary/services/heuristic/heuristicEngine.docs.test.ts new file mode 100644 index 0000000000..25d226035d --- /dev/null +++ b/frontend/editor/src/proprietary/services/heuristic/heuristicEngine.docs.test.ts @@ -0,0 +1,118 @@ +// Regression cases: each feeds the real engine a typical specimen's actual text. + +import { beforeAll, describe, expect, it } from "vitest"; +import { + classifyHeuristic, + ensureRulesLoaded, +} from "@app/services/heuristic/heuristicEngine"; +import type { HeuristicDoc } from "@app/services/heuristic/types"; + +beforeAll(async () => { + await ensureRulesLoaded(); +}); + +function classify( + title: string, + body: string, + fileName = "doc.pdf", + pageCount = 1, +) { + const doc: HeuristicDoc = { + fileName, + pageCount, + meta: {}, + titleZone: title, + firstZone: body, + allZone: body, + }; + return classifyHeuristic(doc); +} + +describe("scoring explanations", () => { + const doc: HeuristicDoc = { + fileName: "invoice_acme.pdf", + pageCount: 1, + meta: {}, + titleZone: "TAX INVOICE", + firstZone: "Invoice Number: INV-9 Invoice Total: 950.00", + allZone: "Invoice Number: INV-9 Invoice Total: 950.00", + }; + + it("returns candidates with per-rule signals when requested", () => { + const r = classifyHeuristic(doc, { explain: true }); + expect(r.labels[0]).toBe("invoice"); + const top = r.explain?.candidates[0]; + expect(top?.id).toBe("invoice"); + expect(top?.score).toBeGreaterThan(0); + expect(top?.signals.some((s) => s.includes('phrase "tax invoice"'))).toBe( + true, + ); + expect(top?.signals.some((s) => s.includes("filename"))).toBe(true); + }); + + it("omits the explanation by default", () => { + expect(classifyHeuristic(doc).explain).toBeUndefined(); + }); +}); + +describe("documents observed lost on upload (engine must label them)", () => { + it("labels a resume", () => { + const body = [ + "CURRICULUM VITAE", + "Jane Doe jane.doe@example.com +44 7700 900123 London, United Kingdom", + "Professional Summary: An experienced software engineer with more than ten years of", + "professional experience building reliable web applications and leading small teams.", + "Career Objective: To take on a senior engineering role where I can apply my skills in", + "distributed systems and mentor other engineers on the team.", + "Professional Experience:", + "Senior Engineer, Northwind Ltd (2019 to present). Led the migration of the billing platform", + "and improved reliability across all of the core services.", + "Software Engineer, Contoso plc (2014 to 2019). Built and maintained customer-facing features", + "used by more than a million people every day.", + "Education: BSc Computer Science, University of Manchester.", + "References available upon request.", + ].join("\n"); + const r = classify("CURRICULUM VITAE", body, "resume_jane_doe.pdf"); + expect(r.labels[0]).toBe("resume"); + }); + + it("labels a purchase order", () => { + const body = [ + "PURCHASE ORDER", + "Purchase Order Number: PO-55231 Requisition Number: REQ-9910 Date: 2 April 2024", + "To: Global Office Supplies Ltd. Please supply the following goods to our warehouse at the", + "address shown below and confirm the expected delivery date by return.", + "Qty Ordered: 20 Item: Ergonomic office chair Unit Price: $180.00", + "Qty Ordered: 15 Item: Height-adjustable desk Unit Price: $420.00", + "Qty Ordered: 50 Item: LED desk lamp Unit Price: $35.00", + "This is an official order. All goods supplied against this purchase order must reference the", + "requisition number on the delivery note and on your invoice.", + "Authorised by: Procurement Department, Northwind Ltd.", + ].join("\n"); + const r = classify("PURCHASE ORDER", body, "purchase_order.pdf"); + expect(r.labels[0]).toBe("purchase-order"); + }); + + it("labels a master services agreement", () => { + const body = [ + "MASTER SERVICES AGREEMENT", + "This Master Services Agreement is made between the Client and the Service Provider and sets", + "out the terms on which the Service Provider will provide services to the Client.", + "1. Engagement. The Client engages the Service Provider to perform the services described in", + "each Statement of Work agreed between the parties from time to time.", + "2. Fees. The Client shall pay the fees set out in the applicable Statement of Work within", + "thirty days of the date of each invoice.", + "3. Term and Termination. This agreement shall continue until terminated by either party on", + "sixty days written notice to the other party.", + "4. Confidentiality. Each party shall keep confidential the confidential information of the", + "other party that it receives under this agreement.", + "We are pleased to act for you and look forward to a productive working relationship.", + ].join("\n"); + const r = classify( + "MASTER SERVICES AGREEMENT", + body, + "service_agreement.pdf", + ); + expect(r.labels[0]).toBe("service-agreement"); + }); +}); diff --git a/frontend/editor/src/proprietary/services/heuristic/heuristicEngine.test.ts b/frontend/editor/src/proprietary/services/heuristic/heuristicEngine.test.ts new file mode 100644 index 0000000000..dbbe78c441 --- /dev/null +++ b/frontend/editor/src/proprietary/services/heuristic/heuristicEngine.test.ts @@ -0,0 +1,125 @@ +// Core fidelity cases for the heuristic engine against the real rules pack. + +import { beforeAll, describe, expect, it } from "vitest"; +import { + classifyHeuristic, + detectEnglish, + ensureRulesLoaded, +} from "@app/services/heuristic/heuristicEngine"; +import type { HeuristicDoc } from "@app/services/heuristic/types"; + +beforeAll(async () => { + await ensureRulesLoaded(); +}); + +function classify(title: string, body: string) { + const doc: HeuristicDoc = { + fileName: "doc.pdf", + pageCount: 1, + meta: {}, + titleZone: title, + firstZone: body, + allZone: body, + }; + return classifyHeuristic(doc); +} + +describe("heuristic engine port fidelity", () => { + it("classifies an invoice as invoice", () => { + const body = [ + "INVOICE", + "Acme Web Services Ltd", + "123 High Street, London, EC1A 4JQ", + "Invoice Number: INV-2024-0117", + "Invoice Date: 14 March 2024", + "Due Date: 13 April 2024", + "Bill To: Northwind Trading Company", + "Description Qty Unit Price Amount", + "Website hosting (annual) 1 480.00 480.00", + "Subtotal: 930.00", + "VAT (20%): 186.00", + "Total Due: 1,116.00", + "Payment Terms: Net 30. Please quote the invoice number with payment.", + ].join("\n"); + const r = classify("INVOICE", body); + expect(r.labels.length).toBeGreaterThan(0); + expect(r.labels[0]).toBe("invoice"); + }); + + it("classifies a curriculum vitae as resume", () => { + const body = [ + "CURRICULUM VITAE", + "Jordan Ellis", + "Bristol, UK | jordan.ellis@example.com | 07700 900123", + "Professional Summary", + "Experienced software engineer with 8 years building web platforms.", + "Work Experience", + "Senior Engineer, Northwind Ltd (2020-present)", + "Education", + "BSc Computer Science, University of Bristol, 2016", + "Skills", + "TypeScript, Java, React, cloud architecture, mentoring", + "References available on request.", + ].join("\n"); + const r = classify("CURRICULUM VITAE", body); + expect(r.labels.length).toBeGreaterThan(0); + expect(r.labels[0]).toBe("resume"); + }); + + it("classifies a boarding pass as ticket", () => { + const body = [ + "BOARDING PASS", + "British Airways", + "Passenger: SMITH/JANE MS", + "Flight: BA 117 Date: 22 APR 2024", + "From: LONDON HEATHROW (LHR) Terminal 5", + "To: NEW YORK JFK (JFK)", + "Departure: 11:20 Boarding Time: 10:35 Gate: B44", + "Seat: 34K Group: 3 Class: Economy", + "Booking Reference: XK9PLQ", + "Please be at the gate 45 minutes before departure.", + ].join("\n"); + const r = classify("BOARDING PASS", body); + expect(r.labels.length).toBeGreaterThan(0); + expect(r.labels[0]).toBe("ticket"); + }); + + it("classifies an NDA as nda", () => { + const body = [ + "NON-DISCLOSURE AGREEMENT", + "This Mutual Non-Disclosure Agreement (the Agreement) is entered into", + "by and between Stirling Systems Ltd and the Receiving Party.", + "1. Confidential Information means any proprietary data disclosed by a party.", + "2. Obligations: The Receiving Party shall hold all Confidential Information", + "in strict confidence and not disclose it to any third party.", + "3. Term: The obligations survive for a period of five (5) years.", + "4. Governing Law: This Agreement is governed by the laws of England and Wales.", + "Accepted and agreed by the authorised representatives of the parties.", + ].join("\n"); + const r = classify("NON-DISCLOSURE AGREEMENT", body); + expect(r.labels.length).toBeGreaterThan(0); + expect(r.labels[0]).toBe("nda"); + }); + + it("does not classify a non-English (Spanish) document", () => { + const body = [ + "CONTRATO DE ARRENDAMIENTO DE VIVIENDA", + "Este contrato de arrendamiento se celebra entre el arrendador y el", + "arrendatario para la vivienda situada en la ciudad.", + "El arrendatario pagara una renta mensual de 1150 euros segun las", + "condiciones que las partes acuerdan por el plazo de doce meses.", + "Ambas partes firman este documento segun la ley aplicable.", + ].join("\n"); + const r = classify("CONTRATO DE ARRENDAMIENTO", body); + expect(r.isEnglish).toBe(false); + expect(r.labels).toHaveLength(0); + }); + + it("detects English prose", () => { + const english = + "This agreement is made between the parties and shall be governed by the laws" + + " of England. The tenant agrees to pay the rent that is due under this" + + " contract for the property."; + expect(detectEnglish(english).isEnglish).toBe(true); + }); +}); diff --git a/frontend/editor/src/proprietary/services/heuristic/heuristicEngine.ts b/frontend/editor/src/proprietary/services/heuristic/heuristicEngine.ts new file mode 100644 index 0000000000..e030a9e93a --- /dev/null +++ b/frontend/editor/src/proprietary/services/heuristic/heuristicEngine.ts @@ -0,0 +1,976 @@ +// Heuristic (non-AI) document classifier: string/regex/structural scoring over +// extracted text, filename and metadata. Rules lazy-load as a separate chunk. + +import type { + HeuristicConfidence, + HeuristicDoc, + HeuristicExplanation, + HeuristicResult, +} from "@app/services/heuristic/types"; + +export type { + HeuristicConfidence, + HeuristicDoc, + HeuristicExplanation, + HeuristicResult, +}; + +// --- scoring constants --- +const ZONE_MULT: Record = { title: 2.0, first: 1.35, any: 1.0 }; +const FLOOR = 18; +const HIGH_SCORE = 45; +const HIGH_MARGIN = 15; +const HIGH_SIGNALS = 3; +const MED_SCORE = 28; +const MED_MARGIN = 8; +const SEC_FLOOR = 28; +const SEC_FRAC = 0.5; +const SEC_SIGNALS = 2; +const SEC_MAX = 4; + +const STOPWORDS = new Set([ + "the", + "and", + "of", + "to", + "in", + "is", + "that", + "for", + "on", + "with", + "as", + "are", + "this", + "be", + "by", + "at", + "from", + "or", + "an", + "not", + "your", + "you", + "we", + "has", + "have", + "will", + "was", + "were", + "been", + "their", + "they", + "which", + "any", + "all", + "may", + "shall", + "if", + "can", + "our", + "its", + "it", + "no", + "but", + "other", + "than", + "these", + "such", + "must", + "each", + "per", + "under", + "more", + "when", + "also", + "into", + "only", + "should", + "would", +]); + +// Non-Latin scripts end English classification outright when they dominate. +const SCRIPT_RANGES: RegExp[] = [ + /[一-鿿぀-ヿ]/g, // CJK + Kana + /[가-힯ᄀ-ᇿ]/g, // Hangul + /[Ѐ-ӿ]/g, // Cyrillic + /[؀-ۿݐ-ݿ]/g, // Arabic + /[Ͱ-Ϳ]/g, // Greek + /[ऀ-ॿ]/g, // Devanagari + /[֐-׿]/g, // Hebrew + /[฀-๿]/g, // Thai +]; + +interface LatinProfile { + words: Set; + dia: RegExp | null; +} + +// Function-word and diacritic profiles for common Latin-script languages. +const LATIN_PROFILES: LatinProfile[] = [ + { + words: new Set([ + "el", + "los", + "las", + "que", + "para", + "una", + "por", + "según", + "más", + ]), + dia: /[áéíóúñ¿¡]/g, + }, + { + words: new Set([ + "le", + "les", + "des", + "une", + "est", + "pour", + "avec", + "dans", + "vous", + "votre", + "être", + "nous", + "cette", + "sont", + "été", + ]), + dia: /[àâçèéêëîïôùûœ]/g, + }, + { + words: new Set([ + "der", + "die", + "das", + "und", + "ist", + "für", + "mit", + "von", + "nicht", + "ein", + "eine", + "werden", + "wird", + "bei", + "sind", + "dem", + ]), + dia: /[äöüß]/g, + }, + { + words: new Set([ + "il", + "di", + "che", + "per", + "con", + "una", + "del", + "della", + "sono", + "questo", + "essere", + "più", + "nel", + "anche", + "gli", + ]), + dia: /[àèéìòù]/g, + }, + { + words: new Set([ + "os", + "as", + "que", + "para", + "com", + "uma", + "por", + "são", + "não", + "você", + "está", + "mais", + ]), + dia: /[ãõçáéíóúâêô]/g, + }, + { + words: new Set([ + "het", + "een", + "van", + "voor", + "met", + "aan", + "niet", + "zijn", + "wordt", + "deze", + "als", + "bij", + "ook", + "naar", + ]), + dia: null, + }, + { + words: new Set([ + "och", + "att", + "det", + "som", + "på", + "är", + "av", + "för", + "med", + "den", + "till", + "inte", + "har", + "ett", + "du", + ]), + dia: /[åäö]/g, + }, + { + words: new Set([ + "nie", + "jest", + "się", + "że", + "oraz", + "dla", + "przez", + "lub", + "być", + "może", + "przy", + "jak", + ]), + dia: /[ąćęłńśźż]/g, + }, + { + words: new Set([ + "ve", + "bir", + "bu", + "için", + "ile", + "olarak", + "olan", + "gibi", + "daha", + "çok", + "her", + "kadar", + "sonra", + ]), + dia: /[çğışöü]/g, + }, +]; + +// detectEnglish helper patterns (global for counting; \p{L} needs the u flag). +const LETTERS = /\p{L}/gu; +const LATIN_LETTER = /[a-z]/gi; +const WORD = /[\p{L}']+/gu; + +// ASCII whitespace plus the no-break spaces pdf.js extraction commonly emits. +// eslint-disable-next-line no-control-regex -- vertical tab is intentional ASCII whitespace +const WHITESPACE = /[\t\n\x0B\f\r \u00A0\u2007\u202F]+/g; + +// Structural signal patterns. Boolean-presence ones stay non-global (safe .test()), +// counting ones are global (used via countAll). Currency symbols are \u-escaped. +const CURRENCY = new RegExp( + "[$£€]\\s?\\d[\\d,.]*|\\d[\\d,.]*\\s?(usd|gbp|eur)\\b", + "gi", +); +const NUMERIC_TOKEN = new RegExp("^[\\d$£€.,%-]+$"); +const DIGIT = /\d/; +const FORM_LABEL = /^[A-Za-z][A-Za-z /()&']{2,30}:\s*$/; +const UNDERSCORE4 = /_{4,}/; +const CHECKBOX = /[☐☑□■]\s/; +const DOT_LEADER = /\.{5,}\s*\d+\s*$/; +const BULLET = /^[•▪◦*-]\s+\S/; +const URL = /https?:\/\/|www\./gi; +const TOC = /table of contents/i; +const SIG1 = /\b(signature|signed by|authorized signature|\/s\/)\b/i; +const SIG2 = /_{6,}\s*\n\s*(date|name|sign)/i; +const REF1 = /\b(references|bibliography)\b/i; +const REF2 = /\[\d{1,3}\]|\(\d{4}\)/; +const EMAIL_FROM = /\bfrom:\s.+\n(.*\n){0,3}?\s*(to|sent|date):\s/i; +const EMAIL_SUBJ = /subject:\s/i; +const ADDRESS = /\b\d{5}(-\d{4})?\b|\b[A-Z]{1,2}\d{1,2}[A-Z]?\s?\d[A-Z]{2}\b/g; + +// --- prepared rule model --- +interface Phrase { + text: string; + weight: number; + where: string; +} +interface Rx { + re: RegExp; + weight: number; + where: string; +} +interface FileRx { + re: RegExp; + weight: number; +} +interface MetaRx { + field: string; + re: RegExp; + weight: number; +} +interface Negative { + text: string | null; + re: RegExp | null; + weight: number; +} +interface Structural { + signal: string; + weight: number; +} +interface PreparedLabel { + id: string; + emit: boolean; + phrases: Phrase[]; + regexes: Rx[]; + filenames: FileRx[]; + metadata: MetaRx[]; + negatives: Negative[]; + structural: Structural[]; +} +interface Prior { + min: number; + max: number | null; +} + +// Raw JSON shapes (loose - the pack is authored by hand). +interface RawRule { + text?: unknown; + pattern?: unknown; + weight?: unknown; + where?: unknown; + flags?: unknown; + field?: unknown; + signal?: unknown; +} +interface RawLabel { + id?: unknown; + emit?: unknown; + phrases?: RawRule[]; + regexes?: RawRule[]; + filenames?: RawRule[]; + metadata?: RawRule[]; + negatives?: RawRule[]; + structural?: RawRule[]; +} +interface RulesFile { + labels?: RawLabel[]; + priors?: Record; +} + +let PREPARED: PreparedLabel[] | null = null; +let PRIORS: Map | null = null; +let loadPromise: Promise | null = null; + +/** Load and prepare the rules pack once. Must resolve before classifyHeuristic. */ +export async function ensureRulesLoaded(): Promise { + if (PREPARED && PRIORS) return; + if (!loadPromise) { + loadPromise = import("@app/services/heuristic/heuristicRules.json").then( + (mod) => { + const root = ((mod as { default?: RulesFile }).default ?? + (mod as RulesFile)) as RulesFile; + PREPARED = prepare(root.labels ?? []); + PRIORS = loadPriors(root.priors ?? {}); + }, + (err) => { + // A failed chunk load (flaky network) must not poison later attempts. + loadPromise = null; + throw err; + }, + ); + } + await loadPromise; +} + +// --- Preparation --- + +function prepare(labels: RawLabel[]): PreparedLabel[] { + const out: PreparedLabel[] = []; + for (const label of labels) { + const id = typeof label.id === "string" ? label.id : ""; + const emit = typeof label.emit !== "boolean" ? true : label.emit; + + const phrases: Phrase[] = []; + for (const p of label.phrases ?? []) { + const text = typeof p.text === "string" ? p.text : ""; + const w = num(p.weight); + if (text.length === 0 || w <= 0) continue; + phrases.push({ + text: normalize(text), + weight: Math.min(w, 40), + where: where(p), + }); + } + + const regexes: Rx[] = []; + for (const r of label.regexes ?? []) { + const re = compileRegex(str(r.pattern), flags(r)); + if (re == null) continue; + regexes.push({ + re, + weight: Math.min(num(r.weight), 30), + where: where(r), + }); + } + + const filenames: FileRx[] = []; + for (const r of label.filenames ?? []) { + const re = compileRegex(str(r.pattern), flags(r)); + if (re == null) continue; + filenames.push({ re, weight: Math.min(num(r.weight), 30) }); + } + + const metadata: MetaRx[] = []; + for (const r of label.metadata ?? []) { + const re = compileRegex(str(r.pattern), flags(r)); + if (re == null) continue; + const field = typeof r.field === "string" && r.field ? r.field : "any"; + metadata.push({ field, re, weight: Math.min(num(r.weight), 20) }); + } + + const negatives: Negative[] = []; + for (const n of label.negatives ?? []) { + const text = n.text != null ? normalize(String(n.text)) : null; + const re = + n.pattern != null ? compileRegex(String(n.pattern), flags(n)) : null; + if (text == null && re == null) continue; + negatives.push({ + text, + re, + weight: Math.min(Math.abs(num(n.weight)), 30), + }); + } + + const structural: Structural[] = []; + for (const s of label.structural ?? []) { + const signal = typeof s.signal === "string" ? s.signal : ""; + const w = num(s.weight); + if (signal.length === 0 || w <= 0) continue; + structural.push({ signal, weight: Math.min(w, 12) }); + } + + out.push({ + id, + emit, + phrases, + regexes, + filenames, + metadata, + negatives, + structural, + }); + } + return out; +} + +function loadPriors(priorsNode: Record): Map { + const out = new Map(); + for (const [key, val] of Object.entries(priorsNode)) { + if (!Array.isArray(val) || val.length === 0) continue; + const min = Math.trunc(num(val[0])); + const max = + val.length > 1 && val[1] != null ? Math.trunc(num(val[1])) : null; + out.set(key, { min, max }); + } + return out; +} + +function where(node: RawRule): string { + const w = typeof node.where === "string" ? node.where : ""; + return w.length === 0 ? "any" : w; +} + +function flags(node: RawRule): string { + return typeof node.flags === "string" ? node.flags : ""; +} + +/** Compile a rule regex to a RegExp, or null when it won't compile. */ +export function compileRegex( + pattern: string | null, + flagStr: string, +): RegExp | null { + if (pattern == null) return null; + try { + const fl = flagStr.length === 0 ? "gi" : flagStr; + let f = "g"; // always global for iterative counting + if (fl.indexOf("i") >= 0) f += "i"; + if (fl.indexOf("m") >= 0) f += "m"; + if (fl.indexOf("s") >= 0) f += "s"; + return new RegExp(pattern, f); + } catch { + return null; + } +} + +// --- Public API --- + +interface ScoredLabel { + label: PreparedLabel; + score: number; + distinct: number; + /** Rule-hit descriptions, collected only when explain is requested. */ + signals: string[] | null; +} + +/** Max candidates and per-candidate signals included in an explanation. */ +const EXPLAIN_CANDIDATES = 6; +const EXPLAIN_SIGNALS = 12; + +const fmt = (n: number) => Math.round(n * 10) / 10; + +function toExplanation( + en: { isEnglish: boolean; lowText: boolean }, + scored: ScoredLabel[], +): HeuristicExplanation { + return { + isEnglish: en.isEnglish, + lowText: en.lowText, + candidates: scored.slice(0, EXPLAIN_CANDIDATES).map((s) => ({ + id: s.label.id, + emit: s.label.emit, + score: fmt(s.score), + distinct: s.distinct, + signals: (s.signals ?? []).slice(0, EXPLAIN_SIGNALS), + })), + }; +} + +/** Classify a document; returns emitted label ids (primary + secondaries, capped at 5). */ +export function classifyHeuristic( + doc: HeuristicDoc, + opts?: { explain?: boolean }, +): HeuristicResult { + if (!PREPARED || !PRIORS) { + throw new Error( + "Heuristic rules not loaded; await ensureRulesLoaded() before classifyHeuristic().", + ); + } + const explain = opts?.explain === true; + + const en = detectEnglish(doc.allZone); + // Non-English with real text: honestly out of scope for the English heuristics. + if (!en.isEnglish && !en.lowText) { + return { + labels: [], + confidence: "none", + score: 0, + isEnglish: false, + ...(explain ? { explain: toExplanation(en, []) } : {}), + }; + } + + const titleRaw = nz(doc.titleZone); + const firstRaw = nz(doc.firstZone); + const anyRaw = nz(doc.allZone); + const titleNorm = normalize(titleRaw); + const firstNorm = normalize(firstRaw); + const anyNorm = normalize(anyRaw); + const fileNameLower = nz(doc.fileName).toLowerCase(); + const meta = doc.meta ?? {}; + const metaAll = Object.values(meta).join(" \n "); + const struct = computeStructural(doc); + + const scored: ScoredLabel[] = []; + for (const label of PREPARED) { + let score = 0; + let distinct = 0; + const sig: string[] | null = explain ? [] : null; + + for (const phrase of label.phrases) { + let best = 0; + let bestZone = ""; + for (const zone of ["title", "first", "any"] as const) { + const hay = + zone === "title" ? titleNorm : zone === "first" ? firstNorm : anyNorm; + const count = countOccurrences(hay, phrase.text); + if (count === 0) continue; + const zf = phrase.where === "any" || phrase.where === zone ? 1 : 0.75; + const value = phrase.weight * ZONE_MULT[zone] * zf * damp(count); + if (value > best) { + best = value; + bestZone = zone; + } + } + if (best > 0) { + score += best; + distinct++; + sig?.push(`phrase "${phrase.text}" +${fmt(best)} (${bestZone})`); + } + } + + for (const rx of label.regexes) { + let best = 0; + let bestZone = ""; + for (const zone of ["title", "first", "any"] as const) { + const hay = + zone === "title" ? titleRaw : zone === "first" ? firstRaw : anyRaw; + const count = countRegex(rx.re, hay); + if (count === 0) continue; + const zf = rx.where === "any" || rx.where === zone ? 1 : 0.75; + const value = rx.weight * ZONE_MULT[zone] * zf * damp(count); + if (value > best) { + best = value; + bestZone = zone; + } + } + if (best > 0) { + score += best; + distinct++; + sig?.push(`regex ${rx.re.source} +${fmt(best)} (${bestZone})`); + } + } + + for (const fn of label.filenames) { + if (countRegex(fn.re, fileNameLower) > 0) { + score += fn.weight; + distinct++; + sig?.push(`filename ${fn.re.source} +${fn.weight}`); + } + } + + for (const md of label.metadata) { + const value = md.field === "any" ? metaAll : (meta[md.field] ?? ""); + if (countRegex(md.re, value) > 0) { + score += md.weight; + distinct++; + sig?.push(`metadata(${md.field}) ${md.re.source} +${md.weight}`); + } + } + + for (const st of label.structural) { + const value = struct[st.signal] ?? 0; + if (value > 0) { + score += st.weight * value; + sig?.push(`structural ${st.signal} +${fmt(st.weight * value)}`); + } + } + + for (const neg of label.negatives) { + const count = + neg.text != null + ? countOccurrences(anyNorm, neg.text) + : countRegex(neg.re, anyRaw); + if (count > 0) { + const value = neg.weight * damp(Math.min(count, 3)); + score -= value; + sig?.push( + `negative ${neg.text != null ? `"${neg.text}"` : (neg.re?.source ?? "")} -${fmt(value)}`, + ); + } + } + + if (score > 0) { + const prior = pagePriorMultiplier(label.id, doc.pageCount); + if (prior !== 1) sig?.push(`page-prior x${fmt(prior)}`); + score *= prior; + scored.push({ label, score, distinct, signals: sig }); + } + } + + // Stable sort by score descending. + scored.sort((a, b) => b.score - a.score); + + const top = scored.length === 0 ? null : scored[0]; + const s1 = top != null ? top.score : 0; + const s2 = scored.length > 1 ? scored[1].score : 0; + const margin = s1 - s2; + + let confidence: HeuristicConfidence = "none"; + if (top != null && s1 >= FLOOR) { + if ( + s1 >= HIGH_SCORE && + margin >= HIGH_MARGIN && + top.distinct >= HIGH_SIGNALS && + s2 <= s1 * 0.65 + ) { + confidence = "high"; + } else if (s1 >= MED_SCORE && margin >= MED_MARGIN) { + confidence = "medium"; + } else { + confidence = "low"; + } + } + + const roundedScore = Math.round(s1); + const explanation = explain ? { explain: toExplanation(en, scored) } : {}; + if (top == null || confidence === "none") { + return { + labels: [], + confidence: "none", + score: roundedScore, + isEnglish: en.isEnglish, + ...explanation, + }; + } + // Internal-only winner (book, menu...): suppress output rather than mislabel. + if (!top.label.emit) { + return { + labels: [], + confidence, + score: roundedScore, + isEnglish: en.isEnglish, + ...explanation, + }; + } + + const labels: string[] = [top.label.id]; + for (let i = 1; i < scored.length && labels.length < 5; i++) { + const s = scored[i]; + if (labels.length - 1 >= SEC_MAX) break; + if ( + s.label.emit && + s.score >= SEC_FLOOR && + s.score >= s1 * SEC_FRAC && + s.distinct >= SEC_SIGNALS + ) { + labels.push(s.label.id); + } + } + return { + labels, + confidence, + score: roundedScore, + isEnglish: en.isEnglish, + ...explanation, + }; +} + +/** True when the top match cleared the high-confidence bar. */ +export function isHighConfidence(r: HeuristicResult): boolean { + return r.confidence === "high"; +} + +/** High confidence AND an emitted label - trustworthy enough to skip the AI engine. */ +export function isDefinitive(r: HeuristicResult): boolean { + return isHighConfidence(r) && r.labels.length > 0; +} + +// --- English detection --- + +interface EnglishResult { + isEnglish: boolean; + lowText: boolean; +} + +export function detectEnglish(text: string): EnglishResult { + const raw = nz(text); + const letters = countAll(LETTERS, raw); + if (letters < 25) return { isEnglish: false, lowText: true }; + + for (const re of SCRIPT_RANGES) { + const hits = countAll(re, raw); + if (hits / letters > 0.25) return { isEnglish: false, lowText: false }; + } + + const latinRatio = countAll(LATIN_LETTER, raw) / letters; + const words = allMatches(WORD, normalize(raw)); + const totalWords = Math.max(words.length, 1); + let enHits = 0; + for (const w of words) if (STOPWORDS.has(w)) enHits++; + const stopRatio = enHits / totalWords; + + let bestScore = 0; + let bestRatio = 0; + let bestDistinct = 0; + let bestDia = 0; + for (const profile of LATIN_PROFILES) { + let hits = 0; + const distinct = new Set(); + for (const w of words) { + if (profile.words.has(w)) { + hits++; + distinct.add(w); + } + } + const diaCount = profile.dia == null ? 0 : countAll(profile.dia, raw); + const ratio = hits / totalWords; + const score = ratio + Math.min(diaCount / totalWords, 0.15) * 6; + if (score > bestScore) { + bestScore = score; + bestRatio = ratio; + bestDistinct = distinct.size; + bestDia = diaCount; + } + } + + const lowText = totalWords < 30; + const nonEnglish = + latinRatio >= 0.7 && + totalWords >= 12 && + (bestDistinct >= 3 || bestDia >= 6) && + (bestDia >= 3 || bestRatio >= 0.1) && + bestScore > stopRatio * 1.2 && + (stopRatio < 0.04 || bestRatio > stopRatio * 1.5); + if (nonEnglish) return { isEnglish: false, lowText }; + + const bar = lowText ? 0.03 : 0.045; + // Data-dense docs (tickets, itineraries, prescriptions) are mostly names and numbers with few + // function words in ANY language; reject stop-poor text only on affirmative foreign evidence. + const foreignEvidence = bestDistinct >= 3 || bestDia >= 6; + return { + isEnglish: latinRatio >= 0.75 && (stopRatio >= bar || !foreignEvidence), + lowText, + }; +} + +// --- Structural signals --- + +function computeStructural(doc: HeuristicDoc): Record { + const all = nz(doc.allZone); + const lines: string[] = []; + for (const l of all.split("\n")) { + const t = l.trim(); + if (t.length > 0) lines.push(t); + } + const tokens: string[] = []; + for (const t of all.split(WHITESPACE)) { + if (t.length > 0) tokens.push(t); + } + const totalTokens = Math.max(tokens.length, 1); + + const currency = countAll(CURRENCY, all); + let numericTokens = 0; + for (const t of tokens) { + if (NUMERIC_TOKEN.test(t) && DIGIT.test(t)) numericTokens++; + } + let formLines = 0; + for (const l of lines) { + if (FORM_LABEL.test(l) || UNDERSCORE4.test(l) || CHECKBOX.test(l)) + formLines++; + } + let dotLeaders = 0; + for (const l of lines) if (DOT_LEADER.test(l)) dotLeaders++; + let bullets = 0; + for (const l of lines) if (BULLET.test(l)) bullets++; + const urls = countAll(URL, all); + const tail = all.length > 2500 ? all.slice(all.length - 2500) : all; + const last4000 = all.length > 4000 ? all.slice(all.length - 4000) : all; + + const s: Record = {}; + s["currency_heavy"] = currency >= 8 ? 1.0 : Math.min(currency / 8.0, 1.0); + s["number_table"] = numericTokens / totalTokens >= 0.22 ? 1.0 : 0.0; + s["form_like"] = formLines >= 6 ? 1.0 : formLines >= 3 ? 0.5 : 0.0; + s["toc"] = TOC.test(all) || dotLeaders >= 5 ? 1.0 : 0.0; + s["signature_block"] = SIG1.test(tail) || SIG2.test(tail) ? 1.0 : 0.0; + s["references_section"] = + REF1.test(last4000) && REF2.test(last4000) ? 1.0 : 0.0; + s["short_doc"] = doc.pageCount > 0 && doc.pageCount <= 2 ? 1.0 : 0.0; + s["long_doc"] = doc.pageCount >= 40 ? 1.0 : 0.0; + s["bullet_heavy"] = bullets >= 12 ? 1.0 : bullets >= 6 ? 0.5 : 0.0; + s["email_headers"] = EMAIL_FROM.test(all) && EMAIL_SUBJ.test(all) ? 1.0 : 0.0; + s["url_heavy"] = urls >= 6 ? 1.0 : 0.0; + s["address_block"] = countAll(ADDRESS, all) >= 2 ? 1.0 : 0.0; + return s; +} + +function pagePriorMultiplier(labelId: string, pageCount: number): number { + const prior = PRIORS!.get(labelId); + if (prior == null || pageCount < 1) return 1; + if (prior.max != null && pageCount > prior.max) { + return Math.max(0.3, prior.max / pageCount); + } + if (pageCount < prior.min) return Math.max(0.3, pageCount / prior.min); + return 1; +} + +// --- Helpers --- + +function nz(s: string | null | undefined): string { + return s == null ? "" : s; +} + +function num(v: unknown): number { + return typeof v === "number" && Number.isFinite(v) ? v : 0; +} + +function str(v: unknown): string | null { + return typeof v === "string" ? v : null; +} + +// Curly apostrophes and fi/fl ligatures survive pdf.js extraction in many PDFs; +// fold them to ASCII so rule phrases authored with ' / fi / fl still match. +const CURLY_APOSTROPHE = /[\u2018\u2019]/g; +const LIGATURE_FI = /\uFB01/g; +const LIGATURE_FL = /\uFB02/g; + +function normalize(text: string | null | undefined): string { + return nz(text) + .toLowerCase() + .replace(CURLY_APOSTROPHE, "'") + .replace(LIGATURE_FI, "fi") + .replace(LIGATURE_FL, "fl") + .replace(WHITESPACE, " "); +} + +function damp(count: number): number { + if (count <= 0) return 0; + return 1 + 0.35 * (Math.log(Math.min(count, 12)) / Math.log(2)); +} + +function countOccurrences(haystack: string, needle: string | null): number { + if (needle == null || needle.length === 0) return 0; + let count = 0; + let idx = haystack.indexOf(needle); + while (idx !== -1 && count < 12) { + count++; + idx = haystack.indexOf(needle, idx + needle.length); + } + return count; +} + +// Non-overlapping matches capped at 12. +function countRegex(re: RegExp | null, text: string | null): number { + if (re == null || text == null || text.length === 0) return 0; + re.lastIndex = 0; + let count = 0; + let m: RegExpExecArray | null; + while (count < 12 && (m = re.exec(text)) !== null) { + count++; + if (m.index === re.lastIndex) re.lastIndex++; // advance past zero-width match + } + return count; +} + +function countAll(re: RegExp, text: string | null): number { + if (text == null || text.length === 0) return 0; + re.lastIndex = 0; + let count = 0; + let m: RegExpExecArray | null; + while ((m = re.exec(text)) !== null) { + count++; + if (m.index === re.lastIndex) re.lastIndex++; + } + return count; +} + +function allMatches(re: RegExp, text: string | null): string[] { + const out: string[] = []; + if (text == null || text.length === 0) return out; + re.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = re.exec(text)) !== null) { + out.push(m[0]); + if (m.index === re.lastIndex) re.lastIndex++; + } + return out; +} diff --git a/frontend/editor/src/proprietary/services/heuristic/heuristicExtractor.ts b/frontend/editor/src/proprietary/services/heuristic/heuristicExtractor.ts new file mode 100644 index 0000000000..426c1133e1 --- /dev/null +++ b/frontend/editor/src/proprietary/services/heuristic/heuristicExtractor.ts @@ -0,0 +1,199 @@ +// pdf.js extraction feeding the engine: page-1 text, a first-5 + last-2 page +// window, Info-dict metadata, and a large-font page-1 "title" zone. + +import { pdfWorkerManager } from "@app/services/pdfWorkerManager"; +import type { + PDFDocumentProxy, + TextItem, +} from "pdfjs-dist/types/src/display/api"; +import type { HeuristicDoc } from "@app/services/heuristic/types"; + +const WINDOW_FIRST = 5; +const WINDOW_LAST = 2; +const PAGE_CHAR_CAP = 8000; +const TITLE_CAP = 400; + +/** One rebuilt text line: baseline y (bottom-origin), its largest font size, and the text. */ +interface Line { + text: string; + size: number; + y: number; +} + +/** Build the engine's input document from a PDF blob. Throws if the PDF can't be read. */ +export async function extractHeuristicDoc( + file: Blob, + fileName: string, +): Promise { + const arrayBuffer = await file.arrayBuffer(); + let pdfDoc: PDFDocumentProxy | null = null; + try { + pdfDoc = await pdfWorkerManager.createDocument(arrayBuffer, { + disableAutoFetch: true, + disableStream: true, + }); + const pageCount = pdfDoc.numPages; + let firstZone = ""; + let titleZone = ""; + if (pageCount >= 1) { + // Page 1 feeds three zones (first, title, window); pump its items once. + const page1 = await pdfDoc.getPage(1); + const items = await pageTextItems(page1); + firstZone = textFromItems(items); + titleZone = titleFromLines( + buildLines(items), + page1.getViewport({ scale: 1 }).height, + ); + } + const parts: string[] = []; + for (const pageNo of windowPages(pageCount)) { + const text = pageNo === 1 ? firstZone : await pageText(pdfDoc, pageNo); + if (text.length > 0) parts.push(text); + } + const meta = await metadata(pdfDoc); + return { + fileName, + pageCount, + meta, + titleZone, + firstZone, + allZone: parts.join("\n"), + }; + } finally { + if (pdfDoc) { + try { + pdfWorkerManager.destroyDocument(pdfDoc); + } catch { + // Best-effort cleanup. + } + } + } +} + +/** First WINDOW_FIRST + last WINDOW_LAST page numbers, deduped, in order. */ +function windowPages(pageCount: number): number[] { + const pages = new Set(); + for (let p = 1; p <= Math.min(WINDOW_FIRST, pageCount); p++) pages.add(p); + for (let p = Math.max(1, pageCount - WINDOW_LAST + 1); p <= pageCount; p++) { + pages.add(p); + } + return [...pages].sort((a, b) => a - b); +} + +function isTextItem(item: unknown): item is TextItem { + return typeof (item as TextItem).str === "string"; +} + +/** + * Pump text items with a plain reader loop: Safari/WebKit cannot async-iterate + * the ReadableStream behind pdf.js getTextContent. + */ +async function pageTextItems( + page: Awaited>, +): Promise { + const reader = page.streamTextContent().getReader(); + const items: unknown[] = []; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + if (Array.isArray(value?.items)) items.push(...value.items); + } + return items; +} + +/** A page's text (items joined, newline on hasEOL), trimmed and capped. */ +async function pageText( + pdfDoc: PDFDocumentProxy, + pageNo: number, +): Promise { + if (pageNo < 1 || pageNo > pdfDoc.numPages) return ""; + const page = await pdfDoc.getPage(pageNo); + return textFromItems(await pageTextItems(page)); +} + +function textFromItems(items: readonly unknown[]): string { + let text = ""; + for (const item of items) { + if (!isTextItem(item)) continue; + text += item.str; + text += item.hasEOL ? "\n" : " "; + } + const trimmed = text.trim(); + return trimmed.length > PAGE_CHAR_CAP + ? trimmed.slice(0, PAGE_CHAR_CAP) + : trimmed; +} + +/** Group items into lines (break on hasEOL), tracking each line's max font size + baseline y. */ +function buildLines(items: readonly unknown[]): Line[] { + const lines: Line[] = []; + let current = ""; + let size = 0; + let y = -1; + const flush = () => { + const text = current.trim(); + if (text.length > 0) lines.push({ text, size, y }); + current = ""; + size = 0; + y = -1; + }; + for (const item of items) { + if (!isTextItem(item)) continue; + const itemSize = Math.hypot(item.transform[0], item.transform[1]); + if (itemSize > size) size = itemSize; + if (y < 0) y = item.transform[5]; + current += item.str; + if (item.hasEOL) flush(); + } + flush(); + return lines; +} + +/** Large-font lines near the top of page 1 approximate the title. */ +function titleFromLines(lines: Line[], pageHeight: number): string { + if (lines.length === 0) return ""; + // pdf.js y is bottom-origin: the top 45% of the page is y > 0.55 * height. + const top = lines.filter((l) => l.y > pageHeight * 0.55); + const pool = top.length > 0 ? top : lines.slice(0, Math.min(8, lines.length)); + let maxSize = 0; + for (const l of pool) maxSize = Math.max(maxSize, l.size); + + const parts: string[] = []; + if (maxSize === 0) { + for (let i = 0; i < Math.min(3, pool.length); i++) parts.push(pool[i].text); + return parts.join("\n"); + } + let taken = 0; + for (const l of pool) { + if (taken >= 6) break; + if (l.size >= maxSize * 0.72) { + parts.push(l.text); + taken++; + } + } + const result = parts.join("\n"); + return result.length > TITLE_CAP ? result.slice(0, TITLE_CAP) : result; +} + +/** Info-dict fields keyed lowercase to match the engine's metadata rules. */ +async function metadata( + pdfDoc: PDFDocumentProxy, +): Promise> { + let info: Record = {}; + try { + const md = await pdfDoc.getMetadata(); + info = (md.info ?? {}) as Record; + } catch { + return {}; + } + const get = (k: string) => + typeof info[k] === "string" ? (info[k] as string) : ""; + return { + title: get("Title"), + author: get("Author"), + subject: get("Subject"), + keywords: get("Keywords"), + creator: get("Creator"), + producer: get("Producer"), + }; +} diff --git a/frontend/editor/src/proprietary/services/heuristic/heuristicRules.json b/frontend/editor/src/proprietary/services/heuristic/heuristicRules.json new file mode 100644 index 0000000000..b34a7bf446 --- /dev/null +++ b/frontend/editor/src/proprietary/services/heuristic/heuristicRules.json @@ -0,0 +1,22352 @@ +{ + "version": 1, + "source": "poc/downloads-classifier (heuristic classifier POC)", + "labels": [ + { + "id": "invoice", + "name": "Invoice", + "emit": true, + "phrases": [ + { + "text": "tax invoice", + "weight": 30, + "where": "title" + }, + { + "text": "invoice", + "weight": 14, + "where": "title" + }, + { + "text": "invoice number", + "weight": 22, + "where": "first" + }, + { + "text": "invoice date", + "weight": 16, + "where": "first" + }, + { + "text": "bill to", + "weight": 12, + "where": "first" + }, + { + "text": "vat invoice", + "weight": 24 + }, + { + "text": "amount due", + "weight": 8 + }, + { + "text": "balance due", + "weight": 14 + }, + { + "text": "invoice total", + "weight": 16 + }, + { + "text": "payment terms", + "weight": 8 + }, + { + "text": "remit payment to", + "weight": 16 + }, + { + "text": "due upon receipt", + "weight": 16 + }, + { + "text": "net 30", + "weight": 12 + }, + { + "text": "total payable", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\binv[-#]?\\d{3,8}\\b", + "weight": 14, + "name": "INV-#### reference" + }, + { + "pattern": "invoice\\s*(?:no\\.?|number|#)\\s*:?\\s*[a-z0-9][a-z0-9-]{2,14}", + "weight": 16, + "name": "Invoice number field" + } + ], + "filenames": [ + { + "pattern": "invoice", + "weight": 26, + "name": "invoice in filename" + }, + { + "pattern": "\\binv[_-]?\\d{3,8}", + "weight": 20, + "name": "INV number in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "invoice", + "weight": 16, + "name": "Invoice in PDF title" + }, + { + "field": "any", + "pattern": "quickbooks|freshbooks|invoice2go|zoho invoice|\\bxero\\b", + "weight": 12, + "name": "Invoicing software producer" + } + ], + "negatives": [ + { + "text": "remittance advice", + "weight": 18, + "name": "remittance heading" + }, + { + "pattern": "packing\\s+slip|delivery\\s+note", + "weight": 14, + "name": "shipping doc wording" + }, + { + "text": "statement period", + "weight": 12, + "name": "statement period" + }, + { + "text": "minimum payment due", + "weight": 14, + "name": "card statement term" + }, + { + "text": "your subscription", + "weight": 16, + "name": "subscription context" + }, + { + "text": "next billing date", + "weight": 18, + "name": "subscription renewal date" + }, + { + "text": "billing period", + "weight": 12, + "name": "recurring billing period" + }, + { + "text": "plan renewal", + "weight": 14, + "name": "subscription plan renewal" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 8 + }, + { + "signal": "number_table", + "weight": 5 + }, + { + "signal": "short_doc", + "weight": 4 + } + ] + }, + { + "id": "receipt", + "name": "Receipt", + "emit": true, + "phrases": [ + { + "text": "sales receipt", + "weight": 28, + "where": "title" + }, + { + "text": "change due", + "weight": 26 + }, + { + "text": "amount tendered", + "weight": 24 + }, + { + "text": "cash tendered", + "weight": 22 + }, + { + "text": "thank you for your purchase", + "weight": 20 + }, + { + "text": "thank you for shopping", + "weight": 20 + }, + { + "text": "receipt number", + "weight": 16 + }, + { + "text": "your receipt", + "weight": 14 + }, + { + "text": "card ending in", + "weight": 14 + }, + { + "text": "payment received", + "weight": 12 + }, + { + "text": "merchant id", + "weight": 12 + }, + { + "text": "approval code", + "weight": 12 + }, + { + "text": "transaction id", + "weight": 8 + }, + { + "text": "total charged", + "weight": 12 + }, + { + "text": "payment confirmation", + "weight": 32, + "where": "title" + }, + { + "text": "your payment was successful", + "weight": 28 + }, + { + "text": "we have received your payment", + "weight": 26 + }, + { + "text": "your payment has been processed", + "weight": 26 + }, + { + "text": "confirmation of payment", + "weight": 24, + "where": "title" + }, + { + "text": "payment successful", + "weight": 22, + "where": "title" + }, + { + "text": "thank you for your payment", + "weight": 22 + }, + { + "text": "proof of payment", + "weight": 16 + }, + { + "text": "payment reference", + "weight": 16 + }, + { + "text": "amount paid", + "weight": 12 + }, + { + "text": "payment date", + "weight": 8 + }, + { + "text": "payment method", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "[x*\\u2022]{4}\\s?-?\\d{4}", + "weight": 12, + "name": "masked card number" + }, + { + "pattern": "auth(?:orization)?\\s*(?:code|#)\\s*:?\\s*[a-z0-9]{4,10}", + "weight": 12, + "name": "authorization code" + }, + { + "pattern": "payment\\s*(?:id|reference|ref\\.?|confirmation)\\s*(?:number|no\\.?|#)?\\s*:?\\s*[a-z0-9][a-z0-9-]{3,24}", + "weight": 14, + "name": "payment reference field" + }, + { + "pattern": "your payment of\\s*(?:usd|eur|gbp|\\$|£|€)", + "weight": 18, + "name": "your payment of " + } + ], + "filenames": [ + { + "pattern": "receipt", + "weight": 26, + "name": "receipt in filename" + }, + { + "pattern": "payment[_-]?confirmation", + "weight": 16, + "name": "payment confirmation filename" + }, + { + "pattern": "payment[-_ ]?confirmation", + "weight": 28, + "name": "payment confirmation filename" + }, + { + "pattern": "proof[-_ ]?of[-_ ]?payment", + "weight": 26, + "name": "proof of payment filename" + }, + { + "pattern": "payment[-_ ]?receipt", + "weight": 18, + "name": "payment receipt filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "receipt", + "weight": 14, + "name": "Receipt in PDF title" + }, + { + "field": "title", + "pattern": "payment (confirmation|receipt)", + "weight": 14, + "name": "payment confirmation in PDF title" + } + ], + "negatives": [ + { + "text": "minimum payment due", + "weight": 18, + "name": "card statement term" + }, + { + "text": "statement period", + "weight": 16, + "name": "statement period" + }, + { + "text": "balance due", + "weight": 12, + "name": "unpaid balance" + }, + { + "text": "your subscription", + "weight": 12, + "name": "subscription context" + }, + { + "text": "next billing date", + "weight": 14, + "name": "subscription renewal date" + }, + { + "text": "renews automatically", + "weight": 14, + "name": "subscription wording" + }, + { + "text": "estimated delivery", + "weight": 12, + "name": "order confirmation field" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 8 + }, + { + "signal": "currency_heavy", + "weight": 6 + } + ] + }, + { + "id": "bank-statement", + "name": "Bank statement", + "emit": true, + "phrases": [ + { + "text": "account statement", + "weight": 18, + "where": "title" + }, + { + "text": "statement of account", + "weight": 14, + "where": "title" + }, + { + "text": "opening balance", + "weight": 20 + }, + { + "text": "closing balance", + "weight": 20 + }, + { + "text": "beginning balance", + "weight": 20 + }, + { + "text": "ending balance", + "weight": 18 + }, + { + "text": "deposits and other credits", + "weight": 24 + }, + { + "text": "withdrawals and other debits", + "weight": 24 + }, + { + "text": "statement period", + "weight": 14 + }, + { + "text": "sort code", + "weight": 12 + }, + { + "text": "money in", + "weight": 12 + }, + { + "text": "money out", + "weight": 12 + }, + { + "text": "available balance", + "weight": 12 + }, + { + "text": "overdraft", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "sort\\s*code\\s*:?\\s*\\d{2}[- ]\\d{2}[- ]\\d{2}", + "weight": 18, + "name": "UK sort code" + }, + { + "pattern": "routing\\s*(?:number|no\\.?|#)\\s*:?\\s*\\d{9}", + "weight": 18, + "name": "US routing number" + }, + { + "pattern": "(?:faster payment|direct debit|standing order)\\b", + "weight": 12, + "name": "UK transaction types" + } + ], + "filenames": [ + { + "pattern": "e?[-_]?statement", + "weight": 16, + "name": "statement in filename" + }, + { + "pattern": "bank[_ -]?statement", + "weight": 24, + "name": "bank statement filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "minimum payment due", + "weight": 22, + "name": "card statement term" + }, + { + "text": "credit limit", + "weight": 18, + "name": "card statement term" + }, + { + "text": "annual percentage rate", + "weight": 14, + "name": "APR wording" + }, + { + "text": "invoice number", + "weight": 10, + "name": "invoice field" + }, + { + "text": "kwh", + "weight": 12, + "name": "utility usage unit" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 10 + }, + { + "signal": "currency_heavy", + "weight": 8 + } + ] + }, + { + "id": "statement-of-account", + "name": "Statement of account", + "emit": true, + "phrases": [ + { + "text": "minimum payment due", + "weight": 28 + }, + { + "text": "late payment warning", + "weight": 26 + }, + { + "text": "minimum payment warning", + "weight": 26 + }, + { + "text": "interest charge calculation", + "weight": 24 + }, + { + "text": "credit limit", + "weight": 20 + }, + { + "text": "available credit", + "weight": 20 + }, + { + "text": "annual percentage rate", + "weight": 20 + }, + { + "text": "purchases and adjustments", + "weight": 18 + }, + { + "text": "statement balance", + "weight": 16 + }, + { + "text": "previous balance", + "weight": 14 + }, + { + "text": "new balance", + "weight": 14 + }, + { + "text": "cash advance", + "weight": 14 + }, + { + "text": "payment due date", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\bAPR\\b", + "flags": "g", + "weight": 12, + "name": "APR (uppercase)" + }, + { + "pattern": "account\\s+ending\\s+(?:in\\s+)?\\d{4}", + "weight": 14, + "name": "account ending ####" + }, + { + "pattern": "\\d{1,2}\\.\\d{2}%\\s+(?:variable\\s+)?apr", + "weight": 14, + "name": "percentage APR rate" + } + ], + "filenames": [ + { + "pattern": "(credit[-_ ]?card|visa|mastercard|amex|barclaycard).{0,12}stat", + "weight": 24, + "name": "card statement filename" + }, + { + "pattern": "card[_-]?statement", + "weight": 22, + "name": "card statement" + } + ], + "metadata": [], + "negatives": [ + { + "text": "sort code", + "weight": 16, + "name": "bank account detail" + }, + { + "text": "routing number", + "weight": 16, + "name": "bank account detail" + }, + { + "text": "deposits and other credits", + "weight": 14, + "name": "checking statement term" + }, + { + "text": "invoice number", + "weight": 12, + "name": "invoice field" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 8 + }, + { + "signal": "number_table", + "weight": 8 + } + ] + }, + { + "id": "purchase-order", + "name": "Purchase order", + "emit": true, + "phrases": [ + { + "text": "purchase order", + "weight": 28, + "where": "title" + }, + { + "text": "please supply the following", + "weight": 24 + }, + { + "text": "official order", + "weight": 20, + "where": "title" + }, + { + "text": "requisition number", + "weight": 18 + }, + { + "text": "qty ordered", + "weight": 16 + }, + { + "text": "quantity ordered", + "weight": 16 + }, + { + "text": "purchase order number", + "weight": 14 + }, + { + "text": "vendor no", + "weight": 12 + }, + { + "text": "po number", + "weight": 10 + }, + { + "text": "delivery date", + "weight": 8 + }, + { + "text": "ship to", + "weight": 8, + "where": "first" + }, + { + "text": "authorized by", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "\\bp\\.?o\\.?\\s*(?:#|no\\.?|number)\\s*:?\\s*[a-z0-9][a-z0-9/-]{2,14}", + "weight": 16, + "name": "PO number field" + }, + { + "pattern": "\\bpo[-/ ]?\\d{4,8}\\b", + "weight": 12, + "name": "PO-#### reference" + }, + { + "pattern": "no\\s+po,?\\s+no\\s+pay", + "weight": 18, + "name": "no PO no pay policy" + } + ], + "filenames": [ + { + "pattern": "purchase[_-]?order", + "weight": 28, + "name": "purchase order filename" + }, + { + "pattern": "\\bpo[_-]?\\d{3,8}", + "weight": 22, + "name": "PO number filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "invoice number", + "weight": 16, + "name": "invoice field" + }, + { + "text": "remittance advice", + "weight": 14, + "name": "remittance heading" + }, + { + "text": "packing slip", + "weight": 12, + "name": "packing slip" + }, + { + "text": "amount due", + "weight": 10, + "name": "billing term" + }, + { + "text": "quotation", + "weight": 10, + "name": "quote wording" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 8 + }, + { + "signal": "short_doc", + "weight": 4 + } + ] + }, + { + "id": "quote", + "name": "Quote", + "emit": true, + "phrases": [ + { + "text": "quotation", + "weight": 22, + "where": "title" + }, + { + "text": "estimate", + "weight": 12, + "where": "title" + }, + { + "text": "we are pleased to quote", + "weight": 26 + }, + { + "text": "this quote is valid", + "weight": 24 + }, + { + "text": "quotation is valid", + "weight": 22 + }, + { + "text": "quotation number", + "weight": 20 + }, + { + "text": "quote number", + "weight": 18 + }, + { + "text": "estimate number", + "weight": 18 + }, + { + "text": "valid until", + "weight": 12 + }, + { + "text": "prices quoted", + "weight": 16 + }, + { + "text": "acceptance signature", + "weight": 16 + }, + { + "text": "to accept this", + "weight": 14 + }, + { + "text": "estimated cost", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "quote\\s*(?:#|no\\.?|number)\\s*:?\\s*[a-z0-9][a-z0-9-]{1,14}", + "weight": 14, + "name": "quote number field" + }, + { + "pattern": "valid\\s+for\\s+\\d{1,3}\\s+days", + "weight": 16, + "name": "valid for N days" + } + ], + "filenames": [ + { + "pattern": "quot(e|ation)", + "weight": 24, + "name": "quote filename" + }, + { + "pattern": "estimate", + "weight": 22, + "name": "estimate filename" + }, + { + "pattern": "\\bqt[_-]?\\d{3,8}", + "weight": 16, + "name": "QT number filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "invoice number", + "weight": 14, + "name": "invoice field" + }, + { + "text": "amount paid", + "weight": 12, + "name": "completed payment" + }, + { + "text": "remittance", + "weight": 12, + "name": "remittance wording" + }, + { + "text": "statement period", + "weight": 12, + "name": "statement period" + }, + { + "text": "executive summary", + "weight": 10, + "name": "proposal wording" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 6 + }, + { + "signal": "short_doc", + "weight": 4 + }, + { + "signal": "number_table", + "weight": 4 + } + ] + }, + { + "id": "remittance-advice", + "name": "Remittance advice", + "emit": true, + "phrases": [ + { + "text": "remittance advice", + "weight": 34, + "where": "title" + }, + { + "text": "payment advice", + "weight": 24, + "where": "title" + }, + { + "text": "amount remitted", + "weight": 24 + }, + { + "text": "invoices paid", + "weight": 20 + }, + { + "text": "bacs payment", + "weight": 20 + }, + { + "text": "in settlement of", + "weight": 18 + }, + { + "text": "the following invoices", + "weight": 16 + }, + { + "text": "payment reference", + "weight": 12 + }, + { + "text": "has been credited", + "weight": 12 + }, + { + "text": "payment date", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "payment\\s+ref(?:erence)?\\s*(?:no\\.?|number|#)?\\s*:?\\s*[a-z0-9][a-z0-9/-]{3,20}", + "weight": 12, + "name": "payment reference field" + } + ], + "filenames": [ + { + "pattern": "remit", + "weight": 26, + "name": "remittance filename" + }, + { + "pattern": "payment[_-]?advice", + "weight": 24, + "name": "payment advice filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "amount due", + "weight": 14, + "name": "unpaid billing term" + }, + { + "text": "minimum payment due", + "weight": 14, + "name": "card statement term" + }, + { + "text": "quotation", + "weight": 12, + "name": "quote wording" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 8 + }, + { + "signal": "short_doc", + "weight": 6 + } + ] + }, + { + "id": "expense-report", + "name": "Expense report", + "emit": true, + "phrases": [ + { + "text": "expense report", + "weight": 32, + "where": "title" + }, + { + "text": "expense claim", + "weight": 28, + "where": "title" + }, + { + "text": "total reimbursable", + "weight": 24 + }, + { + "text": "receipts attached", + "weight": 22 + }, + { + "text": "per diem", + "weight": 20 + }, + { + "text": "employee expense", + "weight": 20 + }, + { + "text": "mileage claimed", + "weight": 20 + }, + { + "text": "business purpose", + "weight": 16 + }, + { + "text": "expenses incurred", + "weight": 14 + }, + { + "text": "subsistence", + "weight": 12 + }, + { + "text": "reimbursement", + "weight": 12 + }, + { + "text": "total claimed", + "weight": 10 + }, + { + "text": "cost center", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\d{1,4}\\s*miles?\\s*(?:@|at)\\s*[$\\u00a3\\u20ac]?\\s?\\d+(?:\\.\\d+)?", + "weight": 14, + "name": "mileage rate line" + } + ], + "filenames": [ + { + "pattern": "expense", + "weight": 24, + "name": "expense filename" + }, + { + "pattern": "reimburs", + "weight": 20, + "name": "reimbursement filename" + } + ], + "metadata": [ + { + "field": "any", + "pattern": "\\bconcur\\b|expensify|zoho expense", + "weight": 14, + "name": "Expense software producer" + } + ], + "negatives": [ + { + "text": "net pay", + "weight": 16, + "name": "payslip term" + }, + { + "text": "gross pay", + "weight": 14, + "name": "payslip term" + }, + { + "text": "policy number", + "weight": 12, + "name": "insurance claim term" + }, + { + "text": "invoice number", + "weight": 10, + "name": "invoice field" + }, + { + "text": "total hours", + "weight": 10, + "name": "timesheet term" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 8 + }, + { + "signal": "currency_heavy", + "weight": 6 + }, + { + "signal": "form_like", + "weight": 5 + } + ] + }, + { + "id": "financial-statement", + "name": "Financial statement", + "emit": true, + "phrases": [ + { + "text": "consolidated financial statements", + "weight": 30 + }, + { + "text": "independent auditor", + "weight": 26 + }, + { + "text": "statement of financial position", + "weight": 26 + }, + { + "text": "notes to the financial statements", + "weight": 24 + }, + { + "text": "of cash flows", + "weight": 22 + }, + { + "text": "balance sheet", + "weight": 20 + }, + { + "text": "income statement", + "weight": 20 + }, + { + "text": "annual report", + "weight": 18, + "where": "title" + }, + { + "text": "retained earnings", + "weight": 18 + }, + { + "text": "cash and cash equivalents", + "weight": 18 + }, + { + "text": "fiscal year ended", + "weight": 16 + }, + { + "text": "earnings per share", + "weight": 16 + }, + { + "text": "profit and loss", + "weight": 14 + }, + { + "text": "quarterly report", + "weight": 14, + "where": "title" + } + ], + "regexes": [ + { + "pattern": "form\\s+10-[kq]\\b", + "weight": 22, + "name": "SEC form 10-K/10-Q" + }, + { + "pattern": "for\\s+the\\s+(?:year|quarter|quarterly\\s+period|three\\s+months|nine\\s+months)\\s+ended", + "weight": 14, + "name": "period ended wording" + }, + { + "pattern": "\\(in\\s+(?:thousands|millions)[^)]{0,40}\\)", + "weight": 16, + "name": "(in thousands/millions)" + } + ], + "filenames": [ + { + "pattern": "annual[_-]?report", + "weight": 26, + "name": "annual report filename" + }, + { + "pattern": "financial[_-]?statements?", + "weight": 22, + "name": "financial statements filename" + }, + { + "pattern": "10-?[kq]\\b", + "weight": 18, + "name": "10-K/10-Q filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "annual report|financial statements", + "weight": 14, + "name": "report title metadata" + } + ], + "negatives": [ + { + "text": "minimum payment due", + "weight": 14, + "name": "card statement term" + }, + { + "text": "sort code", + "weight": 12, + "name": "bank account detail" + }, + { + "text": "invoice number", + "weight": 10, + "name": "invoice field" + }, + { + "text": "business plan", + "weight": 14, + "name": "business plan wording" + } + ], + "structural": [ + { + "signal": "long_doc", + "weight": 8 + }, + { + "signal": "toc", + "weight": 6 + }, + { + "signal": "number_table", + "weight": 6 + } + ] + }, + { + "id": "budget", + "name": "Budget", + "emit": true, + "phrases": [ + { + "text": "annual budget", + "weight": 30, + "where": "title" + }, + { + "text": "budget forecast", + "weight": 30, + "where": "title" + }, + { + "text": "operating budget", + "weight": 26, + "where": "title" + }, + { + "text": "rolling forecast", + "weight": 24 + }, + { + "text": "budget vs actual", + "weight": 26 + }, + { + "text": "actual vs budget", + "weight": 26 + }, + { + "text": "variance analysis", + "weight": 22 + }, + { + "text": "budget variance", + "weight": 20 + }, + { + "text": "forecast assumptions", + "weight": 18 + }, + { + "text": "profit and loss projection", + "weight": 18 + }, + { + "text": "capex", + "weight": 12 + }, + { + "text": "opex", + "weight": 12 + }, + { + "text": "cost center", + "weight": 10 + }, + { + "text": "fiscal year budget", + "weight": 16 + } + ], + "regexes": [ + { + "pattern": "budget\\s+(?:vs\\.?|versus)\\s+actual", + "weight": 18, + "name": "budget vs actual variant" + }, + { + "pattern": "variance\\s+(?:of|is)\\s*[+-]?\\$?[\\d,.]{1,12}", + "weight": 14, + "name": "variance figure" + }, + { + "pattern": "\\bfy\\s?2\\d\\s+budget\\b", + "weight": 14, + "name": "FY budget" + } + ], + "filenames": [ + { + "pattern": "budget", + "weight": 18, + "name": "budget in filename" + }, + { + "pattern": "forecast", + "weight": 18, + "name": "forecast in filename" + }, + { + "pattern": "opex[-_ ]?capex", + "weight": 16, + "name": "opex/capex filename" + }, + { + "pattern": "fy20\\d{2}[-_ ]?budget", + "weight": 22, + "name": "FY budget filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "budget|forecast", + "weight": 12, + "name": "budget/forecast in PDF title" + } + ], + "negatives": [ + { + "text": "independent auditor", + "weight": 18, + "name": "audited financial report term" + }, + { + "text": "consolidated financial statements", + "weight": 16, + "name": "financial report heading" + }, + { + "text": "notes to the financial statements", + "weight": 14, + "name": "financial report term" + }, + { + "text": "reimbursement", + "weight": 12, + "name": "expense report term" + }, + { + "text": "mileage", + "weight": 10, + "name": "expense report term" + }, + { + "text": "employee expense report", + "weight": 14, + "name": "expense report heading" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 8 + }, + { + "signal": "currency_heavy", + "weight": 6 + } + ] + }, + { + "id": "credit-note", + "name": "Credit note", + "emit": true, + "phrases": [ + { + "text": "credit note", + "weight": 34, + "where": "title" + }, + { + "text": "credit memo", + "weight": 30, + "where": "title" + }, + { + "text": "credit note number", + "weight": 28 + }, + { + "text": "reason for credit", + "weight": 26 + }, + { + "text": "credit memo number", + "weight": 24 + }, + { + "text": "amount credited", + "weight": 22 + }, + { + "text": "your account has been credited", + "weight": 22 + }, + { + "text": "total credit", + "weight": 18 + }, + { + "text": "credit note date", + "weight": 18 + }, + { + "text": "original invoice", + "weight": 16 + }, + { + "text": "goods returned", + "weight": 14 + }, + { + "text": "restocking fee", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "(?:relates|applies|refers) to invoice", + "weight": 20, + "name": "relates-to-invoice clause" + }, + { + "pattern": "credit (?:note|memo)\\s*(?:no\\.?|number|#)\\s*:?\\s*[a-z0-9][a-z0-9/-]{2,16}", + "weight": 18, + "name": "credit note number field" + }, + { + "pattern": "\\bcn[-#]\\d{3,8}\\b", + "weight": 14, + "name": "CN-#### reference" + }, + { + "pattern": "(?:-|\\()\\s?(?:[$£€]|usd|gbp|eur)\\s?\\d[\\d,]*(?:\\.\\d{2})?\\)?", + "weight": 12, + "name": "negative currency amount" + } + ], + "filenames": [ + { + "pattern": "credit[-_ ]?note", + "weight": 28, + "name": "credit note filename" + }, + { + "pattern": "credit[-_ ]?memo", + "weight": 26, + "name": "credit memo filename" + }, + { + "pattern": "(^|[^a-z])cn[-_]?\\d{3,}", + "weight": 16, + "name": "CN number in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "credit (note|memo)", + "weight": 16, + "name": "credit note in PDF title" + } + ], + "negatives": [ + { + "text": "remittance advice", + "weight": 20, + "name": "remittance heading" + }, + { + "text": "amount remitted", + "weight": 16, + "name": "remittance language" + }, + { + "text": "change due", + "weight": 16, + "name": "POS receipt term" + }, + { + "text": "amount tendered", + "weight": 14, + "name": "POS receipt term" + }, + { + "text": "balance due", + "weight": 14, + "name": "invoice language" + }, + { + "text": "amount due", + "weight": 12, + "name": "invoice language" + }, + { + "text": "due upon receipt", + "weight": 12, + "name": "invoice language" + }, + { + "text": "estimated delivery", + "weight": 10, + "name": "order confirmation field" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 6 + }, + { + "signal": "short_doc", + "weight": 6 + }, + { + "signal": "number_table", + "weight": 4 + } + ] + }, + { + "id": "loan-agreement", + "name": "Loan agreement", + "emit": true, + "phrases": [ + { + "text": "loan agreement", + "weight": 36, + "where": "title" + }, + { + "text": "credit agreement", + "weight": 32, + "where": "title" + }, + { + "text": "promissory note", + "weight": 34, + "where": "title" + }, + { + "text": "the borrower", + "weight": 18 + }, + { + "text": "the lender", + "weight": 18 + }, + { + "text": "principal amount", + "weight": 18 + }, + { + "text": "repayment schedule", + "weight": 20 + }, + { + "text": "event of default", + "weight": 16 + }, + { + "text": "interest shall accrue", + "weight": 18 + }, + { + "text": "early repayment", + "weight": 10 + }, + { + "text": "outstanding balance", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "interest rate of \\d{1,2}(\\.\\d{1,2})? ?%|\\d{1,2}(\\.\\d{1,2})? ?% per annum", + "flags": "gi", + "weight": 16, + "name": "Interest rate clause" + } + ], + "filenames": [ + { + "pattern": "loan[-_ ]?(agreement|contract|note)|promissory", + "weight": 26, + "name": "Loan filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "mortgage", + "weight": 20, + "name": "Mortgage, not a general loan" + }, + { + "text": "deed of trust", + "weight": 16, + "name": "Mortgage vocabulary" + }, + { + "text": "lease agreement", + "weight": 14, + "name": "Lease" + }, + { + "text": "credit card", + "weight": 14, + "name": "Card statement vocabulary" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 6 + }, + { + "signal": "currency_heavy", + "weight": 4 + } + ] + }, + { + "id": "order-confirmation", + "name": "Order confirmation", + "emit": true, + "phrases": [ + { + "text": "order confirmation", + "weight": 32, + "where": "title" + }, + { + "text": "your order has been placed", + "weight": 28, + "where": "first" + }, + { + "text": "your order is confirmed", + "weight": 28 + }, + { + "text": "thank you for your order", + "weight": 26 + }, + { + "text": "when your order ships", + "weight": 22 + }, + { + "text": "estimated delivery", + "weight": 18 + }, + { + "text": "order placed", + "weight": 16, + "where": "first" + }, + { + "text": "items ordered", + "weight": 16 + }, + { + "text": "order number", + "weight": 14, + "where": "first" + }, + { + "text": "order summary", + "weight": 14 + }, + { + "text": "shipping method", + "weight": 12 + }, + { + "text": "order total", + "weight": 12 + }, + { + "text": "shipping address", + "weight": 6, + "where": "first" + } + ], + "regexes": [ + { + "pattern": "order\\s*(?:no\\.?|number|#)\\s*:?\\s*[a-z0-9][a-z0-9-]{3,20}", + "weight": 14, + "name": "Order number field" + }, + { + "pattern": "\\b\\d{3}-\\d{7}-\\d{7}\\b", + "flags": "g", + "weight": 20, + "name": "Amazon-style order id" + }, + { + "pattern": "arriv(?:es|ing)\\s*:?\\s+(?:mon|tue|wed|thu|fri|sat|sun|by|between)", + "weight": 12, + "name": "arrival estimate line" + } + ], + "filenames": [ + { + "pattern": "order[-_ ]?confirmation", + "weight": 30, + "name": "order confirmation filename" + }, + { + "pattern": "order[-_ #]?\\d{4,}", + "weight": 16, + "name": "order number in filename" + }, + { + "pattern": "your[-_ ]?order", + "weight": 18, + "name": "your order filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "order confirmation", + "weight": 16, + "name": "order confirmation in PDF title" + } + ], + "negatives": [ + { + "text": "invoice number", + "weight": 16, + "name": "invoice field" + }, + { + "text": "packing slip", + "weight": 16, + "name": "shipping doc heading" + }, + { + "text": "your booking is confirmed", + "weight": 16, + "name": "travel booking wording" + }, + { + "text": "check-out date", + "weight": 14, + "name": "hotel confirmation field" + }, + { + "text": "next billing date", + "weight": 14, + "name": "subscription renewal" + }, + { + "text": "minimum payment due", + "weight": 12, + "name": "card statement term" + }, + { + "text": "change due", + "weight": 12, + "name": "POS receipt term" + }, + { + "text": "boarding pass", + "weight": 12, + "name": "boarding pass heading" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 5 + }, + { + "signal": "short_doc", + "weight": 4 + }, + { + "signal": "address_block", + "weight": 4 + } + ] + }, + { + "id": "donation-receipt", + "name": "Donation receipt", + "emit": true, + "phrases": [ + { + "text": "donation receipt", + "weight": 34, + "where": "title" + }, + { + "text": "no goods or services were provided", + "weight": 32 + }, + { + "text": "thank you for your donation", + "weight": 28 + }, + { + "text": "501(c)(3)", + "weight": 26 + }, + { + "text": "charitable contribution", + "weight": 24 + }, + { + "text": "gift aid", + "weight": 24 + }, + { + "text": "tax-deductible", + "weight": 22 + }, + { + "text": "registered charity", + "weight": 22 + }, + { + "text": "thank you for your generous", + "weight": 22 + }, + { + "text": "donation amount", + "weight": 18 + }, + { + "text": "your gift", + "weight": 8 + }, + { + "text": "donor", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "ein\\s*:?\\s*\\d{2}-\\d{7}", + "weight": 14, + "name": "EIN tax id" + }, + { + "pattern": "registered charity (?:no\\.?|number)\\s*:?\\s*\\d{6,8}", + "weight": 18, + "name": "UK charity number" + }, + { + "pattern": "tax[- ]deductible to the (?:full(?:est)? )?extent", + "weight": 18, + "name": "deductible-to-extent clause" + } + ], + "filenames": [ + { + "pattern": "donation", + "weight": 26, + "name": "donation in filename" + }, + { + "pattern": "gift[-_ ]?aid", + "weight": 22, + "name": "gift aid filename" + }, + { + "pattern": "justgiving|gofundme|donorbox", + "weight": 20, + "name": "donation platform filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "donation", + "weight": 14, + "name": "donation in PDF title" + } + ], + "negatives": [ + { + "text": "invoice number", + "weight": 12, + "name": "invoice field" + }, + { + "text": "amount due", + "weight": 12, + "name": "billing term" + }, + { + "text": "minimum payment due", + "weight": 10, + "name": "card statement term" + }, + { + "text": "order confirmation", + "weight": 10, + "name": "retail order heading" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 6 + }, + { + "signal": "address_block", + "weight": 3 + } + ] + }, + { + "id": "letter", + "name": "Letter", + "emit": true, + "phrases": [ + { + "text": "important information about your account", + "weight": 26 + }, + { + "text": "changes to your interest rate", + "weight": 26 + }, + { + "text": "financial services compensation scheme", + "weight": 24 + }, + { + "text": "dear account holder", + "weight": 22 + }, + { + "text": "changes to your account", + "weight": 22 + }, + { + "text": "this letter confirms that", + "weight": 20 + }, + { + "text": "we are writing to inform you", + "weight": 18 + }, + { + "text": "thank you for banking with us", + "weight": 18 + }, + { + "text": "your account with us", + "weight": 16 + }, + { + "text": "dear sir or madam", + "weight": 20, + "where": "first" + }, + { + "text": "to whom it may concern", + "weight": 14, + "where": "first" + }, + { + "text": "yours faithfully", + "weight": 22 + }, + { + "text": "yours sincerely", + "weight": 16 + }, + { + "text": "yours truly", + "weight": 12 + }, + { + "text": "i am writing to inform you", + "weight": 18 + }, + { + "text": "i am writing to complain", + "weight": 22 + }, + { + "text": "with reference to your letter", + "weight": 18 + }, + { + "text": "thank you for your letter", + "weight": 22 + }, + { + "text": "i look forward to hearing from you", + "weight": 12 + }, + { + "text": "please find enclosed", + "weight": 10 + }, + { + "text": "warm regards", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "(?:holds?|maintains?)\\s+an?\\s+account\\s+with", + "weight": 20, + "name": "account verification wording" + }, + { + "pattern": "account\\s+(?:opening|closure)\\s+confirmation", + "weight": 20, + "name": "account opening/closure confirmation" + }, + { + "pattern": "interest\\s+rate\\s+(?:change|is\\s+changing|will\\s+change)", + "weight": 16, + "name": "interest rate change notice" + }, + { + "pattern": "\\bdear (mr|mrs|ms|miss|dr|prof)\\.? [a-z]+", + "flags": "gi", + "weight": 12, + "name": "Personal salutation", + "where": "first" + } + ], + "filenames": [ + { + "pattern": "bank[-_ ]?(confirmation[-_ ]?)?letter", + "weight": 22, + "name": "bank letter filename" + }, + { + "pattern": "account[-_ ]?(verification|confirmation)", + "weight": 20, + "name": "account verification filename" + }, + { + "pattern": "proof[-_ ]?of[-_ ]?(account|funds)", + "weight": 24, + "name": "proof of account/funds filename" + }, + { + "pattern": "^letter[_\\- ]", + "weight": 18, + "name": "Starts with letter_" + }, + { + "pattern": "letter[_\\- ]?(to|from)[_\\- ]", + "weight": 20, + "name": "Letter to/from in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "\\bletter\\b", + "weight": 10, + "name": "Letter in PDF title" + } + ], + "negatives": [ + { + "text": "statement period", + "weight": 22, + "name": "bank statement term" + }, + { + "text": "opening balance", + "weight": 18, + "name": "bank statement term" + }, + { + "text": "closing balance", + "weight": 18, + "name": "bank statement term" + }, + { + "text": "minimum payment due", + "weight": 16, + "name": "card statement term" + }, + { + "text": "deposits and other credits", + "weight": 16, + "name": "checking statement term" + }, + { + "text": "transaction detail", + "weight": 14, + "name": "statement transaction table" + }, + { + "text": "dear hiring manager", + "weight": 22, + "name": "Cover letter salutation" + }, + { + "pattern": "(my|attached|enclosed) (resume|cv|curriculum vitae)", + "flags": "gi", + "weight": 18, + "name": "Resume mention (cover letter)" + }, + { + "pattern": "(pleasure|pleased) to recommend|letter of recommendation", + "flags": "gi", + "weight": 20, + "name": "Reference letter wording" + }, + { + "text": "pleased to offer you the position", + "weight": 22, + "name": "Offer letter wording" + }, + { + "text": "notice is hereby given", + "weight": 18, + "name": "Legal notice wording" + }, + { + "pattern": "patient:? (name|number|id)|re: patient", + "flags": "gi", + "weight": 15, + "name": "Medical letter patient field" + }, + { + "pattern": "internal revenue service|\\bhmrc\\b|department for work and pensions|social security administration", + "flags": "gi", + "weight": 12, + "name": "Government agency letterhead" + } + ], + "structural": [ + { + "signal": "address_block", + "weight": 6 + }, + { + "signal": "short_doc", + "weight": 5 + }, + { + "signal": "signature_block", + "weight": 5 + } + ] + }, + { + "id": "board-report", + "name": "Board report", + "emit": true, + "phrases": [ + { + "text": "board deck", + "weight": 34, + "where": "title" + }, + { + "text": "board pack", + "weight": 32, + "where": "title" + }, + { + "text": "board of directors meeting", + "weight": 26, + "where": "title" + }, + { + "text": "quarterly board meeting", + "weight": 24 + }, + { + "text": "board meeting materials", + "weight": 22 + }, + { + "text": "prepared for the board", + "weight": 20 + }, + { + "text": "distributed to the directors", + "weight": 20 + }, + { + "text": "confidential board materials", + "weight": 24 + }, + { + "text": "for board discussion", + "weight": 18 + }, + { + "text": "kpi review", + "weight": 12 + }, + { + "text": "quarter in review", + "weight": 12 + }, + { + "text": "board resolution to approve", + "weight": 14 + }, + { + "text": "strategic priorities", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "\\bq[1-4]\\s+20\\d{2}\\s+board\\s+(?:deck|pack|meeting)\\b", + "weight": 20, + "name": "Q# 20xx board pack reference" + }, + { + "pattern": "agenda\\s+item\\s+\\d{1,2}\\b", + "weight": 14, + "name": "Agenda item number" + }, + { + "pattern": "\\d{1,2}\\s+in\\s+favou?r,?\\s+\\d{1,2}\\s+against", + "weight": 16, + "name": "Board approval vote tally" + } + ], + "filenames": [ + { + "pattern": "board[_ -]?(?:deck|pack)", + "weight": 28, + "name": "board deck/pack filename" + }, + { + "pattern": "board[_-]?materials", + "weight": 22, + "name": "board materials filename" + }, + { + "pattern": "q[1-4][_-]?20\\d{2}[_-]?board", + "weight": 20, + "name": "Q# board filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "board deck|board pack|board meeting", + "weight": 14, + "name": "Board deck/pack title metadata" + } + ], + "negatives": [ + { + "text": "meeting was called to order", + "weight": 18, + "name": "minutes procedural phrase" + }, + { + "text": "minutes of the previous meeting", + "weight": 18, + "name": "minutes heading" + }, + { + "text": "motion carried", + "weight": 14, + "name": "formal motion vote wording" + }, + { + "text": "the problem we solve", + "weight": 14, + "name": "pitch deck problem framing" + }, + { + "text": "total addressable market", + "weight": 14, + "name": "pitch deck TAM wording" + }, + { + "text": "use of funds", + "weight": 12, + "name": "pitch deck use of funds" + } + ], + "structural": [ + { + "signal": "bullet_heavy", + "weight": 8 + }, + { + "signal": "number_table", + "weight": 6 + }, + { + "signal": "toc", + "weight": 4 + } + ] + }, + { + "id": "pitch-deck", + "name": "Pitch deck", + "emit": true, + "phrases": [ + { + "text": "pitch deck", + "weight": 34, + "where": "title" + }, + { + "text": "the problem we solve", + "weight": 24 + }, + { + "text": "our solution", + "weight": 16 + }, + { + "text": "total addressable market", + "weight": 26 + }, + { + "text": "market size", + "weight": 14 + }, + { + "text": "the ask", + "weight": 22, + "where": "title" + }, + { + "text": "use of funds", + "weight": 22 + }, + { + "text": "traction to date", + "weight": 20 + }, + { + "text": "go-to-market strategy", + "weight": 12 + }, + { + "text": "meet the team", + "weight": 12 + }, + { + "text": "why now", + "weight": 16, + "where": "title" + }, + { + "text": "raising a seed round", + "weight": 22 + }, + { + "text": "series a round", + "weight": 16 + } + ], + "regexes": [ + { + "pattern": "\\btam\\b", + "weight": 14, + "name": "TAM acronym" + }, + { + "pattern": "\\b(?:pre-)?seed\\s+round\\b|\\bseries\\s+[a-e]\\s+round\\b", + "weight": 18, + "name": "Funding round stage mention" + }, + { + "pattern": "raising\\s+\\$\\s?\\d[\\d,.]*\\s*(?:m|k|million|thousand)?\\b", + "weight": 18, + "name": "Raising $X amount" + }, + { + "pattern": "\\bsam\\b\\s*(?:/|and|&)\\s*\\bsom\\b", + "weight": 12, + "name": "SAM/SOM market sizing" + } + ], + "filenames": [ + { + "pattern": "pitch[_-]?deck", + "weight": 30, + "name": "pitch deck filename" + }, + { + "pattern": "seed[_-]?(?:deck|round)", + "weight": 22, + "name": "seed deck/round filename" + }, + { + "pattern": "investor[_-]?deck", + "weight": 20, + "name": "investor deck filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "pitch deck|investor deck", + "weight": 15, + "name": "Pitch deck title metadata" + } + ], + "negatives": [ + { + "text": "board of directors meeting", + "weight": 14, + "name": "board deck wording" + }, + { + "text": "board pack", + "weight": 14, + "name": "board deck wording" + }, + { + "text": "quarter in review", + "weight": 12, + "name": "board KPI review wording" + }, + { + "text": "guidance for the full year", + "weight": 12, + "name": "earnings guidance wording" + } + ], + "structural": [ + { + "signal": "bullet_heavy", + "weight": 8 + }, + { + "signal": "short_doc", + "weight": 5 + } + ] + }, + { + "id": "investment-summary", + "name": "Investment summary", + "emit": true, + "phrases": [ + { + "text": "capitalization table", + "weight": 34, + "where": "title" + }, + { + "text": "cap table", + "weight": 30, + "where": "title" + }, + { + "text": "fully diluted shares outstanding", + "weight": 26 + }, + { + "text": "fully diluted ownership", + "weight": 24 + }, + { + "text": "option pool", + "weight": 18 + }, + { + "text": "preferred stock", + "weight": 12 + }, + { + "text": "common stock", + "weight": 8 + }, + { + "text": "series a preferred", + "weight": 18 + }, + { + "text": "shares outstanding", + "weight": 12 + }, + { + "text": "ownership percentage", + "weight": 14 + }, + { + "text": "as-converted basis", + "weight": 18 + }, + { + "text": "authorized shares", + "weight": 10 + }, + { + "text": "fully diluted basis", + "weight": 16 + }, + { + "text": "investor update", + "weight": 32, + "where": "title" + }, + { + "text": "dear investors", + "weight": 22, + "where": "first" + }, + { + "text": "highlights and lowlights", + "weight": 26 + }, + { + "text": "burn rate", + "weight": 16 + }, + { + "text": "monthly recurring revenue", + "weight": 18 + }, + { + "text": "annual recurring revenue", + "weight": 18 + }, + { + "text": "the ask", + "weight": 12 + }, + { + "text": "key metrics this month", + "weight": 20 + }, + { + "text": "how you can help", + "weight": 18 + }, + { + "text": "months of runway", + "weight": 20 + }, + { + "text": "net new mrr", + "weight": 16 + }, + { + "text": "runway", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "\\d{1,3}(?:\\.\\d{1,2})?\\s?%\\s+(?:fully\\s+diluted|ownership|of\\s+the\\s+company)", + "weight": 16, + "name": "% ownership figure" + }, + { + "pattern": "class\\s+[a-c]\\s+(?:common|preferred)\\s+stock", + "weight": 14, + "name": "Share class label" + }, + { + "pattern": "\\bfd\\s+shares\\b", + "weight": 12, + "name": "FD shares abbreviation" + }, + { + "pattern": "\\bmrr\\b", + "weight": 12, + "name": "MRR acronym" + }, + { + "pattern": "\\barr\\b", + "weight": 10, + "name": "ARR acronym" + }, + { + "pattern": "\\bq[1-4]\\s+20\\d{2}\\s+investor\\s+update\\b", + "weight": 20, + "name": "Q# investor update heading" + } + ], + "filenames": [ + { + "pattern": "cap[_-]?table", + "weight": 30, + "name": "cap table filename" + }, + { + "pattern": "captable", + "weight": 26, + "name": "captable filename" + }, + { + "pattern": "\\bcap[_-]?tbl\\b", + "weight": 16, + "name": "cap tbl abbreviation filename" + }, + { + "pattern": "investor[_-]?update", + "weight": 30, + "name": "investor update filename" + }, + { + "pattern": "monthly[_-]?update", + "weight": 14, + "name": "monthly update filename" + }, + { + "pattern": "investor[_-]?letter", + "weight": 18, + "name": "investor letter filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "cap table|capitalization table|capitalisation table", + "weight": 16, + "name": "Cap table title metadata" + }, + { + "field": "title", + "pattern": "investor update", + "weight": 16, + "name": "Investor update title metadata" + } + ], + "negatives": [ + { + "text": "opening balance", + "weight": 16, + "name": "bank statement wording" + }, + { + "text": "closing balance", + "weight": 14, + "name": "bank statement wording" + }, + { + "text": "sort code", + "weight": 14, + "name": "UK bank account detail" + }, + { + "text": "routing number", + "weight": 14, + "name": "US bank account detail" + }, + { + "text": "balance sheet", + "weight": 10, + "name": "financial report wording" + }, + { + "text": "earnings call", + "weight": 14, + "name": "public earnings wording" + }, + { + "text": "guidance for the full year", + "weight": 12, + "name": "public earnings guidance wording" + }, + { + "text": "diluted earnings per share", + "weight": 12, + "name": "EPS wording" + }, + { + "text": "board pack", + "weight": 10, + "name": "board deck wording" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 10 + }, + { + "signal": "form_like", + "weight": 4 + }, + { + "signal": "short_doc", + "weight": 4 + }, + { + "signal": "bullet_heavy", + "weight": 6 + } + ] + }, + { + "id": "letter-of-intent", + "name": "Letter of intent", + "emit": true, + "phrases": [ + { + "text": "term sheet", + "weight": 32, + "where": "title" + }, + { + "text": "pre-money valuation", + "weight": 28 + }, + { + "text": "post-money valuation", + "weight": 24 + }, + { + "text": "liquidation preference", + "weight": 26 + }, + { + "text": "pro rata rights", + "weight": 20 + }, + { + "text": "anti-dilution protection", + "weight": 18 + }, + { + "text": "drag-along rights", + "weight": 16 + }, + { + "text": "tag-along rights", + "weight": 14 + }, + { + "text": "board seats", + "weight": 10 + }, + { + "text": "this term sheet is non-binding", + "weight": 26 + }, + { + "text": "vesting schedule", + "weight": 8 + }, + { + "text": "right of first refusal", + "weight": 12 + }, + { + "text": "letter of intent", + "weight": 34, + "where": "title" + }, + { + "text": "memorandum of understanding", + "weight": 28, + "where": "title" + }, + { + "text": "this letter of intent", + "weight": 24 + }, + { + "text": "non-binding", + "weight": 20 + }, + { + "text": "indicative terms", + "weight": 22 + }, + { + "text": "exclusivity period", + "weight": 20 + }, + { + "text": "intent to acquire", + "weight": 22 + }, + { + "text": "intent to purchase", + "weight": 20 + }, + { + "text": "subject to definitive agreement", + "weight": 24 + }, + { + "text": "good faith negotiations", + "weight": 14 + }, + { + "text": "no binding obligation", + "weight": 18 + }, + { + "text": "definitive agreement", + "weight": 10 + }, + { + "text": "expression of interest", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "series\\s+[a-e]\\s+preferred\\s+stock", + "weight": 16, + "name": "Series X preferred stock" + }, + { + "pattern": "1x\\s+non-?participating", + "weight": 16, + "name": "1x liquidation preference" + }, + { + "pattern": "\\$\\s?\\d[\\d,.]*\\s*(?:m|million)?\\s+pre-money", + "weight": 16, + "name": "$X pre-money figure" + }, + { + "pattern": "\\bloi\\b", + "weight": 12, + "name": "LOI abbreviation" + }, + { + "pattern": "\\bmou\\b", + "weight": 12, + "name": "MOU abbreviation" + }, + { + "pattern": "subject to (?:the negotiation and execution of )?a definitive (?:purchase |merger )?agreement", + "weight": 20, + "name": "subject to definitive agreement clause" + }, + { + "pattern": "shall not be binding on either party", + "weight": 16, + "name": "non-binding clause" + } + ], + "filenames": [ + { + "pattern": "term[_-]?sheet", + "weight": 30, + "name": "term sheet filename" + }, + { + "pattern": "ts[_-]?series[_-]?[a-e]", + "weight": 18, + "name": "TS series letter filename" + }, + { + "pattern": "letter[-_ ]?of[-_ ]?intent", + "weight": 30, + "name": "letter of intent filename" + }, + { + "pattern": "(^|[^a-z])loi([^a-z]|$)", + "weight": 22, + "name": "LOI in filename" + }, + { + "pattern": "(^|[^a-z])mou([^a-z]|$)", + "weight": 20, + "name": "MOU in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "term sheet", + "weight": 16, + "name": "Term sheet title metadata" + }, + { + "field": "title", + "pattern": "letter of intent|memorandum of understanding", + "weight": 16, + "name": "LOI/MOU in PDF title" + } + ], + "negatives": [ + { + "text": "entire agreement", + "weight": 12, + "name": "formal contract wording" + }, + { + "text": "governing law", + "weight": 10, + "name": "formal contract wording" + }, + { + "text": "fully diluted shares outstanding", + "weight": 10, + "name": "cap table wording" + }, + { + "text": "in witness whereof", + "weight": 16, + "name": "binding contract execution clause" + }, + { + "text": "this agreement is entered into", + "weight": 16, + "name": "definitive agreement wording" + }, + { + "text": "representations and warranties", + "weight": 18, + "name": "M&A definitive agreement term" + }, + { + "text": "purchase price adjustment", + "weight": 14, + "name": "M&A definitive agreement term" + }, + { + "text": "closing conditions", + "weight": 14, + "name": "M&A closing term" + }, + { + "text": "non-disclosure agreement", + "weight": 10, + "name": "NDA heading" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 5 + }, + { + "signal": "short_doc", + "weight": 6 + }, + { + "signal": "signature_block", + "weight": 4 + } + ] + }, + { + "id": "quarterly-report", + "name": "Quarterly report", + "emit": true, + "phrases": [ + { + "text": "quarterly earnings release", + "weight": 28, + "where": "title" + }, + { + "text": "earnings call", + "weight": 24 + }, + { + "text": "conference call to discuss", + "weight": 20 + }, + { + "text": "non-gaap", + "weight": 14 + }, + { + "text": "guidance for the full year", + "weight": 20 + }, + { + "text": "raises full-year guidance", + "weight": 18 + }, + { + "text": "year-over-year", + "weight": 10 + }, + { + "text": "operating margin", + "weight": 12 + }, + { + "text": "diluted earnings per share", + "weight": 18 + }, + { + "text": "beat analyst estimates", + "weight": 20 + }, + { + "text": "consensus estimates", + "weight": 16 + }, + { + "text": "interim results", + "weight": 14, + "where": "title" + }, + { + "text": "half-year results", + "weight": 14, + "where": "title" + } + ], + "regexes": [ + { + "pattern": "reports\\s+(?:first|second|third|fourth|q[1-4])\\s+quarter\\s+(?:and\\s+half-year\\s+)?(?:20\\d{2}\\s+)?(?:financial\\s+)?results", + "weight": 22, + "name": "reports Q# results headline" + }, + { + "pattern": "eps\\s+of\\s+(?:\\$|\\u00a3|\\u20ac)?\\d", + "weight": 16, + "name": "EPS of $X" + }, + { + "pattern": "rais(?:e[sd]?|ing)\\s+(?:its\\s+)?(?:full[- ]year\\s+)?guidance", + "weight": 16, + "name": "raises guidance" + } + ], + "filenames": [ + { + "pattern": "earnings[_-]?(?:release|report)", + "weight": 28, + "name": "earnings release/report filename" + }, + { + "pattern": "q[1-4][_-]?20\\d{2}[_-]?earnings", + "weight": 22, + "name": "Q# earnings filename" + }, + { + "pattern": "interim[_-]?results", + "weight": 18, + "name": "interim results filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "earnings release|earnings report|interim results", + "weight": 12, + "name": "Earnings title metadata" + } + ], + "negatives": [ + { + "text": "consolidated financial statements", + "weight": 14, + "name": "10-K/10-Q financial report wording" + }, + { + "text": "notes to the financial statements", + "weight": 14, + "name": "annual report wording" + }, + { + "text": "form 10-k", + "weight": 12, + "name": "SEC annual filing" + }, + { + "text": "form 10-q", + "weight": 10, + "name": "SEC quarterly filing" + }, + { + "text": "months of runway", + "weight": 10, + "name": "investor update wording" + }, + { + "text": "burn rate", + "weight": 10, + "name": "investor update wording" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 6 + }, + { + "signal": "number_table", + "weight": 6 + } + ] + }, + { + "id": "report", + "name": "Report", + "emit": true, + "phrases": [ + { + "text": "due diligence report", + "weight": 32, + "where": "title" + }, + { + "text": "due diligence findings", + "weight": 26 + }, + { + "text": "red flags", + "weight": 18 + }, + { + "text": "data room", + "weight": 16 + }, + { + "text": "target company", + "weight": 14 + }, + { + "text": "confirmatory due diligence", + "weight": 22 + }, + { + "text": "diligence memo", + "weight": 26, + "where": "title" + }, + { + "text": "legal due diligence", + "weight": 20 + }, + { + "text": "financial due diligence", + "weight": 20 + }, + { + "text": "key risks identified", + "weight": 14 + }, + { + "text": "management interviews", + "weight": 12 + }, + { + "text": "quality of earnings", + "weight": 18 + }, + { + "text": "report to cabinet", + "weight": 32, + "where": "title" + }, + { + "text": "portfolio holder", + "weight": 30 + }, + { + "text": "wards affected", + "weight": 28 + }, + { + "text": "the committee is recommended to", + "weight": 28 + }, + { + "text": "report to the committee", + "weight": 24, + "where": "title" + }, + { + "text": "it is recommended that", + "weight": 20 + }, + { + "text": "key decision", + "weight": 20, + "where": "first" + }, + { + "text": "scrutiny committee", + "weight": 18 + }, + { + "text": "report of the director", + "weight": 22 + }, + { + "text": "reason for the decision", + "weight": 16 + }, + { + "text": "financial implications", + "weight": 12 + }, + { + "text": "legal implications", + "weight": 12 + }, + { + "text": "exempt from publication", + "weight": 14 + } + ], + "regexes": [ + { + "pattern": "due\\s+diligence\\s+(?:report|memo|findings|summary)", + "weight": 20, + "name": "Due diligence heading" + }, + { + "pattern": "\\bqo?e\\s+report\\b", + "weight": 12, + "name": "QoE report abbreviation" + }, + { + "pattern": "agenda\\s+item\\s+(no\\.?\\s*)?\\d{1,2}", + "weight": 14, + "name": "Agenda item number" + }, + { + "pattern": "wards?\\s+affected\\s*:", + "weight": 22, + "name": "Wards affected field" + }, + { + "pattern": "report\\s+(of|by)\\s+the\\s+(director|head|chief|executive|county\\s+administrator|city\\s+manager)", + "weight": 18, + "name": "Officer report line" + }, + { + "pattern": "cabinet\\s+member\\s+for\\s+[a-z]", + "weight": 20, + "name": "Cabinet member portfolio" + }, + { + "pattern": "recommendations?\\s*:?\\s*that\\s+(the\\s+)?(cabinet|committee|council|board)", + "weight": 24, + "name": "Recommendation to body" + } + ], + "filenames": [ + { + "pattern": "due[_-]?diligence", + "weight": 30, + "name": "due diligence filename" + }, + { + "pattern": "\\bdd[_-]?report\\b", + "weight": 18, + "name": "DD report filename" + }, + { + "pattern": "diligence[_-]?memo", + "weight": 22, + "name": "diligence memo filename" + }, + { + "pattern": "(cabinet|committee|council)[-_ ]?report", + "weight": 24, + "name": "Committee report filename" + }, + { + "pattern": "staff[-_ ]?report", + "weight": 20, + "name": "Staff report filename" + }, + { + "pattern": "agenda[-_ ]?item", + "weight": 14, + "name": "Agenda item filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "due diligence", + "weight": 16, + "name": "Due diligence title metadata" + }, + { + "field": "title", + "pattern": "report to (cabinet|council|committee)", + "weight": 14, + "name": "Report-to title" + } + ], + "negatives": [ + { + "text": "entire agreement", + "weight": 10, + "name": "formal contract wording" + }, + { + "text": "case study", + "weight": 10, + "name": "case study wording" + }, + { + "text": "consolidated financial statements", + "weight": 8, + "name": "financial report wording" + }, + { + "text": "minutes of the meeting", + "weight": 26, + "name": "Meeting minutes" + }, + { + "text": "apologies for absence", + "weight": 24, + "name": "Minutes boilerplate" + }, + { + "text": "the meeting was called to order", + "weight": 22, + "name": "US minutes boilerplate" + }, + { + "text": "matters arising", + "weight": 14, + "name": "Minutes section" + }, + { + "text": "notice of meeting", + "weight": 16, + "name": "Meeting agenda cover" + } + ], + "structural": [ + { + "signal": "long_doc", + "weight": 6 + }, + { + "signal": "toc", + "weight": 4 + }, + { + "signal": "references_section", + "weight": 3 + }, + { + "signal": "bullet_heavy", + "weight": 3 + } + ] + }, + { + "id": "regulatory-filing", + "name": "Regulatory filing", + "emit": true, + "phrases": [ + { + "text": "offering memorandum", + "weight": 32, + "where": "title" + }, + { + "text": "prospectus", + "weight": 24, + "where": "title" + }, + { + "text": "private placement memorandum", + "weight": 26, + "where": "title" + }, + { + "text": "risk factors", + "weight": 12 + }, + { + "text": "initial public offering", + "weight": 22 + }, + { + "text": "offering price", + "weight": 18 + }, + { + "text": "these securities have not been registered", + "weight": 28 + }, + { + "text": "underwriters", + "weight": 10 + }, + { + "text": "use of proceeds", + "weight": 18 + }, + { + "text": "prospective investors", + "weight": 14 + }, + { + "text": "subscription agreement", + "weight": 14 + }, + { + "text": "be it enacted", + "weight": 36 + }, + { + "text": "this act may be cited as", + "weight": 32 + }, + { + "text": "a bill to", + "weight": 26, + "where": "title" + }, + { + "text": "an act to", + "weight": 24, + "where": "title" + }, + { + "text": "short title and commencement", + "weight": 26 + }, + { + "text": "in the senate of the united states", + "weight": 26, + "where": "first" + }, + { + "text": "in the house of representatives", + "weight": 22, + "where": "first" + }, + { + "text": "is amended as follows", + "weight": 20 + }, + { + "text": "ordered to be printed", + "weight": 14, + "where": "first" + }, + { + "text": "royal assent", + "weight": 18 + }, + { + "text": "this act extends to", + "weight": 18 + }, + { + "text": "by the authority of parliament", + "weight": 20 + }, + { + "text": "subsection (1)", + "weight": 8 + }, + { + "text": "insert after", + "weight": 6 + }, + { + "text": "these regulations may be cited as", + "weight": 34 + }, + { + "text": "notice of proposed rulemaking", + "weight": 32, + "where": "title" + }, + { + "text": "statutory instruments", + "weight": 30, + "where": "first" + }, + { + "text": "in exercise of the powers conferred", + "weight": 30 + }, + { + "text": "this note is not part of the regulations", + "weight": 30 + }, + { + "text": "federal register", + "weight": 24 + }, + { + "text": "code of federal regulations", + "weight": 22 + }, + { + "text": "final rule", + "weight": 20, + "where": "title" + }, + { + "text": "the secretary hereby", + "weight": 22 + }, + { + "text": "regulatory impact analysis", + "weight": 16 + }, + { + "text": "compliance date", + "weight": 14 + }, + { + "text": "laid before parliament", + "weight": 20 + }, + { + "text": "comments must be received", + "weight": 12 + }, + { + "text": "articles of incorporation", + "weight": 30, + "where": "title" + }, + { + "text": "certificate of incorporation", + "weight": 28, + "where": "title" + }, + { + "text": "articles of association", + "weight": 26, + "where": "title" + }, + { + "text": "memorandum of association", + "weight": 24 + } + ], + "regexes": [ + { + "pattern": "securities\\s+act\\s+of\\s+1933", + "weight": 18, + "name": "Securities Act of 1933 reference" + }, + { + "pattern": "registration\\s+statement\\s+(?:on\\s+form\\s+[a-z]-?\\d)?", + "weight": 12, + "name": "SEC registration statement" + }, + { + "pattern": "offering\\s+price\\s+of\\s+\\$\\d", + "weight": 14, + "name": "Offering price of $X" + }, + { + "pattern": "\\bh\\.\\s?r\\.\\s?\\d{1,5}\\b", + "weight": 20, + "name": "House bill number" + }, + { + "pattern": "\\b1\\d{2}(st|nd|rd|th)\\s+congress\\b", + "weight": 22, + "name": "Congress session" + }, + { + "pattern": "sec(tion)?\\.?\\s*\\d{1,3}\\.\\s+(short title|definitions|purposes)", + "weight": 20, + "name": "Bill section heading" + }, + { + "pattern": "\\b20\\d{2}\\s+chapter\\s+\\d{1,3}\\b", + "weight": 18, + "name": "UK act chapter number" + }, + { + "pattern": "amendments?\\s+(of|to)\\s+the\\s+[a-z ]{3,40}\\s+act\\s+(19|20)\\d{2}", + "weight": 18, + "name": "Amendment of act" + }, + { + "pattern": "\\b\\d{1,2}\\s+cfr\\s+parts?\\s+\\d+", + "weight": 24, + "name": "CFR part citation" + }, + { + "pattern": "\\brin\\s+\\d{4}-[a-z]{2}\\d{2}\\b", + "weight": 22, + "name": "Regulation identifier number" + }, + { + "pattern": "\\bfr\\s+doc\\.?\\s+\\d{4}-\\d{4,6}\\b", + "weight": 20, + "name": "Federal Register doc number" + }, + { + "pattern": "\\b20\\d{2}\\s+no\\.\\s?\\d{1,4}\\b", + "weight": 16, + "name": "SI year and number" + }, + { + "pattern": "docket\\s+(no\\.?|number)\\s*:?\\s*[a-z]{2,6}-", + "weight": 14, + "name": "Rulemaking docket" + } + ], + "filenames": [ + { + "pattern": "prospectus", + "weight": 28, + "name": "prospectus filename" + }, + { + "pattern": "offering[_-]?memorandum", + "weight": 26, + "name": "offering memorandum filename" + }, + { + "pattern": "\\bppm\\b", + "weight": 14, + "name": "PPM abbreviation filename" + }, + { + "pattern": "\\b(hr|s)[-_ ]?\\d{3,5}\\b", + "weight": 16, + "name": "Bill number filename" + }, + { + "pattern": "bill", + "weight": 15, + "name": "Bill filename" + }, + { + "pattern": "(ukpga|act[-_ ]?20\\d{2})", + "weight": 20, + "name": "Act filename" + }, + { + "pattern": "federal[-_ ]?register|final[-_ ]?rule", + "weight": 22, + "name": "Federal Register rule filename" + }, + { + "pattern": "statutory[-_ ]?instrument|uksi", + "weight": 24, + "name": "Statutory instrument filename" + }, + { + "pattern": "\\bcfr\\b|rulemaking", + "weight": 18, + "name": "CFR/rulemaking filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "prospectus|offering memorandum", + "weight": 16, + "name": "Prospectus title metadata" + }, + { + "field": "title", + "pattern": "a bill|an act|public law", + "weight": 12, + "name": "Bill/act in PDF title" + }, + { + "field": "any", + "pattern": "legislation\\.gov\\.uk", + "weight": 14, + "name": "legislation.gov.uk metadata" + }, + { + "field": "title", + "pattern": "federal register|final rule|regulations 20\\d{2}", + "weight": 12, + "name": "Rule in PDF title" + } + ], + "negatives": [ + { + "text": "annual report", + "weight": 10, + "name": "financial report wording" + }, + { + "text": "entire agreement", + "weight": 10, + "name": "formal contract wording" + }, + { + "text": "this term sheet is non-binding", + "weight": 12, + "name": "term sheet wording" + }, + { + "text": "terms and conditions", + "weight": 26, + "name": "T&C legalese confusable" + }, + { + "text": "this agreement", + "weight": 20, + "name": "Contract wording" + }, + { + "text": "privacy policy", + "weight": 16, + "name": "Policy document" + }, + { + "text": "licensee", + "weight": 14, + "name": "License agreement party" + }, + { + "text": "statutory instrument", + "weight": 18, + "name": "Regulation, not primary legislation" + }, + { + "text": "we welcome your views", + "weight": 16, + "name": "Consultation paper wording" + } + ], + "structural": [ + { + "signal": "long_doc", + "weight": 6 + }, + { + "signal": "toc", + "weight": 5 + }, + { + "signal": "references_section", + "weight": 3 + } + ] + }, + { + "id": "appraisal-report", + "name": "Appraisal report", + "emit": true, + "phrases": [ + { + "text": "409a valuation", + "weight": 34, + "where": "title" + }, + { + "text": "valuation report", + "weight": 26, + "where": "title" + }, + { + "text": "fair market value", + "weight": 18 + }, + { + "text": "discounted cash flow", + "weight": 22 + }, + { + "text": "comparable company analysis", + "weight": 22 + }, + { + "text": "enterprise value", + "weight": 14 + }, + { + "text": "guideline public company method", + "weight": 20 + }, + { + "text": "weighted average cost of capital", + "weight": 18 + }, + { + "text": "market approach", + "weight": 8 + }, + { + "text": "income approach", + "weight": 8 + }, + { + "text": "asset approach", + "weight": 8 + }, + { + "text": "valuation date", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "\\b409a\\b", + "weight": 20, + "name": "409A reference" + }, + { + "pattern": "\\bwacc\\b", + "weight": 14, + "name": "WACC acronym" + }, + { + "pattern": "discount\\s+rate\\s+of\\s+\\d{1,2}(?:\\.\\d+)?%", + "weight": 14, + "name": "Discount rate percentage" + } + ], + "filenames": [ + { + "pattern": "409a", + "weight": 28, + "name": "409A filename" + }, + { + "pattern": "valuation[_-]?report", + "weight": 26, + "name": "valuation report filename" + }, + { + "pattern": "business[_-]?valuation", + "weight": 22, + "name": "business valuation filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "409a|valuation report", + "weight": 16, + "name": "Valuation report title metadata" + } + ], + "negatives": [ + { + "text": "balance sheet", + "weight": 8, + "name": "financial report wording" + }, + { + "text": "consolidated financial statements", + "weight": 10, + "name": "financial report wording" + }, + { + "text": "fully diluted shares outstanding", + "weight": 8, + "name": "cap table wording" + }, + { + "text": "due diligence", + "weight": 8, + "name": "due diligence wording" + } + ], + "structural": [ + { + "signal": "long_doc", + "weight": 5 + }, + { + "signal": "number_table", + "weight": 6 + }, + { + "signal": "toc", + "weight": 3 + } + ] + }, + { + "id": "tax-form", + "name": "Tax form", + "emit": true, + "phrases": [ + { + "text": "wage and tax statement", + "weight": 34, + "where": "title" + }, + { + "text": "statement of remuneration paid", + "weight": 30, + "where": "title" + }, + { + "text": "withholding certificate", + "weight": 24, + "where": "title" + }, + { + "text": "request for taxpayer identification number", + "weight": 30, + "where": "title" + }, + { + "text": "p60 end of year certificate", + "weight": 34, + "where": "title" + }, + { + "text": "details of employee leaving work", + "weight": 30, + "where": "title" + }, + { + "text": "pay and income tax details", + "weight": 24, + "where": "first" + }, + { + "text": "nonemployee compensation", + "weight": 24 + }, + { + "text": "social security wages", + "weight": 26 + }, + { + "text": "medicare wages and tips", + "weight": 26 + }, + { + "text": "federal income tax withheld", + "weight": 16 + }, + { + "text": "income tax deducted", + "weight": 14 + }, + { + "text": "employer identification number", + "weight": 8, + "where": "first" + }, + { + "text": "national insurance contributions in this employment", + "weight": 22 + } + ], + "regexes": [ + { + "pattern": "\\bform\\s+w-?2\\b", + "weight": 20, + "name": "Form W-2" + }, + { + "pattern": "\\b1099-(?:misc|nec|int|div|r|g|k|b|s)\\b", + "weight": 22, + "name": "Form 1099 variant" + }, + { + "pattern": "\\bform\\s+w-?9\\b", + "weight": 20, + "name": "Form W-9" + }, + { + "pattern": "\\bform\\s+w-?4\\b", + "weight": 18, + "name": "Form W-4" + }, + { + "pattern": "\\b1095-[abc]\\b", + "weight": 18, + "name": "Form 1095 health coverage" + }, + { + "pattern": "omb no\\.?\\s*1545-\\d{4}", + "weight": 12, + "name": "IRS OMB number" + } + ], + "filenames": [ + { + "pattern": "(?:^|[^a-z0-9])w[-_ ]?2(?:[^0-9]|$)", + "weight": 22, + "name": "W-2 in filename" + }, + { + "pattern": "(?:^|[^0-9])1099(?:[^0-9]|$)", + "weight": 22, + "name": "1099 in filename" + }, + { + "pattern": "(?:^|[^a-z0-9])w[-_ ]?[49](?:[^0-9]|$)", + "weight": 20, + "name": "W-9/W-4 in filename" + }, + { + "pattern": "(?:^|[^a-z0-9])(?:p45|p60|t4)(?:[^0-9a-z]|$)", + "weight": 20, + "name": "P45/P60/T4 in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "w-?[249]\\b|1099|p45|p60|\\bt4\\b", + "weight": 12, + "name": "Tax form id in title" + }, + { + "field": "any", + "pattern": "internal revenue service|hm revenue|canada revenue agency", + "weight": 8, + "name": "Revenue agency in metadata" + } + ], + "negatives": [ + { + "text": "form 1040", + "weight": 15, + "name": "Filed 1040 return" + }, + { + "text": "self assessment", + "weight": 12, + "name": "UK self assessment return" + }, + { + "text": "adjusted gross income", + "weight": 12, + "name": "Return computation line" + }, + { + "text": "net pay", + "weight": 10, + "name": "Payslip vocabulary" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 8 + }, + { + "signal": "short_doc", + "weight": 5 + } + ] + }, + { + "id": "tax-return", + "name": "Tax return", + "emit": true, + "phrases": [ + { + "text": "u.s. individual income tax return", + "weight": 36, + "where": "title" + }, + { + "text": "self assessment tax return", + "weight": 32, + "where": "title" + }, + { + "text": "income tax and benefit return", + "weight": 30, + "where": "title" + }, + { + "text": "resident income tax return", + "weight": 26, + "where": "title" + }, + { + "text": "your tax calculation", + "weight": 24, + "where": "title" + }, + { + "text": "adjusted gross income", + "weight": 22 + }, + { + "text": "total income on which tax is due", + "weight": 26 + }, + { + "text": "amount you owe", + "weight": 16 + }, + { + "text": "filing status", + "weight": 10, + "where": "first" + }, + { + "text": "itemized deductions", + "weight": 16 + }, + { + "text": "unique taxpayer reference", + "weight": 16, + "where": "first" + }, + { + "text": "payment on account", + "weight": 12 + }, + { + "text": "standard deduction", + "weight": 14 + }, + { + "text": "taxable income", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "\\bform\\s+1040\\b", + "weight": 22, + "name": "Form 1040" + }, + { + "pattern": "\\b1040-(?:sr|nr|x|es)\\b", + "weight": 18, + "name": "1040 variant" + }, + { + "pattern": "schedule\\s+(?:[a-e]|se)\\s+\\(form\\s+1040\\)", + "weight": 16, + "name": "1040 schedule" + }, + { + "pattern": "\\bsa(?:100|302)\\b", + "weight": 20, + "name": "HMRC SA form number" + } + ], + "filenames": [ + { + "pattern": "(?:^|[^0-9])1040(?:[^0-9]|$)", + "weight": 20, + "name": "1040 in filename" + }, + { + "pattern": "tax[-_ ]?return", + "weight": 24, + "name": "Tax return in filename" + }, + { + "pattern": "self[-_ ]?assessment|sa100|sa302", + "weight": 22, + "name": "Self assessment in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "1040|tax return|self assessment", + "weight": 12, + "name": "Return in title" + }, + { + "field": "any", + "pattern": "turbotax|taxact|h&r block|taxslayer|freetaxusa|taxcalc", + "weight": 14, + "name": "Tax software producer" + } + ], + "negatives": [ + { + "text": "wage and tax statement", + "weight": 18, + "name": "W-2 heading" + }, + { + "text": "notice of assessment", + "weight": 12, + "name": "Agency notice" + }, + { + "text": "notice date", + "weight": 12, + "name": "Agency notice field" + }, + { + "text": "proposed amount due", + "weight": 14, + "name": "IRS CP2000 notice vocabulary" + }, + { + "text": "net pay", + "weight": 10, + "name": "Payslip vocabulary" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 6 + }, + { + "signal": "number_table", + "weight": 4 + } + ] + }, + { + "id": "tax-statement", + "name": "Tax statement", + "emit": true, + "phrases": [ + { + "text": "notice of deficiency", + "weight": 32 + }, + { + "text": "notice of assessment", + "weight": 30, + "where": "title" + }, + { + "text": "notice of penalty assessment", + "weight": 30, + "where": "title" + }, + { + "text": "notice of intent to levy", + "weight": 30 + }, + { + "text": "amount due immediately", + "weight": 26 + }, + { + "text": "you have unpaid taxes", + "weight": 26 + }, + { + "text": "proposed amount due", + "weight": 24 + }, + { + "text": "late payment penalty", + "weight": 18 + }, + { + "text": "late filing penalty", + "weight": 18 + }, + { + "text": "notice date", + "weight": 12, + "where": "first" + }, + { + "text": "if you do not pay", + "weight": 8 + }, + { + "text": "hm revenue and customs", + "weight": 8, + "where": "first" + }, + { + "text": "hm revenue & customs", + "weight": 8, + "where": "first" + } + ], + "regexes": [ + { + "pattern": "notice\\s+(?:cp|ltr)[- ]?\\d{2,4}", + "weight": 24, + "name": "IRS notice number" + }, + { + "pattern": "failure[ -]to[ -](?:pay|file) penalty", + "weight": 22, + "name": "Failure-to-pay/file penalty" + }, + { + "pattern": "penalt(?:y|ies) and interest", + "weight": 12, + "name": "Penalty and interest" + }, + { + "pattern": "proposed changes to your (?:\\d{4} )?(?:form 1040|tax return)", + "weight": 22, + "name": "CP2000 proposed changes" + } + ], + "filenames": [ + { + "pattern": "irs[-_ ]?(?:notice|letter)|hmrc", + "weight": 18, + "name": "IRS/HMRC in filename" + }, + { + "pattern": "cp[-_ ]?(?:14|49|90|501|503|504|2000)", + "weight": 24, + "name": "IRS CP code in filename" + }, + { + "pattern": "tax[-_ ]?notice|penalty[-_ ]?notice", + "weight": 22, + "name": "Tax notice in filename" + } + ], + "metadata": [ + { + "field": "any", + "pattern": "internal revenue service|hm revenue", + "weight": 10, + "name": "Revenue agency in metadata" + } + ], + "negatives": [ + { + "text": "u.s. individual income tax return", + "weight": 18, + "name": "1040 title" + }, + { + "text": "wage and tax statement", + "weight": 15, + "name": "W-2 heading" + }, + { + "text": "invoice number", + "weight": 15, + "name": "Invoice vocabulary" + } + ], + "structural": [ + { + "signal": "address_block", + "weight": 6 + }, + { + "signal": "short_doc", + "weight": 4 + }, + { + "signal": "currency_heavy", + "weight": 3 + } + ] + }, + { + "id": "payslip", + "name": "Payslip", + "emit": true, + "phrases": [ + { + "text": "earnings statement", + "weight": 26, + "where": "title" + }, + { + "text": "statement of earnings", + "weight": 24, + "where": "title" + }, + { + "text": "pay advice", + "weight": 24, + "where": "title" + }, + { + "text": "payslip", + "weight": 20 + }, + { + "text": "net pay", + "weight": 20 + }, + { + "text": "gross pay", + "weight": 18 + }, + { + "text": "take home pay", + "weight": 18 + }, + { + "text": "pay period", + "weight": 12 + }, + { + "text": "pay date", + "weight": 14, + "where": "first" + }, + { + "text": "basic pay", + "weight": 14 + }, + { + "text": "total deductions", + "weight": 12 + }, + { + "text": "tax code", + "weight": 10, + "where": "first" + }, + { + "text": "year to date", + "weight": 8 + }, + { + "text": "national insurance", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\bytd\\b", + "weight": 12, + "name": "YTD column" + }, + { + "pattern": "\\bfica\\b", + "weight": 14, + "name": "FICA deduction" + }, + { + "pattern": "\\bpaye\\b", + "weight": 12, + "name": "PAYE deduction" + } + ], + "filenames": [ + { + "pattern": "pay[-_ ]?slip", + "weight": 28, + "name": "Payslip in filename" + }, + { + "pattern": "pay[-_ ]?stub", + "weight": 28, + "name": "Paystub in filename" + }, + { + "pattern": "(?:wage|salary)[-_ ]?slip", + "weight": 24, + "name": "Wage/salary slip in filename" + }, + { + "pattern": "payroll|pay[-_ ]?check|earnings[-_ ]?statement", + "weight": 18, + "name": "Payroll in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "payslip|pay slip|pay stub|pay advice|salary slip|earnings statement", + "weight": 14, + "name": "Payslip in title" + }, + { + "field": "any", + "pattern": "\\badp\\b|gusto|paychex|paylocity|moorepay|sage payroll", + "weight": 14, + "name": "Payroll software producer" + } + ], + "negatives": [ + { + "text": "wage and tax statement", + "weight": 16, + "name": "W-2 heading" + }, + { + "pattern": "\\bp60\\b", + "flags": "gi", + "weight": 12, + "name": "P60 certificate" + }, + { + "text": "timesheet", + "weight": 12, + "name": "Timesheet vocabulary" + }, + { + "text": "invoice number", + "weight": 12, + "name": "Invoice vocabulary" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 6 + }, + { + "signal": "short_doc", + "weight": 6 + }, + { + "signal": "form_like", + "weight": 3 + } + ] + }, + { + "id": "benefits-summary", + "name": "Benefits summary", + "emit": true, + "phrases": [ + { + "text": "pension statement", + "weight": 28, + "where": "title" + }, + { + "text": "annual benefit statement", + "weight": 28, + "where": "title" + }, + { + "text": "pension pot", + "weight": 26 + }, + { + "text": "vested balance", + "weight": 26 + }, + { + "text": "superannuation", + "weight": 24 + }, + { + "text": "fund value", + "weight": 18 + }, + { + "text": "lifetime allowance", + "weight": 20 + }, + { + "text": "pension savings", + "weight": 16 + }, + { + "text": "retirement savings", + "weight": 16 + }, + { + "text": "defined contribution", + "weight": 16 + }, + { + "text": "transfer value", + "weight": 16 + }, + { + "text": "defined benefit", + "weight": 14 + }, + { + "text": "employer contributions", + "weight": 14 + }, + { + "text": "retirement age", + "weight": 10 + }, + { + "text": "universal credit", + "weight": 28, + "where": "first" + }, + { + "text": "department for work and pensions", + "weight": 26 + }, + { + "text": "personal independence payment", + "weight": 30 + }, + { + "text": "housing benefit", + "weight": 24 + }, + { + "text": "jobseeker's allowance", + "weight": 26 + }, + { + "text": "employment and support allowance", + "weight": 26 + }, + { + "text": "award notice", + "weight": 22, + "where": "title" + }, + { + "text": "benefit entitlement", + "weight": 22 + }, + { + "text": "social security administration", + "weight": 24, + "where": "first" + }, + { + "text": "supplemental security income", + "weight": 28 + }, + { + "text": "monthly benefit amount", + "weight": 20 + }, + { + "text": "your social security benefits", + "weight": 24 + }, + { + "text": "child benefit", + "weight": 22 + }, + { + "text": "cost of living adjustment", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "\\b401\\(?k\\)?", + "weight": 16, + "name": "401(k) plan" + }, + { + "pattern": "\\b(?:sipp|rrsp)\\b", + "weight": 14, + "name": "Pension account acronym" + }, + { + "pattern": "\\b(?:traditional|roth|sep)[- ]ira\\b", + "weight": 14, + "name": "IRA account type" + }, + { + "pattern": "\\b[a-ceghj-pr-tw-z]{2}\\s?\\d{2}\\s?\\d{2}\\s?\\d{2}\\s?[a-d]\\b", + "weight": 12, + "name": "National Insurance number" + } + ], + "filenames": [ + { + "pattern": "pension", + "weight": 24, + "name": "Pension in filename" + }, + { + "pattern": "401[-_ ]?k", + "weight": 24, + "name": "401k in filename" + }, + { + "pattern": "retirement|superannuation", + "weight": 18, + "name": "Retirement in filename" + }, + { + "pattern": "universal[-_ ]?credit", + "weight": 25, + "name": "Universal Credit filename" + }, + { + "pattern": "benefit", + "weight": 16, + "name": "Benefit filename" + }, + { + "pattern": "award[-_ ]?notice", + "weight": 20, + "name": "Award notice filename" + }, + { + "pattern": "\\bdwp\\b", + "weight": 18, + "name": "DWP filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "pension|retirement|401\\(?k\\)?", + "weight": 12, + "name": "Pension in title" + }, + { + "field": "title", + "pattern": "universal credit|benefit|award notice", + "weight": 12, + "name": "Benefits title" + } + ], + "negatives": [ + { + "text": "net pay", + "weight": 12, + "name": "Payslip vocabulary" + }, + { + "text": "sum insured", + "weight": 12, + "name": "Insurance policy vocabulary" + }, + { + "text": "available balance", + "weight": 10, + "name": "Bank statement vocabulary" + }, + { + "text": "gross pay", + "weight": 14, + "name": "Payslip" + }, + { + "text": "p60", + "weight": 14, + "name": "UK tax certificate" + }, + { + "text": "1099", + "weight": 14, + "name": "US tax form (incl. SSA-1099)" + }, + { + "text": "payslip", + "weight": 16, + "name": "Payslip" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 4 + }, + { + "signal": "number_table", + "weight": 4 + } + ] + }, + { + "id": "contract", + "name": "Contract", + "emit": true, + "phrases": [ + { + "text": "in witness whereof", + "weight": 16 + }, + { + "text": "this agreement is entered into", + "weight": 22, + "where": "first" + }, + { + "text": "hereinafter referred to as", + "weight": 12 + }, + { + "text": "in consideration of the mutual covenants", + "weight": 28 + }, + { + "text": "term and termination", + "weight": 16 + }, + { + "text": "the parties hereto", + "weight": 14 + }, + { + "text": "governing law", + "weight": 8 + }, + { + "text": "entire agreement", + "weight": 8 + }, + { + "text": "indemnify and hold harmless", + "weight": 12 + }, + { + "text": "witnesseth", + "weight": 26, + "where": "first" + }, + { + "text": "by and between", + "weight": 14, + "where": "first" + }, + { + "text": "shall be governed by and construed", + "weight": 12 + }, + { + "text": "agreed terms", + "weight": 16, + "where": "title" + }, + { + "text": "statement of work", + "weight": 8 + }, + { + "text": "data processing agreement", + "weight": 36, + "where": "title" + }, + { + "text": "data processing addendum", + "weight": 32, + "where": "title" + }, + { + "text": "article 28", + "weight": 26 + }, + { + "text": "standard contractual clauses", + "weight": 28 + }, + { + "text": "data controller", + "weight": 18 + }, + { + "text": "data processor", + "weight": 18 + }, + { + "text": "sub-processors", + "weight": 20 + }, + { + "text": "processing of personal data", + "weight": 20 + }, + { + "text": "technical and organisational measures", + "weight": 24 + }, + { + "text": "data subject rights", + "weight": 16 + }, + { + "text": "international data transfer", + "weight": 14 + }, + { + "text": "personal data", + "weight": 8 + }, + { + "text": "gdpr", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "this\\s+agreement\\s+is\\s+made\\s+(on|as\\s+of|by\\s+and\\s+between)", + "flags": "gi", + "weight": 20, + "name": "agreement made on/between", + "where": "first" + }, + { + "pattern": "is\\s+made\\s+and\\s+entered\\s+into", + "flags": "gi", + "weight": 18, + "name": "made and entered into", + "where": "first" + }, + { + "pattern": "article\\s*28(\\(\\d\\))?\\s*(of\\s+the\\s+)?(gdpr|regulation)", + "flags": "gi", + "weight": 20, + "name": "GDPR Article 28 reference" + }, + { + "pattern": "annex\\s+(i{1,3}|1|2|3)\\s*[:\\-]?\\s*(list of parties|description of processing)", + "flags": "gi", + "weight": 16, + "name": "SCC annex heading" + }, + { + "pattern": "processing\\s+instructions", + "flags": "gi", + "weight": 12, + "name": "processing instructions clause" + }, + { + "pattern": "\\bccpa\\b|\\bcpra\\b", + "flags": "gi", + "weight": 12, + "name": "CCPA/CPRA reference" + } + ], + "filenames": [ + { + "pattern": "(contract|agreement)", + "weight": 16, + "name": "contract/agreement in filename" + }, + { + "pattern": "(^|[^a-z0-9])(msa|sow|mou|dpa)([^a-z0-9]|$)", + "weight": 18, + "name": "MSA/SOW/MOU/DPA abbreviation" + }, + { + "pattern": "data[\\s_-]?processing[\\s_-]?agreement", + "weight": 30, + "name": "DPA filename" + }, + { + "pattern": "(^|[^a-z0-9])dpa([^a-z0-9]|$)", + "weight": 24, + "name": "DPA abbreviation filename" + }, + { + "pattern": "standard[\\s_-]?contractual[\\s_-]?clauses|(^|[^a-z0-9])scc([^a-z0-9]|$)", + "weight": 20, + "name": "SCC filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "agreement|contract", + "flags": "gi", + "weight": 10, + "name": "Info title mentions agreement" + }, + { + "field": "title", + "pattern": "data processing agreement|data processing addendum", + "weight": 16, + "name": "Info title DPA" + } + ], + "negatives": [ + { + "text": "non-disclosure agreement", + "weight": 24, + "name": "NDA-specific heading" + }, + { + "text": "disclosing party", + "weight": 16, + "name": "NDA party vocab" + }, + { + "text": "last will and testament", + "weight": 22, + "name": "will vocab" + }, + { + "text": "terms of service", + "weight": 16, + "name": "ToS vocab" + }, + { + "text": "power of attorney", + "weight": 16, + "name": "POA vocab" + }, + { + "pattern": "(employment agreement|contract of employment|offer of employment)", + "flags": "gi", + "weight": 26, + "name": "employment contract heading" + }, + { + "pattern": "(probationary period|annual salary|holiday entitlement|paid time off|hours of work)", + "flags": "gi", + "weight": 16, + "name": "employment terms vocab" + }, + { + "pattern": "\\b(landlord|tenant|tenancy|assured shorthold|security deposit|monthly rent)\\b", + "flags": "gi", + "weight": 24, + "name": "landlord/tenant vocab" + }, + { + "text": "receiving party", + "weight": 14, + "name": "NDA vocab" + }, + { + "text": "by accessing or using", + "weight": 14, + "name": "ToS acceptance clause" + }, + { + "text": "master services agreement", + "weight": 12, + "name": "MSA vocab" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 8 + }, + { + "signal": "long_doc", + "weight": 4 + }, + { + "signal": "toc", + "weight": 3 + } + ] + }, + { + "id": "nda", + "name": "NDA", + "emit": true, + "phrases": [ + { + "text": "non-disclosure agreement", + "weight": 34, + "where": "title" + }, + { + "text": "nondisclosure agreement", + "weight": 32, + "where": "title" + }, + { + "text": "confidentiality agreement", + "weight": 30, + "where": "title" + }, + { + "text": "mutual non-disclosure", + "weight": 30, + "where": "first" + }, + { + "text": "disclosing party", + "weight": 14 + }, + { + "text": "receiving party", + "weight": 14 + }, + { + "text": "confidential information", + "weight": 8 + }, + { + "text": "shall not disclose", + "weight": 14 + }, + { + "text": "return or destroy", + "weight": 14 + }, + { + "text": "permitted purpose", + "weight": 12 + }, + { + "text": "in strict confidence", + "weight": 14 + } + ], + "regexes": [ + { + "pattern": "\\bnda\\b", + "flags": "gi", + "weight": 12, + "name": "NDA abbreviation" + } + ], + "filenames": [ + { + "pattern": "(^|[^a-z0-9])nda([^a-z0-9]|$)", + "weight": 26, + "name": "nda in filename" + }, + { + "pattern": "(non.?disclosure|confidentiality)", + "weight": 24, + "name": "non-disclosure in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "non.?disclosure|confidentiality|\\bnda\\b", + "flags": "gi", + "weight": 14, + "name": "Info title NDA" + } + ], + "negatives": [ + { + "text": "master services agreement", + "weight": 30, + "name": "MSA, not an NDA" + }, + { + "text": "master service agreement", + "weight": 30, + "name": "MSA, not an NDA" + }, + { + "text": "statement of work", + "weight": 20, + "name": "MSA/SOW context" + }, + { + "text": "services agreement", + "weight": 18, + "name": "Services contract" + }, + { + "text": "service levels", + "weight": 14, + "name": "Services contract" + }, + { + "text": "employment agreement", + "weight": 16, + "name": "Employment contract" + }, + { + "text": "purchase agreement", + "weight": 16, + "name": "Purchase agreement" + }, + { + "text": "share purchase", + "weight": 16, + "name": "M&A context" + }, + { + "text": "loan agreement", + "weight": 16, + "name": "Loan agreement" + }, + { + "text": "lease agreement", + "weight": 14, + "name": "Lease" + }, + { + "pattern": "fees (and|&) (payment|charges)|payment terms", + "flags": "gi", + "weight": 12, + "name": "Commercial payment terms (full contract)" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 7 + }, + { + "signal": "short_doc", + "weight": 3 + } + ] + }, + { + "id": "terms-and-conditions", + "name": "Terms and conditions", + "emit": true, + "phrases": [ + { + "text": "terms of service", + "weight": 28, + "where": "title" + }, + { + "text": "terms of use", + "weight": 26, + "where": "title" + }, + { + "text": "terms and conditions", + "weight": 16, + "where": "title" + }, + { + "text": "acceptable use policy", + "weight": 24 + }, + { + "text": "privacy policy", + "weight": 10 + }, + { + "text": "limitation of liability", + "weight": 12 + }, + { + "text": "by accessing or using", + "weight": 28, + "where": "first" + }, + { + "text": "we reserve the right", + "weight": 12 + }, + { + "text": "user agreement", + "weight": 20, + "where": "title" + }, + { + "text": "these terms", + "weight": 8 + }, + { + "text": "you agree to be bound", + "weight": 22 + }, + { + "text": "end user license agreement", + "weight": 22, + "where": "title" + }, + { + "text": "end user licence agreement", + "weight": 22, + "where": "title" + }, + { + "text": "consumer contracts regulations", + "weight": 14 + } + ], + "regexes": [], + "filenames": [ + { + "pattern": "terms.{0,3}(of.{0,3}(service|use)|and.{0,3}conditions)", + "weight": 24, + "name": "terms of service/use in filename" + }, + { + "pattern": "(^|[^a-z0-9])(tos|eula|tnc|t&c)([^a-z0-9]|$)", + "weight": 20, + "name": "ToS/EULA abbreviation" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "terms\\s+(of\\s+(service|use)|and\\s+conditions|&\\s*conditions)", + "flags": "gi", + "weight": 14, + "name": "Info title terms" + } + ], + "negatives": [ + { + "text": "policy number", + "weight": 14, + "name": "insurance policy vocab" + }, + { + "text": "sum insured", + "weight": 14, + "name": "insurance policy vocab" + }, + { + "text": "booking reference", + "weight": 14, + "name": "booking confirmation vocab" + }, + { + "text": "invoice number", + "weight": 12, + "name": "invoice vocab" + } + ], + "structural": [ + { + "signal": "url_heavy", + "weight": 5 + }, + { + "signal": "long_doc", + "weight": 3 + } + ] + }, + { + "id": "court-filing", + "name": "Court filing", + "emit": true, + "phrases": [ + { + "text": "united states district court", + "weight": 22, + "where": "first" + }, + { + "text": "in the circuit court", + "weight": 26, + "where": "first" + }, + { + "text": "in the superior court", + "weight": 26, + "where": "first" + }, + { + "text": "in the high court of justice", + "weight": 28, + "where": "first" + }, + { + "text": "comes now", + "weight": 28 + }, + { + "text": "motion to dismiss", + "weight": 22 + }, + { + "text": "memorandum in support", + "weight": 18 + }, + { + "text": "certificate of service", + "weight": 16 + }, + { + "text": "respectfully submitted", + "weight": 20 + }, + { + "text": "cause of action", + "weight": 14 + }, + { + "text": "prayer for relief", + "weight": 24 + }, + { + "text": "civil action no", + "weight": 24, + "where": "first" + }, + { + "text": "particulars of claim", + "weight": 26, + "where": "first" + }, + { + "text": "statement of truth", + "weight": 20 + } + ], + "regexes": [ + { + "pattern": "\\bcase\\s+(no|number)\\.?\\s*:?\\s*[a-z0-9]", + "flags": "gi", + "weight": 16, + "name": "case number", + "where": "first" + }, + { + "pattern": "\\bdocket\\s+(no|number)\\.?", + "flags": "gi", + "weight": 16, + "name": "docket number" + }, + { + "pattern": "plaintiffs?\\s*,?\\s*v(s)?\\.", + "flags": "gi", + "weight": 18, + "name": "Plaintiff v. caption", + "where": "first" + }, + { + "pattern": "claim\\s+no\\.?\\s*[a-z]{2}-\\d{4}-\\d{4,6}", + "flags": "gi", + "weight": 20, + "name": "UK court claim number" + }, + { + "pattern": "\\d{1,2}:\\d{2}-cv-\\d{4,6}", + "flags": "gi", + "weight": 22, + "name": "federal civil docket (cv)" + } + ], + "filenames": [ + { + "pattern": "(motion|complaint|petition|docket|subpoena|pleading|summons)", + "weight": 16, + "name": "filing type in filename" + }, + { + "pattern": "(^|[^a-z0-9])court([^a-z0-9]|$)", + "weight": 14, + "name": "court in filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "subscribed and sworn to before me", + "weight": 14, + "name": "affidavit jurat" + }, + { + "text": "last will and testament", + "weight": 14, + "name": "will vocab" + }, + { + "text": "memorandum opinion", + "weight": 26, + "name": "Court-issued opinion" + }, + { + "text": "approved judgment", + "weight": 26, + "name": "Court-issued judgment" + }, + { + "text": "opinion of the court", + "weight": 18, + "name": "Court-issued opinion" + }, + { + "text": "it is so ordered", + "weight": 18, + "name": "Court disposition" + }, + { + "text": "per curiam", + "weight": 18, + "name": "Appellate opinion" + }, + { + "text": "for the reasons that follow", + "weight": 14, + "name": "Judicial reasoning" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 3 + } + ] + }, + { + "id": "affidavit", + "name": "Affidavit", + "emit": true, + "phrases": [ + { + "text": "affidavit", + "weight": 22, + "where": "title" + }, + { + "text": "being duly sworn", + "weight": 32 + }, + { + "text": "deposes and says", + "weight": 32 + }, + { + "text": "subscribed and sworn to before me", + "weight": 34 + }, + { + "text": "notary public", + "weight": 8 + }, + { + "text": "my commission expires", + "weight": 10 + }, + { + "text": "under penalty of perjury", + "weight": 12 + }, + { + "text": "statutory declaration", + "weight": 26, + "where": "title" + }, + { + "text": "sworn statement", + "weight": 20 + }, + { + "text": "affiant", + "weight": 26 + }, + { + "text": "deponent", + "weight": 22 + }, + { + "text": "make oath and say", + "weight": 30 + }, + { + "text": "solemnly and sincerely declare", + "weight": 30 + }, + { + "text": "commissioner for oaths", + "weight": 24 + } + ], + "regexes": [ + { + "pattern": "state\\s+of\\s+[a-z]+.{0,30}county\\s+of", + "flags": "gi", + "weight": 16, + "name": "notary venue block" + } + ], + "filenames": [ + { + "pattern": "(affidavit|sworn)", + "weight": 26, + "name": "affidavit in filename" + }, + { + "pattern": "statutory.?declaration", + "weight": 24, + "name": "statutory declaration in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "affidavit|statutory declaration", + "flags": "gi", + "weight": 14, + "name": "Info title affidavit" + } + ], + "negatives": [ + { + "text": "attorney-in-fact", + "weight": 16, + "name": "POA vocab" + }, + { + "text": "last will and testament", + "weight": 16, + "name": "will vocab" + }, + { + "text": "deed of trust", + "weight": 12, + "name": "mortgage vocab" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 7 + }, + { + "signal": "signature_block", + "weight": 8 + } + ] + }, + { + "id": "power-of-attorney", + "name": "Power of attorney", + "emit": true, + "phrases": [ + { + "text": "power of attorney", + "weight": 30, + "where": "title" + }, + { + "text": "attorney-in-fact", + "weight": 30 + }, + { + "text": "durable power of attorney", + "weight": 34 + }, + { + "text": "lasting power of attorney", + "weight": 32 + }, + { + "text": "enduring power of attorney", + "weight": 30 + }, + { + "text": "hereby appoint", + "weight": 20 + }, + { + "text": "grant of authority", + "weight": 18 + }, + { + "text": "office of the public guardian", + "weight": 24 + }, + { + "text": "certificate provider", + "weight": 20 + }, + { + "text": "my agent", + "weight": 8 + }, + { + "text": "on my behalf", + "weight": 4 + }, + { + "text": "disability or incapacity", + "weight": 18 + } + ], + "regexes": [ + { + "pattern": "\\blp1[fh]\\b", + "flags": "gi", + "weight": 16, + "name": "UK LPA form code" + } + ], + "filenames": [ + { + "pattern": "power.?of.?attorney", + "weight": 28, + "name": "power of attorney in filename" + }, + { + "pattern": "(^|[^a-z0-9])(poa|lpa)([^a-z0-9]|$)", + "weight": 20, + "name": "POA/LPA abbreviation" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "power of attorney|\\blpa\\b", + "flags": "gi", + "weight": 14, + "name": "Info title POA" + } + ], + "negatives": [ + { + "text": "last will and testament", + "weight": 18, + "name": "will vocab" + }, + { + "text": "deed of trust", + "weight": 14, + "name": "mortgage vocab" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 8 + }, + { + "signal": "short_doc", + "weight": 4 + }, + { + "signal": "form_like", + "weight": 4 + } + ] + }, + { + "id": "will", + "name": "Will", + "emit": true, + "phrases": [ + { + "text": "last will and testament", + "weight": 36, + "where": "title" + }, + { + "text": "being of sound mind", + "weight": 26 + }, + { + "text": "testator", + "weight": 24 + }, + { + "text": "give, devise and bequeath", + "weight": 32 + }, + { + "text": "bequeath", + "weight": 22 + }, + { + "text": "revocable living trust", + "weight": 30 + }, + { + "text": "residuary estate", + "weight": 28 + }, + { + "text": "per stirpes", + "weight": 28 + }, + { + "text": "codicil", + "weight": 22 + }, + { + "text": "settlor", + "weight": 22 + }, + { + "text": "executor", + "weight": 12 + }, + { + "text": "trustee", + "weight": 6 + }, + { + "text": "declaration of trust", + "weight": 18, + "where": "first" + }, + { + "text": "beneficiaries", + "weight": 4 + } + ], + "regexes": [], + "filenames": [ + { + "pattern": "(last.?will|testament|codicil)", + "weight": 26, + "name": "will in filename" + }, + { + "pattern": "(living|revocable|family).?trust", + "weight": 24, + "name": "trust in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "last will|testament|living trust|declaration of trust", + "flags": "gi", + "weight": 12, + "name": "Info title will/trust" + } + ], + "negatives": [ + { + "text": "deed of trust", + "weight": 24, + "name": "mortgage instrument" + }, + { + "text": "pension scheme", + "weight": 12, + "name": "pension trustee vocab" + }, + { + "text": "unit trust", + "weight": 12, + "name": "investment fund vocab" + }, + { + "text": "attorney-in-fact", + "weight": 10, + "name": "POA vocab" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 8 + } + ] + }, + { + "id": "legal-notice", + "name": "Legal notice", + "emit": true, + "phrases": [ + { + "text": "cease and desist", + "weight": 34 + }, + { + "text": "demand letter", + "weight": 22, + "where": "title" + }, + { + "text": "notice of default", + "weight": 24 + }, + { + "text": "notice of breach", + "weight": 24 + }, + { + "text": "without prejudice", + "weight": 14, + "where": "first" + }, + { + "text": "govern yourself accordingly", + "weight": 32 + }, + { + "text": "letter before action", + "weight": 30 + }, + { + "text": "letter before claim", + "weight": 30 + }, + { + "text": "notice of intent to sue", + "weight": 28 + }, + { + "text": "notice to quit", + "weight": 18 + }, + { + "text": "final demand for payment", + "weight": 18 + }, + { + "text": "failure to comply", + "weight": 8 + }, + { + "text": "further legal action", + "weight": 16 + }, + { + "text": "pre-action protocol", + "weight": 28 + } + ], + "regexes": [ + { + "pattern": "re\\s*:\\s*(cease|demand|notice|outstanding)", + "flags": "gi", + "weight": 12, + "name": "Re: demand/notice subject", + "where": "first" + } + ], + "filenames": [ + { + "pattern": "(cease.?and.?desist|demand.?letter|legal.?notice)", + "weight": 26, + "name": "demand/notice in filename" + }, + { + "pattern": "notice.?of.?(default|breach|termination|claim)", + "weight": 18, + "name": "notice-of in filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "homeowners association", + "weight": 16, + "name": "HOA notice" + }, + { + "text": "internal revenue service", + "weight": 14, + "name": "tax notice" + }, + { + "text": "hmrc", + "weight": 12, + "name": "UK tax notice" + }, + { + "text": "kwh", + "weight": 10, + "name": "utility disconnection notice" + }, + { + "pattern": "\\bcase\\s+no\\b", + "flags": "gi", + "weight": 10, + "name": "court case number present" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 7 + }, + { + "signal": "address_block", + "weight": 6 + } + ] + }, + { + "id": "purchase-agreement", + "name": "Purchase agreement", + "emit": true, + "phrases": [ + { + "text": "agreement and plan of merger", + "weight": 36, + "where": "title" + }, + { + "text": "share purchase agreement", + "weight": 34, + "where": "title" + }, + { + "text": "asset purchase agreement", + "weight": 32, + "where": "title" + }, + { + "text": "merger agreement", + "weight": 28, + "where": "title" + }, + { + "text": "purchase price adjustment", + "weight": 22 + }, + { + "text": "representations and warranties", + "weight": 20 + }, + { + "text": "closing conditions", + "weight": 20 + }, + { + "text": "escrow amount", + "weight": 22 + }, + { + "text": "escrow agent", + "weight": 18 + }, + { + "text": "surviving corporation", + "weight": 20 + }, + { + "text": "material adverse effect", + "weight": 18 + }, + { + "text": "disclosure schedule", + "weight": 16 + }, + { + "text": "target company", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "closing\\s+(?:date|conditions)", + "weight": 12, + "name": "closing date/conditions" + }, + { + "pattern": "escrow\\s+(?:amount|account|agent|fund)", + "weight": 16, + "name": "escrow terms" + }, + { + "pattern": "purchase price\\s+(?:of|shall be|is)\\s*\\$", + "weight": 16, + "name": "purchase price figure" + } + ], + "filenames": [ + { + "pattern": "share[-_ ]?purchase[-_ ]?agreement", + "weight": 28, + "name": "SPA filename" + }, + { + "pattern": "merger[-_ ]?agreement", + "weight": 26, + "name": "merger agreement filename" + }, + { + "pattern": "asset[-_ ]?purchase[-_ ]?agreement", + "weight": 26, + "name": "APA filename" + }, + { + "pattern": "(^|[^a-z])(m&a|spa|apa)([^a-z]|$)", + "weight": 18, + "name": "M&A abbreviation in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "merger agreement|share purchase agreement|plan of merger", + "weight": 16, + "name": "M&A agreement in PDF title" + } + ], + "negatives": [ + { + "text": "letter of intent", + "weight": 16, + "name": "LOI heading" + }, + { + "text": "non-binding", + "weight": 14, + "name": "LOI non-binding wording" + }, + { + "text": "employment agreement", + "weight": 10, + "name": "employment contract heading" + }, + { + "text": "master services agreement", + "weight": 10, + "name": "MSA heading" + }, + { + "text": "statement of work", + "weight": 8, + "name": "SOW heading" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 6 + }, + { + "signal": "long_doc", + "weight": 5 + }, + { + "signal": "number_table", + "weight": 4 + } + ] + }, + { + "id": "shareholder-agreement", + "name": "Shareholder agreement", + "emit": true, + "phrases": [ + { + "text": "shareholders agreement", + "weight": 34, + "where": "title" + }, + { + "text": "shareholders' agreement", + "weight": 34, + "where": "title" + }, + { + "text": "drag-along rights", + "weight": 28 + }, + { + "text": "tag-along rights", + "weight": 28 + }, + { + "text": "pre-emption rights", + "weight": 24 + }, + { + "text": "reserved matters", + "weight": 24 + }, + { + "text": "right of first refusal", + "weight": 18 + }, + { + "text": "transfer of shares", + "weight": 16 + }, + { + "text": "minority shareholder", + "weight": 14 + }, + { + "text": "majority shareholder", + "weight": 12 + }, + { + "text": "anti-dilution", + "weight": 16 + }, + { + "text": "cap table", + "weight": 10 + }, + { + "text": "board composition", + "weight": 12 + }, + { + "text": "voting rights", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "drag[- ]along", + "weight": 18, + "name": "drag-along variant" + }, + { + "pattern": "tag[- ]along", + "weight": 18, + "name": "tag-along variant" + }, + { + "pattern": "pre[- ]emption\\s+rights?", + "weight": 16, + "name": "pre-emption rights variant" + } + ], + "filenames": [ + { + "pattern": "shareholders?[-_ ]?agreement", + "weight": 30, + "name": "shareholders agreement filename" + }, + { + "pattern": "cap[-_ ]?table", + "weight": 18, + "name": "cap table filename" + }, + { + "pattern": "(^|[^a-z])sha([^a-z]|$)", + "weight": 16, + "name": "SHA abbreviation in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "shareholders?.?\\s?agreement", + "weight": 16, + "name": "shareholders agreement in PDF title" + } + ], + "negatives": [ + { + "text": "employment agreement", + "weight": 10, + "name": "employment contract heading" + }, + { + "text": "master services agreement", + "weight": 8, + "name": "MSA heading" + }, + { + "text": "agreement and plan of merger", + "weight": 14, + "name": "merger agreement heading" + }, + { + "text": "purchase price adjustment", + "weight": 12, + "name": "M&A definitive agreement term" + }, + { + "text": "escrow amount", + "weight": 10, + "name": "M&A escrow term" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 6 + }, + { + "signal": "long_doc", + "weight": 4 + } + ] + }, + { + "id": "legal-opinion", + "name": "Legal opinion", + "emit": true, + "phrases": [ + { + "text": "approved judgment", + "weight": 34, + "where": "first" + }, + { + "text": "appeal from the united states district court", + "weight": 24, + "where": "first" + }, + { + "text": "per curiam", + "weight": 24 + }, + { + "text": "the appeal is dismissed", + "weight": 24 + }, + { + "text": "the appeal is allowed", + "weight": 22 + }, + { + "text": "judgment of the court", + "weight": 22 + }, + { + "text": "opinion of the court", + "weight": 22 + }, + { + "text": "it is so ordered", + "weight": 22 + }, + { + "text": "before the honourable", + "weight": 20, + "where": "first" + }, + { + "text": "united states court of appeals", + "weight": 18, + "where": "first" + }, + { + "text": "for the reasons given", + "weight": 14 + }, + { + "text": "handed down", + "weight": 12 + }, + { + "text": "affirmed", + "weight": 6 + }, + { + "text": "remanded", + "weight": 6 + }, + { + "text": "memorandum opinion", + "weight": 32, + "where": "first" + }, + { + "text": "memorandum and order", + "weight": 28, + "where": "first" + }, + { + "text": "findings of fact and conclusions of law", + "weight": 26 + }, + { + "text": "pending before the court", + "weight": 18 + }, + { + "text": "granted in part and denied in part", + "weight": 22 + }, + { + "text": "for the reasons that follow", + "weight": 18 + }, + { + "text": "the motion is granted", + "weight": 14 + }, + { + "text": "the motion is denied", + "weight": 14 + } + ], + "regexes": [ + { + "pattern": "\\[\\d{4}\\]\\s+(ewhc|ewca|uksc|ukpc|ukut|ewfc)\\s+((civ|crim|admin)\\s+)?\\d{1,5}", + "weight": 25, + "name": "UK neutral citation" + }, + { + "pattern": "judgment of the (district|trial) court is (affirmed|reversed|vacated|modified)", + "weight": 22, + "name": "appellate disposition" + }, + { + "pattern": "\\b(mr|mrs|lord|lady)\\s+justice\\s+[a-z]{2,24}", + "weight": 14, + "name": "Mr/Mrs Justice name" + }, + { + "pattern": "\\b\\d{1,3}\\s+f\\.\\s?(2d|3d|4th)\\s+\\d{1,4}\\b", + "weight": 12, + "name": "federal reporter citation" + }, + { + "pattern": "\\bcircuit judges\\b", + "weight": 12, + "name": "circuit judges panel" + } + ], + "filenames": [ + { + "pattern": "judg(e)?ment", + "weight": 26, + "name": "judgment in filename" + }, + { + "pattern": "(^|[^a-z0-9])(ewhc|ewca|uksc)([^a-z0-9]|$)", + "weight": 28, + "name": "UK court code in filename" + }, + { + "pattern": "opinion", + "weight": 16, + "name": "opinion in filename" + }, + { + "pattern": "(^|[^a-z0-9])ruling([^a-z0-9]|$)", + "weight": 15, + "name": "ruling in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "judg(e)?ment|opinion of the court", + "flags": "gi", + "weight": 12, + "name": "Info title judgment/opinion" + } + ], + "negatives": [ + { + "text": "respectfully submitted", + "weight": 20, + "name": "party filing sign-off" + }, + { + "text": "comes now", + "weight": 20, + "name": "party pleading opener" + }, + { + "text": "prayer for relief", + "weight": 16, + "name": "complaint vocab" + }, + { + "text": "particulars of claim", + "weight": 16, + "name": "UK pleading heading" + }, + { + "text": "certificate of service", + "weight": 14, + "name": "filing service certificate" + }, + { + "text": "statement of truth", + "weight": 12, + "name": "UK pleading verification" + }, + { + "pattern": "\\bdoi\\s*:\\s*10\\.\\d{4,5}", + "flags": "gi", + "weight": 14, + "name": "DOI identifier (academic paper)" + } + ], + "structural": [ + { + "signal": "long_doc", + "weight": 5 + }, + { + "signal": "toc", + "weight": 3 + } + ] + }, + { + "id": "subpoena", + "name": "Subpoena", + "emit": true, + "phrases": [ + { + "text": "subpoena duces tecum", + "weight": 34 + }, + { + "text": "witness summons", + "weight": 32, + "where": "title" + }, + { + "text": "you are commanded to", + "weight": 30 + }, + { + "text": "summons in a civil action", + "weight": 28, + "where": "first" + }, + { + "text": "subpoena to testify", + "weight": 26 + }, + { + "text": "subpoena to produce documents", + "weight": 26 + }, + { + "text": "you are hereby summoned", + "weight": 26 + }, + { + "text": "if you do not comply with this summons", + "weight": 24 + }, + { + "text": "place of compliance", + "weight": 18 + }, + { + "text": "return date", + "weight": 14 + }, + { + "text": "to give evidence", + "weight": 12 + }, + { + "text": "failure to comply", + "weight": 8 + }, + { + "text": "clerk of court", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "you are (hereby )?(commanded|summoned|required) to (appear|attend|produce|testify)", + "weight": 24, + "name": "command to appear/produce" + }, + { + "pattern": "\\bao\\s?88[ab]?\\b", + "weight": 16, + "name": "AO 88 subpoena form code" + }, + { + "pattern": "\\bn20\\b", + "weight": 14, + "name": "UK N20 witness summons form" + }, + { + "pattern": "\\brule 45\\b", + "weight": 12, + "name": "FRCP Rule 45 reference" + } + ], + "filenames": [ + { + "pattern": "subpoena", + "weight": 30, + "name": "subpoena in filename" + }, + { + "pattern": "witness[-_ ]?summons", + "weight": 28, + "name": "witness summons in filename" + }, + { + "pattern": "duces[-_ ]?tecum", + "weight": 26, + "name": "duces tecum in filename" + }, + { + "pattern": "(^|[^a-z0-9])summons([^a-z0-9]|$)", + "weight": 22, + "name": "summons in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "subpoena|summons", + "flags": "gi", + "weight": 14, + "name": "Info title subpoena/summons" + } + ], + "negatives": [ + { + "text": "motion to quash", + "weight": 22, + "name": "filing about a subpoena" + }, + { + "text": "respectfully submitted", + "weight": 16, + "name": "party filing sign-off" + }, + { + "text": "memorandum in support", + "weight": 14, + "name": "motion brief vocab" + }, + { + "text": "particulars of claim", + "weight": 14, + "name": "UK pleading heading" + }, + { + "text": "prayer for relief", + "weight": 14, + "name": "complaint vocab" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 7 + }, + { + "signal": "short_doc", + "weight": 6 + }, + { + "signal": "address_block", + "weight": 3 + } + ] + }, + { + "id": "settlement-agreement", + "name": "Settlement agreement", + "emit": true, + "phrases": [ + { + "text": "settlement agreement", + "weight": 30, + "where": "title" + }, + { + "text": "full and final settlement", + "weight": 28 + }, + { + "text": "settlement and release", + "weight": 26 + }, + { + "text": "admission of liability", + "weight": 24 + }, + { + "text": "conciliation officer", + "weight": 24 + }, + { + "text": "release of claims", + "weight": 22 + }, + { + "text": "settlement sum", + "weight": 22 + }, + { + "text": "general release", + "weight": 20 + }, + { + "text": "settlement payment", + "weight": 16 + }, + { + "text": "withdraw the claim", + "weight": 14 + }, + { + "text": "desire to resolve", + "weight": 8 + }, + { + "text": "employment tribunal", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\bcot[- ]?3\\b", + "weight": 22, + "name": "ACAS COT3 form" + }, + { + "pattern": "releases? and forever discharges?", + "weight": 22, + "name": "release and discharge clause" + }, + { + "pattern": "in full and final (settlement|satisfaction) of", + "weight": 20, + "name": "full and final clause" + }, + { + "pattern": "\\bacas\\b", + "weight": 14, + "name": "ACAS conciliation" + } + ], + "filenames": [ + { + "pattern": "settlement", + "weight": 28, + "name": "settlement in filename" + }, + { + "pattern": "(^|[^a-z0-9])cot3([^a-z0-9]|$)", + "weight": 26, + "name": "COT3 in filename" + }, + { + "pattern": "release[-_ ]?agreement", + "weight": 18, + "name": "release agreement in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "settlement", + "flags": "gi", + "weight": 14, + "name": "Info title settlement" + } + ], + "negatives": [ + { + "text": "policy number", + "weight": 16, + "name": "insurance policy vocab" + }, + { + "text": "master services agreement", + "weight": 16, + "name": "MSA heading" + }, + { + "text": "date of loss", + "weight": 14, + "name": "insurance claim vocab" + }, + { + "text": "statement of work", + "weight": 14, + "name": "commercial contract vocab" + }, + { + "text": "non-disclosure agreement", + "weight": 14, + "name": "NDA heading" + }, + { + "text": "purchase price", + "weight": 12, + "name": "purchase/M&A vocab" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 8 + }, + { + "signal": "currency_heavy", + "weight": 3 + } + ] + }, + { + "id": "service-agreement", + "name": "Service agreement", + "emit": true, + "phrases": [ + { + "text": "letter of engagement", + "weight": 32, + "where": "title" + }, + { + "text": "engagement letter", + "weight": 30, + "where": "title" + }, + { + "text": "client care letter", + "weight": 30 + }, + { + "text": "we are pleased to act", + "weight": 28 + }, + { + "text": "we are pleased to confirm our understanding", + "weight": 28 + }, + { + "text": "scope of engagement", + "weight": 26 + }, + { + "text": "thank you for instructing us", + "weight": 26 + }, + { + "text": "terms of engagement", + "weight": 24 + }, + { + "text": "solicitors regulation authority", + "weight": 20 + }, + { + "text": "legal ombudsman", + "weight": 16 + }, + { + "text": "terms of business", + "weight": 14 + }, + { + "text": "our fees", + "weight": 12 + }, + { + "text": "hourly rates", + "weight": 12 + }, + { + "text": "disbursements", + "weight": 8 + }, + { + "text": "master services agreement", + "weight": 38, + "where": "title" + }, + { + "text": "master service agreement", + "weight": 38, + "where": "title" + }, + { + "text": "master agreement", + "weight": 24, + "where": "title" + }, + { + "text": "framework agreement", + "weight": 28, + "where": "title" + }, + { + "text": "statement of work", + "weight": 22 + }, + { + "text": "statements of work", + "weight": 22 + }, + { + "text": "services to be performed", + "weight": 14 + }, + { + "text": "service levels", + "weight": 14 + }, + { + "text": "fees and payment", + "weight": 12 + }, + { + "text": "term and termination", + "weight": 10 + }, + { + "text": "independent contractor", + "weight": 8 + }, + { + "text": "governing law", + "weight": 6 + }, + { + "text": "in witness whereof", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "(will|shall) be (charged|billed) at", + "weight": 12, + "name": "billed/charged at clause" + }, + { + "pattern": "sign and return (the )?(enclosed|attached|a) copy", + "weight": 12, + "name": "sign-and-return instruction" + }, + { + "pattern": "\\bmsa\\b", + "flags": "gi", + "weight": 12, + "name": "MSA abbreviation" + }, + { + "pattern": "\\bsow[s]?\\b", + "flags": "gi", + "weight": 14, + "name": "SOW reference" + }, + { + "pattern": "exhibit [a-e]\\b", + "flags": "gi", + "weight": 10, + "name": "Contract exhibits" + } + ], + "filenames": [ + { + "pattern": "engagement[-_ ]?letter|letter[-_ ]?of[-_ ]?engagement", + "weight": 28, + "name": "engagement letter in filename" + }, + { + "pattern": "client[-_ ]?care", + "weight": 26, + "name": "client care in filename" + }, + { + "pattern": "retainer", + "weight": 20, + "name": "retainer in filename" + }, + { + "pattern": "terms[-_ ]?of[-_ ]?business", + "weight": 18, + "name": "terms of business in filename" + }, + { + "pattern": "\\bmsa\\b|master[-_ ]?servic", + "weight": 26, + "name": "MSA in filename" + }, + { + "pattern": "framework[-_ ]?agreement", + "weight": 22, + "name": "Framework agreement filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "engagement letter|letter of engagement|client care", + "flags": "gi", + "weight": 14, + "name": "Info title engagement" + } + ], + "negatives": [ + { + "text": "pleased to offer you", + "weight": 24, + "name": "job offer wording" + }, + { + "text": "annual salary", + "weight": 20, + "name": "offer/employment vocab" + }, + { + "text": "dear hiring manager", + "weight": 18, + "name": "cover letter salutation" + }, + { + "text": "i am writing to apply", + "weight": 16, + "name": "cover letter opener" + }, + { + "pattern": "my (resume|cv|curriculum vitae)", + "flags": "gi", + "weight": 14, + "name": "applicant materials" + }, + { + "text": "this proposal", + "weight": 12, + "name": "business proposal vocab" + }, + { + "text": "non-disclosure agreement", + "weight": 24, + "name": "Standalone NDA" + }, + { + "text": "last will and testament", + "weight": 20 + }, + { + "text": "lease agreement", + "weight": 16, + "name": "Lease" + }, + { + "text": "employment agreement", + "weight": 16, + "name": "Employment contract" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 6 + }, + { + "signal": "address_block", + "weight": 5 + }, + { + "signal": "signature_block", + "weight": 3 + }, + { + "signal": "toc", + "weight": 4 + } + ] + }, + { + "id": "privacy-policy", + "name": "Privacy policy", + "emit": true, + "phrases": [ + { + "text": "privacy policy", + "weight": 36, + "where": "title" + }, + { + "text": "privacy notice", + "weight": 34, + "where": "title" + }, + { + "text": "personal data we collect", + "weight": 28 + }, + { + "text": "how we use your information", + "weight": 26 + }, + { + "text": "data controller", + "weight": 18 + }, + { + "text": "your rights under", + "weight": 14 + }, + { + "text": "right to erasure", + "weight": 18 + }, + { + "text": "we collect", + "weight": 8 + }, + { + "text": "cookies", + "weight": 8 + }, + { + "text": "third parties", + "weight": 5 + }, + { + "text": "opt out", + "weight": 6 + }, + { + "text": "data protection officer", + "weight": 16 + } + ], + "regexes": [ + { + "pattern": "\\bgdpr\\b|\\bccpa\\b|\\buk gdpr\\b", + "flags": "gi", + "weight": 14, + "name": "Privacy regulation" + }, + { + "pattern": "article \\d{1,2} of the gdpr", + "flags": "gi", + "weight": 16, + "name": "GDPR article reference" + } + ], + "filenames": [ + { + "pattern": "privacy[-_ ]?(policy|notice)", + "weight": 26, + "name": "Privacy policy filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "terms of service", + "weight": 18, + "name": "ToS, not privacy policy" + }, + { + "text": "terms and conditions", + "weight": 16, + "name": "T&C, not privacy policy" + }, + { + "text": "data processing agreement", + "weight": 22, + "name": "DPA (contract), not a policy" + }, + { + "text": "standard contractual clauses", + "weight": 14, + "name": "DPA vocabulary" + } + ], + "structural": [ + { + "signal": "url_heavy", + "weight": 4 + } + ] + }, + { + "id": "resume", + "name": "Resume", + "emit": true, + "phrases": [ + { + "text": "curriculum vitae", + "weight": 32, + "where": "title" + }, + { + "text": "professional experience", + "weight": 18 + }, + { + "text": "work experience", + "weight": 10 + }, + { + "text": "employment history", + "weight": 16 + }, + { + "text": "career history", + "weight": 14 + }, + { + "text": "references available upon request", + "weight": 28 + }, + { + "text": "references available on request", + "weight": 28 + }, + { + "text": "professional summary", + "weight": 20, + "where": "first" + }, + { + "text": "career objective", + "weight": 22, + "where": "first" + }, + { + "text": "personal profile", + "weight": 16, + "where": "first" + }, + { + "text": "core competencies", + "weight": 14 + }, + { + "text": "professional memberships", + "weight": 10 + }, + { + "text": "key skills", + "weight": 10 + }, + { + "text": "technical skills", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "linkedin\\.com/in/[a-z0-9-]{3,40}", + "weight": 10, + "name": "LinkedIn profile URL" + }, + { + "pattern": "(19|20)\\d{2}\\s*[-–]\\s*(present|current)\\b", + "weight": 14, + "name": "Date range ending in Present" + }, + { + "pattern": "^\\s*(work\\s+)?experience\\s*$", + "flags": "im", + "weight": 12, + "name": "Experience section heading" + }, + { + "pattern": "^\\s*education\\s*$", + "flags": "im", + "weight": 10, + "name": "Education section heading" + }, + { + "pattern": "^\\s*(key\\s+|technical\\s+)?skills\\s*$", + "flags": "im", + "weight": 8, + "name": "Skills section heading" + } + ], + "filenames": [ + { + "pattern": "resume", + "weight": 26, + "name": "resume in filename" + }, + { + "pattern": "(^|[^a-z])cv([^a-z]|$)", + "weight": 22, + "name": "cv in filename" + }, + { + "pattern": "curriculum", + "weight": 24, + "name": "curriculum in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "resume|curriculum vitae", + "weight": 14, + "name": "Resume/CV in PDF title" + } + ], + "negatives": [ + { + "text": "dear hiring manager", + "weight": 20, + "name": "cover letter salutation" + }, + { + "text": "we are seeking", + "weight": 18, + "name": "job description language" + }, + { + "text": "pleased to offer you", + "weight": 18, + "name": "offer letter language" + }, + { + "text": "letter of recommendation", + "weight": 15, + "name": "reference letter heading" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 5 + }, + { + "signal": "bullet_heavy", + "weight": 6 + } + ] + }, + { + "id": "cover-letter", + "name": "Cover letter", + "emit": true, + "phrases": [ + { + "text": "dear hiring manager", + "weight": 30, + "where": "first" + }, + { + "text": "i am writing to apply", + "weight": 30 + }, + { + "text": "i am excited to apply", + "weight": 26 + }, + { + "text": "i am writing to express my interest", + "weight": 26 + }, + { + "text": "i wish to apply for", + "weight": 22 + }, + { + "text": "applying for the position of", + "weight": 24 + }, + { + "text": "apply for the above position", + "weight": 22 + }, + { + "text": "thank you for considering my application", + "weight": 28 + }, + { + "text": "my enclosed resume", + "weight": 24 + }, + { + "text": "i enclose my cv", + "weight": 26 + }, + { + "text": "as advertised on", + "weight": 16 + }, + { + "text": "i look forward to the opportunity to discuss", + "weight": 20 + }, + { + "text": "your job posting", + "weight": 16 + } + ], + "regexes": [ + { + "pattern": "\\bre:?\\s+application for\\b", + "weight": 16, + "name": "Re: application for subject line" + } + ], + "filenames": [ + { + "pattern": "cover[_ -]?letter", + "weight": 28, + "name": "cover letter in filename" + }, + { + "pattern": "covering[_ -]?letter", + "weight": 26, + "name": "covering letter in filename" + }, + { + "pattern": "application[_ -]?letter", + "weight": 20, + "name": "application letter in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "cover(ing)? letter", + "weight": 14, + "name": "Cover letter in PDF title" + } + ], + "negatives": [ + { + "text": "curriculum vitae", + "weight": 15, + "name": "CV heading" + }, + { + "text": "we are seeking", + "weight": 18, + "name": "job description language" + }, + { + "text": "pleased to offer you", + "weight": 20, + "name": "offer letter language" + }, + { + "text": "references available upon request", + "weight": 12, + "name": "resume closing line" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 6 + }, + { + "signal": "address_block", + "weight": 4 + } + ] + }, + { + "id": "job-description", + "name": "Job description", + "emit": true, + "phrases": [ + { + "text": "job description", + "weight": 22, + "where": "title" + }, + { + "text": "role profile", + "weight": 24, + "where": "title" + }, + { + "text": "about the role", + "weight": 26, + "where": "title" + }, + { + "text": "position summary", + "weight": 18 + }, + { + "text": "we are seeking", + "weight": 24 + }, + { + "text": "key responsibilities", + "weight": 22 + }, + { + "text": "essential duties", + "weight": 18 + }, + { + "text": "required qualifications", + "weight": 24 + }, + { + "text": "preferred qualifications", + "weight": 26 + }, + { + "text": "the ideal candidate", + "weight": 22 + }, + { + "text": "person specification", + "weight": 28 + }, + { + "text": "job purpose", + "weight": 22, + "where": "title" + }, + { + "text": "main duties and responsibilities", + "weight": 18 + }, + { + "text": "salary range", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "[$£€]\\s?\\d{2,3},\\d{3}\\s*(-|–|to)\\s*[$£€]?\\s?\\d{2,3},\\d{3}", + "weight": 14, + "name": "Salary range figures" + }, + { + "pattern": "what you['’]ll (do|bring|need)", + "weight": 24, + "name": "What you'll do/bring heading (both apostrophes)" + } + ], + "filenames": [ + { + "pattern": "job[_ -]?description", + "weight": 28, + "name": "job description in filename" + }, + { + "pattern": "(^|[^a-z])jd([^a-z]|$)", + "weight": 18, + "name": "jd in filename" + }, + { + "pattern": "role[_ -]?profile", + "weight": 24, + "name": "role profile in filename" + }, + { + "pattern": "(posting|vacancy|advert)", + "weight": 16, + "name": "vacancy/posting in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "job description|role profile", + "weight": 14, + "name": "Job description in PDF title" + } + ], + "negatives": [ + { + "text": "pleased to offer you", + "weight": 22, + "name": "offer letter language" + }, + { + "text": "i am writing to apply", + "weight": 18, + "name": "cover letter language" + }, + { + "text": "curriculum vitae", + "weight": 15, + "name": "CV heading" + }, + { + "text": "this agreement", + "weight": 12, + "name": "contract language" + } + ], + "structural": [ + { + "signal": "bullet_heavy", + "weight": 6 + } + ] + }, + { + "id": "offer-letter", + "name": "Offer letter", + "emit": true, + "phrases": [ + { + "text": "pleased to offer you", + "weight": 30 + }, + { + "text": "offer of employment", + "weight": 30, + "where": "title" + }, + { + "text": "offer you the position of", + "weight": 26 + }, + { + "text": "conditional offer of employment", + "weight": 28 + }, + { + "text": "your start date will be", + "weight": 22, + "where": "first" + }, + { + "text": "annual base salary", + "weight": 22 + }, + { + "text": "accept this offer", + "weight": 28 + }, + { + "text": "this offer is contingent", + "weight": 26 + }, + { + "text": "this offer is conditional", + "weight": 26 + }, + { + "text": "contingent upon", + "weight": 10 + }, + { + "text": "sign and return", + "weight": 14 + }, + { + "text": "we look forward to welcoming you", + "weight": 18 + }, + { + "text": "at-will", + "weight": 8 + }, + { + "text": "offer will remain open", + "weight": 18 + } + ], + "regexes": [ + { + "pattern": "salary of\\s+[$£€]?\\s?\\d{2,3},\\d{3}", + "weight": 16, + "name": "salary of $X figure" + } + ], + "filenames": [ + { + "pattern": "offer[_ -]?letter", + "weight": 28, + "name": "offer letter in filename" + }, + { + "pattern": "offer[_ -]?of[_ -]?employment", + "weight": 28, + "name": "offer of employment in filename" + }, + { + "pattern": "job[_ -]?offer", + "weight": 24, + "name": "job offer in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "offer (letter|of employment)", + "weight": 14, + "name": "Offer letter in PDF title" + } + ], + "negatives": [ + { + "text": "we are seeking", + "weight": 20, + "name": "job description language" + }, + { + "text": "person specification", + "weight": 15, + "name": "UK job description section" + }, + { + "text": "i am writing to apply", + "weight": 18, + "name": "cover letter language" + }, + { + "text": "curriculum vitae", + "weight": 15, + "name": "CV heading" + }, + { + "text": "the employee shall", + "weight": 15, + "name": "employment contract language" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 4 + }, + { + "signal": "signature_block", + "weight": 5 + } + ] + }, + { + "id": "reference-letter", + "name": "Reference letter", + "emit": true, + "phrases": [ + { + "text": "letter of recommendation", + "weight": 34, + "where": "title" + }, + { + "text": "letter of reference", + "weight": 30, + "where": "title" + }, + { + "text": "character reference", + "weight": 30 + }, + { + "text": "it is my pleasure to recommend", + "weight": 32 + }, + { + "text": "i am writing to recommend", + "weight": 32 + }, + { + "text": "i highly recommend", + "weight": 24 + }, + { + "text": "i strongly recommend", + "weight": 20 + }, + { + "text": "i have known", + "weight": 16, + "where": "first" + }, + { + "text": "without reservation", + "weight": 20 + }, + { + "text": "worked under my supervision", + "weight": 24 + }, + { + "text": "in my capacity as", + "weight": 12 + }, + { + "text": "to whom it may concern", + "weight": 10, + "where": "first" + }, + { + "text": "an asset to your", + "weight": 18 + } + ], + "regexes": [ + { + "pattern": "\\brecommend (him|her|them)\\b", + "weight": 16, + "name": "recommend him/her/them" + } + ], + "filenames": [ + { + "pattern": "(reference|recommendation)[_ -]?letter", + "weight": 28, + "name": "reference/recommendation letter in filename" + }, + { + "pattern": "letter[_ -]?of[_ -]?(reference|recommendation)", + "weight": 28, + "name": "letter of reference in filename" + }, + { + "pattern": "char(acter)?[_ -]?ref", + "weight": 22, + "name": "character reference in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "letter of recommendation|reference letter|character reference", + "weight": 14, + "name": "Reference letter in PDF title" + } + ], + "negatives": [ + { + "text": "reference range", + "weight": 24, + "name": "lab report term" + }, + { + "text": "referral", + "weight": 12, + "name": "medical referral term" + }, + { + "text": "i am writing to apply", + "weight": 16, + "name": "cover letter language" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 6 + }, + { + "signal": "signature_block", + "weight": 5 + } + ] + }, + { + "id": "employment-contract", + "name": "Employment contract", + "emit": true, + "phrases": [ + { + "text": "contract of employment", + "weight": 30, + "where": "title" + }, + { + "text": "employment agreement", + "weight": 28, + "where": "title" + }, + { + "text": "terms and conditions of employment", + "weight": 28 + }, + { + "text": "probationary period", + "weight": 22 + }, + { + "text": "notice period", + "weight": 14 + }, + { + "text": "remuneration", + "weight": 16 + }, + { + "text": "the employee shall", + "weight": 22 + }, + { + "text": "the employer shall", + "weight": 22 + }, + { + "text": "termination of employment", + "weight": 18 + }, + { + "text": "hours of work", + "weight": 14 + }, + { + "text": "restrictive covenant", + "weight": 14 + }, + { + "text": "gross misconduct", + "weight": 20 + }, + { + "text": "garden leave", + "weight": 24 + }, + { + "text": "continuous employment", + "weight": 18 + } + ], + "regexes": [ + { + "pattern": "\\((the\\s+)?[\"“]?(employer|employee)[\"”]?\\)", + "weight": 20, + "name": "Employer/Employee defined term" + } + ], + "filenames": [ + { + "pattern": "employment[_ -]?(contract|agreement)", + "weight": 28, + "name": "employment contract in filename" + }, + { + "pattern": "contract[_ -]?of[_ -]?employment", + "weight": 28, + "name": "contract of employment in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "employment (agreement|contract)|contract of employment", + "weight": 14, + "name": "Employment contract in PDF title" + } + ], + "negatives": [ + { + "text": "landlord", + "weight": 24, + "name": "lease agreement term" + }, + { + "text": "tenant", + "weight": 20, + "name": "lease agreement term" + }, + { + "text": "non-disclosure agreement", + "weight": 18, + "name": "NDA heading" + }, + { + "text": "pleased to offer you", + "weight": 16, + "name": "offer letter language" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 6 + }, + { + "signal": "long_doc", + "weight": 4 + }, + { + "signal": "toc", + "weight": 3 + } + ] + }, + { + "id": "timesheet", + "name": "Timesheet", + "emit": true, + "phrases": [ + { + "text": "timesheet", + "weight": 28, + "where": "title" + }, + { + "text": "time sheet", + "weight": 26, + "where": "title" + }, + { + "text": "time and attendance", + "weight": 16 + }, + { + "text": "week ending", + "weight": 24 + }, + { + "text": "hours worked", + "weight": 18 + }, + { + "text": "total hours", + "weight": 14 + }, + { + "text": "regular hours", + "weight": 18 + }, + { + "text": "overtime hours", + "weight": 20 + }, + { + "text": "clock in", + "weight": 20 + }, + { + "text": "clock out", + "weight": 20 + }, + { + "text": "billable hours", + "weight": 12 + }, + { + "text": "employee signature", + "weight": 8 + }, + { + "text": "approved by", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\bmon(day)?\\s+tue(sday)?\\s+wed(nesday)?\\b", + "weight": 16, + "name": "Weekday column headers" + }, + { + "pattern": "\\b\\d{1,2}\\.\\d{2}\\s?(hrs|hours)\\b", + "weight": 12, + "name": "Decimal hours (e.g. 7.50 hrs)" + }, + { + "pattern": "(hours|hrs):\\s?\\d{1,3}\\.\\d{2}\\b", + "weight": 12, + "name": "Hours total (e.g. Hours: 40.00)" + } + ], + "filenames": [ + { + "pattern": "time[_ -]?sheet", + "weight": 30, + "name": "timesheet in filename" + }, + { + "pattern": "(^|[^a-z])hours([^a-z]|$)", + "weight": 12, + "name": "hours in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "time ?sheet", + "weight": 14, + "name": "Timesheet in PDF title" + } + ], + "negatives": [ + { + "text": "invoice number", + "weight": 20, + "name": "invoice field" + }, + { + "text": "net pay", + "weight": 20, + "name": "payslip field" + }, + { + "text": "remittance", + "weight": 15, + "name": "remittance advice term" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 8 + }, + { + "signal": "form_like", + "weight": 5 + }, + { + "signal": "short_doc", + "weight": 5 + } + ] + }, + { + "id": "performance-review", + "name": "Performance review", + "emit": true, + "phrases": [ + { + "text": "performance review", + "weight": 28, + "where": "title" + }, + { + "text": "performance appraisal", + "weight": 30, + "where": "title" + }, + { + "text": "appraisal record", + "weight": 24, + "where": "title" + }, + { + "text": "appraisal meeting", + "weight": 22 + }, + { + "text": "review period", + "weight": 22 + }, + { + "text": "areas for improvement", + "weight": 26 + }, + { + "text": "exceeds expectations", + "weight": 26 + }, + { + "text": "meets expectations", + "weight": 24 + }, + { + "text": "self-assessment", + "weight": 12 + }, + { + "text": "overall rating", + "weight": 22 + }, + { + "text": "development plan", + "weight": 14 + }, + { + "text": "mid-year review", + "weight": 20 + }, + { + "text": "manager comments", + "weight": 18 + } + ], + "regexes": [ + { + "pattern": "\\b5\\s*[-=–]\\s*(exceptional|outstanding|exceeds)", + "weight": 16, + "name": "Rating scale legend (5 = Exceptional)" + }, + { + "pattern": "overall rating\\s*[:=]?\\s*[1-5]\\b", + "weight": 14, + "name": "Overall rating score" + } + ], + "filenames": [ + { + "pattern": "performance[_ -]?review", + "weight": 28, + "name": "performance review in filename" + }, + { + "pattern": "appraisal", + "weight": 18, + "name": "appraisal in filename" + }, + { + "pattern": "perf[_ -]?review", + "weight": 22, + "name": "perf review in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "performance (review|appraisal)|appraisal", + "weight": 12, + "name": "Performance review in PDF title" + } + ], + "negatives": [ + { + "text": "tax return", + "weight": 20, + "name": "self assessment tax confusable" + }, + { + "text": "hmrc", + "weight": 18, + "name": "UK tax authority" + }, + { + "text": "inspection", + "weight": 15, + "name": "inspection report term" + }, + { + "text": "appraised value", + "weight": 20, + "name": "property appraisal term" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 5 + }, + { + "signal": "number_table", + "weight": 3 + } + ] + }, + { + "id": "employee-handbook", + "name": "Employee handbook", + "emit": true, + "phrases": [ + { + "text": "employee handbook", + "weight": 38, + "where": "title" + }, + { + "text": "staff handbook", + "weight": 36, + "where": "title" + }, + { + "text": "code of conduct", + "weight": 16 + }, + { + "text": "dress code", + "weight": 16 + }, + { + "text": "annual leave", + "weight": 12 + }, + { + "text": "sick leave", + "weight": 12 + }, + { + "text": "disciplinary procedure", + "weight": 16 + }, + { + "text": "grievance procedure", + "weight": 16 + }, + { + "text": "equal opportunity", + "weight": 10 + }, + { + "text": "probationary period", + "weight": 8 + }, + { + "text": "working hours", + "weight": 8 + }, + { + "text": "welcome to the company", + "weight": 14 + } + ], + "regexes": [], + "filenames": [ + { + "pattern": "(employee|staff)[-_ ]?handbook", + "weight": 28, + "name": "Handbook filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "employment agreement", + "weight": 18, + "name": "Individual contract, not handbook" + }, + { + "text": "offer of employment", + "weight": 16, + "name": "Offer letter" + }, + { + "text": "curriculum vitae", + "weight": 16, + "name": "CV" + } + ], + "structural": [ + { + "signal": "toc", + "weight": 8 + }, + { + "signal": "long_doc", + "weight": 4 + }, + { + "signal": "bullet_heavy", + "weight": 4 + } + ] + }, + { + "id": "risk-assessment", + "name": "Risk assessment", + "emit": true, + "phrases": [ + { + "text": "penetration test report", + "weight": 38, + "where": "title" + }, + { + "text": "penetration testing report", + "weight": 36, + "where": "title" + }, + { + "text": "penetration test", + "weight": 14, + "where": "title" + }, + { + "text": "vulnerability assessment and penetration test", + "weight": 30 + }, + { + "text": "rules of engagement", + "weight": 22 + }, + { + "text": "attack narrative", + "weight": 20 + }, + { + "text": "exploitability", + "weight": 18 + }, + { + "text": "owasp top 10", + "weight": 16 + }, + { + "text": "remediation recommendations", + "weight": 12 + }, + { + "text": "scope of engagement", + "weight": 12 + }, + { + "text": "critical severity finding", + "weight": 10 + }, + { + "text": "retest results", + "weight": 8 + }, + { + "text": "risk assessment report", + "weight": 34, + "where": "title" + }, + { + "text": "risk register", + "weight": 30, + "where": "title" + }, + { + "text": "likelihood and impact", + "weight": 24 + }, + { + "text": "residual risk", + "weight": 24 + }, + { + "text": "risk heat map", + "weight": 24 + }, + { + "text": "risk owner", + "weight": 20 + }, + { + "text": "inherent risk", + "weight": 18 + }, + { + "text": "mitigation plan", + "weight": 14 + }, + { + "text": "risk appetite", + "weight": 16 + }, + { + "text": "risk rating", + "weight": 12 + }, + { + "text": "threat and vulnerability", + "weight": 16 + }, + { + "text": "control effectiveness", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "cvss\\s*(v?3(\\.\\d)?)?\\s*(score|base score)?\\s*[:=]?\\s*\\d(\\.\\d)?", + "flags": "gi", + "weight": 22, + "name": "CVSS score" + }, + { + "pattern": "\\b(critical|high|medium|low)\\s*[-–]?\\s*(risk|severity)\\b", + "flags": "gi", + "weight": 12, + "name": "severity rating" + }, + { + "pattern": "\\bfinding\\s*#?\\d{1,3}\\b", + "flags": "gi", + "weight": 14, + "name": "finding number" + }, + { + "pattern": "\\ba0[1-9]:20\\d{2}\\b", + "flags": "gi", + "weight": 16, + "name": "OWASP category code" + }, + { + "pattern": "engagement\\s+(dates?|window)", + "flags": "gi", + "weight": 12, + "name": "engagement dates" + }, + { + "pattern": "risk\\s*(score|rating)\\s*[:=]?\\s*(low|medium|high|critical|\\d{1,2})", + "flags": "gi", + "weight": 16, + "name": "risk score/rating" + }, + { + "pattern": "likelihood\\s*[:=]?\\s*(low|medium|high)", + "flags": "gi", + "weight": 12, + "name": "likelihood field" + }, + { + "pattern": "\\d{1,2}\\s*x\\s*\\d{1,2}\\s*risk\\s*matrix", + "flags": "gi", + "weight": 14, + "name": "risk matrix dimensions" + } + ], + "filenames": [ + { + "pattern": "pen[\\s_-]?test|penetration[\\s_-]?test", + "weight": 28, + "name": "pentest in filename" + }, + { + "pattern": "(^|[^a-z0-9])vapt([^a-z0-9]|$)", + "weight": 22, + "name": "VAPT abbreviation" + }, + { + "pattern": "security[\\s_-]?assessment", + "weight": 16, + "name": "security assessment filename" + }, + { + "pattern": "risk[\\s_-]?assessment", + "weight": 28, + "name": "risk assessment filename" + }, + { + "pattern": "risk[\\s_-]?register", + "weight": 26, + "name": "risk register filename" + }, + { + "pattern": "risk[\\s_-]?matrix", + "weight": 18, + "name": "risk matrix filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "penetration test|pentest|vapt", + "weight": 16, + "name": "Info title pentest" + }, + { + "field": "title", + "pattern": "risk assessment|risk register", + "weight": 14, + "name": "Info title risk assessment" + } + ], + "negatives": [ + { + "text": "functional requirements", + "weight": 14, + "name": "tech spec vocab" + }, + { + "text": "api reference", + "weight": 12, + "name": "software doc vocab" + }, + { + "text": "trust services criteria", + "weight": 20, + "name": "SOC 2 report vocab" + }, + { + "text": "auditor's opinion", + "weight": 16, + "name": "SOC 2 opinion vocab" + }, + { + "text": "cvss", + "weight": 14, + "name": "pentest report vocab" + }, + { + "text": "indicators of compromise", + "weight": 12, + "name": "incident report vocab" + }, + { + "text": "containment and eradication", + "weight": 12, + "name": "incident report vocab" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 8 + }, + { + "signal": "toc", + "weight": 4 + }, + { + "signal": "long_doc", + "weight": 3 + }, + { + "signal": "form_like", + "weight": 3 + } + ] + }, + { + "id": "audit-report", + "name": "Audit report", + "emit": true, + "phrases": [ + { + "text": "soc 2 type ii", + "weight": 36, + "where": "title" + }, + { + "text": "soc 2 type 2", + "weight": 34, + "where": "title" + }, + { + "text": "type ii report", + "weight": 20 + }, + { + "text": "system and organization controls", + "weight": 30 + }, + { + "text": "trust services criteria", + "weight": 28 + }, + { + "text": "independent service auditor's report", + "weight": 26 + }, + { + "text": "control objectives", + "weight": 20 + }, + { + "text": "management's assertion", + "weight": 22 + }, + { + "text": "iso/iec 27001", + "weight": 24 + }, + { + "text": "aicpa", + "weight": 16 + }, + { + "text": "exceptions noted", + "weight": 12 + }, + { + "text": "unqualified opinion", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "soc\\s*2\\s*type\\s*(i{1,2}|1|2)", + "flags": "gi", + "weight": 24, + "name": "SOC 2 Type I/II" + }, + { + "pattern": "iso\\s*/?\\s*iec\\s*27001(:\\d{4})?", + "flags": "gi", + "weight": 20, + "name": "ISO 27001 reference" + }, + { + "pattern": "\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{2,4}\\s*(to|through|-)\\s*\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{2,4}", + "flags": "gi", + "weight": 14, + "name": "audit period date range" + } + ], + "filenames": [ + { + "pattern": "soc[\\s_-]?2", + "weight": 28, + "name": "SOC 2 in filename" + }, + { + "pattern": "type[\\s_-]?ii", + "weight": 16, + "name": "Type II in filename" + }, + { + "pattern": "iso[\\s_-]?27001", + "weight": 20, + "name": "ISO 27001 filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "soc\\s*2|type\\s*ii|iso\\s*27001", + "weight": 14, + "name": "Info title SOC 2" + } + ], + "negatives": [ + { + "text": "certificate of liability insurance", + "weight": 22, + "name": "COI heading" + }, + { + "text": "certificate holder", + "weight": 16, + "name": "COI field" + }, + { + "text": "consolidated financial statements", + "weight": 18, + "name": "financial report vocab" + }, + { + "text": "balance sheet", + "weight": 14, + "name": "financial statement vocab" + } + ], + "structural": [ + { + "signal": "long_doc", + "weight": 5 + }, + { + "signal": "toc", + "weight": 4 + }, + { + "signal": "signature_block", + "weight": 3 + } + ] + }, + { + "id": "compliance-document", + "name": "Compliance document", + "emit": true, + "phrases": [ + { + "text": "information security policy", + "weight": 36, + "where": "title" + }, + { + "text": "security policy", + "weight": 20, + "where": "title" + }, + { + "text": "acceptable use policy", + "weight": 20 + }, + { + "text": "data classification policy", + "weight": 26 + }, + { + "text": "access control policy", + "weight": 24 + }, + { + "text": "policy owner", + "weight": 22 + }, + { + "text": "password policy", + "weight": 18 + }, + { + "text": "least privilege", + "weight": 16 + }, + { + "text": "confidentiality, integrity and availability", + "weight": 24 + }, + { + "text": "policy review cycle", + "weight": 14 + }, + { + "text": "information security management system", + "weight": 22 + }, + { + "text": "effective date", + "weight": 6 + }, + { + "text": "business continuity plan", + "weight": 36, + "where": "title" + }, + { + "text": "disaster recovery plan", + "weight": 34, + "where": "title" + }, + { + "text": "recovery time objective", + "weight": 28 + }, + { + "text": "recovery point objective", + "weight": 28 + }, + { + "text": "business impact analysis", + "weight": 26 + }, + { + "text": "failover procedures", + "weight": 20 + }, + { + "text": "crisis management team", + "weight": 20 + }, + { + "text": "continuity of operations", + "weight": 18 + }, + { + "text": "backup and recovery", + "weight": 12 + }, + { + "text": "emergency response plan", + "weight": 16 + }, + { + "text": "alternate site", + "weight": 12 + }, + { + "text": "activation criteria", + "weight": 10 + }, + { + "text": "regulatory compliance report", + "weight": 34, + "where": "title" + }, + { + "text": "compliance report", + "weight": 26, + "where": "title" + }, + { + "text": "compliance assessment", + "weight": 22, + "where": "title" + }, + { + "text": "controls tested", + "weight": 24 + }, + { + "text": "non-compliance", + "weight": 20 + }, + { + "text": "remediation plan", + "weight": 18 + }, + { + "text": "compliance framework", + "weight": 18 + }, + { + "text": "attestation of compliance", + "weight": 28 + }, + { + "text": "gap analysis", + "weight": 14 + }, + { + "text": "corrective action plan", + "weight": 18 + }, + { + "text": "regulatory requirements", + "weight": 10 + }, + { + "text": "audit findings", + "weight": 12 + }, + { + "text": "declaration of conformity", + "weight": 36, + "where": "title" + }, + { + "text": "certificate of conformity", + "weight": 34, + "where": "title" + }, + { + "text": "declare under our sole responsibility", + "weight": 36, + "where": "first" + }, + { + "text": "object of the declaration", + "weight": 30 + }, + { + "text": "notified body", + "weight": 26 + }, + { + "text": "harmonised standards", + "weight": 24 + }, + { + "text": "harmonized standards", + "weight": 24 + }, + { + "text": "ukca", + "weight": 20 + }, + { + "text": "conformity assessment", + "weight": 16 + }, + { + "text": "ce marking", + "weight": 14 + }, + { + "text": "essential requirements", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "policy\\s+(owner|approved by|effective date)\\s*[:]", + "flags": "gi", + "weight": 14, + "name": "policy metadata field" + }, + { + "pattern": "review(ed)?\\s+(annually|every\\s+\\d{1,2}\\s+(months|years))", + "flags": "gi", + "weight": 12, + "name": "policy review cadence" + }, + { + "pattern": "\\biso\\s?27001\\b", + "flags": "gi", + "weight": 12, + "name": "ISO 27001 reference" + }, + { + "pattern": "\\brto\\b", + "flags": "gi", + "weight": 12, + "name": "RTO abbreviation" + }, + { + "pattern": "\\brpo\\b", + "flags": "gi", + "weight": 12, + "name": "RPO abbreviation" + }, + { + "pattern": "recovery\\s+time\\s+objective\\s*[:=]?\\s*\\d{1,3}\\s*(hours?|minutes?|days?)", + "flags": "gi", + "weight": 18, + "name": "RTO with value" + }, + { + "pattern": "tier\\s*[1-4]\\s*(system|application)", + "flags": "gi", + "weight": 12, + "name": "recovery tier classification" + }, + { + "pattern": "pci\\s*dss", + "flags": "gi", + "weight": 18, + "name": "PCI DSS reference" + }, + { + "pattern": "hipaa\\s*(security|privacy)?\\s*rule", + "flags": "gi", + "weight": 16, + "name": "HIPAA rule reference" + }, + { + "pattern": "\\bfedramp\\b", + "flags": "gi", + "weight": 14, + "name": "FedRAMP reference" + }, + { + "pattern": "compliance\\s+(status|rate)\\s*[:=]?\\s*\\d{1,3}\\s*%", + "flags": "gi", + "weight": 14, + "name": "compliance percentage" + }, + { + "pattern": "\\b\\d{4}/\\d{1,3}/(eu|ec|eec)\\b", + "flags": "gi", + "weight": 22, + "name": "eu directive number" + }, + { + "pattern": "regulation\\s?\\(eu\\)\\s?\\d{4}/\\d{3,4}", + "flags": "gi", + "weight": 18, + "name": "eu regulation number" + }, + { + "pattern": "\\b(bs\\s?)?en\\s?(iso\\s?|iec\\s?)?\\d{3,5}([-:]\\d{1,4}){0,3}", + "flags": "gi", + "weight": 12, + "name": "en standard reference" + } + ], + "filenames": [ + { + "pattern": "information[\\s_-]?security[\\s_-]?polic", + "weight": 28, + "name": "information security policy filename" + }, + { + "pattern": "security[\\s_-]?polic(y|ies)", + "weight": 24, + "name": "security policy filename" + }, + { + "pattern": "infosec[\\s_-]?polic", + "weight": 22, + "name": "infosec policy filename" + }, + { + "pattern": "acceptable[\\s_-]?use[\\s_-]?polic", + "weight": 18, + "name": "AUP filename" + }, + { + "pattern": "business[\\s_-]?continuity", + "weight": 28, + "name": "BCP filename" + }, + { + "pattern": "disaster[\\s_-]?recovery", + "weight": 26, + "name": "DR plan filename" + }, + { + "pattern": "(^|[^a-z0-9])(bcp|drp)([^a-z0-9]|$)", + "weight": 20, + "name": "BCP/DRP abbreviation" + }, + { + "pattern": "compliance[\\s_-]?report", + "weight": 28, + "name": "compliance report filename" + }, + { + "pattern": "compliance[\\s_-]?assessment", + "weight": 22, + "name": "compliance assessment filename" + }, + { + "pattern": "(^|[^a-z0-9])(aoc|pci[\\s_-]?dss)([^a-z0-9]|$)", + "weight": 20, + "name": "AOC/PCI DSS abbreviation" + }, + { + "pattern": "declaration[\\s_-]*of[\\s_-]*conformity", + "weight": 26, + "name": "doc filename" + }, + { + "pattern": "(^|[^a-z0-9])(coc|cofc|dofc)([^a-z0-9]|$)", + "weight": 18, + "name": "coc abbreviation" + }, + { + "pattern": "conformity", + "weight": 16, + "name": "conformity keyword" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "information security policy|security policy", + "weight": 14, + "name": "Info title security policy" + }, + { + "field": "title", + "pattern": "business continuity|disaster recovery", + "weight": 14, + "name": "Info title BCP/DR" + }, + { + "field": "title", + "pattern": "compliance report|compliance assessment|attestation of compliance", + "weight": 14, + "name": "Info title compliance report" + } + ], + "negatives": [ + { + "text": "by accessing or using", + "weight": 20, + "name": "ToS acceptance clause" + }, + { + "text": "you agree to be bound", + "weight": 18, + "name": "ToS clause" + }, + { + "text": "these terms", + "weight": 12, + "name": "ToS vocab" + }, + { + "text": "end user license agreement", + "weight": 16, + "name": "EULA vocab" + }, + { + "text": "this document specifies", + "weight": 12, + "name": "tech spec vocab" + }, + { + "text": "functional requirements", + "weight": 12, + "name": "tech spec vocab" + }, + { + "text": "root cause analysis", + "weight": 12, + "name": "incident report vocab" + }, + { + "text": "indicators of compromise", + "weight": 12, + "name": "incident report vocab" + }, + { + "text": "risk register", + "weight": 10, + "name": "risk assessment vocab" + }, + { + "text": "certificate of liability insurance", + "weight": 20, + "name": "COI heading" + }, + { + "text": "certificate holder", + "weight": 16, + "name": "COI field" + }, + { + "text": "consolidated financial statements", + "weight": 16, + "name": "financial report vocab" + }, + { + "text": "trust services criteria", + "weight": 14, + "name": "SOC 2 report vocab" + }, + { + "text": "has successfully completed", + "weight": 24, + "name": "diploma wording" + }, + { + "text": "certificate of insurance", + "weight": 22, + "name": "insurance cert" + }, + { + "text": "proof of purchase", + "weight": 18, + "name": "warranty card" + }, + { + "text": "warranty period", + "weight": 16, + "name": "warranty card" + }, + { + "text": "heat number", + "weight": 18, + "name": "mill cert field" + }, + { + "text": "chemical composition", + "weight": 16, + "name": "mill cert table" + } + ], + "structural": [ + { + "signal": "toc", + "weight": 4 + }, + { + "signal": "long_doc", + "weight": 3 + }, + { + "signal": "bullet_heavy", + "weight": 4 + }, + { + "signal": "number_table", + "weight": 6 + }, + { + "signal": "short_doc", + "weight": 8 + }, + { + "signal": "signature_block", + "weight": 6 + } + ] + }, + { + "id": "incident-report", + "name": "Incident report", + "emit": true, + "phrases": [ + { + "text": "security incident report", + "weight": 36, + "where": "title" + }, + { + "text": "incident response report", + "weight": 32, + "where": "title" + }, + { + "text": "post-incident report", + "weight": 28, + "where": "title" + }, + { + "text": "post-mortem", + "weight": 14 + }, + { + "text": "root cause analysis", + "weight": 22 + }, + { + "text": "timeline of events", + "weight": 20 + }, + { + "text": "affected systems", + "weight": 18 + }, + { + "text": "containment and eradication", + "weight": 26 + }, + { + "text": "indicators of compromise", + "weight": 26 + }, + { + "text": "data breach notification", + "weight": 24 + }, + { + "text": "unauthorized access", + "weight": 14 + }, + { + "text": "lessons learned", + "weight": 8 + }, + { + "text": "severity level", + "weight": 6 + }, + { + "text": "police report", + "weight": 24 + }, + { + "text": "accident report", + "weight": 22, + "where": "title" + }, + { + "text": "reporting officer", + "weight": 20 + } + ], + "regexes": [ + { + "pattern": "incident\\s*(id|number|ref(erence)?)\\s*[:#]?\\s*[a-z0-9-]{3,15}", + "flags": "gi", + "weight": 16, + "name": "incident ID" + }, + { + "pattern": "\\bioc(s)?\\b", + "flags": "gi", + "weight": 12, + "name": "IOC abbreviation" + }, + { + "pattern": "detected\\s+(on|at)\\s+\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{2,4}", + "flags": "gi", + "weight": 14, + "name": "detection date" + }, + { + "pattern": "mean\\s+time\\s+to\\s+(detect|contain|respond)", + "flags": "gi", + "weight": 16, + "name": "MTTD/MTTC/MTTR" + } + ], + "filenames": [ + { + "pattern": "security[\\s_-]?incident", + "weight": 26, + "name": "security incident filename" + }, + { + "pattern": "incident[\\s_-]?report", + "weight": 24, + "name": "incident report filename" + }, + { + "pattern": "breach[\\s_-]?report", + "weight": 22, + "name": "breach report filename" + }, + { + "pattern": "post[\\s_-]?mortem", + "weight": 18, + "name": "postmortem filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "incident report|breach notification|post-incident", + "weight": 14, + "name": "Info title incident report" + } + ], + "negatives": [ + { + "text": "call to order", + "weight": 14, + "name": "meeting minutes vocab" + }, + { + "text": "attendees present", + "weight": 12, + "name": "meeting minutes vocab" + }, + { + "text": "risk register", + "weight": 12, + "name": "risk assessment vocab" + }, + { + "text": "likelihood and impact", + "weight": 10, + "name": "risk assessment vocab" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 6 + }, + { + "signal": "toc", + "weight": 3 + }, + { + "signal": "signature_block", + "weight": 3 + } + ] + }, + { + "id": "questionnaire", + "name": "Questionnaire", + "emit": true, + "phrases": [ + { + "text": "vendor security questionnaire", + "weight": 36, + "where": "title" + }, + { + "text": "third-party risk questionnaire", + "weight": 30, + "where": "title" + }, + { + "text": "standardized information gathering", + "weight": 28 + }, + { + "text": "consensus assessments initiative questionnaire", + "weight": 32 + }, + { + "text": "security questionnaire", + "weight": 20, + "where": "title" + }, + { + "text": "subprocessors", + "weight": 16 + }, + { + "text": "data handling practices", + "weight": 16 + }, + { + "text": "please answer yes or no", + "weight": 18 + }, + { + "text": "vendor risk management", + "weight": 16 + }, + { + "text": "due diligence questionnaire", + "weight": 20 + }, + { + "text": "compensating control", + "weight": 10 + }, + { + "text": "response column", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "\\bsig\\s?(lite|core)\\b", + "flags": "gi", + "weight": 16, + "name": "SIG questionnaire abbreviation" + }, + { + "pattern": "\\bcaiq\\b", + "flags": "gi", + "weight": 18, + "name": "CAIQ abbreviation" + }, + { + "pattern": "yes\\s*/\\s*no\\s*/\\s*n\\s*/\\s*a", + "flags": "gi", + "weight": 12, + "name": "yes/no/na answer format" + }, + { + "pattern": "question\\s*#?\\d{1,3}", + "flags": "gi", + "weight": 12, + "name": "numbered question" + } + ], + "filenames": [ + { + "pattern": "security[\\s_-]?questionnaire", + "weight": 28, + "name": "security questionnaire filename" + }, + { + "pattern": "vendor[\\s_-]?risk", + "weight": 20, + "name": "vendor risk filename" + }, + { + "pattern": "(^|[^a-z0-9])(sig|caiq)([^a-z0-9]|$)", + "weight": 22, + "name": "SIG/CAIQ abbreviation filename" + }, + { + "pattern": "due[\\s_-]?diligence", + "weight": 16, + "name": "due diligence filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "security questionnaire|vendor risk|caiq", + "weight": 14, + "name": "Info title vendor questionnaire" + } + ], + "negatives": [ + { + "text": "cvss", + "weight": 12, + "name": "pentest report vocab" + }, + { + "text": "trust services criteria", + "weight": 14, + "name": "SOC 2 report vocab" + }, + { + "text": "auditor's opinion", + "weight": 12, + "name": "SOC 2 opinion vocab" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 10 + }, + { + "signal": "bullet_heavy", + "weight": 4 + }, + { + "signal": "number_table", + "weight": 4 + } + ] + }, + { + "id": "lab-report", + "name": "Lab report", + "emit": true, + "phrases": [ + { + "text": "laboratory report", + "weight": 28, + "where": "title" + }, + { + "text": "reference range", + "weight": 26 + }, + { + "text": "reference interval", + "weight": 24 + }, + { + "text": "full blood count", + "weight": 24 + }, + { + "text": "comprehensive metabolic panel", + "weight": 24 + }, + { + "text": "complete blood count", + "weight": 22 + }, + { + "text": "specimen type", + "weight": 22 + }, + { + "text": "specimen id", + "weight": 20 + }, + { + "text": "basic metabolic panel", + "weight": 20 + }, + { + "text": "urea and electrolytes", + "weight": 16 + }, + { + "text": "lipid panel", + "weight": 16 + }, + { + "text": "sample type", + "weight": 14, + "where": "first" + }, + { + "text": "date collected", + "weight": 14, + "where": "first" + }, + { + "text": "out of range", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "\\b\\d{1,4}(?:\\.\\d{1,3})?\\s?(?:mg\\/dl|mmol\\/l|g\\/dl|g\\/l|iu\\/l|u\\/l|ng\\/ml|mcg\\/dl|pg\\/ml|miu\\/l|umol\\/l|10\\^(?:9|12)\\/l|x10e(?:3|6)\\/ul)\\b", + "weight": 16, + "name": "Result value with lab units" + }, + { + "pattern": "(?:mg\\/dl|mmol\\/l|g\\/dl|g\\/l|iu\\/l|u\\/l|ng\\/ml|umol\\/l|ug\\/l|fl|%)\\s?\\(\\s?\\d{1,4}(?:\\.\\d{1,3})?\\s?[-–]\\s?\\d{1,4}(?:\\.\\d{1,3})?\\s?\\)", + "weight": 14, + "name": "Bracketed reference range" + }, + { + "pattern": "\\bclia\\s?(?:no\\.?|number|#)?\\s?:?\\s?\\d{2}d\\d{7}\\b", + "weight": 14, + "name": "CLIA number" + }, + { + "pattern": "\\b(?:h(?:a)?emoglobin|h(?:a)?ematocrit|platelet count|white (?:blood )?cell count|creatinine|bilirubin|alkaline phosphatase|hba1c|tsh|egfr|ferritin)\\b", + "weight": 10, + "name": "Common lab analytes" + }, + { + "pattern": "\\bcollected\\s?:?\\s?\\d{1,2}[\\/\\-]", + "weight": 10, + "name": "Collected date field" + } + ], + "filenames": [ + { + "pattern": "lab[-_ .]?(results?|report|work)", + "weight": 26, + "name": "lab results filename" + }, + { + "pattern": "blood[-_ .]?(test|work|results?)", + "weight": 22, + "name": "blood test filename" + }, + { + "pattern": "path(ology)?[-_ .]?(report|results?)", + "weight": 18, + "name": "pathology filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "lab(oratory)?[ _-]?(results?|report)", + "weight": 14, + "name": "lab report in title" + }, + { + "field": "any", + "pattern": "labcorp|quest diagnostics|sonora quest|bioreference", + "weight": 14, + "name": "reference lab vendor" + } + ], + "negatives": [ + { + "text": "explanation of benefits", + "weight": 18, + "name": "EOB language" + }, + { + "text": "certificate of analysis", + "weight": 18, + "name": "industrial CoA, not medical" + }, + { + "text": "amount billed", + "weight": 18, + "name": "billing language" + }, + { + "text": "patient responsibility", + "weight": 18, + "name": "billing language" + }, + { + "text": "allowed amount", + "weight": 16, + "name": "billing language" + }, + { + "text": "plan paid", + "weight": 14, + "name": "EOB payment language" + }, + { + "text": "amount you owe", + "weight": 14, + "name": "billing language" + }, + { + "text": "dispense as written", + "weight": 14, + "name": "prescription language" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 10 + }, + { + "signal": "short_doc", + "weight": 4 + } + ] + }, + { + "id": "prescription", + "name": "Prescription", + "emit": true, + "phrases": [ + { + "text": "dispense as written", + "weight": 32 + }, + { + "text": "repeat prescription", + "weight": 26 + }, + { + "text": "pharmacy stamp", + "weight": 24 + }, + { + "text": "refills remaining", + "weight": 20 + }, + { + "text": "qty dispensed", + "weight": 18 + }, + { + "text": "days supply", + "weight": 16 + }, + { + "text": "take one tablet", + "weight": 14 + }, + { + "text": "take one capsule", + "weight": 14 + }, + { + "text": "sig:", + "weight": 14 + }, + { + "text": "prescriber", + "weight": 12 + }, + { + "text": "controlled substance", + "weight": 10 + }, + { + "text": "pharmacist", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\bdea\\s?(?:no\\.?|number|#)?\\s?:?\\s?[a-z]{2}\\d{7}\\b", + "weight": 25, + "name": "DEA number" + }, + { + "pattern": "\\bndc\\s?:?\\s?\\d{4,5}-\\d{3,4}-\\d{1,2}\\b", + "weight": 20, + "name": "NDC code" + }, + { + "pattern": "\\btake (?:one|two|1|2) (?:tablet|capsule|cap)s? by mouth\\b", + "weight": 18, + "name": "Sig dosing instruction" + }, + { + "pattern": "\\brx\\s?(?:no\\.?|number|#)?\\s?:?\\s?\\d{5,10}\\b", + "weight": 16, + "name": "Rx number" + }, + { + "pattern": "\\brefills?(?: remaining| left| authorized)?\\s?:?\\s?\\d{1,2}\\b", + "weight": 14, + "name": "Refill count" + } + ], + "filenames": [ + { + "pattern": "prescription", + "weight": 26, + "name": "prescription filename" + }, + { + "pattern": "(^|[^a-z])rx([^a-z]|$)", + "weight": 18, + "name": "rx token in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "prescription", + "weight": 12, + "name": "prescription in title" + } + ], + "negatives": [ + { + "text": "explanation of benefits", + "weight": 18, + "name": "EOB language" + }, + { + "text": "reference range", + "weight": 16, + "name": "lab report language" + }, + { + "text": "allowed amount", + "weight": 14, + "name": "billing language" + }, + { + "text": "patient responsibility", + "weight": 12, + "name": "billing language" + }, + { + "text": "formulary", + "weight": 12, + "name": "insurance drug-list language" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 8 + }, + { + "signal": "form_like", + "weight": 4 + } + ] + }, + { + "id": "medical-report", + "name": "Medical report", + "emit": true, + "phrases": [ + { + "text": "discharge summary", + "weight": 26, + "where": "title" + }, + { + "text": "presenting complaint", + "weight": 24 + }, + { + "text": "history of present illness", + "weight": 24 + }, + { + "text": "on examination", + "weight": 20 + }, + { + "text": "past medical history", + "weight": 20 + }, + { + "text": "hospital course", + "weight": 20 + }, + { + "text": "thank you for referring", + "weight": 18, + "where": "first" + }, + { + "text": "treatment plan", + "weight": 14 + }, + { + "text": "diagnosis:", + "weight": 12 + }, + { + "text": "dear dr", + "weight": 12, + "where": "first" + }, + { + "text": "physical examination", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "\\bconsultant (?:cardiologist|physician|dermatologist|neurologist|psychiatrist|surgeon|paediatrician|pediatrician|gastroenterologist|rheumatologist|endocrinologist|oncologist|urologist|ophthalmologist)\\b", + "weight": 16, + "name": "Consultant specialty sign-off" + }, + { + "pattern": "\\bmrn\\s?(?:no\\.?|#)?\\s?:?\\s?[a-z0-9]{5,12}\\b", + "weight": 14, + "name": "Medical record number" + }, + { + "pattern": "\\bnhs\\s?(?:no\\.?|number)\\s?:?\\s?\\d{3}\\s?\\d{3}\\s?\\d{4}\\b", + "weight": 10, + "name": "NHS number" + } + ], + "filenames": [ + { + "pattern": "discharge[-_ .]?summary", + "weight": 24, + "name": "discharge summary filename" + }, + { + "pattern": "(clinic|consult(ant)?|medical|dr)[-_ .]?letter", + "weight": 22, + "name": "clinic letter filename" + }, + { + "pattern": "clinic[-_ .]?note", + "weight": 20, + "name": "clinic note filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "(clinic|discharge) (letter|summary)", + "weight": 12, + "name": "clinic letter in title" + } + ], + "negatives": [ + { + "text": "reason for referral", + "weight": 20, + "name": "referral letter language" + }, + { + "text": "please see this patient", + "weight": 20, + "name": "referral letter language" + }, + { + "text": "i am referring", + "weight": 18, + "name": "referral letter language" + }, + { + "text": "reference range", + "weight": 14, + "name": "lab report language" + }, + { + "text": "explanation of benefits", + "weight": 14, + "name": "EOB language" + } + ], + "structural": [ + { + "signal": "address_block", + "weight": 6 + }, + { + "signal": "signature_block", + "weight": 5 + } + ] + }, + { + "id": "immunization-record", + "name": "Immunization record", + "emit": true, + "phrases": [ + { + "text": "immunization record", + "weight": 34, + "where": "title" + }, + { + "text": "vaccination record", + "weight": 32, + "where": "title" + }, + { + "text": "certificate of vaccination", + "weight": 30, + "where": "title" + }, + { + "text": "immunisation history", + "weight": 26 + }, + { + "text": "date administered", + "weight": 18 + }, + { + "text": "dtap", + "weight": 18 + }, + { + "text": "tdap", + "weight": 18 + }, + { + "text": "booster due", + "weight": 16 + }, + { + "text": "lot number", + "weight": 14 + }, + { + "text": "varicella", + "weight": 14 + }, + { + "text": "mmr", + "weight": 12 + }, + { + "text": "administered by", + "weight": 10 + }, + { + "text": "yellow fever", + "weight": 10 + }, + { + "text": "hepatitis b", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\b(?:comirnaty|spikevax|vaxzevria|nuvaxovid|gardasil|shingrix|havrix|engerix|stamaril|typhim|daptacel|varivax|fluzone|menactra|bexsero|pentacel|infanrix)\\b", + "weight": 16, + "name": "Vaccine brand name" + }, + { + "pattern": "\\b(?:lot|batch)\\s?(?:no\\.?|number|#)?\\s?:?\\s?[a-z0-9][a-z0-9-]{3,11}\\b", + "weight": 10, + "name": "Lot/batch number" + } + ], + "filenames": [ + { + "pattern": "immuni[sz]", + "weight": 26, + "name": "immunization filename" + }, + { + "pattern": "vaccin", + "weight": 24, + "name": "vaccine filename" + }, + { + "pattern": "covid[-_ .]?(pass|cert|vacc|record)", + "weight": 20, + "name": "covid pass filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "immuni[sz]ation|vaccin", + "weight": 12, + "name": "vaccination in title" + } + ], + "negatives": [ + { + "text": "reference range", + "weight": 14, + "name": "lab report language" + }, + { + "text": "explanation of benefits", + "weight": 14, + "name": "EOB language" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 4 + }, + { + "signal": "form_like", + "weight": 4 + }, + { + "signal": "short_doc", + "weight": 5 + } + ] + }, + { + "id": "medical-invoice", + "name": "Medical invoice", + "emit": true, + "phrases": [ + { + "text": "explanation of benefits", + "weight": 34, + "where": "title" + }, + { + "text": "this is not a bill", + "weight": 30, + "where": "first" + }, + { + "text": "patient responsibility", + "weight": 26 + }, + { + "text": "allowed amount", + "weight": 22 + }, + { + "text": "amount billed", + "weight": 20 + }, + { + "text": "plan paid", + "weight": 20 + }, + { + "text": "provider may bill you", + "weight": 18 + }, + { + "text": "coinsurance", + "weight": 16 + }, + { + "text": "copay", + "weight": 12 + }, + { + "text": "date of service", + "weight": 12 + }, + { + "text": "amount you owe", + "weight": 12 + }, + { + "text": "out-of-pocket maximum", + "weight": 10 + }, + { + "text": "guarantor", + "weight": 8 + }, + { + "text": "deductible", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "\\bcpt\\s?(?:code)?\\s?:?\\s?\\d{5}\\b", + "weight": 18, + "name": "CPT code" + }, + { + "pattern": "amount billed[\\s\\S]{0,40}allowed amount", + "weight": 16, + "name": "EOB amount column headers" + }, + { + "pattern": "\\bguarantor\\s?(?:no\\.?|number|#|id)\\s?:?", + "weight": 16, + "name": "Guarantor number" + }, + { + "pattern": "\\bclaim\\s?(?:no\\.?|number|#|id)\\s?:?\\s?[a-z0-9][a-z0-9-]{5,17}\\b", + "weight": 12, + "name": "Claim number" + }, + { + "pattern": "\\b\\d{5}\\s?\\$\\d", + "weight": 12, + "name": "Procedure code before amount" + }, + { + "pattern": "\\bmember id\\s?:?\\s?[a-z0-9]{6,14}\\b", + "weight": 10, + "name": "Member ID" + } + ], + "filenames": [ + { + "pattern": "explanation[-_ .]?of[-_ .]?benefits", + "weight": 28, + "name": "explanation of benefits filename" + }, + { + "pattern": "(^|[^a-z])eob([^a-z]|$)", + "weight": 22, + "name": "eob token in filename" + }, + { + "pattern": "(medical|hospital)[-_ .]?bill", + "weight": 22, + "name": "medical bill filename" + }, + { + "pattern": "billing[-_ .]?statement", + "weight": 18, + "name": "billing statement filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "explanation of benefits|eob|billing statement", + "weight": 12, + "name": "EOB in title" + } + ], + "negatives": [ + { + "text": "summary of benefits and coverage", + "weight": 18, + "name": "insurance policy language" + }, + { + "text": "reference range", + "weight": 16, + "name": "lab report language" + }, + { + "text": "policy period", + "weight": 14, + "name": "insurance policy language" + }, + { + "text": "kwh", + "weight": 14, + "name": "utility bill language" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 9 + }, + { + "signal": "number_table", + "weight": 6 + } + ] + }, + { + "id": "referral-letter", + "name": "Referral letter", + "emit": true, + "phrases": [ + { + "text": "referral letter", + "weight": 28, + "where": "title" + }, + { + "text": "reason for referral", + "weight": 28 + }, + { + "text": "please see this patient", + "weight": 28 + }, + { + "text": "referring physician", + "weight": 26 + }, + { + "text": "i am referring", + "weight": 26 + }, + { + "text": "referring gp", + "weight": 24 + }, + { + "text": "two week wait", + "weight": 24 + }, + { + "text": "grateful if you would see", + "weight": 24 + }, + { + "text": "referring provider", + "weight": 22 + }, + { + "text": "urgent referral", + "weight": 22 + } + ], + "regexes": [ + { + "pattern": "\\bnhs\\s?(?:no\\.?|number)\\s?:?\\s?\\d{3}\\s?\\d{3}\\s?\\d{4}\\b", + "weight": 10, + "name": "NHS number" + } + ], + "filenames": [ + { + "pattern": "referral", + "weight": 26, + "name": "referral filename" + }, + { + "pattern": "refer(red|ring)", + "weight": 16, + "name": "referred/referring filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "referral", + "weight": 14, + "name": "referral in title" + } + ], + "negatives": [ + { + "text": "letter of recommendation", + "weight": 22, + "name": "HR reference letter" + }, + { + "text": "it is my pleasure to recommend", + "weight": 22, + "name": "HR reference letter" + }, + { + "text": "thank you for referring", + "weight": 16, + "name": "consultant reply letter" + }, + { + "text": "reference range", + "weight": 14, + "name": "lab report language" + }, + { + "text": "explanation of benefits", + "weight": 14, + "name": "EOB language" + } + ], + "structural": [ + { + "signal": "address_block", + "weight": 6 + }, + { + "signal": "signature_block", + "weight": 5 + }, + { + "signal": "short_doc", + "weight": 5 + } + ] + }, + { + "id": "consent-form", + "name": "Consent form", + "emit": true, + "phrases": [ + { + "text": "consent form", + "weight": 30, + "where": "title" + }, + { + "text": "informed consent", + "weight": 26, + "where": "title" + }, + { + "text": "i have had the opportunity to ask questions", + "weight": 32 + }, + { + "text": "i consent to", + "weight": 24 + }, + { + "text": "consent to treatment", + "weight": 24 + }, + { + "text": "participant information sheet", + "weight": 24 + }, + { + "text": "person taking consent", + "weight": 26 + }, + { + "text": "withdraw at any time", + "weight": 22 + }, + { + "text": "i give my consent", + "weight": 22 + }, + { + "text": "please initial each box", + "weight": 20 + }, + { + "text": "risks and benefits", + "weight": 18 + }, + { + "text": "patient signature", + "weight": 12 + }, + { + "text": "voluntary", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "i,?\\s+the\\s+undersigned,?\\s+(?:hereby\\s+)?(?:consent|authori[sz]e)", + "weight": 20, + "name": "undersigned consent clause" + }, + { + "pattern": "opportunity to [a-z ,]{0,50}ask questions", + "weight": 18, + "name": "opportunity to ask questions clause" + }, + { + "pattern": "(?:free|right) to withdraw [a-z ]{0,60}at any time", + "weight": 18, + "name": "free to withdraw clause" + }, + { + "pattern": "risks?,? (?:and\\s+)?benefits?,? and alternatives", + "weight": 16, + "name": "risks benefits alternatives triple" + } + ], + "filenames": [ + { + "pattern": "consent[-_ ]?form", + "weight": 28, + "name": "consent form filename" + }, + { + "pattern": "informed[-_ ]?consent", + "weight": 26, + "name": "informed consent filename" + }, + { + "pattern": "consent", + "weight": 18, + "name": "consent in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "consent", + "weight": 14, + "name": "consent in PDF title" + } + ], + "negatives": [ + { + "text": "omb no.", + "weight": 20, + "name": "government form field" + }, + { + "text": "paperwork reduction act", + "weight": 20, + "name": "government form language" + }, + { + "text": "explanation of benefits", + "weight": 16, + "name": "EOB language" + }, + { + "text": "reason for referral", + "weight": 14, + "name": "referral letter language" + }, + { + "text": "reference range", + "weight": 12, + "name": "lab report language" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 8 + }, + { + "signal": "form_like", + "weight": 6 + }, + { + "signal": "short_doc", + "weight": 4 + } + ] + }, + { + "id": "discharge-summary", + "name": "Discharge summary", + "emit": true, + "phrases": [ + { + "text": "discharge summary", + "weight": 34, + "where": "title" + }, + { + "text": "medications on discharge", + "weight": 30 + }, + { + "text": "discharge medications", + "weight": 26 + }, + { + "text": "discharge diagnosis", + "weight": 26 + }, + { + "text": "date of admission", + "weight": 24 + }, + { + "text": "date of discharge", + "weight": 24 + }, + { + "text": "follow-up arrangements", + "weight": 24 + }, + { + "text": "copy to gp", + "weight": 22 + }, + { + "text": "condition at discharge", + "weight": 20 + }, + { + "text": "admission date", + "weight": 18, + "where": "first" + }, + { + "text": "discharge date", + "weight": 18, + "where": "first" + }, + { + "text": "discharge instructions", + "weight": 18 + }, + { + "text": "hospital course", + "weight": 16 + }, + { + "text": "discharged home", + "weight": 14 + } + ], + "regexes": [ + { + "pattern": "date of admission[\\s\\S]{0,60}date of discharge", + "weight": 18, + "name": "admission/discharge date pair (UK)" + }, + { + "pattern": "admi(?:ssion|t) date[\\s\\S]{0,60}discharge date", + "weight": 16, + "name": "admission/discharge date pair (US)" + }, + { + "pattern": "discharged? (?:home|to home|to a (?:nursing|care) home|to rehab)", + "weight": 12, + "name": "discharge destination" + }, + { + "pattern": "\\bnhs\\s?(?:no\\.?|number)\\s?:?\\s?\\d{3}\\s?\\d{3}\\s?\\d{4}\\b", + "weight": 8, + "name": "NHS number" + } + ], + "filenames": [ + { + "pattern": "discharge[-_ ]?summary", + "weight": 28, + "name": "discharge summary filename" + }, + { + "pattern": "discharge", + "weight": 20, + "name": "discharge in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "discharge", + "weight": 14, + "name": "discharge in PDF title" + } + ], + "negatives": [ + { + "text": "reason for referral", + "weight": 22, + "name": "referral letter language" + }, + { + "text": "please see this patient", + "weight": 20, + "name": "referral letter language" + }, + { + "text": "grateful if you would see", + "weight": 18, + "name": "referral letter language" + }, + { + "text": "thank you for referring", + "weight": 16, + "name": "clinic letter language" + }, + { + "text": "explanation of benefits", + "weight": 16, + "name": "EOB language" + }, + { + "text": "patient responsibility", + "weight": 14, + "name": "billing language" + }, + { + "text": "reference range", + "weight": 12, + "name": "lab report language" + } + ], + "structural": [ + { + "signal": "address_block", + "weight": 4 + }, + { + "signal": "signature_block", + "weight": 3 + } + ] + }, + { + "id": "insurance-policy", + "name": "Insurance policy", + "emit": true, + "phrases": [ + { + "text": "policy wording", + "weight": 28, + "where": "title" + }, + { + "text": "insuring agreement", + "weight": 30 + }, + { + "text": "declarations page", + "weight": 26, + "where": "first" + }, + { + "text": "policy schedule", + "weight": 26 + }, + { + "text": "period of insurance", + "weight": 24 + }, + { + "text": "sum insured", + "weight": 22 + }, + { + "text": "limit of indemnity", + "weight": 22 + }, + { + "text": "contract of insurance", + "weight": 20 + }, + { + "text": "general exclusions", + "weight": 18 + }, + { + "text": "policy period", + "weight": 14, + "where": "first" + }, + { + "text": "coverages and limits", + "weight": 12 + }, + { + "text": "named insured", + "weight": 12, + "where": "first" + }, + { + "text": "underwritten by", + "weight": 12 + }, + { + "text": "policyholder", + "weight": 8 + }, + { + "text": "your policy is due for renewal", + "weight": 36, + "where": "first" + }, + { + "text": "renewal invitation", + "weight": 30, + "where": "title" + }, + { + "text": "renewal premium", + "weight": 28 + }, + { + "text": "your new premium", + "weight": 26 + }, + { + "text": "renewal notice", + "weight": 20, + "where": "title" + }, + { + "text": "renewal offer", + "weight": 20, + "where": "title" + }, + { + "text": "renewal term", + "weight": 18 + }, + { + "text": "insurance premium tax", + "weight": 16 + }, + { + "text": "no claims discount", + "weight": 14 + }, + { + "text": "no claims bonus", + "weight": 14 + }, + { + "text": "renewal date", + "weight": 12, + "where": "first" + }, + { + "text": "continuous payment authority", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "policy\\s*(?:no|number|num|#)[.:# ]\\s*[a-z0-9][a-z0-9/-]{4,19}", + "flags": "gi", + "weight": 12, + "name": "Policy number" + }, + { + "pattern": "(?:last year(?:'|’)?s|previous(?: annual)?) premium", + "flags": "gi", + "weight": 20, + "name": "Premium comparison" + }, + { + "pattern": "renewal (?:date|premium)\\s*:", + "flags": "gi", + "weight": 12, + "name": "Renewal field with colon" + }, + { + "pattern": "renew(?:s|ed)? automatically|auto(?:matic(?:ally)?)?[ -]?renew", + "flags": "gi", + "weight": 12, + "name": "Auto-renew variants" + } + ], + "filenames": [ + { + "pattern": "insurance[ _-]?policy", + "weight": 26, + "name": "insurance policy filename" + }, + { + "pattern": "policy[ _-]?(schedule|wording|document|booklet|declarations?)", + "weight": 24, + "name": "policy document filename" + }, + { + "pattern": "(?:^|[\\W_])ipid(?:[\\W_]|$)", + "weight": 18, + "name": "IPID filename" + }, + { + "pattern": "(?:^|[\\W_])policy", + "weight": 8, + "name": "policy in filename" + }, + { + "pattern": "(insurance|policy|motor|home|auto)[ _-]?renewal", + "weight": 26, + "name": "insurance renewal filename" + }, + { + "pattern": "renewal[ _-]?(notice|invitation|offer)", + "weight": 26, + "name": "renewal notice filename" + }, + { + "pattern": "renewal", + "weight": 12, + "name": "renewal in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "insurance policy|policy (schedule|wording|booklet|document)", + "weight": 12, + "name": "policy document title" + }, + { + "field": "title", + "pattern": "renewal (notice|invitation|offer)|policy renewal", + "weight": 12, + "name": "renewal document title" + } + ], + "negatives": [ + { + "text": "certificate of liability insurance", + "weight": 25, + "name": "COI heading" + }, + { + "text": "certificate holder", + "weight": 18, + "name": "COI field" + }, + { + "text": "date of loss", + "weight": 15, + "name": "claim document" + }, + { + "text": "privacy policy", + "weight": 16, + "name": "web/ToS privacy policy" + }, + { + "text": "limited warranty", + "weight": 12, + "name": "product warranty terms" + }, + { + "text": "employee handbook", + "weight": 12, + "name": "HR policy handbook" + }, + { + "text": "your subscription", + "weight": 18, + "name": "subscription renewal" + }, + { + "text": "registration renewal", + "weight": 16, + "name": "vehicle registration renewal" + }, + { + "text": "lease agreement", + "weight": 14, + "name": "lease renewal" + }, + { + "text": "domain name", + "weight": 12, + "name": "domain renewal notice" + }, + { + "text": "membership", + "weight": 10, + "name": "membership renewal" + } + ], + "structural": [ + { + "signal": "long_doc", + "weight": 5 + }, + { + "signal": "toc", + "weight": 3 + }, + { + "signal": "short_doc", + "weight": 4 + }, + { + "signal": "currency_heavy", + "weight": 3 + } + ] + }, + { + "id": "insurance-certificate", + "name": "Insurance certificate", + "emit": true, + "phrases": [ + { + "text": "certificate of liability insurance", + "weight": 38, + "where": "title" + }, + { + "text": "this certificate is issued as a matter of information", + "weight": 36, + "where": "first" + }, + { + "text": "should any of the above described policies be cancelled", + "weight": 34 + }, + { + "text": "certificate of insurance", + "weight": 30, + "where": "title" + }, + { + "text": "certificate holder", + "weight": 30 + }, + { + "text": "certificate of motor insurance", + "weight": 30, + "where": "title" + }, + { + "text": "certificate of employers", + "weight": 28, + "where": "title" + }, + { + "text": "certificate must be displayed", + "weight": 26 + }, + { + "text": "evidence of property insurance", + "weight": 24, + "where": "first" + }, + { + "text": "commercial general liability", + "weight": 18 + }, + { + "text": "general aggregate", + "weight": 18 + }, + { + "text": "evidence of insurance", + "weight": 16 + }, + { + "text": "acord", + "weight": 14 + }, + { + "text": "additional insured", + "weight": 14 + } + ], + "regexes": [ + { + "pattern": "acord\\s*2[0-9]{1,2}", + "flags": "gi", + "weight": 22, + "name": "ACORD form number" + }, + { + "pattern": "each occurrence\\s*\\$\\s*[\\d,]{7,15}", + "flags": "gi", + "weight": 18, + "name": "Occurrence limit amount" + } + ], + "filenames": [ + { + "pattern": "certificate[ _-]?of[ _-]?insurance", + "weight": 28, + "name": "certificate of insurance filename" + }, + { + "pattern": "acord", + "weight": 25, + "name": "acord filename" + }, + { + "pattern": "(?:^|[\\W_])coi(?:[\\W_]|$)", + "weight": 22, + "name": "coi abbreviation" + }, + { + "pattern": "cert.{0,20}(insurance|liab)", + "weight": 20, + "name": "insurance certificate filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "certificate of .{0,20}insurance|acord", + "weight": 16, + "name": "certificate title" + } + ], + "negatives": [ + { + "text": "policy wording", + "weight": 18, + "name": "full policy booklet" + }, + { + "text": "has successfully completed", + "weight": 20, + "name": "diploma/certificate of completion" + }, + { + "text": "certificate of completion", + "weight": 20, + "name": "course completion certificate" + }, + { + "text": "certificate of origin", + "weight": 18, + "name": "customs/trade certificate" + }, + { + "text": "renewal notice", + "weight": 12, + "name": "renewal document" + }, + { + "text": "date of loss", + "weight": 12, + "name": "claim document" + }, + { + "text": "table of contents", + "weight": 8, + "name": "long booklet" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 8 + }, + { + "signal": "form_like", + "weight": 5 + } + ] + }, + { + "id": "insurance-claim", + "name": "Insurance claim", + "emit": true, + "phrases": [ + { + "text": "date of loss", + "weight": 28, + "where": "first" + }, + { + "text": "proof of loss", + "weight": 26 + }, + { + "text": "first notice of loss", + "weight": 26 + }, + { + "text": "claims adjuster", + "weight": 24 + }, + { + "text": "loss adjuster", + "weight": 24 + }, + { + "text": "we have received your claim", + "weight": 24 + }, + { + "text": "notice of claim", + "weight": 22, + "where": "title" + }, + { + "text": "claim reference", + "weight": 20, + "where": "first" + }, + { + "text": "cause of loss", + "weight": 20 + }, + { + "text": "claim number", + "weight": 16, + "where": "first" + }, + { + "text": "your claim has been", + "weight": 16 + }, + { + "text": "claim form", + "weight": 14, + "where": "title" + }, + { + "text": "amount claimed", + "weight": 10 + }, + { + "text": "claimant", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "claim\\s*(?:no|number|num|ref|reference|#)[.:# ]\\s*[a-z0-9][a-z0-9/-]{3,19}", + "flags": "gi", + "weight": 15, + "name": "Claim number" + }, + { + "pattern": "date of loss\\s*[:.]?\\s*\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{2,4}", + "flags": "gi", + "weight": 18, + "name": "Date of loss field" + } + ], + "filenames": [ + { + "pattern": "insurance[ _-]?claim", + "weight": 26, + "name": "insurance claim filename" + }, + { + "pattern": "claim[ _-]?form", + "weight": 22, + "name": "claim form filename" + }, + { + "pattern": "(?:^|[\\W_])fnol(?:[\\W_]|$)", + "weight": 22, + "name": "first notice of loss abbreviation" + }, + { + "pattern": "(?:^|[\\W_])claim", + "weight": 12, + "name": "claim in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "claim (form|summary|letter)|insurance claim", + "weight": 12, + "name": "claim document title" + } + ], + "negatives": [ + { + "text": "explanation of benefits", + "weight": 20, + "name": "health EOB (medical bill)" + }, + { + "text": "this is not a bill", + "weight": 15, + "name": "EOB boilerplate" + }, + { + "text": "policy wording", + "weight": 15, + "name": "full policy booklet" + }, + { + "text": "expense report", + "weight": 15, + "name": "expense claim" + }, + { + "text": "small claims", + "weight": 16, + "name": "small claims court" + }, + { + "pattern": "in the (county|high) court", + "weight": 18, + "name": "UK court heading" + }, + { + "pattern": "\\bplaintiff\\b", + "weight": 14, + "name": "US court party" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 4 + } + ] + }, + { + "id": "research-paper", + "name": "Research paper", + "emit": true, + "phrases": [ + { + "text": "abstract", + "weight": 8, + "where": "title" + }, + { + "text": "keywords:", + "weight": 12, + "where": "first" + }, + { + "text": "index terms", + "weight": 20, + "where": "first" + }, + { + "text": "in this paper", + "weight": 16 + }, + { + "text": "we propose", + "weight": 8 + }, + { + "text": "et al.", + "weight": 8 + }, + { + "text": "related work", + "weight": 14 + }, + { + "text": "corresponding author", + "weight": 20 + }, + { + "text": "received in revised form", + "weight": 24, + "where": "first" + }, + { + "text": "manuscript received", + "weight": 18, + "where": "first" + }, + { + "text": "proceedings of the", + "weight": 14 + }, + { + "text": "contents lists available at sciencedirect", + "weight": 24, + "where": "first" + } + ], + "regexes": [ + { + "pattern": "\\b10\\.\\d{4,9}/[a-z0-9.()/:;_-]{4,60}", + "weight": 20, + "name": "DOI" + }, + { + "pattern": "arxiv[:\\s]\\d{4}\\.\\d{4,5}", + "weight": 25, + "name": "arXiv identifier" + }, + { + "pattern": "\\[\\d{1,3}(,\\s?\\d{1,3}){1,6}\\]", + "weight": 10, + "name": "Numeric citation brackets" + }, + { + "pattern": "vol\\.\\s?\\d{1,3},\\s?no\\.\\s?\\d{1,3}", + "weight": 14, + "name": "Volume/issue line" + }, + { + "pattern": "issn[:\\s]*\\d{4}-?\\d{3}[\\dx]", + "weight": 10, + "name": "ISSN" + } + ], + "filenames": [ + { + "pattern": "arxiv", + "weight": 24, + "name": "arxiv in filename" + }, + { + "pattern": "\\d{4}\\.\\d{4,5}(v\\d{1,2})?\\.pdf", + "weight": 22, + "name": "arXiv id filename" + }, + { + "pattern": "1-s2\\.0-s\\d", + "weight": 25, + "name": "Elsevier download filename" + }, + { + "pattern": "(preprint|manuscript)", + "weight": 16, + "name": "preprint/manuscript filename" + } + ], + "metadata": [ + { + "field": "producer", + "pattern": "latex|pdftex|xetex|luatex", + "weight": 14, + "name": "LaTeX producer" + }, + { + "field": "any", + "pattern": "elsevier|springer|ieee|arxiv", + "weight": 12, + "name": "Academic publisher" + } + ], + "negatives": [ + { + "text": "in partial fulfillment", + "weight": 22, + "name": "Thesis submission clause" + }, + { + "text": "in partial fulfilment", + "weight": 22, + "name": "Thesis submission clause (UK)" + }, + { + "text": "white paper", + "weight": 16, + "name": "Business white paper" + }, + { + "text": "what is claimed is", + "weight": 20, + "name": "Patent claims wording" + }, + { + "text": "this is to certify", + "weight": 12, + "name": "Certificate wording" + } + ], + "structural": [ + { + "signal": "references_section", + "weight": 10 + } + ] + }, + { + "id": "thesis", + "name": "Thesis", + "emit": true, + "phrases": [ + { + "text": "in partial fulfillment", + "weight": 34, + "where": "first" + }, + { + "text": "in partial fulfilment", + "weight": 34, + "where": "first" + }, + { + "text": "a thesis submitted", + "weight": 32, + "where": "first" + }, + { + "text": "a dissertation submitted", + "weight": 32, + "where": "first" + }, + { + "text": "submitted in accordance with the requirements", + "weight": 26, + "where": "first" + }, + { + "text": "submitted to the faculty", + "weight": 22, + "where": "first" + }, + { + "text": "for the degree of", + "weight": 18, + "where": "first" + }, + { + "text": "doctor of philosophy", + "weight": 16, + "where": "first" + }, + { + "text": "doctoral committee", + "weight": 24, + "where": "first" + }, + { + "text": "thesis supervisor", + "weight": 18 + }, + { + "text": "i hereby declare that this thesis", + "weight": 30 + }, + { + "text": "declaration of authorship", + "weight": 24 + }, + { + "text": "graduate school", + "weight": 8 + }, + { + "text": "acknowledgements", + "weight": 5 + } + ], + "regexes": [ + { + "pattern": "(doctoral|ph\\.?d\\.?|master'?s|msc|bachelor'?s|undergraduate)\\s(thesis|dissertation)", + "weight": 22, + "name": "Degree-level thesis mention" + }, + { + "pattern": "\\bchapter\\s[1-9]\\b", + "weight": 6, + "name": "Chapter headings" + } + ], + "filenames": [ + { + "pattern": "(thesis|dissertation)", + "weight": 28, + "name": "thesis/dissertation filename" + }, + { + "pattern": "phd", + "weight": 12, + "name": "phd in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "thesis|dissertation", + "weight": 16, + "name": "Thesis in Info title" + }, + { + "field": "producer", + "pattern": "latex|pdftex|xetex", + "weight": 6, + "name": "LaTeX producer" + } + ], + "negatives": [ + { + "text": "received in revised form", + "weight": 15, + "name": "Journal article header" + }, + { + "text": "contents lists available at", + "weight": 15, + "name": "ScienceDirect header" + }, + { + "pattern": "isbn[:\\s]", + "weight": 10, + "name": "ISBN (book)" + } + ], + "structural": [ + { + "signal": "toc", + "weight": 9 + }, + { + "signal": "long_doc", + "weight": 10 + }, + { + "signal": "references_section", + "weight": 5 + } + ] + }, + { + "id": "certificate", + "name": "Certificate", + "emit": true, + "phrases": [ + { + "text": "certificate of completion", + "weight": 36, + "where": "title" + }, + { + "text": "certificate of achievement", + "weight": 32, + "where": "title" + }, + { + "text": "certificate of attendance", + "weight": 26 + }, + { + "text": "certificate of participation", + "weight": 24 + }, + { + "text": "this is to certify that", + "weight": 32 + }, + { + "text": "has successfully completed", + "weight": 30 + }, + { + "text": "is hereby awarded", + "weight": 26 + }, + { + "text": "having satisfied the examiners", + "weight": 26 + }, + { + "text": "with all the rights and privileges", + "weight": 26 + }, + { + "text": "awarded the degree of", + "weight": 22 + }, + { + "text": "verify this certificate", + "weight": 20 + }, + { + "text": "continuing education credits", + "weight": 16 + }, + { + "text": "in recognition of", + "weight": 12 + }, + { + "text": "professional certificate", + "weight": 14 + }, + { + "text": "certificate of live birth", + "weight": 32, + "where": "title" + }, + { + "text": "certificate of birth", + "weight": 30, + "where": "title" + }, + { + "text": "certificate of marriage", + "weight": 30, + "where": "title" + }, + { + "text": "certificate of death", + "weight": 30, + "where": "title" + }, + { + "text": "registration district", + "weight": 16 + } + ], + "regexes": [ + { + "pattern": "certificate\\s(no|number|id)[.:\\s]", + "weight": 14, + "name": "Certificate number" + }, + { + "pattern": "cpd\\s?(hours|credits|points|units)", + "weight": 14, + "name": "CPD hours" + } + ], + "filenames": [ + { + "pattern": "(certificate|diploma)", + "weight": 22, + "name": "certificate/diploma filename" + }, + { + "pattern": "(^|[^a-z])cert[_-]", + "weight": 14, + "name": "cert_ prefix" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "certificate|diploma", + "weight": 12, + "name": "Certificate in Info title" + } + ], + "negatives": [ + { + "text": "certificate of insurance", + "weight": 30, + "name": "Insurance certificate" + }, + { + "text": "certificate of liability", + "weight": 30, + "name": "Liability insurance certificate" + }, + { + "text": "issued as a matter of information", + "weight": 30, + "name": "ACORD COI wording" + }, + { + "text": "certificate of origin", + "weight": 22, + "name": "Customs certificate" + }, + { + "text": "certificate of incorporation", + "weight": 24, + "name": "Company registration certificate" + }, + { + "text": "certificate of analysis", + "weight": 20, + "name": "Lab/QC certificate" + }, + { + "text": "cumulative gpa", + "weight": 15, + "name": "Transcript wording" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 12 + }, + { + "signal": "signature_block", + "weight": 5 + } + ] + }, + { + "id": "transcript", + "name": "Transcript", + "emit": true, + "phrases": [ + { + "text": "official transcript", + "weight": 32, + "where": "title" + }, + { + "text": "academic transcript", + "weight": 32, + "where": "title" + }, + { + "text": "statement of results", + "weight": 24, + "where": "title" + }, + { + "text": "statement of marks", + "weight": 22, + "where": "title" + }, + { + "text": "academic record", + "weight": 16, + "where": "title" + }, + { + "text": "cumulative gpa", + "weight": 26 + }, + { + "text": "semester gpa", + "weight": 22 + }, + { + "text": "grade point average", + "weight": 20 + }, + { + "text": "credits attempted", + "weight": 22 + }, + { + "text": "credits earned", + "weight": 20 + }, + { + "text": "credits awarded", + "weight": 18 + }, + { + "text": "office of the registrar", + "weight": 26 + }, + { + "text": "dean's list", + "weight": 16 + }, + { + "text": "credit hours", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "\\bgpa[:\\s]+[0-4]\\.\\d{1,2}", + "weight": 18, + "name": "GPA value" + }, + { + "pattern": "\\b[a-f][+-]?\\s\\d{1,2}\\.\\d{2}\\b", + "weight": 8, + "name": "Letter grade row" + }, + { + "pattern": "\\b(10|15|20|30|40|60|120)\\scredits\\b", + "weight": 12, + "name": "Module credit value (UK)" + }, + { + "pattern": "\\b\\d{1,3}(\\.\\d)?\\s?ects\\b", + "weight": 16, + "name": "ECTS credits (EU)" + }, + { + "pattern": "\\b(fall|spring|summer|autumn|winter)\\s(semester|term)\\s20\\d{2}\\b", + "weight": 6, + "name": "Semester heading" + } + ], + "filenames": [ + { + "pattern": "transcript", + "weight": 22, + "name": "transcript filename" + }, + { + "pattern": "academic[_ -]?record", + "weight": 18, + "name": "academic record filename" + }, + { + "pattern": "mark[_ -]?sheet", + "weight": 20, + "name": "marksheet filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "transcript", + "weight": 12, + "name": "Transcript in Info title" + } + ], + "negatives": [ + { + "pattern": "(court|hearing|deposition|interview)\\s?transcript", + "weight": 22, + "name": "Non-academic transcript" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 8 + } + ] + }, + { + "id": "course-syllabus", + "name": "Course syllabus", + "emit": true, + "phrases": [ + { + "text": "course syllabus", + "weight": 32, + "where": "title" + }, + { + "text": "syllabus", + "weight": 18, + "where": "title" + }, + { + "text": "course outline", + "weight": 24, + "where": "title" + }, + { + "text": "module handbook", + "weight": 24 + }, + { + "text": "lecture notes", + "weight": 24 + }, + { + "text": "learning objectives", + "weight": 22 + }, + { + "text": "learning outcomes", + "weight": 20 + }, + { + "text": "required textbook", + "weight": 24 + }, + { + "text": "course description", + "weight": 20, + "where": "first" + }, + { + "text": "grading policy", + "weight": 20 + }, + { + "text": "office hours", + "weight": 14, + "where": "first" + }, + { + "text": "teaching assistant", + "weight": 12 + }, + { + "text": "academic integrity", + "weight": 12 + }, + { + "text": "recommended reading", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "\\blecture\\s\\d{1,2}\\b", + "weight": 12, + "name": "Lecture number" + }, + { + "pattern": "\\bweek\\s\\d{1,2}[:\\s]", + "weight": 8, + "name": "Week schedule" + } + ], + "filenames": [ + { + "pattern": "syllabus", + "weight": 28, + "name": "syllabus filename" + }, + { + "pattern": "lecture", + "weight": 18, + "name": "lecture filename" + }, + { + "pattern": "(week|module|unit)[_ -]?\\d{1,2}", + "weight": 14, + "name": "week/module filename" + }, + { + "pattern": "(^|[^a-z])lec[_-]?\\d", + "weight": 14, + "name": "lecNN filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "syllabus|lecture", + "weight": 12, + "name": "Syllabus/lecture in Info title" + } + ], + "negatives": [ + { + "text": "show your work", + "weight": 14, + "name": "Assignment wording" + }, + { + "text": "answer all questions", + "weight": 14, + "name": "Exam/worksheet wording" + }, + { + "pattern": "[\\[(]\\d{1,2}\\s?marks?[\\])]", + "weight": 12, + "name": "Marks brackets (worksheet)" + } + ], + "structural": [ + { + "signal": "bullet_heavy", + "weight": 4 + }, + { + "signal": "toc", + "weight": 3 + } + ] + }, + { + "id": "assignment-brief", + "name": "Assignment brief", + "emit": true, + "phrases": [ + { + "text": "show your work", + "weight": 28 + }, + { + "text": "show all your working", + "weight": 26 + }, + { + "text": "answer all questions", + "weight": 22 + }, + { + "text": "problem set", + "weight": 22, + "where": "title" + }, + { + "text": "question paper", + "weight": 16, + "where": "title" + }, + { + "text": "total marks", + "weight": 20 + }, + { + "text": "points possible", + "weight": 18 + }, + { + "text": "homework", + "weight": 16, + "where": "title" + }, + { + "text": "worksheet", + "weight": 16, + "where": "title" + }, + { + "text": "submit your solutions", + "weight": 16 + }, + { + "text": "late submissions will", + "weight": 14 + }, + { + "text": "total points", + "weight": 14 + }, + { + "text": "this assignment", + "weight": 10 + }, + { + "text": "due date", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "[\\[(]\\d{1,2}\\s?marks?[\\])]", + "weight": 20, + "name": "Marks brackets" + }, + { + "pattern": "\\(\\d{1,3}\\s?(points|pts)\\)", + "weight": 16, + "name": "Points parenthetical" + }, + { + "pattern": "(question|problem)\\s\\d{1,2}\\s?[.:(]", + "weight": 10, + "name": "Numbered questions" + }, + { + "pattern": "candidate\\s(number|no\\.?)[:\\s]", + "weight": 14, + "name": "Exam candidate header" + }, + { + "pattern": "due\\s(date:?\\s)?(by\\s)?(monday|tuesday|wednesday|thursday|friday|saturday|sunday)", + "weight": 12, + "name": "Due weekday" + } + ], + "filenames": [ + { + "pattern": "(homework|assignment|worksheet|problem[_ -]?set|pset)", + "weight": 24, + "name": "homework/worksheet filename" + }, + { + "pattern": "(^|[^a-z])hw[_ -]?\\d", + "weight": 18, + "name": "hwN filename" + }, + { + "pattern": "(^|[^a-z])(quiz|midterm|exams?)[^a-z]", + "weight": 16, + "name": "quiz/exam filename" + }, + { + "pattern": "exercise", + "weight": 10, + "name": "exercise filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "homework|assignment|worksheet", + "weight": 10, + "name": "Assignment in Info title" + } + ], + "negatives": [ + { + "text": "invoice", + "weight": 18, + "name": "Invoice (due date confusable)" + }, + { + "text": "amount due", + "weight": 16, + "name": "Billing wording" + }, + { + "text": "internal revenue service", + "weight": 18, + "name": "IRS tax worksheet" + }, + { + "pattern": "assignment\\s(of|and assumption)\\s", + "weight": 20, + "name": "Legal assignment" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 4 + }, + { + "signal": "form_like", + "weight": 3 + } + ] + }, + { + "id": "grade-report", + "name": "Grade report", + "emit": true, + "phrases": [ + { + "text": "report card", + "weight": 32, + "where": "title" + }, + { + "text": "working towards the expected standard", + "weight": 30 + }, + { + "text": "working at greater depth", + "weight": 28 + }, + { + "text": "school report", + "weight": 26, + "where": "title" + }, + { + "text": "end of year report", + "weight": 26 + }, + { + "text": "effort grade", + "weight": 24 + }, + { + "text": "teacher comments", + "weight": 20 + }, + { + "text": "teacher's comments", + "weight": 20 + }, + { + "text": "promoted to grade", + "weight": 18 + }, + { + "text": "attainment", + "weight": 16 + }, + { + "text": "exceeds standards", + "weight": 16 + }, + { + "text": "meets expectations", + "weight": 14 + }, + { + "text": "attendance", + "weight": 6 + }, + { + "text": "pupil", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "attendance\\s*:?\\s*\\d{1,3}(?:\\.\\d)?\\s*%", + "weight": 20, + "name": "attendance percentage" + }, + { + "pattern": "(?:working towards|working at|exceeding) (?:the )?expected standard", + "weight": 16, + "name": "UK attainment band" + }, + { + "pattern": "(?:autumn|spring|summer|fall) (?:term|trimester|semester)\\s+(?:report|20\\d{2})", + "weight": 14, + "name": "term report header" + }, + { + "pattern": "\\b(?:wts|exs|gds)\\b", + "weight": 10, + "name": "UK attainment codes" + } + ], + "filenames": [ + { + "pattern": "report[-_ ]?card", + "weight": 28, + "name": "report card filename" + }, + { + "pattern": "school[-_ ]?report", + "weight": 24, + "name": "school report filename" + }, + { + "pattern": "(end[-_ ]?of[-_ ]?(year|term)|summer|autumn)[-_ ]?report", + "weight": 20, + "name": "end of year/term report filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "report card|school report", + "weight": 14, + "name": "report card in PDF title" + } + ], + "negatives": [ + { + "text": "cumulative gpa", + "weight": 24, + "name": "transcript language" + }, + { + "text": "office of the registrar", + "weight": 22, + "name": "transcript issuer" + }, + { + "text": "credits attempted", + "weight": 20, + "name": "transcript language" + }, + { + "text": "official transcript", + "weight": 20, + "name": "transcript heading" + }, + { + "text": "grade point average", + "weight": 16, + "name": "transcript language" + }, + { + "text": "show your work", + "weight": 14, + "name": "worksheet language" + }, + { + "text": "credit hours", + "weight": 12, + "name": "transcript language" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 5 + }, + { + "signal": "number_table", + "weight": 4 + } + ] + }, + { + "id": "registration-form", + "name": "Registration form", + "emit": true, + "phrases": [ + { + "text": "enrollment form", + "weight": 32, + "where": "title" + }, + { + "text": "enrolment form", + "weight": 32, + "where": "title" + }, + { + "text": "new student enrollment", + "weight": 28, + "where": "title" + }, + { + "text": "admission form", + "weight": 26, + "where": "title" + }, + { + "text": "pupil details", + "weight": 26 + }, + { + "text": "admission application", + "weight": 24 + }, + { + "text": "student registration", + "weight": 22 + }, + { + "text": "previous school", + "weight": 22 + }, + { + "text": "home language survey", + "weight": 22 + }, + { + "text": "proof of residency", + "weight": 18 + }, + { + "text": "parent/carer", + "weight": 16 + }, + { + "text": "emergency contacts", + "weight": 14 + }, + { + "text": "year group", + "weight": 14 + }, + { + "text": "parent/guardian", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "entering grade\\s*:?\\s*(?:k|pre-k|[1-9]|1[0-2])\\b", + "weight": 16, + "name": "US entering grade field" + }, + { + "pattern": "previous school (?:attended|or nursery)", + "weight": 14, + "name": "previous school field" + }, + { + "pattern": "year of entry\\s*:?\\s*20\\d{2}", + "weight": 14, + "name": "year of entry field" + }, + { + "pattern": "parent\\s*/\\s*(?:guardian|carer)", + "weight": 12, + "name": "parent/guardian slash field" + } + ], + "filenames": [ + { + "pattern": "enrol{1,2}ment", + "weight": 26, + "name": "enrollment filename" + }, + { + "pattern": "admission[-_ ]?(form|application|pack)", + "weight": 24, + "name": "admission form filename" + }, + { + "pattern": "registration[-_ ]?(form|packet)", + "weight": 18, + "name": "registration form filename" + }, + { + "pattern": "new[-_ ]?student", + "weight": 16, + "name": "new student filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "enrol|admission|registration", + "weight": 12, + "name": "enrollment terms in PDF title" + } + ], + "negatives": [ + { + "text": "omb no.", + "weight": 22, + "name": "government form field" + }, + { + "text": "paperwork reduction act", + "weight": 20, + "name": "government form language" + }, + { + "text": "position applied for", + "weight": 16, + "name": "job application language" + }, + { + "text": "cumulative gpa", + "weight": 14, + "name": "transcript language" + }, + { + "text": "official transcript", + "weight": 14, + "name": "transcript heading" + }, + { + "text": "course syllabus", + "weight": 12, + "name": "course material heading" + }, + { + "text": "risks and benefits", + "weight": 12, + "name": "consent form language" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 8 + }, + { + "signal": "address_block", + "weight": 4 + } + ] + }, + { + "id": "lesson-plan", + "name": "Lesson plan", + "emit": true, + "phrases": [ + { + "text": "lesson plan", + "weight": 34, + "where": "title" + }, + { + "text": "starter activity", + "weight": 30 + }, + { + "text": "plenary", + "weight": 26 + }, + { + "text": "assessment for learning", + "weight": 26 + }, + { + "text": "key vocabulary", + "weight": 24 + }, + { + "text": "success criteria", + "weight": 24 + }, + { + "text": "curriculum links", + "weight": 24 + }, + { + "text": "differentiation", + "weight": 22 + }, + { + "text": "lesson objective", + "weight": 22 + }, + { + "text": "exit ticket", + "weight": 20 + }, + { + "text": "main activity", + "weight": 16 + }, + { + "text": "guided practice", + "weight": 16 + }, + { + "text": "learning objectives", + "weight": 14 + } + ], + "regexes": [ + { + "pattern": "lesson\\s+\\d{1,2}\\s+of\\s+\\d{1,2}", + "weight": 14, + "name": "lesson X of Y" + }, + { + "pattern": "\\b(?:ks[1-4]|key stage [1-4])\\b", + "weight": 14, + "name": "UK key stage" + }, + { + "pattern": "\\bngss\\b|common core (?:state )?standards", + "weight": 14, + "name": "US standards alignment" + }, + { + "pattern": "\\(\\s?\\d{1,2}\\s?min(?:s|utes)?\\s?\\)", + "weight": 12, + "name": "timed activity segments" + } + ], + "filenames": [ + { + "pattern": "lesson[-_ ]?plan", + "weight": 30, + "name": "lesson plan filename" + }, + { + "pattern": "scheme[-_ ]?of[-_ ]?work", + "weight": 20, + "name": "scheme of work filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "lesson plan", + "weight": 14, + "name": "lesson plan in PDF title" + } + ], + "negatives": [ + { + "text": "course syllabus", + "weight": 22, + "name": "course material heading" + }, + { + "text": "required textbook", + "weight": 18, + "name": "course material language" + }, + { + "text": "any other business", + "weight": 16, + "name": "meeting agenda language" + }, + { + "text": "grading policy", + "weight": 16, + "name": "course material language" + }, + { + "text": "apologies for absence", + "weight": 14, + "name": "meeting agenda language" + }, + { + "text": "office hours", + "weight": 14, + "name": "course material language" + }, + { + "text": "show your work", + "weight": 12, + "name": "worksheet language" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 5 + }, + { + "signal": "bullet_heavy", + "weight": 4 + }, + { + "signal": "form_like", + "weight": 3 + } + ] + }, + { + "id": "lease-agreement", + "name": "Lease agreement", + "emit": true, + "phrases": [ + { + "text": "residential lease agreement", + "weight": 34, + "where": "title" + }, + { + "text": "assured shorthold tenancy", + "weight": 32 + }, + { + "text": "tenancy agreement", + "weight": 28, + "where": "title" + }, + { + "text": "lease agreement", + "weight": 20, + "where": "title" + }, + { + "text": "security deposit", + "weight": 20 + }, + { + "text": "deposit protection", + "weight": 20 + }, + { + "text": "holding deposit", + "weight": 18 + }, + { + "text": "landlord and tenant", + "weight": 16 + }, + { + "text": "monthly rent", + "weight": 16 + }, + { + "text": "quiet enjoyment", + "weight": 14 + }, + { + "text": "tenant shall", + "weight": 12 + }, + { + "text": "lessee", + "weight": 12 + }, + { + "text": "the premises", + "weight": 6 + }, + { + "text": "sublet", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "term of (?:the |this )?(?:lease|tenancy)", + "weight": 18, + "name": "term-of-lease clause" + }, + { + "pattern": "[$£€]\\s?\\d[\\d,]{2,9}(?:\\.\\d{2})?\\s*(?:per calendar month|per month|pcm|/month|/mo)", + "weight": 16, + "name": "rent amount per month" + }, + { + "pattern": "deposit protection service|tenancy deposit scheme|mydeposits", + "weight": 16, + "name": "UK deposit scheme" + } + ], + "filenames": [ + { + "pattern": "rental[-_ ]?agreement", + "weight": 28, + "name": "rental agreement filename" + }, + { + "pattern": "tenancy", + "weight": 26, + "name": "tenancy filename" + }, + { + "pattern": "(?:^|[^a-z])lease", + "weight": 24, + "name": "lease filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "lease|tenancy", + "weight": 12, + "name": "lease in doc title" + } + ], + "negatives": [ + { + "text": "closing disclosure", + "weight": 20, + "name": "mortgage closing doc" + }, + { + "text": "loan estimate", + "weight": 18, + "name": "mortgage loan estimate" + }, + { + "text": "borrower", + "weight": 14, + "name": "mortgage borrower term" + }, + { + "text": "service charge demand", + "weight": 14, + "name": "hoa service charge demand" + }, + { + "text": "grantor", + "weight": 12, + "name": "deed grantor term" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 6 + } + ] + }, + { + "id": "mortgage-document", + "name": "Mortgage document", + "emit": true, + "phrases": [ + { + "text": "closing disclosure", + "weight": 34, + "where": "title" + }, + { + "text": "loan estimate", + "weight": 32, + "where": "title" + }, + { + "text": "mortgage offer", + "weight": 28, + "where": "title" + }, + { + "text": "deed of trust", + "weight": 26 + }, + { + "text": "this security instrument", + "weight": 26 + }, + { + "text": "your home may be repossessed", + "weight": 26 + }, + { + "text": "mortgage deed", + "weight": 22 + }, + { + "text": "early repayment charge", + "weight": 20 + }, + { + "text": "escrow account", + "weight": 18 + }, + { + "text": "standard variable rate", + "weight": 18 + }, + { + "text": "promissory note", + "weight": 16 + }, + { + "text": "loan-to-value", + "weight": 16 + }, + { + "text": "amortization schedule", + "weight": 14 + } + ], + "regexes": [ + { + "pattern": "loan (?:number|no\\.?|id|#)\\s*#?\\s*:?\\s*\\d{6,12}", + "weight": 14, + "name": "loan number" + }, + { + "pattern": "interest rate\\s*(?:of|:)?\\s*\\d{1,2}\\.\\d{1,3}\\s?%", + "weight": 12, + "name": "interest rate figure" + }, + { + "pattern": "nmls\\s*(?:id)?\\s*#?\\s*:?\\s*\\d{4,8}", + "weight": 15, + "name": "NMLS identifier" + }, + { + "pattern": "principal (?:and|&) interest", + "weight": 18, + "name": "principal-and-interest" + } + ], + "filenames": [ + { + "pattern": "closing[-_ ]?disclosure", + "weight": 30, + "name": "closing disclosure filename" + }, + { + "pattern": "loan[-_ ]?estimate", + "weight": 28, + "name": "loan estimate filename" + }, + { + "pattern": "mortgage", + "weight": 26, + "name": "mortgage filename" + }, + { + "pattern": "deed[-_ ]?of[-_ ]?trust", + "weight": 26, + "name": "deed of trust filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "closing disclosure|loan estimate|mortgage", + "weight": 14, + "name": "mortgage doc title" + }, + { + "field": "creator", + "pattern": "encompass|calyx", + "weight": 10, + "name": "loan origination software" + } + ], + "negatives": [ + { + "text": "minimum payment due", + "weight": 16, + "name": "credit card statement" + }, + { + "text": "monthly rent", + "weight": 14, + "name": "lease rent term" + }, + { + "text": "security deposit", + "weight": 12, + "name": "lease deposit term" + }, + { + "text": "statement period", + "weight": 12, + "name": "bank statement" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 4 + }, + { + "signal": "number_table", + "weight": 4 + } + ] + }, + { + "id": "deed", + "name": "Deed", + "emit": true, + "phrases": [ + { + "text": "warranty deed", + "weight": 34, + "where": "title" + }, + { + "text": "quitclaim deed", + "weight": 34, + "where": "title" + }, + { + "text": "quit claim deed", + "weight": 32, + "where": "title" + }, + { + "text": "grant deed", + "weight": 30, + "where": "title" + }, + { + "text": "hm land registry", + "weight": 28 + }, + { + "text": "grants and conveys", + "weight": 22 + }, + { + "text": "bargain, sell and convey", + "weight": 22 + }, + { + "text": "register of title", + "weight": 22 + }, + { + "text": "title absolute", + "weight": 22 + }, + { + "text": "in fee simple", + "weight": 20 + }, + { + "text": "grantee", + "weight": 18 + }, + { + "text": "county recorder", + "weight": 18 + }, + { + "text": "grantor", + "weight": 16 + }, + { + "text": "legal description", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "title number\\s*:?\\s*[a-z]{1,3}\\d{4,7}", + "weight": 18, + "name": "UK title number" + }, + { + "pattern": "book\\s+\\d{1,6},?\\s+page\\s+\\d{1,6}", + "weight": 16, + "name": "recorder book/page" + }, + { + "pattern": "(?:apn|parcel (?:id|no\\.?|number))\\s*[:#]?\\s*\\d[\\d-]{4,14}", + "weight": 14, + "name": "assessor parcel number" + } + ], + "filenames": [ + { + "pattern": "quitclaim|warranty[-_ ]?deed", + "weight": 28, + "name": "deed type filename" + }, + { + "pattern": "title[-_ ]?deed", + "weight": 26, + "name": "title deed filename" + }, + { + "pattern": "land[-_ ]?registry|official[-_ ]?copy", + "weight": 22, + "name": "land registry filename" + }, + { + "pattern": "(?:^|[^a-z])deed", + "weight": 20, + "name": "deed filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "deed|land registry", + "weight": 10, + "name": "deed in doc title" + } + ], + "negatives": [ + { + "text": "deed of trust", + "weight": 20, + "name": "mortgage instrument" + }, + { + "text": "last will and testament", + "weight": 18, + "name": "will document" + }, + { + "text": "promissory note", + "weight": 14, + "name": "loan note" + }, + { + "text": "loan estimate", + "weight": 14, + "name": "mortgage doc" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 6 + }, + { + "signal": "short_doc", + "weight": 5 + } + ] + }, + { + "id": "property-listing", + "name": "Property listing", + "emit": true, + "phrases": [ + { + "text": "offers in excess of", + "weight": 26 + }, + { + "text": "guide price", + "weight": 24 + }, + { + "text": "council tax band", + "weight": 24 + }, + { + "text": "asking price", + "weight": 22 + }, + { + "text": "strictly by appointment", + "weight": 20 + }, + { + "text": "epc rating", + "weight": 20 + }, + { + "text": "schedule a showing", + "weight": 20 + }, + { + "text": "open house", + "weight": 16 + }, + { + "text": "estate agent", + "weight": 14 + }, + { + "text": "fitted kitchen", + "weight": 14 + }, + { + "text": "realtor", + "weight": 12 + }, + { + "text": "off-street parking", + "weight": 10 + }, + { + "text": "equal housing opportunity", + "weight": 8 + }, + { + "text": "sq ft", + "weight": 5 + } + ], + "regexes": [ + { + "pattern": "mls\\s*(?:#|no\\.?|number)?\\s*:?\\s*[a-z]?\\d{6,10}", + "weight": 18, + "name": "MLS number" + }, + { + "pattern": "\\d\\s*(?:bed(?:room)?s?|bd)\\s*[|,•·/]?\\s*\\d(?:\\.\\d)?\\s*(?:bath(?:room)?s?|ba)\\b", + "weight": 16, + "name": "beds/baths pattern" + }, + { + "pattern": "[\\d,]{3,6}\\s*(?:sq\\.?\\s?ft|sqft|square feet)", + "weight": 10, + "name": "square footage" + } + ], + "filenames": [ + { + "pattern": "rightmove|zoopla|zillow|redfin", + "weight": 26, + "name": "listing portal filename" + }, + { + "pattern": "listing", + "weight": 20, + "name": "listing filename" + }, + { + "pattern": "property[-_ ]?details|particulars", + "weight": 20, + "name": "property details filename" + } + ], + "metadata": [ + { + "field": "any", + "pattern": "rightmove|zoopla|zillow|redfin|matterport", + "weight": 14, + "name": "listing portal metadata" + }, + { + "field": "title", + "pattern": "for sale|property details", + "weight": 10, + "name": "for-sale title" + } + ], + "negatives": [ + { + "text": "at the time of inspection", + "weight": 16, + "name": "inspection report" + }, + { + "text": "appraisal report", + "weight": 14, + "name": "appraisal doc" + }, + { + "text": "security deposit", + "weight": 10, + "name": "lease term" + } + ], + "structural": [ + { + "signal": "address_block", + "weight": 5 + }, + { + "signal": "url_heavy", + "weight": 4 + }, + { + "signal": "short_doc", + "weight": 3 + } + ] + }, + { + "id": "inspection-report", + "name": "Inspection report", + "emit": true, + "phrases": [ + { + "text": "home inspection report", + "weight": 36, + "where": "title" + }, + { + "text": "property inspection report", + "weight": 32, + "where": "title" + }, + { + "text": "homebuyer report", + "weight": 30, + "where": "title" + }, + { + "text": "building survey", + "weight": 22, + "where": "title" + }, + { + "text": "chartered surveyor", + "weight": 20 + }, + { + "text": "condition rating", + "weight": 18 + }, + { + "text": "recommend evaluation", + "weight": 18 + }, + { + "text": "gfci", + "weight": 16 + }, + { + "text": "roof covering", + "weight": 14 + }, + { + "text": "crawl space", + "weight": 12 + }, + { + "text": "downspouts", + "weight": 10 + }, + { + "text": "deficiencies", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "at the time of (?:the )?inspection", + "weight": 24, + "name": "time-of-inspection phrasing" + }, + { + "pattern": "condition rating[:\\s]+[123]\\b", + "weight": 14, + "name": "RICS condition rating" + }, + { + "pattern": "internachi|ashi (?:certified|member)", + "weight": 14, + "name": "inspector association" + } + ], + "filenames": [ + { + "pattern": "home[-_ ]?inspection", + "weight": 28, + "name": "home inspection filename" + }, + { + "pattern": "homebuyer|building[-_ ]?survey", + "weight": 24, + "name": "survey report filename" + }, + { + "pattern": "inspection", + "weight": 22, + "name": "inspection filename" + } + ], + "metadata": [ + { + "field": "creator", + "pattern": "spectora|homegauge|home inspector pro", + "weight": 12, + "name": "inspection software" + } + ], + "negatives": [ + { + "text": "safety data sheet", + "weight": 22, + "name": "SDS document" + }, + { + "text": "reference range", + "weight": 14, + "name": "lab report" + }, + { + "text": "asking price", + "weight": 12, + "name": "property listing" + }, + { + "text": "certificate of insurance", + "weight": 12, + "name": "insurance cert" + } + ], + "structural": [ + { + "signal": "long_doc", + "weight": 5 + }, + { + "signal": "toc", + "weight": 4 + }, + { + "signal": "bullet_heavy", + "weight": 3 + } + ] + }, + { + "id": "hoa-document", + "name": "HOA document", + "emit": true, + "phrases": [ + { + "text": "homeowners association", + "weight": 26 + }, + { + "text": "homeowners' association", + "weight": 26 + }, + { + "text": "hoa dues", + "weight": 24 + }, + { + "text": "service charge demand", + "weight": 24 + }, + { + "text": "cc&rs", + "weight": 24 + }, + { + "text": "architectural review", + "weight": 22 + }, + { + "text": "residents association", + "weight": 20 + }, + { + "text": "community association", + "weight": 18 + }, + { + "text": "special assessment", + "weight": 18 + }, + { + "text": "annual assessment", + "weight": 16 + }, + { + "text": "ground rent", + "weight": 16 + }, + { + "text": "managing agent", + "weight": 14 + }, + { + "text": "reserve fund", + "weight": 12 + }, + { + "text": "common areas", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "covenants,? conditions,? (?:and|&) restrictions", + "weight": 22, + "name": "CC&R spelled out" + }, + { + "pattern": "(?:dues|assessment)s?\\s+(?:of|are|in the amount of)\\s+[$£]", + "weight": 12, + "name": "dues amount" + }, + { + "pattern": "\\bhoa\\b", + "weight": 12, + "name": "HOA acronym" + } + ], + "filenames": [ + { + "pattern": "hoa[-_. ]|[-_. ]hoa", + "weight": 24, + "name": "hoa filename" + }, + { + "pattern": "homeowners", + "weight": 20, + "name": "homeowners filename" + }, + { + "pattern": "service[-_ ]?charge", + "weight": 18, + "name": "service charge filename" + }, + { + "pattern": "assessment[-_ ]?notice", + "weight": 18, + "name": "assessment notice filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "tenancy agreement", + "weight": 16, + "name": "lease agreement" + }, + { + "text": "taxable value", + "weight": 14, + "name": "tax assessment notice" + }, + { + "text": "policy number", + "weight": 12, + "name": "insurance doc" + }, + { + "text": "invoice number", + "weight": 10, + "name": "invoice" + } + ], + "structural": [ + { + "signal": "address_block", + "weight": 4 + }, + { + "signal": "short_doc", + "weight": 3 + } + ] + }, + { + "id": "application-form", + "name": "Application form", + "emit": true, + "phrases": [ + { + "text": "omb no.", + "weight": 26, + "where": "first" + }, + { + "text": "omb control number", + "weight": 26, + "where": "first" + }, + { + "text": "paperwork reduction act", + "weight": 30 + }, + { + "text": "for official use only", + "weight": 16 + }, + { + "text": "for office use only", + "weight": 20 + }, + { + "text": "do not write below this line", + "weight": 24 + }, + { + "text": "please complete in black ink", + "weight": 22 + }, + { + "text": "block capitals", + "weight": 12 + }, + { + "text": "print or type", + "weight": 14 + }, + { + "text": "form approved", + "weight": 14, + "where": "first" + }, + { + "text": "check the appropriate box", + "weight": 12 + }, + { + "text": "continue on a separate sheet", + "weight": 12 + }, + { + "text": "offence to make a false statement", + "weight": 18 + }, + { + "text": "national insurance number", + "weight": 8 + }, + { + "text": "town and country planning act", + "weight": 30 + }, + { + "text": "planning application", + "weight": 28, + "where": "title" + }, + { + "text": "planning permission", + "weight": 26 + }, + { + "text": "board of zoning appeals", + "weight": 26 + }, + { + "text": "zoning variance", + "weight": 26 + }, + { + "text": "proposed development", + "weight": 22 + }, + { + "text": "building permit application", + "weight": 24 + }, + { + "text": "case officer", + "weight": 20 + }, + { + "text": "planning officer", + "weight": 20 + }, + { + "text": "erection of", + "weight": 14 + }, + { + "text": "site plan", + "weight": 12 + }, + { + "text": "conservation area", + "weight": 10 + }, + { + "text": "setback", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "omb\\s+(no\\.?|control\\s+no\\.?|control\\s+number)\\s*:?\\s*\\d{4}-\\d{4}", + "weight": 24, + "name": "OMB control number", + "where": "first" + }, + { + "pattern": "\\bform\\s+[a-z]{1,4}[- ]?\\d{1,4}[a-z]?\\b", + "weight": 14, + "name": "Agency form number" + }, + { + "pattern": "\\b(department|bureau|ministry|office) of (health|state|motor vehicles|public safety|homeland security|veterans affairs|vital statistics|revenue|labor|labour|transportation|transport|human services|social services|home affairs|the interior|consular affairs)\\b", + "weight": 14, + "name": "Government agency letterhead", + "where": "first" + }, + { + "pattern": "\\(rev\\.?\\s*\\d{1,2}[\\/. -]\\d{2,4}\\)", + "weight": 12, + "name": "Form revision marker", + "where": "first" + }, + { + "pattern": "application\\s+(no\\.?|number|ref(erence)?)\\s*:?\\s*\\d{2}\\/\\d{4,5}\\/[a-z]{2,6}", + "weight": 24, + "name": "UK planning reference" + }, + { + "pattern": "(grant|refuse|approve)\\s+(full\\s+)?planning\\s+permission", + "weight": 22, + "name": "Planning decision wording" + }, + { + "pattern": "parcel\\s+(id|number|no\\.?)", + "weight": 14, + "name": "Parcel identifier" + }, + { + "pattern": "\\bzon(ed|ing)\\s+(district|classification)\\b", + "weight": 14, + "name": "Zoning district" + } + ], + "filenames": [ + { + "pattern": "(application|registration)[-_ ]?form", + "weight": 18, + "name": "Application form filename" + }, + { + "pattern": "\\bform[-_ ]?[a-z]{0,3}-?\\d{1,4}\\b", + "weight": 16, + "name": "Form number filename" + }, + { + "pattern": "planning[-_ ]?(application|permission|statement|decision)", + "weight": 26, + "name": "Planning filename" + }, + { + "pattern": "zoning|variance", + "weight": 20, + "name": "Zoning filename" + }, + { + "pattern": "site[-_ ]?plan", + "weight": 16, + "name": "Site plan filename" + } + ], + "metadata": [ + { + "field": "producer", + "pattern": "livecycle", + "weight": 10, + "name": "LiveCycle form producer" + }, + { + "field": "title", + "pattern": "\\bform\\b", + "weight": 8, + "name": "Form in title" + }, + { + "field": "title", + "pattern": "planning (application|permission)|zoning", + "weight": 12, + "name": "Planning in PDF title" + } + ], + "negatives": [ + { + "text": "internal revenue service", + "weight": 30, + "name": "IRS form is a tax form" + }, + { + "text": "form 1040", + "weight": 20, + "name": "US tax return form" + }, + { + "text": "self assessment", + "weight": 16, + "name": "HMRC tax form" + }, + { + "text": "citizenship and immigration services", + "weight": 16, + "name": "USCIS belongs to visa-immigration" + }, + { + "text": "universal credit", + "weight": 14, + "name": "Benefits paperwork" + }, + { + "pattern": "\\bform w-[249]\\b|withholding (allowance|certificate)|employee's withholding", + "flags": "gi", + "weight": 22, + "name": "Withholding / W-form (tax-form)" + }, + { + "text": "premises licence", + "weight": 20, + "name": "Licensing, not planning" + }, + { + "text": "this permit must be displayed", + "weight": 20, + "name": "Issued permit certificate" + }, + { + "text": "licensing act 2003", + "weight": 18, + "name": "Alcohol licensing" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 12 + }, + { + "signal": "address_block", + "weight": 3 + } + ] + }, + { + "id": "government-notice", + "name": "Government notice", + "emit": true, + "phrases": [ + { + "text": "quote this reference", + "weight": 18 + }, + { + "text": "an executive agency of", + "weight": 26 + }, + { + "text": "on behalf of the secretary of state", + "weight": 28 + }, + { + "text": "penalty for private use", + "weight": 26 + }, + { + "text": "your national insurance number", + "weight": 14 + }, + { + "text": "our reference", + "weight": 10, + "where": "first" + }, + { + "text": "your reference", + "weight": 8, + "where": "first" + }, + { + "text": "electoral register", + "weight": 20 + }, + { + "text": "gov.uk", + "weight": 10 + }, + { + "text": "crown copyright", + "weight": 12 + }, + { + "text": "borough council", + "weight": 12 + }, + { + "text": "county council", + "weight": 12 + }, + { + "text": "official business", + "weight": 8 + }, + { + "text": "yours faithfully", + "weight": 5 + }, + { + "text": "freedom of information request", + "weight": 34, + "where": "title" + }, + { + "text": "your request for information", + "weight": 30 + }, + { + "text": "freedom of information act 2000", + "weight": 28 + }, + { + "text": "records responsive to your request", + "weight": 28 + }, + { + "text": "duty to confirm or deny", + "weight": 26 + }, + { + "text": "information commissioner's office", + "weight": 24 + }, + { + "text": "public records request", + "weight": 22 + }, + { + "text": "right to an internal review", + "weight": 22 + }, + { + "text": "this exemption is subject to a public interest test", + "weight": 24 + }, + { + "text": "fee waiver", + "weight": 8 + }, + { + "text": "environmental information regulations", + "weight": 20 + }, + { + "text": "we have decided to withhold", + "weight": 18 + }, + { + "text": "consultation paper", + "weight": 32, + "where": "title" + }, + { + "text": "we welcome your views", + "weight": 30 + }, + { + "text": "this consultation closes", + "weight": 30 + }, + { + "text": "consultation document", + "weight": 26, + "where": "title" + }, + { + "text": "call for evidence", + "weight": 26 + }, + { + "text": "responses to this consultation", + "weight": 24 + }, + { + "text": "green paper", + "weight": 24, + "where": "title" + }, + { + "text": "consultation period", + "weight": 20 + }, + { + "text": "have your say", + "weight": 20 + }, + { + "text": "how to respond", + "weight": 16 + }, + { + "text": "request for public comment", + "weight": 18 + }, + { + "text": "consultation questions", + "weight": 18 + }, + { + "text": "respondents", + "weight": 7 + }, + { + "text": "supersedes circular", + "weight": 32 + }, + { + "text": "this circular provides guidance", + "weight": 28 + }, + { + "text": "dear chief executive", + "weight": 28, + "where": "first" + }, + { + "text": "guidance to local authorities", + "weight": 26 + }, + { + "text": "chief executives of local authorities", + "weight": 24 + }, + { + "text": "cancels and replaces", + "weight": 22 + }, + { + "text": "this circular", + "weight": 20 + }, + { + "text": "heads of executive departments and agencies", + "weight": 26 + }, + { + "text": "all heads of department", + "weight": 20 + }, + { + "text": "this directive", + "weight": 18 + }, + { + "text": "action required by", + "weight": 16 + }, + { + "text": "circular", + "weight": 14, + "where": "title" + }, + { + "text": "with immediate effect", + "weight": 10 + }, + { + "text": "contract award notice", + "weight": 36, + "where": "title" + }, + { + "text": "notice of award", + "weight": 30, + "where": "title" + }, + { + "text": "notice of intent to award", + "weight": 28 + }, + { + "text": "standstill period", + "weight": 28 + }, + { + "text": "unsuccessful tenderers", + "weight": 24 + }, + { + "text": "successful tenderer", + "weight": 22 + }, + { + "text": "successful bidder", + "weight": 20 + }, + { + "text": "has been awarded to", + "weight": 20 + }, + { + "text": "award of contract", + "weight": 16 + }, + { + "text": "total value of the contract", + "weight": 14 + }, + { + "text": "debriefing", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "(our|your)\\s+ref(erence)?\\s*[:.]\\s*[a-z0-9][a-z0-9\\/\\-]{3,}", + "weight": 12, + "name": "Reference number field" + }, + { + "pattern": "5\\s*u\\.?s\\.?c\\.?\\s*(§|section)?\\s*552", + "weight": 24, + "name": "5 USC 552 citation" + }, + { + "pattern": "\\bfoia?\\b", + "weight": 16, + "name": "FOI/FOIA abbreviation" + }, + { + "pattern": "section\\s+\\d{1,2}(\\(\\d\\))?(\\([a-z]\\))?\\s+of the freedom of information act", + "weight": 24, + "name": "FOI exemption citation" + }, + { + "pattern": "exemption\\s+(b\\s*)?\\(?[b(]\\s*\\d\\)?", + "weight": 14, + "name": "FOIA (b) exemption" + }, + { + "pattern": "(request|reference)\\s+(number|no\\.?)\\s*:?\\s*(foi|foia|epa|doj)[-\\s]?[a-z0-9-]{2,}", + "weight": 20, + "name": "FOI request reference" + }, + { + "pattern": "consultation\\s+(closes|ends|period\\s+closes)\\s+(at|on)", + "weight": 22, + "name": "Consultation close date" + }, + { + "pattern": "comments\\s+must\\s+be\\s+(received|submitted)\\s+(on\\s+or\\s+before|by|no\\s+later\\s+than)", + "weight": 18, + "name": "Comment deadline" + }, + { + "pattern": "question\\s+\\d{1,2}\\s*[:.]\\s*(do|should|to\\s+what|how|what)", + "weight": 16, + "name": "Numbered consultation question" + }, + { + "pattern": "circular\\s+(no\\.?\\s*)?\\d{1,3}\\/\\d{2,4}", + "weight": 26, + "name": "Circular number" + }, + { + "pattern": "\\bomb\\s+circular\\s+(no\\.?\\s*)?a-\\d{2,3}\\b", + "weight": 26, + "name": "OMB circular" + }, + { + "pattern": "supersedes\\s+(circular|directive|memorandum|guidance)", + "weight": 22, + "name": "Supersession line" + }, + { + "pattern": "\\bm-\\d{2}-\\d{2}\\b", + "weight": 16, + "name": "OMB memorandum number" + }, + { + "pattern": "ocds-[a-z0-9]{6}-[a-z0-9]{1,12}", + "weight": 20, + "name": "OCDS notice id" + }, + { + "pattern": "unsuccessful (?:tenderers|bidders|proposers|offerors)", + "weight": 16, + "name": "Unsuccessful parties notice" + }, + { + "pattern": "number of (?:tenders|bids|proposals) received\\s*:?\\s*\\d{1,4}", + "weight": 14, + "name": "Tender count line" + } + ], + "filenames": [ + { + "pattern": "(gov|council|federal|ministry|dwp|home[-_ ]?office)[-_ ]?letter", + "weight": 18, + "name": "Government letter filename" + }, + { + "pattern": "electoral", + "weight": 15, + "name": "Electoral correspondence filename" + }, + { + "pattern": "foia?[-_ ]?(request|response|reply)", + "weight": 28, + "name": "FOI request/response filename" + }, + { + "pattern": "public[-_ ]?records[-_ ]?request", + "weight": 24, + "name": "Public records request filename" + }, + { + "pattern": "(^|[^a-z])foia?([^a-z]|$)", + "weight": 18, + "name": "FOI in filename" + }, + { + "pattern": "consultation", + "weight": 26, + "name": "Consultation filename" + }, + { + "pattern": "green[-_ ]?paper", + "weight": 22, + "name": "Green paper filename" + }, + { + "pattern": "call[-_ ]?for[-_ ]?evidence", + "weight": 20, + "name": "Call for evidence filename" + }, + { + "pattern": "circular", + "weight": 26, + "name": "Circular filename" + }, + { + "pattern": "directive", + "weight": 18, + "name": "Directive filename" + }, + { + "pattern": "award[-_ ]?notice|notice[-_ ]?of[-_ ]?award|intent[-_ ]?to[-_ ]?award", + "weight": 28, + "name": "Award notice filename" + }, + { + "pattern": "contract[-_ ]?award", + "weight": 26, + "name": "Contract award filename" + }, + { + "pattern": "\\bnoia\\b", + "weight": 15, + "name": "NOIA filename" + } + ], + "metadata": [ + { + "field": "any", + "pattern": "gov\\.uk", + "weight": 10, + "name": "gov.uk in metadata" + }, + { + "field": "title", + "pattern": "freedom of information|foia", + "weight": 14, + "name": "FOI in PDF title" + }, + { + "field": "title", + "pattern": "consultation|green paper", + "weight": 14, + "name": "Consultation in PDF title" + }, + { + "field": "title", + "pattern": "circular|directive", + "weight": 14, + "name": "Circular in PDF title" + }, + { + "field": "title", + "pattern": "award notice|notice of (?:intent to )?award", + "weight": 16, + "name": "Award notice in PDF title" + } + ], + "negatives": [ + { + "text": "hm revenue", + "weight": 18, + "name": "HMRC letters are tax notices" + }, + { + "text": "tax return", + "weight": 14, + "name": "Tax correspondence" + }, + { + "text": "universal credit", + "weight": 16, + "name": "Benefits statement territory" + }, + { + "text": "council tax", + "weight": 12, + "name": "Council tax bill" + }, + { + "text": "leave to remain", + "weight": 16, + "name": "Immigration decision letter" + }, + { + "text": "subject access request", + "weight": 20, + "name": "GDPR SAR, not FOI" + }, + { + "text": "planning permission", + "weight": 12, + "name": "Planning application territory" + }, + { + "text": "final rule", + "weight": 18, + "name": "Adopted regulation" + }, + { + "text": "white paper", + "weight": 16, + "name": "White paper report" + }, + { + "text": "minutes of the meeting", + "weight": 14, + "name": "Meeting minutes" + }, + { + "text": "for immediate release", + "weight": 16, + "name": "Press release header" + }, + { + "text": "apologies for absence", + "weight": 12, + "name": "Meeting minutes" + }, + { + "text": "in witness whereof", + "weight": 18, + "name": "Contract execution language" + }, + { + "text": "grant offer", + "weight": 14, + "name": "Grant award letter language" + }, + { + "text": "we are pleased to submit", + "weight": 14, + "name": "Bid submission language" + }, + { + "text": "budget period", + "weight": 16, + "name": "Grant NoA budget period" + }, + { + "text": "selected for funding", + "weight": 14, + "name": "Grant funding decision" + }, + { + "pattern": "\\bcfda\\b|assistance listing|2 cfr part 200", + "weight": 16, + "name": "Federal grant award markers" + } + ], + "structural": [ + { + "signal": "address_block", + "weight": 6 + }, + { + "signal": "signature_block", + "weight": 4 + }, + { + "signal": "short_doc", + "weight": 4 + }, + { + "signal": "bullet_heavy", + "weight": 3 + }, + { + "signal": "url_heavy", + "weight": 3 + } + ] + }, + { + "id": "visa-document", + "name": "Visa document", + "emit": true, + "phrases": [ + { + "text": "u.s. citizenship and immigration services", + "weight": 34, + "where": "first" + }, + { + "text": "uscis", + "weight": 26 + }, + { + "text": "uk visas and immigration", + "weight": 28 + }, + { + "text": "petition for a nonimmigrant worker", + "weight": 30 + }, + { + "text": "biometric residence permit", + "weight": 30 + }, + { + "text": "indefinite leave to remain", + "weight": 32 + }, + { + "text": "leave to remain", + "weight": 24 + }, + { + "text": "immigration status", + "weight": 18 + }, + { + "text": "visa application", + "weight": 20 + }, + { + "text": "home office reference", + "weight": 24, + "where": "first" + }, + { + "text": "certificate of sponsorship", + "weight": 24 + }, + { + "text": "alien registration number", + "weight": 28 + }, + { + "text": "i-94", + "weight": 24 + }, + { + "text": "schengen visa", + "weight": 26 + } + ], + "regexes": [ + { + "pattern": "\\b(eac|wac|lin|src|msc|ioe|nbc|ysc)[-\\s]?\\d{2}[-\\s]?\\d{3}[-\\s]?\\d{5}\\b", + "weight": 25, + "name": "USCIS receipt number" + }, + { + "pattern": "\\bform\\s+i-\\d{2,3}[a-z]?\\b", + "weight": 22, + "name": "USCIS I-series form" + }, + { + "pattern": "\\bds-(11|82|160|260)\\b", + "weight": 20, + "name": "State Dept DS form" + }, + { + "pattern": "\\b(h-1b|h-2a|l-1|f-1|j-1|o-1|e-2|tn)\\s+(visa|status|classification|petition)\\b", + "weight": 18, + "name": "Visa classification" + }, + { + "pattern": "\\ba#\\s*\\d{2,3}[- ]?\\d{3}[- ]?\\d{3}\\b", + "weight": 18, + "name": "Alien number" + } + ], + "filenames": [ + { + "pattern": "visa|uscis|immigration", + "weight": 24, + "name": "Visa/immigration filename" + }, + { + "pattern": "\\bi-?(20|94|129|130|485|539|765|797)", + "weight": 20, + "name": "I-series form filename" + }, + { + "pattern": "passport", + "weight": 18, + "name": "Passport filename" + }, + { + "pattern": "home[-_ ]?office", + "weight": 16, + "name": "Home Office filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "visa|uscis|immigration", + "weight": 14, + "name": "Immigration title" + } + ], + "negatives": [ + { + "text": "boarding pass", + "weight": 20, + "name": "Travel boarding pass" + }, + { + "text": "boarding time", + "weight": 16, + "name": "Travel boarding pass" + }, + { + "text": "itinerary", + "weight": 12, + "name": "Travel itinerary" + }, + { + "text": "e-ticket", + "weight": 14, + "name": "Travel e-ticket" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 3 + } + ] + }, + { + "id": "license", + "name": "License", + "emit": true, + "phrases": [ + { + "text": "licence number", + "weight": 14 + }, + { + "text": "license number", + "weight": 14 + }, + { + "text": "permit holder", + "weight": 22 + }, + { + "text": "is hereby licensed", + "weight": 26 + }, + { + "text": "is hereby authorised", + "weight": 14 + }, + { + "text": "is hereby authorized", + "weight": 14 + }, + { + "text": "this permit must be displayed", + "weight": 28 + }, + { + "text": "licensing authority", + "weight": 22 + }, + { + "text": "licensing act 2003", + "weight": 28 + }, + { + "text": "premises licence", + "weight": 26, + "where": "title" + }, + { + "text": "driving licence", + "weight": 22 + }, + { + "text": "driver's license", + "weight": 10 + }, + { + "text": "subject to the following conditions", + "weight": 12 + }, + { + "text": "is authorized to", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "(licence|license|permit)\\s+(no\\.?|number|#)\\s*:?\\s*[a-z0-9][a-z0-9\\/\\-]{2,}", + "weight": 16, + "name": "License/permit number" + } + ], + "filenames": [ + { + "pattern": "licen[cs]e|permit", + "weight": 20, + "name": "License/permit filename" + }, + { + "pattern": "dvla", + "weight": 20, + "name": "DVLA filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "licen[cs]e|permit", + "weight": 12, + "name": "License/permit title" + } + ], + "negatives": [ + { + "text": "end user license agreement", + "weight": 28, + "name": "Software EULA" + }, + { + "text": "license agreement", + "weight": 18, + "name": "Software/IP contract" + }, + { + "text": "licensor", + "weight": 16, + "name": "Licensing contract party" + }, + { + "text": "certificate of liability insurance", + "weight": 20, + "name": "Insurance certificate" + }, + { + "text": "has successfully completed", + "weight": 14, + "name": "Diploma/certificate wording" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 5 + }, + { + "signal": "signature_block", + "weight": 3 + } + ] + }, + { + "id": "public-notice", + "name": "Public notice", + "emit": true, + "phrases": [ + { + "text": "the london gazette", + "weight": 34, + "where": "first" + }, + { + "text": "official gazette", + "weight": 28 + }, + { + "text": "gazette notice", + "weight": 28 + }, + { + "text": "published by authority", + "weight": 26, + "where": "first" + }, + { + "text": "notice of public hearing", + "weight": 26, + "where": "title" + }, + { + "text": "public notice", + "weight": 22, + "where": "title" + }, + { + "text": "legal advertisement", + "weight": 22 + }, + { + "text": "notice is hereby given", + "weight": 16 + }, + { + "text": "all interested persons", + "weight": 14 + }, + { + "text": "affidavit of publication", + "weight": 20 + }, + { + "text": "will hold a public hearing", + "weight": 18 + }, + { + "text": "notice of intent", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "gazette\\s+(issue|no\\.?|number)\\s*:?\\s*\\d+", + "weight": 22, + "name": "Gazette issue number" + }, + { + "pattern": "notice\\s+(no\\.?|number|id)\\s*:?\\s*\\d{4,}", + "weight": 14, + "name": "Notice number" + }, + { + "pattern": "published\\s+in\\s+the\\s+[a-z ]{3,30}\\s+(gazette|herald|tribune|times|journal)", + "weight": 16, + "name": "Publication in named paper" + } + ], + "filenames": [ + { + "pattern": "gazette", + "weight": 26, + "name": "Gazette filename" + }, + { + "pattern": "public[-_ ]?notice", + "weight": 24, + "name": "Public notice filename" + }, + { + "pattern": "notice[-_ ]?of[-_ ]?hearing", + "weight": 20, + "name": "Notice of hearing filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "gazette|public notice", + "weight": 14, + "name": "Gazette/notice in PDF title" + } + ], + "negatives": [ + { + "text": "plaintiff", + "weight": 20, + "name": "Court filing party" + }, + { + "text": "summons", + "weight": 18, + "name": "Court process document" + }, + { + "text": "notice to quit", + "weight": 16, + "name": "Legal notice to tenant" + }, + { + "text": "case no", + "weight": 14, + "name": "Court case number" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 4 + } + ] + }, + { + "id": "analytics-report", + "name": "Analytics report", + "emit": true, + "phrases": [ + { + "text": "statistical bulletin", + "weight": 32, + "where": "title" + }, + { + "text": "statistical release", + "weight": 32, + "where": "title" + }, + { + "text": "office for national statistics", + "weight": 30 + }, + { + "text": "bureau of labor statistics", + "weight": 30 + }, + { + "text": "seasonally adjusted", + "weight": 26 + }, + { + "text": "census bureau", + "weight": 24 + }, + { + "text": "quarter on quarter", + "weight": 22 + }, + { + "text": "national statistics", + "weight": 18 + }, + { + "text": "next publication date", + "weight": 18 + }, + { + "text": "sampling error", + "weight": 14 + }, + { + "text": "year on year", + "weight": 12 + }, + { + "text": "data tables", + "weight": 10 + }, + { + "text": "margin of error", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "\\b(increased|decreased|rose|fell)\\s+(by\\s+)?\\d{1,2}(\\.\\d)?\\s?(per\\s?cent|percent|percentage\\s+points|%)", + "weight": 16, + "name": "Percent change wording" + }, + { + "pattern": "\\bq[1-4]\\s+20\\d{2}\\b", + "weight": 12, + "name": "Quarter reference" + }, + { + "pattern": "(three\\s+months|quarter)\\s+(to|ending)\\s+[a-z]+\\s+20\\d{2}", + "weight": 16, + "name": "Reference period" + } + ], + "filenames": [ + { + "pattern": "statistical[-_ ]?(release|bulletin)", + "weight": 28, + "name": "Statistical release filename" + }, + { + "pattern": "(^|[^a-z])(ons|bls|cpi|gdp)([^a-z]|$)", + "weight": 18, + "name": "Stats agency/series filename" + }, + { + "pattern": "labou?r[-_ ]?market|unemployment", + "weight": 16, + "name": "Labour market filename" + } + ], + "metadata": [ + { + "field": "any", + "pattern": "office for national statistics|census bureau|bureau of labor statistics", + "weight": 14, + "name": "Stats agency metadata" + } + ], + "negatives": [ + { + "text": "consolidated financial statements", + "weight": 24, + "name": "Company financial report" + }, + { + "text": "earnings per share", + "weight": 22, + "name": "Company results" + }, + { + "text": "balance sheet", + "weight": 18, + "name": "Financial statements" + }, + { + "text": "for immediate release", + "weight": 16, + "name": "Press release header" + }, + { + "text": "fiscal year ended", + "weight": 14, + "name": "Company reporting period" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 8 + } + ] + }, + { + "id": "grant-agreement", + "name": "Grant agreement", + "emit": true, + "phrases": [ + { + "text": "notice of award", + "weight": 34, + "where": "title" + }, + { + "text": "federal award identification number", + "weight": 32 + }, + { + "text": "grant offer letter", + "weight": 30, + "where": "title" + }, + { + "text": "period of performance", + "weight": 28 + }, + { + "text": "your application for funding has been successful", + "weight": 28 + }, + { + "text": "assistance listing number", + "weight": 26 + }, + { + "text": "terms and conditions of the grant", + "weight": 24 + }, + { + "text": "grant number", + "weight": 22 + }, + { + "text": "authorized organizational representative", + "weight": 22 + }, + { + "text": "award amount", + "weight": 20 + }, + { + "text": "grant agreement", + "weight": 14 + }, + { + "text": "match funding", + "weight": 12 + }, + { + "text": "drawdown", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "\\bcfda\\s+(no\\.?|number)?\\s*:?\\s*\\d{2}\\.\\d{3}\\b", + "weight": 24, + "name": "CFDA number" + }, + { + "pattern": "\\bfain\\b", + "weight": 18, + "name": "FAIN abbreviation" + }, + { + "pattern": "period\\s+of\\s+performance\\s*:?\\s*\\d{1,2}\\/\\d{1,2}\\/\\d{2,4}", + "weight": 20, + "name": "Period of performance dates" + }, + { + "pattern": "(award|grant)\\s+(no\\.?|number)\\s*:?\\s*[a-z0-9][a-z0-9\\/\\-]{4,}", + "weight": 16, + "name": "Award/grant number" + } + ], + "filenames": [ + { + "pattern": "notice[-_ ]?of[-_ ]?award", + "weight": 28, + "name": "Notice of award filename" + }, + { + "pattern": "grant[-_ ]?(award|offer|agreement|letter)", + "weight": 24, + "name": "Grant letter filename" + }, + { + "pattern": "(^|[^a-z])noa([^a-z]|$)", + "weight": 15, + "name": "NoA abbreviation filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "notice of award|grant (offer|award)", + "weight": 14, + "name": "Award in PDF title" + } + ], + "negatives": [ + { + "text": "scholarship", + "weight": 16, + "name": "Education scholarship award" + }, + { + "text": "purchase order", + "weight": 14, + "name": "Procurement document" + }, + { + "text": "invoice number", + "weight": 14, + "name": "Invoice" + }, + { + "text": "universal credit", + "weight": 14, + "name": "Benefits statement" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 5 + }, + { + "signal": "signature_block", + "weight": 3 + } + ] + }, + { + "id": "tender-document", + "name": "Tender document", + "emit": true, + "phrases": [ + { + "text": "invitation to tender", + "weight": 34, + "where": "title" + }, + { + "text": "request for proposal", + "weight": 28, + "where": "title" + }, + { + "text": "request for quotation", + "weight": 26, + "where": "title" + }, + { + "text": "instructions to tenderers", + "weight": 26 + }, + { + "text": "instructions to bidders", + "weight": 22 + }, + { + "text": "pre-qualification questionnaire", + "weight": 24 + }, + { + "text": "most economically advantageous tender", + "weight": 26 + }, + { + "text": "evaluation criteria", + "weight": 14 + }, + { + "text": "scope of requirements", + "weight": 16 + }, + { + "text": "submission deadline", + "weight": 12 + }, + { + "text": "clarification questions", + "weight": 12 + }, + { + "text": "tender documents", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "\\b(?:itt|rfp|rfq|rft)\\s*(?:no\\.?|ref(?:erence)?|#)?\\s*[:#-]?\\s*\\d{2,6}\\b", + "weight": 16, + "name": "ITT/RFP/RFQ reference number" + }, + { + "pattern": "deadline for (?:receipt|submission) of (?:tenders|bids|proposals|quotations)", + "weight": 20, + "name": "Tender deadline line" + }, + { + "pattern": "\\bfar\\s+(?:part\\s+\\d{1,2}|\\d{2}\\.\\d{3}(?:-\\d{1,2})?)\\b", + "weight": 18, + "name": "FAR clause/part reference" + }, + { + "pattern": "public contracts regulations 2015|\\bpcr\\s*2015\\b|find a tender|contracts finder", + "weight": 18, + "name": "UK procurement regime" + }, + { + "pattern": "solicitation\\s*(?:no\\.?|number|#)\\s*:?\\s*[a-z0-9][a-z0-9-]{2,18}", + "weight": 16, + "name": "Solicitation number" + }, + { + "pattern": "sealed (?:bids|proposals|tenders)", + "weight": 12, + "name": "Sealed submissions" + } + ], + "filenames": [ + { + "pattern": "invitation[-_ ]?to[-_ ]?tender|request[-_ ]?for[-_ ]?(?:proposal|quotation|tender)s?", + "weight": 28, + "name": "ITT/RFP/RFQ filename" + }, + { + "pattern": "\\b(?:itt|rfp|rfq|rft|pqq)\\b(?![-_ ]?(?:response|reply|submission|return))", + "weight": 22, + "name": "Solicitation acronym filename" + }, + { + "pattern": "tender(?![-_ ]?(?:response|return|submission))", + "weight": 16, + "name": "Tender filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "invitation to tender|request for (?:proposal|quotation|tender)", + "weight": 16, + "name": "Solicitation in PDF title" + } + ], + "negatives": [ + { + "text": "we are pleased to submit", + "weight": 18, + "name": "Bidder submission language" + }, + { + "text": "in response to your invitation", + "weight": 18, + "name": "Bid response opener" + }, + { + "text": "has been awarded to", + "weight": 14, + "name": "Award notice language" + }, + { + "text": "our proposed approach", + "weight": 14, + "name": "Seller proposal language" + }, + { + "text": "this quotation is valid", + "weight": 12, + "name": "Quote validity language" + } + ], + "structural": [ + { + "signal": "toc", + "weight": 4 + }, + { + "signal": "long_doc", + "weight": 3 + } + ] + }, + { + "id": "sales-proposal", + "name": "Sales proposal", + "emit": true, + "phrases": [ + { + "text": "in response to your invitation", + "weight": 30 + }, + { + "text": "we are pleased to submit", + "weight": 26 + }, + { + "text": "tender response", + "weight": 26, + "where": "title" + }, + { + "text": "compliance matrix", + "weight": 24 + }, + { + "text": "pricing schedule", + "weight": 18 + }, + { + "text": "technical response", + "weight": 16 + }, + { + "text": "form of tender", + "weight": 16 + }, + { + "text": "method statement", + "weight": 10 + }, + { + "text": "technical proposal", + "weight": 14 + }, + { + "text": "our bid", + "weight": 8 + }, + { + "text": "fully compliant", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "in response to (?:your |the )?(?:itt|rfp|rfq|rft|invitation to tender|request for (?:proposal|quotation|tender)s?)", + "weight": 22, + "name": "Response to solicitation" + }, + { + "pattern": "(?:technical|cost|price|pricing) (?:proposal|volume|envelope)", + "weight": 12, + "name": "Bid volume naming" + }, + { + "pattern": "comply\\b.{0,40}partially comply", + "weight": 16, + "name": "Compliance matrix headings" + }, + { + "pattern": "(?:tender|bid|offer) (?:remains|is|shall remain) (?:open|valid)", + "weight": 12, + "name": "Tender validity commitment" + } + ], + "filenames": [ + { + "pattern": "(?:tender|bid|rfp|itt)[-_ ]?(?:response|submission|return)", + "weight": 28, + "name": "Tender response filename" + }, + { + "pattern": "response[-_ ]?to[-_ ]?(?:tender|rfp|itt|rfq)", + "weight": 26, + "name": "Response-to-solicitation filename" + }, + { + "pattern": "compliance[-_ ]?matrix", + "weight": 18, + "name": "Compliance matrix filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "tender response|bid response|technical proposal", + "weight": 14, + "name": "Bid response in PDF title" + } + ], + "negatives": [ + { + "text": "instructions to tenderers", + "weight": 18, + "name": "Buyer-side ITT language" + }, + { + "text": "instructions to bidders", + "weight": 16, + "name": "Buyer-side solicitation language" + }, + { + "text": "notice of award", + "weight": 14, + "name": "Award notice language" + }, + { + "text": "purchase order number", + "weight": 12, + "name": "Purchase order language" + }, + { + "text": "sequence of operations", + "weight": 14, + "name": "Standalone method statement (RAMS)" + }, + { + "pattern": "\\brams\\b|risk assessment and method statement", + "weight": 14, + "name": "RAMS document markers" + } + ], + "structural": [ + { + "signal": "toc", + "weight": 3 + }, + { + "signal": "signature_block", + "weight": 3 + }, + { + "signal": "currency_heavy", + "weight": 2 + } + ] + }, + { + "id": "supply-order", + "name": "Supply order", + "emit": true, + "phrases": [ + { + "text": "purchase requisition", + "weight": 36, + "where": "title" + }, + { + "text": "requisition for purchase", + "weight": 24 + }, + { + "text": "requisition number", + "weight": 24 + }, + { + "text": "justification for purchase", + "weight": 26 + }, + { + "text": "budget holder", + "weight": 22 + }, + { + "text": "requisition date", + "weight": 16 + }, + { + "text": "cost centre", + "weight": 14 + }, + { + "text": "cost center", + "weight": 14 + }, + { + "text": "preferred supplier", + "weight": 14 + }, + { + "text": "requested by", + "weight": 12, + "where": "first" + }, + { + "text": "budget code", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "\\breq(?:uisition)?\\.?\\s*(?:no\\.?|number|#)\\s*:?\\s*[a-z0-9][a-z0-9/-]{1,14}", + "weight": 18, + "name": "Requisition number field" + }, + { + "pattern": "cost cent(?:re|er)\\s*:?\\s*[a-z0-9]", + "weight": 12, + "name": "Cost centre code" + }, + { + "pattern": "justification\\s*(?:for (?:purchase|request))?\\s*:", + "weight": 14, + "name": "Justification field" + }, + { + "pattern": "\\bpr[-#]?\\d{4,8}\\b", + "weight": 12, + "name": "PR number reference" + } + ], + "filenames": [ + { + "pattern": "requisition", + "weight": 28, + "name": "Requisition filename" + }, + { + "pattern": "purchase[-_ ]?req", + "weight": 22, + "name": "Purchase req filename" + }, + { + "pattern": "\\bpr[-_]?\\d{4,8}", + "weight": 15, + "name": "PR number filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "requisition", + "weight": 16, + "name": "Requisition in PDF title" + } + ], + "negatives": [ + { + "text": "purchase order number", + "weight": 20, + "name": "Issued PO language" + }, + { + "text": "this purchase order", + "weight": 16, + "name": "PO document language" + }, + { + "text": "ship to", + "weight": 12, + "name": "PO shipping field" + }, + { + "text": "invoice number", + "weight": 12, + "name": "Invoice language" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 8 + }, + { + "signal": "short_doc", + "weight": 5 + }, + { + "signal": "number_table", + "weight": 3 + } + ] + }, + { + "id": "grant-application", + "name": "Grant application", + "emit": true, + "phrases": [ + { + "text": "grant application", + "weight": 30, + "where": "title" + }, + { + "text": "application for funding", + "weight": 30, + "where": "title" + }, + { + "text": "funding application", + "weight": 22, + "where": "title" + }, + { + "text": "applicant organisation", + "weight": 24 + }, + { + "text": "applicant organization", + "weight": 22 + }, + { + "text": "match funding", + "weight": 24 + }, + { + "text": "amount requested", + "weight": 20 + }, + { + "text": "outcomes and milestones", + "weight": 20 + }, + { + "text": "matching funds", + "weight": 16 + }, + { + "text": "we are applying for", + "weight": 16 + }, + { + "text": "project summary", + "weight": 12 + }, + { + "text": "funding opportunity", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "(?:amount|funding|total) (?:requested|sought)\\s*:?\\s*[$£€]?\\s*[\\d.,]{1,12}", + "weight": 16, + "name": "Requested amount field" + }, + { + "pattern": "\\bsf-?424\\b|grants\\.gov|funding opportunity (?:number|announcement|title)", + "weight": 22, + "name": "US federal grant application markers" + }, + { + "pattern": "charity (?:no\\.?|number)\\s*:?\\s*\\d{6,8}", + "weight": 12, + "name": "UK charity number" + } + ], + "filenames": [ + { + "pattern": "grant[-_ ]?application|funding[-_ ]?application|application[-_ ]?for[-_ ]?funding", + "weight": 28, + "name": "Grant application filename" + }, + { + "pattern": "\\bsf[-_]?424", + "weight": 22, + "name": "SF-424 filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "grant application|application for funding", + "weight": 14, + "name": "Grant application in PDF title" + } + ], + "negatives": [ + { + "text": "your application has been successful", + "weight": 20, + "name": "Funder decision language" + }, + { + "text": "pleased to inform you", + "weight": 18, + "name": "Award letter opener" + }, + { + "text": "terms and conditions of grant", + "weight": 14, + "name": "Grant offer conditions" + }, + { + "text": "swot analysis", + "weight": 12, + "name": "Business plan language" + }, + { + "text": "market analysis", + "weight": 10, + "name": "Business plan language" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 5 + } + ] + }, + { + "id": "technical-drawing", + "name": "Technical drawing", + "emit": true, + "phrases": [ + { + "text": "do not scale", + "weight": 30 + }, + { + "text": "angle projection", + "weight": 30 + }, + { + "text": "work to figured dimensions", + "weight": 26 + }, + { + "text": "break all sharp edges", + "weight": 22 + }, + { + "text": "general arrangement", + "weight": 18 + }, + { + "text": "drawn by", + "weight": 16 + }, + { + "text": "checked by", + "weight": 12 + }, + { + "text": "issued for construction", + "weight": 12 + }, + { + "text": "unless noted otherwise", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "\\b(dwg|drg|drawing)\\s?(no\\.?|number|#)\\s?:?\\s?[a-z0-9][a-z0-9/_.-]{2,24}", + "flags": "gi", + "weight": 20, + "name": "drawing number" + }, + { + "pattern": "all\\s?dimensions\\s?(are\\s?)?in\\s?(mm|millimet|inches)", + "flags": "gi", + "weight": 22, + "name": "dimensions in mm note" + }, + { + "pattern": "\\bscale\\s?:?\\s?\\d{1,2}\\s?:\\s?\\d{1,4}\\b", + "flags": "gi", + "weight": 18, + "name": "scale ratio" + }, + { + "pattern": "\\bsheet\\s?\\d{1,3}\\s?of\\s?\\d{1,3}\\b", + "flags": "gi", + "weight": 12, + "name": "sheet x of y" + }, + { + "pattern": "[ø⌀]\\s?\\d{1,3}(\\.\\d{1,2})?", + "flags": "gi", + "weight": 12, + "name": "diameter callout" + } + ], + "filenames": [ + { + "pattern": "(^|[^a-z0-9])(dwg|drg)([^a-z0-9]|$)", + "weight": 20, + "name": "dwg abbreviation" + }, + { + "pattern": "drawing", + "weight": 15, + "name": "drawing keyword" + }, + { + "pattern": "general[\\s_-]?arrangement|(^|[^a-z0-9])ga[\\s_-]?plan", + "weight": 16, + "name": "ga plan filename" + } + ], + "metadata": [ + { + "field": "producer", + "pattern": "autocad|dwg to pdf|solidworks|revit|microstation|inventor|draftsight|bricscad", + "weight": 20, + "name": "cad producer" + }, + { + "field": "creator", + "pattern": "autocad|solidworks|revit|catia|creo|archicad|tekla|vectorworks", + "weight": 18, + "name": "cad creator" + } + ], + "negatives": [ + { + "text": "absolute maximum ratings", + "weight": 22, + "name": "datasheet section" + }, + { + "text": "electrical characteristics", + "weight": 16, + "name": "datasheet section" + }, + { + "text": "safety data sheet", + "weight": 20, + "name": "sds heading" + }, + { + "text": "inspection report", + "weight": 14, + "name": "inspection doc referencing drawings" + }, + { + "text": "table of contents", + "weight": 10, + "name": "manual or spec toc" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 8 + } + ] + }, + { + "id": "bill-of-materials", + "name": "Bill of materials", + "emit": true, + "phrases": [ + { + "text": "bill of quantities", + "weight": 36, + "where": "title" + }, + { + "text": "bills of quantities", + "weight": 32 + }, + { + "text": "carried to collection", + "weight": 30 + }, + { + "text": "provisional sum", + "weight": 26 + }, + { + "text": "measured works", + "weight": 26 + }, + { + "text": "prime cost sum", + "weight": 24 + }, + { + "text": "preliminaries", + "weight": 20 + }, + { + "text": "form of tender", + "weight": 16 + }, + { + "text": "daywork", + "weight": 14 + }, + { + "text": "main summary", + "weight": 12 + }, + { + "text": "bill of material", + "weight": 32, + "where": "title" + }, + { + "text": "reference designator", + "weight": 30 + }, + { + "text": "qty per assembly", + "weight": 28 + }, + { + "text": "manufacturer part number", + "weight": 26 + }, + { + "text": "parts list", + "weight": 20, + "where": "title" + }, + { + "text": "top level assembly", + "weight": 18 + }, + { + "text": "do not populate", + "weight": 16 + }, + { + "text": "assembly drawing", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\b(smm7|nrm\\s?[12]|cesmm\\s?\\d?)\\b", + "flags": "gi", + "weight": 22, + "name": "measurement standard" + }, + { + "pattern": "description\\s+(unit\\s+)?(qty|quantity)\\s+(unit\\s+)?rate\\s+(£\\s+)?amount", + "flags": "gi", + "weight": 20, + "name": "boq column header" + }, + { + "pattern": "\\bmpn\\b|mfr\\.?\\s?p/?n|mfg\\.?\\s?part\\s?(no\\.?|number)", + "flags": "gi", + "weight": 16, + "name": "mpn abbreviation" + }, + { + "pattern": "\\b[rcldqu]\\d{1,3}\\s?,\\s?[rcldqu]\\d{1,3}\\b", + "flags": "gi", + "weight": 16, + "name": "reference designator list" + }, + { + "pattern": "\\bdnp\\b", + "flags": "gi", + "weight": 12, + "name": "dnp marker" + } + ], + "filenames": [ + { + "pattern": "(^|[^a-z0-9])boq([^a-z0-9]|$)", + "weight": 24, + "name": "boq abbreviation" + }, + { + "pattern": "bill[\\s_-]*of[\\s_-]*quantities", + "weight": 26, + "name": "boq filename" + }, + { + "pattern": "pricing[\\s_-]?document|tender[\\s_-]?sum", + "weight": 15, + "name": "tender pricing filename" + }, + { + "pattern": "(^|[^a-z0-9])bom([^a-z0-9]|$)", + "weight": 24, + "name": "bom abbreviation" + }, + { + "pattern": "bill[\\s_-]*of[\\s_-]*materials?", + "weight": 24, + "name": "bom filename" + }, + { + "pattern": "parts[\\s_-]?list", + "weight": 18, + "name": "parts list filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "packing slip", + "weight": 24, + "name": "shipping doc" + }, + { + "text": "purchase order", + "weight": 14, + "name": "procurement doc" + }, + { + "text": "ship to", + "weight": 14, + "name": "shipping address" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 8 + }, + { + "signal": "number_table", + "weight": 6 + }, + { + "signal": "long_doc", + "weight": 4 + } + ] + }, + { + "id": "safety-procedure", + "name": "Safety procedure", + "emit": true, + "phrases": [ + { + "text": "method statement", + "weight": 32, + "where": "title" + }, + { + "text": "safe system of work", + "weight": 28 + }, + { + "text": "sequence of work", + "weight": 26 + }, + { + "text": "permit to work", + "weight": 24 + }, + { + "text": "toolbox talk", + "weight": 22 + }, + { + "text": "control measures", + "weight": 16 + }, + { + "text": "banksman", + "weight": 16 + }, + { + "text": "first aid arrangements", + "weight": 16 + }, + { + "text": "personal protective equipment", + "weight": 14 + }, + { + "text": "site induction", + "weight": 14 + }, + { + "text": "exclusion zone", + "weight": 12 + }, + { + "text": "standard operating procedure", + "weight": 30 + } + ], + "regexes": [ + { + "pattern": "\\brams\\b", + "flags": "gi", + "weight": 16, + "name": "rams acronym" + }, + { + "pattern": "(likelihood|probability)\\s?[x×*]\\s?(severity|consequence)", + "flags": "gi", + "weight": 18, + "name": "risk matrix formula" + }, + { + "pattern": "\\bppe\\b", + "flags": "gi", + "weight": 10, + "name": "ppe acronym" + } + ], + "filenames": [ + { + "pattern": "method[\\s_-]?statement", + "weight": 26, + "name": "method statement filename" + }, + { + "pattern": "(^|[^a-z0-9])rams([^a-z0-9]|$)", + "weight": 20, + "name": "rams filename" + }, + { + "pattern": "(^|[^a-z0-9])ms[-_ ]?\\d{2,4}", + "weight": 15, + "name": "ms number filename" + }, + { + "pattern": "standard[-_ ]?operating[-_ ]?procedure|\\bsop\\b", + "weight": 16, + "name": "SOP filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "information security", + "weight": 24, + "name": "infosec risk assessment" + }, + { + "text": "27001", + "weight": 22, + "name": "iso 27001" + }, + { + "text": "penetration test", + "weight": 18, + "name": "security testing doc" + }, + { + "text": "vulnerability assessment", + "weight": 16, + "name": "infosec assessment" + }, + { + "text": "access control policy", + "weight": 14, + "name": "security policy" + } + ], + "structural": [ + { + "signal": "bullet_heavy", + "weight": 6 + }, + { + "signal": "toc", + "weight": 4 + } + ] + }, + { + "id": "test-report", + "name": "Test report", + "emit": true, + "phrases": [ + { + "text": "material test certificate", + "weight": 34, + "where": "title" + }, + { + "text": "mill test certificate", + "weight": 34, + "where": "title" + }, + { + "text": "heat number", + "weight": 30 + }, + { + "text": "mill certificate", + "weight": 24 + }, + { + "text": "chemical composition", + "weight": 24 + }, + { + "text": "test certificate", + "weight": 22, + "where": "title" + }, + { + "text": "cast number", + "weight": 22 + }, + { + "text": "inspection certificate", + "weight": 18, + "where": "title" + }, + { + "text": "tensile strength", + "weight": 18 + }, + { + "text": "charpy", + "weight": 18 + }, + { + "text": "yield strength", + "weight": 16 + }, + { + "text": "batch tested", + "weight": 16 + }, + { + "text": "elongation", + "weight": 10 + }, + { + "text": "calibration certificate", + "weight": 36, + "where": "title" + }, + { + "text": "certificate of calibration", + "weight": 34, + "where": "title" + }, + { + "text": "traceable to national standards", + "weight": 30 + }, + { + "text": "calibration due", + "weight": 26 + }, + { + "text": "measurement uncertainty", + "weight": 24 + }, + { + "text": "uncertainty of measurement", + "weight": 24 + }, + { + "text": "calibration laboratory", + "weight": 18 + }, + { + "text": "calibration procedure", + "weight": 16 + }, + { + "text": "unit under test", + "weight": 16 + }, + { + "text": "test equipment used", + "weight": 14 + }, + { + "text": "calibrated by", + "weight": 14 + }, + { + "text": "calibration technician", + "weight": 14 + }, + { + "text": "environmental conditions", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "en\\s?10204", + "flags": "gi", + "weight": 25, + "name": "en 10204 reference" + }, + { + "pattern": "\\b3\\.[12]\\s?(certificate|cert)\\b|certificate\\s?(type\\s?)?3\\.[12]", + "flags": "gi", + "weight": 16, + "name": "type 3.1 certificate" + }, + { + "pattern": "\\b\\d{3,4}\\s?(n/mm2|n/mm²|mpa)\\b", + "flags": "gi", + "weight": 14, + "name": "strength units" + }, + { + "pattern": "\\bheat\\s?no\\.?\\b", + "flags": "gi", + "weight": 14, + "name": "heat no abbreviation" + }, + { + "pattern": "\\bas[- ](found|left)\\b", + "flags": "gi", + "weight": 20, + "name": "as-found as-left" + }, + { + "pattern": "coverage\\s?factor\\s?\\(?k\\)?\\s?=?\\s?2", + "flags": "gi", + "weight": 22, + "name": "coverage factor k=2" + }, + { + "pattern": "iso\\s?/?\\s?(iec\\s?)?17025", + "flags": "gi", + "weight": 22, + "name": "iso 17025" + }, + { + "pattern": "\\b(ukas|nist|npl|ptb|nabl|dakks)\\b", + "flags": "gi", + "weight": 14, + "name": "national metrology body" + } + ], + "filenames": [ + { + "pattern": "mill[\\s_-]?(test[\\s_-]?)?cert", + "weight": 24, + "name": "mill cert filename" + }, + { + "pattern": "(^|[^a-z0-9])mtc([^a-z0-9]|$)", + "weight": 20, + "name": "mtc abbreviation" + }, + { + "pattern": "test[\\s_-]?cert", + "weight": 18, + "name": "test cert filename" + }, + { + "pattern": "(^|[^a-z0-9])3\\.1([^a-z0-9]|$)", + "weight": 15, + "name": "3.1 in filename" + }, + { + "pattern": "cal[\\s_-]?cert", + "weight": 22, + "name": "cal cert filename" + }, + { + "pattern": "calibration", + "weight": 22, + "name": "calibration keyword" + }, + { + "pattern": "(^|[^a-z0-9])cal[-_ ]?\\d{3,}", + "weight": 15, + "name": "cal number filename" + } + ], + "metadata": [ + { + "field": "producer", + "pattern": "met/?cal|procal|indysoft|gagetrak", + "weight": 14, + "name": "calibration software" + } + ], + "negatives": [ + { + "text": "patient", + "weight": 24, + "name": "medical lab report" + }, + { + "text": "reference range", + "weight": 20, + "name": "medical lab values" + }, + { + "text": "sole responsibility", + "weight": 14, + "name": "declaration of conformity" + }, + { + "text": "calibration", + "weight": 14, + "name": "calibration cert" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 8 + }, + { + "signal": "short_doc", + "weight": 4 + } + ] + }, + { + "id": "quality-report", + "name": "Quality report", + "emit": true, + "phrases": [ + { + "text": "first article inspection", + "weight": 34, + "where": "title" + }, + { + "text": "incoming inspection", + "weight": 30, + "where": "title" + }, + { + "text": "dimensional inspection", + "weight": 26 + }, + { + "text": "final inspection", + "weight": 24 + }, + { + "text": "non-conformance", + "weight": 22 + }, + { + "text": "inspection lot", + "weight": 22 + }, + { + "text": "inspection report", + "weight": 20, + "where": "title" + }, + { + "text": "sampling plan", + "weight": 20 + }, + { + "text": "out of tolerance", + "weight": 16 + }, + { + "text": "inspected by", + "weight": 14 + }, + { + "text": "acceptance criteria", + "weight": 12 + }, + { + "text": "disposition", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "\\bncr[- ]?\\d{2,6}\\b", + "flags": "gi", + "weight": 20, + "name": "ncr number" + }, + { + "pattern": "\\baql\\b", + "flags": "gi", + "weight": 16, + "name": "aql" + }, + { + "pattern": "nominal\\s+(actual|measured)|nominal\\s+\\d+(\\.\\d+)?\\s+measured", + "flags": "gi", + "weight": 16, + "name": "nominal vs actual columns" + }, + { + "pattern": "accept(ed)?\\s?/\\s?reject(ed)?", + "flags": "gi", + "weight": 14, + "name": "accept reject column" + }, + { + "pattern": "non[- ]?conform(ance|ing|ity|ities)", + "flags": "gi", + "weight": 14, + "name": "nonconformance variants" + } + ], + "filenames": [ + { + "pattern": "(incoming|final|first[\\s_-]?article)[\\s_-]?inspection", + "weight": 24, + "name": "inspection stage filename" + }, + { + "pattern": "(^|[^a-z0-9])(fai|ncr)([^a-z0-9]|$)", + "weight": 18, + "name": "fai or ncr abbreviation" + }, + { + "pattern": "inspection[\\s_-]?report", + "weight": 15, + "name": "inspection report filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "home inspection", + "weight": 28, + "name": "property inspection" + }, + { + "text": "property address", + "weight": 18, + "name": "property inspection" + }, + { + "text": "crawl space", + "weight": 16, + "name": "property inspection" + }, + { + "text": "patient", + "weight": 18, + "name": "medical report" + }, + { + "text": "heat number", + "weight": 14, + "name": "mill cert field" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 8 + }, + { + "signal": "form_like", + "weight": 4 + } + ] + }, + { + "id": "itinerary", + "name": "Itinerary", + "emit": true, + "phrases": [ + { + "text": "travel itinerary", + "weight": 30, + "where": "title" + }, + { + "text": "trip itinerary", + "weight": 30, + "where": "title" + }, + { + "text": "flight itinerary", + "weight": 28, + "where": "title" + }, + { + "text": "your itinerary", + "weight": 22, + "where": "first" + }, + { + "text": "itinerary", + "weight": 10, + "where": "title" + }, + { + "text": "trip summary", + "weight": 24, + "where": "first" + }, + { + "text": "layover", + "weight": 18 + }, + { + "text": "stopover", + "weight": 14 + }, + { + "text": "connecting flight", + "weight": 14 + }, + { + "text": "outbound flight", + "weight": 16 + }, + { + "text": "return flight", + "weight": 12 + }, + { + "text": "total duration", + "weight": 10 + }, + { + "text": "passenger details", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "\\((?!USD|EUR|GBP|CHF|JPY|CAD|AUD|VAT|LLC|LTD|INC|FAQ|PDF|USA)[A-Z]{3}\\)", + "flags": "g", + "weight": 10, + "name": "IATA airport code in parentheses (currency/legal codes excluded)" + }, + { + "pattern": "\\b\\d{1,2}h\\s?\\d{2}m\\b", + "flags": "gi", + "weight": 12, + "name": "duration like 2h 45m" + }, + { + "pattern": "(?:depart|departure)s?\\s*:?\\s*\\d{1,2}:\\d{2}", + "flags": "gi", + "weight": 14, + "name": "departure time field" + }, + { + "pattern": "(?:arrive|arrival)s?\\s*:?\\s*\\d{1,2}:\\d{2}", + "flags": "gi", + "weight": 12, + "name": "arrival time field" + }, + { + "pattern": "travell?er details", + "flags": "gi", + "weight": 14, + "name": "traveler/traveller details (US+UK)" + } + ], + "filenames": [ + { + "pattern": "itinerar", + "weight": 28, + "name": "itinerary in filename" + }, + { + "pattern": "trip[-_ ]?(?:plan|details|summary)", + "weight": 20, + "name": "trip plan/details/summary" + }, + { + "pattern": "travel[-_ ]?plan", + "weight": 18, + "name": "travel plan" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "itinerar", + "weight": 14, + "name": "itinerary in PDF title" + } + ], + "negatives": [ + { + "text": "itinerary receipt", + "weight": 14, + "name": "e-ticket receipt heading" + }, + { + "text": "fare calculation", + "weight": 16, + "name": "e-ticket fare calculation" + }, + { + "text": "fare basis", + "weight": 12, + "name": "e-ticket fare basis" + }, + { + "text": "boarding pass", + "weight": 18, + "name": "boarding pass heading" + }, + { + "text": "check-out date", + "weight": 10, + "name": "hotel confirmation field" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 4 + }, + { + "signal": "short_doc", + "weight": 3 + } + ] + }, + { + "id": "ticket", + "name": "Ticket", + "emit": true, + "phrases": [ + { + "text": "boarding pass", + "weight": 36, + "where": "title" + }, + { + "text": "mobile boarding pass", + "weight": 26 + }, + { + "text": "boarding time", + "weight": 24, + "where": "first" + }, + { + "text": "gate closes", + "weight": 24, + "where": "first" + }, + { + "text": "boarding group", + "weight": 20, + "where": "first" + }, + { + "text": "boarding zone", + "weight": 16, + "where": "first" + }, + { + "text": "boarding begins", + "weight": 14, + "where": "first" + }, + { + "text": "gate subject to change", + "weight": 18 + }, + { + "text": "priority boarding", + "weight": 12 + }, + { + "text": "have a nice flight", + "weight": 14 + }, + { + "text": "gate", + "weight": 5, + "where": "first" + }, + { + "text": "seat", + "weight": 4, + "where": "first" + }, + { + "text": "electronic ticket receipt", + "weight": 32, + "where": "title" + }, + { + "text": "e-ticket receipt", + "weight": 30, + "where": "title" + }, + { + "text": "itinerary receipt", + "weight": 26, + "where": "title" + }, + { + "text": "electronic ticket", + "weight": 24, + "where": "title" + }, + { + "text": "e-ticket", + "weight": 14 + }, + { + "text": "fare basis", + "weight": 26 + }, + { + "text": "fare calculation", + "weight": 26 + }, + { + "text": "baggage allowance", + "weight": 18 + }, + { + "text": "not valid before", + "weight": 18 + }, + { + "text": "not valid after", + "weight": 16 + }, + { + "text": "ticket number", + "weight": 14, + "where": "first" + }, + { + "text": "form of payment", + "weight": 12 + }, + { + "text": "endorsements", + "weight": 12 + }, + { + "text": "booking class", + "weight": 10 + }, + { + "text": "admit one", + "weight": 30 + }, + { + "text": "this ticket admits", + "weight": 28 + }, + { + "text": "doors open", + "weight": 24 + }, + { + "text": "gates open", + "weight": 20 + }, + { + "text": "general admission", + "weight": 24 + }, + { + "text": "no re-entry", + "weight": 24 + }, + { + "text": "valid for one admission", + "weight": 26 + }, + { + "text": "will call", + "weight": 8 + }, + { + "text": "box office", + "weight": 12 + }, + { + "text": "ticket holder", + "weight": 16 + }, + { + "text": "rain or shine", + "weight": 18 + }, + { + "text": "booking reference", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "seat\\s*:?\\s*\\d{1,2}[a-k]\\b", + "flags": "gi", + "weight": 16, + "name": "seat assignment" + }, + { + "pattern": "gate\\s*:?\\s*[a-z]?\\d{1,3}\\b", + "flags": "gi", + "weight": 12, + "name": "gate number" + }, + { + "pattern": "(?:flight|flt)\\s*:?\\s*[a-z]{2}\\s?\\d{1,4}\\b", + "flags": "gi", + "weight": 8, + "name": "flight number" + }, + { + "pattern": "(?:pnr|record locator|booking ref(?:erence)?)\\s*:?\\s*[a-z0-9]{5,7}\\b", + "flags": "gi", + "weight": 12, + "name": "record locator (PNR)" + }, + { + "pattern": "(?:ticket|tkt)\\s*(?:number|no\\.?|#|nbr)?\\s*:?\\s*\\d{3}[- ]?\\d{10}", + "flags": "gi", + "weight": 22, + "name": "ticket number field with 13 digits" + }, + { + "pattern": "\\b\\d{3}[- ]?\\d{10}\\b", + "flags": "g", + "weight": 12, + "name": "13-digit ticket number" + }, + { + "pattern": "\\b(?:etkt|e-tkt)\\b", + "flags": "gi", + "weight": 18, + "name": "ETKT marker" + }, + { + "pattern": "\\bnuc\\s?\\d{1,6}(?:\\.\\d{1,2})?\\b", + "flags": "gi", + "weight": 18, + "name": "NUC amount in fare calculation" + }, + { + "pattern": "sec(tion)?\\W{0,4}[a-z0-9]{1,6}\\W{1,12}row\\W{0,4}[a-z0-9]{1,4}\\W{1,12}seats?\\W{0,4}[a-z0-9]{1,4}", + "flags": "gi", + "weight": 22, + "name": "Section/Row/Seat block" + }, + { + "pattern": "\\brow [a-z0-9]{1,3}\\W{1,10}seats? [a-z0-9]{1,3}\\b", + "flags": "gi", + "weight": 16, + "name": "Row/Seat pair" + }, + { + "pattern": "doors:? ?\\d{1,2}[:.]\\d{2} ?(am|pm)?", + "flags": "gi", + "weight": 16, + "name": "Doors time" + }, + { + "pattern": "print[\\- ]at[\\- ]home", + "flags": "gi", + "weight": 18, + "name": "Print-at-home ticket" + } + ], + "filenames": [ + { + "pattern": "boarding", + "weight": 26, + "name": "boarding in filename" + }, + { + "pattern": "e[-_]?ticket|etkt", + "weight": 28, + "name": "e-ticket filename" + }, + { + "pattern": "ticket[-_ ]?receipt", + "weight": 22, + "name": "ticket receipt filename" + }, + { + "pattern": "ticket", + "weight": 18, + "name": "Ticket in filename" + }, + { + "pattern": "ticketmaster|eventbrite|axs|seetickets|dice[_\\- ]", + "weight": 26, + "name": "Ticketing platform in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "boarding", + "weight": 16, + "name": "boarding in PDF title" + }, + { + "field": "title", + "pattern": "e-?ticket|electronic ticket|itinerary receipt", + "weight": 16, + "name": "e-ticket in PDF title" + }, + { + "field": "any", + "pattern": "amadeus|sabre|travelport|galileo", + "weight": 8, + "name": "GDS software in metadata" + }, + { + "field": "any", + "pattern": "ticketmaster|eventbrite|axs tickets|see tickets", + "weight": 15, + "name": "Ticketing platform metadata" + } + ], + "negatives": [ + { + "text": "check-out date", + "weight": 12, + "name": "hotel confirmation field" + }, + { + "text": "invoice", + "weight": 10, + "name": "invoice wording" + }, + { + "text": "room type", + "weight": 12, + "name": "hotel confirmation field" + }, + { + "text": "flight", + "weight": 12, + "name": "Flight (travel)" + }, + { + "text": "itinerary", + "weight": 15, + "name": "Itinerary (travel)" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 10 + }, + { + "signal": "form_like", + "weight": 3 + }, + { + "signal": "currency_heavy", + "weight": 3 + } + ] + }, + { + "id": "booking-confirmation", + "name": "Booking confirmation", + "emit": true, + "phrases": [ + { + "text": "booking confirmation", + "weight": 32, + "where": "title" + }, + { + "text": "reservation confirmation", + "weight": 30, + "where": "title" + }, + { + "text": "your booking is confirmed", + "weight": 30, + "where": "first" + }, + { + "text": "your reservation is confirmed", + "weight": 28, + "where": "first" + }, + { + "text": "confirmation number", + "weight": 16, + "where": "first" + }, + { + "text": "check-in date", + "weight": 22 + }, + { + "text": "check-out date", + "weight": 24 + }, + { + "text": "room type", + "weight": 20 + }, + { + "text": "number of guests", + "weight": 18 + }, + { + "text": "cancellation policy", + "weight": 14 + }, + { + "text": "we look forward to welcoming you", + "weight": 22 + }, + { + "text": "booking reference", + "weight": 8, + "where": "first" + }, + { + "text": "pick-up date", + "weight": 10 + }, + { + "text": "drop-off date", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "check[- ]?in\\s*:?\\s*(?:mon|tue|wed|thu|fri|sat|sun|\\d{1,2}[/.-])", + "flags": "gi", + "weight": 14, + "name": "check-in date field" + }, + { + "pattern": "check[- ]?out\\s*:?\\s*(?:mon|tue|wed|thu|fri|sat|sun|\\d{1,2}[/.-])", + "flags": "gi", + "weight": 14, + "name": "check-out date field" + }, + { + "pattern": "confirmation\\s*(?:number|no\\.?|code|#)\\s*:?\\s*[a-z0-9.-]{5,20}", + "flags": "gi", + "weight": 14, + "name": "confirmation number field" + }, + { + "pattern": "\\b\\d{1,3}\\s+nights?\\b", + "flags": "gi", + "weight": 8, + "name": "nights count" + }, + { + "pattern": "\\b\\d{1,2}\\s+(?:adults?|guests?)\\b", + "flags": "gi", + "weight": 10, + "name": "guest count" + }, + { + "pattern": "(?:pick[- ]?up|drop[- ]?off)\\s*(?:date|time|location)\\s*:?", + "flags": "gi", + "weight": 12, + "name": "pick-up/drop-off field (car rental)" + } + ], + "filenames": [ + { + "pattern": "booking[-_ ]?confirmation", + "weight": 30, + "name": "booking confirmation filename" + }, + { + "pattern": "confirmation", + "weight": 16, + "name": "confirmation in filename" + }, + { + "pattern": "hotel|reservation|booking", + "weight": 16, + "name": "hotel/reservation/booking in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "booking|reservation|confirmation", + "weight": 12, + "name": "booking in PDF title" + } + ], + "negatives": [ + { + "text": "boarding pass", + "weight": 18, + "name": "boarding pass heading" + }, + { + "text": "boarding time", + "weight": 16, + "name": "boarding pass field" + }, + { + "text": "fare basis", + "weight": 14, + "name": "e-ticket fare basis" + }, + { + "text": "security deposit", + "weight": 14, + "name": "lease agreement wording" + }, + { + "text": "order confirmation", + "weight": 16, + "name": "retail order confirmation heading" + } + ], + "structural": [ + { + "signal": "address_block", + "weight": 5 + }, + { + "signal": "currency_heavy", + "weight": 3 + }, + { + "signal": "short_doc", + "weight": 3 + } + ] + }, + { + "id": "utility-bill", + "name": "Utility bill", + "emit": true, + "phrases": [ + { + "text": "meter reading", + "weight": 24 + }, + { + "text": "standing charge", + "weight": 26 + }, + { + "text": "kwh", + "weight": 12 + }, + { + "text": "per kwh", + "weight": 16 + }, + { + "text": "billing period", + "weight": 6, + "where": "first" + }, + { + "text": "supply address", + "weight": 20, + "where": "first" + }, + { + "text": "estimated reading", + "weight": 20 + }, + { + "text": "energy statement", + "weight": 24, + "where": "title" + }, + { + "text": "energy bill", + "weight": 16, + "where": "title" + }, + { + "text": "gas and electricity", + "weight": 20 + }, + { + "text": "meter number", + "weight": 18 + }, + { + "text": "water and sewer", + "weight": 18 + }, + { + "text": "unit rate", + "weight": 16 + }, + { + "text": "average daily usage", + "weight": 14 + }, + { + "text": "line rental", + "weight": 26 + }, + { + "text": "call charges", + "weight": 20 + }, + { + "text": "data usage", + "weight": 16 + }, + { + "text": "roaming charges", + "weight": 22 + }, + { + "text": "plan charge", + "weight": 16 + }, + { + "text": "unlimited data", + "weight": 14 + }, + { + "text": "itemised calls", + "weight": 24 + }, + { + "text": "itemized calls", + "weight": 24 + }, + { + "text": "minutes used", + "weight": 16 + }, + { + "text": "governmental surcharges", + "weight": 20 + }, + { + "text": "data allowance", + "weight": 16 + }, + { + "text": "broadband", + "weight": 10 + }, + { + "text": "mobile number", + "weight": 10, + "where": "first" + }, + { + "text": "airtime", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "\\b\\d[\\d,]{0,6}(\\.\\d{1,3})?\\s*kwh\\b", + "flags": "gi", + "weight": 15, + "name": "kWh amount" + }, + { + "pattern": "(previous|present|current)\\s+reading", + "flags": "gi", + "weight": 12, + "name": "meter reading row" + }, + { + "pattern": "\\bmp[ar]n\\b", + "flags": "gi", + "weight": 20, + "name": "UK meter point number (MPAN/MPRN)" + }, + { + "pattern": "\\b\\d{1,6}\\s*(ccf|hcf|therms)\\b", + "flags": "gi", + "weight": 13, + "name": "gas/water volume units" + }, + { + "pattern": "electric(ity)?\\s+charges", + "flags": "gi", + "weight": 16, + "name": "electric/electricity charges heading" + }, + { + "pattern": "\\b\\d{1,4}(\\.\\d{1,2})?\\s*gb\\s+(of\\s+)?data\\b", + "flags": "gi", + "weight": 14, + "name": "data allowance in GB" + }, + { + "pattern": "\\bused\\s+\\d{1,4}(\\.\\d{1,2})?\\s*gb\\b", + "flags": "gi", + "weight": 13, + "name": "GB used" + }, + { + "pattern": "\\b\\d{1,4}\\s*(minutes|mins)\\s+(used|included|remaining)\\b", + "flags": "gi", + "weight": 12, + "name": "minutes usage" + }, + { + "pattern": "talk,?\\s*text\\s*(and|&)\\s*data", + "flags": "gi", + "weight": 12, + "name": "talk text and data" + } + ], + "filenames": [ + { + "pattern": "(electric|gas|water|energy|utility|utilities|power)[-_ ]?(bill|statement)", + "weight": 25, + "name": "utility bill filename" + }, + { + "pattern": "(british[-_ ]?gas|octopus[-_ ]?energy|thames[-_ ]?water|scottish[-_ ]?power|national[-_ ]?grid|duke[-_ ]?energy|con[-_ ]?edison)", + "weight": 18, + "name": "utility provider filename" + }, + { + "pattern": "(pg&e|pacific[-_ ]?gas|eon[-_ ]?next|e\\.on|sse[-_ ]?energy|edf[-_ ]?energy|severn[-_ ]?trent|united[-_ ]?utilities|american[-_ ]?water|centerpoint[-_ ]?energy)", + "weight": 18, + "name": "utility provider filename (extended)" + }, + { + "pattern": "(phone|mobile|broadband|wireless|telecom|cell)[-_ ]?(bill|statement|invoice)", + "weight": 25, + "name": "telecom bill filename" + }, + { + "pattern": "(verizon|vodafone|t-?mobile|at&t|xfinity|comcast|spectrum[-_ ]?bill|virgin[-_ ]?media|o2[-_ ]?bill|ee[-_ ]?bill|sky[-_ ]?bill|three[-_ ]?bill|giffgaff|mint[-_ ]?mobile)", + "weight": 18, + "name": "telecom provider filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "(energy|electricity|gas|water|utility)\\s*(bill|statement)", + "weight": 14, + "name": "utility bill title" + }, + { + "field": "author", + "pattern": "(british gas|octopus energy|edf energy|e\\.on|duke energy|con edison|national grid|thames water|pacific gas)", + "weight": 12, + "name": "utility provider author" + }, + { + "field": "title", + "pattern": "(phone|mobile|wireless|broadband|telecom)\\s*(bill|statement)", + "weight": 14, + "name": "telecom bill title" + }, + { + "field": "author", + "pattern": "(verizon|vodafone|at&t|t-mobile|comcast|xfinity|virgin media|bt group)", + "weight": 12, + "name": "telecom provider author" + } + ], + "negatives": [ + { + "text": "patient", + "weight": 14, + "name": "medical bill vocabulary" + }, + { + "text": "policy number", + "weight": 12, + "name": "insurance vocabulary" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 4 + }, + { + "signal": "number_table", + "weight": 5 + }, + { + "signal": "address_block", + "weight": 4 + } + ] + }, + { + "id": "subscription-confirmation", + "name": "Subscription confirmation", + "emit": true, + "phrases": [ + { + "text": "next billing date", + "weight": 28 + }, + { + "text": "subscription renewal", + "weight": 26 + }, + { + "text": "your subscription", + "weight": 22 + }, + { + "text": "manage your subscription", + "weight": 26 + }, + { + "text": "renews on", + "weight": 20 + }, + { + "text": "will automatically renew", + "weight": 26 + }, + { + "text": "renews automatically", + "weight": 22 + }, + { + "text": "unless you cancel", + "weight": 16 + }, + { + "text": "thank you for subscribing", + "weight": 24 + }, + { + "text": "your receipt from apple", + "weight": 32 + }, + { + "text": "google play", + "weight": 10 + }, + { + "text": "free trial", + "weight": 8 + }, + { + "text": "cancel anytime", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "(monthly|annual|yearly)\\s+(subscription|membership|plan)", + "flags": "gi", + "weight": 14, + "name": "plan cadence" + }, + { + "pattern": "next\\s+(billing|payment|renewal)\\s+date", + "flags": "gi", + "weight": 15, + "name": "next billing/payment date field" + }, + { + "pattern": "subscription\\s+(fee|charge|period)", + "flags": "gi", + "weight": 12, + "name": "subscription fee/charge/period" + }, + { + "pattern": "billed\\s+(monthly|annually|yearly)", + "flags": "gi", + "weight": 16, + "name": "billed monthly/annually" + }, + { + "pattern": "auto[- ]?renew(al|s|ing)?", + "flags": "gi", + "weight": 14, + "name": "auto-renew variants" + }, + { + "pattern": "(cancel|manage)\\s+your\\s+(subscription|membership|plan)", + "flags": "gi", + "weight": 15, + "name": "cancel/manage your subscription" + } + ], + "filenames": [ + { + "pattern": "subscription", + "weight": 20, + "name": "subscription filename" + }, + { + "pattern": "(netflix|spotify|patreon|hulu|audible|crunchyroll|disney[-_ ]?plus|disneyplus)", + "weight": 18, + "name": "subscription service filename" + }, + { + "pattern": "(adobe|dropbox|icloud)[-_ ]?(receipt|invoice|renewal|subscription)", + "weight": 20, + "name": "software subscription receipt filename" + }, + { + "pattern": "(apple|itunes|google[-_ ]?play|app[-_ ]?store)[-_ ]?(receipt|invoice|order)", + "weight": 20, + "name": "app store receipt filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "subscription", + "weight": 12, + "name": "subscription title" + }, + { + "field": "author", + "pattern": "(stripe|paddle|recurly|chargebee|fastspring)", + "weight": 10, + "name": "billing platform author" + } + ], + "negatives": [ + { + "text": "policy number", + "weight": 15, + "name": "insurance renewal" + }, + { + "text": "insured", + "weight": 12, + "name": "insurance vocabulary" + }, + { + "text": "kwh", + "weight": 10, + "name": "utility usage" + }, + { + "text": "minutes used", + "weight": 10, + "name": "telecom usage" + }, + { + "text": "roaming", + "weight": 10, + "name": "telecom roaming" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 6 + }, + { + "signal": "currency_heavy", + "weight": 3 + } + ] + }, + { + "id": "waybill", + "name": "Waybill", + "emit": true, + "phrases": [ + { + "text": "shipping label", + "weight": 26, + "where": "title" + }, + { + "text": "ship from", + "weight": 12, + "where": "first" + }, + { + "text": "tracking number", + "weight": 8 + }, + { + "text": "tracking #", + "weight": 14, + "where": "first" + }, + { + "text": "usps priority mail", + "weight": 22 + }, + { + "text": "usps ground advantage", + "weight": 26 + }, + { + "text": "ups ground", + "weight": 12 + }, + { + "text": "fedex ground", + "weight": 12 + }, + { + "text": "billing: p/p", + "weight": 22 + }, + { + "text": "postage paid", + "weight": 14 + }, + { + "text": "royal mail tracked", + "weight": 24 + }, + { + "text": "tracked 48", + "weight": 18 + }, + { + "text": "tracked 24", + "weight": 18 + }, + { + "text": "fold here", + "weight": 16 + }, + { + "text": "bill of lading", + "weight": 28, + "where": "title" + }, + { + "text": "air waybill", + "weight": 26, + "where": "title" + }, + { + "text": "waybill", + "weight": 22, + "where": "title" + } + ], + "regexes": [ + { + "pattern": "\\b1z[\\s-]?[0-9a-z]{3}[\\s-]?[0-9a-z]{3}[\\s-]?[0-9a-z]{2}[\\s-]?[0-9a-z]{4}[\\s-]?[0-9a-z]{4}\\b", + "weight": 24, + "name": "UPS tracking number" + }, + { + "pattern": "\\b9[1-5]\\d{2}[\\s-]?\\d{4}[\\s-]?\\d{4}[\\s-]?\\d{4}[\\s-]?\\d{4}(?:[\\s-]?\\d{2})?\\b", + "weight": 20, + "name": "USPS tracking number" + }, + { + "pattern": "\\b[a-z]{2}\\s?\\d{4}\\s?\\d{4}\\s?\\d\\s?gb\\b", + "weight": 18, + "name": "Royal Mail tracking number" + }, + { + "pattern": "\\b(?:wt|wgt|weight)[:\\s]+\\d+(?:\\.\\d+)?\\s?(?:lbs?|kg|oz)\\b", + "weight": 6, + "name": "Package weight line" + } + ], + "filenames": [ + { + "pattern": "shipping[-_ ]?label", + "weight": 28, + "name": "shipping label filename" + }, + { + "pattern": "(ups|usps|fedex|dhl|evri|dpd|hermes|royal[-_ ]?mail)[-_ ]?(shipping[-_ ]?)?label", + "weight": 26, + "name": "carrier label filename" + }, + { + "pattern": "(^|[\\\\/])labels?( ?\\(\\d+\\))?\\.pdf$", + "weight": 16, + "name": "label.pdf filename" + }, + { + "pattern": "way[-_ ]?bill|bill[-_ ]?of[-_ ]?lading|\\bawb\\b", + "weight": 18, + "name": "waybill / bill of lading filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "shipping label", + "weight": 14, + "name": "shipping label in title" + }, + { + "field": "any", + "pattern": "shipstation|endicia|stamps\\.com|pirate ?ship|easypost|shippo|click ?& ?drop|sendcloud|packlink", + "weight": 15, + "name": "label platform producer" + } + ], + "negatives": [ + { + "text": "return label", + "weight": 20, + "name": "return label doc" + }, + { + "text": "return authorization", + "weight": 16, + "name": "return authorization doc" + }, + { + "pattern": "\\brma\\s?(?:number|no\\.?|#)?[-\\s#:]*\\d", + "weight": 14, + "name": "RMA number present" + }, + { + "text": "packing slip", + "weight": 14, + "name": "packing slip doc" + }, + { + "text": "commercial invoice", + "weight": 12, + "name": "customs invoice doc" + }, + { + "text": "proof of delivery", + "weight": 16, + "name": "POD doc" + }, + { + "text": "delivery note", + "weight": 12, + "name": "delivery note doc" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 10 + }, + { + "signal": "address_block", + "weight": 6 + } + ] + }, + { + "id": "packing-slip", + "name": "Packing slip", + "emit": true, + "phrases": [ + { + "text": "packing slip", + "weight": 34, + "where": "title" + }, + { + "text": "packing list", + "weight": 22, + "where": "title" + }, + { + "text": "qty shipped", + "weight": 24 + }, + { + "text": "qty ordered", + "weight": 20 + }, + { + "text": "quantity shipped", + "weight": 22 + }, + { + "text": "items in this shipment", + "weight": 24 + }, + { + "text": "in this shipment", + "weight": 12 + }, + { + "text": "this is not an invoice", + "weight": 22 + }, + { + "text": "backordered", + "weight": 12 + }, + { + "text": "qty to follow", + "weight": 16 + }, + { + "text": "number of cartons", + "weight": 12 + }, + { + "text": "picked by", + "weight": 14 + }, + { + "text": "order number", + "weight": 5, + "where": "first" + }, + { + "text": "ship date", + "weight": 8, + "where": "first" + } + ], + "regexes": [ + { + "pattern": "\\b\\d{3}-\\d{7}-\\d{7}\\b", + "weight": 10, + "name": "Amazon order ID" + } + ], + "filenames": [ + { + "pattern": "packing[-_ ]?slip", + "weight": 28, + "name": "packing slip filename" + }, + { + "pattern": "packing[-_ ]?list", + "weight": 24, + "name": "packing list filename" + }, + { + "pattern": "pack[-_ ]?slip", + "weight": 20, + "name": "packslip filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "packing (slip|list)", + "weight": 14, + "name": "packing slip in title" + } + ], + "negatives": [ + { + "text": "amount due", + "weight": 14, + "name": "invoice amount due" + }, + { + "text": "subtotal", + "weight": 10, + "name": "priced document" + }, + { + "text": "unit price", + "weight": 8, + "name": "priced line items" + }, + { + "text": "commercial invoice", + "weight": 14, + "name": "customs invoice doc" + }, + { + "pattern": "\\binvoice\\s?(number|no\\.?|#)", + "weight": 8, + "name": "invoice number present" + }, + { + "text": "delivery note", + "weight": 12, + "name": "delivery note doc" + }, + { + "text": "proof of delivery", + "weight": 12, + "name": "POD doc" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 6 + }, + { + "signal": "short_doc", + "weight": 5 + } + ] + }, + { + "id": "customs-declaration", + "name": "Customs declaration", + "emit": true, + "phrases": [ + { + "text": "customs declaration", + "weight": 34, + "where": "title" + }, + { + "text": "commercial invoice", + "weight": 18, + "where": "title" + }, + { + "text": "hs code", + "weight": 20 + }, + { + "text": "hs tariff number", + "weight": 28 + }, + { + "text": "tariff code", + "weight": 18 + }, + { + "text": "country of origin", + "weight": 14 + }, + { + "text": "declared value", + "weight": 22 + }, + { + "text": "reason for export", + "weight": 26 + }, + { + "text": "for customs purposes only", + "weight": 28 + }, + { + "text": "may be opened officially", + "weight": 26 + }, + { + "text": "country of manufacture", + "weight": 18 + }, + { + "text": "air waybill", + "weight": 10 + }, + { + "text": "incoterms", + "weight": 14 + } + ], + "regexes": [ + { + "pattern": "\\bcn\\s?2[23]\\b", + "weight": 25, + "name": "CN22/CN23 form code" + }, + { + "pattern": "\\beori\\b", + "weight": 18, + "name": "EORI mention" + }, + { + "pattern": "\\beori(?:\\s?(?:number|no\\.?))?[:\\s#]*[a-z]{2}\\s?\\d{9,15}\\b", + "weight": 20, + "name": "EORI number" + }, + { + "pattern": "harmoni[sz]ed\\s+(?:system|tariff|code|commodity)", + "weight": 16, + "name": "Harmonized system mention" + }, + { + "pattern": "\\b\\d{4}\\.\\d{2}(?:\\.\\d{2,4})?\\b", + "weight": 6, + "name": "HS code format" + } + ], + "filenames": [ + { + "pattern": "cn[-_ ]?2[23]", + "weight": 28, + "name": "CN22/CN23 filename" + }, + { + "pattern": "customs", + "weight": 24, + "name": "customs filename" + }, + { + "pattern": "commercial[-_ ]?invoice", + "weight": 24, + "name": "commercial invoice filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "customs|commercial invoice", + "weight": 12, + "name": "customs in title" + } + ], + "negatives": [ + { + "text": "amount due", + "weight": 10, + "name": "payment invoice" + }, + { + "text": "remittance advice", + "weight": 12, + "name": "remittance doc" + }, + { + "text": "packing list", + "weight": 10, + "name": "export packing list doc" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 8 + }, + { + "signal": "number_table", + "weight": 4 + } + ] + }, + { + "id": "delivery-note", + "name": "Delivery note", + "emit": true, + "phrases": [ + { + "text": "delivery note", + "weight": 32, + "where": "title" + }, + { + "text": "proof of delivery", + "weight": 30, + "where": "title" + }, + { + "text": "despatch note", + "weight": 24, + "where": "title" + }, + { + "text": "dispatch note", + "weight": 20, + "where": "title" + }, + { + "text": "advice note", + "weight": 16, + "where": "title" + }, + { + "text": "goods received note", + "weight": 22 + }, + { + "text": "received by", + "weight": 16 + }, + { + "text": "delivered on", + "weight": 14 + }, + { + "text": "received in good condition", + "weight": 26 + }, + { + "text": "goods received", + "weight": 18 + }, + { + "text": "signed for by", + "weight": 22 + }, + { + "text": "signature of recipient", + "weight": 20 + }, + { + "text": "waybill number", + "weight": 12 + }, + { + "text": "consignment", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\bpod\\s?(?:no\\.?|number|#)\\b", + "weight": 15, + "name": "POD number" + }, + { + "pattern": "delivered\\s(?:on\\s)?\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{2,4}", + "weight": 14, + "name": "Delivered date line" + } + ], + "filenames": [ + { + "pattern": "delivery[-_ ]?note", + "weight": 28, + "name": "delivery note filename" + }, + { + "pattern": "proof[-_ ]?of[-_ ]?delivery", + "weight": 26, + "name": "proof of delivery filename" + }, + { + "pattern": "(^|[\\\\/_-])pod[_-]?\\d+", + "weight": 20, + "name": "POD number filename" + }, + { + "pattern": "de?spatch[-_ ]?note", + "weight": 22, + "name": "despatch note filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "delivery note|proof of delivery", + "weight": 12, + "name": "delivery note in title" + } + ], + "negatives": [ + { + "text": "packing slip", + "weight": 14, + "name": "packing slip doc" + }, + { + "text": "amount due", + "weight": 12, + "name": "payment invoice" + }, + { + "text": "unit price", + "weight": 8, + "name": "priced line items" + } + ], + "structural": [ + { + "signal": "signature_block", + "weight": 10 + }, + { + "signal": "short_doc", + "weight": 5 + }, + { + "signal": "address_block", + "weight": 4 + } + ] + }, + { + "id": "return-authorization", + "name": "Return authorization", + "emit": true, + "phrases": [ + { + "text": "return label", + "weight": 28 + }, + { + "text": "return mailing label", + "weight": 30, + "where": "title" + }, + { + "text": "return shipping label", + "weight": 30 + }, + { + "text": "returns label", + "weight": 26 + }, + { + "text": "return authorization", + "weight": 26 + }, + { + "text": "return authorisation", + "weight": 26 + }, + { + "text": "return merchandise authorization", + "weight": 32 + }, + { + "text": "rma number", + "weight": 30 + }, + { + "text": "return instructions", + "weight": 22 + }, + { + "text": "affix this label", + "weight": 24 + }, + { + "text": "prepaid return", + "weight": 22 + }, + { + "text": "returns department", + "weight": 18 + }, + { + "text": "return reason", + "weight": 14 + } + ], + "regexes": [ + { + "pattern": "\\brma\\s?(?:number|no\\.?|#)?[-\\s#:]*\\d{4,12}\\b", + "weight": 22, + "name": "RMA number" + }, + { + "pattern": "\\breturns? ?cent(?:er|re)\\b", + "weight": 14, + "name": "Returns center/centre" + }, + { + "pattern": "\\b(?:1z[0-9a-z]{16}|9[1-5]\\d{20})\\b", + "weight": 12, + "name": "Carrier tracking number" + } + ], + "filenames": [ + { + "pattern": "returns?[-_ ]?(mailing[-_ ]?)?(shipping[-_ ]?)?label", + "weight": 30, + "name": "return label filename" + }, + { + "pattern": "(^|[\\\\/_-])rma[_-]?\\d+", + "weight": 24, + "name": "RMA filename" + }, + { + "pattern": "returns?[-_ ]?auth", + "weight": 24, + "name": "return authorization filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "return label|rma", + "weight": 12, + "name": "return label in title" + } + ], + "negatives": [ + { + "text": "tax return", + "weight": 25, + "name": "tax document" + }, + { + "text": "return on investment", + "weight": 14, + "name": "business ROI doc" + }, + { + "text": "annual return", + "weight": 14, + "name": "companies house filing" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 8 + } + ] + }, + { + "id": "proposal", + "name": "Proposal", + "emit": true, + "phrases": [ + { + "text": "proposal for", + "weight": 18, + "where": "title" + }, + { + "text": "this proposal", + "weight": 16 + }, + { + "text": "scope of work", + "weight": 18 + }, + { + "text": "statement of work", + "weight": 18 + }, + { + "text": "prepared for", + "weight": 12, + "where": "first" + }, + { + "text": "deliverables", + "weight": 8 + }, + { + "text": "investment summary", + "weight": 14 + }, + { + "text": "total investment", + "weight": 12 + }, + { + "text": "proposal is valid", + "weight": 24 + }, + { + "text": "acceptance of this proposal", + "weight": 26 + }, + { + "text": "our proposed approach", + "weight": 18 + }, + { + "text": "request for proposal", + "weight": 12 + }, + { + "text": "project timeline", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "proposal\\s*(?:no\\.?|number|#|ref)\\s*[:#]?\\s*[a-z0-9/-]{2,20}", + "weight": 16, + "name": "Proposal reference number" + }, + { + "pattern": "(?:proposal|offer) (?:is |remains )?valid (?:for|until)", + "weight": 10, + "name": "Proposal validity period" + } + ], + "filenames": [ + { + "pattern": "proposal", + "weight": 26, + "name": "'proposal' in filename" + }, + { + "pattern": "\\bsow\\b|statement[-_ ]of[-_ ]work", + "weight": 18, + "name": "Statement of work filename" + }, + { + "pattern": "rfp[-_ ]?(?:response|reply)", + "weight": 20, + "name": "RFP response filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "proposal", + "weight": 14, + "name": "Proposal in PDF title" + } + ], + "negatives": [ + { + "text": "quotation", + "weight": 12, + "name": "Quote document language" + }, + { + "pattern": "invoice\\s*(?:no\\.?|number|#)", + "weight": 14, + "name": "Invoice number present" + }, + { + "text": "in witness whereof", + "weight": 16, + "name": "Contract execution language" + }, + { + "text": "purchase order", + "weight": 10, + "name": "Purchase order language" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 3 + }, + { + "signal": "signature_block", + "weight": 3 + } + ] + }, + { + "id": "presentation", + "name": "Presentation", + "emit": true, + "phrases": [ + { + "text": "agenda", + "weight": 8, + "where": "title" + }, + { + "text": "key takeaways", + "weight": 14 + }, + { + "text": "thank you for your attention", + "weight": 24 + }, + { + "text": "questions?", + "weight": 6 + }, + { + "text": "q&a", + "weight": 8 + }, + { + "text": "in this presentation", + "weight": 16 + }, + { + "text": "quarterly business review", + "weight": 16 + } + ], + "regexes": [ + { + "pattern": "\\bslide\\s+\\d{1,3}\\b", + "weight": 10, + "name": "Slide numbering" + } + ], + "filenames": [ + { + "pattern": "slides|deck|presentation", + "weight": 22, + "name": "Slides/deck in filename" + }, + { + "pattern": "\\.pptx?|powerpoint|keynote", + "weight": 24, + "name": "Exported presentation file" + }, + { + "pattern": "webinar", + "weight": 14, + "name": "Webinar filename" + } + ], + "metadata": [ + { + "field": "any", + "pattern": "powerpoint", + "weight": 18, + "name": "PowerPoint metadata" + }, + { + "field": "creator", + "pattern": "keynote|impress|google slides", + "weight": 15, + "name": "Presentation software creator" + }, + { + "field": "producer", + "pattern": "slides|keynote|impress", + "weight": 12, + "name": "Presentation software producer" + } + ], + "negatives": [ + { + "text": "minutes of the meeting", + "weight": 16, + "name": "Meeting minutes language" + }, + { + "text": "table of contents", + "weight": 8, + "name": "Report-style ToC" + } + ], + "structural": [ + { + "signal": "bullet_heavy", + "weight": 9 + } + ] + }, + { + "id": "press-release", + "name": "Press release", + "emit": true, + "phrases": [ + { + "text": "for immediate release", + "weight": 38, + "where": "first" + }, + { + "text": "press release", + "weight": 24, + "where": "title" + }, + { + "text": "media contact", + "weight": 22 + }, + { + "text": "press contact", + "weight": 20 + }, + { + "text": "today announced", + "weight": 20 + }, + { + "text": "is pleased to announce", + "weight": 16 + }, + { + "text": "notes to editors", + "weight": 26 + }, + { + "text": "embargoed until", + "weight": 26, + "where": "first" + }, + { + "text": "headquartered in", + "weight": 8 + }, + { + "text": "###", + "weight": 14 + }, + { + "text": "for further information please contact", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "\\((?:nasdaq|nyse|lse|tsx|asx|euronext|aim)\\s*:\\s*[a-z.]{1,6}\\)", + "weight": 16, + "name": "Stock exchange ticker" + }, + { + "pattern": "media (?:inquiries|enquiries)", + "weight": 14, + "name": "Media inquiries line" + }, + { + "pattern": "pr newswire|business wire|globe newswire|accesswire", + "weight": 16, + "name": "Wire service name" + } + ], + "filenames": [ + { + "pattern": "press[-_ ]?release", + "weight": 28, + "name": "Press release filename" + }, + { + "pattern": "\\bpr[-_]\\d{2,6}", + "weight": 12, + "name": "Numbered PR filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "press release", + "weight": 16, + "name": "Press release in title" + } + ], + "negatives": [ + { + "text": "in this issue", + "weight": 16, + "name": "Newsletter language" + }, + { + "text": "unsubscribe", + "weight": 12, + "name": "Newsletter footer" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 4 + } + ] + }, + { + "id": "newsletter", + "name": "Newsletter", + "emit": true, + "phrases": [ + { + "text": "in this issue", + "weight": 30 + }, + { + "text": "in this edition", + "weight": 22 + }, + { + "text": "newsletter", + "weight": 16, + "where": "title" + }, + { + "text": "unsubscribe", + "weight": 14 + }, + { + "text": "from the editor", + "weight": 20 + }, + { + "text": "upcoming events", + "weight": 14 + }, + { + "text": "dates for your diary", + "weight": 22 + }, + { + "text": "monthly newsletter", + "weight": 24 + }, + { + "text": "quarterly newsletter", + "weight": 22 + }, + { + "text": "message from the president", + "weight": 16 + }, + { + "text": "save the date", + "weight": 10 + }, + { + "text": "in our next issue", + "weight": 20 + } + ], + "regexes": [ + { + "pattern": "vol(?:ume|\\.)?\\s*\\d{1,3}[,.]?\\s*(?:issue|no\\.?|number)\\s*\\d{1,3}", + "weight": 12, + "name": "Volume/issue numbering" + }, + { + "pattern": "issue\\s*(?:#|no\\.?)?\\s*\\d{1,3}\\s*[|,•-]\\s*(?:spring|summer|autumn|fall|winter|january|february|march|april|may|june|july|august|september|october|november|december)", + "weight": 18, + "name": "Issue with season/month" + } + ], + "filenames": [ + { + "pattern": "newsletter", + "weight": 28, + "name": "Newsletter filename" + }, + { + "pattern": "bulletin", + "weight": 16, + "name": "Bulletin filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "newsletter", + "weight": 14, + "name": "Newsletter in title" + } + ], + "negatives": [ + { + "text": "for immediate release", + "weight": 20, + "name": "Press release header" + }, + { + "text": "invoice", + "weight": 10, + "name": "Billing language" + } + ], + "structural": [ + { + "signal": "url_heavy", + "weight": 3 + } + ] + }, + { + "id": "brochure", + "name": "Brochure", + "emit": true, + "phrases": [ + { + "text": "contact us today", + "weight": 18 + }, + { + "text": "call us today", + "weight": 18 + }, + { + "text": "call today", + "weight": 12 + }, + { + "text": "why choose us", + "weight": 24 + }, + { + "text": "free consultation", + "weight": 16 + }, + { + "text": "free quote", + "weight": 12 + }, + { + "text": "limited time offer", + "weight": 16 + }, + { + "text": "satisfaction guaranteed", + "weight": 16 + }, + { + "text": "features and benefits", + "weight": 16 + }, + { + "text": "no obligation", + "weight": 10 + }, + { + "text": "book your free", + "weight": 18 + }, + { + "text": "bring this flyer", + "weight": 24 + }, + { + "text": "visit our website", + "weight": 6 + }, + { + "text": "follow us on", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "(?:\\d{1,2}% off|free (?:day pass|trial|estimate|gift|tour)|offer ends|no joining fee|grand opening)", + "weight": 12, + "name": "Promotional offer" + } + ], + "filenames": [ + { + "pattern": "brochure|flyer|flier|leaflet|pamphlet", + "weight": 26, + "name": "Brochure/flyer filename" + }, + { + "pattern": "one[-_ ]?pager", + "weight": 16, + "name": "One-pager filename" + } + ], + "metadata": [], + "negatives": [ + { + "text": "for immediate release", + "weight": 15, + "name": "Press release header" + }, + { + "text": "unsubscribe", + "weight": 10, + "name": "Newsletter footer" + }, + { + "pattern": "mls\\s*#|offers over|guide price", + "weight": 12, + "name": "Property listing language" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 8 + }, + { + "signal": "url_heavy", + "weight": 3 + }, + { + "signal": "address_block", + "weight": 3 + } + ] + }, + { + "id": "meeting-minutes", + "name": "Meeting minutes", + "emit": true, + "phrases": [ + { + "text": "minutes of the meeting", + "weight": 30 + }, + { + "text": "meeting minutes", + "weight": 28, + "where": "title" + }, + { + "text": "minutes of the", + "weight": 14, + "where": "title" + }, + { + "text": "apologies for absence", + "weight": 28 + }, + { + "text": "matters arising", + "weight": 26 + }, + { + "text": "the meeting was called to order", + "weight": 28 + }, + { + "text": "motion carried", + "weight": 22 + }, + { + "text": "seconded by", + "weight": 16 + }, + { + "text": "approval of minutes", + "weight": 20 + }, + { + "text": "minutes were approved", + "weight": 20 + }, + { + "text": "date of next meeting", + "weight": 18 + }, + { + "text": "the meeting adjourned", + "weight": 22 + }, + { + "text": "any other business", + "weight": 16 + } + ], + "regexes": [ + { + "pattern": "present:\\s*[a-z]", + "weight": 8, + "name": "Present list" + }, + { + "pattern": "(?:adjourned|closed) at \\d{1,2}[:.]\\d{2}", + "weight": 16, + "name": "Adjournment time" + }, + { + "pattern": "(?:a )?quorum (?:was|being|is) (?:present|established|met)", + "weight": 14, + "name": "Quorum statement" + }, + { + "pattern": "(?:old|new|unfinished) business", + "weight": 10, + "name": "Old/new business section" + } + ], + "filenames": [ + { + "pattern": "minutes", + "weight": 24, + "name": "Minutes filename" + }, + { + "pattern": "\\bmom[-_]|minutes[-_ ]of[-_ ]meeting", + "weight": 14, + "name": "Minutes-of-meeting filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "minutes", + "weight": 12, + "name": "Minutes in title" + } + ], + "negatives": [ + { + "pattern": "plaintiff|defendant", + "weight": 14, + "name": "Court filing parties" + }, + { + "text": "for immediate release", + "weight": 12, + "name": "Press release header" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 3 + } + ] + }, + { + "id": "memo", + "name": "Memo", + "emit": true, + "phrases": [ + { + "text": "memorandum", + "weight": 26, + "where": "title" + }, + { + "text": "interoffice memorandum", + "weight": 32 + }, + { + "text": "interoffice memo", + "weight": 28 + }, + { + "text": "internal memorandum", + "weight": 26 + }, + { + "text": "this memo", + "weight": 16 + }, + { + "text": "this memorandum", + "weight": 16 + }, + { + "text": "memo", + "weight": 8, + "where": "title" + }, + { + "text": "executive briefing", + "weight": 32, + "where": "title" + }, + { + "text": "briefing note", + "weight": 28, + "where": "title" + }, + { + "text": "decision required", + "weight": 26 + }, + { + "text": "bottom line up front", + "weight": 22 + }, + { + "text": "for decision", + "weight": 20 + }, + { + "text": "key recommendation", + "weight": 20 + }, + { + "text": "action requested", + "weight": 18 + }, + { + "text": "prepared for the executive team", + "weight": 22 + }, + { + "text": "prepared for the board", + "weight": 18 + }, + { + "text": "strategic recommendation", + "weight": 16 + }, + { + "text": "summary for leadership", + "weight": 16 + }, + { + "text": "this briefing", + "weight": 14 + } + ], + "regexes": [ + { + "pattern": "\\bto:\\s*\\S[\\s\\S]{0,120}?\\bfrom:\\s*\\S[\\s\\S]{0,160}?\\b(?:date|re|subject):", + "weight": 18, + "name": "To/From/Re header block" + }, + { + "pattern": "memo\\s*(?:no\\.?|number|#)\\s*[:#]?\\s*[a-z0-9-]{2,15}", + "weight": 14, + "name": "Memo number" + }, + { + "pattern": "\\bbluf\\b", + "weight": 12, + "name": "BLUF abbreviation" + }, + { + "pattern": "decision\\s+(?:requested|required)\\s+by", + "weight": 16, + "name": "decision required by" + }, + { + "pattern": "recommend(?:ation)?:\\s", + "weight": 10, + "name": "recommendation label" + } + ], + "filenames": [ + { + "pattern": "memo(?:randum)?(?:[-_ .0-9]|$)", + "weight": 22, + "name": "Memo filename" + }, + { + "pattern": "executive[-_ ]?briefing", + "weight": 28, + "name": "executive briefing filename" + }, + { + "pattern": "exec[-_ ]?summary", + "weight": 24, + "name": "exec summary filename" + }, + { + "pattern": "board[-_ ]?brief", + "weight": 20, + "name": "board brief filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "memo(randum)?", + "weight": 12, + "name": "Memo in title" + }, + { + "field": "title", + "pattern": "executive briefing|briefing note", + "weight": 14, + "name": "executive briefing in PDF title" + } + ], + "negatives": [ + { + "text": "memorandum of understanding", + "weight": 24, + "name": "MoU contract" + }, + { + "text": "memorandum of agreement", + "weight": 22, + "name": "MoA contract" + }, + { + "pattern": "memorandum of law|memorandum in support|memorandum opinion", + "weight": 22, + "name": "Legal brief memorandum" + }, + { + "pattern": "offering memorandum|private placement memorandum", + "weight": 24, + "name": "Securities offering memorandum" + }, + { + "pattern": "\\bsent:\\s", + "weight": 14, + "name": "Email Sent header" + }, + { + "pattern": "\\bdear\\s+(?:mr|ms|mrs|dr|sir|madam)", + "weight": 12, + "name": "Letter salutation" + }, + { + "text": "key findings", + "weight": 12, + "name": "white paper/report term" + }, + { + "text": "methodology", + "weight": 10, + "name": "white paper/report term" + }, + { + "text": "survey respondents", + "weight": 10, + "name": "white paper/report term" + }, + { + "text": "funding requirements", + "weight": 10, + "name": "business plan funding term" + }, + { + "text": "market analysis", + "weight": 8, + "name": "business plan market term" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 6 + }, + { + "signal": "bullet_heavy", + "weight": 4 + } + ] + }, + { + "id": "business-plan", + "name": "Business plan", + "emit": true, + "phrases": [ + { + "text": "business plan", + "weight": 30, + "where": "title" + }, + { + "text": "executive summary", + "weight": 8, + "where": "first" + }, + { + "text": "market analysis", + "weight": 18 + }, + { + "text": "financial projections", + "weight": 22 + }, + { + "text": "competitive landscape", + "weight": 14 + }, + { + "text": "swot analysis", + "weight": 20 + }, + { + "text": "target market", + "weight": 12 + }, + { + "text": "funding requirements", + "weight": 20 + }, + { + "text": "break-even analysis", + "weight": 20 + }, + { + "text": "revenue model", + "weight": 14 + }, + { + "text": "company description", + "weight": 14 + }, + { + "text": "marketing strategy", + "weight": 10 + }, + { + "text": "management team", + "weight": 6 + }, + { + "text": "strategic plan", + "weight": 32, + "where": "title" + }, + { + "text": "corporate strategy", + "weight": 26, + "where": "title" + }, + { + "text": "strategic pillars", + "weight": 26 + }, + { + "text": "strategic priorities", + "weight": 22 + }, + { + "text": "strategic objectives", + "weight": 20 + }, + { + "text": "north star", + "weight": 20 + }, + { + "text": "strategic roadmap", + "weight": 20 + }, + { + "text": "key initiatives", + "weight": 16 + }, + { + "text": "vision statement", + "weight": 14 + }, + { + "text": "mission statement", + "weight": 12 + }, + { + "text": "long-term strategy", + "weight": 14 + }, + { + "text": "three-year strategy", + "weight": 18 + } + ], + "regexes": [ + { + "pattern": "year\\s*(?:one|two|three|1|2|3)\\s*(?:revenue|turnover|sales)", + "weight": 12, + "name": "Year-by-year projection" + }, + { + "pattern": "start[- ]?up (?:costs|funding|capital)", + "weight": 12, + "name": "Startup costs language" + }, + { + "pattern": "\\b\\d{1,2}[- ]year strategic plan", + "weight": 18, + "name": "N-year strategic plan" + }, + { + "pattern": "strategy\\s+20\\d{2}\\s*[-–]\\s*20\\d{2}", + "weight": 16, + "name": "strategy year range" + } + ], + "filenames": [ + { + "pattern": "business[-_ ]?plan", + "weight": 28, + "name": "Business plan filename" + }, + { + "pattern": "bizplan", + "weight": 20, + "name": "Bizplan filename" + }, + { + "pattern": "strategic[-_ ]?plan", + "weight": 28, + "name": "strategic plan filename" + }, + { + "pattern": "corporate[-_ ]?strategy", + "weight": 24, + "name": "corporate strategy filename" + }, + { + "pattern": "strategy[-_ ]?20\\d{2}", + "weight": 18, + "name": "strategy year filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "business plan", + "weight": 16, + "name": "Business plan in title" + }, + { + "field": "title", + "pattern": "strategic plan|corporate strategy", + "weight": 16, + "name": "strategic plan in PDF title" + } + ], + "negatives": [ + { + "text": "consolidated financial statements", + "weight": 16, + "name": "Financial report language" + }, + { + "text": "for the year ended", + "weight": 12, + "name": "Statutory accounts language" + }, + { + "text": "case study", + "weight": 12, + "name": "Case study language" + }, + { + "text": "start-up costs", + "weight": 12, + "name": "business plan startup term" + }, + { + "text": "acceptance of this proposal", + "weight": 14, + "name": "proposal acceptance clause" + }, + { + "text": "total investment", + "weight": 10, + "name": "proposal investment term" + } + ], + "structural": [ + { + "signal": "toc", + "weight": 5 + }, + { + "signal": "currency_heavy", + "weight": 3 + }, + { + "signal": "long_doc", + "weight": 3 + }, + { + "signal": "bullet_heavy", + "weight": 5 + } + ] + }, + { + "id": "white-paper", + "name": "White paper", + "emit": true, + "phrases": [ + { + "text": "white paper", + "weight": 28, + "where": "title" + }, + { + "text": "whitepaper", + "weight": 26 + }, + { + "text": "this white paper", + "weight": 26 + }, + { + "text": "key findings", + "weight": 18 + }, + { + "text": "executive summary", + "weight": 8, + "where": "first" + }, + { + "text": "methodology", + "weight": 10 + }, + { + "text": "conclusions and recommendations", + "weight": 16 + }, + { + "text": "research report", + "weight": 14 + }, + { + "text": "industry report", + "weight": 14 + }, + { + "text": "about this report", + "weight": 16 + }, + { + "text": "survey respondents", + "weight": 12 + }, + { + "text": "the findings", + "weight": 6 + }, + { + "text": "in partnership with", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "\\b\\d{2,3}% of (?:respondents|organi[sz]ations|companies|businesses|leaders)", + "weight": 16, + "name": "Survey percentage finding" + }, + { + "pattern": "surveyed\\s+[\\d,]{3,6}\\s+", + "weight": 12, + "name": "Survey sample size" + } + ], + "filenames": [ + { + "pattern": "white[-_ ]?paper", + "weight": 28, + "name": "Whitepaper filename" + }, + { + "pattern": "state[-_ ]of[-_ ]|industry[-_ ]report|research[-_ ]report", + "weight": 16, + "name": "Industry report filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "white\\s?paper", + "weight": 16, + "name": "White paper in title" + } + ], + "negatives": [ + { + "pattern": "doi\\.org|arxiv", + "weight": 16, + "name": "Academic identifiers" + }, + { + "text": "for immediate release", + "weight": 14, + "name": "Press release header" + }, + { + "text": "abstract", + "weight": 8, + "name": "Academic abstract" + }, + { + "pattern": "independent auditor", + "weight": 16, + "name": "Audited annual report language" + }, + { + "text": "for the year ended", + "weight": 12, + "name": "Statutory accounts language" + } + ], + "structural": [ + { + "signal": "toc", + "weight": 5 + }, + { + "signal": "long_doc", + "weight": 4 + } + ] + }, + { + "id": "case-study", + "name": "Case study", + "emit": true, + "phrases": [ + { + "text": "case study", + "weight": 30, + "where": "title" + }, + { + "text": "customer case study", + "weight": 32 + }, + { + "text": "success story", + "weight": 20 + }, + { + "text": "customer success story", + "weight": 26 + }, + { + "text": "customer story", + "weight": 22 + }, + { + "text": "the challenge", + "weight": 12 + }, + { + "text": "the solution", + "weight": 10 + }, + { + "text": "the results", + "weight": 10 + }, + { + "text": "about the client", + "weight": 18 + }, + { + "text": "about the customer", + "weight": 16 + }, + { + "text": "products used", + "weight": 16 + }, + { + "text": "key results", + "weight": 12 + }, + { + "text": "at a glance", + "weight": 10 + }, + { + "text": "return on investment", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "\\d{1,3}%\\s+(?:increase|reduction|decrease|improvement)\\s+in", + "weight": 14, + "name": "Quantified outcome" + } + ], + "filenames": [ + { + "pattern": "case[-_ ]?study", + "weight": 28, + "name": "Case study filename" + }, + { + "pattern": "success[-_ ]?story", + "weight": 20, + "name": "Success story filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "case study", + "weight": 16, + "name": "Case study in title" + } + ], + "negatives": [ + { + "text": "the patient", + "weight": 12, + "name": "Clinical case report" + }, + { + "pattern": "plaintiff|court of appeals", + "weight": 12, + "name": "Legal case" + }, + { + "pattern": "doi\\.org", + "weight": 10, + "name": "Academic DOI" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 3 + } + ] + }, + { + "id": "organization-chart", + "name": "Organization chart", + "emit": true, + "phrases": [ + { + "text": "organizational chart", + "weight": 34, + "where": "title" + }, + { + "text": "organisation chart", + "weight": 34, + "where": "title" + }, + { + "text": "org chart", + "weight": 28, + "where": "title" + }, + { + "text": "reporting lines", + "weight": 24 + }, + { + "text": "reporting structure", + "weight": 22 + }, + { + "text": "direct reports", + "weight": 18 + }, + { + "text": "reports to", + "weight": 14 + }, + { + "text": "chain of command", + "weight": 18 + }, + { + "text": "headcount", + "weight": 14 + }, + { + "text": "department structure", + "weight": 14 + }, + { + "text": "team structure", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "org(?:ani[sz]ational)?\\s*chart", + "weight": 16, + "name": "org chart variant" + }, + { + "pattern": "reports?\\s+to\\s+(?:the\\s+)?(?:ceo|cfo|coo|cto|vp|director|head of|chief)", + "weight": 14, + "name": "reports to executive role" + } + ], + "filenames": [ + { + "pattern": "org(?:anizational|anisational)?[-_ ]?chart", + "weight": 30, + "name": "org chart filename" + }, + { + "pattern": "reporting[-_ ]?structure", + "weight": 20, + "name": "reporting structure filename" + }, + { + "pattern": "leadership[-_ ]?structure", + "weight": 16, + "name": "leadership structure filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "organi[sz]ational chart|org chart", + "weight": 14, + "name": "org chart in PDF title" + } + ], + "negatives": [ + { + "text": "essential duties and responsibilities", + "weight": 14, + "name": "job description heading" + }, + { + "text": "job summary", + "weight": 12, + "name": "job description term" + }, + { + "text": "duties and responsibilities", + "weight": 12, + "name": "job description term" + }, + { + "text": "qualifications", + "weight": 8, + "name": "job description term" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 6 + }, + { + "signal": "bullet_heavy", + "weight": 4 + } + ] + }, + { + "id": "meeting-agenda", + "name": "Meeting agenda", + "emit": true, + "phrases": [ + { + "text": "meeting agenda", + "weight": 32, + "where": "title" + }, + { + "text": "adoption of the agenda", + "weight": 26 + }, + { + "text": "any other business", + "weight": 24 + }, + { + "text": "apologies for absence", + "weight": 24 + }, + { + "text": "call to order", + "weight": 22 + }, + { + "text": "agenda item", + "weight": 18 + }, + { + "text": "approval of minutes", + "weight": 16 + }, + { + "text": "old business", + "weight": 16 + }, + { + "text": "new business", + "weight": 14 + }, + { + "text": "discussion items", + "weight": 14 + }, + { + "text": "agenda", + "weight": 12, + "where": "title" + }, + { + "text": "next meeting", + "weight": 8 + }, + { + "text": "adjournment", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\bitem\\s+\\d{1,2}\\s*[.:)]", + "weight": 10, + "name": "numbered agenda items" + }, + { + "pattern": "agenda\\s+item\\s+requests?", + "weight": 14, + "name": "agenda item requests" + } + ], + "filenames": [ + { + "pattern": "meeting[-_ ]?agenda", + "weight": 30, + "name": "meeting agenda filename" + }, + { + "pattern": "agenda", + "weight": 26, + "name": "agenda in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "agenda", + "weight": 14, + "name": "agenda in PDF title" + } + ], + "negatives": [ + { + "text": "motion carried", + "weight": 22, + "name": "minutes narrative" + }, + { + "text": "seconded by", + "weight": 18, + "name": "minutes narrative" + }, + { + "text": "respectfully submitted", + "weight": 18, + "name": "minutes sign-off" + }, + { + "text": "the meeting was adjourned", + "weight": 18, + "name": "minutes past-tense closing" + }, + { + "pattern": "minutes\\s+(?:recorded|prepared|taken)\\s+by", + "weight": 16, + "name": "minutes recorder credit" + } + ], + "structural": [ + { + "signal": "bullet_heavy", + "weight": 6 + }, + { + "signal": "short_doc", + "weight": 4 + } + ] + }, + { + "id": "statement-of-work", + "name": "Statement of work", + "emit": true, + "phrases": [ + { + "text": "statement of work", + "weight": 34, + "where": "title" + }, + { + "text": "this statement of work", + "weight": 26 + }, + { + "text": "period of performance", + "weight": 24 + }, + { + "text": "acceptance criteria", + "weight": 18 + }, + { + "text": "scope of work", + "weight": 16 + }, + { + "text": "time and materials", + "weight": 16 + }, + { + "text": "master services agreement", + "weight": 14 + }, + { + "text": "change order", + "weight": 14 + }, + { + "text": "deliverables", + "weight": 12 + }, + { + "text": "work product", + "weight": 12 + }, + { + "text": "milestones", + "weight": 8 + }, + { + "text": "fixed fee", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "statement of work\\s*(?:no\\.?|number|#)\\s*\\d{1,4}", + "weight": 20, + "name": "SOW number" + }, + { + "pattern": "statement of work\\s*\\(\\s*[\"']?sow[\"']?\\s*\\)", + "weight": 22, + "name": "SOW defined-term clause" + }, + { + "pattern": "\\bmsa\\b", + "flags": "g", + "weight": 8, + "name": "MSA abbreviation" + } + ], + "filenames": [ + { + "pattern": "statement[-_ ]?of[-_ ]?work", + "weight": 30, + "name": "statement of work filename" + }, + { + "pattern": "(^|[^a-z])sow[-_ ]?\\d{0,3}([^a-z0-9]|$)", + "weight": 20, + "name": "SOW in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "statement of work|\\bsow\\b", + "weight": 14, + "name": "SOW in PDF title" + } + ], + "negatives": [ + { + "text": "we are pleased to submit", + "weight": 16, + "name": "proposal wording" + }, + { + "text": "this proposal", + "weight": 14, + "name": "proposal wording" + }, + { + "text": "estimate is valid for", + "weight": 14, + "name": "quote validity clause" + }, + { + "text": "employment agreement", + "weight": 14, + "name": "employment contract heading" + }, + { + "text": "in witness whereof", + "weight": 12, + "name": "general contract execution clause" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 4 + }, + { + "signal": "signature_block", + "weight": 3 + } + ] + }, + { + "id": "user-guide", + "name": "User guide", + "emit": true, + "phrases": [ + { + "text": "owner's manual", + "weight": 28, + "where": "title" + }, + { + "text": "instruction manual", + "weight": 26, + "where": "title" + }, + { + "text": "user manual", + "weight": 24, + "where": "title" + }, + { + "text": "instructions for use", + "weight": 22, + "where": "title" + }, + { + "text": "user guide", + "weight": 14, + "where": "title" + }, + { + "text": "operating instructions", + "weight": 24 + }, + { + "text": "important safety instructions", + "weight": 30, + "where": "first" + }, + { + "text": "keep these instructions", + "weight": 22 + }, + { + "text": "before first use", + "weight": 12 + }, + { + "text": "troubleshooting", + "weight": 8 + }, + { + "text": "quick start guide", + "weight": 18, + "where": "title" + }, + { + "text": "risk of electric shock", + "weight": 20 + }, + { + "text": "can be used by children aged", + "weight": 26 + }, + { + "text": "supply cord is damaged", + "weight": 24 + } + ], + "regexes": [ + { + "pattern": "model\\s?(no\\.?|number|#)?\\s?:?\\s?[a-z]{0,4}-?[0-9][a-z0-9/-]{2,}", + "flags": "gi", + "weight": 12, + "name": "model number" + }, + { + "pattern": "(do not|never)\\s(disassemble|immerse|attempt to repair|expose this)", + "flags": "gi", + "weight": 14, + "name": "safety warning" + }, + { + "pattern": "owner[’']?s\\s(manual|guide)", + "flags": "gi", + "weight": 18, + "name": "owners manual apostrophe variants" + } + ], + "filenames": [ + { + "pattern": "(user|owner'?s?|instruction|operating)[\\s_-]*(manual|guide|instructions)", + "weight": 26, + "name": "manual filename" + }, + { + "pattern": "manual", + "weight": 15, + "name": "manual keyword" + }, + { + "pattern": "quick[\\s_-]*start", + "weight": 16, + "name": "quick start filename" + }, + { + "pattern": "(^|[^a-z0-9])ifu([^a-z0-9]|$)|instructions[\\s_-]?for[\\s_-]?use", + "weight": 16, + "name": "instructions for use filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "manual|user guide", + "weight": 12, + "name": "manual in title" + } + ], + "negatives": [ + { + "text": "safety data sheet", + "weight": 28, + "name": "sds heading" + }, + { + "text": "absolute maximum ratings", + "weight": 16, + "name": "datasheet section" + }, + { + "text": "api reference", + "weight": 14, + "name": "software docs" + }, + { + "text": "release notes", + "weight": 12, + "name": "software release notes" + } + ], + "structural": [ + { + "signal": "toc", + "weight": 6 + } + ] + }, + { + "id": "datasheet", + "name": "Datasheet", + "emit": true, + "phrases": [ + { + "text": "absolute maximum ratings", + "weight": 34 + }, + { + "text": "electrical characteristics", + "weight": 28 + }, + { + "text": "recommended operating conditions", + "weight": 28 + }, + { + "text": "typical application circuit", + "weight": 26 + }, + { + "text": "typical performance characteristics", + "weight": 24 + }, + { + "text": "pin configuration", + "weight": 24 + }, + { + "text": "pin description", + "weight": 20 + }, + { + "text": "technical data sheet", + "weight": 20, + "where": "title" + }, + { + "text": "datasheet", + "weight": 18, + "where": "title" + }, + { + "text": "ordering information", + "weight": 14 + }, + { + "text": "package outline", + "weight": 14 + }, + { + "text": "operating temperature range", + "weight": 14 + }, + { + "text": "ripple & noise", + "weight": 20 + }, + { + "text": "derating curve", + "weight": 18 + } + ], + "regexes": [ + { + "pattern": "\\b\\d+(\\.\\d+)?\\s?(mv|µa|ua|pf|nf|µf|uf|mhz|khz|ghz|kω|mω|vdc|vac)", + "flags": "gi", + "weight": 14, + "name": "electrical units" + }, + { + "pattern": "[-–]\\s?\\d{1,3}\\s?°?\\s?c?\\s?(to|~)\\s?\\+?\\d{1,3}\\s?°\\s?c", + "flags": "gi", + "weight": 14, + "name": "temperature range" + } + ], + "filenames": [ + { + "pattern": "datasheet", + "weight": 24, + "name": "datasheet filename" + }, + { + "pattern": "spec[\\s_-]?sheet", + "weight": 18, + "name": "spec sheet filename" + }, + { + "pattern": "(^|[^a-z0-9])tds([^a-z0-9]|$)", + "weight": 16, + "name": "tds filename" + }, + { + "pattern": "(^|[^a-z0-9])ds\\d{4,}", + "weight": 15, + "name": "vendor ds-number filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "datasheet|data sheet", + "weight": 12, + "name": "datasheet title" + }, + { + "field": "any", + "pattern": "texas instruments|stmicroelectronics|analog devices|nxp semiconductors|infineon|microchip technology|onsemi|renesas|vishay|rohm", + "weight": 14, + "name": "semiconductor vendor" + } + ], + "negatives": [ + { + "text": "safety data sheet", + "weight": 30, + "name": "sds heading" + }, + { + "text": "first aid measures", + "weight": 20, + "name": "sds section" + }, + { + "text": "hazards identification", + "weight": 16, + "name": "sds section 2" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 8 + } + ] + }, + { + "id": "technical-specification", + "name": "Technical specification", + "emit": true, + "phrases": [ + { + "text": "technical specification", + "weight": 26, + "where": "title" + }, + { + "text": "requirements specification", + "weight": 28, + "where": "title" + }, + { + "text": "functional requirements", + "weight": 22 + }, + { + "text": "non-functional requirements", + "weight": 28 + }, + { + "text": "external interface requirements", + "weight": 20 + }, + { + "text": "this document specifies", + "weight": 24 + }, + { + "text": "normative references", + "weight": 26 + }, + { + "text": "shall comply with", + "weight": 14 + }, + { + "text": "issued for construction", + "weight": 12 + }, + { + "text": "revision history", + "weight": 8 + }, + { + "text": "acceptance criteria", + "weight": 10 + }, + { + "text": "intended audience", + "weight": 8 + }, + { + "text": "for the purposes of this document", + "weight": 16 + }, + { + "text": "terms and definitions", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "the\\s(system|software|application|device|product|supplier|contractor|equipment|installation)\\sshall\\b", + "flags": "gi", + "weight": 16, + "name": "shall requirement" + }, + { + "pattern": "\\b(req|srs|nfr)[-_]\\d{1,4}\\b", + "flags": "gi", + "weight": 16, + "name": "requirement id" + }, + { + "pattern": "\\b(iso|iec|astm|ansi|bs en)\\s?\\d{2,5}\\b", + "flags": "gi", + "weight": 12, + "name": "standards reference" + } + ], + "filenames": [ + { + "pattern": "(technical|functional|requirements?|system)[\\s_-]?spec", + "weight": 22, + "name": "spec filename" + }, + { + "pattern": "specification", + "weight": 15, + "name": "specification keyword" + }, + { + "pattern": "(^|[^a-z0-9])srs([^a-z0-9]|$)", + "weight": 16, + "name": "srs filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "specification|requirements", + "weight": 10, + "name": "specification title" + } + ], + "negatives": [ + { + "text": "in witness whereof", + "weight": 22, + "name": "contract execution" + }, + { + "text": "what is claimed is", + "weight": 20, + "name": "patent claims" + }, + { + "text": "absolute maximum ratings", + "weight": 14, + "name": "datasheet section" + } + ], + "structural": [ + { + "signal": "toc", + "weight": 5 + }, + { + "signal": "long_doc", + "weight": 3 + } + ] + }, + { + "id": "patent", + "name": "Patent", + "emit": true, + "phrases": [ + { + "text": "united states patent", + "weight": 34, + "where": "first" + }, + { + "text": "patent application publication", + "weight": 34, + "where": "first" + }, + { + "text": "european patent application", + "weight": 30, + "where": "first" + }, + { + "text": "date of patent", + "weight": 24, + "where": "first" + }, + { + "text": "field of the invention", + "weight": 32 + }, + { + "text": "background of the invention", + "weight": 30 + }, + { + "text": "summary of the invention", + "weight": 30 + }, + { + "text": "brief description of the drawings", + "weight": 32 + }, + { + "text": "what is claimed is", + "weight": 36 + }, + { + "text": "the present invention", + "weight": 22 + }, + { + "text": "preferred embodiment", + "weight": 24 + }, + { + "text": "prior art", + "weight": 20 + }, + { + "text": "in one embodiment", + "weight": 18 + }, + { + "text": "references cited", + "weight": 14 + } + ], + "regexes": [ + { + "pattern": "\\bus\\s?\\d{1,2},\\d{3},\\d{3}\\s?[ab]\\d?\\b", + "flags": "gi", + "weight": 20, + "name": "us patent number" + }, + { + "pattern": "\\bus\\s?\\d{4}/\\d{7}\\s?a\\d\\b", + "flags": "gi", + "weight": 18, + "name": "us publication number" + }, + { + "pattern": "\\bep\\s?\\d[\\s.]?\\d{3}[\\s.]?\\d{3}\\s?[ab]\\d\\b", + "flags": "gi", + "weight": 18, + "name": "ep publication number" + }, + { + "pattern": "\\bwo\\s?\\d{4}/\\d{5,6}(\\s?a\\d)?\\b", + "flags": "gi", + "weight": 14, + "name": "wo publication number" + }, + { + "pattern": "(of|according to)\\sclaim\\s\\d{1,3}", + "flags": "gi", + "weight": 18, + "name": "claim reference" + }, + { + "pattern": "int\\.?\\s?cl\\.?\\s?:?\\s?[a-h]\\d{2}[a-z]", + "flags": "gi", + "weight": 18, + "name": "ipc classification" + } + ], + "filenames": [ + { + "pattern": "patent", + "weight": 22, + "name": "patent keyword" + }, + { + "pattern": "(^|[^a-z0-9])us\\d{7,8}[ab]?\\d?", + "weight": 20, + "name": "us patent number filename" + }, + { + "pattern": "(^|[^a-z0-9])ep\\d{6,7}", + "weight": 18, + "name": "ep number filename" + }, + { + "pattern": "(^|[^a-z0-9])wo\\d{4}", + "weight": 16, + "name": "wo publication filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "patent", + "weight": 12, + "name": "patent in title" + } + ], + "negatives": [ + { + "text": "in witness whereof", + "weight": 18, + "name": "patent license contract" + } + ], + "structural": [ + { + "signal": "long_doc", + "weight": 3 + }, + { + "signal": "references_section", + "weight": 3 + } + ] + }, + { + "id": "api-documentation", + "name": "API documentation", + "emit": true, + "phrases": [ + { + "text": "api reference", + "weight": 26, + "where": "title" + }, + { + "text": "release notes", + "weight": 24, + "where": "title" + }, + { + "text": "developer guide", + "weight": 22, + "where": "title" + }, + { + "text": "administration guide", + "weight": 18, + "where": "title" + }, + { + "text": "request body", + "weight": 20 + }, + { + "text": "query parameters", + "weight": 20 + }, + { + "text": "response body", + "weight": 18 + }, + { + "text": "environment variable", + "weight": 16 + }, + { + "text": "api key", + "weight": 14 + }, + { + "text": "changelog", + "weight": 18 + }, + { + "text": "known issues", + "weight": 14 + }, + { + "text": "localhost", + "weight": 12 + }, + { + "text": "configuration file", + "weight": 12 + }, + { + "text": "command line", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\b(get|post|put|delete|patch)\\s/[a-z0-9_{}/.:-]{3,}", + "flags": "gi", + "weight": 18, + "name": "http endpoint" + }, + { + "pattern": "\\$\\s?(sudo|npm|pip|git|docker|curl|apt|brew|systemctl|make)\\b", + "flags": "gi", + "weight": 16, + "name": "shell command" + }, + { + "pattern": "\"[a-z_]{2,24}\"\\s?:\\s?(\"|\\d|true|false|\\{|\\[)", + "flags": "g", + "weight": 12, + "name": "json snippet" + }, + { + "pattern": "\\bv\\d+\\.\\d+\\.\\d+\\b", + "flags": "gi", + "weight": 12, + "name": "semantic version" + } + ], + "filenames": [ + { + "pattern": "(api|developer|dev|admin)[\\s_-]?(reference|guide|docs|documentation)", + "weight": 24, + "name": "api guide filename" + }, + { + "pattern": "release[\\s_-]?notes|changelog", + "weight": 22, + "name": "release notes filename" + }, + { + "pattern": "readme", + "weight": 16, + "name": "readme filename" + } + ], + "metadata": [ + { + "field": "any", + "pattern": "sphinx|mkdocs|doxygen|gitbook|asciidoctor|docusaurus", + "weight": 14, + "name": "docs generator" + } + ], + "negatives": [ + { + "text": "important safety instructions", + "weight": 16, + "name": "hardware manual" + }, + { + "text": "safety data sheet", + "weight": 20, + "name": "sds heading" + } + ], + "structural": [ + { + "signal": "url_heavy", + "weight": 6 + }, + { + "signal": "toc", + "weight": 3 + } + ] + }, + { + "id": "safety-data-sheet", + "name": "Safety data sheet", + "emit": true, + "phrases": [ + { + "text": "safety data sheet", + "weight": 38, + "where": "title" + }, + { + "text": "hazards identification", + "weight": 30 + }, + { + "text": "hazard identification", + "weight": 26 + }, + { + "text": "first aid measures", + "weight": 28 + }, + { + "text": "firefighting measures", + "weight": 26 + }, + { + "text": "accidental release measures", + "weight": 32 + }, + { + "text": "handling and storage", + "weight": 20 + }, + { + "text": "exposure controls", + "weight": 24 + }, + { + "text": "toxicological information", + "weight": 28 + }, + { + "text": "ecological information", + "weight": 24 + }, + { + "text": "disposal considerations", + "weight": 26 + }, + { + "text": "precautionary statements", + "weight": 24 + }, + { + "text": "signal word", + "weight": 22 + }, + { + "text": "flash point", + "weight": 14 + } + ], + "regexes": [ + { + "pattern": "\\b\\d{2,7}-\\d{2}-\\d\\b", + "flags": "g", + "weight": 14, + "name": "cas number" + }, + { + "pattern": "\\b[hp][1-5]\\d{2}\\b", + "flags": "gi", + "weight": 12, + "name": "ghs hazard code" + }, + { + "pattern": "regulation\\s\\(ec\\)\\s?no\\.?\\s?1907/2006", + "flags": "gi", + "weight": 20, + "name": "reach regulation" + }, + { + "pattern": "\\bun\\s?\\d{4}\\b", + "flags": "gi", + "weight": 12, + "name": "un number" + } + ], + "filenames": [ + { + "pattern": "(^|[^a-z0-9])m?sds([^a-z0-9]|$)", + "weight": 26, + "name": "sds filename" + }, + { + "pattern": "safety[\\s_-]?data[\\s_-]?sheet", + "weight": 28, + "name": "safety data sheet filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "safety data sheet|msds", + "weight": 16, + "name": "sds title" + } + ], + "negatives": [ + { + "text": "absolute maximum ratings", + "weight": 16, + "name": "datasheet section" + }, + { + "text": "electrical characteristics", + "weight": 12, + "name": "datasheet section" + }, + { + "text": "important safety instructions", + "weight": 12, + "name": "appliance manual heading" + } + ], + "structural": [ + { + "signal": "number_table", + "weight": 3 + } + ] + }, + { + "id": "warranty-document", + "name": "Warranty document", + "emit": true, + "phrases": [ + { + "text": "warranty certificate", + "weight": 32, + "where": "title" + }, + { + "text": "warranty card", + "weight": 30, + "where": "title" + }, + { + "text": "limited warranty", + "weight": 28 + }, + { + "text": "this warranty covers", + "weight": 26 + }, + { + "text": "this warranty does not cover", + "weight": 26 + }, + { + "text": "warranty period", + "weight": 24 + }, + { + "text": "warranty registration", + "weight": 24 + }, + { + "text": "manufacturer's warranty", + "weight": 22 + }, + { + "text": "from the date of purchase", + "weight": 20 + }, + { + "text": "proof of purchase", + "weight": 18 + }, + { + "text": "implied warranties", + "weight": 16 + }, + { + "text": "normal wear and tear", + "weight": 14 + }, + { + "text": "extended warranty", + "weight": 14 + }, + { + "text": "repair or replace", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "\\b\\d{1,2}[- ]year(?:s)?\\s+(?:limited\\s+)?(?:warranty|guarantee)", + "weight": 18, + "name": "N-year warranty" + }, + { + "pattern": "warrant(?:y|ies)\\s+(?:is|are)\\s+void", + "weight": 14, + "name": "warranty void clause" + } + ], + "filenames": [ + { + "pattern": "warranty", + "weight": 28, + "name": "warranty in filename" + }, + { + "pattern": "guarantee", + "weight": 18, + "name": "guarantee in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "warranty|guarantee", + "weight": 12, + "name": "warranty in PDF title" + } + ], + "negatives": [ + { + "text": "policy number", + "weight": 16, + "name": "insurance policy field" + }, + { + "text": "certificate of insurance", + "weight": 14, + "name": "COI heading" + }, + { + "text": "troubleshooting", + "weight": 14, + "name": "user manual section" + }, + { + "text": "premium", + "weight": 12, + "name": "insurance term" + }, + { + "text": "table of contents", + "weight": 10, + "name": "manual TOC" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 4 + } + ] + }, + { + "id": "license-agreement", + "name": "License agreement", + "emit": true, + "phrases": [ + { + "text": "license certificate", + "weight": 30, + "where": "title" + }, + { + "text": "license key", + "weight": 28 + }, + { + "text": "product key", + "weight": 26 + }, + { + "text": "activation code", + "weight": 26 + }, + { + "text": "activation instructions", + "weight": 24 + }, + { + "text": "perpetual license", + "weight": 22 + }, + { + "text": "number of seats", + "weight": 20 + }, + { + "text": "maintenance expires", + "weight": 20 + }, + { + "text": "licensed to", + "weight": 18 + }, + { + "text": "how to activate", + "weight": 18 + }, + { + "text": "your license", + "weight": 14 + }, + { + "text": "license type", + "weight": 14 + }, + { + "text": "serial number", + "weight": 6 + } + ], + "regexes": [ + { + "pattern": "\\b[a-z0-9]{4,5}(?:-[a-z0-9]{4,5}){3,6}\\b", + "flags": "gi", + "weight": 14, + "name": "hyphenated product key" + }, + { + "pattern": "(?:license|licence|product|activation)\\s*(?:key|code)\\s*:?\\s*[a-z0-9][a-z0-9-]{9,39}", + "weight": 18, + "name": "license key field" + } + ], + "filenames": [ + { + "pattern": "licen[cs]e[-_ ]?key", + "weight": 28, + "name": "license key filename" + }, + { + "pattern": "licen[cs]e[-_ ]?certificate", + "weight": 26, + "name": "license certificate filename" + }, + { + "pattern": "product[-_ ]?key", + "weight": 24, + "name": "product key filename" + }, + { + "pattern": "activation", + "weight": 16, + "name": "activation in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "license (certificate|key)", + "weight": 14, + "name": "license certificate in PDF title" + } + ], + "negatives": [ + { + "text": "end user license agreement", + "weight": 20, + "name": "EULA legal text" + }, + { + "text": "by installing or using", + "weight": 18, + "name": "EULA acceptance clause" + }, + { + "text": "department of motor vehicles", + "weight": 14, + "name": "government license context" + }, + { + "text": "terms and conditions", + "weight": 10, + "name": "T&C legal doc" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 8 + }, + { + "signal": "form_like", + "weight": 3 + } + ] + }, + { + "id": "email-thread", + "name": "Email thread", + "emit": true, + "phrases": [ + { + "text": "original message", + "weight": 16 + }, + { + "text": "forwarded message", + "weight": 18 + }, + { + "text": "begin forwarded message", + "weight": 24 + }, + { + "text": "wrote:", + "weight": 6 + }, + { + "text": "sent from my iphone", + "weight": 24 + }, + { + "text": "reply all", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "from:\\s?[^\\n]{0,60}@[a-z0-9.\\-]+", + "flags": "gi", + "weight": 14, + "name": "From: with email address", + "where": "first" + }, + { + "pattern": "sent:\\s?(monday|tuesday|wednesday|thursday|friday|saturday|sunday)", + "flags": "gi", + "weight": 20, + "name": "Sent: weekday header" + }, + { + "pattern": "subject:\\s?(re|fw|fwd):", + "flags": "gi", + "weight": 22, + "name": "Re:/Fwd: subject header" + }, + { + "pattern": "<[a-z0-9._%+\\-]+@[a-z0-9.\\-]+\\.[a-z]{2,}>", + "flags": "gi", + "weight": 8, + "name": "Angle-bracket email address" + }, + { + "pattern": "\\bon [a-z]{3}, (\\d{1,2} [a-z]{3}|[a-z]{3} \\d{1,2}),? \\d{4}[^\\n]{0,60}wrote:", + "flags": "gi", + "weight": 16, + "name": "Quoted reply header" + }, + { + "pattern": "sent from my (iphone|ipad|galaxy|android|mobile device)|get outlook for (ios|android)", + "flags": "gi", + "weight": 18, + "name": "Mobile signature" + } + ], + "filenames": [ + { + "pattern": "^gmail - ", + "weight": 26, + "name": "Gmail print filename" + }, + { + "pattern": "(^|[\\\\/])(fw|fwd|re)[_ ]", + "weight": 16, + "name": "FW/RE filename prefix" + }, + { + "pattern": "email", + "weight": 14, + "name": "Email in filename" + } + ], + "metadata": [ + { + "field": "creator", + "pattern": "outlook|thunderbird", + "weight": 14, + "name": "Mail client creator" + }, + { + "field": "title", + "pattern": "^(re|fw|fwd):", + "weight": 14, + "name": "Re:/Fwd: in PDF title" + }, + { + "field": "title", + "pattern": "^gmail - ", + "weight": 16, + "name": "Gmail print title" + } + ], + "negatives": [ + { + "text": "memorandum", + "weight": 18, + "name": "Memo heading" + }, + { + "text": "for immediate release", + "weight": 20, + "name": "Press release wording" + } + ], + "structural": [ + { + "signal": "email_headers", + "weight": 12 + } + ] + }, + { + "id": "recipe", + "name": "Recipe", + "phrases": [ + { + "text": "ingredients", + "weight": 10, + "where": "title" + }, + { + "text": "preheat the oven", + "weight": 28 + }, + { + "text": "preheat oven to", + "weight": 26 + }, + { + "text": "prep time", + "weight": 22 + }, + { + "text": "cook time", + "weight": 20 + }, + { + "text": "servings", + "weight": 10 + }, + { + "text": "until golden brown", + "weight": 18 + }, + { + "text": "over medium heat", + "weight": 16 + }, + { + "text": "in a large bowl", + "weight": 14 + }, + { + "text": "season with salt and pepper", + "weight": 18 + }, + { + "text": "method", + "weight": 4, + "where": "title" + } + ], + "regexes": [ + { + "pattern": "\\b\\d+ ?(cups?|tablespoons?|teaspoons?|tbsp|tsp)\\b", + "flags": "gi", + "weight": 16, + "name": "Cooking measurement" + }, + { + "pattern": "\\d{2,3} ?° ?[cf]\\b|\\b\\d{2,3} ?degrees ?[cf]?\\b", + "flags": "gi", + "weight": 10, + "name": "Oven temperature" + }, + { + "pattern": "\\bserves \\d{1,2}\\b", + "flags": "gi", + "weight": 14, + "name": "Serves N" + }, + { + "pattern": "\\bbring to (a|the) (boil|simmer)\\b", + "flags": "gi", + "weight": 12, + "name": "Bring to a boil/simmer" + } + ], + "filenames": [ + { + "pattern": "recipe", + "weight": 25, + "name": "Recipe in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "recipe", + "weight": 12, + "name": "Recipe in PDF title" + } + ], + "negatives": [ + { + "text": "safety data sheet", + "weight": 20, + "name": "SDS ingredients section" + }, + { + "text": "please inform your server", + "weight": 15, + "name": "Menu wording" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 4 + }, + { + "signal": "bullet_heavy", + "weight": 4 + } + ], + "emit": false + }, + { + "id": "menu", + "name": "Menu", + "phrases": [ + { + "text": "starters", + "weight": 14, + "where": "title" + }, + { + "text": "appetizers", + "weight": 16, + "where": "title" + }, + { + "text": "main courses", + "weight": 16, + "where": "title" + }, + { + "text": "mains", + "weight": 10, + "where": "title" + }, + { + "text": "desserts", + "weight": 12, + "where": "title" + }, + { + "text": "wine list", + "weight": 18 + }, + { + "text": "tasting menu", + "weight": 16 + }, + { + "text": "please inform your server", + "weight": 26 + }, + { + "text": "ask your server", + "weight": 20 + }, + { + "text": "market price", + "weight": 18 + }, + { + "text": "food allergy or intolerance", + "weight": 22 + }, + { + "text": "allergen information is available", + "weight": 22 + }, + { + "text": "service charge", + "weight": 8 + } + ], + "regexes": [ + { + "pattern": "\\((v|vg|ve|gf)\\)", + "flags": "g", + "weight": 10, + "name": "Dietary marker (v)/(vg)" + }, + { + "pattern": "\\.{3,} ?(£|\\$|€)?\\d{1,3}(\\.\\d{2})?", + "flags": "g", + "weight": 12, + "name": "Dotted price leader" + } + ], + "filenames": [ + { + "pattern": "menu", + "weight": 24, + "name": "Menu in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "menu", + "weight": 12, + "name": "Menu in PDF title" + } + ], + "negatives": [ + { + "pattern": "\\b\\d+ ?(cups?|tablespoons?|teaspoons?|tbsp|tsp)\\b", + "flags": "gi", + "weight": 14, + "name": "Cooking measurements (recipe)" + }, + { + "text": "invoice number", + "weight": 15, + "name": "Invoice field" + }, + { + "text": "subtotal", + "weight": 12, + "name": "Receipt field" + } + ], + "structural": [ + { + "signal": "currency_heavy", + "weight": 6 + }, + { + "signal": "short_doc", + "weight": 4 + } + ], + "emit": false + }, + { + "id": "book-ebook", + "name": "Book / eBook", + "phrases": [ + { + "text": "no part of this publication may be reproduced", + "weight": 24, + "where": "first" + }, + { + "text": "no part of this book", + "weight": 26, + "where": "first" + }, + { + "text": "first published", + "weight": 20, + "where": "first" + }, + { + "text": "first edition", + "weight": 14, + "where": "first" + }, + { + "text": "printed in the united states of america", + "weight": 18, + "where": "first" + }, + { + "text": "library of congress", + "weight": 22, + "where": "first" + }, + { + "text": "catalogue record for this book", + "weight": 26, + "where": "first" + }, + { + "text": "this is a work of fiction", + "weight": 22, + "where": "first" + }, + { + "text": "about the author", + "weight": 14 + }, + { + "text": "prologue", + "weight": 12 + }, + { + "text": "epilogue", + "weight": 14 + }, + { + "text": "all rights reserved", + "weight": 6, + "where": "first" + }, + { + "text": "table of contents", + "weight": 5 + } + ], + "regexes": [ + { + "pattern": "isbn(-1[03])?:? ?[0-9][0-9\\- ]{8,16}[0-9x]", + "flags": "gi", + "weight": 24, + "name": "ISBN" + }, + { + "pattern": "\\bchapter (one|two|three|\\d{1,2})\\b", + "flags": "gi", + "weight": 10, + "name": "Chapter heading" + } + ], + "filenames": [ + { + "pattern": "ebook|e-book", + "weight": 22, + "name": "eBook in filename" + }, + { + "pattern": "(^|[^a-z])book([^a-z]|$)", + "weight": 10, + "name": "Book in filename" + } + ], + "metadata": [ + { + "field": "producer", + "pattern": "calibre", + "weight": 18, + "name": "calibre producer (ebook tool)" + } + ], + "negatives": [ + { + "text": "a thesis submitted", + "weight": 25, + "name": "Thesis wording" + }, + { + "text": "user manual", + "weight": 18, + "name": "User manual" + }, + { + "text": "instruction manual", + "weight": 15, + "name": "Instruction manual" + }, + { + "pattern": "doi:? ?10\\.\\d{4}", + "flags": "gi", + "weight": 15, + "name": "DOI (academic paper)" + } + ], + "structural": [ + { + "signal": "long_doc", + "weight": 12 + }, + { + "signal": "toc", + "weight": 5 + } + ], + "emit": false + }, + { + "id": "magazine-article", + "name": "Magazine article", + "phrases": [ + { + "text": "continued on page", + "weight": 24 + }, + { + "text": "photography by", + "weight": 20 + }, + { + "text": "words by", + "weight": 22 + }, + { + "text": "illustration by", + "weight": 16 + }, + { + "text": "contributing editor", + "weight": 20 + }, + { + "text": "editor-in-chief", + "weight": 14 + }, + { + "text": "in this issue", + "weight": 20 + }, + { + "text": "cover story", + "weight": 18 + }, + { + "text": "exclusive interview", + "weight": 16 + } + ], + "regexes": [ + { + "pattern": "\\bissue \\d{1,3}\\b", + "flags": "gi", + "weight": 8, + "name": "Issue number" + }, + { + "pattern": "vol\\.? ?\\d{1,3},? (no\\.? ?\\d{1,3}|issue \\d{1,3})", + "flags": "gi", + "weight": 8, + "name": "Volume/issue" + } + ], + "filenames": [ + { + "pattern": "magazine", + "weight": 20, + "name": "Magazine in filename" + }, + { + "pattern": "article|feature", + "weight": 10, + "name": "Article/feature in filename" + } + ], + "metadata": [ + { + "field": "creator", + "pattern": "indesign", + "weight": 8, + "name": "InDesign creator" + } + ], + "negatives": [ + { + "pattern": "doi:? ?10\\.\\d{4}", + "flags": "gi", + "weight": 20, + "name": "DOI (academic paper)" + }, + { + "text": "abstract", + "weight": 12, + "name": "Abstract (academic)" + }, + { + "text": "for immediate release", + "weight": 25, + "name": "Press release wording" + }, + { + "text": "newsletter", + "weight": 18, + "name": "Newsletter" + } + ], + "structural": [], + "emit": false + }, + { + "id": "form", + "name": "Form", + "emit": true, + "phrases": [ + { + "text": "please print clearly", + "weight": 24 + }, + { + "text": "check all that apply", + "weight": 24 + }, + { + "text": "please tick", + "weight": 18 + }, + { + "text": "for office use only", + "weight": 24 + }, + { + "text": "please complete all sections", + "weight": 22 + }, + { + "text": "do not write below this line", + "weight": 26 + }, + { + "text": "block capitals", + "weight": 20 + }, + { + "text": "attach additional sheets", + "weight": 20 + }, + { + "text": "permission slip", + "weight": 24 + }, + { + "text": "i give permission for my child", + "weight": 26 + }, + { + "text": "signature of applicant", + "weight": 14 + }, + { + "text": "parent/guardian", + "weight": 14 + }, + { + "text": "emergency contact", + "weight": 10 + }, + { + "text": "date of birth", + "weight": 5 + } + ], + "regexes": [ + { + "pattern": "_{6,}", + "flags": "g", + "weight": 12, + "name": "Fill-in blanks" + }, + { + "pattern": "☐|❏|❑|\\[ ?\\]", + "flags": "g", + "weight": 12, + "name": "Checkbox marks" + }, + { + "pattern": "_{2,} ?\\/ ?_{2,}", + "flags": "g", + "weight": 12, + "name": "Date fill-in blank" + } + ], + "filenames": [ + { + "pattern": "permission[_\\- ]slip|consent[_\\- ]form", + "weight": 24, + "name": "Permission/consent form filename" + }, + { + "pattern": "form", + "weight": 12, + "name": "Form in filename" + }, + { + "pattern": "application", + "weight": 10, + "name": "Application in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "\\bform\\b", + "weight": 8, + "name": "Form in PDF title" + } + ], + "negatives": [ + { + "pattern": "omb (approval )?(control )?(no\\.?|number|#):? ?\\d{4}-\\d{4}", + "flags": "gi", + "weight": 28, + "name": "OMB control number (US federal form)" + }, + { + "pattern": "internal revenue service|\\birs\\b|form 1040|form w-[249]\\b|\\bhmrc\\b|self assessment|employer identification number|\\bp45\\b|\\bp60\\b", + "flags": "gi", + "weight": 24, + "name": "Tax authority form" + }, + { + "pattern": "uscis|department of homeland security|form (i|n|g|ds|ar)-\\d{1,4}\\b|alien registration number|permanent resident", + "flags": "gi", + "weight": 24, + "name": "Immigration/government form" + }, + { + "pattern": "\\(rev\\.? ?(\\d{1,2}[-/.]\\d{2,4}|[a-z]{3,9} \\d{4})\\)|cat\\.? no\\.? ?\\d{4,6}[a-z]?", + "flags": "gi", + "weight": 16, + "name": "Official form revision/catalog stamp" + }, + { + "pattern": "\\b(department|bureau|ministry|division|office|borough) of (public safety|health|state|motor vehicles|homeland security|veterans affairs|vital statistics|revenue|labor|labour|transportation|transport|human services|social services|home affairs|the interior|consular affairs)\\b|driver license division|driving licence", + "flags": "gi", + "weight": 22, + "name": "Government agency letterhead" + }, + { + "pattern": "national insurance number|social security number|\\bssn\\b|\\bnino\\b", + "flags": "gi", + "weight": 16, + "name": "National ID number field" + }, + { + "pattern": "penalt(y|ies) of perjury", + "flags": "gi", + "weight": 18, + "name": "Perjury declaration (tax/gov form)" + }, + { + "text": "policy number", + "weight": 12, + "name": "Insurance claim form" + }, + { + "pattern": "\\bform\\s+[a-z]{1,4}[- ]?\\d{1,4}[a-z]?\\b", + "flags": "gi", + "weight": 20, + "name": "Official agency form number" + } + ], + "structural": [ + { + "signal": "form_like", + "weight": 12 + }, + { + "signal": "short_doc", + "weight": 3 + } + ] + }, + { + "id": "invitation", + "name": "Invitation", + "emit": true, + "phrases": [ + { + "text": "you are cordially invited", + "weight": 34 + }, + { + "text": "request the pleasure of your company", + "weight": 34 + }, + { + "text": "cordially invite", + "weight": 28 + }, + { + "text": "celebrate the marriage of", + "weight": 28 + }, + { + "text": "save the date", + "weight": 26 + }, + { + "text": "reception to follow", + "weight": 26 + }, + { + "text": "you are invited", + "weight": 22 + }, + { + "text": "kindly respond by", + "weight": 22 + }, + { + "text": "invites you to", + "weight": 20 + }, + { + "text": "rsvp", + "weight": 18 + }, + { + "text": "guest of honor", + "weight": 14 + }, + { + "text": "invitation", + "weight": 10, + "where": "title" + }, + { + "text": "dress code", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "rsvp\\s+(?:by|to|at|before)", + "weight": 16, + "name": "RSVP by/to" + }, + { + "pattern": "black[- ]tie", + "weight": 12, + "name": "black tie dress code" + } + ], + "filenames": [ + { + "pattern": "invitation", + "weight": 26, + "name": "invitation in filename" + }, + { + "pattern": "save[-_ ]?the[-_ ]?date", + "weight": 26, + "name": "save the date filename" + }, + { + "pattern": "invite", + "weight": 16, + "name": "invite in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "invitation|save the date", + "weight": 12, + "name": "invitation in PDF title" + } + ], + "negatives": [ + { + "text": "admit one", + "weight": 20, + "name": "event ticket wording" + }, + { + "text": "general admission", + "weight": 18, + "name": "event ticket wording" + }, + { + "text": "no re-entry", + "weight": 16, + "name": "event ticket wording" + }, + { + "text": "boarding pass", + "weight": 14, + "name": "boarding pass heading" + }, + { + "text": "order confirmation", + "weight": 12, + "name": "retail order heading" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 8 + } + ] + }, + { + "id": "gift-certificate", + "name": "Gift certificate", + "emit": true, + "phrases": [ + { + "text": "gift voucher", + "weight": 30 + }, + { + "text": "gift certificate", + "weight": 28 + }, + { + "text": "this voucher entitles", + "weight": 28 + }, + { + "text": "gift card", + "weight": 24 + }, + { + "text": "voucher code", + "weight": 24 + }, + { + "text": "redeem this voucher", + "weight": 24 + }, + { + "text": "not redeemable for cash", + "weight": 24 + }, + { + "text": "gift card number", + "weight": 22 + }, + { + "text": "redeemable at", + "weight": 22 + }, + { + "text": "no cash value", + "weight": 20 + }, + { + "text": "to redeem", + "weight": 14 + }, + { + "text": "promo code", + "weight": 10 + } + ], + "regexes": [ + { + "pattern": "(?:gift\\s*card|voucher|certificate)\\s*(?:number|no\\.?|code|#)\\s*:?\\s*[a-z0-9][a-z0-9 -]{5,24}", + "weight": 16, + "name": "gift card/voucher code field" + }, + { + "pattern": "(?:check|view)\\s+your\\s+balance", + "weight": 12, + "name": "check your balance" + } + ], + "filenames": [ + { + "pattern": "gift[-_ ]?card", + "weight": 28, + "name": "gift card filename" + }, + { + "pattern": "gift[-_ ]?certificate", + "weight": 26, + "name": "gift certificate filename" + }, + { + "pattern": "voucher", + "weight": 24, + "name": "voucher in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "gift (card|certificate|voucher)", + "weight": 12, + "name": "gift card in PDF title" + } + ], + "negatives": [ + { + "text": "boarding pass", + "weight": 16, + "name": "boarding pass heading" + }, + { + "text": "admit one", + "weight": 14, + "name": "event ticket wording" + }, + { + "text": "fare basis", + "weight": 12, + "name": "e-ticket term" + }, + { + "text": "invoice number", + "weight": 10, + "name": "invoice field" + }, + { + "text": "your subscription", + "weight": 10, + "name": "subscription context" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 8 + } + ] + }, + { + "id": "confirmation-letter", + "name": "Confirmation letter", + "emit": true, + "phrases": [ + { + "text": "appointment confirmation", + "weight": 34, + "where": "title" + }, + { + "text": "your appointment has been confirmed", + "weight": 30 + }, + { + "text": "your appointment is scheduled", + "weight": 28 + }, + { + "text": "appointment reminder", + "weight": 28, + "where": "title" + }, + { + "text": "your upcoming appointment", + "weight": 26 + }, + { + "text": "if you are unable to attend", + "weight": 20 + }, + { + "text": "cancel or reschedule", + "weight": 20 + }, + { + "text": "please arrive", + "weight": 16 + }, + { + "text": "failure to attend", + "weight": 16 + }, + { + "text": "to reschedule", + "weight": 14 + }, + { + "text": "appointment date", + "weight": 12 + }, + { + "text": "appointment time", + "weight": 12 + } + ], + "regexes": [ + { + "pattern": "appointment\\s*(?:date|time|ref(?:erence)?)\\s*:?", + "weight": 10, + "name": "appointment field labels" + }, + { + "pattern": "(?:cancel|change|reschedule)\\s+your\\s+appointment", + "weight": 16, + "name": "cancel/reschedule your appointment" + }, + { + "pattern": "please arrive\\s+\\d{1,2}\\s+minutes", + "weight": 14, + "name": "arrive N minutes early" + } + ], + "filenames": [ + { + "pattern": "appointment", + "weight": 28, + "name": "appointment in filename" + }, + { + "pattern": "appt", + "weight": 18, + "name": "appt abbreviation in filename" + } + ], + "metadata": [ + { + "field": "title", + "pattern": "appointment", + "weight": 14, + "name": "appointment in PDF title" + } + ], + "negatives": [ + { + "text": "check-out date", + "weight": 16, + "name": "hotel confirmation field" + }, + { + "text": "your booking is confirmed", + "weight": 14, + "name": "travel booking wording" + }, + { + "text": "boarding pass", + "weight": 12, + "name": "boarding pass heading" + }, + { + "text": "prescription", + "weight": 12, + "name": "prescription wording" + }, + { + "text": "referral", + "weight": 10, + "name": "referral letter wording" + } + ], + "structural": [ + { + "signal": "short_doc", + "weight": 8 + }, + { + "signal": "address_block", + "weight": 4 + } + ] + } + ], + "priors": { + "resume": [1, 6], + "cover-letter": [1, 3], + "letter": [1, 5], + "memo": [1, 5], + "ticket": [1, 4], + "invitation": [1, 3], + "receipt": [1, 3], + "invoice": [1, 6], + "quote": [1, 10], + "purchase-order": [1, 8], + "remittance-advice": [1, 3], + "payslip": [1, 3], + "tax-form": [1, 10], + "tax-statement": [1, 8], + "subscription-confirmation": [1, 3], + "order-confirmation": [1, 4], + "donation-receipt": [1, 2], + "confirmation-letter": [1, 2], + "gift-certificate": [1, 2], + "waybill": [1, 2], + "return-authorization": [1, 2], + "packing-slip": [1, 3], + "delivery-note": [1, 3], + "prescription": [1, 2], + "referral-letter": [1, 3], + "immunization-record": [1, 4], + "medical-invoice": [1, 6], + "certificate": [1, 2], + "insurance-certificate": [1, 4], + "license": [1, 5], + "utility-bill": [1, 10], + "benefits-summary": [1, 8], + "government-notice": [1, 6], + "hoa-document": [1, 5], + "legal-notice": [1, 6], + "menu": [1, 8], + "recipe": [1, 6], + "timesheet": [1, 3], + "organization-chart": [1, 5], + "meeting-agenda": [1, 5], + "job-description": [1, 6], + "offer-letter": [1, 5], + "reference-letter": [1, 3], + "email-thread": [1, 8], + "nda": [1, 12], + "affidavit": [1, 8], + "investment-summary": [1, 12], + "letter-of-intent": [1, 15], + "credit-note": [1, 3], + "public-notice": [1, 4], + "subpoena": [1, 6], + "service-agreement": [1, 6], + "consent-form": [1, 4], + "discharge-summary": [1, 6], + "grade-report": [1, 4], + "registration-form": [1, 6], + "supply-order": [1, 3], + "grant-agreement": [1, 4], + "technical-drawing": [1, 12], + "compliance-document": [1, 3], + "test-report": [1, 4], + "book-ebook": [30, null], + "thesis": [20, null], + "regulatory-filing": [12, null], + "audit-report": [10, null], + "business-plan": [8, null], + "user-guide": [5, null], + "employee-handbook": [8, null] + } +} diff --git a/frontend/editor/src/proprietary/services/heuristic/heuristicRules.lint.test.ts b/frontend/editor/src/proprietary/services/heuristic/heuristicRules.lint.test.ts new file mode 100644 index 0000000000..d21d9a79bd --- /dev/null +++ b/frontend/editor/src/proprietary/services/heuristic/heuristicRules.lint.test.ts @@ -0,0 +1,173 @@ +// Structural lint for the rules pack: the engine silently drops or clamps +// malformed rules, so authoring mistakes must fail here instead. + +import { describe, expect, it } from "vitest"; +import { compileRegex } from "@app/services/heuristic/heuristicEngine"; +import rules from "@app/services/heuristic/heuristicRules.json"; + +interface RawRule { + text?: string; + pattern?: string; + weight?: number; + where?: string; + flags?: string; + field?: string; + signal?: string; +} +interface RawLabel { + id: string; + emit?: boolean; + phrases?: RawRule[]; + regexes?: RawRule[]; + filenames?: RawRule[]; + metadata?: RawRule[]; + negatives?: RawRule[]; + structural?: RawRule[]; +} +const labels = (rules as { labels: RawLabel[] }).labels; +const priors = (rules as { priors: Record }).priors; + +// Mirrors computeStructural's emitted keys; extend together with the engine. +const SIGNALS = new Set([ + "currency_heavy", + "number_table", + "form_like", + "toc", + "signature_block", + "references_section", + "short_doc", + "long_doc", + "bullet_heavy", + "email_headers", + "url_heavy", + "address_block", +]); +const ZONES = new Set(["title", "first", "any"]); +const META_FIELDS = new Set([ + "title", + "author", + "subject", + "keywords", + "creator", + "producer", + "any", +]); +// The engine clamps at these; authoring past them is a hidden no-op, so fail instead. +const WEIGHT_CAPS = { + phrases: 40, + regexes: 30, + filenames: 30, + metadata: 20, + negatives: 30, + structural: 12, +} as const; + +describe("heuristicRules.json pack lint", () => { + it("has unique, non-empty label ids", () => { + const ids = labels.map((l) => l.id); + expect(ids.every((id) => typeof id === "string" && id.length > 0)).toBe( + true, + ); + expect(new Set(ids).size).toBe(ids.length); + }); + + it("keys every page prior to an existing label id", () => { + const ids = new Set(labels.map((l) => l.id)); + const orphans = Object.keys(priors).filter((k) => !ids.has(k)); + expect(orphans, "priors keyed to no label never apply").toEqual([]); + }); + + it("compiles every regex/filename/metadata/negative pattern", () => { + const broken: string[] = []; + for (const label of labels) { + for (const kind of ["regexes", "filenames", "metadata"] as const) { + for (const r of label[kind] ?? []) { + if (compileRegex(r.pattern ?? null, r.flags ?? "") == null) { + broken.push(`${label.id} ${kind}: ${r.pattern}`); + } + } + } + for (const n of label.negatives ?? []) { + if ( + n.pattern != null && + compileRegex(n.pattern, n.flags ?? "") == null + ) { + broken.push(`${label.id} negative: ${n.pattern}`); + } + } + } + expect(broken, "non-compiling patterns are silently dropped").toEqual([]); + }); + + it("uses only structural signals the engine computes", () => { + const unknown: string[] = []; + for (const label of labels) { + for (const s of label.structural ?? []) { + if (!SIGNALS.has(s.signal ?? "")) { + unknown.push(`${label.id}: ${s.signal}`); + } + } + } + expect(unknown, "unknown signals always score 0").toEqual([]); + }); + + it("uses only zones and metadata fields the engine reads", () => { + const bad: string[] = []; + for (const label of labels) { + for (const kind of ["phrases", "regexes"] as const) { + for (const r of label[kind] ?? []) { + if (r.where != null && !ZONES.has(r.where)) { + bad.push(`${label.id} ${kind} where: ${r.where}`); + } + } + } + for (const m of label.metadata ?? []) { + if (m.field != null && !META_FIELDS.has(m.field)) { + bad.push(`${label.id} metadata field: ${m.field}`); + } + } + } + expect(bad).toEqual([]); + }); + + it("keeps every weight positive and within the engine's clamp", () => { + const bad: string[] = []; + for (const label of labels) { + for (const kind of [ + "phrases", + "regexes", + "filenames", + "metadata", + "negatives", + "structural", + ] as const) { + for (const r of label[kind] ?? []) { + const w = r.weight; + if (typeof w !== "number" || !Number.isFinite(w) || w <= 0) { + bad.push(`${label.id} ${kind}: weight ${String(w)}`); + } else if (Math.abs(w) > WEIGHT_CAPS[kind]) { + bad.push(`${label.id} ${kind}: weight ${w} over cap`); + } + } + } + } + expect(bad).toEqual([]); + }); + + it("gives every phrase a non-empty text and every negative a matcher", () => { + const bad: string[] = []; + for (const label of labels) { + for (const p of label.phrases ?? []) { + if (typeof p.text !== "string" || p.text.trim().length === 0) { + bad.push(`${label.id} phrase with empty text`); + } + } + for (const n of label.negatives ?? []) { + if (n.text == null && n.pattern == null) { + bad.push(`${label.id} negative with neither text nor pattern`); + } + } + } + expect(bad).toEqual([]); + }); +}); diff --git a/frontend/editor/src/proprietary/services/heuristic/types.ts b/frontend/editor/src/proprietary/services/heuristic/types.ts new file mode 100644 index 0000000000..fbd8128ba4 --- /dev/null +++ b/frontend/editor/src/proprietary/services/heuristic/types.ts @@ -0,0 +1,42 @@ +// Shared types for the client-side heuristic (non-AI) document classifier. + +/** Input document for the heuristic engine. */ +export interface HeuristicDoc { + fileName: string; + pageCount: number; + meta: Record; + titleZone: string; + firstZone: string; + allZone: string; +} + +// "none" = no match or non-English; a real runtime value, not just a type state. +export type HeuristicConfidence = "none" | "low" | "medium" | "high"; + +/** One scored candidate label with the rule hits that produced its score (debug only). */ +export interface LabelScoreExplanation { + id: string; + emit: boolean; + score: number; + distinct: number; + /** Human-readable contributions, e.g. `phrase "tax invoice" +60 (title)`. */ + signals: string[]; +} + +/** Why a document scored the way it did; produced only when explain is requested. */ +export interface HeuristicExplanation { + isEnglish: boolean; + lowText: boolean; + /** Top candidates by score, best first. Empty when rejected as non-English. */ + candidates: LabelScoreExplanation[]; +} + +/** Classification outcome: emitted vocabulary label ids (primary first, capped at 5). */ +export interface HeuristicResult { + labels: string[]; + confidence: HeuristicConfidence; + score: number; + isEnglish: boolean; + /** Present only when classify was called with `{ explain: true }`. */ + explain?: HeuristicExplanation; +} diff --git a/frontend/editor/src/proprietary/utils/scheduleIdle.ts b/frontend/editor/src/proprietary/utils/scheduleIdle.ts new file mode 100644 index 0000000000..8d3e5bec96 --- /dev/null +++ b/frontend/editor/src/proprietary/utils/scheduleIdle.ts @@ -0,0 +1,11 @@ +// Idle-time scheduling shared by the classification/backfill passes. + +/** Schedule work for the browser's idle time (or soon after, as a fallback). */ +export function scheduleIdle(task: () => void): () => void { + if (typeof requestIdleCallback === "function") { + const handle = requestIdleCallback(task, { timeout: 2000 }); + return () => cancelIdleCallback(handle); + } + const timer = window.setTimeout(task, 200); + return () => window.clearTimeout(timer); +} diff --git a/frontend/editor/src/saas/hooks/useClassificationEnabled.ts b/frontend/editor/src/saas/hooks/useClassificationEnabled.ts deleted file mode 100644 index a776f77986..0000000000 --- a/frontend/editor/src/saas/hooks/useClassificationEnabled.ts +++ /dev/null @@ -1,11 +0,0 @@ -// SaaS override of the classification-enabled seam: classification is available -// exactly when the AI engine is on for this tenant. Off → the sidebar grouping, -// group-picker, per-file label chips and the file-details Classification section -// all stay hidden, so an AI-disabled SaaS tenant sees the plain flat file list -// with no hint the feature exists. - -import { useAiEngineEnabled } from "@app/hooks/useAiEngineEnabled"; - -export function useClassificationEnabled(): boolean { - return useAiEngineEnabled(); -}