Stirling-Tools/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/Stirling-Tools/Stirling-PDF.git synced 2026-09-03 05:10:16 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix form detection bundle size, model hosting and engine dispatch

Browse Source
This commit is contained in:
Anthony Stirling
2026-08-22 15:05:03 +01:00
parent ffed9ec54a
commit 5a0f7e88cf
18 changed files with 345 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -74,6 +74,7 @@ app/core/src/main/resources/static/favicon.png
app/core/src/main/resources/static/safari-pinned-tab.svg
app/core/src/main/resources/static/pdfium/
app/core/src/main/resources/static/pdfjs/
app/core/src/main/resources/static/ort/
app/core/src/main/resources/static/vendor/
app/core/src/main/resources/static/**/*.gz
app/core/src/main/resources/static/**/*.br
app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java
+4 -4
View File
@@ -464,10 +464,10 @@ public class ApplicationProperties {
private String modelDir = "";
/**
* Read-only dir of models baked into the image (e.g. the Docker server image pre-downloads
* FFDNet-S here). On startup any {@code <catalogId>.onnx} found here is copied into the
* writable model dir if not already present, and activated if no model is active - so the
* feature works out-of-the-box. Blank (default) disables seeding.
* Read-only dir of models baked into the image (the air-gapped image bakes one here). On
* startup any {@code <catalogId>.onnx} found here is activated if no model is active, so
* the feature works out-of-the-box. The file is read in place rather than copied into the
* writable model dir, so it is not stored twice. Blank (default) disables seeding.
*/
private String preinstalledModelDir = "";
}
app/core/build.gradle
+2 -1
View File
@@ -207,7 +207,8 @@ def generatedFrontendPaths = [
'samples',
'pdfium',
'vendor',
'pdfjs'
'pdfjs',
'ort'
]
tasks.register('npmInstall', Exec) {
app/core/src/main/resources/settings.yml.template
+1 -1
View File
@@ -411,7 +411,7 @@ formDetection:
executionMode: auto # Where detection runs: 'auto' (browser first, server fallback), 'browser' (in-browser WASM only, PDF never leaves the device), or 'server' (backend inference)
activeModelId: "" # Id of the installed Auto Form Detection model (set automatically after an admin installs one)
modelDir: "" # Optional override directory for downloaded .onnx models; blank uses <configs>/models/form-detection
preinstalledModelDir: "" # Read-only dir of models baked into the image (Docker pre-downloads FFDNet-S here); seeded into the model dir on startup. Blank disables.
preinstalledModelDir: "" # Read-only dir of models baked into the image (the air-gapped image bakes one here); activated on startup without being copied. Blank disables.
policies:
# Folder automations can read from and write to the directories you allow here, so treat this as a
app/proprietary/src/main/java/stirling/software/proprietary/formdetection/model/ModelStatusResponse.java
+1 -1
View File
@@ -23,7 +23,7 @@ public class ModelStatusResponse {
/** Id of the active/usable model, or blank when none. */
private String activeModelId;
/** Model ids that currently have an .onnx file on disk. */
/** Model ids usable right now: downloaded, plus any the image baked in. */
private List<String> installed;
/** Last error message, or null. */
app/proprietary/src/main/java/stirling/software/proprietary/formdetection/service/FormDetectionModelManager.java
+72 -62
View File
@@ -426,7 +426,9 @@ public class FormDetectionModelManager {
if (StringUtils.isBlank(id) || !SAFE_ID.matcher(id).matches()) {
return;
}
Optional<Path> file = installedModelFile(id);
// Only the downloaded copy is ours to remove; an image-baked file is read-only and is
// retired by the tombstone below instead.
Optional<Path> file = modelFileIn(modelDir(), id);
if (file.isPresent()) {
try {
Files.deleteIfExists(file.get());
@@ -461,15 +463,11 @@ public class FormDetectionModelManager {
public ModelStatusResponse status() {
Path dir = modelDir();
List<String> installed = new ArrayList<>();
if (Files.isDirectory(dir)) {
try (DirectoryStream<Path> s = Files.newDirectoryStream(dir, "*.onnx")) {
for (Path p : s) {
String fn = p.getFileName().toString();
installed.add(fn.substring(0, fn.length() - ".onnx".length()));
}
} catch (IOException e) {
log.debug("Could not list installed models in {}", dir, e);
List<String> installed = new ArrayList<>(listModelIds(dir));
// Image-baked models count as installed even though they were never copied here.
for (String id : listModelIds(preinstalledDir())) {
if (!installed.contains(id) && !isTombstoned(id)) {
installed.add(id);
}
}
return new ModelStatusResponse(
@@ -491,15 +489,24 @@ public class FormDetectionModelManager {
}
/**
* Locate an installed model by listing the model dir, so the path handed to the file API comes
* from the directory itself and can never escape it via the supplied id.
* Locate an installed model by listing a directory, so the path handed to the file API comes
* from the directory itself and can never escape it via the supplied id. The writable model dir
* wins; an image-baked copy is read in place rather than duplicated into it.
*/
private Optional<Path> installedModelFile(String id) {
if (StringUtils.isBlank(id) || !SAFE_ID.matcher(id).matches()) {
return Optional.empty();
}
Path dir = modelDir();
if (!Files.isDirectory(dir)) {
Optional<Path> downloaded = modelFileIn(modelDir(), id);
if (downloaded.isPresent()) {
return downloaded;
}
// An uninstalled model must stay uninstalled even though the image copy is still there.
return isTombstoned(id) ? Optional.empty() : modelFileIn(preinstalledDir(), id);
}
private Optional<Path> modelFileIn(Path dir, String id) {
if (dir == null || !Files.isDirectory(dir)) {
return Optional.empty();
}
String wanted = id + ".onnx";
@@ -510,11 +517,38 @@ public class FormDetectionModelManager {
}
}
} catch (IOException e) {
log.debug("Could not list installed models in {}", dir, e);
log.debug("Could not list models in {}", dir, e);
}
return Optional.empty();
}
/** Ids of the {@code <id>.onnx} files in a directory; empty when it is unset or missing. */
private List<String> listModelIds(Path dir) {
List<String> ids = new ArrayList<>();
if (dir == null || !Files.isDirectory(dir)) {
return ids;
}
try (DirectoryStream<Path> s = Files.newDirectoryStream(dir, "*.onnx")) {
for (Path p : s) {
String fn = p.getFileName().toString();
ids.add(fn.substring(0, fn.length() - ".onnx".length()));
}
} catch (IOException e) {
log.debug("Could not list models in {}", dir, e);
}
return ids;
}
private boolean isTombstoned(String id) {
return SAFE_ID.matcher(id).matches() && Files.exists(tombstoneFor(id));
}
/** Read-only dir of image-baked models, or null when the deployment bakes none. */
private Path preinstalledDir() {
String dir = applicationProperties.getFormDetection().getPreinstalledModelDir();
return StringUtils.isBlank(dir) ? null : Paths.get(dir);
}
public Optional<ModelCatalogEntry> getActiveEntry() {
return catalog.getById(activeModelId());
}
@@ -538,58 +572,34 @@ public class FormDetectionModelManager {
}
/**
* Copy any image-baked models (see {@code formDetection.preinstalledModelDir}) into the
* writable model dir if not already present, and activate one when nothing is active yet. Lets
* the Docker server image ship with FFDNet-S ready without an admin install. No-op when the dir
* is unset or missing (desktop/local).
* Activate an image-baked model (see {@code formDetection.preinstalledModelDir}) when nothing
* is active yet, so the air-gapped image works without an admin install. The file is read where
* the image put it - copying it into the writable model dir would store the same ~37MB twice on
* every running container. No-op when the dir is unset or missing (desktop/local).
*/
private void seedPreinstalledModels() {
String preDir = applicationProperties.getFormDetection().getPreinstalledModelDir();
if (StringUtils.isBlank(preDir)) {
Path src = preinstalledDir();
if (src == null || !Files.isDirectory(src)) {
return;
}
Path src = Paths.get(preDir);
if (!Files.isDirectory(src)) {
return;
}
Path dir = modelDir();
try {
Files.createDirectories(dir);
} catch (IOException e) {
log.warn("Cannot create model dir to seed pre-installed models: {}", e.getMessage());
return;
}
if (!isWritable(dir)) {
log.warn("Model dir {} not writable; skipping pre-installed model seeding", dir);
return;
}
try (DirectoryStream<Path> models = Files.newDirectoryStream(src, "*.onnx")) {
for (Path p : models) {
String fn = p.getFileName().toString();
String id = fn.substring(0, fn.length() - ".onnx".length());
if (!SAFE_ID.matcher(id).matches() || catalog.getById(id).isEmpty()) {
continue;
}
if (Files.exists(tombstoneFor(id))) {
log.info("Skipping pre-installed model '{}': an admin uninstalled it", id);
continue;
}
Path target = dir.resolve(id + ".onnx");
if (!Files.exists(target)) {
Files.copy(p, target, StandardCopyOption.COPY_ATTRIBUTES);
log.info("Seeded pre-installed Auto Form Detection model '{}'", id);
}
if (StringUtils.isBlank(activeModelId())) {
applicationProperties.getFormDetection().setActiveModelId(id);
try {
GeneralUtils.saveKeyToSettings("formDetection.activeModelId", id);
} catch (IOException e) {
log.warn("Could not persist seeded activeModelId: {}", e.getMessage());
}
}
for (String id : listModelIds(src)) {
if (!SAFE_ID.matcher(id).matches() || catalog.getById(id).isEmpty()) {
continue;
}
} catch (IOException e) {
log.warn("Failed to seed pre-installed models from {}: {}", src, e.getMessage());
if (isTombstoned(id)) {
log.info("Skipping pre-installed model '{}': an admin uninstalled it", id);
continue;
}
if (StringUtils.isNotBlank(activeModelId())) {
continue;
}
applicationProperties.getFormDetection().setActiveModelId(id);
try {
GeneralUtils.saveKeyToSettings("formDetection.activeModelId", id);
} catch (IOException e) {
log.warn("Could not persist seeded activeModelId: {}", e.getMessage());
}
log.info("Activated pre-installed Auto Form Detection model '{}'", id);
}
}
app/proprietary/src/main/resources/formdetection/NOTICE
+5 -3
View File
@@ -1,8 +1,9 @@
Auto Form Detection - third-party model attribution
===================================================
Stirling-PDF bundles no model weights. Models are downloaded on demand from the URLs in
model-catalog.json and verified against the SHA-256 recorded there.
Models are downloaded on demand from the URLs in model-catalog.json and verified against the
SHA-256 recorded there. The air-gapped image (docker/embedded/Dockerfile.fat) additionally bakes
the ffdetr weights in at build time, since it is the deployment that cannot fetch them later.
This NOTICE covers weights Stirling itself publishes or redistributes. It exists because
Apache-2.0 section 4 attaches attribution and NOTICE obligations to whoever conveys the work,
@@ -14,6 +15,7 @@ ffdetr
FFDetr, a form-field detector trained on the CommonForms dataset.
Weights https://huggingface.co/jbarrow/FFDetr Apache-2.0
Our export https://huggingface.co/Frooodle/ffdetr-int8 Apache-2.0
Base model Roboflow RF-DETR (rf-detr-medium) Apache-2.0
Framework https://github.com/roboflow/rf-detr Apache-2.0
Backbone DINOv2 (Meta AI) Apache-2.0
@@ -23,7 +25,7 @@ FFDetr, a form-field detector trained on the CommonForms dataset.
The .onnx Stirling distributes is not the upstream artifact: the publisher releases a PyTorch
.pth only. Stirling exports it to ONNX and quantises it to int8 via
scripts/export-ffdetr-onnx.py, which pins the source revision and its checksum. No weights are
modified beyond that conversion.
modified beyond that conversion, and the result is republished under the same licence.
Not vendored: the `commonforms` reference wrapper (github.com/jbarrow/commonforms) carries no
licence file and depends on Ultralytics, so none of its code is used. The pre/post-processing in
app/proprietary/src/main/resources/formdetection/model-catalog.json
+4 -4
View File
@@ -3,10 +3,10 @@
"id": "ffdetr",
"displayName": "CommonForms FFDetr (Apache-2.0)",
"description": "Permissively licensed detector trained on the same CommonForms data. Finds text inputs, checkboxes and signature fields. Quantised to int8, so it is the smallest option as well as the only one with a licence grant.",
"license": "Apache-2.0. Weights https://huggingface.co/jbarrow/FFDetr, base model Roboflow RF-DETR, CommonForms dataset - all Apache-2.0. Converted to ONNX by Stirling via scripts/export-ffdetr-onnx.py.",
"sizeBytes": 37116508,
"onnxUrl": "",
"sha256": "2323b426456887fd8befc3e0dd508497c2084d7936e7da046bc023188f119982",
"license": "Apache-2.0. Weights https://huggingface.co/jbarrow/FFDetr, base model Roboflow RF-DETR, CommonForms dataset - all Apache-2.0. Converted to ONNX by Stirling via scripts/export-ffdetr-onnx.py and republished at https://huggingface.co/Frooodle/ffdetr-int8; see the NOTICE beside this file.",
"sizeBytes": 37116529,
"onnxUrl": "https://huggingface.co/Frooodle/ffdetr-int8/resolve/ad3223d7c3ae25ce1d1ec8fe972c8ff7630aff45/ffdetr-int8.onnx",
"sha256": "de43bb39adb08459fe62fa1a7be6bfb4f2d397bb034c46055f3b8fa3d04fc8f4",
"decoder": "rfdetr",
"inputSize": 1024,
"resizeMode": "stretch",
app/proprietary/src/test/java/stirling/software/proprietary/formdetection/service/FormDetectionModelManagerTest.java
+13 -6
View File
@@ -225,11 +225,18 @@ class FormDetectionModelManagerTest {
FormDetectionModelManager m =
manager(modelDir, e, Mockito.mock(EndpointConfiguration.class), props);
m.init();
assertTrue(Files.exists(modelDir.resolve("test-model.onnx")), "seeded on first boot");
assertEquals("test-model", m.status().getActiveModelId());
assertEquals("test-model", m.status().getActiveModelId(), "seeded on first boot");
assertEquals("ready", m.status().getStatus());
assertTrue(m.status().getInstalled().contains("test-model"));
assertFalse(
Files.exists(modelDir.resolve("test-model.onnx")),
"image-baked model is read in place, never duplicated into the writable dir");
m.deleteModel("test-model");
assertFalse(Files.exists(modelDir.resolve("test-model.onnx")));
assertEquals("not_installed", m.status().getStatus());
assertTrue(
Files.exists(preDir.resolve("test-model.onnx")),
"uninstall must not touch the read-only image copy");
assertTrue(
Files.exists(modelDir.resolve("test-model.onnx.removed")),
"uninstall records a tombstone");
@@ -240,10 +247,10 @@ class FormDetectionModelManagerTest {
FormDetectionModelManager m2 =
manager(modelDir, e, Mockito.mock(EndpointConfiguration.class), props2);
m2.init();
assertFalse(
Files.exists(modelDir.resolve("test-model.onnx")),
"tombstoned model must not be re-seeded");
assertEquals("not_installed", m2.status().getStatus());
assertFalse(
m2.status().getInstalled().contains("test-model"),
"tombstoned model must not be re-seeded");
// An explicit reinstall clears the tombstone and seeding works again afterwards.
m2.startInstall("test-model");
docker/embedded/Dockerfile
+5 -3
View File
@@ -22,9 +22,11 @@ RUN apt-get update \
# Optionally bake an Auto Form Detection model into the image, seeded into the writable configs
# model dir at startup by FormDetectionModelManager (FORMDETECTION_PREINSTALLEDMODELDIR below).
#
# Empty by default ON PURPOSE. Baking a model in makes us its distributor, so only weights we
# hold a redistribution grant for may go here - which the FFDNet checkpoints are not, as their
# publisher declares no licence at all. Set all three args to pre-bundle a model you may ship:
# Empty by default ON PURPOSE, and unlike Dockerfile.fat this image is meant to stay that way:
# it has internet access, so an admin install costs 0MB in the image against ~27MB of pull for
# every user who never opens the tool. Baking a model in also makes us its distributor, so only
# weights we hold a redistribution grant for may go here - which the FFDNet checkpoints are not,
# as their publisher declares no licence at all. Set all three args to pre-bundle a model you may ship:
# --build-arg FORM_DETECTION_MODEL_URL=... \
# --build-arg FORM_DETECTION_MODEL_SHA256=... \
# --build-arg FORM_DETECTION_MODEL_ID=ffdetr
docker/embedded/Dockerfile.fat
+10 -10
View File
@@ -20,17 +20,17 @@ RUN apt-get update \
&& rm /tmp/task.deb \
&& rm -rf /var/lib/apt/lists/*
# Optionally bake an Auto Form Detection model into this air-gapped image, seeded into the
# writable configs model dir at startup.
# Bake an Auto Form Detection model into this air-gapped image, seeded into the writable configs
# model dir at startup. This is the one image where baking earns its cost: it is the deployment
# that cannot download a model later. The internet-connected images leave it to an admin install.
#
# Empty by default ON PURPOSE - see the note in docker/embedded/Dockerfile. Baking a model in
# makes us its distributor, and it is exactly the air-gapped case that makes an unlicensed
# bundle hardest to undo. Set all three args to pre-bundle weights you may ship:
# --build-arg FORM_DETECTION_MODEL_URL=... \
# --build-arg FORM_DETECTION_MODEL_SHA256=... \
# --build-arg FORM_DETECTION_MODEL_ID=ffdetr
ARG FORM_DETECTION_MODEL_URL=""
ARG FORM_DETECTION_MODEL_SHA256=""
# Only weights we hold a redistribution grant for may go here - which the FFDNet checkpoints are
# not, as their publisher declares no licence at all. FFDetr is Apache-2.0 end to end, so it is
# the default: our own int8 export from scripts/export-ffdetr-onnx.py, pinned to a revision SHA.
# Costs ~27MB of pull and ~37MB on disk. Keep these three in step with the ffdetr entry in
# model-catalog.json - a mismatch fails the build at the sha256sum check below, by design.
ARG FORM_DETECTION_MODEL_URL="https://huggingface.co/Frooodle/ffdetr-int8/resolve/ad3223d7c3ae25ce1d1ec8fe972c8ff7630aff45/ffdetr-int8.onnx"
ARG FORM_DETECTION_MODEL_SHA256="de43bb39adb08459fe62fa1a7be6bfb4f2d397bb034c46055f3b8fa3d04fc8f4"
ARG FORM_DETECTION_MODEL_ID="ffdetr"
RUN mkdir -p /preinstalled-models \
&& if [ -n "${FORM_DETECTION_MODEL_URL}" ]; then \
frontend/editor/src/core/hooks/tools/autoFormDetection/useAutoFormDetectionOperation.ts
+32 -2
View File
@@ -93,6 +93,7 @@ async function browserDetect(
parameters: AutoFormDetectionParameters,
file: File,
entry: FormDetectionCatalogEntry,
pageBudgetMs?: number,
): Promise<File> {
emitStage({ kind: "starting", engine: "browser" });
const { runBrowserDetection } =
@@ -103,6 +104,7 @@ async function browserDetect(
entry,
resolveConfidence(parameters),
emitStage,
{ pageBudgetMs },
);
emitSummary(summarizeFields(fields, "browser"));
return new File([new Uint8Array(appliedPdf)], outputName(file), {
@@ -128,13 +130,41 @@ async function processAutoFormDetection(
return { files: [await serverDetect(parameters, file)] };
}
if (mode === "browser") {
// Strict: no budget and no fallback. The point of this mode is that the PDF never leaves the
// device, so a slow device waits rather than silently uploading.
return { files: [await browserDetect(parameters, file, activeEntry)] };
}
// auto: prefer the device. Keeping detection local spares the backend from running every page
// of every user's document, and keeps the file on the machine it came from. Only a device that
// looks too weak to finish in reasonable time starts on the server instead.
const {
isUnderpoweredForBrowserEngine,
describeDevice,
BROWSER_PAGE_BUDGET_MS,
} = await import("@app/services/formDetection/deviceCapability");
if (isUnderpoweredForBrowserEngine()) {
console.debug(
`[AutoFormDetection] ${describeDevice()} - using the server engine`,
);
return { files: [await serverDetect(parameters, file)] };
}
try {
return { files: [await browserDetect(parameters, file, activeEntry)] };
return {
files: [
await browserDetect(
parameters,
file,
activeEntry,
BROWSER_PAGE_BUDGET_MS,
),
],
};
} catch (e) {
// Covers both a hard failure and the budget check: the capability hints are advisory, so the
// first page is the real measurement and the honest place to change our mind.
console.warn(
"[AutoFormDetection] in-browser engine failed; falling back to server",
"[AutoFormDetection] in-browser engine did not complete; falling back to server",
e,
);
return { files: [await serverDetect(parameters, file)] };
frontend/editor/src/core/services/formDetection/deviceCapability.test.ts
+64
View File
@@ -0,0 +1,64 @@
import { describe, expect, it } from "vitest";
import {
describeDevice,
isUnderpoweredForBrowserEngine,
} from "@app/services/formDetection/deviceCapability";
describe("isUnderpoweredForBrowserEngine", () => {
it("keeps a normal machine on the browser engine", () => {
expect(isUnderpoweredForBrowserEngine({ cores: 8, memoryGb: 8 })).toBe(
false,
);
});
// The floors are inclusive: exactly 4 cores / 4GB is still considered capable, so the common
// low-end-but-usable laptop keeps its document on the device.
it("keeps a device that sits exactly on the floor", () => {
expect(isUnderpoweredForBrowserEngine({ cores: 4, memoryGb: 4 })).toBe(
false,
);
});
it("sends a device below the core floor to the server", () => {
expect(isUnderpoweredForBrowserEngine({ cores: 3, memoryGb: 8 })).toBe(
true,
);
expect(isUnderpoweredForBrowserEngine({ cores: 2, memoryGb: 8 })).toBe(
true,
);
expect(isUnderpoweredForBrowserEngine({ cores: 1 })).toBe(true);
});
it("sends a device below the memory floor to the server", () => {
expect(isUnderpoweredForBrowserEngine({ cores: 8, memoryGb: 2 })).toBe(
true,
);
// deviceMemory reports fractions on very small devices.
expect(isUnderpoweredForBrowserEngine({ cores: 8, memoryGb: 0.5 })).toBe(
true,
);
});
// deviceMemory is Chromium-only; Firefox and Safari report nothing. Treating "unknown" as weak
// would quietly move every non-Chromium user's document to the backend.
it("treats unknown hints as capable, not weak", () => {
expect(isUnderpoweredForBrowserEngine({})).toBe(false);
expect(isUnderpoweredForBrowserEngine({ cores: 8 })).toBe(false);
expect(isUnderpoweredForBrowserEngine({ memoryGb: 8 })).toBe(false);
});
// navigator.hardwareConcurrency can be 0 or absent; readHints() maps those to undefined, so a
// bogus zero must not read as the weakest possible machine.
it("does not treat a zero reading as a weak device", () => {
expect(isUnderpoweredForBrowserEngine({ cores: undefined })).toBe(false);
});
it("describes what it saw, including gaps", () => {
expect(describeDevice({ cores: 8, memoryGb: 16 })).toBe(
"8 cores, 16GB RAM",
);
expect(describeDevice({ cores: 8 })).toBe("8 cores, RAM unknown");
expect(describeDevice({})).toBe("cores unknown, RAM unknown");
});
});
frontend/editor/src/core/services/formDetection/deviceCapability.ts
+64
View File
@@ -0,0 +1,64 @@
// Decides whether `auto` mode should start in the browser or go straight to the server.
//
// Detection is ~15s per page in WASM against ~1.6s in Java, so a weak device is a genuinely bad
// place to run it. But pushing everyone to the server would put every page of every user's document
// through one backend, and give up the property that the PDF never leaves the device - so the bias
// is deliberately towards the browser. Only clearly-underpowered devices are sent to the server.
/** How long one page may take in the browser before `auto` gives up and uses the server. */
export const BROWSER_PAGE_BUDGET_MS = 20_000;
/**
* What a device must report to keep detection local. Both are floors, not targets - a machine at
* exactly 4/4 still runs in the browser.
*
* `deviceMemory` only ever reports 0.25/0.5/1/2/4/8 (Chromium caps it at 8 to limit fingerprinting),
* so 4 is a real step on that scale rather than an arbitrary number.
*/
const MIN_CORES = 4;
const MIN_MEMORY_GB = 4;
interface CapabilityHints {
cores?: number;
memoryGb?: number;
}
function readHints(): CapabilityHints {
const nav = navigator as Navigator & { deviceMemory?: number };
// Both are advisory and capped by browsers; deviceMemory is Chromium-only and undefined
// elsewhere, which must not read as "weak".
return {
cores:
typeof nav.hardwareConcurrency === "number" && nav.hardwareConcurrency > 0
? nav.hardwareConcurrency
: undefined,
memoryGb:
typeof nav.deviceMemory === "number" && nav.deviceMemory > 0
? nav.deviceMemory
: undefined,
};
}
/**
* True when the device looks too weak to run detection locally in reasonable time. Conservative on
* purpose: unknown means capable, so an unreported browser keeps the on-device path.
*/
export function isUnderpoweredForBrowserEngine(
hints: CapabilityHints = readHints(),
): boolean {
const { cores, memoryGb } = hints;
if (cores !== undefined && cores < MIN_CORES) return true;
if (memoryGb !== undefined && memoryGb < MIN_MEMORY_GB) return true;
return false;
}
export function describeDevice(hints: CapabilityHints = readHints()): string {
const parts: string[] = [];
parts.push(
hints.cores !== undefined ? `${hints.cores} cores` : "cores unknown",
);
parts.push(
hints.memoryGb !== undefined ? `${hints.memoryGb}GB RAM` : "RAM unknown",
);
return parts.join(", ");
}
frontend/editor/src/core/services/formDetection/onnxSession.ts
+11 -1
View File
@@ -2,8 +2,13 @@
// runs single-threaded (the app sets no COOP/COEP so SharedArrayBuffer threading is unavailable),
// and caches one session per model checksum. Output is returned in the same flat layout the
// backend uses so decode.ts can interpret it identically.
//
// The /wasm subpath entry is deliberate: the package root also bundles the WebGPU and WebGL
// backends, which this code never selects - and its WebGPU runtime is a second 26MB .wasm that
// Rollup emits alongside the CPU one. vite.config.ts pins the subpath to onnxruntime's
// extern-wasm build, so exactly one runtime ships: the copy under /ort/.
import * as ort from "onnxruntime-web";
import * as ort from "onnxruntime-web/wasm";
import { RawOutput } from "@app/services/formDetection/types";
@@ -11,6 +16,11 @@ let configured = false;
function configureOrt(): void {
if (configured) return;
ort.env.wasm.numThreads = 1;
// Run the session on a worker thread. Inference is ~15s per page and would otherwise block the
// main thread outright - a frozen tab with a stalled progress bar. This does not make it faster,
// it keeps the app responsive while it runs. (numThreads stays 1: multi-threading needs
// SharedArrayBuffer, which needs COOP/COEP headers the app does not set.)
ort.env.wasm.proxy = true;
// The CPU SIMD .wasm + its loader are copied next to the app under /ort/ by vite.config.ts.
ort.env.wasm.wasmPaths = new URL("ort/", document.baseURI).href;
configured = true;
frontend/editor/src/core/services/formDetection/runBrowserPipeline.ts
+30 -1
View File
@@ -34,11 +34,31 @@ export interface BrowserDetectResult {
pageCount: number;
}
/** Thrown when a page blows the caller's time budget, so `auto` can hand off to the server. */
export class BrowserEngineTooSlowError extends Error {
constructor(readonly pageMs: number) {
super(
`In-browser detection took ${Math.round(pageMs)}ms on the first page`,
);
this.name = "BrowserEngineTooSlowError";
}
}
export interface BrowserDetectOptions {
/**
* Abandon the run if the first page takes longer than this and pages remain. Only the caller that
* has somewhere to fall back to should set it - `browser` mode passes nothing, because bailing to
* the server is exactly what that mode exists to prevent.
*/
pageBudgetMs?: number;
}
export async function runBrowserDetection(
pdfBytes: ArrayBuffer,
activeEntry: FormDetectionCatalogEntry,
confThreshold?: number,
onStage?: (stage: DetectionStage) => void,
options?: BrowserDetectOptions,
): Promise<BrowserDetectResult> {
const spec = resolveSpec(activeEntry);
const score =
@@ -78,14 +98,23 @@ export async function runBrowserDetection(
},
);
let fields: DetectedField[] = [];
for (const page of pages) {
for (const [index, page] of pages.entries()) {
onStage?.({
kind: "analyzing",
page: page.pageIndex + 1,
pageCount: pages.length,
});
const startedAt = performance.now();
const pre = preprocess(page.rgba, page.widthPx, page.heightPx, spec);
const out = await runInference(session, pre.chw, spec.inputSize);
// Measure the first page and bail before paying the same cost for every remaining one. The
// work already done is discarded rather than merged, so the server sees the whole document and
// the result cannot be a mix of two engines.
const pageMs = performance.now() - startedAt;
const budget = options?.pageBudgetMs;
if (index === 0 && budget && pageMs > budget && pages.length > 1) {
throw new BrowserEngineTooSlowError(pageMs);
}
for (const d of decodeFor(spec, out, pre, score)) {
const rect = toPdfPoints(d, page);
if (rect.w <= 0 || rect.h <= 0) {
frontend/editor/vite.config.ts
+19 -3
View File
@@ -320,9 +320,12 @@ export default defineConfig(async ({ mode, command }) => {
dest: "pdfjs/standard_fonts",
},
{
// onnxruntime-web CPU SIMD runtime + loader for the in-browser Auto Form
// Detection engine. Single-thread (the app sets no COOP/COEP); the heavier
// WebGPU/JSEP and asyncify variants are intentionally not copied.
// onnxruntime-web CPU SIMD runtime + its loader for the in-browser Auto Form Detection
// engine. Always shipped: desktop reaches a backend that has the feature in SaaS and
// self-hosted modes, and the in-browser engine is the whole point there - the PDF never
// leaves the device. Single-thread (the app sets no COOP/COEP); the WebGPU/JSEP and
// asyncify variants are intentionally not copied, and the resolve.alias below stops
// onnxruntime pulling the 26MB WebGPU one in behind us.
src: "../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.{wasm,mjs}",
dest: "ort",
},
@@ -342,6 +345,19 @@ export default defineConfig(async ({ mode, command }) => {
compressStaticCopyPlugin(),
prerenderOgPlugin(effectiveMode === "saas"),
],
resolve: {
alias: {
// Pin onnxruntime-web to its extern-wasm build. The other entries contain
// `new URL("ort-wasm-simd-threaded[.jsep].wasm", import.meta.url)`, which Rollup resolves
// into an emitted asset - so a build ends up shipping the 26MB WebGPU runtime, or a second
// copy of the CPU one, on top of the /ort/ files above. This variant names only the loader,
// so the copy under /ort/ is the only runtime that ships.
"onnxruntime-web/wasm": resolve(
import.meta.dirname,
"../node_modules/onnxruntime-web/dist/ort.wasm.min.mjs",
),
},
},
// Worker bundles are a separate Rollup pass and do NOT inherit `plugins`,
// so without this `@app/*` resolves in the app and fails in a worker.
worker: {
scripts/export-ffdetr-onnx.py
+7 -3
View File
@@ -11,7 +11,7 @@ with onnxruntime alone, exactly as it loads any other model in the catalogue.
python scripts/export-ffdetr-onnx.py --out build/ffdetr
Emits `ffdetr-int8.onnx` (~37MB, the one to host) plus the fp32 graph it came from, and prints
the sha256 for `model-catalog.json`. int8 measures smaller than FFDNet-S at 38.4MB and, on a
the sha256 for `model-catalog.json` and docker/embedded/Dockerfile.fat. int8 measures smaller than FFDNet-S at 38.4MB and, on a
form page, returns detections indistinguishable from fp32.
"""
@@ -24,7 +24,10 @@ import sys
import warnings
from pathlib import Path
# Pinned so a re-export is byte-reproducible; bump deliberately, not incidentally.
# Pinned so a re-export uses the same checkpoint; bump deliberately, not incidentally. The OUTPUT
# is not byte-reproducible - onnxruntime writes its registered opset_import domains into the graph,
# so a different runtime version shifts the tail and the sha256. Check equivalence by running both
# graphs on one input, not by comparing hashes; update model-catalog.json when you republish.
REPO = "jbarrow/FFDetr"
REVISION = "56f4e4235e28dcb2953513dc020bb191a2f54cfe"
CHECKPOINT_SHA256 = "f852e1bac18c8f435b82270fc8ff8e2ca4a2cd8869c411fa8f473f16e69585ef"
@@ -141,7 +144,8 @@ def main() -> int:
print(f" sha256 {sha256_of(int8)}")
print()
print("Host this file, then set onnxUrl/sha256/sizeBytes on the ffdetr entry in")
print("app/proprietary/src/main/resources/formdetection/model-catalog.json.")
print("app/proprietary/src/main/resources/formdetection/model-catalog.json, and the matching")
print("FORM_DETECTION_MODEL_* build args in docker/embedded/Dockerfile.fat.")
return 0