Stirling-Tools/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/Stirling-Tools/Stirling-PDF.git synced 2026-09-03 05:10:16 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add the Apache-2.0 FFDetr detector and stop bundling unlicensed weights

Browse Source
This commit is contained in:
Anthony Stirling
2026-08-13 14:38:43 +01:00
parent 59f331e702
commit 548da09768
21 changed files with 1590 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/proprietary/src/main/java/stirling/software/proprietary/formdetection/controller/FormDetectionController.java
+19 -2
View File
@@ -29,6 +29,7 @@ import stirling.software.common.util.FormUtils.NewFormFieldDefinition;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.WebResponseUtils;
import stirling.software.proprietary.formdetection.inference.OnnxFormDetector;
import stirling.software.proprietary.formdetection.inference.RfDetr;
import stirling.software.proprietary.formdetection.inference.Yolo;
import stirling.software.proprietary.formdetection.model.DetectedField;
import stirling.software.proprietary.formdetection.model.ModelCatalogEntry;
@@ -119,8 +120,8 @@ public class FormDetectionController {
for (PageRasterizer.RasterPage page : pages) {
Yolo.Preprocessed pre =
Yolo.preprocess(page.rgba(), page.widthPx(), page.heightPx(), spec);
Yolo.RawOutput out = detector.infer(pre.chw(), spec.getInputSize());
for (Yolo.Detection d : Yolo.decode(out, spec, pre, score)) {
Map<String, Yolo.RawOutput> out = detector.infer(pre.chw(), spec.getInputSize());
for (Yolo.Detection d : decodeFor(spec, out, pre, score)) {
DetectedField.RectPt rect = CoordinateMapper.toPdfPoints(d, page);
if (rect.w() <= 0 || rect.h() <= 0) {
continue;
@@ -165,6 +166,22 @@ public class FormDetectionController {
return ResponseEntity.ok(new DetectResponse(detections));
}
/**
* Pick the decoder the model's head needs. Unknown values fall back to YOLO, which is what
* every catalogue entry was before a second head shape existed.
*/
private static List<Yolo.Detection> decodeFor(
ModelCatalogEntry spec,
Map<String, Yolo.RawOutput> outputs,
Yolo.Preprocessed pre,
float score) {
if ("rfdetr".equalsIgnoreCase(spec.getDecoder())) {
return RfDetr.decode(outputs, spec, pre, score);
}
// Single-output head: take the sole tensor whatever the graph happens to call it.
return Yolo.decode(outputs.values().iterator().next(), spec, pre, score);
}
private static String fieldType(ModelCatalogEntry spec, int classId) {
List<String> types = spec.getClassFieldTypes();
if (types != null && classId >= 0 && classId < types.size()) {
app/proprietary/src/main/java/stirling/software/proprietary/formdetection/inference/OnnxFormDetector.java
+37 -14
View File
@@ -3,6 +3,8 @@ package stirling.software.proprietary.formdetection.inference;
import java.nio.FloatBuffer;
import java.nio.file.Path;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.Semaphore;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -45,7 +47,15 @@ public class OnnxFormDetector {
private volatile String loadedModelId;
private volatile String inputName;
public Yolo.RawOutput infer(float[] chw, int inputSize) {
/**
* Run the model and return every output tensor keyed by its graph name, in graph order.
*
* <p>Keyed by name rather than position because a query-based head emits two tensors of the
* SAME shape - RF-DETR's {@code dets} and {@code labels} are both [1, 300, 4] when there are
* three classes, since 4 box values and 3 classes + 1 no-object slot coincide. Picking by index
* would silently decode logits as boxes.
*/
public Map<String, Yolo.RawOutput> infer(float[] chw, int inputSize) {
ensureLoaded();
concurrency.acquireUninterruptibly();
lock.readLock().lock();
@@ -55,21 +65,14 @@ public class OnnxFormDetector {
try (OnnxTensor tensor = OnnxTensor.createTensor(env, FloatBuffer.wrap(chw), shape);
OrtSession.Result results =
session.run(Collections.singletonMap(inputName, tensor))) {
OnnxValue value = results.get(0);
Object raw = value.getValue();
if (!(raw instanceof float[][][] out3) || out3.length == 0) {
throw new IllegalStateException(
"Unexpected ONNX output type: "
+ (raw == null ? "null" : raw.getClass()));
Map<String, Yolo.RawOutput> outputs = new LinkedHashMap<>();
for (Map.Entry<String, OnnxValue> entry : results) {
outputs.put(entry.getKey(), toRawOutput(entry.getKey(), entry.getValue()));
}
float[][] m = out3[0];
int d1 = m.length;
int d2 = d1 > 0 ? m[0].length : 0;
float[] flat = new float[d1 * d2];
for (int i = 0; i < d1; i++) {
System.arraycopy(m[i], 0, flat, i * d2, d2);
if (outputs.isEmpty()) {
throw new IllegalStateException("ONNX session returned no outputs");
}
return new Yolo.RawOutput(flat, d1, d2);
return outputs;
}
} catch (OrtException e) {
throw new IllegalStateException("ONNX inference failed: " + e.getMessage(), e);
@@ -79,6 +82,26 @@ public class OnnxFormDetector {
}
}
/** Flatten a [1, d1, d2] output into the row-major buffer the decoders index into. */
private static Yolo.RawOutput toRawOutput(String name, OnnxValue value) throws OrtException {
Object raw = value.getValue();
if (!(raw instanceof float[][][] out3) || out3.length == 0) {
throw new IllegalStateException(
"Unexpected ONNX output type for '"
+ name
+ "': "
+ (raw == null ? "null" : raw.getClass()));
}
float[][] m = out3[0];
int d1 = m.length;
int d2 = d1 > 0 ? m[0].length : 0;
float[] flat = new float[d1 * d2];
for (int i = 0; i < d1; i++) {
System.arraycopy(m[i], 0, flat, i * d2, d2);
}
return new Yolo.RawOutput(flat, d1, d2);
}
/** Force the next inference to reload from disk (called after install/uninstall). */
public void unload() {
lock.writeLock().lock();
app/proprietary/src/main/java/stirling/software/proprietary/formdetection/inference/RfDetr.java
+120
View File
@@ -0,0 +1,120 @@
package stirling.software.proprietary.formdetection.inference;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import stirling.software.proprietary.formdetection.model.ModelCatalogEntry;
/**
* Decoder for RF-DETR style query-based heads, as used by the Apache-2.0 FFDetr checkpoint.
*
* <p>Differs from {@link Yolo} in every respect that matters downstream: two output tensors instead
* of one, a fixed set of queries instead of an anchor grid, boxes normalised to [0,1] instead of
* input pixels, and raw logits instead of activated scores. Preprocessing is shared - {@link
* Yolo#preprocess} already honours the spec's channel order and mean/std, which is all RF-DETR
* needs (RGB, ImageNet normalisation).
*/
@Slf4j
public final class RfDetr {
private RfDetr() {}
/** Normalised box centres/sizes. */
private static final String BOXES = "dets";
/** Per-class logits, plus a trailing no-object column. */
private static final String LOGITS = "labels";
/**
* Decode two named outputs into detections in original-bitmap pixels.
*
* <p>Outputs are looked up by name, never by position: with three classes both tensors are
* [300, 4] - four box values against three classes plus the no-object slot - so they cannot be
* told apart by shape.
*/
public static List<Yolo.Detection> decode(
Map<String, Yolo.RawOutput> outputs,
ModelCatalogEntry spec,
Yolo.Preprocessed pre,
float scoreThreshold) {
int numClasses = spec.getClassNames() == null ? 0 : spec.getClassNames().size();
if (numClasses == 0) {
return List.of();
}
Yolo.RawOutput boxes = outputs.get(BOXES);
Yolo.RawOutput logits = outputs.get(LOGITS);
if (boxes == null || logits == null) {
log.warn(
"rfdetr decoder expects outputs '{}' and '{}'; got {}",
BOXES,
LOGITS,
outputs.keySet());
return List.of();
}
if (boxes.d2() < 4) {
log.warn("rfdetr '{}' has {} columns, expected >= 4", BOXES, boxes.d2());
return List.of();
}
// The head emits one logit per class plus a no-object slot; anything narrower means the
// model was trained for a different class set than the catalogue entry claims.
if (logits.d2() < numClasses) {
log.warn(
"rfdetr '{}' has {} columns for {} classes; skipping",
LOGITS,
logits.d2(),
numClasses);
return List.of();
}
int queries = Math.min(boxes.d1(), logits.d1());
List<Yolo.Detection> dets = new ArrayList<>();
for (int q = 0; q < queries; q++) {
int bestClass = -1;
float bestScore = 0f;
// Deliberately stops at numClasses, dropping the trailing no-object column.
for (int c = 0; c < numClasses; c++) {
float score = sigmoid(logits.data()[q * logits.d2() + c]);
if (score > bestScore) {
bestScore = score;
bestClass = c;
}
}
if (bestClass < 0 || bestScore < scoreThreshold) {
continue;
}
int base = q * boxes.d2();
// Normalised centre form -> input-space pixels, so the un-projection below is the
// same arithmetic the YOLO path uses.
float cx = boxes.data()[base] * pre.inputSize();
float cy = boxes.data()[base + 1] * pre.inputSize();
float w = boxes.data()[base + 2] * pre.inputSize();
float h = boxes.data()[base + 3] * pre.inputSize();
float x1 = cx - w / 2f;
float y1 = cy - h / 2f;
float ox = (x1 - pre.padX()) / pre.scaleX();
float oy = (y1 - pre.padY()) / pre.scaleY();
float ow = w / pre.scaleX();
float oh = h / pre.scaleY();
float clampedX = Math.max(0, Math.min(ox, pre.srcW()));
float clampedY = Math.max(0, Math.min(oy, pre.srcH()));
ow = Math.max(0, Math.min(ow, pre.srcW() - clampedX));
oh = Math.max(0, Math.min(oh, pre.srcH() - clampedY));
if (ow <= 0 || oh <= 0) {
continue;
}
dets.add(new Yolo.Detection(bestClass, bestScore, clampedX, clampedY, ow, oh));
}
return Yolo.nms(dets, spec.getNms(), spec.getIou());
}
private static float sigmoid(float x) {
return (float) (1.0 / (1.0 + Math.exp(-x)));
}
}
app/proprietary/src/main/java/stirling/software/proprietary/formdetection/inference/Yolo.java
+2 -1
View File
@@ -176,7 +176,8 @@ public final class Yolo {
return ncFirst ? data[c * anchors + a] : data[a * channels + c];
}
private static List<Detection> nms(List<Detection> dets, String mode, float iouThreshold) {
/** Shared with {@link RfDetr}: identical suppression whatever head produced the boxes. */
static List<Detection> nms(List<Detection> dets, String mode, float iouThreshold) {
if (dets.size() < 2 || "none".equalsIgnoreCase(mode)) {
return dets;
}
app/proprietary/src/main/java/stirling/software/proprietary/formdetection/model/ModelCatalogEntry.java
+13
View File
@@ -54,6 +54,19 @@ public class ModelCatalogEntry {
private float[] normStd = {1f, 1f, 1f};
// --- Post-processing (parity-critical) ---------------------------------------
/**
* Which head shape the model emits, and so how its output is read.
*
* <ul>
* <li>{@code yolo} - one anchor-grid tensor, boxes already in input pixels, scores already
* through their activation. Described by {@link #outputLayout}/{@link #hasObjectness}.
* <li>{@code rfdetr} - two named tensors, {@code dets} (normalised cxcywh) and {@code labels}
* (raw logits, one column per class plus a trailing no-object slot). Query based, so
* there is no anchor grid and the two fields above do not apply.
* </ul>
*/
private String decoder = "yolo";
/** "nc_first" => output [1, 4+nc, anchors]; "anchors_first" => [1, anchors, 4+nc]. */
private String outputLayout = "nc_first";
app/proprietary/src/main/resources/formdetection/NOTICE
+44
View File
@@ -0,0 +1,44 @@
Auto Form Detection - third-party model attribution
===================================================
Stirling-PDF bundles no model weights. Models are downloaded on demand from the URLs in
model-catalog.json and verified against the SHA-256 recorded there.
This NOTICE covers weights Stirling itself publishes or redistributes. It exists because
Apache-2.0 section 4 attaches attribution and NOTICE obligations to whoever conveys the work,
and converting a checkpoint to ONNX and hosting the result makes us that party.
ffdetr
------
FFDetr, a form-field detector trained on the CommonForms dataset.
Weights https://huggingface.co/jbarrow/FFDetr Apache-2.0
Base model Roboflow RF-DETR (rf-detr-medium) Apache-2.0
Framework https://github.com/roboflow/rf-detr Apache-2.0
Backbone DINOv2 (Meta AI) Apache-2.0
Dataset https://huggingface.co/datasets/jbarrow/CommonForms Apache-2.0
Paper CommonForms, arXiv:2509.16506
The .onnx Stirling distributes is not the upstream artifact: the publisher releases a PyTorch
.pth only. Stirling exports it to ONNX and quantises it to int8 via
scripts/export-ffdetr-onnx.py, which pins the source revision and its checksum. No weights are
modified beyond that conversion.
Not vendored: the `commonforms` reference wrapper (github.com/jbarrow/commonforms) carries no
licence file and depends on Ultralytics, so none of its code is used. The pre/post-processing in
RfDetr.java and decode.ts is written from the model's own input/output contract.
ffdnet-s, ffdnet-l
------------------
NOT REDISTRIBUTED BY STIRLING, and deliberately so.
The FFDNet checkpoints (huggingface.co/jbarrow/FFDNet-S-cpu and FFDNet-L-cpu) declare no
licence at all, which under default copyright means all rights reserved. They are also trained
with Ultralytics YOLO11, whose AGPL-3.0 terms their publisher asserts no grant over.
They remain in the catalogue so an operator who has their own arrangement with the publisher can
install them, but the download comes from the publisher's own URL, they are never baked into a
Stirling image, and they must not become a shipped default. See the notes on
FORM_DETECTION_MODEL_URL in docker/embedded/Dockerfile.
app/proprietary/src/main/resources/formdetection/model-catalog.json
+91 -10
View File
@@ -1,4 +1,45 @@
[
{
"id": "ffdetr",
"displayName": "CommonForms FFDetr (Apache-2.0)",
"description": "Permissively licensed detector trained on the same CommonForms data. Finds text inputs, checkboxes and signature fields. Quantised to int8, so it is the smallest option as well as the only one with a licence grant.",
"license": "Apache-2.0. Weights https://huggingface.co/jbarrow/FFDetr, base model Roboflow RF-DETR, CommonForms dataset - all Apache-2.0. Converted to ONNX by Stirling via scripts/export-ffdetr-onnx.py.",
"sizeBytes": 37116508,
"onnxUrl": "",
"sha256": "2323b426456887fd8befc3e0dd508497c2084d7936e7da046bc023188f119982",
"decoder": "rfdetr",
"inputSize": 1024,
"resizeMode": "stretch",
"padColor": [
114,
114,
114
],
"channelOrder": "rgb",
"normMean": [
0.485,
0.456,
0.406
],
"normStd": [
0.229,
0.224,
0.225
],
"classNames": [
"text",
"choice",
"signature"
],
"classFieldTypes": [
"text",
"checkbox",
"signature"
],
"scoreThreshold": 0.3,
"nms": "classAgnostic",
"iou": 0.45
},
{
"id": "ffdnet-s",
"displayName": "CommonForms FFDNet-S (Small)",
@@ -9,14 +50,34 @@
"sha256": "93bccf47c048f9f947f9b1b52d002edf144a8a583dae39f164d9e5725321acc0",
"inputSize": 1216,
"resizeMode": "stretch",
"padColor": [114, 114, 114],
"padColor": [
114,
114,
114
],
"channelOrder": "bgr",
"normMean": [0.0, 0.0, 0.0],
"normStd": [1.0, 1.0, 1.0],
"normMean": [
0.0,
0.0,
0.0
],
"normStd": [
1.0,
1.0,
1.0
],
"outputLayout": "nc_first",
"hasObjectness": false,
"classNames": ["text", "choice", "signature"],
"classFieldTypes": ["text", "checkbox", "signature"],
"classNames": [
"text",
"choice",
"signature"
],
"classFieldTypes": [
"text",
"checkbox",
"signature"
],
"scoreThreshold": 0.3,
"nms": "perClass",
"iou": 0.45
@@ -31,14 +92,34 @@
"sha256": "e00c59edd9a5275dab5847d38f042c8ecc827063650c8aac22b0e486c414cd35",
"inputSize": 1216,
"resizeMode": "stretch",
"padColor": [114, 114, 114],
"padColor": [
114,
114,
114
],
"channelOrder": "bgr",
"normMean": [0.0, 0.0, 0.0],
"normStd": [1.0, 1.0, 1.0],
"normMean": [
0.0,
0.0,
0.0
],
"normStd": [
1.0,
1.0,
1.0
],
"outputLayout": "nc_first",
"hasObjectness": false,
"classNames": ["text", "choice", "signature"],
"classFieldTypes": ["text", "checkbox", "signature"],
"classNames": [
"text",
"choice",
"signature"
],
"classFieldTypes": [
"text",
"checkbox",
"signature"
],
"scoreThreshold": 0.3,
"nms": "perClass",
"iou": 0.45
app/proprietary/src/test/java/stirling/software/proprietary/formdetection/catalog/ModelCatalogServiceTest.java
+34 -6
View File
@@ -29,14 +29,42 @@ class ModelCatalogServiceTest {
assertEquals(3, l.getClassFieldTypes().size());
assertTrue(l.getInputSize() > 0);
// Model-free distribution: the jar bundles no weights. Every entry instead carries a
// download URL and a SHA-256 so the model is fetched and integrity-verified on demand.
// Model-free distribution: the jar bundles no weights, they are fetched on demand. What
// must never happen is downloading without a checksum to verify against, so any entry
// that declares a URL must also declare a SHA-256. An entry may legitimately carry
// neither yet - the admin panel renders it as not-installable (see `installable` in
// AdminFormDetectionSection) - which is how a model we have not published lands here.
for (ModelCatalogEntry e : all) {
assertNotNull(e.getOnnxUrl(), e.getId() + " must declare a download URL");
assertFalse(e.getOnnxUrl().isBlank(), e.getId() + " must declare a download URL");
assertNotNull(e.getSha256(), e.getId() + " must declare a SHA-256 checksum");
assertFalse(e.getSha256().isBlank(), e.getId() + " must declare a SHA-256 checksum");
assertNotNull(e.getOnnxUrl(), e.getId() + " must declare a URL field, even if blank");
assertNotNull(e.getSha256(), e.getId() + " must declare a checksum");
if (!e.getOnnxUrl().isBlank()) {
assertFalse(
e.getSha256().isBlank(),
e.getId() + " declares a download URL so it must declare a SHA-256");
}
}
assertTrue(
all.stream().anyMatch(e -> !e.getOnnxUrl().isBlank()),
"at least one entry must actually be installable");
}
@Test
void ffdetrIsTheApacheLicensedEntryAndUsesTheQueryHeadDecoder() {
ModelCatalogService service = new ModelCatalogService(JsonMapper.builder().build());
service.load();
ModelCatalogEntry ffdetr = service.getById("ffdetr").orElseThrow();
assertEquals("rfdetr", ffdetr.getDecoder(), "FFDetr has a query head, not an anchor grid");
assertEquals(1024, ffdetr.getInputSize());
assertTrue(ffdetr.getLicense().startsWith("Apache-2.0"), ffdetr.getLicense());
// RF-DETR expects ImageNet normalisation over RGB, unlike the FFDNet entries' /255 BGR.
assertEquals("rgb", ffdetr.getChannelOrder());
assertEquals(0.485f, ffdetr.getNormMean()[0], 1e-6);
// The FFDNet entries keep the anchor-grid decoder; a wrong default here would silently
// decode one family with the other's maths.
assertEquals("yolo", service.getById("ffdnet-s").orElseThrow().getDecoder());
assertEquals("yolo", service.getById("ffdnet-l").orElseThrow().getDecoder());
}
@Test
app/proprietary/src/test/java/stirling/software/proprietary/formdetection/inference/RfDetrTest.java
+188
View File
@@ -0,0 +1,188 @@
package stirling.software.proprietary.formdetection.inference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import stirling.software.proprietary.formdetection.model.ModelCatalogEntry;
/**
* Parity tests for the RF-DETR decode path.
*
* <p>The fixture holds real tensors captured from the exported FFDetr ONNX, together with the
* detections the reference Python decode produced from them. That makes this a genuine parity check
* against the model rather than a restatement of the Java code: if the Java decode forgets the
* sigmoid, reads the no-object column as a class, or treats the boxes as pixels rather than
* normalised, the expected values below stop matching.
*/
class RfDetrTest {
private static final String FIXTURE = "/formdetection/rfdetr-reference.json";
private record Fixture(
Map<String, Yolo.RawOutput> outputs,
List<JsonNode> expected,
int inputSize,
float thr) {}
private static Fixture load() throws Exception {
ObjectMapper mapper = new ObjectMapper();
JsonNode root;
try (InputStream in = RfDetrTest.class.getResourceAsStream(FIXTURE)) {
assertNotNull(in, "missing fixture " + FIXTURE);
root = mapper.readTree(in);
}
JsonNode queries = root.get("queries");
int n = queries.size();
int boxCols = queries.get(0).get("dets").size();
int logitCols = queries.get(0).get("labels").size();
float[] boxes = new float[n * boxCols];
float[] logits = new float[n * logitCols];
for (int i = 0; i < n; i++) {
for (int c = 0; c < boxCols; c++) {
boxes[i * boxCols + c] = (float) queries.get(i).get("dets").get(c).asDouble();
}
for (int c = 0; c < logitCols; c++) {
logits[i * logitCols + c] = (float) queries.get(i).get("labels").get(c).asDouble();
}
}
Map<String, Yolo.RawOutput> outputs = new LinkedHashMap<>();
outputs.put("dets", new Yolo.RawOutput(boxes, n, boxCols));
outputs.put("labels", new Yolo.RawOutput(logits, n, logitCols));
List<JsonNode> expected = new ArrayList<>();
// Re-index expectations onto the trimmed query list the fixture actually carries.
for (JsonNode e : root.get("expected")) {
expected.add(e);
}
return new Fixture(
outputs,
expected,
root.get("inputSize").asInt(),
(float) root.get("scoreThreshold").asDouble());
}
/**
* Identity mapping: model input space == source bitmap, so decode output is directly
* comparable.
*/
private static Yolo.Preprocessed identityPre(int inputSize) {
return new Yolo.Preprocessed(new float[0], inputSize, 1f, 1f, 0, 0, inputSize, inputSize);
}
private static ModelCatalogEntry spec() {
ModelCatalogEntry spec = new ModelCatalogEntry();
spec.setDecoder("rfdetr");
spec.setInputSize(1024);
spec.setClassNames(List.of("text", "choice", "signature"));
spec.setNms("none");
return spec;
}
@Test
void decodesTheExportedModelsRealOutputToTheReferenceDetections() throws Exception {
Fixture f = load();
List<Yolo.Detection> got =
RfDetr.decode(f.outputs(), spec(), identityPre(f.inputSize()), f.thr());
assertEquals(
f.expected().size(),
got.size(),
"detection count must match the Python reference decode");
// Compare as sets keyed on rounded geometry: NMS is off, so order is query order in both.
for (int i = 0; i < got.size(); i++) {
JsonNode want = f.expected().get(i);
Yolo.Detection d = got.get(i);
assertEquals(want.get("cls").asInt(), d.classId(), "class at index " + i);
assertEquals(want.get("score").asDouble(), d.score(), 1e-4, "score at index " + i);
assertEquals(want.get("x").asDouble(), d.x(), 0.05, "x at index " + i);
assertEquals(want.get("y").asDouble(), d.y(), 0.05, "y at index " + i);
assertEquals(want.get("w").asDouble(), d.w(), 0.05, "w at index " + i);
assertEquals(want.get("h").asDouble(), d.h(), 0.05, "h at index " + i);
}
}
@Test
void findsTheFormsEightTextTwoChoiceAndOneSignature() throws Exception {
Fixture f = load();
List<Yolo.Detection> got =
RfDetr.decode(f.outputs(), spec(), identityPre(f.inputSize()), f.thr());
long text = got.stream().filter(d -> d.classId() == 0).count();
long choice = got.stream().filter(d -> d.classId() == 1).count();
long signature = got.stream().filter(d -> d.classId() == 2).count();
assertEquals(8, text, "text fields");
assertEquals(2, choice, "checkboxes");
assertEquals(1, signature, "signature line");
}
@Test
void bindsOutputsByNameNotPosition() throws Exception {
Fixture f = load();
// Both tensors are [n, 4] here, so a positional reader cannot tell them apart. Swapping
// insertion order must change nothing.
Map<String, Yolo.RawOutput> swapped = new LinkedHashMap<>();
swapped.put("labels", f.outputs().get("labels"));
swapped.put("dets", f.outputs().get("dets"));
List<Yolo.Detection> normal =
RfDetr.decode(f.outputs(), spec(), identityPre(f.inputSize()), f.thr());
List<Yolo.Detection> reordered =
RfDetr.decode(swapped, spec(), identityPre(f.inputSize()), f.thr());
assertEquals(normal.toString(), reordered.toString());
}
@Test
void fixtureCarriesTheAmbiguousFourColumnShape() throws Exception {
Fixture f = load();
assertEquals(
4,
f.outputs().get("labels").d2(),
"3 classes + 1 no-object slot - the case where labels and dets share a shape");
assertEquals(4, f.outputs().get("dets").d2());
for (Yolo.Detection d :
RfDetr.decode(f.outputs(), spec(), identityPre(f.inputSize()), f.thr())) {
assertTrue(d.classId() >= 0 && d.classId() < 3, "classId out of range: " + d.classId());
}
}
/**
* Hand-built because the exported model never lets the no-object column win - its highest
* sigmoid across all 300 queries is 0.0014 - so real tensors cannot exercise this guard.
* Without it a dominant 4th column would yield classId 3 and index past classNames.
*/
@Test
void neverClassifiesAQueryAsTheNoObjectColumn() {
// One query: no-object logit is overwhelmingly the largest, real classes are weak.
float[] logits = {-4.0f, -3.0f, -5.0f, 9.0f};
float[] boxes = {0.5f, 0.5f, 0.2f, 0.1f};
Map<String, Yolo.RawOutput> outputs = new LinkedHashMap<>();
outputs.put("dets", new Yolo.RawOutput(boxes, 1, 4));
outputs.put("labels", new Yolo.RawOutput(logits, 1, 4));
List<Yolo.Detection> got = RfDetr.decode(outputs, spec(), identityPre(1024), 0.30f);
assertTrue(
got.isEmpty(),
"a query dominated by the no-object slot must be dropped, got: " + got);
}
@Test
void returnsNothingWhenAnExpectedOutputIsAbsent() throws Exception {
Fixture f = load();
Map<String, Yolo.RawOutput> onlyBoxes = Map.of("dets", f.outputs().get("dets"));
assertTrue(RfDetr.decode(onlyBoxes, spec(), identityPre(f.inputSize()), f.thr()).isEmpty());
}
}
app/proprietary/src/test/resources/formdetection/rfdetr-reference.json
+318
View File
@@ -0,0 +1,318 @@
{
"note": "Captured from ffdetr-int8.onnx on a synthetic form page via scripts/export-ffdetr-onnx.py.",
"inputSize": 1024,
"scoreThreshold": 0.3,
"queries": [
{
"q": 0,
"dets": [
0.594326,
0.253922,
0.575291,
0.03704
],
"labels": [
1.494305,
-4.254563,
-2.83488,
-8.221176
]
},
{
"q": 1,
"dets": [
0.593039,
0.305142,
0.574628,
0.036171
],
"labels": [
1.501225,
-4.211613,
-2.82295,
-8.282976
]
},
{
"q": 2,
"dets": [
0.594104,
0.355328,
0.573003,
0.036946
],
"labels": [
1.550975,
-4.177611,
-3.121929,
-8.416718
]
},
{
"q": 3,
"dets": [
0.593218,
0.204408,
0.573657,
0.036336
],
"labels": [
1.483687,
-4.080855,
-2.717842,
-8.144939
]
},
{
"q": 4,
"dets": [
0.593249,
0.102776,
0.573464,
0.03605
],
"labels": [
1.52258,
-4.080616,
-2.611064,
-8.24098
]
},
{
"q": 5,
"dets": [
0.593204,
0.153512,
0.572868,
0.036364
],
"labels": [
1.489175,
-4.014163,
-2.746595,
-8.259592
]
},
{
"q": 6,
"dets": [
0.593207,
0.406301,
0.575204,
0.036417
],
"labels": [
1.523296,
-4.076083,
-2.997732,
-8.246349
]
},
{
"q": 7,
"dets": [
0.336551,
0.487273,
0.021658,
0.016205
],
"labels": [
-2.577461,
1.216892,
-4.798167,
-7.703153
]
},
{
"q": 8,
"dets": [
0.824501,
0.570058,
0.114489,
0.03537
],
"labels": [
1.210955,
-3.528949,
-3.637566,
-8.277011
]
},
{
"q": 9,
"dets": [
0.499909,
0.487268,
0.021303,
0.016313
],
"labels": [
-2.650476,
1.275709,
-4.80449,
-7.729639
]
},
{
"q": 10,
"dets": [
0.480432,
0.569626,
0.345508,
0.03706
],
"labels": [
-0.667743,
-4.025378,
-0.391272,
-7.142419
]
},
{
"q": 99,
"dets": [
0.787065,
0.572109,
0.041269,
0.032919
],
"labels": [
-4.071043,
-4.381146,
-5.294834,
-8.117142
]
},
{
"q": 150,
"dets": [
0.617099,
0.154866,
0.457747,
0.034707
],
"labels": [
-3.982041,
-5.252909,
-4.907092,
-8.302184
]
},
{
"q": 299,
"dets": [
0.595358,
0.708946,
0.331338,
0.039393
],
"labels": [
-4.61758,
-5.568352,
-4.882038,
-7.22629
]
}
],
"expected": [
{
"q": 0,
"cls": 0,
"score": 0.81672,
"x": 314.041,
"y": 241.051,
"w": 589.098,
"h": 37.929
},
{
"q": 1,
"cls": 0,
"score": 0.81776,
"x": 313.062,
"y": 293.945,
"w": 588.419,
"h": 37.039
},
{
"q": 2,
"cls": 0,
"score": 0.82505,
"x": 314.985,
"y": 344.94,
"w": 586.755,
"h": 37.833
},
{
"q": 3,
"cls": 0,
"score": 0.81513,
"x": 313.743,
"y": 190.709,
"w": 587.424,
"h": 37.208
},
{
"q": 4,
"cls": 0,
"score": 0.82092,
"x": 313.873,
"y": 86.785,
"w": 587.227,
"h": 36.915
},
{
"q": 5,
"cls": 0,
"score": 0.81595,
"x": 314.133,
"y": 138.578,
"w": 586.617,
"h": 37.237
},
{
"q": 6,
"cls": 0,
"score": 0.82102,
"x": 312.94,
"y": 397.407,
"w": 589.009,
"h": 37.291
},
{
"q": 7,
"cls": 1,
"score": 0.77152,
"x": 333.539,
"y": 490.671,
"w": 22.178,
"h": 16.594
},
{
"q": 8,
"cls": 0,
"score": 0.77047,
"x": 785.671,
"y": 565.63,
"w": 117.237,
"h": 36.219
},
{
"q": 9,
"cls": 1,
"score": 0.78172,
"x": 500.999,
"y": 490.61,
"w": 21.814,
"h": 16.705
},
{
"q": 10,
"cls": 2,
"score": 0.40341,
"x": 315.062,
"y": 564.322,
"w": 353.801,
"h": 37.95
}
]
}
docker/embedded/Dockerfile
+20 -7
View File
@@ -19,14 +19,27 @@ RUN apt-get update \
&& rm /tmp/task.deb \
&& rm -rf /var/lib/apt/lists/*
# Pre-download the default Auto Form Detection model (FFDNet-S, ~37MB) so the feature works
# out-of-the-box. Verified by checksum here; seeded into the writable configs model dir at startup
# by FormDetectionModelManager (FORMDETECTION_PREINSTALLEDMODELDIR below).
ARG FORM_DETECTION_MODEL_URL=https://huggingface.co/jbarrow/FFDNet-S-cpu/resolve/d6cb18cb4bf31d7f5adfe8fbc7cc3744b516d1fe/FFDNet-S.onnx
ARG FORM_DETECTION_MODEL_SHA256=93bccf47c048f9f947f9b1b52d002edf144a8a583dae39f164d9e5725321acc0
# Optionally bake an Auto Form Detection model into the image, seeded into the writable configs
# model dir at startup by FormDetectionModelManager (FORMDETECTION_PREINSTALLEDMODELDIR below).
#
# Empty by default ON PURPOSE. Baking a model in makes us its distributor, so only weights we
# hold a redistribution grant for may go here - which the FFDNet checkpoints are not, as their
# publisher declares no licence at all. Set all three args to pre-bundle a model you may ship:
# --build-arg FORM_DETECTION_MODEL_URL=... \
# --build-arg FORM_DETECTION_MODEL_SHA256=... \
# --build-arg FORM_DETECTION_MODEL_ID=ffdetr
# Without them the image ships no model and the tile stays dependency-disabled until an admin
# installs one from the catalogue.
ARG FORM_DETECTION_MODEL_URL=""
ARG FORM_DETECTION_MODEL_SHA256=""
ARG FORM_DETECTION_MODEL_ID="ffdetr"
RUN mkdir -p /preinstalled-models \
&& curl -fSL "${FORM_DETECTION_MODEL_URL}" -o /preinstalled-models/ffdnet-s.onnx \
&& echo "${FORM_DETECTION_MODEL_SHA256} /preinstalled-models/ffdnet-s.onnx" | sha256sum -c -
&& if [ -n "${FORM_DETECTION_MODEL_URL}" ]; then \
curl -fSL "${FORM_DETECTION_MODEL_URL}" -o "/preinstalled-models/${FORM_DETECTION_MODEL_ID}.onnx" \
&& echo "${FORM_DETECTION_MODEL_SHA256} /preinstalled-models/${FORM_DETECTION_MODEL_ID}.onnx" | sha256sum -c -; \
else \
echo "No form-detection model pre-bundled (FORM_DETECTION_MODEL_URL unset)"; \
fi
# JDK 25+: --add-exports is no longer accepted via JAVA_TOOL_OPTIONS; use JDK_JAVA_OPTIONS instead
ENV JDK_JAVA_OPTIONS="--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
docker/embedded/Dockerfile.fat
+18 -7
View File
@@ -20,14 +20,25 @@ RUN apt-get update \
&& rm /tmp/task.deb \
&& rm -rf /var/lib/apt/lists/*
# Pre-download the default Auto Form Detection model (FFDNet-S, ~37MB) so the feature works
# out-of-the-box - especially important for this air-gapped fat image (no runtime internet).
# Verified by checksum; seeded into the writable configs model dir at startup.
ARG FORM_DETECTION_MODEL_URL=https://huggingface.co/jbarrow/FFDNet-S-cpu/resolve/d6cb18cb4bf31d7f5adfe8fbc7cc3744b516d1fe/FFDNet-S.onnx
ARG FORM_DETECTION_MODEL_SHA256=93bccf47c048f9f947f9b1b52d002edf144a8a583dae39f164d9e5725321acc0
# Optionally bake an Auto Form Detection model into this air-gapped image, seeded into the
# writable configs model dir at startup.
#
# Empty by default ON PURPOSE - see the note in docker/embedded/Dockerfile. Baking a model in
# makes us its distributor, and it is exactly the air-gapped case that makes an unlicensed
# bundle hardest to undo. Set all three args to pre-bundle weights you may ship:
# --build-arg FORM_DETECTION_MODEL_URL=... \
# --build-arg FORM_DETECTION_MODEL_SHA256=... \
# --build-arg FORM_DETECTION_MODEL_ID=ffdetr
ARG FORM_DETECTION_MODEL_URL=""
ARG FORM_DETECTION_MODEL_SHA256=""
ARG FORM_DETECTION_MODEL_ID="ffdetr"
RUN mkdir -p /preinstalled-models \
&& curl -fSL "${FORM_DETECTION_MODEL_URL}" -o /preinstalled-models/ffdnet-s.onnx \
&& echo "${FORM_DETECTION_MODEL_SHA256} /preinstalled-models/ffdnet-s.onnx" | sha256sum -c -
&& if [ -n "${FORM_DETECTION_MODEL_URL}" ]; then \
curl -fSL "${FORM_DETECTION_MODEL_URL}" -o "/preinstalled-models/${FORM_DETECTION_MODEL_ID}.onnx" \
&& echo "${FORM_DETECTION_MODEL_SHA256} /preinstalled-models/${FORM_DETECTION_MODEL_ID}.onnx" | sha256sum -c -; \
else \
echo "No form-detection model pre-bundled (FORM_DETECTION_MODEL_URL unset)"; \
fi
# JDK 25+: --add-exports is no longer accepted via JAVA_TOOL_OPTIONS; use JDK_JAVA_OPTIONS instead
ENV JDK_JAVA_OPTIONS="--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
frontend/editor/src/core/hooks/useFormDetectionModelStatus.ts
+1
View File
@@ -12,6 +12,7 @@ export interface FormDetectionCatalogEntry {
onnxUrl: string;
sha256: string;
// Pipeline spec (parity with the backend ModelCatalogEntry) - drives the in-browser engine.
decoder?: string;
inputSize: number;
resizeMode?: string;
padColor?: number[];
frontend/editor/src/core/services/formDetection/__fixtures__/rfdetr-reference.json
+318
View File
@@ -0,0 +1,318 @@
{
"note": "Captured from ffdetr-int8.onnx on a synthetic form page via scripts/export-ffdetr-onnx.py.",
"inputSize": 1024,
"scoreThreshold": 0.3,
"queries": [
{
"q": 0,
"dets": [
0.594326,
0.253922,
0.575291,
0.03704
],
"labels": [
1.494305,
-4.254563,
-2.83488,
-8.221176
]
},
{
"q": 1,
"dets": [
0.593039,
0.305142,
0.574628,
0.036171
],
"labels": [
1.501225,
-4.211613,
-2.82295,
-8.282976
]
},
{
"q": 2,
"dets": [
0.594104,
0.355328,
0.573003,
0.036946
],
"labels": [
1.550975,
-4.177611,
-3.121929,
-8.416718
]
},
{
"q": 3,
"dets": [
0.593218,
0.204408,
0.573657,
0.036336
],
"labels": [
1.483687,
-4.080855,
-2.717842,
-8.144939
]
},
{
"q": 4,
"dets": [
0.593249,
0.102776,
0.573464,
0.03605
],
"labels": [
1.52258,
-4.080616,
-2.611064,
-8.24098
]
},
{
"q": 5,
"dets": [
0.593204,
0.153512,
0.572868,
0.036364
],
"labels": [
1.489175,
-4.014163,
-2.746595,
-8.259592
]
},
{
"q": 6,
"dets": [
0.593207,
0.406301,
0.575204,
0.036417
],
"labels": [
1.523296,
-4.076083,
-2.997732,
-8.246349
]
},
{
"q": 7,
"dets": [
0.336551,
0.487273,
0.021658,
0.016205
],
"labels": [
-2.577461,
1.216892,
-4.798167,
-7.703153
]
},
{
"q": 8,
"dets": [
0.824501,
0.570058,
0.114489,
0.03537
],
"labels": [
1.210955,
-3.528949,
-3.637566,
-8.277011
]
},
{
"q": 9,
"dets": [
0.499909,
0.487268,
0.021303,
0.016313
],
"labels": [
-2.650476,
1.275709,
-4.80449,
-7.729639
]
},
{
"q": 10,
"dets": [
0.480432,
0.569626,
0.345508,
0.03706
],
"labels": [
-0.667743,
-4.025378,
-0.391272,
-7.142419
]
},
{
"q": 99,
"dets": [
0.787065,
0.572109,
0.041269,
0.032919
],
"labels": [
-4.071043,
-4.381146,
-5.294834,
-8.117142
]
},
{
"q": 150,
"dets": [
0.617099,
0.154866,
0.457747,
0.034707
],
"labels": [
-3.982041,
-5.252909,
-4.907092,
-8.302184
]
},
{
"q": 299,
"dets": [
0.595358,
0.708946,
0.331338,
0.039393
],
"labels": [
-4.61758,
-5.568352,
-4.882038,
-7.22629
]
}
],
"expected": [
{
"q": 0,
"cls": 0,
"score": 0.81672,
"x": 314.041,
"y": 241.051,
"w": 589.098,
"h": 37.929
},
{
"q": 1,
"cls": 0,
"score": 0.81776,
"x": 313.062,
"y": 293.945,
"w": 588.419,
"h": 37.039
},
{
"q": 2,
"cls": 0,
"score": 0.82505,
"x": 314.985,
"y": 344.94,
"w": 586.755,
"h": 37.833
},
{
"q": 3,
"cls": 0,
"score": 0.81513,
"x": 313.743,
"y": 190.709,
"w": 587.424,
"h": 37.208
},
{
"q": 4,
"cls": 0,
"score": 0.82092,
"x": 313.873,
"y": 86.785,
"w": 587.227,
"h": 36.915
},
{
"q": 5,
"cls": 0,
"score": 0.81595,
"x": 314.133,
"y": 138.578,
"w": 586.617,
"h": 37.237
},
{
"q": 6,
"cls": 0,
"score": 0.82102,
"x": 312.94,
"y": 397.407,
"w": 589.009,
"h": 37.291
},
{
"q": 7,
"cls": 1,
"score": 0.77152,
"x": 333.539,
"y": 490.671,
"w": 22.178,
"h": 16.594
},
{
"q": 8,
"cls": 0,
"score": 0.77047,
"x": 785.671,
"y": 565.63,
"w": 117.237,
"h": 36.219
},
{
"q": 9,
"cls": 1,
"score": 0.78172,
"x": 500.999,
"y": 490.61,
"w": 21.814,
"h": 16.705
},
{
"q": 10,
"cls": 2,
"score": 0.40341,
"x": 315.062,
"y": 564.322,
"w": 353.801,
"h": 37.95
}
]
}
frontend/editor/src/core/services/formDetection/decode.test.ts
+1
View File
@@ -13,6 +13,7 @@ import {
// produce identical detections.
const spec: ModelPipelineSpec = {
decoder: "yolo",
inputSize: 10,
resizeMode: "letterbox",
padColor: [114, 114, 114],
frontend/editor/src/core/services/formDetection/decode.ts
+63
View File
@@ -121,3 +121,66 @@ export function decode(
}
return nms(dets, spec.nms, spec.iou);
}
/**
* Decode an RF-DETR style query head, the browser mirror of RfDetr.java.
*
* Outputs are looked up by name, never by position: with three classes both tensors are
* [queries, 4] - four box values against three classes plus a no-object slot - so they are
* indistinguishable by shape and a positional read would decode logits as boxes.
*/
export function decodeRfDetr(
outputs: Record<string, RawOutput>,
spec: ModelPipelineSpec,
pre: Preprocessed,
scoreThreshold: number,
): Detection[] {
const numClasses = spec.classNames?.length ?? 0;
if (numClasses === 0) return [];
const boxes = outputs["dets"];
const logits = outputs["labels"];
if (!boxes || !logits) return [];
if (boxes.d2 < 4 || logits.d2 < numClasses) return [];
const queries = Math.min(boxes.d1, logits.d1);
const size = spec.inputSize;
const dets: Detection[] = [];
for (let q = 0; q < queries; q++) {
let bestClass = -1;
let bestScore = 0;
// Stops at numClasses on purpose, dropping the trailing no-object column.
for (let c = 0; c < numClasses; c++) {
const s = 1 / (1 + Math.exp(-logits.data[q * logits.d2 + c]));
if (s > bestScore) {
bestScore = s;
bestClass = c;
}
}
if (bestClass < 0 || bestScore < scoreThreshold) continue;
const base = q * boxes.d2;
// Normalised centre form -> input pixels, so the un-projection matches the YOLO path.
const cx = boxes.data[base] * size;
const cy = boxes.data[base + 1] * size;
const w = boxes.data[base + 2] * size;
const h = boxes.data[base + 3] * size;
const ox = (cx - w / 2 - pre.padX) / pre.scaleX;
const oy = (cy - h / 2 - pre.padY) / pre.scaleY;
let ow = w / pre.scaleX;
let oh = h / pre.scaleY;
const cxl = Math.max(0, Math.min(ox, pre.srcW));
const cyl = Math.max(0, Math.min(oy, pre.srcH));
ow = Math.max(0, Math.min(ow, pre.srcW - cxl));
oh = Math.max(0, Math.min(oh, pre.srcH - cyl));
if (ow <= 0 || oh <= 0) continue;
dets.push({
classId: bestClass,
score: bestScore,
x: cxl,
y: cyl,
w: ow,
h: oh,
});
}
return nms(dets, spec.nms, spec.iou);
}
frontend/editor/src/core/services/formDetection/decodeRfDetr.test.ts
+108
View File
@@ -0,0 +1,108 @@
import { describe, expect, test } from "vitest";
import { decodeRfDetr } from "@app/services/formDetection/decode";
import {
Detection,
ModelPipelineSpec,
Preprocessed,
RawOutput,
} from "@app/services/formDetection/types";
import reference from "@app/services/formDetection/__fixtures__/rfdetr-reference.json";
// The same fixture RfDetrTest.java consumes: real tensors from the exported FFDetr ONNX plus the
// detections the reference Python decode produced. Sharing it is the point - it proves the
// browser and server decoders agree with each other AND with the model, rather than merely
// agreeing with themselves.
const spec: ModelPipelineSpec = {
decoder: "rfdetr",
inputSize: reference.inputSize,
resizeMode: "stretch",
padColor: [114, 114, 114],
channelOrder: "rgb",
normMean: [0.485, 0.456, 0.406],
normStd: [0.229, 0.224, 0.225],
outputLayout: "nc_first",
hasObjectness: false,
classNames: ["text", "choice", "signature"],
classFieldTypes: ["text", "checkbox", "signature"],
scoreThreshold: reference.scoreThreshold,
nms: "none",
iou: 0.45,
};
/** Identity mapping so decode output is directly comparable to the reference numbers. */
const pre: Preprocessed = {
chw: new Float32Array(0),
inputSize: reference.inputSize,
scaleX: 1,
scaleY: 1,
padX: 0,
padY: 0,
srcW: reference.inputSize,
srcH: reference.inputSize,
};
function outputs(): Record<string, RawOutput> {
const rows = reference.queries;
const boxCols = rows[0].dets.length;
const logitCols = rows[0].labels.length;
const dets = new Float32Array(rows.length * boxCols);
const labels = new Float32Array(rows.length * logitCols);
rows.forEach((row, i) => {
row.dets.forEach((v, c) => (dets[i * boxCols + c] = v));
row.labels.forEach((v, c) => (labels[i * logitCols + c] = v));
});
return {
dets: { data: dets, d1: rows.length, d2: boxCols },
labels: { data: labels, d1: rows.length, d2: logitCols },
};
}
describe("decodeRfDetr", () => {
test("matches the Python reference decode of the exported model", () => {
const got = decodeRfDetr(outputs(), spec, pre, spec.scoreThreshold);
expect(got).toHaveLength(reference.expected.length);
got.forEach((d: Detection, i: number) => {
const want = reference.expected[i];
expect(d.classId).toBe(want.cls);
expect(d.score).toBeCloseTo(want.score, 4);
expect(d.x).toBeCloseTo(want.x, 1);
expect(d.y).toBeCloseTo(want.y, 1);
expect(d.w).toBeCloseTo(want.w, 1);
expect(d.h).toBeCloseTo(want.h, 1);
});
});
test("finds the form's 8 text, 2 choice and 1 signature", () => {
const got = decodeRfDetr(outputs(), spec, pre, spec.scoreThreshold);
const byClass = (c: number) => got.filter((d) => d.classId === c).length;
expect(byClass(0)).toBe(8);
expect(byClass(1)).toBe(2);
expect(byClass(2)).toBe(1);
});
test("binds outputs by name, not position", () => {
const o = outputs();
const swapped = { labels: o.labels, dets: o.dets };
expect(decodeRfDetr(swapped, spec, pre, spec.scoreThreshold)).toEqual(
decodeRfDetr(o, spec, pre, spec.scoreThreshold),
);
});
// The exported model never lets the no-object column win (max sigmoid 0.0014 over 300
// queries), so this case has to be built by hand to lock the guard in.
test("never classifies a query as the trailing no-object column", () => {
const o: Record<string, RawOutput> = {
dets: { data: new Float32Array([0.5, 0.5, 0.2, 0.1]), d1: 1, d2: 4 },
labels: { data: new Float32Array([-4, -3, -5, 9]), d1: 1, d2: 4 },
};
expect(decodeRfDetr(o, spec, pre, 0.3)).toEqual([]);
});
test("returns nothing when an expected output is absent", () => {
const o = outputs();
expect(
decodeRfDetr({ dets: o.dets }, spec, pre, spec.scoreThreshold),
).toEqual([]);
});
});
frontend/editor/src/core/services/formDetection/onnxSession.ts
+13 -7
View File
@@ -47,14 +47,20 @@ export async function runInference(
s: ort.InferenceSession,
chw: Float32Array,
inputSize: number,
): Promise<RawOutput> {
): Promise<Record<string, RawOutput>> {
const inputName = s.inputNames[0];
const tensor = new ort.Tensor("float32", chw, [1, 3, inputSize, inputSize]);
const result = await s.run({ [inputName]: tensor });
const out = result[s.outputNames[0]];
const dims = out.dims;
// Expect [1, d1, d2]; data is flat row-major so data[i*d2 + j] == out[0][i][j].
const d1 = dims.length >= 2 ? Number(dims[1]) : 0;
const d2 = dims.length >= 3 ? Number(dims[2]) : 0;
return { data: out.data as Float32Array, d1, d2 };
// Keyed by name, in graph order: a query head emits two tensors of the SAME shape (dets and
// labels are both [1, 300, 4] at three classes), so position cannot tell them apart.
const outputs: Record<string, RawOutput> = {};
for (const name of s.outputNames) {
const out = result[name];
const dims = out.dims;
// Expect [1, d1, d2]; data is flat row-major so data[i*d2 + j] == out[0][i][j].
const d1 = dims.length >= 2 ? Number(dims[1]) : 0;
const d2 = dims.length >= 3 ? Number(dims[2]) : 0;
outputs[name] = { data: out.data as Float32Array, d1, d2 };
}
return outputs;
}
frontend/editor/src/core/services/formDetection/runBrowserPipeline.ts
+27 -3
View File
@@ -6,7 +6,7 @@ import { FormDetectionCatalogEntry } from "@app/hooks/useFormDetectionModelStatu
import { applyFields } from "@app/services/formDetection/applyFields";
import { toPdfPoints } from "@app/services/formDetection/coordinateMapping";
import { decode } from "@app/services/formDetection/decode";
import { decode, decodeRfDetr } from "@app/services/formDetection/decode";
import { loadModelBytes } from "@app/services/formDetection/modelCache";
import {
getSession,
@@ -14,7 +14,14 @@ import {
} from "@app/services/formDetection/onnxSession";
import { renderPages } from "@app/services/formDetection/pdfRender";
import { preprocess } from "@app/services/formDetection/preprocess";
import { DetectedField, resolveSpec } from "@app/services/formDetection/types";
import {
DetectedField,
Detection,
ModelPipelineSpec,
Preprocessed,
RawOutput,
resolveSpec,
} from "@app/services/formDetection/types";
import { DetectionStage } from "@app/services/formDetection/progress";
// Kept in lockstep with FormDetectionController.MAX_PAGES / MAX_FIELDS.
@@ -79,7 +86,7 @@ export async function runBrowserDetection(
});
const pre = preprocess(page.rgba, page.widthPx, page.heightPx, spec);
const out = await runInference(session, pre.chw, spec.inputSize);
for (const d of decode(out, spec, pre, score)) {
for (const d of decodeFor(spec, out, pre, score)) {
const rect = toPdfPoints(d, page);
if (rect.w <= 0 || rect.h <= 0) {
continue;
@@ -103,3 +110,20 @@ export async function runBrowserDetection(
const appliedPdf = await applyFields(pdfBytes.slice(0), fields);
return { fields, appliedPdf, pageCount: pages.length };
}
/**
* Pick the decoder the model's head needs, mirroring FormDetectionController.decodeFor.
* Unknown values fall back to YOLO, which is what every entry was before a second head existed.
*/
function decodeFor(
spec: ModelPipelineSpec,
outputs: Record<string, RawOutput>,
pre: Preprocessed,
score: number,
): Detection[] {
if ((spec.decoder ?? "").toLowerCase() === "rfdetr") {
return decodeRfDetr(outputs, spec, pre, score);
}
// Single-output head: take the sole tensor whatever the graph calls it.
return decode(Object.values(outputs)[0], spec, pre, score);
}
frontend/editor/src/core/services/formDetection/types.ts
+2
View File
@@ -5,6 +5,7 @@ import { FormDetectionCatalogEntry } from "@app/hooks/useFormDetectionModelStatu
/** Pipeline spec resolved from the active catalog entry (with backend defaults applied). */
export interface ModelPipelineSpec {
decoder: string; // "yolo" | "rfdetr"
inputSize: number;
resizeMode: string; // "stretch" | "letterbox"
padColor: number[];
@@ -69,6 +70,7 @@ export function resolveSpec(
entry: FormDetectionCatalogEntry,
): ModelPipelineSpec {
return {
decoder: entry.decoder || "yolo",
inputSize: entry.inputSize > 0 ? entry.inputSize : 1216,
resizeMode: entry.resizeMode ?? "letterbox",
padColor: entry.padColor ?? [114, 114, 114],
scripts/export-ffdetr-onnx.py
+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env python3
"""Export the Apache-2.0 FFDetr form-field detector to the ONNX we ship.
Its publisher releases only a PyTorch `.pth`, so unlike the FFDNet checkpoints there is no
`.onnx` to point a catalogue entry at. This does the conversion they skipped. Run it once on a
workstation or a CI job; torch is needed HERE and nowhere else - the product loads the result
with onnxruntime alone, exactly as it loads any other model in the catalogue.
pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
pip install rfdetr onnx onnxruntime onnxconverter-common
python scripts/export-ffdetr-onnx.py --out build/ffdetr
Emits `ffdetr-int8.onnx` (~37MB, the one to host) plus the fp32 graph it came from, and prints
the sha256 for `model-catalog.json`. int8 measures smaller than FFDNet-S at 38.4MB and, on a
form page, returns detections indistinguishable from fp32.
"""
from __future__ import annotations
import argparse
import hashlib
import shutil
import sys
import warnings
from pathlib import Path
# Pinned so a re-export is byte-reproducible; bump deliberately, not incidentally.
REPO = "jbarrow/FFDetr"
REVISION = "56f4e4235e28dcb2953513dc020bb191a2f54cfe"
CHECKPOINT_SHA256 = "f852e1bac18c8f435b82270fc8ff8e2ca4a2cd8869c411fa8f473f16e69585ef"
# Must match model-catalog.json. 1024 is RF-DETR's native resolution and has to stay divisible
# by 32 (patch_size 16 x num_windows 2).
INPUT_SIZE = 1024
NUM_CLASSES = 3
OPSET = 17
def sha256_of(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1 << 20), b""):
digest.update(chunk)
return digest.hexdigest()
def fetch_checkpoint(dest: Path) -> Path:
"""Download the pinned .pth and refuse to continue if it is not the one we vetted."""
import urllib.request
if not dest.exists():
url = f"https://huggingface.co/{REPO}/resolve/{REVISION}/FFDetr.pth"
print(f"downloading {url}")
with urllib.request.urlopen(url) as response, dest.open("wb") as out:
shutil.copyfileobj(response, out)
actual = sha256_of(dest)
if actual != CHECKPOINT_SHA256:
raise SystemExit(
f"checkpoint sha256 mismatch\n expected {CHECKPOINT_SHA256}\n actual {actual}"
)
print(f"checkpoint verified ({dest.stat().st_size / 1e6:.1f} MB)")
return dest
def export_fp32(checkpoint: Path, out_dir: Path) -> Path:
from rfdetr import RFDETRMedium
# An absolute path matters: a relative one is resolved against ~/.roboflow/models and 404s.
model = RFDETRMedium(
pretrain_weights=str(checkpoint.resolve()),
num_classes=NUM_CLASSES,
trust_checkpoint=True,
)
# fp16=False keeps the graph single-precision; we quantise to int8 below instead, which is
# both smaller and - unlike the fp16 converter - produces a graph onnxruntime will load.
produced = model.export(
output_dir=str(out_dir),
shape=(INPUT_SIZE, INPUT_SIZE),
opset_version=OPSET,
fp16=False,
verbose=False,
)
return Path(produced)
def quantize(fp32: Path, out: Path) -> Path:
from onnxruntime.quantization import QuantType, quantize_dynamic
from onnxruntime.quantization.shape_inference import quant_pre_process
prepared = fp32.with_name("ffdetr-prep.onnx")
quant_pre_process(str(fp32), str(prepared), skip_symbolic_shape=False)
quantize_dynamic(str(prepared), str(out), weight_type=QuantType.QInt8)
prepared.unlink(missing_ok=True)
return out
def verify(model: Path) -> None:
"""Load under the same onnxruntime the product uses and assert the contract the decoder relies on."""
import numpy as np
import onnx
import onnxruntime as ort
graph = onnx.load(str(model))
custom = sorted({n.domain for n in graph.graph.node if n.domain not in ("", "ai.onnx")})
if custom:
raise SystemExit(f"refusing to ship: graph needs custom op domains {custom}")
session = ort.InferenceSession(str(model), providers=["CPUExecutionProvider"])
names = [o.name for o in session.get_outputs()]
if names != ["dets", "labels"]:
raise SystemExit(f"unexpected output names {names}; the decoder binds by name")
outputs = session.run(
None, {session.get_inputs()[0].name: np.zeros((1, 3, INPUT_SIZE, INPUT_SIZE), np.float32)}
)
for name, value in zip(names, outputs):
if np.isnan(value).any():
raise SystemExit(f"output {name} contains NaN")
# Both outputs are [1,300,4]: 4 box values vs NUM_CLASSES + 1 no-object logit. They are
# indistinguishable by shape, which is exactly why everything downstream binds by name.
print(f"verified: ops all standard, outputs {list(zip(names, (o.shape for o in outputs)))}")
def main() -> int:
warnings.filterwarnings("ignore")
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--out", default="build/ffdetr", help="output directory")
args = parser.parse_args()
out_dir = Path(args.out)
out_dir.mkdir(parents=True, exist_ok=True)
checkpoint = fetch_checkpoint(out_dir / "FFDetr.pth")
fp32 = export_fp32(checkpoint, out_dir)
print(f"fp32: {fp32.stat().st_size / 1e6:.1f} MB")
int8 = quantize(fp32, out_dir / "ffdetr-int8.onnx")
verify(int8)
size = int8.stat().st_size
print()
print(f" file {int8}")
print(f" sizeBytes {size}")
print(f" sha256 {sha256_of(int8)}")
print()
print("Host this file, then set onnxUrl/sha256/sizeBytes on the ffdetr entry in")
print("app/proprietary/src/main/resources/formdetection/model-catalog.json.")
return 0
if __name__ == "__main__":
sys.exit(main())