Compare commits

...
Author SHA1 Message Date
Anthony Stirling b19e85698e Guard heading detection on unspaced scripts and drop dead hyphen repair 2026-08-30 10:28:13 +01:00
Anthony Stirling f097450883 Merge remote-tracking branch 'origin/main' into improve-pdf-markdown-extraction 2026-08-30 10:08:28 +01:00
Anthony Stirling 941051648e Condense comments to two lines or under 2026-08-27 12:09:22 +01:00
Anthony Stirling 33a22db712 Split AdvancedPdfMarkdownConverter into per-stage classes 2026-08-27 11:55:47 +01:00
Anthony Stirling 1e36895561 Merge remote-tracking branch 'origin/main' into sweep/pr7303 2026-08-26 08:11:06 +01:00
Anthony Stirling ea05a3f576 Merge remote-tracking branch 'origin/main' into sweep/pr7303 2026-08-26 07:11:01 +01:00
Anthony Stirling 1c3d216e70 Move the advanced markdown extraction engine into proprietary 2026-08-14 19:27:12 +01:00
Anthony Stirling 7159b1d179 Improve pdf markdown extraction 2026-08-14 16:21:14 +01:00
Anthony Stirling 8f0b3a6d94 Guard the merged jpdfium redaction entry points 2026-08-13 13:54:07 +01:00
Anthony Stirling 8b1a9c8902 Merge remote-tracking branch 'origin/main' into improve-pdf-markdown-extraction 2026-08-13 13:50:45 +01:00
Anthony Stirling c35b252092 Raise jpdfium lock timeouts instead of reporting no tables found 2026-08-13 12:11:44 +01:00
Anthony Stirling ebe216ee8d Bound ruled-table partitioning and guard stirling.md property parsing 2026-08-13 10:28:30 +01:00
Anthony Stirling 84af8476bc Bound the jpdfium lock wait and map the timeout to 503 2026-08-13 10:28:16 +01:00
Anthony Stirling 7e0ef7cfae Copy the CSV upload instead of moving it so billing still page-counts it 2026-08-13 10:27:59 +01:00
Anthony Stirling ffc9052961 Merge origin/main into improve-pdf-markdown-extraction 2026-08-13 09:51:32 +01:00
Anthony Stirling 532e2a675a Condense comments added by this branch to at most two lines 2026-08-05 15:12:39 +01:00
Anthony Stirling 72c67520ef Scope CSV table fallback to the requested pages and translate jpdfium errors globally 2026-08-05 14:04:58 +01:00
Anthony Stirling 8b1c045349 Fall back to word-grid table detection when lattice mode finds nothing 2026-08-05 13:11:42 +01:00
Anthony Stirling dc70c0b821 Detect tables from ruling lines and rewrite heading and column detection 2026-08-05 13:11:38 +01:00
Anthony Stirling 6f4a951291 Map JPDFiumException to typed errors so responses stop leaking temp paths 2026-08-05 13:11:19 +01:00
Anthony Stirling 4dac016b64 Serialise jpdfium native access to stop concurrent requests wedging the JVM 2026-08-05 13:11:18 +01:00
30 changed files with 4741 additions and 19 deletions
@@ -0,0 +1,17 @@
package stirling.software.common.pdf;
import java.io.IOException;
import org.springframework.stereotype.Service;
import stirling.software.jpdfium.PdfDocument;
/** Built-in Markdown conversion, used whenever no richer extractor is registered. */
@Service
public class BasicPdfMarkdownExtractor implements PdfMarkdownExtractor {
@Override
public String convert(PdfDocument doc) throws IOException {
return new PdfMarkdownConverter().convert(doc);
}
}
@@ -0,0 +1,14 @@
package stirling.software.common.pdf;
import java.io.IOException;
import stirling.software.jpdfium.PdfDocument;
/**
* Seam for PDF to Markdown conversion. The proprietary module supplies a layout-aware
* implementation that takes precedence on the classpath.
*/
public interface PdfMarkdownExtractor {
String convert(PdfDocument doc) throws IOException;
}
@@ -19,7 +19,7 @@ import stirling.software.common.enumeration.ResourceWeight;
import stirling.software.common.model.api.PDFFile;
import stirling.software.common.model.tool.ToolFormat;
import stirling.software.common.model.tool.ToolIO;
import stirling.software.common.pdf.PdfMarkdownConverter;
import stirling.software.common.pdf.PdfMarkdownExtractor;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.WebResponseUtils;
@@ -30,6 +30,7 @@ import stirling.software.jpdfium.PdfDocument;
public class ConvertPDFToMarkdown {
private final TempFileManager tempFileManager;
private final PdfMarkdownExtractor markdownExtractor;
@AutoJobPostMapping(
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
@@ -54,7 +55,7 @@ public class ConvertPDFToMarkdown {
try (TempFile tempInput = new TempFile(tempFileManager, ".pdf")) {
inputFile.transferTo(tempInput.getFile());
try (PdfDocument doc = PdfDocument.open(tempInput.getPath())) {
markdown = new PdfMarkdownConverter().convert(doc);
markdown = markdownExtractor.convert(doc);
}
}
@@ -23,14 +23,14 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import stirling.software.common.pdf.PdfMarkdownConverter;
import stirling.software.common.pdf.PdfMarkdownExtractor;
import stirling.software.common.util.TempFile;
import stirling.software.jpdfium.PdfDocument;
class ConvertPDFToMarkdownTest {
private MockMvc mockMvc() {
return MockMvcBuilders.standaloneSetup(new ConvertPDFToMarkdown(null))
private MockMvc mockMvc(PdfMarkdownExtractor extractor) {
return MockMvcBuilders.standaloneSetup(new ConvertPDFToMarkdown(null, extractor))
.setControllerAdvice(new GlobalErrorHandler())
.build();
}
@@ -61,11 +61,10 @@ class ConvertPDFToMarkdownTest {
when(mock.getFile()).thenReturn(tmpFile);
when(mock.getPath()).thenReturn(tmpFile.toPath());
});
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class);
MockedConstruction<PdfMarkdownConverter> converterMock =
Mockito.mockConstruction(
PdfMarkdownConverter.class,
(mock, ctx) -> when(mock.convert(any())).thenReturn(expectedMd))) {
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class)) {
PdfMarkdownExtractor extractor = Mockito.mock(PdfMarkdownExtractor.class);
when(extractor.convert(any())).thenReturn(expectedMd);
PdfDocument mockDoc = Mockito.mock(PdfDocument.class);
docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc);
@@ -74,7 +73,7 @@ class ConvertPDFToMarkdownTest {
new MockMultipartFile(
"fileInput", "input.pdf", "application/pdf", new byte[] {1, 2, 3});
mockMvc()
mockMvc(extractor)
.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
.andExpect(status().isOk())
.andExpect(header().string("Content-Type", "text/markdown"))
@@ -94,13 +93,10 @@ class ConvertPDFToMarkdownTest {
when(mock.getFile()).thenReturn(tmpFile);
when(mock.getPath()).thenReturn(tmpFile.toPath());
});
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class);
MockedConstruction<PdfMarkdownConverter> converterMock =
Mockito.mockConstruction(
PdfMarkdownConverter.class,
(mock, ctx) ->
when(mock.convert(any()))
.thenThrow(new RuntimeException("boom")))) {
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class)) {
PdfMarkdownExtractor extractor = Mockito.mock(PdfMarkdownExtractor.class);
when(extractor.convert(any())).thenThrow(new RuntimeException("boom"));
PdfDocument mockDoc = Mockito.mock(PdfDocument.class);
docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc);
@@ -109,7 +105,7 @@ class ConvertPDFToMarkdownTest {
new MockMultipartFile(
"fileInput", "x.pdf", "application/pdf", new byte[] {0x01});
mockMvc()
mockMvc(extractor)
.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
.andExpect(status().isInternalServerError());
}
+8
View File
@@ -98,6 +98,14 @@ dependencies {
testImplementation "org.testcontainers:junit-jupiter:${testcontainersMinioVersion}"
}
// Golden extraction fixtures live in :common; the advanced converter is graded against the same
// corpus as the built-in one rather than a forked copy of it.
processTestResources {
from(project(':common').file('src/test/resources/pdf-ingestion-fixtures')) {
into 'pdf-ingestion-fixtures'
}
}
tasks.register('prepareKotlinBuildScriptModel') {}
tasks.register('type3SignatureTool', JavaExec) {
@@ -0,0 +1,269 @@
package stirling.software.proprietary.pdf;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.pdf.PdfMarkdownExtractor;
import stirling.software.jpdfium.PdfDocument;
import stirling.software.jpdfium.PdfPage;
import stirling.software.jpdfium.text.PageText;
import stirling.software.jpdfium.text.PdfTextExtractor;
import stirling.software.jpdfium.text.TextLine;
/**
* Converts a PDF to Markdown from PDFium {@link TextLine}s. Orchestration only: each stage of the
* pipeline lives in its own class in this package.
*/
@Slf4j
@Service
@Primary
public class AdvancedPdfMarkdownConverter implements PdfMarkdownExtractor {
@Override
public String convert(PdfDocument doc) throws IOException {
List<String> rendered = new ArrayList<>();
for (Object e : buildElements(doc)) {
rendered.add(e instanceof TableBlock tb ? tb.render() : (String) e);
}
return MarkdownText.normaliseHeadingLevels(String.join("\n\n", rendered));
}
private List<Object> buildElements(PdfDocument doc) throws IOException {
List<PageText> allPageText = PdfTextExtractor.extractAll(doc);
float medianSize = HeadingDetector.medianFontSize(allPageText);
float medianHeight = HeadingDetector.medianLineHeight(allPageText);
String bodyFont = HeadingDetector.bodyFont(allPageText);
int pageCount = doc.pageCount();
// Tables stay structured until after the page loop so one split across a page break can
// be stitched back together before rendering.
List<Object> output = new ArrayList<>();
// Header of a table that ended the previous page, for spotting a continuation; null if
// none.
String prevPageTrailingTableHeader = null;
for (int pageIndex = 0; pageIndex < pageCount; pageIndex++) {
PageLines page = pageLines(doc, allPageText, pageIndex);
if (page.lines().isEmpty()) {
PageImages.emit(doc, pageIndex, output);
prevPageTrailingTableHeader = null;
continue;
}
List<Object> pageItems =
buildPageItems(
doc,
page,
pageIndex,
medianSize,
medianHeight,
bodyFont,
prevPageTrailingTableHeader);
if (pageItems.isEmpty()) {
continue;
}
PageStitcher.mergeAcrossPageBoundary(output, pageItems);
output.addAll(pageItems);
prevPageTrailingTableHeader = PageStitcher.trailingTableHeader(pageItems);
}
// Stitch tables split across page breaks; callers decide how to realise the elements.
return PageStitcher.stitchTables(output);
}
/**
* One page's lines plus its layout verdict, which must be taken before text repair: merging
* reduces the line count the two-column guard scales with.
*/
private record PageLines(List<Line> lines, List<Float> gutters) {
boolean twoColumnLayout() {
return !gutters.isEmpty();
}
}
/** Assembled lines for one page, sorted top-to-bottom (PDF y=0 is the page bottom). */
private static PageLines pageLines(PdfDocument doc, List<PageText> allPageText, int pageIndex) {
List<TextLine> rawLines =
pageIndex < allPageText.size() ? allPageText.get(pageIndex).lines() : List.of();
List<Line> stitched = GlyphStitcher.stitchGlyphs(rawLines);
List<Float> gutters = ColumnLayout.detectGutters(stitched);
List<Line> lines = LineMerger.mergeLineFragments(stitched, gutters);
lines.addAll(FormValues.lines(doc, pageIndex, lines));
lines.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
return new PageLines(lines, gutters);
}
/**
* One page's elements: paragraph strings interleaved with {@link TableBlock}s in reading order.
* {@code continuationHeader} is the previous page's trailing table header, or null.
*/
private List<Object> buildPageItems(
PdfDocument doc,
PageLines page,
int pageIndex,
float medianSize,
float medianHeight,
String bodyFont,
String continuationHeader)
throws IOException {
List<Line> lines = page.lines();
// Only genuine two-column prose is split: a table's column gutters must not read as a page
// gutter, and a table continuing from the previous page is not a new two-column layout.
boolean tableContinuation =
continuationHeader != null
&& lines.stream()
.anyMatch(
l ->
MarkdownText.normaliseSpace(l.text)
.equals(continuationHeader));
// Merging widens lines, so re-check the pre-repair verdict here: ordering by a gutter the
// finished lines no longer respect is worse than not splitting at all.
List<Float> gutters = tableContinuation ? List.of() : page.gutters();
boolean twoColumn = !gutters.isEmpty();
boolean respected = twoColumn && ColumnLayout.gutterRespected(lines, gutters);
// Two detectors: ruling lines give exact boundaries and see single-word cells; the word
// grid covers what the rules do not, i.e. borderless and whitespace-aligned tables.
Set<String> tableRowTexts = new HashSet<>();
PageRules rules = readRules(doc, pageIndex);
List<TableBlock> blocks = TableFinder.find(lines, rules, pageIndex + 1);
if (log.isDebugEnabled()) {
log.debug(
"p{} lines={} hRules={} vRules={} twoColumn={} blocks={}",
pageIndex,
lines.size(),
rules.horizontal().size(),
rules.vertical().size(),
!gutters.isEmpty(),
blocks.size());
for (TableBlock b : blocks) {
List<String[]> cs = b.cells();
log.debug(
"block top={} bot={} ruled={} src={} rows={} cells={}x{} "
+ "grid={} spans={} owns={}",
b.top(),
b.bottom(),
b.ruled(),
b.rowSource(),
b.rows().size(),
cs.size(),
cs.isEmpty() ? 0 : cs.get(0).length,
TableShape.looksLikeGrid(b),
TableShape.spansPage(b, lines),
TableShape.ownsItsBand(b, lines));
for (List<Line> row : b.rows()) {
log.debug(" row: {}", PageStitcher.rowText(row));
}
}
}
if (twoColumn) {
// On a multi-column page only a full-width block is a table; anything narrower sits
// inside a column. A ruled block owning its own band has no column layout to sit in.
blocks =
blocks.stream()
.filter(
b ->
(b.ruled() || TableShape.looksLikeGrid(b))
&& (TableShape.spansPage(b, lines)
|| (b.ruled()
&& TableShape.ownsItsBand(
b, lines))))
.toList();
}
Set<Line> tableLines = new HashSet<>();
for (TableBlock b : blocks) {
for (List<Line> row : b.rows()) {
for (Line l : row) {
tableLines.add(l);
tableRowTexts.add(l.text.strip());
}
}
}
List<Object> pageItems = new ArrayList<>();
List<List<Line>> segments = segmentsAround(lines, blocks, tableLines);
if (twoColumn) {
// A full-width table interrupts both columns, so splitting at its own vertical band
// keeps the prose above and below it in column order.
for (int s = 0; s < segments.size(); s++) {
List<List<Line>> groups =
respected
? ColumnLayout.orderByBand(segments.get(s), gutters)
: ColumnLayout.legacySplit(segments.get(s));
for (List<Line> col : groups) {
List<String> paras = new ArrayList<>();
ParagraphAssembler.assembleParagraphs(
col, medianSize, medianHeight, bodyFont, paras, tableRowTexts);
pageItems.addAll(paras);
}
if (s < blocks.size()) {
pageItems.add(blocks.get(s));
}
}
} else {
// Interleave tables with text by vertical position: each block gets a slot,
// and non-table lines fall into the slot for their y, keeping tables separate.
for (int s = 0; s <= blocks.size(); s++) {
List<String> paras = new ArrayList<>();
ParagraphAssembler.assembleParagraphs(
segments.get(s), medianSize, medianHeight, bodyFont, paras, tableRowTexts);
pageItems.addAll(paras);
if (s < blocks.size()) {
pageItems.add(blocks.get(s));
}
}
}
PageImages.emit(doc, pageIndex, pageItems);
return pageItems;
}
/**
* Splits a page's non-table lines into the bands between its table blocks, band {@code s}
* holding the lines above block {@code s}. Blocks must be in top-to-bottom order.
*/
private static List<List<Line>> segmentsAround(
List<Line> lines, List<TableBlock> blocks, Set<Line> tableLines) {
List<List<Line>> segments = new ArrayList<>();
for (int s = 0; s <= blocks.size(); s++) {
segments.add(new ArrayList<>());
}
for (Line l : lines) {
if (tableLines.contains(l)) {
continue;
}
int slot = 0;
for (TableBlock b : blocks) {
if (b.bottom() > l.y) {
slot++;
}
}
segments.get(slot).add(l);
}
return segments;
}
/** Ruling lines of one page, or {@link PageRules#EMPTY} if the page cannot be opened. */
private static PageRules readRules(PdfDocument doc, int pageIndex) {
try (PdfPage page = doc.page(pageIndex)) {
return PageRules.of(page);
} catch (Exception e) {
log.debug(
"Page {} ruling lines unreadable; falling back to word-grid tables",
pageIndex,
e);
return PageRules.EMPTY;
}
}
}
@@ -0,0 +1,150 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import stirling.software.jpdfium.text.TextWord;
/**
* Incremental {@link ColumnRanges#find(List)} for a stitched table: appending a page costs O(page),
* bit-for-bit identical to re-projecting the lot.
*/
final class ColumnAccumulator {
private int lineCount;
private float minX = Float.MAX_VALUE;
private float maxX = -Float.MAX_VALUE;
private double totalWidth;
private int totalChars;
/** Coverage counts, cov[i] = lines covering absolute x-bucket covBase + i. */
private int[] cov = new int[0];
private int covBase;
/** Set once the x-span exceeds what findColumnRanges accepts; no histogram is then kept. */
private boolean oversized;
private boolean[] scratch = new boolean[0];
static ColumnAccumulator of(List<List<Line>> rows) {
ColumnAccumulator a = new ColumnAccumulator();
for (List<Line> row : rows) {
for (Line l : row) {
a.addLine(l);
}
}
return a;
}
void addLine(Line l) {
lineCount++;
List<TextWord> words = l.words();
int lineLo = Integer.MAX_VALUE;
int lineHi = Integer.MIN_VALUE;
for (TextWord w : words) {
float x0 = w.x();
float x1 = x0 + w.width();
minX = Math.min(minX, x0);
maxX = Math.max(maxX, x1);
totalWidth += w.width();
totalChars += Math.max(1, w.text().strip().length());
int a = (int) Math.floor(x0);
int b = (int) Math.ceil(x1);
if (a < lineLo) {
lineLo = a;
}
if (b > lineHi) {
lineHi = b;
}
}
// Mirrors ColumnRanges.find's guard: past this span it returns no columns, so the
// histogram is dead weight and (with crafted coordinates) unboundedly large.
if (!oversized && (maxX - minX) > 2000f) {
oversized = true;
cov = null;
scratch = null;
}
if (oversized || lineHi <= lineLo) {
return;
}
ensureRange(lineLo, lineHi);
int n = lineHi - lineLo;
if (scratch.length < n) {
scratch = new boolean[n];
} else {
Arrays.fill(scratch, 0, n, false);
}
for (TextWord w : words) {
int a = (int) Math.floor(w.x()) - lineLo;
int b = (int) Math.ceil(w.x() + w.width()) - lineLo;
for (int x = a; x < b; x++) {
scratch[x] = true;
}
}
int off = lineLo - covBase;
for (int x = 0; x < n; x++) {
if (scratch[x]) {
cov[off + x]++;
}
}
}
private void ensureRange(int lo, int hi) {
if (cov.length == 0) {
covBase = lo - 32;
cov = new int[(hi - lo) + 64];
return;
}
int have0 = covBase;
int have1 = covBase + cov.length;
if (lo >= have0 && hi <= have1) {
return;
}
int newBase = Math.min(have0, lo) - 32;
int newEnd = Math.max(have1, hi) + 32;
int[] nc = new int[newEnd - newBase];
System.arraycopy(cov, 0, nc, have0 - newBase, cov.length);
cov = nc;
covBase = newBase;
}
/** Exactly what {@link ColumnRanges#find(List)} would return for the accumulated lines. */
List<float[]> columns() {
if (oversized || maxX <= minX || (maxX - minX) > 2000f) {
return List.of();
}
int lo = (int) Math.floor(minX);
int span = Math.min((int) Math.ceil(maxX) - lo + 1, 2001);
int support = Math.max(2, Math.round(lineCount * 0.35f));
List<float[]> columns = new ArrayList<>();
int start = -1;
for (int x = 0; x < span; x++) {
int idx = lo + x - covBase;
int c = (idx >= 0 && idx < cov.length) ? cov[idx] : 0;
boolean isColumn = c >= support;
if (isColumn && start < 0) {
start = x;
} else if (!isColumn && start >= 0) {
columns.add(new float[] {lo + start, lo + x});
start = -1;
}
}
if (start >= 0) {
columns.add(new float[] {(float) (lo + start), (float) (lo + span)});
}
float charWidth = totalChars == 0 ? 6f : (float) (totalWidth / totalChars);
float minGutter = Math.max(10f, charWidth * 2.5f);
List<float[]> merged = new ArrayList<>();
for (float[] band : columns) {
if (!merged.isEmpty() && band[0] - merged.get(merged.size() - 1)[1] < minGutter) {
merged.get(merged.size() - 1)[1] = band[1];
} else {
merged.add(new float[] {band[0], band[1]});
}
}
return merged;
}
}
@@ -0,0 +1,334 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import stirling.software.jpdfium.text.TextLine;
/**
* Multi-column page layout: finding the gutters between columns of prose, splitting a page's lines
* at them, and emitting the columns in reading order.
*/
final class ColumnLayout {
private ColumnLayout() {}
/** Narrowest run of near-empty x that can separate two columns of prose. */
private static final float MIN_GUTTER = 10f;
/** Narrowest column worth splitting out; below this a "gutter" is just a ragged margin. */
private static final float MIN_COLUMN = 70f;
/** Fraction of a page's lines that may cross a gutter and still leave it a gutter. */
private static final float MAX_CROSSING = 0.15f;
/** Most columns recognised on one page. Beyond this the geometry is a table, not a layout. */
private static final int MAX_COLUMNS = 4;
/**
* Finds the page's column gutters, or empty for a single column. Scans the 5th to 95th
* percentile of line edges so one degenerate box cannot drag it off the page.
*/
static List<Float> detectGutters(List<Line> lines) {
if (lines.size() < 8) {
return List.of();
}
int n = lines.size();
float[] los = new float[n];
float[] his = new float[n];
for (int i = 0; i < n; i++) {
los[i] = lines.get(i).left();
his[i] = lines.get(i).right();
}
float[] sortedLo = los.clone();
float[] sortedHi = his.clone();
Arrays.sort(sortedLo);
Arrays.sort(sortedHi);
float lo = sortedLo[(int) (n * 0.05f)];
float hi = sortedHi[Math.min(n - 1, (int) (n * 0.95f))];
if (hi - lo < 2 * MIN_COLUMN + MIN_GUTTER || !plausibleSpan(lo, hi)) {
return List.of();
}
int maxCrossing = (int) (n * MAX_CROSSING);
int start = -1;
List<float[]> bands = new ArrayList<>();
// Stepped as an int: past 2^24 a float can no longer represent x + 1, so a float counter
// over a crafted coordinate stops advancing and spins forever.
int scanFrom = (int) Math.floor(lo + MIN_COLUMN);
int scanTo = (int) Math.ceil(hi - MIN_COLUMN);
for (int xi = scanFrom; xi <= scanTo; xi++) {
float x = xi;
int crossing = 0;
for (int i = 0; i < n; i++) {
if (los[i] < x - 2f && his[i] > x + 2f) {
crossing++;
}
}
if (crossing <= maxCrossing) {
if (start < 0) {
start = (int) x;
}
} else if (start >= 0) {
bands.add(new float[] {start, x});
start = -1;
}
}
if (start >= 0) {
bands.add(new float[] {start, hi - MIN_COLUMN});
}
// Widest first, so the strongest separation wins; then keep only bands MIN_COLUMN apart.
bands.sort(Comparator.comparingDouble((float[] b) -> b[1] - b[0]).reversed());
List<Float> gutters = new ArrayList<>();
for (float[] b : bands) {
if (b[1] - b[0] < MIN_GUTTER || gutters.size() >= MAX_COLUMNS - 1) {
continue;
}
float mid = (b[0] + b[1]) / 2f;
boolean tooClose = mid - lo < MIN_COLUMN || hi - mid < MIN_COLUMN;
for (float g : gutters) {
tooClose |= Math.abs(g - mid) < MIN_COLUMN;
}
if (!tooClose) {
gutters.add(mid);
}
}
gutters.sort(Comparator.naturalOrder());
if (!gutters.isEmpty() && columnsLookLikeText(lines, gutters)) {
return gutters;
}
return centralGutter(lines, los, his, lo, hi);
}
/**
* Rejects geometry too wide to be real: past 2^24 a float cannot represent x + 1, so a
* constant-step scan stops advancing.
*/
private static boolean plausibleSpan(float lo, float hi) {
return Float.isFinite(lo) && Float.isFinite(hi) && (hi - lo) <= 2000f;
}
/** Fallback: accepts halves of scattered labels, which read as columns but not as prose. */
private static List<Float> centralGutter(
List<Line> lines, float[] los, float[] his, float lo, float hi) {
int n = lines.size();
float centreLo = lo + (hi - lo) * 0.35f;
float centreHi = lo + (hi - lo) * 0.65f;
int bestCrossing = Integer.MAX_VALUE;
float bestAt = 0f;
int bestLeft = 0;
int bestRight = 0;
for (int gi = (int) Math.floor(centreLo); gi <= (int) Math.ceil(centreHi); gi += 2) {
float gutter = gi;
int crossing = 0;
int left = 0;
int right = 0;
for (int i = 0; i < n; i++) {
if (los[i] < gutter - 5f && his[i] > gutter + 5f) {
crossing++;
} else if (his[i] <= gutter) {
left++;
} else {
right++;
}
}
if (crossing < bestCrossing) {
bestCrossing = crossing;
bestAt = gutter;
bestLeft = left;
bestRight = right;
}
}
boolean ok = bestLeft >= 4 && bestRight >= 4 && bestCrossing <= (int) (n * 0.25f);
return ok ? List.of(bestAt) : List.of();
}
/** Lines of at least this fraction of a column's width count as that column's body text. */
private static final float BODY_LINE_WIDTH = 0.5f;
/** Body lines a column must hold before it is accepted as a column. */
private static final int BODY_LINES = 4;
/**
* True when every carved-out column reads as running text; projection alone cannot tell prose
* from any other empty lane, such as a bar chart's label gaps.
*/
private static boolean columnsLookLikeText(List<Line> lines, List<Float> gutters) {
// Judge only lines inside a column: a spanning line is assigned to one by its centre, and
// its width would set a measure no real body line could reach.
List<Line> inside =
lines.stream().filter(l -> !spansGutter(l, gutters)).collect(Collectors.toList());
List<List<Line>> columns = splitIntoColumns(inside, gutters);
if (columns.size() < 2) {
return false;
}
for (List<Line> column : columns) {
float lo = Float.MAX_VALUE;
float hi = -Float.MAX_VALUE;
for (Line l : column) {
lo = Math.min(lo, l.left());
hi = Math.max(hi, l.right());
}
float measure = hi - lo;
int body = 0;
for (Line l : column) {
if (l.right() - l.left() >= measure * BODY_LINE_WIDTH) {
body++;
}
}
if (body < BODY_LINES || measure < MIN_COLUMN) {
return false;
}
}
return true;
}
/**
* Splits lines into columns at the given gutters. A line crossing one goes to the column its
* centre falls in; band ordering then places it correctly.
*/
static List<List<Line>> splitIntoColumns(List<Line> lines, List<Float> gutters) {
if (gutters.isEmpty()) {
return List.of(lines);
}
List<List<Line>> columns = new ArrayList<>(gutters.size() + 1);
for (int i = 0; i <= gutters.size(); i++) {
columns.add(new ArrayList<>());
}
for (Line l : lines) {
columns.get(columnOf(l, gutters)).add(l);
}
columns.removeIf(List::isEmpty);
return columns;
}
private static int columnOf(Line l, List<Float> gutters) {
float centre = (l.left() + l.right()) / 2f;
int col = 0;
while (col < gutters.size() && centre > gutters.get(col)) {
col++;
}
return col;
}
/**
* True when the finished lines still respect the gutters the unmerged lines showed; merging
* widens lines, and band-ordering a straddled gutter interleaves the columns.
*/
static boolean gutterRespected(List<Line> lines, List<Float> gutters) {
List<Line> real = lines.stream().filter(l -> !l.synthetic).toList();
if (real.isEmpty()) {
return false;
}
long spanning = real.stream().filter(l -> spansGutter(l, gutters)).count();
return spanning <= real.size() * BAND_CROSSING;
}
/** Fraction of the finished lines that may straddle a gutter and still allow band ordering. */
private static final float BAND_CROSSING = 0.35f;
/** Fallback column split: cut at the widest gap between the lines' left edges. */
static List<List<Line>> legacySplit(List<Line> lines) {
List<Float> xs =
lines.stream()
.filter(l -> l.width >= 40f)
.map(l -> l.x)
.sorted()
.collect(Collectors.toList());
if (xs.isEmpty()) {
return List.of(lines);
}
float splitAt = (xs.getFirst() + xs.getLast()) / 2f;
float biggestGap = 0;
for (int i = 1; i < xs.size(); i++) {
float gap = xs.get(i) - xs.get(i - 1);
if (gap > biggestGap) {
biggestGap = gap;
splitAt = (xs.get(i - 1) + xs.get(i)) / 2f;
}
}
List<Line> left = new ArrayList<>();
List<Line> right = new ArrayList<>();
for (Line l : lines) {
(l.x < splitAt ? left : right).add(l);
}
if (left.isEmpty()) {
return List.of(right);
}
if (right.isEmpty()) {
return List.of(left);
}
return List.of(left, right);
}
/** Longest a line may be and still be a line of a heading rather than of a paragraph. */
private static final int HEADING_LENGTH_WORDS = 12;
/**
* True when a spanning line is short enough to be one line of a full-width banner heading,
* which {@link #orderByBand} keeps in a single group.
*/
private static boolean headingLength(Line l) {
return MarkdownText.wordCount(l.text) <= HEADING_LENGTH_WORDS;
}
/** True when a line straddles a gutter, i.e. it belongs to no single column. */
static boolean spansGutter(Line l, List<Float> gutters) {
float left = l.left();
float right = l.right();
for (float g : gutters) {
if (left < g - 2f && right > g + 2f) {
return true;
}
}
return false;
}
/**
* Orders a multi-column region as a one-level XY cut: spanning lines cut it into bands, and
* each band's columns are emitted in turn.
*/
static List<List<Line>> orderByBand(List<Line> lines, List<Float> gutters) {
List<Line> ordered = new ArrayList<>(lines);
ordered.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
List<List<Line>> out = new ArrayList<>();
List<Line> band = new ArrayList<>();
List<Line> spanning = new ArrayList<>();
for (Line l : ordered) {
if (spansGutter(l, gutters)) {
if (spanning.isEmpty()) {
out.addAll(splitIntoColumns(band, gutters));
band = new ArrayList<>();
} else if (!headingLength(l) || !headingLength(spanning.get(spanning.size() - 1))) {
// Only heading-length lines are kept together: a full-width paragraph or list
// is also a run of spanning lines, and merging those runs its items together.
out.add(new ArrayList<>(spanning));
spanning.clear();
}
spanning.add(l);
} else {
if (!spanning.isEmpty()) {
out.add(new ArrayList<>(spanning));
spanning.clear();
}
band.add(l);
}
}
if (!spanning.isEmpty()) {
out.add(new ArrayList<>(spanning));
}
out.addAll(splitIntoColumns(band, gutters));
out.removeIf(List::isEmpty);
return out;
}
/** Visible for testing: as {@link ColumnRanges#fromTextLines(List)}, for gutter detection. */
static List<Float> guttersFromTextLines(List<TextLine> rows) {
return detectGutters(rows.stream().map(Line::new).collect(Collectors.toList()));
}
}
@@ -0,0 +1,117 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import stirling.software.jpdfium.text.TextLine;
import stirling.software.jpdfium.text.TextWord;
/**
* Finds a table's column x-ranges by vertical-whitespace projection: a column is a contiguous
* x-band occupied by enough rows, and the gaps between bands are the gutters.
*/
final class ColumnRanges {
private ColumnRanges() {}
/** Character widths of clear space that separate two columns of an unruled block. */
static final float GUTTER_CHARS = 2.5f;
/** Absolute floor, in points, on an unruled block's column gutter. */
static final float GUTTER_FLOOR = 10f;
/** As {@link #GUTTER_CHARS}, for a block the page's rules already declare to be a table. */
static final float RULED_GUTTER_CHARS = 1.2f;
/** As {@link #GUTTER_FLOOR}, for a block the page's rules already declare to be a table. */
static final float RULED_GUTTER_FLOOR = 4f;
static List<float[]> find(List<Line> rows) {
return find(rows, GUTTER_CHARS, GUTTER_FLOOR);
}
/** As {@link #find(List)}, with the gutter thresholds given explicitly. */
static List<float[]> find(List<Line> rows, float gutterChars, float gutterFloor) {
return find(rows, gutterChars, gutterFloor, 0);
}
/**
* As above, but {@code minSupport} overrides how many rows must occupy an x-band; zero keeps
* the row-count-scaled default.
*/
static List<float[]> find(
List<Line> rows, float gutterChars, float gutterFloor, int minSupport) {
float minX = Float.MAX_VALUE;
float maxX = -Float.MAX_VALUE;
for (Line l : rows) {
for (TextWord w : l.words()) {
minX = Math.min(minX, w.x());
maxX = Math.max(maxX, w.x() + w.width());
}
}
// Real pages are under ~2000pt wide; anything larger is a malformed/crafted coordinate
// that would allocate a multi-GB array or produce a negative span on overflow.
if (maxX <= minX || (maxX - minX) > 2000f) {
return List.of();
}
int lo = (int) Math.floor(minX);
int span = Math.min((int) Math.ceil(maxX) - lo + 1, 2001);
int[] coverage = new int[span];
for (Line l : rows) {
boolean[] covered = new boolean[span];
for (TextWord w : l.words()) {
int a = Math.max(0, (int) Math.floor(w.x()) - lo);
int b = Math.min(span, (int) Math.ceil(w.x() + w.width()) - lo);
for (int x = a; x < b; x++) {
covered[x] = true;
}
}
for (int x = 0; x < span; x++) {
if (covered[x]) {
coverage[x]++;
}
}
}
// A column band must be occupied by at least this many rows; below it is gutter.
int support = minSupport > 0 ? minSupport : Math.max(2, Math.round(rows.size() * 0.35f));
List<float[]> columns = new ArrayList<>();
int start = -1;
for (int x = 0; x < span; x++) {
boolean isColumn = coverage[x] >= support;
if (isColumn && start < 0) {
start = x;
} else if (!isColumn && start >= 0) {
columns.add(new float[] {lo + start, lo + x});
start = -1;
}
}
if (start >= 0) {
columns.add(new float[] {(float) (lo + start), (float) (lo + span)});
}
// Merge bands closer than a real column separator: the gaps inside a multi-word cell are
// about one character, and would otherwise split "January 20th, 2026" into three columns.
float charWidth = WordGeometry.averageCharWidth(rows);
float minGutter = Math.max(gutterFloor, charWidth * gutterChars);
List<float[]> merged = new ArrayList<>();
for (float[] band : columns) {
if (!merged.isEmpty() && band[0] - merged.getLast()[1] < minGutter) {
merged.getLast()[1] = band[1];
} else {
merged.add(new float[] {band[0], band[1]});
}
}
return merged;
}
/**
* Visible for testing: column detection depends only on word geometry, so tests can exercise
* degenerate coordinates without a binary fixture.
*/
static List<float[]> fromTextLines(List<TextLine> rows) {
return find(rows.stream().map(Line::new).collect(Collectors.toList()));
}
}
@@ -0,0 +1,120 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.List;
import stirling.software.jpdfium.PdfDocument;
import stirling.software.jpdfium.PdfPage;
import stirling.software.jpdfium.doc.FormField;
import stirling.software.jpdfium.doc.FormFieldType;
import stirling.software.jpdfium.doc.PdfFormReader;
import stirling.software.jpdfium.model.Rect;
import stirling.software.jpdfium.text.TextChar;
import stirling.software.jpdfium.text.TextLine;
import stirling.software.jpdfium.text.TextWord;
/**
* Recovers AcroForm values that live only in a field's {@code /V}, as pseudo lines at their widget
* rectangles so they land in reading order.
*/
final class FormValues {
private FormValues() {}
/**
* Pseudo text lines for {@code /V}-only AcroForm values, placed at their widget rectangles;
* values already in the content stream are skipped.
*/
static List<Line> lines(PdfDocument doc, int pageIndex, List<Line> existing) {
List<FormField> fields;
try (PdfPage page = doc.page(pageIndex)) {
fields = PdfFormReader.readPage(page.rawDocHandle(), page.rawHandle(), pageIndex);
} catch (RuntimeException e) {
// A malformed AcroForm must not sink the whole conversion; body text still stands.
return List.of();
}
List<Line> out = new ArrayList<>();
for (FormField f : fields) {
String value = fieldText(f);
if (value == null || value.isBlank()) {
continue;
}
Rect r = f.rect();
if (r == null || r.width() <= 0 || r.height() <= 0) {
continue;
}
if (alreadyInContent(existing, value, r)) {
continue;
}
out.add(syntheticLine(value, r));
}
return out;
}
/** The text a filled field contributes, or null when the field contributes nothing. */
private static String fieldText(FormField f) {
FormFieldType type = f.type();
if (type == FormFieldType.PUSHBUTTON
|| type == FormFieldType.SIGNATURE
|| type == FormFieldType.UNKNOWN) {
return null;
}
if (type == FormFieldType.CHECKBOX || type == FormFieldType.RADIO) {
return f.checked() ? "[x]" : null;
}
String value = f.value();
if (value == null || "Off".equals(value)) {
return null;
}
return value.replace('\r', ' ').replace('\n', ' ').strip();
}
/** True when the extractor already found this value inside the widget's own rectangle. */
private static boolean alreadyInContent(List<Line> lines, String value, Rect r) {
String needle = MarkdownText.normaliseSpace(value);
for (Line l : lines) {
boolean overlaps =
l.x < r.x() + r.width()
&& l.x + l.width > r.x()
&& l.y < r.y() + r.height()
&& l.y + l.height > r.y();
if (overlaps && MarkdownText.normaliseSpace(l.text).contains(needle)) {
return true;
}
}
return false;
}
/**
* Wraps a field value as a one-word-per-token {@link TextLine} at the widget rectangle, so
* downstream stages treat it like any other text.
*/
private static Line syntheticLine(String value, Rect r) {
String[] tokens = value.split("\\s+");
float height = Math.min(r.height(), 14f);
float advance = tokens.length == 0 ? r.width() : r.width() / tokens.length;
List<TextWord> words = new ArrayList<>(tokens.length);
for (int i = 0; i < tokens.length; i++) {
float wx = r.x() + advance * i;
List<TextChar> chars = new ArrayList<>(tokens[i].length());
float charWidth = tokens[i].isEmpty() ? advance : advance / tokens[i].length();
for (int c = 0; c < tokens[i].length(); c++) {
chars.add(
new TextChar(
c,
tokens[i].charAt(c),
wx + charWidth * c,
r.y(),
charWidth,
height,
"",
0f));
}
words.add(new TextWord(chars, wx, r.y(), advance * 0.95f, height));
}
TextLine line = new TextLine(words, r.x(), r.y(), r.width(), height);
Line out = new Line(line, value);
out.synthetic = true;
return out;
}
}
@@ -0,0 +1,55 @@
package stirling.software.proprietary.pdf;
import java.util.List;
/** Renders a resolved cell grid as a GitHub-Flavored Markdown table. */
final class GfmTable {
private GfmTable() {}
static String render(List<String[]> rows, int cols) {
if (rows.isEmpty()) {
return "";
}
int[] widths = new int[cols];
for (int c = 0; c < cols; c++) {
widths[c] = 3;
}
for (String[] row : rows) {
for (int c = 0; c < cols; c++) {
if (c < row.length) {
widths[c] = Math.max(widths[c], escapeCell(row[c]).length());
}
}
}
StringBuilder sb = new StringBuilder();
sb.append(buildGfmRow(rows.getFirst(), widths, cols)).append('\n');
sb.append('|');
for (int c = 0; c < cols; c++) {
sb.append('-').append("-".repeat(widths[c])).append('-').append('|');
}
for (int r = 1; r < rows.size(); r++) {
sb.append('\n').append(buildGfmRow(rows.get(r), widths, cols));
}
return sb.toString();
}
private static String buildGfmRow(String[] row, int[] widths, int cols) {
StringBuilder sb = new StringBuilder().append('|');
for (int c = 0; c < cols; c++) {
String cell = c < row.length ? escapeCell(row[c]) : "";
sb.append(' ').append(padRight(cell, widths[c])).append(' ').append('|');
}
return sb.toString();
}
private static String escapeCell(String cell) {
// Cell content is inline context: escape inline markdown (including the column delimiter)
// but not leading block markers, which have no meaning inside a table cell.
return MarkdownText.escapeMarkdownInline(cell);
}
private static String padRight(String s, int width) {
return s.length() >= width ? s : s + " ".repeat(width - s.length());
}
}
@@ -0,0 +1,144 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import stirling.software.jpdfium.text.TextLine;
/**
* Rebuilds assembled {@link Line}s from the extractor's {@link TextLine}s, folding in the narrow
* glyph fragments PDFium emits for apostrophes, markers and bullets.
*/
final class GlyphStitcher {
private GlyphStitcher() {}
/** Width below which a TextLine is treated as a stray glyph fragment to be stitched. */
private static final float GLYPH_WIDTH = 7.5f;
/**
* Merges narrow glyph fragments into the line they belong to: inline between two same-baseline
* fragments, or appended/prepended at a line's edge.
*/
static List<Line> stitchGlyphs(List<TextLine> raw) {
List<TextLine> hosts = new ArrayList<>();
List<TextLine> glyphs = new ArrayList<>();
for (TextLine l : raw) {
String t = stripSoftHyphens(l.text()).strip();
if (t.isEmpty()) {
continue;
}
if (l.width() < GLYPH_WIDTH && t.length() <= 2) {
glyphs.add(l);
} else {
hosts.add(l);
}
}
List<Line> lines =
hosts.stream()
.map(l -> new Line(l, stripSoftHyphens(l.text())))
.collect(Collectors.toList());
for (TextLine g : glyphs) {
String gt = stripSoftHyphens(g.text()).strip();
if (isBulletGlyph(gt)) {
attachBullet(g, gt, lines);
} else {
attachInlineGlyph(g, gt, lines);
}
}
return lines;
}
/**
* Removes U+00AD SOFT HYPHEN, a break-opportunity marker PDFium hands back verbatim as {@code
* ar<AD>e}.
*/
private static String stripSoftHyphens(String text) {
if (text.indexOf('­') < 0) {
return text;
}
return text.replace("­", "");
}
private static boolean isBulletGlyph(String gt) {
return "".equals(gt) || "".equals(gt) || "".equals(gt);
}
/**
* Attaches a bullet glyph to the line it introduces: the closest line beginning to its right,
* at roughly the same height or just below.
*/
private static void attachBullet(TextLine g, String gt, List<Line> lines) {
Line best = null;
float bestScore = Float.MAX_VALUE;
for (Line h : lines) {
if (h.x < g.x() - 2f) {
continue;
}
float dy = g.y() - h.y;
if (dy < -4f || dy > 28f) {
continue;
}
float score = Math.abs(dy) + (h.x - g.x()) * 0.2f;
if (score < bestScore) {
bestScore = score;
best = h;
}
}
if (best != null && !best.text.startsWith("")) {
best.text = "" + best.text;
best.x = g.x();
} else {
lines.add(new Line(g, gt));
}
}
/**
* Stitches a narrow inline glyph into its line: between two same-baseline fragments, appended
* to the line ending at it, or prepended to the one starting at it.
*/
private static void attachInlineGlyph(TextLine g, String gt, List<Line> lines) {
Line left = null;
Line right = null;
float lb = 7f;
float rb = 7f;
for (Line h : lines) {
boolean sameBaseline = g.y() >= h.y - 4f && g.y() <= h.y + h.height + 5f;
if (!sameBaseline) {
continue;
}
float rightEdge = h.x + h.width;
float dxLeft = Math.abs(rightEdge - g.x());
if (dxLeft < lb) {
lb = dxLeft;
left = h;
}
float dxRight = Math.abs(h.x - g.x());
if (dxRight < rb) {
rb = dxRight;
right = h;
}
}
if (left != null && right != null && left != right && Math.abs(left.y - right.y) < 6f) {
left.text = left.text + gt + right.text;
left.width = (right.x + right.width) - left.x;
left.absorb(g);
left.absorb(right);
lines.remove(right);
} else if (left != null) {
left.text = left.text + gt;
left.width = Math.max(left.width, g.x() + g.width() - left.x);
left.absorb(g);
} else if (right != null) {
right.text = gt + right.text;
right.x = g.x();
right.absorb(g);
} else {
lines.add(new Line(g, gt));
}
}
}
@@ -0,0 +1,485 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Pattern;
import stirling.software.jpdfium.text.PageText;
import stirling.software.jpdfium.text.TextChar;
import stirling.software.jpdfium.text.TextLine;
import stirling.software.jpdfium.text.TextWord;
final class HeadingDetector {
private HeadingDetector() {}
/** A heading is at most this many words; longer lines are treated as body text. */
private static final int MAX_HEADING_WORDS = 12;
/** A heading in a script written without word spaces is at most this many characters. */
private static final int MAX_HEADING_UNSPACED_CHARS = 30;
/** Six-letter subset tag PDF writers prepend to embedded font names. */
private static final Pattern SUBSET_TAG = Pattern.compile("^[A-Z]{6}\\+");
/** PostScript name fragments denoting a weight heavier than the regular face. */
private static final String[] BOLD_TOKENS = {
"bold", "black", "heavy", "semibold", "demi", "ultra", "extrabold"
};
/**
* URW/Nimbus bold is "-Medi", TeX bold extended is CMBX/CMSSBX. "Medium" is excluded: matching
* it makes whole CJK paragraphs bold.
*/
private static final Pattern OTHER_BOLD = Pattern.compile("medi(?!um)|cm(ss)?bx");
/**
* A float label followed by its number: set like a heading but naming an illustration, not a
* section.
*/
private static final Pattern CAPTION =
Pattern.compile(
"^(table|figure|fig|chart|exhibit|plate|scheme|graph|diagram|illustration)"
+ "\\s*\\.?\\s*\\d",
Pattern.CASE_INSENSITIVE);
/** A numbered section clause: {@code 3.}, {@code 6.2.}, {@code 7.2.1} followed by a name. */
private static final Pattern CLAUSE = Pattern.compile("^\\d{1,2}(\\.\\d{1,2})*\\.?\\s+\\p{Lu}");
/**
* Two sentences in a row: a bold run-in lead-in, not a heading. The lower-case letters before
* the stop keep it off section numbers.
*/
private static final Pattern RUNS_ON = Pattern.compile("\\p{Ll}{3}[.!?][\\s\\u00a0]+\\p{Lu}");
/** Size ratio at which a line is a level-1 heading on size alone. */
private static final float H1_RATIO = 1.4f;
/** Size ratio at which a line is a level-2 heading on size alone. */
private static final float H2_RATIO = 1.3f;
/**
* A section number ending in a period. Stricter than {@link #CLAUSE}: only the period tells a
* clause from a header's page number.
*/
private static final Pattern NUMBERED_CLAUSE =
Pattern.compile("^\\d{1,2}(\\.\\d{1,2})*\\.\\s+\\p{Lu}");
/**
* True when every cased letter is a capital; digits and uncased scripts do not count, so {@code
* BIO 181} qualifies.
*/
private static boolean isAllCaps(String text) {
int upper = 0;
for (int i = 0; i < text.length(); i++) {
char c = text.charAt(i);
if (Character.isLowerCase(c)) {
return false;
}
if (Character.isUpperCase(c)) {
upper++;
}
}
return upper >= 2;
}
/**
* Markdown heading prefix from size, brevity, isolation and weight, never text matching. Bold
* is vetoed on the body face itself.
*/
static String headingPrefix(
TextLine line,
float medianBodySize,
float medianBodyHeight,
String bodyFont,
boolean isolated) {
return headingPrefix(
line.text(),
line.height(),
line.words(),
medianBodySize,
medianBodyHeight,
bodyFont,
isolated);
}
/**
* Geometry-only overload, judging a merged line on its merged text, height and words. Boldness
* counts only when the face differs from {@code bodyFont}.
*/
static String headingPrefix(
String lineText,
float lineHeight,
List<TextWord> words,
float medianBodySize,
float medianBodyHeight,
String bodyFont,
boolean isolated) {
String text = lineText.strip();
if (text.isEmpty() || tooLongForHeading(text)) {
return "";
}
float ratio = sizeRatio(lineHeight, words, medianBodySize, medianBodyHeight);
if (ratio < 0f) {
return "";
}
// A heading names something. A line with no word in it is a value, an equation fragment or
// a chart label, however large it is set; a caption names a float, not a section.
if (CAPTION.matcher(text).find() || !hasWord(text)) {
return "";
}
if (RUNS_ON.matcher(text).find()) {
return "";
}
// Bold only marks a heading when it stands out from the body face. Some documents set
// the whole body in a bold-named font, where boldness carries no structural meaning.
boolean bold = isBold(words) && !normalisedFont(words).equals(normalise(bodyFont));
// A line that reads as a sentence is prose, unless it carries heading typography.
if (endsLikeSentence(text) && !bold && ratio <= H2_RATIO) {
return "";
}
if (ratio > H1_RATIO) {
return "# ";
}
if (ratio > H2_RATIO) {
return "## ";
}
// A numbered clause: the section number is the structure, so it needs no blank line above
// it to be one. Requiring a capital after the number keeps ordinary list items out.
if (bold && CLAUSE.matcher(text).find()) {
return "### ";
}
// Same size as the body but bold and starting its own block: a run-in section heading.
if (bold && isolated && hasWord(text)) {
return "### ";
}
// Some documents give a heading no size and no weight, only capitals. A short, isolated
// line set entirely in capitals is one of those.
if (isolated
&& !endsLikeSentence(text)
&& wordCount(text) >= 3
&& isAllCaps(text)
&& NUMBERED_CLAUSE.matcher(text).find()) {
return "### ";
}
return "";
}
private static float sizeRatio(
float lineHeight, List<TextWord> words, float medianBodySize, float medianBodyHeight) {
float dominant = dominantFontSize(words);
float value;
float baseline;
if (dominant > 2f && medianBodySize > 2f) {
value = dominant;
baseline = medianBodySize;
} else {
float glyph = glyphHeight(words);
value = glyph > 0f ? glyph : lineHeight;
baseline = medianBodyHeight;
}
return baseline <= 0f ? -1f : value / baseline;
}
/**
* Quantile of a line's glyph heights taken as its size: high enough for the cap band, low
* enough that one rogue glyph box cannot set it.
*/
private static final float GLYPH_HEIGHT_QUANTILE = 0.8f;
/**
* A line's type size from its glyphs, for PDFs encoding visual size in the text matrix; the
* line box runs ascender to descender.
*/
private static float glyphHeight(List<TextWord> words) {
int capacity = 0;
for (TextWord word : words) {
capacity += word.chars().size();
}
if (capacity == 0) {
return 0f;
}
float[] heights = new float[capacity];
int n = 0;
for (TextWord word : words) {
for (TextChar ch : word.chars()) {
if (ch.isWhitespace() || ch.isNewline()) {
continue;
}
// Letters and digits only: brackets and maths operators are drawn taller than the
// cap height, so an equation would measure as display type.
if (!Character.isLetterOrDigit(ch.toChar())) {
continue;
}
float h = ch.height();
if (Float.isFinite(h) && h > 0f) {
heights[n++] = h;
}
}
}
if (n == 0) {
return 0f;
}
Arrays.sort(heights, 0, n);
return heights[(int) (GLYPH_HEIGHT_QUANTILE * (n - 1))];
}
private static String normalise(String fontName) {
return SUBSET_TAG.matcher(fontName == null ? "" : fontName).replaceFirst("");
}
private static String normalisedFont(List<TextWord> words) {
return normalise(dominantFontName(words));
}
/** Most-used font across the document, weighted by glyph count: the body face. */
static String bodyFont(List<PageText> allPages) {
Map<String, Integer> counts = new HashMap<>();
for (PageText page : allPages) {
for (TextChar ch : page.chars()) {
if (ch.isWhitespace() || ch.isNewline()) {
continue;
}
String name = ch.fontName();
if (name != null && !name.isBlank()) {
counts.merge(normalise(name), 1, Integer::sum);
}
}
}
String dominant = "";
int max = -1;
for (Map.Entry<String, Integer> e : counts.entrySet()) {
if (e.getValue() > max) {
max = e.getValue();
dominant = e.getKey();
}
}
return dominant;
}
/** A heading is made of words: at least two letters, and letters at least half the glyphs. */
private static boolean hasWord(String text) {
int letters = 0;
int glyphs = 0;
for (int i = 0; i < text.length(); i++) {
char c = text.charAt(i);
if (Character.isWhitespace(c)) {
continue;
}
glyphs++;
if (Character.isLetter(c)) {
letters++;
}
}
return letters >= 2 && letters * 2 >= glyphs;
}
/**
* True when a line should be emphasised as bold rather than promoted: bold, short, and not a
* full sentence.
*/
static boolean isBoldLabel(String lineText, List<TextWord> words) {
String text = lineText.strip();
if (text.isEmpty() || tooLongForHeading(text) || endsLikeSentence(text)) {
return false;
}
if (RUNS_ON.matcher(text).find()) {
return false;
}
return hasWord(text) && isBold(words);
}
private static int wordCount(String text) {
return text.split("\\s+").length;
}
/** Whitespace tokens do not measure a line written in a script with no word spaces. */
private static boolean tooLongForHeading(String text) {
return wordCount(text) > MAX_HEADING_WORDS
|| unspacedScriptChars(text) > MAX_HEADING_UNSPACED_CHARS;
}
private static int unspacedScriptChars(String text) {
int n = 0;
for (int i = 0; i < text.length(); ) {
int cp = text.codePointAt(i);
if (isUnspacedScript(cp)) {
n++;
}
i += Character.charCount(cp);
}
return n;
}
private static boolean isUnspacedScript(int cp) {
Character.UnicodeScript s = Character.UnicodeScript.of(cp);
return s == Character.UnicodeScript.HAN
|| s == Character.UnicodeScript.HIRAGANA
|| s == Character.UnicodeScript.KATAKANA
|| s == Character.UnicodeScript.THAI
|| s == Character.UnicodeScript.LAO
|| s == Character.UnicodeScript.KHMER
|| s == Character.UnicodeScript.MYANMAR;
}
private static boolean endsLikeSentence(String text) {
char last = text.charAt(text.length() - 1);
// Ideographic and full-width stops end a sentence exactly as the ASCII ones do.
return last == '.'
|| last == '!'
|| last == '?'
|| last == '\u3002'
|| last == '\uff01'
|| last == '\uff1f'
|| last == '\uff61';
}
/** True when the line's dominant font is bold, inferred from PostScript font names. */
private static boolean isBold(List<TextWord> words) {
String lower = normalisedFont(words).toLowerCase(Locale.ROOT);
for (String token : BOLD_TOKENS) {
if (lower.contains(token)) {
return true;
}
}
return OTHER_BOLD.matcher(lower).find();
}
private static String dominantFontName(List<TextWord> words) {
Map<String, Integer> counts = new HashMap<>();
for (TextWord word : words) {
for (TextChar ch : word.chars()) {
if (ch.isWhitespace() || ch.isNewline()) {
continue;
}
String name = ch.fontName();
if (name != null && !name.isBlank()) {
counts.merge(name, 1, Integer::sum);
}
}
}
String dominantFont = "";
int max = -1;
for (Map.Entry<String, Integer> e : counts.entrySet()) {
if (e.getValue() > max) {
max = e.getValue();
dominantFont = e.getKey();
}
}
return dominantFont;
}
/** Computes the median glyph font size across all pages. */
static float medianFontSize(List<PageText> allPages) {
List<Float> sizes = new ArrayList<>();
for (PageText page : allPages) {
for (TextChar ch : page.chars()) {
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
sizes.add(ch.fontSize());
}
}
}
return median(sizes, 12f);
}
/**
* Body baseline for the height path: median per-line {@link #glyphHeight} weighted by glyph
* count, so tiny axis labels cannot drag it down.
*/
static float medianLineHeight(List<PageText> allPages) {
List<float[]> weighted = new ArrayList<>();
double total = 0;
for (PageText page : allPages) {
for (TextLine line : page.lines()) {
if (line.text().isBlank()) {
continue;
}
float h = glyphHeight(line.words());
if (h <= 0f) {
h = line.height();
}
if (h <= 0f) {
continue;
}
float w = glyphCount(line);
weighted.add(new float[] {h, w});
total += w;
}
}
if (weighted.isEmpty()) {
return 12f;
}
weighted.sort(Comparator.comparingDouble(p -> p[0]));
double half = total / 2d;
double seen = 0;
for (float[] p : weighted) {
seen += p[1];
if (seen >= half) {
return p[0];
}
}
return weighted.get(weighted.size() - 1)[0];
}
/** How much text a line carries, in glyphs; at least one so an empty line still counts. */
private static float glyphCount(TextLine line) {
int glyphs = 0;
for (TextWord word : line.words()) {
for (TextChar ch : word.chars()) {
if (!ch.isWhitespace() && !ch.isNewline()) {
glyphs++;
}
}
}
return Math.max(1, glyphs);
}
private static float median(List<Float> values, float fallback) {
if (values.isEmpty()) {
return fallback;
}
Collections.sort(values);
int mid = values.size() / 2;
if (values.size() % 2 == 0) {
return (values.get(mid - 1) + values.get(mid)) / 2f;
}
return values.get(mid);
}
/**
* The font size appearing most often by character count in the line; ties go to the larger
* size.
*/
private static float dominantFontSize(List<TextWord> words) {
Map<Float, Integer> counts = new HashMap<>();
for (TextWord word : words) {
for (TextChar ch : word.chars()) {
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
counts.merge(ch.fontSize(), 1, Integer::sum);
}
}
}
if (counts.isEmpty()) {
return 0f;
}
float dominant = 0f;
int maxCount = -1;
for (Map.Entry<Float, Integer> entry : counts.entrySet()) {
int count = entry.getValue();
float size = entry.getKey();
if (count > maxCount || (count == maxCount && size > dominant)) {
maxCount = count;
dominant = size;
}
}
return dominant;
}
}
@@ -0,0 +1,129 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import stirling.software.jpdfium.text.TextChar;
import stirling.software.jpdfium.text.TextLine;
import stirling.software.jpdfium.text.TextWord;
/**
* A mutable assembled line: text plus geometry. {@link #left()}/{@link #right()} come from the word
* boxes, {@link #glyphLeft()}/{@link #glyphRight()} from the glyphs.
*/
final class Line {
String text;
float x;
float y;
float width;
float height;
final TextLine source;
/** Extra extractor fragments merged into this line; empty for an unmerged line. */
final List<TextLine> merged = new ArrayList<>();
/** True for a line synthesised from an AcroForm value rather than page content. */
boolean synthetic;
Line(TextLine src) {
this(src, src.text());
}
Line(TextLine src, String text) {
this.source = src;
this.text = text;
this.x = src.x();
this.y = src.y();
this.width = src.width();
this.height = src.height();
}
/** Every word on the line, in x order, across all merged fragments. */
List<TextWord> words() {
if (merged.isEmpty()) {
return source.words();
}
List<TextWord> all = new ArrayList<>(source.words());
for (TextLine extra : merged) {
all.addAll(extra.words());
}
all.sort(Comparator.comparingDouble(TextWord::x));
return all;
}
/** Text for the heading/bold classifiers; an unmerged line keeps the extractor's own string. */
String detectText() {
return merged.isEmpty() ? source.text() : text;
}
float detectHeight() {
return merged.isEmpty() ? source.height() : height;
}
/** Top edge; PDF y grows upwards, so this is the larger of the two vertical bounds. */
float top() {
return y + height;
}
float centreY() {
return y + height / 2f;
}
float centreX() {
return x + width / 2f;
}
/** Left edge of the line's words, falling back to its bounding box when it has none. */
float left() {
float edge = Float.MAX_VALUE;
for (TextWord w : words()) {
edge = Math.min(edge, w.x());
}
return edge == Float.MAX_VALUE ? x : edge;
}
float right() {
float edge = -Float.MAX_VALUE;
for (TextWord w : words()) {
edge = Math.max(edge, w.x() + w.width());
}
return edge == -Float.MAX_VALUE ? x + width : edge;
}
/** Left edge of the line's glyphs, ignoring any space a word box carries. */
float glyphLeft() {
float edge = Float.MAX_VALUE;
for (TextWord w : words()) {
for (TextChar c : w.chars()) {
if (!c.isWhitespace() && !c.isNewline()) {
edge = Math.min(edge, c.x());
}
}
}
return edge == Float.MAX_VALUE ? x : edge;
}
float glyphRight() {
float edge = -Float.MAX_VALUE;
for (TextWord w : words()) {
for (TextChar c : w.chars()) {
if (!c.isWhitespace() && !c.isNewline()) {
edge = Math.max(edge, c.x() + c.width());
}
}
}
return edge == -Float.MAX_VALUE ? x + width : edge;
}
/** Records a fragment folded into this line so its word list still covers the whole extent. */
void absorb(TextLine fragment) {
merged.add(fragment);
}
void absorb(Line fragment) {
merged.add(fragment.source);
merged.addAll(fragment.merged);
}
}
@@ -0,0 +1,155 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import stirling.software.jpdfium.text.TextChar;
import stirling.software.jpdfium.text.TextWord;
/**
* Rejoins extractor fragments that are really one visual line. Fragments are grouped into rows
* first, then joined left to right.
*/
final class LineMerger {
private LineMerger() {}
/** Gap above this many average character widths is a real layout gap, so never merged. */
private static final float MAX_MERGE_GAP = 1.60f;
/**
* Rejoins fragments of one visual line: PDFium splits on bounding box, so a run with no
* ascender ({@code rou}) lands apart from the rest.
*/
static List<Line> mergeLineFragments(List<Line> lines, List<Float> gutters) {
if (lines.size() < 2) {
return lines;
}
// Merge within each column, so a line ending at the gutter never joins the next column's.
if (!gutters.isEmpty()) {
List<List<Line>> columns = ColumnLayout.splitIntoColumns(lines, gutters);
if (columns.size() > 1) {
List<Line> out = new ArrayList<>(lines.size());
for (List<Line> column : columns) {
out.addAll(mergeRows(column));
}
return out;
}
}
return mergeRows(lines);
}
private static List<Line> mergeRows(List<Line> lines) {
if (lines.size() < 2) {
return new ArrayList<>(lines);
}
List<Line> ordered = new ArrayList<>(lines);
// Top edge first, so fragments of one visual line arrive together whatever their heights.
ordered.sort(Comparator.comparingDouble((Line l) -> -(l.y + l.height)));
// Group into rows first: a fragment's continuation is its right-hand neighbour on the same
// row, not whichever line the extractor happened to emit next.
List<List<Line>> rows = new ArrayList<>();
for (Line line : ordered) {
List<Line> row = null;
for (int i = rows.size() - 1; i >= 0 && i >= rows.size() - 3; i--) {
if (overlapsRow(rows.get(i), line)) {
row = rows.get(i);
break;
}
}
if (row == null) {
row = new ArrayList<>();
rows.add(row);
}
row.add(line);
}
List<Line> out = new ArrayList<>();
for (List<Line> row : rows) {
row.sort(Comparator.comparingDouble((Line l) -> l.x));
Line host = null;
for (Line line : row) {
if (host != null && adjacentOnRow(host, line)) {
appendFragment(host, line);
} else {
out.add(line);
host = line;
}
}
}
return out;
}
/** True when a line shares a row with the lines already in it (vertical overlap). */
private static boolean overlapsRow(List<Line> row, Line line) {
for (Line member : row) {
float overlap =
Math.min(member.y + member.height, line.y + line.height)
- Math.max(member.y, line.y);
float minHeight = Math.min(member.height, line.height);
if (minHeight > 0f && overlap >= minHeight * 0.5f) {
return true;
}
}
return false;
}
/** True when {@code next} sits close enough to {@code host} to be the same visual line. */
private static boolean adjacentOnRow(Line host, Line next) {
// Merging concatenates left to right, which is reading order for LTR only; RTL fragments
// would be joined back to front.
if (hasStrongRtl(host.text) || hasStrongRtl(next.text)) {
return false;
}
float gap = next.glyphLeft() - host.glyphRight();
float charWidth = fragmentCharWidth(host, next);
return gap > -charWidth && gap < charWidth * MAX_MERGE_GAP;
}
/** True when the text contains a Hebrew, Arabic, Syriac or Thaana character. */
private static boolean hasStrongRtl(String text) {
for (int i = 0; i < text.length(); i++) {
byte dir = Character.getDirectionality(text.charAt(i));
if (dir == Character.DIRECTIONALITY_RIGHT_TO_LEFT
|| dir == Character.DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC) {
return true;
}
}
return false;
}
private static void appendFragment(Line host, Line next) {
float gap = next.glyphLeft() - host.glyphRight();
float charWidth = fragmentCharWidth(host, next);
String left = host.text.stripTrailing();
String right = next.text.stripLeading();
boolean space = gap >= charWidth * WordGeometry.NO_SPACE_GAP;
host.text = left + (space ? " " : "") + right;
host.merged.add(next.source);
host.merged.addAll(next.merged);
float right0 = Math.max(host.x + host.width, next.x + next.width);
float top = Math.max(host.y + host.height, next.y + next.height);
host.x = Math.min(host.x, next.x);
host.y = Math.min(host.y, next.y);
host.width = right0 - host.x;
host.height = top - host.y;
}
private static float fragmentCharWidth(Line a, Line b) {
double total = 0;
int chars = 0;
for (Line l : List.of(a, b)) {
for (TextWord w : l.words()) {
for (TextChar c : w.chars()) {
if (!c.isWhitespace() && !c.isNewline() && c.width() > 0f) {
total += c.width();
chars++;
}
}
}
}
return chars == 0 ? 6f : (float) (total / chars);
}
}
@@ -0,0 +1,120 @@
package stirling.software.proprietary.pdf;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Markdown text utilities: escaping extracted text and rebasing the finished document's heading
* levels.
*/
final class MarkdownText {
private MarkdownText() {}
/** A Markdown ATX heading at the start of a line, with its level in group 1. */
private static final Pattern ATX_HEADING = Pattern.compile("(?m)^(#{1,6}) (?=\\S)");
/**
* Rebases headings so the strongest is level 1 and no level is skipped: levels only mean
* anything against the other headings in the same document.
*/
static String normaliseHeadingLevels(String markdown) {
Set<Integer> levels = new TreeSet<>();
Matcher m = ATX_HEADING.matcher(markdown);
while (m.find()) {
levels.add(m.group(1).length());
}
if (levels.isEmpty() || (levels.contains(1) && levels.size() == maxOf(levels))) {
return markdown;
}
Map<Integer, String> rebased = new HashMap<>();
int rank = 1;
for (int level : levels) {
rebased.put(level, "#".repeat(rank++));
}
return m.reset().replaceAll(r -> rebased.get(r.group(1).length()) + " ");
}
private static int maxOf(Set<Integer> levels) {
int max = 0;
for (int level : levels) {
max = Math.max(max, level);
}
return max;
}
static int wordCount(String text) {
return text.isBlank() ? 0 : text.strip().split("\\s+").length;
}
/**
* Escapes Markdown control characters so extracted text is emitted literally. Output is still
* untrusted: this is defence-in-depth, not safe rendering.
*/
static String escapeMarkdown(String text) {
if (text.isEmpty()) {
return text;
}
String inline = escapeMarkdownInline(text);
return escapeLeadingBlockMarker(inline, text);
}
/** Escapes inline-significant Markdown characters anywhere in the string. */
static String escapeMarkdownInline(String text) {
StringBuilder sb = new StringBuilder(text.length() + 8);
for (int i = 0; i < text.length(); i++) {
char c = text.charAt(i);
switch (c) {
case '\\', '`', '*', '_', '[', ']', '<', '>', '|', '~' -> sb.append('\\').append(c);
default -> sb.append(c);
}
}
return sb.toString();
}
/**
* Escapes markers significant only at line start: {@code #}, {@code -}, {@code +} and
* ordered-list numbers. {@code original} is unescaped, so positions line up.
*/
private static String escapeLeadingBlockMarker(String escaped, String original) {
char c0 = original.charAt(0);
if (c0 == '#' || c0 == '-' || c0 == '+') {
return "\\" + escaped;
}
int i = 0;
while (i < original.length() && Character.isDigit(original.charAt(i))) {
i++;
}
if (i > 0 && i < original.length()) {
char delim = original.charAt(i);
if (delim == '.' || delim == ')') {
return escaped.substring(0, i) + "\\" + escaped.substring(i);
}
}
return escaped;
}
static String normaliseSpace(String s) {
return s.strip().replaceAll("\\s+", " ");
}
static void flushParagraph(StringBuilder para, List<String> out) {
if (!para.isEmpty()) {
out.add(escapeMarkdown(para.toString()));
para.setLength(0);
}
}
static boolean endsWithSentencePunctuation(String s) {
if (s.isEmpty()) {
return false;
}
char last = s.charAt(s.length() - 1);
return last == '.' || last == '?' || last == '!' || last == ':';
}
}
@@ -0,0 +1,70 @@
package stirling.software.proprietary.pdf;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import stirling.software.jpdfium.PdfDocument;
import stirling.software.jpdfium.PdfPage;
import stirling.software.jpdfium.doc.ExtractedImage;
import stirling.software.jpdfium.doc.PdfImageExtractor;
import stirling.software.jpdfium.model.Rect;
/**
* Emits a placeholder per image on a page; image bytes are deliberately not carried into the
* Markdown.
*/
final class PageImages {
private PageImages() {}
static void emit(PdfDocument doc, int pageIndex, List<Object> pageItems) throws IOException {
try (PdfPage page = doc.page(pageIndex)) {
List<ExtractedImage> images =
PdfImageExtractor.extract(page.rawDocHandle(), page.rawHandle(), pageIndex);
for (ExtractedImage img : images) {
pageItems.add(describe(img));
}
}
}
/**
* Image placeholder annotated with whatever JPDFium exposes: pixels, placement, DPI, format,
* colour space, depth. Missing fields are omitted.
*/
private static String describe(ExtractedImage img) {
List<String> parts = new ArrayList<>();
if (img.width() > 0 && img.height() > 0) {
parts.add(img.width() + "x" + img.height() + "px");
}
Rect b = img.bounds();
if (b != null && b.width() > 0 && b.height() > 0) {
parts.add(String.format("%.0fx%.0fpt", b.width(), b.height()));
if (img.width() > 0) {
float dpiX = img.width() / (b.width() / 72f);
float dpiY = img.height() / (b.height() / 72f);
if (Float.isFinite(dpiX) && dpiX > 0) {
parts.add(String.format("~%.0fdpi", (dpiX + dpiY) / 2f));
}
}
}
String ext = img.suggestedExtension();
if (ext != null && !ext.isBlank()) {
parts.add(ext.replaceFirst("^\\.", "").toUpperCase(Locale.ROOT));
}
if (img.colorSpace() != null) {
parts.add(img.colorSpace().toString());
}
if (img.bitsPerPixel() > 0) {
parts.add(img.bitsPerPixel() + "bpp");
}
StringBuilder sb = new StringBuilder("<image redacted");
if (!parts.isEmpty()) {
sb.append(": ").append(String.join(", ", parts));
}
sb.append('>');
return sb.toString();
}
}
@@ -0,0 +1,128 @@
package stirling.software.proprietary.pdf;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import stirling.software.jpdfium.PdfPage;
import stirling.software.jpdfium.doc.PageContentSummary;
import stirling.software.jpdfium.doc.PageObject;
import stirling.software.jpdfium.doc.PageObjectType;
import stirling.software.jpdfium.doc.PdfPageObjects;
import stirling.software.jpdfium.model.Rect;
/**
* Ruling lines of one page, from its {@link PageObject} paths: a long thin box is a rule, and a box
* large in both axes contributes its edges.
*/
final class PageRules {
/** A path this thin in its short axis is a rule rather than a filled area. */
private static final float RULE_THICKNESS = 3f;
/** A rule shorter than this is decoration (tick marks, bullets, glyph art). */
private static final float MIN_RULE_LENGTH = 8f;
/** Boxes larger than this in either axis are page furniture, not table structure. */
private static final float MAX_BOX = 1500f;
/**
* Object count past which the page is skipped: a page drawing this many is a chart or an
* operator flood, never a readable grid.
*/
private static final int MAX_PAGE_OBJECTS = 20_000;
/** A straight rule: {@code pos} is its y (horizontal) or x (vertical), spanning lo..hi. */
record Rule(float pos, float lo, float hi) {}
private final List<Rule> horizontal;
private final List<Rule> vertical;
private PageRules(List<Rule> horizontal, List<Rule> vertical) {
this.horizontal = horizontal;
this.vertical = vertical;
}
static final PageRules EMPTY = new PageRules(List.of(), List.of());
List<Rule> horizontal() {
return horizontal;
}
List<Rule> vertical() {
return vertical;
}
boolean isEmpty() {
return horizontal.isEmpty() && vertical.isEmpty();
}
/** Reads the ruling lines of an already-open page. */
static PageRules of(PdfPage page) throws IOException {
List<Rule> h = new ArrayList<>();
List<Rule> v = new ArrayList<>();
List<PageObject> objects;
try {
// Counting is far cheaper than materialising every object, so decide from the summary
// whether the page is worth enumerating at all.
PageContentSummary summary = PdfPageObjects.summarize(page.rawHandle());
if (summary.pathObjectCount() < 2 || summary.totalObjects() > MAX_PAGE_OBJECTS) {
return EMPTY;
}
objects = PdfPageObjects.list(page.rawHandle());
} catch (RuntimeException e) {
// Path enumeration is an optimisation, never a correctness requirement: a page whose
// objects cannot be read simply falls back to word-grid detection.
return EMPTY;
}
float pageW = 0f;
float pageH = 0f;
try {
pageW = page.size().width();
pageH = page.size().height();
} catch (RuntimeException ignored) {
// Fall through with 0,0: the on-page check below is then skipped.
}
for (PageObject o : objects) {
if (o.type() != PageObjectType.PATH) {
continue;
}
Rect b = o.bounds();
if (b == null) {
continue;
}
float w = b.width();
float ht = b.height();
if (!Float.isFinite(w) || !Float.isFinite(ht) || w < 0 || ht < 0) {
continue;
}
if (w > MAX_BOX || ht > MAX_BOX) {
continue;
}
// Paths that run off the page are chart clipping or decoration, never table structure.
if (pageW > 0
&& (b.x() < -1f
|| b.y() < -1f
|| b.x() + w > pageW + 1f
|| b.y() + ht > pageH + 1f)) {
continue;
}
if (ht <= RULE_THICKNESS && w >= MIN_RULE_LENGTH) {
h.add(new Rule(b.y() + ht / 2f, b.x(), b.x() + w));
} else if (w <= RULE_THICKNESS && ht >= MIN_RULE_LENGTH) {
v.add(new Rule(b.x() + w / 2f, b.y(), b.y() + ht));
} else if (w >= MIN_RULE_LENGTH && ht >= MIN_RULE_LENGTH) {
// A box: a table border, a cell outline or a shaded row fill. Its edges bound cells
// exactly as drawn rules do, and many generators draw grids as per-cell rectangles.
h.add(new Rule(b.y(), b.x(), b.x() + w));
h.add(new Rule(b.y() + ht, b.x(), b.x() + w));
v.add(new Rule(b.x(), b.y(), b.y() + ht));
v.add(new Rule(b.x() + w, b.y(), b.y() + ht));
}
}
h.sort(Comparator.comparingDouble(Rule::pos).reversed());
v.sort(Comparator.comparingDouble(Rule::pos));
return new PageRules(h, v);
}
}
@@ -0,0 +1,144 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
/**
* Joins what a page break split: a sentence running into the next page, and a table whose rows
* continue on it.
*/
final class PageStitcher {
private PageStitcher() {}
static void mergeAcrossPageBoundary(List<Object> output, List<Object> pageItems) {
if (output.isEmpty() || pageItems.isEmpty()) {
return;
}
// Only merge a sentence continuation between two text paragraphs, never into/out of a
// table.
if (!(output.getLast() instanceof String last)
|| !(pageItems.getFirst() instanceof String first)) {
return;
}
if (!first.isEmpty()
&& Character.isLowerCase(first.charAt(0))
&& !MarkdownText.endsWithSentencePunctuation(last)) {
output.set(output.size() - 1, last + " " + first);
pageItems.remove(0);
}
}
/**
* Joins tables split across a page break: two consecutive blocks with no text between them
* merge when their column layouts match, dropping a repeated header.
*/
static List<Object> stitchTables(List<Object> elements) {
List<Object> out = new ArrayList<>();
// Column geometry of the trailing TableBlock in `out`, carried forward across merges so a
// table running page-to-page is not re-projected from every accumulated row at each break.
ColumnAccumulator acc = null;
// Row list we own and may append to in place; null while the trailing block still holds a
// list belonging to `elements`.
List<List<Line>> ownedRows = null;
for (Object e : elements) {
if (e instanceof TableBlock tb
&& !out.isEmpty()
&& out.getLast() instanceof TableBlock prev) {
if (acc == null) {
acc = ColumnAccumulator.of(prev.rows());
}
if (columnsMatch(acc.columns(), ColumnRanges.find(flatten(tb.rows())))) {
List<List<Line>> merged;
if (ownedRows == null) {
merged = new ArrayList<>(prev.rows());
ownedRows = merged;
} else {
merged = ownedRows;
}
List<List<Line>> tail = tb.rows();
if (!tail.isEmpty()
&& !prev.rows().isEmpty()
&& rowText(tail.getFirst()).equals(rowText(prev.rows().getFirst()))) {
tail = tail.subList(1, tail.size());
}
for (List<Line> row : tail) {
for (Line l : row) {
acc.addLine(l);
}
}
merged.addAll(tail);
// A stitched table belongs to where it started, so keep the earlier block's
// page and columns; its ruling lines are dropped as they are one page's only.
out.set(
out.size() - 1,
new TableBlock(
merged,
prev.top(),
tb.bottom(),
prev.cols(),
prev.ruled(),
prev.rowSource(),
prev.page()));
continue;
}
}
out.add(e);
acc = null;
ownedRows = null;
}
return out;
}
private static List<Line> flatten(List<List<Line>> rows) {
return rows.stream().flatMap(List::stream).collect(Collectors.toList());
}
/**
* Header text of a table at the very bottom of a page, or null. Trailing image placeholders are
* skipped; any other text means it is not a continuation.
*/
static String trailingTableHeader(List<Object> pageItems) {
for (int i = pageItems.size() - 1; i >= 0; i--) {
Object e = pageItems.get(i);
if (e instanceof String s && s.strip().startsWith("<image redacted")) {
continue;
}
if (e instanceof TableBlock tb && !tb.rows().isEmpty()) {
return rowText(tb.rows().getFirst());
}
return null;
}
return null;
}
static String rowText(List<Line> row) {
List<Line> ordered = new ArrayList<>(row);
ordered.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
StringBuilder sb = new StringBuilder();
for (Line l : ordered) {
if (!sb.isEmpty()) {
sb.append(' ');
}
sb.append(l.text);
}
return MarkdownText.normaliseSpace(sb.toString());
}
/** True when two table blocks have the same number of columns at near-identical x-centres. */
private static boolean columnsMatch(List<float[]> ca, List<float[]> cb) {
if (ca.size() < 2 || ca.size() != cb.size()) {
return false;
}
for (int i = 0; i < ca.size(); i++) {
float centreA = (ca.get(i)[0] + ca.get(i)[1]) / 2f;
float centreB = (cb.get(i)[0] + cb.get(i)[1]) / 2f;
if (Math.abs(centreA - centreB) > 15f) {
return false;
}
}
return true;
}
}
@@ -0,0 +1,252 @@
package stirling.software.proprietary.pdf;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
/**
* Turns a column's lines into Markdown blocks: headings (including wrapped ones), bullets, bold
* labels and paragraphs. Contents lists are recognised and left alone.
*/
final class ParagraphAssembler {
private ParagraphAssembler() {}
static void assembleParagraphs(
List<Line> lines,
float medianSize,
float medianHeight,
String bodyFont,
List<String> out,
Set<String> tableRowTexts) {
StringBuilder para = new StringBuilder();
float prevBottomY = Float.MAX_VALUE;
float prevHeight = 0f;
boolean[] inContents = contentsRun(lines);
for (int i = 0; i < lines.size(); i++) {
Line line = lines.get(i);
String text = line.text.strip();
if (text.isEmpty()) {
continue;
}
if (tableRowTexts.contains(text)) {
continue;
}
float blockTop = line.y + line.height;
float gap = prevBottomY - blockTop;
boolean paragraphBreak = prevHeight > 0f && gap > prevHeight * 0.8f;
// A contents entry carries the typography of the section it points at without being
// that section, so nothing on one is promoted or emphasised.
boolean structural = inContents[i];
// A field value is data, never a heading: its widget box is taller than a text line
// and would otherwise be promoted purely on height.
String prefix =
line.synthetic || structural
? ""
: HeadingDetector.headingPrefix(
line.detectText(),
line.detectHeight(),
line.words(),
medianSize,
medianHeight,
bodyFont,
prevHeight <= 0f || paragraphBreak);
if (prefix.isEmpty() && !structural && contentsTitle(lines, inContents, i)) {
// The line a contents list runs on from is its heading: a contents page is often
// set in one face, leaving no size or weight to promote it on.
prefix = "# ";
}
boolean isBullet = startsWithBullet(text);
// A line that opens with a list marker is an item of a list, whatever it is set in.
boolean isHeading = !prefix.isEmpty() && !isBullet;
if (isHeading) {
MarkdownText.flushParagraph(para, out);
StringBuilder heading = new StringBuilder(MarkdownText.escapeMarkdown(text));
int words = MarkdownText.wordCount(text);
int j = i;
int k = i + 1;
while (k < lines.size() && words < MAX_WRAPPED_HEADING_WORDS) {
Line next = lines.get(k);
String nt = next.text.strip();
if (nt.isEmpty()) {
// An empty extractor record is not a break in the text; the vertical
// gap below decides whether the heading ended.
k++;
continue;
}
if (inContents[k] || tableRowTexts.contains(nt)) {
break;
}
if (!wrapsHeading(
lines.get(j), next, prefix, medianSize, medianHeight, bodyFont)) {
break;
}
heading.append(' ').append(MarkdownText.escapeMarkdown(nt));
words += MarkdownText.wordCount(nt);
j = k;
k++;
}
out.add(prefix + heading);
if (j > i) {
i = j;
line = lines.get(j);
}
} else if (isBullet) {
MarkdownText.flushParagraph(para, out);
out.add(MarkdownText.escapeMarkdown(text));
} else if (!line.synthetic
&& !structural
&& HeadingDetector.isBoldLabel(line.detectText(), line.words())) {
// Bold but not large enough to be a heading → emphasise as bold, don't promote.
MarkdownText.flushParagraph(para, out);
out.add("**" + MarkdownText.escapeMarkdown(text) + "**");
} else if (paragraphBreak) {
MarkdownText.flushParagraph(para, out);
para.append(text);
} else {
if (!para.isEmpty()) {
char fc = text.charAt(0);
boolean noSpace = fc == '\'' || fc == '' || fc == '' || fc == '"';
if (!noSpace) {
para.append(' ');
}
}
para.append(text);
}
prevBottomY = line.y;
prevHeight = line.height;
}
MarkdownText.flushParagraph(para, out);
}
/** Glyphs a document may set its list markers in beyond the three already recognised. */
private static final String EXTRA_BULLETS = "‣⁃▶●○■□" + "◆⮚➢➣➤";
private static boolean startsWithBullet(String text) {
if (text.isEmpty()) {
return false;
}
if (text.startsWith("") || text.startsWith("") || text.startsWith("")) {
return true;
}
return EXTRA_BULLETS.indexOf(text.charAt(0)) >= 0;
}
/** Longest a heading may grow to by absorbing its continuation lines, in words. */
private static final int MAX_WRAPPED_HEADING_WORDS = 24;
/** How far a continuation line's type size may differ from the line it continues. */
private static final float WRAP_SIZE_TOLERANCE = 0.2f;
/** A full stop that a further sentence follows: the shape of prose, not of a heading. */
private static final Pattern SENTENCE_BREAK = Pattern.compile("[.!?]\\s+\\p{Lu}");
/**
* True when {@code next} continues a wrapped heading: each visual line arrives separately, so
* an unjoined heading emits as several spurious ones.
*/
private static boolean wrapsHeading(
Line head,
Line next,
String prefix,
float medianSize,
float medianHeight,
String bodyFont) {
if (next.synthetic) {
return false;
}
float height = head.detectHeight();
if (height <= 0f) {
return false;
}
// The next baseline down, not the next block. The same 0.8 the paragraph assembler uses,
// so a heading absorbs exactly what the converter already calls one block.
float gap = head.y - (next.y + next.height);
if (gap > height * 0.8f || gap < -height * 0.5f) {
return false;
}
float nextHeight = next.detectHeight();
if (Math.abs(nextHeight - height) > WRAP_SIZE_TOLERANCE * Math.max(nextHeight, height)) {
return false;
}
// Same column: an x-range that misses the heading's belongs to another block entirely.
if (next.x >= head.x + head.width || head.x >= next.x + next.width) {
return false;
}
// A heading does not run to a full stop and then start another sentence; the bold run-in
// lead-in below it does, and nothing else tells the two apart.
if (SENTENCE_BREAK.matcher(next.text).find()) {
return false;
}
String nextPrefix =
HeadingDetector.headingPrefix(
next.detectText(),
next.detectHeight(),
next.words(),
medianSize,
medianHeight,
bodyFont,
false);
// Either the continuation is display type in its own right, or it is the bold remainder of
// a run-in heading, which cannot be promoted on its own because no gap precedes it.
return nextPrefix.equals(prefix)
|| (nextPrefix.isEmpty()
&& HeadingDetector.isBoldLabel(next.detectText(), next.words()));
}
/** A leader run: the dots that carry the eye from a contents entry to its page number. */
private static final Pattern LEADER = Pattern.compile("([.][ ]?){4,}|[.\u00b7]{3,}|\u2026{2,}");
/** Entries this many lines long make a contents list rather than a coincidence. */
private static final int MIN_CONTENTS_RUN = 3;
/**
* Marks the lines of a contents list: titles joined to page numbers by leader dots, carrying
* the typography of the sections they point at.
*/
private static boolean[] contentsRun(List<Line> lines) {
boolean[] entry = new boolean[lines.size()];
int run = 0;
for (int i = 0; i < lines.size(); i++) {
String t = lines.get(i).text;
if (LEADER.matcher(t).find() && endsWithNumber(t)) {
entry[i] = true;
run++;
} else {
if (run < MIN_CONTENTS_RUN) {
clear(entry, i - run, i);
}
run = 0;
}
}
if (run < MIN_CONTENTS_RUN) {
clear(entry, lines.size() - run, lines.size());
}
return entry;
}
private static void clear(boolean[] flags, int from, int to) {
for (int i = Math.max(0, from); i < to; i++) {
flags[i] = false;
}
}
private static boolean endsWithNumber(String text) {
String t = text.strip();
return !t.isEmpty() && Character.isDigit(t.charAt(t.length() - 1));
}
/** True for the short line a contents list runs on from: the list's own heading. */
private static boolean contentsTitle(List<Line> lines, boolean[] inContents, int index) {
if (index + 1 >= lines.size() || inContents[index] || !inContents[index + 1]) {
return false;
}
String t = lines.get(index).text.strip();
return !t.isEmpty() && t.split(" +").length <= 6 && !endsWithNumber(t);
}
}
@@ -0,0 +1,18 @@
package stirling.software.proprietary.pdf;
/**
* How much of a block's row structure the page itself drew; stronger evidence means weaker
* false-positive guards.
*/
enum RowSource {
/** Rows inferred from word geometry alone; nothing on the page confirms a table. */
WORDS,
/** Rows sit inside a region fenced by drawn rules, but the rules do not delimit them. */
RULE_BOUNDED,
/** Every row boundary is a drawn rule running the table's own width. */
LATTICE;
boolean ruleConfirmed() {
return this != WORDS;
}
}
@@ -0,0 +1,153 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
/**
* A page's ruling lines reduced to a grid: rules merged into levels, levels grouped into the
* components that each describe one table. Both steps are bounded.
*/
@Slf4j
final class RuleGrid {
/** Rules within this distance are the same drawn line (double strokes, overdraw). */
static final float LEVEL_TOLERANCE = 2.5f;
/** Slack when testing whether a horizontal and a vertical rule touch. */
private static final float TOUCH = 3f;
/** Segments at one position further apart than this belong to different tables. */
private static final float CONTIGUOUS_GAP = 8f;
/** Crossing tests past which a page is an operator flood rather than a readable grid. */
private static final long MAX_CROSSING_TESTS = 4_000_000L;
/** Rule components past which the extra blocks cannot be real tables. */
private static final int MAX_COMPONENTS = 256;
private RuleGrid() {}
/** A group of rules at the same position: {@code pos} with the union of their extents. */
record Level(float pos, float lo, float hi) {}
/** One connected component of crossing rules: the levels of each family it spans. */
record Component(List<Level> h, List<Level> v) {}
/**
* Merges rules at the same position into levels, but only while contiguous, so two tables
* ruling at the same x are not bridged into one region.
*/
static List<Level> cluster(List<PageRules.Rule> rules) {
List<PageRules.Rule> sorted = new ArrayList<>(rules);
sorted.sort(
Comparator.comparingDouble(PageRules.Rule::pos)
.thenComparingDouble(PageRules.Rule::lo));
List<Level> out = new ArrayList<>();
int i = 0;
while (i < sorted.size()) {
float pos = sorted.get(i).pos();
int j = i;
while (j < sorted.size() && sorted.get(j).pos() - pos <= LEVEL_TOLERANCE) {
j++;
}
List<PageRules.Rule> same = new ArrayList<>(sorted.subList(i, j));
same.sort(Comparator.comparingDouble(PageRules.Rule::lo));
float lo = same.get(0).lo();
float hi = same.get(0).hi();
for (int k = 1; k < same.size(); k++) {
if (same.get(k).lo() <= hi + CONTIGUOUS_GAP) {
hi = Math.max(hi, same.get(k).hi());
} else {
out.add(new Level(pos, lo, hi));
lo = same.get(k).lo();
hi = same.get(k).hi();
}
}
out.add(new Level(pos, lo, hi));
i = j;
}
return out;
}
/**
* Connected components of crossing rules, read from one union-find array: an id array per
* component is O(components x levels) a rule flood can exhaust.
*/
static List<Component> partition(List<Level> hLevels, List<Level> vLevels) {
int n = hLevels.size() + vLevels.size();
if ((long) hLevels.size() * vLevels.size() > MAX_CROSSING_TESTS) {
log.debug(
"ruled-table partition skipped: {}x{} rule levels",
hLevels.size(),
vLevels.size());
return List.of();
}
int[] parent = new int[n];
for (int i = 0; i < n; i++) {
parent[i] = i;
}
for (int i = 0; i < hLevels.size(); i++) {
Level h = hLevels.get(i);
for (int j = 0; j < vLevels.size(); j++) {
Level v = vLevels.get(j);
boolean crosses =
v.pos() >= h.lo() - TOUCH
&& v.pos() <= h.hi() + TOUCH
&& h.pos() >= v.lo() - TOUCH
&& h.pos() <= v.hi() + TOUCH;
if (crosses) {
union(parent, i, hLevels.size() + j);
}
}
}
Map<Integer, Component> byRoot = new LinkedHashMap<>();
for (int i = 0; i < n; i++) {
int root = find(parent, i);
Component c = byRoot.get(root);
if (c == null) {
// Past the cap the page is line art, not tables; keep the components already
// found whole rather than truncating them mid-scan.
if (byRoot.size() >= MAX_COMPONENTS) {
continue;
}
c = new Component(new ArrayList<>(), new ArrayList<>());
byRoot.put(root, c);
}
if (i < hLevels.size()) {
c.h().add(hLevels.get(i));
} else {
c.v().add(vLevels.get(i - hLevels.size()));
}
}
return List.copyOf(byRoot.values());
}
private static int find(int[] parent, int x) {
while (parent[x] != x) {
parent[x] = parent[parent[x]];
x = parent[x];
}
return x;
}
private static void union(int[] parent, int a, int b) {
int ra = find(parent, a);
int rb = find(parent, b);
if (ra != rb) {
parent[rb] = ra;
}
}
/**
* Visible for testing: partitioning depends only on rule geometry, so tests can drive it from
* synthetic rules.
*/
static int componentCount(List<PageRules.Rule> horizontal, List<PageRules.Rule> vertical) {
return partition(cluster(horizontal), cluster(vertical)).size();
}
}
@@ -0,0 +1,154 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import stirling.software.jpdfium.text.TextWord;
/**
* Groups the lines inside a ruled region into rows, and reads its column bands off the vertical
* rules.
*/
final class RuledRows {
private RuledRows() {}
/** A vertical rule must cover this fraction of a region's height to be a column boundary. */
private static final float COLUMN_COVERAGE = 0.5f;
/**
* Splits a band whose every baseline is a complete row back into those rows; a wrapped cell
* leaves the other columns empty, a run of rows does not.
*/
static List<List<Line>> splitCompleteBands(List<List<Line>> bands, List<float[]> cols) {
if (cols == null || cols.size() < 2) {
return bands;
}
List<List<Line>> out = new ArrayList<>();
for (List<Line> band : bands) {
List<List<Line>> baselines = baselineRows(band);
if (baselines.size() < 2 || !allRowsComplete(baselines, cols)) {
out.add(band);
continue;
}
out.addAll(baselines);
}
return out;
}
/** True when every baseline group puts a word in every column band. */
private static boolean allRowsComplete(List<List<Line>> baselines, List<float[]> cols) {
for (List<Line> row : baselines) {
boolean[] hit = new boolean[cols.size()];
for (Line l : row) {
for (TextWord w : l.words()) {
if (w.text().strip().isEmpty()) {
continue;
}
int c = TableGrid.containingColumn(w.x() + w.width() / 2f, cols);
if (c >= 0 && c < hit.length) {
hit[c] = true;
}
}
}
for (boolean h : hit) {
if (!h) {
return false;
}
}
}
return true;
}
/**
* Column bands from the vertical rules spanning the region; null when no interior rule
* survives, as whitespace projection guesses better.
*/
static List<float[]> columns(
List<RuleGrid.Level> vLevels, float left, float right, float top, float bottom) {
float height = top - bottom;
// Per-cell strokes give one rule per row, and a row that draws no boxes breaks the run
// in two, so strokes at one x are measured together rather than as separate runs.
List<RuleGrid.Level> sorted = new ArrayList<>(vLevels);
sorted.sort(Comparator.comparingDouble(RuleGrid.Level::pos));
List<Float> xs = new ArrayList<>();
int at = 0;
while (at < sorted.size()) {
float pos = sorted.get(at).pos();
float covered = 0f;
int end = at;
while (end < sorted.size() && sorted.get(end).pos() - pos <= RuleGrid.LEVEL_TOLERANCE) {
RuleGrid.Level v = sorted.get(end);
covered += Math.max(0f, Math.min(top, v.hi()) - Math.max(bottom, v.lo()));
end++;
}
if (covered >= height * COLUMN_COVERAGE) {
xs.add(pos);
}
at = end;
}
List<Float> bounds = new ArrayList<>();
bounds.add(left);
for (float x : xs) {
if (x > bounds.get(bounds.size() - 1) + RuleGrid.LEVEL_TOLERANCE
&& x < right - RuleGrid.LEVEL_TOLERANCE) {
bounds.add(x);
}
}
if (bounds.size() < 2) {
return null;
}
bounds.add(right);
List<float[]> cols = new ArrayList<>();
for (int i = 1; i < bounds.size(); i++) {
cols.add(new float[] {bounds.get(i - 1), bounds.get(i)});
}
return cols;
}
/** Rows delimited by horizontal rules; this is what keeps a wrapped cell as one row. */
static List<List<Line>> latticeRows(List<Float> bands, List<Line> inside) {
List<List<Line>> rows = new ArrayList<>();
for (int i = 1; i < bands.size(); i++) {
float hi = bands.get(i - 1);
float lo = bands.get(i);
List<Line> band = new ArrayList<>();
for (Line l : inside) {
float cy = l.y + l.height / 2f;
if (cy > lo && cy <= hi) {
band.add(l);
}
}
if (!band.isEmpty()) {
rows.add(band);
}
}
return rows;
}
/** Rows by baseline proximity, for a table ruled between its columns but not its rows. */
static List<List<Line>> baselineRows(List<Line> inside) {
List<Line> sorted = new ArrayList<>(inside);
sorted.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
List<Float> heights = sorted.stream().map(l -> l.height).sorted().toList();
float sameRow = Math.max(2f, heights.get(heights.size() / 2) * 0.6f);
List<List<Line>> rows = new ArrayList<>();
List<Line> current = new ArrayList<>();
float anchor = 0f;
for (Line l : sorted) {
if (current.isEmpty()) {
anchor = l.y;
} else if (anchor - l.y > sameRow) {
rows.add(current);
current = new ArrayList<>();
anchor = l.y;
}
current.add(l);
}
if (!current.isEmpty()) {
rows.add(current);
}
return rows;
}
}
@@ -0,0 +1,385 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
/**
* Builds table blocks from a page's ruling lines: whitespace projection cannot see single-word or
* wrapped cells, as they leave no wide gap.
*/
@Slf4j
final class RuledTables {
private RuledTables() {}
/** Largest vertical gap between two rules of one rows-only table. */
private static final float ROWS_ONLY_GAP = 150f;
/** How far two rules of one rows-only table may differ at either end. */
private static final float EXTENT_TOLERANCE = 8f;
/** Lines a rows-only group needs before two rules alone are enough to call it a table. */
private static final int ROWS_ONLY_LINES = 4;
/** Fraction of a lattice's row bands that must contain text for it to be a real table. */
private static final float FILLED_BANDS = 0.6f;
/** Fraction of the table's width an interior rule must run to be a row boundary. */
private static final float ROW_RULE_SPAN = 0.8f;
/**
* Interior row rules needed before drawn bands beat text baselines; bands keep a multi-line
* cell whole where baselines split it.
*/
private static final int MIN_INTERIOR_RULES = 1;
/** Fraction of a region's width every rule must run for its rows to be a drawn lattice. */
private static final float FULL_WIDTH_RULE = 0.8f;
private static TableBlock dbgNull(String why) {
log.debug("ruled-table build rejected: {}", why);
return null;
}
static List<TableBlock> find(List<Line> lines, PageRules rules, int page) {
if (rules == null || rules.isEmpty() || lines.isEmpty()) {
return List.of();
}
// Synthetic AcroForm values carry no glyphs of their own, so a ruled grid must not
// claim them: they would seed rows and columns the content stream never drew.
lines = lines.stream().filter(l -> !l.synthetic).toList();
if (lines.isEmpty()) {
return List.of();
}
List<RuleGrid.Level> hLevels = RuleGrid.cluster(rules.horizontal());
List<RuleGrid.Level> vLevels = RuleGrid.cluster(rules.vertical());
if (hLevels.size() < 2) {
return List.of();
}
List<TableBlock> blocks = new ArrayList<>();
for (RuleGrid.Component part : RuleGrid.partition(hLevels, vLevels)) {
TableBlock b = build(part.h(), part.v(), lines, page);
if (b != null) {
blocks.add(b);
}
}
// Horizontal rules no grid block claimed can still be a booktabs table: rows ruled,
// columns not drawn at all. Whatever the grid did not take is offered to that reading.
List<RuleGrid.Level> unclaimed = new ArrayList<>();
for (RuleGrid.Level h : hLevels) {
boolean claimed = false;
for (TableBlock b : blocks) {
if (h.pos() >= b.bottom() - RuleGrid.LEVEL_TOLERANCE
&& h.pos() <= b.top() + RuleGrid.LEVEL_TOLERANCE) {
claimed = true;
break;
}
}
if (!claimed) {
unclaimed.add(h);
}
}
if (unclaimed.size() >= 2) {
for (TableBlock b : rowsOnly(unclaimed, lines, page)) {
boolean overlaps = false;
for (TableBlock existing : blocks) {
if (TableFinder.covers(existing, b)) {
overlaps = true;
break;
}
}
if (!overlaps) {
blocks.add(b);
}
}
}
blocks.sort(Comparator.comparingDouble(TableBlock::top).reversed());
return blocks;
}
/**
* Blocks for a page ruled only across its rows (booktabs): no column geometry to recover, so
* these only find a table the word grid missed.
*/
private static List<TableBlock> rowsOnly(
List<RuleGrid.Level> levels, List<Line> lines, int page) {
List<RuleGrid.Level> hLevels = new ArrayList<>(levels);
hLevels.sort(Comparator.comparingDouble(RuleGrid.Level::pos).reversed());
List<TableBlock> blocks = new ArrayList<>();
List<List<RuleGrid.Level>> groups = new ArrayList<>();
List<RuleGrid.Level> current = new ArrayList<>();
current.add(hLevels.get(0));
for (int i = 1; i < hLevels.size(); i++) {
RuleGrid.Level prev = current.get(current.size() - 1);
RuleGrid.Level l = hLevels.get(i);
// One booktabs table rules to a single extent; two stacked tables differ in width,
// and grouping them would project their columns into a single band.
if (prev.pos() - l.pos() > ROWS_ONLY_GAP
|| Math.abs(prev.lo() - l.lo()) > EXTENT_TOLERANCE
|| Math.abs(prev.hi() - l.hi()) > EXTENT_TOLERANCE) {
groups.add(current);
current = new ArrayList<>();
}
current.add(l);
}
groups.add(current);
for (List<RuleGrid.Level> g : groups) {
if (g.size() < 2) {
continue;
}
float top = g.get(0).pos();
float bottom = g.get(g.size() - 1).pos();
float left = Float.MAX_VALUE;
float right = -Float.MAX_VALUE;
for (RuleGrid.Level l : g) {
left = Math.min(left, l.lo());
right = Math.max(right, l.hi());
}
List<Line> inside = new ArrayList<>();
for (Line l : lines) {
float cy = l.y + l.height / 2f;
float cx = l.x + l.width / 2f;
if (cy > bottom && cy < top && cx > left - 5f && cx < right + 5f) {
inside.add(l);
}
}
// Enough text to be a table: several rows, or for a two-row table a third rule,
// the header separator a lone pair of decorative rules does not draw.
if (inside.size() < 2 || (inside.size() < ROWS_ONLY_LINES && g.size() < 3)) {
continue;
}
List<List<Line>> rows = RuledRows.baselineRows(inside);
if (rows.size() < 2 || TableGrid.render(rows, null, RowSource.RULE_BOUNDED).isBlank()) {
continue;
}
blocks.add(new TableBlock(rows, top, bottom, null, page));
}
blocks.sort(Comparator.comparingDouble(TableBlock::top).reversed());
return blocks;
}
private static TableBlock build(
List<RuleGrid.Level> hL, List<RuleGrid.Level> vL, List<Line> lines, int page) {
log.debug("ruled-table build hL={} vL={}", hL.size(), vL.size());
if (hL.size() < 2 || vL.size() < 2) {
return dbgNull("hL/vL < 2");
}
hL.sort(Comparator.comparingDouble(RuleGrid.Level::pos).reversed());
vL.sort(Comparator.comparingDouble(RuleGrid.Level::pos));
// The extent is the union of both families: a table ruled only between its columns
// takes its top and bottom from the verticals, and vice versa.
float top = hL.get(0).pos();
float bottom = hL.get(hL.size() - 1).pos();
float left = vL.get(0).pos();
float right = vL.get(vL.size() - 1).pos();
for (RuleGrid.Level v : vL) {
top = Math.max(top, v.hi());
bottom = Math.min(bottom, v.lo());
}
for (RuleGrid.Level h : hL) {
left = Math.min(left, h.lo());
right = Math.max(right, h.hi());
}
if (top - bottom < 6f || right - left < 20f) {
return dbgNull("too small");
}
List<Line> inside = new ArrayList<>();
for (Line l : lines) {
float cy = l.y + l.height / 2f;
float cx = l.x + l.width / 2f;
if (cy > bottom && cy < top && cx > left - 5f && cx < right + 5f) {
inside.add(l);
}
}
if (inside.size() < 2) {
return dbgNull("inside<2");
}
// Null columns mean the grid is ruled between its rows only; the block is still worth
// building, but its columns then come from whitespace projection.
List<float[]> cols = RuledRows.columns(vL, left, right, top, bottom);
// A row boundary is a y position, not a segment, and runs the table's width: per-cell
// rectangles report it once per cell and also box each wrapped line inside a cell.
float rowRuleWidth = (right - left) * ROW_RULE_SPAN;
List<Float> interiorH = new ArrayList<>();
List<RuleGrid.Level> bandRules = new ArrayList<>();
float prevWide = top;
int i = 0;
while (i < hL.size()) {
float pos = hL.get(i).pos();
int j = i;
RuleGrid.Level widest = hL.get(i);
while (j < hL.size() && Math.abs(hL.get(j).pos() - pos) <= RuleGrid.LEVEL_TOLERANCE) {
if (hL.get(j).hi() - hL.get(j).lo() > widest.hi() - widest.lo()) {
widest = hL.get(j);
}
j++;
}
i = j;
if (pos <= bottom + RuleGrid.LEVEL_TOLERANCE || pos >= top - RuleGrid.LEVEL_TOLERANCE) {
bandRules.add(widest);
continue;
}
boolean wide = widest.hi() - widest.lo() >= rowRuleWidth;
boolean keep =
wide || spanningNeighbour(widest, vL, inside, pos, prevWide, top - bottom);
if (!keep) {
continue;
}
interiorH.add(pos);
bandRules.add(widest);
if (wide) {
prevWide = pos;
}
}
List<List<Line>> rows;
RowSource source = RowSource.RULE_BOUNDED;
if (interiorH.size() >= MIN_INTERIOR_RULES) {
List<Float> bands = new ArrayList<>();
bands.add(top);
bands.addAll(interiorH);
bands.add(bottom);
List<List<Line>> filled = RuledRows.latticeRows(bands, inside);
// Most bands must carry text: a chart's axis ticks or a zebra table's stripes rule
// many empty bands, and reading those as a table steals lines from the prose.
if (filled.size() < (bands.size() - 1) * FILLED_BANDS) {
return dbgNull("filled " + filled.size() + " of bands " + (bands.size() - 1));
}
rows = RuledRows.splitCompleteBands(filled, cols);
if (fullWidthRules(bandRules, left, right)) {
source = RowSource.LATTICE;
}
} else {
rows = RuledRows.baselineRows(inside);
}
if (rows.size() < 2) {
return dbgNull("rows<2");
}
// A grid is often ruled around its body only, leaving the header just above the top
// rule; take it when it fits the grid's width and resolves into its columns.
if (cols != null) {
// The header's cells are separate lines when they sit far apart, so the whole
// band above the grid is taken, not the nearest line.
List<Line> hdr = new ArrayList<>();
float band = Float.MAX_VALUE;
for (Line l : lines) {
if (l.y <= top
|| l.y - top > TableFinder.HEADER_RULE_GAP * Math.max(l.height, 1f)
|| l.x < left - 5f
|| l.x + l.width > right + 5f) {
continue;
}
band = Math.min(band, l.y);
}
for (Line l : lines) {
if (band < Float.MAX_VALUE
&& l.y >= band
&& l.y <= band + 2f
&& l.x >= left - 5f
&& l.x + l.width <= right + 5f) {
hdr.add(l);
}
}
if (!hdr.isEmpty()) {
List<List<Line>> withHeader = new ArrayList<>();
withHeader.add(hdr);
withHeader.addAll(rows);
List<String[]> grown = TableGrid.cells(withHeader, cols, source);
if (!grown.isEmpty()
&& TableGrid.filledCells(grown.get(0)) >= grown.get(0).length - 1
&& TableGrid.filledCells(grown.get(0)) >= 2
&& TableFinder.wordGroups(hdr) == TableGrid.filledCells(grown.get(0))) {
rows = withHeader;
top = band + hdr.get(0).height;
}
}
}
TableBlock block = new TableBlock(rows, top, bottom, cols, true, source, page);
// A block that fails the shared false-positive guards is not a table; leaving its lines
// unclaimed lets the word-grid detector or ordinary paragraph assembly handle them.
if (TableGrid.render(rows, cols, source).isBlank()) {
return dbgNull(
"guards rejected: rows="
+ rows.size()
+ " cols="
+ (cols == null ? -1 : cols.size()));
}
return block;
}
/** How near a rule end must be to a vertical rule to count as landing on it. */
private static final float COLUMN_SNAP = 2.5f;
/** Fraction of the table's height a vertical must run to be a column boundary. */
private static final float COLUMN_RUN = 0.5f;
/**
* True when a rule narrower than the table is still a row boundary: it ends on the grid's
* verticals and a spanning cell's text sits beside it.
*/
private static boolean spanningNeighbour(
RuleGrid.Level rule,
List<RuleGrid.Level> vL,
List<Line> inside,
float pos,
float above,
float height) {
// The vertical must run the table, not merely be there: a line box inside a wrapped
// cell draws its own short verticals at its inset edges.
float columnRun = height * COLUMN_RUN;
boolean loOnRule = false;
boolean hiOnRule = false;
for (RuleGrid.Level v : vL) {
if (v.hi() - v.lo() < columnRun) {
continue;
}
if (Math.abs(v.pos() - rule.lo()) <= COLUMN_SNAP) {
loOnRule = true;
}
if (Math.abs(v.pos() - rule.hi()) <= COLUMN_SNAP) {
hiOnRule = true;
}
}
if (!loOnRule || !hiOnRule) {
return false;
}
// The spanning cell's text must sit beside the rule anywhere in the row the last
// full-width boundary opened: it is written once, at the top of the span.
for (Line l : inside) {
float cy = l.y + l.height / 2f;
float cx = l.x + l.width / 2f;
if (cy > pos && cy < above && (cx < rule.lo() || cx > rule.hi())) {
return true;
}
}
return false;
}
/**
* True when every horizontal rule runs nearly the region's full width; legend swatches and
* per-cell outlines do not.
*/
private static boolean fullWidthRules(List<RuleGrid.Level> hL, float left, float right) {
float width = right - left;
if (width <= 0f) {
return false;
}
for (RuleGrid.Level h : hL) {
if (h.hi() - h.lo() < width * FULL_WIDTH_RULE) {
return false;
}
}
return true;
}
}
@@ -0,0 +1,33 @@
package stirling.software.proprietary.pdf;
import java.util.List;
/**
* A detected table. Each row is a list of source lines: usually one, but more when a cell wraps.
*/
record TableBlock(
List<List<Line>> rows,
float top,
float bottom,
List<float[]> cols,
boolean ruled,
RowSource rowSource,
int page) {
TableBlock(List<List<Line>> rows, float top, float bottom, int page) {
this(rows, top, bottom, null, false, RowSource.WORDS, page);
}
/** A rules-derived block whose rows are not a drawn lattice. */
TableBlock(List<List<Line>> rows, float top, float bottom, List<float[]> cols, int page) {
this(rows, top, bottom, cols, true, RowSource.RULE_BOUNDED, page);
}
String render() {
return TableGrid.render(rows, cols, rowSource);
}
/** Cell grid for the layout guards; empty when the block fails the table guards. */
List<String[]> cells() {
return TableGrid.cells(rows, cols, rowSource);
}
}
@@ -0,0 +1,299 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import stirling.software.jpdfium.text.TextWord;
/**
* Finds one page's table blocks and reconciles the two detectors: where both see a table, rows come
* from the text and columns from the rules.
*/
final class TableFinder {
private TableFinder() {}
/**
* Fraction of the word grid's rows a ruled grid must also find before its rows are trusted;
* below it the rules would merge several rows into one band.
*/
private static final float COMPLETE_LATTICE = 0.5f;
/**
* Detects a page's table blocks: ruled blocks first, then word-grid blocks over whatever lines
* the rules did not claim.
*/
static List<TableBlock> find(List<Line> lines, PageRules rules, int page) {
List<TableBlock> ruled = RuledTables.find(lines, rules, page);
List<TableBlock> word = fromWordGrid(lines, page);
if (ruled.isEmpty()) {
return word;
}
// Where both detectors see the same table, keep the word-grid's rows (read from the text)
// but take the columns from the rules, which are exact where projection only guesses.
List<TableBlock> all = new ArrayList<>();
Set<TableBlock> usedRules = new HashSet<>();
for (TableBlock w : word) {
TableBlock match = null;
for (TableBlock r : ruled) {
// Only a grid with real column rules can improve on the word-grid; one ruled
// across its rows alone contributes detection, never geometry.
if (r.cols() != null && covers(w, r)) {
match = r;
break;
}
}
if (match == null) {
// No column rules, but a rules-only grid over the same lines still confirms that a
// table is here. The word-grid's own reading of it stands, now rule-backed.
TableBlock evidence = null;
for (TableBlock r : ruled) {
if (r.cols() == null && w.top() > r.bottom() && w.bottom() < r.top()) {
evidence = r;
break;
}
}
if (evidence == null) {
all.add(w);
} else {
usedRules.add(evidence);
all.add(
new TableBlock(
w.rows(),
w.top(),
w.bottom(),
null,
true,
RowSource.WORDS,
w.page()));
}
} else if (match.rows().size() >= w.rows().size() * COMPLETE_LATTICE) {
// The rules cover nearly every row, so take the whole grid from them; one ruled
// grid can span several word-grid blocks, so emit it only once.
if (usedRules.add(match)) {
all.add(match);
}
} else {
// Only some row boundaries are drawn: rows from the text, columns from the rules.
usedRules.add(match);
all.add(
new TableBlock(
w.rows(),
w.top(),
w.bottom(),
match.cols(),
true,
RowSource.WORDS,
w.page()));
}
}
// A ruled table the word-grid never saw (single-word or wrapped cells leave it no wide gap
// to anchor on) is emitted from its rules alone.
for (TableBlock r : ruled) {
if (usedRules.contains(r)) {
continue;
}
boolean covered =
all.stream().anyMatch(b -> b.top() > r.bottom() && b.bottom() < r.top());
if (!covered) {
all.add(r);
}
}
all.sort(Comparator.comparingDouble(TableBlock::top).reversed());
return all;
}
/**
* Detects table blocks from word geometry: anchor rows grouped into contiguous runs, with
* non-anchor lines inside a run absorbed as wrapped cells.
*/
private static List<TableBlock> fromWordGrid(List<Line> lines, int page) {
List<Line> cands =
lines.stream()
.filter(l -> !l.synthetic && isTableCandidate(l.words()))
.sorted(Comparator.comparingDouble((Line l) -> l.y).reversed())
.collect(Collectors.toList());
if (cands.size() < 2) {
return List.of();
}
List<Float> gaps = new ArrayList<>();
for (int i = 1; i < cands.size(); i++) {
gaps.add(cands.get(i - 1).y - cands.get(i).y);
}
List<Float> sorted = new ArrayList<>(gaps);
sorted.sort(Comparator.naturalOrder());
float medianGap = sorted.get(sorted.size() / 2);
float splitThreshold = Math.max(medianGap * 2.5f, medianGap + 6f);
List<List<Line>> anchorGroups = new ArrayList<>();
List<Line> current = new ArrayList<>();
current.add(cands.getFirst());
for (int i = 1; i < cands.size(); i++) {
float gap = cands.get(i - 1).y - cands.get(i).y;
if (gap > splitThreshold) {
anchorGroups.add(current);
current = new ArrayList<>();
}
current.add(cands.get(i));
}
anchorGroups.add(current);
// Synthetic form values are kept out of the table path: they must not seed a column layout
// or be absorbed as wrapped cells, as they were never in the content stream.
List<Line> nonCandidates =
lines.stream()
.filter(l -> !l.synthetic && !isTableCandidate(l.words()))
.collect(Collectors.toList());
List<TableBlock> blocks = new ArrayList<>();
for (List<Line> anchors : anchorGroups) {
if (anchors.size() < 2) {
continue;
}
float top = anchors.getFirst().y;
float bottom = anchors.getLast().y;
// Each anchor seeds a row; absorb wrapped continuation lines (non-anchors within the
// run's vertical span, with a little slack below the last row) into the anchor above.
List<List<Line>> rows = new ArrayList<>();
for (Line a : anchors) {
List<Line> row = new ArrayList<>();
row.add(a);
rows.add(row);
}
for (Line nc : nonCandidates) {
if (nc.y > top || nc.y < bottom - medianGap) {
continue;
}
int owner = 0;
float bestDelta = Float.MAX_VALUE;
for (int i = 0; i < anchors.size(); i++) {
float delta = anchors.get(i).y - nc.y; // positive when anchor is above nc
if (delta >= -1f && delta < bestDelta) {
bestDelta = delta;
owner = i;
}
}
rows.get(owner).add(nc);
}
List<String[]> base = TableGrid.cells(rows, null, RowSource.WORDS);
if (base.isEmpty()) {
continue;
}
// A header row often has no wide gap between its cells, so the anchor test misses it.
// The line above is kept only if its grid has the same shape, excluding captions.
Line header = headerAbove(nonCandidates, top, medianGap);
if (header != null) {
List<List<Line>> withHeader = new ArrayList<>();
withHeader.add(new ArrayList<>(List.of(header)));
withHeader.addAll(rows);
List<String[]> grown = TableGrid.cells(withHeader, null, RowSource.WORDS);
if (!grown.isEmpty()
&& grown.get(0).length == base.get(0).length
&& TableGrid.filledCells(grown.get(0)) >= base.get(0).length) {
rows = withHeader;
top = header.y;
}
}
blocks.add(new TableBlock(rows, top, bottom, page));
}
return blocks;
}
/** Vertical gaps, in median row gaps, within which a line above a block can be its header. */
private static final float HEADER_GAP = 1.6f;
/**
* Runs of words separated by more than a cell gutter: a header row has one per cell, a caption
* written across the table is a single run.
*/
static int wordGroups(List<Line> row) {
List<TextWord> words = new ArrayList<>();
for (Line line : row) {
for (TextWord w : line.words()) {
if (!w.text().strip().isEmpty()) {
words.add(w);
}
}
}
if (words.isEmpty()) {
return 0;
}
words.sort(Comparator.comparingDouble(TextWord::x));
float chars = 0;
float width = 0;
for (TextWord w : words) {
width += w.width();
chars += Math.max(1, w.text().strip().length());
}
float gutter =
Math.max(
ColumnRanges.RULED_GUTTER_FLOOR,
(width / chars) * ColumnRanges.RULED_GUTTER_CHARS);
int groups = 1;
for (int i = 1; i < words.size(); i++) {
float gap = words.get(i).x() - (words.get(i - 1).x() + words.get(i - 1).width());
if (gap >= gutter) {
groups++;
}
}
return groups;
}
/** Line heights within which a line above a ruled grid can be its header row. */
static final float HEADER_RULE_GAP = 2.5f;
/** The nearest line above {@code top} close enough to be the block's header row. */
private static Line headerAbove(List<Line> lines, float top, float medianGap) {
Line best = null;
for (Line l : lines) {
if (l.y <= top || l.y - top > medianGap * HEADER_GAP || l.words().size() < 2) {
continue;
}
if (best == null || l.y < best.y) {
best = l;
}
}
return best;
}
/**
* True when a line has two words separated by a gap far wider than word spacing. The threshold
* comes from the line's own character width, not a document font size.
*/
private static boolean isTableCandidate(List<TextWord> words) {
if (words.size() < 2) {
return false;
}
double totalWidth = 0;
int totalChars = 0;
for (TextWord w : words) {
totalWidth += w.width();
totalChars += Math.max(1, w.text().strip().length());
}
float charWidth = (float) (totalWidth / Math.max(1, totalChars));
// A deliberate cell gap is several blank characters wide; ordinary word spaces are ~a third
// of a character. Floor at 8pt so tiny fonts still need a real gap.
float cellGap = Math.max(8f, charWidth * 3f);
for (int i = 1; i < words.size(); i++) {
TextWord prev = words.get(i - 1);
float gap = words.get(i).x() - (prev.x() + prev.width());
if (gap >= cellGap) {
return true;
}
}
return false;
}
/** True when two blocks overlap vertically, i.e. they describe the same table. */
static boolean covers(TableBlock a, TableBlock b) {
return Math.min(a.top(), b.top()) > Math.max(a.bottom(), b.bottom());
}
}
@@ -0,0 +1,248 @@
package stirling.software.proprietary.pdf;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import stirling.software.jpdfium.text.TextWord;
/**
* Resolves a detected table block into a cell grid and renders it. The false-positive guards live
* here, so every consumer sees the same cells and the same verdict.
*/
final class TableGrid {
private TableGrid() {}
/**
* Renders a table block; {@code ruledColumns} are exact bands from vertical rules, or null to
* project the columns from whitespace instead.
*/
static String render(
List<List<Line>> rowGroups, List<float[]> ruledColumns, RowSource rowSource) {
List<String[]> rows = cells(rowGroups, ruledColumns, rowSource);
return rows.isEmpty() ? "" : GfmTable.render(rows, rows.get(0).length);
}
/**
* Resolves a table block into a cell grid, or empty when it fails the false-positive guards.
*/
static List<String[]> cells(
List<List<Line>> rowGroups, List<float[]> ruledColumns, RowSource rowSource) {
// Columns come from cross-row whitespace projection, not a 1-D gap threshold on pooled word
// x's, which is fragile with right-aligned numbers or sparse cells in their own band.
List<Line> flat = rowGroups.stream().flatMap(List::stream).collect(Collectors.toList());
// Inside a region the rules already declare a table, a narrower gutter still separates
// columns: the wide floor only exists to stop word spacing splitting an unruled block.
List<float[]> columns =
ruledColumns != null
? ruledColumns
: ColumnRanges.find(
flat,
rowSource.ruleConfirmed()
? ColumnRanges.RULED_GUTTER_CHARS
: ColumnRanges.GUTTER_CHARS,
rowSource.ruleConfirmed()
? ColumnRanges.RULED_GUTTER_FLOOR
: ColumnRanges.GUTTER_FLOOR);
// A column only the header occupies is invisible to the projection, which needs a band
// shared by several rows; but inside a ruled region a blank answer column is still one.
boolean headerOnlyColumn = false;
if (columns.size() < 2 && ruledColumns == null && rowSource.ruleConfirmed()) {
List<float[]> retry =
ColumnRanges.find(
flat,
ColumnRanges.RULED_GUTTER_CHARS,
ColumnRanges.RULED_GUTTER_FLOOR,
1);
// Only the worksheet shape: exactly one row, the first, reaches past the supported
// column. Anything else would invent a column and swallow the headings around it.
if (retry.size() >= 2 && retry.size() <= 15) {
float edge = retry.get(0)[1];
int beyond = 0;
int firstBeyond = -1;
for (int r = 0; r < rowGroups.size(); r++) {
boolean out = false;
for (Line l : rowGroups.get(r)) {
for (TextWord w : l.words()) {
if (!w.text().strip().isEmpty() && w.x() + w.width() / 2f > edge) {
out = true;
}
}
}
if (out) {
beyond++;
if (firstBeyond < 0) {
firstBeyond = r;
}
}
}
if (beyond == 1 && firstBeyond == 0 && rowGroups.size() >= 3) {
columns = retry;
headerOnlyColumn = true;
}
}
}
// Only a drawn lattice can be a one-column table; inferred from whitespace it is just a
// run of centred lines.
int minColumns = rowSource == RowSource.LATTICE ? 1 : 2;
if (columns.size() < minColumns || columns.size() > 15) {
return List.of();
}
float[] centers = new float[columns.size()];
for (int i = 0; i < columns.size(); i++) {
centers[i] = (columns.get(i)[0] + columns.get(i)[1]) / 2f;
}
int cols = centers.length;
List<String[]> rows = new ArrayList<>();
for (List<Line> rowLines : rowGroups) {
String[] row = new String[cols];
TextWord[] lastWord = new TextWord[cols];
String[] lastText = new String[cols];
boolean[] boundMark = new boolean[cols];
for (int i = 0; i < cols; i++) {
row[i] = "";
lastText[i] = "";
}
// Top line first so a wrapped cell's words stay in reading order within the cell.
rowLines.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
for (Line line : rowLines) {
for (TextWord word : line.words()) {
String wt = word.text().strip();
if (wt.isEmpty()) {
continue;
}
float mid = word.x() + word.width() / 2f;
// Ruled columns are real boundaries, so a word belongs to the band that
// contains it; projected columns are only approximate centres, so nearest wins.
int col =
ruledColumns != null
? containingColumn(mid, columns)
: nearestColumn(mid, centers);
// A mark that closed up against the word on its left closes up against the
// word on its right too, so Party - List does not settle at "Party- List".
boolean bind =
!row[col].isEmpty()
&& (boundMark[col]
|| (WordGeometry.isBindingMark(wt)
|| WordGeometry.isBindingMark(
lastText[col]))
&& !WordGeometry.separated(
lastWord[col], word));
row[col] = row[col].isEmpty() ? wt : row[col] + (bind ? "" : " ") + wt;
boundMark[col] = bind && WordGeometry.isBindingMark(wt);
lastWord[col] = word;
lastText[col] = wt;
}
}
for (int c = 0; c < cols; c++) {
row[c] = WordGeometry.rejoinContractions(row[c]);
}
rows.add(row);
}
// Guard against false positives while tolerating uneven rows: require an anchor row that
// nearly fills the grid, and that most rows are genuinely multi-column.
if (ruledColumns != null) {
// A rule that is not a column separator (a cell outline, a shading edge) leaves an
// empty column; drop those rather than emitting them across every row.
List<Integer> keep = new ArrayList<>();
for (int c = 0; c < cols; c++) {
final int col = c;
if (rows.stream().anyMatch(r -> !r[col].isEmpty())) {
keep.add(c);
}
}
// Two is the floor whatever the rows say: one filled column means the rules drew a box
// round a single block of text, not a table.
if (keep.size() < 2) {
return List.of();
}
if (keep.size() < cols) {
List<String[]> trimmed = new ArrayList<>(rows.size());
for (String[] r : rows) {
String[] t = new String[keep.size()];
for (int i = 0; i < keep.size(); i++) {
t[i] = r[keep.get(i)];
}
trimmed.add(t);
}
rows = trimmed;
cols = keep.size();
List<float[]> kept = new ArrayList<>(keep.size());
for (int idx : keep) {
kept.add(columns.get(idx));
}
columns = kept;
}
}
if (cols == 1) {
// A one-column table has no cross-row alignment to check, so the evidence is the rules
// plus the shape of the run: enough rows, nearly all carrying text.
long filled = rows.stream().filter(r -> !r[0].isEmpty()).count();
return rows.size() >= SINGLE_COLUMN_ROWS && filled >= rows.size() * SINGLE_COLUMN_FILLED
? rows
: List.of();
}
int anchorWidth = Math.max(2, Math.round(cols * 0.6f));
long anchorRows = rows.stream().filter(r -> filledCells(r) >= anchorWidth).count();
long multiColumnRows = rows.stream().filter(r -> filledCells(r) >= 2).count();
// The multi-column tests ask whether a grid inferred from whitespace is real; when rows
// and columns are both drawn there is nothing to infer, and a blank worksheet would fail.
boolean drawnGrid =
headerOnlyColumn || (ruledColumns != null && rowSource == RowSource.LATTICE);
if (drawnGrid
? anchorRows < 1
: (anchorRows < 1 || multiColumnRows < 2 || multiColumnRows < rows.size() * 0.5)) {
return List.of();
}
if (ruledColumns == null && TableShape.isProseNotTable(rows, cols)) {
return List.of();
}
return rows;
}
/** Rows a single-column ruled table needs before it is a table rather than a run of lines. */
private static final int SINGLE_COLUMN_ROWS = 3;
/** Fraction of a single-column table's rows that must carry text. */
private static final float SINGLE_COLUMN_FILLED = 0.8f;
/** Index of the column band containing x, clamped to the first/last band outside the grid. */
static int containingColumn(float x, List<float[]> columns) {
for (int i = 0; i < columns.size(); i++) {
if (x < columns.get(i)[1]) {
return i;
}
}
return columns.size() - 1;
}
private static int nearestColumn(float x, float[] centers) {
int best = 0;
float bestDist = Float.MAX_VALUE;
for (int i = 0; i < centers.length; i++) {
float d = Math.abs(x - centers[i]);
if (d < bestDist) {
bestDist = d;
best = i;
}
}
return best;
}
static int filledCells(String[] row) {
int count = 0;
for (String cell : row) {
if (!cell.isEmpty()) {
count++;
}
}
return count;
}
}
@@ -0,0 +1,195 @@
package stirling.software.proprietary.pdf;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
/**
* False-positive guards: whether a block is really a table, and whether it is wide enough to
* outrank the page's own column layout.
*/
final class TableShape {
private TableShape() {}
/** Fraction of the page's text width a table must span to override two-column layout. */
private static final float FULL_WIDTH = 0.6f;
/** Rows of a two-column block that must end in a page number for it to be a contents list. */
private static final float TOC_ROWS = 0.65f;
/** Mean filled-cell length above which a two-column block reads as prose, not cells. */
private static final float PROSE_CELL = 40f;
private static final Pattern PAGE_NUMBER = Pattern.compile("[0-9]{1,4}|[ivxlcdmIVXLCDM]{1,7}");
/** A run of spaced or solid dots, the leader of a contents line. */
private static final Pattern DOT_LEADER = Pattern.compile("(\\.\\s*){4,}|…");
/**
* True when a block is running text the word grid mistook for a table: a contents list, or two
* columns of prose whose cells are whole sentences.
*/
static boolean isProseNotTable(List<String[]> rows, int cols) {
if (rows.isEmpty()) {
return false;
}
for (String[] row : rows) {
for (String cell : row) {
if (DOT_LEADER.matcher(cell).find()) {
return true;
}
}
}
if (cols != 2) {
return everyColumnIsProse(rows, cols);
}
int folios = 0;
int length = 0;
int filled = 0;
for (String[] row : rows) {
String last = "";
for (String cell : row) {
if (!cell.isEmpty()) {
length += cell.length();
filled++;
last = cell;
}
}
if (PAGE_NUMBER.matcher(last).matches() && !PAGE_NUMBER.matcher(row[0]).matches()) {
folios++;
}
}
if (folios >= rows.size() * TOC_ROWS) {
return true;
}
return filled > 0 && (float) length / filled >= PROSE_CELL;
}
/** Mean cell length at or above which a column carries sentences rather than values. */
private static final float PROSE_COLUMN = 20f;
/** Fraction of neighbouring cells that must continue each other's sentence to read as prose. */
private static final float PROSE_RUN_ON = 0.5f;
/** A cell that ends a sentence or clause, so the cell after it starts something new. */
private static final Pattern CELL_ENDS_CLAUSE = Pattern.compile("[.!?:;,]$");
/**
* True when a wide block is multi-column prose read across, not a table: no column keys the
* rows, and the cells continue each other's sentences.
*/
static boolean everyColumnIsProse(List<String[]> rows, int cols) {
if (cols < 3) {
return false;
}
for (int c = 0; c < cols; c++) {
int length = 0;
int filled = 0;
for (String[] row : rows) {
if (c < row.length && !row[c].isEmpty()) {
length += row[c].length();
filled++;
}
}
if (filled == 0 || (float) length / filled < PROSE_COLUMN) {
return false;
}
}
return runsOnAcrossCells(rows);
}
/**
* Fraction of side-by-side filled cells where the right one continues the left one's clause.
*/
private static boolean runsOnAcrossCells(List<String[]> rows) {
int pairs = 0;
int runOn = 0;
for (String[] row : rows) {
String previous = null;
for (String cell : row) {
if (cell.isEmpty()) {
continue;
}
if (previous != null) {
pairs++;
if (!CELL_ENDS_CLAUSE.matcher(previous).find()
&& Character.isLowerCase(cell.charAt(0))) {
runOn++;
}
}
previous = cell;
}
}
return pairs > 0 && (float) runOn / pairs > PROSE_RUN_ON;
}
/**
* True when no text outside the block sits in its vertical band, so it cannot be one column of
* a two-column layout.
*/
static boolean ownsItsBand(TableBlock block, List<Line> lines) {
Set<Line> own = new HashSet<>();
for (List<Line> row : block.rows()) {
own.addAll(row);
}
for (Line l : lines) {
if (own.contains(l)) {
continue;
}
float centre = l.y + l.height / 2f;
if (centre > block.bottom() && centre < block.top()) {
return false;
}
}
return true;
}
/** Columns a full-width unruled block needs before it can outrank the page's column layout. */
private static final int GRID_COLUMNS = 3;
/** Mean filled-cell length above which a full-width unruled block is prose read across. */
private static final float GRID_CELL = 25f;
/**
* True when an unruled full-width block is really a table: a data table's cells are short
* values, a page gutter's are sentences.
*/
static boolean looksLikeGrid(TableBlock block) {
List<String[]> cells = block.cells();
if (cells.isEmpty() || cells.get(0).length < GRID_COLUMNS) {
return false;
}
int length = 0;
int filled = 0;
for (String[] row : cells) {
for (String cell : row) {
if (!cell.isEmpty()) {
length += cell.length();
filled++;
}
}
}
return filled > 0 && (float) length / filled <= GRID_CELL;
}
/** True when a table block is wide enough to be a full-width table, not one inside a column. */
static boolean spansPage(TableBlock block, List<Line> lines) {
float pageLo = Float.MAX_VALUE;
float pageHi = -Float.MAX_VALUE;
for (Line l : lines) {
pageLo = Math.min(pageLo, l.x);
pageHi = Math.max(pageHi, l.x + l.width);
}
float lo = Float.MAX_VALUE;
float hi = -Float.MAX_VALUE;
for (List<Line> row : block.rows()) {
for (Line l : row) {
lo = Math.min(lo, l.x);
hi = Math.max(hi, l.x + l.width);
}
}
return pageHi > pageLo && (hi - lo) >= (pageHi - pageLo) * FULL_WIDTH;
}
}
@@ -0,0 +1,106 @@
package stirling.software.proprietary.pdf;
import java.util.List;
import java.util.regex.Pattern;
import stirling.software.jpdfium.text.TextChar;
import stirling.software.jpdfium.text.TextWord;
/**
* Word-level geometry and spacing. PDFium splits words on its own bounding boxes, so both real
* spaces and edges are re-derived from the glyphs.
*/
final class WordGeometry {
private WordGeometry() {}
/** Gap below this many average character widths reads as no space at all (mid-word split). */
static final float NO_SPACE_GAP = 0.30f;
/**
* Punctuation that binds to the words on both sides. Closing words up is only considered around
* one of these, because dropping a real space corrupts the text.
*/
private static final String BINDING_MARKS = "'’ʼ´`-‐‑";
/** True for a lone apostrophe or hyphen, as in {@code firm}, {@code '}, {@code s}. */
static boolean isBindingMark(String word) {
return word.length() == 1 && BINDING_MARKS.indexOf(word.charAt(0)) >= 0;
}
/**
* A contraction whose apostrophe the extractor padded on both sides. English suffixes only: a
* spaced lone apostrophe is an opening quote.
*/
private static final Pattern SPLIT_CONTRACTION =
Pattern.compile("(\\p{L})\\s*(['ʼ´`])\\s*(s|t|d|m|re|ve|ll)\\b");
/** Closes up an apostrophe the extractor left standing alone inside a cell. */
static String rejoinContractions(String cell) {
return cell.indexOf(' ') < 0 ? cell : SPLIT_CONTRACTION.matcher(cell).replaceAll("$1$2$3");
}
/**
* True when two words of a cell are far enough apart to be separated by a space; punctuation
* set tight against its neighbour arrives as its own word.
*/
static boolean separated(TextWord previous, TextWord current) {
if (previous == null) {
return true;
}
float gap = leftEdge(current) - rightEdge(previous);
if (gap < 0f) {
// Overlapping or out of order (a second line of a wrapped cell): keep the space.
return true;
}
float charWidth = wordCharWidth(previous, current);
return charWidth <= 0f || gap >= charWidth * NO_SPACE_GAP;
}
/** Mean glyph width across two words, used to size the space test above. */
private static float wordCharWidth(TextWord a, TextWord b) {
float width = 0f;
int chars = 0;
for (TextWord w : List.of(a, b)) {
for (TextChar c : w.chars()) {
if (!c.isWhitespace() && !c.isNewline()) {
width += c.width();
chars++;
}
}
}
return chars == 0 ? 0f : width / chars;
}
static float rightEdge(TextWord w) {
float edge = -Float.MAX_VALUE;
for (TextChar c : w.chars()) {
if (!c.isWhitespace() && !c.isNewline()) {
edge = Math.max(edge, c.x() + c.width());
}
}
return edge == -Float.MAX_VALUE ? w.x() + w.width() : edge;
}
static float leftEdge(TextWord w) {
float edge = Float.MAX_VALUE;
for (TextChar c : w.chars()) {
if (!c.isWhitespace() && !c.isNewline()) {
edge = Math.min(edge, c.x());
}
}
return edge == Float.MAX_VALUE ? w.x() : edge;
}
static float averageCharWidth(List<Line> rows) {
double totalWidth = 0;
int totalChars = 0;
for (Line l : rows) {
for (TextWord w : l.words()) {
totalWidth += w.width();
totalChars += Math.max(1, w.text().strip().length());
}
}
return totalChars == 0 ? 6f : (float) (totalWidth / totalChars);
}
}
@@ -0,0 +1,423 @@
package stirling.software.proprietary.pdf;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import stirling.software.jpdfium.PdfDocument;
import stirling.software.jpdfium.text.TextChar;
import stirling.software.jpdfium.text.TextLine;
import stirling.software.jpdfium.text.TextWord;
/**
* Accuracy and robustness tests comparing output against hand-authored golden Markdown. {@link
* #gatedFixtures()} gates CI; {@link #wipFixtures()} is disabled.
*/
class AdvancedPdfMarkdownConverterTest {
/** Accuracy threshold: output must share at least this fraction of content with the golden. */
private static final double THRESHOLD = 0.95;
@TempDir Path tmp;
/** Fixtures that meet the accuracy threshold today and therefore gate CI. */
static Stream<Arguments> gatedFixtures() {
return Stream.of(
Arguments.of("multi-column-test_lorem.pdf", "multi-column-test_lorem.md"),
Arguments.of("bordered-table-test_widget.pdf", "bordered-table-test_widget.md"),
Arguments.of("many-tables-test_stress.pdf", "many-tables-test_stress.md"));
}
/** Fixtures still below the threshold; tracked here, enable locally to iterate. */
static Stream<Arguments> wipFixtures() {
return Stream.of(
Arguments.of(
"wrapped-cell-test_expense-report.pdf",
"wrapped-cell-test_expense-report.md"));
}
@ParameterizedTest(name = "{0}")
@MethodSource("gatedFixtures")
void convertMatchesGoldenMarkdown(String pdfName, String mdName) throws IOException {
assertConversionMatchesGolden(pdfName, mdName);
}
@Disabled("WIP fixtures below the accuracy threshold; enable locally to iterate")
@ParameterizedTest(name = "{0}")
@MethodSource("wipFixtures")
void convertMatchesGoldenMarkdownWip(String pdfName, String mdName) throws IOException {
assertConversionMatchesGolden(pdfName, mdName);
}
/**
* Degenerate geometry must not crash the converter: a text matrix can place a word past {@link
* Integer#MAX_VALUE}, which used to size an {@code int[]} from the span.
*/
@Test
void columnDetectionSurvivesDegenerateGeometry() {
// x ≈ 2.5e9 is past Integer.MAX_VALUE; combined with a near-origin word it yields an
// implausible span that the pre-fix code turned into a fatal array allocation.
List<TextLine> rows = new ArrayList<>();
for (int r = 0; r < 4; r++) {
float y = 400f - r * 12f;
TextWord near = new TextWord(List.of(), 50f, y, 30f, 10f);
TextWord far = new TextWord(List.of(), 2_500_000_000f, y, 30f, 10f);
rows.add(new TextLine(List.of(near, far), 50f, y, 2_499_999_980f, 10f));
}
List<float[]> columns = assertDoesNotThrow(() -> ColumnRanges.fromTextLines(rows));
assertTrue(
columns.isEmpty(),
"implausible page span should disable column detection, not allocate from it");
}
@Test
@Timeout(20)
void gutterScanTerminatesOnCoordinatesBeyondFloatPrecision() {
// Past 2^24 a float cannot represent x + 1, so a float-stepped scan over a crafted text
// matrix stops advancing and spins forever - wedging the process-wide jpdfium lock with it.
List<TextLine> rows = new ArrayList<>();
for (int r = 0; r < 10; r++) {
float y = 400f - r * 12f;
float x = 20_000_000f;
TextWord w = new TextWord(List.of(), x, y, 200f, 10f);
rows.add(new TextLine(List.of(w), x, y, 200f, 10f));
}
List<Float> gutters = assertDoesNotThrow(() -> ColumnLayout.guttersFromTextLines(rows));
assertTrue(
gutters.isEmpty(),
"every candidate band is crossed by every line, so no gutter is found");
}
/**
* Three-column prose aligns across rows exactly as cells do. What tells them apart is a keying
* column of short values and cells that do not run on.
*/
@Test
void multiColumnProseIsNotATable() {
List<String[]> prose =
List.of(
new String[] {
"The SS Pack can reduce the information acquisition time by",
"returning all the information that matches",
"the user's search intent and the query behind it"
},
new String[] {
"Unlike existing search systems that only return information",
"limited to the entered search keywords, this pack",
"returns all relevant data meeting the search intent"
});
assertTrue(
TableShape.everyColumnIsProse(prose, 3),
"three columns of running sentences are a page layout, not a table");
}
@Test
void wideTableWithLongCellsStaysATable() {
// The prose test must not fire on a real table just because one column runs long: the
// short "Jurisdiction" and yes/no columns are what key the rows.
List<String[]> table =
List.of(
new String[] {
"Argentina",
"Y",
"Prohibition on ownership of property that contains or borders water"
},
new String[] {
"Australia",
"N",
"Approval is needed from the Treasurer if the acquisition is large"
});
assertTrue(!TableShape.everyColumnIsProse(table, 3), "a keyed table is a table");
}
@Test
void splitApostropheIsClosedUpInCells() {
// PDFium splits on its own bounding boxes, so a tight apostrophe arrives as its own word.
assertEquals("the firm's returns", WordGeometry.rejoinContractions("the firm ' s returns"));
assertEquals("Dont know", WordGeometry.rejoinContractions("Don t know"));
// An opening quote has real space around it and must keep it.
assertEquals("he said ' hello", WordGeometry.rejoinContractions("he said ' hello"));
}
@Test
void headingLevelsAreRebasedOnTheStrongestHeadingPresent() {
// A document whose headings are body-size and bold scores every one of them level 3;
// relative to each other they are its top level, so they must render as level 1.
assertEquals("# CONTENTS\n", MarkdownText.normaliseHeadingLevels("### CONTENTS\n"));
// A real two-level document keeps two levels, with no gap between them.
assertEquals(
"# Title\n\ntext\n\n## Section\n",
MarkdownText.normaliseHeadingLevels("# Title\n\ntext\n\n### Section\n"));
// Already rooted at level 1 with no gaps: left alone.
String unchanged = "# Title\n\n## Section\n";
assertEquals(unchanged, MarkdownText.normaliseHeadingLevels(unchanged));
}
/** A line of Han text has no spaces, so the word-count heading guard cannot measure it. */
private static List<TextWord> cjkWords(String text, String font) {
List<TextChar> chars = new ArrayList<>(text.length());
for (int i = 0; i < text.length(); i++) {
chars.add(new TextChar(i, text.charAt(i), 50f + i * 12f, 400f, 12f, 12f, font, 12f));
}
return List.of(new TextWord(chars, 50f, 400f, text.length() * 12f, 12f));
}
@Test
void boldCjkParagraphIsNotPromotedToAHeading() {
String paragraph =
"\u672c\u898f\u7d04\u306f\u3001\u5f53\u793e\u304c\u63d0\u4f9b\u3059\u308b"
+ "\u672c\u30b5\u30fc\u30d3\u30b9\u306e\u5229\u7528\u6761\u4ef6\u3092"
+ "\u5b9a\u3081\u308b\u3082\u306e\u3067\u3042\u308a\u3001\u5229\u7528"
+ "\u8005\u306e\u7686\u3055\u307e\u306b\u306f\u672c\u898f\u7d04\u306b"
+ "\u5f93\u3063\u3066\u3054\u5229\u7528\u3044\u305f\u3060\u304d\u307e\u3059\u3002";
List<TextWord> words = cjkWords(paragraph, "NotoSansCJKjp-Bold");
assertEquals(
"",
HeadingDetector.headingPrefix(
paragraph, 12f, words, 12f, 12f, "NotoSansCJKjp-Regular", true),
"a bold paragraph in a script with no word spaces is body text, not a heading");
assertFalse(
HeadingDetector.isBoldLabel(paragraph, words),
"a paragraph ending in an ideographic stop is a sentence, not a bold label");
// The guard must not cost the short headings it is meant to keep.
String heading = "\u7b2c\u4e09\u7ae0 \u5b9f\u88c5\u306e\u6982\u8981";
assertEquals(
"### ",
HeadingDetector.headingPrefix(
heading,
12f,
cjkWords(heading, "NotoSansCJKjp-Bold"),
12f,
12f,
"NotoSansCJKjp-Regular",
true),
"a short isolated bold CJK line is still a heading");
}
/**
* A crafted PDF can draw thousands of disjoint rules; partitioning used to cost O(N^2) retained
* memory, so it must stay linear and bounded.
*/
@Test
@Timeout(20)
void ruledTablePartitionSurvivesPathologicalGrid() {
// 4000 rules that never cross, so every one is its own component: the shape that made the
// old code allocate 4000 arrays of 4001 ints, kept under the crossing-test budget.
List<PageRules.Rule> horizontal = new ArrayList<>();
List<PageRules.Rule> vertical = new ArrayList<>();
for (int i = 0; i < 2_000; i++) {
horizontal.add(new PageRules.Rule(i * 10f, 0f, 20f));
vertical.add(new PageRules.Rule(1_000_000f + i * 10f, -50f, -30f));
}
int components = assertDoesNotThrow(() -> RuleGrid.componentCount(horizontal, vertical));
// 4000 disjoint rules would be 4000 components; the cap is what keeps this bounded.
assertEquals(256, components, "component count must stay bounded");
}
@Test
@Timeout(20)
void ruledTablePartitionBailsOutOnOperatorFlood() {
// Enough levels that the pairwise crossing scan alone would dominate the request.
List<PageRules.Rule> horizontal = new ArrayList<>();
List<PageRules.Rule> vertical = new ArrayList<>();
for (int i = 0; i < 20_000; i++) {
horizontal.add(new PageRules.Rule(i * 10f, 0f, 20f));
vertical.add(new PageRules.Rule(1_000_000f + i * 10f, -50f, -30f));
}
assertTrue(
RuleGrid.componentCount(horizontal, vertical) == 0,
"a rule flood should disable ruled-table detection, not scan it");
}
private void assertConversionMatchesGolden(String pdfName, String mdName) throws IOException {
Path pdfPath = tmp.resolve(pdfName);
try (InputStream in =
getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + pdfName)) {
if (in == null) {
fail("Fixture not found on classpath: /pdf-ingestion-fixtures/" + pdfName);
}
Files.copy(in, pdfPath);
}
String actual;
try (PdfDocument doc = PdfDocument.open(pdfPath)) {
actual = new AdvancedPdfMarkdownConverter().convert(doc);
}
String expected;
try (InputStream in = getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + mdName)) {
if (in == null) {
fail("Golden file not found on classpath: /pdf-ingestion-fixtures/" + mdName);
}
expected = new String(in.readAllBytes(), StandardCharsets.UTF_8);
}
// Image placeholders are not scored: their body text is a TODO rather than real content, so
// comparing it would penalise output for matching a placeholder we intend to replace.
expected = stripImagePlaceholders(expected);
actual = stripImagePlaceholders(actual);
double similarity = similarity(expected, actual);
if (similarity < THRESHOLD) {
fail(
String.format(
"Markdown output differs from golden file '%s' by %.1f%% (threshold %.0f%%):%n%s",
mdName,
(1.0 - similarity) * 100,
(1.0 - THRESHOLD) * 100,
unifiedDiff(expected, actual)));
}
}
/** Substring identifying an image-placeholder line, which is excluded from scoring. */
private static final String IMAGE_PLACEHOLDER_MARKER = "Image intentionally redacted";
/**
* Removes non-content lines: image placeholders, and GFM separator rows whose exact dash count
* is cosmetic.
*/
private static String stripImagePlaceholders(String md) {
StringBuilder sb = new StringBuilder();
for (String line : md.split("\n", -1)) {
if (line.contains(IMAGE_PLACEHOLDER_MARKER)
|| line.strip().startsWith("<image redacted")
|| isTableSeparatorRow(line)) {
continue;
}
if (!sb.isEmpty()) {
sb.append('\n');
}
sb.append(line);
}
return sb.toString();
}
/** True for a GFM table separator row, e.g. {@code |---|:--:|---|} (only |, -, :, space). */
private static boolean isTableSeparatorRow(String line) {
String t = line.strip();
if (!t.contains("-")) {
return false;
}
return t.chars().allMatch(c -> c == '|' || c == '-' || c == ':' || c == ' ');
}
/**
* Character-level similarity: the fraction of expected characters in the LCS. O(n*m), fine for
* small goldens.
*/
private static double similarity(String expected, String actual) {
if (expected.isEmpty() && actual.isEmpty()) return 1.0;
if (expected.isEmpty() || actual.isEmpty()) return 0.0;
// Strip all whitespace for a content-focused comparison
String e = expected.replaceAll("\\s+", " ").strip();
String a = actual.replaceAll("\\s+", " ").strip();
int lcs = lcsLength(e, a);
return (double) lcs / Math.max(e.length(), a.length());
}
private static int lcsLength(String a, String b) {
// Use two-row DP to keep memory reasonable
int m = a.length(), n = b.length();
int[] prev = new int[n + 1];
int[] curr = new int[n + 1];
for (int i = 1; i <= m; i++) {
for (int j = 1; j <= n; j++) {
if (a.charAt(i - 1) == b.charAt(j - 1)) {
curr[j] = prev[j - 1] + 1;
} else {
curr[j] = Math.max(curr[j - 1], prev[j]);
}
}
int[] tmp = prev;
prev = curr;
curr = tmp;
java.util.Arrays.fill(curr, 0);
}
return prev[n];
}
private static String unifiedDiff(String expected, String actual) {
String[] expectedLines = expected.split("\n", -1);
String[] actualLines = actual.split("\n", -1);
List<String> diff = new ArrayList<>();
diff.add("--- expected");
diff.add("+++ actual");
int maxLines = Math.max(expectedLines.length, actualLines.length);
int context = 3;
boolean inHunk = false;
int hunkStart = -1;
List<String> hunkLines = new ArrayList<>();
for (int i = 0; i < maxLines; i++) {
String exp = i < expectedLines.length ? expectedLines[i] : null;
String act = i < actualLines.length ? actualLines[i] : null;
boolean changed = exp == null || act == null || !exp.equals(act);
if (changed) {
if (!inHunk) {
inHunk = true;
hunkStart = Math.max(0, i - context);
// add context lines before change
for (int c = hunkStart; c < i; c++) {
hunkLines.add(" " + (c < expectedLines.length ? expectedLines[c] : ""));
}
}
if (exp != null) hunkLines.add("-" + exp);
if (act != null) hunkLines.add("+" + act);
} else {
if (inHunk) {
hunkLines.add(" " + exp);
// check if we're far enough past the last change to close the hunk
boolean moreChanges = false;
for (int j = i + 1; j < Math.min(i + context, maxLines); j++) {
String e2 = j < expectedLines.length ? expectedLines[j] : null;
String a2 = j < actualLines.length ? actualLines[j] : null;
if (e2 == null || a2 == null || !e2.equals(a2)) {
moreChanges = true;
break;
}
}
if (!moreChanges && (i - hunkStart) >= context) {
diff.add("@@ -" + (hunkStart + 1) + " @@");
diff.addAll(hunkLines);
hunkLines.clear();
inHunk = false;
}
}
}
}
if (inHunk && !hunkLines.isEmpty()) {
diff.add("@@ -" + (hunkStart + 1) + " @@");
diff.addAll(hunkLines);
}
return String.join("\n", diff);
}
}