mirror of
https://github.com/Stirling-Tools/Stirling-PDF.git
synced 2026-09-03 05:10:16 +03:00
Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b19e85698e | ||
|
|
f097450883 | ||
|
|
941051648e | ||
|
|
33a22db712 | ||
|
|
1e36895561 | ||
|
|
ea05a3f576 | ||
|
|
1c3d216e70 | ||
|
|
7159b1d179 | ||
|
|
8f0b3a6d94 | ||
|
|
8b1a9c8902 | ||
|
|
c35b252092 | ||
|
|
ebe216ee8d | ||
|
|
84af8476bc | ||
|
|
7e0ef7cfae | ||
|
|
ffc9052961 | ||
|
|
532e2a675a | ||
|
|
72c67520ef | ||
|
|
8b1c045349 | ||
|
|
dc70c0b821 | ||
|
|
6f4a951291 | ||
|
|
4dac016b64 |
@@ -0,0 +1,17 @@
|
||||
package stirling.software.common.pdf;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
|
||||
/** Built-in Markdown conversion, used whenever no richer extractor is registered. */
|
||||
@Service
|
||||
public class BasicPdfMarkdownExtractor implements PdfMarkdownExtractor {
|
||||
|
||||
@Override
|
||||
public String convert(PdfDocument doc) throws IOException {
|
||||
return new PdfMarkdownConverter().convert(doc);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package stirling.software.common.pdf;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
|
||||
/**
|
||||
* Seam for PDF to Markdown conversion. The proprietary module supplies a layout-aware
|
||||
* implementation that takes precedence on the classpath.
|
||||
*/
|
||||
public interface PdfMarkdownExtractor {
|
||||
|
||||
String convert(PdfDocument doc) throws IOException;
|
||||
}
|
||||
+3
-2
@@ -19,7 +19,7 @@ import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
import stirling.software.common.model.tool.ToolFormat;
|
||||
import stirling.software.common.model.tool.ToolIO;
|
||||
import stirling.software.common.pdf.PdfMarkdownConverter;
|
||||
import stirling.software.common.pdf.PdfMarkdownExtractor;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
@@ -30,6 +30,7 @@ import stirling.software.jpdfium.PdfDocument;
|
||||
public class ConvertPDFToMarkdown {
|
||||
|
||||
private final TempFileManager tempFileManager;
|
||||
private final PdfMarkdownExtractor markdownExtractor;
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
@@ -54,7 +55,7 @@ public class ConvertPDFToMarkdown {
|
||||
try (TempFile tempInput = new TempFile(tempFileManager, ".pdf")) {
|
||||
inputFile.transferTo(tempInput.getFile());
|
||||
try (PdfDocument doc = PdfDocument.open(tempInput.getPath())) {
|
||||
markdown = new PdfMarkdownConverter().convert(doc);
|
||||
markdown = markdownExtractor.convert(doc);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+13
-17
@@ -23,14 +23,14 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||
|
||||
import stirling.software.common.pdf.PdfMarkdownConverter;
|
||||
import stirling.software.common.pdf.PdfMarkdownExtractor;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
|
||||
class ConvertPDFToMarkdownTest {
|
||||
|
||||
private MockMvc mockMvc() {
|
||||
return MockMvcBuilders.standaloneSetup(new ConvertPDFToMarkdown(null))
|
||||
private MockMvc mockMvc(PdfMarkdownExtractor extractor) {
|
||||
return MockMvcBuilders.standaloneSetup(new ConvertPDFToMarkdown(null, extractor))
|
||||
.setControllerAdvice(new GlobalErrorHandler())
|
||||
.build();
|
||||
}
|
||||
@@ -61,11 +61,10 @@ class ConvertPDFToMarkdownTest {
|
||||
when(mock.getFile()).thenReturn(tmpFile);
|
||||
when(mock.getPath()).thenReturn(tmpFile.toPath());
|
||||
});
|
||||
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class);
|
||||
MockedConstruction<PdfMarkdownConverter> converterMock =
|
||||
Mockito.mockConstruction(
|
||||
PdfMarkdownConverter.class,
|
||||
(mock, ctx) -> when(mock.convert(any())).thenReturn(expectedMd))) {
|
||||
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class)) {
|
||||
|
||||
PdfMarkdownExtractor extractor = Mockito.mock(PdfMarkdownExtractor.class);
|
||||
when(extractor.convert(any())).thenReturn(expectedMd);
|
||||
|
||||
PdfDocument mockDoc = Mockito.mock(PdfDocument.class);
|
||||
docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc);
|
||||
@@ -74,7 +73,7 @@ class ConvertPDFToMarkdownTest {
|
||||
new MockMultipartFile(
|
||||
"fileInput", "input.pdf", "application/pdf", new byte[] {1, 2, 3});
|
||||
|
||||
mockMvc()
|
||||
mockMvc(extractor)
|
||||
.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(header().string("Content-Type", "text/markdown"))
|
||||
@@ -94,13 +93,10 @@ class ConvertPDFToMarkdownTest {
|
||||
when(mock.getFile()).thenReturn(tmpFile);
|
||||
when(mock.getPath()).thenReturn(tmpFile.toPath());
|
||||
});
|
||||
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class);
|
||||
MockedConstruction<PdfMarkdownConverter> converterMock =
|
||||
Mockito.mockConstruction(
|
||||
PdfMarkdownConverter.class,
|
||||
(mock, ctx) ->
|
||||
when(mock.convert(any()))
|
||||
.thenThrow(new RuntimeException("boom")))) {
|
||||
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class)) {
|
||||
|
||||
PdfMarkdownExtractor extractor = Mockito.mock(PdfMarkdownExtractor.class);
|
||||
when(extractor.convert(any())).thenThrow(new RuntimeException("boom"));
|
||||
|
||||
PdfDocument mockDoc = Mockito.mock(PdfDocument.class);
|
||||
docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc);
|
||||
@@ -109,7 +105,7 @@ class ConvertPDFToMarkdownTest {
|
||||
new MockMultipartFile(
|
||||
"fileInput", "x.pdf", "application/pdf", new byte[] {0x01});
|
||||
|
||||
mockMvc()
|
||||
mockMvc(extractor)
|
||||
.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
|
||||
.andExpect(status().isInternalServerError());
|
||||
}
|
||||
|
||||
@@ -98,6 +98,14 @@ dependencies {
|
||||
testImplementation "org.testcontainers:junit-jupiter:${testcontainersMinioVersion}"
|
||||
}
|
||||
|
||||
// Golden extraction fixtures live in :common; the advanced converter is graded against the same
|
||||
// corpus as the built-in one rather than a forked copy of it.
|
||||
processTestResources {
|
||||
from(project(':common').file('src/test/resources/pdf-ingestion-fixtures')) {
|
||||
into 'pdf-ingestion-fixtures'
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register('prepareKotlinBuildScriptModel') {}
|
||||
|
||||
tasks.register('type3SignatureTool', JavaExec) {
|
||||
|
||||
+269
@@ -0,0 +1,269 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.pdf.PdfMarkdownExtractor;
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.PdfPage;
|
||||
import stirling.software.jpdfium.text.PageText;
|
||||
import stirling.software.jpdfium.text.PdfTextExtractor;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
|
||||
/**
|
||||
* Converts a PDF to Markdown from PDFium {@link TextLine}s. Orchestration only: each stage of the
|
||||
* pipeline lives in its own class in this package.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@Primary
|
||||
public class AdvancedPdfMarkdownConverter implements PdfMarkdownExtractor {
|
||||
|
||||
@Override
|
||||
public String convert(PdfDocument doc) throws IOException {
|
||||
List<String> rendered = new ArrayList<>();
|
||||
for (Object e : buildElements(doc)) {
|
||||
rendered.add(e instanceof TableBlock tb ? tb.render() : (String) e);
|
||||
}
|
||||
return MarkdownText.normaliseHeadingLevels(String.join("\n\n", rendered));
|
||||
}
|
||||
|
||||
private List<Object> buildElements(PdfDocument doc) throws IOException {
|
||||
List<PageText> allPageText = PdfTextExtractor.extractAll(doc);
|
||||
float medianSize = HeadingDetector.medianFontSize(allPageText);
|
||||
float medianHeight = HeadingDetector.medianLineHeight(allPageText);
|
||||
String bodyFont = HeadingDetector.bodyFont(allPageText);
|
||||
|
||||
int pageCount = doc.pageCount();
|
||||
// Tables stay structured until after the page loop so one split across a page break can
|
||||
// be stitched back together before rendering.
|
||||
List<Object> output = new ArrayList<>();
|
||||
// Header of a table that ended the previous page, for spotting a continuation; null if
|
||||
// none.
|
||||
String prevPageTrailingTableHeader = null;
|
||||
|
||||
for (int pageIndex = 0; pageIndex < pageCount; pageIndex++) {
|
||||
PageLines page = pageLines(doc, allPageText, pageIndex);
|
||||
if (page.lines().isEmpty()) {
|
||||
PageImages.emit(doc, pageIndex, output);
|
||||
prevPageTrailingTableHeader = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
List<Object> pageItems =
|
||||
buildPageItems(
|
||||
doc,
|
||||
page,
|
||||
pageIndex,
|
||||
medianSize,
|
||||
medianHeight,
|
||||
bodyFont,
|
||||
prevPageTrailingTableHeader);
|
||||
if (pageItems.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
PageStitcher.mergeAcrossPageBoundary(output, pageItems);
|
||||
output.addAll(pageItems);
|
||||
prevPageTrailingTableHeader = PageStitcher.trailingTableHeader(pageItems);
|
||||
}
|
||||
|
||||
// Stitch tables split across page breaks; callers decide how to realise the elements.
|
||||
return PageStitcher.stitchTables(output);
|
||||
}
|
||||
|
||||
/**
|
||||
* One page's lines plus its layout verdict, which must be taken before text repair: merging
|
||||
* reduces the line count the two-column guard scales with.
|
||||
*/
|
||||
private record PageLines(List<Line> lines, List<Float> gutters) {
|
||||
boolean twoColumnLayout() {
|
||||
return !gutters.isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
/** Assembled lines for one page, sorted top-to-bottom (PDF y=0 is the page bottom). */
|
||||
private static PageLines pageLines(PdfDocument doc, List<PageText> allPageText, int pageIndex) {
|
||||
List<TextLine> rawLines =
|
||||
pageIndex < allPageText.size() ? allPageText.get(pageIndex).lines() : List.of();
|
||||
List<Line> stitched = GlyphStitcher.stitchGlyphs(rawLines);
|
||||
List<Float> gutters = ColumnLayout.detectGutters(stitched);
|
||||
List<Line> lines = LineMerger.mergeLineFragments(stitched, gutters);
|
||||
lines.addAll(FormValues.lines(doc, pageIndex, lines));
|
||||
lines.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
|
||||
return new PageLines(lines, gutters);
|
||||
}
|
||||
|
||||
/**
|
||||
* One page's elements: paragraph strings interleaved with {@link TableBlock}s in reading order.
|
||||
* {@code continuationHeader} is the previous page's trailing table header, or null.
|
||||
*/
|
||||
private List<Object> buildPageItems(
|
||||
PdfDocument doc,
|
||||
PageLines page,
|
||||
int pageIndex,
|
||||
float medianSize,
|
||||
float medianHeight,
|
||||
String bodyFont,
|
||||
String continuationHeader)
|
||||
throws IOException {
|
||||
List<Line> lines = page.lines();
|
||||
// Only genuine two-column prose is split: a table's column gutters must not read as a page
|
||||
// gutter, and a table continuing from the previous page is not a new two-column layout.
|
||||
boolean tableContinuation =
|
||||
continuationHeader != null
|
||||
&& lines.stream()
|
||||
.anyMatch(
|
||||
l ->
|
||||
MarkdownText.normaliseSpace(l.text)
|
||||
.equals(continuationHeader));
|
||||
|
||||
// Merging widens lines, so re-check the pre-repair verdict here: ordering by a gutter the
|
||||
// finished lines no longer respect is worse than not splitting at all.
|
||||
List<Float> gutters = tableContinuation ? List.of() : page.gutters();
|
||||
boolean twoColumn = !gutters.isEmpty();
|
||||
boolean respected = twoColumn && ColumnLayout.gutterRespected(lines, gutters);
|
||||
|
||||
// Two detectors: ruling lines give exact boundaries and see single-word cells; the word
|
||||
// grid covers what the rules do not, i.e. borderless and whitespace-aligned tables.
|
||||
Set<String> tableRowTexts = new HashSet<>();
|
||||
PageRules rules = readRules(doc, pageIndex);
|
||||
List<TableBlock> blocks = TableFinder.find(lines, rules, pageIndex + 1);
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug(
|
||||
"p{} lines={} hRules={} vRules={} twoColumn={} blocks={}",
|
||||
pageIndex,
|
||||
lines.size(),
|
||||
rules.horizontal().size(),
|
||||
rules.vertical().size(),
|
||||
!gutters.isEmpty(),
|
||||
blocks.size());
|
||||
for (TableBlock b : blocks) {
|
||||
List<String[]> cs = b.cells();
|
||||
log.debug(
|
||||
"block top={} bot={} ruled={} src={} rows={} cells={}x{} "
|
||||
+ "grid={} spans={} owns={}",
|
||||
b.top(),
|
||||
b.bottom(),
|
||||
b.ruled(),
|
||||
b.rowSource(),
|
||||
b.rows().size(),
|
||||
cs.size(),
|
||||
cs.isEmpty() ? 0 : cs.get(0).length,
|
||||
TableShape.looksLikeGrid(b),
|
||||
TableShape.spansPage(b, lines),
|
||||
TableShape.ownsItsBand(b, lines));
|
||||
for (List<Line> row : b.rows()) {
|
||||
log.debug(" row: {}", PageStitcher.rowText(row));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (twoColumn) {
|
||||
// On a multi-column page only a full-width block is a table; anything narrower sits
|
||||
// inside a column. A ruled block owning its own band has no column layout to sit in.
|
||||
blocks =
|
||||
blocks.stream()
|
||||
.filter(
|
||||
b ->
|
||||
(b.ruled() || TableShape.looksLikeGrid(b))
|
||||
&& (TableShape.spansPage(b, lines)
|
||||
|| (b.ruled()
|
||||
&& TableShape.ownsItsBand(
|
||||
b, lines))))
|
||||
.toList();
|
||||
}
|
||||
Set<Line> tableLines = new HashSet<>();
|
||||
for (TableBlock b : blocks) {
|
||||
for (List<Line> row : b.rows()) {
|
||||
for (Line l : row) {
|
||||
tableLines.add(l);
|
||||
tableRowTexts.add(l.text.strip());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<Object> pageItems = new ArrayList<>();
|
||||
List<List<Line>> segments = segmentsAround(lines, blocks, tableLines);
|
||||
if (twoColumn) {
|
||||
// A full-width table interrupts both columns, so splitting at its own vertical band
|
||||
// keeps the prose above and below it in column order.
|
||||
for (int s = 0; s < segments.size(); s++) {
|
||||
List<List<Line>> groups =
|
||||
respected
|
||||
? ColumnLayout.orderByBand(segments.get(s), gutters)
|
||||
: ColumnLayout.legacySplit(segments.get(s));
|
||||
for (List<Line> col : groups) {
|
||||
List<String> paras = new ArrayList<>();
|
||||
ParagraphAssembler.assembleParagraphs(
|
||||
col, medianSize, medianHeight, bodyFont, paras, tableRowTexts);
|
||||
pageItems.addAll(paras);
|
||||
}
|
||||
if (s < blocks.size()) {
|
||||
pageItems.add(blocks.get(s));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Interleave tables with text by vertical position: each block gets a slot,
|
||||
// and non-table lines fall into the slot for their y, keeping tables separate.
|
||||
for (int s = 0; s <= blocks.size(); s++) {
|
||||
List<String> paras = new ArrayList<>();
|
||||
ParagraphAssembler.assembleParagraphs(
|
||||
segments.get(s), medianSize, medianHeight, bodyFont, paras, tableRowTexts);
|
||||
pageItems.addAll(paras);
|
||||
if (s < blocks.size()) {
|
||||
pageItems.add(blocks.get(s));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PageImages.emit(doc, pageIndex, pageItems);
|
||||
return pageItems;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits a page's non-table lines into the bands between its table blocks, band {@code s}
|
||||
* holding the lines above block {@code s}. Blocks must be in top-to-bottom order.
|
||||
*/
|
||||
private static List<List<Line>> segmentsAround(
|
||||
List<Line> lines, List<TableBlock> blocks, Set<Line> tableLines) {
|
||||
List<List<Line>> segments = new ArrayList<>();
|
||||
for (int s = 0; s <= blocks.size(); s++) {
|
||||
segments.add(new ArrayList<>());
|
||||
}
|
||||
for (Line l : lines) {
|
||||
if (tableLines.contains(l)) {
|
||||
continue;
|
||||
}
|
||||
int slot = 0;
|
||||
for (TableBlock b : blocks) {
|
||||
if (b.bottom() > l.y) {
|
||||
slot++;
|
||||
}
|
||||
}
|
||||
segments.get(slot).add(l);
|
||||
}
|
||||
return segments;
|
||||
}
|
||||
|
||||
/** Ruling lines of one page, or {@link PageRules#EMPTY} if the page cannot be opened. */
|
||||
private static PageRules readRules(PdfDocument doc, int pageIndex) {
|
||||
try (PdfPage page = doc.page(pageIndex)) {
|
||||
return PageRules.of(page);
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"Page {} ruling lines unreadable; falling back to word-grid tables",
|
||||
pageIndex,
|
||||
e);
|
||||
return PageRules.EMPTY;
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Incremental {@link ColumnRanges#find(List)} for a stitched table: appending a page costs O(page),
|
||||
* bit-for-bit identical to re-projecting the lot.
|
||||
*/
|
||||
final class ColumnAccumulator {
|
||||
|
||||
private int lineCount;
|
||||
private float minX = Float.MAX_VALUE;
|
||||
private float maxX = -Float.MAX_VALUE;
|
||||
private double totalWidth;
|
||||
private int totalChars;
|
||||
|
||||
/** Coverage counts, cov[i] = lines covering absolute x-bucket covBase + i. */
|
||||
private int[] cov = new int[0];
|
||||
|
||||
private int covBase;
|
||||
|
||||
/** Set once the x-span exceeds what findColumnRanges accepts; no histogram is then kept. */
|
||||
private boolean oversized;
|
||||
|
||||
private boolean[] scratch = new boolean[0];
|
||||
|
||||
static ColumnAccumulator of(List<List<Line>> rows) {
|
||||
ColumnAccumulator a = new ColumnAccumulator();
|
||||
for (List<Line> row : rows) {
|
||||
for (Line l : row) {
|
||||
a.addLine(l);
|
||||
}
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
void addLine(Line l) {
|
||||
lineCount++;
|
||||
List<TextWord> words = l.words();
|
||||
int lineLo = Integer.MAX_VALUE;
|
||||
int lineHi = Integer.MIN_VALUE;
|
||||
for (TextWord w : words) {
|
||||
float x0 = w.x();
|
||||
float x1 = x0 + w.width();
|
||||
minX = Math.min(minX, x0);
|
||||
maxX = Math.max(maxX, x1);
|
||||
totalWidth += w.width();
|
||||
totalChars += Math.max(1, w.text().strip().length());
|
||||
int a = (int) Math.floor(x0);
|
||||
int b = (int) Math.ceil(x1);
|
||||
if (a < lineLo) {
|
||||
lineLo = a;
|
||||
}
|
||||
if (b > lineHi) {
|
||||
lineHi = b;
|
||||
}
|
||||
}
|
||||
// Mirrors ColumnRanges.find's guard: past this span it returns no columns, so the
|
||||
// histogram is dead weight and (with crafted coordinates) unboundedly large.
|
||||
if (!oversized && (maxX - minX) > 2000f) {
|
||||
oversized = true;
|
||||
cov = null;
|
||||
scratch = null;
|
||||
}
|
||||
if (oversized || lineHi <= lineLo) {
|
||||
return;
|
||||
}
|
||||
ensureRange(lineLo, lineHi);
|
||||
int n = lineHi - lineLo;
|
||||
if (scratch.length < n) {
|
||||
scratch = new boolean[n];
|
||||
} else {
|
||||
Arrays.fill(scratch, 0, n, false);
|
||||
}
|
||||
for (TextWord w : words) {
|
||||
int a = (int) Math.floor(w.x()) - lineLo;
|
||||
int b = (int) Math.ceil(w.x() + w.width()) - lineLo;
|
||||
for (int x = a; x < b; x++) {
|
||||
scratch[x] = true;
|
||||
}
|
||||
}
|
||||
int off = lineLo - covBase;
|
||||
for (int x = 0; x < n; x++) {
|
||||
if (scratch[x]) {
|
||||
cov[off + x]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureRange(int lo, int hi) {
|
||||
if (cov.length == 0) {
|
||||
covBase = lo - 32;
|
||||
cov = new int[(hi - lo) + 64];
|
||||
return;
|
||||
}
|
||||
int have0 = covBase;
|
||||
int have1 = covBase + cov.length;
|
||||
if (lo >= have0 && hi <= have1) {
|
||||
return;
|
||||
}
|
||||
int newBase = Math.min(have0, lo) - 32;
|
||||
int newEnd = Math.max(have1, hi) + 32;
|
||||
int[] nc = new int[newEnd - newBase];
|
||||
System.arraycopy(cov, 0, nc, have0 - newBase, cov.length);
|
||||
cov = nc;
|
||||
covBase = newBase;
|
||||
}
|
||||
|
||||
/** Exactly what {@link ColumnRanges#find(List)} would return for the accumulated lines. */
|
||||
List<float[]> columns() {
|
||||
if (oversized || maxX <= minX || (maxX - minX) > 2000f) {
|
||||
return List.of();
|
||||
}
|
||||
int lo = (int) Math.floor(minX);
|
||||
int span = Math.min((int) Math.ceil(maxX) - lo + 1, 2001);
|
||||
int support = Math.max(2, Math.round(lineCount * 0.35f));
|
||||
List<float[]> columns = new ArrayList<>();
|
||||
int start = -1;
|
||||
for (int x = 0; x < span; x++) {
|
||||
int idx = lo + x - covBase;
|
||||
int c = (idx >= 0 && idx < cov.length) ? cov[idx] : 0;
|
||||
boolean isColumn = c >= support;
|
||||
if (isColumn && start < 0) {
|
||||
start = x;
|
||||
} else if (!isColumn && start >= 0) {
|
||||
columns.add(new float[] {lo + start, lo + x});
|
||||
start = -1;
|
||||
}
|
||||
}
|
||||
if (start >= 0) {
|
||||
columns.add(new float[] {(float) (lo + start), (float) (lo + span)});
|
||||
}
|
||||
|
||||
float charWidth = totalChars == 0 ? 6f : (float) (totalWidth / totalChars);
|
||||
float minGutter = Math.max(10f, charWidth * 2.5f);
|
||||
List<float[]> merged = new ArrayList<>();
|
||||
for (float[] band : columns) {
|
||||
if (!merged.isEmpty() && band[0] - merged.get(merged.size() - 1)[1] < minGutter) {
|
||||
merged.get(merged.size() - 1)[1] = band[1];
|
||||
} else {
|
||||
merged.add(new float[] {band[0], band[1]});
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
|
||||
/**
|
||||
* Multi-column page layout: finding the gutters between columns of prose, splitting a page's lines
|
||||
* at them, and emitting the columns in reading order.
|
||||
*/
|
||||
final class ColumnLayout {
|
||||
|
||||
private ColumnLayout() {}
|
||||
|
||||
/** Narrowest run of near-empty x that can separate two columns of prose. */
|
||||
private static final float MIN_GUTTER = 10f;
|
||||
|
||||
/** Narrowest column worth splitting out; below this a "gutter" is just a ragged margin. */
|
||||
private static final float MIN_COLUMN = 70f;
|
||||
|
||||
/** Fraction of a page's lines that may cross a gutter and still leave it a gutter. */
|
||||
private static final float MAX_CROSSING = 0.15f;
|
||||
|
||||
/** Most columns recognised on one page. Beyond this the geometry is a table, not a layout. */
|
||||
private static final int MAX_COLUMNS = 4;
|
||||
|
||||
/**
|
||||
* Finds the page's column gutters, or empty for a single column. Scans the 5th to 95th
|
||||
* percentile of line edges so one degenerate box cannot drag it off the page.
|
||||
*/
|
||||
static List<Float> detectGutters(List<Line> lines) {
|
||||
if (lines.size() < 8) {
|
||||
return List.of();
|
||||
}
|
||||
int n = lines.size();
|
||||
float[] los = new float[n];
|
||||
float[] his = new float[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
los[i] = lines.get(i).left();
|
||||
his[i] = lines.get(i).right();
|
||||
}
|
||||
float[] sortedLo = los.clone();
|
||||
float[] sortedHi = his.clone();
|
||||
Arrays.sort(sortedLo);
|
||||
Arrays.sort(sortedHi);
|
||||
float lo = sortedLo[(int) (n * 0.05f)];
|
||||
float hi = sortedHi[Math.min(n - 1, (int) (n * 0.95f))];
|
||||
if (hi - lo < 2 * MIN_COLUMN + MIN_GUTTER || !plausibleSpan(lo, hi)) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
int maxCrossing = (int) (n * MAX_CROSSING);
|
||||
int start = -1;
|
||||
List<float[]> bands = new ArrayList<>();
|
||||
// Stepped as an int: past 2^24 a float can no longer represent x + 1, so a float counter
|
||||
// over a crafted coordinate stops advancing and spins forever.
|
||||
int scanFrom = (int) Math.floor(lo + MIN_COLUMN);
|
||||
int scanTo = (int) Math.ceil(hi - MIN_COLUMN);
|
||||
for (int xi = scanFrom; xi <= scanTo; xi++) {
|
||||
float x = xi;
|
||||
int crossing = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (los[i] < x - 2f && his[i] > x + 2f) {
|
||||
crossing++;
|
||||
}
|
||||
}
|
||||
if (crossing <= maxCrossing) {
|
||||
if (start < 0) {
|
||||
start = (int) x;
|
||||
}
|
||||
} else if (start >= 0) {
|
||||
bands.add(new float[] {start, x});
|
||||
start = -1;
|
||||
}
|
||||
}
|
||||
if (start >= 0) {
|
||||
bands.add(new float[] {start, hi - MIN_COLUMN});
|
||||
}
|
||||
|
||||
// Widest first, so the strongest separation wins; then keep only bands MIN_COLUMN apart.
|
||||
bands.sort(Comparator.comparingDouble((float[] b) -> b[1] - b[0]).reversed());
|
||||
List<Float> gutters = new ArrayList<>();
|
||||
for (float[] b : bands) {
|
||||
if (b[1] - b[0] < MIN_GUTTER || gutters.size() >= MAX_COLUMNS - 1) {
|
||||
continue;
|
||||
}
|
||||
float mid = (b[0] + b[1]) / 2f;
|
||||
boolean tooClose = mid - lo < MIN_COLUMN || hi - mid < MIN_COLUMN;
|
||||
for (float g : gutters) {
|
||||
tooClose |= Math.abs(g - mid) < MIN_COLUMN;
|
||||
}
|
||||
if (!tooClose) {
|
||||
gutters.add(mid);
|
||||
}
|
||||
}
|
||||
gutters.sort(Comparator.naturalOrder());
|
||||
|
||||
if (!gutters.isEmpty() && columnsLookLikeText(lines, gutters)) {
|
||||
return gutters;
|
||||
}
|
||||
return centralGutter(lines, los, his, lo, hi);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rejects geometry too wide to be real: past 2^24 a float cannot represent x + 1, so a
|
||||
* constant-step scan stops advancing.
|
||||
*/
|
||||
private static boolean plausibleSpan(float lo, float hi) {
|
||||
return Float.isFinite(lo) && Float.isFinite(hi) && (hi - lo) <= 2000f;
|
||||
}
|
||||
|
||||
/** Fallback: accepts halves of scattered labels, which read as columns but not as prose. */
|
||||
private static List<Float> centralGutter(
|
||||
List<Line> lines, float[] los, float[] his, float lo, float hi) {
|
||||
int n = lines.size();
|
||||
float centreLo = lo + (hi - lo) * 0.35f;
|
||||
float centreHi = lo + (hi - lo) * 0.65f;
|
||||
int bestCrossing = Integer.MAX_VALUE;
|
||||
float bestAt = 0f;
|
||||
int bestLeft = 0;
|
||||
int bestRight = 0;
|
||||
for (int gi = (int) Math.floor(centreLo); gi <= (int) Math.ceil(centreHi); gi += 2) {
|
||||
float gutter = gi;
|
||||
int crossing = 0;
|
||||
int left = 0;
|
||||
int right = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (los[i] < gutter - 5f && his[i] > gutter + 5f) {
|
||||
crossing++;
|
||||
} else if (his[i] <= gutter) {
|
||||
left++;
|
||||
} else {
|
||||
right++;
|
||||
}
|
||||
}
|
||||
if (crossing < bestCrossing) {
|
||||
bestCrossing = crossing;
|
||||
bestAt = gutter;
|
||||
bestLeft = left;
|
||||
bestRight = right;
|
||||
}
|
||||
}
|
||||
boolean ok = bestLeft >= 4 && bestRight >= 4 && bestCrossing <= (int) (n * 0.25f);
|
||||
return ok ? List.of(bestAt) : List.of();
|
||||
}
|
||||
|
||||
/** Lines of at least this fraction of a column's width count as that column's body text. */
|
||||
private static final float BODY_LINE_WIDTH = 0.5f;
|
||||
|
||||
/** Body lines a column must hold before it is accepted as a column. */
|
||||
private static final int BODY_LINES = 4;
|
||||
|
||||
/**
|
||||
* True when every carved-out column reads as running text; projection alone cannot tell prose
|
||||
* from any other empty lane, such as a bar chart's label gaps.
|
||||
*/
|
||||
private static boolean columnsLookLikeText(List<Line> lines, List<Float> gutters) {
|
||||
// Judge only lines inside a column: a spanning line is assigned to one by its centre, and
|
||||
// its width would set a measure no real body line could reach.
|
||||
List<Line> inside =
|
||||
lines.stream().filter(l -> !spansGutter(l, gutters)).collect(Collectors.toList());
|
||||
List<List<Line>> columns = splitIntoColumns(inside, gutters);
|
||||
if (columns.size() < 2) {
|
||||
return false;
|
||||
}
|
||||
for (List<Line> column : columns) {
|
||||
float lo = Float.MAX_VALUE;
|
||||
float hi = -Float.MAX_VALUE;
|
||||
for (Line l : column) {
|
||||
lo = Math.min(lo, l.left());
|
||||
hi = Math.max(hi, l.right());
|
||||
}
|
||||
float measure = hi - lo;
|
||||
int body = 0;
|
||||
for (Line l : column) {
|
||||
if (l.right() - l.left() >= measure * BODY_LINE_WIDTH) {
|
||||
body++;
|
||||
}
|
||||
}
|
||||
if (body < BODY_LINES || measure < MIN_COLUMN) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits lines into columns at the given gutters. A line crossing one goes to the column its
|
||||
* centre falls in; band ordering then places it correctly.
|
||||
*/
|
||||
static List<List<Line>> splitIntoColumns(List<Line> lines, List<Float> gutters) {
|
||||
if (gutters.isEmpty()) {
|
||||
return List.of(lines);
|
||||
}
|
||||
List<List<Line>> columns = new ArrayList<>(gutters.size() + 1);
|
||||
for (int i = 0; i <= gutters.size(); i++) {
|
||||
columns.add(new ArrayList<>());
|
||||
}
|
||||
for (Line l : lines) {
|
||||
columns.get(columnOf(l, gutters)).add(l);
|
||||
}
|
||||
columns.removeIf(List::isEmpty);
|
||||
return columns;
|
||||
}
|
||||
|
||||
private static int columnOf(Line l, List<Float> gutters) {
|
||||
float centre = (l.left() + l.right()) / 2f;
|
||||
int col = 0;
|
||||
while (col < gutters.size() && centre > gutters.get(col)) {
|
||||
col++;
|
||||
}
|
||||
return col;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the finished lines still respect the gutters the unmerged lines showed; merging
|
||||
* widens lines, and band-ordering a straddled gutter interleaves the columns.
|
||||
*/
|
||||
static boolean gutterRespected(List<Line> lines, List<Float> gutters) {
|
||||
List<Line> real = lines.stream().filter(l -> !l.synthetic).toList();
|
||||
if (real.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
long spanning = real.stream().filter(l -> spansGutter(l, gutters)).count();
|
||||
return spanning <= real.size() * BAND_CROSSING;
|
||||
}
|
||||
|
||||
/** Fraction of the finished lines that may straddle a gutter and still allow band ordering. */
|
||||
private static final float BAND_CROSSING = 0.35f;
|
||||
|
||||
/** Fallback column split: cut at the widest gap between the lines' left edges. */
|
||||
static List<List<Line>> legacySplit(List<Line> lines) {
|
||||
List<Float> xs =
|
||||
lines.stream()
|
||||
.filter(l -> l.width >= 40f)
|
||||
.map(l -> l.x)
|
||||
.sorted()
|
||||
.collect(Collectors.toList());
|
||||
if (xs.isEmpty()) {
|
||||
return List.of(lines);
|
||||
}
|
||||
float splitAt = (xs.getFirst() + xs.getLast()) / 2f;
|
||||
float biggestGap = 0;
|
||||
for (int i = 1; i < xs.size(); i++) {
|
||||
float gap = xs.get(i) - xs.get(i - 1);
|
||||
if (gap > biggestGap) {
|
||||
biggestGap = gap;
|
||||
splitAt = (xs.get(i - 1) + xs.get(i)) / 2f;
|
||||
}
|
||||
}
|
||||
List<Line> left = new ArrayList<>();
|
||||
List<Line> right = new ArrayList<>();
|
||||
for (Line l : lines) {
|
||||
(l.x < splitAt ? left : right).add(l);
|
||||
}
|
||||
if (left.isEmpty()) {
|
||||
return List.of(right);
|
||||
}
|
||||
if (right.isEmpty()) {
|
||||
return List.of(left);
|
||||
}
|
||||
return List.of(left, right);
|
||||
}
|
||||
|
||||
/** Longest a line may be and still be a line of a heading rather than of a paragraph. */
|
||||
private static final int HEADING_LENGTH_WORDS = 12;
|
||||
|
||||
/**
|
||||
* True when a spanning line is short enough to be one line of a full-width banner heading,
|
||||
* which {@link #orderByBand} keeps in a single group.
|
||||
*/
|
||||
private static boolean headingLength(Line l) {
|
||||
return MarkdownText.wordCount(l.text) <= HEADING_LENGTH_WORDS;
|
||||
}
|
||||
|
||||
/** True when a line straddles a gutter, i.e. it belongs to no single column. */
|
||||
static boolean spansGutter(Line l, List<Float> gutters) {
|
||||
float left = l.left();
|
||||
float right = l.right();
|
||||
for (float g : gutters) {
|
||||
if (left < g - 2f && right > g + 2f) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Orders a multi-column region as a one-level XY cut: spanning lines cut it into bands, and
|
||||
* each band's columns are emitted in turn.
|
||||
*/
|
||||
static List<List<Line>> orderByBand(List<Line> lines, List<Float> gutters) {
|
||||
List<Line> ordered = new ArrayList<>(lines);
|
||||
ordered.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
|
||||
List<List<Line>> out = new ArrayList<>();
|
||||
List<Line> band = new ArrayList<>();
|
||||
List<Line> spanning = new ArrayList<>();
|
||||
for (Line l : ordered) {
|
||||
if (spansGutter(l, gutters)) {
|
||||
if (spanning.isEmpty()) {
|
||||
out.addAll(splitIntoColumns(band, gutters));
|
||||
band = new ArrayList<>();
|
||||
} else if (!headingLength(l) || !headingLength(spanning.get(spanning.size() - 1))) {
|
||||
// Only heading-length lines are kept together: a full-width paragraph or list
|
||||
// is also a run of spanning lines, and merging those runs its items together.
|
||||
out.add(new ArrayList<>(spanning));
|
||||
spanning.clear();
|
||||
}
|
||||
spanning.add(l);
|
||||
} else {
|
||||
if (!spanning.isEmpty()) {
|
||||
out.add(new ArrayList<>(spanning));
|
||||
spanning.clear();
|
||||
}
|
||||
band.add(l);
|
||||
}
|
||||
}
|
||||
if (!spanning.isEmpty()) {
|
||||
out.add(new ArrayList<>(spanning));
|
||||
}
|
||||
out.addAll(splitIntoColumns(band, gutters));
|
||||
out.removeIf(List::isEmpty);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Visible for testing: as {@link ColumnRanges#fromTextLines(List)}, for gutter detection. */
|
||||
static List<Float> guttersFromTextLines(List<TextLine> rows) {
|
||||
return detectGutters(rows.stream().map(Line::new).collect(Collectors.toList()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Finds a table's column x-ranges by vertical-whitespace projection: a column is a contiguous
|
||||
* x-band occupied by enough rows, and the gaps between bands are the gutters.
|
||||
*/
|
||||
final class ColumnRanges {
|
||||
|
||||
private ColumnRanges() {}
|
||||
|
||||
/** Character widths of clear space that separate two columns of an unruled block. */
|
||||
static final float GUTTER_CHARS = 2.5f;
|
||||
|
||||
/** Absolute floor, in points, on an unruled block's column gutter. */
|
||||
static final float GUTTER_FLOOR = 10f;
|
||||
|
||||
/** As {@link #GUTTER_CHARS}, for a block the page's rules already declare to be a table. */
|
||||
static final float RULED_GUTTER_CHARS = 1.2f;
|
||||
|
||||
/** As {@link #GUTTER_FLOOR}, for a block the page's rules already declare to be a table. */
|
||||
static final float RULED_GUTTER_FLOOR = 4f;
|
||||
|
||||
static List<float[]> find(List<Line> rows) {
|
||||
return find(rows, GUTTER_CHARS, GUTTER_FLOOR);
|
||||
}
|
||||
|
||||
/** As {@link #find(List)}, with the gutter thresholds given explicitly. */
|
||||
static List<float[]> find(List<Line> rows, float gutterChars, float gutterFloor) {
|
||||
return find(rows, gutterChars, gutterFloor, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* As above, but {@code minSupport} overrides how many rows must occupy an x-band; zero keeps
|
||||
* the row-count-scaled default.
|
||||
*/
|
||||
static List<float[]> find(
|
||||
List<Line> rows, float gutterChars, float gutterFloor, int minSupport) {
|
||||
float minX = Float.MAX_VALUE;
|
||||
float maxX = -Float.MAX_VALUE;
|
||||
for (Line l : rows) {
|
||||
for (TextWord w : l.words()) {
|
||||
minX = Math.min(minX, w.x());
|
||||
maxX = Math.max(maxX, w.x() + w.width());
|
||||
}
|
||||
}
|
||||
// Real pages are under ~2000pt wide; anything larger is a malformed/crafted coordinate
|
||||
// that would allocate a multi-GB array or produce a negative span on overflow.
|
||||
if (maxX <= minX || (maxX - minX) > 2000f) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
int lo = (int) Math.floor(minX);
|
||||
int span = Math.min((int) Math.ceil(maxX) - lo + 1, 2001);
|
||||
int[] coverage = new int[span];
|
||||
for (Line l : rows) {
|
||||
boolean[] covered = new boolean[span];
|
||||
for (TextWord w : l.words()) {
|
||||
int a = Math.max(0, (int) Math.floor(w.x()) - lo);
|
||||
int b = Math.min(span, (int) Math.ceil(w.x() + w.width()) - lo);
|
||||
for (int x = a; x < b; x++) {
|
||||
covered[x] = true;
|
||||
}
|
||||
}
|
||||
for (int x = 0; x < span; x++) {
|
||||
if (covered[x]) {
|
||||
coverage[x]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A column band must be occupied by at least this many rows; below it is gutter.
|
||||
int support = minSupport > 0 ? minSupport : Math.max(2, Math.round(rows.size() * 0.35f));
|
||||
List<float[]> columns = new ArrayList<>();
|
||||
int start = -1;
|
||||
for (int x = 0; x < span; x++) {
|
||||
boolean isColumn = coverage[x] >= support;
|
||||
if (isColumn && start < 0) {
|
||||
start = x;
|
||||
} else if (!isColumn && start >= 0) {
|
||||
columns.add(new float[] {lo + start, lo + x});
|
||||
start = -1;
|
||||
}
|
||||
}
|
||||
if (start >= 0) {
|
||||
columns.add(new float[] {(float) (lo + start), (float) (lo + span)});
|
||||
}
|
||||
|
||||
// Merge bands closer than a real column separator: the gaps inside a multi-word cell are
|
||||
// about one character, and would otherwise split "January 20th, 2026" into three columns.
|
||||
float charWidth = WordGeometry.averageCharWidth(rows);
|
||||
float minGutter = Math.max(gutterFloor, charWidth * gutterChars);
|
||||
List<float[]> merged = new ArrayList<>();
|
||||
for (float[] band : columns) {
|
||||
if (!merged.isEmpty() && band[0] - merged.getLast()[1] < minGutter) {
|
||||
merged.getLast()[1] = band[1];
|
||||
} else {
|
||||
merged.add(new float[] {band[0], band[1]});
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Visible for testing: column detection depends only on word geometry, so tests can exercise
|
||||
* degenerate coordinates without a binary fixture.
|
||||
*/
|
||||
static List<float[]> fromTextLines(List<TextLine> rows) {
|
||||
return find(rows.stream().map(Line::new).collect(Collectors.toList()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.PdfPage;
|
||||
import stirling.software.jpdfium.doc.FormField;
|
||||
import stirling.software.jpdfium.doc.FormFieldType;
|
||||
import stirling.software.jpdfium.doc.PdfFormReader;
|
||||
import stirling.software.jpdfium.model.Rect;
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Recovers AcroForm values that live only in a field's {@code /V}, as pseudo lines at their widget
|
||||
* rectangles so they land in reading order.
|
||||
*/
|
||||
final class FormValues {
|
||||
|
||||
private FormValues() {}
|
||||
|
||||
/**
|
||||
* Pseudo text lines for {@code /V}-only AcroForm values, placed at their widget rectangles;
|
||||
* values already in the content stream are skipped.
|
||||
*/
|
||||
static List<Line> lines(PdfDocument doc, int pageIndex, List<Line> existing) {
|
||||
List<FormField> fields;
|
||||
try (PdfPage page = doc.page(pageIndex)) {
|
||||
fields = PdfFormReader.readPage(page.rawDocHandle(), page.rawHandle(), pageIndex);
|
||||
} catch (RuntimeException e) {
|
||||
// A malformed AcroForm must not sink the whole conversion; body text still stands.
|
||||
return List.of();
|
||||
}
|
||||
List<Line> out = new ArrayList<>();
|
||||
for (FormField f : fields) {
|
||||
String value = fieldText(f);
|
||||
if (value == null || value.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
Rect r = f.rect();
|
||||
if (r == null || r.width() <= 0 || r.height() <= 0) {
|
||||
continue;
|
||||
}
|
||||
if (alreadyInContent(existing, value, r)) {
|
||||
continue;
|
||||
}
|
||||
out.add(syntheticLine(value, r));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The text a filled field contributes, or null when the field contributes nothing. */
|
||||
private static String fieldText(FormField f) {
|
||||
FormFieldType type = f.type();
|
||||
if (type == FormFieldType.PUSHBUTTON
|
||||
|| type == FormFieldType.SIGNATURE
|
||||
|| type == FormFieldType.UNKNOWN) {
|
||||
return null;
|
||||
}
|
||||
if (type == FormFieldType.CHECKBOX || type == FormFieldType.RADIO) {
|
||||
return f.checked() ? "[x]" : null;
|
||||
}
|
||||
String value = f.value();
|
||||
if (value == null || "Off".equals(value)) {
|
||||
return null;
|
||||
}
|
||||
return value.replace('\r', ' ').replace('\n', ' ').strip();
|
||||
}
|
||||
|
||||
/** True when the extractor already found this value inside the widget's own rectangle. */
|
||||
private static boolean alreadyInContent(List<Line> lines, String value, Rect r) {
|
||||
String needle = MarkdownText.normaliseSpace(value);
|
||||
for (Line l : lines) {
|
||||
boolean overlaps =
|
||||
l.x < r.x() + r.width()
|
||||
&& l.x + l.width > r.x()
|
||||
&& l.y < r.y() + r.height()
|
||||
&& l.y + l.height > r.y();
|
||||
if (overlaps && MarkdownText.normaliseSpace(l.text).contains(needle)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a field value as a one-word-per-token {@link TextLine} at the widget rectangle, so
|
||||
* downstream stages treat it like any other text.
|
||||
*/
|
||||
private static Line syntheticLine(String value, Rect r) {
|
||||
String[] tokens = value.split("\\s+");
|
||||
float height = Math.min(r.height(), 14f);
|
||||
float advance = tokens.length == 0 ? r.width() : r.width() / tokens.length;
|
||||
List<TextWord> words = new ArrayList<>(tokens.length);
|
||||
for (int i = 0; i < tokens.length; i++) {
|
||||
float wx = r.x() + advance * i;
|
||||
List<TextChar> chars = new ArrayList<>(tokens[i].length());
|
||||
float charWidth = tokens[i].isEmpty() ? advance : advance / tokens[i].length();
|
||||
for (int c = 0; c < tokens[i].length(); c++) {
|
||||
chars.add(
|
||||
new TextChar(
|
||||
c,
|
||||
tokens[i].charAt(c),
|
||||
wx + charWidth * c,
|
||||
r.y(),
|
||||
charWidth,
|
||||
height,
|
||||
"",
|
||||
0f));
|
||||
}
|
||||
words.add(new TextWord(chars, wx, r.y(), advance * 0.95f, height));
|
||||
}
|
||||
TextLine line = new TextLine(words, r.x(), r.y(), r.width(), height);
|
||||
Line out = new Line(line, value);
|
||||
out.synthetic = true;
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** Renders a resolved cell grid as a GitHub-Flavored Markdown table. */
|
||||
final class GfmTable {
|
||||
|
||||
private GfmTable() {}
|
||||
|
||||
static String render(List<String[]> rows, int cols) {
|
||||
if (rows.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
int[] widths = new int[cols];
|
||||
for (int c = 0; c < cols; c++) {
|
||||
widths[c] = 3;
|
||||
}
|
||||
for (String[] row : rows) {
|
||||
for (int c = 0; c < cols; c++) {
|
||||
if (c < row.length) {
|
||||
widths[c] = Math.max(widths[c], escapeCell(row[c]).length());
|
||||
}
|
||||
}
|
||||
}
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(buildGfmRow(rows.getFirst(), widths, cols)).append('\n');
|
||||
sb.append('|');
|
||||
for (int c = 0; c < cols; c++) {
|
||||
sb.append('-').append("-".repeat(widths[c])).append('-').append('|');
|
||||
}
|
||||
for (int r = 1; r < rows.size(); r++) {
|
||||
sb.append('\n').append(buildGfmRow(rows.get(r), widths, cols));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static String buildGfmRow(String[] row, int[] widths, int cols) {
|
||||
StringBuilder sb = new StringBuilder().append('|');
|
||||
for (int c = 0; c < cols; c++) {
|
||||
String cell = c < row.length ? escapeCell(row[c]) : "";
|
||||
sb.append(' ').append(padRight(cell, widths[c])).append(' ').append('|');
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static String escapeCell(String cell) {
|
||||
// Cell content is inline context: escape inline markdown (including the column delimiter)
|
||||
// but not leading block markers, which have no meaning inside a table cell.
|
||||
return MarkdownText.escapeMarkdownInline(cell);
|
||||
}
|
||||
|
||||
private static String padRight(String s, int width) {
|
||||
return s.length() >= width ? s : s + " ".repeat(width - s.length());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
|
||||
/**
|
||||
* Rebuilds assembled {@link Line}s from the extractor's {@link TextLine}s, folding in the narrow
|
||||
* glyph fragments PDFium emits for apostrophes, markers and bullets.
|
||||
*/
|
||||
final class GlyphStitcher {
|
||||
|
||||
private GlyphStitcher() {}
|
||||
|
||||
/** Width below which a TextLine is treated as a stray glyph fragment to be stitched. */
|
||||
private static final float GLYPH_WIDTH = 7.5f;
|
||||
|
||||
/**
|
||||
* Merges narrow glyph fragments into the line they belong to: inline between two same-baseline
|
||||
* fragments, or appended/prepended at a line's edge.
|
||||
*/
|
||||
static List<Line> stitchGlyphs(List<TextLine> raw) {
|
||||
List<TextLine> hosts = new ArrayList<>();
|
||||
List<TextLine> glyphs = new ArrayList<>();
|
||||
for (TextLine l : raw) {
|
||||
String t = stripSoftHyphens(l.text()).strip();
|
||||
if (t.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (l.width() < GLYPH_WIDTH && t.length() <= 2) {
|
||||
glyphs.add(l);
|
||||
} else {
|
||||
hosts.add(l);
|
||||
}
|
||||
}
|
||||
|
||||
List<Line> lines =
|
||||
hosts.stream()
|
||||
.map(l -> new Line(l, stripSoftHyphens(l.text())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
for (TextLine g : glyphs) {
|
||||
String gt = stripSoftHyphens(g.text()).strip();
|
||||
if (isBulletGlyph(gt)) {
|
||||
attachBullet(g, gt, lines);
|
||||
} else {
|
||||
attachInlineGlyph(g, gt, lines);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes U+00AD SOFT HYPHEN, a break-opportunity marker PDFium hands back verbatim as {@code
|
||||
* ar<AD>e}.
|
||||
*/
|
||||
private static String stripSoftHyphens(String text) {
|
||||
if (text.indexOf('') < 0) {
|
||||
return text;
|
||||
}
|
||||
return text.replace("", "");
|
||||
}
|
||||
|
||||
private static boolean isBulletGlyph(String gt) {
|
||||
return "•".equals(gt) || "▪".equals(gt) || "◦".equals(gt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Attaches a bullet glyph to the line it introduces: the closest line beginning to its right,
|
||||
* at roughly the same height or just below.
|
||||
*/
|
||||
private static void attachBullet(TextLine g, String gt, List<Line> lines) {
|
||||
Line best = null;
|
||||
float bestScore = Float.MAX_VALUE;
|
||||
for (Line h : lines) {
|
||||
if (h.x < g.x() - 2f) {
|
||||
continue;
|
||||
}
|
||||
float dy = g.y() - h.y;
|
||||
if (dy < -4f || dy > 28f) {
|
||||
continue;
|
||||
}
|
||||
float score = Math.abs(dy) + (h.x - g.x()) * 0.2f;
|
||||
if (score < bestScore) {
|
||||
bestScore = score;
|
||||
best = h;
|
||||
}
|
||||
}
|
||||
if (best != null && !best.text.startsWith("•")) {
|
||||
best.text = "• " + best.text;
|
||||
best.x = g.x();
|
||||
} else {
|
||||
lines.add(new Line(g, gt));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stitches a narrow inline glyph into its line: between two same-baseline fragments, appended
|
||||
* to the line ending at it, or prepended to the one starting at it.
|
||||
*/
|
||||
private static void attachInlineGlyph(TextLine g, String gt, List<Line> lines) {
|
||||
Line left = null;
|
||||
Line right = null;
|
||||
float lb = 7f;
|
||||
float rb = 7f;
|
||||
for (Line h : lines) {
|
||||
boolean sameBaseline = g.y() >= h.y - 4f && g.y() <= h.y + h.height + 5f;
|
||||
if (!sameBaseline) {
|
||||
continue;
|
||||
}
|
||||
float rightEdge = h.x + h.width;
|
||||
float dxLeft = Math.abs(rightEdge - g.x());
|
||||
if (dxLeft < lb) {
|
||||
lb = dxLeft;
|
||||
left = h;
|
||||
}
|
||||
float dxRight = Math.abs(h.x - g.x());
|
||||
if (dxRight < rb) {
|
||||
rb = dxRight;
|
||||
right = h;
|
||||
}
|
||||
}
|
||||
|
||||
if (left != null && right != null && left != right && Math.abs(left.y - right.y) < 6f) {
|
||||
left.text = left.text + gt + right.text;
|
||||
left.width = (right.x + right.width) - left.x;
|
||||
left.absorb(g);
|
||||
left.absorb(right);
|
||||
lines.remove(right);
|
||||
} else if (left != null) {
|
||||
left.text = left.text + gt;
|
||||
left.width = Math.max(left.width, g.x() + g.width() - left.x);
|
||||
left.absorb(g);
|
||||
} else if (right != null) {
|
||||
right.text = gt + right.text;
|
||||
right.x = g.x();
|
||||
right.absorb(g);
|
||||
} else {
|
||||
lines.add(new Line(g, gt));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,485 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import stirling.software.jpdfium.text.PageText;
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
final class HeadingDetector {
|
||||
|
||||
private HeadingDetector() {}
|
||||
|
||||
/** A heading is at most this many words; longer lines are treated as body text. */
|
||||
private static final int MAX_HEADING_WORDS = 12;
|
||||
|
||||
/** A heading in a script written without word spaces is at most this many characters. */
|
||||
private static final int MAX_HEADING_UNSPACED_CHARS = 30;
|
||||
|
||||
/** Six-letter subset tag PDF writers prepend to embedded font names. */
|
||||
private static final Pattern SUBSET_TAG = Pattern.compile("^[A-Z]{6}\\+");
|
||||
|
||||
/** PostScript name fragments denoting a weight heavier than the regular face. */
|
||||
private static final String[] BOLD_TOKENS = {
|
||||
"bold", "black", "heavy", "semibold", "demi", "ultra", "extrabold"
|
||||
};
|
||||
|
||||
/**
|
||||
* URW/Nimbus bold is "-Medi", TeX bold extended is CMBX/CMSSBX. "Medium" is excluded: matching
|
||||
* it makes whole CJK paragraphs bold.
|
||||
*/
|
||||
private static final Pattern OTHER_BOLD = Pattern.compile("medi(?!um)|cm(ss)?bx");
|
||||
|
||||
/**
|
||||
* A float label followed by its number: set like a heading but naming an illustration, not a
|
||||
* section.
|
||||
*/
|
||||
private static final Pattern CAPTION =
|
||||
Pattern.compile(
|
||||
"^(table|figure|fig|chart|exhibit|plate|scheme|graph|diagram|illustration)"
|
||||
+ "\\s*\\.?\\s*\\d",
|
||||
Pattern.CASE_INSENSITIVE);
|
||||
|
||||
/** A numbered section clause: {@code 3.}, {@code 6.2.}, {@code 7.2.1} followed by a name. */
|
||||
private static final Pattern CLAUSE = Pattern.compile("^\\d{1,2}(\\.\\d{1,2})*\\.?\\s+\\p{Lu}");
|
||||
|
||||
/**
|
||||
* Two sentences in a row: a bold run-in lead-in, not a heading. The lower-case letters before
|
||||
* the stop keep it off section numbers.
|
||||
*/
|
||||
private static final Pattern RUNS_ON = Pattern.compile("\\p{Ll}{3}[.!?][\\s\\u00a0]+\\p{Lu}");
|
||||
|
||||
/** Size ratio at which a line is a level-1 heading on size alone. */
|
||||
private static final float H1_RATIO = 1.4f;
|
||||
|
||||
/** Size ratio at which a line is a level-2 heading on size alone. */
|
||||
private static final float H2_RATIO = 1.3f;
|
||||
|
||||
/**
|
||||
* A section number ending in a period. Stricter than {@link #CLAUSE}: only the period tells a
|
||||
* clause from a header's page number.
|
||||
*/
|
||||
private static final Pattern NUMBERED_CLAUSE =
|
||||
Pattern.compile("^\\d{1,2}(\\.\\d{1,2})*\\.\\s+\\p{Lu}");
|
||||
|
||||
/**
|
||||
* True when every cased letter is a capital; digits and uncased scripts do not count, so {@code
|
||||
* BIO 181} qualifies.
|
||||
*/
|
||||
private static boolean isAllCaps(String text) {
|
||||
int upper = 0;
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
char c = text.charAt(i);
|
||||
if (Character.isLowerCase(c)) {
|
||||
return false;
|
||||
}
|
||||
if (Character.isUpperCase(c)) {
|
||||
upper++;
|
||||
}
|
||||
}
|
||||
return upper >= 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* Markdown heading prefix from size, brevity, isolation and weight, never text matching. Bold
|
||||
* is vetoed on the body face itself.
|
||||
*/
|
||||
static String headingPrefix(
|
||||
TextLine line,
|
||||
float medianBodySize,
|
||||
float medianBodyHeight,
|
||||
String bodyFont,
|
||||
boolean isolated) {
|
||||
return headingPrefix(
|
||||
line.text(),
|
||||
line.height(),
|
||||
line.words(),
|
||||
medianBodySize,
|
||||
medianBodyHeight,
|
||||
bodyFont,
|
||||
isolated);
|
||||
}
|
||||
|
||||
/**
|
||||
* Geometry-only overload, judging a merged line on its merged text, height and words. Boldness
|
||||
* counts only when the face differs from {@code bodyFont}.
|
||||
*/
|
||||
static String headingPrefix(
|
||||
String lineText,
|
||||
float lineHeight,
|
||||
List<TextWord> words,
|
||||
float medianBodySize,
|
||||
float medianBodyHeight,
|
||||
String bodyFont,
|
||||
boolean isolated) {
|
||||
String text = lineText.strip();
|
||||
if (text.isEmpty() || tooLongForHeading(text)) {
|
||||
return "";
|
||||
}
|
||||
float ratio = sizeRatio(lineHeight, words, medianBodySize, medianBodyHeight);
|
||||
if (ratio < 0f) {
|
||||
return "";
|
||||
}
|
||||
// A heading names something. A line with no word in it is a value, an equation fragment or
|
||||
// a chart label, however large it is set; a caption names a float, not a section.
|
||||
if (CAPTION.matcher(text).find() || !hasWord(text)) {
|
||||
return "";
|
||||
}
|
||||
if (RUNS_ON.matcher(text).find()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
// Bold only marks a heading when it stands out from the body face. Some documents set
|
||||
// the whole body in a bold-named font, where boldness carries no structural meaning.
|
||||
boolean bold = isBold(words) && !normalisedFont(words).equals(normalise(bodyFont));
|
||||
// A line that reads as a sentence is prose, unless it carries heading typography.
|
||||
if (endsLikeSentence(text) && !bold && ratio <= H2_RATIO) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (ratio > H1_RATIO) {
|
||||
return "# ";
|
||||
}
|
||||
if (ratio > H2_RATIO) {
|
||||
return "## ";
|
||||
}
|
||||
// A numbered clause: the section number is the structure, so it needs no blank line above
|
||||
// it to be one. Requiring a capital after the number keeps ordinary list items out.
|
||||
if (bold && CLAUSE.matcher(text).find()) {
|
||||
return "### ";
|
||||
}
|
||||
// Same size as the body but bold and starting its own block: a run-in section heading.
|
||||
if (bold && isolated && hasWord(text)) {
|
||||
return "### ";
|
||||
}
|
||||
// Some documents give a heading no size and no weight, only capitals. A short, isolated
|
||||
// line set entirely in capitals is one of those.
|
||||
if (isolated
|
||||
&& !endsLikeSentence(text)
|
||||
&& wordCount(text) >= 3
|
||||
&& isAllCaps(text)
|
||||
&& NUMBERED_CLAUSE.matcher(text).find()) {
|
||||
return "### ";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private static float sizeRatio(
|
||||
float lineHeight, List<TextWord> words, float medianBodySize, float medianBodyHeight) {
|
||||
float dominant = dominantFontSize(words);
|
||||
float value;
|
||||
float baseline;
|
||||
if (dominant > 2f && medianBodySize > 2f) {
|
||||
value = dominant;
|
||||
baseline = medianBodySize;
|
||||
} else {
|
||||
float glyph = glyphHeight(words);
|
||||
value = glyph > 0f ? glyph : lineHeight;
|
||||
baseline = medianBodyHeight;
|
||||
}
|
||||
return baseline <= 0f ? -1f : value / baseline;
|
||||
}
|
||||
|
||||
/**
|
||||
* Quantile of a line's glyph heights taken as its size: high enough for the cap band, low
|
||||
* enough that one rogue glyph box cannot set it.
|
||||
*/
|
||||
private static final float GLYPH_HEIGHT_QUANTILE = 0.8f;
|
||||
|
||||
/**
|
||||
* A line's type size from its glyphs, for PDFs encoding visual size in the text matrix; the
|
||||
* line box runs ascender to descender.
|
||||
*/
|
||||
private static float glyphHeight(List<TextWord> words) {
|
||||
int capacity = 0;
|
||||
for (TextWord word : words) {
|
||||
capacity += word.chars().size();
|
||||
}
|
||||
if (capacity == 0) {
|
||||
return 0f;
|
||||
}
|
||||
float[] heights = new float[capacity];
|
||||
int n = 0;
|
||||
for (TextWord word : words) {
|
||||
for (TextChar ch : word.chars()) {
|
||||
if (ch.isWhitespace() || ch.isNewline()) {
|
||||
continue;
|
||||
}
|
||||
// Letters and digits only: brackets and maths operators are drawn taller than the
|
||||
// cap height, so an equation would measure as display type.
|
||||
if (!Character.isLetterOrDigit(ch.toChar())) {
|
||||
continue;
|
||||
}
|
||||
float h = ch.height();
|
||||
if (Float.isFinite(h) && h > 0f) {
|
||||
heights[n++] = h;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (n == 0) {
|
||||
return 0f;
|
||||
}
|
||||
Arrays.sort(heights, 0, n);
|
||||
return heights[(int) (GLYPH_HEIGHT_QUANTILE * (n - 1))];
|
||||
}
|
||||
|
||||
private static String normalise(String fontName) {
|
||||
return SUBSET_TAG.matcher(fontName == null ? "" : fontName).replaceFirst("");
|
||||
}
|
||||
|
||||
private static String normalisedFont(List<TextWord> words) {
|
||||
return normalise(dominantFontName(words));
|
||||
}
|
||||
|
||||
/** Most-used font across the document, weighted by glyph count: the body face. */
|
||||
static String bodyFont(List<PageText> allPages) {
|
||||
Map<String, Integer> counts = new HashMap<>();
|
||||
for (PageText page : allPages) {
|
||||
for (TextChar ch : page.chars()) {
|
||||
if (ch.isWhitespace() || ch.isNewline()) {
|
||||
continue;
|
||||
}
|
||||
String name = ch.fontName();
|
||||
if (name != null && !name.isBlank()) {
|
||||
counts.merge(normalise(name), 1, Integer::sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
String dominant = "";
|
||||
int max = -1;
|
||||
for (Map.Entry<String, Integer> e : counts.entrySet()) {
|
||||
if (e.getValue() > max) {
|
||||
max = e.getValue();
|
||||
dominant = e.getKey();
|
||||
}
|
||||
}
|
||||
return dominant;
|
||||
}
|
||||
|
||||
/** A heading is made of words: at least two letters, and letters at least half the glyphs. */
|
||||
private static boolean hasWord(String text) {
|
||||
int letters = 0;
|
||||
int glyphs = 0;
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
char c = text.charAt(i);
|
||||
if (Character.isWhitespace(c)) {
|
||||
continue;
|
||||
}
|
||||
glyphs++;
|
||||
if (Character.isLetter(c)) {
|
||||
letters++;
|
||||
}
|
||||
}
|
||||
return letters >= 2 && letters * 2 >= glyphs;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a line should be emphasised as bold rather than promoted: bold, short, and not a
|
||||
* full sentence.
|
||||
*/
|
||||
static boolean isBoldLabel(String lineText, List<TextWord> words) {
|
||||
String text = lineText.strip();
|
||||
if (text.isEmpty() || tooLongForHeading(text) || endsLikeSentence(text)) {
|
||||
return false;
|
||||
}
|
||||
if (RUNS_ON.matcher(text).find()) {
|
||||
return false;
|
||||
}
|
||||
return hasWord(text) && isBold(words);
|
||||
}
|
||||
|
||||
private static int wordCount(String text) {
|
||||
return text.split("\\s+").length;
|
||||
}
|
||||
|
||||
/** Whitespace tokens do not measure a line written in a script with no word spaces. */
|
||||
private static boolean tooLongForHeading(String text) {
|
||||
return wordCount(text) > MAX_HEADING_WORDS
|
||||
|| unspacedScriptChars(text) > MAX_HEADING_UNSPACED_CHARS;
|
||||
}
|
||||
|
||||
private static int unspacedScriptChars(String text) {
|
||||
int n = 0;
|
||||
for (int i = 0; i < text.length(); ) {
|
||||
int cp = text.codePointAt(i);
|
||||
if (isUnspacedScript(cp)) {
|
||||
n++;
|
||||
}
|
||||
i += Character.charCount(cp);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
private static boolean isUnspacedScript(int cp) {
|
||||
Character.UnicodeScript s = Character.UnicodeScript.of(cp);
|
||||
return s == Character.UnicodeScript.HAN
|
||||
|| s == Character.UnicodeScript.HIRAGANA
|
||||
|| s == Character.UnicodeScript.KATAKANA
|
||||
|| s == Character.UnicodeScript.THAI
|
||||
|| s == Character.UnicodeScript.LAO
|
||||
|| s == Character.UnicodeScript.KHMER
|
||||
|| s == Character.UnicodeScript.MYANMAR;
|
||||
}
|
||||
|
||||
private static boolean endsLikeSentence(String text) {
|
||||
char last = text.charAt(text.length() - 1);
|
||||
// Ideographic and full-width stops end a sentence exactly as the ASCII ones do.
|
||||
return last == '.'
|
||||
|| last == '!'
|
||||
|| last == '?'
|
||||
|| last == '\u3002'
|
||||
|| last == '\uff01'
|
||||
|| last == '\uff1f'
|
||||
|| last == '\uff61';
|
||||
}
|
||||
|
||||
/** True when the line's dominant font is bold, inferred from PostScript font names. */
|
||||
private static boolean isBold(List<TextWord> words) {
|
||||
String lower = normalisedFont(words).toLowerCase(Locale.ROOT);
|
||||
for (String token : BOLD_TOKENS) {
|
||||
if (lower.contains(token)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return OTHER_BOLD.matcher(lower).find();
|
||||
}
|
||||
|
||||
private static String dominantFontName(List<TextWord> words) {
|
||||
Map<String, Integer> counts = new HashMap<>();
|
||||
for (TextWord word : words) {
|
||||
for (TextChar ch : word.chars()) {
|
||||
if (ch.isWhitespace() || ch.isNewline()) {
|
||||
continue;
|
||||
}
|
||||
String name = ch.fontName();
|
||||
if (name != null && !name.isBlank()) {
|
||||
counts.merge(name, 1, Integer::sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
String dominantFont = "";
|
||||
int max = -1;
|
||||
for (Map.Entry<String, Integer> e : counts.entrySet()) {
|
||||
if (e.getValue() > max) {
|
||||
max = e.getValue();
|
||||
dominantFont = e.getKey();
|
||||
}
|
||||
}
|
||||
return dominantFont;
|
||||
}
|
||||
|
||||
/** Computes the median glyph font size across all pages. */
|
||||
static float medianFontSize(List<PageText> allPages) {
|
||||
List<Float> sizes = new ArrayList<>();
|
||||
for (PageText page : allPages) {
|
||||
for (TextChar ch : page.chars()) {
|
||||
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
|
||||
sizes.add(ch.fontSize());
|
||||
}
|
||||
}
|
||||
}
|
||||
return median(sizes, 12f);
|
||||
}
|
||||
|
||||
/**
|
||||
* Body baseline for the height path: median per-line {@link #glyphHeight} weighted by glyph
|
||||
* count, so tiny axis labels cannot drag it down.
|
||||
*/
|
||||
static float medianLineHeight(List<PageText> allPages) {
|
||||
List<float[]> weighted = new ArrayList<>();
|
||||
double total = 0;
|
||||
for (PageText page : allPages) {
|
||||
for (TextLine line : page.lines()) {
|
||||
if (line.text().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
float h = glyphHeight(line.words());
|
||||
if (h <= 0f) {
|
||||
h = line.height();
|
||||
}
|
||||
if (h <= 0f) {
|
||||
continue;
|
||||
}
|
||||
float w = glyphCount(line);
|
||||
weighted.add(new float[] {h, w});
|
||||
total += w;
|
||||
}
|
||||
}
|
||||
if (weighted.isEmpty()) {
|
||||
return 12f;
|
||||
}
|
||||
weighted.sort(Comparator.comparingDouble(p -> p[0]));
|
||||
double half = total / 2d;
|
||||
double seen = 0;
|
||||
for (float[] p : weighted) {
|
||||
seen += p[1];
|
||||
if (seen >= half) {
|
||||
return p[0];
|
||||
}
|
||||
}
|
||||
return weighted.get(weighted.size() - 1)[0];
|
||||
}
|
||||
|
||||
/** How much text a line carries, in glyphs; at least one so an empty line still counts. */
|
||||
private static float glyphCount(TextLine line) {
|
||||
int glyphs = 0;
|
||||
for (TextWord word : line.words()) {
|
||||
for (TextChar ch : word.chars()) {
|
||||
if (!ch.isWhitespace() && !ch.isNewline()) {
|
||||
glyphs++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return Math.max(1, glyphs);
|
||||
}
|
||||
|
||||
private static float median(List<Float> values, float fallback) {
|
||||
if (values.isEmpty()) {
|
||||
return fallback;
|
||||
}
|
||||
Collections.sort(values);
|
||||
int mid = values.size() / 2;
|
||||
if (values.size() % 2 == 0) {
|
||||
return (values.get(mid - 1) + values.get(mid)) / 2f;
|
||||
}
|
||||
return values.get(mid);
|
||||
}
|
||||
|
||||
/**
|
||||
* The font size appearing most often by character count in the line; ties go to the larger
|
||||
* size.
|
||||
*/
|
||||
private static float dominantFontSize(List<TextWord> words) {
|
||||
Map<Float, Integer> counts = new HashMap<>();
|
||||
for (TextWord word : words) {
|
||||
for (TextChar ch : word.chars()) {
|
||||
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
|
||||
counts.merge(ch.fontSize(), 1, Integer::sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (counts.isEmpty()) {
|
||||
return 0f;
|
||||
}
|
||||
float dominant = 0f;
|
||||
int maxCount = -1;
|
||||
for (Map.Entry<Float, Integer> entry : counts.entrySet()) {
|
||||
int count = entry.getValue();
|
||||
float size = entry.getKey();
|
||||
if (count > maxCount || (count == maxCount && size > dominant)) {
|
||||
maxCount = count;
|
||||
dominant = size;
|
||||
}
|
||||
}
|
||||
return dominant;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* A mutable assembled line: text plus geometry. {@link #left()}/{@link #right()} come from the word
|
||||
* boxes, {@link #glyphLeft()}/{@link #glyphRight()} from the glyphs.
|
||||
*/
|
||||
final class Line {
|
||||
|
||||
String text;
|
||||
float x;
|
||||
float y;
|
||||
float width;
|
||||
float height;
|
||||
final TextLine source;
|
||||
|
||||
/** Extra extractor fragments merged into this line; empty for an unmerged line. */
|
||||
final List<TextLine> merged = new ArrayList<>();
|
||||
|
||||
/** True for a line synthesised from an AcroForm value rather than page content. */
|
||||
boolean synthetic;
|
||||
|
||||
Line(TextLine src) {
|
||||
this(src, src.text());
|
||||
}
|
||||
|
||||
Line(TextLine src, String text) {
|
||||
this.source = src;
|
||||
this.text = text;
|
||||
this.x = src.x();
|
||||
this.y = src.y();
|
||||
this.width = src.width();
|
||||
this.height = src.height();
|
||||
}
|
||||
|
||||
/** Every word on the line, in x order, across all merged fragments. */
|
||||
List<TextWord> words() {
|
||||
if (merged.isEmpty()) {
|
||||
return source.words();
|
||||
}
|
||||
List<TextWord> all = new ArrayList<>(source.words());
|
||||
for (TextLine extra : merged) {
|
||||
all.addAll(extra.words());
|
||||
}
|
||||
all.sort(Comparator.comparingDouble(TextWord::x));
|
||||
return all;
|
||||
}
|
||||
|
||||
/** Text for the heading/bold classifiers; an unmerged line keeps the extractor's own string. */
|
||||
String detectText() {
|
||||
return merged.isEmpty() ? source.text() : text;
|
||||
}
|
||||
|
||||
float detectHeight() {
|
||||
return merged.isEmpty() ? source.height() : height;
|
||||
}
|
||||
|
||||
/** Top edge; PDF y grows upwards, so this is the larger of the two vertical bounds. */
|
||||
float top() {
|
||||
return y + height;
|
||||
}
|
||||
|
||||
float centreY() {
|
||||
return y + height / 2f;
|
||||
}
|
||||
|
||||
float centreX() {
|
||||
return x + width / 2f;
|
||||
}
|
||||
|
||||
/** Left edge of the line's words, falling back to its bounding box when it has none. */
|
||||
float left() {
|
||||
float edge = Float.MAX_VALUE;
|
||||
for (TextWord w : words()) {
|
||||
edge = Math.min(edge, w.x());
|
||||
}
|
||||
return edge == Float.MAX_VALUE ? x : edge;
|
||||
}
|
||||
|
||||
float right() {
|
||||
float edge = -Float.MAX_VALUE;
|
||||
for (TextWord w : words()) {
|
||||
edge = Math.max(edge, w.x() + w.width());
|
||||
}
|
||||
return edge == -Float.MAX_VALUE ? x + width : edge;
|
||||
}
|
||||
|
||||
/** Left edge of the line's glyphs, ignoring any space a word box carries. */
|
||||
float glyphLeft() {
|
||||
float edge = Float.MAX_VALUE;
|
||||
for (TextWord w : words()) {
|
||||
for (TextChar c : w.chars()) {
|
||||
if (!c.isWhitespace() && !c.isNewline()) {
|
||||
edge = Math.min(edge, c.x());
|
||||
}
|
||||
}
|
||||
}
|
||||
return edge == Float.MAX_VALUE ? x : edge;
|
||||
}
|
||||
|
||||
float glyphRight() {
|
||||
float edge = -Float.MAX_VALUE;
|
||||
for (TextWord w : words()) {
|
||||
for (TextChar c : w.chars()) {
|
||||
if (!c.isWhitespace() && !c.isNewline()) {
|
||||
edge = Math.max(edge, c.x() + c.width());
|
||||
}
|
||||
}
|
||||
}
|
||||
return edge == -Float.MAX_VALUE ? x + width : edge;
|
||||
}
|
||||
|
||||
/** Records a fragment folded into this line so its word list still covers the whole extent. */
|
||||
void absorb(TextLine fragment) {
|
||||
merged.add(fragment);
|
||||
}
|
||||
|
||||
void absorb(Line fragment) {
|
||||
merged.add(fragment.source);
|
||||
merged.addAll(fragment.merged);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Rejoins extractor fragments that are really one visual line. Fragments are grouped into rows
|
||||
* first, then joined left to right.
|
||||
*/
|
||||
final class LineMerger {
|
||||
|
||||
private LineMerger() {}
|
||||
|
||||
/** Gap above this many average character widths is a real layout gap, so never merged. */
|
||||
private static final float MAX_MERGE_GAP = 1.60f;
|
||||
|
||||
/**
|
||||
* Rejoins fragments of one visual line: PDFium splits on bounding box, so a run with no
|
||||
* ascender ({@code rou}) lands apart from the rest.
|
||||
*/
|
||||
static List<Line> mergeLineFragments(List<Line> lines, List<Float> gutters) {
|
||||
if (lines.size() < 2) {
|
||||
return lines;
|
||||
}
|
||||
// Merge within each column, so a line ending at the gutter never joins the next column's.
|
||||
if (!gutters.isEmpty()) {
|
||||
List<List<Line>> columns = ColumnLayout.splitIntoColumns(lines, gutters);
|
||||
if (columns.size() > 1) {
|
||||
List<Line> out = new ArrayList<>(lines.size());
|
||||
for (List<Line> column : columns) {
|
||||
out.addAll(mergeRows(column));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
return mergeRows(lines);
|
||||
}
|
||||
|
||||
private static List<Line> mergeRows(List<Line> lines) {
|
||||
if (lines.size() < 2) {
|
||||
return new ArrayList<>(lines);
|
||||
}
|
||||
List<Line> ordered = new ArrayList<>(lines);
|
||||
// Top edge first, so fragments of one visual line arrive together whatever their heights.
|
||||
ordered.sort(Comparator.comparingDouble((Line l) -> -(l.y + l.height)));
|
||||
|
||||
// Group into rows first: a fragment's continuation is its right-hand neighbour on the same
|
||||
// row, not whichever line the extractor happened to emit next.
|
||||
List<List<Line>> rows = new ArrayList<>();
|
||||
for (Line line : ordered) {
|
||||
List<Line> row = null;
|
||||
for (int i = rows.size() - 1; i >= 0 && i >= rows.size() - 3; i--) {
|
||||
if (overlapsRow(rows.get(i), line)) {
|
||||
row = rows.get(i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (row == null) {
|
||||
row = new ArrayList<>();
|
||||
rows.add(row);
|
||||
}
|
||||
row.add(line);
|
||||
}
|
||||
|
||||
List<Line> out = new ArrayList<>();
|
||||
for (List<Line> row : rows) {
|
||||
row.sort(Comparator.comparingDouble((Line l) -> l.x));
|
||||
Line host = null;
|
||||
for (Line line : row) {
|
||||
if (host != null && adjacentOnRow(host, line)) {
|
||||
appendFragment(host, line);
|
||||
} else {
|
||||
out.add(line);
|
||||
host = line;
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** True when a line shares a row with the lines already in it (vertical overlap). */
|
||||
private static boolean overlapsRow(List<Line> row, Line line) {
|
||||
for (Line member : row) {
|
||||
float overlap =
|
||||
Math.min(member.y + member.height, line.y + line.height)
|
||||
- Math.max(member.y, line.y);
|
||||
float minHeight = Math.min(member.height, line.height);
|
||||
if (minHeight > 0f && overlap >= minHeight * 0.5f) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** True when {@code next} sits close enough to {@code host} to be the same visual line. */
|
||||
private static boolean adjacentOnRow(Line host, Line next) {
|
||||
// Merging concatenates left to right, which is reading order for LTR only; RTL fragments
|
||||
// would be joined back to front.
|
||||
if (hasStrongRtl(host.text) || hasStrongRtl(next.text)) {
|
||||
return false;
|
||||
}
|
||||
float gap = next.glyphLeft() - host.glyphRight();
|
||||
float charWidth = fragmentCharWidth(host, next);
|
||||
return gap > -charWidth && gap < charWidth * MAX_MERGE_GAP;
|
||||
}
|
||||
|
||||
/** True when the text contains a Hebrew, Arabic, Syriac or Thaana character. */
|
||||
private static boolean hasStrongRtl(String text) {
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
byte dir = Character.getDirectionality(text.charAt(i));
|
||||
if (dir == Character.DIRECTIONALITY_RIGHT_TO_LEFT
|
||||
|| dir == Character.DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static void appendFragment(Line host, Line next) {
|
||||
float gap = next.glyphLeft() - host.glyphRight();
|
||||
float charWidth = fragmentCharWidth(host, next);
|
||||
String left = host.text.stripTrailing();
|
||||
String right = next.text.stripLeading();
|
||||
boolean space = gap >= charWidth * WordGeometry.NO_SPACE_GAP;
|
||||
host.text = left + (space ? " " : "") + right;
|
||||
host.merged.add(next.source);
|
||||
host.merged.addAll(next.merged);
|
||||
float right0 = Math.max(host.x + host.width, next.x + next.width);
|
||||
float top = Math.max(host.y + host.height, next.y + next.height);
|
||||
host.x = Math.min(host.x, next.x);
|
||||
host.y = Math.min(host.y, next.y);
|
||||
host.width = right0 - host.x;
|
||||
host.height = top - host.y;
|
||||
}
|
||||
|
||||
private static float fragmentCharWidth(Line a, Line b) {
|
||||
double total = 0;
|
||||
int chars = 0;
|
||||
for (Line l : List.of(a, b)) {
|
||||
for (TextWord w : l.words()) {
|
||||
for (TextChar c : w.chars()) {
|
||||
if (!c.isWhitespace() && !c.isNewline() && c.width() > 0f) {
|
||||
total += c.width();
|
||||
chars++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return chars == 0 ? 6f : (float) (total / chars);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Markdown text utilities: escaping extracted text and rebasing the finished document's heading
|
||||
* levels.
|
||||
*/
|
||||
final class MarkdownText {
|
||||
|
||||
private MarkdownText() {}
|
||||
|
||||
/** A Markdown ATX heading at the start of a line, with its level in group 1. */
|
||||
private static final Pattern ATX_HEADING = Pattern.compile("(?m)^(#{1,6}) (?=\\S)");
|
||||
|
||||
/**
|
||||
* Rebases headings so the strongest is level 1 and no level is skipped: levels only mean
|
||||
* anything against the other headings in the same document.
|
||||
*/
|
||||
static String normaliseHeadingLevels(String markdown) {
|
||||
Set<Integer> levels = new TreeSet<>();
|
||||
Matcher m = ATX_HEADING.matcher(markdown);
|
||||
while (m.find()) {
|
||||
levels.add(m.group(1).length());
|
||||
}
|
||||
if (levels.isEmpty() || (levels.contains(1) && levels.size() == maxOf(levels))) {
|
||||
return markdown;
|
||||
}
|
||||
Map<Integer, String> rebased = new HashMap<>();
|
||||
int rank = 1;
|
||||
for (int level : levels) {
|
||||
rebased.put(level, "#".repeat(rank++));
|
||||
}
|
||||
return m.reset().replaceAll(r -> rebased.get(r.group(1).length()) + " ");
|
||||
}
|
||||
|
||||
private static int maxOf(Set<Integer> levels) {
|
||||
int max = 0;
|
||||
for (int level : levels) {
|
||||
max = Math.max(max, level);
|
||||
}
|
||||
return max;
|
||||
}
|
||||
|
||||
static int wordCount(String text) {
|
||||
return text.isBlank() ? 0 : text.strip().split("\\s+").length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Escapes Markdown control characters so extracted text is emitted literally. Output is still
|
||||
* untrusted: this is defence-in-depth, not safe rendering.
|
||||
*/
|
||||
static String escapeMarkdown(String text) {
|
||||
if (text.isEmpty()) {
|
||||
return text;
|
||||
}
|
||||
String inline = escapeMarkdownInline(text);
|
||||
return escapeLeadingBlockMarker(inline, text);
|
||||
}
|
||||
|
||||
/** Escapes inline-significant Markdown characters anywhere in the string. */
|
||||
static String escapeMarkdownInline(String text) {
|
||||
StringBuilder sb = new StringBuilder(text.length() + 8);
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
char c = text.charAt(i);
|
||||
switch (c) {
|
||||
case '\\', '`', '*', '_', '[', ']', '<', '>', '|', '~' -> sb.append('\\').append(c);
|
||||
default -> sb.append(c);
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Escapes markers significant only at line start: {@code #}, {@code -}, {@code +} and
|
||||
* ordered-list numbers. {@code original} is unescaped, so positions line up.
|
||||
*/
|
||||
private static String escapeLeadingBlockMarker(String escaped, String original) {
|
||||
char c0 = original.charAt(0);
|
||||
if (c0 == '#' || c0 == '-' || c0 == '+') {
|
||||
return "\\" + escaped;
|
||||
}
|
||||
int i = 0;
|
||||
while (i < original.length() && Character.isDigit(original.charAt(i))) {
|
||||
i++;
|
||||
}
|
||||
if (i > 0 && i < original.length()) {
|
||||
char delim = original.charAt(i);
|
||||
if (delim == '.' || delim == ')') {
|
||||
return escaped.substring(0, i) + "\\" + escaped.substring(i);
|
||||
}
|
||||
}
|
||||
return escaped;
|
||||
}
|
||||
|
||||
static String normaliseSpace(String s) {
|
||||
return s.strip().replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
static void flushParagraph(StringBuilder para, List<String> out) {
|
||||
if (!para.isEmpty()) {
|
||||
out.add(escapeMarkdown(para.toString()));
|
||||
para.setLength(0);
|
||||
}
|
||||
}
|
||||
|
||||
static boolean endsWithSentencePunctuation(String s) {
|
||||
if (s.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
char last = s.charAt(s.length() - 1);
|
||||
return last == '.' || last == '?' || last == '!' || last == ':';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.PdfPage;
|
||||
import stirling.software.jpdfium.doc.ExtractedImage;
|
||||
import stirling.software.jpdfium.doc.PdfImageExtractor;
|
||||
import stirling.software.jpdfium.model.Rect;
|
||||
|
||||
/**
|
||||
* Emits a placeholder per image on a page; image bytes are deliberately not carried into the
|
||||
* Markdown.
|
||||
*/
|
||||
final class PageImages {
|
||||
|
||||
private PageImages() {}
|
||||
|
||||
static void emit(PdfDocument doc, int pageIndex, List<Object> pageItems) throws IOException {
|
||||
try (PdfPage page = doc.page(pageIndex)) {
|
||||
List<ExtractedImage> images =
|
||||
PdfImageExtractor.extract(page.rawDocHandle(), page.rawHandle(), pageIndex);
|
||||
for (ExtractedImage img : images) {
|
||||
pageItems.add(describe(img));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Image placeholder annotated with whatever JPDFium exposes: pixels, placement, DPI, format,
|
||||
* colour space, depth. Missing fields are omitted.
|
||||
*/
|
||||
private static String describe(ExtractedImage img) {
|
||||
List<String> parts = new ArrayList<>();
|
||||
if (img.width() > 0 && img.height() > 0) {
|
||||
parts.add(img.width() + "x" + img.height() + "px");
|
||||
}
|
||||
Rect b = img.bounds();
|
||||
if (b != null && b.width() > 0 && b.height() > 0) {
|
||||
parts.add(String.format("%.0fx%.0fpt", b.width(), b.height()));
|
||||
if (img.width() > 0) {
|
||||
float dpiX = img.width() / (b.width() / 72f);
|
||||
float dpiY = img.height() / (b.height() / 72f);
|
||||
if (Float.isFinite(dpiX) && dpiX > 0) {
|
||||
parts.add(String.format("~%.0fdpi", (dpiX + dpiY) / 2f));
|
||||
}
|
||||
}
|
||||
}
|
||||
String ext = img.suggestedExtension();
|
||||
if (ext != null && !ext.isBlank()) {
|
||||
parts.add(ext.replaceFirst("^\\.", "").toUpperCase(Locale.ROOT));
|
||||
}
|
||||
if (img.colorSpace() != null) {
|
||||
parts.add(img.colorSpace().toString());
|
||||
}
|
||||
if (img.bitsPerPixel() > 0) {
|
||||
parts.add(img.bitsPerPixel() + "bpp");
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder("<image redacted");
|
||||
if (!parts.isEmpty()) {
|
||||
sb.append(": ").append(String.join(", ", parts));
|
||||
}
|
||||
sb.append('>');
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.jpdfium.PdfPage;
|
||||
import stirling.software.jpdfium.doc.PageContentSummary;
|
||||
import stirling.software.jpdfium.doc.PageObject;
|
||||
import stirling.software.jpdfium.doc.PageObjectType;
|
||||
import stirling.software.jpdfium.doc.PdfPageObjects;
|
||||
import stirling.software.jpdfium.model.Rect;
|
||||
|
||||
/**
|
||||
* Ruling lines of one page, from its {@link PageObject} paths: a long thin box is a rule, and a box
|
||||
* large in both axes contributes its edges.
|
||||
*/
|
||||
final class PageRules {
|
||||
|
||||
/** A path this thin in its short axis is a rule rather than a filled area. */
|
||||
private static final float RULE_THICKNESS = 3f;
|
||||
|
||||
/** A rule shorter than this is decoration (tick marks, bullets, glyph art). */
|
||||
private static final float MIN_RULE_LENGTH = 8f;
|
||||
|
||||
/** Boxes larger than this in either axis are page furniture, not table structure. */
|
||||
private static final float MAX_BOX = 1500f;
|
||||
|
||||
/**
|
||||
* Object count past which the page is skipped: a page drawing this many is a chart or an
|
||||
* operator flood, never a readable grid.
|
||||
*/
|
||||
private static final int MAX_PAGE_OBJECTS = 20_000;
|
||||
|
||||
/** A straight rule: {@code pos} is its y (horizontal) or x (vertical), spanning lo..hi. */
|
||||
record Rule(float pos, float lo, float hi) {}
|
||||
|
||||
private final List<Rule> horizontal;
|
||||
private final List<Rule> vertical;
|
||||
|
||||
private PageRules(List<Rule> horizontal, List<Rule> vertical) {
|
||||
this.horizontal = horizontal;
|
||||
this.vertical = vertical;
|
||||
}
|
||||
|
||||
static final PageRules EMPTY = new PageRules(List.of(), List.of());
|
||||
|
||||
List<Rule> horizontal() {
|
||||
return horizontal;
|
||||
}
|
||||
|
||||
List<Rule> vertical() {
|
||||
return vertical;
|
||||
}
|
||||
|
||||
boolean isEmpty() {
|
||||
return horizontal.isEmpty() && vertical.isEmpty();
|
||||
}
|
||||
|
||||
/** Reads the ruling lines of an already-open page. */
|
||||
static PageRules of(PdfPage page) throws IOException {
|
||||
List<Rule> h = new ArrayList<>();
|
||||
List<Rule> v = new ArrayList<>();
|
||||
List<PageObject> objects;
|
||||
try {
|
||||
// Counting is far cheaper than materialising every object, so decide from the summary
|
||||
// whether the page is worth enumerating at all.
|
||||
PageContentSummary summary = PdfPageObjects.summarize(page.rawHandle());
|
||||
if (summary.pathObjectCount() < 2 || summary.totalObjects() > MAX_PAGE_OBJECTS) {
|
||||
return EMPTY;
|
||||
}
|
||||
objects = PdfPageObjects.list(page.rawHandle());
|
||||
} catch (RuntimeException e) {
|
||||
// Path enumeration is an optimisation, never a correctness requirement: a page whose
|
||||
// objects cannot be read simply falls back to word-grid detection.
|
||||
return EMPTY;
|
||||
}
|
||||
float pageW = 0f;
|
||||
float pageH = 0f;
|
||||
try {
|
||||
pageW = page.size().width();
|
||||
pageH = page.size().height();
|
||||
} catch (RuntimeException ignored) {
|
||||
// Fall through with 0,0: the on-page check below is then skipped.
|
||||
}
|
||||
for (PageObject o : objects) {
|
||||
if (o.type() != PageObjectType.PATH) {
|
||||
continue;
|
||||
}
|
||||
Rect b = o.bounds();
|
||||
if (b == null) {
|
||||
continue;
|
||||
}
|
||||
float w = b.width();
|
||||
float ht = b.height();
|
||||
if (!Float.isFinite(w) || !Float.isFinite(ht) || w < 0 || ht < 0) {
|
||||
continue;
|
||||
}
|
||||
if (w > MAX_BOX || ht > MAX_BOX) {
|
||||
continue;
|
||||
}
|
||||
// Paths that run off the page are chart clipping or decoration, never table structure.
|
||||
if (pageW > 0
|
||||
&& (b.x() < -1f
|
||||
|| b.y() < -1f
|
||||
|| b.x() + w > pageW + 1f
|
||||
|| b.y() + ht > pageH + 1f)) {
|
||||
continue;
|
||||
}
|
||||
if (ht <= RULE_THICKNESS && w >= MIN_RULE_LENGTH) {
|
||||
h.add(new Rule(b.y() + ht / 2f, b.x(), b.x() + w));
|
||||
} else if (w <= RULE_THICKNESS && ht >= MIN_RULE_LENGTH) {
|
||||
v.add(new Rule(b.x() + w / 2f, b.y(), b.y() + ht));
|
||||
} else if (w >= MIN_RULE_LENGTH && ht >= MIN_RULE_LENGTH) {
|
||||
// A box: a table border, a cell outline or a shaded row fill. Its edges bound cells
|
||||
// exactly as drawn rules do, and many generators draw grids as per-cell rectangles.
|
||||
h.add(new Rule(b.y(), b.x(), b.x() + w));
|
||||
h.add(new Rule(b.y() + ht, b.x(), b.x() + w));
|
||||
v.add(new Rule(b.x(), b.y(), b.y() + ht));
|
||||
v.add(new Rule(b.x() + w, b.y(), b.y() + ht));
|
||||
}
|
||||
}
|
||||
h.sort(Comparator.comparingDouble(Rule::pos).reversed());
|
||||
v.sort(Comparator.comparingDouble(Rule::pos));
|
||||
return new PageRules(h, v);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Joins what a page break split: a sentence running into the next page, and a table whose rows
|
||||
* continue on it.
|
||||
*/
|
||||
final class PageStitcher {
|
||||
|
||||
private PageStitcher() {}
|
||||
|
||||
static void mergeAcrossPageBoundary(List<Object> output, List<Object> pageItems) {
|
||||
if (output.isEmpty() || pageItems.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
// Only merge a sentence continuation between two text paragraphs, never into/out of a
|
||||
// table.
|
||||
if (!(output.getLast() instanceof String last)
|
||||
|| !(pageItems.getFirst() instanceof String first)) {
|
||||
return;
|
||||
}
|
||||
if (!first.isEmpty()
|
||||
&& Character.isLowerCase(first.charAt(0))
|
||||
&& !MarkdownText.endsWithSentencePunctuation(last)) {
|
||||
output.set(output.size() - 1, last + " " + first);
|
||||
pageItems.remove(0);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Joins tables split across a page break: two consecutive blocks with no text between them
|
||||
* merge when their column layouts match, dropping a repeated header.
|
||||
*/
|
||||
static List<Object> stitchTables(List<Object> elements) {
|
||||
List<Object> out = new ArrayList<>();
|
||||
// Column geometry of the trailing TableBlock in `out`, carried forward across merges so a
|
||||
// table running page-to-page is not re-projected from every accumulated row at each break.
|
||||
ColumnAccumulator acc = null;
|
||||
// Row list we own and may append to in place; null while the trailing block still holds a
|
||||
// list belonging to `elements`.
|
||||
List<List<Line>> ownedRows = null;
|
||||
for (Object e : elements) {
|
||||
if (e instanceof TableBlock tb
|
||||
&& !out.isEmpty()
|
||||
&& out.getLast() instanceof TableBlock prev) {
|
||||
if (acc == null) {
|
||||
acc = ColumnAccumulator.of(prev.rows());
|
||||
}
|
||||
if (columnsMatch(acc.columns(), ColumnRanges.find(flatten(tb.rows())))) {
|
||||
List<List<Line>> merged;
|
||||
if (ownedRows == null) {
|
||||
merged = new ArrayList<>(prev.rows());
|
||||
ownedRows = merged;
|
||||
} else {
|
||||
merged = ownedRows;
|
||||
}
|
||||
List<List<Line>> tail = tb.rows();
|
||||
if (!tail.isEmpty()
|
||||
&& !prev.rows().isEmpty()
|
||||
&& rowText(tail.getFirst()).equals(rowText(prev.rows().getFirst()))) {
|
||||
tail = tail.subList(1, tail.size());
|
||||
}
|
||||
for (List<Line> row : tail) {
|
||||
for (Line l : row) {
|
||||
acc.addLine(l);
|
||||
}
|
||||
}
|
||||
merged.addAll(tail);
|
||||
// A stitched table belongs to where it started, so keep the earlier block's
|
||||
// page and columns; its ruling lines are dropped as they are one page's only.
|
||||
out.set(
|
||||
out.size() - 1,
|
||||
new TableBlock(
|
||||
merged,
|
||||
prev.top(),
|
||||
tb.bottom(),
|
||||
prev.cols(),
|
||||
prev.ruled(),
|
||||
prev.rowSource(),
|
||||
prev.page()));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
out.add(e);
|
||||
acc = null;
|
||||
ownedRows = null;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static List<Line> flatten(List<List<Line>> rows) {
|
||||
return rows.stream().flatMap(List::stream).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* Header text of a table at the very bottom of a page, or null. Trailing image placeholders are
|
||||
* skipped; any other text means it is not a continuation.
|
||||
*/
|
||||
static String trailingTableHeader(List<Object> pageItems) {
|
||||
for (int i = pageItems.size() - 1; i >= 0; i--) {
|
||||
Object e = pageItems.get(i);
|
||||
if (e instanceof String s && s.strip().startsWith("<image redacted")) {
|
||||
continue;
|
||||
}
|
||||
if (e instanceof TableBlock tb && !tb.rows().isEmpty()) {
|
||||
return rowText(tb.rows().getFirst());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static String rowText(List<Line> row) {
|
||||
List<Line> ordered = new ArrayList<>(row);
|
||||
ordered.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (Line l : ordered) {
|
||||
if (!sb.isEmpty()) {
|
||||
sb.append(' ');
|
||||
}
|
||||
sb.append(l.text);
|
||||
}
|
||||
return MarkdownText.normaliseSpace(sb.toString());
|
||||
}
|
||||
|
||||
/** True when two table blocks have the same number of columns at near-identical x-centres. */
|
||||
private static boolean columnsMatch(List<float[]> ca, List<float[]> cb) {
|
||||
if (ca.size() < 2 || ca.size() != cb.size()) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < ca.size(); i++) {
|
||||
float centreA = (ca.get(i)[0] + ca.get(i)[1]) / 2f;
|
||||
float centreB = (cb.get(i)[0] + cb.get(i)[1]) / 2f;
|
||||
if (Math.abs(centreA - centreB) > 15f) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
+252
@@ -0,0 +1,252 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Turns a column's lines into Markdown blocks: headings (including wrapped ones), bullets, bold
|
||||
* labels and paragraphs. Contents lists are recognised and left alone.
|
||||
*/
|
||||
final class ParagraphAssembler {
|
||||
|
||||
private ParagraphAssembler() {}
|
||||
|
||||
static void assembleParagraphs(
|
||||
List<Line> lines,
|
||||
float medianSize,
|
||||
float medianHeight,
|
||||
String bodyFont,
|
||||
List<String> out,
|
||||
Set<String> tableRowTexts) {
|
||||
StringBuilder para = new StringBuilder();
|
||||
float prevBottomY = Float.MAX_VALUE;
|
||||
float prevHeight = 0f;
|
||||
boolean[] inContents = contentsRun(lines);
|
||||
|
||||
for (int i = 0; i < lines.size(); i++) {
|
||||
Line line = lines.get(i);
|
||||
String text = line.text.strip();
|
||||
if (text.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (tableRowTexts.contains(text)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
float blockTop = line.y + line.height;
|
||||
float gap = prevBottomY - blockTop;
|
||||
boolean paragraphBreak = prevHeight > 0f && gap > prevHeight * 0.8f;
|
||||
// A contents entry carries the typography of the section it points at without being
|
||||
// that section, so nothing on one is promoted or emphasised.
|
||||
boolean structural = inContents[i];
|
||||
|
||||
// A field value is data, never a heading: its widget box is taller than a text line
|
||||
// and would otherwise be promoted purely on height.
|
||||
String prefix =
|
||||
line.synthetic || structural
|
||||
? ""
|
||||
: HeadingDetector.headingPrefix(
|
||||
line.detectText(),
|
||||
line.detectHeight(),
|
||||
line.words(),
|
||||
medianSize,
|
||||
medianHeight,
|
||||
bodyFont,
|
||||
prevHeight <= 0f || paragraphBreak);
|
||||
if (prefix.isEmpty() && !structural && contentsTitle(lines, inContents, i)) {
|
||||
// The line a contents list runs on from is its heading: a contents page is often
|
||||
// set in one face, leaving no size or weight to promote it on.
|
||||
prefix = "# ";
|
||||
}
|
||||
boolean isBullet = startsWithBullet(text);
|
||||
// A line that opens with a list marker is an item of a list, whatever it is set in.
|
||||
boolean isHeading = !prefix.isEmpty() && !isBullet;
|
||||
|
||||
if (isHeading) {
|
||||
MarkdownText.flushParagraph(para, out);
|
||||
StringBuilder heading = new StringBuilder(MarkdownText.escapeMarkdown(text));
|
||||
int words = MarkdownText.wordCount(text);
|
||||
int j = i;
|
||||
int k = i + 1;
|
||||
while (k < lines.size() && words < MAX_WRAPPED_HEADING_WORDS) {
|
||||
Line next = lines.get(k);
|
||||
String nt = next.text.strip();
|
||||
if (nt.isEmpty()) {
|
||||
// An empty extractor record is not a break in the text; the vertical
|
||||
// gap below decides whether the heading ended.
|
||||
k++;
|
||||
continue;
|
||||
}
|
||||
if (inContents[k] || tableRowTexts.contains(nt)) {
|
||||
break;
|
||||
}
|
||||
if (!wrapsHeading(
|
||||
lines.get(j), next, prefix, medianSize, medianHeight, bodyFont)) {
|
||||
break;
|
||||
}
|
||||
heading.append(' ').append(MarkdownText.escapeMarkdown(nt));
|
||||
words += MarkdownText.wordCount(nt);
|
||||
j = k;
|
||||
k++;
|
||||
}
|
||||
out.add(prefix + heading);
|
||||
if (j > i) {
|
||||
i = j;
|
||||
line = lines.get(j);
|
||||
}
|
||||
} else if (isBullet) {
|
||||
MarkdownText.flushParagraph(para, out);
|
||||
out.add(MarkdownText.escapeMarkdown(text));
|
||||
} else if (!line.synthetic
|
||||
&& !structural
|
||||
&& HeadingDetector.isBoldLabel(line.detectText(), line.words())) {
|
||||
// Bold but not large enough to be a heading → emphasise as bold, don't promote.
|
||||
MarkdownText.flushParagraph(para, out);
|
||||
out.add("**" + MarkdownText.escapeMarkdown(text) + "**");
|
||||
} else if (paragraphBreak) {
|
||||
MarkdownText.flushParagraph(para, out);
|
||||
para.append(text);
|
||||
} else {
|
||||
if (!para.isEmpty()) {
|
||||
char fc = text.charAt(0);
|
||||
boolean noSpace = fc == '\'' || fc == '’' || fc == '‘' || fc == '"';
|
||||
if (!noSpace) {
|
||||
para.append(' ');
|
||||
}
|
||||
}
|
||||
para.append(text);
|
||||
}
|
||||
|
||||
prevBottomY = line.y;
|
||||
prevHeight = line.height;
|
||||
}
|
||||
MarkdownText.flushParagraph(para, out);
|
||||
}
|
||||
|
||||
/** Glyphs a document may set its list markers in beyond the three already recognised. */
|
||||
private static final String EXTRA_BULLETS = "‣⁃▶●○■□" + "◆⮚➢➣➤";
|
||||
|
||||
private static boolean startsWithBullet(String text) {
|
||||
if (text.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
if (text.startsWith("•") || text.startsWith("▪") || text.startsWith("◦")) {
|
||||
return true;
|
||||
}
|
||||
return EXTRA_BULLETS.indexOf(text.charAt(0)) >= 0;
|
||||
}
|
||||
|
||||
/** Longest a heading may grow to by absorbing its continuation lines, in words. */
|
||||
private static final int MAX_WRAPPED_HEADING_WORDS = 24;
|
||||
|
||||
/** How far a continuation line's type size may differ from the line it continues. */
|
||||
private static final float WRAP_SIZE_TOLERANCE = 0.2f;
|
||||
|
||||
/** A full stop that a further sentence follows: the shape of prose, not of a heading. */
|
||||
private static final Pattern SENTENCE_BREAK = Pattern.compile("[.!?]\\s+\\p{Lu}");
|
||||
|
||||
/**
|
||||
* True when {@code next} continues a wrapped heading: each visual line arrives separately, so
|
||||
* an unjoined heading emits as several spurious ones.
|
||||
*/
|
||||
private static boolean wrapsHeading(
|
||||
Line head,
|
||||
Line next,
|
||||
String prefix,
|
||||
float medianSize,
|
||||
float medianHeight,
|
||||
String bodyFont) {
|
||||
if (next.synthetic) {
|
||||
return false;
|
||||
}
|
||||
float height = head.detectHeight();
|
||||
if (height <= 0f) {
|
||||
return false;
|
||||
}
|
||||
// The next baseline down, not the next block. The same 0.8 the paragraph assembler uses,
|
||||
// so a heading absorbs exactly what the converter already calls one block.
|
||||
float gap = head.y - (next.y + next.height);
|
||||
if (gap > height * 0.8f || gap < -height * 0.5f) {
|
||||
return false;
|
||||
}
|
||||
float nextHeight = next.detectHeight();
|
||||
if (Math.abs(nextHeight - height) > WRAP_SIZE_TOLERANCE * Math.max(nextHeight, height)) {
|
||||
return false;
|
||||
}
|
||||
// Same column: an x-range that misses the heading's belongs to another block entirely.
|
||||
if (next.x >= head.x + head.width || head.x >= next.x + next.width) {
|
||||
return false;
|
||||
}
|
||||
// A heading does not run to a full stop and then start another sentence; the bold run-in
|
||||
// lead-in below it does, and nothing else tells the two apart.
|
||||
if (SENTENCE_BREAK.matcher(next.text).find()) {
|
||||
return false;
|
||||
}
|
||||
String nextPrefix =
|
||||
HeadingDetector.headingPrefix(
|
||||
next.detectText(),
|
||||
next.detectHeight(),
|
||||
next.words(),
|
||||
medianSize,
|
||||
medianHeight,
|
||||
bodyFont,
|
||||
false);
|
||||
// Either the continuation is display type in its own right, or it is the bold remainder of
|
||||
// a run-in heading, which cannot be promoted on its own because no gap precedes it.
|
||||
return nextPrefix.equals(prefix)
|
||||
|| (nextPrefix.isEmpty()
|
||||
&& HeadingDetector.isBoldLabel(next.detectText(), next.words()));
|
||||
}
|
||||
|
||||
/** A leader run: the dots that carry the eye from a contents entry to its page number. */
|
||||
private static final Pattern LEADER = Pattern.compile("([.][ ]?){4,}|[.\u00b7]{3,}|\u2026{2,}");
|
||||
|
||||
/** Entries this many lines long make a contents list rather than a coincidence. */
|
||||
private static final int MIN_CONTENTS_RUN = 3;
|
||||
|
||||
/**
|
||||
* Marks the lines of a contents list: titles joined to page numbers by leader dots, carrying
|
||||
* the typography of the sections they point at.
|
||||
*/
|
||||
private static boolean[] contentsRun(List<Line> lines) {
|
||||
boolean[] entry = new boolean[lines.size()];
|
||||
int run = 0;
|
||||
for (int i = 0; i < lines.size(); i++) {
|
||||
String t = lines.get(i).text;
|
||||
if (LEADER.matcher(t).find() && endsWithNumber(t)) {
|
||||
entry[i] = true;
|
||||
run++;
|
||||
} else {
|
||||
if (run < MIN_CONTENTS_RUN) {
|
||||
clear(entry, i - run, i);
|
||||
}
|
||||
run = 0;
|
||||
}
|
||||
}
|
||||
if (run < MIN_CONTENTS_RUN) {
|
||||
clear(entry, lines.size() - run, lines.size());
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
private static void clear(boolean[] flags, int from, int to) {
|
||||
for (int i = Math.max(0, from); i < to; i++) {
|
||||
flags[i] = false;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean endsWithNumber(String text) {
|
||||
String t = text.strip();
|
||||
return !t.isEmpty() && Character.isDigit(t.charAt(t.length() - 1));
|
||||
}
|
||||
|
||||
/** True for the short line a contents list runs on from: the list's own heading. */
|
||||
private static boolean contentsTitle(List<Line> lines, boolean[] inContents, int index) {
|
||||
if (index + 1 >= lines.size() || inContents[index] || !inContents[index + 1]) {
|
||||
return false;
|
||||
}
|
||||
String t = lines.get(index).text.strip();
|
||||
return !t.isEmpty() && t.split(" +").length <= 6 && !endsWithNumber(t);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
/**
|
||||
* How much of a block's row structure the page itself drew; stronger evidence means weaker
|
||||
* false-positive guards.
|
||||
*/
|
||||
enum RowSource {
|
||||
/** Rows inferred from word geometry alone; nothing on the page confirms a table. */
|
||||
WORDS,
|
||||
/** Rows sit inside a region fenced by drawn rules, but the rules do not delimit them. */
|
||||
RULE_BOUNDED,
|
||||
/** Every row boundary is a drawn rule running the table's own width. */
|
||||
LATTICE;
|
||||
|
||||
boolean ruleConfirmed() {
|
||||
return this != WORDS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* A page's ruling lines reduced to a grid: rules merged into levels, levels grouped into the
|
||||
* components that each describe one table. Both steps are bounded.
|
||||
*/
|
||||
@Slf4j
|
||||
final class RuleGrid {
|
||||
|
||||
/** Rules within this distance are the same drawn line (double strokes, overdraw). */
|
||||
static final float LEVEL_TOLERANCE = 2.5f;
|
||||
|
||||
/** Slack when testing whether a horizontal and a vertical rule touch. */
|
||||
private static final float TOUCH = 3f;
|
||||
|
||||
/** Segments at one position further apart than this belong to different tables. */
|
||||
private static final float CONTIGUOUS_GAP = 8f;
|
||||
|
||||
/** Crossing tests past which a page is an operator flood rather than a readable grid. */
|
||||
private static final long MAX_CROSSING_TESTS = 4_000_000L;
|
||||
|
||||
/** Rule components past which the extra blocks cannot be real tables. */
|
||||
private static final int MAX_COMPONENTS = 256;
|
||||
|
||||
private RuleGrid() {}
|
||||
|
||||
/** A group of rules at the same position: {@code pos} with the union of their extents. */
|
||||
record Level(float pos, float lo, float hi) {}
|
||||
|
||||
/** One connected component of crossing rules: the levels of each family it spans. */
|
||||
record Component(List<Level> h, List<Level> v) {}
|
||||
|
||||
/**
|
||||
* Merges rules at the same position into levels, but only while contiguous, so two tables
|
||||
* ruling at the same x are not bridged into one region.
|
||||
*/
|
||||
static List<Level> cluster(List<PageRules.Rule> rules) {
|
||||
List<PageRules.Rule> sorted = new ArrayList<>(rules);
|
||||
sorted.sort(
|
||||
Comparator.comparingDouble(PageRules.Rule::pos)
|
||||
.thenComparingDouble(PageRules.Rule::lo));
|
||||
List<Level> out = new ArrayList<>();
|
||||
int i = 0;
|
||||
while (i < sorted.size()) {
|
||||
float pos = sorted.get(i).pos();
|
||||
int j = i;
|
||||
while (j < sorted.size() && sorted.get(j).pos() - pos <= LEVEL_TOLERANCE) {
|
||||
j++;
|
||||
}
|
||||
List<PageRules.Rule> same = new ArrayList<>(sorted.subList(i, j));
|
||||
same.sort(Comparator.comparingDouble(PageRules.Rule::lo));
|
||||
float lo = same.get(0).lo();
|
||||
float hi = same.get(0).hi();
|
||||
for (int k = 1; k < same.size(); k++) {
|
||||
if (same.get(k).lo() <= hi + CONTIGUOUS_GAP) {
|
||||
hi = Math.max(hi, same.get(k).hi());
|
||||
} else {
|
||||
out.add(new Level(pos, lo, hi));
|
||||
lo = same.get(k).lo();
|
||||
hi = same.get(k).hi();
|
||||
}
|
||||
}
|
||||
out.add(new Level(pos, lo, hi));
|
||||
i = j;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Connected components of crossing rules, read from one union-find array: an id array per
|
||||
* component is O(components x levels) a rule flood can exhaust.
|
||||
*/
|
||||
static List<Component> partition(List<Level> hLevels, List<Level> vLevels) {
|
||||
int n = hLevels.size() + vLevels.size();
|
||||
if ((long) hLevels.size() * vLevels.size() > MAX_CROSSING_TESTS) {
|
||||
log.debug(
|
||||
"ruled-table partition skipped: {}x{} rule levels",
|
||||
hLevels.size(),
|
||||
vLevels.size());
|
||||
return List.of();
|
||||
}
|
||||
int[] parent = new int[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
parent[i] = i;
|
||||
}
|
||||
for (int i = 0; i < hLevels.size(); i++) {
|
||||
Level h = hLevels.get(i);
|
||||
for (int j = 0; j < vLevels.size(); j++) {
|
||||
Level v = vLevels.get(j);
|
||||
boolean crosses =
|
||||
v.pos() >= h.lo() - TOUCH
|
||||
&& v.pos() <= h.hi() + TOUCH
|
||||
&& h.pos() >= v.lo() - TOUCH
|
||||
&& h.pos() <= v.hi() + TOUCH;
|
||||
if (crosses) {
|
||||
union(parent, i, hLevels.size() + j);
|
||||
}
|
||||
}
|
||||
}
|
||||
Map<Integer, Component> byRoot = new LinkedHashMap<>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
int root = find(parent, i);
|
||||
Component c = byRoot.get(root);
|
||||
if (c == null) {
|
||||
// Past the cap the page is line art, not tables; keep the components already
|
||||
// found whole rather than truncating them mid-scan.
|
||||
if (byRoot.size() >= MAX_COMPONENTS) {
|
||||
continue;
|
||||
}
|
||||
c = new Component(new ArrayList<>(), new ArrayList<>());
|
||||
byRoot.put(root, c);
|
||||
}
|
||||
if (i < hLevels.size()) {
|
||||
c.h().add(hLevels.get(i));
|
||||
} else {
|
||||
c.v().add(vLevels.get(i - hLevels.size()));
|
||||
}
|
||||
}
|
||||
return List.copyOf(byRoot.values());
|
||||
}
|
||||
|
||||
private static int find(int[] parent, int x) {
|
||||
while (parent[x] != x) {
|
||||
parent[x] = parent[parent[x]];
|
||||
x = parent[x];
|
||||
}
|
||||
return x;
|
||||
}
|
||||
|
||||
private static void union(int[] parent, int a, int b) {
|
||||
int ra = find(parent, a);
|
||||
int rb = find(parent, b);
|
||||
if (ra != rb) {
|
||||
parent[rb] = ra;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Visible for testing: partitioning depends only on rule geometry, so tests can drive it from
|
||||
* synthetic rules.
|
||||
*/
|
||||
static int componentCount(List<PageRules.Rule> horizontal, List<PageRules.Rule> vertical) {
|
||||
return partition(cluster(horizontal), cluster(vertical)).size();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Groups the lines inside a ruled region into rows, and reads its column bands off the vertical
|
||||
* rules.
|
||||
*/
|
||||
final class RuledRows {
|
||||
|
||||
private RuledRows() {}
|
||||
|
||||
/** A vertical rule must cover this fraction of a region's height to be a column boundary. */
|
||||
private static final float COLUMN_COVERAGE = 0.5f;
|
||||
|
||||
/**
|
||||
* Splits a band whose every baseline is a complete row back into those rows; a wrapped cell
|
||||
* leaves the other columns empty, a run of rows does not.
|
||||
*/
|
||||
static List<List<Line>> splitCompleteBands(List<List<Line>> bands, List<float[]> cols) {
|
||||
if (cols == null || cols.size() < 2) {
|
||||
return bands;
|
||||
}
|
||||
List<List<Line>> out = new ArrayList<>();
|
||||
for (List<Line> band : bands) {
|
||||
List<List<Line>> baselines = baselineRows(band);
|
||||
if (baselines.size() < 2 || !allRowsComplete(baselines, cols)) {
|
||||
out.add(band);
|
||||
continue;
|
||||
}
|
||||
out.addAll(baselines);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** True when every baseline group puts a word in every column band. */
|
||||
private static boolean allRowsComplete(List<List<Line>> baselines, List<float[]> cols) {
|
||||
for (List<Line> row : baselines) {
|
||||
boolean[] hit = new boolean[cols.size()];
|
||||
for (Line l : row) {
|
||||
for (TextWord w : l.words()) {
|
||||
if (w.text().strip().isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
int c = TableGrid.containingColumn(w.x() + w.width() / 2f, cols);
|
||||
if (c >= 0 && c < hit.length) {
|
||||
hit[c] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (boolean h : hit) {
|
||||
if (!h) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Column bands from the vertical rules spanning the region; null when no interior rule
|
||||
* survives, as whitespace projection guesses better.
|
||||
*/
|
||||
static List<float[]> columns(
|
||||
List<RuleGrid.Level> vLevels, float left, float right, float top, float bottom) {
|
||||
float height = top - bottom;
|
||||
// Per-cell strokes give one rule per row, and a row that draws no boxes breaks the run
|
||||
// in two, so strokes at one x are measured together rather than as separate runs.
|
||||
List<RuleGrid.Level> sorted = new ArrayList<>(vLevels);
|
||||
sorted.sort(Comparator.comparingDouble(RuleGrid.Level::pos));
|
||||
List<Float> xs = new ArrayList<>();
|
||||
int at = 0;
|
||||
while (at < sorted.size()) {
|
||||
float pos = sorted.get(at).pos();
|
||||
float covered = 0f;
|
||||
int end = at;
|
||||
while (end < sorted.size() && sorted.get(end).pos() - pos <= RuleGrid.LEVEL_TOLERANCE) {
|
||||
RuleGrid.Level v = sorted.get(end);
|
||||
covered += Math.max(0f, Math.min(top, v.hi()) - Math.max(bottom, v.lo()));
|
||||
end++;
|
||||
}
|
||||
if (covered >= height * COLUMN_COVERAGE) {
|
||||
xs.add(pos);
|
||||
}
|
||||
at = end;
|
||||
}
|
||||
List<Float> bounds = new ArrayList<>();
|
||||
bounds.add(left);
|
||||
for (float x : xs) {
|
||||
if (x > bounds.get(bounds.size() - 1) + RuleGrid.LEVEL_TOLERANCE
|
||||
&& x < right - RuleGrid.LEVEL_TOLERANCE) {
|
||||
bounds.add(x);
|
||||
}
|
||||
}
|
||||
if (bounds.size() < 2) {
|
||||
return null;
|
||||
}
|
||||
bounds.add(right);
|
||||
List<float[]> cols = new ArrayList<>();
|
||||
for (int i = 1; i < bounds.size(); i++) {
|
||||
cols.add(new float[] {bounds.get(i - 1), bounds.get(i)});
|
||||
}
|
||||
return cols;
|
||||
}
|
||||
|
||||
/** Rows delimited by horizontal rules; this is what keeps a wrapped cell as one row. */
|
||||
static List<List<Line>> latticeRows(List<Float> bands, List<Line> inside) {
|
||||
List<List<Line>> rows = new ArrayList<>();
|
||||
for (int i = 1; i < bands.size(); i++) {
|
||||
float hi = bands.get(i - 1);
|
||||
float lo = bands.get(i);
|
||||
List<Line> band = new ArrayList<>();
|
||||
for (Line l : inside) {
|
||||
float cy = l.y + l.height / 2f;
|
||||
if (cy > lo && cy <= hi) {
|
||||
band.add(l);
|
||||
}
|
||||
}
|
||||
if (!band.isEmpty()) {
|
||||
rows.add(band);
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** Rows by baseline proximity, for a table ruled between its columns but not its rows. */
|
||||
static List<List<Line>> baselineRows(List<Line> inside) {
|
||||
List<Line> sorted = new ArrayList<>(inside);
|
||||
sorted.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
|
||||
List<Float> heights = sorted.stream().map(l -> l.height).sorted().toList();
|
||||
float sameRow = Math.max(2f, heights.get(heights.size() / 2) * 0.6f);
|
||||
List<List<Line>> rows = new ArrayList<>();
|
||||
List<Line> current = new ArrayList<>();
|
||||
float anchor = 0f;
|
||||
for (Line l : sorted) {
|
||||
if (current.isEmpty()) {
|
||||
anchor = l.y;
|
||||
} else if (anchor - l.y > sameRow) {
|
||||
rows.add(current);
|
||||
current = new ArrayList<>();
|
||||
anchor = l.y;
|
||||
}
|
||||
current.add(l);
|
||||
}
|
||||
if (!current.isEmpty()) {
|
||||
rows.add(current);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Builds table blocks from a page's ruling lines: whitespace projection cannot see single-word or
|
||||
* wrapped cells, as they leave no wide gap.
|
||||
*/
|
||||
@Slf4j
|
||||
final class RuledTables {
|
||||
|
||||
private RuledTables() {}
|
||||
|
||||
/** Largest vertical gap between two rules of one rows-only table. */
|
||||
private static final float ROWS_ONLY_GAP = 150f;
|
||||
|
||||
/** How far two rules of one rows-only table may differ at either end. */
|
||||
private static final float EXTENT_TOLERANCE = 8f;
|
||||
|
||||
/** Lines a rows-only group needs before two rules alone are enough to call it a table. */
|
||||
private static final int ROWS_ONLY_LINES = 4;
|
||||
|
||||
/** Fraction of a lattice's row bands that must contain text for it to be a real table. */
|
||||
private static final float FILLED_BANDS = 0.6f;
|
||||
|
||||
/** Fraction of the table's width an interior rule must run to be a row boundary. */
|
||||
private static final float ROW_RULE_SPAN = 0.8f;
|
||||
|
||||
/**
|
||||
* Interior row rules needed before drawn bands beat text baselines; bands keep a multi-line
|
||||
* cell whole where baselines split it.
|
||||
*/
|
||||
private static final int MIN_INTERIOR_RULES = 1;
|
||||
|
||||
/** Fraction of a region's width every rule must run for its rows to be a drawn lattice. */
|
||||
private static final float FULL_WIDTH_RULE = 0.8f;
|
||||
|
||||
private static TableBlock dbgNull(String why) {
|
||||
log.debug("ruled-table build rejected: {}", why);
|
||||
return null;
|
||||
}
|
||||
|
||||
static List<TableBlock> find(List<Line> lines, PageRules rules, int page) {
|
||||
if (rules == null || rules.isEmpty() || lines.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
// Synthetic AcroForm values carry no glyphs of their own, so a ruled grid must not
|
||||
// claim them: they would seed rows and columns the content stream never drew.
|
||||
lines = lines.stream().filter(l -> !l.synthetic).toList();
|
||||
if (lines.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<RuleGrid.Level> hLevels = RuleGrid.cluster(rules.horizontal());
|
||||
List<RuleGrid.Level> vLevels = RuleGrid.cluster(rules.vertical());
|
||||
if (hLevels.size() < 2) {
|
||||
return List.of();
|
||||
}
|
||||
List<TableBlock> blocks = new ArrayList<>();
|
||||
for (RuleGrid.Component part : RuleGrid.partition(hLevels, vLevels)) {
|
||||
TableBlock b = build(part.h(), part.v(), lines, page);
|
||||
if (b != null) {
|
||||
blocks.add(b);
|
||||
}
|
||||
}
|
||||
|
||||
// Horizontal rules no grid block claimed can still be a booktabs table: rows ruled,
|
||||
// columns not drawn at all. Whatever the grid did not take is offered to that reading.
|
||||
List<RuleGrid.Level> unclaimed = new ArrayList<>();
|
||||
for (RuleGrid.Level h : hLevels) {
|
||||
boolean claimed = false;
|
||||
for (TableBlock b : blocks) {
|
||||
if (h.pos() >= b.bottom() - RuleGrid.LEVEL_TOLERANCE
|
||||
&& h.pos() <= b.top() + RuleGrid.LEVEL_TOLERANCE) {
|
||||
claimed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!claimed) {
|
||||
unclaimed.add(h);
|
||||
}
|
||||
}
|
||||
if (unclaimed.size() >= 2) {
|
||||
for (TableBlock b : rowsOnly(unclaimed, lines, page)) {
|
||||
boolean overlaps = false;
|
||||
for (TableBlock existing : blocks) {
|
||||
if (TableFinder.covers(existing, b)) {
|
||||
overlaps = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!overlaps) {
|
||||
blocks.add(b);
|
||||
}
|
||||
}
|
||||
}
|
||||
blocks.sort(Comparator.comparingDouble(TableBlock::top).reversed());
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Blocks for a page ruled only across its rows (booktabs): no column geometry to recover, so
|
||||
* these only find a table the word grid missed.
|
||||
*/
|
||||
private static List<TableBlock> rowsOnly(
|
||||
List<RuleGrid.Level> levels, List<Line> lines, int page) {
|
||||
List<RuleGrid.Level> hLevels = new ArrayList<>(levels);
|
||||
hLevels.sort(Comparator.comparingDouble(RuleGrid.Level::pos).reversed());
|
||||
List<TableBlock> blocks = new ArrayList<>();
|
||||
List<List<RuleGrid.Level>> groups = new ArrayList<>();
|
||||
List<RuleGrid.Level> current = new ArrayList<>();
|
||||
current.add(hLevels.get(0));
|
||||
for (int i = 1; i < hLevels.size(); i++) {
|
||||
RuleGrid.Level prev = current.get(current.size() - 1);
|
||||
RuleGrid.Level l = hLevels.get(i);
|
||||
// One booktabs table rules to a single extent; two stacked tables differ in width,
|
||||
// and grouping them would project their columns into a single band.
|
||||
if (prev.pos() - l.pos() > ROWS_ONLY_GAP
|
||||
|| Math.abs(prev.lo() - l.lo()) > EXTENT_TOLERANCE
|
||||
|| Math.abs(prev.hi() - l.hi()) > EXTENT_TOLERANCE) {
|
||||
groups.add(current);
|
||||
current = new ArrayList<>();
|
||||
}
|
||||
current.add(l);
|
||||
}
|
||||
groups.add(current);
|
||||
|
||||
for (List<RuleGrid.Level> g : groups) {
|
||||
if (g.size() < 2) {
|
||||
continue;
|
||||
}
|
||||
float top = g.get(0).pos();
|
||||
float bottom = g.get(g.size() - 1).pos();
|
||||
float left = Float.MAX_VALUE;
|
||||
float right = -Float.MAX_VALUE;
|
||||
for (RuleGrid.Level l : g) {
|
||||
left = Math.min(left, l.lo());
|
||||
right = Math.max(right, l.hi());
|
||||
}
|
||||
List<Line> inside = new ArrayList<>();
|
||||
for (Line l : lines) {
|
||||
float cy = l.y + l.height / 2f;
|
||||
float cx = l.x + l.width / 2f;
|
||||
if (cy > bottom && cy < top && cx > left - 5f && cx < right + 5f) {
|
||||
inside.add(l);
|
||||
}
|
||||
}
|
||||
// Enough text to be a table: several rows, or for a two-row table a third rule,
|
||||
// the header separator a lone pair of decorative rules does not draw.
|
||||
if (inside.size() < 2 || (inside.size() < ROWS_ONLY_LINES && g.size() < 3)) {
|
||||
continue;
|
||||
}
|
||||
List<List<Line>> rows = RuledRows.baselineRows(inside);
|
||||
if (rows.size() < 2 || TableGrid.render(rows, null, RowSource.RULE_BOUNDED).isBlank()) {
|
||||
continue;
|
||||
}
|
||||
blocks.add(new TableBlock(rows, top, bottom, null, page));
|
||||
}
|
||||
blocks.sort(Comparator.comparingDouble(TableBlock::top).reversed());
|
||||
return blocks;
|
||||
}
|
||||
|
||||
private static TableBlock build(
|
||||
List<RuleGrid.Level> hL, List<RuleGrid.Level> vL, List<Line> lines, int page) {
|
||||
log.debug("ruled-table build hL={} vL={}", hL.size(), vL.size());
|
||||
if (hL.size() < 2 || vL.size() < 2) {
|
||||
return dbgNull("hL/vL < 2");
|
||||
}
|
||||
hL.sort(Comparator.comparingDouble(RuleGrid.Level::pos).reversed());
|
||||
vL.sort(Comparator.comparingDouble(RuleGrid.Level::pos));
|
||||
|
||||
// The extent is the union of both families: a table ruled only between its columns
|
||||
// takes its top and bottom from the verticals, and vice versa.
|
||||
float top = hL.get(0).pos();
|
||||
float bottom = hL.get(hL.size() - 1).pos();
|
||||
float left = vL.get(0).pos();
|
||||
float right = vL.get(vL.size() - 1).pos();
|
||||
for (RuleGrid.Level v : vL) {
|
||||
top = Math.max(top, v.hi());
|
||||
bottom = Math.min(bottom, v.lo());
|
||||
}
|
||||
for (RuleGrid.Level h : hL) {
|
||||
left = Math.min(left, h.lo());
|
||||
right = Math.max(right, h.hi());
|
||||
}
|
||||
if (top - bottom < 6f || right - left < 20f) {
|
||||
return dbgNull("too small");
|
||||
}
|
||||
|
||||
List<Line> inside = new ArrayList<>();
|
||||
for (Line l : lines) {
|
||||
float cy = l.y + l.height / 2f;
|
||||
float cx = l.x + l.width / 2f;
|
||||
if (cy > bottom && cy < top && cx > left - 5f && cx < right + 5f) {
|
||||
inside.add(l);
|
||||
}
|
||||
}
|
||||
if (inside.size() < 2) {
|
||||
return dbgNull("inside<2");
|
||||
}
|
||||
|
||||
// Null columns mean the grid is ruled between its rows only; the block is still worth
|
||||
// building, but its columns then come from whitespace projection.
|
||||
List<float[]> cols = RuledRows.columns(vL, left, right, top, bottom);
|
||||
|
||||
// A row boundary is a y position, not a segment, and runs the table's width: per-cell
|
||||
// rectangles report it once per cell and also box each wrapped line inside a cell.
|
||||
float rowRuleWidth = (right - left) * ROW_RULE_SPAN;
|
||||
List<Float> interiorH = new ArrayList<>();
|
||||
List<RuleGrid.Level> bandRules = new ArrayList<>();
|
||||
float prevWide = top;
|
||||
int i = 0;
|
||||
while (i < hL.size()) {
|
||||
float pos = hL.get(i).pos();
|
||||
int j = i;
|
||||
RuleGrid.Level widest = hL.get(i);
|
||||
while (j < hL.size() && Math.abs(hL.get(j).pos() - pos) <= RuleGrid.LEVEL_TOLERANCE) {
|
||||
if (hL.get(j).hi() - hL.get(j).lo() > widest.hi() - widest.lo()) {
|
||||
widest = hL.get(j);
|
||||
}
|
||||
j++;
|
||||
}
|
||||
i = j;
|
||||
if (pos <= bottom + RuleGrid.LEVEL_TOLERANCE || pos >= top - RuleGrid.LEVEL_TOLERANCE) {
|
||||
bandRules.add(widest);
|
||||
continue;
|
||||
}
|
||||
boolean wide = widest.hi() - widest.lo() >= rowRuleWidth;
|
||||
boolean keep =
|
||||
wide || spanningNeighbour(widest, vL, inside, pos, prevWide, top - bottom);
|
||||
if (!keep) {
|
||||
continue;
|
||||
}
|
||||
interiorH.add(pos);
|
||||
bandRules.add(widest);
|
||||
if (wide) {
|
||||
prevWide = pos;
|
||||
}
|
||||
}
|
||||
|
||||
List<List<Line>> rows;
|
||||
RowSource source = RowSource.RULE_BOUNDED;
|
||||
if (interiorH.size() >= MIN_INTERIOR_RULES) {
|
||||
List<Float> bands = new ArrayList<>();
|
||||
bands.add(top);
|
||||
bands.addAll(interiorH);
|
||||
bands.add(bottom);
|
||||
List<List<Line>> filled = RuledRows.latticeRows(bands, inside);
|
||||
// Most bands must carry text: a chart's axis ticks or a zebra table's stripes rule
|
||||
// many empty bands, and reading those as a table steals lines from the prose.
|
||||
if (filled.size() < (bands.size() - 1) * FILLED_BANDS) {
|
||||
return dbgNull("filled " + filled.size() + " of bands " + (bands.size() - 1));
|
||||
}
|
||||
rows = RuledRows.splitCompleteBands(filled, cols);
|
||||
if (fullWidthRules(bandRules, left, right)) {
|
||||
source = RowSource.LATTICE;
|
||||
}
|
||||
} else {
|
||||
rows = RuledRows.baselineRows(inside);
|
||||
}
|
||||
if (rows.size() < 2) {
|
||||
return dbgNull("rows<2");
|
||||
}
|
||||
|
||||
// A grid is often ruled around its body only, leaving the header just above the top
|
||||
// rule; take it when it fits the grid's width and resolves into its columns.
|
||||
if (cols != null) {
|
||||
// The header's cells are separate lines when they sit far apart, so the whole
|
||||
// band above the grid is taken, not the nearest line.
|
||||
List<Line> hdr = new ArrayList<>();
|
||||
float band = Float.MAX_VALUE;
|
||||
for (Line l : lines) {
|
||||
if (l.y <= top
|
||||
|| l.y - top > TableFinder.HEADER_RULE_GAP * Math.max(l.height, 1f)
|
||||
|| l.x < left - 5f
|
||||
|| l.x + l.width > right + 5f) {
|
||||
continue;
|
||||
}
|
||||
band = Math.min(band, l.y);
|
||||
}
|
||||
for (Line l : lines) {
|
||||
if (band < Float.MAX_VALUE
|
||||
&& l.y >= band
|
||||
&& l.y <= band + 2f
|
||||
&& l.x >= left - 5f
|
||||
&& l.x + l.width <= right + 5f) {
|
||||
hdr.add(l);
|
||||
}
|
||||
}
|
||||
if (!hdr.isEmpty()) {
|
||||
List<List<Line>> withHeader = new ArrayList<>();
|
||||
withHeader.add(hdr);
|
||||
withHeader.addAll(rows);
|
||||
List<String[]> grown = TableGrid.cells(withHeader, cols, source);
|
||||
if (!grown.isEmpty()
|
||||
&& TableGrid.filledCells(grown.get(0)) >= grown.get(0).length - 1
|
||||
&& TableGrid.filledCells(grown.get(0)) >= 2
|
||||
&& TableFinder.wordGroups(hdr) == TableGrid.filledCells(grown.get(0))) {
|
||||
rows = withHeader;
|
||||
top = band + hdr.get(0).height;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TableBlock block = new TableBlock(rows, top, bottom, cols, true, source, page);
|
||||
// A block that fails the shared false-positive guards is not a table; leaving its lines
|
||||
// unclaimed lets the word-grid detector or ordinary paragraph assembly handle them.
|
||||
if (TableGrid.render(rows, cols, source).isBlank()) {
|
||||
return dbgNull(
|
||||
"guards rejected: rows="
|
||||
+ rows.size()
|
||||
+ " cols="
|
||||
+ (cols == null ? -1 : cols.size()));
|
||||
}
|
||||
return block;
|
||||
}
|
||||
|
||||
/** How near a rule end must be to a vertical rule to count as landing on it. */
|
||||
private static final float COLUMN_SNAP = 2.5f;
|
||||
|
||||
/** Fraction of the table's height a vertical must run to be a column boundary. */
|
||||
private static final float COLUMN_RUN = 0.5f;
|
||||
|
||||
/**
|
||||
* True when a rule narrower than the table is still a row boundary: it ends on the grid's
|
||||
* verticals and a spanning cell's text sits beside it.
|
||||
*/
|
||||
private static boolean spanningNeighbour(
|
||||
RuleGrid.Level rule,
|
||||
List<RuleGrid.Level> vL,
|
||||
List<Line> inside,
|
||||
float pos,
|
||||
float above,
|
||||
float height) {
|
||||
// The vertical must run the table, not merely be there: a line box inside a wrapped
|
||||
// cell draws its own short verticals at its inset edges.
|
||||
float columnRun = height * COLUMN_RUN;
|
||||
boolean loOnRule = false;
|
||||
boolean hiOnRule = false;
|
||||
for (RuleGrid.Level v : vL) {
|
||||
if (v.hi() - v.lo() < columnRun) {
|
||||
continue;
|
||||
}
|
||||
if (Math.abs(v.pos() - rule.lo()) <= COLUMN_SNAP) {
|
||||
loOnRule = true;
|
||||
}
|
||||
if (Math.abs(v.pos() - rule.hi()) <= COLUMN_SNAP) {
|
||||
hiOnRule = true;
|
||||
}
|
||||
}
|
||||
if (!loOnRule || !hiOnRule) {
|
||||
return false;
|
||||
}
|
||||
// The spanning cell's text must sit beside the rule anywhere in the row the last
|
||||
// full-width boundary opened: it is written once, at the top of the span.
|
||||
for (Line l : inside) {
|
||||
float cy = l.y + l.height / 2f;
|
||||
float cx = l.x + l.width / 2f;
|
||||
if (cy > pos && cy < above && (cx < rule.lo() || cx > rule.hi())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when every horizontal rule runs nearly the region's full width; legend swatches and
|
||||
* per-cell outlines do not.
|
||||
*/
|
||||
private static boolean fullWidthRules(List<RuleGrid.Level> hL, float left, float right) {
|
||||
float width = right - left;
|
||||
if (width <= 0f) {
|
||||
return false;
|
||||
}
|
||||
for (RuleGrid.Level h : hL) {
|
||||
if (h.hi() - h.lo() < width * FULL_WIDTH_RULE) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* A detected table. Each row is a list of source lines: usually one, but more when a cell wraps.
|
||||
*/
|
||||
record TableBlock(
|
||||
List<List<Line>> rows,
|
||||
float top,
|
||||
float bottom,
|
||||
List<float[]> cols,
|
||||
boolean ruled,
|
||||
RowSource rowSource,
|
||||
int page) {
|
||||
TableBlock(List<List<Line>> rows, float top, float bottom, int page) {
|
||||
this(rows, top, bottom, null, false, RowSource.WORDS, page);
|
||||
}
|
||||
|
||||
/** A rules-derived block whose rows are not a drawn lattice. */
|
||||
TableBlock(List<List<Line>> rows, float top, float bottom, List<float[]> cols, int page) {
|
||||
this(rows, top, bottom, cols, true, RowSource.RULE_BOUNDED, page);
|
||||
}
|
||||
|
||||
String render() {
|
||||
return TableGrid.render(rows, cols, rowSource);
|
||||
}
|
||||
|
||||
/** Cell grid for the layout guards; empty when the block fails the table guards. */
|
||||
List<String[]> cells() {
|
||||
return TableGrid.cells(rows, cols, rowSource);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Finds one page's table blocks and reconciles the two detectors: where both see a table, rows come
|
||||
* from the text and columns from the rules.
|
||||
*/
|
||||
final class TableFinder {
|
||||
|
||||
private TableFinder() {}
|
||||
|
||||
/**
|
||||
* Fraction of the word grid's rows a ruled grid must also find before its rows are trusted;
|
||||
* below it the rules would merge several rows into one band.
|
||||
*/
|
||||
private static final float COMPLETE_LATTICE = 0.5f;
|
||||
|
||||
/**
|
||||
* Detects a page's table blocks: ruled blocks first, then word-grid blocks over whatever lines
|
||||
* the rules did not claim.
|
||||
*/
|
||||
static List<TableBlock> find(List<Line> lines, PageRules rules, int page) {
|
||||
List<TableBlock> ruled = RuledTables.find(lines, rules, page);
|
||||
List<TableBlock> word = fromWordGrid(lines, page);
|
||||
if (ruled.isEmpty()) {
|
||||
return word;
|
||||
}
|
||||
|
||||
// Where both detectors see the same table, keep the word-grid's rows (read from the text)
|
||||
// but take the columns from the rules, which are exact where projection only guesses.
|
||||
List<TableBlock> all = new ArrayList<>();
|
||||
Set<TableBlock> usedRules = new HashSet<>();
|
||||
for (TableBlock w : word) {
|
||||
TableBlock match = null;
|
||||
for (TableBlock r : ruled) {
|
||||
// Only a grid with real column rules can improve on the word-grid; one ruled
|
||||
// across its rows alone contributes detection, never geometry.
|
||||
if (r.cols() != null && covers(w, r)) {
|
||||
match = r;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (match == null) {
|
||||
// No column rules, but a rules-only grid over the same lines still confirms that a
|
||||
// table is here. The word-grid's own reading of it stands, now rule-backed.
|
||||
TableBlock evidence = null;
|
||||
for (TableBlock r : ruled) {
|
||||
if (r.cols() == null && w.top() > r.bottom() && w.bottom() < r.top()) {
|
||||
evidence = r;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (evidence == null) {
|
||||
all.add(w);
|
||||
} else {
|
||||
usedRules.add(evidence);
|
||||
all.add(
|
||||
new TableBlock(
|
||||
w.rows(),
|
||||
w.top(),
|
||||
w.bottom(),
|
||||
null,
|
||||
true,
|
||||
RowSource.WORDS,
|
||||
w.page()));
|
||||
}
|
||||
} else if (match.rows().size() >= w.rows().size() * COMPLETE_LATTICE) {
|
||||
// The rules cover nearly every row, so take the whole grid from them; one ruled
|
||||
// grid can span several word-grid blocks, so emit it only once.
|
||||
if (usedRules.add(match)) {
|
||||
all.add(match);
|
||||
}
|
||||
} else {
|
||||
// Only some row boundaries are drawn: rows from the text, columns from the rules.
|
||||
usedRules.add(match);
|
||||
all.add(
|
||||
new TableBlock(
|
||||
w.rows(),
|
||||
w.top(),
|
||||
w.bottom(),
|
||||
match.cols(),
|
||||
true,
|
||||
RowSource.WORDS,
|
||||
w.page()));
|
||||
}
|
||||
}
|
||||
// A ruled table the word-grid never saw (single-word or wrapped cells leave it no wide gap
|
||||
// to anchor on) is emitted from its rules alone.
|
||||
for (TableBlock r : ruled) {
|
||||
if (usedRules.contains(r)) {
|
||||
continue;
|
||||
}
|
||||
boolean covered =
|
||||
all.stream().anyMatch(b -> b.top() > r.bottom() && b.bottom() < r.top());
|
||||
if (!covered) {
|
||||
all.add(r);
|
||||
}
|
||||
}
|
||||
all.sort(Comparator.comparingDouble(TableBlock::top).reversed());
|
||||
return all;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects table blocks from word geometry: anchor rows grouped into contiguous runs, with
|
||||
* non-anchor lines inside a run absorbed as wrapped cells.
|
||||
*/
|
||||
private static List<TableBlock> fromWordGrid(List<Line> lines, int page) {
|
||||
List<Line> cands =
|
||||
lines.stream()
|
||||
.filter(l -> !l.synthetic && isTableCandidate(l.words()))
|
||||
.sorted(Comparator.comparingDouble((Line l) -> l.y).reversed())
|
||||
.collect(Collectors.toList());
|
||||
if (cands.size() < 2) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
List<Float> gaps = new ArrayList<>();
|
||||
for (int i = 1; i < cands.size(); i++) {
|
||||
gaps.add(cands.get(i - 1).y - cands.get(i).y);
|
||||
}
|
||||
List<Float> sorted = new ArrayList<>(gaps);
|
||||
sorted.sort(Comparator.naturalOrder());
|
||||
float medianGap = sorted.get(sorted.size() / 2);
|
||||
float splitThreshold = Math.max(medianGap * 2.5f, medianGap + 6f);
|
||||
|
||||
List<List<Line>> anchorGroups = new ArrayList<>();
|
||||
List<Line> current = new ArrayList<>();
|
||||
current.add(cands.getFirst());
|
||||
for (int i = 1; i < cands.size(); i++) {
|
||||
float gap = cands.get(i - 1).y - cands.get(i).y;
|
||||
if (gap > splitThreshold) {
|
||||
anchorGroups.add(current);
|
||||
current = new ArrayList<>();
|
||||
}
|
||||
current.add(cands.get(i));
|
||||
}
|
||||
anchorGroups.add(current);
|
||||
|
||||
// Synthetic form values are kept out of the table path: they must not seed a column layout
|
||||
// or be absorbed as wrapped cells, as they were never in the content stream.
|
||||
List<Line> nonCandidates =
|
||||
lines.stream()
|
||||
.filter(l -> !l.synthetic && !isTableCandidate(l.words()))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
List<TableBlock> blocks = new ArrayList<>();
|
||||
for (List<Line> anchors : anchorGroups) {
|
||||
if (anchors.size() < 2) {
|
||||
continue;
|
||||
}
|
||||
float top = anchors.getFirst().y;
|
||||
float bottom = anchors.getLast().y;
|
||||
|
||||
// Each anchor seeds a row; absorb wrapped continuation lines (non-anchors within the
|
||||
// run's vertical span, with a little slack below the last row) into the anchor above.
|
||||
List<List<Line>> rows = new ArrayList<>();
|
||||
for (Line a : anchors) {
|
||||
List<Line> row = new ArrayList<>();
|
||||
row.add(a);
|
||||
rows.add(row);
|
||||
}
|
||||
for (Line nc : nonCandidates) {
|
||||
if (nc.y > top || nc.y < bottom - medianGap) {
|
||||
continue;
|
||||
}
|
||||
int owner = 0;
|
||||
float bestDelta = Float.MAX_VALUE;
|
||||
for (int i = 0; i < anchors.size(); i++) {
|
||||
float delta = anchors.get(i).y - nc.y; // positive when anchor is above nc
|
||||
if (delta >= -1f && delta < bestDelta) {
|
||||
bestDelta = delta;
|
||||
owner = i;
|
||||
}
|
||||
}
|
||||
rows.get(owner).add(nc);
|
||||
}
|
||||
|
||||
List<String[]> base = TableGrid.cells(rows, null, RowSource.WORDS);
|
||||
if (base.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
// A header row often has no wide gap between its cells, so the anchor test misses it.
|
||||
// The line above is kept only if its grid has the same shape, excluding captions.
|
||||
Line header = headerAbove(nonCandidates, top, medianGap);
|
||||
if (header != null) {
|
||||
List<List<Line>> withHeader = new ArrayList<>();
|
||||
withHeader.add(new ArrayList<>(List.of(header)));
|
||||
withHeader.addAll(rows);
|
||||
List<String[]> grown = TableGrid.cells(withHeader, null, RowSource.WORDS);
|
||||
if (!grown.isEmpty()
|
||||
&& grown.get(0).length == base.get(0).length
|
||||
&& TableGrid.filledCells(grown.get(0)) >= base.get(0).length) {
|
||||
rows = withHeader;
|
||||
top = header.y;
|
||||
}
|
||||
}
|
||||
blocks.add(new TableBlock(rows, top, bottom, page));
|
||||
}
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/** Vertical gaps, in median row gaps, within which a line above a block can be its header. */
|
||||
private static final float HEADER_GAP = 1.6f;
|
||||
|
||||
/**
|
||||
* Runs of words separated by more than a cell gutter: a header row has one per cell, a caption
|
||||
* written across the table is a single run.
|
||||
*/
|
||||
static int wordGroups(List<Line> row) {
|
||||
List<TextWord> words = new ArrayList<>();
|
||||
for (Line line : row) {
|
||||
for (TextWord w : line.words()) {
|
||||
if (!w.text().strip().isEmpty()) {
|
||||
words.add(w);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (words.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
words.sort(Comparator.comparingDouble(TextWord::x));
|
||||
float chars = 0;
|
||||
float width = 0;
|
||||
for (TextWord w : words) {
|
||||
width += w.width();
|
||||
chars += Math.max(1, w.text().strip().length());
|
||||
}
|
||||
float gutter =
|
||||
Math.max(
|
||||
ColumnRanges.RULED_GUTTER_FLOOR,
|
||||
(width / chars) * ColumnRanges.RULED_GUTTER_CHARS);
|
||||
int groups = 1;
|
||||
for (int i = 1; i < words.size(); i++) {
|
||||
float gap = words.get(i).x() - (words.get(i - 1).x() + words.get(i - 1).width());
|
||||
if (gap >= gutter) {
|
||||
groups++;
|
||||
}
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
/** Line heights within which a line above a ruled grid can be its header row. */
|
||||
static final float HEADER_RULE_GAP = 2.5f;
|
||||
|
||||
/** The nearest line above {@code top} close enough to be the block's header row. */
|
||||
private static Line headerAbove(List<Line> lines, float top, float medianGap) {
|
||||
Line best = null;
|
||||
for (Line l : lines) {
|
||||
if (l.y <= top || l.y - top > medianGap * HEADER_GAP || l.words().size() < 2) {
|
||||
continue;
|
||||
}
|
||||
if (best == null || l.y < best.y) {
|
||||
best = l;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a line has two words separated by a gap far wider than word spacing. The threshold
|
||||
* comes from the line's own character width, not a document font size.
|
||||
*/
|
||||
private static boolean isTableCandidate(List<TextWord> words) {
|
||||
if (words.size() < 2) {
|
||||
return false;
|
||||
}
|
||||
double totalWidth = 0;
|
||||
int totalChars = 0;
|
||||
for (TextWord w : words) {
|
||||
totalWidth += w.width();
|
||||
totalChars += Math.max(1, w.text().strip().length());
|
||||
}
|
||||
float charWidth = (float) (totalWidth / Math.max(1, totalChars));
|
||||
// A deliberate cell gap is several blank characters wide; ordinary word spaces are ~a third
|
||||
// of a character. Floor at 8pt so tiny fonts still need a real gap.
|
||||
float cellGap = Math.max(8f, charWidth * 3f);
|
||||
for (int i = 1; i < words.size(); i++) {
|
||||
TextWord prev = words.get(i - 1);
|
||||
float gap = words.get(i).x() - (prev.x() + prev.width());
|
||||
if (gap >= cellGap) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** True when two blocks overlap vertically, i.e. they describe the same table. */
|
||||
static boolean covers(TableBlock a, TableBlock b) {
|
||||
return Math.min(a.top(), b.top()) > Math.max(a.bottom(), b.bottom());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Resolves a detected table block into a cell grid and renders it. The false-positive guards live
|
||||
* here, so every consumer sees the same cells and the same verdict.
|
||||
*/
|
||||
final class TableGrid {
|
||||
|
||||
private TableGrid() {}
|
||||
|
||||
/**
|
||||
* Renders a table block; {@code ruledColumns} are exact bands from vertical rules, or null to
|
||||
* project the columns from whitespace instead.
|
||||
*/
|
||||
static String render(
|
||||
List<List<Line>> rowGroups, List<float[]> ruledColumns, RowSource rowSource) {
|
||||
List<String[]> rows = cells(rowGroups, ruledColumns, rowSource);
|
||||
return rows.isEmpty() ? "" : GfmTable.render(rows, rows.get(0).length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a table block into a cell grid, or empty when it fails the false-positive guards.
|
||||
*/
|
||||
static List<String[]> cells(
|
||||
List<List<Line>> rowGroups, List<float[]> ruledColumns, RowSource rowSource) {
|
||||
// Columns come from cross-row whitespace projection, not a 1-D gap threshold on pooled word
|
||||
// x's, which is fragile with right-aligned numbers or sparse cells in their own band.
|
||||
List<Line> flat = rowGroups.stream().flatMap(List::stream).collect(Collectors.toList());
|
||||
// Inside a region the rules already declare a table, a narrower gutter still separates
|
||||
// columns: the wide floor only exists to stop word spacing splitting an unruled block.
|
||||
List<float[]> columns =
|
||||
ruledColumns != null
|
||||
? ruledColumns
|
||||
: ColumnRanges.find(
|
||||
flat,
|
||||
rowSource.ruleConfirmed()
|
||||
? ColumnRanges.RULED_GUTTER_CHARS
|
||||
: ColumnRanges.GUTTER_CHARS,
|
||||
rowSource.ruleConfirmed()
|
||||
? ColumnRanges.RULED_GUTTER_FLOOR
|
||||
: ColumnRanges.GUTTER_FLOOR);
|
||||
// A column only the header occupies is invisible to the projection, which needs a band
|
||||
// shared by several rows; but inside a ruled region a blank answer column is still one.
|
||||
boolean headerOnlyColumn = false;
|
||||
if (columns.size() < 2 && ruledColumns == null && rowSource.ruleConfirmed()) {
|
||||
List<float[]> retry =
|
||||
ColumnRanges.find(
|
||||
flat,
|
||||
ColumnRanges.RULED_GUTTER_CHARS,
|
||||
ColumnRanges.RULED_GUTTER_FLOOR,
|
||||
1);
|
||||
// Only the worksheet shape: exactly one row, the first, reaches past the supported
|
||||
// column. Anything else would invent a column and swallow the headings around it.
|
||||
if (retry.size() >= 2 && retry.size() <= 15) {
|
||||
float edge = retry.get(0)[1];
|
||||
int beyond = 0;
|
||||
int firstBeyond = -1;
|
||||
for (int r = 0; r < rowGroups.size(); r++) {
|
||||
boolean out = false;
|
||||
for (Line l : rowGroups.get(r)) {
|
||||
for (TextWord w : l.words()) {
|
||||
if (!w.text().strip().isEmpty() && w.x() + w.width() / 2f > edge) {
|
||||
out = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (out) {
|
||||
beyond++;
|
||||
if (firstBeyond < 0) {
|
||||
firstBeyond = r;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (beyond == 1 && firstBeyond == 0 && rowGroups.size() >= 3) {
|
||||
columns = retry;
|
||||
headerOnlyColumn = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Only a drawn lattice can be a one-column table; inferred from whitespace it is just a
|
||||
// run of centred lines.
|
||||
int minColumns = rowSource == RowSource.LATTICE ? 1 : 2;
|
||||
if (columns.size() < minColumns || columns.size() > 15) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
float[] centers = new float[columns.size()];
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
centers[i] = (columns.get(i)[0] + columns.get(i)[1]) / 2f;
|
||||
}
|
||||
|
||||
int cols = centers.length;
|
||||
List<String[]> rows = new ArrayList<>();
|
||||
for (List<Line> rowLines : rowGroups) {
|
||||
String[] row = new String[cols];
|
||||
TextWord[] lastWord = new TextWord[cols];
|
||||
String[] lastText = new String[cols];
|
||||
boolean[] boundMark = new boolean[cols];
|
||||
for (int i = 0; i < cols; i++) {
|
||||
row[i] = "";
|
||||
lastText[i] = "";
|
||||
}
|
||||
// Top line first so a wrapped cell's words stay in reading order within the cell.
|
||||
rowLines.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
|
||||
for (Line line : rowLines) {
|
||||
for (TextWord word : line.words()) {
|
||||
String wt = word.text().strip();
|
||||
if (wt.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
float mid = word.x() + word.width() / 2f;
|
||||
// Ruled columns are real boundaries, so a word belongs to the band that
|
||||
// contains it; projected columns are only approximate centres, so nearest wins.
|
||||
int col =
|
||||
ruledColumns != null
|
||||
? containingColumn(mid, columns)
|
||||
: nearestColumn(mid, centers);
|
||||
// A mark that closed up against the word on its left closes up against the
|
||||
// word on its right too, so Party - List does not settle at "Party- List".
|
||||
boolean bind =
|
||||
!row[col].isEmpty()
|
||||
&& (boundMark[col]
|
||||
|| (WordGeometry.isBindingMark(wt)
|
||||
|| WordGeometry.isBindingMark(
|
||||
lastText[col]))
|
||||
&& !WordGeometry.separated(
|
||||
lastWord[col], word));
|
||||
row[col] = row[col].isEmpty() ? wt : row[col] + (bind ? "" : " ") + wt;
|
||||
boundMark[col] = bind && WordGeometry.isBindingMark(wt);
|
||||
lastWord[col] = word;
|
||||
lastText[col] = wt;
|
||||
}
|
||||
}
|
||||
for (int c = 0; c < cols; c++) {
|
||||
row[c] = WordGeometry.rejoinContractions(row[c]);
|
||||
}
|
||||
rows.add(row);
|
||||
}
|
||||
|
||||
// Guard against false positives while tolerating uneven rows: require an anchor row that
|
||||
// nearly fills the grid, and that most rows are genuinely multi-column.
|
||||
if (ruledColumns != null) {
|
||||
// A rule that is not a column separator (a cell outline, a shading edge) leaves an
|
||||
// empty column; drop those rather than emitting them across every row.
|
||||
List<Integer> keep = new ArrayList<>();
|
||||
for (int c = 0; c < cols; c++) {
|
||||
final int col = c;
|
||||
if (rows.stream().anyMatch(r -> !r[col].isEmpty())) {
|
||||
keep.add(c);
|
||||
}
|
||||
}
|
||||
// Two is the floor whatever the rows say: one filled column means the rules drew a box
|
||||
// round a single block of text, not a table.
|
||||
if (keep.size() < 2) {
|
||||
return List.of();
|
||||
}
|
||||
if (keep.size() < cols) {
|
||||
List<String[]> trimmed = new ArrayList<>(rows.size());
|
||||
for (String[] r : rows) {
|
||||
String[] t = new String[keep.size()];
|
||||
for (int i = 0; i < keep.size(); i++) {
|
||||
t[i] = r[keep.get(i)];
|
||||
}
|
||||
trimmed.add(t);
|
||||
}
|
||||
rows = trimmed;
|
||||
cols = keep.size();
|
||||
List<float[]> kept = new ArrayList<>(keep.size());
|
||||
for (int idx : keep) {
|
||||
kept.add(columns.get(idx));
|
||||
}
|
||||
columns = kept;
|
||||
}
|
||||
}
|
||||
|
||||
if (cols == 1) {
|
||||
// A one-column table has no cross-row alignment to check, so the evidence is the rules
|
||||
// plus the shape of the run: enough rows, nearly all carrying text.
|
||||
long filled = rows.stream().filter(r -> !r[0].isEmpty()).count();
|
||||
return rows.size() >= SINGLE_COLUMN_ROWS && filled >= rows.size() * SINGLE_COLUMN_FILLED
|
||||
? rows
|
||||
: List.of();
|
||||
}
|
||||
|
||||
int anchorWidth = Math.max(2, Math.round(cols * 0.6f));
|
||||
long anchorRows = rows.stream().filter(r -> filledCells(r) >= anchorWidth).count();
|
||||
long multiColumnRows = rows.stream().filter(r -> filledCells(r) >= 2).count();
|
||||
// The multi-column tests ask whether a grid inferred from whitespace is real; when rows
|
||||
// and columns are both drawn there is nothing to infer, and a blank worksheet would fail.
|
||||
boolean drawnGrid =
|
||||
headerOnlyColumn || (ruledColumns != null && rowSource == RowSource.LATTICE);
|
||||
if (drawnGrid
|
||||
? anchorRows < 1
|
||||
: (anchorRows < 1 || multiColumnRows < 2 || multiColumnRows < rows.size() * 0.5)) {
|
||||
return List.of();
|
||||
}
|
||||
if (ruledColumns == null && TableShape.isProseNotTable(rows, cols)) {
|
||||
return List.of();
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** Rows a single-column ruled table needs before it is a table rather than a run of lines. */
|
||||
private static final int SINGLE_COLUMN_ROWS = 3;
|
||||
|
||||
/** Fraction of a single-column table's rows that must carry text. */
|
||||
private static final float SINGLE_COLUMN_FILLED = 0.8f;
|
||||
|
||||
/** Index of the column band containing x, clamped to the first/last band outside the grid. */
|
||||
static int containingColumn(float x, List<float[]> columns) {
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
if (x < columns.get(i)[1]) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return columns.size() - 1;
|
||||
}
|
||||
|
||||
private static int nearestColumn(float x, float[] centers) {
|
||||
int best = 0;
|
||||
float bestDist = Float.MAX_VALUE;
|
||||
for (int i = 0; i < centers.length; i++) {
|
||||
float d = Math.abs(x - centers[i]);
|
||||
if (d < bestDist) {
|
||||
bestDist = d;
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
static int filledCells(String[] row) {
|
||||
int count = 0;
|
||||
for (String cell : row) {
|
||||
if (!cell.isEmpty()) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* False-positive guards: whether a block is really a table, and whether it is wide enough to
|
||||
* outrank the page's own column layout.
|
||||
*/
|
||||
final class TableShape {
|
||||
|
||||
private TableShape() {}
|
||||
|
||||
/** Fraction of the page's text width a table must span to override two-column layout. */
|
||||
private static final float FULL_WIDTH = 0.6f;
|
||||
|
||||
/** Rows of a two-column block that must end in a page number for it to be a contents list. */
|
||||
private static final float TOC_ROWS = 0.65f;
|
||||
|
||||
/** Mean filled-cell length above which a two-column block reads as prose, not cells. */
|
||||
private static final float PROSE_CELL = 40f;
|
||||
|
||||
private static final Pattern PAGE_NUMBER = Pattern.compile("[0-9]{1,4}|[ivxlcdmIVXLCDM]{1,7}");
|
||||
|
||||
/** A run of spaced or solid dots, the leader of a contents line. */
|
||||
private static final Pattern DOT_LEADER = Pattern.compile("(\\.\\s*){4,}|…");
|
||||
|
||||
/**
|
||||
* True when a block is running text the word grid mistook for a table: a contents list, or two
|
||||
* columns of prose whose cells are whole sentences.
|
||||
*/
|
||||
static boolean isProseNotTable(List<String[]> rows, int cols) {
|
||||
if (rows.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
for (String[] row : rows) {
|
||||
for (String cell : row) {
|
||||
if (DOT_LEADER.matcher(cell).find()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (cols != 2) {
|
||||
return everyColumnIsProse(rows, cols);
|
||||
}
|
||||
int folios = 0;
|
||||
int length = 0;
|
||||
int filled = 0;
|
||||
for (String[] row : rows) {
|
||||
String last = "";
|
||||
for (String cell : row) {
|
||||
if (!cell.isEmpty()) {
|
||||
length += cell.length();
|
||||
filled++;
|
||||
last = cell;
|
||||
}
|
||||
}
|
||||
if (PAGE_NUMBER.matcher(last).matches() && !PAGE_NUMBER.matcher(row[0]).matches()) {
|
||||
folios++;
|
||||
}
|
||||
}
|
||||
if (folios >= rows.size() * TOC_ROWS) {
|
||||
return true;
|
||||
}
|
||||
return filled > 0 && (float) length / filled >= PROSE_CELL;
|
||||
}
|
||||
|
||||
/** Mean cell length at or above which a column carries sentences rather than values. */
|
||||
private static final float PROSE_COLUMN = 20f;
|
||||
|
||||
/** Fraction of neighbouring cells that must continue each other's sentence to read as prose. */
|
||||
private static final float PROSE_RUN_ON = 0.5f;
|
||||
|
||||
/** A cell that ends a sentence or clause, so the cell after it starts something new. */
|
||||
private static final Pattern CELL_ENDS_CLAUSE = Pattern.compile("[.!?:;,]$");
|
||||
|
||||
/**
|
||||
* True when a wide block is multi-column prose read across, not a table: no column keys the
|
||||
* rows, and the cells continue each other's sentences.
|
||||
*/
|
||||
static boolean everyColumnIsProse(List<String[]> rows, int cols) {
|
||||
if (cols < 3) {
|
||||
return false;
|
||||
}
|
||||
for (int c = 0; c < cols; c++) {
|
||||
int length = 0;
|
||||
int filled = 0;
|
||||
for (String[] row : rows) {
|
||||
if (c < row.length && !row[c].isEmpty()) {
|
||||
length += row[c].length();
|
||||
filled++;
|
||||
}
|
||||
}
|
||||
if (filled == 0 || (float) length / filled < PROSE_COLUMN) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return runsOnAcrossCells(rows);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fraction of side-by-side filled cells where the right one continues the left one's clause.
|
||||
*/
|
||||
private static boolean runsOnAcrossCells(List<String[]> rows) {
|
||||
int pairs = 0;
|
||||
int runOn = 0;
|
||||
for (String[] row : rows) {
|
||||
String previous = null;
|
||||
for (String cell : row) {
|
||||
if (cell.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (previous != null) {
|
||||
pairs++;
|
||||
if (!CELL_ENDS_CLAUSE.matcher(previous).find()
|
||||
&& Character.isLowerCase(cell.charAt(0))) {
|
||||
runOn++;
|
||||
}
|
||||
}
|
||||
previous = cell;
|
||||
}
|
||||
}
|
||||
return pairs > 0 && (float) runOn / pairs > PROSE_RUN_ON;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when no text outside the block sits in its vertical band, so it cannot be one column of
|
||||
* a two-column layout.
|
||||
*/
|
||||
static boolean ownsItsBand(TableBlock block, List<Line> lines) {
|
||||
Set<Line> own = new HashSet<>();
|
||||
for (List<Line> row : block.rows()) {
|
||||
own.addAll(row);
|
||||
}
|
||||
for (Line l : lines) {
|
||||
if (own.contains(l)) {
|
||||
continue;
|
||||
}
|
||||
float centre = l.y + l.height / 2f;
|
||||
if (centre > block.bottom() && centre < block.top()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Columns a full-width unruled block needs before it can outrank the page's column layout. */
|
||||
private static final int GRID_COLUMNS = 3;
|
||||
|
||||
/** Mean filled-cell length above which a full-width unruled block is prose read across. */
|
||||
private static final float GRID_CELL = 25f;
|
||||
|
||||
/**
|
||||
* True when an unruled full-width block is really a table: a data table's cells are short
|
||||
* values, a page gutter's are sentences.
|
||||
*/
|
||||
static boolean looksLikeGrid(TableBlock block) {
|
||||
List<String[]> cells = block.cells();
|
||||
if (cells.isEmpty() || cells.get(0).length < GRID_COLUMNS) {
|
||||
return false;
|
||||
}
|
||||
int length = 0;
|
||||
int filled = 0;
|
||||
for (String[] row : cells) {
|
||||
for (String cell : row) {
|
||||
if (!cell.isEmpty()) {
|
||||
length += cell.length();
|
||||
filled++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return filled > 0 && (float) length / filled <= GRID_CELL;
|
||||
}
|
||||
|
||||
/** True when a table block is wide enough to be a full-width table, not one inside a column. */
|
||||
static boolean spansPage(TableBlock block, List<Line> lines) {
|
||||
float pageLo = Float.MAX_VALUE;
|
||||
float pageHi = -Float.MAX_VALUE;
|
||||
for (Line l : lines) {
|
||||
pageLo = Math.min(pageLo, l.x);
|
||||
pageHi = Math.max(pageHi, l.x + l.width);
|
||||
}
|
||||
float lo = Float.MAX_VALUE;
|
||||
float hi = -Float.MAX_VALUE;
|
||||
for (List<Line> row : block.rows()) {
|
||||
for (Line l : row) {
|
||||
lo = Math.min(lo, l.x);
|
||||
hi = Math.max(hi, l.x + l.width);
|
||||
}
|
||||
}
|
||||
return pageHi > pageLo && (hi - lo) >= (pageHi - pageLo) * FULL_WIDTH;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Word-level geometry and spacing. PDFium splits words on its own bounding boxes, so both real
|
||||
* spaces and edges are re-derived from the glyphs.
|
||||
*/
|
||||
final class WordGeometry {
|
||||
|
||||
private WordGeometry() {}
|
||||
|
||||
/** Gap below this many average character widths reads as no space at all (mid-word split). */
|
||||
static final float NO_SPACE_GAP = 0.30f;
|
||||
|
||||
/**
|
||||
* Punctuation that binds to the words on both sides. Closing words up is only considered around
|
||||
* one of these, because dropping a real space corrupts the text.
|
||||
*/
|
||||
private static final String BINDING_MARKS = "'’ʼ´`-‐‑";
|
||||
|
||||
/** True for a lone apostrophe or hyphen, as in {@code firm}, {@code '}, {@code s}. */
|
||||
static boolean isBindingMark(String word) {
|
||||
return word.length() == 1 && BINDING_MARKS.indexOf(word.charAt(0)) >= 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* A contraction whose apostrophe the extractor padded on both sides. English suffixes only: a
|
||||
* spaced lone apostrophe is an opening quote.
|
||||
*/
|
||||
private static final Pattern SPLIT_CONTRACTION =
|
||||
Pattern.compile("(\\p{L})\\s*([’'ʼ´`])\\s*(s|t|d|m|re|ve|ll)\\b");
|
||||
|
||||
/** Closes up an apostrophe the extractor left standing alone inside a cell. */
|
||||
static String rejoinContractions(String cell) {
|
||||
return cell.indexOf(' ') < 0 ? cell : SPLIT_CONTRACTION.matcher(cell).replaceAll("$1$2$3");
|
||||
}
|
||||
|
||||
/**
|
||||
* True when two words of a cell are far enough apart to be separated by a space; punctuation
|
||||
* set tight against its neighbour arrives as its own word.
|
||||
*/
|
||||
static boolean separated(TextWord previous, TextWord current) {
|
||||
if (previous == null) {
|
||||
return true;
|
||||
}
|
||||
float gap = leftEdge(current) - rightEdge(previous);
|
||||
if (gap < 0f) {
|
||||
// Overlapping or out of order (a second line of a wrapped cell): keep the space.
|
||||
return true;
|
||||
}
|
||||
float charWidth = wordCharWidth(previous, current);
|
||||
return charWidth <= 0f || gap >= charWidth * NO_SPACE_GAP;
|
||||
}
|
||||
|
||||
/** Mean glyph width across two words, used to size the space test above. */
|
||||
private static float wordCharWidth(TextWord a, TextWord b) {
|
||||
float width = 0f;
|
||||
int chars = 0;
|
||||
for (TextWord w : List.of(a, b)) {
|
||||
for (TextChar c : w.chars()) {
|
||||
if (!c.isWhitespace() && !c.isNewline()) {
|
||||
width += c.width();
|
||||
chars++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return chars == 0 ? 0f : width / chars;
|
||||
}
|
||||
|
||||
static float rightEdge(TextWord w) {
|
||||
float edge = -Float.MAX_VALUE;
|
||||
for (TextChar c : w.chars()) {
|
||||
if (!c.isWhitespace() && !c.isNewline()) {
|
||||
edge = Math.max(edge, c.x() + c.width());
|
||||
}
|
||||
}
|
||||
return edge == -Float.MAX_VALUE ? w.x() + w.width() : edge;
|
||||
}
|
||||
|
||||
static float leftEdge(TextWord w) {
|
||||
float edge = Float.MAX_VALUE;
|
||||
for (TextChar c : w.chars()) {
|
||||
if (!c.isWhitespace() && !c.isNewline()) {
|
||||
edge = Math.min(edge, c.x());
|
||||
}
|
||||
}
|
||||
return edge == Float.MAX_VALUE ? w.x() : edge;
|
||||
}
|
||||
|
||||
static float averageCharWidth(List<Line> rows) {
|
||||
double totalWidth = 0;
|
||||
int totalChars = 0;
|
||||
for (Line l : rows) {
|
||||
for (TextWord w : l.words()) {
|
||||
totalWidth += w.width();
|
||||
totalChars += Math.max(1, w.text().strip().length());
|
||||
}
|
||||
}
|
||||
return totalChars == 0 ? 6f : (float) (totalWidth / totalChars);
|
||||
}
|
||||
}
|
||||
+423
@@ -0,0 +1,423 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.Timeout;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Accuracy and robustness tests comparing output against hand-authored golden Markdown. {@link
|
||||
* #gatedFixtures()} gates CI; {@link #wipFixtures()} is disabled.
|
||||
*/
|
||||
class AdvancedPdfMarkdownConverterTest {
|
||||
|
||||
/** Accuracy threshold: output must share at least this fraction of content with the golden. */
|
||||
private static final double THRESHOLD = 0.95;
|
||||
|
||||
@TempDir Path tmp;
|
||||
|
||||
/** Fixtures that meet the accuracy threshold today and therefore gate CI. */
|
||||
static Stream<Arguments> gatedFixtures() {
|
||||
return Stream.of(
|
||||
Arguments.of("multi-column-test_lorem.pdf", "multi-column-test_lorem.md"),
|
||||
Arguments.of("bordered-table-test_widget.pdf", "bordered-table-test_widget.md"),
|
||||
Arguments.of("many-tables-test_stress.pdf", "many-tables-test_stress.md"));
|
||||
}
|
||||
|
||||
/** Fixtures still below the threshold; tracked here, enable locally to iterate. */
|
||||
static Stream<Arguments> wipFixtures() {
|
||||
return Stream.of(
|
||||
Arguments.of(
|
||||
"wrapped-cell-test_expense-report.pdf",
|
||||
"wrapped-cell-test_expense-report.md"));
|
||||
}
|
||||
|
||||
@ParameterizedTest(name = "{0}")
|
||||
@MethodSource("gatedFixtures")
|
||||
void convertMatchesGoldenMarkdown(String pdfName, String mdName) throws IOException {
|
||||
assertConversionMatchesGolden(pdfName, mdName);
|
||||
}
|
||||
|
||||
@Disabled("WIP fixtures below the accuracy threshold; enable locally to iterate")
|
||||
@ParameterizedTest(name = "{0}")
|
||||
@MethodSource("wipFixtures")
|
||||
void convertMatchesGoldenMarkdownWip(String pdfName, String mdName) throws IOException {
|
||||
assertConversionMatchesGolden(pdfName, mdName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Degenerate geometry must not crash the converter: a text matrix can place a word past {@link
|
||||
* Integer#MAX_VALUE}, which used to size an {@code int[]} from the span.
|
||||
*/
|
||||
@Test
|
||||
void columnDetectionSurvivesDegenerateGeometry() {
|
||||
// x ≈ 2.5e9 is past Integer.MAX_VALUE; combined with a near-origin word it yields an
|
||||
// implausible span that the pre-fix code turned into a fatal array allocation.
|
||||
List<TextLine> rows = new ArrayList<>();
|
||||
for (int r = 0; r < 4; r++) {
|
||||
float y = 400f - r * 12f;
|
||||
TextWord near = new TextWord(List.of(), 50f, y, 30f, 10f);
|
||||
TextWord far = new TextWord(List.of(), 2_500_000_000f, y, 30f, 10f);
|
||||
rows.add(new TextLine(List.of(near, far), 50f, y, 2_499_999_980f, 10f));
|
||||
}
|
||||
|
||||
List<float[]> columns = assertDoesNotThrow(() -> ColumnRanges.fromTextLines(rows));
|
||||
assertTrue(
|
||||
columns.isEmpty(),
|
||||
"implausible page span should disable column detection, not allocate from it");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Timeout(20)
|
||||
void gutterScanTerminatesOnCoordinatesBeyondFloatPrecision() {
|
||||
// Past 2^24 a float cannot represent x + 1, so a float-stepped scan over a crafted text
|
||||
// matrix stops advancing and spins forever - wedging the process-wide jpdfium lock with it.
|
||||
List<TextLine> rows = new ArrayList<>();
|
||||
for (int r = 0; r < 10; r++) {
|
||||
float y = 400f - r * 12f;
|
||||
float x = 20_000_000f;
|
||||
TextWord w = new TextWord(List.of(), x, y, 200f, 10f);
|
||||
rows.add(new TextLine(List.of(w), x, y, 200f, 10f));
|
||||
}
|
||||
|
||||
List<Float> gutters = assertDoesNotThrow(() -> ColumnLayout.guttersFromTextLines(rows));
|
||||
assertTrue(
|
||||
gutters.isEmpty(),
|
||||
"every candidate band is crossed by every line, so no gutter is found");
|
||||
}
|
||||
|
||||
/**
|
||||
* Three-column prose aligns across rows exactly as cells do. What tells them apart is a keying
|
||||
* column of short values and cells that do not run on.
|
||||
*/
|
||||
@Test
|
||||
void multiColumnProseIsNotATable() {
|
||||
List<String[]> prose =
|
||||
List.of(
|
||||
new String[] {
|
||||
"The SS Pack can reduce the information acquisition time by",
|
||||
"returning all the information that matches",
|
||||
"the user's search intent and the query behind it"
|
||||
},
|
||||
new String[] {
|
||||
"Unlike existing search systems that only return information",
|
||||
"limited to the entered search keywords, this pack",
|
||||
"returns all relevant data meeting the search intent"
|
||||
});
|
||||
assertTrue(
|
||||
TableShape.everyColumnIsProse(prose, 3),
|
||||
"three columns of running sentences are a page layout, not a table");
|
||||
}
|
||||
|
||||
@Test
|
||||
void wideTableWithLongCellsStaysATable() {
|
||||
// The prose test must not fire on a real table just because one column runs long: the
|
||||
// short "Jurisdiction" and yes/no columns are what key the rows.
|
||||
List<String[]> table =
|
||||
List.of(
|
||||
new String[] {
|
||||
"Argentina",
|
||||
"Y",
|
||||
"Prohibition on ownership of property that contains or borders water"
|
||||
},
|
||||
new String[] {
|
||||
"Australia",
|
||||
"N",
|
||||
"Approval is needed from the Treasurer if the acquisition is large"
|
||||
});
|
||||
assertTrue(!TableShape.everyColumnIsProse(table, 3), "a keyed table is a table");
|
||||
}
|
||||
|
||||
@Test
|
||||
void splitApostropheIsClosedUpInCells() {
|
||||
// PDFium splits on its own bounding boxes, so a tight apostrophe arrives as its own word.
|
||||
assertEquals("the firm's returns", WordGeometry.rejoinContractions("the firm ' s returns"));
|
||||
assertEquals("Don’t know", WordGeometry.rejoinContractions("Don ’ t know"));
|
||||
// An opening quote has real space around it and must keep it.
|
||||
assertEquals("he said ' hello", WordGeometry.rejoinContractions("he said ' hello"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void headingLevelsAreRebasedOnTheStrongestHeadingPresent() {
|
||||
// A document whose headings are body-size and bold scores every one of them level 3;
|
||||
// relative to each other they are its top level, so they must render as level 1.
|
||||
assertEquals("# CONTENTS\n", MarkdownText.normaliseHeadingLevels("### CONTENTS\n"));
|
||||
// A real two-level document keeps two levels, with no gap between them.
|
||||
assertEquals(
|
||||
"# Title\n\ntext\n\n## Section\n",
|
||||
MarkdownText.normaliseHeadingLevels("# Title\n\ntext\n\n### Section\n"));
|
||||
// Already rooted at level 1 with no gaps: left alone.
|
||||
String unchanged = "# Title\n\n## Section\n";
|
||||
assertEquals(unchanged, MarkdownText.normaliseHeadingLevels(unchanged));
|
||||
}
|
||||
|
||||
/** A line of Han text has no spaces, so the word-count heading guard cannot measure it. */
|
||||
private static List<TextWord> cjkWords(String text, String font) {
|
||||
List<TextChar> chars = new ArrayList<>(text.length());
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
chars.add(new TextChar(i, text.charAt(i), 50f + i * 12f, 400f, 12f, 12f, font, 12f));
|
||||
}
|
||||
return List.of(new TextWord(chars, 50f, 400f, text.length() * 12f, 12f));
|
||||
}
|
||||
|
||||
@Test
|
||||
void boldCjkParagraphIsNotPromotedToAHeading() {
|
||||
String paragraph =
|
||||
"\u672c\u898f\u7d04\u306f\u3001\u5f53\u793e\u304c\u63d0\u4f9b\u3059\u308b"
|
||||
+ "\u672c\u30b5\u30fc\u30d3\u30b9\u306e\u5229\u7528\u6761\u4ef6\u3092"
|
||||
+ "\u5b9a\u3081\u308b\u3082\u306e\u3067\u3042\u308a\u3001\u5229\u7528"
|
||||
+ "\u8005\u306e\u7686\u3055\u307e\u306b\u306f\u672c\u898f\u7d04\u306b"
|
||||
+ "\u5f93\u3063\u3066\u3054\u5229\u7528\u3044\u305f\u3060\u304d\u307e\u3059\u3002";
|
||||
List<TextWord> words = cjkWords(paragraph, "NotoSansCJKjp-Bold");
|
||||
assertEquals(
|
||||
"",
|
||||
HeadingDetector.headingPrefix(
|
||||
paragraph, 12f, words, 12f, 12f, "NotoSansCJKjp-Regular", true),
|
||||
"a bold paragraph in a script with no word spaces is body text, not a heading");
|
||||
assertFalse(
|
||||
HeadingDetector.isBoldLabel(paragraph, words),
|
||||
"a paragraph ending in an ideographic stop is a sentence, not a bold label");
|
||||
|
||||
// The guard must not cost the short headings it is meant to keep.
|
||||
String heading = "\u7b2c\u4e09\u7ae0 \u5b9f\u88c5\u306e\u6982\u8981";
|
||||
assertEquals(
|
||||
"### ",
|
||||
HeadingDetector.headingPrefix(
|
||||
heading,
|
||||
12f,
|
||||
cjkWords(heading, "NotoSansCJKjp-Bold"),
|
||||
12f,
|
||||
12f,
|
||||
"NotoSansCJKjp-Regular",
|
||||
true),
|
||||
"a short isolated bold CJK line is still a heading");
|
||||
}
|
||||
|
||||
/**
|
||||
* A crafted PDF can draw thousands of disjoint rules; partitioning used to cost O(N^2) retained
|
||||
* memory, so it must stay linear and bounded.
|
||||
*/
|
||||
@Test
|
||||
@Timeout(20)
|
||||
void ruledTablePartitionSurvivesPathologicalGrid() {
|
||||
// 4000 rules that never cross, so every one is its own component: the shape that made the
|
||||
// old code allocate 4000 arrays of 4001 ints, kept under the crossing-test budget.
|
||||
List<PageRules.Rule> horizontal = new ArrayList<>();
|
||||
List<PageRules.Rule> vertical = new ArrayList<>();
|
||||
for (int i = 0; i < 2_000; i++) {
|
||||
horizontal.add(new PageRules.Rule(i * 10f, 0f, 20f));
|
||||
vertical.add(new PageRules.Rule(1_000_000f + i * 10f, -50f, -30f));
|
||||
}
|
||||
|
||||
int components = assertDoesNotThrow(() -> RuleGrid.componentCount(horizontal, vertical));
|
||||
// 4000 disjoint rules would be 4000 components; the cap is what keeps this bounded.
|
||||
assertEquals(256, components, "component count must stay bounded");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Timeout(20)
|
||||
void ruledTablePartitionBailsOutOnOperatorFlood() {
|
||||
// Enough levels that the pairwise crossing scan alone would dominate the request.
|
||||
List<PageRules.Rule> horizontal = new ArrayList<>();
|
||||
List<PageRules.Rule> vertical = new ArrayList<>();
|
||||
for (int i = 0; i < 20_000; i++) {
|
||||
horizontal.add(new PageRules.Rule(i * 10f, 0f, 20f));
|
||||
vertical.add(new PageRules.Rule(1_000_000f + i * 10f, -50f, -30f));
|
||||
}
|
||||
|
||||
assertTrue(
|
||||
RuleGrid.componentCount(horizontal, vertical) == 0,
|
||||
"a rule flood should disable ruled-table detection, not scan it");
|
||||
}
|
||||
|
||||
private void assertConversionMatchesGolden(String pdfName, String mdName) throws IOException {
|
||||
Path pdfPath = tmp.resolve(pdfName);
|
||||
try (InputStream in =
|
||||
getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + pdfName)) {
|
||||
if (in == null) {
|
||||
fail("Fixture not found on classpath: /pdf-ingestion-fixtures/" + pdfName);
|
||||
}
|
||||
Files.copy(in, pdfPath);
|
||||
}
|
||||
|
||||
String actual;
|
||||
try (PdfDocument doc = PdfDocument.open(pdfPath)) {
|
||||
actual = new AdvancedPdfMarkdownConverter().convert(doc);
|
||||
}
|
||||
|
||||
String expected;
|
||||
try (InputStream in = getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + mdName)) {
|
||||
if (in == null) {
|
||||
fail("Golden file not found on classpath: /pdf-ingestion-fixtures/" + mdName);
|
||||
}
|
||||
expected = new String(in.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
// Image placeholders are not scored: their body text is a TODO rather than real content, so
|
||||
// comparing it would penalise output for matching a placeholder we intend to replace.
|
||||
expected = stripImagePlaceholders(expected);
|
||||
actual = stripImagePlaceholders(actual);
|
||||
|
||||
double similarity = similarity(expected, actual);
|
||||
if (similarity < THRESHOLD) {
|
||||
fail(
|
||||
String.format(
|
||||
"Markdown output differs from golden file '%s' by %.1f%% (threshold %.0f%%):%n%s",
|
||||
mdName,
|
||||
(1.0 - similarity) * 100,
|
||||
(1.0 - THRESHOLD) * 100,
|
||||
unifiedDiff(expected, actual)));
|
||||
}
|
||||
}
|
||||
|
||||
/** Substring identifying an image-placeholder line, which is excluded from scoring. */
|
||||
private static final String IMAGE_PLACEHOLDER_MARKER = "Image intentionally redacted";
|
||||
|
||||
/**
|
||||
* Removes non-content lines: image placeholders, and GFM separator rows whose exact dash count
|
||||
* is cosmetic.
|
||||
*/
|
||||
private static String stripImagePlaceholders(String md) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (String line : md.split("\n", -1)) {
|
||||
if (line.contains(IMAGE_PLACEHOLDER_MARKER)
|
||||
|| line.strip().startsWith("<image redacted")
|
||||
|| isTableSeparatorRow(line)) {
|
||||
continue;
|
||||
}
|
||||
if (!sb.isEmpty()) {
|
||||
sb.append('\n');
|
||||
}
|
||||
sb.append(line);
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/** True for a GFM table separator row, e.g. {@code |---|:--:|---|} (only |, -, :, space). */
|
||||
private static boolean isTableSeparatorRow(String line) {
|
||||
String t = line.strip();
|
||||
if (!t.contains("-")) {
|
||||
return false;
|
||||
}
|
||||
return t.chars().allMatch(c -> c == '|' || c == '-' || c == ':' || c == ' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Character-level similarity: the fraction of expected characters in the LCS. O(n*m), fine for
|
||||
* small goldens.
|
||||
*/
|
||||
private static double similarity(String expected, String actual) {
|
||||
if (expected.isEmpty() && actual.isEmpty()) return 1.0;
|
||||
if (expected.isEmpty() || actual.isEmpty()) return 0.0;
|
||||
// Strip all whitespace for a content-focused comparison
|
||||
String e = expected.replaceAll("\\s+", " ").strip();
|
||||
String a = actual.replaceAll("\\s+", " ").strip();
|
||||
int lcs = lcsLength(e, a);
|
||||
return (double) lcs / Math.max(e.length(), a.length());
|
||||
}
|
||||
|
||||
private static int lcsLength(String a, String b) {
|
||||
// Use two-row DP to keep memory reasonable
|
||||
int m = a.length(), n = b.length();
|
||||
int[] prev = new int[n + 1];
|
||||
int[] curr = new int[n + 1];
|
||||
for (int i = 1; i <= m; i++) {
|
||||
for (int j = 1; j <= n; j++) {
|
||||
if (a.charAt(i - 1) == b.charAt(j - 1)) {
|
||||
curr[j] = prev[j - 1] + 1;
|
||||
} else {
|
||||
curr[j] = Math.max(curr[j - 1], prev[j]);
|
||||
}
|
||||
}
|
||||
int[] tmp = prev;
|
||||
prev = curr;
|
||||
curr = tmp;
|
||||
java.util.Arrays.fill(curr, 0);
|
||||
}
|
||||
return prev[n];
|
||||
}
|
||||
|
||||
private static String unifiedDiff(String expected, String actual) {
|
||||
String[] expectedLines = expected.split("\n", -1);
|
||||
String[] actualLines = actual.split("\n", -1);
|
||||
|
||||
List<String> diff = new ArrayList<>();
|
||||
diff.add("--- expected");
|
||||
diff.add("+++ actual");
|
||||
|
||||
int maxLines = Math.max(expectedLines.length, actualLines.length);
|
||||
int context = 3;
|
||||
boolean inHunk = false;
|
||||
int hunkStart = -1;
|
||||
List<String> hunkLines = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < maxLines; i++) {
|
||||
String exp = i < expectedLines.length ? expectedLines[i] : null;
|
||||
String act = i < actualLines.length ? actualLines[i] : null;
|
||||
|
||||
boolean changed = exp == null || act == null || !exp.equals(act);
|
||||
if (changed) {
|
||||
if (!inHunk) {
|
||||
inHunk = true;
|
||||
hunkStart = Math.max(0, i - context);
|
||||
// add context lines before change
|
||||
for (int c = hunkStart; c < i; c++) {
|
||||
hunkLines.add(" " + (c < expectedLines.length ? expectedLines[c] : ""));
|
||||
}
|
||||
}
|
||||
if (exp != null) hunkLines.add("-" + exp);
|
||||
if (act != null) hunkLines.add("+" + act);
|
||||
} else {
|
||||
if (inHunk) {
|
||||
hunkLines.add(" " + exp);
|
||||
// check if we're far enough past the last change to close the hunk
|
||||
boolean moreChanges = false;
|
||||
for (int j = i + 1; j < Math.min(i + context, maxLines); j++) {
|
||||
String e2 = j < expectedLines.length ? expectedLines[j] : null;
|
||||
String a2 = j < actualLines.length ? actualLines[j] : null;
|
||||
if (e2 == null || a2 == null || !e2.equals(a2)) {
|
||||
moreChanges = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!moreChanges && (i - hunkStart) >= context) {
|
||||
diff.add("@@ -" + (hunkStart + 1) + " @@");
|
||||
diff.addAll(hunkLines);
|
||||
hunkLines.clear();
|
||||
inHunk = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (inHunk && !hunkLines.isEmpty()) {
|
||||
diff.add("@@ -" + (hunkStart + 1) + " @@");
|
||||
diff.addAll(hunkLines);
|
||||
}
|
||||
|
||||
return String.join("\n", diff);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user