mirror of
https://github.com/Stirling-Tools/Stirling-PDF.git
synced 2026-09-03 05:10:16 +03:00
Compare commits
38
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b19e85698e | ||
|
|
f097450883 | ||
|
|
34694c6f5e | ||
|
|
0b7b4e02c2 | ||
|
|
74be5bf0ad | ||
|
|
8c00fffe18 | ||
|
|
c5da4177c4 | ||
|
|
ddc0ac0ced | ||
|
|
8bdd00b2fa | ||
|
|
3718af45ff | ||
|
|
41cbd97b48 | ||
|
|
993adaa3cd | ||
|
|
ead8a536d2 | ||
|
|
c22d9ecf58 | ||
|
|
d3708c1e63 | ||
|
|
1c055f3d18 | ||
|
|
4ab2505a6c | ||
|
|
658aa54c20 | ||
|
|
0a3f0c1814 | ||
|
|
941051648e | ||
|
|
33a22db712 | ||
|
|
1e36895561 | ||
|
|
ea05a3f576 | ||
|
|
1c3d216e70 | ||
|
|
7159b1d179 | ||
|
|
8f0b3a6d94 | ||
|
|
8b1a9c8902 | ||
|
|
c35b252092 | ||
|
|
ebe216ee8d | ||
|
|
84af8476bc | ||
|
|
7e0ef7cfae | ||
|
|
ffc9052961 | ||
|
|
532e2a675a | ||
|
|
72c67520ef | ||
|
|
8b1c045349 | ||
|
|
dc70c0b821 | ||
|
|
6f4a951291 | ||
|
|
4dac016b64 |
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/claude-code-settings.json",
|
||||
"hooks": {
|
||||
"Stop": [
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node",
|
||||
"args": [
|
||||
"${CLAUDE_PROJECT_DIR}/scripts/lint/comment-lint-hook.mjs"
|
||||
],
|
||||
"timeout": 60
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ Closes #(issue_number)
|
||||
- [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable)
|
||||
- [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable)
|
||||
- [ ] I have performed a self-review of my own code
|
||||
- [ ] Every comment I added says something the code does not ([guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/CODE_COMMENTS.md))
|
||||
- [ ] My changes generate no new warnings
|
||||
|
||||
### Documentation
|
||||
|
||||
@@ -34,7 +34,6 @@ jobs:
|
||||
cache-dependency-glob: |
|
||||
engine/pyproject.toml
|
||||
engine/uv.lock
|
||||
cache-suffix: ai-engine
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
|
||||
@@ -42,7 +42,6 @@ jobs:
|
||||
cache-dependency-glob: |
|
||||
engine/pyproject.toml
|
||||
engine/uv.lock
|
||||
cache-suffix: generated-models
|
||||
|
||||
- name: Restore cache Gradle User Home
|
||||
if: inputs.use_shared_cache
|
||||
|
||||
@@ -31,10 +31,14 @@ jobs:
|
||||
cache-dependency-glob: |
|
||||
engine/pyproject.toml
|
||||
engine/uv.lock
|
||||
cache-suffix: pre-commit
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
|
||||
- name: Run pre-commit checks
|
||||
run: task pre-commit
|
||||
|
||||
# The fixture corpus checks the comment rules themselves, so it runs here
|
||||
# rather than on every local commit.
|
||||
- name: Check the comment-lint fixture corpus
|
||||
run: task pre-commit:comment-lint:selftest
|
||||
|
||||
@@ -59,7 +59,6 @@ jobs:
|
||||
cache-dependency-glob: |
|
||||
engine/pyproject.toml
|
||||
engine/uv.lock
|
||||
cache-suffix: sync-files
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
|
||||
+7
-2
@@ -298,8 +298,13 @@ docs/type3/signatures/
|
||||
|
||||
**/application-dev-local.properties
|
||||
|
||||
# Claude
|
||||
.claude/
|
||||
# Claude. Contents are ignored so personal config stays local, with the two
|
||||
# shared pieces re-included: settings.json (the comment-lint hook) and skills/.
|
||||
# The directory itself cannot be ignored or git will not look inside it.
|
||||
.claude/*
|
||||
!.claude/settings.json
|
||||
!.claude/skills/
|
||||
.claude/settings.local.json
|
||||
|
||||
# Playwright MCP screenshots / traces
|
||||
.playwright-mcp/
|
||||
|
||||
@@ -11,6 +11,7 @@ vars:
|
||||
'.github/scripts/*.py'
|
||||
'app/core/src/main/resources/static/python/*.py'
|
||||
':(exclude)*split_photos.py'
|
||||
':(exclude)scripts/lint/fixtures/*'
|
||||
SPELL_FILES: >-
|
||||
'*.html'
|
||||
'*.css'
|
||||
@@ -59,6 +60,7 @@ tasks:
|
||||
- task: gitleaks
|
||||
- task: whitespace
|
||||
- task: toml-sort
|
||||
- task: comment-lint
|
||||
|
||||
fix:
|
||||
desc: "Auto-fix formatting, spelling, and secrets issues across the repo"
|
||||
@@ -75,6 +77,7 @@ tasks:
|
||||
vars: { FIX: '1' }
|
||||
- task: codespell
|
||||
- task: gitleaks
|
||||
- task: comment-lint
|
||||
|
||||
install:
|
||||
desc: "Install the pinned pre-commit Python tools"
|
||||
@@ -130,6 +133,85 @@ tasks:
|
||||
cmds:
|
||||
- "{{.GITLEAKS_BIN}} git --pre-commit --redact --staged --verbose"
|
||||
|
||||
comment-lint:
|
||||
desc: "Check comment quality on the lines this branch adds"
|
||||
summary: |
|
||||
Blocks a comment that restates the code below it, a section banner, or a
|
||||
block of commented-out code. Everything else it reports is advisory.
|
||||
|
||||
Scoped to added lines, so touching an old file never surfaces the standing
|
||||
backlog. The standard is devGuide/CODE_COMMENTS.md.
|
||||
|
||||
With no arguments it diffs the working tree against HEAD, which is what a
|
||||
pre-commit run wants: the lines you are about to commit. On a CI pull request
|
||||
it diffs against the target branch instead, via GITHUB_BASE_REF.
|
||||
|
||||
To ask what a whole branch adds instead, use the branch variant, which
|
||||
needs no argument passing:
|
||||
task comment-lint:branch
|
||||
|
||||
Full tree (report only): task pre-commit:comment-lint:all
|
||||
Fixture corpus: task pre-commit:comment-lint:selftest
|
||||
# Depends on the frontend install because the .ts/.tsx half of the rule set
|
||||
# runs as an oxlint plugin. Without it the TS engine warns and skips, which
|
||||
# would leave the frontend silently unchecked on CI.
|
||||
deps: [":frontend:install"]
|
||||
cmds:
|
||||
- node scripts/lint/comment-lint.mjs {{.CLI_ARGS}}
|
||||
|
||||
comment-lint:branch:
|
||||
desc: "Check comment quality on everything this branch adds over its base"
|
||||
summary: |
|
||||
Like `task comment-lint`, but scoped to the whole branch rather than to
|
||||
uncommitted work, so it still reports after you commit.
|
||||
|
||||
Exists as its own task because passing `-- --since origin/main` through Task
|
||||
is not portable: with the npm build of Task the launcher is a PowerShell
|
||||
script, and PowerShell strips the `--` before Task sees it, leaving Task to
|
||||
print its own usage.
|
||||
|
||||
Override the base with BASE=<ref>.
|
||||
vars:
|
||||
BASE: '{{.BASE | default "origin/main"}}'
|
||||
deps: [":frontend:install"]
|
||||
cmds:
|
||||
- node scripts/lint/comment-lint.mjs --since {{.BASE}}
|
||||
|
||||
comment-lint:ci:
|
||||
desc: "Comment gate as CI runs it: fixture corpus, then the diff"
|
||||
summary: |
|
||||
The corpus checks the rules themselves rather than the code under review, so
|
||||
it belongs on CI and not on every local commit. Run this before changing a
|
||||
rule, and let CI run it on every pull request.
|
||||
deps: [":frontend:install"]
|
||||
cmds:
|
||||
- node scripts/lint/comment-lint.mjs --selftest
|
||||
- node scripts/lint/comment-lint.mjs {{.CLI_ARGS}}
|
||||
|
||||
comment-lint:hook:
|
||||
desc: "Comment gate for the editor hook: everything this turn changed"
|
||||
summary: |
|
||||
Same scope as `task comment-lint`, kept as its own name so the hook has a
|
||||
stable entry point and the taskfile shows every way the linter is invoked.
|
||||
|
||||
Not in the frontend-install dependency chain on purpose: this runs at the end
|
||||
of every turn, so it stays as short as it can be. If oxlint is missing the TS
|
||||
half warns and skips.
|
||||
cmds:
|
||||
- node scripts/lint/comment-lint.mjs
|
||||
|
||||
comment-lint:all:
|
||||
desc: "Report every comment finding in the tree (never fails)"
|
||||
deps: [":frontend:install"]
|
||||
cmds:
|
||||
- node scripts/lint/comment-lint.mjs --all
|
||||
|
||||
comment-lint:selftest:
|
||||
desc: "Check both comment-lint engines against the fixture corpus"
|
||||
deps: [":frontend:install"]
|
||||
cmds:
|
||||
- node scripts/lint/comment-lint.mjs --selftest
|
||||
|
||||
gitleaks-bin:
|
||||
internal: true
|
||||
desc: "Ensure the pinned, checksum-verified gitleaks binary is cached in .task/bin"
|
||||
|
||||
@@ -21,6 +21,43 @@ Task `desc:` fields should describe **what** the task does, not **how** it does
|
||||
- `task docker:build` — build standard Docker image
|
||||
- `task docker:up` — start Docker compose stack
|
||||
|
||||
## Comments
|
||||
|
||||
A comment must carry information the code cannot. If a reader could derive it from the code in front of them, delete it.
|
||||
|
||||
Comment the current state. Not what the code used to do, not what changed, not why it changed: git holds that. Where history explains the shape, state the reason instead, so "this used to reimplement the modal internals" becomes "thin wrapper over the shared Modal: duplicating its portal and focus trap is how dialogs drift apart". Future state goes in a TODO with an issue.
|
||||
|
||||
Write a comment when it does one of these four jobs:
|
||||
|
||||
- **Contract.** What a caller must know that the signature cannot say: preconditions, invariants, units, ownership and lifetime, thread-safety, error semantics, side effects. Document the contract of everything a caller outside the file can reach, and nothing else. Goes on the type/method/module as Javadoc, JSDoc, or a docstring.
|
||||
- **Why.** The constraint the code satisfies, the bug it avoids, the alternative rejected and the reason.
|
||||
- **Hazard.** "Must stay in sync with X", "order matters because Y", "do not remove, it prevents Z".
|
||||
- **Map.** A short orientation at the top of a genuinely complex file: what it owns, and what it deliberately does not.
|
||||
|
||||
Never write:
|
||||
|
||||
- A comment that restates the next line. `// Handle drag start` above `handleDragStart` is noise.
|
||||
- Section banners or position markers: `// --- Types ---`, `// Helpers`, `// =====`.
|
||||
- Step narration in a function body (`// Step 1:`, `// Then we`). If the steps need labels they need names: extract functions. Numbering a genuinely numbered thing, like a wizard step, is fine.
|
||||
- Commented-out code. Delete it.
|
||||
- Doc tags that restate the signature. `@param blob - The blob` says nothing; omit the tag rather than pad it.
|
||||
- Docs on self-explanatory members with no constraint to state.
|
||||
|
||||
Two tests before keeping a comment:
|
||||
|
||||
- **Delete it.** Is any information lost? If not, it stays deleted.
|
||||
- **Could a name carry it instead?** A better identifier, an extracted function, or a named constant beats a comment. Prefer the code change.
|
||||
|
||||
A comment at the end of a line usually decodes that line, and that is worth keeping: `{0x25, 0x50} // "%PDF"`, `50L * 1024 * 1024 // 50 MB`. The rules that compare a comment against the code below it do not apply there, but a trailing TODO or a trailing bit of history is judged like any other.
|
||||
|
||||
A reference is supplementary, never load-bearing: the comment must survive deleting it. `// See #1234` is a dead end; `// saving first loses every annotation (#6865)` is not. Prefer a spec (`RFC 3161`) or CVE where one applies.
|
||||
|
||||
A TODO needs an issue, not an owner: `// TODO(#1234): re-enable the gate once account syncing lands`. If it is not worth an issue, it is not worth a TODO. A question is not a TODO.
|
||||
|
||||
A comment block over ~12 lines outside a file or type header usually means the code needs restructuring, or that the prose is product documentation and belongs in the docs repo.
|
||||
|
||||
`task comment-lint` checks the mechanical part of this on the lines you add, and runs inside `task pre-commit`. Reasoning, worked examples and the linter's own rules: @devGuide/CODE_COMMENTS.md
|
||||
|
||||
## Common Development Commands
|
||||
|
||||
### Build and Test
|
||||
@@ -70,7 +107,7 @@ The project structure is defined in `engine/pyproject.toml`. Any new dependencie
|
||||
- Avoid nested functions and nested classes unless the language construct requires them.
|
||||
- Prefer composition to inheritance when combining concepts.
|
||||
- Avoid speculative abstractions. Add a layer only when it removes real duplication or clarifies lifecycle.
|
||||
- Add comments sparingly and only when they explain non-obvious intent.
|
||||
- Comments follow the repo-wide rules in the "Comments" section above.
|
||||
|
||||
#### Python Typing and Models
|
||||
- Deserialize into Pydantic models as early as possible.
|
||||
|
||||
@@ -42,6 +42,7 @@ Please make sure your Pull Request adheres to the following guidelines:
|
||||
- Keep commits atomic. One commit should contain one change. If you want to make multiple changes, submit multiple Pull Requests.
|
||||
- Commits should be clear, concise, and easy to understand.
|
||||
- References to the Issue number in the Pull Request and/or Commit message.
|
||||
- Every comment in the diff should say something the code does not. See [Code comments](devGuide/CODE_COMMENTS.md); `task comment-lint` checks the mechanical part.
|
||||
|
||||
## Translations
|
||||
|
||||
|
||||
@@ -266,6 +266,20 @@ tasks:
|
||||
cmds:
|
||||
- task: frontend:lint
|
||||
- task: engine:lint
|
||||
- task: comment-lint
|
||||
|
||||
comment-lint:
|
||||
desc: "Check comment quality on the lines this branch adds"
|
||||
aliases: [comments]
|
||||
cmds:
|
||||
- task: pre-commit:comment-lint
|
||||
vars: { CLI_ARGS: '{{.CLI_ARGS}}' }
|
||||
|
||||
comment-lint:branch:
|
||||
desc: "Check comment quality on everything this branch adds over its base"
|
||||
cmds:
|
||||
- task: pre-commit:comment-lint:branch
|
||||
vars: { BASE: '{{.BASE}}' }
|
||||
|
||||
fix:
|
||||
desc: "Auto-fix all components"
|
||||
|
||||
@@ -48,7 +48,7 @@ public class EndpointConfiguration {
|
||||
private final ApplicationProperties applicationProperties;
|
||||
@Getter private Map<String, Boolean> endpointStatuses = new ConcurrentHashMap<>();
|
||||
private Map<String, Set<String>> endpointGroups = new ConcurrentHashMap<>();
|
||||
private Set<String> disabledGroups = new HashSet<>();
|
||||
private Set<String> disabledGroups = ConcurrentHashMap.newKeySet();
|
||||
private Map<String, DisableReason> endpointDisableReasons = new ConcurrentHashMap<>();
|
||||
private Map<String, DisableReason> groupDisableReasons = new ConcurrentHashMap<>();
|
||||
private Map<String, Set<String>> endpointAlternatives = new ConcurrentHashMap<>();
|
||||
|
||||
@@ -237,7 +237,7 @@ public class TabulaTableParser implements TableParser {
|
||||
score -= 0.3f;
|
||||
}
|
||||
|
||||
return Math.max(0f, Math.min(1f, score));
|
||||
return Math.clamp(score, 0f, 1f);
|
||||
}
|
||||
|
||||
private Bounds tableBounds(Table table) {
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package stirling.software.common.pdf;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
|
||||
/** Built-in Markdown conversion, used whenever no richer extractor is registered. */
|
||||
@Service
|
||||
public class BasicPdfMarkdownExtractor implements PdfMarkdownExtractor {
|
||||
|
||||
@Override
|
||||
public String convert(PdfDocument doc) throws IOException {
|
||||
return new PdfMarkdownConverter().convert(doc);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package stirling.software.common.pdf;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
|
||||
/**
|
||||
* Seam for PDF to Markdown conversion. The proprietary module supplies a layout-aware
|
||||
* implementation that takes precedence on the classpath.
|
||||
*/
|
||||
public interface PdfMarkdownExtractor {
|
||||
|
||||
String convert(PdfDocument doc) throws IOException;
|
||||
}
|
||||
+2
-1
@@ -15,7 +15,8 @@ public class StringToMapPropertyEditor extends PropertyEditorSupport {
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
try {
|
||||
TypeReference<HashMap<String, String>> typeRef = new TypeReference<>() {};
|
||||
TypeReference<HashMap<String, String>> typeRef =
|
||||
new TypeReference<HashMap<String, String>>() {};
|
||||
Map<String, String> map = objectMapper.readValue(text, typeRef);
|
||||
setValue(map);
|
||||
} catch (Exception e) {
|
||||
|
||||
+1
-1
@@ -237,7 +237,7 @@ public class EditTextController {
|
||||
|
||||
Matcher matcher = edit.pattern().matcher(joined);
|
||||
List<MatchSpan> spans = new ArrayList<>();
|
||||
StringBuffer interpolation = new StringBuffer();
|
||||
StringBuilder interpolation = new StringBuilder();
|
||||
int previousAppendPosition = 0;
|
||||
while (matcher.find()) {
|
||||
if (matcher.start() == matcher.end()) {
|
||||
|
||||
@@ -95,7 +95,8 @@ public class UIDataController {
|
||||
|
||||
try (InputStream is = resource.getInputStream()) {
|
||||
Map<String, List<Dependency>> licenseData =
|
||||
objectMapper.readValue(is, new TypeReference<>() {});
|
||||
objectMapper.readValue(
|
||||
is, new TypeReference<Map<String, List<Dependency>>>() {});
|
||||
data.setDependencies(licenseData.get("dependencies"));
|
||||
} catch (IOException e) {
|
||||
log.error("Failed to load licenses data", e);
|
||||
|
||||
+6
-3
@@ -25,12 +25,15 @@ final class FormPayloadParser {
|
||||
private static final String KEY_VALUE = "value";
|
||||
private static final String KEY_DEFAULT_VALUE = "defaultValue";
|
||||
|
||||
private static final TypeReference<Map<String, Object>> MAP_TYPE = new TypeReference<>() {};
|
||||
private static final TypeReference<Map<String, Object>> MAP_TYPE =
|
||||
new TypeReference<Map<String, Object>>() {};
|
||||
private static final TypeReference<List<FormUtils.ModifyFormFieldDefinition>>
|
||||
MODIFY_FIELD_LIST_TYPE = new TypeReference<>() {};
|
||||
MODIFY_FIELD_LIST_TYPE =
|
||||
new TypeReference<List<FormUtils.ModifyFormFieldDefinition>>() {};
|
||||
private static final TypeReference<List<FormUtils.NewFormFieldDefinition>> NEW_FIELD_LIST_TYPE =
|
||||
new TypeReference<>() {};
|
||||
private static final TypeReference<List<String>> STRING_LIST_TYPE = new TypeReference<>() {};
|
||||
private static final TypeReference<List<String>> STRING_LIST_TYPE =
|
||||
new TypeReference<List<String>>() {};
|
||||
|
||||
private FormPayloadParser() {}
|
||||
|
||||
|
||||
+3
-1
@@ -96,7 +96,9 @@ public class AddCommentsController {
|
||||
|
||||
List<CommentSpecDto> dtos;
|
||||
try {
|
||||
dtos = objectMapper.readValue(commentsJson, new TypeReference<>() {});
|
||||
dtos =
|
||||
objectMapper.readValue(
|
||||
commentsJson, new TypeReference<List<CommentSpecDto>>() {});
|
||||
} catch (JacksonException e) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST, "comments must be a JSON array of CommentSpec objects");
|
||||
|
||||
+3
-2
@@ -19,7 +19,7 @@ import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
import stirling.software.common.model.tool.ToolFormat;
|
||||
import stirling.software.common.model.tool.ToolIO;
|
||||
import stirling.software.common.pdf.PdfMarkdownConverter;
|
||||
import stirling.software.common.pdf.PdfMarkdownExtractor;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
@@ -30,6 +30,7 @@ import stirling.software.jpdfium.PdfDocument;
|
||||
public class ConvertPDFToMarkdown {
|
||||
|
||||
private final TempFileManager tempFileManager;
|
||||
private final PdfMarkdownExtractor markdownExtractor;
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
@@ -54,7 +55,7 @@ public class ConvertPDFToMarkdown {
|
||||
try (TempFile tempInput = new TempFile(tempFileManager, ".pdf")) {
|
||||
inputFile.transferTo(tempInput.getFile());
|
||||
try (PdfDocument doc = PdfDocument.open(tempInput.getPath())) {
|
||||
markdown = new PdfMarkdownConverter().convert(doc);
|
||||
markdown = markdownExtractor.convert(doc);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+11
-8
@@ -4,7 +4,9 @@ import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -21,7 +23,7 @@ public class WeeklyActiveUsersService {
|
||||
private final Map<String, Instant> activeBrowsers = new ConcurrentHashMap<>();
|
||||
|
||||
// Track total unique browsers seen (overall)
|
||||
private long totalUniqueBrowsers = 0;
|
||||
private final AtomicLong totalUniqueBrowsers = new AtomicLong(0);
|
||||
|
||||
// Application start time
|
||||
private final Instant startTime = Instant.now();
|
||||
@@ -36,12 +38,12 @@ public class WeeklyActiveUsersService {
|
||||
return;
|
||||
}
|
||||
|
||||
boolean isNewBrowser = !activeBrowsers.containsKey(browserId);
|
||||
activeBrowsers.put(browserId, Instant.now());
|
||||
Instant now = Instant.now();
|
||||
Instant previous = activeBrowsers.put(browserId, now);
|
||||
|
||||
if (isNewBrowser) {
|
||||
totalUniqueBrowsers++;
|
||||
log.debug("New browser recorded: {} (Total: {})", browserId, totalUniqueBrowsers);
|
||||
if (previous == null) {
|
||||
long total = totalUniqueBrowsers.incrementAndGet();
|
||||
log.debug("New browser recorded: {} (Total: {})", browserId, total);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,7 +63,7 @@ public class WeeklyActiveUsersService {
|
||||
* @return Total unique browsers count
|
||||
*/
|
||||
public long getTotalUniqueBrowsers() {
|
||||
return totalUniqueBrowsers;
|
||||
return totalUniqueBrowsers.get();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -88,7 +90,8 @@ public class WeeklyActiveUsersService {
|
||||
activeBrowsers.entrySet().removeIf(entry -> entry.getValue().isBefore(sevenDaysAgo));
|
||||
}
|
||||
|
||||
/** Manual cleanup trigger (can be called by scheduled task if needed) */
|
||||
/** Scheduled cleanup trigger running every hour */
|
||||
@Scheduled(fixedRate = 3600000)
|
||||
public void performCleanup() {
|
||||
int sizeBefore = activeBrowsers.size();
|
||||
cleanupOldEntries();
|
||||
|
||||
+13
-17
@@ -23,14 +23,14 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||
|
||||
import stirling.software.common.pdf.PdfMarkdownConverter;
|
||||
import stirling.software.common.pdf.PdfMarkdownExtractor;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
|
||||
class ConvertPDFToMarkdownTest {
|
||||
|
||||
private MockMvc mockMvc() {
|
||||
return MockMvcBuilders.standaloneSetup(new ConvertPDFToMarkdown(null))
|
||||
private MockMvc mockMvc(PdfMarkdownExtractor extractor) {
|
||||
return MockMvcBuilders.standaloneSetup(new ConvertPDFToMarkdown(null, extractor))
|
||||
.setControllerAdvice(new GlobalErrorHandler())
|
||||
.build();
|
||||
}
|
||||
@@ -61,11 +61,10 @@ class ConvertPDFToMarkdownTest {
|
||||
when(mock.getFile()).thenReturn(tmpFile);
|
||||
when(mock.getPath()).thenReturn(tmpFile.toPath());
|
||||
});
|
||||
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class);
|
||||
MockedConstruction<PdfMarkdownConverter> converterMock =
|
||||
Mockito.mockConstruction(
|
||||
PdfMarkdownConverter.class,
|
||||
(mock, ctx) -> when(mock.convert(any())).thenReturn(expectedMd))) {
|
||||
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class)) {
|
||||
|
||||
PdfMarkdownExtractor extractor = Mockito.mock(PdfMarkdownExtractor.class);
|
||||
when(extractor.convert(any())).thenReturn(expectedMd);
|
||||
|
||||
PdfDocument mockDoc = Mockito.mock(PdfDocument.class);
|
||||
docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc);
|
||||
@@ -74,7 +73,7 @@ class ConvertPDFToMarkdownTest {
|
||||
new MockMultipartFile(
|
||||
"fileInput", "input.pdf", "application/pdf", new byte[] {1, 2, 3});
|
||||
|
||||
mockMvc()
|
||||
mockMvc(extractor)
|
||||
.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(header().string("Content-Type", "text/markdown"))
|
||||
@@ -94,13 +93,10 @@ class ConvertPDFToMarkdownTest {
|
||||
when(mock.getFile()).thenReturn(tmpFile);
|
||||
when(mock.getPath()).thenReturn(tmpFile.toPath());
|
||||
});
|
||||
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class);
|
||||
MockedConstruction<PdfMarkdownConverter> converterMock =
|
||||
Mockito.mockConstruction(
|
||||
PdfMarkdownConverter.class,
|
||||
(mock, ctx) ->
|
||||
when(mock.convert(any()))
|
||||
.thenThrow(new RuntimeException("boom")))) {
|
||||
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class)) {
|
||||
|
||||
PdfMarkdownExtractor extractor = Mockito.mock(PdfMarkdownExtractor.class);
|
||||
when(extractor.convert(any())).thenThrow(new RuntimeException("boom"));
|
||||
|
||||
PdfDocument mockDoc = Mockito.mock(PdfDocument.class);
|
||||
docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc);
|
||||
@@ -109,7 +105,7 @@ class ConvertPDFToMarkdownTest {
|
||||
new MockMultipartFile(
|
||||
"fileInput", "x.pdf", "application/pdf", new byte[] {0x01});
|
||||
|
||||
mockMvc()
|
||||
mockMvc(extractor)
|
||||
.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
|
||||
.andExpect(status().isInternalServerError());
|
||||
}
|
||||
|
||||
@@ -98,6 +98,14 @@ dependencies {
|
||||
testImplementation "org.testcontainers:junit-jupiter:${testcontainersMinioVersion}"
|
||||
}
|
||||
|
||||
// Golden extraction fixtures live in :common; the advanced converter is graded against the same
|
||||
// corpus as the built-in one rather than a forked copy of it.
|
||||
processTestResources {
|
||||
from(project(':common').file('src/test/resources/pdf-ingestion-fixtures')) {
|
||||
into 'pdf-ingestion-fixtures'
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register('prepareKotlinBuildScriptModel') {}
|
||||
|
||||
tasks.register('type3SignatureTool', JavaExec) {
|
||||
|
||||
@@ -59,7 +59,7 @@ public enum AuditLevel {
|
||||
*/
|
||||
public static AuditLevel fromInt(int level) {
|
||||
// Ensure level is within valid bounds
|
||||
int boundedLevel = Math.min(Math.max(level, 0), 3);
|
||||
int boundedLevel = Math.clamp(level, 0, 3);
|
||||
|
||||
for (AuditLevel auditLevel : values()) {
|
||||
if (auditLevel.level == boundedLevel) {
|
||||
|
||||
+11
-9
@@ -17,16 +17,16 @@ import org.springframework.data.redis.core.ScanOptions;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.JobStore;
|
||||
import stirling.software.common.cluster.JobStoreEntry;
|
||||
|
||||
import tools.jackson.core.JacksonException;
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Valkey-backed {@link JobStore}. Each job is one hash; a reverse index maps fileId to jobId.
|
||||
*
|
||||
@@ -44,8 +44,10 @@ public class ValkeyJobStore implements JobStore {
|
||||
private static final String FILE_INDEX_PREFIX = "stirling:file2job:";
|
||||
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
private static final TypeReference<List<String>> LIST_STRING = new TypeReference<>() {};
|
||||
private static final TypeReference<Map<String, String>> MAP_STRING = new TypeReference<>() {};
|
||||
private static final TypeReference<List<String>> LIST_STRING =
|
||||
new TypeReference<List<String>>() {};
|
||||
private static final TypeReference<Map<String, String>> MAP_STRING =
|
||||
new TypeReference<Map<String, String>>() {};
|
||||
|
||||
private final StringRedisTemplate template;
|
||||
|
||||
@@ -265,7 +267,7 @@ public class ValkeyJobStore implements JobStore {
|
||||
}
|
||||
try {
|
||||
return MAPPER.readValue(v.toString(), MAP_STRING);
|
||||
} catch (JsonProcessingException e) {
|
||||
} catch (JacksonException e) {
|
||||
log.warn(
|
||||
"JobStore {} field 'resultMeta' is not valid JSON '{}' - treating as empty",
|
||||
key,
|
||||
@@ -277,7 +279,7 @@ public class ValkeyJobStore implements JobStore {
|
||||
private static String writeJson(Object value) {
|
||||
try {
|
||||
return MAPPER.writeValueAsString(value);
|
||||
} catch (JsonProcessingException e) {
|
||||
} catch (JacksonException e) {
|
||||
throw new IllegalStateException("Failed to JSON-serialize JobStore field", e);
|
||||
}
|
||||
}
|
||||
@@ -286,7 +288,7 @@ public class ValkeyJobStore implements JobStore {
|
||||
try {
|
||||
List<String> parsed = MAPPER.readValue(json, LIST_STRING);
|
||||
return parsed == null ? new ArrayList<>() : parsed;
|
||||
} catch (JsonProcessingException e) {
|
||||
} catch (JacksonException e) {
|
||||
log.warn(
|
||||
"JobStore {} field 'fileIds' is not valid JSON '{}' - treating as empty",
|
||||
key,
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ public class AuditConfigurationProperties {
|
||||
|
||||
// Ensure level is within valid bounds (0-3)
|
||||
int configLevel = auditConfig.getLevel();
|
||||
this.level = Math.min(Math.max(configLevel, 0), 3);
|
||||
this.level = Math.clamp(configLevel, 0, 3);
|
||||
|
||||
// Retention days (0 means infinite)
|
||||
this.retentionDays = auditConfig.getRetentionDays();
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ public class UsageRestController {
|
||||
@RequestParam(value = "dataType", defaultValue = "all") String dataType,
|
||||
@RequestParam(value = "days", defaultValue = "30") Integer days) {
|
||||
|
||||
int lookbackDays = Math.max(1, Math.min(days, 365));
|
||||
int lookbackDays = Math.clamp(days, 1, 365);
|
||||
|
||||
// Get audit events filtered by type
|
||||
List<PersistentAuditEvent> events = getEventsByDataType(dataType, lookbackDays);
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package stirling.software.proprietary.model;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
|
||||
import jakarta.persistence.*;
|
||||
@@ -19,7 +20,7 @@ import lombok.*;
|
||||
@ToString
|
||||
public class UserLicenseSettings implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
@Serial private static final long serialVersionUID = 1L;
|
||||
|
||||
public static final Long SINGLETON_ID = 1L;
|
||||
|
||||
|
||||
+269
@@ -0,0 +1,269 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.pdf.PdfMarkdownExtractor;
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.PdfPage;
|
||||
import stirling.software.jpdfium.text.PageText;
|
||||
import stirling.software.jpdfium.text.PdfTextExtractor;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
|
||||
/**
|
||||
* Converts a PDF to Markdown from PDFium {@link TextLine}s. Orchestration only: each stage of the
|
||||
* pipeline lives in its own class in this package.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@Primary
|
||||
public class AdvancedPdfMarkdownConverter implements PdfMarkdownExtractor {
|
||||
|
||||
@Override
|
||||
public String convert(PdfDocument doc) throws IOException {
|
||||
List<String> rendered = new ArrayList<>();
|
||||
for (Object e : buildElements(doc)) {
|
||||
rendered.add(e instanceof TableBlock tb ? tb.render() : (String) e);
|
||||
}
|
||||
return MarkdownText.normaliseHeadingLevels(String.join("\n\n", rendered));
|
||||
}
|
||||
|
||||
private List<Object> buildElements(PdfDocument doc) throws IOException {
|
||||
List<PageText> allPageText = PdfTextExtractor.extractAll(doc);
|
||||
float medianSize = HeadingDetector.medianFontSize(allPageText);
|
||||
float medianHeight = HeadingDetector.medianLineHeight(allPageText);
|
||||
String bodyFont = HeadingDetector.bodyFont(allPageText);
|
||||
|
||||
int pageCount = doc.pageCount();
|
||||
// Tables stay structured until after the page loop so one split across a page break can
|
||||
// be stitched back together before rendering.
|
||||
List<Object> output = new ArrayList<>();
|
||||
// Header of a table that ended the previous page, for spotting a continuation; null if
|
||||
// none.
|
||||
String prevPageTrailingTableHeader = null;
|
||||
|
||||
for (int pageIndex = 0; pageIndex < pageCount; pageIndex++) {
|
||||
PageLines page = pageLines(doc, allPageText, pageIndex);
|
||||
if (page.lines().isEmpty()) {
|
||||
PageImages.emit(doc, pageIndex, output);
|
||||
prevPageTrailingTableHeader = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
List<Object> pageItems =
|
||||
buildPageItems(
|
||||
doc,
|
||||
page,
|
||||
pageIndex,
|
||||
medianSize,
|
||||
medianHeight,
|
||||
bodyFont,
|
||||
prevPageTrailingTableHeader);
|
||||
if (pageItems.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
PageStitcher.mergeAcrossPageBoundary(output, pageItems);
|
||||
output.addAll(pageItems);
|
||||
prevPageTrailingTableHeader = PageStitcher.trailingTableHeader(pageItems);
|
||||
}
|
||||
|
||||
// Stitch tables split across page breaks; callers decide how to realise the elements.
|
||||
return PageStitcher.stitchTables(output);
|
||||
}
|
||||
|
||||
/**
|
||||
* One page's lines plus its layout verdict, which must be taken before text repair: merging
|
||||
* reduces the line count the two-column guard scales with.
|
||||
*/
|
||||
private record PageLines(List<Line> lines, List<Float> gutters) {
|
||||
boolean twoColumnLayout() {
|
||||
return !gutters.isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
/** Assembled lines for one page, sorted top-to-bottom (PDF y=0 is the page bottom). */
|
||||
private static PageLines pageLines(PdfDocument doc, List<PageText> allPageText, int pageIndex) {
|
||||
List<TextLine> rawLines =
|
||||
pageIndex < allPageText.size() ? allPageText.get(pageIndex).lines() : List.of();
|
||||
List<Line> stitched = GlyphStitcher.stitchGlyphs(rawLines);
|
||||
List<Float> gutters = ColumnLayout.detectGutters(stitched);
|
||||
List<Line> lines = LineMerger.mergeLineFragments(stitched, gutters);
|
||||
lines.addAll(FormValues.lines(doc, pageIndex, lines));
|
||||
lines.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
|
||||
return new PageLines(lines, gutters);
|
||||
}
|
||||
|
||||
/**
|
||||
* One page's elements: paragraph strings interleaved with {@link TableBlock}s in reading order.
|
||||
* {@code continuationHeader} is the previous page's trailing table header, or null.
|
||||
*/
|
||||
private List<Object> buildPageItems(
|
||||
PdfDocument doc,
|
||||
PageLines page,
|
||||
int pageIndex,
|
||||
float medianSize,
|
||||
float medianHeight,
|
||||
String bodyFont,
|
||||
String continuationHeader)
|
||||
throws IOException {
|
||||
List<Line> lines = page.lines();
|
||||
// Only genuine two-column prose is split: a table's column gutters must not read as a page
|
||||
// gutter, and a table continuing from the previous page is not a new two-column layout.
|
||||
boolean tableContinuation =
|
||||
continuationHeader != null
|
||||
&& lines.stream()
|
||||
.anyMatch(
|
||||
l ->
|
||||
MarkdownText.normaliseSpace(l.text)
|
||||
.equals(continuationHeader));
|
||||
|
||||
// Merging widens lines, so re-check the pre-repair verdict here: ordering by a gutter the
|
||||
// finished lines no longer respect is worse than not splitting at all.
|
||||
List<Float> gutters = tableContinuation ? List.of() : page.gutters();
|
||||
boolean twoColumn = !gutters.isEmpty();
|
||||
boolean respected = twoColumn && ColumnLayout.gutterRespected(lines, gutters);
|
||||
|
||||
// Two detectors: ruling lines give exact boundaries and see single-word cells; the word
|
||||
// grid covers what the rules do not, i.e. borderless and whitespace-aligned tables.
|
||||
Set<String> tableRowTexts = new HashSet<>();
|
||||
PageRules rules = readRules(doc, pageIndex);
|
||||
List<TableBlock> blocks = TableFinder.find(lines, rules, pageIndex + 1);
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug(
|
||||
"p{} lines={} hRules={} vRules={} twoColumn={} blocks={}",
|
||||
pageIndex,
|
||||
lines.size(),
|
||||
rules.horizontal().size(),
|
||||
rules.vertical().size(),
|
||||
!gutters.isEmpty(),
|
||||
blocks.size());
|
||||
for (TableBlock b : blocks) {
|
||||
List<String[]> cs = b.cells();
|
||||
log.debug(
|
||||
"block top={} bot={} ruled={} src={} rows={} cells={}x{} "
|
||||
+ "grid={} spans={} owns={}",
|
||||
b.top(),
|
||||
b.bottom(),
|
||||
b.ruled(),
|
||||
b.rowSource(),
|
||||
b.rows().size(),
|
||||
cs.size(),
|
||||
cs.isEmpty() ? 0 : cs.get(0).length,
|
||||
TableShape.looksLikeGrid(b),
|
||||
TableShape.spansPage(b, lines),
|
||||
TableShape.ownsItsBand(b, lines));
|
||||
for (List<Line> row : b.rows()) {
|
||||
log.debug(" row: {}", PageStitcher.rowText(row));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (twoColumn) {
|
||||
// On a multi-column page only a full-width block is a table; anything narrower sits
|
||||
// inside a column. A ruled block owning its own band has no column layout to sit in.
|
||||
blocks =
|
||||
blocks.stream()
|
||||
.filter(
|
||||
b ->
|
||||
(b.ruled() || TableShape.looksLikeGrid(b))
|
||||
&& (TableShape.spansPage(b, lines)
|
||||
|| (b.ruled()
|
||||
&& TableShape.ownsItsBand(
|
||||
b, lines))))
|
||||
.toList();
|
||||
}
|
||||
Set<Line> tableLines = new HashSet<>();
|
||||
for (TableBlock b : blocks) {
|
||||
for (List<Line> row : b.rows()) {
|
||||
for (Line l : row) {
|
||||
tableLines.add(l);
|
||||
tableRowTexts.add(l.text.strip());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<Object> pageItems = new ArrayList<>();
|
||||
List<List<Line>> segments = segmentsAround(lines, blocks, tableLines);
|
||||
if (twoColumn) {
|
||||
// A full-width table interrupts both columns, so splitting at its own vertical band
|
||||
// keeps the prose above and below it in column order.
|
||||
for (int s = 0; s < segments.size(); s++) {
|
||||
List<List<Line>> groups =
|
||||
respected
|
||||
? ColumnLayout.orderByBand(segments.get(s), gutters)
|
||||
: ColumnLayout.legacySplit(segments.get(s));
|
||||
for (List<Line> col : groups) {
|
||||
List<String> paras = new ArrayList<>();
|
||||
ParagraphAssembler.assembleParagraphs(
|
||||
col, medianSize, medianHeight, bodyFont, paras, tableRowTexts);
|
||||
pageItems.addAll(paras);
|
||||
}
|
||||
if (s < blocks.size()) {
|
||||
pageItems.add(blocks.get(s));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Interleave tables with text by vertical position: each block gets a slot,
|
||||
// and non-table lines fall into the slot for their y, keeping tables separate.
|
||||
for (int s = 0; s <= blocks.size(); s++) {
|
||||
List<String> paras = new ArrayList<>();
|
||||
ParagraphAssembler.assembleParagraphs(
|
||||
segments.get(s), medianSize, medianHeight, bodyFont, paras, tableRowTexts);
|
||||
pageItems.addAll(paras);
|
||||
if (s < blocks.size()) {
|
||||
pageItems.add(blocks.get(s));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PageImages.emit(doc, pageIndex, pageItems);
|
||||
return pageItems;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits a page's non-table lines into the bands between its table blocks, band {@code s}
|
||||
* holding the lines above block {@code s}. Blocks must be in top-to-bottom order.
|
||||
*/
|
||||
private static List<List<Line>> segmentsAround(
|
||||
List<Line> lines, List<TableBlock> blocks, Set<Line> tableLines) {
|
||||
List<List<Line>> segments = new ArrayList<>();
|
||||
for (int s = 0; s <= blocks.size(); s++) {
|
||||
segments.add(new ArrayList<>());
|
||||
}
|
||||
for (Line l : lines) {
|
||||
if (tableLines.contains(l)) {
|
||||
continue;
|
||||
}
|
||||
int slot = 0;
|
||||
for (TableBlock b : blocks) {
|
||||
if (b.bottom() > l.y) {
|
||||
slot++;
|
||||
}
|
||||
}
|
||||
segments.get(slot).add(l);
|
||||
}
|
||||
return segments;
|
||||
}
|
||||
|
||||
/** Ruling lines of one page, or {@link PageRules#EMPTY} if the page cannot be opened. */
|
||||
private static PageRules readRules(PdfDocument doc, int pageIndex) {
|
||||
try (PdfPage page = doc.page(pageIndex)) {
|
||||
return PageRules.of(page);
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"Page {} ruling lines unreadable; falling back to word-grid tables",
|
||||
pageIndex,
|
||||
e);
|
||||
return PageRules.EMPTY;
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Incremental {@link ColumnRanges#find(List)} for a stitched table: appending a page costs O(page),
|
||||
* bit-for-bit identical to re-projecting the lot.
|
||||
*/
|
||||
final class ColumnAccumulator {
|
||||
|
||||
private int lineCount;
|
||||
private float minX = Float.MAX_VALUE;
|
||||
private float maxX = -Float.MAX_VALUE;
|
||||
private double totalWidth;
|
||||
private int totalChars;
|
||||
|
||||
/** Coverage counts, cov[i] = lines covering absolute x-bucket covBase + i. */
|
||||
private int[] cov = new int[0];
|
||||
|
||||
private int covBase;
|
||||
|
||||
/** Set once the x-span exceeds what findColumnRanges accepts; no histogram is then kept. */
|
||||
private boolean oversized;
|
||||
|
||||
private boolean[] scratch = new boolean[0];
|
||||
|
||||
static ColumnAccumulator of(List<List<Line>> rows) {
|
||||
ColumnAccumulator a = new ColumnAccumulator();
|
||||
for (List<Line> row : rows) {
|
||||
for (Line l : row) {
|
||||
a.addLine(l);
|
||||
}
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
void addLine(Line l) {
|
||||
lineCount++;
|
||||
List<TextWord> words = l.words();
|
||||
int lineLo = Integer.MAX_VALUE;
|
||||
int lineHi = Integer.MIN_VALUE;
|
||||
for (TextWord w : words) {
|
||||
float x0 = w.x();
|
||||
float x1 = x0 + w.width();
|
||||
minX = Math.min(minX, x0);
|
||||
maxX = Math.max(maxX, x1);
|
||||
totalWidth += w.width();
|
||||
totalChars += Math.max(1, w.text().strip().length());
|
||||
int a = (int) Math.floor(x0);
|
||||
int b = (int) Math.ceil(x1);
|
||||
if (a < lineLo) {
|
||||
lineLo = a;
|
||||
}
|
||||
if (b > lineHi) {
|
||||
lineHi = b;
|
||||
}
|
||||
}
|
||||
// Mirrors ColumnRanges.find's guard: past this span it returns no columns, so the
|
||||
// histogram is dead weight and (with crafted coordinates) unboundedly large.
|
||||
if (!oversized && (maxX - minX) > 2000f) {
|
||||
oversized = true;
|
||||
cov = null;
|
||||
scratch = null;
|
||||
}
|
||||
if (oversized || lineHi <= lineLo) {
|
||||
return;
|
||||
}
|
||||
ensureRange(lineLo, lineHi);
|
||||
int n = lineHi - lineLo;
|
||||
if (scratch.length < n) {
|
||||
scratch = new boolean[n];
|
||||
} else {
|
||||
Arrays.fill(scratch, 0, n, false);
|
||||
}
|
||||
for (TextWord w : words) {
|
||||
int a = (int) Math.floor(w.x()) - lineLo;
|
||||
int b = (int) Math.ceil(w.x() + w.width()) - lineLo;
|
||||
for (int x = a; x < b; x++) {
|
||||
scratch[x] = true;
|
||||
}
|
||||
}
|
||||
int off = lineLo - covBase;
|
||||
for (int x = 0; x < n; x++) {
|
||||
if (scratch[x]) {
|
||||
cov[off + x]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureRange(int lo, int hi) {
|
||||
if (cov.length == 0) {
|
||||
covBase = lo - 32;
|
||||
cov = new int[(hi - lo) + 64];
|
||||
return;
|
||||
}
|
||||
int have0 = covBase;
|
||||
int have1 = covBase + cov.length;
|
||||
if (lo >= have0 && hi <= have1) {
|
||||
return;
|
||||
}
|
||||
int newBase = Math.min(have0, lo) - 32;
|
||||
int newEnd = Math.max(have1, hi) + 32;
|
||||
int[] nc = new int[newEnd - newBase];
|
||||
System.arraycopy(cov, 0, nc, have0 - newBase, cov.length);
|
||||
cov = nc;
|
||||
covBase = newBase;
|
||||
}
|
||||
|
||||
/** Exactly what {@link ColumnRanges#find(List)} would return for the accumulated lines. */
|
||||
List<float[]> columns() {
|
||||
if (oversized || maxX <= minX || (maxX - minX) > 2000f) {
|
||||
return List.of();
|
||||
}
|
||||
int lo = (int) Math.floor(minX);
|
||||
int span = Math.min((int) Math.ceil(maxX) - lo + 1, 2001);
|
||||
int support = Math.max(2, Math.round(lineCount * 0.35f));
|
||||
List<float[]> columns = new ArrayList<>();
|
||||
int start = -1;
|
||||
for (int x = 0; x < span; x++) {
|
||||
int idx = lo + x - covBase;
|
||||
int c = (idx >= 0 && idx < cov.length) ? cov[idx] : 0;
|
||||
boolean isColumn = c >= support;
|
||||
if (isColumn && start < 0) {
|
||||
start = x;
|
||||
} else if (!isColumn && start >= 0) {
|
||||
columns.add(new float[] {lo + start, lo + x});
|
||||
start = -1;
|
||||
}
|
||||
}
|
||||
if (start >= 0) {
|
||||
columns.add(new float[] {(float) (lo + start), (float) (lo + span)});
|
||||
}
|
||||
|
||||
float charWidth = totalChars == 0 ? 6f : (float) (totalWidth / totalChars);
|
||||
float minGutter = Math.max(10f, charWidth * 2.5f);
|
||||
List<float[]> merged = new ArrayList<>();
|
||||
for (float[] band : columns) {
|
||||
if (!merged.isEmpty() && band[0] - merged.get(merged.size() - 1)[1] < minGutter) {
|
||||
merged.get(merged.size() - 1)[1] = band[1];
|
||||
} else {
|
||||
merged.add(new float[] {band[0], band[1]});
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
|
||||
/**
|
||||
* Multi-column page layout: finding the gutters between columns of prose, splitting a page's lines
|
||||
* at them, and emitting the columns in reading order.
|
||||
*/
|
||||
final class ColumnLayout {
|
||||
|
||||
private ColumnLayout() {}
|
||||
|
||||
/** Narrowest run of near-empty x that can separate two columns of prose. */
|
||||
private static final float MIN_GUTTER = 10f;
|
||||
|
||||
/** Narrowest column worth splitting out; below this a "gutter" is just a ragged margin. */
|
||||
private static final float MIN_COLUMN = 70f;
|
||||
|
||||
/** Fraction of a page's lines that may cross a gutter and still leave it a gutter. */
|
||||
private static final float MAX_CROSSING = 0.15f;
|
||||
|
||||
/** Most columns recognised on one page. Beyond this the geometry is a table, not a layout. */
|
||||
private static final int MAX_COLUMNS = 4;
|
||||
|
||||
/**
|
||||
* Finds the page's column gutters, or empty for a single column. Scans the 5th to 95th
|
||||
* percentile of line edges so one degenerate box cannot drag it off the page.
|
||||
*/
|
||||
static List<Float> detectGutters(List<Line> lines) {
|
||||
if (lines.size() < 8) {
|
||||
return List.of();
|
||||
}
|
||||
int n = lines.size();
|
||||
float[] los = new float[n];
|
||||
float[] his = new float[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
los[i] = lines.get(i).left();
|
||||
his[i] = lines.get(i).right();
|
||||
}
|
||||
float[] sortedLo = los.clone();
|
||||
float[] sortedHi = his.clone();
|
||||
Arrays.sort(sortedLo);
|
||||
Arrays.sort(sortedHi);
|
||||
float lo = sortedLo[(int) (n * 0.05f)];
|
||||
float hi = sortedHi[Math.min(n - 1, (int) (n * 0.95f))];
|
||||
if (hi - lo < 2 * MIN_COLUMN + MIN_GUTTER || !plausibleSpan(lo, hi)) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
int maxCrossing = (int) (n * MAX_CROSSING);
|
||||
int start = -1;
|
||||
List<float[]> bands = new ArrayList<>();
|
||||
// Stepped as an int: past 2^24 a float can no longer represent x + 1, so a float counter
|
||||
// over a crafted coordinate stops advancing and spins forever.
|
||||
int scanFrom = (int) Math.floor(lo + MIN_COLUMN);
|
||||
int scanTo = (int) Math.ceil(hi - MIN_COLUMN);
|
||||
for (int xi = scanFrom; xi <= scanTo; xi++) {
|
||||
float x = xi;
|
||||
int crossing = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (los[i] < x - 2f && his[i] > x + 2f) {
|
||||
crossing++;
|
||||
}
|
||||
}
|
||||
if (crossing <= maxCrossing) {
|
||||
if (start < 0) {
|
||||
start = (int) x;
|
||||
}
|
||||
} else if (start >= 0) {
|
||||
bands.add(new float[] {start, x});
|
||||
start = -1;
|
||||
}
|
||||
}
|
||||
if (start >= 0) {
|
||||
bands.add(new float[] {start, hi - MIN_COLUMN});
|
||||
}
|
||||
|
||||
// Widest first, so the strongest separation wins; then keep only bands MIN_COLUMN apart.
|
||||
bands.sort(Comparator.comparingDouble((float[] b) -> b[1] - b[0]).reversed());
|
||||
List<Float> gutters = new ArrayList<>();
|
||||
for (float[] b : bands) {
|
||||
if (b[1] - b[0] < MIN_GUTTER || gutters.size() >= MAX_COLUMNS - 1) {
|
||||
continue;
|
||||
}
|
||||
float mid = (b[0] + b[1]) / 2f;
|
||||
boolean tooClose = mid - lo < MIN_COLUMN || hi - mid < MIN_COLUMN;
|
||||
for (float g : gutters) {
|
||||
tooClose |= Math.abs(g - mid) < MIN_COLUMN;
|
||||
}
|
||||
if (!tooClose) {
|
||||
gutters.add(mid);
|
||||
}
|
||||
}
|
||||
gutters.sort(Comparator.naturalOrder());
|
||||
|
||||
if (!gutters.isEmpty() && columnsLookLikeText(lines, gutters)) {
|
||||
return gutters;
|
||||
}
|
||||
return centralGutter(lines, los, his, lo, hi);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rejects geometry too wide to be real: past 2^24 a float cannot represent x + 1, so a
|
||||
* constant-step scan stops advancing.
|
||||
*/
|
||||
private static boolean plausibleSpan(float lo, float hi) {
|
||||
return Float.isFinite(lo) && Float.isFinite(hi) && (hi - lo) <= 2000f;
|
||||
}
|
||||
|
||||
/** Fallback: accepts halves of scattered labels, which read as columns but not as prose. */
|
||||
private static List<Float> centralGutter(
|
||||
List<Line> lines, float[] los, float[] his, float lo, float hi) {
|
||||
int n = lines.size();
|
||||
float centreLo = lo + (hi - lo) * 0.35f;
|
||||
float centreHi = lo + (hi - lo) * 0.65f;
|
||||
int bestCrossing = Integer.MAX_VALUE;
|
||||
float bestAt = 0f;
|
||||
int bestLeft = 0;
|
||||
int bestRight = 0;
|
||||
for (int gi = (int) Math.floor(centreLo); gi <= (int) Math.ceil(centreHi); gi += 2) {
|
||||
float gutter = gi;
|
||||
int crossing = 0;
|
||||
int left = 0;
|
||||
int right = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (los[i] < gutter - 5f && his[i] > gutter + 5f) {
|
||||
crossing++;
|
||||
} else if (his[i] <= gutter) {
|
||||
left++;
|
||||
} else {
|
||||
right++;
|
||||
}
|
||||
}
|
||||
if (crossing < bestCrossing) {
|
||||
bestCrossing = crossing;
|
||||
bestAt = gutter;
|
||||
bestLeft = left;
|
||||
bestRight = right;
|
||||
}
|
||||
}
|
||||
boolean ok = bestLeft >= 4 && bestRight >= 4 && bestCrossing <= (int) (n * 0.25f);
|
||||
return ok ? List.of(bestAt) : List.of();
|
||||
}
|
||||
|
||||
/** Lines of at least this fraction of a column's width count as that column's body text. */
|
||||
private static final float BODY_LINE_WIDTH = 0.5f;
|
||||
|
||||
/** Body lines a column must hold before it is accepted as a column. */
|
||||
private static final int BODY_LINES = 4;
|
||||
|
||||
/**
|
||||
* True when every carved-out column reads as running text; projection alone cannot tell prose
|
||||
* from any other empty lane, such as a bar chart's label gaps.
|
||||
*/
|
||||
private static boolean columnsLookLikeText(List<Line> lines, List<Float> gutters) {
|
||||
// Judge only lines inside a column: a spanning line is assigned to one by its centre, and
|
||||
// its width would set a measure no real body line could reach.
|
||||
List<Line> inside =
|
||||
lines.stream().filter(l -> !spansGutter(l, gutters)).collect(Collectors.toList());
|
||||
List<List<Line>> columns = splitIntoColumns(inside, gutters);
|
||||
if (columns.size() < 2) {
|
||||
return false;
|
||||
}
|
||||
for (List<Line> column : columns) {
|
||||
float lo = Float.MAX_VALUE;
|
||||
float hi = -Float.MAX_VALUE;
|
||||
for (Line l : column) {
|
||||
lo = Math.min(lo, l.left());
|
||||
hi = Math.max(hi, l.right());
|
||||
}
|
||||
float measure = hi - lo;
|
||||
int body = 0;
|
||||
for (Line l : column) {
|
||||
if (l.right() - l.left() >= measure * BODY_LINE_WIDTH) {
|
||||
body++;
|
||||
}
|
||||
}
|
||||
if (body < BODY_LINES || measure < MIN_COLUMN) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits lines into columns at the given gutters. A line crossing one goes to the column its
|
||||
* centre falls in; band ordering then places it correctly.
|
||||
*/
|
||||
static List<List<Line>> splitIntoColumns(List<Line> lines, List<Float> gutters) {
|
||||
if (gutters.isEmpty()) {
|
||||
return List.of(lines);
|
||||
}
|
||||
List<List<Line>> columns = new ArrayList<>(gutters.size() + 1);
|
||||
for (int i = 0; i <= gutters.size(); i++) {
|
||||
columns.add(new ArrayList<>());
|
||||
}
|
||||
for (Line l : lines) {
|
||||
columns.get(columnOf(l, gutters)).add(l);
|
||||
}
|
||||
columns.removeIf(List::isEmpty);
|
||||
return columns;
|
||||
}
|
||||
|
||||
private static int columnOf(Line l, List<Float> gutters) {
|
||||
float centre = (l.left() + l.right()) / 2f;
|
||||
int col = 0;
|
||||
while (col < gutters.size() && centre > gutters.get(col)) {
|
||||
col++;
|
||||
}
|
||||
return col;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the finished lines still respect the gutters the unmerged lines showed; merging
|
||||
* widens lines, and band-ordering a straddled gutter interleaves the columns.
|
||||
*/
|
||||
static boolean gutterRespected(List<Line> lines, List<Float> gutters) {
|
||||
List<Line> real = lines.stream().filter(l -> !l.synthetic).toList();
|
||||
if (real.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
long spanning = real.stream().filter(l -> spansGutter(l, gutters)).count();
|
||||
return spanning <= real.size() * BAND_CROSSING;
|
||||
}
|
||||
|
||||
/** Fraction of the finished lines that may straddle a gutter and still allow band ordering. */
|
||||
private static final float BAND_CROSSING = 0.35f;
|
||||
|
||||
/** Fallback column split: cut at the widest gap between the lines' left edges. */
|
||||
static List<List<Line>> legacySplit(List<Line> lines) {
|
||||
List<Float> xs =
|
||||
lines.stream()
|
||||
.filter(l -> l.width >= 40f)
|
||||
.map(l -> l.x)
|
||||
.sorted()
|
||||
.collect(Collectors.toList());
|
||||
if (xs.isEmpty()) {
|
||||
return List.of(lines);
|
||||
}
|
||||
float splitAt = (xs.getFirst() + xs.getLast()) / 2f;
|
||||
float biggestGap = 0;
|
||||
for (int i = 1; i < xs.size(); i++) {
|
||||
float gap = xs.get(i) - xs.get(i - 1);
|
||||
if (gap > biggestGap) {
|
||||
biggestGap = gap;
|
||||
splitAt = (xs.get(i - 1) + xs.get(i)) / 2f;
|
||||
}
|
||||
}
|
||||
List<Line> left = new ArrayList<>();
|
||||
List<Line> right = new ArrayList<>();
|
||||
for (Line l : lines) {
|
||||
(l.x < splitAt ? left : right).add(l);
|
||||
}
|
||||
if (left.isEmpty()) {
|
||||
return List.of(right);
|
||||
}
|
||||
if (right.isEmpty()) {
|
||||
return List.of(left);
|
||||
}
|
||||
return List.of(left, right);
|
||||
}
|
||||
|
||||
/** Longest a line may be and still be a line of a heading rather than of a paragraph. */
|
||||
private static final int HEADING_LENGTH_WORDS = 12;
|
||||
|
||||
/**
|
||||
* True when a spanning line is short enough to be one line of a full-width banner heading,
|
||||
* which {@link #orderByBand} keeps in a single group.
|
||||
*/
|
||||
private static boolean headingLength(Line l) {
|
||||
return MarkdownText.wordCount(l.text) <= HEADING_LENGTH_WORDS;
|
||||
}
|
||||
|
||||
/** True when a line straddles a gutter, i.e. it belongs to no single column. */
|
||||
static boolean spansGutter(Line l, List<Float> gutters) {
|
||||
float left = l.left();
|
||||
float right = l.right();
|
||||
for (float g : gutters) {
|
||||
if (left < g - 2f && right > g + 2f) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Orders a multi-column region as a one-level XY cut: spanning lines cut it into bands, and
|
||||
* each band's columns are emitted in turn.
|
||||
*/
|
||||
static List<List<Line>> orderByBand(List<Line> lines, List<Float> gutters) {
|
||||
List<Line> ordered = new ArrayList<>(lines);
|
||||
ordered.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
|
||||
List<List<Line>> out = new ArrayList<>();
|
||||
List<Line> band = new ArrayList<>();
|
||||
List<Line> spanning = new ArrayList<>();
|
||||
for (Line l : ordered) {
|
||||
if (spansGutter(l, gutters)) {
|
||||
if (spanning.isEmpty()) {
|
||||
out.addAll(splitIntoColumns(band, gutters));
|
||||
band = new ArrayList<>();
|
||||
} else if (!headingLength(l) || !headingLength(spanning.get(spanning.size() - 1))) {
|
||||
// Only heading-length lines are kept together: a full-width paragraph or list
|
||||
// is also a run of spanning lines, and merging those runs its items together.
|
||||
out.add(new ArrayList<>(spanning));
|
||||
spanning.clear();
|
||||
}
|
||||
spanning.add(l);
|
||||
} else {
|
||||
if (!spanning.isEmpty()) {
|
||||
out.add(new ArrayList<>(spanning));
|
||||
spanning.clear();
|
||||
}
|
||||
band.add(l);
|
||||
}
|
||||
}
|
||||
if (!spanning.isEmpty()) {
|
||||
out.add(new ArrayList<>(spanning));
|
||||
}
|
||||
out.addAll(splitIntoColumns(band, gutters));
|
||||
out.removeIf(List::isEmpty);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Visible for testing: as {@link ColumnRanges#fromTextLines(List)}, for gutter detection. */
|
||||
static List<Float> guttersFromTextLines(List<TextLine> rows) {
|
||||
return detectGutters(rows.stream().map(Line::new).collect(Collectors.toList()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Finds a table's column x-ranges by vertical-whitespace projection: a column is a contiguous
|
||||
* x-band occupied by enough rows, and the gaps between bands are the gutters.
|
||||
*/
|
||||
final class ColumnRanges {
|
||||
|
||||
private ColumnRanges() {}
|
||||
|
||||
/** Character widths of clear space that separate two columns of an unruled block. */
|
||||
static final float GUTTER_CHARS = 2.5f;
|
||||
|
||||
/** Absolute floor, in points, on an unruled block's column gutter. */
|
||||
static final float GUTTER_FLOOR = 10f;
|
||||
|
||||
/** As {@link #GUTTER_CHARS}, for a block the page's rules already declare to be a table. */
|
||||
static final float RULED_GUTTER_CHARS = 1.2f;
|
||||
|
||||
/** As {@link #GUTTER_FLOOR}, for a block the page's rules already declare to be a table. */
|
||||
static final float RULED_GUTTER_FLOOR = 4f;
|
||||
|
||||
static List<float[]> find(List<Line> rows) {
|
||||
return find(rows, GUTTER_CHARS, GUTTER_FLOOR);
|
||||
}
|
||||
|
||||
/** As {@link #find(List)}, with the gutter thresholds given explicitly. */
|
||||
static List<float[]> find(List<Line> rows, float gutterChars, float gutterFloor) {
|
||||
return find(rows, gutterChars, gutterFloor, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* As above, but {@code minSupport} overrides how many rows must occupy an x-band; zero keeps
|
||||
* the row-count-scaled default.
|
||||
*/
|
||||
static List<float[]> find(
|
||||
List<Line> rows, float gutterChars, float gutterFloor, int minSupport) {
|
||||
float minX = Float.MAX_VALUE;
|
||||
float maxX = -Float.MAX_VALUE;
|
||||
for (Line l : rows) {
|
||||
for (TextWord w : l.words()) {
|
||||
minX = Math.min(minX, w.x());
|
||||
maxX = Math.max(maxX, w.x() + w.width());
|
||||
}
|
||||
}
|
||||
// Real pages are under ~2000pt wide; anything larger is a malformed/crafted coordinate
|
||||
// that would allocate a multi-GB array or produce a negative span on overflow.
|
||||
if (maxX <= minX || (maxX - minX) > 2000f) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
int lo = (int) Math.floor(minX);
|
||||
int span = Math.min((int) Math.ceil(maxX) - lo + 1, 2001);
|
||||
int[] coverage = new int[span];
|
||||
for (Line l : rows) {
|
||||
boolean[] covered = new boolean[span];
|
||||
for (TextWord w : l.words()) {
|
||||
int a = Math.max(0, (int) Math.floor(w.x()) - lo);
|
||||
int b = Math.min(span, (int) Math.ceil(w.x() + w.width()) - lo);
|
||||
for (int x = a; x < b; x++) {
|
||||
covered[x] = true;
|
||||
}
|
||||
}
|
||||
for (int x = 0; x < span; x++) {
|
||||
if (covered[x]) {
|
||||
coverage[x]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A column band must be occupied by at least this many rows; below it is gutter.
|
||||
int support = minSupport > 0 ? minSupport : Math.max(2, Math.round(rows.size() * 0.35f));
|
||||
List<float[]> columns = new ArrayList<>();
|
||||
int start = -1;
|
||||
for (int x = 0; x < span; x++) {
|
||||
boolean isColumn = coverage[x] >= support;
|
||||
if (isColumn && start < 0) {
|
||||
start = x;
|
||||
} else if (!isColumn && start >= 0) {
|
||||
columns.add(new float[] {lo + start, lo + x});
|
||||
start = -1;
|
||||
}
|
||||
}
|
||||
if (start >= 0) {
|
||||
columns.add(new float[] {(float) (lo + start), (float) (lo + span)});
|
||||
}
|
||||
|
||||
// Merge bands closer than a real column separator: the gaps inside a multi-word cell are
|
||||
// about one character, and would otherwise split "January 20th, 2026" into three columns.
|
||||
float charWidth = WordGeometry.averageCharWidth(rows);
|
||||
float minGutter = Math.max(gutterFloor, charWidth * gutterChars);
|
||||
List<float[]> merged = new ArrayList<>();
|
||||
for (float[] band : columns) {
|
||||
if (!merged.isEmpty() && band[0] - merged.getLast()[1] < minGutter) {
|
||||
merged.getLast()[1] = band[1];
|
||||
} else {
|
||||
merged.add(new float[] {band[0], band[1]});
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Visible for testing: column detection depends only on word geometry, so tests can exercise
|
||||
* degenerate coordinates without a binary fixture.
|
||||
*/
|
||||
static List<float[]> fromTextLines(List<TextLine> rows) {
|
||||
return find(rows.stream().map(Line::new).collect(Collectors.toList()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.PdfPage;
|
||||
import stirling.software.jpdfium.doc.FormField;
|
||||
import stirling.software.jpdfium.doc.FormFieldType;
|
||||
import stirling.software.jpdfium.doc.PdfFormReader;
|
||||
import stirling.software.jpdfium.model.Rect;
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Recovers AcroForm values that live only in a field's {@code /V}, as pseudo lines at their widget
|
||||
* rectangles so they land in reading order.
|
||||
*/
|
||||
final class FormValues {
|
||||
|
||||
private FormValues() {}
|
||||
|
||||
/**
|
||||
* Pseudo text lines for {@code /V}-only AcroForm values, placed at their widget rectangles;
|
||||
* values already in the content stream are skipped.
|
||||
*/
|
||||
static List<Line> lines(PdfDocument doc, int pageIndex, List<Line> existing) {
|
||||
List<FormField> fields;
|
||||
try (PdfPage page = doc.page(pageIndex)) {
|
||||
fields = PdfFormReader.readPage(page.rawDocHandle(), page.rawHandle(), pageIndex);
|
||||
} catch (RuntimeException e) {
|
||||
// A malformed AcroForm must not sink the whole conversion; body text still stands.
|
||||
return List.of();
|
||||
}
|
||||
List<Line> out = new ArrayList<>();
|
||||
for (FormField f : fields) {
|
||||
String value = fieldText(f);
|
||||
if (value == null || value.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
Rect r = f.rect();
|
||||
if (r == null || r.width() <= 0 || r.height() <= 0) {
|
||||
continue;
|
||||
}
|
||||
if (alreadyInContent(existing, value, r)) {
|
||||
continue;
|
||||
}
|
||||
out.add(syntheticLine(value, r));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The text a filled field contributes, or null when the field contributes nothing. */
|
||||
private static String fieldText(FormField f) {
|
||||
FormFieldType type = f.type();
|
||||
if (type == FormFieldType.PUSHBUTTON
|
||||
|| type == FormFieldType.SIGNATURE
|
||||
|| type == FormFieldType.UNKNOWN) {
|
||||
return null;
|
||||
}
|
||||
if (type == FormFieldType.CHECKBOX || type == FormFieldType.RADIO) {
|
||||
return f.checked() ? "[x]" : null;
|
||||
}
|
||||
String value = f.value();
|
||||
if (value == null || "Off".equals(value)) {
|
||||
return null;
|
||||
}
|
||||
return value.replace('\r', ' ').replace('\n', ' ').strip();
|
||||
}
|
||||
|
||||
/** True when the extractor already found this value inside the widget's own rectangle. */
|
||||
private static boolean alreadyInContent(List<Line> lines, String value, Rect r) {
|
||||
String needle = MarkdownText.normaliseSpace(value);
|
||||
for (Line l : lines) {
|
||||
boolean overlaps =
|
||||
l.x < r.x() + r.width()
|
||||
&& l.x + l.width > r.x()
|
||||
&& l.y < r.y() + r.height()
|
||||
&& l.y + l.height > r.y();
|
||||
if (overlaps && MarkdownText.normaliseSpace(l.text).contains(needle)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a field value as a one-word-per-token {@link TextLine} at the widget rectangle, so
|
||||
* downstream stages treat it like any other text.
|
||||
*/
|
||||
private static Line syntheticLine(String value, Rect r) {
|
||||
String[] tokens = value.split("\\s+");
|
||||
float height = Math.min(r.height(), 14f);
|
||||
float advance = tokens.length == 0 ? r.width() : r.width() / tokens.length;
|
||||
List<TextWord> words = new ArrayList<>(tokens.length);
|
||||
for (int i = 0; i < tokens.length; i++) {
|
||||
float wx = r.x() + advance * i;
|
||||
List<TextChar> chars = new ArrayList<>(tokens[i].length());
|
||||
float charWidth = tokens[i].isEmpty() ? advance : advance / tokens[i].length();
|
||||
for (int c = 0; c < tokens[i].length(); c++) {
|
||||
chars.add(
|
||||
new TextChar(
|
||||
c,
|
||||
tokens[i].charAt(c),
|
||||
wx + charWidth * c,
|
||||
r.y(),
|
||||
charWidth,
|
||||
height,
|
||||
"",
|
||||
0f));
|
||||
}
|
||||
words.add(new TextWord(chars, wx, r.y(), advance * 0.95f, height));
|
||||
}
|
||||
TextLine line = new TextLine(words, r.x(), r.y(), r.width(), height);
|
||||
Line out = new Line(line, value);
|
||||
out.synthetic = true;
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** Renders a resolved cell grid as a GitHub-Flavored Markdown table. */
|
||||
final class GfmTable {
|
||||
|
||||
private GfmTable() {}
|
||||
|
||||
static String render(List<String[]> rows, int cols) {
|
||||
if (rows.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
int[] widths = new int[cols];
|
||||
for (int c = 0; c < cols; c++) {
|
||||
widths[c] = 3;
|
||||
}
|
||||
for (String[] row : rows) {
|
||||
for (int c = 0; c < cols; c++) {
|
||||
if (c < row.length) {
|
||||
widths[c] = Math.max(widths[c], escapeCell(row[c]).length());
|
||||
}
|
||||
}
|
||||
}
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(buildGfmRow(rows.getFirst(), widths, cols)).append('\n');
|
||||
sb.append('|');
|
||||
for (int c = 0; c < cols; c++) {
|
||||
sb.append('-').append("-".repeat(widths[c])).append('-').append('|');
|
||||
}
|
||||
for (int r = 1; r < rows.size(); r++) {
|
||||
sb.append('\n').append(buildGfmRow(rows.get(r), widths, cols));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static String buildGfmRow(String[] row, int[] widths, int cols) {
|
||||
StringBuilder sb = new StringBuilder().append('|');
|
||||
for (int c = 0; c < cols; c++) {
|
||||
String cell = c < row.length ? escapeCell(row[c]) : "";
|
||||
sb.append(' ').append(padRight(cell, widths[c])).append(' ').append('|');
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static String escapeCell(String cell) {
|
||||
// Cell content is inline context: escape inline markdown (including the column delimiter)
|
||||
// but not leading block markers, which have no meaning inside a table cell.
|
||||
return MarkdownText.escapeMarkdownInline(cell);
|
||||
}
|
||||
|
||||
private static String padRight(String s, int width) {
|
||||
return s.length() >= width ? s : s + " ".repeat(width - s.length());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
|
||||
/**
|
||||
* Rebuilds assembled {@link Line}s from the extractor's {@link TextLine}s, folding in the narrow
|
||||
* glyph fragments PDFium emits for apostrophes, markers and bullets.
|
||||
*/
|
||||
final class GlyphStitcher {
|
||||
|
||||
private GlyphStitcher() {}
|
||||
|
||||
/** Width below which a TextLine is treated as a stray glyph fragment to be stitched. */
|
||||
private static final float GLYPH_WIDTH = 7.5f;
|
||||
|
||||
/**
|
||||
* Merges narrow glyph fragments into the line they belong to: inline between two same-baseline
|
||||
* fragments, or appended/prepended at a line's edge.
|
||||
*/
|
||||
static List<Line> stitchGlyphs(List<TextLine> raw) {
|
||||
List<TextLine> hosts = new ArrayList<>();
|
||||
List<TextLine> glyphs = new ArrayList<>();
|
||||
for (TextLine l : raw) {
|
||||
String t = stripSoftHyphens(l.text()).strip();
|
||||
if (t.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (l.width() < GLYPH_WIDTH && t.length() <= 2) {
|
||||
glyphs.add(l);
|
||||
} else {
|
||||
hosts.add(l);
|
||||
}
|
||||
}
|
||||
|
||||
List<Line> lines =
|
||||
hosts.stream()
|
||||
.map(l -> new Line(l, stripSoftHyphens(l.text())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
for (TextLine g : glyphs) {
|
||||
String gt = stripSoftHyphens(g.text()).strip();
|
||||
if (isBulletGlyph(gt)) {
|
||||
attachBullet(g, gt, lines);
|
||||
} else {
|
||||
attachInlineGlyph(g, gt, lines);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes U+00AD SOFT HYPHEN, a break-opportunity marker PDFium hands back verbatim as {@code
|
||||
* ar<AD>e}.
|
||||
*/
|
||||
private static String stripSoftHyphens(String text) {
|
||||
if (text.indexOf('') < 0) {
|
||||
return text;
|
||||
}
|
||||
return text.replace("", "");
|
||||
}
|
||||
|
||||
private static boolean isBulletGlyph(String gt) {
|
||||
return "•".equals(gt) || "▪".equals(gt) || "◦".equals(gt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Attaches a bullet glyph to the line it introduces: the closest line beginning to its right,
|
||||
* at roughly the same height or just below.
|
||||
*/
|
||||
private static void attachBullet(TextLine g, String gt, List<Line> lines) {
|
||||
Line best = null;
|
||||
float bestScore = Float.MAX_VALUE;
|
||||
for (Line h : lines) {
|
||||
if (h.x < g.x() - 2f) {
|
||||
continue;
|
||||
}
|
||||
float dy = g.y() - h.y;
|
||||
if (dy < -4f || dy > 28f) {
|
||||
continue;
|
||||
}
|
||||
float score = Math.abs(dy) + (h.x - g.x()) * 0.2f;
|
||||
if (score < bestScore) {
|
||||
bestScore = score;
|
||||
best = h;
|
||||
}
|
||||
}
|
||||
if (best != null && !best.text.startsWith("•")) {
|
||||
best.text = "• " + best.text;
|
||||
best.x = g.x();
|
||||
} else {
|
||||
lines.add(new Line(g, gt));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stitches a narrow inline glyph into its line: between two same-baseline fragments, appended
|
||||
* to the line ending at it, or prepended to the one starting at it.
|
||||
*/
|
||||
private static void attachInlineGlyph(TextLine g, String gt, List<Line> lines) {
|
||||
Line left = null;
|
||||
Line right = null;
|
||||
float lb = 7f;
|
||||
float rb = 7f;
|
||||
for (Line h : lines) {
|
||||
boolean sameBaseline = g.y() >= h.y - 4f && g.y() <= h.y + h.height + 5f;
|
||||
if (!sameBaseline) {
|
||||
continue;
|
||||
}
|
||||
float rightEdge = h.x + h.width;
|
||||
float dxLeft = Math.abs(rightEdge - g.x());
|
||||
if (dxLeft < lb) {
|
||||
lb = dxLeft;
|
||||
left = h;
|
||||
}
|
||||
float dxRight = Math.abs(h.x - g.x());
|
||||
if (dxRight < rb) {
|
||||
rb = dxRight;
|
||||
right = h;
|
||||
}
|
||||
}
|
||||
|
||||
if (left != null && right != null && left != right && Math.abs(left.y - right.y) < 6f) {
|
||||
left.text = left.text + gt + right.text;
|
||||
left.width = (right.x + right.width) - left.x;
|
||||
left.absorb(g);
|
||||
left.absorb(right);
|
||||
lines.remove(right);
|
||||
} else if (left != null) {
|
||||
left.text = left.text + gt;
|
||||
left.width = Math.max(left.width, g.x() + g.width() - left.x);
|
||||
left.absorb(g);
|
||||
} else if (right != null) {
|
||||
right.text = gt + right.text;
|
||||
right.x = g.x();
|
||||
right.absorb(g);
|
||||
} else {
|
||||
lines.add(new Line(g, gt));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,485 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import stirling.software.jpdfium.text.PageText;
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
final class HeadingDetector {
|
||||
|
||||
private HeadingDetector() {}
|
||||
|
||||
/** A heading is at most this many words; longer lines are treated as body text. */
|
||||
private static final int MAX_HEADING_WORDS = 12;
|
||||
|
||||
/** A heading in a script written without word spaces is at most this many characters. */
|
||||
private static final int MAX_HEADING_UNSPACED_CHARS = 30;
|
||||
|
||||
/** Six-letter subset tag PDF writers prepend to embedded font names. */
|
||||
private static final Pattern SUBSET_TAG = Pattern.compile("^[A-Z]{6}\\+");
|
||||
|
||||
/** PostScript name fragments denoting a weight heavier than the regular face. */
|
||||
private static final String[] BOLD_TOKENS = {
|
||||
"bold", "black", "heavy", "semibold", "demi", "ultra", "extrabold"
|
||||
};
|
||||
|
||||
/**
|
||||
* URW/Nimbus bold is "-Medi", TeX bold extended is CMBX/CMSSBX. "Medium" is excluded: matching
|
||||
* it makes whole CJK paragraphs bold.
|
||||
*/
|
||||
private static final Pattern OTHER_BOLD = Pattern.compile("medi(?!um)|cm(ss)?bx");
|
||||
|
||||
/**
|
||||
* A float label followed by its number: set like a heading but naming an illustration, not a
|
||||
* section.
|
||||
*/
|
||||
private static final Pattern CAPTION =
|
||||
Pattern.compile(
|
||||
"^(table|figure|fig|chart|exhibit|plate|scheme|graph|diagram|illustration)"
|
||||
+ "\\s*\\.?\\s*\\d",
|
||||
Pattern.CASE_INSENSITIVE);
|
||||
|
||||
/** A numbered section clause: {@code 3.}, {@code 6.2.}, {@code 7.2.1} followed by a name. */
|
||||
private static final Pattern CLAUSE = Pattern.compile("^\\d{1,2}(\\.\\d{1,2})*\\.?\\s+\\p{Lu}");
|
||||
|
||||
/**
|
||||
* Two sentences in a row: a bold run-in lead-in, not a heading. The lower-case letters before
|
||||
* the stop keep it off section numbers.
|
||||
*/
|
||||
private static final Pattern RUNS_ON = Pattern.compile("\\p{Ll}{3}[.!?][\\s\\u00a0]+\\p{Lu}");
|
||||
|
||||
/** Size ratio at which a line is a level-1 heading on size alone. */
|
||||
private static final float H1_RATIO = 1.4f;
|
||||
|
||||
/** Size ratio at which a line is a level-2 heading on size alone. */
|
||||
private static final float H2_RATIO = 1.3f;
|
||||
|
||||
/**
|
||||
* A section number ending in a period. Stricter than {@link #CLAUSE}: only the period tells a
|
||||
* clause from a header's page number.
|
||||
*/
|
||||
private static final Pattern NUMBERED_CLAUSE =
|
||||
Pattern.compile("^\\d{1,2}(\\.\\d{1,2})*\\.\\s+\\p{Lu}");
|
||||
|
||||
/**
|
||||
* True when every cased letter is a capital; digits and uncased scripts do not count, so {@code
|
||||
* BIO 181} qualifies.
|
||||
*/
|
||||
private static boolean isAllCaps(String text) {
|
||||
int upper = 0;
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
char c = text.charAt(i);
|
||||
if (Character.isLowerCase(c)) {
|
||||
return false;
|
||||
}
|
||||
if (Character.isUpperCase(c)) {
|
||||
upper++;
|
||||
}
|
||||
}
|
||||
return upper >= 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* Markdown heading prefix from size, brevity, isolation and weight, never text matching. Bold
|
||||
* is vetoed on the body face itself.
|
||||
*/
|
||||
static String headingPrefix(
|
||||
TextLine line,
|
||||
float medianBodySize,
|
||||
float medianBodyHeight,
|
||||
String bodyFont,
|
||||
boolean isolated) {
|
||||
return headingPrefix(
|
||||
line.text(),
|
||||
line.height(),
|
||||
line.words(),
|
||||
medianBodySize,
|
||||
medianBodyHeight,
|
||||
bodyFont,
|
||||
isolated);
|
||||
}
|
||||
|
||||
/**
|
||||
* Geometry-only overload, judging a merged line on its merged text, height and words. Boldness
|
||||
* counts only when the face differs from {@code bodyFont}.
|
||||
*/
|
||||
static String headingPrefix(
|
||||
String lineText,
|
||||
float lineHeight,
|
||||
List<TextWord> words,
|
||||
float medianBodySize,
|
||||
float medianBodyHeight,
|
||||
String bodyFont,
|
||||
boolean isolated) {
|
||||
String text = lineText.strip();
|
||||
if (text.isEmpty() || tooLongForHeading(text)) {
|
||||
return "";
|
||||
}
|
||||
float ratio = sizeRatio(lineHeight, words, medianBodySize, medianBodyHeight);
|
||||
if (ratio < 0f) {
|
||||
return "";
|
||||
}
|
||||
// A heading names something. A line with no word in it is a value, an equation fragment or
|
||||
// a chart label, however large it is set; a caption names a float, not a section.
|
||||
if (CAPTION.matcher(text).find() || !hasWord(text)) {
|
||||
return "";
|
||||
}
|
||||
if (RUNS_ON.matcher(text).find()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
// Bold only marks a heading when it stands out from the body face. Some documents set
|
||||
// the whole body in a bold-named font, where boldness carries no structural meaning.
|
||||
boolean bold = isBold(words) && !normalisedFont(words).equals(normalise(bodyFont));
|
||||
// A line that reads as a sentence is prose, unless it carries heading typography.
|
||||
if (endsLikeSentence(text) && !bold && ratio <= H2_RATIO) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (ratio > H1_RATIO) {
|
||||
return "# ";
|
||||
}
|
||||
if (ratio > H2_RATIO) {
|
||||
return "## ";
|
||||
}
|
||||
// A numbered clause: the section number is the structure, so it needs no blank line above
|
||||
// it to be one. Requiring a capital after the number keeps ordinary list items out.
|
||||
if (bold && CLAUSE.matcher(text).find()) {
|
||||
return "### ";
|
||||
}
|
||||
// Same size as the body but bold and starting its own block: a run-in section heading.
|
||||
if (bold && isolated && hasWord(text)) {
|
||||
return "### ";
|
||||
}
|
||||
// Some documents give a heading no size and no weight, only capitals. A short, isolated
|
||||
// line set entirely in capitals is one of those.
|
||||
if (isolated
|
||||
&& !endsLikeSentence(text)
|
||||
&& wordCount(text) >= 3
|
||||
&& isAllCaps(text)
|
||||
&& NUMBERED_CLAUSE.matcher(text).find()) {
|
||||
return "### ";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private static float sizeRatio(
|
||||
float lineHeight, List<TextWord> words, float medianBodySize, float medianBodyHeight) {
|
||||
float dominant = dominantFontSize(words);
|
||||
float value;
|
||||
float baseline;
|
||||
if (dominant > 2f && medianBodySize > 2f) {
|
||||
value = dominant;
|
||||
baseline = medianBodySize;
|
||||
} else {
|
||||
float glyph = glyphHeight(words);
|
||||
value = glyph > 0f ? glyph : lineHeight;
|
||||
baseline = medianBodyHeight;
|
||||
}
|
||||
return baseline <= 0f ? -1f : value / baseline;
|
||||
}
|
||||
|
||||
/**
|
||||
* Quantile of a line's glyph heights taken as its size: high enough for the cap band, low
|
||||
* enough that one rogue glyph box cannot set it.
|
||||
*/
|
||||
private static final float GLYPH_HEIGHT_QUANTILE = 0.8f;
|
||||
|
||||
/**
|
||||
* A line's type size from its glyphs, for PDFs encoding visual size in the text matrix; the
|
||||
* line box runs ascender to descender.
|
||||
*/
|
||||
private static float glyphHeight(List<TextWord> words) {
|
||||
int capacity = 0;
|
||||
for (TextWord word : words) {
|
||||
capacity += word.chars().size();
|
||||
}
|
||||
if (capacity == 0) {
|
||||
return 0f;
|
||||
}
|
||||
float[] heights = new float[capacity];
|
||||
int n = 0;
|
||||
for (TextWord word : words) {
|
||||
for (TextChar ch : word.chars()) {
|
||||
if (ch.isWhitespace() || ch.isNewline()) {
|
||||
continue;
|
||||
}
|
||||
// Letters and digits only: brackets and maths operators are drawn taller than the
|
||||
// cap height, so an equation would measure as display type.
|
||||
if (!Character.isLetterOrDigit(ch.toChar())) {
|
||||
continue;
|
||||
}
|
||||
float h = ch.height();
|
||||
if (Float.isFinite(h) && h > 0f) {
|
||||
heights[n++] = h;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (n == 0) {
|
||||
return 0f;
|
||||
}
|
||||
Arrays.sort(heights, 0, n);
|
||||
return heights[(int) (GLYPH_HEIGHT_QUANTILE * (n - 1))];
|
||||
}
|
||||
|
||||
private static String normalise(String fontName) {
|
||||
return SUBSET_TAG.matcher(fontName == null ? "" : fontName).replaceFirst("");
|
||||
}
|
||||
|
||||
private static String normalisedFont(List<TextWord> words) {
|
||||
return normalise(dominantFontName(words));
|
||||
}
|
||||
|
||||
/** Most-used font across the document, weighted by glyph count: the body face. */
|
||||
static String bodyFont(List<PageText> allPages) {
|
||||
Map<String, Integer> counts = new HashMap<>();
|
||||
for (PageText page : allPages) {
|
||||
for (TextChar ch : page.chars()) {
|
||||
if (ch.isWhitespace() || ch.isNewline()) {
|
||||
continue;
|
||||
}
|
||||
String name = ch.fontName();
|
||||
if (name != null && !name.isBlank()) {
|
||||
counts.merge(normalise(name), 1, Integer::sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
String dominant = "";
|
||||
int max = -1;
|
||||
for (Map.Entry<String, Integer> e : counts.entrySet()) {
|
||||
if (e.getValue() > max) {
|
||||
max = e.getValue();
|
||||
dominant = e.getKey();
|
||||
}
|
||||
}
|
||||
return dominant;
|
||||
}
|
||||
|
||||
/** A heading is made of words: at least two letters, and letters at least half the glyphs. */
|
||||
private static boolean hasWord(String text) {
|
||||
int letters = 0;
|
||||
int glyphs = 0;
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
char c = text.charAt(i);
|
||||
if (Character.isWhitespace(c)) {
|
||||
continue;
|
||||
}
|
||||
glyphs++;
|
||||
if (Character.isLetter(c)) {
|
||||
letters++;
|
||||
}
|
||||
}
|
||||
return letters >= 2 && letters * 2 >= glyphs;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a line should be emphasised as bold rather than promoted: bold, short, and not a
|
||||
* full sentence.
|
||||
*/
|
||||
static boolean isBoldLabel(String lineText, List<TextWord> words) {
|
||||
String text = lineText.strip();
|
||||
if (text.isEmpty() || tooLongForHeading(text) || endsLikeSentence(text)) {
|
||||
return false;
|
||||
}
|
||||
if (RUNS_ON.matcher(text).find()) {
|
||||
return false;
|
||||
}
|
||||
return hasWord(text) && isBold(words);
|
||||
}
|
||||
|
||||
private static int wordCount(String text) {
|
||||
return text.split("\\s+").length;
|
||||
}
|
||||
|
||||
/** Whitespace tokens do not measure a line written in a script with no word spaces. */
|
||||
private static boolean tooLongForHeading(String text) {
|
||||
return wordCount(text) > MAX_HEADING_WORDS
|
||||
|| unspacedScriptChars(text) > MAX_HEADING_UNSPACED_CHARS;
|
||||
}
|
||||
|
||||
private static int unspacedScriptChars(String text) {
|
||||
int n = 0;
|
||||
for (int i = 0; i < text.length(); ) {
|
||||
int cp = text.codePointAt(i);
|
||||
if (isUnspacedScript(cp)) {
|
||||
n++;
|
||||
}
|
||||
i += Character.charCount(cp);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
private static boolean isUnspacedScript(int cp) {
|
||||
Character.UnicodeScript s = Character.UnicodeScript.of(cp);
|
||||
return s == Character.UnicodeScript.HAN
|
||||
|| s == Character.UnicodeScript.HIRAGANA
|
||||
|| s == Character.UnicodeScript.KATAKANA
|
||||
|| s == Character.UnicodeScript.THAI
|
||||
|| s == Character.UnicodeScript.LAO
|
||||
|| s == Character.UnicodeScript.KHMER
|
||||
|| s == Character.UnicodeScript.MYANMAR;
|
||||
}
|
||||
|
||||
private static boolean endsLikeSentence(String text) {
|
||||
char last = text.charAt(text.length() - 1);
|
||||
// Ideographic and full-width stops end a sentence exactly as the ASCII ones do.
|
||||
return last == '.'
|
||||
|| last == '!'
|
||||
|| last == '?'
|
||||
|| last == '\u3002'
|
||||
|| last == '\uff01'
|
||||
|| last == '\uff1f'
|
||||
|| last == '\uff61';
|
||||
}
|
||||
|
||||
/** True when the line's dominant font is bold, inferred from PostScript font names. */
|
||||
private static boolean isBold(List<TextWord> words) {
|
||||
String lower = normalisedFont(words).toLowerCase(Locale.ROOT);
|
||||
for (String token : BOLD_TOKENS) {
|
||||
if (lower.contains(token)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return OTHER_BOLD.matcher(lower).find();
|
||||
}
|
||||
|
||||
private static String dominantFontName(List<TextWord> words) {
|
||||
Map<String, Integer> counts = new HashMap<>();
|
||||
for (TextWord word : words) {
|
||||
for (TextChar ch : word.chars()) {
|
||||
if (ch.isWhitespace() || ch.isNewline()) {
|
||||
continue;
|
||||
}
|
||||
String name = ch.fontName();
|
||||
if (name != null && !name.isBlank()) {
|
||||
counts.merge(name, 1, Integer::sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
String dominantFont = "";
|
||||
int max = -1;
|
||||
for (Map.Entry<String, Integer> e : counts.entrySet()) {
|
||||
if (e.getValue() > max) {
|
||||
max = e.getValue();
|
||||
dominantFont = e.getKey();
|
||||
}
|
||||
}
|
||||
return dominantFont;
|
||||
}
|
||||
|
||||
/** Computes the median glyph font size across all pages. */
|
||||
static float medianFontSize(List<PageText> allPages) {
|
||||
List<Float> sizes = new ArrayList<>();
|
||||
for (PageText page : allPages) {
|
||||
for (TextChar ch : page.chars()) {
|
||||
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
|
||||
sizes.add(ch.fontSize());
|
||||
}
|
||||
}
|
||||
}
|
||||
return median(sizes, 12f);
|
||||
}
|
||||
|
||||
/**
|
||||
* Body baseline for the height path: median per-line {@link #glyphHeight} weighted by glyph
|
||||
* count, so tiny axis labels cannot drag it down.
|
||||
*/
|
||||
static float medianLineHeight(List<PageText> allPages) {
|
||||
List<float[]> weighted = new ArrayList<>();
|
||||
double total = 0;
|
||||
for (PageText page : allPages) {
|
||||
for (TextLine line : page.lines()) {
|
||||
if (line.text().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
float h = glyphHeight(line.words());
|
||||
if (h <= 0f) {
|
||||
h = line.height();
|
||||
}
|
||||
if (h <= 0f) {
|
||||
continue;
|
||||
}
|
||||
float w = glyphCount(line);
|
||||
weighted.add(new float[] {h, w});
|
||||
total += w;
|
||||
}
|
||||
}
|
||||
if (weighted.isEmpty()) {
|
||||
return 12f;
|
||||
}
|
||||
weighted.sort(Comparator.comparingDouble(p -> p[0]));
|
||||
double half = total / 2d;
|
||||
double seen = 0;
|
||||
for (float[] p : weighted) {
|
||||
seen += p[1];
|
||||
if (seen >= half) {
|
||||
return p[0];
|
||||
}
|
||||
}
|
||||
return weighted.get(weighted.size() - 1)[0];
|
||||
}
|
||||
|
||||
/** How much text a line carries, in glyphs; at least one so an empty line still counts. */
|
||||
private static float glyphCount(TextLine line) {
|
||||
int glyphs = 0;
|
||||
for (TextWord word : line.words()) {
|
||||
for (TextChar ch : word.chars()) {
|
||||
if (!ch.isWhitespace() && !ch.isNewline()) {
|
||||
glyphs++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return Math.max(1, glyphs);
|
||||
}
|
||||
|
||||
private static float median(List<Float> values, float fallback) {
|
||||
if (values.isEmpty()) {
|
||||
return fallback;
|
||||
}
|
||||
Collections.sort(values);
|
||||
int mid = values.size() / 2;
|
||||
if (values.size() % 2 == 0) {
|
||||
return (values.get(mid - 1) + values.get(mid)) / 2f;
|
||||
}
|
||||
return values.get(mid);
|
||||
}
|
||||
|
||||
/**
|
||||
* The font size appearing most often by character count in the line; ties go to the larger
|
||||
* size.
|
||||
*/
|
||||
private static float dominantFontSize(List<TextWord> words) {
|
||||
Map<Float, Integer> counts = new HashMap<>();
|
||||
for (TextWord word : words) {
|
||||
for (TextChar ch : word.chars()) {
|
||||
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
|
||||
counts.merge(ch.fontSize(), 1, Integer::sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (counts.isEmpty()) {
|
||||
return 0f;
|
||||
}
|
||||
float dominant = 0f;
|
||||
int maxCount = -1;
|
||||
for (Map.Entry<Float, Integer> entry : counts.entrySet()) {
|
||||
int count = entry.getValue();
|
||||
float size = entry.getKey();
|
||||
if (count > maxCount || (count == maxCount && size > dominant)) {
|
||||
maxCount = count;
|
||||
dominant = size;
|
||||
}
|
||||
}
|
||||
return dominant;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* A mutable assembled line: text plus geometry. {@link #left()}/{@link #right()} come from the word
|
||||
* boxes, {@link #glyphLeft()}/{@link #glyphRight()} from the glyphs.
|
||||
*/
|
||||
final class Line {
|
||||
|
||||
String text;
|
||||
float x;
|
||||
float y;
|
||||
float width;
|
||||
float height;
|
||||
final TextLine source;
|
||||
|
||||
/** Extra extractor fragments merged into this line; empty for an unmerged line. */
|
||||
final List<TextLine> merged = new ArrayList<>();
|
||||
|
||||
/** True for a line synthesised from an AcroForm value rather than page content. */
|
||||
boolean synthetic;
|
||||
|
||||
Line(TextLine src) {
|
||||
this(src, src.text());
|
||||
}
|
||||
|
||||
Line(TextLine src, String text) {
|
||||
this.source = src;
|
||||
this.text = text;
|
||||
this.x = src.x();
|
||||
this.y = src.y();
|
||||
this.width = src.width();
|
||||
this.height = src.height();
|
||||
}
|
||||
|
||||
/** Every word on the line, in x order, across all merged fragments. */
|
||||
List<TextWord> words() {
|
||||
if (merged.isEmpty()) {
|
||||
return source.words();
|
||||
}
|
||||
List<TextWord> all = new ArrayList<>(source.words());
|
||||
for (TextLine extra : merged) {
|
||||
all.addAll(extra.words());
|
||||
}
|
||||
all.sort(Comparator.comparingDouble(TextWord::x));
|
||||
return all;
|
||||
}
|
||||
|
||||
/** Text for the heading/bold classifiers; an unmerged line keeps the extractor's own string. */
|
||||
String detectText() {
|
||||
return merged.isEmpty() ? source.text() : text;
|
||||
}
|
||||
|
||||
float detectHeight() {
|
||||
return merged.isEmpty() ? source.height() : height;
|
||||
}
|
||||
|
||||
/** Top edge; PDF y grows upwards, so this is the larger of the two vertical bounds. */
|
||||
float top() {
|
||||
return y + height;
|
||||
}
|
||||
|
||||
float centreY() {
|
||||
return y + height / 2f;
|
||||
}
|
||||
|
||||
float centreX() {
|
||||
return x + width / 2f;
|
||||
}
|
||||
|
||||
/** Left edge of the line's words, falling back to its bounding box when it has none. */
|
||||
float left() {
|
||||
float edge = Float.MAX_VALUE;
|
||||
for (TextWord w : words()) {
|
||||
edge = Math.min(edge, w.x());
|
||||
}
|
||||
return edge == Float.MAX_VALUE ? x : edge;
|
||||
}
|
||||
|
||||
float right() {
|
||||
float edge = -Float.MAX_VALUE;
|
||||
for (TextWord w : words()) {
|
||||
edge = Math.max(edge, w.x() + w.width());
|
||||
}
|
||||
return edge == -Float.MAX_VALUE ? x + width : edge;
|
||||
}
|
||||
|
||||
/** Left edge of the line's glyphs, ignoring any space a word box carries. */
|
||||
float glyphLeft() {
|
||||
float edge = Float.MAX_VALUE;
|
||||
for (TextWord w : words()) {
|
||||
for (TextChar c : w.chars()) {
|
||||
if (!c.isWhitespace() && !c.isNewline()) {
|
||||
edge = Math.min(edge, c.x());
|
||||
}
|
||||
}
|
||||
}
|
||||
return edge == Float.MAX_VALUE ? x : edge;
|
||||
}
|
||||
|
||||
float glyphRight() {
|
||||
float edge = -Float.MAX_VALUE;
|
||||
for (TextWord w : words()) {
|
||||
for (TextChar c : w.chars()) {
|
||||
if (!c.isWhitespace() && !c.isNewline()) {
|
||||
edge = Math.max(edge, c.x() + c.width());
|
||||
}
|
||||
}
|
||||
}
|
||||
return edge == -Float.MAX_VALUE ? x + width : edge;
|
||||
}
|
||||
|
||||
/** Records a fragment folded into this line so its word list still covers the whole extent. */
|
||||
void absorb(TextLine fragment) {
|
||||
merged.add(fragment);
|
||||
}
|
||||
|
||||
void absorb(Line fragment) {
|
||||
merged.add(fragment.source);
|
||||
merged.addAll(fragment.merged);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Rejoins extractor fragments that are really one visual line. Fragments are grouped into rows
|
||||
* first, then joined left to right.
|
||||
*/
|
||||
final class LineMerger {
|
||||
|
||||
private LineMerger() {}
|
||||
|
||||
/** Gap above this many average character widths is a real layout gap, so never merged. */
|
||||
private static final float MAX_MERGE_GAP = 1.60f;
|
||||
|
||||
/**
|
||||
* Rejoins fragments of one visual line: PDFium splits on bounding box, so a run with no
|
||||
* ascender ({@code rou}) lands apart from the rest.
|
||||
*/
|
||||
static List<Line> mergeLineFragments(List<Line> lines, List<Float> gutters) {
|
||||
if (lines.size() < 2) {
|
||||
return lines;
|
||||
}
|
||||
// Merge within each column, so a line ending at the gutter never joins the next column's.
|
||||
if (!gutters.isEmpty()) {
|
||||
List<List<Line>> columns = ColumnLayout.splitIntoColumns(lines, gutters);
|
||||
if (columns.size() > 1) {
|
||||
List<Line> out = new ArrayList<>(lines.size());
|
||||
for (List<Line> column : columns) {
|
||||
out.addAll(mergeRows(column));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
return mergeRows(lines);
|
||||
}
|
||||
|
||||
private static List<Line> mergeRows(List<Line> lines) {
|
||||
if (lines.size() < 2) {
|
||||
return new ArrayList<>(lines);
|
||||
}
|
||||
List<Line> ordered = new ArrayList<>(lines);
|
||||
// Top edge first, so fragments of one visual line arrive together whatever their heights.
|
||||
ordered.sort(Comparator.comparingDouble((Line l) -> -(l.y + l.height)));
|
||||
|
||||
// Group into rows first: a fragment's continuation is its right-hand neighbour on the same
|
||||
// row, not whichever line the extractor happened to emit next.
|
||||
List<List<Line>> rows = new ArrayList<>();
|
||||
for (Line line : ordered) {
|
||||
List<Line> row = null;
|
||||
for (int i = rows.size() - 1; i >= 0 && i >= rows.size() - 3; i--) {
|
||||
if (overlapsRow(rows.get(i), line)) {
|
||||
row = rows.get(i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (row == null) {
|
||||
row = new ArrayList<>();
|
||||
rows.add(row);
|
||||
}
|
||||
row.add(line);
|
||||
}
|
||||
|
||||
List<Line> out = new ArrayList<>();
|
||||
for (List<Line> row : rows) {
|
||||
row.sort(Comparator.comparingDouble((Line l) -> l.x));
|
||||
Line host = null;
|
||||
for (Line line : row) {
|
||||
if (host != null && adjacentOnRow(host, line)) {
|
||||
appendFragment(host, line);
|
||||
} else {
|
||||
out.add(line);
|
||||
host = line;
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** True when a line shares a row with the lines already in it (vertical overlap). */
|
||||
private static boolean overlapsRow(List<Line> row, Line line) {
|
||||
for (Line member : row) {
|
||||
float overlap =
|
||||
Math.min(member.y + member.height, line.y + line.height)
|
||||
- Math.max(member.y, line.y);
|
||||
float minHeight = Math.min(member.height, line.height);
|
||||
if (minHeight > 0f && overlap >= minHeight * 0.5f) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** True when {@code next} sits close enough to {@code host} to be the same visual line. */
|
||||
private static boolean adjacentOnRow(Line host, Line next) {
|
||||
// Merging concatenates left to right, which is reading order for LTR only; RTL fragments
|
||||
// would be joined back to front.
|
||||
if (hasStrongRtl(host.text) || hasStrongRtl(next.text)) {
|
||||
return false;
|
||||
}
|
||||
float gap = next.glyphLeft() - host.glyphRight();
|
||||
float charWidth = fragmentCharWidth(host, next);
|
||||
return gap > -charWidth && gap < charWidth * MAX_MERGE_GAP;
|
||||
}
|
||||
|
||||
/** True when the text contains a Hebrew, Arabic, Syriac or Thaana character. */
|
||||
private static boolean hasStrongRtl(String text) {
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
byte dir = Character.getDirectionality(text.charAt(i));
|
||||
if (dir == Character.DIRECTIONALITY_RIGHT_TO_LEFT
|
||||
|| dir == Character.DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static void appendFragment(Line host, Line next) {
|
||||
float gap = next.glyphLeft() - host.glyphRight();
|
||||
float charWidth = fragmentCharWidth(host, next);
|
||||
String left = host.text.stripTrailing();
|
||||
String right = next.text.stripLeading();
|
||||
boolean space = gap >= charWidth * WordGeometry.NO_SPACE_GAP;
|
||||
host.text = left + (space ? " " : "") + right;
|
||||
host.merged.add(next.source);
|
||||
host.merged.addAll(next.merged);
|
||||
float right0 = Math.max(host.x + host.width, next.x + next.width);
|
||||
float top = Math.max(host.y + host.height, next.y + next.height);
|
||||
host.x = Math.min(host.x, next.x);
|
||||
host.y = Math.min(host.y, next.y);
|
||||
host.width = right0 - host.x;
|
||||
host.height = top - host.y;
|
||||
}
|
||||
|
||||
private static float fragmentCharWidth(Line a, Line b) {
|
||||
double total = 0;
|
||||
int chars = 0;
|
||||
for (Line l : List.of(a, b)) {
|
||||
for (TextWord w : l.words()) {
|
||||
for (TextChar c : w.chars()) {
|
||||
if (!c.isWhitespace() && !c.isNewline() && c.width() > 0f) {
|
||||
total += c.width();
|
||||
chars++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return chars == 0 ? 6f : (float) (total / chars);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Markdown text utilities: escaping extracted text and rebasing the finished document's heading
|
||||
* levels.
|
||||
*/
|
||||
final class MarkdownText {
|
||||
|
||||
private MarkdownText() {}
|
||||
|
||||
/** A Markdown ATX heading at the start of a line, with its level in group 1. */
|
||||
private static final Pattern ATX_HEADING = Pattern.compile("(?m)^(#{1,6}) (?=\\S)");
|
||||
|
||||
/**
|
||||
* Rebases headings so the strongest is level 1 and no level is skipped: levels only mean
|
||||
* anything against the other headings in the same document.
|
||||
*/
|
||||
static String normaliseHeadingLevels(String markdown) {
|
||||
Set<Integer> levels = new TreeSet<>();
|
||||
Matcher m = ATX_HEADING.matcher(markdown);
|
||||
while (m.find()) {
|
||||
levels.add(m.group(1).length());
|
||||
}
|
||||
if (levels.isEmpty() || (levels.contains(1) && levels.size() == maxOf(levels))) {
|
||||
return markdown;
|
||||
}
|
||||
Map<Integer, String> rebased = new HashMap<>();
|
||||
int rank = 1;
|
||||
for (int level : levels) {
|
||||
rebased.put(level, "#".repeat(rank++));
|
||||
}
|
||||
return m.reset().replaceAll(r -> rebased.get(r.group(1).length()) + " ");
|
||||
}
|
||||
|
||||
private static int maxOf(Set<Integer> levels) {
|
||||
int max = 0;
|
||||
for (int level : levels) {
|
||||
max = Math.max(max, level);
|
||||
}
|
||||
return max;
|
||||
}
|
||||
|
||||
static int wordCount(String text) {
|
||||
return text.isBlank() ? 0 : text.strip().split("\\s+").length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Escapes Markdown control characters so extracted text is emitted literally. Output is still
|
||||
* untrusted: this is defence-in-depth, not safe rendering.
|
||||
*/
|
||||
static String escapeMarkdown(String text) {
|
||||
if (text.isEmpty()) {
|
||||
return text;
|
||||
}
|
||||
String inline = escapeMarkdownInline(text);
|
||||
return escapeLeadingBlockMarker(inline, text);
|
||||
}
|
||||
|
||||
/** Escapes inline-significant Markdown characters anywhere in the string. */
|
||||
static String escapeMarkdownInline(String text) {
|
||||
StringBuilder sb = new StringBuilder(text.length() + 8);
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
char c = text.charAt(i);
|
||||
switch (c) {
|
||||
case '\\', '`', '*', '_', '[', ']', '<', '>', '|', '~' -> sb.append('\\').append(c);
|
||||
default -> sb.append(c);
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Escapes markers significant only at line start: {@code #}, {@code -}, {@code +} and
|
||||
* ordered-list numbers. {@code original} is unescaped, so positions line up.
|
||||
*/
|
||||
private static String escapeLeadingBlockMarker(String escaped, String original) {
|
||||
char c0 = original.charAt(0);
|
||||
if (c0 == '#' || c0 == '-' || c0 == '+') {
|
||||
return "\\" + escaped;
|
||||
}
|
||||
int i = 0;
|
||||
while (i < original.length() && Character.isDigit(original.charAt(i))) {
|
||||
i++;
|
||||
}
|
||||
if (i > 0 && i < original.length()) {
|
||||
char delim = original.charAt(i);
|
||||
if (delim == '.' || delim == ')') {
|
||||
return escaped.substring(0, i) + "\\" + escaped.substring(i);
|
||||
}
|
||||
}
|
||||
return escaped;
|
||||
}
|
||||
|
||||
static String normaliseSpace(String s) {
|
||||
return s.strip().replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
static void flushParagraph(StringBuilder para, List<String> out) {
|
||||
if (!para.isEmpty()) {
|
||||
out.add(escapeMarkdown(para.toString()));
|
||||
para.setLength(0);
|
||||
}
|
||||
}
|
||||
|
||||
static boolean endsWithSentencePunctuation(String s) {
|
||||
if (s.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
char last = s.charAt(s.length() - 1);
|
||||
return last == '.' || last == '?' || last == '!' || last == ':';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.PdfPage;
|
||||
import stirling.software.jpdfium.doc.ExtractedImage;
|
||||
import stirling.software.jpdfium.doc.PdfImageExtractor;
|
||||
import stirling.software.jpdfium.model.Rect;
|
||||
|
||||
/**
|
||||
* Emits a placeholder per image on a page; image bytes are deliberately not carried into the
|
||||
* Markdown.
|
||||
*/
|
||||
final class PageImages {
|
||||
|
||||
private PageImages() {}
|
||||
|
||||
static void emit(PdfDocument doc, int pageIndex, List<Object> pageItems) throws IOException {
|
||||
try (PdfPage page = doc.page(pageIndex)) {
|
||||
List<ExtractedImage> images =
|
||||
PdfImageExtractor.extract(page.rawDocHandle(), page.rawHandle(), pageIndex);
|
||||
for (ExtractedImage img : images) {
|
||||
pageItems.add(describe(img));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Image placeholder annotated with whatever JPDFium exposes: pixels, placement, DPI, format,
|
||||
* colour space, depth. Missing fields are omitted.
|
||||
*/
|
||||
private static String describe(ExtractedImage img) {
|
||||
List<String> parts = new ArrayList<>();
|
||||
if (img.width() > 0 && img.height() > 0) {
|
||||
parts.add(img.width() + "x" + img.height() + "px");
|
||||
}
|
||||
Rect b = img.bounds();
|
||||
if (b != null && b.width() > 0 && b.height() > 0) {
|
||||
parts.add(String.format("%.0fx%.0fpt", b.width(), b.height()));
|
||||
if (img.width() > 0) {
|
||||
float dpiX = img.width() / (b.width() / 72f);
|
||||
float dpiY = img.height() / (b.height() / 72f);
|
||||
if (Float.isFinite(dpiX) && dpiX > 0) {
|
||||
parts.add(String.format("~%.0fdpi", (dpiX + dpiY) / 2f));
|
||||
}
|
||||
}
|
||||
}
|
||||
String ext = img.suggestedExtension();
|
||||
if (ext != null && !ext.isBlank()) {
|
||||
parts.add(ext.replaceFirst("^\\.", "").toUpperCase(Locale.ROOT));
|
||||
}
|
||||
if (img.colorSpace() != null) {
|
||||
parts.add(img.colorSpace().toString());
|
||||
}
|
||||
if (img.bitsPerPixel() > 0) {
|
||||
parts.add(img.bitsPerPixel() + "bpp");
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder("<image redacted");
|
||||
if (!parts.isEmpty()) {
|
||||
sb.append(": ").append(String.join(", ", parts));
|
||||
}
|
||||
sb.append('>');
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.jpdfium.PdfPage;
|
||||
import stirling.software.jpdfium.doc.PageContentSummary;
|
||||
import stirling.software.jpdfium.doc.PageObject;
|
||||
import stirling.software.jpdfium.doc.PageObjectType;
|
||||
import stirling.software.jpdfium.doc.PdfPageObjects;
|
||||
import stirling.software.jpdfium.model.Rect;
|
||||
|
||||
/**
|
||||
* Ruling lines of one page, from its {@link PageObject} paths: a long thin box is a rule, and a box
|
||||
* large in both axes contributes its edges.
|
||||
*/
|
||||
final class PageRules {
|
||||
|
||||
/** A path this thin in its short axis is a rule rather than a filled area. */
|
||||
private static final float RULE_THICKNESS = 3f;
|
||||
|
||||
/** A rule shorter than this is decoration (tick marks, bullets, glyph art). */
|
||||
private static final float MIN_RULE_LENGTH = 8f;
|
||||
|
||||
/** Boxes larger than this in either axis are page furniture, not table structure. */
|
||||
private static final float MAX_BOX = 1500f;
|
||||
|
||||
/**
|
||||
* Object count past which the page is skipped: a page drawing this many is a chart or an
|
||||
* operator flood, never a readable grid.
|
||||
*/
|
||||
private static final int MAX_PAGE_OBJECTS = 20_000;
|
||||
|
||||
/** A straight rule: {@code pos} is its y (horizontal) or x (vertical), spanning lo..hi. */
|
||||
record Rule(float pos, float lo, float hi) {}
|
||||
|
||||
private final List<Rule> horizontal;
|
||||
private final List<Rule> vertical;
|
||||
|
||||
private PageRules(List<Rule> horizontal, List<Rule> vertical) {
|
||||
this.horizontal = horizontal;
|
||||
this.vertical = vertical;
|
||||
}
|
||||
|
||||
static final PageRules EMPTY = new PageRules(List.of(), List.of());
|
||||
|
||||
List<Rule> horizontal() {
|
||||
return horizontal;
|
||||
}
|
||||
|
||||
List<Rule> vertical() {
|
||||
return vertical;
|
||||
}
|
||||
|
||||
boolean isEmpty() {
|
||||
return horizontal.isEmpty() && vertical.isEmpty();
|
||||
}
|
||||
|
||||
/** Reads the ruling lines of an already-open page. */
|
||||
static PageRules of(PdfPage page) throws IOException {
|
||||
List<Rule> h = new ArrayList<>();
|
||||
List<Rule> v = new ArrayList<>();
|
||||
List<PageObject> objects;
|
||||
try {
|
||||
// Counting is far cheaper than materialising every object, so decide from the summary
|
||||
// whether the page is worth enumerating at all.
|
||||
PageContentSummary summary = PdfPageObjects.summarize(page.rawHandle());
|
||||
if (summary.pathObjectCount() < 2 || summary.totalObjects() > MAX_PAGE_OBJECTS) {
|
||||
return EMPTY;
|
||||
}
|
||||
objects = PdfPageObjects.list(page.rawHandle());
|
||||
} catch (RuntimeException e) {
|
||||
// Path enumeration is an optimisation, never a correctness requirement: a page whose
|
||||
// objects cannot be read simply falls back to word-grid detection.
|
||||
return EMPTY;
|
||||
}
|
||||
float pageW = 0f;
|
||||
float pageH = 0f;
|
||||
try {
|
||||
pageW = page.size().width();
|
||||
pageH = page.size().height();
|
||||
} catch (RuntimeException ignored) {
|
||||
// Fall through with 0,0: the on-page check below is then skipped.
|
||||
}
|
||||
for (PageObject o : objects) {
|
||||
if (o.type() != PageObjectType.PATH) {
|
||||
continue;
|
||||
}
|
||||
Rect b = o.bounds();
|
||||
if (b == null) {
|
||||
continue;
|
||||
}
|
||||
float w = b.width();
|
||||
float ht = b.height();
|
||||
if (!Float.isFinite(w) || !Float.isFinite(ht) || w < 0 || ht < 0) {
|
||||
continue;
|
||||
}
|
||||
if (w > MAX_BOX || ht > MAX_BOX) {
|
||||
continue;
|
||||
}
|
||||
// Paths that run off the page are chart clipping or decoration, never table structure.
|
||||
if (pageW > 0
|
||||
&& (b.x() < -1f
|
||||
|| b.y() < -1f
|
||||
|| b.x() + w > pageW + 1f
|
||||
|| b.y() + ht > pageH + 1f)) {
|
||||
continue;
|
||||
}
|
||||
if (ht <= RULE_THICKNESS && w >= MIN_RULE_LENGTH) {
|
||||
h.add(new Rule(b.y() + ht / 2f, b.x(), b.x() + w));
|
||||
} else if (w <= RULE_THICKNESS && ht >= MIN_RULE_LENGTH) {
|
||||
v.add(new Rule(b.x() + w / 2f, b.y(), b.y() + ht));
|
||||
} else if (w >= MIN_RULE_LENGTH && ht >= MIN_RULE_LENGTH) {
|
||||
// A box: a table border, a cell outline or a shaded row fill. Its edges bound cells
|
||||
// exactly as drawn rules do, and many generators draw grids as per-cell rectangles.
|
||||
h.add(new Rule(b.y(), b.x(), b.x() + w));
|
||||
h.add(new Rule(b.y() + ht, b.x(), b.x() + w));
|
||||
v.add(new Rule(b.x(), b.y(), b.y() + ht));
|
||||
v.add(new Rule(b.x() + w, b.y(), b.y() + ht));
|
||||
}
|
||||
}
|
||||
h.sort(Comparator.comparingDouble(Rule::pos).reversed());
|
||||
v.sort(Comparator.comparingDouble(Rule::pos));
|
||||
return new PageRules(h, v);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Joins what a page break split: a sentence running into the next page, and a table whose rows
|
||||
* continue on it.
|
||||
*/
|
||||
final class PageStitcher {
|
||||
|
||||
private PageStitcher() {}
|
||||
|
||||
static void mergeAcrossPageBoundary(List<Object> output, List<Object> pageItems) {
|
||||
if (output.isEmpty() || pageItems.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
// Only merge a sentence continuation between two text paragraphs, never into/out of a
|
||||
// table.
|
||||
if (!(output.getLast() instanceof String last)
|
||||
|| !(pageItems.getFirst() instanceof String first)) {
|
||||
return;
|
||||
}
|
||||
if (!first.isEmpty()
|
||||
&& Character.isLowerCase(first.charAt(0))
|
||||
&& !MarkdownText.endsWithSentencePunctuation(last)) {
|
||||
output.set(output.size() - 1, last + " " + first);
|
||||
pageItems.remove(0);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Joins tables split across a page break: two consecutive blocks with no text between them
|
||||
* merge when their column layouts match, dropping a repeated header.
|
||||
*/
|
||||
static List<Object> stitchTables(List<Object> elements) {
|
||||
List<Object> out = new ArrayList<>();
|
||||
// Column geometry of the trailing TableBlock in `out`, carried forward across merges so a
|
||||
// table running page-to-page is not re-projected from every accumulated row at each break.
|
||||
ColumnAccumulator acc = null;
|
||||
// Row list we own and may append to in place; null while the trailing block still holds a
|
||||
// list belonging to `elements`.
|
||||
List<List<Line>> ownedRows = null;
|
||||
for (Object e : elements) {
|
||||
if (e instanceof TableBlock tb
|
||||
&& !out.isEmpty()
|
||||
&& out.getLast() instanceof TableBlock prev) {
|
||||
if (acc == null) {
|
||||
acc = ColumnAccumulator.of(prev.rows());
|
||||
}
|
||||
if (columnsMatch(acc.columns(), ColumnRanges.find(flatten(tb.rows())))) {
|
||||
List<List<Line>> merged;
|
||||
if (ownedRows == null) {
|
||||
merged = new ArrayList<>(prev.rows());
|
||||
ownedRows = merged;
|
||||
} else {
|
||||
merged = ownedRows;
|
||||
}
|
||||
List<List<Line>> tail = tb.rows();
|
||||
if (!tail.isEmpty()
|
||||
&& !prev.rows().isEmpty()
|
||||
&& rowText(tail.getFirst()).equals(rowText(prev.rows().getFirst()))) {
|
||||
tail = tail.subList(1, tail.size());
|
||||
}
|
||||
for (List<Line> row : tail) {
|
||||
for (Line l : row) {
|
||||
acc.addLine(l);
|
||||
}
|
||||
}
|
||||
merged.addAll(tail);
|
||||
// A stitched table belongs to where it started, so keep the earlier block's
|
||||
// page and columns; its ruling lines are dropped as they are one page's only.
|
||||
out.set(
|
||||
out.size() - 1,
|
||||
new TableBlock(
|
||||
merged,
|
||||
prev.top(),
|
||||
tb.bottom(),
|
||||
prev.cols(),
|
||||
prev.ruled(),
|
||||
prev.rowSource(),
|
||||
prev.page()));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
out.add(e);
|
||||
acc = null;
|
||||
ownedRows = null;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static List<Line> flatten(List<List<Line>> rows) {
|
||||
return rows.stream().flatMap(List::stream).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* Header text of a table at the very bottom of a page, or null. Trailing image placeholders are
|
||||
* skipped; any other text means it is not a continuation.
|
||||
*/
|
||||
static String trailingTableHeader(List<Object> pageItems) {
|
||||
for (int i = pageItems.size() - 1; i >= 0; i--) {
|
||||
Object e = pageItems.get(i);
|
||||
if (e instanceof String s && s.strip().startsWith("<image redacted")) {
|
||||
continue;
|
||||
}
|
||||
if (e instanceof TableBlock tb && !tb.rows().isEmpty()) {
|
||||
return rowText(tb.rows().getFirst());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static String rowText(List<Line> row) {
|
||||
List<Line> ordered = new ArrayList<>(row);
|
||||
ordered.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (Line l : ordered) {
|
||||
if (!sb.isEmpty()) {
|
||||
sb.append(' ');
|
||||
}
|
||||
sb.append(l.text);
|
||||
}
|
||||
return MarkdownText.normaliseSpace(sb.toString());
|
||||
}
|
||||
|
||||
/** True when two table blocks have the same number of columns at near-identical x-centres. */
|
||||
private static boolean columnsMatch(List<float[]> ca, List<float[]> cb) {
|
||||
if (ca.size() < 2 || ca.size() != cb.size()) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < ca.size(); i++) {
|
||||
float centreA = (ca.get(i)[0] + ca.get(i)[1]) / 2f;
|
||||
float centreB = (cb.get(i)[0] + cb.get(i)[1]) / 2f;
|
||||
if (Math.abs(centreA - centreB) > 15f) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
+252
@@ -0,0 +1,252 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Turns a column's lines into Markdown blocks: headings (including wrapped ones), bullets, bold
|
||||
* labels and paragraphs. Contents lists are recognised and left alone.
|
||||
*/
|
||||
final class ParagraphAssembler {
|
||||
|
||||
private ParagraphAssembler() {}
|
||||
|
||||
static void assembleParagraphs(
|
||||
List<Line> lines,
|
||||
float medianSize,
|
||||
float medianHeight,
|
||||
String bodyFont,
|
||||
List<String> out,
|
||||
Set<String> tableRowTexts) {
|
||||
StringBuilder para = new StringBuilder();
|
||||
float prevBottomY = Float.MAX_VALUE;
|
||||
float prevHeight = 0f;
|
||||
boolean[] inContents = contentsRun(lines);
|
||||
|
||||
for (int i = 0; i < lines.size(); i++) {
|
||||
Line line = lines.get(i);
|
||||
String text = line.text.strip();
|
||||
if (text.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (tableRowTexts.contains(text)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
float blockTop = line.y + line.height;
|
||||
float gap = prevBottomY - blockTop;
|
||||
boolean paragraphBreak = prevHeight > 0f && gap > prevHeight * 0.8f;
|
||||
// A contents entry carries the typography of the section it points at without being
|
||||
// that section, so nothing on one is promoted or emphasised.
|
||||
boolean structural = inContents[i];
|
||||
|
||||
// A field value is data, never a heading: its widget box is taller than a text line
|
||||
// and would otherwise be promoted purely on height.
|
||||
String prefix =
|
||||
line.synthetic || structural
|
||||
? ""
|
||||
: HeadingDetector.headingPrefix(
|
||||
line.detectText(),
|
||||
line.detectHeight(),
|
||||
line.words(),
|
||||
medianSize,
|
||||
medianHeight,
|
||||
bodyFont,
|
||||
prevHeight <= 0f || paragraphBreak);
|
||||
if (prefix.isEmpty() && !structural && contentsTitle(lines, inContents, i)) {
|
||||
// The line a contents list runs on from is its heading: a contents page is often
|
||||
// set in one face, leaving no size or weight to promote it on.
|
||||
prefix = "# ";
|
||||
}
|
||||
boolean isBullet = startsWithBullet(text);
|
||||
// A line that opens with a list marker is an item of a list, whatever it is set in.
|
||||
boolean isHeading = !prefix.isEmpty() && !isBullet;
|
||||
|
||||
if (isHeading) {
|
||||
MarkdownText.flushParagraph(para, out);
|
||||
StringBuilder heading = new StringBuilder(MarkdownText.escapeMarkdown(text));
|
||||
int words = MarkdownText.wordCount(text);
|
||||
int j = i;
|
||||
int k = i + 1;
|
||||
while (k < lines.size() && words < MAX_WRAPPED_HEADING_WORDS) {
|
||||
Line next = lines.get(k);
|
||||
String nt = next.text.strip();
|
||||
if (nt.isEmpty()) {
|
||||
// An empty extractor record is not a break in the text; the vertical
|
||||
// gap below decides whether the heading ended.
|
||||
k++;
|
||||
continue;
|
||||
}
|
||||
if (inContents[k] || tableRowTexts.contains(nt)) {
|
||||
break;
|
||||
}
|
||||
if (!wrapsHeading(
|
||||
lines.get(j), next, prefix, medianSize, medianHeight, bodyFont)) {
|
||||
break;
|
||||
}
|
||||
heading.append(' ').append(MarkdownText.escapeMarkdown(nt));
|
||||
words += MarkdownText.wordCount(nt);
|
||||
j = k;
|
||||
k++;
|
||||
}
|
||||
out.add(prefix + heading);
|
||||
if (j > i) {
|
||||
i = j;
|
||||
line = lines.get(j);
|
||||
}
|
||||
} else if (isBullet) {
|
||||
MarkdownText.flushParagraph(para, out);
|
||||
out.add(MarkdownText.escapeMarkdown(text));
|
||||
} else if (!line.synthetic
|
||||
&& !structural
|
||||
&& HeadingDetector.isBoldLabel(line.detectText(), line.words())) {
|
||||
// Bold but not large enough to be a heading → emphasise as bold, don't promote.
|
||||
MarkdownText.flushParagraph(para, out);
|
||||
out.add("**" + MarkdownText.escapeMarkdown(text) + "**");
|
||||
} else if (paragraphBreak) {
|
||||
MarkdownText.flushParagraph(para, out);
|
||||
para.append(text);
|
||||
} else {
|
||||
if (!para.isEmpty()) {
|
||||
char fc = text.charAt(0);
|
||||
boolean noSpace = fc == '\'' || fc == '’' || fc == '‘' || fc == '"';
|
||||
if (!noSpace) {
|
||||
para.append(' ');
|
||||
}
|
||||
}
|
||||
para.append(text);
|
||||
}
|
||||
|
||||
prevBottomY = line.y;
|
||||
prevHeight = line.height;
|
||||
}
|
||||
MarkdownText.flushParagraph(para, out);
|
||||
}
|
||||
|
||||
/** Glyphs a document may set its list markers in beyond the three already recognised. */
|
||||
private static final String EXTRA_BULLETS = "‣⁃▶●○■□" + "◆⮚➢➣➤";
|
||||
|
||||
private static boolean startsWithBullet(String text) {
|
||||
if (text.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
if (text.startsWith("•") || text.startsWith("▪") || text.startsWith("◦")) {
|
||||
return true;
|
||||
}
|
||||
return EXTRA_BULLETS.indexOf(text.charAt(0)) >= 0;
|
||||
}
|
||||
|
||||
/** Longest a heading may grow to by absorbing its continuation lines, in words. */
|
||||
private static final int MAX_WRAPPED_HEADING_WORDS = 24;
|
||||
|
||||
/** How far a continuation line's type size may differ from the line it continues. */
|
||||
private static final float WRAP_SIZE_TOLERANCE = 0.2f;
|
||||
|
||||
/** A full stop that a further sentence follows: the shape of prose, not of a heading. */
|
||||
private static final Pattern SENTENCE_BREAK = Pattern.compile("[.!?]\\s+\\p{Lu}");
|
||||
|
||||
/**
|
||||
* True when {@code next} continues a wrapped heading: each visual line arrives separately, so
|
||||
* an unjoined heading emits as several spurious ones.
|
||||
*/
|
||||
private static boolean wrapsHeading(
|
||||
Line head,
|
||||
Line next,
|
||||
String prefix,
|
||||
float medianSize,
|
||||
float medianHeight,
|
||||
String bodyFont) {
|
||||
if (next.synthetic) {
|
||||
return false;
|
||||
}
|
||||
float height = head.detectHeight();
|
||||
if (height <= 0f) {
|
||||
return false;
|
||||
}
|
||||
// The next baseline down, not the next block. The same 0.8 the paragraph assembler uses,
|
||||
// so a heading absorbs exactly what the converter already calls one block.
|
||||
float gap = head.y - (next.y + next.height);
|
||||
if (gap > height * 0.8f || gap < -height * 0.5f) {
|
||||
return false;
|
||||
}
|
||||
float nextHeight = next.detectHeight();
|
||||
if (Math.abs(nextHeight - height) > WRAP_SIZE_TOLERANCE * Math.max(nextHeight, height)) {
|
||||
return false;
|
||||
}
|
||||
// Same column: an x-range that misses the heading's belongs to another block entirely.
|
||||
if (next.x >= head.x + head.width || head.x >= next.x + next.width) {
|
||||
return false;
|
||||
}
|
||||
// A heading does not run to a full stop and then start another sentence; the bold run-in
|
||||
// lead-in below it does, and nothing else tells the two apart.
|
||||
if (SENTENCE_BREAK.matcher(next.text).find()) {
|
||||
return false;
|
||||
}
|
||||
String nextPrefix =
|
||||
HeadingDetector.headingPrefix(
|
||||
next.detectText(),
|
||||
next.detectHeight(),
|
||||
next.words(),
|
||||
medianSize,
|
||||
medianHeight,
|
||||
bodyFont,
|
||||
false);
|
||||
// Either the continuation is display type in its own right, or it is the bold remainder of
|
||||
// a run-in heading, which cannot be promoted on its own because no gap precedes it.
|
||||
return nextPrefix.equals(prefix)
|
||||
|| (nextPrefix.isEmpty()
|
||||
&& HeadingDetector.isBoldLabel(next.detectText(), next.words()));
|
||||
}
|
||||
|
||||
/** A leader run: the dots that carry the eye from a contents entry to its page number. */
|
||||
private static final Pattern LEADER = Pattern.compile("([.][ ]?){4,}|[.\u00b7]{3,}|\u2026{2,}");
|
||||
|
||||
/** Entries this many lines long make a contents list rather than a coincidence. */
|
||||
private static final int MIN_CONTENTS_RUN = 3;
|
||||
|
||||
/**
|
||||
* Marks the lines of a contents list: titles joined to page numbers by leader dots, carrying
|
||||
* the typography of the sections they point at.
|
||||
*/
|
||||
private static boolean[] contentsRun(List<Line> lines) {
|
||||
boolean[] entry = new boolean[lines.size()];
|
||||
int run = 0;
|
||||
for (int i = 0; i < lines.size(); i++) {
|
||||
String t = lines.get(i).text;
|
||||
if (LEADER.matcher(t).find() && endsWithNumber(t)) {
|
||||
entry[i] = true;
|
||||
run++;
|
||||
} else {
|
||||
if (run < MIN_CONTENTS_RUN) {
|
||||
clear(entry, i - run, i);
|
||||
}
|
||||
run = 0;
|
||||
}
|
||||
}
|
||||
if (run < MIN_CONTENTS_RUN) {
|
||||
clear(entry, lines.size() - run, lines.size());
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
private static void clear(boolean[] flags, int from, int to) {
|
||||
for (int i = Math.max(0, from); i < to; i++) {
|
||||
flags[i] = false;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean endsWithNumber(String text) {
|
||||
String t = text.strip();
|
||||
return !t.isEmpty() && Character.isDigit(t.charAt(t.length() - 1));
|
||||
}
|
||||
|
||||
/** True for the short line a contents list runs on from: the list's own heading. */
|
||||
private static boolean contentsTitle(List<Line> lines, boolean[] inContents, int index) {
|
||||
if (index + 1 >= lines.size() || inContents[index] || !inContents[index + 1]) {
|
||||
return false;
|
||||
}
|
||||
String t = lines.get(index).text.strip();
|
||||
return !t.isEmpty() && t.split(" +").length <= 6 && !endsWithNumber(t);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
/**
|
||||
* How much of a block's row structure the page itself drew; stronger evidence means weaker
|
||||
* false-positive guards.
|
||||
*/
|
||||
enum RowSource {
|
||||
/** Rows inferred from word geometry alone; nothing on the page confirms a table. */
|
||||
WORDS,
|
||||
/** Rows sit inside a region fenced by drawn rules, but the rules do not delimit them. */
|
||||
RULE_BOUNDED,
|
||||
/** Every row boundary is a drawn rule running the table's own width. */
|
||||
LATTICE;
|
||||
|
||||
boolean ruleConfirmed() {
|
||||
return this != WORDS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* A page's ruling lines reduced to a grid: rules merged into levels, levels grouped into the
|
||||
* components that each describe one table. Both steps are bounded.
|
||||
*/
|
||||
@Slf4j
|
||||
final class RuleGrid {
|
||||
|
||||
/** Rules within this distance are the same drawn line (double strokes, overdraw). */
|
||||
static final float LEVEL_TOLERANCE = 2.5f;
|
||||
|
||||
/** Slack when testing whether a horizontal and a vertical rule touch. */
|
||||
private static final float TOUCH = 3f;
|
||||
|
||||
/** Segments at one position further apart than this belong to different tables. */
|
||||
private static final float CONTIGUOUS_GAP = 8f;
|
||||
|
||||
/** Crossing tests past which a page is an operator flood rather than a readable grid. */
|
||||
private static final long MAX_CROSSING_TESTS = 4_000_000L;
|
||||
|
||||
/** Rule components past which the extra blocks cannot be real tables. */
|
||||
private static final int MAX_COMPONENTS = 256;
|
||||
|
||||
private RuleGrid() {}
|
||||
|
||||
/** A group of rules at the same position: {@code pos} with the union of their extents. */
|
||||
record Level(float pos, float lo, float hi) {}
|
||||
|
||||
/** One connected component of crossing rules: the levels of each family it spans. */
|
||||
record Component(List<Level> h, List<Level> v) {}
|
||||
|
||||
/**
|
||||
* Merges rules at the same position into levels, but only while contiguous, so two tables
|
||||
* ruling at the same x are not bridged into one region.
|
||||
*/
|
||||
static List<Level> cluster(List<PageRules.Rule> rules) {
|
||||
List<PageRules.Rule> sorted = new ArrayList<>(rules);
|
||||
sorted.sort(
|
||||
Comparator.comparingDouble(PageRules.Rule::pos)
|
||||
.thenComparingDouble(PageRules.Rule::lo));
|
||||
List<Level> out = new ArrayList<>();
|
||||
int i = 0;
|
||||
while (i < sorted.size()) {
|
||||
float pos = sorted.get(i).pos();
|
||||
int j = i;
|
||||
while (j < sorted.size() && sorted.get(j).pos() - pos <= LEVEL_TOLERANCE) {
|
||||
j++;
|
||||
}
|
||||
List<PageRules.Rule> same = new ArrayList<>(sorted.subList(i, j));
|
||||
same.sort(Comparator.comparingDouble(PageRules.Rule::lo));
|
||||
float lo = same.get(0).lo();
|
||||
float hi = same.get(0).hi();
|
||||
for (int k = 1; k < same.size(); k++) {
|
||||
if (same.get(k).lo() <= hi + CONTIGUOUS_GAP) {
|
||||
hi = Math.max(hi, same.get(k).hi());
|
||||
} else {
|
||||
out.add(new Level(pos, lo, hi));
|
||||
lo = same.get(k).lo();
|
||||
hi = same.get(k).hi();
|
||||
}
|
||||
}
|
||||
out.add(new Level(pos, lo, hi));
|
||||
i = j;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Connected components of crossing rules, read from one union-find array: an id array per
|
||||
* component is O(components x levels) a rule flood can exhaust.
|
||||
*/
|
||||
static List<Component> partition(List<Level> hLevels, List<Level> vLevels) {
|
||||
int n = hLevels.size() + vLevels.size();
|
||||
if ((long) hLevels.size() * vLevels.size() > MAX_CROSSING_TESTS) {
|
||||
log.debug(
|
||||
"ruled-table partition skipped: {}x{} rule levels",
|
||||
hLevels.size(),
|
||||
vLevels.size());
|
||||
return List.of();
|
||||
}
|
||||
int[] parent = new int[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
parent[i] = i;
|
||||
}
|
||||
for (int i = 0; i < hLevels.size(); i++) {
|
||||
Level h = hLevels.get(i);
|
||||
for (int j = 0; j < vLevels.size(); j++) {
|
||||
Level v = vLevels.get(j);
|
||||
boolean crosses =
|
||||
v.pos() >= h.lo() - TOUCH
|
||||
&& v.pos() <= h.hi() + TOUCH
|
||||
&& h.pos() >= v.lo() - TOUCH
|
||||
&& h.pos() <= v.hi() + TOUCH;
|
||||
if (crosses) {
|
||||
union(parent, i, hLevels.size() + j);
|
||||
}
|
||||
}
|
||||
}
|
||||
Map<Integer, Component> byRoot = new LinkedHashMap<>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
int root = find(parent, i);
|
||||
Component c = byRoot.get(root);
|
||||
if (c == null) {
|
||||
// Past the cap the page is line art, not tables; keep the components already
|
||||
// found whole rather than truncating them mid-scan.
|
||||
if (byRoot.size() >= MAX_COMPONENTS) {
|
||||
continue;
|
||||
}
|
||||
c = new Component(new ArrayList<>(), new ArrayList<>());
|
||||
byRoot.put(root, c);
|
||||
}
|
||||
if (i < hLevels.size()) {
|
||||
c.h().add(hLevels.get(i));
|
||||
} else {
|
||||
c.v().add(vLevels.get(i - hLevels.size()));
|
||||
}
|
||||
}
|
||||
return List.copyOf(byRoot.values());
|
||||
}
|
||||
|
||||
private static int find(int[] parent, int x) {
|
||||
while (parent[x] != x) {
|
||||
parent[x] = parent[parent[x]];
|
||||
x = parent[x];
|
||||
}
|
||||
return x;
|
||||
}
|
||||
|
||||
private static void union(int[] parent, int a, int b) {
|
||||
int ra = find(parent, a);
|
||||
int rb = find(parent, b);
|
||||
if (ra != rb) {
|
||||
parent[rb] = ra;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Visible for testing: partitioning depends only on rule geometry, so tests can drive it from
|
||||
* synthetic rules.
|
||||
*/
|
||||
static int componentCount(List<PageRules.Rule> horizontal, List<PageRules.Rule> vertical) {
|
||||
return partition(cluster(horizontal), cluster(vertical)).size();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Groups the lines inside a ruled region into rows, and reads its column bands off the vertical
|
||||
* rules.
|
||||
*/
|
||||
final class RuledRows {
|
||||
|
||||
private RuledRows() {}
|
||||
|
||||
/** A vertical rule must cover this fraction of a region's height to be a column boundary. */
|
||||
private static final float COLUMN_COVERAGE = 0.5f;
|
||||
|
||||
/**
|
||||
* Splits a band whose every baseline is a complete row back into those rows; a wrapped cell
|
||||
* leaves the other columns empty, a run of rows does not.
|
||||
*/
|
||||
static List<List<Line>> splitCompleteBands(List<List<Line>> bands, List<float[]> cols) {
|
||||
if (cols == null || cols.size() < 2) {
|
||||
return bands;
|
||||
}
|
||||
List<List<Line>> out = new ArrayList<>();
|
||||
for (List<Line> band : bands) {
|
||||
List<List<Line>> baselines = baselineRows(band);
|
||||
if (baselines.size() < 2 || !allRowsComplete(baselines, cols)) {
|
||||
out.add(band);
|
||||
continue;
|
||||
}
|
||||
out.addAll(baselines);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** True when every baseline group puts a word in every column band. */
|
||||
private static boolean allRowsComplete(List<List<Line>> baselines, List<float[]> cols) {
|
||||
for (List<Line> row : baselines) {
|
||||
boolean[] hit = new boolean[cols.size()];
|
||||
for (Line l : row) {
|
||||
for (TextWord w : l.words()) {
|
||||
if (w.text().strip().isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
int c = TableGrid.containingColumn(w.x() + w.width() / 2f, cols);
|
||||
if (c >= 0 && c < hit.length) {
|
||||
hit[c] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (boolean h : hit) {
|
||||
if (!h) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Column bands from the vertical rules spanning the region; null when no interior rule
|
||||
* survives, as whitespace projection guesses better.
|
||||
*/
|
||||
static List<float[]> columns(
|
||||
List<RuleGrid.Level> vLevels, float left, float right, float top, float bottom) {
|
||||
float height = top - bottom;
|
||||
// Per-cell strokes give one rule per row, and a row that draws no boxes breaks the run
|
||||
// in two, so strokes at one x are measured together rather than as separate runs.
|
||||
List<RuleGrid.Level> sorted = new ArrayList<>(vLevels);
|
||||
sorted.sort(Comparator.comparingDouble(RuleGrid.Level::pos));
|
||||
List<Float> xs = new ArrayList<>();
|
||||
int at = 0;
|
||||
while (at < sorted.size()) {
|
||||
float pos = sorted.get(at).pos();
|
||||
float covered = 0f;
|
||||
int end = at;
|
||||
while (end < sorted.size() && sorted.get(end).pos() - pos <= RuleGrid.LEVEL_TOLERANCE) {
|
||||
RuleGrid.Level v = sorted.get(end);
|
||||
covered += Math.max(0f, Math.min(top, v.hi()) - Math.max(bottom, v.lo()));
|
||||
end++;
|
||||
}
|
||||
if (covered >= height * COLUMN_COVERAGE) {
|
||||
xs.add(pos);
|
||||
}
|
||||
at = end;
|
||||
}
|
||||
List<Float> bounds = new ArrayList<>();
|
||||
bounds.add(left);
|
||||
for (float x : xs) {
|
||||
if (x > bounds.get(bounds.size() - 1) + RuleGrid.LEVEL_TOLERANCE
|
||||
&& x < right - RuleGrid.LEVEL_TOLERANCE) {
|
||||
bounds.add(x);
|
||||
}
|
||||
}
|
||||
if (bounds.size() < 2) {
|
||||
return null;
|
||||
}
|
||||
bounds.add(right);
|
||||
List<float[]> cols = new ArrayList<>();
|
||||
for (int i = 1; i < bounds.size(); i++) {
|
||||
cols.add(new float[] {bounds.get(i - 1), bounds.get(i)});
|
||||
}
|
||||
return cols;
|
||||
}
|
||||
|
||||
/** Rows delimited by horizontal rules; this is what keeps a wrapped cell as one row. */
|
||||
static List<List<Line>> latticeRows(List<Float> bands, List<Line> inside) {
|
||||
List<List<Line>> rows = new ArrayList<>();
|
||||
for (int i = 1; i < bands.size(); i++) {
|
||||
float hi = bands.get(i - 1);
|
||||
float lo = bands.get(i);
|
||||
List<Line> band = new ArrayList<>();
|
||||
for (Line l : inside) {
|
||||
float cy = l.y + l.height / 2f;
|
||||
if (cy > lo && cy <= hi) {
|
||||
band.add(l);
|
||||
}
|
||||
}
|
||||
if (!band.isEmpty()) {
|
||||
rows.add(band);
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** Rows by baseline proximity, for a table ruled between its columns but not its rows. */
|
||||
static List<List<Line>> baselineRows(List<Line> inside) {
|
||||
List<Line> sorted = new ArrayList<>(inside);
|
||||
sorted.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
|
||||
List<Float> heights = sorted.stream().map(l -> l.height).sorted().toList();
|
||||
float sameRow = Math.max(2f, heights.get(heights.size() / 2) * 0.6f);
|
||||
List<List<Line>> rows = new ArrayList<>();
|
||||
List<Line> current = new ArrayList<>();
|
||||
float anchor = 0f;
|
||||
for (Line l : sorted) {
|
||||
if (current.isEmpty()) {
|
||||
anchor = l.y;
|
||||
} else if (anchor - l.y > sameRow) {
|
||||
rows.add(current);
|
||||
current = new ArrayList<>();
|
||||
anchor = l.y;
|
||||
}
|
||||
current.add(l);
|
||||
}
|
||||
if (!current.isEmpty()) {
|
||||
rows.add(current);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Builds table blocks from a page's ruling lines: whitespace projection cannot see single-word or
|
||||
* wrapped cells, as they leave no wide gap.
|
||||
*/
|
||||
@Slf4j
|
||||
final class RuledTables {
|
||||
|
||||
private RuledTables() {}
|
||||
|
||||
/** Largest vertical gap between two rules of one rows-only table. */
|
||||
private static final float ROWS_ONLY_GAP = 150f;
|
||||
|
||||
/** How far two rules of one rows-only table may differ at either end. */
|
||||
private static final float EXTENT_TOLERANCE = 8f;
|
||||
|
||||
/** Lines a rows-only group needs before two rules alone are enough to call it a table. */
|
||||
private static final int ROWS_ONLY_LINES = 4;
|
||||
|
||||
/** Fraction of a lattice's row bands that must contain text for it to be a real table. */
|
||||
private static final float FILLED_BANDS = 0.6f;
|
||||
|
||||
/** Fraction of the table's width an interior rule must run to be a row boundary. */
|
||||
private static final float ROW_RULE_SPAN = 0.8f;
|
||||
|
||||
/**
|
||||
* Interior row rules needed before drawn bands beat text baselines; bands keep a multi-line
|
||||
* cell whole where baselines split it.
|
||||
*/
|
||||
private static final int MIN_INTERIOR_RULES = 1;
|
||||
|
||||
/** Fraction of a region's width every rule must run for its rows to be a drawn lattice. */
|
||||
private static final float FULL_WIDTH_RULE = 0.8f;
|
||||
|
||||
private static TableBlock dbgNull(String why) {
|
||||
log.debug("ruled-table build rejected: {}", why);
|
||||
return null;
|
||||
}
|
||||
|
||||
static List<TableBlock> find(List<Line> lines, PageRules rules, int page) {
|
||||
if (rules == null || rules.isEmpty() || lines.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
// Synthetic AcroForm values carry no glyphs of their own, so a ruled grid must not
|
||||
// claim them: they would seed rows and columns the content stream never drew.
|
||||
lines = lines.stream().filter(l -> !l.synthetic).toList();
|
||||
if (lines.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<RuleGrid.Level> hLevels = RuleGrid.cluster(rules.horizontal());
|
||||
List<RuleGrid.Level> vLevels = RuleGrid.cluster(rules.vertical());
|
||||
if (hLevels.size() < 2) {
|
||||
return List.of();
|
||||
}
|
||||
List<TableBlock> blocks = new ArrayList<>();
|
||||
for (RuleGrid.Component part : RuleGrid.partition(hLevels, vLevels)) {
|
||||
TableBlock b = build(part.h(), part.v(), lines, page);
|
||||
if (b != null) {
|
||||
blocks.add(b);
|
||||
}
|
||||
}
|
||||
|
||||
// Horizontal rules no grid block claimed can still be a booktabs table: rows ruled,
|
||||
// columns not drawn at all. Whatever the grid did not take is offered to that reading.
|
||||
List<RuleGrid.Level> unclaimed = new ArrayList<>();
|
||||
for (RuleGrid.Level h : hLevels) {
|
||||
boolean claimed = false;
|
||||
for (TableBlock b : blocks) {
|
||||
if (h.pos() >= b.bottom() - RuleGrid.LEVEL_TOLERANCE
|
||||
&& h.pos() <= b.top() + RuleGrid.LEVEL_TOLERANCE) {
|
||||
claimed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!claimed) {
|
||||
unclaimed.add(h);
|
||||
}
|
||||
}
|
||||
if (unclaimed.size() >= 2) {
|
||||
for (TableBlock b : rowsOnly(unclaimed, lines, page)) {
|
||||
boolean overlaps = false;
|
||||
for (TableBlock existing : blocks) {
|
||||
if (TableFinder.covers(existing, b)) {
|
||||
overlaps = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!overlaps) {
|
||||
blocks.add(b);
|
||||
}
|
||||
}
|
||||
}
|
||||
blocks.sort(Comparator.comparingDouble(TableBlock::top).reversed());
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Blocks for a page ruled only across its rows (booktabs): no column geometry to recover, so
|
||||
* these only find a table the word grid missed.
|
||||
*/
|
||||
private static List<TableBlock> rowsOnly(
|
||||
List<RuleGrid.Level> levels, List<Line> lines, int page) {
|
||||
List<RuleGrid.Level> hLevels = new ArrayList<>(levels);
|
||||
hLevels.sort(Comparator.comparingDouble(RuleGrid.Level::pos).reversed());
|
||||
List<TableBlock> blocks = new ArrayList<>();
|
||||
List<List<RuleGrid.Level>> groups = new ArrayList<>();
|
||||
List<RuleGrid.Level> current = new ArrayList<>();
|
||||
current.add(hLevels.get(0));
|
||||
for (int i = 1; i < hLevels.size(); i++) {
|
||||
RuleGrid.Level prev = current.get(current.size() - 1);
|
||||
RuleGrid.Level l = hLevels.get(i);
|
||||
// One booktabs table rules to a single extent; two stacked tables differ in width,
|
||||
// and grouping them would project their columns into a single band.
|
||||
if (prev.pos() - l.pos() > ROWS_ONLY_GAP
|
||||
|| Math.abs(prev.lo() - l.lo()) > EXTENT_TOLERANCE
|
||||
|| Math.abs(prev.hi() - l.hi()) > EXTENT_TOLERANCE) {
|
||||
groups.add(current);
|
||||
current = new ArrayList<>();
|
||||
}
|
||||
current.add(l);
|
||||
}
|
||||
groups.add(current);
|
||||
|
||||
for (List<RuleGrid.Level> g : groups) {
|
||||
if (g.size() < 2) {
|
||||
continue;
|
||||
}
|
||||
float top = g.get(0).pos();
|
||||
float bottom = g.get(g.size() - 1).pos();
|
||||
float left = Float.MAX_VALUE;
|
||||
float right = -Float.MAX_VALUE;
|
||||
for (RuleGrid.Level l : g) {
|
||||
left = Math.min(left, l.lo());
|
||||
right = Math.max(right, l.hi());
|
||||
}
|
||||
List<Line> inside = new ArrayList<>();
|
||||
for (Line l : lines) {
|
||||
float cy = l.y + l.height / 2f;
|
||||
float cx = l.x + l.width / 2f;
|
||||
if (cy > bottom && cy < top && cx > left - 5f && cx < right + 5f) {
|
||||
inside.add(l);
|
||||
}
|
||||
}
|
||||
// Enough text to be a table: several rows, or for a two-row table a third rule,
|
||||
// the header separator a lone pair of decorative rules does not draw.
|
||||
if (inside.size() < 2 || (inside.size() < ROWS_ONLY_LINES && g.size() < 3)) {
|
||||
continue;
|
||||
}
|
||||
List<List<Line>> rows = RuledRows.baselineRows(inside);
|
||||
if (rows.size() < 2 || TableGrid.render(rows, null, RowSource.RULE_BOUNDED).isBlank()) {
|
||||
continue;
|
||||
}
|
||||
blocks.add(new TableBlock(rows, top, bottom, null, page));
|
||||
}
|
||||
blocks.sort(Comparator.comparingDouble(TableBlock::top).reversed());
|
||||
return blocks;
|
||||
}
|
||||
|
||||
private static TableBlock build(
|
||||
List<RuleGrid.Level> hL, List<RuleGrid.Level> vL, List<Line> lines, int page) {
|
||||
log.debug("ruled-table build hL={} vL={}", hL.size(), vL.size());
|
||||
if (hL.size() < 2 || vL.size() < 2) {
|
||||
return dbgNull("hL/vL < 2");
|
||||
}
|
||||
hL.sort(Comparator.comparingDouble(RuleGrid.Level::pos).reversed());
|
||||
vL.sort(Comparator.comparingDouble(RuleGrid.Level::pos));
|
||||
|
||||
// The extent is the union of both families: a table ruled only between its columns
|
||||
// takes its top and bottom from the verticals, and vice versa.
|
||||
float top = hL.get(0).pos();
|
||||
float bottom = hL.get(hL.size() - 1).pos();
|
||||
float left = vL.get(0).pos();
|
||||
float right = vL.get(vL.size() - 1).pos();
|
||||
for (RuleGrid.Level v : vL) {
|
||||
top = Math.max(top, v.hi());
|
||||
bottom = Math.min(bottom, v.lo());
|
||||
}
|
||||
for (RuleGrid.Level h : hL) {
|
||||
left = Math.min(left, h.lo());
|
||||
right = Math.max(right, h.hi());
|
||||
}
|
||||
if (top - bottom < 6f || right - left < 20f) {
|
||||
return dbgNull("too small");
|
||||
}
|
||||
|
||||
List<Line> inside = new ArrayList<>();
|
||||
for (Line l : lines) {
|
||||
float cy = l.y + l.height / 2f;
|
||||
float cx = l.x + l.width / 2f;
|
||||
if (cy > bottom && cy < top && cx > left - 5f && cx < right + 5f) {
|
||||
inside.add(l);
|
||||
}
|
||||
}
|
||||
if (inside.size() < 2) {
|
||||
return dbgNull("inside<2");
|
||||
}
|
||||
|
||||
// Null columns mean the grid is ruled between its rows only; the block is still worth
|
||||
// building, but its columns then come from whitespace projection.
|
||||
List<float[]> cols = RuledRows.columns(vL, left, right, top, bottom);
|
||||
|
||||
// A row boundary is a y position, not a segment, and runs the table's width: per-cell
|
||||
// rectangles report it once per cell and also box each wrapped line inside a cell.
|
||||
float rowRuleWidth = (right - left) * ROW_RULE_SPAN;
|
||||
List<Float> interiorH = new ArrayList<>();
|
||||
List<RuleGrid.Level> bandRules = new ArrayList<>();
|
||||
float prevWide = top;
|
||||
int i = 0;
|
||||
while (i < hL.size()) {
|
||||
float pos = hL.get(i).pos();
|
||||
int j = i;
|
||||
RuleGrid.Level widest = hL.get(i);
|
||||
while (j < hL.size() && Math.abs(hL.get(j).pos() - pos) <= RuleGrid.LEVEL_TOLERANCE) {
|
||||
if (hL.get(j).hi() - hL.get(j).lo() > widest.hi() - widest.lo()) {
|
||||
widest = hL.get(j);
|
||||
}
|
||||
j++;
|
||||
}
|
||||
i = j;
|
||||
if (pos <= bottom + RuleGrid.LEVEL_TOLERANCE || pos >= top - RuleGrid.LEVEL_TOLERANCE) {
|
||||
bandRules.add(widest);
|
||||
continue;
|
||||
}
|
||||
boolean wide = widest.hi() - widest.lo() >= rowRuleWidth;
|
||||
boolean keep =
|
||||
wide || spanningNeighbour(widest, vL, inside, pos, prevWide, top - bottom);
|
||||
if (!keep) {
|
||||
continue;
|
||||
}
|
||||
interiorH.add(pos);
|
||||
bandRules.add(widest);
|
||||
if (wide) {
|
||||
prevWide = pos;
|
||||
}
|
||||
}
|
||||
|
||||
List<List<Line>> rows;
|
||||
RowSource source = RowSource.RULE_BOUNDED;
|
||||
if (interiorH.size() >= MIN_INTERIOR_RULES) {
|
||||
List<Float> bands = new ArrayList<>();
|
||||
bands.add(top);
|
||||
bands.addAll(interiorH);
|
||||
bands.add(bottom);
|
||||
List<List<Line>> filled = RuledRows.latticeRows(bands, inside);
|
||||
// Most bands must carry text: a chart's axis ticks or a zebra table's stripes rule
|
||||
// many empty bands, and reading those as a table steals lines from the prose.
|
||||
if (filled.size() < (bands.size() - 1) * FILLED_BANDS) {
|
||||
return dbgNull("filled " + filled.size() + " of bands " + (bands.size() - 1));
|
||||
}
|
||||
rows = RuledRows.splitCompleteBands(filled, cols);
|
||||
if (fullWidthRules(bandRules, left, right)) {
|
||||
source = RowSource.LATTICE;
|
||||
}
|
||||
} else {
|
||||
rows = RuledRows.baselineRows(inside);
|
||||
}
|
||||
if (rows.size() < 2) {
|
||||
return dbgNull("rows<2");
|
||||
}
|
||||
|
||||
// A grid is often ruled around its body only, leaving the header just above the top
|
||||
// rule; take it when it fits the grid's width and resolves into its columns.
|
||||
if (cols != null) {
|
||||
// The header's cells are separate lines when they sit far apart, so the whole
|
||||
// band above the grid is taken, not the nearest line.
|
||||
List<Line> hdr = new ArrayList<>();
|
||||
float band = Float.MAX_VALUE;
|
||||
for (Line l : lines) {
|
||||
if (l.y <= top
|
||||
|| l.y - top > TableFinder.HEADER_RULE_GAP * Math.max(l.height, 1f)
|
||||
|| l.x < left - 5f
|
||||
|| l.x + l.width > right + 5f) {
|
||||
continue;
|
||||
}
|
||||
band = Math.min(band, l.y);
|
||||
}
|
||||
for (Line l : lines) {
|
||||
if (band < Float.MAX_VALUE
|
||||
&& l.y >= band
|
||||
&& l.y <= band + 2f
|
||||
&& l.x >= left - 5f
|
||||
&& l.x + l.width <= right + 5f) {
|
||||
hdr.add(l);
|
||||
}
|
||||
}
|
||||
if (!hdr.isEmpty()) {
|
||||
List<List<Line>> withHeader = new ArrayList<>();
|
||||
withHeader.add(hdr);
|
||||
withHeader.addAll(rows);
|
||||
List<String[]> grown = TableGrid.cells(withHeader, cols, source);
|
||||
if (!grown.isEmpty()
|
||||
&& TableGrid.filledCells(grown.get(0)) >= grown.get(0).length - 1
|
||||
&& TableGrid.filledCells(grown.get(0)) >= 2
|
||||
&& TableFinder.wordGroups(hdr) == TableGrid.filledCells(grown.get(0))) {
|
||||
rows = withHeader;
|
||||
top = band + hdr.get(0).height;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TableBlock block = new TableBlock(rows, top, bottom, cols, true, source, page);
|
||||
// A block that fails the shared false-positive guards is not a table; leaving its lines
|
||||
// unclaimed lets the word-grid detector or ordinary paragraph assembly handle them.
|
||||
if (TableGrid.render(rows, cols, source).isBlank()) {
|
||||
return dbgNull(
|
||||
"guards rejected: rows="
|
||||
+ rows.size()
|
||||
+ " cols="
|
||||
+ (cols == null ? -1 : cols.size()));
|
||||
}
|
||||
return block;
|
||||
}
|
||||
|
||||
/** How near a rule end must be to a vertical rule to count as landing on it. */
|
||||
private static final float COLUMN_SNAP = 2.5f;
|
||||
|
||||
/** Fraction of the table's height a vertical must run to be a column boundary. */
|
||||
private static final float COLUMN_RUN = 0.5f;
|
||||
|
||||
/**
|
||||
* True when a rule narrower than the table is still a row boundary: it ends on the grid's
|
||||
* verticals and a spanning cell's text sits beside it.
|
||||
*/
|
||||
private static boolean spanningNeighbour(
|
||||
RuleGrid.Level rule,
|
||||
List<RuleGrid.Level> vL,
|
||||
List<Line> inside,
|
||||
float pos,
|
||||
float above,
|
||||
float height) {
|
||||
// The vertical must run the table, not merely be there: a line box inside a wrapped
|
||||
// cell draws its own short verticals at its inset edges.
|
||||
float columnRun = height * COLUMN_RUN;
|
||||
boolean loOnRule = false;
|
||||
boolean hiOnRule = false;
|
||||
for (RuleGrid.Level v : vL) {
|
||||
if (v.hi() - v.lo() < columnRun) {
|
||||
continue;
|
||||
}
|
||||
if (Math.abs(v.pos() - rule.lo()) <= COLUMN_SNAP) {
|
||||
loOnRule = true;
|
||||
}
|
||||
if (Math.abs(v.pos() - rule.hi()) <= COLUMN_SNAP) {
|
||||
hiOnRule = true;
|
||||
}
|
||||
}
|
||||
if (!loOnRule || !hiOnRule) {
|
||||
return false;
|
||||
}
|
||||
// The spanning cell's text must sit beside the rule anywhere in the row the last
|
||||
// full-width boundary opened: it is written once, at the top of the span.
|
||||
for (Line l : inside) {
|
||||
float cy = l.y + l.height / 2f;
|
||||
float cx = l.x + l.width / 2f;
|
||||
if (cy > pos && cy < above && (cx < rule.lo() || cx > rule.hi())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when every horizontal rule runs nearly the region's full width; legend swatches and
|
||||
* per-cell outlines do not.
|
||||
*/
|
||||
private static boolean fullWidthRules(List<RuleGrid.Level> hL, float left, float right) {
|
||||
float width = right - left;
|
||||
if (width <= 0f) {
|
||||
return false;
|
||||
}
|
||||
for (RuleGrid.Level h : hL) {
|
||||
if (h.hi() - h.lo() < width * FULL_WIDTH_RULE) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* A detected table. Each row is a list of source lines: usually one, but more when a cell wraps.
|
||||
*/
|
||||
record TableBlock(
|
||||
List<List<Line>> rows,
|
||||
float top,
|
||||
float bottom,
|
||||
List<float[]> cols,
|
||||
boolean ruled,
|
||||
RowSource rowSource,
|
||||
int page) {
|
||||
TableBlock(List<List<Line>> rows, float top, float bottom, int page) {
|
||||
this(rows, top, bottom, null, false, RowSource.WORDS, page);
|
||||
}
|
||||
|
||||
/** A rules-derived block whose rows are not a drawn lattice. */
|
||||
TableBlock(List<List<Line>> rows, float top, float bottom, List<float[]> cols, int page) {
|
||||
this(rows, top, bottom, cols, true, RowSource.RULE_BOUNDED, page);
|
||||
}
|
||||
|
||||
String render() {
|
||||
return TableGrid.render(rows, cols, rowSource);
|
||||
}
|
||||
|
||||
/** Cell grid for the layout guards; empty when the block fails the table guards. */
|
||||
List<String[]> cells() {
|
||||
return TableGrid.cells(rows, cols, rowSource);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Finds one page's table blocks and reconciles the two detectors: where both see a table, rows come
|
||||
* from the text and columns from the rules.
|
||||
*/
|
||||
final class TableFinder {
|
||||
|
||||
private TableFinder() {}
|
||||
|
||||
/**
|
||||
* Fraction of the word grid's rows a ruled grid must also find before its rows are trusted;
|
||||
* below it the rules would merge several rows into one band.
|
||||
*/
|
||||
private static final float COMPLETE_LATTICE = 0.5f;
|
||||
|
||||
/**
|
||||
* Detects a page's table blocks: ruled blocks first, then word-grid blocks over whatever lines
|
||||
* the rules did not claim.
|
||||
*/
|
||||
static List<TableBlock> find(List<Line> lines, PageRules rules, int page) {
|
||||
List<TableBlock> ruled = RuledTables.find(lines, rules, page);
|
||||
List<TableBlock> word = fromWordGrid(lines, page);
|
||||
if (ruled.isEmpty()) {
|
||||
return word;
|
||||
}
|
||||
|
||||
// Where both detectors see the same table, keep the word-grid's rows (read from the text)
|
||||
// but take the columns from the rules, which are exact where projection only guesses.
|
||||
List<TableBlock> all = new ArrayList<>();
|
||||
Set<TableBlock> usedRules = new HashSet<>();
|
||||
for (TableBlock w : word) {
|
||||
TableBlock match = null;
|
||||
for (TableBlock r : ruled) {
|
||||
// Only a grid with real column rules can improve on the word-grid; one ruled
|
||||
// across its rows alone contributes detection, never geometry.
|
||||
if (r.cols() != null && covers(w, r)) {
|
||||
match = r;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (match == null) {
|
||||
// No column rules, but a rules-only grid over the same lines still confirms that a
|
||||
// table is here. The word-grid's own reading of it stands, now rule-backed.
|
||||
TableBlock evidence = null;
|
||||
for (TableBlock r : ruled) {
|
||||
if (r.cols() == null && w.top() > r.bottom() && w.bottom() < r.top()) {
|
||||
evidence = r;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (evidence == null) {
|
||||
all.add(w);
|
||||
} else {
|
||||
usedRules.add(evidence);
|
||||
all.add(
|
||||
new TableBlock(
|
||||
w.rows(),
|
||||
w.top(),
|
||||
w.bottom(),
|
||||
null,
|
||||
true,
|
||||
RowSource.WORDS,
|
||||
w.page()));
|
||||
}
|
||||
} else if (match.rows().size() >= w.rows().size() * COMPLETE_LATTICE) {
|
||||
// The rules cover nearly every row, so take the whole grid from them; one ruled
|
||||
// grid can span several word-grid blocks, so emit it only once.
|
||||
if (usedRules.add(match)) {
|
||||
all.add(match);
|
||||
}
|
||||
} else {
|
||||
// Only some row boundaries are drawn: rows from the text, columns from the rules.
|
||||
usedRules.add(match);
|
||||
all.add(
|
||||
new TableBlock(
|
||||
w.rows(),
|
||||
w.top(),
|
||||
w.bottom(),
|
||||
match.cols(),
|
||||
true,
|
||||
RowSource.WORDS,
|
||||
w.page()));
|
||||
}
|
||||
}
|
||||
// A ruled table the word-grid never saw (single-word or wrapped cells leave it no wide gap
|
||||
// to anchor on) is emitted from its rules alone.
|
||||
for (TableBlock r : ruled) {
|
||||
if (usedRules.contains(r)) {
|
||||
continue;
|
||||
}
|
||||
boolean covered =
|
||||
all.stream().anyMatch(b -> b.top() > r.bottom() && b.bottom() < r.top());
|
||||
if (!covered) {
|
||||
all.add(r);
|
||||
}
|
||||
}
|
||||
all.sort(Comparator.comparingDouble(TableBlock::top).reversed());
|
||||
return all;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects table blocks from word geometry: anchor rows grouped into contiguous runs, with
|
||||
* non-anchor lines inside a run absorbed as wrapped cells.
|
||||
*/
|
||||
private static List<TableBlock> fromWordGrid(List<Line> lines, int page) {
|
||||
List<Line> cands =
|
||||
lines.stream()
|
||||
.filter(l -> !l.synthetic && isTableCandidate(l.words()))
|
||||
.sorted(Comparator.comparingDouble((Line l) -> l.y).reversed())
|
||||
.collect(Collectors.toList());
|
||||
if (cands.size() < 2) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
List<Float> gaps = new ArrayList<>();
|
||||
for (int i = 1; i < cands.size(); i++) {
|
||||
gaps.add(cands.get(i - 1).y - cands.get(i).y);
|
||||
}
|
||||
List<Float> sorted = new ArrayList<>(gaps);
|
||||
sorted.sort(Comparator.naturalOrder());
|
||||
float medianGap = sorted.get(sorted.size() / 2);
|
||||
float splitThreshold = Math.max(medianGap * 2.5f, medianGap + 6f);
|
||||
|
||||
List<List<Line>> anchorGroups = new ArrayList<>();
|
||||
List<Line> current = new ArrayList<>();
|
||||
current.add(cands.getFirst());
|
||||
for (int i = 1; i < cands.size(); i++) {
|
||||
float gap = cands.get(i - 1).y - cands.get(i).y;
|
||||
if (gap > splitThreshold) {
|
||||
anchorGroups.add(current);
|
||||
current = new ArrayList<>();
|
||||
}
|
||||
current.add(cands.get(i));
|
||||
}
|
||||
anchorGroups.add(current);
|
||||
|
||||
// Synthetic form values are kept out of the table path: they must not seed a column layout
|
||||
// or be absorbed as wrapped cells, as they were never in the content stream.
|
||||
List<Line> nonCandidates =
|
||||
lines.stream()
|
||||
.filter(l -> !l.synthetic && !isTableCandidate(l.words()))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
List<TableBlock> blocks = new ArrayList<>();
|
||||
for (List<Line> anchors : anchorGroups) {
|
||||
if (anchors.size() < 2) {
|
||||
continue;
|
||||
}
|
||||
float top = anchors.getFirst().y;
|
||||
float bottom = anchors.getLast().y;
|
||||
|
||||
// Each anchor seeds a row; absorb wrapped continuation lines (non-anchors within the
|
||||
// run's vertical span, with a little slack below the last row) into the anchor above.
|
||||
List<List<Line>> rows = new ArrayList<>();
|
||||
for (Line a : anchors) {
|
||||
List<Line> row = new ArrayList<>();
|
||||
row.add(a);
|
||||
rows.add(row);
|
||||
}
|
||||
for (Line nc : nonCandidates) {
|
||||
if (nc.y > top || nc.y < bottom - medianGap) {
|
||||
continue;
|
||||
}
|
||||
int owner = 0;
|
||||
float bestDelta = Float.MAX_VALUE;
|
||||
for (int i = 0; i < anchors.size(); i++) {
|
||||
float delta = anchors.get(i).y - nc.y; // positive when anchor is above nc
|
||||
if (delta >= -1f && delta < bestDelta) {
|
||||
bestDelta = delta;
|
||||
owner = i;
|
||||
}
|
||||
}
|
||||
rows.get(owner).add(nc);
|
||||
}
|
||||
|
||||
List<String[]> base = TableGrid.cells(rows, null, RowSource.WORDS);
|
||||
if (base.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
// A header row often has no wide gap between its cells, so the anchor test misses it.
|
||||
// The line above is kept only if its grid has the same shape, excluding captions.
|
||||
Line header = headerAbove(nonCandidates, top, medianGap);
|
||||
if (header != null) {
|
||||
List<List<Line>> withHeader = new ArrayList<>();
|
||||
withHeader.add(new ArrayList<>(List.of(header)));
|
||||
withHeader.addAll(rows);
|
||||
List<String[]> grown = TableGrid.cells(withHeader, null, RowSource.WORDS);
|
||||
if (!grown.isEmpty()
|
||||
&& grown.get(0).length == base.get(0).length
|
||||
&& TableGrid.filledCells(grown.get(0)) >= base.get(0).length) {
|
||||
rows = withHeader;
|
||||
top = header.y;
|
||||
}
|
||||
}
|
||||
blocks.add(new TableBlock(rows, top, bottom, page));
|
||||
}
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/** Vertical gaps, in median row gaps, within which a line above a block can be its header. */
|
||||
private static final float HEADER_GAP = 1.6f;
|
||||
|
||||
/**
|
||||
* Runs of words separated by more than a cell gutter: a header row has one per cell, a caption
|
||||
* written across the table is a single run.
|
||||
*/
|
||||
static int wordGroups(List<Line> row) {
|
||||
List<TextWord> words = new ArrayList<>();
|
||||
for (Line line : row) {
|
||||
for (TextWord w : line.words()) {
|
||||
if (!w.text().strip().isEmpty()) {
|
||||
words.add(w);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (words.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
words.sort(Comparator.comparingDouble(TextWord::x));
|
||||
float chars = 0;
|
||||
float width = 0;
|
||||
for (TextWord w : words) {
|
||||
width += w.width();
|
||||
chars += Math.max(1, w.text().strip().length());
|
||||
}
|
||||
float gutter =
|
||||
Math.max(
|
||||
ColumnRanges.RULED_GUTTER_FLOOR,
|
||||
(width / chars) * ColumnRanges.RULED_GUTTER_CHARS);
|
||||
int groups = 1;
|
||||
for (int i = 1; i < words.size(); i++) {
|
||||
float gap = words.get(i).x() - (words.get(i - 1).x() + words.get(i - 1).width());
|
||||
if (gap >= gutter) {
|
||||
groups++;
|
||||
}
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
/** Line heights within which a line above a ruled grid can be its header row. */
|
||||
static final float HEADER_RULE_GAP = 2.5f;
|
||||
|
||||
/** The nearest line above {@code top} close enough to be the block's header row. */
|
||||
private static Line headerAbove(List<Line> lines, float top, float medianGap) {
|
||||
Line best = null;
|
||||
for (Line l : lines) {
|
||||
if (l.y <= top || l.y - top > medianGap * HEADER_GAP || l.words().size() < 2) {
|
||||
continue;
|
||||
}
|
||||
if (best == null || l.y < best.y) {
|
||||
best = l;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a line has two words separated by a gap far wider than word spacing. The threshold
|
||||
* comes from the line's own character width, not a document font size.
|
||||
*/
|
||||
private static boolean isTableCandidate(List<TextWord> words) {
|
||||
if (words.size() < 2) {
|
||||
return false;
|
||||
}
|
||||
double totalWidth = 0;
|
||||
int totalChars = 0;
|
||||
for (TextWord w : words) {
|
||||
totalWidth += w.width();
|
||||
totalChars += Math.max(1, w.text().strip().length());
|
||||
}
|
||||
float charWidth = (float) (totalWidth / Math.max(1, totalChars));
|
||||
// A deliberate cell gap is several blank characters wide; ordinary word spaces are ~a third
|
||||
// of a character. Floor at 8pt so tiny fonts still need a real gap.
|
||||
float cellGap = Math.max(8f, charWidth * 3f);
|
||||
for (int i = 1; i < words.size(); i++) {
|
||||
TextWord prev = words.get(i - 1);
|
||||
float gap = words.get(i).x() - (prev.x() + prev.width());
|
||||
if (gap >= cellGap) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** True when two blocks overlap vertically, i.e. they describe the same table. */
|
||||
static boolean covers(TableBlock a, TableBlock b) {
|
||||
return Math.min(a.top(), b.top()) > Math.max(a.bottom(), b.bottom());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Resolves a detected table block into a cell grid and renders it. The false-positive guards live
|
||||
* here, so every consumer sees the same cells and the same verdict.
|
||||
*/
|
||||
final class TableGrid {
|
||||
|
||||
private TableGrid() {}
|
||||
|
||||
/**
|
||||
* Renders a table block; {@code ruledColumns} are exact bands from vertical rules, or null to
|
||||
* project the columns from whitespace instead.
|
||||
*/
|
||||
static String render(
|
||||
List<List<Line>> rowGroups, List<float[]> ruledColumns, RowSource rowSource) {
|
||||
List<String[]> rows = cells(rowGroups, ruledColumns, rowSource);
|
||||
return rows.isEmpty() ? "" : GfmTable.render(rows, rows.get(0).length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a table block into a cell grid, or empty when it fails the false-positive guards.
|
||||
*/
|
||||
static List<String[]> cells(
|
||||
List<List<Line>> rowGroups, List<float[]> ruledColumns, RowSource rowSource) {
|
||||
// Columns come from cross-row whitespace projection, not a 1-D gap threshold on pooled word
|
||||
// x's, which is fragile with right-aligned numbers or sparse cells in their own band.
|
||||
List<Line> flat = rowGroups.stream().flatMap(List::stream).collect(Collectors.toList());
|
||||
// Inside a region the rules already declare a table, a narrower gutter still separates
|
||||
// columns: the wide floor only exists to stop word spacing splitting an unruled block.
|
||||
List<float[]> columns =
|
||||
ruledColumns != null
|
||||
? ruledColumns
|
||||
: ColumnRanges.find(
|
||||
flat,
|
||||
rowSource.ruleConfirmed()
|
||||
? ColumnRanges.RULED_GUTTER_CHARS
|
||||
: ColumnRanges.GUTTER_CHARS,
|
||||
rowSource.ruleConfirmed()
|
||||
? ColumnRanges.RULED_GUTTER_FLOOR
|
||||
: ColumnRanges.GUTTER_FLOOR);
|
||||
// A column only the header occupies is invisible to the projection, which needs a band
|
||||
// shared by several rows; but inside a ruled region a blank answer column is still one.
|
||||
boolean headerOnlyColumn = false;
|
||||
if (columns.size() < 2 && ruledColumns == null && rowSource.ruleConfirmed()) {
|
||||
List<float[]> retry =
|
||||
ColumnRanges.find(
|
||||
flat,
|
||||
ColumnRanges.RULED_GUTTER_CHARS,
|
||||
ColumnRanges.RULED_GUTTER_FLOOR,
|
||||
1);
|
||||
// Only the worksheet shape: exactly one row, the first, reaches past the supported
|
||||
// column. Anything else would invent a column and swallow the headings around it.
|
||||
if (retry.size() >= 2 && retry.size() <= 15) {
|
||||
float edge = retry.get(0)[1];
|
||||
int beyond = 0;
|
||||
int firstBeyond = -1;
|
||||
for (int r = 0; r < rowGroups.size(); r++) {
|
||||
boolean out = false;
|
||||
for (Line l : rowGroups.get(r)) {
|
||||
for (TextWord w : l.words()) {
|
||||
if (!w.text().strip().isEmpty() && w.x() + w.width() / 2f > edge) {
|
||||
out = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (out) {
|
||||
beyond++;
|
||||
if (firstBeyond < 0) {
|
||||
firstBeyond = r;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (beyond == 1 && firstBeyond == 0 && rowGroups.size() >= 3) {
|
||||
columns = retry;
|
||||
headerOnlyColumn = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Only a drawn lattice can be a one-column table; inferred from whitespace it is just a
|
||||
// run of centred lines.
|
||||
int minColumns = rowSource == RowSource.LATTICE ? 1 : 2;
|
||||
if (columns.size() < minColumns || columns.size() > 15) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
float[] centers = new float[columns.size()];
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
centers[i] = (columns.get(i)[0] + columns.get(i)[1]) / 2f;
|
||||
}
|
||||
|
||||
int cols = centers.length;
|
||||
List<String[]> rows = new ArrayList<>();
|
||||
for (List<Line> rowLines : rowGroups) {
|
||||
String[] row = new String[cols];
|
||||
TextWord[] lastWord = new TextWord[cols];
|
||||
String[] lastText = new String[cols];
|
||||
boolean[] boundMark = new boolean[cols];
|
||||
for (int i = 0; i < cols; i++) {
|
||||
row[i] = "";
|
||||
lastText[i] = "";
|
||||
}
|
||||
// Top line first so a wrapped cell's words stay in reading order within the cell.
|
||||
rowLines.sort(Comparator.comparingDouble((Line l) -> l.y).reversed());
|
||||
for (Line line : rowLines) {
|
||||
for (TextWord word : line.words()) {
|
||||
String wt = word.text().strip();
|
||||
if (wt.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
float mid = word.x() + word.width() / 2f;
|
||||
// Ruled columns are real boundaries, so a word belongs to the band that
|
||||
// contains it; projected columns are only approximate centres, so nearest wins.
|
||||
int col =
|
||||
ruledColumns != null
|
||||
? containingColumn(mid, columns)
|
||||
: nearestColumn(mid, centers);
|
||||
// A mark that closed up against the word on its left closes up against the
|
||||
// word on its right too, so Party - List does not settle at "Party- List".
|
||||
boolean bind =
|
||||
!row[col].isEmpty()
|
||||
&& (boundMark[col]
|
||||
|| (WordGeometry.isBindingMark(wt)
|
||||
|| WordGeometry.isBindingMark(
|
||||
lastText[col]))
|
||||
&& !WordGeometry.separated(
|
||||
lastWord[col], word));
|
||||
row[col] = row[col].isEmpty() ? wt : row[col] + (bind ? "" : " ") + wt;
|
||||
boundMark[col] = bind && WordGeometry.isBindingMark(wt);
|
||||
lastWord[col] = word;
|
||||
lastText[col] = wt;
|
||||
}
|
||||
}
|
||||
for (int c = 0; c < cols; c++) {
|
||||
row[c] = WordGeometry.rejoinContractions(row[c]);
|
||||
}
|
||||
rows.add(row);
|
||||
}
|
||||
|
||||
// Guard against false positives while tolerating uneven rows: require an anchor row that
|
||||
// nearly fills the grid, and that most rows are genuinely multi-column.
|
||||
if (ruledColumns != null) {
|
||||
// A rule that is not a column separator (a cell outline, a shading edge) leaves an
|
||||
// empty column; drop those rather than emitting them across every row.
|
||||
List<Integer> keep = new ArrayList<>();
|
||||
for (int c = 0; c < cols; c++) {
|
||||
final int col = c;
|
||||
if (rows.stream().anyMatch(r -> !r[col].isEmpty())) {
|
||||
keep.add(c);
|
||||
}
|
||||
}
|
||||
// Two is the floor whatever the rows say: one filled column means the rules drew a box
|
||||
// round a single block of text, not a table.
|
||||
if (keep.size() < 2) {
|
||||
return List.of();
|
||||
}
|
||||
if (keep.size() < cols) {
|
||||
List<String[]> trimmed = new ArrayList<>(rows.size());
|
||||
for (String[] r : rows) {
|
||||
String[] t = new String[keep.size()];
|
||||
for (int i = 0; i < keep.size(); i++) {
|
||||
t[i] = r[keep.get(i)];
|
||||
}
|
||||
trimmed.add(t);
|
||||
}
|
||||
rows = trimmed;
|
||||
cols = keep.size();
|
||||
List<float[]> kept = new ArrayList<>(keep.size());
|
||||
for (int idx : keep) {
|
||||
kept.add(columns.get(idx));
|
||||
}
|
||||
columns = kept;
|
||||
}
|
||||
}
|
||||
|
||||
if (cols == 1) {
|
||||
// A one-column table has no cross-row alignment to check, so the evidence is the rules
|
||||
// plus the shape of the run: enough rows, nearly all carrying text.
|
||||
long filled = rows.stream().filter(r -> !r[0].isEmpty()).count();
|
||||
return rows.size() >= SINGLE_COLUMN_ROWS && filled >= rows.size() * SINGLE_COLUMN_FILLED
|
||||
? rows
|
||||
: List.of();
|
||||
}
|
||||
|
||||
int anchorWidth = Math.max(2, Math.round(cols * 0.6f));
|
||||
long anchorRows = rows.stream().filter(r -> filledCells(r) >= anchorWidth).count();
|
||||
long multiColumnRows = rows.stream().filter(r -> filledCells(r) >= 2).count();
|
||||
// The multi-column tests ask whether a grid inferred from whitespace is real; when rows
|
||||
// and columns are both drawn there is nothing to infer, and a blank worksheet would fail.
|
||||
boolean drawnGrid =
|
||||
headerOnlyColumn || (ruledColumns != null && rowSource == RowSource.LATTICE);
|
||||
if (drawnGrid
|
||||
? anchorRows < 1
|
||||
: (anchorRows < 1 || multiColumnRows < 2 || multiColumnRows < rows.size() * 0.5)) {
|
||||
return List.of();
|
||||
}
|
||||
if (ruledColumns == null && TableShape.isProseNotTable(rows, cols)) {
|
||||
return List.of();
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** Rows a single-column ruled table needs before it is a table rather than a run of lines. */
|
||||
private static final int SINGLE_COLUMN_ROWS = 3;
|
||||
|
||||
/** Fraction of a single-column table's rows that must carry text. */
|
||||
private static final float SINGLE_COLUMN_FILLED = 0.8f;
|
||||
|
||||
/** Index of the column band containing x, clamped to the first/last band outside the grid. */
|
||||
static int containingColumn(float x, List<float[]> columns) {
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
if (x < columns.get(i)[1]) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return columns.size() - 1;
|
||||
}
|
||||
|
||||
private static int nearestColumn(float x, float[] centers) {
|
||||
int best = 0;
|
||||
float bestDist = Float.MAX_VALUE;
|
||||
for (int i = 0; i < centers.length; i++) {
|
||||
float d = Math.abs(x - centers[i]);
|
||||
if (d < bestDist) {
|
||||
bestDist = d;
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
static int filledCells(String[] row) {
|
||||
int count = 0;
|
||||
for (String cell : row) {
|
||||
if (!cell.isEmpty()) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* False-positive guards: whether a block is really a table, and whether it is wide enough to
|
||||
* outrank the page's own column layout.
|
||||
*/
|
||||
final class TableShape {
|
||||
|
||||
private TableShape() {}
|
||||
|
||||
/** Fraction of the page's text width a table must span to override two-column layout. */
|
||||
private static final float FULL_WIDTH = 0.6f;
|
||||
|
||||
/** Rows of a two-column block that must end in a page number for it to be a contents list. */
|
||||
private static final float TOC_ROWS = 0.65f;
|
||||
|
||||
/** Mean filled-cell length above which a two-column block reads as prose, not cells. */
|
||||
private static final float PROSE_CELL = 40f;
|
||||
|
||||
private static final Pattern PAGE_NUMBER = Pattern.compile("[0-9]{1,4}|[ivxlcdmIVXLCDM]{1,7}");
|
||||
|
||||
/** A run of spaced or solid dots, the leader of a contents line. */
|
||||
private static final Pattern DOT_LEADER = Pattern.compile("(\\.\\s*){4,}|…");
|
||||
|
||||
/**
|
||||
* True when a block is running text the word grid mistook for a table: a contents list, or two
|
||||
* columns of prose whose cells are whole sentences.
|
||||
*/
|
||||
static boolean isProseNotTable(List<String[]> rows, int cols) {
|
||||
if (rows.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
for (String[] row : rows) {
|
||||
for (String cell : row) {
|
||||
if (DOT_LEADER.matcher(cell).find()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (cols != 2) {
|
||||
return everyColumnIsProse(rows, cols);
|
||||
}
|
||||
int folios = 0;
|
||||
int length = 0;
|
||||
int filled = 0;
|
||||
for (String[] row : rows) {
|
||||
String last = "";
|
||||
for (String cell : row) {
|
||||
if (!cell.isEmpty()) {
|
||||
length += cell.length();
|
||||
filled++;
|
||||
last = cell;
|
||||
}
|
||||
}
|
||||
if (PAGE_NUMBER.matcher(last).matches() && !PAGE_NUMBER.matcher(row[0]).matches()) {
|
||||
folios++;
|
||||
}
|
||||
}
|
||||
if (folios >= rows.size() * TOC_ROWS) {
|
||||
return true;
|
||||
}
|
||||
return filled > 0 && (float) length / filled >= PROSE_CELL;
|
||||
}
|
||||
|
||||
/** Mean cell length at or above which a column carries sentences rather than values. */
|
||||
private static final float PROSE_COLUMN = 20f;
|
||||
|
||||
/** Fraction of neighbouring cells that must continue each other's sentence to read as prose. */
|
||||
private static final float PROSE_RUN_ON = 0.5f;
|
||||
|
||||
/** A cell that ends a sentence or clause, so the cell after it starts something new. */
|
||||
private static final Pattern CELL_ENDS_CLAUSE = Pattern.compile("[.!?:;,]$");
|
||||
|
||||
/**
|
||||
* True when a wide block is multi-column prose read across, not a table: no column keys the
|
||||
* rows, and the cells continue each other's sentences.
|
||||
*/
|
||||
static boolean everyColumnIsProse(List<String[]> rows, int cols) {
|
||||
if (cols < 3) {
|
||||
return false;
|
||||
}
|
||||
for (int c = 0; c < cols; c++) {
|
||||
int length = 0;
|
||||
int filled = 0;
|
||||
for (String[] row : rows) {
|
||||
if (c < row.length && !row[c].isEmpty()) {
|
||||
length += row[c].length();
|
||||
filled++;
|
||||
}
|
||||
}
|
||||
if (filled == 0 || (float) length / filled < PROSE_COLUMN) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return runsOnAcrossCells(rows);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fraction of side-by-side filled cells where the right one continues the left one's clause.
|
||||
*/
|
||||
private static boolean runsOnAcrossCells(List<String[]> rows) {
|
||||
int pairs = 0;
|
||||
int runOn = 0;
|
||||
for (String[] row : rows) {
|
||||
String previous = null;
|
||||
for (String cell : row) {
|
||||
if (cell.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (previous != null) {
|
||||
pairs++;
|
||||
if (!CELL_ENDS_CLAUSE.matcher(previous).find()
|
||||
&& Character.isLowerCase(cell.charAt(0))) {
|
||||
runOn++;
|
||||
}
|
||||
}
|
||||
previous = cell;
|
||||
}
|
||||
}
|
||||
return pairs > 0 && (float) runOn / pairs > PROSE_RUN_ON;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when no text outside the block sits in its vertical band, so it cannot be one column of
|
||||
* a two-column layout.
|
||||
*/
|
||||
static boolean ownsItsBand(TableBlock block, List<Line> lines) {
|
||||
Set<Line> own = new HashSet<>();
|
||||
for (List<Line> row : block.rows()) {
|
||||
own.addAll(row);
|
||||
}
|
||||
for (Line l : lines) {
|
||||
if (own.contains(l)) {
|
||||
continue;
|
||||
}
|
||||
float centre = l.y + l.height / 2f;
|
||||
if (centre > block.bottom() && centre < block.top()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Columns a full-width unruled block needs before it can outrank the page's column layout. */
|
||||
private static final int GRID_COLUMNS = 3;
|
||||
|
||||
/** Mean filled-cell length above which a full-width unruled block is prose read across. */
|
||||
private static final float GRID_CELL = 25f;
|
||||
|
||||
/**
|
||||
* True when an unruled full-width block is really a table: a data table's cells are short
|
||||
* values, a page gutter's are sentences.
|
||||
*/
|
||||
static boolean looksLikeGrid(TableBlock block) {
|
||||
List<String[]> cells = block.cells();
|
||||
if (cells.isEmpty() || cells.get(0).length < GRID_COLUMNS) {
|
||||
return false;
|
||||
}
|
||||
int length = 0;
|
||||
int filled = 0;
|
||||
for (String[] row : cells) {
|
||||
for (String cell : row) {
|
||||
if (!cell.isEmpty()) {
|
||||
length += cell.length();
|
||||
filled++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return filled > 0 && (float) length / filled <= GRID_CELL;
|
||||
}
|
||||
|
||||
/** True when a table block is wide enough to be a full-width table, not one inside a column. */
|
||||
static boolean spansPage(TableBlock block, List<Line> lines) {
|
||||
float pageLo = Float.MAX_VALUE;
|
||||
float pageHi = -Float.MAX_VALUE;
|
||||
for (Line l : lines) {
|
||||
pageLo = Math.min(pageLo, l.x);
|
||||
pageHi = Math.max(pageHi, l.x + l.width);
|
||||
}
|
||||
float lo = Float.MAX_VALUE;
|
||||
float hi = -Float.MAX_VALUE;
|
||||
for (List<Line> row : block.rows()) {
|
||||
for (Line l : row) {
|
||||
lo = Math.min(lo, l.x);
|
||||
hi = Math.max(hi, l.x + l.width);
|
||||
}
|
||||
}
|
||||
return pageHi > pageLo && (hi - lo) >= (pageHi - pageLo) * FULL_WIDTH;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Word-level geometry and spacing. PDFium splits words on its own bounding boxes, so both real
|
||||
* spaces and edges are re-derived from the glyphs.
|
||||
*/
|
||||
final class WordGeometry {
|
||||
|
||||
private WordGeometry() {}
|
||||
|
||||
/** Gap below this many average character widths reads as no space at all (mid-word split). */
|
||||
static final float NO_SPACE_GAP = 0.30f;
|
||||
|
||||
/**
|
||||
* Punctuation that binds to the words on both sides. Closing words up is only considered around
|
||||
* one of these, because dropping a real space corrupts the text.
|
||||
*/
|
||||
private static final String BINDING_MARKS = "'’ʼ´`-‐‑";
|
||||
|
||||
/** True for a lone apostrophe or hyphen, as in {@code firm}, {@code '}, {@code s}. */
|
||||
static boolean isBindingMark(String word) {
|
||||
return word.length() == 1 && BINDING_MARKS.indexOf(word.charAt(0)) >= 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* A contraction whose apostrophe the extractor padded on both sides. English suffixes only: a
|
||||
* spaced lone apostrophe is an opening quote.
|
||||
*/
|
||||
private static final Pattern SPLIT_CONTRACTION =
|
||||
Pattern.compile("(\\p{L})\\s*([’'ʼ´`])\\s*(s|t|d|m|re|ve|ll)\\b");
|
||||
|
||||
/** Closes up an apostrophe the extractor left standing alone inside a cell. */
|
||||
static String rejoinContractions(String cell) {
|
||||
return cell.indexOf(' ') < 0 ? cell : SPLIT_CONTRACTION.matcher(cell).replaceAll("$1$2$3");
|
||||
}
|
||||
|
||||
/**
|
||||
* True when two words of a cell are far enough apart to be separated by a space; punctuation
|
||||
* set tight against its neighbour arrives as its own word.
|
||||
*/
|
||||
static boolean separated(TextWord previous, TextWord current) {
|
||||
if (previous == null) {
|
||||
return true;
|
||||
}
|
||||
float gap = leftEdge(current) - rightEdge(previous);
|
||||
if (gap < 0f) {
|
||||
// Overlapping or out of order (a second line of a wrapped cell): keep the space.
|
||||
return true;
|
||||
}
|
||||
float charWidth = wordCharWidth(previous, current);
|
||||
return charWidth <= 0f || gap >= charWidth * NO_SPACE_GAP;
|
||||
}
|
||||
|
||||
/** Mean glyph width across two words, used to size the space test above. */
|
||||
private static float wordCharWidth(TextWord a, TextWord b) {
|
||||
float width = 0f;
|
||||
int chars = 0;
|
||||
for (TextWord w : List.of(a, b)) {
|
||||
for (TextChar c : w.chars()) {
|
||||
if (!c.isWhitespace() && !c.isNewline()) {
|
||||
width += c.width();
|
||||
chars++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return chars == 0 ? 0f : width / chars;
|
||||
}
|
||||
|
||||
static float rightEdge(TextWord w) {
|
||||
float edge = -Float.MAX_VALUE;
|
||||
for (TextChar c : w.chars()) {
|
||||
if (!c.isWhitespace() && !c.isNewline()) {
|
||||
edge = Math.max(edge, c.x() + c.width());
|
||||
}
|
||||
}
|
||||
return edge == -Float.MAX_VALUE ? w.x() + w.width() : edge;
|
||||
}
|
||||
|
||||
static float leftEdge(TextWord w) {
|
||||
float edge = Float.MAX_VALUE;
|
||||
for (TextChar c : w.chars()) {
|
||||
if (!c.isWhitespace() && !c.isNewline()) {
|
||||
edge = Math.min(edge, c.x());
|
||||
}
|
||||
}
|
||||
return edge == Float.MAX_VALUE ? w.x() : edge;
|
||||
}
|
||||
|
||||
static float averageCharWidth(List<Line> rows) {
|
||||
double totalWidth = 0;
|
||||
int totalChars = 0;
|
||||
for (Line l : rows) {
|
||||
for (TextWord w : l.words()) {
|
||||
totalWidth += w.width();
|
||||
totalChars += Math.max(1, w.text().strip().length());
|
||||
}
|
||||
}
|
||||
return totalChars == 0 ? 6f : (float) (totalWidth / totalChars);
|
||||
}
|
||||
}
|
||||
+17
-15
@@ -70,21 +70,23 @@ public class CustomLogoutSuccessHandler extends SimpleUrlLogoutSuccessHandler {
|
||||
|
||||
if (!response.isCommitted()) {
|
||||
if (authentication != null) {
|
||||
if (authentication instanceof Saml2Authentication samlAuthentication) {
|
||||
// Handle SAML2 logout redirection
|
||||
getRedirect_saml2(request, response, samlAuthentication);
|
||||
} else if (authentication instanceof OAuth2AuthenticationToken oAuthToken) {
|
||||
// Handle OAuth2 logout redirection
|
||||
getRedirect_oauth2(request, response, oAuthToken);
|
||||
} else if (authentication instanceof UsernamePasswordAuthenticationToken) {
|
||||
// Handle Username/Password logout
|
||||
getRedirectStrategy().sendRedirect(request, response, LOGOUT_PATH);
|
||||
} else {
|
||||
// Handle unknown authentication types
|
||||
log.error(
|
||||
"Authentication class unknown: {}",
|
||||
authentication.getClass().getSimpleName());
|
||||
getRedirectStrategy().sendRedirect(request, response, LOGOUT_PATH);
|
||||
switch (authentication) {
|
||||
case Saml2Authentication samlAuthentication ->
|
||||
// Handle SAML2 logout redirection
|
||||
getRedirect_saml2(request, response, samlAuthentication);
|
||||
case OAuth2AuthenticationToken oAuthToken ->
|
||||
// Handle OAuth2 logout redirection
|
||||
getRedirect_oauth2(request, response, oAuthToken);
|
||||
case UsernamePasswordAuthenticationToken usernamePasswordAuthenticationToken ->
|
||||
// Handle Username/Password logout
|
||||
getRedirectStrategy().sendRedirect(request, response, LOGOUT_PATH);
|
||||
default -> {
|
||||
// Handle unknown authentication types
|
||||
log.error(
|
||||
"Authentication class unknown: {}",
|
||||
authentication.getClass().getSimpleName());
|
||||
getRedirectStrategy().sendRedirect(request, response, LOGOUT_PATH);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (jwtService != null) {
|
||||
|
||||
+34
-34
@@ -392,40 +392,40 @@ public class SecurityConfiguration {
|
||||
// Handle OAUTH2 Logins
|
||||
if (securityProperties.isOauth2Active()) {
|
||||
http.oauth2Login(
|
||||
oauth2 -> {
|
||||
oauth2.loginPage("/login")
|
||||
.authorizationEndpoint(
|
||||
authorizationEndpoint -> {
|
||||
if (clientRegistrationRepository != null) {
|
||||
authorizationEndpoint
|
||||
.authorizationRequestResolver(
|
||||
new TauriAuthorizationRequestResolver(
|
||||
clientRegistrationRepository));
|
||||
}
|
||||
})
|
||||
.successHandler(
|
||||
new CustomOAuth2AuthenticationSuccessHandler(
|
||||
loginAttemptService,
|
||||
securityProperties.getOauth2(),
|
||||
userService,
|
||||
jwtService,
|
||||
licenseSettingsService,
|
||||
applicationProperties))
|
||||
.failureHandler(new CustomOAuth2AuthenticationFailureHandler())
|
||||
// Add existing Authorities from the database
|
||||
.userInfoEndpoint(
|
||||
userInfoEndpoint ->
|
||||
userInfoEndpoint
|
||||
.oidcUserService(
|
||||
new CustomOAuth2UserService(
|
||||
securityProperties
|
||||
.getOauth2(),
|
||||
userService,
|
||||
loginAttemptService))
|
||||
.userAuthoritiesMapper(
|
||||
oAuth2userAuthoritiesMapper))
|
||||
.permitAll();
|
||||
});
|
||||
oauth2 ->
|
||||
oauth2.loginPage("/login")
|
||||
.authorizationEndpoint(
|
||||
authorizationEndpoint -> {
|
||||
if (clientRegistrationRepository != null) {
|
||||
authorizationEndpoint
|
||||
.authorizationRequestResolver(
|
||||
new TauriAuthorizationRequestResolver(
|
||||
clientRegistrationRepository));
|
||||
}
|
||||
})
|
||||
.successHandler(
|
||||
new CustomOAuth2AuthenticationSuccessHandler(
|
||||
loginAttemptService,
|
||||
securityProperties.getOauth2(),
|
||||
userService,
|
||||
jwtService,
|
||||
licenseSettingsService,
|
||||
applicationProperties))
|
||||
.failureHandler(
|
||||
new CustomOAuth2AuthenticationFailureHandler())
|
||||
// Add existing Authorities from the database
|
||||
.userInfoEndpoint(
|
||||
userInfoEndpoint ->
|
||||
userInfoEndpoint
|
||||
.oidcUserService(
|
||||
new CustomOAuth2UserService(
|
||||
securityProperties
|
||||
.getOauth2(),
|
||||
userService,
|
||||
loginAttemptService))
|
||||
.userAuthoritiesMapper(
|
||||
oAuth2userAuthoritiesMapper))
|
||||
.permitAll());
|
||||
}
|
||||
// Handle SAML
|
||||
if (securityProperties.isSaml2Active() && runningProOrHigher) {
|
||||
|
||||
+12
-11
@@ -703,17 +703,18 @@ public class AuthController {
|
||||
}
|
||||
|
||||
private long extractEpochMillis(Object claimValue) {
|
||||
if (claimValue == null) {
|
||||
return -1L;
|
||||
}
|
||||
|
||||
if (claimValue instanceof java.util.Date date) {
|
||||
return date.getTime();
|
||||
}
|
||||
|
||||
if (claimValue instanceof Number number) {
|
||||
long epochSeconds = number.longValue();
|
||||
return epochSeconds * 1000L;
|
||||
switch (claimValue) {
|
||||
case null -> {
|
||||
return -1L;
|
||||
}
|
||||
case java.util.Date date -> {
|
||||
return date.getTime();
|
||||
}
|
||||
case Number number -> {
|
||||
long epochSeconds = number.longValue();
|
||||
return epochSeconds * 1000L;
|
||||
}
|
||||
default -> {}
|
||||
}
|
||||
|
||||
return -1L;
|
||||
|
||||
+8
-8
@@ -760,14 +760,14 @@ public class UserController {
|
||||
for (Object principal : principals) {
|
||||
List<SessionInformation> sessionsInformation =
|
||||
sessionRegistry.getAllSessions(principal, false);
|
||||
if (principal instanceof UserDetails detailsUser) {
|
||||
userNameP = detailsUser.getUsername();
|
||||
} else if (principal instanceof OAuth2User oAuth2User) {
|
||||
userNameP = oAuth2User.getName();
|
||||
} else if (principal instanceof CustomSaml2AuthenticatedPrincipal saml2User) {
|
||||
userNameP = saml2User.name();
|
||||
} else if (principal instanceof String stringUser) {
|
||||
userNameP = stringUser;
|
||||
switch (principal) {
|
||||
case null -> {}
|
||||
case UserDetails detailsUser -> userNameP = detailsUser.getUsername();
|
||||
case OAuth2User oAuth2User -> userNameP = oAuth2User.getName();
|
||||
case CustomSaml2AuthenticatedPrincipal saml2User ->
|
||||
userNameP = saml2User.name();
|
||||
case String stringUser -> userNameP = stringUser;
|
||||
default -> {}
|
||||
}
|
||||
if (userNameP.equalsIgnoreCase(username)) {
|
||||
for (SessionInformation sessionInfo : sessionsInformation) {
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package stirling.software.proprietary.security.model;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
|
||||
import org.springframework.security.core.GrantedAuthority;
|
||||
@@ -28,7 +29,7 @@ import lombok.Setter;
|
||||
@Setter
|
||||
public class Authority implements GrantedAuthority, Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
@Serial private static final long serialVersionUID = 1L;
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package stirling.software.proprietary.security.model;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@@ -18,7 +19,7 @@ import lombok.Setter;
|
||||
@Setter
|
||||
public class InviteToken implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
@Serial private static final long serialVersionUID = 1L;
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
|
||||
+52
-47
@@ -36,57 +36,62 @@ public class CustomOAuth2AuthenticationFailureHandler
|
||||
AuthenticationException exception)
|
||||
throws IOException, ServletException {
|
||||
|
||||
if (exception instanceof BadCredentialsException) {
|
||||
log.error("BadCredentialsException", exception);
|
||||
getRedirectStrategy().sendRedirect(request, response, "/login?error=badCredentials");
|
||||
return;
|
||||
}
|
||||
if (exception instanceof DisabledException) {
|
||||
log.error("User is deactivated: ", exception);
|
||||
getRedirectStrategy().sendRedirect(request, response, "/logout?userIsDisabled=true");
|
||||
return;
|
||||
}
|
||||
if (exception instanceof LockedException) {
|
||||
log.error("Account locked: ", exception);
|
||||
getRedirectStrategy().sendRedirect(request, response, "/logout?error=locked");
|
||||
return;
|
||||
}
|
||||
if (exception instanceof OAuth2AuthenticationException oAuth2Exception) {
|
||||
OAuth2Error error = oAuth2Exception.getError();
|
||||
|
||||
String errorCode = error.getErrorCode();
|
||||
|
||||
if ("Password must not be null".equals(error.getErrorCode())) {
|
||||
errorCode = "userAlreadyExistsWeb";
|
||||
switch (exception) {
|
||||
case BadCredentialsException badCredentialsException -> {
|
||||
log.error("BadCredentialsException", exception);
|
||||
getRedirectStrategy()
|
||||
.sendRedirect(request, response, "/login?error=badCredentials");
|
||||
return;
|
||||
}
|
||||
case DisabledException disabledException -> {
|
||||
log.error("User is deactivated: ", exception);
|
||||
getRedirectStrategy()
|
||||
.sendRedirect(request, response, "/logout?userIsDisabled=true");
|
||||
return;
|
||||
}
|
||||
case LockedException lockedException -> {
|
||||
log.error("Account locked: ", exception);
|
||||
getRedirectStrategy().sendRedirect(request, response, "/logout?error=locked");
|
||||
return;
|
||||
}
|
||||
case OAuth2AuthenticationException oAuth2Exception -> {
|
||||
OAuth2Error error = oAuth2Exception.getError();
|
||||
|
||||
log.error(
|
||||
"OAuth2 Authentication error: {}",
|
||||
errorCode != null ? errorCode : exception.getMessage(),
|
||||
exception);
|
||||
String errorValue = errorCode != null ? errorCode : "oauth2AuthenticationError";
|
||||
clearRedirectCookie(response);
|
||||
boolean tauriState = TauriOAuthUtils.isTauriState(request);
|
||||
String redirectUrl;
|
||||
if (tauriState) {
|
||||
String basePath =
|
||||
TauriOAuthUtils.defaultTauriCallbackPath(request.getContextPath());
|
||||
redirectUrl = basePath;
|
||||
String stateParam = request.getParameter("state");
|
||||
if (stateParam != null && !stateParam.isBlank()) {
|
||||
redirectUrl = appendQueryParam(redirectUrl, "state", stateParam);
|
||||
// Extract and pass nonce for CSRF validation
|
||||
String nonce = TauriOAuthUtils.extractNonceFromState(stateParam);
|
||||
if (nonce != null) {
|
||||
redirectUrl = appendQueryParam(redirectUrl, "nonce", nonce);
|
||||
}
|
||||
String errorCode = error.getErrorCode();
|
||||
|
||||
if ("Password must not be null".equals(error.getErrorCode())) {
|
||||
errorCode = "userAlreadyExistsWeb";
|
||||
}
|
||||
redirectUrl = appendQueryParam(redirectUrl, "errorOAuth", errorValue);
|
||||
} else {
|
||||
redirectUrl = buildFailureRedirectUrl(request, errorValue);
|
||||
|
||||
log.error(
|
||||
"OAuth2 Authentication error: {}",
|
||||
errorCode != null ? errorCode : exception.getMessage(),
|
||||
exception);
|
||||
String errorValue = errorCode != null ? errorCode : "oauth2AuthenticationError";
|
||||
clearRedirectCookie(response);
|
||||
boolean tauriState = TauriOAuthUtils.isTauriState(request);
|
||||
String redirectUrl;
|
||||
if (tauriState) {
|
||||
String basePath =
|
||||
TauriOAuthUtils.defaultTauriCallbackPath(request.getContextPath());
|
||||
redirectUrl = basePath;
|
||||
String stateParam = request.getParameter("state");
|
||||
if (stateParam != null && !stateParam.isBlank()) {
|
||||
redirectUrl = appendQueryParam(redirectUrl, "state", stateParam);
|
||||
// Extract and pass nonce for CSRF validation
|
||||
String nonce = TauriOAuthUtils.extractNonceFromState(stateParam);
|
||||
if (nonce != null) {
|
||||
redirectUrl = appendQueryParam(redirectUrl, "nonce", nonce);
|
||||
}
|
||||
}
|
||||
redirectUrl = appendQueryParam(redirectUrl, "errorOAuth", errorValue);
|
||||
} else {
|
||||
redirectUrl = buildFailureRedirectUrl(request, errorValue);
|
||||
}
|
||||
getRedirectStrategy().sendRedirect(request, response, redirectUrl);
|
||||
return;
|
||||
}
|
||||
getRedirectStrategy().sendRedirect(request, response, redirectUrl);
|
||||
return;
|
||||
default -> {}
|
||||
}
|
||||
log.error("Unhandled authentication exception", exception);
|
||||
super.onAuthenticationFailure(request, response, exception);
|
||||
|
||||
+6
-1
@@ -61,7 +61,12 @@ public class CustomSaml2ResponseAuthenticationConverter
|
||||
|
||||
@Override
|
||||
public Saml2Authentication convert(ResponseToken responseToken) {
|
||||
Assertion assertion = responseToken.getResponse().getAssertions().getFirst();
|
||||
List<Assertion> assertions = responseToken.getResponse().getAssertions();
|
||||
if (assertions == null || assertions.isEmpty()) {
|
||||
log.error("SAML response contains no assertions");
|
||||
return null;
|
||||
}
|
||||
Assertion assertion = assertions.getFirst();
|
||||
Map<String, List<Object>> attributes = extractAttributes(assertion);
|
||||
|
||||
// Debug log with actual values
|
||||
|
||||
+5
-2
@@ -213,8 +213,11 @@ public class CustomOAuth2UserService implements OAuth2UserService<OidcUserReques
|
||||
}
|
||||
|
||||
sb.append(
|
||||
"\nWARNING: this block contains PII. Set security.oauth2.debugLogging=false once"
|
||||
+ " troubleshooting is complete.\n");
|
||||
"""
|
||||
|
||||
WARNING: this block contains PII. Set security.oauth2.debugLogging=false once\
|
||||
troubleshooting is complete.
|
||||
""");
|
||||
sb.append("========== [/OAUTH2 DEBUG] ==========");
|
||||
|
||||
if (failure) {
|
||||
|
||||
+3
-1
@@ -132,7 +132,9 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
|
||||
verifyingKeyCache.put(
|
||||
key.getKeyId(), new JwtVerificationKey(key.getKeyId(), key.getVerifyingKey()));
|
||||
}
|
||||
activeKey = new JwtVerificationKey(keys.get(0).getKeyId(), keys.get(0).getVerifyingKey());
|
||||
activeKey =
|
||||
new JwtVerificationKey(
|
||||
keys.getFirst().getKeyId(), keys.getFirst().getVerifyingKey());
|
||||
log.info("Loaded {} JWT key(s) from DB, active key: {}", keys.size(), activeKey.getKeyId());
|
||||
}
|
||||
|
||||
|
||||
+8
-8
@@ -640,14 +640,14 @@ public class UserService implements UserServiceInterface {
|
||||
for (Object principal : sessionRegistry.getAllPrincipals()) {
|
||||
for (SessionInformation sessionsInformation :
|
||||
sessionRegistry.getAllSessions(principal, false)) {
|
||||
if (principal instanceof UserDetails detailsUser) {
|
||||
usernameP = detailsUser.getUsername();
|
||||
} else if (principal instanceof OAuth2User oAuth2User) {
|
||||
usernameP = oAuth2User.getName();
|
||||
} else if (principal instanceof CustomSaml2AuthenticatedPrincipal saml2User) {
|
||||
usernameP = saml2User.name();
|
||||
} else if (principal instanceof String stringUser) {
|
||||
usernameP = stringUser;
|
||||
switch (principal) {
|
||||
case null -> {}
|
||||
case UserDetails detailsUser -> usernameP = detailsUser.getUsername();
|
||||
case OAuth2User oAuth2User -> usernameP = oAuth2User.getName();
|
||||
case CustomSaml2AuthenticatedPrincipal saml2User ->
|
||||
usernameP = saml2User.name();
|
||||
case String stringUser -> usernameP = stringUser;
|
||||
default -> {}
|
||||
}
|
||||
if (usernameP.equalsIgnoreCase(username)) {
|
||||
sessionRegistry.expireSession(sessionsInformation.getSessionId());
|
||||
|
||||
+14
-16
@@ -47,14 +47,13 @@ public class SessionPersistentRegistry implements SessionRegistry {
|
||||
List<SessionInformation> sessionInformations = new ArrayList<>();
|
||||
String principalName = null;
|
||||
|
||||
if (principal instanceof UserDetails detailsUser) {
|
||||
principalName = detailsUser.getUsername();
|
||||
} else if (principal instanceof OAuth2User oAuth2User) {
|
||||
principalName = oAuth2User.getName();
|
||||
} else if (principal instanceof CustomSaml2AuthenticatedPrincipal saml2User) {
|
||||
principalName = saml2User.name();
|
||||
} else if (principal instanceof String stringUser) {
|
||||
principalName = stringUser;
|
||||
switch (principal) {
|
||||
case null -> {}
|
||||
case UserDetails detailsUser -> principalName = detailsUser.getUsername();
|
||||
case OAuth2User oAuth2User -> principalName = oAuth2User.getName();
|
||||
case CustomSaml2AuthenticatedPrincipal saml2User -> principalName = saml2User.name();
|
||||
case String stringUser -> principalName = stringUser;
|
||||
default -> {}
|
||||
}
|
||||
|
||||
if (principalName != null) {
|
||||
@@ -78,14 +77,13 @@ public class SessionPersistentRegistry implements SessionRegistry {
|
||||
public void registerNewSession(String sessionId, Object principal) {
|
||||
String principalName = null;
|
||||
|
||||
if (principal instanceof UserDetails detailsUser) {
|
||||
principalName = detailsUser.getUsername();
|
||||
} else if (principal instanceof OAuth2User oAuth2User) {
|
||||
principalName = oAuth2User.getName();
|
||||
} else if (principal instanceof CustomSaml2AuthenticatedPrincipal saml2User) {
|
||||
principalName = saml2User.name();
|
||||
} else if (principal instanceof String stringUser) {
|
||||
principalName = stringUser;
|
||||
switch (principal) {
|
||||
case null -> {}
|
||||
case UserDetails detailsUser -> principalName = detailsUser.getUsername();
|
||||
case OAuth2User oAuth2User -> principalName = oAuth2User.getName();
|
||||
case CustomSaml2AuthenticatedPrincipal saml2User -> principalName = saml2User.name();
|
||||
case String stringUser -> principalName = stringUser;
|
||||
default -> {}
|
||||
}
|
||||
|
||||
if (principalName != null) {
|
||||
|
||||
+8
-8
@@ -3,16 +3,16 @@ package stirling.software.proprietary.storage.converter;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import jakarta.persistence.AttributeConverter;
|
||||
import jakarta.persistence.Converter;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.core.JacksonException;
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* JPA AttributeConverter for storing Map<String, Object> as JSON in database columns.
|
||||
*
|
||||
@@ -33,7 +33,7 @@ public class JsonMapConverter implements AttributeConverter<Map<String, Object>,
|
||||
|
||||
try {
|
||||
return objectMapper.writeValueAsString(attribute);
|
||||
} catch (JsonProcessingException e) {
|
||||
} catch (JacksonException e) {
|
||||
log.error("Failed to convert map to JSON", e);
|
||||
throw new RuntimeException("Failed to convert map to JSON", e);
|
||||
}
|
||||
@@ -48,7 +48,7 @@ public class JsonMapConverter implements AttributeConverter<Map<String, Object>,
|
||||
try {
|
||||
// Try normal parsing first
|
||||
return objectMapper.readValue(dbData, new TypeReference<Map<String, Object>>() {});
|
||||
} catch (JsonProcessingException e) {
|
||||
} catch (JacksonException e) {
|
||||
// Fallback: try double-parsing for legacy double-encoded data
|
||||
// This handles data that was stored as JSON strings instead of JSON objects
|
||||
log.debug("Attempting double-decode fallback for legacy metadata format");
|
||||
@@ -69,7 +69,7 @@ public class JsonMapConverter implements AttributeConverter<Map<String, Object>,
|
||||
return objectMapper.readValue(
|
||||
node.asText(), new TypeReference<Map<String, Object>>() {});
|
||||
}
|
||||
} catch (JsonProcessingException e2) {
|
||||
} catch (JacksonException e2) {
|
||||
log.error("Failed to parse metadata even with double-decode fallback", e2);
|
||||
}
|
||||
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package stirling.software.proprietary.storage.model;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@@ -46,7 +47,7 @@ import stirling.software.proprietary.security.model.User;
|
||||
@Setter
|
||||
public class FileShare implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
@Serial private static final long serialVersionUID = 1L;
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package stirling.software.proprietary.storage.model;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@@ -39,7 +40,7 @@ import stirling.software.proprietary.security.model.User;
|
||||
@Setter
|
||||
public class FileShareAccess implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
@Serial private static final long serialVersionUID = 1L;
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package stirling.software.proprietary.storage.model;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@@ -24,7 +25,7 @@ import lombok.Setter;
|
||||
@Setter
|
||||
public class StorageCleanupEntry implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
@Serial private static final long serialVersionUID = 1L;
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package stirling.software.proprietary.storage.model;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.HashSet;
|
||||
@@ -45,7 +46,7 @@ import stirling.software.proprietary.workflow.model.WorkflowSession;
|
||||
@Setter
|
||||
public class StoredFile implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
@Serial private static final long serialVersionUID = 1L;
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package stirling.software.proprietary.storage.model;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
@@ -19,7 +20,7 @@ import lombok.Setter;
|
||||
@Setter
|
||||
public class StoredFileBlob implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
@Serial private static final long serialVersionUID = 1L;
|
||||
|
||||
@Id
|
||||
@Column(name = "storage_key", nullable = false, length = 128)
|
||||
|
||||
@@ -7,6 +7,7 @@ import org.slf4j.MDC;
|
||||
import org.springframework.core.Ordered;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.security.core.Authentication;
|
||||
import org.springframework.security.core.GrantedAuthority;
|
||||
import org.springframework.security.core.context.SecurityContextHolder;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.filter.OncePerRequestFilter;
|
||||
@@ -64,7 +65,7 @@ public class AuditWebFilter extends OncePerRequestFilter {
|
||||
if (auth != null && auth.getAuthorities() != null) {
|
||||
String roles =
|
||||
auth.getAuthorities().stream()
|
||||
.map(a -> a.getAuthority())
|
||||
.map(GrantedAuthority::getAuthority)
|
||||
.reduce((a, b) -> a + "," + b)
|
||||
.orElse("");
|
||||
MDC.put("userRoles", roles);
|
||||
|
||||
+6
-3
@@ -20,8 +20,6 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
@@ -39,11 +37,14 @@ import stirling.software.proprietary.workflow.dto.CertificateInfo;
|
||||
import stirling.software.proprietary.workflow.dto.CertificateValidationResponse;
|
||||
import stirling.software.proprietary.workflow.dto.ParticipantRequest;
|
||||
import stirling.software.proprietary.workflow.dto.WorkflowCreationRequest;
|
||||
import stirling.software.proprietary.workflow.model.WorkflowParticipant;
|
||||
import stirling.software.proprietary.workflow.model.WorkflowSession;
|
||||
import stirling.software.proprietary.workflow.service.CertificateSubmissionValidator;
|
||||
import stirling.software.proprietary.workflow.service.SigningFinalizationService;
|
||||
import stirling.software.proprietary.workflow.service.WorkflowSessionService;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/security")
|
||||
@@ -259,7 +260,9 @@ public class SigningSessionController {
|
||||
+ "database until manual cleanup.",
|
||||
sessionId,
|
||||
session.getParticipants() != null
|
||||
? session.getParticipants().stream().map(p -> p.getEmail()).toList()
|
||||
? session.getParticipants().stream()
|
||||
.map(WorkflowParticipant::getEmail)
|
||||
.toList()
|
||||
: "unknown",
|
||||
e);
|
||||
throw new ResponseStatusException(
|
||||
|
||||
+2
-1
@@ -5,6 +5,7 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.http.ContentDisposition;
|
||||
@@ -429,7 +430,7 @@ public class WorkflowParticipantController {
|
||||
java.util.List<Map<String, Object>> wetSigs =
|
||||
objectMapper.readValue(
|
||||
request.getWetSignaturesData(),
|
||||
new TypeReference<java.util.List<Map<String, Object>>>() {});
|
||||
new TypeReference<List<Map<String, Object>>>() {});
|
||||
if (wetSigs.size() > WetSignatureMetadata.MAX_SIGNATURES_PER_PARTICIPANT) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST, "Too many wet signatures submitted");
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package stirling.software.proprietary.workflow.model;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
@@ -51,7 +52,7 @@ import stirling.software.proprietary.storage.model.ShareAccessRole;
|
||||
@Setter
|
||||
public class WorkflowParticipant implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
@Serial private static final long serialVersionUID = 1L;
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package stirling.software.proprietary.workflow.model;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
@@ -53,7 +54,7 @@ import stirling.software.proprietary.storage.model.StoredFile;
|
||||
@Setter
|
||||
public class WorkflowSession implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
@Serial private static final long serialVersionUID = 1L;
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
|
||||
+4
-10
@@ -217,16 +217,13 @@ public class SigningFinalizationService {
|
||||
wetSignatures.size(),
|
||||
session.getSessionId());
|
||||
|
||||
PDDocument document = pdfDocumentFactory.load(new ByteArrayInputStream(pdfBytes));
|
||||
try {
|
||||
try (PDDocument document = pdfDocumentFactory.load(new ByteArrayInputStream(pdfBytes))) {
|
||||
for (WetSignatureMetadata wetSig : wetSignatures) {
|
||||
applyWetSignatureToPage(document, wetSig);
|
||||
}
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
document.save(baos);
|
||||
return baos.toByteArray();
|
||||
} finally {
|
||||
document.close();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -242,11 +239,10 @@ public class SigningFinalizationService {
|
||||
}
|
||||
|
||||
PDPage page = document.getPage(pageIndex);
|
||||
PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true);
|
||||
|
||||
try {
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
// Use WetSignatureMetadata.extractBase64Data() to strip data URL prefix
|
||||
String base64Data = wetSig.extractBase64Data();
|
||||
if (base64Data == null || base64Data.isBlank()) {
|
||||
@@ -279,8 +275,6 @@ public class SigningFinalizationService {
|
||||
pdfY,
|
||||
width,
|
||||
height);
|
||||
} finally {
|
||||
contentStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+16
-15
@@ -954,21 +954,22 @@ public class WorkflowSessionService {
|
||||
Object pemObject = pemParser.readObject();
|
||||
JcaPEMKeyConverter converter = new JcaPEMKeyConverter().setProvider("BC");
|
||||
PrivateKeyInfo keyInfo;
|
||||
if (pemObject instanceof PKCS8EncryptedPrivateKeyInfo encrypted) {
|
||||
InputDecryptorProvider decryptor =
|
||||
new JceOpenSSLPKCS8DecryptorProviderBuilder().build(password);
|
||||
keyInfo = encrypted.decryptPrivateKeyInfo(decryptor);
|
||||
} else if (pemObject instanceof PEMEncryptedKeyPair encryptedKeyPair) {
|
||||
PEMDecryptorProvider decryptor =
|
||||
new JcePEMDecryptorProviderBuilder().build(password);
|
||||
keyInfo = encryptedKeyPair.decryptKeyPair(decryptor).getPrivateKeyInfo();
|
||||
} else if (pemObject instanceof PEMKeyPair keyPair) {
|
||||
keyInfo = keyPair.getPrivateKeyInfo();
|
||||
} else if (pemObject instanceof PrivateKeyInfo info) {
|
||||
keyInfo = info;
|
||||
} else {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST, "Unsupported PEM private key format");
|
||||
switch (pemObject) {
|
||||
case PKCS8EncryptedPrivateKeyInfo encrypted -> {
|
||||
InputDecryptorProvider decryptor =
|
||||
new JceOpenSSLPKCS8DecryptorProviderBuilder().build(password);
|
||||
keyInfo = encrypted.decryptPrivateKeyInfo(decryptor);
|
||||
}
|
||||
case PEMEncryptedKeyPair encryptedKeyPair -> {
|
||||
PEMDecryptorProvider decryptor =
|
||||
new JcePEMDecryptorProviderBuilder().build(password);
|
||||
keyInfo = encryptedKeyPair.decryptKeyPair(decryptor).getPrivateKeyInfo();
|
||||
}
|
||||
case PEMKeyPair keyPair -> keyInfo = keyPair.getPrivateKeyInfo();
|
||||
case PrivateKeyInfo info -> keyInfo = info;
|
||||
case null, default ->
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST, "Unsupported PEM private key format");
|
||||
}
|
||||
return converter.getPrivateKey(keyInfo);
|
||||
}
|
||||
|
||||
+2
-2
@@ -4,14 +4,14 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import stirling.software.proprietary.workflow.dto.ParticipantResponse;
|
||||
import stirling.software.proprietary.workflow.dto.WetSignatureMetadata;
|
||||
import stirling.software.proprietary.workflow.dto.WorkflowSessionResponse;
|
||||
import stirling.software.proprietary.workflow.model.WorkflowParticipant;
|
||||
import stirling.software.proprietary.workflow.model.WorkflowSession;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Utility class for mapping workflow entities to DTOs. Centralizes conversion logic for consistent
|
||||
* API responses.
|
||||
|
||||
+423
@@ -0,0 +1,423 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.Timeout;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.text.TextChar;
|
||||
import stirling.software.jpdfium.text.TextLine;
|
||||
import stirling.software.jpdfium.text.TextWord;
|
||||
|
||||
/**
|
||||
* Accuracy and robustness tests comparing output against hand-authored golden Markdown. {@link
|
||||
* #gatedFixtures()} gates CI; {@link #wipFixtures()} is disabled.
|
||||
*/
|
||||
class AdvancedPdfMarkdownConverterTest {
|
||||
|
||||
/** Accuracy threshold: output must share at least this fraction of content with the golden. */
|
||||
private static final double THRESHOLD = 0.95;
|
||||
|
||||
@TempDir Path tmp;
|
||||
|
||||
/** Fixtures that meet the accuracy threshold today and therefore gate CI. */
|
||||
static Stream<Arguments> gatedFixtures() {
|
||||
return Stream.of(
|
||||
Arguments.of("multi-column-test_lorem.pdf", "multi-column-test_lorem.md"),
|
||||
Arguments.of("bordered-table-test_widget.pdf", "bordered-table-test_widget.md"),
|
||||
Arguments.of("many-tables-test_stress.pdf", "many-tables-test_stress.md"));
|
||||
}
|
||||
|
||||
/** Fixtures still below the threshold; tracked here, enable locally to iterate. */
|
||||
static Stream<Arguments> wipFixtures() {
|
||||
return Stream.of(
|
||||
Arguments.of(
|
||||
"wrapped-cell-test_expense-report.pdf",
|
||||
"wrapped-cell-test_expense-report.md"));
|
||||
}
|
||||
|
||||
@ParameterizedTest(name = "{0}")
|
||||
@MethodSource("gatedFixtures")
|
||||
void convertMatchesGoldenMarkdown(String pdfName, String mdName) throws IOException {
|
||||
assertConversionMatchesGolden(pdfName, mdName);
|
||||
}
|
||||
|
||||
@Disabled("WIP fixtures below the accuracy threshold; enable locally to iterate")
|
||||
@ParameterizedTest(name = "{0}")
|
||||
@MethodSource("wipFixtures")
|
||||
void convertMatchesGoldenMarkdownWip(String pdfName, String mdName) throws IOException {
|
||||
assertConversionMatchesGolden(pdfName, mdName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Degenerate geometry must not crash the converter: a text matrix can place a word past {@link
|
||||
* Integer#MAX_VALUE}, which used to size an {@code int[]} from the span.
|
||||
*/
|
||||
@Test
|
||||
void columnDetectionSurvivesDegenerateGeometry() {
|
||||
// x ≈ 2.5e9 is past Integer.MAX_VALUE; combined with a near-origin word it yields an
|
||||
// implausible span that the pre-fix code turned into a fatal array allocation.
|
||||
List<TextLine> rows = new ArrayList<>();
|
||||
for (int r = 0; r < 4; r++) {
|
||||
float y = 400f - r * 12f;
|
||||
TextWord near = new TextWord(List.of(), 50f, y, 30f, 10f);
|
||||
TextWord far = new TextWord(List.of(), 2_500_000_000f, y, 30f, 10f);
|
||||
rows.add(new TextLine(List.of(near, far), 50f, y, 2_499_999_980f, 10f));
|
||||
}
|
||||
|
||||
List<float[]> columns = assertDoesNotThrow(() -> ColumnRanges.fromTextLines(rows));
|
||||
assertTrue(
|
||||
columns.isEmpty(),
|
||||
"implausible page span should disable column detection, not allocate from it");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Timeout(20)
|
||||
void gutterScanTerminatesOnCoordinatesBeyondFloatPrecision() {
|
||||
// Past 2^24 a float cannot represent x + 1, so a float-stepped scan over a crafted text
|
||||
// matrix stops advancing and spins forever - wedging the process-wide jpdfium lock with it.
|
||||
List<TextLine> rows = new ArrayList<>();
|
||||
for (int r = 0; r < 10; r++) {
|
||||
float y = 400f - r * 12f;
|
||||
float x = 20_000_000f;
|
||||
TextWord w = new TextWord(List.of(), x, y, 200f, 10f);
|
||||
rows.add(new TextLine(List.of(w), x, y, 200f, 10f));
|
||||
}
|
||||
|
||||
List<Float> gutters = assertDoesNotThrow(() -> ColumnLayout.guttersFromTextLines(rows));
|
||||
assertTrue(
|
||||
gutters.isEmpty(),
|
||||
"every candidate band is crossed by every line, so no gutter is found");
|
||||
}
|
||||
|
||||
/**
|
||||
* Three-column prose aligns across rows exactly as cells do. What tells them apart is a keying
|
||||
* column of short values and cells that do not run on.
|
||||
*/
|
||||
@Test
|
||||
void multiColumnProseIsNotATable() {
|
||||
List<String[]> prose =
|
||||
List.of(
|
||||
new String[] {
|
||||
"The SS Pack can reduce the information acquisition time by",
|
||||
"returning all the information that matches",
|
||||
"the user's search intent and the query behind it"
|
||||
},
|
||||
new String[] {
|
||||
"Unlike existing search systems that only return information",
|
||||
"limited to the entered search keywords, this pack",
|
||||
"returns all relevant data meeting the search intent"
|
||||
});
|
||||
assertTrue(
|
||||
TableShape.everyColumnIsProse(prose, 3),
|
||||
"three columns of running sentences are a page layout, not a table");
|
||||
}
|
||||
|
||||
@Test
|
||||
void wideTableWithLongCellsStaysATable() {
|
||||
// The prose test must not fire on a real table just because one column runs long: the
|
||||
// short "Jurisdiction" and yes/no columns are what key the rows.
|
||||
List<String[]> table =
|
||||
List.of(
|
||||
new String[] {
|
||||
"Argentina",
|
||||
"Y",
|
||||
"Prohibition on ownership of property that contains or borders water"
|
||||
},
|
||||
new String[] {
|
||||
"Australia",
|
||||
"N",
|
||||
"Approval is needed from the Treasurer if the acquisition is large"
|
||||
});
|
||||
assertTrue(!TableShape.everyColumnIsProse(table, 3), "a keyed table is a table");
|
||||
}
|
||||
|
||||
@Test
|
||||
void splitApostropheIsClosedUpInCells() {
|
||||
// PDFium splits on its own bounding boxes, so a tight apostrophe arrives as its own word.
|
||||
assertEquals("the firm's returns", WordGeometry.rejoinContractions("the firm ' s returns"));
|
||||
assertEquals("Don’t know", WordGeometry.rejoinContractions("Don ’ t know"));
|
||||
// An opening quote has real space around it and must keep it.
|
||||
assertEquals("he said ' hello", WordGeometry.rejoinContractions("he said ' hello"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void headingLevelsAreRebasedOnTheStrongestHeadingPresent() {
|
||||
// A document whose headings are body-size and bold scores every one of them level 3;
|
||||
// relative to each other they are its top level, so they must render as level 1.
|
||||
assertEquals("# CONTENTS\n", MarkdownText.normaliseHeadingLevels("### CONTENTS\n"));
|
||||
// A real two-level document keeps two levels, with no gap between them.
|
||||
assertEquals(
|
||||
"# Title\n\ntext\n\n## Section\n",
|
||||
MarkdownText.normaliseHeadingLevels("# Title\n\ntext\n\n### Section\n"));
|
||||
// Already rooted at level 1 with no gaps: left alone.
|
||||
String unchanged = "# Title\n\n## Section\n";
|
||||
assertEquals(unchanged, MarkdownText.normaliseHeadingLevels(unchanged));
|
||||
}
|
||||
|
||||
/** A line of Han text has no spaces, so the word-count heading guard cannot measure it. */
|
||||
private static List<TextWord> cjkWords(String text, String font) {
|
||||
List<TextChar> chars = new ArrayList<>(text.length());
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
chars.add(new TextChar(i, text.charAt(i), 50f + i * 12f, 400f, 12f, 12f, font, 12f));
|
||||
}
|
||||
return List.of(new TextWord(chars, 50f, 400f, text.length() * 12f, 12f));
|
||||
}
|
||||
|
||||
@Test
|
||||
void boldCjkParagraphIsNotPromotedToAHeading() {
|
||||
String paragraph =
|
||||
"\u672c\u898f\u7d04\u306f\u3001\u5f53\u793e\u304c\u63d0\u4f9b\u3059\u308b"
|
||||
+ "\u672c\u30b5\u30fc\u30d3\u30b9\u306e\u5229\u7528\u6761\u4ef6\u3092"
|
||||
+ "\u5b9a\u3081\u308b\u3082\u306e\u3067\u3042\u308a\u3001\u5229\u7528"
|
||||
+ "\u8005\u306e\u7686\u3055\u307e\u306b\u306f\u672c\u898f\u7d04\u306b"
|
||||
+ "\u5f93\u3063\u3066\u3054\u5229\u7528\u3044\u305f\u3060\u304d\u307e\u3059\u3002";
|
||||
List<TextWord> words = cjkWords(paragraph, "NotoSansCJKjp-Bold");
|
||||
assertEquals(
|
||||
"",
|
||||
HeadingDetector.headingPrefix(
|
||||
paragraph, 12f, words, 12f, 12f, "NotoSansCJKjp-Regular", true),
|
||||
"a bold paragraph in a script with no word spaces is body text, not a heading");
|
||||
assertFalse(
|
||||
HeadingDetector.isBoldLabel(paragraph, words),
|
||||
"a paragraph ending in an ideographic stop is a sentence, not a bold label");
|
||||
|
||||
// The guard must not cost the short headings it is meant to keep.
|
||||
String heading = "\u7b2c\u4e09\u7ae0 \u5b9f\u88c5\u306e\u6982\u8981";
|
||||
assertEquals(
|
||||
"### ",
|
||||
HeadingDetector.headingPrefix(
|
||||
heading,
|
||||
12f,
|
||||
cjkWords(heading, "NotoSansCJKjp-Bold"),
|
||||
12f,
|
||||
12f,
|
||||
"NotoSansCJKjp-Regular",
|
||||
true),
|
||||
"a short isolated bold CJK line is still a heading");
|
||||
}
|
||||
|
||||
/**
|
||||
* A crafted PDF can draw thousands of disjoint rules; partitioning used to cost O(N^2) retained
|
||||
* memory, so it must stay linear and bounded.
|
||||
*/
|
||||
@Test
|
||||
@Timeout(20)
|
||||
void ruledTablePartitionSurvivesPathologicalGrid() {
|
||||
// 4000 rules that never cross, so every one is its own component: the shape that made the
|
||||
// old code allocate 4000 arrays of 4001 ints, kept under the crossing-test budget.
|
||||
List<PageRules.Rule> horizontal = new ArrayList<>();
|
||||
List<PageRules.Rule> vertical = new ArrayList<>();
|
||||
for (int i = 0; i < 2_000; i++) {
|
||||
horizontal.add(new PageRules.Rule(i * 10f, 0f, 20f));
|
||||
vertical.add(new PageRules.Rule(1_000_000f + i * 10f, -50f, -30f));
|
||||
}
|
||||
|
||||
int components = assertDoesNotThrow(() -> RuleGrid.componentCount(horizontal, vertical));
|
||||
// 4000 disjoint rules would be 4000 components; the cap is what keeps this bounded.
|
||||
assertEquals(256, components, "component count must stay bounded");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Timeout(20)
|
||||
void ruledTablePartitionBailsOutOnOperatorFlood() {
|
||||
// Enough levels that the pairwise crossing scan alone would dominate the request.
|
||||
List<PageRules.Rule> horizontal = new ArrayList<>();
|
||||
List<PageRules.Rule> vertical = new ArrayList<>();
|
||||
for (int i = 0; i < 20_000; i++) {
|
||||
horizontal.add(new PageRules.Rule(i * 10f, 0f, 20f));
|
||||
vertical.add(new PageRules.Rule(1_000_000f + i * 10f, -50f, -30f));
|
||||
}
|
||||
|
||||
assertTrue(
|
||||
RuleGrid.componentCount(horizontal, vertical) == 0,
|
||||
"a rule flood should disable ruled-table detection, not scan it");
|
||||
}
|
||||
|
||||
private void assertConversionMatchesGolden(String pdfName, String mdName) throws IOException {
|
||||
Path pdfPath = tmp.resolve(pdfName);
|
||||
try (InputStream in =
|
||||
getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + pdfName)) {
|
||||
if (in == null) {
|
||||
fail("Fixture not found on classpath: /pdf-ingestion-fixtures/" + pdfName);
|
||||
}
|
||||
Files.copy(in, pdfPath);
|
||||
}
|
||||
|
||||
String actual;
|
||||
try (PdfDocument doc = PdfDocument.open(pdfPath)) {
|
||||
actual = new AdvancedPdfMarkdownConverter().convert(doc);
|
||||
}
|
||||
|
||||
String expected;
|
||||
try (InputStream in = getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + mdName)) {
|
||||
if (in == null) {
|
||||
fail("Golden file not found on classpath: /pdf-ingestion-fixtures/" + mdName);
|
||||
}
|
||||
expected = new String(in.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
// Image placeholders are not scored: their body text is a TODO rather than real content, so
|
||||
// comparing it would penalise output for matching a placeholder we intend to replace.
|
||||
expected = stripImagePlaceholders(expected);
|
||||
actual = stripImagePlaceholders(actual);
|
||||
|
||||
double similarity = similarity(expected, actual);
|
||||
if (similarity < THRESHOLD) {
|
||||
fail(
|
||||
String.format(
|
||||
"Markdown output differs from golden file '%s' by %.1f%% (threshold %.0f%%):%n%s",
|
||||
mdName,
|
||||
(1.0 - similarity) * 100,
|
||||
(1.0 - THRESHOLD) * 100,
|
||||
unifiedDiff(expected, actual)));
|
||||
}
|
||||
}
|
||||
|
||||
/** Substring identifying an image-placeholder line, which is excluded from scoring. */
|
||||
private static final String IMAGE_PLACEHOLDER_MARKER = "Image intentionally redacted";
|
||||
|
||||
/**
|
||||
* Removes non-content lines: image placeholders, and GFM separator rows whose exact dash count
|
||||
* is cosmetic.
|
||||
*/
|
||||
private static String stripImagePlaceholders(String md) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (String line : md.split("\n", -1)) {
|
||||
if (line.contains(IMAGE_PLACEHOLDER_MARKER)
|
||||
|| line.strip().startsWith("<image redacted")
|
||||
|| isTableSeparatorRow(line)) {
|
||||
continue;
|
||||
}
|
||||
if (!sb.isEmpty()) {
|
||||
sb.append('\n');
|
||||
}
|
||||
sb.append(line);
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/** True for a GFM table separator row, e.g. {@code |---|:--:|---|} (only |, -, :, space). */
|
||||
private static boolean isTableSeparatorRow(String line) {
|
||||
String t = line.strip();
|
||||
if (!t.contains("-")) {
|
||||
return false;
|
||||
}
|
||||
return t.chars().allMatch(c -> c == '|' || c == '-' || c == ':' || c == ' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Character-level similarity: the fraction of expected characters in the LCS. O(n*m), fine for
|
||||
* small goldens.
|
||||
*/
|
||||
private static double similarity(String expected, String actual) {
|
||||
if (expected.isEmpty() && actual.isEmpty()) return 1.0;
|
||||
if (expected.isEmpty() || actual.isEmpty()) return 0.0;
|
||||
// Strip all whitespace for a content-focused comparison
|
||||
String e = expected.replaceAll("\\s+", " ").strip();
|
||||
String a = actual.replaceAll("\\s+", " ").strip();
|
||||
int lcs = lcsLength(e, a);
|
||||
return (double) lcs / Math.max(e.length(), a.length());
|
||||
}
|
||||
|
||||
private static int lcsLength(String a, String b) {
|
||||
// Use two-row DP to keep memory reasonable
|
||||
int m = a.length(), n = b.length();
|
||||
int[] prev = new int[n + 1];
|
||||
int[] curr = new int[n + 1];
|
||||
for (int i = 1; i <= m; i++) {
|
||||
for (int j = 1; j <= n; j++) {
|
||||
if (a.charAt(i - 1) == b.charAt(j - 1)) {
|
||||
curr[j] = prev[j - 1] + 1;
|
||||
} else {
|
||||
curr[j] = Math.max(curr[j - 1], prev[j]);
|
||||
}
|
||||
}
|
||||
int[] tmp = prev;
|
||||
prev = curr;
|
||||
curr = tmp;
|
||||
java.util.Arrays.fill(curr, 0);
|
||||
}
|
||||
return prev[n];
|
||||
}
|
||||
|
||||
private static String unifiedDiff(String expected, String actual) {
|
||||
String[] expectedLines = expected.split("\n", -1);
|
||||
String[] actualLines = actual.split("\n", -1);
|
||||
|
||||
List<String> diff = new ArrayList<>();
|
||||
diff.add("--- expected");
|
||||
diff.add("+++ actual");
|
||||
|
||||
int maxLines = Math.max(expectedLines.length, actualLines.length);
|
||||
int context = 3;
|
||||
boolean inHunk = false;
|
||||
int hunkStart = -1;
|
||||
List<String> hunkLines = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < maxLines; i++) {
|
||||
String exp = i < expectedLines.length ? expectedLines[i] : null;
|
||||
String act = i < actualLines.length ? actualLines[i] : null;
|
||||
|
||||
boolean changed = exp == null || act == null || !exp.equals(act);
|
||||
if (changed) {
|
||||
if (!inHunk) {
|
||||
inHunk = true;
|
||||
hunkStart = Math.max(0, i - context);
|
||||
// add context lines before change
|
||||
for (int c = hunkStart; c < i; c++) {
|
||||
hunkLines.add(" " + (c < expectedLines.length ? expectedLines[c] : ""));
|
||||
}
|
||||
}
|
||||
if (exp != null) hunkLines.add("-" + exp);
|
||||
if (act != null) hunkLines.add("+" + act);
|
||||
} else {
|
||||
if (inHunk) {
|
||||
hunkLines.add(" " + exp);
|
||||
// check if we're far enough past the last change to close the hunk
|
||||
boolean moreChanges = false;
|
||||
for (int j = i + 1; j < Math.min(i + context, maxLines); j++) {
|
||||
String e2 = j < expectedLines.length ? expectedLines[j] : null;
|
||||
String a2 = j < actualLines.length ? actualLines[j] : null;
|
||||
if (e2 == null || a2 == null || !e2.equals(a2)) {
|
||||
moreChanges = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!moreChanges && (i - hunkStart) >= context) {
|
||||
diff.add("@@ -" + (hunkStart + 1) + " @@");
|
||||
diff.addAll(hunkLines);
|
||||
hunkLines.clear();
|
||||
inHunk = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (inHunk && !hunkLines.isEmpty()) {
|
||||
diff.add("@@ -" + (hunkStart + 1) + " @@");
|
||||
diff.addAll(hunkLines);
|
||||
}
|
||||
|
||||
return String.join("\n", diff);
|
||||
}
|
||||
}
|
||||
@@ -26,8 +26,8 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.core.JacksonException;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
@@ -167,7 +167,7 @@ public class AiCreateController {
|
||||
if (request.constraints() != null) {
|
||||
try {
|
||||
constraintsPayload = objectMapper.writeValueAsString(request.constraints());
|
||||
} catch (JsonProcessingException exc) {
|
||||
} catch (JacksonException exc) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST, "Invalid constraints payload", exc);
|
||||
}
|
||||
@@ -202,7 +202,7 @@ public class AiCreateController {
|
||||
String payload;
|
||||
try {
|
||||
payload = objectMapper.writeValueAsString(request.draftSections());
|
||||
} catch (JsonProcessingException exc) {
|
||||
} catch (JacksonException exc) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST, "Invalid draft sections payload", exc);
|
||||
}
|
||||
@@ -392,7 +392,7 @@ public class AiCreateController {
|
||||
objectMapper
|
||||
.getTypeFactory()
|
||||
.constructCollectionType(List.class, DraftSection.class));
|
||||
} catch (JsonProcessingException exc) {
|
||||
} catch (JacksonException exc) {
|
||||
log.warn("Failed to parse draft sections payload", exc);
|
||||
return null;
|
||||
}
|
||||
@@ -408,7 +408,7 @@ public class AiCreateController {
|
||||
objectMapper
|
||||
.getTypeFactory()
|
||||
.constructMapType(Map.class, String.class, Object.class));
|
||||
} catch (JsonProcessingException exc) {
|
||||
} catch (JacksonException exc) {
|
||||
log.warn("Failed to parse outline constraints payload", exc);
|
||||
return null;
|
||||
}
|
||||
|
||||
+6
-6
@@ -14,8 +14,8 @@ import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.core.JacksonException;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
@@ -61,7 +61,7 @@ public class AiCreateInternalController {
|
||||
try {
|
||||
outlineConstraintsPayload =
|
||||
objectMapper.writeValueAsString(request.outlineConstraints());
|
||||
} catch (JsonProcessingException exc) {
|
||||
} catch (JacksonException exc) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST, "Invalid outline constraints payload", exc);
|
||||
}
|
||||
@@ -70,7 +70,7 @@ public class AiCreateInternalController {
|
||||
if (request.draftSections() != null) {
|
||||
try {
|
||||
draftSectionsPayload = objectMapper.writeValueAsString(request.draftSections());
|
||||
} catch (JsonProcessingException exc) {
|
||||
} catch (JacksonException exc) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST, "Invalid draft sections payload", exc);
|
||||
}
|
||||
@@ -136,7 +136,7 @@ public class AiCreateInternalController {
|
||||
.getTypeFactory()
|
||||
.constructCollectionType(
|
||||
List.class, AiCreateController.DraftSection.class));
|
||||
} catch (JsonProcessingException exc) {
|
||||
} catch (JacksonException exc) {
|
||||
log.warn("Failed to parse draft sections payload", exc);
|
||||
return null;
|
||||
}
|
||||
@@ -152,7 +152,7 @@ public class AiCreateInternalController {
|
||||
objectMapper
|
||||
.getTypeFactory()
|
||||
.constructMapType(Map.class, String.class, Object.class));
|
||||
} catch (JsonProcessingException exc) {
|
||||
} catch (JacksonException exc) {
|
||||
log.warn("Failed to parse outline constraints payload", exc);
|
||||
return null;
|
||||
}
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Profile;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
|
||||
@@ -13,8 +13,8 @@ import java.util.regex.Pattern;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
|
||||
@@ -56,28 +56,26 @@ public class LegalDocumentRegistry {
|
||||
subprocessorUrl = root.path("subprocessorUrl").asText("");
|
||||
eulaUrl = root.path("eulaUrl").asText("");
|
||||
JsonNode docs = root.path("documents");
|
||||
docs.fieldNames()
|
||||
.forEachRemaining(
|
||||
id -> {
|
||||
JsonNode d = docs.get(id);
|
||||
List<String> parts =
|
||||
objectMapper.convertValue(
|
||||
d.path("parts"),
|
||||
objectMapper
|
||||
.getTypeFactory()
|
||||
.constructCollectionType(
|
||||
List.class, String.class));
|
||||
documents.put(
|
||||
docs.forEachEntry(
|
||||
(id, d) -> {
|
||||
List<String> parts =
|
||||
objectMapper.convertValue(
|
||||
d.path("parts"),
|
||||
objectMapper
|
||||
.getTypeFactory()
|
||||
.constructCollectionType(
|
||||
List.class, String.class));
|
||||
documents.put(
|
||||
id,
|
||||
new LegalDocumentMeta(
|
||||
id,
|
||||
new LegalDocumentMeta(
|
||||
id,
|
||||
d.path("label").asText(id),
|
||||
d.path("displayName").asText(id),
|
||||
d.path("version").asText("0"),
|
||||
d.path("effectiveDate").asText(""),
|
||||
d.path("status").asText("draft"),
|
||||
parts == null ? List.of() : parts));
|
||||
});
|
||||
d.path("label").asText(id),
|
||||
d.path("displayName").asText(id),
|
||||
d.path("version").asText("0"),
|
||||
d.path("effectiveDate").asText(""),
|
||||
d.path("status").asText("draft"),
|
||||
parts == null ? List.of() : parts));
|
||||
});
|
||||
log.info("[legal] loaded {} document(s) from {}", documents.size(), MANIFEST);
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.method.HandlerMethod;
|
||||
import org.springframework.web.servlet.HandlerInterceptor;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
import io.micrometer.core.instrument.Counter;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ import java.util.Map;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
+2
-2
@@ -16,8 +16,8 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Profile;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
|
||||
+3
-3
@@ -10,8 +10,8 @@ import org.springframework.context.annotation.Profile;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.core.JacksonException;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@@ -707,7 +707,7 @@ public class ProcurementService {
|
||||
private String writeLineItems(QuoteBreakdown breakdown) {
|
||||
try {
|
||||
return OBJECT_MAPPER.writeValueAsString(breakdown.lineItems());
|
||||
} catch (JsonProcessingException e) {
|
||||
} catch (JacksonException e) {
|
||||
log.warn("[procurement] failed to serialise line items", e);
|
||||
return "[]";
|
||||
}
|
||||
|
||||
@@ -113,19 +113,13 @@ public class RateLimitService {
|
||||
public void cleanupExpiredBuckets() {
|
||||
long now = System.currentTimeMillis();
|
||||
|
||||
int hourlyRemoved =
|
||||
(int)
|
||||
hourlyLimits.entrySet().stream()
|
||||
.filter(e -> e.getValue().getResetTime() < now)
|
||||
.peek(e -> hourlyLimits.remove(e.getKey()))
|
||||
.count();
|
||||
int hourlyBefore = hourlyLimits.size();
|
||||
hourlyLimits.entrySet().removeIf(e -> e.getValue().getResetTime() < now);
|
||||
int hourlyRemoved = hourlyBefore - hourlyLimits.size();
|
||||
|
||||
int dailyRemoved =
|
||||
(int)
|
||||
dailyLimits.entrySet().stream()
|
||||
.filter(e -> e.getValue().getResetTime() < now)
|
||||
.peek(e -> dailyLimits.remove(e.getKey()))
|
||||
.count();
|
||||
int dailyBefore = dailyLimits.size();
|
||||
dailyLimits.entrySet().removeIf(e -> e.getValue().getResetTime() < now);
|
||||
int dailyRemoved = dailyBefore - dailyLimits.size();
|
||||
|
||||
if (hourlyRemoved + dailyRemoved > 0) {
|
||||
log.debug(
|
||||
|
||||
+2
-2
@@ -28,8 +28,8 @@ import org.springframework.security.core.context.SecurityContextHolder;
|
||||
import org.springframework.security.oauth2.jwt.Jwt;
|
||||
import org.springframework.web.method.HandlerMethod;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
# Code comments
|
||||
|
||||
A comment must carry information the code cannot. If a reader could derive it from
|
||||
the code in front of them, delete it: a redundant comment still has to be
|
||||
maintained, will eventually contradict the code, and dilutes the comments that
|
||||
matter.
|
||||
|
||||
The operative rules are in `AGENTS.md`, kept short so they stay in an agent's
|
||||
context. This document is the reasoning and the worked examples behind them, plus
|
||||
how to run the linter.
|
||||
|
||||
## Comment the current state
|
||||
|
||||
Describe the code as it is. Not what it used to be, not what changed, not why it
|
||||
changed. A comment that narrates history is stale the moment the next change
|
||||
lands, and git already holds that record.
|
||||
|
||||
When you know the history and it explains the shape of the code, the useful half is
|
||||
the reason, not the sequence. State the reason:
|
||||
|
||||
```java
|
||||
// Don't:
|
||||
// This used to reimplement the modal internals, which is how the procurement
|
||||
// dialogs drifted from the billing ones.
|
||||
|
||||
// Do:
|
||||
// Thin wrapper over the shared Modal: duplicating its portal and focus trap is
|
||||
// how dialogs drift apart.
|
||||
```
|
||||
|
||||
Future state is the exception, and it belongs in a TODO with an issue.
|
||||
|
||||
## The four jobs
|
||||
|
||||
**Contract.** What a caller must know that the signature cannot say:
|
||||
preconditions, invariants, units, ownership and lifetime, thread-safety, error
|
||||
semantics, side effects.
|
||||
|
||||
The bound is the surface, not the volume: document the contract of everything a
|
||||
caller outside the file can reach, and nothing else. Inside that surface say
|
||||
whatever a caller needs; outside it a comment earns its place on the same terms as
|
||||
any other.
|
||||
|
||||
```java
|
||||
/**
|
||||
* Authority on which filesystem locations a policy may read or write. Fail-closed
|
||||
* in order: denied entirely under the saas profile; Stirling's own config dir is
|
||||
* always rejected; the path must resolve within policies.allowedFolderRoots.
|
||||
*
|
||||
* <p>Compared after normalisation so {@code ..} cannot escape a root. Symlink
|
||||
* escape is not defended: an operator who roots an allowlist on a symlink to a
|
||||
* sensitive location is trusted.
|
||||
*/
|
||||
```
|
||||
|
||||
**Why.** The constraint the code satisfies, the bug it avoids, the alternative
|
||||
rejected and the reason.
|
||||
|
||||
```java
|
||||
// whenComplete runs on the worker thread after the run finishes, so the
|
||||
// terminal event never races the step events.
|
||||
handle.completion()
|
||||
```
|
||||
|
||||
A reference is supplementary, never load-bearing: the comment must survive
|
||||
deleting it. `// See #1234` is a dead end.
|
||||
|
||||
```java
|
||||
// flatten() reads the annotation list that save() clears, so saving first loses
|
||||
// every annotation (#6865).
|
||||
document.flatten(annotations);
|
||||
```
|
||||
|
||||
Prefer a spec (`RFC 3161`, `ISO 4217`) or a CVE where one applies. Both are
|
||||
immutable; a ticket can be closed, moved or made private.
|
||||
|
||||
**Hazard.** "Must stay in sync with X." "Order matters because Y." "Do not remove,
|
||||
it prevents Z."
|
||||
|
||||
**Map.** A short orientation at the head of a genuinely complex file: what it owns,
|
||||
and what it deliberately does not.
|
||||
|
||||
## The test that decides it
|
||||
|
||||
A comment earns its place when it sits at a different level of detail than the line
|
||||
below it: lower, stating a precise fact the code implies but does not say, or
|
||||
higher, giving intent a reader would otherwise assemble from ten lines.
|
||||
Same-altitude is the definition of redundant.
|
||||
|
||||
- **Delete it.** Is any information lost? If not, it stays deleted.
|
||||
- **Could a name carry it instead?** A better identifier, an extracted function or
|
||||
a named constant beats a comment. Prefer the code change.
|
||||
|
||||
## What not to write
|
||||
|
||||
| Don't | Instead |
|
||||
| --- | --- |
|
||||
| `// Handle drag start` above `handleDragStart` | Nothing. The name already says it. |
|
||||
| `// ─── Types ───`, `// Helpers`, `// ====` | If a file needs internal signposting, split the file. |
|
||||
| `// Step 1:` narrating a function body | Extract functions. If the steps need labels they need names. |
|
||||
| `// No longer needed`, `// Previously this used X` | State why the code is as it is now, or nothing. |
|
||||
| Commented-out code | Delete it. Git remembers. |
|
||||
| `@param blob - The blob to download` | Omit the tag rather than pad it. |
|
||||
| Docs on a self-explanatory member | Nothing, unless there is a real constraint to state. |
|
||||
|
||||
Step numbering is fine where it labels a genuinely numbered thing, such as a wizard
|
||||
step or a step in a written test procedure. It is narration when it numbers the
|
||||
lines of one function.
|
||||
|
||||
## Comments at the end of a line
|
||||
|
||||
A trailing comment usually does a different job from one above the code: it decodes
|
||||
the line it sits on. Those are worth keeping, and the linter leaves them alone.
|
||||
|
||||
```java
|
||||
byte[] pdfBytes = {0x25, 0x50, 0x44, 0x46}; // "%PDF"
|
||||
long maxAttachmentSize = 50L * 1024 * 1024; // 50 MB
|
||||
double buffer = 0.10; // 10% headroom
|
||||
default -> toBytes(value, 2); // MB
|
||||
```
|
||||
|
||||
Each overlaps in words with the code and each adds the interpretation the code
|
||||
leaves implicit, which is the lower-altitude case the test above asks for. So
|
||||
`CMT001` does not judge trailing comments; on this codebase it would have been
|
||||
wrong about roughly six in seven of them.
|
||||
|
||||
What still applies is anything that does not depend on the code below: a trailing
|
||||
`// TODO fix this` is as unowned as one on its own line, and a trailing
|
||||
`// this used to run before the flush` narrates history wherever it sits.
|
||||
|
||||
A comment block over about 12 lines, outside a file or type header, is usually a
|
||||
sign the code needs restructuring. If it is genuinely product documentation, it
|
||||
belongs in the docs repo.
|
||||
|
||||
## TODOs
|
||||
|
||||
A TODO needs an issue, because an issue is the only part that will close it:
|
||||
|
||||
```java
|
||||
// TODO(#1234): re-enable the checkout gate once account syncing lands
|
||||
```
|
||||
|
||||
An owner is not a substitute: a username goes stale when someone changes team and
|
||||
means nothing to an outside contributor. If the work is not worth an issue it is
|
||||
not worth a TODO, and the options are to do it now or leave the code alone. A
|
||||
question is not a TODO.
|
||||
|
||||
## Per language
|
||||
|
||||
**Java.** Google Java Style, which this repo already formats to. Its §7.3.1
|
||||
exception applies: omit Javadoc on a self-explanatory member where there is
|
||||
genuinely nothing to add, but do not cite it to skip something a reader needs.
|
||||
Summary fragments are noun or verb phrases, not sentences starting "This method
|
||||
returns".
|
||||
|
||||
**TypeScript.** JSDoc on the `@app/*` seams, exported hooks, and anything crossing
|
||||
a layer boundary. No `@param`/`@returns` that restates a typed signature. JSX
|
||||
comments follow the same rules as any other.
|
||||
|
||||
**Python.** Docstrings on modules, public functions and Pydantic models where the
|
||||
contract is not obvious from the type.
|
||||
|
||||
## The linter
|
||||
|
||||
```bash
|
||||
task comment-lint # what the working tree adds over HEAD
|
||||
task comment-lint:branch # what the branch adds over origin/main (BASE=<ref> to change)
|
||||
task pre-commit:comment-lint:ci # the fixture corpus, then the diff
|
||||
```
|
||||
|
||||
`comment-lint` is the pre-commit question, so it reports nothing once you have
|
||||
committed; on a CI pull request it compares against the target branch via
|
||||
`GITHUB_BASE_REF`. `comment-lint:branch` is the review question. The corpus checks
|
||||
the rules themselves rather than the code under review, so it runs on CI and before
|
||||
a rule change, not on every local commit.
|
||||
|
||||
`task comment-lint` also runs inside `task pre-commit`, and as a Claude Code `Stop`
|
||||
hook, so an agent is told before it finishes a turn and fixes the comment inside
|
||||
that turn. Stop rather than per file write: a run costs the same for one file as for
|
||||
twenty-five, and half of all writes in a turn go to a file already written in it.
|
||||
|
||||
Findings are scoped to comment text that is new, not to lines git calls new, so
|
||||
reindenting or moving code does not resurface comments you did not write.
|
||||
|
||||
The rules are the `RULES` object in
|
||||
[`scripts/lint/comment-rules.mjs`](../scripts/lint/comment-rules.mjs); the exact
|
||||
condition for each is the predicate of the same name in that file, with the
|
||||
readings it deliberately excludes beside it.
|
||||
|
||||
**Every rule blocks.** A rule that only warns is a rule nobody acts on. So a
|
||||
finding you believe is wrong is a bug in the rule, not something to live with:
|
||||
narrow the rule, or mark the line and say why.
|
||||
|
||||
Every comment form the repo writes is covered: `//` and `/* */`, Javadoc and JSDoc,
|
||||
JSX comments, `#`, and Python docstrings. `CMT007` reads all three parameter
|
||||
conventions in use here, Javadoc/JSDoc `@param`, Sphinx `:param name:` and Google
|
||||
`name: description` under `Args:`.
|
||||
|
||||
Two engines, one rule set. `.ts`/`.tsx`/`.mjs` go to an oxlint JS plugin, so
|
||||
comments come from the parser: a `//` inside a string is not a comment, and JSX
|
||||
`{/* … */}` is. `.java`/`.py` go to a line scanner. Neither reads the other's
|
||||
files, so they cannot disagree about one file. `scripts/lint/fixtures/` is the
|
||||
corpus that keeps them meaning the same thing.
|
||||
|
||||
### When a finding is wrong
|
||||
|
||||
Name the rule on the line above:
|
||||
|
||||
```ts
|
||||
// comment-lint-allow: CMT002
|
||||
// ─── kept deliberately, because <reason> ───
|
||||
```
|
||||
|
||||
There is no form that disables every rule, and the directive has to earn its
|
||||
place. `CMT008` reports one that names something which is not a rule, and one that
|
||||
silences nothing, so a typo does not read as a suppression and a stale
|
||||
suppression does not sit there blinding the line. The whole comment must be the
|
||||
directive; prose that mentions the syntax is just prose.
|
||||
|
||||
If you reach for this more than occasionally the rule is wrong: fix it in
|
||||
`comment-rules.mjs` and update the fixture corpus in the same commit, so the diff
|
||||
shows what moved.
|
||||
|
||||
### The existing backlog
|
||||
|
||||
`task pre-commit:comment-lint:all` reports the whole tree and never fails. There is
|
||||
a standing backlog being cleared by directory; diff scoping is what keeps it off
|
||||
whoever touches a file first.
|
||||
|
||||
To turn the editor hook off, put `{ "env": { "COMMENT_LINT_HOOK": "0" } }` in
|
||||
`.claude/settings.local.json`. The commit-time gate still applies, so you lose the
|
||||
early warning rather than the check.
|
||||
@@ -8,6 +8,7 @@ This directory contains all development-related documentation for Stirling PDF.
|
||||
- **[DeveloperGuide.md](../DeveloperGuide.md)** - Main developer setup and architecture guide (in repo root)
|
||||
- **[Taskfile.yml](../Taskfile.yml)** - Unified task runner for all build/dev/test/lint commands
|
||||
- **[EXCEPTION_HANDLING_GUIDE.md](./EXCEPTION_HANDLING_GUIDE.md)** - Exception handling patterns and i18n best practices
|
||||
- **[CODE_COMMENTS.md](./CODE_COMMENTS.md)** - What a comment is for, what not to write, and the `task comment-lint` rules
|
||||
- **[HowToAddNewLanguage.md](./HowToAddNewLanguage.md)** - Internationalization and translation guide
|
||||
- **[STORAGE_ENCRYPTION_AT_REST.md](./STORAGE_ENCRYPTION_AT_REST.md)** - Encryption at rest for stored files: key setup, migration, revocation, rotation
|
||||
|
||||
|
||||
@@ -725,6 +725,9 @@ class OcrPdfParams(ApiModel):
|
||||
)
|
||||
ocr_type: OcrType = Field(..., description="Specify the OCR type, e.g., 'skip-text', 'force-ocr', or 'Normal'")
|
||||
remove_images_after: bool | None = Field(None, description="Remove images from the output PDF if set to true")
|
||||
rotate_pages: bool | None = Field(
|
||||
None, description="Auto-correct page orientation (90/180/270) using Tesseract OSD if set to true"
|
||||
)
|
||||
sidecar: bool | None = Field(None, description="Include OCR text in a sidecar text file if set to true")
|
||||
|
||||
|
||||
|
||||
@@ -3526,7 +3526,6 @@ label = "إحداثي Y"
|
||||
|
||||
[crop.error]
|
||||
failed = "فشل قصّ PDF"
|
||||
invalidArea = "منطقة القص تتجاوز حدود PDF"
|
||||
|
||||
[crop.preview]
|
||||
title = "معاينة منطقة القص"
|
||||
|
||||
@@ -3526,7 +3526,6 @@ label = "Y mövqeyi"
|
||||
|
||||
[crop.error]
|
||||
failed = "PDF-i kəsmək alınmadı"
|
||||
invalidArea = "Kəsmə sahəsi PDF sərhədlərini aşır"
|
||||
|
||||
[crop.preview]
|
||||
title = "Kəsmə sahəsinin seçimi"
|
||||
|
||||
@@ -3526,7 +3526,6 @@ label = "Y позиция"
|
||||
|
||||
[crop.error]
|
||||
failed = "Неуспешно изрязване на PDF"
|
||||
invalidArea = "Областта за изрязване излиза извън границите на PDF"
|
||||
|
||||
[crop.preview]
|
||||
title = "Избор на област за изрязване"
|
||||
|
||||
@@ -3526,7 +3526,6 @@ label = "Yཡི་གནས་བབ།"
|
||||
|
||||
[crop.error]
|
||||
failed = "སོན་བཟང་མ་འདང་བ། PDF"
|
||||
invalidArea = "སོན་འདེབས་རྒྱ་ཁྱོན་དེ་PDFམཚམས་ཐིག་ལས་བརྒལ་ཡོད།"
|
||||
|
||||
[crop.preview]
|
||||
title = "སོན་བཟང་ཁུལ་འདེམས་པ།"
|
||||
|
||||
@@ -3526,7 +3526,6 @@ label = "Posició Y"
|
||||
|
||||
[crop.error]
|
||||
failed = "No s'ha pogut retallar el PDF"
|
||||
invalidArea = "L'àrea de retall s'estén més enllà dels límits del PDF"
|
||||
|
||||
[crop.preview]
|
||||
title = "Selecció de l'àrea de retall"
|
||||
|
||||
@@ -3526,7 +3526,6 @@ label = "Pozice Y"
|
||||
|
||||
[crop.error]
|
||||
failed = "Oříznutí PDF se nezdařilo"
|
||||
invalidArea = "Oblast ořezu přesahuje hranice PDF"
|
||||
|
||||
[crop.preview]
|
||||
title = "Výběr oblasti ořezu"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user