From 271212d61fc2940df8829fe35b5347340d8ce922 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:39:56 +0100 Subject: [PATCH] Avoid renderer OOM when adding large PDFs to the workbench --- .../src/core/contexts/file/fileActions.ts | 19 ++- .../editor/src/core/services/fileStorage.ts | 9 +- .../editor/src/core/utils/thumbnailUtils.ts | 142 +++++++++++++++++- 3 files changed, 159 insertions(+), 11 deletions(-) diff --git a/frontend/editor/src/core/contexts/file/fileActions.ts b/frontend/editor/src/core/contexts/file/fileActions.ts index 437c90711c..77447003f6 100644 --- a/frontend/editor/src/core/contexts/file/fileActions.ts +++ b/frontend/editor/src/core/contexts/file/fileActions.ts @@ -13,7 +13,7 @@ import { ProcessedFileMetadata, } from "@app/types/fileContext"; import { FileId, ToolOperation } from "@app/types/file"; -import { generateThumbnailWithMetadata } from "@app/utils/thumbnailUtils"; +import { generateThumbnailPairWithMetadata } from "@app/utils/thumbnailUtils"; import { FileLifecycleManager } from "@app/contexts/file/lifecycle"; import { buildQuickKeySet } from "@app/contexts/file/fileSelectors"; import { StirlingFile } from "@app/types/fileContext"; @@ -125,11 +125,20 @@ export async function generateProcessedFileMetadata( } try { - // Generate unrotated thumbnails for PageEditor (rotation applied via CSS) - const unrotatedResult = await generateThumbnailWithMetadata(file, false); + // One parse produces both variants: unrotated thumbnails for PageEditor + // (rotation applied via CSS) and the rotated one for file manager display. + const { unrotated: unrotatedResult, rotated: rotatedResult } = + await generateThumbnailPairWithMetadata(file); - // Generate rotated thumbnail for file manager display - const rotatedResult = await generateThumbnailWithMetadata(file, true); + // Large PDF whose linearized-prefix attempt failed: report "no metadata" + // (the tolerated failure shape) rather than a bogus zero-page document. + if ( + !unrotatedResult.thumbnail && + unrotatedResult.pageCount === 0 && + !unrotatedResult.isEncrypted + ) { + return undefined; + } const processedFile = createProcessedFile( unrotatedResult.pageCount, diff --git a/frontend/editor/src/core/services/fileStorage.ts b/frontend/editor/src/core/services/fileStorage.ts index 4cdecca9f1..4f82af3a8f 100644 --- a/frontend/editor/src/core/services/fileStorage.ts +++ b/frontend/editor/src/core/services/fileStorage.ts @@ -23,7 +23,9 @@ import { const THUMBNAIL_TTL_MS = 30 * 24 * 60 * 60 * 1000; // 30 days export interface StoredStirlingFileRecord extends BaseFileMetadata { - data: ArrayBuffer; + // Blob since the large-file OOM fix (stored by reference, no JS-side copy); + // ArrayBuffer records predate it and are still readable. + data: ArrayBuffer | Blob; fileId: FileId; // Matches runtime StirlingFile.fileId exactly quickKey: string; // Matches runtime StirlingFile.quickKey exactly thumbnail?: string; @@ -118,7 +120,6 @@ class FileStorageService { stub: StirlingFileStub, ): Promise { const db = await this.getDatabase(); - const arrayBuffer = await stirlingFile.arrayBuffer(); const record: StoredStirlingFileRecord = { id: stirlingFile.fileId, @@ -129,7 +130,9 @@ class FileStorageService { size: stirlingFile.size, lastModified: stirlingFile.lastModified, createdAt: stub.createdAt, - data: arrayBuffer, + // Store the File (a Blob) itself: IndexedDB persists it by reference and + // streams to disk, so multi-GB files never materialize in JS memory. + data: stirlingFile, thumbnail: stub.thumbnailUrl, thumbnailStoredAt: stub.thumbnailUrl ? Date.now() : undefined, isLeaf: stub.isLeaf ?? true, diff --git a/frontend/editor/src/core/utils/thumbnailUtils.ts b/frontend/editor/src/core/utils/thumbnailUtils.ts index 838992e38b..84b4e7146f 100644 --- a/frontend/editor/src/core/utils/thumbnailUtils.ts +++ b/frontend/editor/src/core/utils/thumbnailUtils.ts @@ -31,6 +31,14 @@ export function calculateScaleFromFileSize(fileSize: number): number { /** PDFium error code 4 = password required (encrypted PDF). */ const PDFIUM_ERR_PASSWORD = 4; +/** PDFs at or above this size never get a full-buffer client-side parse + * (renderer OOM) - only the linearized-prefix attempt below. */ +export const LARGE_PDF_PARSE_LIMIT = 100 * 1024 * 1024; + +/** Linearized PDFs keep page 1 + hint tables in the first bytes, so a small + * prefix is often enough to render a thumbnail without reading the file. */ +const LINEARIZED_PREFIX_BYTES = 2 * 1024 * 1024; + interface PdfiumRenderResult { thumbnail: string; pageCount: number; @@ -112,6 +120,72 @@ async function renderPdfThumbnailPdfium( } } +/** + * Render both thumbnail variants (upright + rotation-baked) from a single + * document open - halves the parse and memory cost of the add-files path. + */ +async function renderPdfThumbnailPairPdfium( + data: ArrayBuffer, + scale: number, + collectAllPagesMetadata: boolean, +): Promise<{ unrotated: PdfiumRenderResult; rotated: PdfiumRenderResult }> { + const m = await getPdfiumModule(); + let docPtr: number; + try { + docPtr = await openRawDocumentSafe(data); + } catch (error) { + if ( + error instanceof Error && + new RegExp(`error ${PDFIUM_ERR_PASSWORD}`).test(error.message) + ) { + const encrypted: PdfiumRenderResult = { + thumbnail: "", + pageCount: 1, + pageRotations: [], + pageDimensions: [], + isEncrypted: true, + }; + return { unrotated: encrypted, rotated: { ...encrypted } }; + } + throw error; + } + + try { + const pageCount = m.FPDF_GetPageCount(docPtr); + const unrotatedThumb = await renderPdfiumPageDataUrl(docPtr, 0, scale, { + applyRotation: false, + }); + const rotatedThumb = await renderPdfiumPageDataUrl(docPtr, 0, scale, { + applyRotation: true, + }); + if (!unrotatedThumb || !rotatedThumb) { + throw new Error("PDFium: failed to render page 0"); + } + + const firstMeta = await readPdfiumPageMetadata(docPtr, 0); + const pageRotations: number[] = [firstMeta?.rotation ?? 0]; + const pageDimensions: Array<{ width: number; height: number }> = [ + { width: firstMeta?.width ?? 0, height: firstMeta?.height ?? 0 }, + ]; + if (collectAllPagesMetadata) { + for (let i = 1; i < pageCount; i++) { + const meta = await readPdfiumPageMetadata(docPtr, i); + if (!meta) continue; + pageRotations[i] = meta.rotation; + pageDimensions[i] = { width: meta.width, height: meta.height }; + } + } + + const base = { pageCount, pageRotations, pageDimensions }; + return { + unrotated: { thumbnail: unrotatedThumb, ...base }, + rotated: { thumbnail: rotatedThumb, ...base }, + }; + } finally { + await closeRawDocument(docPtr); + } +} + async function generatePDFThumbnail( arrayBuffer: ArrayBuffer, scale: number, @@ -133,7 +207,7 @@ async function generatePDFThumbnail( */ export async function generateThumbnailForFile(file: File): Promise { // Very large PDFs skip thumbnail generation — SVG icon shown in UI instead - if (file.size >= 100 * 1024 * 1024) { + if (file.size >= LARGE_PDF_PARSE_LIMIT) { return ""; } @@ -152,8 +226,7 @@ export async function generateThumbnailForFile(file: File): Promise { const scale = calculateScaleFromFileSize(file.size); // Only read first 2MB for thumbnail generation to save memory - const chunkSize = 2 * 1024 * 1024; // 2MB - const chunk = file.slice(0, Math.min(chunkSize, file.size)); + const chunk = file.slice(0, Math.min(LINEARIZED_PREFIX_BYTES, file.size)); const arrayBuffer = await chunk.arrayBuffer(); try { @@ -193,6 +266,31 @@ export async function generateThumbnailWithMetadata( const scale = calculateScaleFromFileSize(file.size); + // Never full-parse huge PDFs client-side - the renderer process OOMs long + // before system RAM runs out. The prefix succeeds for linearized PDFs. + if (file.size >= LARGE_PDF_PARSE_LIMIT) { + try { + const chunk = await file.slice(0, LINEARIZED_PREFIX_BYTES).arrayBuffer(); + const result = await renderPdfThumbnailPdfium( + chunk, + scale, + applyRotation, + false, + ); + if (result.isEncrypted) { + return { thumbnail: "", pageCount: 1, isEncrypted: true }; + } + return { + thumbnail: result.thumbnail, + pageCount: result.pageCount, + pageRotations: result.pageRotations, + pageDimensions: result.pageDimensions, + }; + } catch { + return { thumbnail: "", pageCount: 0 }; + } + } + try { const arrayBuffer = await file.arrayBuffer(); // Always read per-page rotation: PageEditor renders thumbnails upright and @@ -222,3 +320,41 @@ export async function generateThumbnailWithMetadata( return { thumbnail: "", pageCount: 1 }; } } + +/** + * Both thumbnail variants + page metadata from ONE full parse instead of two. + * Large PDFs only get the linearized-prefix attempt; if that fails, both + * variants are empty placeholders and page metadata is omitted. + */ +export async function generateThumbnailPairWithMetadata(file: File): Promise<{ + unrotated: ThumbnailWithMetadata; + rotated: ThumbnailWithMetadata; +}> { + const scale = calculateScaleFromFileSize(file.size); + try { + const isLarge = file.size >= LARGE_PDF_PARSE_LIMIT; + const buffer = isLarge + ? await file.slice(0, LINEARIZED_PREFIX_BYTES).arrayBuffer() + : await file.arrayBuffer(); + const pair = await renderPdfThumbnailPairPdfium(buffer, scale, !isLarge); + + const toPublic = (r: PdfiumRenderResult): ThumbnailWithMetadata => + r.isEncrypted + ? { thumbnail: "", pageCount: 1, isEncrypted: true } + : { + thumbnail: r.thumbnail, + pageCount: r.pageCount, + pageRotations: r.pageRotations, + pageDimensions: r.pageDimensions, + }; + return { + unrotated: toPublic(pair.unrotated), + rotated: toPublic(pair.rotated), + }; + } catch { + return { + unrotated: { thumbnail: "", pageCount: 0 }, + rotated: { thumbnail: "", pageCount: 0 }, + }; + } +}