mirror of
https://github.com/Stirling-Tools/Stirling-PDF.git
synced 2026-09-03 05:10:16 +03:00
Compare commits
49
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b63078a4b1 | ||
|
|
75170a2335 | ||
|
|
56b66e588b | ||
|
|
6a820888a7 | ||
|
|
93addc0a4f | ||
|
|
a2da7f43e3 | ||
|
|
0160fd7b80 | ||
|
|
73b0450f1a | ||
|
|
ffa85c2a74 | ||
|
|
b7c9065a4e | ||
|
|
d580f565b7 | ||
|
|
fb6c4d45af | ||
|
|
0d5f21bed8 | ||
|
|
365ebaf46d | ||
|
|
f715d74227 | ||
|
|
f3f2a2fddc | ||
|
|
bc6979a659 | ||
|
|
4e75e11714 | ||
|
|
0a9e07c5bd | ||
|
|
e8a97ff27c | ||
|
|
650dcc8465 | ||
|
|
a90b66d216 | ||
|
|
835ea60769 | ||
|
|
d29c044e3c | ||
|
|
4d3ce35aee | ||
|
|
ecd72dbb19 | ||
|
|
b77f570089 | ||
|
|
94b09152c4 | ||
|
|
60a19f609d | ||
|
|
1004be8cb5 | ||
|
|
ef76aa0dc3 | ||
|
|
d1945fe9b9 | ||
|
|
663763142a | ||
|
|
b81255642c | ||
|
|
5c5297c944 | ||
|
|
84b3a51e4a | ||
|
|
7eb213056b | ||
|
|
8067437654 | ||
|
|
81a68bfe0d | ||
|
|
67dd095476 | ||
|
|
f208579be7 | ||
|
|
2e4ce484dc | ||
|
|
82183ad4f8 | ||
|
|
25933298d3 | ||
|
|
125c82a9b3 | ||
|
|
3e462cdb53 | ||
|
|
04178efcc1 | ||
|
|
7e88f38b9e | ||
|
|
2993719e72 |
+14
-2
@@ -10,6 +10,13 @@ vars:
|
||||
# fails at launch with UnsupportedClassVersionError. Enforced by jlink:verify.
|
||||
REQUIRED_JAVA: "25"
|
||||
|
||||
# Which backend the bundled JAR is: "true" builds core only, "false" includes
|
||||
# the proprietary module (the `with-login` shape the release matrix builds).
|
||||
# Defaults to core, matching the default desktop release variant; override via
|
||||
# the DISABLE_ADDITIONAL_FEATURES env to run the desktop app against a backend
|
||||
# that carries the proprietary endpoints.
|
||||
DISABLE_ADDITIONAL_FEATURES: '{{.DISABLE_ADDITIONAL_FEATURES | default "true"}}'
|
||||
|
||||
# Override via JPDFIUM_PLATFORMS env (csv of platform keys, or 'all').
|
||||
JPDFIUM_PLATFORMS:
|
||||
sh: |
|
||||
@@ -128,17 +135,22 @@ tasks:
|
||||
run: once
|
||||
dir: ..
|
||||
env:
|
||||
DISABLE_ADDITIONAL_FEATURES: "true"
|
||||
DISABLE_ADDITIONAL_FEATURES: "{{.DISABLE_ADDITIONAL_FEATURES}}"
|
||||
cmds:
|
||||
- echo "Building bootJar with JPDFium natives for {{.JPDFIUM_PLATFORMS}}"
|
||||
- echo "Building bootJar (additional features disabled={{.DISABLE_ADDITIONAL_FEATURES}}) with JPDFium natives for {{.JPDFIUM_PLATFORMS}}"
|
||||
- cmd: cmd /c gradlew.bat bootJar --no-daemon -PjpdfiumPlatforms={{.JPDFIUM_PLATFORMS}}
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew bootJar --no-daemon -PjpdfiumPlatforms={{.JPDFIUM_PLATFORMS}}
|
||||
platforms: [linux, darwin]
|
||||
- mkdir -p frontend/editor/src-tauri/libs
|
||||
- cp app/core/build/libs/stirling-pdf-*.jar frontend/editor/src-tauri/libs/
|
||||
# Record which backend the bundled JAR is, so switching variant rebuilds
|
||||
# rather than silently reusing the other one — the same staleness trap
|
||||
# jlink:verify exists to catch for the JRE.
|
||||
- echo "{{.DISABLE_ADDITIONAL_FEATURES}}" > frontend/editor/src-tauri/libs/.variant
|
||||
status:
|
||||
- test -f frontend/editor/src-tauri/libs/stirling-pdf-*.jar
|
||||
- test "$(cat frontend/editor/src-tauri/libs/.variant 2>/dev/null)" = "{{.DISABLE_ADDITIONAL_FEATURES}}"
|
||||
|
||||
jlink:runtime:
|
||||
desc: "Create custom JRE with jlink"
|
||||
|
||||
@@ -215,6 +215,16 @@ public class ApplicationProperties {
|
||||
*/
|
||||
private List<String> allowedFolderRoots = new java.util.ArrayList<>();
|
||||
|
||||
/**
|
||||
* How many of one sweep's runs may execute at once; further runs queue (visible as pending)
|
||||
* and start as slots free up. Sweeps fan out one run per file, and a folder of documents
|
||||
* dispatched all at once piles up at the pipeline's slowest tool — nothing visibly finishes
|
||||
* until the end. The cap keeps completions arriving steadily; the default suits API-bound
|
||||
* pipelines (classification is one fast-model call per document). Turn it down for a
|
||||
* heavyweight local engine, 0 = unbounded.
|
||||
*/
|
||||
private int sweepConcurrency = 6;
|
||||
|
||||
/** How often (seconds) the schedule trigger checks for policies whose schedule is due. */
|
||||
private long scheduleSweepSeconds = 60;
|
||||
|
||||
|
||||
+15
-3
@@ -21,6 +21,7 @@ import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RequestPart;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.context.request.RequestContextHolder;
|
||||
@@ -203,15 +204,20 @@ public class PolicyController {
|
||||
summary = "List the caller's stored-policy runs",
|
||||
description =
|
||||
"Returns the caller's in-flight and recently-finished stored-policy runs (within"
|
||||
+ " the run-retention window). The frontend reconciles these on load so a"
|
||||
+ " run started before a refresh/crash is rediscovered and its outputs"
|
||||
+ " the run-retention window), optionally narrowed to one policy via"
|
||||
+ " `policyId` — a client following a single sweep polls this every"
|
||||
+ " second, and the unfiltered list grows with every other policy's"
|
||||
+ " runs. The frontend reconciles the unfiltered list on load so a run"
|
||||
+ " started before a refresh/crash is rediscovered and its outputs"
|
||||
+ " collected, rather than orphaned on the backend. Ad-hoc runs (no"
|
||||
+ " policy id) are excluded.")
|
||||
public List<PolicyRunView> listRuns() {
|
||||
public List<PolicyRunView> listRuns(
|
||||
@RequestParam(name = "policyId", required = false) String policyId) {
|
||||
// Local runs first (they carry live step state); keyed by runId to dedupe shared entries.
|
||||
Map<String, PolicyRunView> byRunId = new LinkedHashMap<>();
|
||||
runRegistry.all().stream()
|
||||
.filter(run -> run.getPolicyId() != null)
|
||||
.filter(run -> policyId == null || policyId.equals(run.getPolicyId()))
|
||||
.filter(run -> ownedByCurrentUser(run.getRunId()))
|
||||
.forEach(run -> byRunId.put(run.getRunId(), PolicyRunView.of(run)));
|
||||
// Then runs from other nodes, read from the shared job store.
|
||||
@@ -223,6 +229,9 @@ public class PolicyController {
|
||||
if (meta == null || !meta.containsKey("policyId")) {
|
||||
continue; // ad-hoc job, not a stored-policy run
|
||||
}
|
||||
if (policyId != null && !policyId.equals(meta.get("policyId"))) {
|
||||
continue;
|
||||
}
|
||||
if (ownedByCurrentUser(entry.jobId())) {
|
||||
byRunId.put(entry.jobId(), PolicyRunView.ofEntry(entry));
|
||||
}
|
||||
@@ -456,6 +465,9 @@ public class PolicyController {
|
||||
+ " values.")
|
||||
public List<Policy> listPolicies() {
|
||||
return policyAccessGuard.visibleFrom(policyStore).stream()
|
||||
// Processing folders share the engine but are the editor's own surface,
|
||||
// served exclusively by ProcessingFolderController.
|
||||
.filter(policy -> !ProcessingFolderController.isProcessingFolder(policy))
|
||||
.map(PolicyController::withMaskedOutputSecrets)
|
||||
.toList();
|
||||
}
|
||||
|
||||
+556
@@ -0,0 +1,556 @@
|
||||
package stirling.software.proprietary.policy.controller;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.config.FolderAccessGuard;
|
||||
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.PolicyValidator;
|
||||
import stirling.software.proprietary.policy.engine.SweepOutcome;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineInput;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.TriggerConfig;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.storage.model.Folder;
|
||||
import stirling.software.proprietary.storage.repository.FolderRepository;
|
||||
import stirling.software.proprietary.storage.service.FileStorageService;
|
||||
|
||||
/**
|
||||
* Processing folders: a storage folder with a pipeline attached, so any file that lands in it is
|
||||
* processed. One processing folder is a pair of records — a {@code storage-folder} source and a
|
||||
* policy — composed and torn down together here so neither can exist half-configured. The pair is
|
||||
* marked with {@link #SURFACE} and served only by this route: the portal's policies and pipelines
|
||||
* surfaces filter it out, and this route serves nothing else.
|
||||
*
|
||||
* <p>Unlike org policies this is a personal, per-user feature: any authenticated user may create
|
||||
* processing folders on folders they own; there is no team-leader gate. Records are still stamped
|
||||
* with the caller's team so the engine's scoping holds.
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/processing-folders")
|
||||
@RequiredArgsConstructor
|
||||
@Tag(name = "Processing Folders", description = "Folders that process any file added to them.")
|
||||
public class ProcessingFolderController {
|
||||
|
||||
/** Marker in the policy's output options separating this surface from policies/pipelines. */
|
||||
public static final String SURFACE_OPTION = "surface";
|
||||
|
||||
public static final String SURFACE = "processing-folder";
|
||||
|
||||
/** The paired source's type; the policies/pipelines surfaces hide sources of this type too. */
|
||||
public static final String SOURCE_TYPE = "storage-folder";
|
||||
|
||||
/** A processing folder over a directory on the server's disk (desktop / self-hosted). */
|
||||
static final String DISK_SOURCE_TYPE = FolderAccessGuard.FOLDER_TYPE;
|
||||
|
||||
/**
|
||||
* How many files one sweep of a disk-backed folder takes on. A Downloads directory can hold
|
||||
* thousands; the cap keeps a first run bounded and predictable, and everything beyond it keeps
|
||||
* its place in the ledger and is picked up by later sweeps rather than dropped.
|
||||
*/
|
||||
static final int DISK_SWEEP_LIMIT = 100;
|
||||
|
||||
/** Where a disk-backed folder's results land, relative to the directory it watches. */
|
||||
static final String DISK_OUTPUT_SUBDIR = "Stirling Processed";
|
||||
|
||||
/** The trigger that watches a directory for arrivals. */
|
||||
static final String WATCH_TRIGGER = "folder-watch";
|
||||
|
||||
private final PolicyStore policyStore;
|
||||
private final SourceStore sourceStore;
|
||||
private final PolicyValidator policyValidator;
|
||||
private final PolicyRunner policyRunner;
|
||||
private final PolicyTriggerManager policyTriggerManager;
|
||||
private final ProcessedLedger processedLedger;
|
||||
private final FolderRepository folderRepository;
|
||||
private final FileStorageService fileStorageService;
|
||||
private final PolicyAccessGuard policyAccessGuard;
|
||||
private final FolderAccessGuard folderAccessGuard;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
/** What a processing folder looks like to the editor client. */
|
||||
public record ProcessingFolderView(
|
||||
String id,
|
||||
String folderId,
|
||||
String directory,
|
||||
String name,
|
||||
boolean enabled,
|
||||
List<PipelineStep> steps,
|
||||
Map<String, Object> output,
|
||||
/** Runs the creating sweep started; 0 means there was nothing new to process. */
|
||||
int startedRuns,
|
||||
/** Files the sweep skipped because this folder had already processed them. */
|
||||
int alreadyProcessed) {}
|
||||
|
||||
/**
|
||||
* Create/update payload. A null id creates; a present id updates the caller's own record.
|
||||
* Exactly one of {@code folderId} (a folder in app storage) or {@code directory} (a directory
|
||||
* on the server's disk — on a desktop or self-hosted install, the user's own machine) says
|
||||
* where the folder watches.
|
||||
*/
|
||||
public record SaveProcessingFolderRequest(
|
||||
String id,
|
||||
String folderId,
|
||||
String directory,
|
||||
Boolean enabled,
|
||||
List<PipelineStep> steps,
|
||||
Map<String, Object> output) {}
|
||||
|
||||
/**
|
||||
* What the Downloads offer should say. The browser cannot see the machine's paths, so the
|
||||
* server names its own Downloads directory and counts what is waiting there.
|
||||
*/
|
||||
public record DownloadsSuggestion(
|
||||
String directory, boolean available, int pdfCount, int limit) {}
|
||||
|
||||
@GetMapping("/downloads-suggestion")
|
||||
@Operation(
|
||||
summary = "The server's Downloads directory and how many PDFs are waiting in it",
|
||||
description =
|
||||
"Backs the offer to process a user's Downloads. `available` is false when the"
|
||||
+ " directory does not exist or is outside the permitted folder roots,"
|
||||
+ " so the offer is never made where it could only fail.")
|
||||
public DownloadsSuggestion downloadsSuggestion() {
|
||||
currentUserOrNull();
|
||||
Path downloads = Path.of(System.getProperty("user.home", ""), "Downloads");
|
||||
if (!Files.isDirectory(downloads)) {
|
||||
return new DownloadsSuggestion(downloads.toString(), false, 0, DISK_SWEEP_LIMIT);
|
||||
}
|
||||
try {
|
||||
folderAccessGuard.requirePermitted(downloads);
|
||||
} catch (RuntimeException notPermitted) {
|
||||
return new DownloadsSuggestion(downloads.toString(), false, 0, DISK_SWEEP_LIMIT);
|
||||
}
|
||||
int pdfCount = 0;
|
||||
try (Stream<Path> entries = Files.list(downloads)) {
|
||||
pdfCount =
|
||||
(int)
|
||||
entries.filter(Files::isRegularFile)
|
||||
.filter(
|
||||
path ->
|
||||
path.getFileName()
|
||||
.toString()
|
||||
.toLowerCase(Locale.ROOT)
|
||||
.endsWith(".pdf"))
|
||||
.limit(DISK_SWEEP_LIMIT * 10L)
|
||||
.count();
|
||||
} catch (IOException e) {
|
||||
log.debug("Could not count PDFs in {}: {}", downloads, e.getMessage());
|
||||
}
|
||||
return new DownloadsSuggestion(downloads.toString(), true, pdfCount, DISK_SWEEP_LIMIT);
|
||||
}
|
||||
|
||||
@GetMapping
|
||||
@Operation(summary = "List the caller's processing folders")
|
||||
public List<ProcessingFolderView> list() {
|
||||
User user = currentUserOrNull();
|
||||
return policyAccessGuard.visibleFrom(policyStore).stream()
|
||||
.filter(ProcessingFolderController::isProcessingFolder)
|
||||
.filter(policy -> ownedBy(policy, user))
|
||||
.map(this::toView)
|
||||
.toList();
|
||||
}
|
||||
|
||||
@PostMapping(consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
summary = "Create or update a processing folder",
|
||||
description =
|
||||
"Composes the folder's source + pipeline pair, validated like any policy save."
|
||||
+ " Creating one immediately processes the folder's existing files (the"
|
||||
+ " ledger keeps already-processed files from re-running).")
|
||||
public ResponseEntity<ProcessingFolderView> save(
|
||||
@RequestBody SaveProcessingFolderRequest request) {
|
||||
User user = currentUserOrNull();
|
||||
boolean onDisk = request.directory() != null && !request.directory().isBlank();
|
||||
if (onDisk == (request.folderId() != null && !request.folderId().isBlank())) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST,
|
||||
"a processing folder needs either a folderId or a directory, not both");
|
||||
}
|
||||
Folder folder = onDisk ? null : requireOwnedFolder(request.folderId(), user);
|
||||
boolean requestedCreate = request.id() == null || request.id().isBlank();
|
||||
// A place carries at most one processing folder per user: a create against a place that
|
||||
// already has one adopts it — same policy, same ledger — instead of composing a duplicate
|
||||
// pair whose empty ledger would re-process everything the original already did. The
|
||||
// adopted create still runs the backlog sweep below; the kept ledger makes it pick up
|
||||
// only what is genuinely new.
|
||||
Policy existing =
|
||||
requestedCreate ? existingForPlace(request, user) : requireOwn(request.id(), user);
|
||||
String name = onDisk ? diskFolderName(request.directory()) : folder.getName();
|
||||
|
||||
// Held for rollback: the source is written before the policy validates, and a rejected
|
||||
// save must not leave the pair half-updated (source mutated, policy old).
|
||||
String existingSourceId = existing == null ? null : soleSourceId(existing);
|
||||
Source priorSource =
|
||||
existingSourceId == null ? null : sourceStore.get(existingSourceId).orElse(null);
|
||||
|
||||
Source source =
|
||||
sourceStore.save(
|
||||
new Source(
|
||||
existing == null ? null : soleSourceId(existing),
|
||||
name,
|
||||
onDisk ? DISK_SOURCE_TYPE : SOURCE_TYPE,
|
||||
onDisk
|
||||
? diskSourceOptions(request.directory())
|
||||
: Map.of("folderId", folder.getId().toString()),
|
||||
request.enabled() == null || request.enabled(),
|
||||
policyAccessGuard.ownerForNewPolicy(),
|
||||
policyAccessGuard.teamForNewPolicy()));
|
||||
Policy policy =
|
||||
new Policy(
|
||||
existing == null ? null : existing.id(),
|
||||
"Processing folder: " + name,
|
||||
policyAccessGuard.ownerForNewPolicy(),
|
||||
request.enabled() == null || request.enabled(),
|
||||
// A disk directory is watched, so the folder reacts to arrivals on its own.
|
||||
// A null trigger would make the input manual-only: the create-time backlog
|
||||
// sweep would run and nothing would ever process again. Storage-backed
|
||||
// folders stay manual until the storage arrival trigger exists —
|
||||
// folder-watch only supports directory sources.
|
||||
List.of(
|
||||
new PipelineInput(
|
||||
source.id(),
|
||||
onDisk
|
||||
? new TriggerConfig(WATCH_TRIGGER, Map.of())
|
||||
: null)),
|
||||
request.steps() == null ? List.of() : request.steps(),
|
||||
outputSpecFor(request, folder),
|
||||
policyAccessGuard.teamForNewPolicy());
|
||||
try {
|
||||
policyValidator.validate(policy);
|
||||
} catch (IllegalArgumentException e) {
|
||||
if (existing == null) {
|
||||
sourceStore.delete(source.id());
|
||||
} else if (priorSource != null) {
|
||||
sourceStore.save(priorSource);
|
||||
}
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
|
||||
}
|
||||
Policy saved = policyStore.save(policy);
|
||||
policyTriggerManager.notifyPoliciesChanged();
|
||||
if (!requestedCreate) {
|
||||
return ResponseEntity.ok(toView(saved));
|
||||
}
|
||||
// Process the backlog: everything already in the folder runs once, now. The counts go back
|
||||
// to the caller so a client can report real progress — and can tell "nothing new to do"
|
||||
// apart from "work started", instead of waiting for runs that were never going to appear.
|
||||
SweepOutcome outcome = policyRunner.run(saved);
|
||||
log.debug(
|
||||
"Processing folder {} created; backlog sweep started {} runs ({} already processed,"
|
||||
+ " {} listed)",
|
||||
saved.id(),
|
||||
outcome.runIds().size(),
|
||||
outcome.alreadyProcessed(),
|
||||
outcome.filesListed());
|
||||
return ResponseEntity.ok(
|
||||
toView(saved, outcome.runIds().size(), outcome.alreadyProcessed()));
|
||||
}
|
||||
|
||||
/** One file in a mounted directory, as the file manager needs to list it. */
|
||||
public record MountedFileView(String name, long sizeBytes, long lastModified) {}
|
||||
|
||||
@GetMapping("/{id}/files")
|
||||
@Operation(
|
||||
summary = "List the files in a disk-backed processing folder",
|
||||
description =
|
||||
"The directory itself is the source of truth — nothing is mirrored into app"
|
||||
+ " storage — so the file manager reads its contents through here."
|
||||
+ " Empty for a storage-backed folder, whose files are ordinary stored"
|
||||
+ " files.")
|
||||
public List<MountedFileView> files(@PathVariable String id) {
|
||||
User user = currentUserOrNull();
|
||||
Policy policy = requireOwn(id, user);
|
||||
Path directory = watchedDirectory(policy);
|
||||
if (directory == null) {
|
||||
return List.of();
|
||||
}
|
||||
// Re-check on read: the permitted roots may have narrowed since the folder was created.
|
||||
Path permitted = folderAccessGuard.requirePermitted(directory);
|
||||
try (Stream<Path> entries = Files.list(permitted)) {
|
||||
return entries.filter(Files::isRegularFile)
|
||||
.filter(path -> !path.getFileName().toString().startsWith("."))
|
||||
.map(ProcessingFolderController::toMountedFile)
|
||||
.filter(Objects::nonNull)
|
||||
.toList();
|
||||
} catch (IOException e) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_GATEWAY, "Could not read " + permitted + ": " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static MountedFileView toMountedFile(Path path) {
|
||||
try {
|
||||
return new MountedFileView(
|
||||
path.getFileName().toString(),
|
||||
Files.size(path),
|
||||
Files.getLastModifiedTime(path).toMillis());
|
||||
} catch (IOException vanished) {
|
||||
return null; // listed then removed; the next read tells the truth
|
||||
}
|
||||
}
|
||||
|
||||
/** The disk directory a processing folder watches, or null when it is storage-backed. */
|
||||
private Path watchedDirectory(Policy policy) {
|
||||
String sourceId = soleSourceId(policy);
|
||||
if (sourceId == null) {
|
||||
return null;
|
||||
}
|
||||
return sourceStore
|
||||
.get(sourceId)
|
||||
.filter(source -> DISK_SOURCE_TYPE.equals(source.type()))
|
||||
.map(source -> source.options().get("directory"))
|
||||
.filter(Objects::nonNull)
|
||||
.map(directory -> Path.of(directory.toString()))
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
@PostMapping("/{id}/sweep")
|
||||
@Operation(summary = "Run the folder's pipeline against its current contents now")
|
||||
public ResponseEntity<SweepOutcome> sweep(@PathVariable String id) {
|
||||
User user = currentUserOrNull();
|
||||
Policy policy = requireOwn(id, user);
|
||||
return ResponseEntity.accepted().body(policyRunner.run(policy));
|
||||
}
|
||||
|
||||
@DeleteMapping("/{id}")
|
||||
@Operation(
|
||||
summary = "Delete a processing folder",
|
||||
description =
|
||||
"Removes the pipeline and its source. The storage folder and every file in it"
|
||||
+ " are untouched.")
|
||||
public ResponseEntity<Void> delete(@PathVariable String id) {
|
||||
User user = currentUserOrNull();
|
||||
Policy policy = requireOwn(id, user);
|
||||
policyStore.delete(policy.id());
|
||||
policy.inputs().stream().map(PipelineInput::sourceId).forEach(sourceStore::delete);
|
||||
processedLedger.clearPolicy(policy.id());
|
||||
policyTriggerManager.notifyPoliciesChanged();
|
||||
return ResponseEntity.noContent().build();
|
||||
}
|
||||
|
||||
/**
|
||||
* The caller's processing folder already watching the requested place, if any. Paths compare
|
||||
* normalized (and by the platform's own case rules), so the same directory spelled two ways is
|
||||
* still one place.
|
||||
*/
|
||||
private Policy existingForPlace(SaveProcessingFolderRequest request, User user) {
|
||||
boolean onDisk = request.directory() != null && !request.directory().isBlank();
|
||||
Path directory = onDisk ? Path.of(request.directory().trim()).normalize() : null;
|
||||
return policyAccessGuard.visibleFrom(policyStore).stream()
|
||||
.filter(ProcessingFolderController::isProcessingFolder)
|
||||
.filter(policy -> ownedBy(policy, user))
|
||||
.filter(
|
||||
policy -> {
|
||||
String sourceId = soleSourceId(policy);
|
||||
Source source =
|
||||
sourceId == null
|
||||
? null
|
||||
: sourceStore.get(sourceId).orElse(null);
|
||||
if (source == null) {
|
||||
return false;
|
||||
}
|
||||
if (onDisk) {
|
||||
Object watched = source.options().get("directory");
|
||||
return watched != null
|
||||
&& Path.of(watched.toString())
|
||||
.normalize()
|
||||
.equals(directory);
|
||||
}
|
||||
return String.valueOf(source.options().get("folderId"))
|
||||
.equals(request.folderId());
|
||||
})
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
/** The pair's policy record, only if it is a processing folder the caller owns. */
|
||||
private Policy requireOwn(String id, User user) {
|
||||
return policyStore
|
||||
.get(id)
|
||||
.filter(policyAccessGuard::canAccess)
|
||||
.filter(ProcessingFolderController::isProcessingFolder)
|
||||
.filter(policy -> ownedBy(policy, user))
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new ResponseStatusException(
|
||||
HttpStatus.NOT_FOUND, "No processing folder: " + id));
|
||||
}
|
||||
|
||||
/** The storage folder, only if the caller owns it — the authorization boundary here. */
|
||||
private Folder requireOwnedFolder(String rawFolderId, User user) {
|
||||
UUID folderId;
|
||||
try {
|
||||
folderId = UUID.fromString(String.valueOf(rawFolderId));
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST, "a processing folder needs a folderId");
|
||||
}
|
||||
Folder folder =
|
||||
folderRepository
|
||||
.findById(folderId)
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new ResponseStatusException(
|
||||
HttpStatus.NOT_FOUND, "No folder: " + rawFolderId));
|
||||
if (folder.getOwner() == null || !Objects.equals(folder.getOwner().getId(), user.getId())) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No folder: " + rawFolderId);
|
||||
}
|
||||
return folder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-user ownership on top of the guard's team scoping: processing folders are personal, so a
|
||||
* teammate's records are invisible here even though the engine treats them as team records.
|
||||
* Login disabled (null owner) matches everything.
|
||||
*/
|
||||
private static boolean ownedBy(Policy policy, User user) {
|
||||
if (user == null) {
|
||||
// No accounts on this install: the local operator owns everything.
|
||||
return true;
|
||||
}
|
||||
return policy.owner() == null || Objects.equals(policy.owner(), user.getUsername());
|
||||
}
|
||||
|
||||
/**
|
||||
* The caller, or null on an install with no accounts (desktop, single-user self-host), where
|
||||
* there is no principal to demand and the local operator is the only user. Mirrors {@link
|
||||
* PolicyAccessGuard#ownerForNewPolicy()}, which stamps a null owner in the same case —
|
||||
* requiring a principal here would make the whole surface 401 on those installs.
|
||||
*/
|
||||
private User currentUserOrNull() {
|
||||
if (!applicationProperties.getSecurity().isEnableLogin()) {
|
||||
return null;
|
||||
}
|
||||
return fileStorageService.requireAuthenticatedUser();
|
||||
}
|
||||
|
||||
/** The pair's source id; the compose invariant is exactly one source per processing folder. */
|
||||
private static String soleSourceId(Policy policy) {
|
||||
return policy.inputs().isEmpty() ? null : policy.inputs().get(0).sourceId();
|
||||
}
|
||||
|
||||
/** Whether a policy record belongs to this surface (and so is hidden from the others). */
|
||||
public static boolean isProcessingFolder(Policy policy) {
|
||||
return policy.output() != null
|
||||
&& SURFACE.equals(policy.output().options().get(SURFACE_OPTION));
|
||||
}
|
||||
|
||||
/**
|
||||
* A processing folder never consumes its input directory. The user owns that folder — their
|
||||
* Downloads, a scanner drop — so the source is pinned to {@code track}: claim each file once
|
||||
* per version through the ledger and leave it exactly where they put it. The disk source's
|
||||
* default mode deletes processed files, which must never be what a processing folder does.
|
||||
*/
|
||||
private static Map<String, Object> diskSourceOptions(String directory) {
|
||||
return Map.of(
|
||||
"directory",
|
||||
directory.trim(),
|
||||
"mode",
|
||||
"track",
|
||||
"identity",
|
||||
"hash",
|
||||
"recursive",
|
||||
false,
|
||||
"limit",
|
||||
DISK_SWEEP_LIMIT);
|
||||
}
|
||||
|
||||
/** The trailing path segment ("Downloads"), or the raw path when it has none. */
|
||||
private static String diskFolderName(String directory) {
|
||||
Path path = Path.of(directory.trim());
|
||||
Path fileName = path.getFileName();
|
||||
return fileName == null ? path.toString() : fileName.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Where results go. A storage-backed folder writes back into app storage. A disk-backed one
|
||||
* writes into a subdirectory of the directory it watches, so the user's own files are never
|
||||
* rewritten and the results sit next to them. That subdirectory is outside the source's
|
||||
* (non-recursive) scan, and the sink records each output in the ledger, so a run's results are
|
||||
* never mistaken for new work.
|
||||
*
|
||||
* <p>Writing to disk rather than app storage is what makes this work on an install with no
|
||||
* accounts and no file storage — a desktop app, where the server is the user's own machine and
|
||||
* there is nothing to store a file against.
|
||||
*
|
||||
* <p>TEMPORARY: the subdirectory name is fixed. It is expected to become a per-folder option.
|
||||
*/
|
||||
private OutputSpec outputSpecFor(SaveProcessingFolderRequest request, Folder folder) {
|
||||
Map<String, Object> options =
|
||||
new HashMap<>(request.output() == null ? Map.of() : request.output());
|
||||
options.put(SURFACE_OPTION, SURFACE);
|
||||
if (folder != null) {
|
||||
options.putIfAbsent("folderId", folder.getId().toString());
|
||||
return new OutputSpec("storage", options);
|
||||
}
|
||||
options.put(
|
||||
"directory",
|
||||
Path.of(request.directory().trim()).resolve(DISK_OUTPUT_SUBDIR).toString());
|
||||
return new OutputSpec("folder", options);
|
||||
}
|
||||
|
||||
private ProcessingFolderView toView(Policy policy) {
|
||||
return toView(policy, 0, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* What the folder watches comes from its source, never from its output. Reading it off the
|
||||
* output made a disk-backed folder advertise the subdirectory its results go into as its own
|
||||
* address, and a client that groups by folderId pick up the output folder instead of the
|
||||
* watched one.
|
||||
*/
|
||||
private ProcessingFolderView toView(Policy policy, int startedRuns, int alreadyProcessed) {
|
||||
Map<String, Object> output = new HashMap<>(policy.output().options());
|
||||
output.remove(SURFACE_OPTION);
|
||||
String sourceId = soleSourceId(policy);
|
||||
Source source = sourceId == null ? null : sourceStore.get(sourceId).orElse(null);
|
||||
Object folderId = source == null ? null : source.options().get("folderId");
|
||||
Object directory = source == null ? null : source.options().get("directory");
|
||||
return new ProcessingFolderView(
|
||||
policy.id(),
|
||||
folderId == null ? null : folderId.toString(),
|
||||
directory == null ? null : directory.toString(),
|
||||
policy.name(),
|
||||
policy.enabled(),
|
||||
policy.steps(),
|
||||
output,
|
||||
startedRuns,
|
||||
alreadyProcessed);
|
||||
}
|
||||
}
|
||||
+48
-5
@@ -7,6 +7,7 @@ import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Semaphore;
|
||||
|
||||
import org.slf4j.MDC;
|
||||
import org.springframework.core.io.Resource;
|
||||
@@ -130,7 +131,7 @@ public class PolicyEngine {
|
||||
// worker.
|
||||
String principal = currentActingPrincipal();
|
||||
return submitForPrincipal(
|
||||
principal, principal, policyId, definition, inputs, listener, null, null);
|
||||
principal, principal, policyId, definition, inputs, listener, null, null, null);
|
||||
}
|
||||
|
||||
/** Run a stored policy on demand. {@code enabled} gates triggers, not explicit runs. */
|
||||
@@ -151,6 +152,23 @@ public class PolicyEngine {
|
||||
PolicyProgressListener listener,
|
||||
String sourceId,
|
||||
String fileIdentity) {
|
||||
return runPolicy(policy, inputs, listener, sourceId, fileIdentity, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* As above, additionally pacing execution through {@code admission}: the run is registered and
|
||||
* visible immediately (pending), but its work only proceeds while holding a permit. A sweep
|
||||
* passes one gate for all its runs so a folderful of files executes a few at a time — the
|
||||
* pipeline's slowest tool serializes them anyway, and paced runs finish steadily instead of all
|
||||
* sitting in-flight until the end. Null means ungated.
|
||||
*/
|
||||
public PolicyRunHandle runPolicy(
|
||||
Policy policy,
|
||||
PolicyInputs inputs,
|
||||
PolicyProgressListener listener,
|
||||
String sourceId,
|
||||
String fileIdentity,
|
||||
Semaphore admission) {
|
||||
// Bill the policy owner: trigger-fired runs have no security context, and the async worker
|
||||
// doesn't inherit the caller's, so the owner (stamped at policy creation) is the reliable
|
||||
// billing identity — and for org-wide policies the org/owner is meant to pay. But own the
|
||||
@@ -179,7 +197,8 @@ public class PolicyEngine {
|
||||
resolved,
|
||||
listener,
|
||||
sourceId,
|
||||
fileIdentity);
|
||||
fileIdentity,
|
||||
admission);
|
||||
}
|
||||
|
||||
private PolicyRunHandle submitForPrincipal(
|
||||
@@ -190,7 +209,8 @@ public class PolicyEngine {
|
||||
PolicyInputs inputs,
|
||||
PolicyProgressListener listener,
|
||||
String sourceId,
|
||||
String fileIdentity) {
|
||||
String fileIdentity,
|
||||
Semaphore admission) {
|
||||
// Scope the run id to the current user (this request thread) so the file-download
|
||||
// ownership check passes. No-op when security is off.
|
||||
String runId = jobOwnershipService.createScopedJobKey(UUID.randomUUID().toString());
|
||||
@@ -214,7 +234,27 @@ public class PolicyEngine {
|
||||
billingPrincipal,
|
||||
fileOwner,
|
||||
definition.name(),
|
||||
() -> runToCompletion(run, inputs, tracking, completion));
|
||||
() -> {
|
||||
// Pacing gate: park (cheap on a virtual thread, and the run
|
||||
// honestly reads as pending) until a slot frees up.
|
||||
if (admission != null) {
|
||||
try {
|
||||
admission.acquire();
|
||||
} catch (InterruptedException e) {
|
||||
// Shutdown while parked: never ran, never will.
|
||||
Thread.currentThread().interrupt();
|
||||
completion.completeExceptionally(e);
|
||||
return;
|
||||
}
|
||||
}
|
||||
try {
|
||||
runToCompletion(run, inputs, tracking, completion);
|
||||
} finally {
|
||||
if (admission != null) {
|
||||
admission.release();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// One admission unit per run; steps run synchronously within it, so this gates heavy work
|
||||
// without the pool-within-pool risk of queueing each tool call.
|
||||
@@ -284,7 +324,10 @@ public class PolicyEngine {
|
||||
outputs.addAll(
|
||||
sinkFor(destination)
|
||||
.deliver(
|
||||
new OutputDelivery(runId, run.getPolicyId()),
|
||||
// The inputs travel with the delivery: a storage sink
|
||||
// anchors ownership and placement on the file the run
|
||||
// came from.
|
||||
new OutputDelivery(runId, run.getPolicyId(), inputs),
|
||||
result.files(),
|
||||
destination));
|
||||
}
|
||||
|
||||
+39
-6
@@ -3,6 +3,7 @@ package stirling.software.proprietary.policy.engine;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -10,6 +11,7 @@ import org.springframework.stereotype.Service;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.input.InputSource;
|
||||
import stirling.software.proprietary.policy.input.ResolvedInput;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
@@ -42,6 +44,18 @@ public class PolicyRunner {
|
||||
private final SourceStore sourceStore;
|
||||
private final SourceDocCounter docCounter;
|
||||
private final ProcessedLedger processedLedger;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
/**
|
||||
* One admission gate per sweep: every run still starts (and is visible) immediately, but only
|
||||
* this many execute at once. A sweep fans out one run per file, and a folderful dispatched all
|
||||
* at once just queues at the pipeline's slowest tool — same total time, nothing visibly done
|
||||
* until the end. Paced, completions arrive steadily from the first file onward.
|
||||
*/
|
||||
private Semaphore sweepAdmission() {
|
||||
int concurrency = applicationProperties.getPolicies().getSweepConcurrency();
|
||||
return concurrency > 0 ? new Semaphore(concurrency) : null;
|
||||
}
|
||||
|
||||
/** Full-listing sweep over every input: resolve each source, then reconcile the ledger. */
|
||||
public SweepOutcome run(Policy policy) {
|
||||
@@ -74,13 +88,21 @@ public class PolicyRunner {
|
||||
public SweepOutcome run(Policy policy, List<PipelineInput> inputs, SweepKind sweep) {
|
||||
long sweepStart = System.currentTimeMillis();
|
||||
PolicySweep context = new PolicySweep(policy.id(), sweep, processedLedger);
|
||||
Semaphore admission = sweepAdmission();
|
||||
List<String> runIds = new ArrayList<>();
|
||||
if (inputs.isEmpty()) {
|
||||
// Generator pipeline: one run with no input. Still fall through to the cleanup
|
||||
// below so rows recorded for its folder outputs are pruned like anything else,
|
||||
// instead of accumulating until the policy is deleted.
|
||||
// Generator pipeline: no input, so neither a source nor a document to attribute to.
|
||||
runIds.add(startRun(policy, null, null, PolicyInputs.of(List.of()), unused -> {}));
|
||||
runIds.add(
|
||||
startRun(
|
||||
policy,
|
||||
null,
|
||||
null,
|
||||
PolicyInputs.of(List.of()),
|
||||
unused -> {},
|
||||
admission));
|
||||
}
|
||||
for (PipelineInput input : inputs) {
|
||||
String sourceId = input.sourceId();
|
||||
@@ -100,7 +122,7 @@ public class PolicyRunner {
|
||||
context.vetoCleanup();
|
||||
continue;
|
||||
}
|
||||
runIds.addAll(pullAndRun(policy, sourceId, source.toInputSpec(), context));
|
||||
runIds.addAll(pullAndRun(policy, sourceId, source.toInputSpec(), context, admission));
|
||||
}
|
||||
boolean fullPolicy = inputs.size() == policy.inputs().size();
|
||||
if (fullPolicy && context.cleanupAllowed()) {
|
||||
@@ -140,7 +162,11 @@ public class PolicyRunner {
|
||||
* this sweep's ledger cleanup.
|
||||
*/
|
||||
private List<String> pullAndRun(
|
||||
Policy policy, String sourceId, InputSpec spec, PolicySweep context) {
|
||||
Policy policy,
|
||||
String sourceId,
|
||||
InputSpec spec,
|
||||
PolicySweep context,
|
||||
Semaphore admission) {
|
||||
InputSource source = sourceFor(spec);
|
||||
if (source == null) {
|
||||
log.warn(
|
||||
@@ -174,7 +200,8 @@ public class PolicyRunner {
|
||||
sourceId,
|
||||
unit.fileIdentity(),
|
||||
unit.inputs(),
|
||||
unit.onComplete()));
|
||||
unit.onComplete(),
|
||||
admission));
|
||||
docsFed += unit.inputs().primary().size();
|
||||
}
|
||||
docCounter.record(sourceId, docsFed);
|
||||
@@ -186,11 +213,17 @@ public class PolicyRunner {
|
||||
String sourceId,
|
||||
String fileIdentity,
|
||||
PolicyInputs inputs,
|
||||
Consumer<Boolean> onComplete) {
|
||||
Consumer<Boolean> onComplete,
|
||||
Semaphore admission) {
|
||||
log.info("Running policy {} ({})", policy.id(), policy.name());
|
||||
PolicyRunHandle handle =
|
||||
policyEngine.runPolicy(
|
||||
policy, inputs, PolicyProgressListener.NOOP, sourceId, fileIdentity);
|
||||
policy,
|
||||
inputs,
|
||||
PolicyProgressListener.NOOP,
|
||||
sourceId,
|
||||
fileIdentity,
|
||||
admission);
|
||||
handle.completion()
|
||||
.whenComplete((run, throwable) -> onComplete.accept(succeeded(run, throwable)));
|
||||
return handle.runId();
|
||||
|
||||
+72
-10
@@ -9,6 +9,7 @@ import java.nio.file.Path;
|
||||
import java.nio.file.SimpleFileVisitor;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
@@ -32,11 +33,13 @@ import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
* {@link ResolveContext} ledger rather than moved aside, so nothing accumulates in a work
|
||||
* directory. Options: "mode" is "consume" (default: a processed file is removed once every policy
|
||||
* that claimed it has settled successfully and it is still the version that ran; failures stay in
|
||||
* place and are not retried until they change) or "snapshot" (stateless, every run sees the full
|
||||
* set); "recursive" descends into subdirectories; "identity" is "stat" (default, any size/mtime
|
||||
* change is a new version) or "hash" (content-verified, so a touch does not reprocess). Hidden
|
||||
* files and directories, including the legacy {@code .stirling} work dir, are never picked up, and
|
||||
* files mid-write are skipped by the readiness check.
|
||||
* place and are not retried until they change), "track" (the same claim-once-per-version tracking
|
||||
* with no removal, for a directory the user owns and expects to stay intact - their Downloads, a
|
||||
* scanner drop), or "snapshot" (stateless, every run sees the full set); "recursive" descends into
|
||||
* subdirectories; "identity" is "stat" (default, any size/mtime change is a new version) or "hash"
|
||||
* (content-verified, so a touch does not reprocess). Hidden files and directories, including the
|
||||
* legacy {@code .stirling} work dir, are never picked up, and files mid-write are skipped by the
|
||||
* readiness check.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -81,6 +84,10 @@ public class FolderInputSource implements InputSource {
|
||||
}
|
||||
Path canonicalDir = FolderIdentities.canonicalDir(inputDir);
|
||||
List<Path> present = listFiles(inputDir, config.recursive());
|
||||
// Smallest first: a sweep's first results should appear within seconds of it starting,
|
||||
// not after the largest document in the folder. Unsizeable entries (vanished mid-listing)
|
||||
// sort last and resolve their own fate at claim time.
|
||||
present.sort(Comparator.comparingLong(FolderInputSource::sizeForOrdering));
|
||||
|
||||
if (config.snapshot()) {
|
||||
List<ResolvedInput> work = new ArrayList<>();
|
||||
@@ -99,6 +106,17 @@ public class FolderInputSource implements InputSource {
|
||||
|
||||
List<ResolvedInput> work = new ArrayList<>();
|
||||
for (Path file : present) {
|
||||
// "limit" caps how much one sweep takes on, not what it observes: the full listing is
|
||||
// still reported above so presence cleanup stays honest, and the files beyond the cap
|
||||
// keep their ledger rows and are picked up by later sweeps.
|
||||
if (config.limit() > 0 && work.size() >= config.limit()) {
|
||||
log.debug(
|
||||
"Folder {} has more ready files than this sweep's limit of {}; the rest"
|
||||
+ " follow on later sweeps",
|
||||
inputDir,
|
||||
config.limit());
|
||||
break;
|
||||
}
|
||||
if (!readinessChecker.isReady(file)) {
|
||||
continue;
|
||||
}
|
||||
@@ -117,13 +135,29 @@ public class FolderInputSource implements InputSource {
|
||||
if (!claimed) {
|
||||
continue;
|
||||
}
|
||||
String claimedGate = gate;
|
||||
work.add(
|
||||
ResolvedInput.forFile(
|
||||
PolicyInputs.of(List.of(fileResource(file))),
|
||||
identity,
|
||||
success ->
|
||||
completeConsumed(
|
||||
ctx, identity, file, gate, contentHash, success)));
|
||||
success -> {
|
||||
if (config.track()) {
|
||||
// Track mode never removes the input: the directory belongs to
|
||||
// the user (their Downloads, a scan drop), so a processed file
|
||||
// is recorded and left exactly where they put it. The hash is
|
||||
// taken at the claimed version only — a file replaced or
|
||||
// removed mid-run settles null rather than recording the
|
||||
// replacement's bytes under the old gate.
|
||||
ctx.settle(
|
||||
identity,
|
||||
claimedGate,
|
||||
claimedHash(file, claimedGate, contentHash),
|
||||
success);
|
||||
return;
|
||||
}
|
||||
completeConsumed(
|
||||
ctx, identity, file, claimedGate, contentHash, success);
|
||||
}));
|
||||
}
|
||||
return work;
|
||||
}
|
||||
@@ -209,6 +243,15 @@ public class FolderInputSource implements InputSource {
|
||||
}
|
||||
|
||||
/** Every non-hidden regular file in the source, readable or not. */
|
||||
/** The file's size for sweep ordering; unreadable reads as largest, sorting it last. */
|
||||
private static long sizeForOrdering(Path file) {
|
||||
try {
|
||||
return Files.size(file);
|
||||
} catch (IOException e) {
|
||||
return Long.MAX_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
private static List<Path> listFiles(Path inputDir, boolean recursive) throws IOException {
|
||||
List<Path> files = new ArrayList<>();
|
||||
if (!recursive) {
|
||||
@@ -271,15 +314,23 @@ public class FolderInputSource implements InputSource {
|
||||
};
|
||||
}
|
||||
|
||||
record FolderConfig(Path directory, boolean snapshot, boolean recursive, boolean hashIdentity) {
|
||||
record FolderConfig(
|
||||
Path directory,
|
||||
boolean snapshot,
|
||||
boolean track,
|
||||
boolean recursive,
|
||||
boolean hashIdentity,
|
||||
int limit) {
|
||||
|
||||
private static final String DIRECTORY_OPTION = "directory";
|
||||
private static final String MODE_OPTION = "mode";
|
||||
private static final String MODE_SNAPSHOT = "snapshot";
|
||||
private static final String MODE_TRACK = "track";
|
||||
private static final String RECURSIVE_OPTION = "recursive";
|
||||
private static final String IDENTITY_OPTION = "identity";
|
||||
private static final String IDENTITY_STAT = "stat";
|
||||
private static final String IDENTITY_HASH = "hash";
|
||||
private static final String LIMIT_OPTION = "limit";
|
||||
|
||||
static FolderConfig from(Map<String, Object> options) {
|
||||
Object directory = options.get(DIRECTORY_OPTION);
|
||||
@@ -288,6 +339,7 @@ public class FolderInputSource implements InputSource {
|
||||
}
|
||||
Object mode = options.get(MODE_OPTION);
|
||||
boolean snapshot = mode != null && MODE_SNAPSHOT.equals(mode.toString());
|
||||
boolean track = mode != null && MODE_TRACK.equals(mode.toString());
|
||||
Object recursive = options.get(RECURSIVE_OPTION);
|
||||
boolean recurse = recursive != null && Boolean.parseBoolean(recursive.toString());
|
||||
Object identity = options.get(IDENTITY_OPTION);
|
||||
@@ -298,7 +350,17 @@ public class FolderInputSource implements InputSource {
|
||||
throw new IllegalArgumentException(
|
||||
"folder input 'identity' must be 'stat' or 'hash'");
|
||||
}
|
||||
return new FolderConfig(Path.of(directory.toString()), snapshot, recurse, hash);
|
||||
Object limit = options.get(LIMIT_OPTION);
|
||||
int max = 0;
|
||||
if (limit != null && !limit.toString().isBlank()) {
|
||||
try {
|
||||
max = Math.max(0, Integer.parseInt(limit.toString().trim()));
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IllegalArgumentException("folder input 'limit' must be a number", e);
|
||||
}
|
||||
}
|
||||
return new FolderConfig(
|
||||
Path.of(directory.toString()), snapshot, track, recurse, hash, max);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+214
@@ -0,0 +1,214 @@
|
||||
package stirling.software.proprietary.policy.input;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.core.io.AbstractResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.ledger.StorageFileIdentities;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.storage.model.FilePurpose;
|
||||
import stirling.software.proprietary.storage.model.StoredFile;
|
||||
import stirling.software.proprietary.storage.provider.StorageProvider;
|
||||
import stirling.software.proprietary.storage.repository.FolderRepository;
|
||||
import stirling.software.proprietary.storage.repository.StoredFileRepository;
|
||||
|
||||
/**
|
||||
* Reads input files from a folder in the app's file storage — the input side of a processing
|
||||
* folder. Each stored file is one unit of work, claimed through the ledger at its current content
|
||||
* version ({@code updatedAt} + size), so an unchanged file never reruns while a re-uploaded or
|
||||
* edited one is picked up again. Files are tracked in place and never deleted.
|
||||
*
|
||||
* <p>A run whose output replaces the file's content in place bumps that version; the completion
|
||||
* hook settles the ledger at the file's post-run version so the next sweep does not re-ingest the
|
||||
* run's own output. Purpose-specific files (signing artifacts etc.) are never picked up.
|
||||
*
|
||||
* <p>Options: {@code folderId} — the storage folder's UUID.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class StorageFolderInputSource implements InputSource {
|
||||
|
||||
private static final String TYPE = "storage-folder";
|
||||
|
||||
private final StoredFileRepository storedFileRepository;
|
||||
private final FolderRepository folderRepository;
|
||||
private final StorageProvider storageProvider;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean supports(InputSpec spec) {
|
||||
return spec != null && TYPE.equals(spec.type());
|
||||
}
|
||||
|
||||
/** Fails fast at save time: storage must be on and the folder must exist. */
|
||||
@Override
|
||||
public void validate(InputSpec spec) {
|
||||
if (!applicationProperties.getSecurity().isEnableLogin()
|
||||
|| !applicationProperties.getStorage().isEnabled()) {
|
||||
throw new IllegalArgumentException("file storage is not enabled on this server");
|
||||
}
|
||||
if (!folderRepository.existsById(folderId(spec))) {
|
||||
throw new IllegalArgumentException(
|
||||
"unknown storage folder: " + spec.options().get("folderId"));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResolvedInput> resolve(InputSpec spec, ResolveContext ctx) throws IOException {
|
||||
UUID folderId = folderId(spec);
|
||||
List<StoredFile> files =
|
||||
storedFileRepository.findAllByFolderId(folderId).stream()
|
||||
.filter(StorageFolderInputSource::ingestible)
|
||||
.toList();
|
||||
|
||||
ctx.reportPresent(files.stream().map(StorageFolderInputSource::identity).toList());
|
||||
|
||||
List<ResolvedInput> work = new ArrayList<>();
|
||||
for (StoredFile file : files) {
|
||||
String identity = identity(file);
|
||||
String gate = gate(file);
|
||||
// The hash tier turns metadata-only gate bumps (a folder move, a rename) into a gate
|
||||
// refresh instead of a reprocess; only genuinely new content runs again.
|
||||
boolean claimed;
|
||||
try {
|
||||
claimed =
|
||||
ctx.claim(
|
||||
identity,
|
||||
gate,
|
||||
() -> StorageFileIdentities.contentHash(storageProvider, file));
|
||||
} catch (RuntimeException e) {
|
||||
// One unreadable blob (missing key, provider hiccup) skips that file — never the
|
||||
// whole sweep. Unclaimed, so the next sweep tries it again.
|
||||
log.debug("Could not read {} for its content hash: {}", identity, e.getMessage());
|
||||
continue;
|
||||
}
|
||||
if (!claimed) {
|
||||
continue;
|
||||
}
|
||||
Long fileId = file.getId();
|
||||
work.add(
|
||||
ResolvedInput.forFile(
|
||||
PolicyInputs.of(List.of(new StoredFileResource(storageProvider, file))),
|
||||
identity,
|
||||
success ->
|
||||
settleAtCurrentVersion(ctx, fileId, identity, gate, success)));
|
||||
}
|
||||
return work;
|
||||
}
|
||||
|
||||
/**
|
||||
* Settle at whatever version the file carries after the run, not the one that was claimed: an
|
||||
* in-place output bumped {@code updatedAt}, and settling at the old gate would make the next
|
||||
* sweep read the run's own output as a fresh edit. A file deleted mid-run settles at the
|
||||
* claimed gate; presence cleanup prunes its row.
|
||||
*/
|
||||
private void settleAtCurrentVersion(
|
||||
ResolveContext ctx, Long fileId, String identity, String claimedGate, boolean success) {
|
||||
StoredFile current = storedFileRepository.findById(fileId).orElse(null);
|
||||
if (current == null) {
|
||||
ctx.settle(identity, claimedGate, null, success);
|
||||
return;
|
||||
}
|
||||
// Settle with the content hash so a later metadata-only bump (move/rename) refreshes the
|
||||
// gate instead of reprocessing. Hash failures fall back to gate-only semantics.
|
||||
String finalContentHash = null;
|
||||
try {
|
||||
finalContentHash = StorageFileIdentities.contentHash(storageProvider, current);
|
||||
} catch (RuntimeException e) {
|
||||
log.debug("Could not hash {} at settle: {}", identity, e.getMessage());
|
||||
}
|
||||
ctx.settle(identity, gate(current), finalContentHash, success);
|
||||
}
|
||||
|
||||
/** Only generic user files are processed — purpose-bound artifacts belong to their feature. */
|
||||
private static boolean ingestible(StoredFile file) {
|
||||
return file.getPurpose() == null || file.getPurpose() == FilePurpose.GENERIC;
|
||||
}
|
||||
|
||||
private static String identity(StoredFile file) {
|
||||
return StorageFileIdentities.identity(file);
|
||||
}
|
||||
|
||||
private static String gate(StoredFile file) {
|
||||
return StorageFileIdentities.gate(file);
|
||||
}
|
||||
|
||||
private static UUID folderId(InputSpec spec) {
|
||||
Object raw = spec.options().get("folderId");
|
||||
try {
|
||||
return UUID.fromString(String.valueOf(raw));
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new IllegalArgumentException("storage-folder source needs a folderId", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Streams the stored blob on demand through the storage provider, presenting the user-visible
|
||||
* filename (the storage key is opaque). Content is not version-pinned: a concurrent in-place
|
||||
* replace is read as-is and reconciled by the gate on the next sweep.
|
||||
*/
|
||||
private static final class StoredFileResource extends AbstractResource
|
||||
implements StoredFileBacked {
|
||||
|
||||
private final StorageProvider storageProvider;
|
||||
private final Long fileId;
|
||||
private final String storageKey;
|
||||
private final String filename;
|
||||
private final long sizeBytes;
|
||||
|
||||
private StoredFileResource(StorageProvider storageProvider, StoredFile file) {
|
||||
this.storageProvider = storageProvider;
|
||||
this.fileId = file.getId();
|
||||
this.storageKey = file.getStorageKey();
|
||||
this.filename = file.getOriginalFilename();
|
||||
this.sizeBytes = file.getSizeBytes();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long storedFileId() {
|
||||
return fileId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream getInputStream() throws IOException {
|
||||
return storageProvider.load(storageKey).getInputStream();
|
||||
}
|
||||
|
||||
/** Listed just now; readers get a precise error from {@link #getInputStream} instead. */
|
||||
@Override
|
||||
public boolean exists() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long contentLength() {
|
||||
return sizeBytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFilename() {
|
||||
return filename;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "stored file " + filename + " (" + storageKey + ")";
|
||||
}
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
package stirling.software.proprietary.policy.input;
|
||||
|
||||
/**
|
||||
* Marks an input {@link org.springframework.core.io.Resource} as backed by a row in app storage, so
|
||||
* an output sink writing back to storage (a new version of the input) can find the origin file.
|
||||
*/
|
||||
public interface StoredFileBacked {
|
||||
|
||||
Long storedFileId();
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package stirling.software.proprietary.policy.ledger;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.security.DigestInputStream;
|
||||
import java.security.MessageDigest;
|
||||
|
||||
import stirling.software.proprietary.billing.ContentHasher;
|
||||
import stirling.software.proprietary.storage.model.StoredFile;
|
||||
import stirling.software.proprietary.storage.provider.StorageProvider;
|
||||
|
||||
/**
|
||||
* Ledger identity and version tiers for files in app storage, shared by the storage-folder input
|
||||
* source and the storage output sink so a produced file is recorded in exactly the shape the next
|
||||
* sweep computes. Identity is the immutable row id. The cheap gate pairs {@code updatedAt} with
|
||||
* size — but metadata-only writes (a folder move, a rename) bump {@code updatedAt} too, so the gate
|
||||
* over-triggers by design and the content hash is the second tier that turns those into a gate
|
||||
* refresh instead of a reprocess.
|
||||
*/
|
||||
public final class StorageFileIdentities {
|
||||
|
||||
private StorageFileIdentities() {}
|
||||
|
||||
public static String identity(StoredFile file) {
|
||||
return "storage:" + file.getId();
|
||||
}
|
||||
|
||||
public static String gate(StoredFile file) {
|
||||
return file.getUpdatedAt() + ":" + file.getSizeBytes();
|
||||
}
|
||||
|
||||
/** SHA-256 of the stored blob; {@link UncheckedIOException} on read failure (propagates). */
|
||||
public static String contentHash(StorageProvider storageProvider, StoredFile file) {
|
||||
MessageDigest digest = ContentHasher.newSha256();
|
||||
try (InputStream is =
|
||||
new DigestInputStream(
|
||||
storageProvider.load(file.getStorageKey()).getInputStream(), digest)) {
|
||||
is.transferTo(java.io.OutputStream.nullOutputStream());
|
||||
return ContentHasher.toHex(digest.digest());
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException("could not hash stored file " + identity(file), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
+24
-3
@@ -21,7 +21,13 @@ public record PolicyRunView(
|
||||
Boolean errorSubscribed,
|
||||
List<ResultFile> outputs,
|
||||
/** When the run was created, epoch millis, so a rediscovered run shows its real age. */
|
||||
long createdAt) {
|
||||
long createdAt,
|
||||
/**
|
||||
* The input document's display name, when the run's source recorded one — a client showing
|
||||
* live runs needs something to call them before any output exists. Null for uploads and
|
||||
* cross-node views, whose identity is not name-shaped.
|
||||
*/
|
||||
String fileName) {
|
||||
|
||||
public static PolicyRunView of(PolicyRun run) {
|
||||
return new PolicyRunView(
|
||||
@@ -34,7 +40,21 @@ public record PolicyRunView(
|
||||
run.getErrorCode(),
|
||||
run.getErrorSubscribed(),
|
||||
run.getOutputs(),
|
||||
run.getCreatedAt().toEpochMilli());
|
||||
run.getCreatedAt().toEpochMilli(),
|
||||
fileNameOf(run.getFileIdentity()));
|
||||
}
|
||||
|
||||
/**
|
||||
* The trailing path segment of a path-shaped file identity (a folder source's identity is the
|
||||
* document's absolute path). Identities that aren't path-shaped pass through whole — for a
|
||||
* storage source that is still a recognisable reference, and null stays null.
|
||||
*/
|
||||
private static String fileNameOf(String fileIdentity) {
|
||||
if (fileIdentity == null || fileIdentity.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
int cut = Math.max(fileIdentity.lastIndexOf('/'), fileIdentity.lastIndexOf('\\'));
|
||||
return cut < 0 ? fileIdentity : fileIdentity.substring(cut + 1);
|
||||
}
|
||||
|
||||
/** Cross-node view from a shared job-store entry; step cursor is node-local so it reads 0. */
|
||||
@@ -63,6 +83,7 @@ public record PolicyRunView(
|
||||
null,
|
||||
null,
|
||||
outputs,
|
||||
createdAt);
|
||||
createdAt,
|
||||
null);
|
||||
}
|
||||
}
|
||||
|
||||
+16
-2
@@ -1,8 +1,22 @@
|
||||
package stirling.software.proprietary.policy.output;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
|
||||
/**
|
||||
* Context for one run's output delivery. {@code policyId} is null for ad-hoc pipelines; when
|
||||
* present, sinks record outputs in the processed-file ledger so the producing policy does not
|
||||
* re-ingest them.
|
||||
* re-ingest them. {@code inputs} carries the run's inputs so a sink that writes back to where the
|
||||
* input lives (e.g. a new version of a stored file) can correlate output to origin.
|
||||
*/
|
||||
public record OutputDelivery(String runId, String policyId) {}
|
||||
public record OutputDelivery(String runId, String policyId, PolicyInputs inputs) {
|
||||
|
||||
public OutputDelivery {
|
||||
inputs = inputs == null ? PolicyInputs.of(List.of()) : inputs;
|
||||
}
|
||||
|
||||
public OutputDelivery(String runId, String policyId) {
|
||||
this(runId, policyId, null);
|
||||
}
|
||||
}
|
||||
|
||||
+277
@@ -0,0 +1,277 @@
|
||||
package stirling.software.proprietary.policy.output;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.MediaTypeFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.job.ResultFile;
|
||||
import stirling.software.proprietary.policy.input.StoredFileBacked;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
import stirling.software.proprietary.policy.ledger.StorageFileIdentities;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.storage.model.Folder;
|
||||
import stirling.software.proprietary.storage.model.StoredFile;
|
||||
import stirling.software.proprietary.storage.provider.StorageProvider;
|
||||
import stirling.software.proprietary.storage.repository.FolderRepository;
|
||||
import stirling.software.proprietary.storage.repository.StoredFileRepository;
|
||||
import stirling.software.proprietary.storage.service.FileStorageService;
|
||||
|
||||
/**
|
||||
* Writes a run's outputs back into app storage — the output side of a processing folder. Two modes,
|
||||
* chosen per policy via {@code mode}:
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@code new_version} (default): the single output replaces the input file's content in
|
||||
* place, under the input's own name. The producing source settles the ledger at the bumped
|
||||
* version, so the folder does not re-ingest the run's own output.
|
||||
* <li>{@code new_file}: each output is stored as a new file and placed in the folder given by
|
||||
* {@code folderId} (default: the input file's folder). The file is stored unplaced first and
|
||||
* recorded in the processed-file ledger before it becomes visible in the folder, so a sweep
|
||||
* can never claim the producing policy's own output.
|
||||
* </ul>
|
||||
*
|
||||
* <p>Ownership follows the input: outputs are stored as the input file's owner, within their quota.
|
||||
* A run fed from outside storage — a directory on disk — has no such anchor, so it is stored as the
|
||||
* owner of the {@code folderId} its outputs are placed in, and must name one.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class StorageOutputSink implements PolicyOutputSink {
|
||||
|
||||
static final String TYPE = "storage";
|
||||
static final String MODE_OPTION = "mode";
|
||||
static final String FOLDER_OPTION = "folderId";
|
||||
static final String NEW_VERSION = "new_version";
|
||||
static final String NEW_FILE = "new_file";
|
||||
|
||||
private final StoredFileRepository storedFileRepository;
|
||||
private final FolderRepository folderRepository;
|
||||
private final FileStorageService fileStorageService;
|
||||
private final ProcessedLedger processedLedger;
|
||||
private final StorageProvider storageProvider;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean supports(OutputSpec spec) {
|
||||
return spec != null && TYPE.equals(spec.type());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(OutputSpec spec) {
|
||||
if (!applicationProperties.getSecurity().isEnableLogin()
|
||||
|| !applicationProperties.getStorage().isEnabled()) {
|
||||
throw new IllegalArgumentException("file storage is not enabled on this server");
|
||||
}
|
||||
String mode = modeOf(spec);
|
||||
if (!NEW_VERSION.equals(mode) && !NEW_FILE.equals(mode)) {
|
||||
throw new IllegalArgumentException("unknown storage output mode: " + mode);
|
||||
}
|
||||
UUID folderId = folderIdOf(spec);
|
||||
if (folderId != null && !folderRepository.existsById(folderId)) {
|
||||
throw new IllegalArgumentException("unknown storage folder: " + folderId);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ResultFile> deliver(
|
||||
OutputDelivery delivery, List<Resource> outputs, OutputSpec spec) throws IOException {
|
||||
StoredFile origin = originOf(delivery);
|
||||
UUID folderId = folderIdOf(spec);
|
||||
User owner = ownerFor(origin, folderId);
|
||||
List<ResultFile> results = new ArrayList<>();
|
||||
|
||||
// Replacing in place needs a stored row to replace, which a run fed from disk has not got.
|
||||
boolean replaceInPlace =
|
||||
origin != null && NEW_VERSION.equals(modeOf(spec)) && outputs.size() == 1;
|
||||
for (int i = 0; i < outputs.size(); i++) {
|
||||
Resource output = outputs.get(i);
|
||||
StoredFile stored;
|
||||
if (replaceInPlace) {
|
||||
// The output takes the input's place — same row, same name, new content, and
|
||||
// replaceFile keeps the row in whatever folder the user put it in.
|
||||
stored =
|
||||
fileStorageService.replaceFile(
|
||||
origin.getOwner(),
|
||||
origin,
|
||||
new ResourceMultipartFile(output, origin.getOriginalFilename()));
|
||||
} else {
|
||||
stored = storeIntoFolder(delivery, output, i, owner, origin, folderId);
|
||||
}
|
||||
results.add(
|
||||
ResultFile.builder()
|
||||
.fileId(String.valueOf(stored.getId()))
|
||||
.fileName(stored.getOriginalFilename())
|
||||
.contentType(stored.getContentType())
|
||||
.fileSize(stored.getSizeBytes())
|
||||
.build());
|
||||
log.debug(
|
||||
"Wrote policy run {} output to stored file {}",
|
||||
delivery.runId(),
|
||||
stored.getId());
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store first (unplaced — invisible to any folder sweep), record the ledger row, then place
|
||||
* into the folder. The row therefore exists before the file is discoverable, mirroring the disk
|
||||
* folder sink's stage-record-rename order.
|
||||
*/
|
||||
private StoredFile storeIntoFolder(
|
||||
OutputDelivery delivery,
|
||||
Resource output,
|
||||
int index,
|
||||
User owner,
|
||||
StoredFile origin,
|
||||
UUID folderId)
|
||||
throws IOException {
|
||||
String name = OutputNames.safeName(output.getFilename(), index);
|
||||
StoredFile stored =
|
||||
fileStorageService.storeFile(owner, new ResourceMultipartFile(output, name));
|
||||
// Read the origin's placement as a plain id: it is detached here, so touching its lazy
|
||||
// folder association would fail.
|
||||
UUID targetFolder = folderId;
|
||||
if (targetFolder == null && origin != null) {
|
||||
targetFolder = storedFileRepository.findFolderIdByFileId(origin.getId()).orElse(null);
|
||||
}
|
||||
if (targetFolder == null) {
|
||||
return stored;
|
||||
}
|
||||
if (delivery.policyId() != null) {
|
||||
// The placement save below bumps updatedAt past this gate; the content hash is what
|
||||
// lets the next sweep read that bump as "already processed" rather than fresh work.
|
||||
processedLedger.recordOutput(
|
||||
delivery.policyId(),
|
||||
StorageFileIdentities.identity(stored),
|
||||
StorageFileIdentities.gate(stored),
|
||||
StorageFileIdentities.contentHash(storageProvider, stored));
|
||||
}
|
||||
stored.setFolder(folderRepository.getReferenceById(targetFolder));
|
||||
return storedFileRepository.save(stored);
|
||||
}
|
||||
|
||||
/**
|
||||
* The stored file the run's primary input came from, or null when the input came from outside
|
||||
* storage (a directory on disk). Storage outputs anchor to it whenever it exists.
|
||||
*/
|
||||
private StoredFile originOf(OutputDelivery delivery) {
|
||||
return delivery.inputs().primary().stream()
|
||||
.filter(StoredFileBacked.class::isInstance)
|
||||
.map(resource -> ((StoredFileBacked) resource).storedFileId())
|
||||
.flatMap(id -> storedFileRepository.findById(id).stream())
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Who the outputs are stored as, and therefore whose quota they count against. A storage-backed
|
||||
* run follows its input's owner. A run fed from disk has nobody to follow, so it takes the
|
||||
* owner of the folder it is writing into — which is why such a policy must name one.
|
||||
*/
|
||||
private User ownerFor(StoredFile origin, UUID folderId) {
|
||||
if (origin != null) {
|
||||
return origin.getOwner();
|
||||
}
|
||||
if (folderId == null) {
|
||||
throw new IllegalStateException(
|
||||
"storage output from a non-storage input needs a folderId to anchor ownership");
|
||||
}
|
||||
return folderRepository
|
||||
.findById(folderId)
|
||||
.map(Folder::getOwner)
|
||||
.orElseThrow(
|
||||
() -> new IllegalStateException("unknown storage folder: " + folderId));
|
||||
}
|
||||
|
||||
private static String modeOf(OutputSpec spec) {
|
||||
Object mode = spec.options().get(MODE_OPTION);
|
||||
return mode == null || String.valueOf(mode).isBlank() ? NEW_VERSION : String.valueOf(mode);
|
||||
}
|
||||
|
||||
private static UUID folderIdOf(OutputSpec spec) {
|
||||
Object raw = spec.options().get(FOLDER_OPTION);
|
||||
if (raw == null || String.valueOf(raw).isBlank()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return UUID.fromString(String.valueOf(raw));
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new IllegalArgumentException("invalid storage output folderId: " + raw, e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Streams a run output into the storage service's upload seam without buffering it. */
|
||||
private record ResourceMultipartFile(Resource resource, String filename)
|
||||
implements MultipartFile {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "file";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getOriginalFilename() {
|
||||
return filename;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getContentType() {
|
||||
return MediaTypeFactory.getMediaType(filename)
|
||||
.orElse(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
return getSize() == 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSize() {
|
||||
try {
|
||||
return resource.contentLength();
|
||||
} catch (IOException e) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getBytes() throws IOException {
|
||||
try (InputStream is = resource.getInputStream()) {
|
||||
return is.readAllBytes();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream getInputStream() throws IOException {
|
||||
return resource.getInputStream();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void transferTo(java.io.File dest) throws IOException {
|
||||
try (InputStream is = resource.getInputStream()) {
|
||||
java.nio.file.Files.copy(
|
||||
is, dest.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
@@ -11,6 +11,7 @@ import org.springframework.stereotype.Service;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
|
||||
import stirling.software.proprietary.policy.controller.ProcessingFolderController;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
@@ -38,9 +39,12 @@ public class PolicyOverviewService {
|
||||
private final SourceAccessGuard sourceAccessGuard;
|
||||
|
||||
public PoliciesOverviewResponse overview() {
|
||||
// Processing folders are the editor's own surface (ProcessingFolderController); the
|
||||
// portal's pipelines overview never sees them.
|
||||
List<Policy> policies =
|
||||
policyAccessGuard.visibleFrom(policyStore).stream()
|
||||
.filter(PolicyOverviewService::isPipeline)
|
||||
.filter(policy -> !ProcessingFolderController.isProcessingFolder(policy))
|
||||
.toList();
|
||||
Map<String, String> sourceNames = sourceNames();
|
||||
|
||||
|
||||
+26
-2
@@ -7,12 +7,14 @@ import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
|
||||
import stirling.software.proprietary.policy.controller.ProcessingFolderController;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
import stirling.software.proprietary.util.SecretMasker;
|
||||
@@ -34,8 +36,30 @@ public class SourceOverviewService {
|
||||
private final SourceDocCounter docCounter;
|
||||
|
||||
public SourcesResponse overview() {
|
||||
List<Source> sources = sourceAccessGuard.visibleFrom(sourceStore);
|
||||
List<Policy> policies = policyAccessGuard.visibleFrom(policyStore);
|
||||
// Processing folders (source + policy pairs) are the editor's own surface, served by
|
||||
// ProcessingFolderController; the portal's sources/pipelines views never see them. The
|
||||
// pair's source is hidden by reference, not by type alone: a disk-backed processing
|
||||
// folder's source shares its type with ordinary folder-watch sources, and hiding those
|
||||
// wholesale would take a real portal feature with it. The type filter stays as a backstop
|
||||
// for a pair-half orphaned by a deleted policy.
|
||||
List<Policy> visiblePolicies = policyAccessGuard.visibleFrom(policyStore);
|
||||
Set<String> processingFolderSourceIds =
|
||||
visiblePolicies.stream()
|
||||
.filter(ProcessingFolderController::isProcessingFolder)
|
||||
.flatMap(policy -> policy.sourceIds().stream())
|
||||
.collect(Collectors.toSet());
|
||||
List<Source> sources =
|
||||
sourceAccessGuard.visibleFrom(sourceStore).stream()
|
||||
.filter(
|
||||
source ->
|
||||
!ProcessingFolderController.SOURCE_TYPE.equals(
|
||||
source.type()))
|
||||
.filter(source -> !processingFolderSourceIds.contains(source.id()))
|
||||
.toList();
|
||||
List<Policy> policies =
|
||||
visiblePolicies.stream()
|
||||
.filter(policy -> !ProcessingFolderController.isProcessingFolder(policy))
|
||||
.toList();
|
||||
|
||||
Map<String, List<Policy>> referencesBySource = referencesBySource(policies);
|
||||
Map<String, DocStats> docStats =
|
||||
|
||||
+11
@@ -2,6 +2,7 @@ package stirling.software.proprietary.storage.repository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
@@ -60,6 +61,16 @@ public interface StoredFileRepository extends JpaRepository<StoredFile, Long> {
|
||||
|
||||
List<StoredFile> findAllByOwner(User owner);
|
||||
|
||||
/** Every file placed in the given storage folder — the working set of a processing folder. */
|
||||
List<StoredFile> findAllByFolderId(UUID folderId);
|
||||
|
||||
/**
|
||||
* A file's folder placement as a plain id. Reads the FK directly so callers outside a
|
||||
* transaction never touch the lazy {@code folder} association.
|
||||
*/
|
||||
@Query("SELECT sf.folder.id FROM StoredFile sf WHERE sf.id = :fileId")
|
||||
Optional<UUID> findFolderIdByFileId(@Param("fileId") Long fileId);
|
||||
|
||||
/**
|
||||
* Bulk lookup used by the folder-placement controller. Returns only files owned by {@code
|
||||
* owner}; ids that don't exist or that belong to another user are silently dropped so the
|
||||
|
||||
+12
@@ -49,6 +49,7 @@ import stirling.software.proprietary.storage.provider.StorageProvider;
|
||||
import stirling.software.proprietary.storage.provider.StoredObject;
|
||||
import stirling.software.proprietary.storage.repository.FileShareAccessRepository;
|
||||
import stirling.software.proprietary.storage.repository.FileShareRepository;
|
||||
import stirling.software.proprietary.storage.repository.FolderRepository;
|
||||
import stirling.software.proprietary.storage.repository.StorageCleanupEntryRepository;
|
||||
import stirling.software.proprietary.storage.repository.StoredFileRepository;
|
||||
|
||||
@@ -63,6 +64,7 @@ public class FileStorageService {
|
||||
Pattern.compile("^[^\\s@]+@[^\\s@]+\\.[^\\s@]{2,}$");
|
||||
|
||||
private final StoredFileRepository storedFileRepository;
|
||||
private final FolderRepository folderRepository;
|
||||
private final FileShareRepository fileShareRepository;
|
||||
private final FileShareAccessRepository fileShareAccessRepository;
|
||||
private final UserRepository userRepository;
|
||||
@@ -218,6 +220,16 @@ public class FileStorageService {
|
||||
applyAuditMetadata(existing, auditObject);
|
||||
}
|
||||
|
||||
// The entity is often detached (policy runs deliver on worker threads with no open
|
||||
// persistence context), and its lazy folder association is then an unreadable proxy
|
||||
// from a closed session — merging that drops the FK, silently moving the file to the
|
||||
// file-manager root. Re-anchor the placement as a fresh reference, read by plain id.
|
||||
existing.setFolder(
|
||||
storedFileRepository
|
||||
.findFolderIdByFileId(existing.getId())
|
||||
.map(folderRepository::getReferenceById)
|
||||
.orElse(null));
|
||||
|
||||
StoredFile updated;
|
||||
try {
|
||||
updated = storedFileRepository.save(existing);
|
||||
|
||||
+1
-1
@@ -359,7 +359,7 @@ class PolicyControllerTest {
|
||||
when(jobOwnershipService.createScopedJobKey("owned")).thenReturn("owned");
|
||||
when(jobOwnershipService.createScopedJobKey("other")).thenReturn("scoped-other");
|
||||
|
||||
List<PolicyRunView> views = controller.listRuns();
|
||||
List<PolicyRunView> views = controller.listRuns(null);
|
||||
|
||||
assertThat(views).hasSize(1);
|
||||
assertThat(views.get(0).runId()).isEqualTo("owned");
|
||||
|
||||
+342
@@ -0,0 +1,342 @@
|
||||
package stirling.software.proprietary.policy.controller;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.proprietary.policy.config.FolderAccessGuard;
|
||||
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
|
||||
import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.PolicyValidator;
|
||||
import stirling.software.proprietary.policy.engine.SweepOutcome;
|
||||
import stirling.software.proprietary.policy.input.InputSource;
|
||||
import stirling.software.proprietary.policy.input.StorageFolderInputSource;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.output.PolicyOutputSink;
|
||||
import stirling.software.proprietary.policy.output.StorageOutputSink;
|
||||
import stirling.software.proprietary.policy.source.InProcessSourceStore;
|
||||
import stirling.software.proprietary.policy.store.InProcessPolicyStore;
|
||||
import stirling.software.proprietary.policy.trigger.PolicyTrigger;
|
||||
import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
|
||||
import stirling.software.proprietary.security.model.User;
|
||||
import stirling.software.proprietary.storage.model.Folder;
|
||||
import stirling.software.proprietary.storage.provider.StorageProvider;
|
||||
import stirling.software.proprietary.storage.repository.FolderRepository;
|
||||
import stirling.software.proprietary.storage.repository.StoredFileRepository;
|
||||
import stirling.software.proprietary.storage.service.FileStorageService;
|
||||
|
||||
/**
|
||||
* Tests for {@link ProcessingFolderController}: the source + policy pair composes and tears down
|
||||
* together, an invalid pipeline rolls the pair back, only the caller's own folders qualify, and the
|
||||
* records stay invisible to the policies surface.
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class ProcessingFolderControllerTest {
|
||||
|
||||
private static final UUID FOLDER_ID = UUID.randomUUID();
|
||||
|
||||
@Mock private PolicyRunner policyRunner;
|
||||
@Mock private PolicyTriggerManager policyTriggerManager;
|
||||
@Mock private ProcessedLedger processedLedger;
|
||||
@Mock private FolderRepository folderRepository;
|
||||
@Mock private FileStorageService fileStorageService;
|
||||
@Mock private StoredFileRepository storedFileRepository;
|
||||
@Mock private StorageProvider storageProvider;
|
||||
@Mock private UserServiceInterface userService;
|
||||
@Mock private PolicyManagementAuthority policyManagementAuthority;
|
||||
@Mock private FolderAccessGuard folderAccessGuard;
|
||||
@Mock private PolicyTrigger folderWatchTrigger;
|
||||
@Mock private InputSource diskFolderSource;
|
||||
@Mock private stirling.software.proprietary.policy.asset.PolicyAssetStore assetStore;
|
||||
@Mock private stirling.software.common.service.ToolChainValidator toolChainValidator;
|
||||
@Mock private PolicyOutputSink diskFolderSink;
|
||||
|
||||
private final InProcessPolicyStore policyStore = new InProcessPolicyStore();
|
||||
private final InProcessSourceStore sourceStore = new InProcessSourceStore();
|
||||
|
||||
private User user;
|
||||
private Folder folder;
|
||||
private ProcessingFolderController controller;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
ApplicationProperties properties = new ApplicationProperties();
|
||||
properties.getSecurity().setEnableLogin(true);
|
||||
properties.getStorage().setEnabled(true);
|
||||
|
||||
user = new User();
|
||||
user.setId(7L);
|
||||
user.setUsername("reece");
|
||||
folder = new Folder();
|
||||
folder.setId(FOLDER_ID);
|
||||
folder.setName("Contracts");
|
||||
folder.setOwner(user);
|
||||
|
||||
lenient().when(fileStorageService.requireAuthenticatedUser()).thenReturn(user);
|
||||
// A disk-backed folder creates a storage folder to deliver its results into, then looks it
|
||||
// up again on the next save. The double has to remember what it stored for that second
|
||||
// lookup to find anything — otherwise every save mints a fresh folder.
|
||||
Map<UUID, Folder> folders = new HashMap<>();
|
||||
folders.put(FOLDER_ID, folder);
|
||||
lenient()
|
||||
.when(folderRepository.saveAndFlush(any(Folder.class)))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
Folder saved = invocation.getArgument(0);
|
||||
folders.put(saved.getId(), saved);
|
||||
return saved;
|
||||
});
|
||||
lenient()
|
||||
.when(folderRepository.findById(any(UUID.class)))
|
||||
.thenAnswer(
|
||||
invocation -> Optional.ofNullable(folders.get(invocation.getArgument(0))));
|
||||
lenient()
|
||||
.when(folderRepository.existsById(any(UUID.class)))
|
||||
.thenAnswer(invocation -> folders.containsKey(invocation.getArgument(0)));
|
||||
lenient().when(userService.getCurrentUsername()).thenReturn("reece");
|
||||
lenient().when(policyManagementAuthority.currentUserTeamId()).thenReturn(3L);
|
||||
lenient()
|
||||
.when(policyRunner.run(any()))
|
||||
.thenReturn(new SweepOutcome(List.of("run-1"), 1, 0, 0, 0));
|
||||
|
||||
PolicyAccessGuard accessGuard =
|
||||
new PolicyAccessGuard(userService, properties, policyManagementAuthority);
|
||||
// The real FolderWatchTrigger is a bean; without one registered the validator reads
|
||||
// "folder-watch" as an unknown trigger type.
|
||||
lenient().when(folderWatchTrigger.type()).thenReturn("folder-watch");
|
||||
// Likewise the disk folder source and sink: real beans in the app, stubbed here so a
|
||||
// disk-backed folder validates without touching the filesystem.
|
||||
lenient().when(diskFolderSource.supports(any())).thenReturn(true);
|
||||
lenient().when(diskFolderSink.supports(any())).thenReturn(true);
|
||||
PolicyValidator validator =
|
||||
new PolicyValidator(
|
||||
List.of(folderWatchTrigger),
|
||||
List.of(
|
||||
new StorageFolderInputSource(
|
||||
storedFileRepository,
|
||||
folderRepository,
|
||||
storageProvider,
|
||||
properties),
|
||||
diskFolderSource),
|
||||
List.of(
|
||||
new StorageOutputSink(
|
||||
storedFileRepository,
|
||||
folderRepository,
|
||||
fileStorageService,
|
||||
processedLedger,
|
||||
storageProvider,
|
||||
properties),
|
||||
diskFolderSink),
|
||||
List.of(),
|
||||
sourceStore,
|
||||
assetStore,
|
||||
toolChainValidator);
|
||||
controller =
|
||||
new ProcessingFolderController(
|
||||
policyStore,
|
||||
sourceStore,
|
||||
validator,
|
||||
policyRunner,
|
||||
policyTriggerManager,
|
||||
processedLedger,
|
||||
folderRepository,
|
||||
fileStorageService,
|
||||
accessGuard,
|
||||
folderAccessGuard,
|
||||
properties);
|
||||
}
|
||||
|
||||
@Test
|
||||
void createComposesAValidatedPairAndSweepsTheBacklog() {
|
||||
var view = controller.save(request(null, "new_version")).getBody();
|
||||
|
||||
assertThat(view.folderId()).isEqualTo(FOLDER_ID.toString());
|
||||
assertThat(view.enabled()).isTrue();
|
||||
Policy stored = policyStore.get(view.id()).orElseThrow();
|
||||
assertThat(ProcessingFolderController.isProcessingFolder(stored)).isTrue();
|
||||
assertThat(stored.owner()).isEqualTo("reece");
|
||||
assertThat(stored.teamId()).isEqualTo(3L);
|
||||
assertThat(stored.inputs()).hasSize(1);
|
||||
var source = sourceStore.get(stored.inputs().get(0).sourceId()).orElseThrow();
|
||||
assertThat(source.type()).isEqualTo("storage-folder");
|
||||
assertThat(source.options()).containsEntry("folderId", FOLDER_ID.toString());
|
||||
verify(policyRunner).run(stored);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aDiskFolderIsWatchedSoArrivalsProcessThemselves() {
|
||||
var view =
|
||||
controller
|
||||
.save(
|
||||
new ProcessingFolderController.SaveProcessingFolderRequest(
|
||||
null,
|
||||
null,
|
||||
"/tmp/Downloads",
|
||||
true,
|
||||
List.of(
|
||||
new PipelineStep(
|
||||
"/api/v1/misc/flatten",
|
||||
Map.of("flattenOnlyForms", false),
|
||||
Map.of())),
|
||||
Map.of()))
|
||||
.getBody();
|
||||
|
||||
Policy stored = policyStore.get(view.id()).orElseThrow();
|
||||
// Without a trigger the engine treats the policy as manual-only: the creating sweep would
|
||||
// run and the directory would never be processed again.
|
||||
assertThat(stored.inputs()).hasSize(1);
|
||||
assertThat(stored.inputs().get(0).trigger()).isNotNull();
|
||||
assertThat(stored.inputs().get(0).trigger().type()).isEqualTo("folder-watch");
|
||||
var source = sourceStore.get(stored.inputs().get(0).sourceId()).orElseThrow();
|
||||
assertThat(source.type()).isEqualTo("folder");
|
||||
// Never "consume": the directory is the user's own and must stay intact.
|
||||
assertThat(source.options()).containsEntry("mode", "track");
|
||||
assertThat(source.options()).containsEntry("limit", 100);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aDiskFolderWritesItsResultsBesideTheOriginals() {
|
||||
var view =
|
||||
controller
|
||||
.save(
|
||||
new ProcessingFolderController.SaveProcessingFolderRequest(
|
||||
null,
|
||||
null,
|
||||
"/tmp/Downloads",
|
||||
true,
|
||||
List.of(
|
||||
new PipelineStep(
|
||||
"/api/v1/misc/flatten",
|
||||
Map.of("flattenOnlyForms", false),
|
||||
Map.of())),
|
||||
Map.of()))
|
||||
.getBody();
|
||||
|
||||
Policy stored = policyStore.get(view.id()).orElseThrow();
|
||||
// Disk, not app storage: an install with no accounts and no file storage has nothing to
|
||||
// store a result against, and the watched directory is the one place that always exists.
|
||||
assertThat(stored.output().type()).isEqualTo("folder");
|
||||
assertThat(stored.output().options().get("directory").toString())
|
||||
.endsWith("Stirling Processed");
|
||||
// The originals themselves are never written over.
|
||||
assertThat(stored.output().options().get("directory").toString())
|
||||
.isNotEqualTo("/tmp/Downloads");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aDiskFolderReportsTheDirectoryItWatchesNotWhereResultsGo() {
|
||||
var view =
|
||||
controller
|
||||
.save(
|
||||
new ProcessingFolderController.SaveProcessingFolderRequest(
|
||||
null,
|
||||
null,
|
||||
"/tmp/Downloads",
|
||||
true,
|
||||
List.of(
|
||||
new PipelineStep(
|
||||
"/api/v1/misc/flatten",
|
||||
Map.of("flattenOnlyForms", false),
|
||||
Map.of())),
|
||||
Map.of()))
|
||||
.getBody();
|
||||
|
||||
// The client shows this as the folder's address, so it has to be the watched directory —
|
||||
// reading it off the output made the folder advertise its own results subdirectory.
|
||||
assertThat(view.directory()).isEqualTo("/tmp/Downloads");
|
||||
assertThat(view.folderId()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aStorageFolderStaysManualUntilTheArrivalTriggerExists() {
|
||||
var view = controller.save(request(null, "new_version")).getBody();
|
||||
|
||||
assertThat(policyStore.get(view.id()).orElseThrow().inputs().get(0).trigger()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void anInvalidPipelineRollsBackTheSource() {
|
||||
assertThatThrownBy(() -> controller.save(request(null, "no_such_mode")))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.hasMessageContaining("mode");
|
||||
|
||||
assertThat(sourceStore.all()).isEmpty();
|
||||
assertThat(policyStore.all()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void anotherUsersFolderReadsAsNotFound() {
|
||||
User stranger = new User();
|
||||
stranger.setId(8L);
|
||||
folder.setOwner(stranger);
|
||||
|
||||
assertThatThrownBy(() -> controller.save(request(null, "new_version")))
|
||||
.isInstanceOf(ResponseStatusException.class)
|
||||
.hasMessageContaining("No folder");
|
||||
assertThat(sourceStore.all()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteTearsDownThePairAndItsHistory() {
|
||||
var view = controller.save(request(null, "new_version")).getBody();
|
||||
|
||||
controller.delete(view.id());
|
||||
|
||||
assertThat(policyStore.all()).isEmpty();
|
||||
assertThat(sourceStore.all()).isEmpty();
|
||||
verify(processedLedger).clearPolicy(view.id());
|
||||
}
|
||||
|
||||
@Test
|
||||
void listShowsOnlyProcessingFolders() {
|
||||
controller.save(request(null, "new_version"));
|
||||
// An org policy in the same team is not a processing folder and stays invisible here.
|
||||
policyStore.save(
|
||||
new Policy(
|
||||
null,
|
||||
"Security Policy",
|
||||
"reece",
|
||||
true,
|
||||
List.of(),
|
||||
List.of(),
|
||||
stirling.software.proprietary.policy.model.OutputSpec.inline(),
|
||||
3L));
|
||||
|
||||
assertThat(controller.list()).hasSize(1);
|
||||
}
|
||||
|
||||
private static ProcessingFolderController.SaveProcessingFolderRequest request(
|
||||
String id, String mode) {
|
||||
return new ProcessingFolderController.SaveProcessingFolderRequest(
|
||||
id,
|
||||
FOLDER_ID.toString(),
|
||||
null,
|
||||
true,
|
||||
List.of(
|
||||
new PipelineStep(
|
||||
"/api/v1/misc/flatten",
|
||||
Map.of("flattenOnlyForms", false),
|
||||
Map.of())),
|
||||
Map.of("mode", mode));
|
||||
}
|
||||
}
|
||||
+16
-12
@@ -29,6 +29,7 @@ import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.input.InputSource;
|
||||
import stirling.software.proprietary.policy.input.ResolveContext;
|
||||
import stirling.software.proprietary.policy.input.ResolvedInput;
|
||||
@@ -72,19 +73,20 @@ class PolicyRunnerTest {
|
||||
List.of(folderSource),
|
||||
sourceStore,
|
||||
docCounter,
|
||||
processedLedger);
|
||||
processedLedger,
|
||||
new ApplicationProperties());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runsOnceWithNoFilesWhenThePolicyHasNoSources() {
|
||||
Policy policy = policy(List.of());
|
||||
when(policyEngine.runPolicy(eq(policy), any(), any(), any(), any()))
|
||||
when(policyEngine.runPolicy(eq(policy), any(), any(), any(), any(), any()))
|
||||
.thenReturn(new PolicyRunHandle("r", new CompletableFuture<>()));
|
||||
|
||||
runner.run(policy);
|
||||
|
||||
ArgumentCaptor<PolicyInputs> inputs = ArgumentCaptor.forClass(PolicyInputs.class);
|
||||
verify(policyEngine).runPolicy(eq(policy), inputs.capture(), any(), any(), any());
|
||||
verify(policyEngine).runPolicy(eq(policy), inputs.capture(), any(), any(), any(), any());
|
||||
assertTrue(inputs.getValue().primary().isEmpty());
|
||||
// Ledger hygiene still runs: rows recorded for a generator policy's folder outputs
|
||||
// are pruned by its own sweeps rather than accumulating until the policy is deleted.
|
||||
@@ -100,7 +102,8 @@ class PolicyRunnerTest {
|
||||
List.of(folderSource),
|
||||
sourceStore,
|
||||
new InProcessSourceDocCounter(),
|
||||
ledger);
|
||||
ledger,
|
||||
new ApplicationProperties());
|
||||
InputSpec spec = InputSpec.folder("/in");
|
||||
Policy policy = policy(List.of(spec));
|
||||
// One file already processed at its current version, one parked by a failed run.
|
||||
@@ -137,12 +140,12 @@ class PolicyRunnerTest {
|
||||
List.of(
|
||||
ResolvedInput.of(PolicyInputs.of(List.of())),
|
||||
ResolvedInput.of(PolicyInputs.of(List.of()))));
|
||||
when(policyEngine.runPolicy(any(), any(), any(), any(), any()))
|
||||
when(policyEngine.runPolicy(any(), any(), any(), any(), any(), any()))
|
||||
.thenReturn(new PolicyRunHandle("r", new CompletableFuture<>()));
|
||||
|
||||
runner.run(policy);
|
||||
|
||||
verify(policyEngine, times(2)).runPolicy(eq(policy), any(), any(), any(), any());
|
||||
verify(policyEngine, times(2)).runPolicy(eq(policy), any(), any(), any(), any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -154,7 +157,7 @@ class PolicyRunnerTest {
|
||||
when(folderSource.supports(spec)).thenReturn(true);
|
||||
when(folderSource.resolve(eq(spec), any())).thenReturn(List.of(unit));
|
||||
CompletableFuture<PolicyRun> completion = new CompletableFuture<>();
|
||||
when(policyEngine.runPolicy(any(), any(), any(), any(), any()))
|
||||
when(policyEngine.runPolicy(any(), any(), any(), any(), any(), any()))
|
||||
.thenReturn(new PolicyRunHandle("r", completion));
|
||||
|
||||
runner.run(policy);
|
||||
@@ -175,7 +178,7 @@ class PolicyRunnerTest {
|
||||
when(folderSource.supports(spec)).thenReturn(true);
|
||||
when(folderSource.resolve(eq(spec), any())).thenReturn(List.of(unit));
|
||||
CompletableFuture<PolicyRun> completion = new CompletableFuture<>();
|
||||
when(policyEngine.runPolicy(any(), any(), any(), any(), any()))
|
||||
when(policyEngine.runPolicy(any(), any(), any(), any(), any(), any()))
|
||||
.thenReturn(new PolicyRunHandle("r", completion));
|
||||
|
||||
runner.run(policy);
|
||||
@@ -224,12 +227,12 @@ class PolicyRunnerTest {
|
||||
when(folderSource.supports(spec)).thenReturn(true);
|
||||
when(folderSource.resolve(eq(spec), any()))
|
||||
.thenReturn(List.of(ResolvedInput.of(PolicyInputs.of(List.of()))));
|
||||
when(policyEngine.runPolicy(any(), any(), any(), any(), any()))
|
||||
when(policyEngine.runPolicy(any(), any(), any(), any(), any(), any()))
|
||||
.thenReturn(new PolicyRunHandle("r", new CompletableFuture<>()));
|
||||
|
||||
runner.run(policy, SweepKind.LIGHT);
|
||||
|
||||
verify(policyEngine).runPolicy(eq(policy), any(), any(), any(), any());
|
||||
verify(policyEngine).runPolicy(eq(policy), any(), any(), any(), any(), any());
|
||||
verify(processedLedger, never()).markSeen(any(), any());
|
||||
verify(processedLedger, never()).deleteUnseen(any(), anyLong());
|
||||
}
|
||||
@@ -244,13 +247,14 @@ class PolicyRunnerTest {
|
||||
when(folderSource.resolve(eq(broken), any())).thenThrow(new IOException("mount gone"));
|
||||
when(folderSource.resolve(eq(healthy), any()))
|
||||
.thenReturn(List.of(ResolvedInput.of(PolicyInputs.of(List.of()))));
|
||||
when(policyEngine.runPolicy(any(), any(), any(), any(), any()))
|
||||
when(policyEngine.runPolicy(any(), any(), any(), any(), any(), any()))
|
||||
.thenReturn(new PolicyRunHandle("r", new CompletableFuture<>()));
|
||||
|
||||
runner.run(policy);
|
||||
|
||||
verify(policyEngine)
|
||||
.runPolicy(eq(policy), any(), any(), any(), any()); // healthy source still ran
|
||||
.runPolicy(
|
||||
eq(policy), any(), any(), any(), any(), any()); // healthy source still ran
|
||||
verify(processedLedger, never()).deleteUnseen(any(), anyLong()); // history preserved
|
||||
}
|
||||
|
||||
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
package stirling.software.proprietary.policy.input;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.storage.model.FilePurpose;
|
||||
import stirling.software.proprietary.storage.model.StoredFile;
|
||||
import stirling.software.proprietary.storage.provider.StorageProvider;
|
||||
import stirling.software.proprietary.storage.repository.FolderRepository;
|
||||
import stirling.software.proprietary.storage.repository.StoredFileRepository;
|
||||
|
||||
/**
|
||||
* Tests for {@link StorageFolderInputSource}: files are claimed once per content version, an
|
||||
* in-place output settles at its own post-run version instead of re-triggering, a genuine edit is
|
||||
* picked up again, and purpose-bound files are never ingested.
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class StorageFolderInputSourceTest {
|
||||
|
||||
private static final String POLICY = "p1";
|
||||
private static final UUID FOLDER = UUID.randomUUID();
|
||||
private static final LocalDateTime T1 = LocalDateTime.of(2026, 7, 1, 10, 0);
|
||||
private static final LocalDateTime T2 = LocalDateTime.of(2026, 7, 1, 10, 5);
|
||||
|
||||
@Mock private StoredFileRepository storedFileRepository;
|
||||
@Mock private FolderRepository folderRepository;
|
||||
@Mock private StorageProvider storageProvider;
|
||||
|
||||
private StorageFolderInputSource source;
|
||||
private InProcessProcessedLedger ledger;
|
||||
private RecordingContext ctx;
|
||||
// Stands in for the blob store: hashing reads whatever this currently holds.
|
||||
private byte[] blobContent = "content-v1".getBytes();
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws IOException {
|
||||
lenient()
|
||||
.when(storageProvider.load(anyString()))
|
||||
.thenAnswer(invocation -> new ByteArrayResource(blobContent));
|
||||
source =
|
||||
new StorageFolderInputSource(
|
||||
storedFileRepository,
|
||||
folderRepository,
|
||||
storageProvider,
|
||||
storageEnabledProperties());
|
||||
ledger = new InProcessProcessedLedger();
|
||||
ctx = new RecordingContext();
|
||||
}
|
||||
|
||||
@Test
|
||||
void claimsEachFileOncePerContentVersion() throws IOException {
|
||||
StoredFile file = storedFile(1L, "doc.pdf", T1);
|
||||
when(storedFileRepository.findAllByFolderId(FOLDER)).thenReturn(List.of(file));
|
||||
when(storedFileRepository.findById(1L)).thenReturn(Optional.of(file));
|
||||
|
||||
List<ResolvedInput> work = source.resolve(spec(), ctx);
|
||||
|
||||
assertEquals(1, work.size());
|
||||
assertEquals("doc.pdf", work.get(0).inputs().primary().get(0).getFilename());
|
||||
// In flight: a second sweep does not pick it up again.
|
||||
assertTrue(source.resolve(spec(), ctx).isEmpty());
|
||||
|
||||
// Settled at an unchanged version: still nothing new to do.
|
||||
work.get(0).onComplete().accept(true);
|
||||
assertTrue(source.resolve(spec(), ctx).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void anInPlaceOutputDoesNotRetriggerTheFolder() throws IOException {
|
||||
StoredFile file = storedFile(1L, "doc.pdf", T1);
|
||||
when(storedFileRepository.findAllByFolderId(FOLDER)).thenReturn(List.of(file));
|
||||
|
||||
List<ResolvedInput> work = source.resolve(spec(), ctx);
|
||||
|
||||
// The run replaces the file's content in place before completion fires.
|
||||
file.setUpdatedAt(T2);
|
||||
when(storedFileRepository.findById(1L)).thenReturn(Optional.of(file));
|
||||
work.get(0).onComplete().accept(true);
|
||||
|
||||
// The next sweep sees the bumped version already settled — no self-feeding loop.
|
||||
assertTrue(source.resolve(spec(), ctx).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aGenuineEditIsPickedUpAgain() throws IOException {
|
||||
StoredFile file = storedFile(1L, "doc.pdf", T1);
|
||||
when(storedFileRepository.findAllByFolderId(FOLDER)).thenReturn(List.of(file));
|
||||
when(storedFileRepository.findById(1L)).thenReturn(Optional.of(file));
|
||||
|
||||
source.resolve(spec(), ctx).get(0).onComplete().accept(true);
|
||||
|
||||
// The user re-uploads: gate and content both change — fresh work.
|
||||
file.setUpdatedAt(T2);
|
||||
blobContent = "content-v2".getBytes();
|
||||
assertEquals(1, source.resolve(spec(), ctx).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aMetadataOnlyBumpDoesNotReprocess() throws IOException {
|
||||
StoredFile file = storedFile(1L, "doc.pdf", T1);
|
||||
when(storedFileRepository.findAllByFolderId(FOLDER)).thenReturn(List.of(file));
|
||||
when(storedFileRepository.findById(1L)).thenReturn(Optional.of(file));
|
||||
|
||||
source.resolve(spec(), ctx).get(0).onComplete().accept(true);
|
||||
|
||||
// A folder move / rename bumps updatedAt but not the content: the hash tier refreshes the
|
||||
// gate instead of reprocessing.
|
||||
file.setUpdatedAt(T2);
|
||||
assertTrue(source.resolve(spec(), ctx).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void purposeBoundFilesAreNeverIngested() throws IOException {
|
||||
StoredFile signing = storedFile(2L, "contract.pdf", T1);
|
||||
signing.setPurpose(FilePurpose.SIGNING_ORIGINAL);
|
||||
when(storedFileRepository.findAllByFolderId(FOLDER)).thenReturn(List.of(signing));
|
||||
|
||||
assertTrue(source.resolve(spec(), ctx).isEmpty());
|
||||
assertTrue(ctx.present.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFailedRunLeavesTheFileForItsNextVersion() throws IOException {
|
||||
StoredFile file = storedFile(1L, "doc.pdf", T1);
|
||||
when(storedFileRepository.findAllByFolderId(FOLDER)).thenReturn(List.of(file));
|
||||
when(storedFileRepository.findById(1L)).thenReturn(Optional.of(file));
|
||||
|
||||
source.resolve(spec(), ctx).get(0).onComplete().accept(false);
|
||||
|
||||
// Failed at this version: not retried until the content changes.
|
||||
assertTrue(source.resolve(spec(), ctx).isEmpty());
|
||||
file.setUpdatedAt(T2);
|
||||
blobContent = "content-v2".getBytes();
|
||||
assertEquals(1, source.resolve(spec(), ctx).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateRejectsAnUnknownFolder() {
|
||||
when(folderRepository.existsById(FOLDER)).thenReturn(false);
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> source.validate(spec()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateRejectsAMissingFolderId() {
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> source.validate(new InputSpec("storage-folder", Map.of())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateRejectsWhenStorageIsDisabled() {
|
||||
StorageFolderInputSource disabled =
|
||||
new StorageFolderInputSource(
|
||||
storedFileRepository,
|
||||
folderRepository,
|
||||
storageProvider,
|
||||
new ApplicationProperties());
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> disabled.validate(spec()));
|
||||
}
|
||||
|
||||
private static InputSpec spec() {
|
||||
return new InputSpec("storage-folder", Map.of("folderId", FOLDER.toString()));
|
||||
}
|
||||
|
||||
private static StoredFile storedFile(Long id, String name, LocalDateTime updatedAt) {
|
||||
StoredFile file = new StoredFile();
|
||||
file.setId(id);
|
||||
file.setOriginalFilename(name);
|
||||
file.setStorageKey("key-" + id);
|
||||
file.setSizeBytes(100);
|
||||
file.setUpdatedAt(updatedAt);
|
||||
return file;
|
||||
}
|
||||
|
||||
private static ApplicationProperties storageEnabledProperties() {
|
||||
ApplicationProperties properties = new ApplicationProperties();
|
||||
properties.getSecurity().setEnableLogin(true);
|
||||
properties.getStorage().setEnabled(true);
|
||||
return properties;
|
||||
}
|
||||
|
||||
private class RecordingContext implements ResolveContext {
|
||||
|
||||
private final List<String> present = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public boolean claim(String identity, String gate, Supplier<String> contentHash) {
|
||||
return ledger.claim(POLICY, identity, gate, contentHash);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void settle(
|
||||
String identity, String finalGate, String finalContentHash, boolean success) {
|
||||
ledger.settle(POLICY, identity, finalGate, finalContentHash, success);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean allSettledDone(String identity) {
|
||||
return ledger.allSettledDone(identity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reportPresent(Collection<String> identities) {
|
||||
present.addAll(identities);
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
@@ -41,6 +41,7 @@ import stirling.software.proprietary.storage.model.StoredFile;
|
||||
import stirling.software.proprietary.storage.provider.StorageProvider;
|
||||
import stirling.software.proprietary.storage.repository.FileShareAccessRepository;
|
||||
import stirling.software.proprietary.storage.repository.FileShareRepository;
|
||||
import stirling.software.proprietary.storage.repository.FolderRepository;
|
||||
import stirling.software.proprietary.storage.repository.StorageCleanupEntryRepository;
|
||||
import stirling.software.proprietary.storage.repository.StoredFileRepository;
|
||||
import stirling.software.proprietary.workflow.model.WorkflowSession;
|
||||
@@ -52,6 +53,7 @@ import stirling.software.proprietary.workflow.model.WorkflowSession;
|
||||
class FileStorageServiceMoreTest {
|
||||
|
||||
@Mock private StoredFileRepository storedFileRepository;
|
||||
@Mock private FolderRepository folderRepository;
|
||||
@Mock private FileShareRepository fileShareRepository;
|
||||
@Mock private FileShareAccessRepository fileShareAccessRepository;
|
||||
@Mock private UserRepository userRepository;
|
||||
@@ -71,6 +73,7 @@ class FileStorageServiceMoreTest {
|
||||
service =
|
||||
new FileStorageService(
|
||||
storedFileRepository,
|
||||
folderRepository,
|
||||
fileShareRepository,
|
||||
fileShareAccessRepository,
|
||||
userRepository,
|
||||
|
||||
+3
@@ -35,6 +35,7 @@ import stirling.software.proprietary.storage.provider.StorageProvider;
|
||||
import stirling.software.proprietary.storage.provider.StoredObject;
|
||||
import stirling.software.proprietary.storage.repository.FileShareAccessRepository;
|
||||
import stirling.software.proprietary.storage.repository.FileShareRepository;
|
||||
import stirling.software.proprietary.storage.repository.FolderRepository;
|
||||
import stirling.software.proprietary.storage.repository.StorageCleanupEntryRepository;
|
||||
import stirling.software.proprietary.storage.repository.StoredFileRepository;
|
||||
import stirling.software.proprietary.workflow.model.WorkflowSession;
|
||||
@@ -44,6 +45,7 @@ import stirling.software.proprietary.workflow.model.WorkflowSession;
|
||||
class FileStorageServiceTest {
|
||||
|
||||
@Mock private StoredFileRepository storedFileRepository;
|
||||
@Mock private FolderRepository folderRepository;
|
||||
@Mock private FileShareRepository fileShareRepository;
|
||||
@Mock private FileShareAccessRepository fileShareAccessRepository;
|
||||
@Mock private UserRepository userRepository;
|
||||
@@ -64,6 +66,7 @@ class FileStorageServiceTest {
|
||||
service =
|
||||
new FileStorageService(
|
||||
storedFileRepository,
|
||||
folderRepository,
|
||||
fileShareRepository,
|
||||
fileShareAccessRepository,
|
||||
userRepository,
|
||||
|
||||
@@ -3935,6 +3935,8 @@ backToFolder = "Back to {{folder}}"
|
||||
backToMyFiles = "Back to My Files"
|
||||
breadcrumbs = "Folder path"
|
||||
cancel = "Cancel"
|
||||
categoryHint = "Categories: {{labels}}"
|
||||
categoryHintGrouped = "{{families}} — {{labels}}"
|
||||
classification = "Classification"
|
||||
clearSelection = "Clear selection"
|
||||
closeDetails = "Close details"
|
||||
@@ -3978,13 +3980,19 @@ inPath = "in {{path}}"
|
||||
inWorkspace = "Open"
|
||||
inWorkspaceAria = "Already in workspace"
|
||||
loading = "Loading…"
|
||||
localFolderManagedByDisk = "This folder is managed by its directory on disk."
|
||||
localFoldersUnavailable = "Folders are cloud-only - save a file to the cloud to organize it."
|
||||
moveAcrossKindsBlocked = "These folders live in different places, so one can't go inside the other."
|
||||
moveIntoLocalBlocked = "Files can't be moved into a folder that mirrors a directory on disk."
|
||||
moveIntoVirtualCloudSkipped_one = "{{count}} server file was left in place — server files can't live in browser-only folders."
|
||||
moveIntoVirtualCloudSkipped_other = "{{count}} server files were left in place — server files can't live in browser-only folders."
|
||||
moveSkippedRemote_one = "{{count}} file couldn't be moved on the server (no permission or already deleted)."
|
||||
moveSkippedRemote_other = "{{count}} files couldn't be moved on the server (no permission or already deleted)."
|
||||
moveTo = "Move to…"
|
||||
myFiles = "My Files"
|
||||
newFolder = "New folder"
|
||||
newFolderStorageDisabled = "Server folder storage isn't enabled. Ask your admin to turn it on."
|
||||
newFolderInLocalUnavailable = "This folder mirrors a directory on disk — create subfolders in your file explorer."
|
||||
newFolderStorageDisabled = "Server folder storage isn't enabled."
|
||||
newFolderTabUnavailable = "Switch to All or Cloud to create folders."
|
||||
offlineNoFolderEdits = "Server folder sync unavailable - folder changes are disabled. Check sign-in and storage configuration."
|
||||
open = "Open"
|
||||
@@ -3992,6 +4000,7 @@ openVersionInWorkspace = "Open in workspace"
|
||||
originFilter = "Filter by source"
|
||||
refresh = "Refresh from server"
|
||||
remove = "Delete"
|
||||
removeLocalFolder = "Remove (files stay on disk)"
|
||||
removeVersion = "Remove this version"
|
||||
rename = "Rename"
|
||||
renamed = "Renamed"
|
||||
@@ -4029,6 +4038,10 @@ icon = "Icon"
|
||||
title = "Appearance"
|
||||
useColour = "Use color {{c}}"
|
||||
|
||||
[filesPage.categoryFilter]
|
||||
all = "All categories"
|
||||
label = "Filter by category"
|
||||
|
||||
[filesPage.column]
|
||||
modified = "Modified"
|
||||
name = "Name"
|
||||
@@ -4104,12 +4117,24 @@ totalSize = "Total size"
|
||||
type = "Type"
|
||||
versionHistory = "Version journey"
|
||||
|
||||
[filesPage.folderKind]
|
||||
local = "Local folder"
|
||||
virtual = "Browser folder"
|
||||
|
||||
[filesPage.folderKindChoice]
|
||||
localUnavailable = "Available in the desktop app, which can see your disk."
|
||||
|
||||
[filesPage.folderName]
|
||||
cancel = "Cancel"
|
||||
error = "Could not save folder. Try again."
|
||||
label = "Folder name"
|
||||
placeholder = "Folder name"
|
||||
|
||||
[filesPage.folderOrigin]
|
||||
diskHint = "A folder mounted from a directory on your disk"
|
||||
serverHint = "A folder stored on the Stirling server"
|
||||
virtualHint = "A folder that lives only in this browser"
|
||||
|
||||
[filesPage.moveDialog]
|
||||
cancel = "Cancel"
|
||||
confirm = "Move here"
|
||||
@@ -4123,19 +4148,46 @@ newFolderPlaceholder = "Folder name"
|
||||
newFolderToggle = "Create new folder…"
|
||||
title = "Move to folder"
|
||||
|
||||
[filesPage.newFolderMenu]
|
||||
addExisting = "Add folder from this computer…"
|
||||
addExistingHint = "Its files stay exactly where they are."
|
||||
device = "New folder on this device"
|
||||
deviceHint = "Lives only on this device. Works offline."
|
||||
server = "New folder on the server"
|
||||
serverHint = "Synced to your account, available wherever you sign in."
|
||||
|
||||
[filesPage.origin]
|
||||
all = "All sources"
|
||||
cloud = "Cloud"
|
||||
cloudHint = "Stored on the Stirling server"
|
||||
disk = "On disk"
|
||||
diskHint = "A file in the mounted folder on your disk"
|
||||
local = "Local"
|
||||
localHint = "Only stored in this browser"
|
||||
shared = "Shared"
|
||||
sharedHint = "Shared with you via link"
|
||||
|
||||
[filesPage.processing]
|
||||
active = "Processing folder"
|
||||
paused = "Processing paused"
|
||||
start = "Process files in this folder…"
|
||||
stop = "Stop processing this folder"
|
||||
sweep = "Process files now"
|
||||
|
||||
[filesPage.processingSections]
|
||||
inputs = "Inputs"
|
||||
inputsHint = "Your originals — never changed"
|
||||
outputs = "Outputs"
|
||||
outputsHint = "Processed results"
|
||||
processing = "Processing"
|
||||
processingHint = "Being processed right now"
|
||||
running = "Processing…"
|
||||
runStep = "Step {{current}} of {{total}}"
|
||||
|
||||
[filesPage.search]
|
||||
clear = "Clear filter"
|
||||
label = "Filter files by name"
|
||||
placeholder = "Filter files…"
|
||||
placeholder = "Filter by name or category…"
|
||||
|
||||
[filesPage.sort]
|
||||
modifiedAsc = "Oldest first"
|
||||
@@ -8871,6 +8923,24 @@ unlimited = "{{used}} · Unlimited"
|
||||
[printFile]
|
||||
title = "Print File"
|
||||
|
||||
[processingFolders.downloads]
|
||||
approve = "Process my Downloads"
|
||||
capped = "You have {{found}} PDFs; the first {{limit}} are processed now and the rest follow."
|
||||
close = "Done"
|
||||
explain = "Stirling can classify the {{count}} PDFs already in your Downloads folder and open the results here."
|
||||
failed = "Could not set that up. Your files have not been changed."
|
||||
finished = "Classified {{count}} files and opened {{opened}} of them here, ready to work on."
|
||||
keepsOriginals = "Your files stay where they are — originals are never moved or deleted."
|
||||
nothingNew = "Nothing new to process — these {{count}} files have already been through."
|
||||
notNow = "Not now"
|
||||
outputs = "Results are saved into a \"{{subdir}}\" folder alongside them."
|
||||
progress = "Processing {{done}} of {{total}} files…"
|
||||
someFailed = "{{count}} could not be processed and were left untouched."
|
||||
stillRunning = "Some files are still being processed in the background."
|
||||
title = "Organise your Downloads?"
|
||||
trigger = "Process {{count}} PDFs in Downloads"
|
||||
working = "Processing…"
|
||||
|
||||
[provider.googledrive]
|
||||
name = "Google Drive"
|
||||
scope = "File Import"
|
||||
|
||||
@@ -39,6 +39,14 @@
|
||||
"identifier": "fs:allow-read-file",
|
||||
"allow": [{ "path": "**" }]
|
||||
},
|
||||
{
|
||||
"identifier": "fs:allow-read-dir",
|
||||
"allow": [{ "path": "**" }]
|
||||
},
|
||||
{
|
||||
"identifier": "fs:allow-stat",
|
||||
"allow": [{ "path": "**" }]
|
||||
},
|
||||
{
|
||||
"identifier": "fs:allow-write-file",
|
||||
"allow": [{ "path": "**" }]
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
false
|
||||
@@ -0,0 +1,76 @@
|
||||
import React from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Tooltip } from "@mantine/core";
|
||||
import { LocalIcon } from "@app/components/shared/LocalIcon";
|
||||
import {
|
||||
useFamilyBadges,
|
||||
useLabelBadges,
|
||||
} from "@app/components/shared/fileSidebarGrouping";
|
||||
|
||||
/** At most this many icons on a card; the hover names everything. */
|
||||
const MAX_ICONS = 3;
|
||||
|
||||
/**
|
||||
* A classified file's categories, worn as the sidebar's own family icons in
|
||||
* the same cycled accents — no text, and the hover names the group first and
|
||||
* its labels after. The label-level icons only stand in when no visible
|
||||
* family claims the labels (a hidden category), so a tagged file is never
|
||||
* entirely unmarked. Renders nothing for an unclassified file (or in builds
|
||||
* without classification): absence of the badge IS the "no category" state.
|
||||
*/
|
||||
export function FileCategoryBadge({ labels }: { labels?: string[] | null }) {
|
||||
const { t } = useTranslation();
|
||||
const families = useFamilyBadges(labels);
|
||||
const labelBadges = useLabelBadges(labels);
|
||||
const badges = families.length > 0 ? families : labelBadges;
|
||||
if (badges.length === 0) return null;
|
||||
const hover =
|
||||
families.length > 0
|
||||
? t("filesPage.categoryHintGrouped", {
|
||||
families: families.map((badge) => badge.name).join(", "),
|
||||
labels: labelBadges.map((badge) => badge.name).join(", "),
|
||||
defaultValue: "{{families}} — {{labels}}",
|
||||
})
|
||||
: t("filesPage.categoryHint", {
|
||||
labels: labelBadges.map((badge) => badge.name).join(", "),
|
||||
defaultValue: "Categories: {{labels}}",
|
||||
});
|
||||
return (
|
||||
<Tooltip label={hover} withinPortal>
|
||||
<span
|
||||
style={{
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
gap: "0.25rem",
|
||||
padding: "0.1rem 0.4rem",
|
||||
borderRadius: "999px",
|
||||
lineHeight: 1.2,
|
||||
// Mixed into the surface, not transparency — the badge sits on top
|
||||
// of thumbnails, where a see-through backer is illegible.
|
||||
background:
|
||||
"color-mix(in srgb, var(--c-text-subtle) 16%, var(--c-surface))",
|
||||
}}
|
||||
>
|
||||
{badges.slice(0, MAX_ICONS).map((badge) => (
|
||||
<LocalIcon
|
||||
key={badge.id}
|
||||
icon={badge.icon}
|
||||
width="0.85rem"
|
||||
style={badge.color ? { color: badge.color } : undefined}
|
||||
/>
|
||||
))}
|
||||
{badges.length > MAX_ICONS && (
|
||||
<span
|
||||
style={{
|
||||
fontSize: "0.68rem",
|
||||
fontWeight: 600,
|
||||
color: "var(--c-text-muted)",
|
||||
}}
|
||||
>
|
||||
+{badges.length - MAX_ICONS}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -10,8 +10,10 @@ import { useLocation, useNavigate } from "react-router-dom";
|
||||
import {
|
||||
Drawer,
|
||||
Group,
|
||||
Menu,
|
||||
MultiSelect,
|
||||
Select,
|
||||
Text,
|
||||
TextInput,
|
||||
Tooltip,
|
||||
} from "@mantine/core";
|
||||
@@ -32,6 +34,9 @@ import OpenInNewIcon from "@mui/icons-material/OpenInNew";
|
||||
import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined";
|
||||
import CloudUploadIcon from "@mui/icons-material/CloudUpload";
|
||||
import KeyboardArrowRightIcon from "@mui/icons-material/KeyboardArrowRight";
|
||||
import ArrowDropDownIcon from "@mui/icons-material/ArrowDropDown";
|
||||
import DriveFolderUploadIcon from "@mui/icons-material/DriveFolderUpload";
|
||||
import CloudIcon from "@mui/icons-material/Cloud";
|
||||
import RefreshIcon from "@mui/icons-material/Refresh";
|
||||
|
||||
import { stripBasePath } from "@app/constants/app";
|
||||
@@ -41,6 +46,10 @@ import { useFolders } from "@app/contexts/FolderContext";
|
||||
import { useFileActions } from "@app/contexts/file/fileHooks";
|
||||
import { useAllFiles } from "@app/contexts/FileContext";
|
||||
import { useFileHandler } from "@app/hooks/useFileHandler";
|
||||
import { useFileManagement } from "@app/contexts/FileContext";
|
||||
import { getCachedDiskThumbnail } from "@app/hooks/useLazyThumbnail";
|
||||
import { readClassificationLabelsFromFile } from "@app/services/fileClassification";
|
||||
import { fileStorage } from "@app/services/fileStorage";
|
||||
import {
|
||||
useNavigationActions,
|
||||
useNavigationGuard,
|
||||
@@ -56,15 +65,41 @@ import { getFileOrigin } from "@app/components/filesPage/fileOrigin";
|
||||
|
||||
import { FileId } from "@app/types/file";
|
||||
import { StirlingFileStub } from "@app/types/fileContext";
|
||||
import { FolderId, ROOT_FOLDER_ID } from "@app/types/folder";
|
||||
import { FolderId, ROOT_FOLDER_ID, folderKind } from "@app/types/folder";
|
||||
|
||||
import { FileGrid, FilesPageEntry } from "@app/components/filesPage/FileGrid";
|
||||
import {
|
||||
FileGrid,
|
||||
FilesPageEntry,
|
||||
PROCESSING_SECTION_LABELS,
|
||||
ProcessingSectionId,
|
||||
} from "@app/components/filesPage/FileGrid";
|
||||
import {
|
||||
useProcessingFolders,
|
||||
type ProcessingRunInfo,
|
||||
} from "@app/hooks/useProcessingFolders";
|
||||
import {
|
||||
useCategoryFilterOptions,
|
||||
useLabelSearchMatcher,
|
||||
} from "@app/components/shared/fileSidebarGrouping";
|
||||
import { LocalIcon } from "@app/components/shared/LocalIcon";
|
||||
import {
|
||||
fileMatchesFilters,
|
||||
type FileFilterContext,
|
||||
type FileFilters,
|
||||
} from "@app/components/filesPage/fileFilters";
|
||||
import SuperSearch from "@app/components/shared/superSearch/SuperSearch";
|
||||
import { useEditorSearchScopes } from "@app/hooks/useSuperSearch";
|
||||
import { FileDetailsPanel } from "@app/components/filesPage/FileDetailsPanel";
|
||||
import BulkUploadToServerModal from "@app/components/shared/BulkUploadToServerModal";
|
||||
import MobileUploadModal from "@app/components/shared/MobileUploadModal";
|
||||
import { useAppConfig } from "@app/contexts/AppConfigContext";
|
||||
import { canPickDirectory, pickDirectory } from "@app/services/directoryPicker";
|
||||
import {
|
||||
canListDirectory,
|
||||
listDirectory,
|
||||
readDiskFile,
|
||||
type DiskFileEntry,
|
||||
} from "@app/services/localFolderContents";
|
||||
import { useIsMobile } from "@app/hooks/useIsMobile";
|
||||
import { MoveToFolderDialog } from "@app/components/filesPage/MoveToFolderDialog";
|
||||
import { FolderNameDialog } from "@app/components/filesPage/FolderNameDialog";
|
||||
@@ -159,6 +194,8 @@ export default function FileManagerView() {
|
||||
setOriginFilter,
|
||||
typeFilter,
|
||||
setTypeFilter,
|
||||
categoryFilter,
|
||||
setCategoryFilter,
|
||||
currentTab,
|
||||
setCurrentTab,
|
||||
folderNameDialog,
|
||||
@@ -360,19 +397,33 @@ export default function FileManagerView() {
|
||||
}
|
||||
}, [availableTypes, typeFilter, setTypeFilter]);
|
||||
|
||||
// Category filter over the classification families the sidebar groups by;
|
||||
// empty (core, classification off) means the dropdown never renders.
|
||||
const categoryOptions = useCategoryFilterOptions();
|
||||
const labelsMatchText = useLabelSearchMatcher();
|
||||
const categoryLabelKeys = useMemo(() => {
|
||||
if (categoryFilter === "all") return null;
|
||||
const option = categoryOptions.find((c) => c.id === categoryFilter);
|
||||
return option ? new Set(option.labelKeys) : null;
|
||||
}, [categoryFilter, categoryOptions]);
|
||||
|
||||
const visibleFiles = useMemo(() => {
|
||||
const filtered = filesInCurrentFolder
|
||||
.filter((f) =>
|
||||
search ? f.name.toLowerCase().includes(search.toLowerCase()) : true,
|
||||
)
|
||||
.filter((f) =>
|
||||
originFilter === "all" ? true : getFileOrigin(f) === originFilter,
|
||||
)
|
||||
.filter((f) => {
|
||||
if (typeFilter.length === 0) return true;
|
||||
const ext = (f.name.split(".").pop() ?? "").toUpperCase();
|
||||
return typeFilter.includes(ext);
|
||||
});
|
||||
// One unified filter pass: text (names + classification), origin, type,
|
||||
// category — see fileFilters.ts, where new facets belong.
|
||||
const filters: FileFilters = {
|
||||
text: search,
|
||||
origin: originFilter,
|
||||
types: typeFilter,
|
||||
category: categoryFilter,
|
||||
};
|
||||
const ctx: FileFilterContext = {
|
||||
originOf: getFileOrigin,
|
||||
categoryLabelKeys,
|
||||
labelsMatchText,
|
||||
};
|
||||
const filtered = filesInCurrentFolder.filter((f) =>
|
||||
fileMatchesFilters(f, filters, ctx),
|
||||
);
|
||||
const sorted = [...filtered];
|
||||
sorted.sort((a, b) => {
|
||||
switch (sortMode) {
|
||||
@@ -392,7 +443,16 @@ export default function FileManagerView() {
|
||||
}
|
||||
});
|
||||
return sorted;
|
||||
}, [filesInCurrentFolder, search, sortMode, originFilter, typeFilter]);
|
||||
}, [
|
||||
filesInCurrentFolder,
|
||||
search,
|
||||
sortMode,
|
||||
originFilter,
|
||||
typeFilter,
|
||||
categoryFilter,
|
||||
categoryLabelKeys,
|
||||
labelsMatchText,
|
||||
]);
|
||||
|
||||
/**
|
||||
* Resolve a folder id to its breadcrumb path (e.g. "Receipts / 2024 / Q1").
|
||||
@@ -418,12 +478,259 @@ export default function FileManagerView() {
|
||||
[foldersById],
|
||||
);
|
||||
|
||||
// ─── read-through listing for a mounted local folder ────────────────────
|
||||
// The directory is the source of truth: its contents are read fresh off
|
||||
// the disk whenever the user is inside the folder, never ingested to show.
|
||||
const currentFolder = currentFolderId
|
||||
? folders.foldersById.get(currentFolderId)
|
||||
: undefined;
|
||||
const currentLocalDirectory =
|
||||
currentFolder && folderKind(currentFolder) === "local"
|
||||
? currentFolder.directory
|
||||
: undefined;
|
||||
const { setError: setFolderError } = folders;
|
||||
const [diskEntries, setDiskEntries] = useState<DiskFileEntry[]>([]);
|
||||
const [diskLoading, setDiskLoading] = useState(false);
|
||||
useEffect(() => {
|
||||
if (!currentLocalDirectory || !canListDirectory) {
|
||||
setDiskEntries([]);
|
||||
// Also stand the loading flag down: when the user navigates OUT of a
|
||||
// mount mid-listing, the in-flight finally skips its reset (cancelled),
|
||||
// and this branch is the only code that runs — without the reset the
|
||||
// skeleton covers every folder for the rest of the session.
|
||||
setDiskLoading(false);
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
setDiskLoading(true);
|
||||
listDirectory(currentLocalDirectory)
|
||||
.then((listed) => {
|
||||
if (!cancelled) setDiskEntries(listed ?? []);
|
||||
})
|
||||
.catch((err) => {
|
||||
console.warn("[FileManagerView] disk listing failed", err);
|
||||
if (!cancelled) {
|
||||
setDiskEntries([]);
|
||||
setFolderError(
|
||||
err instanceof Error
|
||||
? `Could not read the folder: ${err.message}`
|
||||
: "Could not read the folder.",
|
||||
);
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) setDiskLoading(false);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
// The stable setter, not the context object: that changes identity on
|
||||
// every folder mutation — including the setError call above, which would
|
||||
// make a failing listing re-trigger itself.
|
||||
}, [currentLocalDirectory, setFolderError]);
|
||||
|
||||
// ─── processing-folder sections (Inputs / Outputs / Processing) ─────────
|
||||
// A mount with processing attached presents as a master folder of three
|
||||
// fixed sections instead of a flat listing: the untouched originals, the
|
||||
// processed results (wherever the record says they land), and what is
|
||||
// running right now. Pure presentation — no stored folder backs a section.
|
||||
const processingApi = useProcessingFolders();
|
||||
const currentProcessing = currentFolder
|
||||
? processingApi.stateFor(currentFolder)
|
||||
: undefined;
|
||||
const outputDirectory = currentLocalDirectory
|
||||
? currentProcessing?.outputDirectory
|
||||
: undefined;
|
||||
const rawSection = new URLSearchParams(location.search).get("section");
|
||||
const processingSection: ProcessingSectionId | null =
|
||||
outputDirectory &&
|
||||
(rawSection === "inputs" ||
|
||||
rawSection === "outputs" ||
|
||||
rawSection === "processing")
|
||||
? rawSection
|
||||
: null;
|
||||
|
||||
const [outputEntries, setOutputEntries] = useState<DiskFileEntry[]>([]);
|
||||
const [outputLoading, setOutputLoading] = useState(false);
|
||||
useEffect(() => {
|
||||
if (!outputDirectory || !canListDirectory) {
|
||||
setOutputEntries([]);
|
||||
setOutputLoading(false);
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
setOutputLoading(true);
|
||||
listDirectory(outputDirectory)
|
||||
.then((listed) => {
|
||||
if (!cancelled) setOutputEntries(listed ?? []);
|
||||
})
|
||||
.catch(() => {
|
||||
// The output directory only exists once a run has delivered into it,
|
||||
// so unreadable reads as empty rather than as an error.
|
||||
if (!cancelled) setOutputEntries([]);
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) setOutputLoading(false);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [outputDirectory]);
|
||||
|
||||
// What is running right now — polled while the master folder is open so
|
||||
// the Processing section and its count stay live.
|
||||
const [activeRuns, setActiveRuns] = useState<ProcessingRunInfo[] | null>(
|
||||
null,
|
||||
);
|
||||
const processingRecordId = currentProcessing?.id;
|
||||
const { listActiveRuns } = processingApi;
|
||||
useEffect(() => {
|
||||
if (!outputDirectory || !processingRecordId) {
|
||||
setActiveRuns(null);
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
const tick = async () => {
|
||||
const runs = await listActiveRuns(processingRecordId);
|
||||
if (!cancelled) setActiveRuns(runs);
|
||||
};
|
||||
void tick();
|
||||
const timer = setInterval(() => void tick(), 3000);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(timer);
|
||||
};
|
||||
}, [outputDirectory, processingRecordId, listActiveRuns]);
|
||||
|
||||
const openProcessingSection = useCallback(
|
||||
(id: ProcessingSectionId) => {
|
||||
if (!currentFolderId) return;
|
||||
navigate(`/files/${currentFolderId}?section=${id}`);
|
||||
},
|
||||
[navigate, currentFolderId],
|
||||
);
|
||||
const clearProcessingSection = useCallback(() => {
|
||||
if (!currentFolderId) return;
|
||||
navigate(`/files/${currentFolderId}`);
|
||||
}, [navigate, currentFolderId]);
|
||||
|
||||
// Opening a disk file loads its bytes into the workbench — the one moment
|
||||
// anything leaves the disk, and only because the user asked to work on it.
|
||||
const { updateStirlingFileStub } = useFileManagement();
|
||||
const openDiskFile = useCallback(
|
||||
async (entry: DiskFileEntry) => {
|
||||
try {
|
||||
const file = await readDiskFile(entry);
|
||||
if (!file) return;
|
||||
clearFilesPageReturnRoute();
|
||||
// The listing usually rendered this file's thumbnail already; hand it
|
||||
// through so the workbench adopts it instead of rasterising again.
|
||||
const cachedThumb = getCachedDiskThumbnail(entry);
|
||||
const added = await addFiles([file], {
|
||||
selectFiles: true,
|
||||
...(cachedThumb
|
||||
? { precomputedThumbnails: new Map([[file as File, cachedThumb]]) }
|
||||
: {}),
|
||||
});
|
||||
// A processed file carries its labels in its own metadata; stamping
|
||||
// them here puts it in its category the moment it appears, instead of
|
||||
// whenever the lazy backfill gets around to re-reading the PDF.
|
||||
const stirlingFile = added[0];
|
||||
if (stirlingFile) {
|
||||
const labels = await readClassificationLabelsFromFile(file);
|
||||
if (labels && labels.length > 0) {
|
||||
const updates = { classificationLabels: labels };
|
||||
updateStirlingFileStub(stirlingFile.fileId, updates);
|
||||
void fileStorage.updateFileMetadata(stirlingFile.fileId, updates);
|
||||
}
|
||||
}
|
||||
navActions.setWorkbench("viewer");
|
||||
navigate("/");
|
||||
} catch (err) {
|
||||
folders.setError(
|
||||
err instanceof Error
|
||||
? `Could not open ${entry.name}: ${err.message}`
|
||||
: `Could not open ${entry.name}.`,
|
||||
);
|
||||
}
|
||||
},
|
||||
[addFiles, updateStirlingFileStub, navActions, navigate, folders],
|
||||
);
|
||||
|
||||
const entries = useMemo<FilesPageEntry[]>(() => {
|
||||
// When searching, items may come from anywhere in the subtree, so we
|
||||
// expose a "parentPath" subtitle whenever the item's parent differs from
|
||||
// currentFolderId. When no search is active, every item is in the
|
||||
// current folder by definition and the subtitle is suppressed.
|
||||
const inSearch = search.length > 0;
|
||||
// Inside a mounted folder the listing IS the directory; storage rows and
|
||||
// subfolders don't apply there.
|
||||
if (currentLocalDirectory) {
|
||||
const needle = search.toLowerCase();
|
||||
const compare: Record<
|
||||
string,
|
||||
(a: DiskFileEntry, b: DiskFileEntry) => number
|
||||
> = {
|
||||
"name-asc": (a, b) => a.name.localeCompare(b.name),
|
||||
"name-desc": (a, b) => b.name.localeCompare(a.name),
|
||||
"size-asc": (a, b) => a.sizeBytes - b.sizeBytes,
|
||||
"size-desc": (a, b) => b.sizeBytes - a.sizeBytes,
|
||||
"modified-asc": (a, b) => a.lastModified - b.lastModified,
|
||||
"modified-desc": (a, b) => b.lastModified - a.lastModified,
|
||||
};
|
||||
const toDiskEntries = (list: DiskFileEntry[]) =>
|
||||
list
|
||||
.filter((disk) => !needle || disk.name.toLowerCase().includes(needle))
|
||||
.sort(compare[filesPage.sortMode] ?? compare["modified-desc"]!)
|
||||
.map<FilesPageEntry>((disk) => ({ kind: "diskFile", disk }));
|
||||
// A processing folder's root is its three sections; a search cuts
|
||||
// through them straight to the originals.
|
||||
if (outputDirectory && processingSection === null && !inSearch) {
|
||||
return [
|
||||
{
|
||||
kind: "section",
|
||||
section: {
|
||||
id: "inputs",
|
||||
count: diskLoading ? null : diskEntries.length,
|
||||
},
|
||||
},
|
||||
{
|
||||
kind: "section",
|
||||
section: {
|
||||
id: "outputs",
|
||||
count: outputLoading ? null : outputEntries.length,
|
||||
},
|
||||
},
|
||||
{
|
||||
kind: "section",
|
||||
section: {
|
||||
id: "processing",
|
||||
count: activeRuns === null ? null : activeRuns.length,
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
if (processingSection === "outputs") {
|
||||
return toDiskEntries(outputEntries);
|
||||
}
|
||||
if (processingSection === "processing") {
|
||||
return (activeRuns ?? [])
|
||||
.filter(
|
||||
(run) =>
|
||||
!needle || (run.fileName ?? "").toLowerCase().includes(needle),
|
||||
)
|
||||
.map<FilesPageEntry>((run) => ({
|
||||
kind: "run",
|
||||
run: {
|
||||
runId: run.runId,
|
||||
fileName: run.fileName ?? "…",
|
||||
currentStep: run.currentStep,
|
||||
stepCount: run.stepCount,
|
||||
},
|
||||
}));
|
||||
}
|
||||
return toDiskEntries(diskEntries);
|
||||
}
|
||||
return [
|
||||
...visibleFolders.map<FilesPageEntry>((folder) => ({
|
||||
kind: "folder",
|
||||
@@ -449,6 +756,15 @@ export default function FileManagerView() {
|
||||
filesPage.fileCountsByFolder,
|
||||
search,
|
||||
currentFolderId,
|
||||
currentLocalDirectory,
|
||||
diskEntries,
|
||||
diskLoading,
|
||||
outputDirectory,
|
||||
processingSection,
|
||||
outputEntries,
|
||||
outputLoading,
|
||||
activeRuns,
|
||||
filesPage.sortMode,
|
||||
pathForFolderId,
|
||||
]);
|
||||
|
||||
@@ -816,13 +1132,47 @@ export default function FileManagerView() {
|
||||
[selectedFiles, fileMap],
|
||||
);
|
||||
|
||||
// Per-destination availability for the New-folder menu. The reasons render
|
||||
// inline as the disabled item's caption — the reason IS the information.
|
||||
const serverFolderDisabledReason =
|
||||
signInRequiredReason ??
|
||||
(!uploadEnabled || !folders.serverReachable
|
||||
? t(
|
||||
"filesPage.newFolderStorageDisabled",
|
||||
"Server folder storage isn't enabled.",
|
||||
)
|
||||
: undefined);
|
||||
const addExistingDisabledReason = canPickDirectory
|
||||
? undefined
|
||||
: t(
|
||||
"filesPage.folderKindChoice.localUnavailable",
|
||||
"Available in the desktop app, which can see your disk.",
|
||||
);
|
||||
|
||||
// "Add an existing folder" needs no dialog at all: the native picker is the
|
||||
// whole interaction, and the directory's name is the folder's name. Landing
|
||||
// inside the fresh mount is the confirmation.
|
||||
const addExistingFolder = useCallback(async () => {
|
||||
try {
|
||||
const picked = await pickDirectory();
|
||||
if (!picked) return;
|
||||
const record = await folders.mountLocalFolder(picked.path, picked.name);
|
||||
// The URL is the source of truth for folder selection (the pathname →
|
||||
// state effect owns currentFolderId). Setting state directly here races
|
||||
// that effect — it re-runs on the same commit's foldersById change with
|
||||
// the old pathname and snaps the selection back to root.
|
||||
navigate(`/files/${record.id}`);
|
||||
} catch (err) {
|
||||
folders.setError(
|
||||
err instanceof Error
|
||||
? `Could not add the folder: ${err.message}`
|
||||
: "Could not add the folder.",
|
||||
);
|
||||
}
|
||||
}, [folders, navigate]);
|
||||
|
||||
// null = New folder actionable; string = disabled tooltip reason.
|
||||
const newFolderDisabledReason: string | null = useMemo(() => {
|
||||
// Guests can't use cloud folders at all - say so before any tab/storage
|
||||
// hint, since switching tabs wouldn't help them.
|
||||
if (signInRequiredReason) {
|
||||
return signInRequiredReason;
|
||||
}
|
||||
if (currentTab === "local") {
|
||||
return t(
|
||||
"filesPage.localFoldersUnavailable",
|
||||
@@ -839,20 +1189,46 @@ export default function FileManagerView() {
|
||||
"Switch to All or Cloud to create folders.",
|
||||
);
|
||||
}
|
||||
if (!folders.serverReachable) {
|
||||
// Inside a mounted folder there is nothing to create: its contents ARE
|
||||
// the directory, and subfolders are made in the file explorer.
|
||||
if (currentLocalDirectory) {
|
||||
return t(
|
||||
"filesPage.newFolderStorageDisabled",
|
||||
"Server folder storage isn't enabled. Ask your admin to turn it on.",
|
||||
"filesPage.newFolderInLocalUnavailable",
|
||||
"This folder mirrors a directory on disk — create subfolders in your file explorer.",
|
||||
);
|
||||
}
|
||||
// Inside a server folder the subfolder inherits kind server, so the
|
||||
// server-side blockers apply to the button itself — otherwise the dialog
|
||||
// opens only to fail at submit with a raw error.
|
||||
if (
|
||||
currentFolder &&
|
||||
folderKind(currentFolder) === "server" &&
|
||||
serverFolderDisabledReason
|
||||
) {
|
||||
return serverFolderDisabledReason;
|
||||
}
|
||||
// Reachability and storage no longer disable the button: those only rule
|
||||
// out the server option, which the dialog now greys out individually —
|
||||
// browser and disk folders remain creatable regardless.
|
||||
return null;
|
||||
}, [signInRequiredReason, currentTab, folders.serverReachable, t]);
|
||||
}, [
|
||||
currentTab,
|
||||
currentLocalDirectory,
|
||||
currentFolder,
|
||||
serverFolderDisabledReason,
|
||||
t,
|
||||
]);
|
||||
|
||||
return (
|
||||
<div className="files-page" ref={dropZoneRef}>
|
||||
<header className="files-page-header">
|
||||
{/* Breadcrumb only for folder-rooted tabs. */}
|
||||
{(currentTab === "all" || currentTab === "cloud") && <Breadcrumbs />}
|
||||
{(currentTab === "all" || currentTab === "cloud") && (
|
||||
<Breadcrumbs
|
||||
section={processingSection}
|
||||
onClearSection={clearProcessingSection}
|
||||
/>
|
||||
)}
|
||||
{(currentTab === "local" ||
|
||||
currentTab === "recent" ||
|
||||
currentTab === "shared" ||
|
||||
@@ -948,7 +1324,9 @@ export default function FileManagerView() {
|
||||
</Button>
|
||||
</span>
|
||||
</Tooltip>
|
||||
) : (
|
||||
) : folders.currentFolderId !== null ? (
|
||||
// Inside a folder there is nothing to choose: the subfolder
|
||||
// inherits its parent's kind, so plain click → name dialog.
|
||||
<Button
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
@@ -957,6 +1335,73 @@ export default function FileManagerView() {
|
||||
>
|
||||
{t("filesPage.newFolder", "New folder")}
|
||||
</Button>
|
||||
) : (
|
||||
// Root: the button IS the menu. The three destinations are
|
||||
// peers — none deserves to be the hidden one behind a
|
||||
// chevron — so every click shows all of them.
|
||||
<Menu shadow="md" position="bottom-end" withinPortal>
|
||||
<Menu.Target>
|
||||
<Button
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
leftSection={<CreateNewFolderIcon fontSize="small" />}
|
||||
rightSection={<ArrowDropDownIcon fontSize="small" />}
|
||||
>
|
||||
{t("filesPage.newFolder", "New folder")}
|
||||
</Button>
|
||||
</Menu.Target>
|
||||
<Menu.Dropdown>
|
||||
<Menu.Item
|
||||
leftSection={<CreateNewFolderIcon fontSize="small" />}
|
||||
onClick={() => openNewFolderDialog(null, "virtual")}
|
||||
>
|
||||
{t(
|
||||
"filesPage.newFolderMenu.device",
|
||||
"New folder on this device",
|
||||
)}
|
||||
<Text size="xs" c="dimmed">
|
||||
{t(
|
||||
"filesPage.newFolderMenu.deviceHint",
|
||||
"Lives only on this device. Works offline.",
|
||||
)}
|
||||
</Text>
|
||||
</Menu.Item>
|
||||
<Menu.Item
|
||||
leftSection={<DriveFolderUploadIcon fontSize="small" />}
|
||||
disabled={Boolean(addExistingDisabledReason)}
|
||||
onClick={() => void addExistingFolder()}
|
||||
>
|
||||
{t(
|
||||
"filesPage.newFolderMenu.addExisting",
|
||||
"Add folder from this computer…",
|
||||
)}
|
||||
<Text size="xs" c="dimmed">
|
||||
{addExistingDisabledReason ??
|
||||
t(
|
||||
"filesPage.newFolderMenu.addExistingHint",
|
||||
"Its files stay exactly where they are.",
|
||||
)}
|
||||
</Text>
|
||||
</Menu.Item>
|
||||
<Menu.Item
|
||||
leftSection={<CloudIcon fontSize="small" />}
|
||||
disabled={Boolean(serverFolderDisabledReason)}
|
||||
onClick={() => openNewFolderDialog(null, "server")}
|
||||
>
|
||||
{t(
|
||||
"filesPage.newFolderMenu.server",
|
||||
"New folder on the server",
|
||||
)}
|
||||
<Text size="xs" c="dimmed">
|
||||
{serverFolderDisabledReason ??
|
||||
t(
|
||||
"filesPage.newFolderMenu.serverHint",
|
||||
"Synced to your account, available wherever you sign in.",
|
||||
)}
|
||||
</Text>
|
||||
</Menu.Item>
|
||||
</Menu.Dropdown>
|
||||
</Menu>
|
||||
)}
|
||||
<Button
|
||||
size="sm"
|
||||
@@ -1336,6 +1781,72 @@ export default function FileManagerView() {
|
||||
style={{ width: 140 }}
|
||||
aria-label={t("filesPage.originFilter", "Filter by source")}
|
||||
/>
|
||||
{categoryOptions.length > 0 && (
|
||||
<Select
|
||||
size="xs"
|
||||
value={categoryFilter}
|
||||
onChange={(value) => setCategoryFilter(value ?? "all")}
|
||||
data={[
|
||||
{
|
||||
value: "all",
|
||||
label: t(
|
||||
"filesPage.categoryFilter.all",
|
||||
"All categories",
|
||||
),
|
||||
},
|
||||
...categoryOptions.map((category) => ({
|
||||
value: category.id,
|
||||
label: category.name,
|
||||
})),
|
||||
]}
|
||||
renderOption={({ option }) => {
|
||||
const category = categoryOptions.find(
|
||||
(c) => c.id === option.value,
|
||||
);
|
||||
return (
|
||||
<span
|
||||
style={{
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
gap: "0.4rem",
|
||||
}}
|
||||
>
|
||||
{category && (
|
||||
<LocalIcon
|
||||
icon={category.icon}
|
||||
width="0.95rem"
|
||||
style={
|
||||
category.color
|
||||
? { color: category.color }
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{option.label}
|
||||
</span>
|
||||
);
|
||||
}}
|
||||
leftSection={(() => {
|
||||
const selected = categoryOptions.find(
|
||||
(c) => c.id === categoryFilter,
|
||||
);
|
||||
return selected ? (
|
||||
<LocalIcon
|
||||
icon={selected.icon}
|
||||
width="0.95rem"
|
||||
style={
|
||||
selected.color ? { color: selected.color } : undefined
|
||||
}
|
||||
/>
|
||||
) : undefined;
|
||||
})()}
|
||||
style={{ width: 165 }}
|
||||
aria-label={t(
|
||||
"filesPage.categoryFilter.label",
|
||||
"Filter by category",
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
{availableTypes.length > 1 && (
|
||||
<MultiSelect
|
||||
size="xs"
|
||||
@@ -1361,7 +1872,10 @@ export default function FileManagerView() {
|
||||
size="xs"
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.currentTarget.value)}
|
||||
placeholder={t("filesPage.search.placeholder", "Filter files…")}
|
||||
placeholder={t(
|
||||
"filesPage.search.placeholder",
|
||||
"Filter by name or category…",
|
||||
)}
|
||||
leftSection={<SearchIcon sx={{ fontSize: "1rem" }} />}
|
||||
rightSection={
|
||||
search ? (
|
||||
@@ -1467,7 +1981,16 @@ export default function FileManagerView() {
|
||||
>
|
||||
<FileGrid
|
||||
entries={entries}
|
||||
loading={loading}
|
||||
loading={
|
||||
loading ||
|
||||
// The master view's section cards render instantly (their
|
||||
// counts fill in); only a section's own listing skeletons.
|
||||
(outputDirectory && processingSection === null
|
||||
? false
|
||||
: processingSection === "outputs"
|
||||
? outputLoading
|
||||
: diskLoading)
|
||||
}
|
||||
currentTab={currentTab}
|
||||
searchActive={search.trim().length > 0}
|
||||
serverReachable={folders.serverReachable}
|
||||
@@ -1479,6 +2002,8 @@ export default function FileManagerView() {
|
||||
onSelectFile={handleSelectFile}
|
||||
onSetSelection={setSelectedFileIds}
|
||||
onOpenFolder={handleOpenFolder}
|
||||
onOpenSection={openProcessingSection}
|
||||
onOpenDiskFile={(entry) => void openDiskFile(entry)}
|
||||
onOpenFile={handleOpenFile}
|
||||
onMoveFiles={moveFilesTo}
|
||||
onMoveFolder={moveFolderTo}
|
||||
@@ -1509,7 +2034,15 @@ export default function FileManagerView() {
|
||||
// (disabled tooltips, native file picker, dialog) is
|
||||
// identical regardless of where the user clicks from.
|
||||
onEmptyUpload={() => fileInputRef.current?.click()}
|
||||
onEmptyCreateFolder={() => openNewFolderDialog()}
|
||||
onEmptyCreateFolder={() =>
|
||||
// At the root the kind must be said out loud — the context's
|
||||
// default prefers the server, which a guest can't use. On this
|
||||
// surface there is no menu, so the never-fails kind wins.
|
||||
openNewFolderDialog(
|
||||
folders.currentFolderId,
|
||||
folders.currentFolderId === null ? "virtual" : undefined,
|
||||
)
|
||||
}
|
||||
newFolderDisabledReason={newFolderDisabledReason}
|
||||
/>
|
||||
{isDraggingExternal && (
|
||||
@@ -1692,7 +2225,14 @@ export default function FileManagerView() {
|
||||
);
|
||||
}
|
||||
|
||||
function Breadcrumbs() {
|
||||
function Breadcrumbs({
|
||||
section,
|
||||
onClearSection,
|
||||
}: {
|
||||
/** Active processing-folder section, appended as a trailing crumb. */
|
||||
section?: ProcessingSectionId | null;
|
||||
onClearSection?: () => void;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const folders = useFolders();
|
||||
const filesPage = useFilesPage();
|
||||
@@ -1703,13 +2243,21 @@ function Breadcrumbs() {
|
||||
aria-label={t("filesPage.breadcrumbs", "Folder path")}
|
||||
>
|
||||
{trail.map((entry, idx) => {
|
||||
const isLast = idx === trail.length - 1;
|
||||
const isLast = idx === trail.length - 1 && !section;
|
||||
// The current folder's crumb with a section open must clear the
|
||||
// section: re-selecting the already-current folder is a no-op, so
|
||||
// navigation is the only way back to the master view.
|
||||
const isSectionParent = idx === trail.length - 1 && Boolean(section);
|
||||
return (
|
||||
<React.Fragment key={entry.id ?? "root"}>
|
||||
<Button
|
||||
variant="tertiary"
|
||||
className={`files-page-breadcrumb${isLast ? " is-current" : ""}`}
|
||||
onClick={() => folders.setCurrentFolderId(entry.id)}
|
||||
onClick={() =>
|
||||
isSectionParent
|
||||
? onClearSection?.()
|
||||
: folders.setCurrentFolderId(entry.id)
|
||||
}
|
||||
onDragOver={(e) => {
|
||||
if (e.dataTransfer.types.includes(FILES_PAGE_DRAG_TYPE)) {
|
||||
e.preventDefault();
|
||||
@@ -1777,6 +2325,14 @@ function Breadcrumbs() {
|
||||
</React.Fragment>
|
||||
);
|
||||
})}
|
||||
{section && (
|
||||
<Button variant="tertiary" className="files-page-breadcrumb is-current">
|
||||
{t(
|
||||
PROCESSING_SECTION_LABELS[section].key,
|
||||
PROCESSING_SECTION_LABELS[section].fallback,
|
||||
)}
|
||||
</Button>
|
||||
)}
|
||||
</nav>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -11,6 +11,11 @@ interface FileOriginBadgeProps {
|
||||
origin: FileOrigin;
|
||||
/** Compact (icon-only) vs full (icon + text). */
|
||||
compact?: boolean;
|
||||
/**
|
||||
* Override the hover text. The defaults are phrased for files; a folder
|
||||
* wearing the same badge needs its own wording.
|
||||
*/
|
||||
tooltip?: string;
|
||||
}
|
||||
|
||||
const styles = {
|
||||
@@ -26,17 +31,20 @@ const styles = {
|
||||
letterSpacing: "0.04em",
|
||||
lineHeight: 1.2,
|
||||
},
|
||||
// Tints are mixed into the surface colour, never transparency: these badges
|
||||
// sit on top of thumbnails, where a see-through backer makes them illegible.
|
||||
local: {
|
||||
background: "color-mix(in srgb, var(--c-text-subtle) 16%, transparent)",
|
||||
background:
|
||||
"color-mix(in srgb, var(--c-text-subtle) 16%, var(--c-surface))",
|
||||
color: "var(--c-text-muted)",
|
||||
},
|
||||
cloud: {
|
||||
background: "color-mix(in srgb, var(--c-primary) 16%, transparent)",
|
||||
background: "color-mix(in srgb, var(--c-primary) 16%, var(--c-surface))",
|
||||
color: "var(--c-accent-text)",
|
||||
},
|
||||
shared: {
|
||||
background:
|
||||
"color-mix(in srgb, var(--mantine-color-orange-6) 16%, transparent)",
|
||||
"color-mix(in srgb, var(--mantine-color-orange-6) 16%, var(--c-surface))",
|
||||
color: "var(--color-amber-dark)",
|
||||
},
|
||||
};
|
||||
@@ -44,6 +52,7 @@ const styles = {
|
||||
export function FileOriginBadge({
|
||||
origin,
|
||||
compact = false,
|
||||
tooltip,
|
||||
}: FileOriginBadgeProps) {
|
||||
const { t } = useTranslation();
|
||||
|
||||
@@ -88,7 +97,7 @@ export function FileOriginBadge({
|
||||
);
|
||||
|
||||
return (
|
||||
<Tooltip label={config.tooltip} withinPortal>
|
||||
<Tooltip label={tooltip ?? config.tooltip} withinPortal>
|
||||
{badge}
|
||||
</Tooltip>
|
||||
);
|
||||
|
||||
@@ -476,6 +476,32 @@
|
||||
gap: 0.4rem;
|
||||
}
|
||||
|
||||
/* Marks a folder that runs a pipeline over anything added to it. Replaces the
|
||||
item count so a processing folder reads as a different kind of thing. */
|
||||
.files-page-processing-tag {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.3rem;
|
||||
padding: 0.05rem 0.4rem;
|
||||
border-radius: var(--radius-sm, 4px);
|
||||
font-size: 0.72rem;
|
||||
font-weight: 600;
|
||||
color: var(--c-success-text, var(--c-primary));
|
||||
background: color-mix(
|
||||
in srgb,
|
||||
var(--c-success, var(--c-primary)) 14%,
|
||||
transparent
|
||||
);
|
||||
}
|
||||
|
||||
.files-page-processing-tag::before {
|
||||
content: "";
|
||||
width: 0.4rem;
|
||||
height: 0.4rem;
|
||||
border-radius: 50%;
|
||||
background: currentColor;
|
||||
}
|
||||
|
||||
/* Parent-folder breadcrumb shown on cards/rows during recursive search so
|
||||
the user can tell which folder each hit lives in without navigating. */
|
||||
.files-page-card-path {
|
||||
@@ -554,8 +580,13 @@
|
||||
position: absolute;
|
||||
bottom: 0.4rem;
|
||||
left: 0.4rem;
|
||||
/* The overlay itself stays transparent to the card's clicks and drags, but
|
||||
the badge inside must catch hover or its tooltip can never open. */
|
||||
pointer-events: none;
|
||||
}
|
||||
.files-page-card-origin > * {
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
/* "Open" badge - file is currently loaded in the active workspace.
|
||||
Solid pill with white text so it reads against any thumbnail
|
||||
|
||||
@@ -16,6 +16,7 @@ import { useFolders } from "@app/contexts/FolderContext";
|
||||
import { FileId } from "@app/types/file";
|
||||
import {
|
||||
FolderId,
|
||||
folderKind,
|
||||
FolderRecord,
|
||||
FolderTreeNode,
|
||||
ROOT_FOLDER_ID,
|
||||
@@ -260,6 +261,12 @@ function TreeNodeRow({
|
||||
}: TreeNodeRowProps) {
|
||||
const { t } = useTranslation();
|
||||
const { serverReachable, setError } = useFolders();
|
||||
// Server folders need the server; a virtual folder is browser-owned and a
|
||||
// local one is managed by its directory, so its edit items disable with a
|
||||
// kind-specific hint instead of a wrong "offline" excuse.
|
||||
const kind = folderKind(node.folder);
|
||||
const editsDisabled =
|
||||
kind === "local" || (kind === "server" && !serverReachable);
|
||||
const { currentTab } = useFilesPage();
|
||||
const offlineHint = t(
|
||||
"filesPage.offlineNoFolderEdits",
|
||||
@@ -433,8 +440,17 @@ function TreeNodeRow({
|
||||
e.stopPropagation();
|
||||
onRenameFolder(node.folder);
|
||||
}}
|
||||
disabled={!serverReachable}
|
||||
title={!serverReachable ? offlineHint : undefined}
|
||||
disabled={editsDisabled}
|
||||
title={
|
||||
kind === "local"
|
||||
? t(
|
||||
"filesPage.localFolderManagedByDisk",
|
||||
"This folder is managed by its directory on disk.",
|
||||
)
|
||||
: editsDisabled
|
||||
? offlineHint
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{t("filesPage.treeMenu.rename", "Rename")}
|
||||
</Menu.Item>
|
||||
@@ -444,8 +460,17 @@ function TreeNodeRow({
|
||||
e.stopPropagation();
|
||||
onRequestNewFolder(node.folder.id);
|
||||
}}
|
||||
disabled={!serverReachable}
|
||||
title={!serverReachable ? offlineHint : undefined}
|
||||
disabled={editsDisabled}
|
||||
title={
|
||||
kind === "local"
|
||||
? t(
|
||||
"filesPage.localFolderManagedByDisk",
|
||||
"This folder is managed by its directory on disk.",
|
||||
)
|
||||
: editsDisabled
|
||||
? offlineHint
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{t("filesPage.treeMenu.newSubfolder", "New subfolder")}
|
||||
</Menu.Item>
|
||||
@@ -457,10 +482,20 @@ function TreeNodeRow({
|
||||
e.stopPropagation();
|
||||
onDeleteFolder(node.folder);
|
||||
}}
|
||||
disabled={!serverReachable}
|
||||
title={!serverReachable ? offlineHint : undefined}
|
||||
// Removing is supported for every kind — a mount's removal
|
||||
// deletes the record and nothing on disk — so only the server
|
||||
// kind's reachability gate applies here.
|
||||
disabled={kind === "server" && !serverReachable}
|
||||
title={
|
||||
kind === "server" && !serverReachable ? offlineHint : undefined
|
||||
}
|
||||
>
|
||||
{t("filesPage.treeMenu.delete", "Delete folder")}
|
||||
{kind === "local"
|
||||
? t(
|
||||
"filesPage.removeLocalFolder",
|
||||
"Remove (files stay on disk)",
|
||||
)
|
||||
: t("filesPage.treeMenu.delete", "Delete folder")}
|
||||
</Menu.Item>
|
||||
</Menu.Dropdown>
|
||||
</Menu>
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* The files page's one filter model. Every toolbar control writes one field
|
||||
* here, and visibility is decided in a single pass — so a new facet extends
|
||||
* this model instead of adding another ad-hoc `.filter` chain, and the text
|
||||
* box is one unified filter over everything we know about a file rather than
|
||||
* a name-only sub-search.
|
||||
*/
|
||||
|
||||
import type { StirlingFileStub } from "@app/types/fileContext";
|
||||
import type { FileOrigin } from "@app/components/filesPage/fileOrigin";
|
||||
import type { FilesPageOriginFilter } from "@app/contexts/FilesPageContext";
|
||||
|
||||
export interface FileFilters {
|
||||
/**
|
||||
* Free text. Matches the file's name and, where classification exists, its
|
||||
* label and category names — typing "user guide" or "finance" finds the
|
||||
* files so tagged, not just files named that way.
|
||||
*/
|
||||
text: string;
|
||||
origin: FilesPageOriginFilter;
|
||||
/** Uppercase extensions to keep; empty keeps every type. */
|
||||
types: string[];
|
||||
/** Category (label family) id, or "all". */
|
||||
category: string;
|
||||
}
|
||||
|
||||
/** What the pure matcher needs from the environment. */
|
||||
export interface FileFilterContext {
|
||||
originOf: (stub: StirlingFileStub) => FileOrigin;
|
||||
/** Labels the selected category rolls up; null when no category is chosen. */
|
||||
categoryLabelKeys: ReadonlySet<string> | null;
|
||||
/** Whether a file's labels satisfy the text needle (never, without classification). */
|
||||
labelsMatchText: (
|
||||
labels: string[] | null | undefined,
|
||||
needle: string,
|
||||
) => boolean;
|
||||
}
|
||||
|
||||
export function fileMatchesFilters(
|
||||
stub: StirlingFileStub,
|
||||
filters: FileFilters,
|
||||
ctx: FileFilterContext,
|
||||
): boolean {
|
||||
const needle = filters.text.trim().toLowerCase();
|
||||
if (
|
||||
needle &&
|
||||
!stub.name.toLowerCase().includes(needle) &&
|
||||
!ctx.labelsMatchText(stub.classificationLabels, needle)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (filters.origin !== "all" && ctx.originOf(stub) !== filters.origin) {
|
||||
return false;
|
||||
}
|
||||
if (filters.types.length > 0) {
|
||||
const ext = (stub.name.split(".").pop() ?? "").toUpperCase();
|
||||
if (!filters.types.includes(ext)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (ctx.categoryLabelKeys) {
|
||||
const labels = stub.classificationLabels ?? [];
|
||||
if (!labels.some((label) => ctx.categoryLabelKeys!.has(label))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Core stub for the Downloads processing offer.
|
||||
*
|
||||
* The real implementation lives in
|
||||
* {@code proprietary/components/policies/DownloadsProcessingWizard.tsx} and shadows this stub via
|
||||
* the {@code @app/*} alias cascade in the proprietary build. Core builds have no processing
|
||||
* folders, so this renders nothing and makes no offer.
|
||||
*/
|
||||
export function DownloadsProcessingWizard(_props?: { active?: boolean }) {
|
||||
return null;
|
||||
}
|
||||
@@ -55,6 +55,7 @@ import { getFileOrigin } from "@app/components/filesPage/fileOrigin";
|
||||
import { VersionHistoryModal } from "@app/components/filesPage/VersionHistoryModal";
|
||||
import { DeleteFilesDialog } from "@app/components/filesPage/DeleteFilesDialog";
|
||||
import { SidebarChecklistSlot } from "@app/components/shared/SidebarChecklistSlot";
|
||||
import { SidebarProcessingSlot } from "@app/components/shared/SidebarProcessingSlot";
|
||||
import {
|
||||
deleteServerFile,
|
||||
type DeleteScope,
|
||||
@@ -1088,6 +1089,10 @@ const FileSidebar = forwardRef<HTMLDivElement, FileSidebarProps>(
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Offer to process a folder of files, below the ways of opening
|
||||
one. Empty in builds without a policy engine. */}
|
||||
<SidebarProcessingSlot collapsed={collapsed} />
|
||||
</NavSurface>
|
||||
|
||||
{/* Box 2 — the file tree (this box scrolls). */}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
export interface SidebarProcessingSlotProps {
|
||||
/** Whether the sidebar is collapsed to its narrow rail. */
|
||||
collapsed?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extension point for a processing-folder offer in the sidebar's controls box.
|
||||
* Core has no policy engine to run one, so it renders nothing; builds that
|
||||
* ship processing folders (proprietary/SaaS) shadow this file.
|
||||
*/
|
||||
export function SidebarProcessingSlot(_props: SidebarProcessingSlotProps) {
|
||||
return null;
|
||||
}
|
||||
@@ -24,6 +24,67 @@ export function useFileSidebarGroups(
|
||||
return null;
|
||||
}
|
||||
|
||||
/** One classification label as a file card wears it: its own icon and the
|
||||
* accent its category carries in the sidebar, named on hover. */
|
||||
export interface LabelBadge {
|
||||
id: string;
|
||||
/** Translated display name, for the hover. */
|
||||
name: string;
|
||||
/** Material Symbols icon key (rendered via LocalIcon). */
|
||||
icon: string;
|
||||
/** CSS colour matching the label's sidebar category accent. */
|
||||
color?: string;
|
||||
}
|
||||
|
||||
const NO_BADGES: LabelBadge[] = [];
|
||||
|
||||
/** Badge descriptors for a file's labels; core (no classification) has none. */
|
||||
export function useLabelBadges(_labels?: string[] | null): LabelBadge[] {
|
||||
return NO_BADGES;
|
||||
}
|
||||
|
||||
/**
|
||||
* Badge descriptors for the categories (label families) a file's labels roll
|
||||
* up into — the same identities the sidebar groups by. Core has none.
|
||||
*/
|
||||
export function useFamilyBadges(_labels?: string[] | null): LabelBadge[] {
|
||||
return NO_BADGES;
|
||||
}
|
||||
|
||||
/** One category (label family) as a files-page filter offers it. */
|
||||
export interface CategoryFilterOption {
|
||||
id: string;
|
||||
name: string;
|
||||
/** Material Symbols icon key — the family's own sidebar icon. */
|
||||
icon: string;
|
||||
/** The accent its sidebar group wears. */
|
||||
color?: string;
|
||||
/** Label ids the category rolls up — a file matches if it carries any. */
|
||||
labelKeys: string[];
|
||||
}
|
||||
|
||||
const NO_CATEGORIES: CategoryFilterOption[] = [];
|
||||
|
||||
/** Categories to filter by; core (no classification) offers none. */
|
||||
export function useCategoryFilterOptions(): CategoryFilterOption[] {
|
||||
return NO_CATEGORIES;
|
||||
}
|
||||
|
||||
const NEVER_MATCHES = () => false;
|
||||
|
||||
/**
|
||||
* Text matcher over a file's classification: whether any of its labels' or
|
||||
* their categories' display names contain the needle. Core, which has no
|
||||
* classification, never matches — the files-page text filter then falls back
|
||||
* to names alone.
|
||||
*/
|
||||
export function useLabelSearchMatcher(): (
|
||||
labels: string[] | null | undefined,
|
||||
needle: string,
|
||||
) => boolean {
|
||||
return NEVER_MATCHES;
|
||||
}
|
||||
|
||||
// Header control for customizing the grouping; core has none, an override renders a group picker.
|
||||
export function FileSidebarGroupControls(_props: {
|
||||
stubs: StirlingFileStub[];
|
||||
|
||||
@@ -158,6 +158,7 @@ export default function RightSidebar() {
|
||||
>
|
||||
{/* Headless: enforces enabled policies on every uploaded file. */}
|
||||
{policiesEnabled && <PolicyAutoRunController />}
|
||||
{/* Offers to process the PDFs already in the user's Downloads folder, once. */}
|
||||
{!fullscreenExpanded && !isPanelVisible && !isMobile && (
|
||||
<div className="tool-panel__collapsed-strip">
|
||||
<div className="tool-panel__collapsed-top">
|
||||
|
||||
@@ -267,6 +267,8 @@ function FileContextInner({
|
||||
skipWorkspaceDispatch?: boolean;
|
||||
skipUploadTracking?: boolean;
|
||||
derivedFromTool?: boolean;
|
||||
/** Already-rendered display thumbnails, keyed by File instance. */
|
||||
precomputedThumbnails?: Map<File, string>;
|
||||
},
|
||||
): Promise<StirlingFile[]> => {
|
||||
const stirlingFiles = await addFiles(
|
||||
|
||||
@@ -13,7 +13,13 @@ import { useTranslation } from "react-i18next";
|
||||
|
||||
import { FileId } from "@app/types/file";
|
||||
import { StirlingFileStub } from "@app/types/fileContext";
|
||||
import { FolderId, FolderRecord, ROOT_FOLDER_ID } from "@app/types/folder";
|
||||
import {
|
||||
FolderId,
|
||||
FolderKind,
|
||||
FolderRecord,
|
||||
ROOT_FOLDER_ID,
|
||||
folderKind,
|
||||
} from "@app/types/folder";
|
||||
import { fileStorage } from "@app/services/fileStorage";
|
||||
import { folderSyncService } from "@app/services/folderSyncService";
|
||||
import { uploadHistoryChain } from "@app/services/serverStorageUpload";
|
||||
@@ -28,6 +34,7 @@ import {
|
||||
} from "@app/contexts/IndexedDBContext";
|
||||
import { useFileActions } from "@app/contexts/file/fileHooks";
|
||||
import { useFolders } from "@app/contexts/FolderContext";
|
||||
import { useProcessingFolders } from "@app/hooks/useProcessingFolders";
|
||||
import { useAppConfig } from "@app/contexts/AppConfigContext";
|
||||
import { useAuth } from "@app/auth/UseSession";
|
||||
|
||||
@@ -59,6 +66,8 @@ export type FilesPageTab =
|
||||
export interface FolderNameDialogState {
|
||||
mode: "new" | "rename" | null;
|
||||
parentId?: FolderId | null;
|
||||
/** For a root-level create: the kind the caller chose (menu, not dialog). */
|
||||
kind?: FolderKind;
|
||||
folder?: FolderRecord;
|
||||
}
|
||||
|
||||
@@ -95,6 +104,9 @@ interface FilesPageContextValue {
|
||||
* Empty array = no type filter applied. */
|
||||
typeFilter: string[];
|
||||
setTypeFilter: (next: string[]) => void;
|
||||
/** Selected classification category (label family) id; "all" = no filter. */
|
||||
categoryFilter: string;
|
||||
setCategoryFilter: (id: string) => void;
|
||||
|
||||
/** Active filter-tab. Drives which files appear and which UI affordances enable. */
|
||||
currentTab: FilesPageTab;
|
||||
@@ -102,7 +114,7 @@ interface FilesPageContextValue {
|
||||
|
||||
// Dialog state
|
||||
folderNameDialog: FolderNameDialogState;
|
||||
openNewFolderDialog: (parentId?: FolderId | null) => void;
|
||||
openNewFolderDialog: (parentId?: FolderId | null, kind?: FolderKind) => void;
|
||||
openRenameFolderDialog: (folder: FolderRecord) => void;
|
||||
closeFolderNameDialog: () => void;
|
||||
submitFolderName: (name: string) => Promise<void>;
|
||||
@@ -150,6 +162,7 @@ export function FilesPageProvider({ children }: { children: React.ReactNode }) {
|
||||
const indexedDB = useIndexedDB();
|
||||
const indexedDBRevision = useIndexedDBRevision();
|
||||
const folders = useFolders();
|
||||
const processingFolders = useProcessingFolders();
|
||||
const { actions: fileActions } = useFileActions();
|
||||
const { config: appConfig } = useAppConfig();
|
||||
const { isAnonymous } = useAuth();
|
||||
@@ -236,6 +249,7 @@ export function FilesPageProvider({ children }: { children: React.ReactNode }) {
|
||||
const [originFilter, setOriginFilter] =
|
||||
useState<FilesPageOriginFilter>("all");
|
||||
const [typeFilter, setTypeFilter] = useState<string[]>([]);
|
||||
const [categoryFilter, setCategoryFilter] = useState<string>("all");
|
||||
const [currentTab, setCurrentTab] = useState<FilesPageTab>("all");
|
||||
|
||||
// Dialog: folder name -----------------------------------------------------
|
||||
@@ -243,8 +257,11 @@ export function FilesPageProvider({ children }: { children: React.ReactNode }) {
|
||||
useState<FolderNameDialogState>({ mode: null });
|
||||
|
||||
const openNewFolderDialog = useCallback(
|
||||
(parentId: FolderId | null = folders.currentFolderId) => {
|
||||
setFolderNameDialog({ mode: "new", parentId });
|
||||
(
|
||||
parentId: FolderId | null = folders.currentFolderId,
|
||||
kind?: FolderKind,
|
||||
) => {
|
||||
setFolderNameDialog({ mode: "new", parentId, kind });
|
||||
},
|
||||
[folders.currentFolderId],
|
||||
);
|
||||
@@ -260,9 +277,12 @@ export function FilesPageProvider({ children }: { children: React.ReactNode }) {
|
||||
const submitFolderName = useCallback(
|
||||
async (name: string) => {
|
||||
if (folderNameDialog.mode === "new") {
|
||||
// The kind was chosen before the dialog opened (the New-folder menu);
|
||||
// it only matters at the root — a subfolder inherits its parent's.
|
||||
await folders.createFolder(
|
||||
name,
|
||||
folderNameDialog.parentId ?? folders.currentFolderId,
|
||||
folderNameDialog.kind,
|
||||
);
|
||||
} else if (
|
||||
folderNameDialog.mode === "rename" &&
|
||||
@@ -304,6 +324,47 @@ export function FilesPageProvider({ children }: { children: React.ReactNode }) {
|
||||
// Cloud list is mutated below with newly-promoted local files.
|
||||
const cloudFiles = stubs.filter((s) => s.remoteStorageId != null);
|
||||
|
||||
const targetFolder =
|
||||
folderId === null ? null : folders.foldersById.get(folderId);
|
||||
const targetKind = targetFolder ? folderKind(targetFolder) : null;
|
||||
|
||||
if (targetKind === "local") {
|
||||
// A local folder's contents are whatever its directory contains on
|
||||
// disk; putting an app file there means writing to the filesystem,
|
||||
// which is a different feature from membership, not a move.
|
||||
folders.setError(
|
||||
t(
|
||||
"filesPage.moveIntoLocalBlocked",
|
||||
"Files can't be moved into a folder that mirrors a directory on disk.",
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (targetKind === "virtual") {
|
||||
// A virtual folder is browser-owned, so membership is too: local
|
||||
// files just point their folderId at it — no upload, no server call.
|
||||
// Server files stay out: their folder membership belongs to the
|
||||
// server, and the next sync would silently snap them back.
|
||||
if (cloudFiles.length > 0) {
|
||||
folders.setError(
|
||||
t(
|
||||
"filesPage.moveIntoVirtualCloudSkipped",
|
||||
"{{count}} server file(s) were left in place — server files can't live in browser-only folders.",
|
||||
{ count: cloudFiles.length },
|
||||
),
|
||||
);
|
||||
}
|
||||
if (localOnly.length > 0) {
|
||||
await indexedDB.moveFilesToFolder(
|
||||
localOnly.map((s) => s.id),
|
||||
folderId,
|
||||
);
|
||||
}
|
||||
await refresh();
|
||||
return;
|
||||
}
|
||||
|
||||
if (folderId !== null && localOnly.length > 0) {
|
||||
// Per-file uploadHistoryChain so each gets its own remoteStorageId.
|
||||
try {
|
||||
@@ -379,7 +440,19 @@ export function FilesPageProvider({ children }: { children: React.ReactNode }) {
|
||||
}
|
||||
}
|
||||
|
||||
// Local files moving to ROOT need no cloud write.
|
||||
// Local files moving to the root DO need a write when they are leaving a
|
||||
// folder — their membership is a browser-side folderId that nothing
|
||||
// above has touched (the upload branch only runs for a non-null
|
||||
// target). Without this, a file placed in a virtual folder could never
|
||||
// be taken out of it.
|
||||
if (folderId === null && localOnly.length > 0) {
|
||||
const leaving = localOnly
|
||||
.filter((s) => (s.folderId ?? null) !== null)
|
||||
.map((s) => s.id);
|
||||
if (leaving.length > 0) {
|
||||
await indexedDB.moveFilesToFolder(leaving, null);
|
||||
}
|
||||
}
|
||||
await refresh();
|
||||
},
|
||||
[indexedDB, refresh, fileMap, folders, t, fileActions],
|
||||
@@ -397,6 +470,22 @@ export function FilesPageProvider({ children }: { children: React.ReactNode }) {
|
||||
);
|
||||
return;
|
||||
}
|
||||
// A subtree is one kind throughout (each kind has its own system of
|
||||
// record), so a cross-kind drop is refused here as a message rather
|
||||
// than surfacing as a thrown error from the context.
|
||||
if (newParentId !== null) {
|
||||
const source = folders.foldersById.get(folderId);
|
||||
const target = folders.foldersById.get(newParentId);
|
||||
if (source && target && folderKind(source) !== folderKind(target)) {
|
||||
folders.setError(
|
||||
t(
|
||||
"filesPage.moveAcrossKindsBlocked",
|
||||
"These folders live in different places, so one can't go inside the other.",
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
await folders.moveFolder(folderId, newParentId);
|
||||
},
|
||||
[folders, t],
|
||||
@@ -554,10 +643,29 @@ export function FilesPageProvider({ children }: { children: React.ReactNode }) {
|
||||
|
||||
const promptDeleteFolder = useCallback(
|
||||
(folder: FolderRecord) => {
|
||||
if (folderKind(folder) === "local") {
|
||||
// Removing a mount destroys nothing — the record goes, the directory
|
||||
// and every file in it stay — so there is nothing to warn about and
|
||||
// the delete dialog's "what about the files?" question would be a
|
||||
// scary lie. Remove directly.
|
||||
void (async () => {
|
||||
// Its processing record points at the same directory; left behind,
|
||||
// it would keep processing a folder the app no longer shows.
|
||||
await processingFolders.disable(folder).catch(() => {});
|
||||
await folders.deleteFolder(folder.id);
|
||||
})().catch((err) => {
|
||||
folders.setError(
|
||||
err instanceof Error
|
||||
? `Could not remove folder: ${err.message}`
|
||||
: "Could not remove folder.",
|
||||
);
|
||||
});
|
||||
return;
|
||||
}
|
||||
const fileCount = filesInSubtree(folder.id).length;
|
||||
setDeleteFolderDialog({ folder, fileCount });
|
||||
},
|
||||
[filesInSubtree],
|
||||
[filesInSubtree, folders, processingFolders],
|
||||
);
|
||||
|
||||
const deleteFolder = useCallback(
|
||||
@@ -595,6 +703,8 @@ export function FilesPageProvider({ children }: { children: React.ReactNode }) {
|
||||
setOriginFilter,
|
||||
typeFilter,
|
||||
setTypeFilter,
|
||||
categoryFilter,
|
||||
setCategoryFilter,
|
||||
currentTab,
|
||||
setCurrentTab,
|
||||
folderNameDialog,
|
||||
@@ -631,6 +741,7 @@ export function FilesPageProvider({ children }: { children: React.ReactNode }) {
|
||||
search,
|
||||
originFilter,
|
||||
typeFilter,
|
||||
categoryFilter,
|
||||
currentTab,
|
||||
folderNameDialog,
|
||||
openNewFolderDialog,
|
||||
|
||||
@@ -93,6 +93,26 @@ vi.mock("@app/services/folderStorage", () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
// The virtual store is exercised by its own suite (virtualFolderStorage.test);
|
||||
// here it only needs to exist and be empty so the merged load resolves.
|
||||
vi.mock("@app/services/virtualFolderStorage", () => ({
|
||||
virtualFolderStorage: {
|
||||
getAllFolders: vi.fn(() => Promise.resolve([])),
|
||||
createFolder: vi.fn(),
|
||||
updateFolder: vi.fn(),
|
||||
moveFolder: vi.fn(),
|
||||
deleteFolder: vi.fn(() => Promise.resolve([])),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@app/services/localFolderStorage", () => ({
|
||||
localFolderStorage: {
|
||||
getAllFolders: vi.fn(() => Promise.resolve([])),
|
||||
mountDirectory: vi.fn(),
|
||||
removeFolder: vi.fn(() => Promise.resolve()),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@app/contexts/IndexedDBContext", () => ({
|
||||
useIndexedDB: () => ({
|
||||
clearFolderForFiles: vi.fn().mockResolvedValue(undefined),
|
||||
|
||||
@@ -26,14 +26,18 @@ import React, {
|
||||
} from "react";
|
||||
|
||||
import { folderStorage } from "@app/services/folderStorage";
|
||||
import { virtualFolderStorage } from "@app/services/virtualFolderStorage";
|
||||
import { localFolderStorage } from "@app/services/localFolderStorage";
|
||||
import { folderSyncService } from "@app/services/folderSyncService";
|
||||
import {
|
||||
FolderBreadcrumbEntry,
|
||||
FolderId,
|
||||
FolderKind,
|
||||
FolderRecord,
|
||||
FolderTreeNode,
|
||||
ROOT_FOLDER_ID,
|
||||
createFolderId,
|
||||
folderKind,
|
||||
pickFolderColor,
|
||||
} from "@app/types/folder";
|
||||
import { useIndexedDB } from "@app/contexts/IndexedDBContext";
|
||||
@@ -88,9 +92,17 @@ interface FolderContextValue {
|
||||
ok: boolean;
|
||||
reason?: "endpoint-missing" | "network" | "server" | "client";
|
||||
}>;
|
||||
/**
|
||||
* Create a folder. With a parent, the kind is the parent's — a subtree is
|
||||
* one kind throughout, since each kind has its own system of record and a
|
||||
* mixed chain would mean an ancestry no single store can vouch for. At the
|
||||
* root, `kind` decides (default: server when this install has server-backed
|
||||
* storage, else virtual — organisation shouldn't need an account).
|
||||
*/
|
||||
createFolder: (
|
||||
name: string,
|
||||
parentFolderId?: FolderId | null,
|
||||
kind?: FolderKind,
|
||||
) => Promise<FolderRecord>;
|
||||
renameFolder: (id: FolderId, name: string) => Promise<FolderRecord | null>;
|
||||
moveFolder: (
|
||||
@@ -102,6 +114,12 @@ interface FolderContextValue {
|
||||
appearance: { color?: string; icon?: string | null },
|
||||
) => Promise<FolderRecord | null>;
|
||||
deleteFolder: (id: FolderId) => Promise<FolderId[]>;
|
||||
/**
|
||||
* Mount a directory on the machine as a local folder. Idempotent per
|
||||
* directory. Removing the mount later goes through {@link deleteFolder};
|
||||
* the directory itself is never touched by either.
|
||||
*/
|
||||
mountLocalFolder: (directory: string, name: string) => Promise<FolderRecord>;
|
||||
|
||||
getChildFolderIds: (parentId: FolderId | null) => FolderId[];
|
||||
isDescendant: (candidateId: FolderId, ancestorId: FolderId | null) => boolean;
|
||||
@@ -269,9 +287,15 @@ export function FolderProvider({ children }: FolderProviderProps) {
|
||||
const refresh = useCallback(async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const all = await folderStorage.getAllFolders();
|
||||
// Two systems of record: the server cache and the browser-owned virtual
|
||||
// store. The UI sees one list; kind says which rules each row follows.
|
||||
const [server, virtual, local] = await Promise.all([
|
||||
folderStorage.getAllFolders(),
|
||||
virtualFolderStorage.getAllFolders(),
|
||||
localFolderStorage.getAllFolders(),
|
||||
]);
|
||||
if (!mountedRef.current) return;
|
||||
setFolders(all);
|
||||
setFolders([...server, ...virtual, ...local]);
|
||||
} catch (err) {
|
||||
console.error("[FolderContext] cache read failed", err);
|
||||
if (mountedRef.current) {
|
||||
@@ -342,7 +366,12 @@ export function FolderProvider({ children }: FolderProviderProps) {
|
||||
console.warn("[FolderContext] cache replace failed", cacheErr);
|
||||
}
|
||||
if (mountedRef.current) {
|
||||
setFolders(remote);
|
||||
// Server-wins applies to server rows only: virtual and local folders
|
||||
// have no server copy, so a pull says nothing about them.
|
||||
setFolders((prev) => [
|
||||
...remote,
|
||||
...prev.filter((f) => folderKind(f) !== "server"),
|
||||
]);
|
||||
setServerReachable(true);
|
||||
setError(null);
|
||||
}
|
||||
@@ -519,11 +548,45 @@ export function FolderProvider({ children }: FolderProviderProps) {
|
||||
[bumpFolderRevision, folders, handleStaleFolder],
|
||||
);
|
||||
|
||||
/** The kind of an existing folder, or throw — mutations must never guess. */
|
||||
const requireKind = useCallback(
|
||||
(id: FolderId): FolderKind => {
|
||||
const folder = foldersById.get(id);
|
||||
if (!folder) throw new Error(`Unknown folder: ${id}`);
|
||||
return folderKind(folder);
|
||||
},
|
||||
[foldersById],
|
||||
);
|
||||
|
||||
const createFolder = useCallback(
|
||||
async (
|
||||
name: string,
|
||||
parentFolderId: FolderId | null = currentFolderId,
|
||||
kind?: FolderKind,
|
||||
): Promise<FolderRecord> => {
|
||||
// A child's kind is its parent's, always: one subtree, one system of
|
||||
// record. Only a root-level create gets to choose.
|
||||
const effectiveKind: FolderKind =
|
||||
parentFolderId !== null
|
||||
? requireKind(parentFolderId)
|
||||
: (kind ?? (storageBackedByServer ? "server" : "virtual"));
|
||||
if (effectiveKind === "local") {
|
||||
// Local folders mount a directory that already exists on disk; they
|
||||
// are registered by the feature that watches them, not created here.
|
||||
throw new Error("Cannot create folders inside a local folder");
|
||||
}
|
||||
if (effectiveKind === "virtual") {
|
||||
const record = await virtualFolderStorage.createFolder(
|
||||
name,
|
||||
parentFolderId,
|
||||
);
|
||||
if (mountedRef.current) {
|
||||
setFolders((prev) => [...prev, record]);
|
||||
setError(null);
|
||||
}
|
||||
bumpFolderRevision();
|
||||
return record;
|
||||
}
|
||||
const color = pickFolderColor(name);
|
||||
// Client-side id makes server idempotency check safe on retry.
|
||||
const id = createFolderId();
|
||||
@@ -549,11 +612,43 @@ export function FolderProvider({ children }: FolderProviderProps) {
|
||||
}
|
||||
return result;
|
||||
},
|
||||
[currentFolderId, runFolderMutation],
|
||||
[
|
||||
currentFolderId,
|
||||
requireKind,
|
||||
storageBackedByServer,
|
||||
bumpFolderRevision,
|
||||
runFolderMutation,
|
||||
],
|
||||
);
|
||||
|
||||
/** Apply a mutated non-server record to state; the store already has it. */
|
||||
const applyOwnedRecord = useCallback(
|
||||
(record: FolderRecord | null): FolderRecord | null => {
|
||||
if (record !== null && mountedRef.current) {
|
||||
setFolders((prev) =>
|
||||
prev.map((f) => (f.id === record.id ? record : f)),
|
||||
);
|
||||
setError(null);
|
||||
}
|
||||
bumpFolderRevision();
|
||||
return record;
|
||||
},
|
||||
[bumpFolderRevision],
|
||||
);
|
||||
|
||||
const renameFolder = useCallback(
|
||||
async (id: FolderId, name: string) => {
|
||||
const kind = requireKind(id);
|
||||
if (kind === "local") {
|
||||
// The record's name is the directory's name; renaming the directory
|
||||
// is the filesystem's business, not Stirling's.
|
||||
throw new Error("A local folder takes its name from its directory");
|
||||
}
|
||||
if (kind === "virtual") {
|
||||
return applyOwnedRecord(
|
||||
await virtualFolderStorage.updateFolder(id, { name }),
|
||||
);
|
||||
}
|
||||
return runFolderMutation(
|
||||
() => folderSyncService.update(id, { name }),
|
||||
async (record) => {
|
||||
@@ -565,11 +660,25 @@ export function FolderProvider({ children }: FolderProviderProps) {
|
||||
id,
|
||||
);
|
||||
},
|
||||
[runFolderMutation],
|
||||
[applyOwnedRecord, requireKind, runFolderMutation],
|
||||
);
|
||||
|
||||
const moveFolder = useCallback(
|
||||
async (id: FolderId, newParentId: FolderId | null) => {
|
||||
const kind = requireKind(id);
|
||||
// One subtree, one system of record: a folder can move to the root or
|
||||
// under a parent of its own kind, never across.
|
||||
if (newParentId !== null && requireKind(newParentId) !== kind) {
|
||||
throw new Error("Folders can only move within their own kind");
|
||||
}
|
||||
if (kind === "local") {
|
||||
throw new Error("A local folder sits where its directory sits");
|
||||
}
|
||||
if (kind === "virtual") {
|
||||
return applyOwnedRecord(
|
||||
await virtualFolderStorage.moveFolder(id, newParentId),
|
||||
);
|
||||
}
|
||||
return runFolderMutation(
|
||||
() =>
|
||||
folderSyncService.update(id, {
|
||||
@@ -585,7 +694,7 @@ export function FolderProvider({ children }: FolderProviderProps) {
|
||||
id,
|
||||
);
|
||||
},
|
||||
[runFolderMutation],
|
||||
[applyOwnedRecord, requireKind, runFolderMutation],
|
||||
);
|
||||
|
||||
const updateFolderAppearance = useCallback(
|
||||
@@ -593,6 +702,27 @@ export function FolderProvider({ children }: FolderProviderProps) {
|
||||
id: FolderId,
|
||||
appearance: { color?: string; icon?: string | null },
|
||||
) => {
|
||||
const kind = requireKind(id);
|
||||
if (kind === "local") {
|
||||
// Nothing persists a local folder's cosmetics yet; its record lives
|
||||
// with whichever feature mounted it.
|
||||
throw new Error("Local folders cannot be recoloured yet");
|
||||
}
|
||||
if (kind === "virtual") {
|
||||
// Forward only the fields the picker actually sent: it sends one key
|
||||
// per interaction, and the store's spread persists an explicit
|
||||
// undefined — so passing both keys would erase whichever appearance
|
||||
// field the user did NOT touch. (icon: null means "clear the icon"
|
||||
// and maps to an explicit undefined deliberately.)
|
||||
const updates: { color?: string; icon?: string } = {};
|
||||
if (appearance.color !== undefined) updates.color = appearance.color;
|
||||
if (appearance.icon !== undefined) {
|
||||
updates.icon = appearance.icon ?? undefined;
|
||||
}
|
||||
return applyOwnedRecord(
|
||||
await virtualFolderStorage.updateFolder(id, updates),
|
||||
);
|
||||
}
|
||||
return runFolderMutation(
|
||||
() =>
|
||||
folderSyncService.update(id, {
|
||||
@@ -608,11 +738,47 @@ export function FolderProvider({ children }: FolderProviderProps) {
|
||||
id,
|
||||
);
|
||||
},
|
||||
[runFolderMutation],
|
||||
[applyOwnedRecord, requireKind, runFolderMutation],
|
||||
);
|
||||
|
||||
const deleteFolder = useCallback(
|
||||
async (id: FolderId): Promise<FolderId[]> => {
|
||||
const kind = requireKind(id);
|
||||
if (kind === "local") {
|
||||
// Removing the mount removes the record and nothing else — the
|
||||
// directory on disk is the user's, always.
|
||||
await localFolderStorage.removeFolder(id);
|
||||
if (mountedRef.current) {
|
||||
setError(null);
|
||||
setFolders((prev) => prev.filter((f) => f.id !== id));
|
||||
if (currentFolderId === id) {
|
||||
setCurrentFolderId(ROOT_FOLDER_ID);
|
||||
}
|
||||
}
|
||||
bumpFolderRevision();
|
||||
return [id];
|
||||
}
|
||||
if (kind === "virtual") {
|
||||
// Same shape as the server path below: subtree delete, strand-reset,
|
||||
// then detach the files that pointed at any removed folder.
|
||||
const removed = await virtualFolderStorage.deleteFolder(id);
|
||||
const removedSet = new Set(removed);
|
||||
if (mountedRef.current) {
|
||||
setError(null);
|
||||
setFolders((prev) => prev.filter((f) => !removedSet.has(f.id)));
|
||||
if (
|
||||
currentFolderId &&
|
||||
shouldStrandedReset(currentFolderId, removedSet, folders)
|
||||
) {
|
||||
setCurrentFolderId(ROOT_FOLDER_ID);
|
||||
}
|
||||
}
|
||||
bumpFolderRevision();
|
||||
await clearFolderForFiles(removed).catch((e) =>
|
||||
console.warn("[FolderContext] virtual folder file cleanup", e),
|
||||
);
|
||||
return removed;
|
||||
}
|
||||
// Custom path (not runFolderMutation) because we have two best-effort
|
||||
// cleanups to coordinate, and need to reset currentFolderId BEFORE the
|
||||
// cleanups so the user isn't stranded inside a tombstone if the cache
|
||||
@@ -680,9 +846,26 @@ export function FolderProvider({ children }: FolderProviderProps) {
|
||||
currentFolderId,
|
||||
folders,
|
||||
handleStaleFolder,
|
||||
requireKind,
|
||||
],
|
||||
);
|
||||
|
||||
const mountLocalFolder = useCallback(
|
||||
async (directory: string, name: string): Promise<FolderRecord> => {
|
||||
const record = await localFolderStorage.mountDirectory(directory, name);
|
||||
if (mountedRef.current) {
|
||||
setError(null);
|
||||
// Idempotent mount can hand back a record that's already listed.
|
||||
setFolders((prev) =>
|
||||
prev.some((f) => f.id === record.id) ? prev : [...prev, record],
|
||||
);
|
||||
}
|
||||
bumpFolderRevision();
|
||||
return record;
|
||||
},
|
||||
[bumpFolderRevision],
|
||||
);
|
||||
|
||||
const value = useMemo<FolderContextValue>(
|
||||
() => ({
|
||||
folders,
|
||||
@@ -698,6 +881,7 @@ export function FolderProvider({ children }: FolderProviderProps) {
|
||||
refresh,
|
||||
pullFromServer,
|
||||
createFolder,
|
||||
mountLocalFolder,
|
||||
renameFolder,
|
||||
moveFolder,
|
||||
updateFolderAppearance,
|
||||
@@ -717,6 +901,7 @@ export function FolderProvider({ children }: FolderProviderProps) {
|
||||
refresh,
|
||||
pullFromServer,
|
||||
createFolder,
|
||||
mountLocalFolder,
|
||||
renameFolder,
|
||||
moveFolder,
|
||||
updateFolderAppearance,
|
||||
|
||||
@@ -121,6 +121,10 @@ export function createProcessedFile(
|
||||
*/
|
||||
export async function generateProcessedFileMetadata(
|
||||
file: File,
|
||||
options?: {
|
||||
/** An already-rendered display thumbnail to adopt instead of re-rendering. */
|
||||
precomputedRotatedThumbnail?: string;
|
||||
},
|
||||
): Promise<ProcessedFileMetadata | undefined> {
|
||||
// Only generate metadata for PDF files
|
||||
if (!file.type.startsWith("application/pdf")) {
|
||||
@@ -131,7 +135,7 @@ export async function generateProcessedFileMetadata(
|
||||
// One parse produces both variants: unrotated thumbnails for PageEditor
|
||||
// (rotation applied via CSS) and the rotated one for file manager display.
|
||||
const { unrotated: unrotatedResult, rotated: rotatedResult } =
|
||||
await generateThumbnailPairWithMetadata(file);
|
||||
await generateThumbnailPairWithMetadata(file, options);
|
||||
|
||||
// Large PDF whose linearized-prefix attempt failed: report "no metadata"
|
||||
// (the tolerated failure shape) rather than a bogus zero-page document.
|
||||
@@ -251,6 +255,14 @@ interface AddFileOptions {
|
||||
pageCount?: number;
|
||||
}>;
|
||||
|
||||
/**
|
||||
* Already-rendered display thumbnails, keyed by the exact File instance
|
||||
* being added. Hydration adopts one instead of re-rendering — for files
|
||||
* whose thumbnail another view (a mounted folder's listing) has just
|
||||
* produced. Metadata is still parsed; only the rasterisation is skipped.
|
||||
*/
|
||||
precomputedThumbnails?: Map<File, string>;
|
||||
|
||||
// Insertion position
|
||||
insertAfterPageId?: string;
|
||||
|
||||
@@ -524,8 +536,13 @@ export async function addFiles(
|
||||
// here would just duplicate work. Metadata is refreshed after unlock.
|
||||
processedFileMetadata = fileStub.processedFile;
|
||||
} else {
|
||||
processedFileMetadata =
|
||||
await generateProcessedFileMetadata(targetFile);
|
||||
processedFileMetadata = await generateProcessedFileMetadata(
|
||||
targetFile,
|
||||
{
|
||||
precomputedRotatedThumbnail:
|
||||
options.precomputedThumbnails?.get(targetFile),
|
||||
},
|
||||
);
|
||||
thumbnail = processedFileMetadata?.thumbnailUrl;
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -13,6 +13,8 @@ export const useFileHandler = () => {
|
||||
selectFiles?: boolean;
|
||||
/** Persist to IDB without dispatching to workspace state. */
|
||||
skipWorkspaceDispatch?: boolean;
|
||||
/** Already-rendered display thumbnails, keyed by File instance. */
|
||||
precomputedThumbnails?: Map<File, string>;
|
||||
} = {},
|
||||
): Promise<StirlingFile[]> => {
|
||||
// Merge default options with passed options - passed options take precedence
|
||||
|
||||
@@ -1,8 +1,16 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import {
|
||||
useCallback,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
useSyncExternalStore,
|
||||
} from "react";
|
||||
import type { FileId } from "@app/types/file";
|
||||
import { useFileManagement } from "@app/contexts/FileContext";
|
||||
import { useIndexedDB } from "@app/contexts/IndexedDBContext";
|
||||
import { generateThumbnailForFile } from "@app/utils/thumbnailUtils";
|
||||
import { readDiskFile } from "@app/services/localFolderContents";
|
||||
import { readClassificationLabelsFromFile } from "@app/services/fileClassification";
|
||||
|
||||
const THUMBNAIL_SIZE_LIMIT = 100 * 1024 * 1024; // 100MB
|
||||
|
||||
@@ -15,7 +23,7 @@ const LAZY_THUMB_CONCURRENCY = 2;
|
||||
let activeLazyThumbs = 0;
|
||||
const lazyThumbQueue: Array<() => Promise<void>> = [];
|
||||
|
||||
function scheduleLazyThumb(task: () => Promise<void>): void {
|
||||
export function scheduleLazyThumb(task: () => Promise<void>): void {
|
||||
lazyThumbQueue.push(task);
|
||||
drainLazyThumbQueue();
|
||||
}
|
||||
@@ -80,3 +88,167 @@ export function useLazyThumbnail(
|
||||
|
||||
return thumb;
|
||||
}
|
||||
|
||||
// ─── thumbnails for files listed straight off a mounted directory ─────────
|
||||
|
||||
/**
|
||||
* Cache keyed by path + mtime + size, so an unchanged file never renders
|
||||
* twice and an edited one re-renders. Bounded: a mounted Downloads folder can
|
||||
* list hundreds of files, and each generation reads the file's FULL bytes off
|
||||
* disk, so the cache is what makes revisits and re-sorts free.
|
||||
*/
|
||||
const diskThumbCache = new Map<string, string>();
|
||||
// Image thumbnails are data URLs whose size tracks the source image, so the
|
||||
// cache is bounded by BYTES, not entries — 300 photos would otherwise pin
|
||||
// gigabytes of strings for the process lifetime.
|
||||
const DISK_THUMB_CACHE_MAX_BYTES = 48 * 1024 * 1024;
|
||||
let diskThumbCacheBytes = 0;
|
||||
|
||||
function cacheDiskThumb(key: string, url: string): void {
|
||||
const prior = diskThumbCache.get(key);
|
||||
if (prior !== undefined) diskThumbCacheBytes -= prior.length;
|
||||
while (
|
||||
diskThumbCacheBytes + url.length > DISK_THUMB_CACHE_MAX_BYTES &&
|
||||
diskThumbCache.size > 0
|
||||
) {
|
||||
// Maps iterate in insertion order; evicting the first entry makes this
|
||||
// FIFO — crude, but evicted thumbnails simply re-render on revisit.
|
||||
const oldest = diskThumbCache.keys().next().value!;
|
||||
diskThumbCacheBytes -= diskThumbCache.get(oldest)!.length;
|
||||
diskThumbCache.delete(oldest);
|
||||
}
|
||||
diskThumbCache.set(key, url);
|
||||
diskThumbCacheBytes += url.length;
|
||||
}
|
||||
|
||||
// Reading a file's bytes is the expensive step, so it only happens for types
|
||||
// the generator can actually render — it branches on MIME (PDF and images)
|
||||
// and returns nothing for everything else, which must not cost a full read.
|
||||
const THUMBABLE_EXTENSIONS = new Set([
|
||||
"pdf",
|
||||
"png",
|
||||
"jpg",
|
||||
"jpeg",
|
||||
"gif",
|
||||
"webp",
|
||||
"bmp",
|
||||
"svg",
|
||||
]);
|
||||
|
||||
function canEverThumbnail(name: string): boolean {
|
||||
const ext = name.includes(".") ? name.split(".").pop()!.toLowerCase() : "";
|
||||
return THUMBABLE_EXTENSIONS.has(ext);
|
||||
}
|
||||
|
||||
/**
|
||||
* Classification labels read off disk-listed PDFs, keyed like the thumbnail
|
||||
* cache and filled by the same read: the thumbnail task already holds the
|
||||
* file's bytes, so extracting the embedded labels there costs one metadata
|
||||
* parse instead of a second full read. Listeners let rows already on screen
|
||||
* pick a late-arriving label up.
|
||||
*/
|
||||
const diskLabelCache = new Map<string, string[]>();
|
||||
const diskLabelListeners = new Set<() => void>();
|
||||
const NO_LABELS: string[] = [];
|
||||
|
||||
function cacheDiskLabels(key: string, labels: string[]): void {
|
||||
diskLabelCache.set(key, labels);
|
||||
diskLabelListeners.forEach((listener) => listener());
|
||||
}
|
||||
|
||||
/** Labels for a disk-listed file, once its thumbnail pass has read them. */
|
||||
export function useDiskLabels(entry: {
|
||||
path: string;
|
||||
name: string;
|
||||
sizeBytes: number;
|
||||
lastModified: number;
|
||||
}): string[] {
|
||||
const key = `${entry.path}|${entry.lastModified}|${entry.sizeBytes}`;
|
||||
const subscribe = useCallback((listener: () => void) => {
|
||||
diskLabelListeners.add(listener);
|
||||
return () => {
|
||||
diskLabelListeners.delete(listener);
|
||||
};
|
||||
}, []);
|
||||
return useSyncExternalStore(
|
||||
subscribe,
|
||||
() => diskLabelCache.get(key) ?? NO_LABELS,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The disk-listed file's already-rendered thumbnail, if the listing produced
|
||||
* one — so opening the file elsewhere can adopt it instead of re-rendering.
|
||||
* A cached "" (failed render) is not a thumbnail and reads as absent.
|
||||
*/
|
||||
export function getCachedDiskThumbnail(entry: {
|
||||
path: string;
|
||||
name: string;
|
||||
sizeBytes: number;
|
||||
lastModified: number;
|
||||
}): string | undefined {
|
||||
const hit = diskThumbCache.get(
|
||||
`${entry.path}|${entry.lastModified}|${entry.sizeBytes}`,
|
||||
);
|
||||
return hit ? hit : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Thumbnail for a disk-listed file, through the same generator and the same
|
||||
* concurrency gate as stored files — a mounted folder's rows fill in
|
||||
* progressively alongside everything else instead of stampeding the disk.
|
||||
* Returns undefined while pending, unsupported, or too large (placeholder
|
||||
* icon stays).
|
||||
*/
|
||||
export function useDiskThumbnail(entry: {
|
||||
path: string;
|
||||
name: string;
|
||||
sizeBytes: number;
|
||||
lastModified: number;
|
||||
}): string | undefined {
|
||||
const key = `${entry.path}|${entry.lastModified}|${entry.sizeBytes}`;
|
||||
const [thumb, setThumb] = useState<string | undefined>(() => {
|
||||
const hit = diskThumbCache.get(key);
|
||||
return hit === "" ? undefined : hit;
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
const cached = diskThumbCache.get(key);
|
||||
if (cached !== undefined) {
|
||||
setThumb(cached === "" ? undefined : cached);
|
||||
return;
|
||||
}
|
||||
if (entry.sizeBytes >= THUMBNAIL_SIZE_LIMIT) return;
|
||||
if (!canEverThumbnail(entry.name)) return;
|
||||
let cancelled = false;
|
||||
scheduleLazyThumb(async () => {
|
||||
if (cancelled || diskThumbCache.has(key)) return;
|
||||
try {
|
||||
const file = await readDiskFile(entry);
|
||||
if (!file || cancelled) return;
|
||||
const url = await generateThumbnailForFile(file);
|
||||
// "" is cached too: a failed/oversized render should not retry on
|
||||
// every re-mount of the same row.
|
||||
cacheDiskThumb(key, url);
|
||||
if (!cancelled && url) setThumb(url);
|
||||
// Same bytes, second harvest: a processed PDF names its categories in
|
||||
// its own metadata, and this is the one moment the file is in hand.
|
||||
if (file.type === "application/pdf" && !diskLabelCache.has(key)) {
|
||||
const labels = await readClassificationLabelsFromFile(file).catch(
|
||||
() => null,
|
||||
);
|
||||
cacheDiskLabels(key, labels ?? []);
|
||||
}
|
||||
} catch {
|
||||
cacheDiskThumb(key, "");
|
||||
}
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
// The key encodes every field of `entry` this effect reads.
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [key]);
|
||||
|
||||
return thumb;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { FolderRecord } from "@app/types/folder";
|
||||
|
||||
/** A folder's processing state, as the files page needs to render it. */
|
||||
export interface ProcessingFolderState {
|
||||
/** The processing record's own id — not the folder's. */
|
||||
id: string;
|
||||
enabled: boolean;
|
||||
/** Where a disk-backed folder's results land, when the record names one. */
|
||||
outputDirectory?: string;
|
||||
}
|
||||
|
||||
/** One in-flight run of a processing folder, as the files page shows it. */
|
||||
export interface ProcessingRunInfo {
|
||||
runId: string;
|
||||
/** The document being processed, when the run's source recorded a name. */
|
||||
fileName: string | null;
|
||||
currentStep: number;
|
||||
stepCount: number;
|
||||
}
|
||||
|
||||
export interface ProcessingFoldersApi {
|
||||
/** The folder's processing state; undefined means an ordinary folder. */
|
||||
stateFor: (folder: FolderRecord) => ProcessingFolderState | undefined;
|
||||
/** Server-storage folder ids whose processing is enabled, for id-only callers. */
|
||||
enabledFolderIds: ReadonlySet<string>;
|
||||
/** Whether any processing folder is enabled, whatever it watches. */
|
||||
anyEnabled: boolean;
|
||||
/** The record's runs that are currently executing (or queued to). */
|
||||
listActiveRuns: (recordId: string) => Promise<ProcessingRunInfo[]>;
|
||||
/** Attach the default (classification) pipeline to a folder. */
|
||||
enable: (folder: FolderRecord) => Promise<void>;
|
||||
/** Remove the processing behaviour; the folder and its files stay. */
|
||||
disable: (folder: FolderRecord) => Promise<void>;
|
||||
/** Process the folder's current contents now. */
|
||||
sweep: (folder: FolderRecord) => Promise<void>;
|
||||
}
|
||||
|
||||
const EMPTY_IDS: ReadonlySet<string> = new Set();
|
||||
|
||||
/**
|
||||
* Processing folders — folders that run a pipeline over anything added to
|
||||
* them, whatever kind of folder they are. Inert in core; the proprietary
|
||||
* build shadows this with an implementation backed by
|
||||
* `/api/v1/processing-folders`.
|
||||
*/
|
||||
export function useProcessingFolders(): ProcessingFoldersApi {
|
||||
return {
|
||||
stateFor: () => undefined,
|
||||
enabledFolderIds: EMPTY_IDS,
|
||||
anyEnabled: false,
|
||||
listActiveRuns: async () => [],
|
||||
enable: async () => {},
|
||||
disable: async () => {},
|
||||
sweep: async () => {},
|
||||
};
|
||||
}
|
||||
|
||||
/** Reload the shared list. No-op in core, which has no processing folders. */
|
||||
export function refreshProcessingFolders(): Promise<void> {
|
||||
return Promise.resolve();
|
||||
}
|
||||
@@ -35,6 +35,7 @@ import {
|
||||
useFilesPage,
|
||||
} from "@app/contexts/FilesPageContext";
|
||||
import { useFolders } from "@app/contexts/FolderContext";
|
||||
import { folderKind } from "@app/types/folder";
|
||||
import { useFileHandler } from "@app/hooks/useFileHandler";
|
||||
import { FolderTreePanel } from "@app/components/filesPage/FolderTreePanel";
|
||||
import type { FileSidebarProps } from "@app/components/shared/FileSidebar";
|
||||
@@ -588,12 +589,27 @@ const MyFilesSidebarOverrides = forwardRef<HTMLDivElement, FileSidebarProps>(
|
||||
[addFiles, filesPage, folders.currentFolderId],
|
||||
);
|
||||
|
||||
const newFolderDisabledReason = !folders.serverReachable
|
||||
? t(
|
||||
"filesPage.newFolderStorageDisabled",
|
||||
"Server folder storage isn't enabled. Ask your admin to turn it on.",
|
||||
)
|
||||
// Kind-aware: only a server folder's subfolder needs the server, and a
|
||||
// mounted directory takes no subfolders from here at all. At the root the
|
||||
// rail creates a folder on this device, which nothing can disable.
|
||||
const railCurrentFolder = folders.currentFolderId
|
||||
? folders.foldersById.get(folders.currentFolderId)
|
||||
: undefined;
|
||||
const railCurrentKind = railCurrentFolder
|
||||
? folderKind(railCurrentFolder)
|
||||
: null;
|
||||
const newFolderDisabledReason =
|
||||
railCurrentKind === "local"
|
||||
? t(
|
||||
"filesPage.newFolderInLocalUnavailable",
|
||||
"This folder mirrors a directory on disk — create subfolders in your file explorer.",
|
||||
)
|
||||
: railCurrentKind === "server" && !folders.serverReachable
|
||||
? t(
|
||||
"filesPage.newFolderStorageDisabled",
|
||||
"Server folder storage isn't enabled.",
|
||||
)
|
||||
: null;
|
||||
|
||||
return (
|
||||
<FileSidebar
|
||||
@@ -604,7 +620,11 @@ const MyFilesSidebarOverrides = forwardRef<HTMLDivElement, FileSidebarProps>(
|
||||
extraAction={{
|
||||
icon: <CreateNewFolderIcon />,
|
||||
label: t("filesPage.newFolder", "New folder"),
|
||||
onClick: () => filesPage.openNewFolderDialog(),
|
||||
onClick: () =>
|
||||
filesPage.openNewFolderDialog(
|
||||
folders.currentFolderId,
|
||||
folders.currentFolderId === null ? "virtual" : undefined,
|
||||
),
|
||||
disabled: newFolderDisabledReason !== null,
|
||||
disabledTooltip: newFolderDisabledReason ?? undefined,
|
||||
testId: "files-rail-new-folder",
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/**
|
||||
* Picking a directory on the machine, as a real filesystem path.
|
||||
*
|
||||
* Only an environment that can see the filesystem can offer this — a browser
|
||||
* deliberately cannot reveal paths (the File System Access API deals in
|
||||
* handles, not locations), so core reports the capability absent and the
|
||||
* desktop build shadows this module with the Tauri dialog.
|
||||
*/
|
||||
|
||||
export interface PickedDirectory {
|
||||
/** Absolute path, as the platform writes it. */
|
||||
path: string;
|
||||
/** The directory's own name — the mounted folder's display name. */
|
||||
name: string;
|
||||
}
|
||||
|
||||
/** Whether this build can produce a directory path at all. */
|
||||
export const canPickDirectory = false;
|
||||
|
||||
/** Ask the user for a directory; null when cancelled (or unsupported). */
|
||||
export async function pickDirectory(): Promise<PickedDirectory | null> {
|
||||
return null;
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import { alert } from "@app/components/toast";
|
||||
import { StirlingFileStub, StirlingFile } from "@app/types/fileContext";
|
||||
import { FileId } from "@app/types/fileContext";
|
||||
import { FolderId, parseFolderId } from "@app/types/folder";
|
||||
import { virtualFolderStorage } from "@app/services/virtualFolderStorage";
|
||||
import {
|
||||
isZipBundle,
|
||||
loadShareBundleEntries,
|
||||
@@ -119,6 +120,16 @@ export async function reconcileServerFiles(
|
||||
}
|
||||
|
||||
let combinedStubs: StirlingFileStub[];
|
||||
// Virtual folders are browser-owned, so a stub sitting in one must keep its
|
||||
// membership through the reconcile — the server's folderId (always null for
|
||||
// them) is not an opinion about it. Read best-effort: with the store
|
||||
// unreadable, behave exactly as before the guard existed.
|
||||
const virtualFolderIds = new Set<FolderId>(
|
||||
await virtualFolderStorage
|
||||
.getAllFolders()
|
||||
.then((folders) => folders.map((folder) => folder.id))
|
||||
.catch(() => []),
|
||||
);
|
||||
const localRemoteIds = new Set(
|
||||
localStubs
|
||||
.map((s) => s.remoteStorageId)
|
||||
@@ -202,7 +213,12 @@ export async function reconcileServerFiles(
|
||||
// Server is authoritative for cloud-stored files. Don't fall back to
|
||||
// stub.folderId on null - that would resurrect a stale folder pointer
|
||||
// after the server SET_NULL'd it (e.g. owner deleted the folder).
|
||||
folderId: safeParseFolderId(serverFile.folderId),
|
||||
// EXCEPT when the stub sits in a browser-owned (virtual) folder: the
|
||||
// server has never heard of that folder, so its null says nothing
|
||||
// about the membership and must not eject the file from it.
|
||||
folderId: virtualFolderIds.has((stub.folderId ?? "") as FolderId)
|
||||
? stub.folderId
|
||||
: safeParseFolderId(serverFile.folderId),
|
||||
};
|
||||
});
|
||||
|
||||
|
||||
@@ -11,12 +11,27 @@
|
||||
* are all the server's job now.
|
||||
*/
|
||||
|
||||
import { FolderId, FolderRecord } from "@app/types/folder";
|
||||
import { FolderId, FolderRecord, folderKind } from "@app/types/folder";
|
||||
import {
|
||||
indexedDBManager,
|
||||
DATABASE_CONFIGS,
|
||||
} from "@app/services/indexedDBManager";
|
||||
|
||||
/**
|
||||
* This cache is wiped and rewritten from the server's response on every sync,
|
||||
* so a non-server folder stored here would silently vanish on the next pull.
|
||||
* Virtual folders live in their own store (virtualFolderStorage); local
|
||||
* folders are records of a directory, not cache entries. Refusing loudly here
|
||||
* is what keeps a mis-routed mutation a bug report instead of data loss.
|
||||
*/
|
||||
function requireServerFolder(folder: FolderRecord): void {
|
||||
if (folderKind(folder) !== "server") {
|
||||
throw new Error(
|
||||
`folderStorage caches server folders only; got kind "${folderKind(folder)}" for ${folder.id}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class FolderStorageService {
|
||||
private readonly dbConfig = DATABASE_CONFIGS.FILES;
|
||||
private readonly storeName = "folders";
|
||||
@@ -43,6 +58,7 @@ class FolderStorageService {
|
||||
reject(transaction.error ?? new Error("folder cache replace aborted"));
|
||||
store.clear();
|
||||
for (const folder of folders) {
|
||||
requireServerFolder(folder);
|
||||
store.put(folder);
|
||||
}
|
||||
});
|
||||
@@ -50,6 +66,7 @@ class FolderStorageService {
|
||||
|
||||
/** Insert or overwrite a single folder in the cache. */
|
||||
async upsertFolder(folder: FolderRecord): Promise<void> {
|
||||
requireServerFolder(folder);
|
||||
const db = await this.getDatabase();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const transaction = db.transaction([this.storeName], "readwrite");
|
||||
|
||||
@@ -51,6 +51,9 @@ function toFolderRecord(dto: ServerFolder): FolderRecord {
|
||||
dto.parentFolderId === null ? null : parseFolderId(dto.parentFolderId);
|
||||
return {
|
||||
id,
|
||||
// Everything that comes off this wire is a server folder by definition;
|
||||
// virtual and local folders never round-trip through the server at all.
|
||||
kind: "server",
|
||||
name: dto.name,
|
||||
parentFolderId,
|
||||
color: dto.color ?? undefined,
|
||||
|
||||
@@ -399,4 +399,73 @@ describe("IndexedDB migration (FILES store)", () => {
|
||||
TARGET_VERSION,
|
||||
);
|
||||
});
|
||||
|
||||
test("a v10 profile missing local_folders upgrades to v11 with the full schema", async () => {
|
||||
// v10 briefly existed with only one of the two browser-folder stores.
|
||||
// The cure is the shipped version bump: v11 declares both, so the normal
|
||||
// upgrade path (which only adds what's absent) completes the schema.
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const req = indexedDB.open(DB_NAME, 10);
|
||||
req.onupgradeneeded = () => {
|
||||
const db = req.result;
|
||||
db.createObjectStore("files", { keyPath: "id" });
|
||||
db.createObjectStore("folders", { keyPath: "id" });
|
||||
db.createObjectStore("virtual_folders", { keyPath: "id" });
|
||||
// local_folders deliberately absent.
|
||||
};
|
||||
req.onsuccess = () => {
|
||||
req.result.close();
|
||||
resolve();
|
||||
};
|
||||
req.onerror = () => reject(req.error);
|
||||
});
|
||||
|
||||
const db = await indexedDBManager.openDatabase(DATABASE_CONFIGS.FILES);
|
||||
const names = Array.from(db.objectStoreNames);
|
||||
expect(names).toContain("local_folders");
|
||||
expect(names).toContain("virtual_folders");
|
||||
expect(db.version).toBe(TARGET_VERSION);
|
||||
indexedDBManager.closeDatabase(DB_NAME);
|
||||
});
|
||||
|
||||
test("v9 -> latest adds virtual_folders without touching files or folders", async () => {
|
||||
// Seed a database shaped like the v9 schema: files + folders, no
|
||||
// virtual_folders yet, with a row in each that must survive the upgrade.
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const req = indexedDB.open(DB_NAME, 9);
|
||||
req.onupgradeneeded = () => {
|
||||
const db = req.result;
|
||||
db.createObjectStore("files", { keyPath: "id" });
|
||||
db.createObjectStore("folders", { keyPath: "id" });
|
||||
};
|
||||
req.onsuccess = () => {
|
||||
const db = req.result;
|
||||
const tx = db.transaction(["files", "folders"], "readwrite");
|
||||
tx.objectStore("files").put({ id: "file-1", folderId: null });
|
||||
tx.objectStore("folders").put({ id: "folder-1", name: "Kept" });
|
||||
tx.oncomplete = () => {
|
||||
db.close();
|
||||
resolve();
|
||||
};
|
||||
tx.onerror = () => reject(tx.error);
|
||||
};
|
||||
req.onerror = () => reject(req.error);
|
||||
});
|
||||
|
||||
await indexedDBManager.openDatabase(DATABASE_CONFIGS.FILES);
|
||||
indexedDBManager.closeDatabase(DB_NAME);
|
||||
|
||||
const stores = await getObjectStoreNames();
|
||||
expect(stores).toContain("virtual_folders");
|
||||
expect(stores).toContain("local_folders");
|
||||
expect(stores).toContain("files");
|
||||
expect(stores).toContain("folders");
|
||||
|
||||
const rows = (await readAllFiles()) as Array<Record<string, unknown>>;
|
||||
expect(rows.map((row) => row.id)).toEqual(["file-1"]);
|
||||
|
||||
expect(await indexedDBManager.getDatabaseVersion(DB_NAME)).toBe(
|
||||
TARGET_VERSION,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -465,7 +465,11 @@ class IndexedDBManager {
|
||||
export const DATABASE_CONFIGS = {
|
||||
FILES: {
|
||||
name: "stirling-pdf-files",
|
||||
version: 9,
|
||||
// v10 existed briefly with only one of the two browser-folder stores;
|
||||
// v11 declares both, so every v10 profile upgrades to a full schema.
|
||||
// Never add a store under an already-opened version number — the upgrade
|
||||
// only fires on a version change, so late additions are unreachable.
|
||||
version: 11,
|
||||
stores: [
|
||||
{
|
||||
name: "files",
|
||||
@@ -492,6 +496,33 @@ export const DATABASE_CONFIGS = {
|
||||
{ name: "createdAt", keyPath: "createdAt", unique: false },
|
||||
],
|
||||
},
|
||||
// Browser-owned folders (kind "virtual"), deliberately a separate store
|
||||
// from `folders`: that one is a cache the server sync wipes wholesale on
|
||||
// every pull, and these rows have no server copy to be restored from.
|
||||
// NOT named smart_folders/folder_members/folder_run_states — the upgrade
|
||||
// cleanup above deletes stores by those names.
|
||||
// Folders mounted from a directory on the machine (kind "local"). Flat
|
||||
// by construction — a mount has no parent, and its subdirectories are
|
||||
// the filesystem's business. Same lifecycle reasoning as
|
||||
// virtual_folders: browser-owned, so never in the server-synced cache.
|
||||
{
|
||||
name: "local_folders",
|
||||
keyPath: "id",
|
||||
indexes: [{ name: "name", keyPath: "name", unique: false }],
|
||||
},
|
||||
{
|
||||
name: "virtual_folders",
|
||||
keyPath: "id",
|
||||
indexes: [
|
||||
{
|
||||
name: "parentFolderId",
|
||||
keyPath: "parentFolderId",
|
||||
unique: false,
|
||||
},
|
||||
{ name: "name", keyPath: "name", unique: false },
|
||||
{ name: "createdAt", keyPath: "createdAt", unique: false },
|
||||
],
|
||||
},
|
||||
],
|
||||
} as DatabaseConfig,
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* Reading a mounted local folder's contents straight off the disk.
|
||||
*
|
||||
* A local folder is read-through: the directory is the source of truth and
|
||||
* nothing is ingested to show it — the listing IS the directory, taken fresh
|
||||
* on every look. Only an environment that can see the filesystem can do
|
||||
* this, so core reports the capability absent and the desktop build shadows
|
||||
* this module with the Tauri filesystem plugin.
|
||||
*/
|
||||
|
||||
/** One file inside a mounted directory, as the file manager lists it. */
|
||||
export interface DiskFileEntry {
|
||||
/** Absolute path — the file's identity here; nothing about it is stored. */
|
||||
path: string;
|
||||
name: string;
|
||||
sizeBytes: number;
|
||||
lastModified: number;
|
||||
}
|
||||
|
||||
/** Whether this build can list a directory at all. */
|
||||
export const canListDirectory = false;
|
||||
|
||||
/**
|
||||
* The regular files directly inside `directory` (no recursion — a mount's
|
||||
* subdirectories are the filesystem's business). Null when unsupported.
|
||||
*/
|
||||
export async function listDirectory(
|
||||
_directory: string,
|
||||
): Promise<DiskFileEntry[] | null> {
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Read one listed file's bytes as a File, ready for the workbench. */
|
||||
export async function readDiskFile(
|
||||
_entry: DiskFileEntry,
|
||||
): Promise<File | null> {
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
/**
|
||||
* Local Folder Storage - the record of directories mounted into the file
|
||||
* manager (kind "local").
|
||||
*
|
||||
* A local folder is a pointer at a directory on the machine; the directory
|
||||
* itself is the source of truth for everything else — name, contents,
|
||||
* lifetime — so the record carries only where it is and how to show it.
|
||||
* Mounts are flat by construction: they have no parent, and a directory's
|
||||
* subdirectories are the filesystem's business, not a folder hierarchy for
|
||||
* this store to model. Removing a mount removes the record and nothing else.
|
||||
*/
|
||||
|
||||
import {
|
||||
FolderId,
|
||||
FolderRecord,
|
||||
folderKind,
|
||||
createFolderId,
|
||||
pickFolderColor,
|
||||
} from "@app/types/folder";
|
||||
import {
|
||||
indexedDBManager,
|
||||
DATABASE_CONFIGS,
|
||||
} from "@app/services/indexedDBManager";
|
||||
|
||||
function requireLocalFolder(folder: FolderRecord): void {
|
||||
if (folderKind(folder) !== "local") {
|
||||
throw new Error(
|
||||
`localFolderStorage owns local folders only; got kind "${folderKind(folder)}" for ${folder.id}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class LocalFolderStorageService {
|
||||
private readonly dbConfig = DATABASE_CONFIGS.FILES;
|
||||
private readonly storeName = "local_folders";
|
||||
|
||||
private async getDatabase(): Promise<IDBDatabase> {
|
||||
return indexedDBManager.openDatabase(this.dbConfig);
|
||||
}
|
||||
|
||||
async getAllFolders(): Promise<FolderRecord[]> {
|
||||
const db = await this.getDatabase();
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = db.transaction([this.storeName], "readonly");
|
||||
const store = transaction.objectStore(this.storeName);
|
||||
const request = store.getAll();
|
||||
request.onerror = () => reject(request.error);
|
||||
request.onsuccess = () =>
|
||||
resolve((request.result as FolderRecord[]) ?? []);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Mount a directory. Mounting the same directory twice hands back the
|
||||
* existing record — two rows for one directory would be two names for one
|
||||
* truth, and removing one would lie about the other.
|
||||
*/
|
||||
async mountDirectory(directory: string, name: string): Promise<FolderRecord> {
|
||||
const existing = (await this.getAllFolders()).find(
|
||||
(folder) => folder.directory === directory,
|
||||
);
|
||||
if (existing) return existing;
|
||||
const now = Date.now();
|
||||
const record: FolderRecord = {
|
||||
id: createFolderId(),
|
||||
kind: "local",
|
||||
name,
|
||||
parentFolderId: null,
|
||||
directory,
|
||||
color: pickFolderColor(name),
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
requireLocalFolder(record);
|
||||
const db = await this.getDatabase();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const transaction = db.transaction([this.storeName], "readwrite");
|
||||
const store = transaction.objectStore(this.storeName);
|
||||
const req = store.put(record);
|
||||
req.onerror = () => reject(req.error);
|
||||
req.onsuccess = () => resolve();
|
||||
});
|
||||
return record;
|
||||
}
|
||||
|
||||
/** Remove the mount. The directory on disk is untouched, always. */
|
||||
async removeFolder(id: FolderId): Promise<void> {
|
||||
const db = await this.getDatabase();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const transaction = db.transaction([this.storeName], "readwrite");
|
||||
const store = transaction.objectStore(this.storeName);
|
||||
const req = store.delete(id);
|
||||
req.onerror = () => reject(req.error);
|
||||
req.onsuccess = () => resolve();
|
||||
});
|
||||
}
|
||||
|
||||
async clearAll(): Promise<void> {
|
||||
const db = await this.getDatabase();
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = db.transaction([this.storeName], "readwrite");
|
||||
const store = transaction.objectStore(this.storeName);
|
||||
const request = store.clear();
|
||||
request.onerror = () => reject(request.error);
|
||||
request.onsuccess = () => resolve();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export const localFolderStorage = new LocalFolderStorageService();
|
||||
@@ -0,0 +1,78 @@
|
||||
import { describe, expect, test, beforeEach } from "vitest";
|
||||
import "fake-indexeddb/auto";
|
||||
import { IDBFactory } from "fake-indexeddb";
|
||||
import { virtualFolderStorage } from "@app/services/virtualFolderStorage";
|
||||
import { folderStorage } from "@app/services/folderStorage";
|
||||
import {
|
||||
indexedDBManager,
|
||||
DATABASE_CONFIGS,
|
||||
} from "@app/services/indexedDBManager";
|
||||
|
||||
/**
|
||||
* Virtual folders have no server to be authoritative, so the invariants the
|
||||
* server enforces for its folders (no cycles, bounded depth, subtree deletes
|
||||
* that report every removed id) are this module's own responsibility.
|
||||
*/
|
||||
describe("virtualFolderStorage", () => {
|
||||
beforeEach(() => {
|
||||
indexedDBManager.closeDatabase(DATABASE_CONFIGS.FILES.name);
|
||||
indexedDB = new IDBFactory();
|
||||
});
|
||||
|
||||
test("creates rows stamped virtual, invisible to the server folder cache", async () => {
|
||||
const created = await virtualFolderStorage.createFolder("Research", null);
|
||||
expect(created.kind).toBe("virtual");
|
||||
|
||||
// Same DB, different store: the server cache must not see it, because a
|
||||
// sync wipes that cache wholesale and would silently destroy the row.
|
||||
expect(await folderStorage.getAllFolders()).toEqual([]);
|
||||
expect(await virtualFolderStorage.getAllFolders()).toEqual([created]);
|
||||
});
|
||||
|
||||
test("the server cache refuses a virtual row outright", async () => {
|
||||
const virtual = await virtualFolderStorage.createFolder("Research", null);
|
||||
await expect(folderStorage.upsertFolder(virtual)).rejects.toThrow(
|
||||
/server folders only/,
|
||||
);
|
||||
});
|
||||
|
||||
test("refuses to move a folder into its own subtree", async () => {
|
||||
const parent = await virtualFolderStorage.createFolder("a", null);
|
||||
const child = await virtualFolderStorage.createFolder("b", parent.id);
|
||||
const grandchild = await virtualFolderStorage.createFolder("c", child.id);
|
||||
|
||||
await expect(
|
||||
virtualFolderStorage.moveFolder(parent.id, grandchild.id),
|
||||
).rejects.toThrow(/own subtree/);
|
||||
await expect(
|
||||
virtualFolderStorage.moveFolder(parent.id, parent.id),
|
||||
).rejects.toThrow(/into itself/);
|
||||
});
|
||||
|
||||
test("deleting a folder removes its whole subtree and reports every id", async () => {
|
||||
const parent = await virtualFolderStorage.createFolder("a", null);
|
||||
const child = await virtualFolderStorage.createFolder("b", parent.id);
|
||||
const grandchild = await virtualFolderStorage.createFolder("c", child.id);
|
||||
const bystander = await virtualFolderStorage.createFolder("keep", null);
|
||||
|
||||
const removed = await virtualFolderStorage.deleteFolder(parent.id);
|
||||
|
||||
// Every removed id is reported so the caller can unlink files that
|
||||
// referenced them — the same contract as the server delete.
|
||||
expect([...removed].sort()).toEqual(
|
||||
[parent.id, child.id, grandchild.id].sort(),
|
||||
);
|
||||
expect(await virtualFolderStorage.getAllFolders()).toEqual([bystander]);
|
||||
});
|
||||
|
||||
test("refuses to nest past the depth cap", async () => {
|
||||
let parentId = (await virtualFolderStorage.createFolder("d0", null)).id;
|
||||
for (let i = 1; i < 64; i += 1) {
|
||||
parentId = (await virtualFolderStorage.createFolder(`d${i}`, parentId))
|
||||
.id;
|
||||
}
|
||||
await expect(
|
||||
virtualFolderStorage.createFolder("too-deep", parentId),
|
||||
).rejects.toThrow(/depth limit/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,250 @@
|
||||
/**
|
||||
* Virtual Folder Storage - the system of record for kind "virtual" folders.
|
||||
*
|
||||
* Unlike {@link folderStorage} (a passive cache of the server's folder
|
||||
* hierarchy, wiped and rewritten on every sync), this store OWNS its rows:
|
||||
* a virtual folder exists only in this browser's IndexedDB and has no server
|
||||
* copy to be restored from. That is the point — virtual folders organise
|
||||
* files on installs with no login, no server storage, or no network.
|
||||
*
|
||||
* Because there is no server to be authoritative, the invariants the server
|
||||
* enforces for its folders are enforced here instead: no reparenting a folder
|
||||
* under its own subtree (cycles), and a bounded chain depth. Limits mirror
|
||||
* FolderService so a hierarchy never behaves differently for being virtual.
|
||||
*/
|
||||
|
||||
import {
|
||||
FolderId,
|
||||
FolderProcessingConfig,
|
||||
FolderRecord,
|
||||
folderKind,
|
||||
createFolderId,
|
||||
pickFolderColor,
|
||||
} from "@app/types/folder";
|
||||
import {
|
||||
indexedDBManager,
|
||||
DATABASE_CONFIGS,
|
||||
} from "@app/services/indexedDBManager";
|
||||
|
||||
/** Mirrors FolderService.MAX_FOLDER_DEPTH so virtual trees can't out-nest server ones. */
|
||||
const MAX_FOLDER_DEPTH = 64;
|
||||
|
||||
function requireVirtualFolder(folder: FolderRecord): void {
|
||||
if (folderKind(folder) !== "virtual") {
|
||||
throw new Error(
|
||||
`virtualFolderStorage owns virtual folders only; got kind "${folderKind(folder)}" for ${folder.id}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class VirtualFolderStorageService {
|
||||
private readonly dbConfig = DATABASE_CONFIGS.FILES;
|
||||
private readonly storeName = "virtual_folders";
|
||||
|
||||
private async getDatabase(): Promise<IDBDatabase> {
|
||||
return indexedDBManager.openDatabase(this.dbConfig);
|
||||
}
|
||||
|
||||
async getAllFolders(): Promise<FolderRecord[]> {
|
||||
const db = await this.getDatabase();
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = db.transaction([this.storeName], "readonly");
|
||||
const store = transaction.objectStore(this.storeName);
|
||||
const request = store.getAll();
|
||||
request.onerror = () => reject(request.error);
|
||||
request.onsuccess = () =>
|
||||
resolve((request.result as FolderRecord[]) ?? []);
|
||||
});
|
||||
}
|
||||
|
||||
async getFolder(id: FolderId): Promise<FolderRecord | null> {
|
||||
const db = await this.getDatabase();
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = db.transaction([this.storeName], "readonly");
|
||||
const store = transaction.objectStore(this.storeName);
|
||||
const request = store.get(id);
|
||||
request.onerror = () => reject(request.error);
|
||||
request.onsuccess = () =>
|
||||
resolve((request.result as FolderRecord | undefined) ?? null);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a virtual folder under the given parent (null = root). The parent,
|
||||
* when set, must itself be a virtual folder: a virtual row can't hang off a
|
||||
* server folder, whose lifetime this browser doesn't control — a server-side
|
||||
* delete would orphan the whole virtual subtree with nothing to notice.
|
||||
*/
|
||||
async createFolder(
|
||||
name: string,
|
||||
parentFolderId: FolderId | null,
|
||||
): Promise<FolderRecord> {
|
||||
if (parentFolderId !== null) {
|
||||
await this.requireWithinDepth(parentFolderId);
|
||||
}
|
||||
const now = Date.now();
|
||||
const record: FolderRecord = {
|
||||
id: createFolderId(),
|
||||
kind: "virtual",
|
||||
name,
|
||||
parentFolderId,
|
||||
color: pickFolderColor(name),
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
await this.put(record);
|
||||
return record;
|
||||
}
|
||||
|
||||
/** Attach or replace the folder's processing pipeline; null removes it. */
|
||||
async setProcessing(
|
||||
id: FolderId,
|
||||
config: FolderProcessingConfig | null,
|
||||
): Promise<FolderRecord | null> {
|
||||
const existing = await this.getFolder(id);
|
||||
if (!existing) return null;
|
||||
const next: FolderRecord = {
|
||||
...existing,
|
||||
processing: config ?? undefined,
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
if (!config) delete next.processing;
|
||||
await this.put(next);
|
||||
return next;
|
||||
}
|
||||
|
||||
/** Rename / recolour / re-icon in place. Structure is moveFolder's job. */
|
||||
async updateFolder(
|
||||
id: FolderId,
|
||||
updates: Partial<Pick<FolderRecord, "name" | "color" | "icon">>,
|
||||
): Promise<FolderRecord | null> {
|
||||
const existing = await this.getFolder(id);
|
||||
if (!existing) return null;
|
||||
const next: FolderRecord = {
|
||||
...existing,
|
||||
...updates,
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
await this.put(next);
|
||||
return next;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reparent a folder (null = to root), refusing moves that would make the
|
||||
* tree lie: under itself or its own descendant (a cycle — the subtree would
|
||||
* fall out of every walk), or deeper than the depth cap.
|
||||
*/
|
||||
async moveFolder(
|
||||
id: FolderId,
|
||||
newParentId: FolderId | null,
|
||||
): Promise<FolderRecord | null> {
|
||||
const existing = await this.getFolder(id);
|
||||
if (!existing) return null;
|
||||
if (newParentId !== null) {
|
||||
if (newParentId === id) {
|
||||
throw new Error("Cannot move a folder into itself");
|
||||
}
|
||||
const ancestors = await this.requireWithinDepth(newParentId);
|
||||
if (ancestors.has(id)) {
|
||||
throw new Error("Cannot move a folder into its own subtree");
|
||||
}
|
||||
}
|
||||
const next: FolderRecord = {
|
||||
...existing,
|
||||
parentFolderId: newParentId,
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
await this.put(next);
|
||||
return next;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a folder and its whole virtual subtree, returning every removed id
|
||||
* so the caller can unlink files that referenced them — mirroring the shape
|
||||
* of the server delete, which reports removedFolderIds for the same reason.
|
||||
*/
|
||||
async deleteFolder(id: FolderId): Promise<FolderId[]> {
|
||||
const all = await this.getAllFolders();
|
||||
const childrenByParent = new Map<FolderId | null, FolderRecord[]>();
|
||||
for (const folder of all) {
|
||||
const siblings = childrenByParent.get(folder.parentFolderId) ?? [];
|
||||
siblings.push(folder);
|
||||
childrenByParent.set(folder.parentFolderId, siblings);
|
||||
}
|
||||
const removed: FolderId[] = [];
|
||||
const queue: FolderId[] = [id];
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift()!;
|
||||
removed.push(current);
|
||||
for (const child of childrenByParent.get(current) ?? []) {
|
||||
queue.push(child.id);
|
||||
}
|
||||
}
|
||||
const db = await this.getDatabase();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const transaction = db.transaction([this.storeName], "readwrite");
|
||||
const store = transaction.objectStore(this.storeName);
|
||||
transaction.oncomplete = () => resolve();
|
||||
transaction.onerror = () =>
|
||||
reject(transaction.error ?? new Error("virtual folder delete failed"));
|
||||
transaction.onabort = () =>
|
||||
reject(transaction.error ?? new Error("virtual folder delete aborted"));
|
||||
for (const folderId of removed) store.delete(folderId);
|
||||
});
|
||||
return removed;
|
||||
}
|
||||
|
||||
async clearAll(): Promise<void> {
|
||||
const db = await this.getDatabase();
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = db.transaction([this.storeName], "readwrite");
|
||||
const store = transaction.objectStore(this.storeName);
|
||||
const request = store.clear();
|
||||
request.onerror = () => reject(request.error);
|
||||
request.onsuccess = () => resolve();
|
||||
});
|
||||
}
|
||||
|
||||
private async put(record: FolderRecord): Promise<void> {
|
||||
requireVirtualFolder(record);
|
||||
const db = await this.getDatabase();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const transaction = db.transaction([this.storeName], "readwrite");
|
||||
const store = transaction.objectStore(this.storeName);
|
||||
const req = store.put(record);
|
||||
req.onerror = () => reject(req.error);
|
||||
req.onsuccess = () => resolve();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk from `startId` to the root, returning the ids seen. Throws when the
|
||||
* chain is missing a link (the parent must exist and be virtual), already
|
||||
* cyclic (defensive — a bug or hand-edited DB, not a reachable state), or
|
||||
* too deep to accept another child.
|
||||
*/
|
||||
private async requireWithinDepth(startId: FolderId): Promise<Set<FolderId>> {
|
||||
const seen = new Set<FolderId>();
|
||||
let cursor: FolderId | null = startId;
|
||||
while (cursor !== null) {
|
||||
if (seen.has(cursor)) {
|
||||
throw new Error("Virtual folder hierarchy contains a cycle");
|
||||
}
|
||||
seen.add(cursor);
|
||||
const parent: FolderRecord | null = await this.getFolder(cursor);
|
||||
if (parent === null) {
|
||||
throw new Error(`No virtual folder: ${cursor}`);
|
||||
}
|
||||
cursor = parent.parentFolderId;
|
||||
}
|
||||
// The chain walked is the prospective parent's own ancestry; whatever is
|
||||
// being placed under it sits one level deeper, so a full-depth chain has
|
||||
// no room for a child.
|
||||
if (seen.size >= MAX_FOLDER_DEPTH) {
|
||||
throw new Error(`Folder depth limit reached (max ${MAX_FOLDER_DEPTH})`);
|
||||
}
|
||||
return seen;
|
||||
}
|
||||
}
|
||||
|
||||
export const virtualFolderStorage = new VirtualFolderStorageService();
|
||||
@@ -37,11 +37,47 @@ export const FOLDER_COLOR_PALETTE = [
|
||||
/** Members of {@link FOLDER_COLOR_PALETTE}. Use this rather than `string` to keep callers honest. */
|
||||
export type FolderPaletteColor = (typeof FOLDER_COLOR_PALETTE)[number];
|
||||
|
||||
/**
|
||||
* What kind of thing a folder is — three independent features that happen to
|
||||
* share a shape, not variants of one:
|
||||
*
|
||||
* - `server`: a folder in app storage. Lives in the server's database, synced
|
||||
* down and cached in IndexedDB; needs login + storage to exist.
|
||||
* - `virtual`: an organisation-only folder in this browser's IndexedDB. No
|
||||
* server involvement at all, so it works offline and on installs with
|
||||
* storage disabled.
|
||||
* - `local`: a real directory on the machine, mounted read-through — the
|
||||
* filesystem is the source of truth and Stirling holds no copy of its
|
||||
* contents, only this record of where it is.
|
||||
*/
|
||||
export type FolderKind = "server" | "virtual" | "local";
|
||||
|
||||
/**
|
||||
* A pipeline a folder runs over its files. Only browser-owned (virtual)
|
||||
* folders carry this on their record: server and mounted folders keep their
|
||||
* processing configuration server-side, where their engine runs.
|
||||
*/
|
||||
export interface FolderProcessingConfig {
|
||||
enabled: boolean;
|
||||
/** Tool endpoint paths with their parameters, run in order per file. */
|
||||
steps: Array<{ operation: string; parameters: Record<string, unknown> }>;
|
||||
}
|
||||
|
||||
/** Persisted folder shape stored in IndexedDB. */
|
||||
export interface FolderRecord {
|
||||
id: FolderId;
|
||||
/**
|
||||
* Absent means `server`: kinds arrived after rows already existed in user
|
||||
* databases and on the server wire, and every one of those is a server
|
||||
* folder. Read through {@link folderKind} rather than directly.
|
||||
*/
|
||||
kind?: FolderKind;
|
||||
name: string;
|
||||
parentFolderId: FolderId | null;
|
||||
/** For `local` folders: the directory this record mounts. */
|
||||
directory?: string;
|
||||
/** For `virtual` folders: the pipeline this folder runs over its files. */
|
||||
processing?: FolderProcessingConfig;
|
||||
/** Hex colour - either a palette member or any custom hex from a future picker. */
|
||||
color?: string;
|
||||
icon?: string;
|
||||
@@ -49,6 +85,11 @@ export interface FolderRecord {
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
/** The folder's kind, reading absent as `server` (pre-kinds rows and server DTOs). */
|
||||
export function folderKind(folder: Pick<FolderRecord, "kind">): FolderKind {
|
||||
return folder.kind ?? "server";
|
||||
}
|
||||
|
||||
/**
|
||||
* Folder tree node - derived from FolderRecord[] for rendering the tree
|
||||
* navigator. Children are ordered by name (case-insensitive).
|
||||
|
||||
@@ -157,6 +157,7 @@ async function renderPdfThumbnailPairPdfium(
|
||||
data: ArrayBuffer,
|
||||
scale: number,
|
||||
collectAllPagesMetadata: boolean,
|
||||
precomputedRotatedThumbnail?: string,
|
||||
): Promise<{ unrotated: PdfiumRenderResult; rotated: PdfiumRenderResult }> {
|
||||
const m = await getPdfiumModule();
|
||||
let docPtr: number;
|
||||
@@ -181,17 +182,34 @@ async function renderPdfThumbnailPairPdfium(
|
||||
|
||||
try {
|
||||
const pageCount = m.FPDF_GetPageCount(docPtr);
|
||||
const unrotatedThumb = await renderPdfiumPageDataUrl(docPtr, 0, scale, {
|
||||
applyRotation: false,
|
||||
});
|
||||
const rotatedThumb = await renderPdfiumPageDataUrl(docPtr, 0, scale, {
|
||||
applyRotation: true,
|
||||
});
|
||||
const firstMeta = await readPdfiumPageMetadata(docPtr, 0);
|
||||
|
||||
// A caller that already rendered this document's display thumbnail (the
|
||||
// disk view, whose cache keys never reach this layer) supplies it instead
|
||||
// of paying for the same rasterisation again. It is the rotated variant;
|
||||
// when page 0 carries no rotation the two variants are identical, so one
|
||||
// image serves both and no rendering happens at all.
|
||||
let unrotatedThumb: string | null;
|
||||
let rotatedThumb: string | null;
|
||||
if (precomputedRotatedThumbnail) {
|
||||
rotatedThumb = precomputedRotatedThumbnail;
|
||||
unrotatedThumb =
|
||||
(firstMeta?.rotation ?? 0) === 0
|
||||
? precomputedRotatedThumbnail
|
||||
: await renderPdfiumPageDataUrl(docPtr, 0, scale, {
|
||||
applyRotation: false,
|
||||
});
|
||||
} else {
|
||||
unrotatedThumb = await renderPdfiumPageDataUrl(docPtr, 0, scale, {
|
||||
applyRotation: false,
|
||||
});
|
||||
rotatedThumb = await renderPdfiumPageDataUrl(docPtr, 0, scale, {
|
||||
applyRotation: true,
|
||||
});
|
||||
}
|
||||
if (!unrotatedThumb || !rotatedThumb) {
|
||||
throw new Error("PDFium: failed to render page 0");
|
||||
}
|
||||
|
||||
const firstMeta = await readPdfiumPageMetadata(docPtr, 0);
|
||||
const pageRotations: number[] = [firstMeta?.rotation ?? 0];
|
||||
const pageDimensions: Array<{ width: number; height: number }> = [
|
||||
{ width: firstMeta?.width ?? 0, height: firstMeta?.height ?? 0 },
|
||||
@@ -362,7 +380,13 @@ export async function generateThumbnailWithMetadata(
|
||||
* Large PDFs only get the linearized-prefix attempt; if that fails, both
|
||||
* variants are empty placeholders and page metadata is omitted.
|
||||
*/
|
||||
export async function generateThumbnailPairWithMetadata(file: File): Promise<{
|
||||
export async function generateThumbnailPairWithMetadata(
|
||||
file: File,
|
||||
options?: {
|
||||
/** An already-rendered rotated (display) thumbnail to adopt instead of re-rendering. */
|
||||
precomputedRotatedThumbnail?: string;
|
||||
},
|
||||
): Promise<{
|
||||
unrotated: ThumbnailWithMetadata;
|
||||
rotated: ThumbnailWithMetadata;
|
||||
}> {
|
||||
@@ -382,7 +406,12 @@ export async function generateThumbnailPairWithMetadata(file: File): Promise<{
|
||||
const buffer = isLarge
|
||||
? await file.slice(0, LINEARIZED_PREFIX_BYTES).arrayBuffer()
|
||||
: await file.arrayBuffer();
|
||||
const pair = await renderPdfThumbnailPairPdfium(buffer, scale, !isLarge);
|
||||
const pair = await renderPdfThumbnailPairPdfium(
|
||||
buffer,
|
||||
scale,
|
||||
!isLarge,
|
||||
options?.precomputedRotatedThumbnail,
|
||||
);
|
||||
|
||||
const toPublic = (r: PdfiumRenderResult): ThumbnailWithMetadata =>
|
||||
r.isEncrypted
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Desktop directory picking: the Tauri file dialog hands back a real path,
|
||||
* which is the whole reason local folders are a desktop capability — a
|
||||
* browser can only produce handles, never locations.
|
||||
*/
|
||||
|
||||
import { isTauri } from "@tauri-apps/api/core";
|
||||
import { open } from "@tauri-apps/plugin-dialog";
|
||||
import type { PickedDirectory } from "@core/services/directoryPicker";
|
||||
export type { PickedDirectory };
|
||||
|
||||
// The desktop bundle also runs as a plain web page in dev; only the actual
|
||||
// Tauri webview can open the native dialog.
|
||||
export const canPickDirectory = isTauri();
|
||||
|
||||
export async function pickDirectory(): Promise<PickedDirectory | null> {
|
||||
if (!canPickDirectory) return null;
|
||||
const picked = await open({ directory: true, multiple: false });
|
||||
if (typeof picked !== "string" || picked.length === 0) return null;
|
||||
// The path's last segment, tolerant of either separator and a trailing one.
|
||||
const name =
|
||||
picked
|
||||
.replace(/[\\/]+$/, "")
|
||||
.split(/[\\/]/)
|
||||
.pop() || picked;
|
||||
return { path: picked, name };
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* Desktop read-through for mounted local folders, over the Tauri filesystem
|
||||
* plugin. The listing is taken fresh from the directory on every call —
|
||||
* nothing is copied or ingested to produce it.
|
||||
*/
|
||||
|
||||
import { isTauri } from "@tauri-apps/api/core";
|
||||
import { join } from "@tauri-apps/api/path";
|
||||
import { readDir, readFile, stat } from "@tauri-apps/plugin-fs";
|
||||
import type { DiskFileEntry } from "@core/services/localFolderContents";
|
||||
export type { DiskFileEntry };
|
||||
|
||||
/**
|
||||
* A directory can hold anything; the page shouldn't drown in it. Everything
|
||||
* up to the cap lists; past it, the freshest files win — for a Downloads-like
|
||||
* directory that is also the end the user is looking for.
|
||||
*/
|
||||
const LIST_CAP = 500;
|
||||
|
||||
export const canListDirectory = isTauri();
|
||||
|
||||
export async function listDirectory(
|
||||
directory: string,
|
||||
): Promise<DiskFileEntry[] | null> {
|
||||
if (!canListDirectory) return null;
|
||||
const dirEntries = await readDir(directory);
|
||||
const files: DiskFileEntry[] = [];
|
||||
for (const entry of dirEntries) {
|
||||
// Regular, visible files only: subdirectories are the filesystem's
|
||||
// business, and dotfiles are hidden there for a reason.
|
||||
if (!entry.isFile || entry.name.startsWith(".")) continue;
|
||||
const path = await join(directory, entry.name);
|
||||
try {
|
||||
const info = await stat(path);
|
||||
files.push({
|
||||
path,
|
||||
name: entry.name,
|
||||
sizeBytes: info.size,
|
||||
lastModified: info.mtime ? new Date(info.mtime).getTime() : 0,
|
||||
});
|
||||
} catch {
|
||||
// Vanished or unreadable mid-listing; the next look tells the truth.
|
||||
}
|
||||
}
|
||||
files.sort((a, b) => b.lastModified - a.lastModified);
|
||||
return files.slice(0, LIST_CAP);
|
||||
}
|
||||
|
||||
/**
|
||||
* The filesystem gives back bytes and a name, never a MIME type — but
|
||||
* everything downstream branches on File.type (the thumbnail generator's PDF
|
||||
* path, the workbench's format handling), and an untyped File silently takes
|
||||
* every "unknown format" branch. Recover the type from the extension.
|
||||
*/
|
||||
const MIME_BY_EXTENSION: Record<string, string> = {
|
||||
pdf: "application/pdf",
|
||||
png: "image/png",
|
||||
jpg: "image/jpeg",
|
||||
jpeg: "image/jpeg",
|
||||
gif: "image/gif",
|
||||
webp: "image/webp",
|
||||
bmp: "image/bmp",
|
||||
svg: "image/svg+xml",
|
||||
};
|
||||
|
||||
function mimeForName(name: string): string {
|
||||
const ext = name.includes(".") ? name.split(".").pop()!.toLowerCase() : "";
|
||||
return MIME_BY_EXTENSION[ext] ?? "";
|
||||
}
|
||||
|
||||
export async function readDiskFile(entry: DiskFileEntry): Promise<File | null> {
|
||||
if (!canListDirectory) return null;
|
||||
const bytes = await readFile(entry.path);
|
||||
return new File([new Uint8Array(bytes)], entry.name, {
|
||||
type: mimeForName(entry.name),
|
||||
lastModified: entry.lastModified || undefined,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
.downloads-wizard__title {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.45rem;
|
||||
}
|
||||
|
||||
.downloads-wizard__body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.7rem;
|
||||
font-size: 0.9rem;
|
||||
color: var(--c-text);
|
||||
}
|
||||
|
||||
.downloads-wizard__path {
|
||||
font-family: var(--font-mono, monospace);
|
||||
font-size: 0.8rem;
|
||||
color: var(--c-text-muted);
|
||||
background: var(--c-surface);
|
||||
border: 1px solid var(--c-border-subtle);
|
||||
border-radius: var(--radius-sm, 4px);
|
||||
padding: 0.35rem 0.5rem;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
.downloads-wizard__facts {
|
||||
margin: 0;
|
||||
padding-left: 1.1rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.35rem;
|
||||
font-size: 0.84rem;
|
||||
color: var(--c-text-muted);
|
||||
}
|
||||
|
||||
.downloads-wizard__progress {
|
||||
align-items: center;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
/* Fills as runs settle; width is driven inline from the settled/total ratio. */
|
||||
.downloads-wizard__bar {
|
||||
width: 100%;
|
||||
height: 0.35rem;
|
||||
border-radius: 999px;
|
||||
background: var(--c-border-subtle);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.downloads-wizard__bar > span {
|
||||
display: block;
|
||||
height: 100%;
|
||||
border-radius: 999px;
|
||||
background: var(--c-primary);
|
||||
transition: width 0.3s ease;
|
||||
}
|
||||
|
||||
.downloads-wizard__tick {
|
||||
color: var(--c-success, var(--c-primary));
|
||||
}
|
||||
|
||||
.downloads-wizard__warn {
|
||||
color: var(--c-danger-text, var(--c-text));
|
||||
font-size: 0.84rem;
|
||||
}
|
||||
|
||||
.downloads-wizard__foot {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
/* The entry point: a compact row in the side panel holding the button that opens the offer. */
|
||||
.downloads-wizard__trigger {
|
||||
display: flex;
|
||||
padding: 0.5rem;
|
||||
}
|
||||
|
||||
.downloads-wizard__trigger > * {
|
||||
width: 100%;
|
||||
justify-content: flex-start;
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Loader } from "@mantine/core";
|
||||
import CheckCircleIcon from "@mui/icons-material/CheckCircle";
|
||||
import FolderSpecialIcon from "@mui/icons-material/FolderSpecial";
|
||||
import { Button } from "@app/ui/Button";
|
||||
import { Modal } from "@app/ui/Modal";
|
||||
import {
|
||||
CLASSIFY_OPERATION,
|
||||
fetchDownloadsSuggestion,
|
||||
saveProcessingFolder,
|
||||
type DownloadsSuggestion,
|
||||
} from "@app/services/processingFolderApi";
|
||||
import { deliverSweepResults } from "@app/services/processingRunDelivery";
|
||||
import { refreshProcessingFolders } from "@app/hooks/useProcessingFolders";
|
||||
import { useFileHandler } from "@app/hooks/useFileHandler";
|
||||
import { useFolders } from "@app/contexts/FolderContext";
|
||||
import { canListDirectory } from "@app/services/localFolderContents";
|
||||
import "@app/components/policies/DownloadsProcessingWizard.css";
|
||||
|
||||
type Phase = "asking" | "working" | "done" | "failed";
|
||||
|
||||
interface DownloadsProcessingWizardProps {
|
||||
/** Renders nothing until true, so the offer never competes with a first load. */
|
||||
active?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Offers to process the PDFs already in the user's Downloads folder, then shows what it is doing.
|
||||
*
|
||||
* <p>Renders as a button; the offer opens on click. The server names its own Downloads directory
|
||||
* (the browser cannot see the machine's paths) and counts what is waiting; approving composes a
|
||||
* processing folder over it. The first sweep is capped server-side, and anything beyond the cap is
|
||||
* picked up by later sweeps rather than dropped.
|
||||
*/
|
||||
export function DownloadsProcessingWizard({
|
||||
active = true,
|
||||
}: DownloadsProcessingWizardProps) {
|
||||
const { t } = useTranslation();
|
||||
const [suggestion, setSuggestion] = useState<DownloadsSuggestion | null>(
|
||||
null,
|
||||
);
|
||||
const [open, setOpen] = useState(false);
|
||||
const [phase, setPhase] = useState<Phase>("asking");
|
||||
const [processed, setProcessed] = useState(0);
|
||||
const [failed, setFailed] = useState(0);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [started, setStarted] = useState(0);
|
||||
const [skipped, setSkipped] = useState(0);
|
||||
const [stalled, setStalled] = useState(false);
|
||||
const [opened, setOpened] = useState(0);
|
||||
const { addFiles } = useFileHandler();
|
||||
const { mountLocalFolder } = useFolders();
|
||||
|
||||
// Only offer where it can actually work: Downloads must exist, be a permitted folder root, and
|
||||
// have something in it worth processing.
|
||||
//
|
||||
// Asked repeatedly rather than once, because the window can open before the backend is
|
||||
// reachable — on a desktop install the app and its bundled server start together, and the UI
|
||||
// always wins that race. A single attempt would fail on every cold start and the offer would
|
||||
// simply never appear. Gives up after a bounded wait so an install where the answer is a
|
||||
// genuine "no" stops asking.
|
||||
useEffect(() => {
|
||||
if (!active) return;
|
||||
let cancelled = false;
|
||||
let attempts = 0;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
|
||||
const ask = () => {
|
||||
void fetchDownloadsSuggestion()
|
||||
.then((next) => {
|
||||
if (cancelled) return;
|
||||
if (next.available && next.pdfCount > 0) {
|
||||
setSuggestion(next);
|
||||
return;
|
||||
}
|
||||
// A definite answer: Downloads is missing, not permitted, or empty. Nothing to wait for.
|
||||
})
|
||||
.catch(() => {
|
||||
// Backend not up yet, storage/folder access off, or not authenticated. Only the first of
|
||||
// those resolves itself, so retry a while before concluding there is no offer.
|
||||
if (cancelled || (attempts += 1) >= 20) return;
|
||||
timer = setTimeout(ask, 1500);
|
||||
});
|
||||
};
|
||||
ask();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
}, [active]);
|
||||
|
||||
/** Closing resets to the question, so the offer can be reopened and re-run. */
|
||||
const close = () => {
|
||||
setOpen(false);
|
||||
setPhase("asking");
|
||||
setProcessed(0);
|
||||
setFailed(0);
|
||||
setError(null);
|
||||
setStarted(0);
|
||||
setSkipped(0);
|
||||
setStalled(false);
|
||||
setOpened(0);
|
||||
};
|
||||
|
||||
/**
|
||||
* Deliver the sweep's results into the workbench as they settle, mirroring
|
||||
* the shared delivery's progress into this dialog's own display state.
|
||||
*/
|
||||
const trackRuns = useCallback(
|
||||
async (policyId: string, expected: number) => {
|
||||
await deliverSweepResults(policyId, expected, addFiles, (progress) => {
|
||||
setProcessed(progress.processed);
|
||||
setFailed(progress.failed);
|
||||
setOpened(progress.opened);
|
||||
if (progress.stalled) setStalled(true);
|
||||
});
|
||||
},
|
||||
[addFiles],
|
||||
);
|
||||
|
||||
const approve = async () => {
|
||||
if (!suggestion) return;
|
||||
setPhase("working");
|
||||
try {
|
||||
const folder = await saveProcessingFolder({
|
||||
directory: suggestion.directory,
|
||||
enabled: true,
|
||||
steps: [{ operation: CLASSIFY_OPERATION, parameters: {}, assets: {} }],
|
||||
});
|
||||
// Mount the directory as a local folder too, so Downloads exists in the
|
||||
// file manager as a real folder — the processing record attaches to it
|
||||
// there — rather than results appearing from nowhere. Only where this
|
||||
// build can actually read the directory (the desktop app, where the
|
||||
// server's Downloads IS this machine's): a plain browser mounting the
|
||||
// server's path would show a folder that is forever empty. Idempotent,
|
||||
// and best-effort: the sweep's results matter more than the bookmark.
|
||||
if (canListDirectory) {
|
||||
const segments = suggestion.directory.split(/[/\\]/).filter(Boolean);
|
||||
await mountLocalFolder(
|
||||
suggestion.directory,
|
||||
segments[segments.length - 1] ?? suggestion.directory,
|
||||
).catch(() => {});
|
||||
}
|
||||
// The server reports what it actually started; 0 means everything there was already
|
||||
// processed, which is a finished state, not something to wait for.
|
||||
setStarted(folder.startedRuns);
|
||||
setSkipped(folder.alreadyProcessed);
|
||||
// The new folder was created outside the hook's own actions; refresh the shared list so the
|
||||
// files page and any other consumer pick it up without a reload.
|
||||
void refreshProcessingFolders();
|
||||
if (folder.startedRuns > 0) {
|
||||
await trackRuns(folder.id, folder.startedRuns);
|
||||
}
|
||||
// One sweep, not a standing watch: the offer's promise is "sort out what is already in
|
||||
// Downloads", so the folder is stood down once it has. Leaving it enabled would keep
|
||||
// opening files into the workbench every time anything landed in Downloads.
|
||||
await saveProcessingFolder({
|
||||
id: folder.id,
|
||||
directory: suggestion.directory,
|
||||
enabled: false,
|
||||
steps: [{ operation: CLASSIFY_OPERATION, parameters: {}, assets: {} }],
|
||||
}).catch(() => {
|
||||
// The results are already in; a folder left running is a nuisance, not a failure.
|
||||
});
|
||||
void refreshProcessingFolders();
|
||||
setPhase("done");
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
setPhase("failed");
|
||||
}
|
||||
};
|
||||
|
||||
if (!suggestion) return null;
|
||||
|
||||
const capped = suggestion.pdfCount > suggestion.limit;
|
||||
const total = Math.min(suggestion.pdfCount, suggestion.limit);
|
||||
|
||||
if (!open) {
|
||||
return (
|
||||
<div className="downloads-wizard__trigger">
|
||||
<Button
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={() => setOpen(true)}
|
||||
leftSection={<FolderSpecialIcon fontSize="small" />}
|
||||
>
|
||||
{t("processingFolders.downloads.trigger", {
|
||||
count: suggestion.pdfCount,
|
||||
defaultValue: "Process {{count}} PDFs in Downloads",
|
||||
})}
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Modal
|
||||
open
|
||||
onClose={phase === "working" ? () => {} : close}
|
||||
width="sm"
|
||||
title={
|
||||
<span className="downloads-wizard__title">
|
||||
<FolderSpecialIcon fontSize="small" />
|
||||
{t("processingFolders.downloads.title", "Organise your Downloads?")}
|
||||
</span>
|
||||
}
|
||||
footer={
|
||||
<div className="downloads-wizard__foot">
|
||||
{phase === "asking" && (
|
||||
<>
|
||||
<Button variant="tertiary" size="sm" onClick={close}>
|
||||
{t("processingFolders.downloads.notNow", "Not now")}
|
||||
</Button>
|
||||
<Button size="sm" onClick={() => void approve()}>
|
||||
{t(
|
||||
"processingFolders.downloads.approve",
|
||||
"Process my Downloads",
|
||||
)}
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
{phase === "working" && (
|
||||
<Button size="sm" disabled loading>
|
||||
{t("processingFolders.downloads.working", "Processing…")}
|
||||
</Button>
|
||||
)}
|
||||
{(phase === "done" || phase === "failed") && (
|
||||
<Button size="sm" onClick={close}>
|
||||
{t("processingFolders.downloads.close", "Done")}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
}
|
||||
>
|
||||
{phase === "asking" && (
|
||||
<div className="downloads-wizard__body">
|
||||
<p>
|
||||
{t("processingFolders.downloads.explain", {
|
||||
count: total,
|
||||
defaultValue:
|
||||
"Stirling can classify the {{count}} PDFs already in your Downloads folder and open the results here.",
|
||||
})}
|
||||
</p>
|
||||
<p className="downloads-wizard__path">{suggestion.directory}</p>
|
||||
<ul className="downloads-wizard__facts">
|
||||
<li>
|
||||
{t(
|
||||
"processingFolders.downloads.keepsOriginals",
|
||||
"Your files stay where they are — originals are never moved or deleted.",
|
||||
)}
|
||||
</li>
|
||||
<li>
|
||||
{t("processingFolders.downloads.outputs", {
|
||||
subdir: "Stirling Processed",
|
||||
defaultValue:
|
||||
'Results are saved into a "{{subdir}}" folder alongside them.',
|
||||
})}
|
||||
</li>
|
||||
{capped && (
|
||||
<li>
|
||||
{t("processingFolders.downloads.capped", {
|
||||
limit: suggestion.limit,
|
||||
found: suggestion.pdfCount,
|
||||
defaultValue:
|
||||
"You have {{found}} PDFs; the first {{limit}} are processed now and the rest follow.",
|
||||
})}
|
||||
</li>
|
||||
)}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{phase === "working" && (
|
||||
<div className="downloads-wizard__body downloads-wizard__progress">
|
||||
<Loader size="sm" />
|
||||
<p>
|
||||
{t("processingFolders.downloads.progress", {
|
||||
done: processed + failed,
|
||||
total: started || total,
|
||||
defaultValue: "Processing {{done}} of {{total}} files…",
|
||||
})}
|
||||
</p>
|
||||
<div className="downloads-wizard__bar" role="progressbar">
|
||||
<span
|
||||
style={{
|
||||
width: `${
|
||||
(started || total) === 0
|
||||
? 0
|
||||
: Math.round(
|
||||
((processed + failed) / (started || total)) * 100,
|
||||
)
|
||||
}%`,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{phase === "done" && (
|
||||
<div className="downloads-wizard__body downloads-wizard__progress">
|
||||
<CheckCircleIcon className="downloads-wizard__tick" />
|
||||
{started === 0 ? (
|
||||
<p>
|
||||
{t("processingFolders.downloads.nothingNew", {
|
||||
count: skipped,
|
||||
defaultValue:
|
||||
"Nothing new to process — these {{count}} files have already been through.",
|
||||
})}
|
||||
</p>
|
||||
) : (
|
||||
<p>
|
||||
{t("processingFolders.downloads.finished", {
|
||||
count: processed,
|
||||
opened,
|
||||
defaultValue:
|
||||
"Classified {{count}} files and opened {{opened}} of them here, ready to work on.",
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
{failed > 0 && (
|
||||
<p className="downloads-wizard__warn">
|
||||
{t("processingFolders.downloads.someFailed", {
|
||||
count: failed,
|
||||
defaultValue:
|
||||
"{{count}} could not be processed and were left untouched.",
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
{stalled && (
|
||||
<p className="downloads-wizard__warn">
|
||||
{t(
|
||||
"processingFolders.downloads.stillRunning",
|
||||
"Some files are still being processed in the background.",
|
||||
)}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{phase === "failed" && (
|
||||
<div className="downloads-wizard__body">
|
||||
<p className="downloads-wizard__warn">
|
||||
{error ??
|
||||
t(
|
||||
"processingFolders.downloads.failed",
|
||||
"Could not set that up. Your files have not been changed.",
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { usePolicyAutoRun } from "@app/components/policies/usePolicyAutoRun";
|
||||
import { useClientSideClassification } from "@app/components/policies/useClientSideClassification";
|
||||
import { useVirtualFolderProcessing } from "@app/components/policies/useVirtualFolderProcessing";
|
||||
|
||||
/**
|
||||
* Headless controller that drives policy auto-run (enforce every enabled policy
|
||||
@@ -10,5 +11,7 @@ export function PolicyAutoRunController() {
|
||||
usePolicyAutoRun();
|
||||
// Non-AI systems classify uploads in the browser; inert when the AI engine is on.
|
||||
useClientSideClassification();
|
||||
// Virtual processing folders run their pipelines from the browser; inert when AI is off.
|
||||
useVirtualFolderProcessing();
|
||||
return null;
|
||||
}
|
||||
|
||||
+12
@@ -54,6 +54,18 @@ vi.mock("@app/hooks/usePolicies", () => ({
|
||||
},
|
||||
}),
|
||||
}));
|
||||
// No processing folders in these cases: the org-wide Classification policy above is what
|
||||
// activates the loop. Stubbed out so the hook's fetch never lands mid-assertion.
|
||||
vi.mock("@app/hooks/useProcessingFolders", () => ({
|
||||
useProcessingFolders: () => ({
|
||||
stateFor: () => undefined,
|
||||
enabledFolderIds: new Set<string>(),
|
||||
anyEnabled: false,
|
||||
enable: async () => {},
|
||||
disable: async () => {},
|
||||
sweep: async () => {},
|
||||
}),
|
||||
}));
|
||||
vi.mock("@app/contexts/FileContext", () => ({
|
||||
useAllFiles: () => ({ fileStubs: mocks.workspace }),
|
||||
useFileManagement: () => ({
|
||||
|
||||
@@ -10,6 +10,7 @@ import { useClassificationEnabled } from "@app/hooks/useClassificationEnabled";
|
||||
import { useAiEngineEnabled } from "@app/hooks/useAiEngineEnabled";
|
||||
import { scheduleIdle } from "@app/utils/scheduleIdle";
|
||||
import { usePolicies } from "@app/hooks/usePolicies";
|
||||
import { useProcessingFolders } from "@app/hooks/useProcessingFolders";
|
||||
import { classifyFileHeuristically } from "@app/services/heuristic/heuristicClassification";
|
||||
import { meterClassificationRun } from "@app/services/classificationMeter";
|
||||
import {
|
||||
@@ -67,9 +68,26 @@ export function useClientSideClassification(): void {
|
||||
policy.sources.length === 0 ||
|
||||
policy.sources.includes("editor")),
|
||||
);
|
||||
// Pausing Classification stops it everywhere: a processing folder may activate this loop where
|
||||
// no org-wide policy exists, but it must never resurrect a capability an admin has paused.
|
||||
const classificationPaused = policy?.status === "paused";
|
||||
// A processing folder classifies whatever lands in it, on exactly the same terms: the server
|
||||
// does it when AI is on, and this loop does it when AI is off. So a file sitting in an enabled
|
||||
// processing folder is in scope even with no org-wide Classification policy.
|
||||
const { enabledFolderIds, anyEnabled } = useProcessingFolders();
|
||||
const inEnabledProcessingFolder = (stub: StirlingFileStub) => {
|
||||
const folderId = stub.folderId as string | null | undefined;
|
||||
return Boolean(folderId && enabledFolderIds.has(folderId));
|
||||
};
|
||||
const anyProcessingFolder = !classificationPaused && anyEnabled;
|
||||
|
||||
useEffect(() => {
|
||||
if (configLoading || !classificationEnabled || aiEnabled || !active) {
|
||||
if (
|
||||
configLoading ||
|
||||
!classificationEnabled ||
|
||||
aiEnabled ||
|
||||
(!active && !anyProcessingFolder)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const claimKey = (s: StirlingFileStub) =>
|
||||
@@ -80,7 +98,8 @@ export function useClientSideClassification(): void {
|
||||
(s) =>
|
||||
!s.derivedFromTool &&
|
||||
s.classificationLabels == null &&
|
||||
!claimed.current.has(claimKey(s)),
|
||||
!claimed.current.has(claimKey(s)) &&
|
||||
(active || (!classificationPaused && inEnabledProcessingFolder(s))),
|
||||
)
|
||||
.slice(0, CLASSIFY_BATCH);
|
||||
if (pending.length === 0) return;
|
||||
@@ -121,6 +140,9 @@ export function useClientSideClassification(): void {
|
||||
}, [
|
||||
fileStubs,
|
||||
active,
|
||||
classificationPaused,
|
||||
anyProcessingFolder,
|
||||
enabledFolderIds,
|
||||
classificationEnabled,
|
||||
aiEnabled,
|
||||
configLoading,
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
/**
|
||||
* Client-side engine for virtual (browser-owned) processing folders.
|
||||
*
|
||||
* A virtual folder's files live only in this browser's IndexedDB, so the
|
||||
* server's folder watchers can never reach them. When such a folder has
|
||||
* processing enabled, this loop plays the watcher: it finds the folder's
|
||||
* unprocessed files, uploads each through an ad-hoc pipeline run
|
||||
* (`POST /api/v1/policies/run` — the same engine stored policies use), and
|
||||
* delivers the output back into IndexedDB as a new version of the input.
|
||||
* The versioned child inherits the input's folderId, so results stay in the
|
||||
* folder they came from.
|
||||
*
|
||||
* Runs only while the AI engine is on: the pipeline's steps execute
|
||||
* server-side (classification needs the engine), and with AI off the
|
||||
* browser-side classifier (useClientSideClassification) covers these folders
|
||||
* instead — the same split the org-wide Classification policy uses.
|
||||
*
|
||||
* Each (folder, file) pair is dispatched once, tracked in the shared
|
||||
* dispatched-markers store; outputs are stamped `derivedFromTool`, the durable
|
||||
* guard that stops the loop re-processing its own results.
|
||||
*/
|
||||
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { useFolders } from "@app/contexts/FolderContext";
|
||||
import { useAllFiles, useFileContext } from "@app/contexts/FileContext";
|
||||
import {
|
||||
useIndexedDB,
|
||||
useIndexedDBRevision,
|
||||
} from "@app/contexts/IndexedDBContext";
|
||||
import { useAiEngineEnabled } from "@app/hooks/useAiEngineEnabled";
|
||||
import { fileStorage } from "@app/services/fileStorage";
|
||||
import {
|
||||
downloadPolicyOutput,
|
||||
getPolicyRun,
|
||||
resolvePolicyRunTarget,
|
||||
runPolicyPipeline,
|
||||
} from "@app/services/policyApi";
|
||||
import type { PolicyRunView } from "@app/services/policyPipeline";
|
||||
import { readClassificationLabelsFromFile } from "@app/services/fileClassification";
|
||||
import { createStirlingFilesAndStubs } from "@app/services/fileStubHelpers";
|
||||
import {
|
||||
isDispatched,
|
||||
markDispatched,
|
||||
} from "@app/components/policies/policyRunStore";
|
||||
import { folderKind, type FolderRecord } from "@app/types/folder";
|
||||
import type { FileId } from "@app/types/file";
|
||||
import type { StirlingFile, StirlingFileStub } from "@app/types/fileContext";
|
||||
|
||||
const POLL_MS = 2000;
|
||||
/** Per-step budget mirroring the server's own step timeout, plus slack. */
|
||||
const STEP_TIMEOUT_MS = 300_000;
|
||||
const POLL_GRACE_MS = 30_000;
|
||||
|
||||
const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/** Dispatch-marker namespace: one category per processing folder. */
|
||||
function categoryFor(folderId: string): string {
|
||||
return `processing-folder:${folderId}`;
|
||||
}
|
||||
|
||||
/** Poll an ad-hoc run to a terminal state, or null if the budget runs out. */
|
||||
async function waitForRun(runId: string): Promise<PolicyRunView | null> {
|
||||
let budgetMs = STEP_TIMEOUT_MS + POLL_GRACE_MS;
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < budgetMs) {
|
||||
await delay(POLL_MS);
|
||||
let view: PolicyRunView;
|
||||
try {
|
||||
view = await getPolicyRun(runId);
|
||||
} catch {
|
||||
continue; // transient; the budget bounds it
|
||||
}
|
||||
if (view.stepCount > 0) {
|
||||
budgetMs = view.stepCount * STEP_TIMEOUT_MS + POLL_GRACE_MS;
|
||||
}
|
||||
if (
|
||||
view.status === "COMPLETED" ||
|
||||
view.status === "FAILED" ||
|
||||
view.status === "CANCELLED"
|
||||
) {
|
||||
return view;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
interface DeliveryContext {
|
||||
/** Live workspace stubs, read at delivery time (never a dependency). */
|
||||
workspaceStubs: () => ReadonlyArray<StirlingFileStub>;
|
||||
consumeFiles: (
|
||||
inputFileIds: FileId[],
|
||||
outputs: StirlingFile[],
|
||||
stubs: StirlingFileStub[],
|
||||
options?: { silent?: boolean },
|
||||
) => Promise<unknown>;
|
||||
bumpRevision: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Version the input with the run's outputs — in the workspace when the file is
|
||||
* open there (so the views update in place), else directly at the storage
|
||||
* layer. Labels are read off the output PDF and stamped on the child stub so
|
||||
* the sidebar groups it immediately.
|
||||
*/
|
||||
async function deliverOutputs(
|
||||
stub: StirlingFileStub,
|
||||
view: PolicyRunView,
|
||||
ctx: DeliveryContext,
|
||||
): Promise<void> {
|
||||
const target = resolvePolicyRunTarget();
|
||||
const files: File[] = [];
|
||||
for (const output of view.outputs) {
|
||||
const blob = await downloadPolicyOutput(output.fileId, target);
|
||||
files.push(
|
||||
new File([blob], stub.name, { type: blob.type || "application/pdf" }),
|
||||
);
|
||||
}
|
||||
if (files.length === 0) return;
|
||||
const parentStub = (await fileStorage.getStirlingFileStub(stub.id)) ?? stub;
|
||||
const { stirlingFiles, stubs } = await createStirlingFilesAndStubs(
|
||||
files,
|
||||
parentStub,
|
||||
"automate",
|
||||
);
|
||||
const finalStubs = await Promise.all(
|
||||
stubs.map(async (child, i) => {
|
||||
const labels =
|
||||
(await readClassificationLabelsFromFile(files[i]!)) ?? undefined;
|
||||
return {
|
||||
...child,
|
||||
derivedFromTool: true,
|
||||
...(labels ? { classificationLabels: labels } : {}),
|
||||
};
|
||||
}),
|
||||
);
|
||||
const inWorkspace = ctx
|
||||
.workspaceStubs()
|
||||
.some((w) => (w.id as string) === (stub.id as string));
|
||||
if (inWorkspace) {
|
||||
await ctx.consumeFiles([stub.id], stirlingFiles, finalStubs, {
|
||||
silent: true,
|
||||
});
|
||||
} else {
|
||||
await fileStorage.persistVersionedOutputs(
|
||||
[stub.id],
|
||||
stirlingFiles,
|
||||
finalStubs,
|
||||
);
|
||||
ctx.bumpRevision();
|
||||
}
|
||||
}
|
||||
|
||||
export function useVirtualFolderProcessing(): void {
|
||||
const { folders } = useFolders();
|
||||
const { fileStubs } = useAllFiles();
|
||||
const { consumeFiles } = useFileContext();
|
||||
const { bumpRevision } = useIndexedDB();
|
||||
const revision = useIndexedDBRevision();
|
||||
const aiEnabled = useAiEngineEnabled();
|
||||
|
||||
// Workspace stubs read via a ref: delivery mutates them, and depending on
|
||||
// them would make the scan re-trigger on its own deliveries.
|
||||
const fileStubsRef = useRef(fileStubs);
|
||||
fileStubsRef.current = fileStubs;
|
||||
// One scan at a time. A running scan's own deliveries bump the revision and
|
||||
// re-fire the effect, so the re-fire queues a follow-up scan instead of
|
||||
// cancelling the one in flight — cancelling there would strand every file
|
||||
// after the first delivery until some unrelated write happened along.
|
||||
const scanning = useRef(false);
|
||||
const rescanQueued = useRef(false);
|
||||
const unmounted = useRef(false);
|
||||
const [tick, setTick] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
unmounted.current = false;
|
||||
return () => {
|
||||
unmounted.current = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!aiEnabled) return;
|
||||
const enabled = folders.filter(
|
||||
(folder) =>
|
||||
folderKind(folder) === "virtual" &&
|
||||
folder.processing?.enabled &&
|
||||
folder.processing.steps.length > 0,
|
||||
);
|
||||
if (enabled.length === 0) return;
|
||||
if (scanning.current) {
|
||||
rescanQueued.current = true;
|
||||
return;
|
||||
}
|
||||
scanning.current = true;
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const all = await fileStorage.getAllStirlingFileStubs();
|
||||
for (const folder of enabled) {
|
||||
const category = categoryFor(folder.id as string);
|
||||
const pending = all.filter(
|
||||
(stub) =>
|
||||
(stub.folderId ?? null) === (folder.id as string) &&
|
||||
stub.isLeaf &&
|
||||
!stub.derivedFromTool &&
|
||||
!isDispatched(category, stub.id as string),
|
||||
);
|
||||
for (const stub of pending) {
|
||||
if (unmounted.current) return;
|
||||
await processOne(folder, stub, category, {
|
||||
workspaceStubs: () => fileStubsRef.current,
|
||||
consumeFiles,
|
||||
bumpRevision,
|
||||
});
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
scanning.current = false;
|
||||
// Anything that changed mid-scan (deliveries included) gets one full
|
||||
// follow-up pass; a clean follow-up finds nothing pending and stops.
|
||||
if (!unmounted.current && rescanQueued.current) {
|
||||
rescanQueued.current = false;
|
||||
setTick((n) => n + 1);
|
||||
}
|
||||
}
|
||||
})();
|
||||
// `revision` re-scans after any IndexedDB write — that is how a file
|
||||
// moved or uploaded into the folder gets picked up. No cleanup cancels
|
||||
// the loop: it must outlive re-renders its own deliveries cause, and
|
||||
// only unmount stops it.
|
||||
}, [folders, revision, aiEnabled, tick, consumeFiles, bumpRevision]);
|
||||
}
|
||||
|
||||
/** Run one file through its folder's pipeline and deliver the result. */
|
||||
async function processOne(
|
||||
folder: FolderRecord,
|
||||
stub: StirlingFileStub,
|
||||
category: string,
|
||||
ctx: DeliveryContext,
|
||||
): Promise<void> {
|
||||
const file = await fileStorage.getStirlingFile(stub.id).catch(() => null);
|
||||
if (!file) {
|
||||
// Removed since listing; never coming back under this id.
|
||||
markDispatched(category, stub.id as string);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const runId = await runPolicyPipeline(
|
||||
{
|
||||
name: `Processing folder: ${folder.name}`,
|
||||
steps: folder.processing!.steps.map((step) => ({
|
||||
operation: step.operation,
|
||||
parameters: step.parameters,
|
||||
})),
|
||||
outputs: [{ type: "inline", options: {} }],
|
||||
},
|
||||
[file],
|
||||
);
|
||||
// Marked at dispatch (not delivery): a delivery failure must not re-run
|
||||
// the pipeline — the run happened, and re-firing it would double-process.
|
||||
markDispatched(category, stub.id as string);
|
||||
const view = await waitForRun(runId);
|
||||
if (view?.status === "COMPLETED") {
|
||||
await deliverOutputs(stub, view, ctx);
|
||||
} else if (view) {
|
||||
console.warn(
|
||||
`[VirtualFolderProcessing] run for ${stub.name} ended ${view.status}`,
|
||||
view.error,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
// Dispatch or delivery failed. Marked either way so a broken file can't
|
||||
// wedge the folder in a re-dispatch loop; a new version retries naturally.
|
||||
markDispatched(category, stub.id as string);
|
||||
console.warn(
|
||||
`[VirtualFolderProcessing] could not process ${stub.name}`,
|
||||
err,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { type SidebarProcessingSlotProps } from "@core/components/shared/SidebarProcessingSlot";
|
||||
export { type SidebarProcessingSlotProps };
|
||||
|
||||
import { DownloadsProcessingWizard } from "@app/components/policies/DownloadsProcessingWizard";
|
||||
|
||||
/**
|
||||
* The offer to process the user's Downloads, alongside the sidebar's other
|
||||
* file-entry actions — it is one more way of getting files in, so it belongs
|
||||
* with "Open from computer" rather than in the tool panel.
|
||||
*
|
||||
* Deliberately not gated on whether policies are available. A processing
|
||||
* folder is its own surface: it happens to run on the policy engine, but a
|
||||
* user never meets the word, and the builds where the portal's Policies rail
|
||||
* makes sense are not the builds where a Downloads folder exists. The offer
|
||||
* gates itself instead — it asks the server whether there is a Downloads
|
||||
* directory it is allowed to read, and renders nothing when there is not.
|
||||
*
|
||||
* Hidden on the collapsed rail: the offer is a sentence, not an icon, and the
|
||||
* wizard makes no sense reduced to a glyph.
|
||||
*/
|
||||
export function SidebarProcessingSlot({ collapsed }: SidebarProcessingSlotProps) {
|
||||
if (collapsed) return null;
|
||||
return <DownloadsProcessingWizard />;
|
||||
}
|
||||
@@ -22,9 +22,20 @@ import { buildLabelGroups } from "@app/components/shared/fileSidebarGroupingLogi
|
||||
import { scheduleIdle } from "@app/utils/scheduleIdle";
|
||||
import type { FileId } from "@app/types/file";
|
||||
import type { StirlingFileStub } from "@app/types/fileContext";
|
||||
import type { FileSidebarGroup } from "@core/components/shared/fileSidebarGrouping";
|
||||
import type {
|
||||
CategoryFilterOption,
|
||||
FileSidebarGroup,
|
||||
LabelBadge,
|
||||
} from "@core/components/shared/fileSidebarGrouping";
|
||||
import { DEFAULT_CLASSIFICATION_LABELS } from "@app/data/classificationLabels";
|
||||
import { DEFAULT_LABEL_ICON } from "@app/data/labelIcons";
|
||||
import { accentColor, accentCycleColor } from "@app/utils/accentColors";
|
||||
|
||||
export type { FileSidebarGroup };
|
||||
export type {
|
||||
CategoryFilterOption,
|
||||
LabelBadge,
|
||||
} from "@core/components/shared/fileSidebarGrouping";
|
||||
// Pure grouping logic lives in a component-free module so tests don't drag in the picker's UI deps.
|
||||
export {
|
||||
buildLabelGroups,
|
||||
@@ -107,3 +118,149 @@ export function useFileSidebarGroups(
|
||||
[enabled, stubs, t, categories],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The visible categories as filter options, in the sidebar's own display
|
||||
* order — the files-page category filter and the sidebar groups must name
|
||||
* and order the world identically.
|
||||
*/
|
||||
export function useCategoryFilterOptions(): CategoryFilterOption[] {
|
||||
const categories = useSyncExternalStore(
|
||||
subscribeSidebarCategories,
|
||||
getSidebarCategories,
|
||||
);
|
||||
return useMemo(
|
||||
() =>
|
||||
categories
|
||||
.filter((category) => !category.hidden)
|
||||
.sort((a, b) =>
|
||||
a.name.localeCompare(b.name, undefined, { sensitivity: "base" }),
|
||||
)
|
||||
.map((category, index) => ({
|
||||
id: category.id,
|
||||
name: category.name,
|
||||
icon: category.icon,
|
||||
color: accentCycleColor(index),
|
||||
labelKeys: [...category.labelKeys],
|
||||
})),
|
||||
[categories],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Text matcher over classification: a file matches when any of its labels'
|
||||
* display names — or the names of the categories those labels roll up into —
|
||||
* contain the needle. The index is built once per vocabulary/category state,
|
||||
* so per-file checks during filtering are set lookups, not string assembly.
|
||||
*/
|
||||
export function useLabelSearchMatcher(): (
|
||||
labels: string[] | null | undefined,
|
||||
needle: string,
|
||||
) => boolean {
|
||||
const { t } = useTranslation();
|
||||
const categories = useSyncExternalStore(
|
||||
subscribeSidebarCategories,
|
||||
getSidebarCategories,
|
||||
);
|
||||
return useMemo(() => {
|
||||
const familyNameByLabel = new Map<string, string>();
|
||||
for (const category of categories) {
|
||||
for (const key of category.labelKeys) {
|
||||
if (!familyNameByLabel.has(key)) {
|
||||
familyNameByLabel.set(key, category.name.toLowerCase());
|
||||
}
|
||||
}
|
||||
}
|
||||
const searchableByLabel = new Map<string, string>();
|
||||
for (const label of DEFAULT_CLASSIFICATION_LABELS) {
|
||||
const name = t(
|
||||
`classification.labels.${label.id}`,
|
||||
label.name,
|
||||
).toLowerCase();
|
||||
const family = familyNameByLabel.get(label.id) ?? "";
|
||||
searchableByLabel.set(label.id, `${name} ${family}`);
|
||||
}
|
||||
return (labels: string[] | null | undefined, needle: string) => {
|
||||
if (!labels || labels.length === 0 || !needle) return false;
|
||||
return labels.some((id) =>
|
||||
(searchableByLabel.get(id) ?? id).includes(needle),
|
||||
);
|
||||
};
|
||||
}, [categories, t]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Badge descriptors for the categories a file's labels roll up into: each
|
||||
* visible family's own icon, wearing the same cycled accent its sidebar group
|
||||
* does. Deduped and in sidebar display order; labels only under hidden
|
||||
* categories contribute nothing (their files read as "Other").
|
||||
*/
|
||||
export function useFamilyBadges(labels?: string[] | null): LabelBadge[] {
|
||||
const { t } = useTranslation();
|
||||
const categories = useSyncExternalStore(
|
||||
subscribeSidebarCategories,
|
||||
getSidebarCategories,
|
||||
);
|
||||
return useMemo(() => {
|
||||
if (!labels || labels.length === 0) return [];
|
||||
const carried = new Set(labels);
|
||||
return categories
|
||||
.filter((category) => !category.hidden)
|
||||
.sort((a, b) =>
|
||||
a.name.localeCompare(b.name, undefined, { sensitivity: "base" }),
|
||||
)
|
||||
.map((category, index) => ({ category, index }))
|
||||
.filter(({ category }) =>
|
||||
category.labelKeys.some((key) => carried.has(key)),
|
||||
)
|
||||
.map(({ category, index }) => ({
|
||||
id: category.id,
|
||||
name: category.name,
|
||||
icon: category.icon,
|
||||
color: accentCycleColor(index),
|
||||
}));
|
||||
}, [labels, categories, t]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Badge descriptors for a file's labels: each label's own icon from the
|
||||
* classification vocabulary, coloured with the accent its category cycles to
|
||||
* in the sidebar (visible categories in display order — the same order the
|
||||
* groups render in, so a badge and its group read as one colour). Labels
|
||||
* under a hidden category wear the same neutral grey as the "Other" group.
|
||||
*/
|
||||
export function useLabelBadges(labels?: string[] | null): LabelBadge[] {
|
||||
const { t } = useTranslation();
|
||||
const categories = useSyncExternalStore(
|
||||
subscribeSidebarCategories,
|
||||
getSidebarCategories,
|
||||
);
|
||||
return useMemo(() => {
|
||||
if (!labels || labels.length === 0) return [];
|
||||
const visible = categories
|
||||
.filter((category) => !category.hidden)
|
||||
.sort((a, b) =>
|
||||
a.name.localeCompare(b.name, undefined, { sensitivity: "base" }),
|
||||
);
|
||||
const accentByLabel = new Map<string, string>();
|
||||
visible.forEach((category, index) => {
|
||||
for (const labelKey of category.labelKeys) {
|
||||
if (!accentByLabel.has(labelKey)) {
|
||||
accentByLabel.set(labelKey, accentCycleColor(index));
|
||||
}
|
||||
}
|
||||
});
|
||||
const byId = new Map(
|
||||
DEFAULT_CLASSIFICATION_LABELS.map((label) => [label.id, label]),
|
||||
);
|
||||
return labels.map((id) => {
|
||||
const label = byId.get(id);
|
||||
return {
|
||||
id,
|
||||
name: t(`classification.labels.${id}`, label?.name ?? id),
|
||||
icon: label?.icon ?? DEFAULT_LABEL_ICON,
|
||||
color: accentByLabel.get(id) ?? accentColor("gray"),
|
||||
};
|
||||
});
|
||||
}, [labels, categories, t]);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
import { useCallback, useEffect, useMemo, useSyncExternalStore } from "react";
|
||||
import {
|
||||
CLASSIFY_OPERATION,
|
||||
classificationDefaults,
|
||||
deleteProcessingFolder,
|
||||
fetchProcessingFolderRuns,
|
||||
fetchProcessingFolders,
|
||||
saveProcessingFolder,
|
||||
sweepProcessingFolder,
|
||||
type ProcessingFolder,
|
||||
} from "@app/services/processingFolderApi";
|
||||
import { useFolders } from "@app/contexts/FolderContext";
|
||||
import { useFileHandler } from "@app/hooks/useFileHandler";
|
||||
import { deliverSweepResults } from "@app/services/processingRunDelivery";
|
||||
import { virtualFolderStorage } from "@app/services/virtualFolderStorage";
|
||||
import { folderKind, type FolderRecord } from "@app/types/folder";
|
||||
// The core stub declares the contract this shadows; import it from @core
|
||||
// explicitly, since @app/hooks/useProcessingFolders resolves back to this file.
|
||||
import type {
|
||||
ProcessingFolderState,
|
||||
ProcessingFoldersApi,
|
||||
ProcessingRunInfo,
|
||||
} from "@core/hooks/useProcessingFolders";
|
||||
|
||||
// Consumers import the contract's types from @app, which resolves here in
|
||||
// builds that carry this shadow — so it must re-export what the stub declares.
|
||||
export type {
|
||||
ProcessingFolderState,
|
||||
ProcessingFoldersApi,
|
||||
ProcessingRunInfo,
|
||||
} from "@core/hooks/useProcessingFolders";
|
||||
|
||||
/**
|
||||
* One shared list for every consumer. The files page calls this hook once per folder row, on top of
|
||||
* the wizard and the classification loop, so per-instance state would mean one request per row and
|
||||
* a mutation in one row leaving the others stale until they remounted.
|
||||
*/
|
||||
let folders: ProcessingFolder[] = [];
|
||||
let inFlight: Promise<void> | null = null;
|
||||
const listeners = new Set<() => void>();
|
||||
|
||||
function subscribe(listener: () => void): () => void {
|
||||
listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Snapshot identity only changes when the list is replaced, so consumers re-render on real news. */
|
||||
function getSnapshot(): ProcessingFolder[] {
|
||||
return folders;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the list, sharing one request across concurrent callers. `force` bypasses an existing
|
||||
* in-flight read so a mutation always observes its own effect.
|
||||
*/
|
||||
function load(force = false): Promise<void> {
|
||||
if (inFlight && !force) return inFlight;
|
||||
const request = fetchProcessingFolders()
|
||||
.then((next) => {
|
||||
folders = next;
|
||||
})
|
||||
.catch(() => {
|
||||
// Storage or login disabled, or not authenticated: nothing to show, and the files page
|
||||
// still works without processing folders.
|
||||
folders = [];
|
||||
})
|
||||
.finally(() => {
|
||||
if (inFlight === request) inFlight = null;
|
||||
listeners.forEach((listener) => listener());
|
||||
});
|
||||
inFlight = request;
|
||||
return request;
|
||||
}
|
||||
|
||||
/**
|
||||
* A directory as a comparison key. A mount and its processing record are
|
||||
* created from the same picker string, but one side may carry a trailing
|
||||
* separator the other lost to trimming.
|
||||
*/
|
||||
function directoryKey(directory: string): string {
|
||||
return directory.trim().replace(/[/\\]+$/, "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Processing folders for the files page: which folders run a pipeline, and the actions to attach,
|
||||
* detach, or re-run one. The record's identity is kind-shaped — a server folder is matched by its
|
||||
* storage folderId, a mounted folder by the directory it mirrors — so the same folder row finds its
|
||||
* processing state whichever side of that split it lives on. Backed by
|
||||
* `/api/v1/processing-folders`, which composes the source + policy pair.
|
||||
*
|
||||
* Every mutation reloads rather than patching locally, so the list always reflects what the server
|
||||
* actually composed — and because the list is shared, every consumer sees it at once.
|
||||
*/
|
||||
export function useProcessingFolders(): ProcessingFoldersApi {
|
||||
const current = useSyncExternalStore(subscribe, getSnapshot, getSnapshot);
|
||||
// Virtual folders keep their processing config on their own record, so the
|
||||
// folder list is this hook's second system of record (and its refresh is how
|
||||
// a virtual mutation becomes visible).
|
||||
const { folders: allFolders, refresh: refreshFolders } = useFolders();
|
||||
const { addFiles } = useFileHandler();
|
||||
|
||||
useEffect(() => {
|
||||
void load();
|
||||
}, []);
|
||||
|
||||
const recordFor = useCallback(
|
||||
(folder: FolderRecord): ProcessingFolder | undefined => {
|
||||
switch (folderKind(folder)) {
|
||||
case "local": {
|
||||
if (!folder.directory) return undefined;
|
||||
const key = directoryKey(folder.directory);
|
||||
return current.find(
|
||||
(record) =>
|
||||
record.directory && directoryKey(record.directory) === key,
|
||||
);
|
||||
}
|
||||
case "virtual":
|
||||
// Browser-owned folders process client-side; the server has no record of them.
|
||||
return undefined;
|
||||
default:
|
||||
return current.find((record) => record.folderId === folder.id);
|
||||
}
|
||||
},
|
||||
[current],
|
||||
);
|
||||
|
||||
const stateFor = useCallback(
|
||||
(folder: FolderRecord): ProcessingFolderState | undefined => {
|
||||
if (folderKind(folder) === "virtual") {
|
||||
return folder.processing
|
||||
? { id: folder.id as string, enabled: folder.processing.enabled }
|
||||
: undefined;
|
||||
}
|
||||
const record = recordFor(folder);
|
||||
if (!record) return undefined;
|
||||
const outputDirectory = record.output?.["directory"];
|
||||
return {
|
||||
id: record.id,
|
||||
enabled: record.enabled,
|
||||
outputDirectory:
|
||||
typeof outputDirectory === "string" && outputDirectory
|
||||
? outputDirectory
|
||||
: undefined,
|
||||
};
|
||||
},
|
||||
[recordFor],
|
||||
);
|
||||
|
||||
const enabledFolderIds = useMemo(() => {
|
||||
const ids = new Set<string>();
|
||||
for (const record of current) {
|
||||
if (record.enabled && record.folderId) ids.add(record.folderId);
|
||||
}
|
||||
// Virtual processing folders belong here too: with the AI engine off, the
|
||||
// browser-side classifier is their engine, and this set is what it scopes by.
|
||||
for (const folder of allFolders) {
|
||||
if (folderKind(folder) === "virtual" && folder.processing?.enabled) {
|
||||
ids.add(folder.id as string);
|
||||
}
|
||||
}
|
||||
return ids as ReadonlySet<string>;
|
||||
}, [current, allFolders]);
|
||||
|
||||
const anyEnabled = useMemo(
|
||||
() =>
|
||||
current.some((record) => record.enabled) ||
|
||||
allFolders.some(
|
||||
(folder) =>
|
||||
folderKind(folder) === "virtual" && folder.processing?.enabled,
|
||||
),
|
||||
[current, allFolders],
|
||||
);
|
||||
|
||||
const enable = useCallback(
|
||||
async (folder: FolderRecord) => {
|
||||
switch (folderKind(folder)) {
|
||||
case "local": {
|
||||
const saved = await saveProcessingFolder({
|
||||
directory: folder.directory ?? "",
|
||||
enabled: true,
|
||||
steps: [
|
||||
{ operation: CLASSIFY_OPERATION, parameters: {}, assets: {} },
|
||||
],
|
||||
});
|
||||
// The create-time backlog sweep runs server-side; its results land
|
||||
// on disk, so pull them into the workbench as they settle — a
|
||||
// sweep whose results appear nowhere reads as nothing happening.
|
||||
if (saved.startedRuns > 0) {
|
||||
void deliverSweepResults(saved.id, saved.startedRuns, addFiles);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "virtual":
|
||||
// Browser-owned: the config lives on the folder record and the
|
||||
// client-side engine picks it up from there. No server record.
|
||||
await virtualFolderStorage.setProcessing(folder.id, {
|
||||
enabled: true,
|
||||
steps: [{ operation: CLASSIFY_OPERATION, parameters: {} }],
|
||||
});
|
||||
await refreshFolders();
|
||||
return;
|
||||
default:
|
||||
await saveProcessingFolder(classificationDefaults(folder.id));
|
||||
}
|
||||
await load(true);
|
||||
},
|
||||
[refreshFolders, addFiles],
|
||||
);
|
||||
|
||||
const disable = useCallback(
|
||||
async (folder: FolderRecord) => {
|
||||
if (folderKind(folder) === "virtual") {
|
||||
await virtualFolderStorage.setProcessing(folder.id, null);
|
||||
await refreshFolders();
|
||||
return;
|
||||
}
|
||||
const existing = recordFor(folder);
|
||||
if (!existing) return;
|
||||
await deleteProcessingFolder(existing.id);
|
||||
await load(true);
|
||||
},
|
||||
[recordFor, refreshFolders],
|
||||
);
|
||||
|
||||
const listActiveRuns = useCallback(
|
||||
async (recordId: string): Promise<ProcessingRunInfo[]> => {
|
||||
const TERMINAL = ["COMPLETED", "FAILED", "CANCELLED"];
|
||||
const runs = await fetchProcessingFolderRuns(recordId).catch(() => []);
|
||||
return runs
|
||||
.filter((run) => run.runId && !TERMINAL.includes(run.status))
|
||||
.map((run) => ({
|
||||
runId: run.runId!,
|
||||
fileName: run.fileName ?? null,
|
||||
currentStep: run.currentStep ?? 0,
|
||||
stepCount: run.stepCount ?? 0,
|
||||
}));
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const sweep = useCallback(
|
||||
async (folder: FolderRecord) => {
|
||||
if (folderKind(folder) === "virtual") {
|
||||
// The client-side engine is continuous: it processes the folder's
|
||||
// files as they appear, so there is no backlog for a sweep to start.
|
||||
return;
|
||||
}
|
||||
const existing = recordFor(folder);
|
||||
if (!existing) return;
|
||||
const outcome = await sweepProcessingFolder(existing.id);
|
||||
// A mount's results land on disk where nothing shows them; open them
|
||||
// into the workbench as they settle. A storage folder's results replace
|
||||
// its files in place, already visible where the user is looking.
|
||||
if (folderKind(folder) === "local" && outcome.runIds.length > 0) {
|
||||
void deliverSweepResults(existing.id, outcome.runIds.length, addFiles);
|
||||
}
|
||||
},
|
||||
[recordFor, addFiles],
|
||||
);
|
||||
|
||||
return useMemo(
|
||||
() => ({
|
||||
stateFor,
|
||||
enabledFolderIds,
|
||||
anyEnabled,
|
||||
listActiveRuns,
|
||||
enable,
|
||||
disable,
|
||||
sweep,
|
||||
}),
|
||||
[
|
||||
stateFor,
|
||||
enabledFolderIds,
|
||||
anyEnabled,
|
||||
listActiveRuns,
|
||||
enable,
|
||||
disable,
|
||||
sweep,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
/** Reload the shared list — for a caller that created a folder outside these actions. */
|
||||
export function refreshProcessingFolders(): Promise<void> {
|
||||
return load(true);
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
/**
|
||||
* Client for processing folders (`/api/v1/processing-folders`) — a storage
|
||||
* folder with a pipeline attached, so any file added to it is processed. The
|
||||
* backend composes the source + policy pair behind this route; nothing here
|
||||
* deals in policies or sources directly.
|
||||
*/
|
||||
|
||||
import apiClient from "@app/services/apiClient";
|
||||
import { readDiskFile } from "@app/services/localFolderContents";
|
||||
|
||||
/** The classify step: identifies the document's type and tags it. No parameters. */
|
||||
export const CLASSIFY_OPERATION = "/api/v1/ai/tools/classify-and-label";
|
||||
|
||||
export interface ProcessingFolderStep {
|
||||
operation: string;
|
||||
parameters: Record<string, unknown>;
|
||||
assets?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface ProcessingFolder {
|
||||
id: string;
|
||||
/** Set for a storage-backed folder; null when the folder is mounted from disk. */
|
||||
folderId: string | null;
|
||||
/** Set for a disk-backed (mounted) folder; null when it is storage-backed. */
|
||||
directory: string | null;
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
steps: ProcessingFolderStep[];
|
||||
output: Record<string, unknown>;
|
||||
/** Runs the creating sweep started; 0 means there was nothing new to process. */
|
||||
startedRuns: number;
|
||||
/** Files the creating sweep skipped because this folder had already processed them. */
|
||||
alreadyProcessed: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Exactly one of `folderId` (a folder in app storage) or `directory` (a directory on the server's
|
||||
* disk — on a desktop or self-hosted install, the user's own machine) says where a folder watches.
|
||||
*/
|
||||
export interface SaveProcessingFolderRequest {
|
||||
id?: string | null;
|
||||
folderId?: string;
|
||||
directory?: string;
|
||||
enabled?: boolean;
|
||||
steps: ProcessingFolderStep[];
|
||||
output?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Every processing folder the current user owns. */
|
||||
export async function fetchProcessingFolders(): Promise<ProcessingFolder[]> {
|
||||
const res = await apiClient.get<ProcessingFolder[]>(
|
||||
"/api/v1/processing-folders",
|
||||
);
|
||||
return res.data ?? [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Create or update one. Creating immediately processes what is already in the
|
||||
* folder; the backend's ledger keeps already-processed files from re-running.
|
||||
*/
|
||||
export async function saveProcessingFolder(
|
||||
request: SaveProcessingFolderRequest,
|
||||
): Promise<ProcessingFolder> {
|
||||
const res = await apiClient.post<ProcessingFolder>(
|
||||
"/api/v1/processing-folders",
|
||||
request,
|
||||
);
|
||||
return res.data;
|
||||
}
|
||||
|
||||
/** What one sweep took on, as the backend reports it. */
|
||||
export interface SweepOutcome {
|
||||
runIds: string[];
|
||||
filesListed: number;
|
||||
alreadyProcessed: number;
|
||||
}
|
||||
|
||||
/** Run the pipeline over the folder's current contents now. */
|
||||
export async function sweepProcessingFolder(id: string): Promise<SweepOutcome> {
|
||||
const res = await apiClient.post<SweepOutcome>(
|
||||
`/api/v1/processing-folders/${id}/sweep`,
|
||||
);
|
||||
return res.data;
|
||||
}
|
||||
|
||||
/** Remove the processing behaviour. The folder and its files are untouched. */
|
||||
export async function deleteProcessingFolder(id: string): Promise<void> {
|
||||
await apiClient.delete(`/api/v1/processing-folders/${id}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* The default pipeline a folder gets when it is turned into a processing
|
||||
* folder: classification, matching the Classification policy. Outputs replace
|
||||
* the file in place as a new version so the folder does not fill with copies.
|
||||
*/
|
||||
export function classificationDefaults(
|
||||
folderId: string,
|
||||
): SaveProcessingFolderRequest {
|
||||
return {
|
||||
folderId,
|
||||
enabled: true,
|
||||
steps: [{ operation: CLASSIFY_OPERATION, parameters: {}, assets: {} }],
|
||||
output: { mode: "new_version" },
|
||||
};
|
||||
}
|
||||
|
||||
/** The server's Downloads directory and what is waiting in it. */
|
||||
export interface DownloadsSuggestion {
|
||||
directory: string;
|
||||
available: boolean;
|
||||
pdfCount: number;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Where the server's own Downloads directory is and how many PDFs sit in it.
|
||||
* The browser cannot see the machine's paths, so the offer is built from this.
|
||||
*/
|
||||
export async function fetchDownloadsSuggestion(): Promise<DownloadsSuggestion> {
|
||||
const res = await apiClient.get<DownloadsSuggestion>(
|
||||
"/api/v1/processing-folders/downloads-suggestion",
|
||||
);
|
||||
return res.data;
|
||||
}
|
||||
|
||||
/** One file a run produced. Downloadable by id from the general files endpoint. */
|
||||
export interface ProcessingRunOutput {
|
||||
fileId: string;
|
||||
fileName?: string | null;
|
||||
}
|
||||
|
||||
export interface ProcessingFolderRun {
|
||||
runId?: string;
|
||||
status: string;
|
||||
error?: string | null;
|
||||
outputs?: ProcessingRunOutput[] | null;
|
||||
/** The input document's display name, for runs whose source recorded one. */
|
||||
fileName?: string | null;
|
||||
currentStep?: number;
|
||||
stepCount?: number;
|
||||
}
|
||||
|
||||
/** Runs belonging to a processing folder, newest first — drives the progress display. */
|
||||
export async function fetchProcessingFolderRuns(
|
||||
policyId: string,
|
||||
): Promise<ProcessingFolderRun[]> {
|
||||
// Filtered server-side: delivery polls this every second, and the
|
||||
// unfiltered list carries every policy's runs. The client-side filter stays
|
||||
// as a guard against a backend that ignores the parameter.
|
||||
const res = await apiClient.get<
|
||||
(ProcessingFolderRun & { policyId?: string })[]
|
||||
>("/api/v1/policies/runs", { params: { policyId } });
|
||||
return (res.data ?? []).filter((run) => run.policyId === policyId);
|
||||
}
|
||||
|
||||
/** An absolute filesystem path (Windows drive-letter or POSIX rooted). */
|
||||
function isAbsolutePath(value: string): boolean {
|
||||
return /^([A-Za-z]:[\\/]|\/)/.test(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a run output's bytes as a File, ready to hand to the workbench.
|
||||
*
|
||||
* A storage-backed run puts the stored file's own id in `fileId`, so it downloads from the storage
|
||||
* endpoint. The job endpoint (`/api/v1/general/files/{id}`) keys off job-file UUIDs and rejects a
|
||||
* stored-file id outright — the two share a field name but not an id space.
|
||||
*
|
||||
* A disk-backed run delivers to the filesystem instead: its `fileId` is synthetic (nothing serves
|
||||
* it) and `fileName` is the output's absolute path. Only a build that can see the filesystem — the
|
||||
* desktop app, where the server is this machine — can pick those up, by reading the path directly.
|
||||
*/
|
||||
export async function fetchRunOutputFile(
|
||||
output: ProcessingRunOutput,
|
||||
): Promise<File> {
|
||||
const name = output.fileName?.trim() || `${output.fileId}.pdf`;
|
||||
if (isAbsolutePath(name)) {
|
||||
const baseName = name.split(/[\\/]/).pop() || name;
|
||||
const file = await readDiskFile({
|
||||
path: name,
|
||||
name: baseName,
|
||||
sizeBytes: 0,
|
||||
lastModified: 0,
|
||||
});
|
||||
if (!file) {
|
||||
throw new Error(`This build cannot read the run output at ${name}`);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
const res = await apiClient.get(
|
||||
`/api/v1/storage/files/${output.fileId}/download`,
|
||||
{ responseType: "blob" },
|
||||
);
|
||||
return new File([res.data as Blob], name, {
|
||||
type: (res.data as Blob).type || "application/pdf",
|
||||
});
|
||||
}
|
||||
|
||||
/** One file in a mounted (disk-backed) processing folder. */
|
||||
export interface MountedFile {
|
||||
name: string;
|
||||
sizeBytes: number;
|
||||
lastModified: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The contents of a disk-backed processing folder, read from the directory itself — the folder is
|
||||
* mounted, not mirrored, so the filesystem stays the single source of truth.
|
||||
*/
|
||||
export async function fetchMountedFiles(id: string): Promise<MountedFile[]> {
|
||||
const res = await apiClient.get<MountedFile[]>(
|
||||
`/api/v1/processing-folders/${id}/files`,
|
||||
);
|
||||
return res.data ?? [];
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* Delivery of a processing-folder sweep's results into the workbench.
|
||||
*
|
||||
* A sweep runs server-side, so without this the user is left with a finished
|
||||
* job and an unchanged screen — the results exist (on disk or in storage) but
|
||||
* nothing shows them. This polls the folder's runs until the sweep's own runs
|
||||
* have settled, opening each run's results as soon as that run finishes
|
||||
* rather than at the end: a single slow or stuck file would otherwise hold
|
||||
* back everything that already succeeded, and a timeout would throw all of it
|
||||
* away.
|
||||
*/
|
||||
|
||||
import {
|
||||
fetchProcessingFolderRuns,
|
||||
fetchRunOutputFile,
|
||||
type ProcessingRunOutput,
|
||||
} from "@app/services/processingFolderApi";
|
||||
|
||||
const TERMINAL = ["COMPLETED", "FAILED", "CANCELLED"];
|
||||
/** Poll cadence and budget: up to ~15 minutes of 1s polls, as sweeps are per-file jobs. */
|
||||
const POLL_MS = 1000;
|
||||
const MAX_POLLS = 900;
|
||||
|
||||
export interface SweepDeliveryProgress {
|
||||
/** Runs that completed successfully so far. */
|
||||
processed: number;
|
||||
/** Runs that failed or were cancelled so far. */
|
||||
failed: number;
|
||||
/** Result files opened into the workbench so far. */
|
||||
opened: number;
|
||||
/** True when the budget ran out with runs still unsettled. */
|
||||
stalled: boolean;
|
||||
}
|
||||
|
||||
const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/**
|
||||
* Poll `policyId`'s runs until `expected` of them have settled, opening each
|
||||
* completed run's outputs into the workbench via `addFiles`. `expected` is
|
||||
* what the server reported starting, so this never waits on runs that were
|
||||
* never going to appear. Progress is reported after every poll; the final
|
||||
* state is also returned.
|
||||
*/
|
||||
export async function deliverSweepResults(
|
||||
policyId: string,
|
||||
expected: number,
|
||||
addFiles: (
|
||||
files: File[],
|
||||
options?: { selectFiles?: boolean },
|
||||
) => Promise<unknown>,
|
||||
onProgress?: (progress: SweepDeliveryProgress) => void,
|
||||
): Promise<SweepDeliveryProgress> {
|
||||
const alreadyOpened = new Set<string>();
|
||||
const progress: SweepDeliveryProgress = {
|
||||
processed: 0,
|
||||
failed: 0,
|
||||
opened: 0,
|
||||
stalled: false,
|
||||
};
|
||||
|
||||
const openInWorkbench = async (outputs: ProcessingRunOutput[]) => {
|
||||
if (outputs.length === 0) return;
|
||||
const files: File[] = [];
|
||||
for (const output of outputs) {
|
||||
// Downloads are sequential so a hundred results don't open a hundred
|
||||
// parallel requests, and one failure costs one file rather than the
|
||||
// batch — it still exists where the run put it either way.
|
||||
try {
|
||||
files.push(await fetchRunOutputFile(output));
|
||||
} catch (e) {
|
||||
// Logged rather than swallowed: a fetch that fails for every file is
|
||||
// indistinguishable from the pipeline producing nothing, and looks
|
||||
// like the feature simply not working.
|
||||
console.warn(
|
||||
`[processing folders] could not open result ${output.fileId}`,
|
||||
e,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (files.length === 0) return;
|
||||
// Never select what is delivered: a selection isn't meaningful across a
|
||||
// folderful of results, and re-selecting on every batch re-renders the
|
||||
// whole growing file list once a second for the length of the sweep.
|
||||
await addFiles(files);
|
||||
progress.opened += files.length;
|
||||
};
|
||||
|
||||
for (let attempt = 0; attempt < MAX_POLLS; attempt++) {
|
||||
const runs = await fetchProcessingFolderRuns(policyId).catch(() => []);
|
||||
const settled = runs.filter((run) => TERMINAL.includes(run.status));
|
||||
const done = settled.filter((run) => run.status === "COMPLETED");
|
||||
progress.processed = done.length;
|
||||
progress.failed = settled.length - done.length;
|
||||
|
||||
const fresh = done.filter(
|
||||
(run) => run.runId && !alreadyOpened.has(run.runId),
|
||||
);
|
||||
fresh.forEach((run) => alreadyOpened.add(run.runId!));
|
||||
await openInWorkbench(fresh.flatMap((run) => run.outputs ?? []));
|
||||
onProgress?.({ ...progress });
|
||||
|
||||
if (settled.length >= expected) return progress;
|
||||
await delay(POLL_MS);
|
||||
}
|
||||
progress.stalled = true;
|
||||
onProgress?.({ ...progress });
|
||||
return progress;
|
||||
}
|
||||
Reference in New Issue
Block a user