Further improvements to policies file tracking (#6941)

# Description of Changes
Fixes requested in review of #6903

app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileEntity.java

@@ -10,6 +10,7 @@ import jakarta.persistence.EnumType;
 import jakarta.persistence.Enumerated;
 import jakarta.persistence.Id;
 import jakarta.persistence.IdClass;
+import jakarta.persistence.Index;
 import jakarta.persistence.Table;
 import jakarta.persistence.Transient;
 
@@ -25,7 +26,15 @@ import lombok.Setter;
  * violation rather than a silent merge.
  */
 @Entity
-@Table(name = "policy_processed_files")
+@Table(
+        name = "policy_processed_files",
+        indexes = {
+            // presence cleanup: delete this policy's rows unseen since the sweep began
+            @Index(name = "idx_processed_files_policy_seen", columnList = "policy_id, last_seen"),
+            // cross-policy deletion consensus: existsByIdentityHashAndStatusNot filters
+            // identity_hash on its own, so it cannot ride the (policy_id, identity_hash) PK
+            @Index(name = "idx_processed_files_identity", columnList = "identity_hash")
+        })
 @IdClass(ProcessedFileId.class)
 @NoArgsConstructor
 @Getter

app/saas/src/main/resources/db/migration/saas/V32__policy_processed_files.sql

@@ -28,3 +28,9 @@ CREATE TABLE IF NOT EXISTS policy_processed_files (
 
 CREATE INDEX IF NOT EXISTS idx_processed_files_policy_seen
     ON policy_processed_files (policy_id, last_seen);
+
+-- The cross-policy deletion-consensus check (existsByIdentityHashAndStatusNot) filters identity_hash
+-- alone, so it cannot use the (policy_id, identity_hash) primary key; it runs once per successfully
+-- consumed file, so index it to avoid a full scan on the hot path.
+CREATE INDEX IF NOT EXISTS idx_processed_files_identity
+    ON policy_processed_files (identity_hash);

frontend/editor/public/locales/en-US/translation.toml

@@ -7511,6 +7511,12 @@ pause = "Pause"
 resume = "Resume"
 runNow = "Run now"
 
+[portal.policies.detail.clearHistory]
+body = "This policy will forget every file it has already processed and reprocess everything currently in its sources on the next run. The files themselves are not changed. This cannot be undone."
+cancel = "Cancel"
+confirm = "Clear history"
+title = "Clear processed history?"
+
 [portal.policies.detail.emptyActivity]
 description = "Documents will appear here once this policy runs."
 title = "No activity yet"

frontend/editor/src/portal/components/policies/PolicyDetailPanel.tsx

@@ -111,6 +111,7 @@ export function PolicyDetailPanel({
   onRetry,
 }: PolicyDetailPanelProps) {
   const { t } = useTranslation();
+  const [confirmingClear, setConfirmingClear] = useState(false);
   if (!policy) return null;
   const { category, config, state, steps, stats, activity } = policy;
   const isPaused = state.status === "paused";
@@ -136,200 +137,241 @@ export function PolicyDetailPanel({
   }
 
   return (
-    <Modal
-      open
-      onClose={onClose}
-      width="lg"
-      title={t(category.label)}
-      footer={
-        <div className="portal-policies__detail-foot">
-          {canDelete && (
-            <Button
-              variant="tertiary"
-              accent="danger"
-              size="sm"
-              onClick={onDelete}
-              disabled={busy}
-              style={{ marginRight: "auto" }}
-            >
-              {t("portal.policies.detail.actions.delete")}
-            </Button>
-          )}
-          {onRun && (
+    <>
+      <Modal
+        open
+        onClose={onClose}
+        width="lg"
+        title={t(category.label)}
+        footer={
+          <div className="portal-policies__detail-foot">
+            {canDelete && (
+              <Button
+                variant="tertiary"
+                accent="danger"
+                size="sm"
+                onClick={onDelete}
+                disabled={busy}
+                style={{ marginRight: "auto" }}
+              >
+                {t("portal.policies.detail.actions.delete")}
+              </Button>
+            )}
+            {onRun && (
+              <Button
+                variant="secondary"
+                size="sm"
+                onClick={onRun}
+                disabled={busy}
+                style={canDelete ? undefined : { marginRight: "auto" }}
+              >
+                {t("portal.policies.detail.actions.runNow")}
+              </Button>
+            )}
+            {canClearHistory && (
+              <Button
+                variant="secondary"
+                size="sm"
+                onClick={() => setConfirmingClear(true)}
+                disabled={busy}
+              >
+                {t("portal.policies.detail.actions.clearHistory")}
+              </Button>
+            )}
             <Button
               variant="secondary"
               size="sm"
-              onClick={onRun}
-              disabled={busy}
-              style={canDelete ? undefined : { marginRight: "auto" }}
-            >
-              {t("portal.policies.detail.actions.runNow")}
-            </Button>
-          )}
-          {canClearHistory && (
-            <Button
-              variant="secondary"
-              size="sm"
-              onClick={onClearHistory}
+              onClick={onTogglePause}
               disabled={busy}
             >
-              {t("portal.policies.detail.actions.clearHistory")}
+              {isPaused
+                ? t("portal.policies.detail.actions.resume")
+                : t("portal.policies.detail.actions.pause")}
             </Button>
-          )}
-          <Button
-            variant="secondary"
-            size="sm"
-            onClick={onTogglePause}
-            disabled={busy}
+            <Button size="sm" onClick={onEdit} disabled={busy}>
+              {t("portal.policies.detail.actions.editSettings")}
+            </Button>
+          </div>
+        }
+      >
+        {/* Status + trigger strip */}
+        <div className="portal-policies__detail-status">
+          <StatusBadge
+            tone={isPaused ? "warning" : "success"}
+            pulse={!isPaused}
           >
             {isPaused
-              ? t("portal.policies.detail.actions.resume")
-              : t("portal.policies.detail.actions.pause")}
-          </Button>
-          <Button size="sm" onClick={onEdit} disabled={busy}>
-            {t("portal.policies.detail.actions.editSettings")}
-          </Button>
+              ? t("portal.policies.status.paused")
+              : t("portal.policies.status.active")}
+          </StatusBadge>
+          {hasEditorSource && (
+            <>
+              <span className="portal-policies__detail-sep" aria-hidden>
+                ·
+              </span>
+              <span className="portal-policies__detail-meta">{trigger}</span>
+              <span className="portal-policies__detail-sep" aria-hidden>
+                ·
+              </span>
+              <span className="portal-policies__detail-meta">
+                {outputLabel}
+              </span>
+            </>
+          )}
         </div>
-      }
-    >
-      {/* Status + trigger strip */}
-      <div className="portal-policies__detail-status">
-        <StatusBadge tone={isPaused ? "warning" : "success"} pulse={!isPaused}>
-          {isPaused
-            ? t("portal.policies.status.paused")
-            : t("portal.policies.status.active")}
-        </StatusBadge>
-        {hasEditorSource && (
-          <>
-            <span className="portal-policies__detail-sep" aria-hidden>
-              ·
-            </span>
-            <span className="portal-policies__detail-meta">{trigger}</span>
-            <span className="portal-policies__detail-sep" aria-hidden>
-              ·
-            </span>
-            <span className="portal-policies__detail-meta">{outputLabel}</span>
-          </>
-        )}
-      </div>
 
-      {/* Enforces — plain text, no pills */}
-      <div className="portal-policies__detail-inline">
-        <span className="portal-policies__detail-inline-label">
-          {t("portal.policies.detail.enforces")}
-        </span>
-        <span className="portal-policies__detail-inline-value">
-          {enforceItems
-            ? enforceItems.map((op, i) => (
-                <span key={op}>
-                  {i > 0 && (
-                    <span
-                      className="portal-policies__enforce-arrow"
-                      aria-hidden
-                    >
-                      {" "}
-                      →{" "}
-                    </span>
-                  )}
-                  {humanizeEndpoint(op, t)}
-                </span>
-              ))
-            : config.rules.map((r) => t(r)).join(" · ")}
-        </span>
-      </div>
-
-      {/* Sources */}
-      {state.sources.length > 0 && (
+        {/* Enforces — plain text, no pills */}
         <div className="portal-policies__detail-inline">
           <span className="portal-policies__detail-inline-label">
-            {t("portal.policies.detail.sources")}
+            {t("portal.policies.detail.enforces")}
           </span>
           <span className="portal-policies__detail-inline-value">
-            {state.sources.map(sourceLabel).join(" · ")}
+            {enforceItems
+              ? enforceItems.map((op, i) => (
+                  <span key={op}>
+                    {i > 0 && (
+                      <span
+                        className="portal-policies__enforce-arrow"
+                        aria-hidden
+                      >
+                        {" "}
+                        →{" "}
+                      </span>
+                    )}
+                    {humanizeEndpoint(op, t)}
+                  </span>
+                ))
+              : config.rules.map((r) => t(r)).join(" · ")}
           </span>
         </div>
-      )}
 
-      <h3 className="portal-policies__wizard-heading">
-        {t("portal.policies.detail.recentActivity")}
-      </h3>
+        {/* Sources */}
+        {state.sources.length > 0 && (
+          <div className="portal-policies__detail-inline">
+            <span className="portal-policies__detail-inline-label">
+              {t("portal.policies.detail.sources")}
+            </span>
+            <span className="portal-policies__detail-inline-value">
+              {state.sources.map(sourceLabel).join(" · ")}
+            </span>
+          </div>
+        )}
 
-      {activity.length > 0 ? (
-        <Card padding="none">
-          {activity.map((item, i) => (
-            <div
-              key={`${item.doc}-${i}`}
-              className="portal-policies__activity-row"
-            >
-              <span
-                className={`portal-policies__activity-icon portal-policies__activity-icon--${
-                  item.status === "flagged"
-                    ? "warning"
-                    : item.status === "processing"
-                      ? "info"
-                      : "success"
-                }`}
+        <h3 className="portal-policies__wizard-heading">
+          {t("portal.policies.detail.recentActivity")}
+        </h3>
+
+        {activity.length > 0 ? (
+          <Card padding="none">
+            {activity.map((item, i) => (
+              <div
+                key={`${item.doc}-${i}`}
+                className="portal-policies__activity-row"
               >
-                {item.status === "flagged" ? (
-                  <WarnIcon />
-                ) : item.status === "processing" ? (
-                  <SpinIcon />
-                ) : (
-                  <CheckIcon />
-                )}
-              </span>
-              <span className="portal-policies__activity-text">
-                <span className="portal-policies__activity-doc">
-                  {item.doc}
-                </span>
-                <span className="portal-policies__activity-action">
+                <span
+                  className={`portal-policies__activity-icon portal-policies__activity-icon--${
+                    item.status === "flagged"
+                      ? "warning"
+                      : item.status === "processing"
+                        ? "info"
+                        : "success"
+                  }`}
+                >
                   {item.status === "flagged" ? (
-                    <ActivityError message={item.action} />
+                    <WarnIcon />
+                  ) : item.status === "processing" ? (
+                    <SpinIcon />
                   ) : (
-                    item.action
+                    <CheckIcon />
                   )}
                 </span>
-              </span>
-              <span className="portal-policies__activity-time">
-                {item.time}
-              </span>
-              {item.status === "flagged" && onRetry && (
-                <Button
-                  type="button"
-                  variant="quiet"
-                  className="portal-policies__link portal-policies__activity-retry"
-                  onClick={() => onRetry(item)}
-                >
-                  {t("portal.policies.detail.retry")}
-                </Button>
+                <span className="portal-policies__activity-text">
+                  <span className="portal-policies__activity-doc">
+                    {item.doc}
+                  </span>
+                  <span className="portal-policies__activity-action">
+                    {item.status === "flagged" ? (
+                      <ActivityError message={item.action} />
+                    ) : (
+                      item.action
+                    )}
+                  </span>
+                </span>
+                <span className="portal-policies__activity-time">
+                  {item.time}
+                </span>
+                {item.status === "flagged" && onRetry && (
+                  <Button
+                    type="button"
+                    variant="quiet"
+                    className="portal-policies__link portal-policies__activity-retry"
+                    onClick={() => onRetry(item)}
+                  >
+                    {t("portal.policies.detail.retry")}
+                  </Button>
+                )}
+              </div>
+            ))}
+          </Card>
+        ) : (
+          <Card padding="default">
+            <EmptyState
+              size="compact"
+              title={t("portal.policies.detail.emptyActivity.title")}
+              description={t(
+                "portal.policies.detail.emptyActivity.description",
               )}
-            </div>
-          ))}
-        </Card>
-      ) : (
-        <Card padding="default">
-          <EmptyState
-            size="compact"
-            title={t("portal.policies.detail.emptyActivity.title")}
-            description={t("portal.policies.detail.emptyActivity.description")}
+            />
+          </Card>
+        )}
+
+        <Card padding="none" className="portal-policies__detail-stats">
+          <StatTile
+            label={t("portal.policies.stats.docsEnforced")}
+            value={stats.enforced.toLocaleString()}
+          />
+          <StatTile
+            label={t("portal.policies.stats.dataProcessed")}
+            value={stats.dataProcessed}
+          />
+          <StatTile
+            label={t("portal.policies.stats.activeFor")}
+            value={stats.activeFor}
           />
         </Card>
-      )}
-
-      <Card padding="none" className="portal-policies__detail-stats">
-        <StatTile
-          label={t("portal.policies.stats.docsEnforced")}
-          value={stats.enforced.toLocaleString()}
-        />
-        <StatTile
-          label={t("portal.policies.stats.dataProcessed")}
-          value={stats.dataProcessed}
-        />
-        <StatTile
-          label={t("portal.policies.stats.activeFor")}
-          value={stats.activeFor}
-        />
-      </Card>
-    </Modal>
+      </Modal>
+      <Modal
+        open={confirmingClear}
+        onClose={() => setConfirmingClear(false)}
+        width="sm"
+        title={t("portal.policies.detail.clearHistory.title")}
+        footer={
+          <div className="portal-policies__detail-foot">
+            <Button
+              variant="secondary"
+              size="sm"
+              onClick={() => setConfirmingClear(false)}
+              disabled={busy}
+            >
+              {t("portal.policies.detail.clearHistory.cancel")}
+            </Button>
+            <Button
+              variant="primary"
+              accent="danger"
+              size="sm"
+              disabled={busy}
+              onClick={() => {
+                setConfirmingClear(false);
+                onClearHistory?.();
+              }}
+            >
+              {t("portal.policies.detail.clearHistory.confirm")}
+            </Button>
+          </div>
+        }
+      >
+        {t("portal.policies.detail.clearHistory.body")}
+      </Modal>
+    </>
   );
 }