From a5ee329c365397df7404ba3f01957f214c17ac16 Mon Sep 17 00:00:00 2001
From: James Brunton <jbrunton96@gmail.com>
Date: Thu, 9 Jul 2026 16:43:38 +0100
Subject: [PATCH] Further improvements to policies file tracking (#6941)

# Description of Changes
Fixes requested in review of #6903
---
 .../policy/ledger/ProcessedFileEntity.java    |  11 +-
 .../saas/V32__policy_processed_files.sql      |   6 +
 .../public/locales/en-US/translation.toml     |   6 +
 .../components/policies/PolicyDetailPanel.tsx | 380 ++++++++++--------
 4 files changed, 233 insertions(+), 170 deletions(-)

diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileEntity.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileEntity.java
index b05c8b1eec..2e98025aff 100644
--- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileEntity.java
+++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/ledger/ProcessedFileEntity.java
@@ -10,6 +10,7 @@ import jakarta.persistence.EnumType;
 import jakarta.persistence.Enumerated;
 import jakarta.persistence.Id;
 import jakarta.persistence.IdClass;
+import jakarta.persistence.Index;
 import jakarta.persistence.Table;
 import jakarta.persistence.Transient;
 
@@ -25,7 +26,15 @@ import lombok.Setter;
  * violation rather than a silent merge.
  */
 @Entity
-@Table(name = "policy_processed_files")
+@Table(
+        name = "policy_processed_files",
+        indexes = {
+            // presence cleanup: delete this policy's rows unseen since the sweep began
+            @Index(name = "idx_processed_files_policy_seen", columnList = "policy_id, last_seen"),
+            // cross-policy deletion consensus: existsByIdentityHashAndStatusNot filters
+            // identity_hash on its own, so it cannot ride the (policy_id, identity_hash) PK
+            @Index(name = "idx_processed_files_identity", columnList = "identity_hash")
+        })
 @IdClass(ProcessedFileId.class)
 @NoArgsConstructor
 @Getter
diff --git a/app/saas/src/main/resources/db/migration/saas/V32__policy_processed_files.sql b/app/saas/src/main/resources/db/migration/saas/V32__policy_processed_files.sql
index f3a81de753..4be3fa07d8 100644
--- a/app/saas/src/main/resources/db/migration/saas/V32__policy_processed_files.sql
+++ b/app/saas/src/main/resources/db/migration/saas/V32__policy_processed_files.sql
@@ -28,3 +28,9 @@ CREATE TABLE IF NOT EXISTS policy_processed_files (
 
 CREATE INDEX IF NOT EXISTS idx_processed_files_policy_seen
     ON policy_processed_files (policy_id, last_seen);
+
+-- The cross-policy deletion-consensus check (existsByIdentityHashAndStatusNot) filters identity_hash
+-- alone, so it cannot use the (policy_id, identity_hash) primary key; it runs once per successfully
+-- consumed file, so index it to avoid a full scan on the hot path.
+CREATE INDEX IF NOT EXISTS idx_processed_files_identity
+    ON policy_processed_files (identity_hash);
diff --git a/frontend/editor/public/locales/en-US/translation.toml b/frontend/editor/public/locales/en-US/translation.toml
index 6228914b66..1cc2ad8ff1 100644
--- a/frontend/editor/public/locales/en-US/translation.toml
+++ b/frontend/editor/public/locales/en-US/translation.toml
@@ -7511,6 +7511,12 @@ pause = "Pause"
 resume = "Resume"
 runNow = "Run now"
 
+[portal.policies.detail.clearHistory]
+body = "This policy will forget every file it has already processed and reprocess everything currently in its sources on the next run. The files themselves are not changed. This cannot be undone."
+cancel = "Cancel"
+confirm = "Clear history"
+title = "Clear processed history?"
+
 [portal.policies.detail.emptyActivity]
 description = "Documents will appear here once this policy runs."
 title = "No activity yet"
diff --git a/frontend/editor/src/portal/components/policies/PolicyDetailPanel.tsx b/frontend/editor/src/portal/components/policies/PolicyDetailPanel.tsx
index f8da1226b0..e198fd92c8 100644
--- a/frontend/editor/src/portal/components/policies/PolicyDetailPanel.tsx
+++ b/frontend/editor/src/portal/components/policies/PolicyDetailPanel.tsx
@@ -111,6 +111,7 @@ export function PolicyDetailPanel({
   onRetry,
 }: PolicyDetailPanelProps) {
   const { t } = useTranslation();
+  const [confirmingClear, setConfirmingClear] = useState(false);
   if (!policy) return null;
   const { category, config, state, steps, stats, activity } = policy;
   const isPaused = state.status === "paused";
@@ -136,200 +137,241 @@ export function PolicyDetailPanel({
   }
 
   return (
-    <Modal
-      open
-      onClose={onClose}
-      width="lg"
-      title={t(category.label)}
-      footer={
-        <div className="portal-policies__detail-foot">
-          {canDelete && (
-            <Button
-              variant="tertiary"
-              accent="danger"
-              size="sm"
-              onClick={onDelete}
-              disabled={busy}
-              style={{ marginRight: "auto" }}
-            >
-              {t("portal.policies.detail.actions.delete")}
-            </Button>
-          )}
-          {onRun && (
+    <>
+      <Modal
+        open
+        onClose={onClose}
+        width="lg"
+        title={t(category.label)}
+        footer={
+          <div className="portal-policies__detail-foot">
+            {canDelete && (
+              <Button
+                variant="tertiary"
+                accent="danger"
+                size="sm"
+                onClick={onDelete}
+                disabled={busy}
+                style={{ marginRight: "auto" }}
+              >
+                {t("portal.policies.detail.actions.delete")}
+              </Button>
+            )}
+            {onRun && (
+              <Button
+                variant="secondary"
+                size="sm"
+                onClick={onRun}
+                disabled={busy}
+                style={canDelete ? undefined : { marginRight: "auto" }}
+              >
+                {t("portal.policies.detail.actions.runNow")}
+              </Button>
+            )}
+            {canClearHistory && (
+              <Button
+                variant="secondary"
+                size="sm"
+                onClick={() => setConfirmingClear(true)}
+                disabled={busy}
+              >
+                {t("portal.policies.detail.actions.clearHistory")}
+              </Button>
+            )}
             <Button
               variant="secondary"
               size="sm"
-              onClick={onRun}
-              disabled={busy}
-              style={canDelete ? undefined : { marginRight: "auto" }}
-            >
-              {t("portal.policies.detail.actions.runNow")}
-            </Button>
-          )}
-          {canClearHistory && (
-            <Button
-              variant="secondary"
-              size="sm"
-              onClick={onClearHistory}
+              onClick={onTogglePause}
               disabled={busy}
             >
-              {t("portal.policies.detail.actions.clearHistory")}
+              {isPaused
+                ? t("portal.policies.detail.actions.resume")
+                : t("portal.policies.detail.actions.pause")}
             </Button>
-          )}
-          <Button
-            variant="secondary"
-            size="sm"
-            onClick={onTogglePause}
-            disabled={busy}
+            <Button size="sm" onClick={onEdit} disabled={busy}>
+              {t("portal.policies.detail.actions.editSettings")}
+            </Button>
+          </div>
+        }
+      >
+        {/* Status + trigger strip */}
+        <div className="portal-policies__detail-status">
+          <StatusBadge
+            tone={isPaused ? "warning" : "success"}
+            pulse={!isPaused}
           >
             {isPaused
-              ? t("portal.policies.detail.actions.resume")
-              : t("portal.policies.detail.actions.pause")}
-          </Button>
-          <Button size="sm" onClick={onEdit} disabled={busy}>
-            {t("portal.policies.detail.actions.editSettings")}
-          </Button>
+              ? t("portal.policies.status.paused")
+              : t("portal.policies.status.active")}
+          </StatusBadge>
+          {hasEditorSource && (
+            <>
+              <span className="portal-policies__detail-sep" aria-hidden>
+                ·
+              </span>
+              <span className="portal-policies__detail-meta">{trigger}</span>
+              <span className="portal-policies__detail-sep" aria-hidden>
+                ·
+              </span>
+              <span className="portal-policies__detail-meta">
+                {outputLabel}
+              </span>
+            </>
+          )}
         </div>
-      }
-    >
-      {/* Status + trigger strip */}
-      <div className="portal-policies__detail-status">
-        <StatusBadge tone={isPaused ? "warning" : "success"} pulse={!isPaused}>
-          {isPaused
-            ? t("portal.policies.status.paused")
-            : t("portal.policies.status.active")}
-        </StatusBadge>
-        {hasEditorSource && (
-          <>
-            <span className="portal-policies__detail-sep" aria-hidden>
-              ·
-            </span>
-            <span className="portal-policies__detail-meta">{trigger}</span>
-            <span className="portal-policies__detail-sep" aria-hidden>
-              ·
-            </span>
-            <span className="portal-policies__detail-meta">{outputLabel}</span>
-          </>
-        )}
-      </div>
 
-      {/* Enforces — plain text, no pills */}
-      <div className="portal-policies__detail-inline">
-        <span className="portal-policies__detail-inline-label">
-          {t("portal.policies.detail.enforces")}
-        </span>
-        <span className="portal-policies__detail-inline-value">
-          {enforceItems
-            ? enforceItems.map((op, i) => (
-                <span key={op}>
-                  {i > 0 && (
-                    <span
-                      className="portal-policies__enforce-arrow"
-                      aria-hidden
-                    >
-                      {" "}
-                      →{" "}
-                    </span>
-                  )}
-                  {humanizeEndpoint(op, t)}
-                </span>
-              ))
-            : config.rules.map((r) => t(r)).join(" · ")}
-        </span>
-      </div>
-
-      {/* Sources */}
-      {state.sources.length > 0 && (
+        {/* Enforces — plain text, no pills */}
         <div className="portal-policies__detail-inline">
           <span className="portal-policies__detail-inline-label">
-            {t("portal.policies.detail.sources")}
+            {t("portal.policies.detail.enforces")}
           </span>
           <span className="portal-policies__detail-inline-value">
-            {state.sources.map(sourceLabel).join(" · ")}
+            {enforceItems
+              ? enforceItems.map((op, i) => (
+                  <span key={op}>
+                    {i > 0 && (
+                      <span
+                        className="portal-policies__enforce-arrow"
+                        aria-hidden
+                      >
+                        {" "}
+                        →{" "}
+                      </span>
+                    )}
+                    {humanizeEndpoint(op, t)}
+                  </span>
+                ))
+              : config.rules.map((r) => t(r)).join(" · ")}
           </span>
         </div>
-      )}
 
-      <h3 className="portal-policies__wizard-heading">
-        {t("portal.policies.detail.recentActivity")}
-      </h3>
+        {/* Sources */}
+        {state.sources.length > 0 && (
+          <div className="portal-policies__detail-inline">
+            <span className="portal-policies__detail-inline-label">
+              {t("portal.policies.detail.sources")}
+            </span>
+            <span className="portal-policies__detail-inline-value">
+              {state.sources.map(sourceLabel).join(" · ")}
+            </span>
+          </div>
+        )}
 
-      {activity.length > 0 ? (
-        <Card padding="none">
-          {activity.map((item, i) => (
-            <div
-              key={`${item.doc}-${i}`}
-              className="portal-policies__activity-row"
-            >
-              <span
-                className={`portal-policies__activity-icon portal-policies__activity-icon--${
-                  item.status === "flagged"
-                    ? "warning"
-                    : item.status === "processing"
-                      ? "info"
-                      : "success"
-                }`}
+        <h3 className="portal-policies__wizard-heading">
+          {t("portal.policies.detail.recentActivity")}
+        </h3>
+
+        {activity.length > 0 ? (
+          <Card padding="none">
+            {activity.map((item, i) => (
+              <div
+                key={`${item.doc}-${i}`}
+                className="portal-policies__activity-row"
               >
-                {item.status === "flagged" ? (
-                  <WarnIcon />
-                ) : item.status === "processing" ? (
-                  <SpinIcon />
-                ) : (
-                  <CheckIcon />
-                )}
-              </span>
-              <span className="portal-policies__activity-text">
-                <span className="portal-policies__activity-doc">
-                  {item.doc}
-                </span>
-                <span className="portal-policies__activity-action">
+                <span
+                  className={`portal-policies__activity-icon portal-policies__activity-icon--${
+                    item.status === "flagged"
+                      ? "warning"
+                      : item.status === "processing"
+                        ? "info"
+                        : "success"
+                  }`}
+                >
                   {item.status === "flagged" ? (
-                    <ActivityError message={item.action} />
+                    <WarnIcon />
+                  ) : item.status === "processing" ? (
+                    <SpinIcon />
                   ) : (
-                    item.action
+                    <CheckIcon />
                   )}
                 </span>
-              </span>
-              <span className="portal-policies__activity-time">
-                {item.time}
-              </span>
-              {item.status === "flagged" && onRetry && (
-                <Button
-                  type="button"
-                  variant="quiet"
-                  className="portal-policies__link portal-policies__activity-retry"
-                  onClick={() => onRetry(item)}
-                >
-                  {t("portal.policies.detail.retry")}
-                </Button>
+                <span className="portal-policies__activity-text">
+                  <span className="portal-policies__activity-doc">
+                    {item.doc}
+                  </span>
+                  <span className="portal-policies__activity-action">
+                    {item.status === "flagged" ? (
+                      <ActivityError message={item.action} />
+                    ) : (
+                      item.action
+                    )}
+                  </span>
+                </span>
+                <span className="portal-policies__activity-time">
+                  {item.time}
+                </span>
+                {item.status === "flagged" && onRetry && (
+                  <Button
+                    type="button"
+                    variant="quiet"
+                    className="portal-policies__link portal-policies__activity-retry"
+                    onClick={() => onRetry(item)}
+                  >
+                    {t("portal.policies.detail.retry")}
+                  </Button>
+                )}
+              </div>
+            ))}
+          </Card>
+        ) : (
+          <Card padding="default">
+            <EmptyState
+              size="compact"
+              title={t("portal.policies.detail.emptyActivity.title")}
+              description={t(
+                "portal.policies.detail.emptyActivity.description",
               )}
-            </div>
-          ))}
-        </Card>
-      ) : (
-        <Card padding="default">
-          <EmptyState
-            size="compact"
-            title={t("portal.policies.detail.emptyActivity.title")}
-            description={t("portal.policies.detail.emptyActivity.description")}
+            />
+          </Card>
+        )}
+
+        <Card padding="none" className="portal-policies__detail-stats">
+          <StatTile
+            label={t("portal.policies.stats.docsEnforced")}
+            value={stats.enforced.toLocaleString()}
+          />
+          <StatTile
+            label={t("portal.policies.stats.dataProcessed")}
+            value={stats.dataProcessed}
+          />
+          <StatTile
+            label={t("portal.policies.stats.activeFor")}
+            value={stats.activeFor}
           />
         </Card>
-      )}
-
-      <Card padding="none" className="portal-policies__detail-stats">
-        <StatTile
-          label={t("portal.policies.stats.docsEnforced")}
-          value={stats.enforced.toLocaleString()}
-        />
-        <StatTile
-          label={t("portal.policies.stats.dataProcessed")}
-          value={stats.dataProcessed}
-        />
-        <StatTile
-          label={t("portal.policies.stats.activeFor")}
-          value={stats.activeFor}
-        />
-      </Card>
-    </Modal>
+      </Modal>
+      <Modal
+        open={confirmingClear}
+        onClose={() => setConfirmingClear(false)}
+        width="sm"
+        title={t("portal.policies.detail.clearHistory.title")}
+        footer={
+          <div className="portal-policies__detail-foot">
+            <Button
+              variant="secondary"
+              size="sm"
+              onClick={() => setConfirmingClear(false)}
+              disabled={busy}
+            >
+              {t("portal.policies.detail.clearHistory.cancel")}
+            </Button>
+            <Button
+              variant="primary"
+              accent="danger"
+              size="sm"
+              disabled={busy}
+              onClick={() => {
+                setConfirmingClear(false);
+                onClearHistory?.();
+              }}
+            >
+              {t("portal.policies.detail.clearHistory.confirm")}
+            </Button>
+          </div>
+        }
+      >
+        {t("portal.policies.detail.clearHistory.body")}
+      </Modal>
+    </>
   );
 }