feat(ui): Implement play actions

* Discard plays early to avoid unnecessary transforms and existing checks
* Don't throw client scrobble processing, just set play as discarded
* Fix/improve source play processing on failure

package-lock.json

@@ -30,6 +30,7 @@
         "@kenyip/backoff-strategies": "^1.0.4",
         "@keyv/valkey": "^1.0.8",
         "@lukehagar/plexjs": "^0.39.0",
+        "@minisylar/express-typed-router": "^1.9.3",
         "@pierre/diffs": "^1.2.4",
         "@rocksky/sdk": "^0.3.0",
         "@supercharge/promise-pool": "^3.0.0",
@@ -209,6 +210,7 @@
         "typescript-eslint": "^8.59.3",
         "usehooks-ts": "^3.1.1",
         "vite": "^8.0.12",
+        "vite-plugin-svgr": "^5.2.0",
         "with-local-tmp-dir": "^7.0.1",
         "zocker": "^3.0.0",
         "zod": "^4.4.3"
@@ -3000,6 +3002,22 @@
         "react": ">=16"
       }
     },
+    "node_modules/@minisylar/express-typed-router": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/@minisylar/express-typed-router/-/express-typed-router-1.9.3.tgz",
+      "integrity": "sha512-Ek4pMAvQSMLoHftqJD5beNTL1dndPCrFUSE2i9WyLVltB4sZ+W6wD1yzgMm3AcjGSfTKTZzVfLaH0ziLh5ECYw==",
+      "license": "MIT",
+      "dependencies": {
+        "@standard-schema/spec": "^1.1.0",
+        "@standard-schema/utils": "^0.3.0"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      },
+      "peerDependencies": {
+        "express": "^4.18.0 || ^5.0.0"
+      }
+    },
     "node_modules/@mjackson/form-data-parser": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/@mjackson/form-data-parser/-/form-data-parser-0.4.0.tgz",
@@ -5329,7 +5347,6 @@
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/@standard-schema/utils/-/utils-0.3.0.tgz",
       "integrity": "sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g==",
-      "dev": true,
       "license": "MIT"
     },
     "node_modules/@stdlib/error-tools-fmtprodmsg": {
@@ -5813,6 +5830,258 @@
         "node": ">=8"
       }
     },
+    "node_modules/@svgr/babel-plugin-add-jsx-attribute": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-add-jsx-attribute/-/babel-plugin-add-jsx-attribute-8.0.0.tgz",
+      "integrity": "sha512-b9MIk7yhdS1pMCZM8VeNfUlSKVRhsHZNMl5O9SfaX0l0t5wjdgu4IDzGB8bpnGBBOjGST3rRFVsaaEtI4W6f7g==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/gregberge"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
+    "node_modules/@svgr/babel-plugin-remove-jsx-attribute": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-remove-jsx-attribute/-/babel-plugin-remove-jsx-attribute-8.0.0.tgz",
+      "integrity": "sha512-BcCkm/STipKvbCl6b7QFrMh/vx00vIP63k2eM66MfHJzPr6O2U0jYEViXkHJWqXqQYjdeA9cuCl5KWmlwjDvbA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/gregberge"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
+    "node_modules/@svgr/babel-plugin-remove-jsx-empty-expression": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-remove-jsx-empty-expression/-/babel-plugin-remove-jsx-empty-expression-8.0.0.tgz",
+      "integrity": "sha512-5BcGCBfBxB5+XSDSWnhTThfI9jcO5f0Ai2V24gZpG+wXF14BzwxxdDb4g6trdOux0rhibGs385BeFMSmxtS3uA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/gregberge"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
+    "node_modules/@svgr/babel-plugin-replace-jsx-attribute-value": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-replace-jsx-attribute-value/-/babel-plugin-replace-jsx-attribute-value-8.0.0.tgz",
+      "integrity": "sha512-KVQ+PtIjb1BuYT3ht8M5KbzWBhdAjjUPdlMtpuw/VjT8coTrItWX6Qafl9+ji831JaJcu6PJNKCV0bp01lBNzQ==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/gregberge"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
+    "node_modules/@svgr/babel-plugin-svg-dynamic-title": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-svg-dynamic-title/-/babel-plugin-svg-dynamic-title-8.0.0.tgz",
+      "integrity": "sha512-omNiKqwjNmOQJ2v6ge4SErBbkooV2aAWwaPFs2vUY7p7GhVkzRkJ00kILXQvRhA6miHnNpXv7MRnnSjdRjK8og==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/gregberge"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
+    "node_modules/@svgr/babel-plugin-svg-em-dimensions": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-svg-em-dimensions/-/babel-plugin-svg-em-dimensions-8.0.0.tgz",
+      "integrity": "sha512-mURHYnu6Iw3UBTbhGwE/vsngtCIbHE43xCRK7kCw4t01xyGqb2Pd+WXekRRoFOBIY29ZoOhUCTEweDMdrjfi9g==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/gregberge"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
+    "node_modules/@svgr/babel-plugin-transform-react-native-svg": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-transform-react-native-svg/-/babel-plugin-transform-react-native-svg-8.1.0.tgz",
+      "integrity": "sha512-Tx8T58CHo+7nwJ+EhUwx3LfdNSG9R2OKfaIXXs5soiy5HtgoAEkDay9LIimLOcG8dJQH1wPZp/cnAv6S9CrR1Q==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/gregberge"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
+    "node_modules/@svgr/babel-plugin-transform-svg-component": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-transform-svg-component/-/babel-plugin-transform-svg-component-8.0.0.tgz",
+      "integrity": "sha512-DFx8xa3cZXTdb/k3kfPeaixecQLgKh5NVBMwD0AQxOzcZawK4oo1Jh9LbrcACUivsCA7TLG8eeWgrDXjTMhRmw==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/gregberge"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
+    "node_modules/@svgr/babel-preset": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-preset/-/babel-preset-8.1.0.tgz",
+      "integrity": "sha512-7EYDbHE7MxHpv4sxvnVPngw5fuR6pw79SkcrILHJ/iMpuKySNCl5W1qcwPEpU+LgyRXOaAFgH0KhwD18wwg6ug==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@svgr/babel-plugin-add-jsx-attribute": "8.0.0",
+        "@svgr/babel-plugin-remove-jsx-attribute": "8.0.0",
+        "@svgr/babel-plugin-remove-jsx-empty-expression": "8.0.0",
+        "@svgr/babel-plugin-replace-jsx-attribute-value": "8.0.0",
+        "@svgr/babel-plugin-svg-dynamic-title": "8.0.0",
+        "@svgr/babel-plugin-svg-em-dimensions": "8.0.0",
+        "@svgr/babel-plugin-transform-react-native-svg": "8.1.0",
+        "@svgr/babel-plugin-transform-svg-component": "8.0.0"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/gregberge"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
+    "node_modules/@svgr/core": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/@svgr/core/-/core-8.1.0.tgz",
+      "integrity": "sha512-8QqtOQT5ACVlmsvKOJNEaWmRPmcojMOzCz4Hs2BGG/toAp/K38LcsMRyLp349glq5AzJbCEeimEoxaX6v/fLrA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@babel/core": "^7.21.3",
+        "@svgr/babel-preset": "8.1.0",
+        "camelcase": "^6.2.0",
+        "cosmiconfig": "^8.1.3",
+        "snake-case": "^3.0.4"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/gregberge"
+      }
+    },
+    "node_modules/@svgr/core/node_modules/cosmiconfig": {
+      "version": "8.3.6",
+      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz",
+      "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "import-fresh": "^3.3.0",
+        "js-yaml": "^4.1.0",
+        "parse-json": "^5.2.0",
+        "path-type": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/d-fischer"
+      },
+      "peerDependencies": {
+        "typescript": ">=4.9.5"
+      },
+      "peerDependenciesMeta": {
+        "typescript": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@svgr/hast-util-to-babel-ast": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/@svgr/hast-util-to-babel-ast/-/hast-util-to-babel-ast-8.0.0.tgz",
+      "integrity": "sha512-EbDKwO9GpfWP4jN9sGdYwPBU0kdomaPIL2Eu4YwmgP+sJeXT+L7bMwJUBnhzfH8Q2qMBqZ4fJwpCyYsAN3mt2Q==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@babel/types": "^7.21.3",
+        "entities": "^4.4.0"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/gregberge"
+      }
+    },
+    "node_modules/@svgr/plugin-jsx": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/@svgr/plugin-jsx/-/plugin-jsx-8.1.0.tgz",
+      "integrity": "sha512-0xiIyBsLlr8quN+WyuxooNW9RJ0Dpr8uOnH/xrCVO8GLUcwHISwj1AG0k+LFzteTkAA0GbX0kj9q6Dk70PTiPA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@babel/core": "^7.21.3",
+        "@svgr/babel-preset": "8.1.0",
+        "@svgr/hast-util-to-babel-ast": "8.0.0",
+        "svg-parser": "^2.0.4"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/gregberge"
+      },
+      "peerDependencies": {
+        "@svgr/core": "*"
+      }
+    },
     "node_modules/@svrooij/sonos": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/@svrooij/sonos/-/sonos-2.5.0.tgz",
@@ -10317,6 +10586,17 @@
         "csstype": "^3.0.2"
       }
     },
+    "node_modules/dot-case": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.4.tgz",
+      "integrity": "sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "no-case": "^3.0.4",
+        "tslib": "^2.0.3"
+      }
+    },
     "node_modules/dotenv": {
       "version": "10.0.0",
       "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz",
@@ -15100,6 +15380,16 @@
         "get-func-name": "^2.0.1"
       }
     },
+    "node_modules/lower-case": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz",
+      "integrity": "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "tslib": "^2.0.3"
+      }
+    },
     "node_modules/lowercase-keys": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
@@ -15996,6 +16286,17 @@
         "react-dom": "^16.8 || ^17 || ^18 || ^19 || ^19.0.0-rc"
       }
     },
+    "node_modules/no-case": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz",
+      "integrity": "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "lower-case": "^2.0.2",
+        "tslib": "^2.0.3"
+      }
+    },
     "node_modules/node-addon-api": {
       "version": "7.1.1",
       "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz",
@@ -18556,6 +18857,17 @@
         "node": ">=6"
       }
     },
+    "node_modules/snake-case": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/snake-case/-/snake-case-3.0.4.tgz",
+      "integrity": "sha512-LAOh4z89bGQvl9pFfNF8V146i7o7/CqFPbqzYgP+yYzDIDeS9HaNFtXABamRW+AQzEVODcvE79ljJ+8a9YSdMg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "dot-case": "^3.0.4",
+        "tslib": "^2.0.3"
+      }
+    },
     "node_modules/sonic-boom": {
       "version": "4.2.1",
       "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz",
@@ -19021,6 +19333,16 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/svg-parser": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.5.tgz",
+      "integrity": "sha512-FuT74puvVMJQ8sgFxgOKPex5MbxPeltqCXJV3wR9Xq+vPHaF+tTdE1lhJcwspUxjtjvXy1NwqcSLZx9UNwHawg==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/tagged-tag": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/tagged-tag/-/tagged-tag-1.0.0.tgz",
@@ -20174,6 +20496,21 @@
         "picocolors": "^1.1.1"
       }
     },
+    "node_modules/vite-plugin-svgr": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/vite-plugin-svgr/-/vite-plugin-svgr-5.2.0.tgz",
+      "integrity": "sha512-qj2eAKF8C6PZWemVTvQA0xgQIcP1hHU6Buh7fl6BhvayWwnuxE+z417miKxeDvRWbDrupQ1oK99hfxElopJ3sQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@rollup/pluginutils": "^5.3.0",
+        "@svgr/core": "^8.1.0",
+        "@svgr/plugin-jsx": "^8.1.0"
+      },
+      "peerDependencies": {
+        "vite": ">=3.0.0"
+      }
+    },
     "node_modules/vlc-client": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/vlc-client/-/vlc-client-1.1.1.tgz",

package.json

@@ -70,6 +70,7 @@
     "@kenyip/backoff-strategies": "^1.0.4",
     "@keyv/valkey": "^1.0.8",
     "@lukehagar/plexjs": "^0.39.0",
+    "@minisylar/express-typed-router": "^1.9.3",
     "@pierre/diffs": "^1.2.4",
     "@rocksky/sdk": "^0.3.0",
     "@supercharge/promise-pool": "^3.0.0",
@@ -249,6 +250,7 @@
     "typescript-eslint": "^8.59.3",
     "usehooks-ts": "^3.1.1",
     "vite": "^8.0.12",
+    "vite-plugin-svgr": "^5.2.0",
     "with-local-tmp-dir": "^7.0.1",
     "zocker": "^3.0.0",
     "zod": "^4.4.3"

patches/@minisylar+express-typed-router+1.9.3.patch

@@ -0,0 +1,58 @@
+diff --git a/node_modules/@minisylar/express-typed-router/dist/schema-router.d.cts b/node_modules/@minisylar/express-typed-router/dist/schema-router.d.cts
+index 87de99b..228c288 100644
+--- a/node_modules/@minisylar/express-typed-router/dist/schema-router.d.cts
++++ b/node_modules/@minisylar/express-typed-router/dist/schema-router.d.cts
+@@ -319,13 +319,17 @@ declare class TypedRouter<Req extends Record<string, any> = AdditionalReqProps,
+    */
+   docs(options?: DocsOptions): express.Router & express.RequestHandler;
+   get<Path extends string>(path: Path, handler: SchemaRouteHandler<Path, unknown, unknown, Req, Locals>): TypedRouter<Req, Locals>;
+-  get<Path extends string, BodySchema extends AnyStandardSchema | unknown, QuerySchema extends AnyStandardSchema | unknown>(path: Path, options: RouteOptions<BodySchema, QuerySchema>, handler: SchemaRouteHandler<Path, BodySchema, QuerySchema, Req, Locals>): TypedRouter<Req, Locals>;
+-  get<Path extends string, Middleware extends readonly TypedMiddleware<any, any>[]>(path: Path, options: {
+-    middleware: Middleware;
+-  }, handler: SchemaRouteHandler<Path, unknown, unknown, Req & InferMiddlewareProps<Middleware>, Locals & InferMiddlewareLocals<Middleware>>): TypedRouter<Req, Locals>;
+-  get<Path extends string, BodySchema extends AnyStandardSchema | unknown, QuerySchema extends AnyStandardSchema | unknown, M extends TypedMiddleware<any, any>[]>(path: Path, options: RouteOptions<BodySchema, QuerySchema> & {
+-    middleware: [...M];
+-  }, // Using tuple spread pattern
++  get<Path extends string, BodySchema extends AnyStandardSchema | unknown = unknown, QuerySchema extends AnyStandardSchema | unknown = unknown, M extends TypedMiddleware<any, any>[] = []>(path: Path, options: {
++    bodySchema?: BodySchema;
++    querySchema?: QuerySchema;
++    middleware?: [...M];
++    tags?: string[];
++    description?: string;
++    summary?: string;
++    deprecated?: boolean;
++    responseSchema?: AnyStandardSchema;
++    hidden?: boolean;
++  }, // Using tuple spread pattern; kept as a flat object (not intersected with a generic RouteOptions<B,Q>) so TS can infer the M tuple and BodySchema/QuerySchema from the same object literal
+   handler: SchemaRouteHandler<Path, BodySchema, QuerySchema, Req & InferMiddlewareProps<readonly [...M]>, // Make it readonly for type inference
+   // Make it readonly for type inference
+   Locals & InferMiddlewareLocals<readonly [...M]>>): TypedRouter<Req, Locals>;
+diff --git a/node_modules/@minisylar/express-typed-router/dist/schema-router.d.mts b/node_modules/@minisylar/express-typed-router/dist/schema-router.d.mts
+index 87de99b..228c288 100644
+--- a/node_modules/@minisylar/express-typed-router/dist/schema-router.d.mts
++++ b/node_modules/@minisylar/express-typed-router/dist/schema-router.d.mts
+@@ -319,13 +319,17 @@ declare class TypedRouter<Req extends Record<string, any> = AdditionalReqProps,
+    */
+   docs(options?: DocsOptions): express.Router & express.RequestHandler;
+   get<Path extends string>(path: Path, handler: SchemaRouteHandler<Path, unknown, unknown, Req, Locals>): TypedRouter<Req, Locals>;
+-  get<Path extends string, BodySchema extends AnyStandardSchema | unknown, QuerySchema extends AnyStandardSchema | unknown>(path: Path, options: RouteOptions<BodySchema, QuerySchema>, handler: SchemaRouteHandler<Path, BodySchema, QuerySchema, Req, Locals>): TypedRouter<Req, Locals>;
+-  get<Path extends string, Middleware extends readonly TypedMiddleware<any, any>[]>(path: Path, options: {
+-    middleware: Middleware;
+-  }, handler: SchemaRouteHandler<Path, unknown, unknown, Req & InferMiddlewareProps<Middleware>, Locals & InferMiddlewareLocals<Middleware>>): TypedRouter<Req, Locals>;
+-  get<Path extends string, BodySchema extends AnyStandardSchema | unknown, QuerySchema extends AnyStandardSchema | unknown, M extends TypedMiddleware<any, any>[]>(path: Path, options: RouteOptions<BodySchema, QuerySchema> & {
+-    middleware: [...M];
+-  }, // Using tuple spread pattern
++  get<Path extends string, BodySchema extends AnyStandardSchema | unknown = unknown, QuerySchema extends AnyStandardSchema | unknown = unknown, M extends TypedMiddleware<any, any>[] = []>(path: Path, options: {
++    bodySchema?: BodySchema;
++    querySchema?: QuerySchema;
++    middleware?: [...M];
++    tags?: string[];
++    description?: string;
++    summary?: string;
++    deprecated?: boolean;
++    responseSchema?: AnyStandardSchema;
++    hidden?: boolean;
++  }, // Using tuple spread pattern; kept as a flat object (not intersected with a generic RouteOptions<B,Q>) so TS can infer the M tuple and BodySchema/QuerySchema from the same object literal
+   handler: SchemaRouteHandler<Path, BodySchema, QuerySchema, Req & InferMiddlewareProps<readonly [...M]>, // Make it readonly for type inference
+   // Make it readonly for type inference
+   Locals & InferMiddlewareLocals<readonly [...M]>>): TypedRouter<Req, Locals>;

src/backend/common/AbstractComponent.ts

@@ -7,7 +7,7 @@ import {MONITORING_ORIGIN_SYSTEM, MONITORING_ORIGIN_USER, type ComponentType, ty
 import { buildTrackString } from "../../core/StringUtils.ts";
 import type {CommonClientConfig} from "./infrastructure/config/client/index.ts";
 import type {CommonSourceConfig} from "./infrastructure/config/source/index.ts";
-import { mergeSimpleError, SimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.ts";
+import { mergeSimpleError, SimpleError, SkipTransformStageError, StageChangeError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.ts";
 import {
     FLOW_CONTROL_TERM,
     type PlayTransformRules,
@@ -33,7 +33,7 @@ import { getRetentionCompactAfterFromEnv, getRetentionDeleteAfterFromEnv, isComp
 import type {DbConcrete} from "./database/drizzle/drizzleUtils.ts";
 import type {ComponentSelect} from "./database/drizzle/drizzleTypes.ts";
 import { DrizzlePlayRepository } from "./database/drizzle/repositories/PlayRepository.ts";
-import type {ClientType, MonitoringStatus} from "../../core/Atomic.ts";
+import type {ClientType, MonitoringStatus, OptionalCacheUsage} from "../../core/Atomic.ts";
 import type {SourceType} from "../../core/Atomic.ts";
 import { DrizzleComponentRepository } from "./database/drizzle/repositories/ComponentRepository.ts";
 import dayjs, { type Dayjs } from "dayjs";
@@ -221,6 +221,20 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         }
     }
 
+    public abstract stop(opts?: {reason?: string | Error}):  Promise<void>
+    public abstract start(opts?: {forceInit?: boolean}):  Promise<boolean>
+
+    public async restart(opts: { reason?: string | Error, forceInit?: boolean } = {}): Promise<void> {
+        try {
+            await this.stop(opts);
+            await this.start(opts);
+        } catch (e) {
+            const err = new StageChangeError('Failed to restart', { cause: e });
+            this.replaceErrors(err);
+            throw err;
+        }
+    }
+
     public retentionCleanup = async () => {
         if(this.databaseOK !== true) {
             this.logger.warn(`Cannot run retention cleanup because ${this.componentType} database state is not OK`);
@@ -233,7 +247,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             this.setStatus('Retention cleanup finished');
         } catch (e) {
             const retentionErr = new Error('Failed to do retention cleanup', {cause: e});
-            this.warning = retentionErr;
+            this.warnings.push(retentionErr);
             this.logger.warn(retentionErr);
             this.setStatus('Retention cleanup failed');
         }
@@ -248,7 +262,12 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         return partArr.map(x => this.transformManager.parseTransformerConfig(x));
     }
 
-    public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean | 'all') => {
+    public transformPlay = async (play: PlayObject, hookType: TransformHook, transformOpts: {log?: boolean | 'all'} & OptionalCacheUsage = {}) => {
+
+        const {
+            log,
+            useCachedResult = true
+        } = transformOpts;
 
         let logger: Logger;
 
@@ -280,7 +299,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             const shouldLog = log ?? this.config.options?.playTransform?.log ?? isDebugMode();
 
             const transformHash = `playTransform-${hashObject(hook)}-${hashObject(playContentInvariantTransform(play))}`;
-            const cachedSteps = await this.cache.cacheTransform.get<LifecycleStep[]>(transformHash);
+            const cachedSteps = useCachedResult ?  await this.cache.cacheTransform.get<LifecycleStep[]>(transformHash) : undefined;
             if(cachedSteps !== undefined) {
                 logger.trace(`Cache hit for Steps => ${transformHash}`);
                 //return cachedTransformPlay;
@@ -294,7 +313,8 @@ export default abstract class AbstractComponent extends AbstractInitializable {
 
             const opts = {
                 logger,
-                asyncId
+                asyncId,
+                useCachedResult
             }
 
             if(cachedSteps !== undefined) {
@@ -415,17 +435,18 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         }
     }
 
-    protected generateStepFromStage = async (playTruth: PlayObject, hookItem: StageConfig, hookType: TransformHook, opts: { logger?: Logger, asyncId?: string } = {}): Promise<[LifecycleStep, PlayObject]> => {
+    protected generateStepFromStage = async (playTruth: PlayObject, hookItem: StageConfig, hookType: TransformHook, opts: { logger?: Logger, asyncId?: string } & OptionalCacheUsage = {}): Promise<[LifecycleStep, PlayObject]> => {
         const {
             onSuccess = 'continue',
             onFailure = 'stop',
             onSkip = 'continue',
-            failureReturnPartial = false
+            failureReturnPartial = false,
         } = hookItem;
 
         const {
             logger = loggerNoop,
-            asyncId = nanoid(6)
+            asyncId = nanoid(6),
+            useCachedResult
         } = opts;
 
         const {
@@ -446,7 +467,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             stageName: string = 'Unnamed',
             err: Error;
         try {
-            [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, playTruth, asyncId);
+            [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, playTruth, {asyncId, useCachedResult});
             newTransformedPlay = clone(newTransformedPlay);
         } catch (e) {
             err = e;
@@ -533,7 +554,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
 
     public abstract getRunningState(): ComponentState
 
-    public getApiData(): Omit<ComponentCommonApiJson, 'type' | 'countLive' | 'players'> & Pick<ComponentCommonApi, 'state' | 'error' | 'warning'>  {
+    public getApiData(): Omit<ComponentCommonApiJson, 'type' | 'countLive' | 'players' | 'deadLetterPlays' | 'deadLetterPlaysTotal' | 'queued'> & Pick<ComponentCommonApi, 'state' | 'errors' | 'warnings'>  {
         let state: ComponentState;
         if(!this.initializedOnce || this.initializing) {
             state = COMPONENT_STATE.INITIALIZING;
@@ -553,8 +574,8 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             createdAt: this.dbComponent.createdAt?.toISOString(),
             lastReadyAt: this.lastReadyAt?.toISOString(),
             lastActiveAt: this.lastActiveAt?.toISOString(),
-            error: this.error !== undefined && this.error instanceof Error ? serializeError(this.error) : this.error,
-            warning: this.warning !== undefined && this.warning instanceof Error ? serializeError(this.warning) : this.warning,
+            errors: this.errors.map(x => x instanceof Error ? serializeError(x) : x),
+            warnings: this.warnings.map(x => x instanceof Error ? serializeError(x) : x),
             ...this.additionalApiData()
         }
     }
@@ -569,12 +590,20 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         });
     }
 
-    protected emitComponentUpdate = <T extends Partial<ReturnType<typeof this.getApiData>>>(payload: T) => {
-        if('error' in payload && payload.error instanceof Error) {
-            payload.error = serializeError(payload.error);
+    public emitComponentUpdate = <T extends Partial<ReturnType<typeof this.getApiData>>>(payload: T) => {
+        if('errors' in payload) {
+            if(payload.errors.length > 0) {
+                payload.errors = payload.errors.map(x => x instanceof Error ? serializeError(x) : x);
+            } else {
+                payload.errors = [];
+            }
         }
-        if('warning' in payload && payload.warning instanceof Error) {
-            payload.warning = serializeError(payload.warning);
+        if('warnings' in payload) {
+            if(payload.warnings.length > 0) {
+               payload.warnings = payload.warnings.map(x => x instanceof Error ? serializeError(x) : x); 
+            } else {
+                payload.warnings = [];
+            }
         }
         this.emitEvent('componentUpdate', payload);
     }
@@ -590,7 +619,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         this.setStatus(payload.title);
     }
 
-    protected setStatus = (status: string) => {
+    public setStatus = (status: string) => {
         this.status = status;
         this.emitComponentUpdate({status});
     }

src/backend/common/AbstractInitializable.ts

@@ -3,13 +3,14 @@ import {truncateStringToLength } from "../../core/StringUtils.ts";
 import { hasNodeNetworkException } from "./errors/NodeErrors.ts";
 import { hasUpstreamError } from "./errors/UpstreamError.ts";
 import type {WebhookPayload} from "./infrastructure/config/health/webhooks.ts";
-import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, StageError } from "./errors/MSErrors.ts";
-import { messageWithCausesTruncatedDefault } from "../../core/ErrorUtils.ts";
+import { AuthCheckError, AuthError, BuildDataError, ConnectionCheckError, findAuthIssue, ParseCacheError, PostInitError, StageError, type AuthErrorMap } from "./errors/MSErrors.ts";
+import { generateErrorTruthyTest, messageWithCausesTruncatedDefault, type TruthyErrorsOpts } from "../../core/ErrorUtils.ts";
 import { spawn } from 'abort-controller-x';
-
+import { COMPONENT_AUTH_TYPE, type ComponentAuthType } from "../../core/Atomic.ts";
 export default abstract class AbstractInitializable {
     requiresAuth: boolean = false;
     requiresAuthInteraction: boolean = false;
+    authType: ComponentAuthType = COMPONENT_AUTH_TYPE.none;
     authed: boolean = false;
     authFailure?: boolean;
 
@@ -19,8 +20,8 @@ export default abstract class AbstractInitializable {
     databaseOK?: boolean | null;
     connectionOK?: boolean | null;
     cacheOK?: boolean | null;
-    error?: Error;
-    warning?: Error;
+    errors?: Error[] = [];
+    warnings?: Error[] = [];
 
     protected initializedOnce: boolean = false;
     initializing: boolean = false;
@@ -88,15 +89,15 @@ export default abstract class AbstractInitializable {
                 } catch (e) {
                     throw new PostInitError('Error occurred during post-initialization hook', {cause: e});
                 }
-                this.error = undefined;
-                this.warning = undefined;
+                this.errors = [];
+                this.warnings = [];
                 return true;
             } catch(e) {
                 if(notify) {
                     await this.notify({identifier: this.getIdentifier(), title: notifyTitle, message: truncateStringToLength(500)(messageWithCausesTruncatedDefault(e)), priority: 'error'});
                 }
                 const initError = new Error('Initialization failed', {cause: e});
-                this.error = initError;
+                this.errors = [initError];
                 throw initError;
             } finally {
                 this.initializing = false;
@@ -283,17 +284,31 @@ export default abstract class AbstractInitializable {
         return;
     }
 
-    authGated = () => this.requiresAuth && !this.authed
+    authGated = () => this.authType !== COMPONENT_AUTH_TYPE.none && !this.authed
 
-    canTryAuth = () => this.isUsable() && this.authGated() && this.authFailure !== true
+    canTryAuth = () => this.isUsable() && this.authGated() && !this.hasUnrecoverableAuthFailure()
 
-    canAuthUnattended = () => !this.authGated || !this.requiresAuthInteraction || (this.requiresAuthInteraction && !this.authFailure);
+    findAuthIssue = <T extends keyof AuthErrorMap = 'auth'>(opts: Parameters<typeof findAuthIssue<T>>[1] = {}): AuthErrorMap[T] | undefined => {
+        for(const err of this.errors) {
+            const authError = findAuthIssue(err, opts);
+            if(authError !== undefined) {
+                return authError;
+            }
+        }
+        return undefined;
+    }
+
+    hasAuthIssue = () => this.findAuthIssue() !== undefined;
+
+    hasUnrecoverableAuthFailure = () => this.findAuthIssue({unrecoverable: true}) !== undefined;
+
+    canAuthUnattended = () => !this.authGated() || this.authType === COMPONENT_AUTH_TYPE.unattended || (this.authType === COMPONENT_AUTH_TYPE.interactive && !this.hasUnrecoverableAuthFailure()) ;
 
     protected doAuthentication = async (): Promise<boolean> => this.authed
 
     // default init function, should be overridden if auth stage is required
     testAuth = async (force: boolean = false) => {
-        if(!this.requiresAuth) {
+        if(this.authType === COMPONENT_AUTH_TYPE.none) {
             return;
         }
         if(this.authed) {
@@ -303,24 +318,46 @@ export default abstract class AbstractInitializable {
             this.logger.debug('Auth OK but step was forced');
         }
 
-        if(this.authFailure) {
+        // only throw *before* testing if we have previously tested and the error was unrecoverable
+        // there is no reason to constantly retry auth when we already know it won't succeed
+        //
+        // test retries should be forced by api calls and auth callback flows *after* we have made changes to component credentials
+        const unrecoverableAuth = this.findAuthIssue({unrecoverable: true});
+        if(unrecoverableAuth !== undefined) {
+            let unrecoverable: boolean | undefined,
+            cause: Error;
             if(!force) {
-                if(this.requiresAuthInteraction) {
-                    throw new AuthCheckError('Authentication failure: Will not retry auth because user interaction is required for authentication');
+                if(unrecoverableAuth instanceof AuthCheckError) {
+                    // we threw a fallback AuthCheckError
+                    unrecoverable = true;
+                    cause = unrecoverableAuth.cause as Error;
+                } else {
+                    cause = unrecoverableAuth;
                 }
-                throw new AuthCheckError('Authentication failure: Will not retry auth because authentication previously failed and must be reauthenticated');
+                if(this.authType === COMPONENT_AUTH_TYPE.interactive) {
+                    throw new AuthCheckError('Authentication failure: Will not retry auth because user interaction is required for authentication', {cause, unrecoverable});
+                }
+                throw new AuthCheckError('Authentication failure: Will not retry auth because authentication previously failed and must be reauthenticated', {cause, unrecoverable});
             }
             this.logger.debug('Auth previously failed for non upstream/network reasons but retry is being forced');
         }
 
         try {
-            this.authed = await this.doAuthentication();
-            this.authFailure = !this.authed;
+            await this.doAuthentication();
+            this.authed = true;
         } catch (e) {
-            // only signal as auth failure if error was NOT either a node network error or a non-showstopping upstream error
-            this.authFailure = !(hasNodeNetworkException(e) || hasUpstreamError(e, false));
+            let unrecoverableMsg: boolean,
+            unrecoverable: boolean | undefined;
+            if(e instanceof AuthError) {
+                unrecoverableMsg = e.unrecoverable;
+            } else {
+                // if component auth test didn't throw AuthError (it should have!)
+                // then fallback teo determining by these conditions
+                unrecoverable = !(hasNodeNetworkException(e) || hasUpstreamError(e, false));
+                unrecoverableMsg = unrecoverable;
+            }
             this.authed = false;
-            throw new AuthCheckError(`Authentication test failed!${this.authFailure === false ? ' Due to a network issue. Will retry authentication on next heartbeat.' : ''}`, {cause: e});
+            throw new AuthCheckError(`Authentication test failed!${unrecoverableMsg === false ? ' Due to a network issue. Will retry authentication on next heartbeat.' : ''}`, {cause: e, unrecoverable});
         }
     }
 
@@ -349,4 +386,20 @@ export default abstract class AbstractInitializable {
     public additionalApiData(): Record<string, any> {
         return {};
     }
+
+    public clearErrors(opts?: TruthyErrorsOpts) {
+        if (!opts) { this.errors = []; return; }
+        
+        const test = generateErrorTruthyTest(opts);
+        this.errors = this.errors.filter(x => !test(x));
+    }
+
+    public replaceErrors(e: Error, opts?: TruthyErrorsOpts) {
+        if(opts !== undefined) {
+            this.clearErrors(opts);
+        } else {
+            this.clearErrors({instance: e});
+        }
+        this.errors.push(e);
+    }
 }

src/backend/common/database/appMigrations/002_inputHash.ts

@@ -0,0 +1,73 @@
+import type { SqliteDatabase, Migration } from 'sqlite-up';
+import type { MigrateBaseContext } from '../appMigrator.ts';
+import { playInputs, plays as drizzlePlays } from '../drizzle/schema/schema.ts';
+import { eq } from 'drizzle-orm';
+import { playContentBasicInvariantTransform } from '../../../utils/PlayComparisonUtils.ts';
+import { hashObject } from '../../../utils/StringUtils.ts';
+
+
+export const up: Migration<MigrateBaseContext>['up'] = async (db: SqliteDatabase, ctx: MigrateBaseContext): Promise<void> => {
+
+    ctx.logger.info('Generating hashes for Play input data...');
+
+    let more = true;
+    let offset = 0,
+        processed = 0,
+        updated = 0;
+    
+    // first update input hashes
+    while (more) {
+        const inputRows = await ctx.db.select().from(playInputs).limit(100).offset(offset);
+        for (const row of inputRows) {
+            if (row.playHash !== null) {
+                processed++;
+            }
+            try {
+                await ctx.db.update(playInputs).set({
+                    playHash: hashObject(playContentBasicInvariantTransform(row.play).data)
+                }).where(eq(playInputs.id, row.id));
+                updated++;
+                processed++;
+            } catch (e) {
+                ctx.logger.warn(new Error(`Failed to generate hash for Play Input ${row.id}`, { cause: e }));
+            }
+        }
+        offset += 100;
+        ctx.logger.verbose(`Play Input Hash Generation Progress: Processed ${processed} | Updated ${updated}`);
+        if (inputRows.length < 100) {
+            more = false;
+        }
+    }
+
+    ctx.logger.info('Regenerating hashes for Play data...');
+
+    offset = 0;
+    processed = 0;
+    updated = 0;
+    more = true;
+    // then update Plays so hashes reflect Plays after transforms are done 
+    while (more) {
+        const playsRows = await ctx.db.select().from(drizzlePlays).limit(100).offset(offset);
+        for (const row of playsRows) {
+            try {
+                await ctx.db.update(drizzlePlays).set({
+                    playHash: hashObject(playContentBasicInvariantTransform(row.play).data)
+                }).where(eq(drizzlePlays.id, row.id));
+                updated++;
+                processed++;
+            } catch (e) {
+                ctx.logger.warn(new Error(`Failed to generate hash for Play ${row.id} (${row.uid})`, { cause: e }));
+            }
+        }
+        offset += 100;
+        ctx.logger.verbose(`Play Hash Regeneration Progress: Processed ${processed} | Updated ${updated}`);
+        if (playsRows.length < 100) {
+            more = false;
+        }
+    }
+};
+
+export const down: Migration<MigrateBaseContext>['down'] = async (db: SqliteDatabase, ctx: MigrateBaseContext): Promise<void> => {
+    // Rollback code here
+    // context is passed as ctx
+};

src/backend/common/database/appMigrations/003_deadConsolidation.ts

@@ -0,0 +1,76 @@
+import type { SqliteDatabase, Migration } from 'sqlite-up';
+import type { MigrateBaseContext } from '../appMigrator.ts';
+import { queueStates } from '../drizzle/schema/schema.ts';
+import { eq } from 'drizzle-orm';
+
+
+export const up: Migration<MigrateBaseContext>['up'] = async (db: SqliteDatabase, ctx: MigrateBaseContext): Promise<void> => {
+
+    ctx.logger.info('Beginning queue entities consolidation.');
+
+    ctx.logger.verbose('Deleting (now) unused completed queue states...');
+
+    ctx.db.delete(queueStates).where(eq(queueStates.queueStatus,'completed'));
+
+    ctx.logger.verbose('Done with completed queue state deletions');
+
+    ctx.logger.info('Converting dead queue states to ingress states with failure + retries...');
+
+    let more = true;
+    let offset = 0,
+        updated = 0;
+    
+    while (more) {
+        const plays = await ctx.db.query.plays.findMany({
+            where: {
+                queueStates: {
+                    queueName: 'dead'
+                }
+            },
+            with: {
+                queueStates: true
+            },
+            limit: 100,
+            offset
+        });
+
+        for(const p of plays) {
+            const ingress = p.queueStates.find(x => x.queueName === 'ingress');
+            const dead = p.queueStates.find(x => x.queueName === 'dead');
+
+            if(ingress === undefined) {
+                await ctx.db.insert(queueStates).values([{
+                    componentId: p.componentId,
+                    playId: p.id,
+                    queueName: 'ingress',
+                    queueStatus: 'failed',
+                    retries: dead.retries,
+                    error: dead.error,
+                    createdAt: dead.updatedAt,
+                    updatedAt: dead.updatedAt
+                }]);
+            } else {
+                await ctx.db.update(queueStates).set({
+                    queueStatus: 'failed',
+                    retries: dead.retries,
+                    error: dead.error,
+                    updatedAt: dead.updatedAt}).where(eq(queueStates.id, ingress.id));
+            }
+            updated++;
+            await ctx.db.delete(queueStates).where(eq(queueStates.id, dead.id));
+        }
+
+        offset += 100;
+        ctx.logger.verbose(`Conversion Progress: Updated ${updated}`);
+        if (plays.length < 100) {
+            more = false;
+        }
+    }
+
+    ctx.logger.info('Done.');
+};
+
+export const down: Migration<MigrateBaseContext>['down'] = async (db: SqliteDatabase, ctx: MigrateBaseContext): Promise<void> => {
+    // Rollback code here
+    // context is passed as ctx
+};

src/backend/common/database/drizzle/drizzleTypes.ts

@@ -1,5 +1,5 @@
 import type {DBQueryConfig, DBQueryConfigWith, KnownKeysOnly, RelationFieldsFilterInternals, BuildQueryResult, RelationsFilter} from "drizzle-orm";
-import type { components, componentMigrations, playInputs, plays, queueStates, relations, playsHistorical } from "./schema/schema.ts";
+import type { components, componentMigrations, playInputs, plays, queueStates, relations, playsHistorical, playEvents } from "./schema/schema.ts";
 import type {TSchema, TableName} from "./schema/schema.ts";
 
 
@@ -24,6 +24,9 @@ export type PlayNew = typeof plays.$inferInsert;
 export type PlayHistoricalSelect = typeof playsHistorical.$inferSelect;
 export type PlayHistoricalNew = typeof playsHistorical.$inferInsert;
 
+export type PlayEventSelect = typeof playEvents.$inferSelect;
+export type PlayEventNew = typeof playEvents.$inferInsert;
+
 // useful references for building types
 // https://github.com/drizzle-team/drizzle-orm/discussions/2596
 // https://github.com/drizzle-team/drizzle-orm/discussions/1539

src/backend/common/database/drizzle/entityUtils.ts

@@ -1,14 +1,15 @@
 import assert from "node:assert";
-import type {PlayHistoricalNew, PlayHistoricalSelect, PlayNew, PlaySelect, PlaySelectWithQueueStates} from "./drizzleTypes.ts";
+import type {PlayHistoricalNew, PlayHistoricalSelect, PlayNew, PlaySelect, PlaySelectWithQueueStates, QueueStateSelect} from "./drizzleTypes.ts";
 import type {PlayInputNew} from "./drizzleTypes.ts";
 import type {QueueStateNew} from "./drizzleTypes.ts";
 import type {ComponentNew} from "./drizzleTypes.ts";
-import type { MarkOptional } from "ts-essentials";
-import { CLIENT_DEAD_QUEUE, type DeadLetterScrobble, type ErrorLike, type PlayObject } from "../../../../core/Atomic.ts";
+import type { MarkOptional, MarkRequired } from "ts-essentials";
+import { DEAD_QUEUE, type DeadLetterScrobble, type ErrorLike, type LifecycleStep, type PlayObject } from "../../../../core/Atomic.ts";
 import dayjs from "dayjs";
 import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../utils/PlayComparisonUtils.ts";
 import { hashObject } from "../../../utils/StringUtils.ts";
 import { serializeError } from "serialize-error";
+import { PLAY_EVENT_TYPE, type PlayEventDupeCheck, type PlayEventDupeCheckData, type PlayEventPlayStateChange, type PlayEventPlayStateChangeData, type PlayEventQueueStateChange, type PlayEventQueueStateChangeData, type PlayEventScrobbleResult, type PlayEventScrobbleResultData, type PlayEventTransform } from "../../../../core/PlayEvent.ts";
 
 export const generateComponentEntity = (data: MarkOptional<ComponentNew, 'uid'>): ComponentNew => {
     assert(data.name !== undefined, 'Must provide name');
@@ -72,7 +73,7 @@ export const hydratePlaySelect = <T extends PlaySelect | PlayHistoricalSelect>(s
 }
 
 export const playSelectToDeadScrobble = (select: PlaySelectWithQueueStates, serializedError: boolean = false): DeadLetterScrobble<PlayObject> => {
-    const deadQueue = select.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE);
+    const deadQueue = select.queueStates.find(x => x.queueName === DEAD_QUEUE);
     return {
         play: select.play,
         id: select.uid,
@@ -85,9 +86,70 @@ export const playSelectToDeadScrobble = (select: PlaySelectWithQueueStates, seri
 }
 
 export const generateInputEntity = (data: PlayInputNew): PlayInputNew => {
-    return data;
+    const {
+        playHash = hashObject(playContentBasicInvariantTransform(data.play).data)
+    } = data;
+    return {...data, playHash};
 }
 
 export const generateQueueStateEntity = (data: QueueStateNew): QueueStateNew => {
     return data;
-}
+}
+
+export const queueStateToEventData = (qs: QueueStateSelect): PlayEventQueueStateChangeData => {
+    const {
+        queueName,
+        queueStatus,
+        error,
+        retries
+    } = qs;
+    return {
+        queueName,
+        queueStatus,
+        error,
+        retries
+    }
+}
+
+export const transformToPlayEvent = (lifecycle: LifecycleStep[]): Omit<PlayEventTransform, 'playId'> => ({
+    eventName: PLAY_EVENT_TYPE.transform,
+    createdAt: dayjs(lifecycle[0].createdAt),
+    data: lifecycle
+});
+
+export const stateChangeToPlayEvent = (partial: PlayEventPlayStateChangeData): Omit<PlayEventPlayStateChange, 'playId'> => ({
+    eventName: PLAY_EVENT_TYPE.playStateChange,
+    createdAt: dayjs(),
+    data: partial
+})
+
+export const queueStateToPlayEvent = (partial: QueueStateSelect): Omit<PlayEventQueueStateChange, 'playId'> => ({
+    eventName: PLAY_EVENT_TYPE.queueStateChange,
+    createdAt: dayjs(),
+    data: partial
+});
+
+export const dupeCheckToPlayEvent = (partial: MarkRequired<Partial<PlayEventDupeCheckData>, 'match'>): Omit<PlayEventDupeCheck, 'playId'> => {
+    const {match, ...rest} = partial;
+    const data: PlayEventDupeCheckData = {
+        match,
+        score: match ? 1 : 0,
+        breakdowns: [],
+        createdAt: dayjs().toISOString(),
+        ...rest
+    };
+    return {
+        eventName: PLAY_EVENT_TYPE.dupeCheck,
+        createdAt: dayjs(),
+        data
+    }
+}
+
+export const scrobbleToPlayEvent = (data: PlayEventScrobbleResultData): Omit<PlayEventScrobbleResult, 'playId'> => ({
+    eventName: PLAY_EVENT_TYPE.scrobbleResult,
+    createdAt: data.createdAt ?? dayjs(),
+    data
+});
+
+export const entityIsPlayEntity = (obj: object): obj is PlaySelectWithQueueStates => `play` in obj && typeof obj.play === 'object'
+    && `componentId` in obj && typeof obj.componentId === 'number'

src/backend/common/database/drizzle/migrations/20260814181523_abnormal_black_tom/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE `play_inputs` ADD `playHash` text;

src/backend/common/database/drizzle/migrations/20260818175856_daffy_kate_bishop/migration.sql

@@ -0,0 +1,11 @@
+CREATE TABLE `play_events` (
+	`id` integer PRIMARY KEY,
+	`playId` integer NOT NULL,
+	`eventName` text(50) NOT NULL,
+	`data` text,
+	`error` text,
+	`createdAt` number NOT NULL,
+	CONSTRAINT `fk_play_events_playId_plays_id_fk` FOREIGN KEY (`playId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE CASCADE
+);
+--> statement-breakpoint
+CREATE INDEX `play_event_id_idx` ON `play_events` (`playId`);

src/backend/common/database/drizzle/migrations/20260819172112_little_wolfsbane/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE `play_queue_states` ADD `context` text;

src/backend/common/database/drizzle/repositories/BaseRepository.ts

@@ -45,9 +45,9 @@ export abstract class DrizzleBaseRepository<T extends TableName> {
         await this.db.delete(this.table).where(inArray(this.table.id, ids));
     }
 
-    async updateById(id: number, data: Partial<typeof this.table.$inferInsert>): Promise<void> {
+    async updateById(id: number, data: Partial<typeof this.table.$inferInsert>): Promise<typeof this.table.$inferSelect> {
         assert(id !== null && id !== undefined, `${id === null ? 'null' : 'undefined'} given for entity id`);
-        await this.db.update(this.table).set(data).where(eq(this.table.id, id));
+        return (await this.db.update(this.table).set(data).where(eq(this.table.id, id)).returning())[0];
     }
 
     async create(data: typeof this.table.$inferInsert): Promise<typeof this.table.$inferSelect> {

src/backend/common/database/drizzle/repositories/PlayEventsRepository.ts

@@ -0,0 +1,8 @@
+import { DrizzleBaseRepository, type DrizzleRepositoryOpts } from "./BaseRepository.ts";
+import type {DbConcrete} from "../drizzleUtils.ts";
+export class DrizzlePlayEventsRepository extends DrizzleBaseRepository<'playEvents'> {
+
+    constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
+        super(db, 'playEvents', 'Play Events', opts);
+    }
+}

src/backend/common/database/drizzle/repositories/PlayRepository.ts

@@ -3,28 +3,29 @@ import dayjs, { type Dayjs } from "dayjs";
 import { eq, inArray, relationsFilterToSQL, sql } from "drizzle-orm";
 import assert from "node:assert";
 import type { MarkOptional } from "ts-essentials";
-import { type DateLike, type DeepReplaceValue, type PlayObject, type PlayState, type QueueName, TA_DEFAULT_ACCURACY, type TemporalAccuracy } from "../../../../../core/Atomic.ts";
+import { type DateLike, type DeepReplaceValue, type PlayObject, type PlayState, QUEUE_STATUS_QUEUED, type QueueName, SCROBBLE_TS_SOC_END, TA_DEFAULT_ACCURACY, type TemporalAccuracy } from "../../../../../core/Atomic.ts";
 import { removeUndefinedKeys } from '../../../../../core/DataUtils.ts';
 import { shortTodayAwareFormat } from "../../../../../core/TimeUtils.ts";
 import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../../utils/PlayComparisonUtils.ts";
 import { hashObject } from "../../../../utils/StringUtils.ts";
-import { comparePlayTemporally, getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy } from "../../../../utils/TimeUtils.ts";
+import { comparePlayTemporally, getScrobbleTsSOCDateWithContext, getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy } from "../../../../utils/TimeUtils.ts";
 import { type CompactableProperty, type RetentionOptions, retentionPlayTypes } from "../../../infrastructure/config/database.ts";
-import type {SourceType} from "../../../../../core/Atomic.ts";
+import type {ErrorLike, SourceType} from "../../../../../core/Atomic.ts";
 import type {FindMany, FindWhere, FindWith, PlayInputNew, PlayNew, PlaySelect, PlaySelectWithQueueStates, PlayWith, QueueStateSelect, WhereClause} from "../drizzleTypes.ts";
 import { type DbConcrete, runTransaction } from "../drizzleUtils.ts";
-import { generateInputEntity, generatePlayEntity, hydratePlaySelect, type PlayEntityOpts, type PlayHydateOptions } from "../entityUtils.ts";
-import { playInputs, plays, relations } from "../schema/schema.ts";
+import { generateInputEntity, generatePlayEntity, hydratePlaySelect, stateChangeToPlayEvent, transformToPlayEvent, type PlayEntityOpts, type PlayHydateOptions } from "../entityUtils.ts";
+import { playEvents, playInputs, plays, relations, type TSchema } from "../schema/schema.ts";
 import { buildDateCompare, type CompareDateOp, type ComponentConstrainedRepoOpts, DrizzleBaseRepository, type DrizzleRepositoryOpts } from "./BaseRepository.ts";
 import type {PaginatedResponse} from "../../../../../core/Api.ts";
 import type {PaginatedQueryResponse} from "../../../../../core/Api.ts";
-;
+import { type PlayEventTransform } from "../../../../../core/PlayEvent.ts";
 
 // https://github.com/drizzle-team/drizzle-orm/issues/695 may be useful for typing models with relations?
 
 export interface QueueCriteria {
     queueName: QueueName
     queueStatus: QueueStateSelect['queueStatus'][] | QueueStateSelect['queueStatus']
+    retries?: number
 }
 
 export interface PlayWhereOpts<D extends DateLike = Dayjs> {
@@ -38,7 +39,7 @@ export interface PlayWhereOpts<D extends DateLike = Dayjs> {
     text?: string[]
 }
 
-export type WithPlayRelation = 'input' | 'parent' | 'parent-input' | 'queues';
+export type WithPlayRelation = 'input' | 'parent' | 'parent-input' | 'queues' | 'events';
 export interface QueryPlaysOpts<D extends DateLike = Dayjs> extends PlayWhereOpts<D> {
     sort?: 'seenAt' | 'playedAt'
     order?: 'asc' | 'desc'
@@ -92,6 +93,16 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         return res as PlaySelectWithQueueStates;
     }
 
+    findByIdWith = async <K extends keyof TSchema['plays']["relations"]>(id: number, args: WithPlayRelation[]): Promise<PlayWith<K> | undefined> => {
+        const res = await this.db.query.plays.findFirst({
+            where: {
+                id
+            },
+            with: buildPlayWith(args)
+        });
+        return res as unknown as PlayWith<K> | undefined;
+    }
+
     createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[], opts: HydrateOpts = {}): Promise<PlayWith<'input'>[]> => {
 
         const {
@@ -103,11 +114,14 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
 
             const entitiesData = entitiesOpts.map((data) => {
                 const {
-                    play,
+                    play: {
+                        lifecycle,
+                        ...playRest
+                    },
                     input,
                     ...rest
                 } = data;
-                return generatePlayEntity(play, { componentId: this.componentId, ...rest });
+                return generatePlayEntity(playRest, { componentId: this.componentId, ...rest });
             });
 
             const nakedPlays = await this.db.insert(plays).values(entitiesData).returning();
@@ -126,6 +140,23 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
             });
 
             const inputRow = await this.db.insert(playInputs).values(inputDatas);
+
+            const eventData = nakedPlays.map((x, index) => {
+                const {
+                    play: {
+                        lifecycle = [],
+                    } = {}
+                } = entitiesOpts[index];
+
+                if(lifecycle.length > 0) {
+                    return {...transformToPlayEvent(lifecycle), playId: x.id}
+                }
+                return undefined;
+            }).filter(x => x !== undefined);
+            if(eventData.length > 0) {
+                await this.db.insert(playEvents).values(eventData);
+            }
+
             playRows = nakedPlays.map((x, index) => ({...x, play: hydratePlaySelect(x, hydrate), input: inputRow[index]}));
         });
 
@@ -182,6 +213,9 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
                     case 'queues':
                         query.with.queueStates = true;
                         break;
+                    case 'events':
+                        query.with.events = true;
+                        break;
                     default:
                         throw new Error(`Unknown relation ${w}`);
                 }
@@ -408,28 +442,28 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
                     summaryDelStates.push(`No '${retentionType}' Plays older than ${shortTodayAwareFormat(date)}`);
                 } else {
                     for(const id of ids) {
-                        let compactedPlay: PlayObject;
                         if(compactTypes.includes('input')) {
                             await this.db.update(playInputs).set({
                                 data: {removedReason: 'Removed by compaction'}
                             }).where(eq(playInputs.playId, id));
                         }
                         if(compactTypes.includes('transform')) {
-                            const playRow = await this.db.query.plays.findFirst({where: {id: id}});
-                            if(playRow === undefined) {
-                                // uhh shouldn't be
-                                loggerCom.warn(`No Play found with ID ${id}, but it should have been...`);
-                                continue;
-                            }
-
-                            compactedPlay = playRow.play;
-                            if(compactedPlay.lifecycle !== undefined) {
-                                compactedPlay.lifecycle = compactedPlay.lifecycle.map(x => {
-                                    if(x.inputs == undefined) {
+                            const events = await this.db.query.playEvents.findMany({where: {playId: id}});
+                            for(const ev of events) {
+                                if(ev.eventName === 'transform') {
+                                    const transformEvent = ev as PlayEventTransform;
+                                    let compactedInput = false;
+                                    transformEvent.data = transformEvent.data.map(x => {
+                                        if(x.inputs !== undefined && x.inputs.length > 0) {
+                                            compactedInput = true;
+                                            return {...x, inputs: x.inputs.map((y) => ({type: y.type, input: 'Removed by compaction'}))};
+                                        }
                                         return x;
+                                    });
+                                    if(compactedInput) {
+                                        await this.db.update(playEvents).set({data: transformEvent.data}).where(eq(playEvents.id, transformEvent.id));
                                     }
-                                    return {...x, inputs: x.inputs.map(y => ({type: y.type, input: 'Removed by compaction'}))};
-                                });
+                                }
                             }
                         }
 
@@ -437,9 +471,6 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
                         const vals: Parameters<typeof updater.set>[0] = {
                             compacted: compactedFlags.join('-')
                         };
-                        if(compactedPlay !== undefined) {
-                            vals.play = compactedPlay;
-                        }
                         await this.db.update(plays).set(vals).where(eq(plays.id, id));
                     }
                     loggerCom.trace(`Compacted ${ids.length} '${retentionType}' plays`);
@@ -454,65 +485,75 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
     }
 
     protected prepareGetQueueNext = () => this.db.query.plays.findFirst({
+        // https://stackoverflow.com/a/78551796
+        // cannot bind arrays in sqlite
         where: {
             componentId: sql.placeholder('componentId'),
             queueStates: {
                 queueName: sql.placeholder('queueName'),
-                queueStatus: 'queued',
+                queueStatus: sql.placeholder('status'),
                 retries: {
                     lte: sql.placeholder('retries')
                 }
             },
         },
         with: {
-            queueStates: true
-        },
-        orderBy: {
-            seenAt: 'asc'
+            queueStates: {
+                orderBy: {
+                    updatedAt: 'asc'
+                }
+            }
         },
+        // orderBy: {
+        //     seenAt: 'asc'
+        // },
     }).prepare()
 
-    public getQueueNext = async (queueName: string, opts: {order?: 'asc' | 'desc', retries?: number} & ComponentConstrainedRepoOpts = {}): Promise<PlaySelectWithQueueStates | undefined> => {
+    public getQueueNext = async (queueName: string, opts: {order?: 'asc' | 'desc', retries?: number, notIds?: number[], status?: QueueStateSelect['queueStatus']} & ComponentConstrainedRepoOpts = {}): Promise<PlaySelectWithQueueStates | undefined> => {
         const {
-            retries = 0,
+            retries = 1000,
+            notIds,
             order = 'asc',
+            status = QUEUE_STATUS_QUEUED,
             componentId = this.componentId
         } = opts;
 
-        // let where: FindWhere<'plays'> = {
-        //     componentId
-        // }
+        let res: PlaySelectWithQueueStates | undefined;
 
-        // if(retries !== undefined) {
-        //     where.queueStates = {
-        //         queueName,
-        //         queueStatus: 'queued',
-        //         retries: {
-        //             lte: retries
-        //         }
-        //     }
-        // } else {
-        //     where.queueStates = {
-        //         queueName,
-        //         queueStatus: 'queued'
-        //     }
-        // }
+        if (notIds === undefined) {
+            if (this.getQueueNextPrepared === undefined) {
+                this.getQueueNextPrepared = this.prepareGetQueueNext();
+            }
 
-        // const res = await this.db.query.plays.findFirst({
-        //         where: where,
-        //         orderBy: {
-        //             seenAt: order
-        //         },
-        //         with: {
-        //             queueStates: true
-        //         }
-        // });
-
-        if(this.getQueueNextPrepared === undefined) {
-            this.getQueueNextPrepared = this.prepareGetQueueNext();
+            res = await this.getQueueNextPrepared.execute({ queueName, retries, componentId, status });
+        } else {
+            // cannot bind arrays in sqlite so this has to be non-prepared
+            res = await this.db.query.plays.findFirst({
+                where: {
+                    id: {
+                        notIn: notIds
+                    },
+                    componentId: componentId,
+                    queueStates: {
+                        queueName: queueName,
+                        queueStatus: status,
+                        retries: {
+                            lte: retries
+                        }
+                    },
+                },
+                with: {
+                    queueStates: {
+                        orderBy: {
+                            updatedAt: 'asc'
+                        }
+                    }
+                },
+                // orderBy: {
+                //     seenAt: 'asc'
+                // },
+            });
         }
-
-        const res = await this.getQueueNextPrepared.execute({queueName, retries, componentId});
  
         if(res === undefined) {
             return undefined;
@@ -613,12 +654,20 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         return {data: res.map(x => ({...x, play: hydratePlaySelect(x, hydrate)})), meta: {limit, offset}};
     }
 
-    public checkExisting = async (play: PlayObject, opts: {queueName?: string, states?: PlaySelect['state'][], taAccuracy?: TemporalAccuracy[]} & ComponentConstrainedRepoOpts = {}): Promise<PlaySelectWithQueueStates | undefined> => {
+    public checkExisting = async (play: PlayObject, opts: {
+        queueName?: string,
+        states?: PlaySelect['state'][],
+        taAccuracy?: TemporalAccuracy[],
+        inputHash?: string | PlayObject,
+        notId?: number
+    } & ComponentConstrainedRepoOpts = {}): Promise<PlaySelectWithQueueStates | undefined> => {
         const {
             queueName,
             componentId = this.componentId,
             taAccuracy = TA_DEFAULT_ACCURACY,
-            states
+            states,
+            inputHash,
+            notId,
         } = opts;
         const hash = hashObject(playContentBasicInvariantTransform(play).data);
 
@@ -626,19 +675,25 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         // which we can then use with temporal comparison to make sure we are comparing the correct dates
         //
         // this isn't as fast as just comparing playDate directly but its still much faster/cheaper than paginating plays and doing everything in-memory
-        const dateGranularity = getTemporalAccuracyCloseVal(play.meta.source as SourceType);
-        let endRange: Dayjs;
-        if(play.data.playDateCompleted !== undefined) {
-            // this will be present if source reports it
-            // or we tracked it live with MemorySource
-            endRange = play.data.playDateCompleted.add(dateGranularity, 's');
-        } else {
-            endRange = play.data.playDate.add(dateGranularity, 's');
-        }
+        // const dateGranularity = getTemporalAccuracyCloseVal(play.meta.source as SourceType);
+        // let endRange: Dayjs;
+        // if(play.data.playDateCompleted !== undefined) {
+        //     // this will be present if source reports it
+        //     // or we tracked it live with MemorySource
+        //     endRange = play.data.playDateCompleted.add(dateGranularity, 's');
+        // } else {
+        //     endRange = play.data.playDate.add(dateGranularity, 's');
+        // }
         const where: FindWhere<'plays'> = {
             componentId,
-            playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play)),
+            playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play, {useDuration: true})),
         };
+
+        if(notId !== undefined) {
+            where.NOT = {
+                id: notId
+            }
+        }
         
         if(queueName !== undefined) {
             where.queueStates = {
@@ -653,19 +708,20 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         }
 
         const mbidId = playMbidIdentifier(play);
-        if(mbidId !== undefined) {
-            where.AND = [
-                {
-                    OR: [
-                        {
-                            playHash: hash
-                        },
-                        {
-                            mbidIdentifier: mbidId
-                        }
-                    ]
-                }
-            ]
+        if (mbidId !== undefined || inputHash !== undefined) {
+            where.AND = [{
+                OR: [
+                    {
+                        playHash: hash
+                    }
+                ]
+            }];
+            if (mbidId !== undefined) {
+                where.AND[0].OR.push({ mbidIdentifier: mbidId });
+            }
+            if (inputHash !== undefined) {
+                where.AND[0].OR.push({ input: { playHash: typeof inputHash === 'string' ? inputHash : hashObject(playContentBasicInvariantTransform(inputHash).data) } });
+            }
         } else {
             where.playHash = hash;
         }
@@ -673,7 +729,8 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         const res = await this.db.query.plays.findMany({
             where,
             with: {
-                queueStates: true
+                queueStates: true,
+                input: true
             }
         });
         if(res.length === 0) {
@@ -734,26 +791,56 @@ where compacted IS NOT NULL
 group by componentId,compacted;`);
         return res;
     }
+
+    async updateById(id: number, data: Partial<PlayNew> & {event?: boolean, reason?: string, error?: ErrorLike}): Promise<typeof this.table.$inferSelect> {
+        const res = await super.updateById(id, data) as PlaySelect;
+        if(data.event === true) {
+            if(data.state !== undefined) {
+                try {
+                    await this.db.insert(playEvents).values({...stateChangeToPlayEvent(removeUndefinedKeys({state: data.state, reason: data.reason, error: data.error})), playId: id});
+                } catch (e) {
+                    this.logger.warn(new Error(`Failed to create Play Event for state change ${data.state} on Play ${id}`));
+                }
+            }
+        }
+        return res;
+    }
 }
 
-export const getTemporallyCloseDateCompareOp = (play: PlayObject, opts: {bufferTime?: number, useCompleted?: boolean} = {}): CompareDateOp => {
+export const getTemporallyCloseDateCompareOp = (play: PlayObject, opts: {bufferTime?: number, useCompleted?: boolean, useDuration?: boolean} = {}): CompareDateOp => {
     const {
         // use either provided arg or default to using source granularity
         bufferTime = getTemporalAccuracyCloseVal(play.meta.source as SourceType),
-        useCompleted = true
+        useCompleted,
+        useDuration,
     } = opts;
         // we get all plays with a play date between playdate - (buffer) AND (playDateCompleted or playDate) + (buffer)
-        let endRange: Dayjs;
-        if(play.data.playDateCompleted !== undefined && useCompleted) {
+        let startRange: Dayjs,
+        endRange: Dayjs;
+
+        // make sure we use the 
+        const [sotPlayDate, SOT] = getScrobbleTsSOCDateWithContext(play);
+
+        if(useDuration && play.data.duration !== undefined) {
+            if(SOT === SCROBBLE_TS_SOC_END) {
+                endRange = play.data.playDate.add(bufferTime, 's');
+                startRange = play.data.playDate.subtract(play.data.duration + bufferTime,'s')
+            } else {
+                endRange = play.data.playDate.add(play.data.duration + bufferTime, 's');
+                startRange = play.data.playDate.subtract(bufferTime,'s')
+            }
+        } else if(play.data.playDateCompleted !== undefined && useCompleted) {
+            startRange = play.data.playDate.subtract(bufferTime, 's');
             // this will be present if source reports it
             // or we tracked it live with MemorySource
             endRange = play.data.playDateCompleted.add(bufferTime, 's');
         } else {
-            endRange = play.data.playDate.add(bufferTime, 's');
+            startRange = sotPlayDate.subtract(bufferTime, 's');
+            endRange = sotPlayDate.add(bufferTime, 's');
         }
         return {
             type: 'between',
-            range: [play.data.playDate.subtract(bufferTime, 's'), endRange]
+            range: [startRange, endRange]
         }
 }
 
@@ -780,6 +867,9 @@ export const buildPlayWith = (args: WithPlayRelation[] | undefined): FindWith<'p
             case 'queues':
                 qWith.queueStates = true;
                 break;
+            case 'events':
+                qWith.events = true;
+                break;
             default:
                 throw new Error(`Unknown relation ${w}`);
         }
@@ -829,14 +919,26 @@ export const buildPlayWhere = (args: PlayWhereOpts): WhereClause<'plays'> => {
         // or else assigning an array to OR using only `typeof where.queueStates` causes a type error
         const queueWhere: typeof where.queueStates.OR[0][] = [];
         for(const q of queues) {
-            queueWhere.push(
-                {
-                    queueName: q.queueName,
-                    queueStatus: typeof q.queueStatus === 'string' ? q.queueStatus : {
-                        in: q.queueStatus
-                    }
+            const qWhereCriteria: typeof where.queueStates.OR[0] =  {
+                queueName: q.queueName,
+                queueStatus: typeof q.queueStatus === 'string' ? q.queueStatus : {
+                    in: q.queueStatus
                 }
-            )
+            };
+            if(q.retries !== undefined) {
+                qWhereCriteria.retries = {
+                    lte: q.retries
+                }
+            }
+            queueWhere.push(qWhereCriteria);
+            // queueWhere.push(
+            //     {
+            //         queueName: q.queueName,
+            //         queueStatus: typeof q.queueStatus === 'string' ? q.queueStatus : {
+            //             in: q.queueStatus
+            //         }
+            //     }
+            // )
         }
         if(queueWhere.length === 1) {
             where.queueStates = queueWhere[0];

src/backend/common/database/drizzle/repositories/QueueRepository.ts

@@ -1,9 +1,10 @@
-import { eq, and, lte, inArray } from "drizzle-orm";
+import { eq, and, gte, lte, inArray } from "drizzle-orm";
 import { DrizzleBaseRepository, type DrizzleRepositoryOpts } from "./BaseRepository.ts";
 import type {DbConcrete} from "../drizzleUtils.ts";
-import type {QueueStateSelect} from "../drizzleTypes.ts";
-import { queueStates } from "../schema/schema.ts";
-import { CLIENT_DEAD_QUEUE } from "../../../../../core/Atomic.ts";
+import type {PlaySelect, QueueStateSelect} from "../drizzleTypes.ts";
+import { playEvents, queueStates } from "../schema/schema.ts";
+import { DEAD_QUEUE, INGRESS_QUEUE } from "../../../../../core/Atomic.ts";
+import { queueStateToPlayEvent } from "../entityUtils.ts";
 export class DrizzleQueueRepository extends DrizzleBaseRepository<'queueStates'> {
 
     constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
@@ -17,7 +18,7 @@ export class DrizzleQueueRepository extends DrizzleBaseRepository<'queueStates'>
             eq(queueStates.componentId, componentId),
             lte(queueStates.retries, retries),
             eq(queueStates.queueStatus, 'failed'),
-            eq(queueStates.queueName, CLIENT_DEAD_QUEUE)
+            eq(queueStates.queueName, INGRESS_QUEUE)
         ));
     }
 
@@ -27,15 +28,56 @@ export class DrizzleQueueRepository extends DrizzleBaseRepository<'queueStates'>
         }).where(and(
             eq(queueStates.componentId, componentId),
             eq(queueStates.queueStatus, 'queued'),
-            eq(queueStates.queueName, CLIENT_DEAD_QUEUE)
+            eq(queueStates.queueName, DEAD_QUEUE)
         ));
     }
 
-    public getQueueCount = async (componentId: number, queueNames: string[], queueStatus: QueueStateSelect['queueStatus'][] = ['queued']): Promise<number> => {
+    public getQueueCount = async (componentId: number, queueNames: string[], opts: {
+        queueStatus?: QueueStateSelect['queueStatus'][],
+        retries?: number
+        retryEq?: 'lte' | 'gte'
+    } = {}): Promise<number> => {
+        const {
+            queueStatus = ['queued'],
+            retries,
+            retryEq = 'lte'
+        } = opts
+        if(retries === undefined) {
+            return await this.db.$count(queueStates, and(
+                eq(queueStates.componentId, componentId),
+                inArray(queueStates.queueName, queueNames),
+                inArray(queueStates.queueStatus, queueStatus)
+            ));
+        }
         return await this.db.$count(queueStates, and(
             eq(queueStates.componentId, componentId),
             inArray(queueStates.queueName, queueNames),
-            inArray(queueStates.queueStatus, queueStatus)
-        ));
+            inArray(queueStates.queueStatus, queueStatus),
+            retryEq === 'lte' ? lte(queueStates.retries, retries) : gte(queueStates.retries, retries)
+        )); 
+    }
+
+    async create(data: typeof this.table.$inferInsert & {playId?: PlaySelect['id'], event?: boolean}): Promise<typeof this.table.$inferSelect> {
+        const res = await super.create(data) as QueueStateSelect;
+        if(data.event === true && data.playId !== undefined) {
+            try {
+                await this.db.insert(playEvents).values({...queueStateToPlayEvent(res), playId: data.playId});
+            } catch (e) {
+                this.logger.warn(new Error(`Failed to create Play Event for new queue creation on Play ${data.playId}`));
+            }
+        }
+        return res;
+    }
+
+    async updateById(id: number, data: Partial<typeof this.table.$inferInsert> & {playId?: PlaySelect['id'], event?: boolean}): Promise<typeof this.table.$inferSelect> {
+        const res = await super.updateById(id, data) as QueueStateSelect;
+        if(data.event === true && data.playId !== undefined) {
+            try {
+                await this.db.insert(playEvents).values({...queueStateToPlayEvent(res), playId: data.playId});
+            } catch (e) {
+                this.logger.warn(new Error(`Failed to create Play Event for queue ${res.queueName} on Play ${data.playId}`));
+            }
+        }
+        return res;
     }
 }

src/backend/common/database/drizzle/schema/schema.ts

@@ -1,7 +1,7 @@
 import { integer, sqliteTable, text, index, uniqueIndex, customType, type AnySQLiteColumn } from "drizzle-orm/sqlite-core";
 import { defineRelations } from 'drizzle-orm';
 import dayjs, { type Dayjs } from "dayjs";
-import { COMPONENT_TYPE_CLIENT, COMPONENT_TYPE_SOURCE, type ErrorLike, type PlayObject } from "../../../../../core/Atomic.ts";
+import { COMPONENT_TYPE_CLIENT, COMPONENT_TYPE_SOURCE, type ErrorLike, type PlayObject, type QueueContext } from "../../../../../core/Atomic.ts";
 import { asPlayCheap } from "../../../../../core/PlayMarshalUtils.ts";
 import type {ExternalMetadataTerm, PlayTransformPartsConfig, SearchAndReplaceTerm} from "../../../../../core/Transform.ts";
 import type {JobRangeCount, JobRangeTime} from "../../../infrastructure/Job.ts";
@@ -94,6 +94,7 @@ export const playInputs = sqliteTable("play_inputs", {
   playId: integer().notNull().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}),
   data: text({ mode: 'json' }).$type<object>(),
   play: PlayJson('play').notNull(),//text({ mode: 'json' }).notNull().$type<PlayObject>(),
+  playHash: text(),
   createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs())
 }, (table) => [
   uniqueIndex('play_input_id_idx').on(table.playId)
@@ -143,6 +144,7 @@ export const queueStates = sqliteTable("play_queue_states", {
   queueStatus: text({enum: ['queued','completed','failed']}).notNull().default('queued'),
   retries: integer().notNull().default(0),
   error: ErrorLikeJson('error'),
+  context: text({mode: 'json'}).$type<QueueContext & {isRetry?: boolean}>(),
   createdAt: DayjsTimestamp('createdAt').notNull().$defaultFn(() => dayjs()),
   updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()).$onUpdate(() => dayjs())
 }, (table) => [
@@ -211,7 +213,18 @@ export const jobs = sqliteTable("jobs", {
   completedAt: DayjsTimestamp('completedAt')
 });
 
-const playRelations = defineRelations({ plays, queueStates, playInputs, components, jobs, componentMigrations,playsHistorical }, (r) => ({
+export const playEvents = sqliteTable("play_events", {
+  id: integer({ mode: 'number' }).primaryKey(),
+  playId: integer().notNull().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}),
+  eventName: text({length: 50}).notNull(),
+  data: text({ mode: 'json' }),
+  error: ErrorLikeJson('error'),
+  createdAt: DayjsTimestamp('createdAt').notNull().$defaultFn(() => dayjs()),
+}, (table) => [
+  index('play_event_id_idx').on(table.playId)
+]);
+
+const playRelations = defineRelations({ plays, queueStates, playEvents, playInputs, components, jobs, componentMigrations,playsHistorical }, (r) => ({
   plays: {
     queueStates: r.many.queueStates(),
     input: r.one.playInputs({
@@ -233,7 +246,8 @@ const playRelations = defineRelations({ plays, queueStates, playInputs, componen
       from: r.plays.jobId,
       to: r.jobs.id,
       optional: true
-    })
+    }),
+    events: r.many.playEvents(),
   },
   playsHistorical: {
     component: r.one.components({
@@ -265,7 +279,13 @@ const playRelations = defineRelations({ plays, queueStates, playInputs, componen
   },
   jobs: {
     plays: r.many.plays()
-  }
+  },
+  playEvents: {
+    play: r.one.plays({
+      from: r.playEvents.playId,
+      to: r.plays.id
+    }),
+  },
 }));
 
 export const relations = playRelations;
@@ -280,6 +300,8 @@ export const getConfigByTableName = <T extends TableName>(name: T) => {
       return components;
     case 'playInputs':
       return playInputs;
+    case 'playEvents':
+      return playEvents;
     case 'queueStates':
       return queueStates;
     case 'componentMigrations':
@@ -289,7 +311,7 @@ export const getConfigByTableName = <T extends TableName>(name: T) => {
   }
 }
 
-export const schema = {playInputs, plays, components, componentMigrations, queueStates, jobs};
+export const schema = {playInputs, plays, playEvents, components, componentMigrations, queueStates, jobs};
 
 export type TSchema = typeof relations;
 export type Schema = typeof schema;

src/backend/common/errors/MSErrors.ts

@@ -33,16 +33,38 @@ export class ConnectionCheckError extends StageError {
 }
 addKnownErrorConstructor(ConnectionCheckError);
 
+export class AuthError extends NamedError {
+    override name = 'Authentication Error';
+    unrecoverable: boolean = false;
+
+    public constructor(msg?: string, opts: ErrorOptions & {unrecoverable?: boolean} = {}) {
+        super(msg, opts);
+        this.unrecoverable = opts.unrecoverable ?? false;
+    }
+}
+addKnownErrorConstructor(AuthError);
+export const isAuthError = (e: Error): e is AuthError => e.name === 'Authentication Error' || e instanceof AuthError;
 export class AuthCheckError extends StageError {
-    name = 'Authentication Check';
+    override name = 'Authentication Check';
+    unrecoverable?: boolean;
+
+    public constructor(msg?: string, opts: ErrorOptions & {unrecoverable?: boolean} = {}) {
+        super(msg, opts);
+        this.unrecoverable = opts.unrecoverable;
+    }
 }
 addKnownErrorConstructor(AuthCheckError);
+export const isAuthCheckError = (e: Error): e is AuthCheckError => e.name === 'Authentication Check' || e instanceof AuthCheckError;
 
 export class PostInitError extends StageError {
     name = 'Post Initialization';
 }
 addKnownErrorConstructor(PostInitError);
 
+export class StageChangeError extends NamedError {
+    override name = 'Stage Change';
+}
+
 const STACK_AT_REGEX = new RegExp(/[\n\r]\s*at/);
 
 export class SimpleError extends Error implements HasSimpleError {
@@ -200,3 +222,33 @@ export class InvalidRegexError extends SimpleError {
     }
 }
 addKnownErrorConstructor(InvalidRegexError, () => new InvalidRegexError(new RegExp(/1/)));
+
+export type AuthErrorMap = {
+    root: AuthError
+    check: AuthCheckError
+    auth: AuthError | AuthCheckError
+    parent: Error
+}
+
+export const findAuthIssue = <T extends keyof AuthErrorMap = 'auth'>(err: Error, opts: { type?: T, unrecoverable?: boolean } = {}): AuthErrorMap[T] | undefined => {
+    const {
+        type = 'auth',
+        unrecoverable
+    } = opts;
+    const authError = findCauseByFunc(err, (e) => {
+        if (['check','auth'].includes(type) && isAuthCheckError(e)) {
+            return unrecoverable === undefined || e.unrecoverable === unrecoverable;
+        }
+        if (['root','auth'].includes(type) && isAuthError(e)) {
+            return unrecoverable === undefined || e.unrecoverable === unrecoverable;
+        }
+        return false;
+    });
+    if (authError !== undefined) {
+        if (type === 'parent') {
+            return err as AuthErrorMap[T];
+        }
+        return authError as AuthErrorMap[T];
+    }
+    return undefined;
+}

src/backend/common/errors/PlayProcessingError.ts

@@ -0,0 +1,18 @@
+import { NamedError } from "./MSErrors.ts";
+import type { PlayProcessingResult } from "../infrastructure/PlayProcessing.ts";
+
+export interface PlayProcessingErrorData extends PlayProcessingResult {
+    showStopping: boolean
+}
+
+export class PlayProcessingError extends NamedError {
+    override name = 'Play Processing Error';
+    result: PlayProcessingResult
+    showStopping: boolean
+
+    constructor(error: Error, data: PlayProcessingErrorData, opts: ErrorOptions = {}) {
+        super('Error occured while processing Play', {...opts, cause: error});
+        this.result = data;
+        this.showStopping = data.showStopping;
+    }
+}

src/backend/common/infrastructure/MSBackendEventMap.ts

@@ -16,7 +16,7 @@ export interface MSBackendEventMap extends Omit<MSCoreEvents, 'playerUpdate'> {
         scrobbleTo: string[]
         }
     }>]
-    playerUpdate: [EmittedMSEvent<SourcePlayerObj<Dayjs> & { options: { scrobbleTo: string[] } },{},SourceType>]
+    playerUpdate: [EmittedMSEvent<SourcePlayerObj<Dayjs> & { options: { scrobbleTo: string[], wasMonitored?: boolean } },{},SourceType>]
     configError: [Error | string]
     instantiateError: [Error]
 }

src/backend/common/infrastructure/PlayProcessing.ts

@@ -0,0 +1,8 @@
+import type { PlayEvent } from "../../../core/PlayEvent.ts"
+import type { PlaySelectWithQueueStates, QueueStateSelect } from "../database/drizzle/drizzleTypes.ts"
+
+export interface PlayProcessingResult {
+    playEntity: PlaySelectWithQueueStates,
+    events: Omit<PlayEvent, 'playId'>[],
+    queue: QueueStateSelect
+}

src/backend/common/infrastructure/config/client/index.ts

@@ -1,6 +1,6 @@
 import * as z from "zod";
 import {playTransformOptionsSchema} from "../../../../../core/Transform.ts";
-import {commonConfigSchema, requestRetryOptionsSchema, monitorOptionsSchema, type CommonComponentEnvShape} from "../common.ts";
+import {commonConfigSchema, requestRetryOptionsSchema, monitorOptionsSchema, type CommonComponentEnvShape, deadLetterOptionsSchema} from "../common.ts";
 import {retentionConfigDurationValueSchema} from "../database.ts";
 import type { PipeUnwrapDirection } from "../../../../utils/ZodUtils.ts";
 
@@ -124,6 +124,7 @@ export const commonClientOptionsSchema = z.object({
     ...monitorOptionsSchema.shape,
     ...requestRetryOptionsSchema.shape,
     ...upstreamRefreshOptionsSchema.shape,
+    ...deadLetterOptionsSchema.shape,
 
     /**
      * Check client for an existing scrobble at the same recorded time as the "new" track to be scrobbled. If an existing scrobble is found this track is not track scrobbled.
@@ -143,17 +144,6 @@ export const commonClientOptionsSchema = z.object({
     }).optional().meta({
         description: "Options used for increasing verbosity of logging in MS (used for debugging)"
     }),
-    /**
-     * Number of times MS should automatically retry scrobbles in dead letter queue
-     *
-     * @default 3
-     * @examples [3]
-     * */
-    deadLetterRetries: z.number().optional().meta({
-        description: "Number of times MS should automatically retry scrobbles in dead letter queue",
-        default: 3,
-        examples: [3]
-    }),
 
     /** Enhance/correct Play data by applying a transform pipeline */
     playTransform: playTransformOptionsSchema.optional().meta({description: 'Enhance/correct Play data by applying a transform pipeline'}),

src/backend/common/infrastructure/config/common.ts

@@ -155,6 +155,21 @@ export const monitorOptionsSchema = z.object({
 })
 export type MonitorOptions = z.infer<typeof monitorOptionsSchema>;
 
+export const deadLetterOptionsSchema = z.object({
+    /**
+     * Number of times MS should automatically retry Plays in dead letter queue
+     *
+     * @default 3
+     * @examples [3]
+     * */
+    deadLetterRetries: z.number().optional().meta({
+        description: "Number of times MS should automatically retry Plays in dead letter queue",
+        default: 3,
+        examples: [3]
+    })
+})
+export type DeadLetterOptions = z.infer<typeof deadLetterOptionsSchema>;
+
 export type UnparsedConfig<T extends (SourceType | ClientType)> = {config: object, type: T, source?: 'file' | 'aio' | 'env', pos: string};
 
 export const generateConfigLocation = (configType: string, config: UnparsedConfig<any>): string => {

src/backend/common/infrastructure/config/source/deezer.ts

@@ -81,6 +81,14 @@ export type DeezerInternalData = z.infer<typeof deezerInternalDataSchema>;
 
 export const deezerInternalSourceOptionsSchema = z.object({
     ...commonSourceOptionsSchema.shape,
+    /**
+     * Set the heuristic level of duplicate/repeated track pruning from listening history.
+     * 
+     * Some third party integrations (Sonos) cause Deezer to report the same track many times in listening history
+     * 
+     * * `true` => detect plays with play times at exact, close to, or fuzzy (beginning/end of predicated play time + duration)
+     * * `aggressive` => above AND check for plays during previous play times that occurred during the track (50% +/- duration plus playedAt date)
+     */
     fuzzyDiscoveryIgnore: z.union([z.boolean(), z.literal('aggressive')]).optional(),
 });
 

src/backend/common/infrastructure/config/source/index.ts

@@ -1,5 +1,5 @@
 import * as z from "zod";
-import {requestRetryOptionsSchema, commonConfigSchema, monitorOptionsSchema} from "../common.ts";
+import {requestRetryOptionsSchema, commonConfigSchema, monitorOptionsSchema, deadLetterOptionsSchema} from "../common.ts";
 import {retentionConfigDurationValueSchema} from "../database.ts";
 import {playTransformOptionsSchema} from "../../../../../core/Transform.ts";
 import type { PipeUnwrapDirection } from "../../../../utils/ZodUtils.ts";
@@ -73,6 +73,7 @@ export const fileLogOptionsSchema = z.object({
 export const commonSourceOptionsSchema = z.object({
     ...monitorOptionsSchema.shape,
     ...sourceRetryOptionsSchema.shape,
+    ...deadLetterOptionsSchema.shape,
     /**
      * * If this source has INGRESS to MS (sends a payload, rather than MS GETTING requesting a payload) then setting this option to true will make MS log the payload JSON to DEBUG output
      * * If this source is POLLING then it will log the raw data for each unique track/response the first time it is seen

src/backend/common/transforms/AbstractTransformer.ts

@@ -1,5 +1,5 @@
 import { childLogger, type Logger } from "@foxxmd/logging";
-import type { PlayObject, TransformerCommon, TransformerCommonConfig } from "../../../core/Atomic.ts";
+import type { OptionalCacheUsage, PlayObject, TransformerCommon, TransformerCommonConfig } from "../../../core/Atomic.ts";
 import { isStageTyped, testWhenConditions } from "../../utils/PlayTransformUtils.ts";
 import AbstractInitializable from "../AbstractInitializable.ts";
 import type { StageConfig } from "../../../core/Transform.ts";
@@ -61,13 +61,16 @@ export default abstract class AbstractTransformer<T = any, Y extends StageConfig
 
     protected abstract doParseConfig(data: StageConfig): Y;
 
-    public async handle(data: Y, play: PlayObject): Promise<PlayObject> {
+    public async handle(data: Y, play: PlayObject, opts?: OptionalCacheUsage): Promise<PlayObject> {
+        const {
+            useCachedResult = true,
+        } = (opts ?? {});
         const cacheKey = `transformResult-${this.configHash}-${hashObject(data)}-${hashObject(playContentInvariantTransform(play))}`
         try {
-            const cachedTransformData = await this.cache.get<T>(cacheKey);
+            const cachedTransformData = useCachedResult ? await this.cache.get<T>(cacheKey) : undefined;
             if(cachedTransformData !== undefined) {
                 this.logger.debug('Transform cache hit');
-                const transformed = await this.doHandle(data, play, cachedTransformData);
+                const transformed = await this.doHandle(data, play, cachedTransformData, opts);
                 return transformed;
             }
         } catch (e) {
@@ -82,7 +85,7 @@ export default abstract class AbstractTransformer<T = any, Y extends StageConfig
         }
 
         try {
-            await this.handlePreFetch(play, data);
+            await this.handlePreFetch(play, data, opts);
         } catch (e) {
             if(e instanceof SkipTransformStageError) {
                 await this.cache.set(cacheKey, play, this.config.options?.ttl ?? '15s');
@@ -93,13 +96,13 @@ export default abstract class AbstractTransformer<T = any, Y extends StageConfig
         let transformData: T;
         let fetchedTransformData: any;
         try {
-            fetchedTransformData = await this.getTransformerData(play, data);
+            fetchedTransformData = await this.getTransformerData(play, data, opts);
         } catch (e) {
             throw new Error(`Could not fetch transformer data`, { cause: e });
         }
 
         try {
-            transformData = await this.handlePostFetch(play, fetchedTransformData, data);
+            transformData = await this.handlePostFetch(play, fetchedTransformData, data, opts);
         } catch (e) {
             if(e instanceof StagePrerequisiteError) {
                 await this.cache.set(cacheKey, play, this.config.options?.ttl ?? '15s');
@@ -107,22 +110,22 @@ export default abstract class AbstractTransformer<T = any, Y extends StageConfig
             throw new Error('postFetch did not pass', { cause: e });
         }
 
-        const transformed = await this.doHandle(data, play, transformData);
+        const transformed = await this.doHandle(data, play, transformData, opts);
         await this.cache.set(cacheKey, transformData, this.config.options?.ttl ?? '15s');
         return transformed;
     }
 
-    protected abstract doHandle(data: StageConfig, play: PlayObject, transformData: T): Promise<PlayObject>;
+    protected abstract doHandle(data: StageConfig, play: PlayObject, transformData: T, opts?: OptionalCacheUsage): Promise<PlayObject>;
 
-    public async getTransformerData(play: PlayObject, stageConfig: Y): Promise<any> {
+    public async getTransformerData(play: PlayObject, stageConfig: Y, opts?: OptionalCacheUsage): Promise<any> {
         return undefined;
     }
 
-    public async handlePostFetch(play: PlayObject, transformData: any, stageConfig: Y): Promise<T> {
+    public async handlePostFetch(play: PlayObject, transformData: any, stageConfig: Y, opts?: OptionalCacheUsage): Promise<T> {
         return transformData;
     }
 
-    public async handlePreFetch(play: PlayObject, stageConfig: Y): Promise<void> {
+    public async handlePreFetch(play: PlayObject, stageConfig: Y, opts?: OptionalCacheUsage): Promise<void> {
         return
     }
 }

src/backend/common/transforms/MusicbrainzTransformer.ts

@@ -1,4 +1,4 @@
-import { type ArtistCredit, asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, type LifecycleInput, type MBReleaseGroupPrimaryType, type MBReleaseGroupSecondaryType, type MBReleaseStatus, type MissingMbidType, type PlayObject, type TrackMeta, type TransformerCommon, type TransformOptions } from "../../../core/Atomic.ts";
+import { type ArtistCredit, asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, type LifecycleInput, type MBReleaseGroupPrimaryType, type MBReleaseGroupSecondaryType, type MBReleaseStatus, type MissingMbidType, type OptionalCacheUsage, type PlayObject, type TrackMeta, type TransformerCommon, type TransformOptions } from "../../../core/Atomic.ts";
 import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.ts";
 import type {WebhookPayload} from "../infrastructure/config/health/webhooks.ts";
 import type {ExternalMetadataTerm, PlayTransformMetadataStage} from "../../../core/Transform.ts";
@@ -429,7 +429,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
         }
     }
 
-    public async getTransformerData(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async getTransformerData(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts?: OptionalCacheUsage): Promise<IRecordingMSList> {
         
         const {
             // preserve order of search from before searchOrder
@@ -443,25 +443,25 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
             try {
                 switch(searchType) {
                     case 'isrc':
-                        results = await this.searchByIsrc(play, stageConfig);
+                        results = await this.searchByIsrc(play, stageConfig, opts);
                         break;
                     case 'album':
-                        results = await this.searchByAlbum(play, stageConfig);
+                        results = await this.searchByAlbum(play, stageConfig, opts);
                         break;
                     case 'artist':
-                        results = await this.searchByArtist(play, stageConfig);
+                        results = await this.searchByArtist(play, stageConfig, opts);
                         break;
                     case 'basic':
-                        results = await this.searchByBasicFields(play, stageConfig);
+                        results = await this.searchByBasicFields(play, stageConfig, opts);
                         break;
                     case 'freetext':
-                        results = await this.searchByFreeText(play, stageConfig);
+                        results = await this.searchByFreeText(play, stageConfig, opts);
                         break;
                     case 'basicorids':
-                        results = await this.searchByBasicFieldsOrMBIDs(play, stageConfig);
+                        results = await this.searchByBasicFieldsOrMBIDs(play, stageConfig, opts);
                         break;
                     case 'mbidrecording':
-                        results = await this.searchByRecordingMbid(play, stageConfig);
+                        results = await this.searchByRecordingMbid(play, stageConfig, opts);
                         break;
                 }
                 queries.push({type: `mbQuery-${searchType}${results.recordings.length === 0 ? '-empty' : ''}`, input: results.requestQuery});
@@ -486,12 +486,12 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
         return {...results, requestQueries: queries};
     }
 
-    public async searchByBasicFields(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByBasicFields(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         this.logger.debug({labels: ['Basic Search']}, 'Searching by artist/album/track');
-        return await this.api.searchByRecording(play);
+        return await this.api.searchByRecording(play, opts);
     }
 
-    public async searchByBasicFieldsOrMBIDs(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByBasicFieldsOrMBIDs(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         const using: UsingTypes[] = [];
         const {
             data: {
@@ -512,31 +512,31 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
         using.push((brainz.artist ?? []).length > 0 ? 'mbidartist' : 'artist');
 
         this.logger.debug({labels: ['Basic Or MBID Search']}, `Searching using ${using.join(', ')}}`);
-        return await this.api.searchByRecording(play, {using});
+        return await this.api.searchByRecording(play, {using, ...opts});
     }
 
-    public async searchByIsrc(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByIsrc(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         if(play.data.isrc !== undefined) {
             this.logger.debug({labels: ['ISRC Search']},'Searching with ISRC');
-            return await this.api.searchByRecording(play, {using: ['isrc']});
+            return await this.api.searchByRecording(play, {using: ['isrc'], ...opts});
         }
         throw new SearchPrerequisiteError('Play does not have ISRC');
     }
 
-    public async searchByRecordingMbid(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByRecordingMbid(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         if(play.data.meta?.brainz?.recording !== undefined) {
             this.logger.debug({labels: ['MBID Search']},'Searching with Recording MBID');
-            return await this.api.searchByRecording(play, {using: ['mbidrecording']});
+            return await this.api.searchByRecording(play, {using: ['mbidrecording'], ...opts});
         }
         throw new SearchPrerequisiteError('Play does not have recording MBID');
     }
 
-    public async searchByAlbum(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByAlbum(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         // possibly the artist is incorrect (may be combined as one string)
         // if we have an album we can likely still get a decent hit w/o using artist
         if(play.data.album !== undefined && play.data.artists !== undefined && play.data.artists.length > 0) {
             this.logger.debug({labels: ['Album Search']},'Searching with only track+album');
-            return await this.api.searchByRecording(play, {using: ['title','album']});
+            return await this.api.searchByRecording(play, {using: ['title','album'], ...opts});
         }
         if(play.data.album === undefined) {
             throw new SearchPrerequisiteError('Play does not have an album');
@@ -549,7 +549,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
         }
     }
 
-    public async searchByArtist(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByArtist(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         const {
             searchArtistMethod = this.defaults.searchArtistMethod,
         } = stageConfig;
@@ -582,7 +582,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
                     // so we split out all artists by all found delimiters
                     const nativePlay = nativeParse(play, {titleClean: true, delimiters: DELIMITERS});
                     this.logger.debug({labels: ['Parsed Artist Search']},'Searching with aggressive native parsing');
-                    return await this.api.searchByRecording(nativePlay, {using: ['title','artist']});
+                    return await this.api.searchByRecording(nativePlay, {using: ['title','artist'], ...opts});
                 }
             }
 
@@ -594,9 +594,9 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
             }
     }
 
-    public async searchByFreeText(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByFreeText(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         this.logger.debug({labels: ['Freetext Search']},'Trying freetext search');
-        const results = await this.api.searchByRecording(play, {freetext: true}) as IRecordingMSList;
+        const results = await this.api.searchByRecording(play, {freetext: true, ...opts}) as IRecordingMSList;
         results.freeText = true;
         return results;
     }

src/backend/common/transforms/TransformerManager.ts

@@ -1,6 +1,6 @@
 import { childLogger, type Logger } from "@foxxmd/logging";
 import type AbstractTransformer from "./AbstractTransformer.ts";
-import type {TransformerCommonConfig} from "../../../core/Atomic.ts";
+import type {OptionalCacheUsage, TransformerCommonConfig} from "../../../core/Atomic.ts";
 import UserTransformer from "./UserTransformer.ts";
 import type {StageConfig} from "../../../core/Transform.ts";
 import type {PlayObject} from "../../../core/Atomic.ts";
@@ -146,10 +146,10 @@ export default class TransformerManager {
         return config;
     }
 
-    public async handleStage(data: StageConfig, play: PlayObject, asyncId: string = nanoid(6)): Promise<[PlayObject, string]> {
+    public async handleStage(data: StageConfig, play: PlayObject, opts: {asyncId?: string} & OptionalCacheUsage): Promise<[PlayObject, string]> {
         const t: AbstractTransformer = this.getTransformerByStage(data);
         try {
-            const transformedPlay = await this.asyncStore.run(asyncId, async () => {
+            const transformedPlay = await this.asyncStore.run(opts.asyncId ?? nanoid(6), async () => {
                 return await t.handle(data, play);
             });
             return [transformedPlay, t.name];

src/backend/common/vendor/JRiverApiClient.ts

@@ -4,6 +4,7 @@ import xml2js from 'xml2js';
 import { type AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER } from "../infrastructure/Atomic.ts";
 import type {JRiverData} from "../infrastructure/config/source/jriver.ts";
 import AbstractApiClient from "./AbstractApiClient.ts";
+import { AuthError } from '../errors/MSErrors.ts';
 
 const parser = new xml2js.Parser({'async': true});
 
@@ -155,7 +156,7 @@ export class JRiverApiClient extends AbstractApiClient {
             if(this.config.username === undefined || this.config.password === undefined) {
                 msg = 'Authentication failed. No username/password was provided in config! Did you mean to do this?';
             }
-            throw new Error(msg, {cause: e});
+            throw new AuthError(msg, {cause: e});
         }
     }
 

src/backend/common/vendor/KodiApiClient.ts

@@ -9,6 +9,7 @@ import type {KodiData} from "../infrastructure/config/source/kodi.ts";
 import AbstractApiClient from "./AbstractApiClient.ts";
 import { baseFormatPlayObj } from "../../utils/PlayTransformUtils.ts";
 import { artistNamesToCredits } from "../../../core/StringUtils.ts";
+import { AuthError } from "../errors/MSErrors.ts";
 
 interface KodiDuration {
     hours: number
@@ -150,7 +151,7 @@ export class KodiApiClient extends AbstractApiClient {
             if(this.config.username === undefined || this.config.password === undefined) {
                 msg = 'Authentication failed. No username/password was provided in config! Did you mean to do this?';
             }
-            throw new Error(msg, {cause: e});
+            throw new AuthError(msg, {cause: e});
         }
     }
 

src/backend/common/vendor/LastfmApiClient.ts

@@ -13,14 +13,15 @@ import { type AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, type FormatPlayObjec
 import type {LastfmData} from "../infrastructure/config/client/lastfm.ts";
 import AbstractApiClient from "./AbstractApiClient.ts";
 import { normalizeStr, parseArtistCredits } from "../../utils/StringUtils.ts";
-import { LastFMUser, LastFMAuth, LastFMTrack, type LastFMUserGetRecentTracksResponse, type LastFMBooleanNumber, type LastFMUpdateNowPlayingResponse, type LastFMUserGetInfoResponse, type LastFMUserGetRecentTracksParams } from 'lastfm-ts-api';
+import { LastFMUser, LastFMAuth, LastFMTrack, type LastFMUserGetRecentTracksResponse, type LastFMBooleanNumber, type LastFMUpdateNowPlayingResponse, type LastFMUserGetInfoResponse, type LastFMUserGetRecentTracksParams, LastFMResponseError } from 'lastfm-ts-api';
 import clone from 'clone';
 import type { IncomingMessage } from "http";
 import { baseFormatPlayObj } from "../../utils/PlayTransformUtils.ts";
-import { ScrobbleSubmitError, SimpleError } from "../errors/MSErrors.ts";
+import { AuthError, ScrobbleSubmitError, SimpleError } from "../errors/MSErrors.ts";
 import { redactString } from "@foxxmd/redact-string";
 import dns from 'node:dns/promises';
 import xml2js from 'xml2js';
+import { findCauseByFunc } from "../../utils/ErrorUtils.ts";
 
 const badErrors = [
     'api key suspended',
@@ -30,6 +31,12 @@ const badErrors = [
     'invalid parameters'
 ];
 
+const unrecoverableAuthErrorCodes = [
+    10, // invalid api key
+    13, // invalid method signature
+    26, // api key suspended
+]
+
 const retryErrors = [
     'operation failed',
     'service offline',
@@ -247,7 +254,7 @@ export default class LastfmApiClient extends AbstractApiClient implements Pagina
         if (this.sessionKey === undefined) {
             this.logger.warn('No session key found. User interaction for authentication required.');
             this.logger.info(`Redirect URL that will be used on auth callback: '${this.redirectUri}'`);
-            throw new SimpleError('No session key found. User interaction for authentication required.');
+            throw new AuthError('No session key found. User interaction for authentication required.', {unrecoverable: false});
         }
         try {
             // existing lastfm clients are ok with getting user from getInfo
@@ -269,11 +276,32 @@ export default class LastfmApiClient extends AbstractApiClient implements Pagina
             this.logger.info(`Client authorized for user ${this.user}`)
             return true;
         } catch (e) {
+            if(e instanceof AuthError) {
+                throw e;
+            }
             this.logger.error('Testing auth failed');
             if(isNodeNetworkException(e)) {
                 this.logger.error(`Could not communicate with ${this.upstreamName} API`);
+                throw new AuthError('Testing auth failed', {cause: e, unrecoverable: false});
             }
-            throw e;
+            let unrecoverable: boolean;
+            if(e instanceof LastFMResponseError) {
+                try {
+                    const errorContent = JSON.parse(e.content);
+                    if(`error` in errorContent) {
+                        unrecoverable = unrecoverableAuthErrorCodes.includes(errorContent.error);
+                    }
+                } catch (e) {
+                    // swallow
+                }
+            }
+            if(unrecoverable === undefined) {
+                const errorWithMessage = findCauseByFunc(e, (ee) => `response` in ee) as Error & {response: IncomingMessage} | undefined;
+                if(errorWithMessage !== undefined) {
+                    unrecoverable = [401,403].includes(errorWithMessage.response.statusCode);
+                }
+            }
+            throw new AuthError('Testing auth failed', {cause: e, unrecoverable});
         }
     }
 

src/backend/common/vendor/ListenbrainzApiClient.ts

@@ -23,7 +23,7 @@ import { unique } from '../../utils.ts';
 import { removeUndefinedKeys } from '../../../core/DataUtils.ts';
 import type {ListenPayload, ListenResponse, ListenType, SubmitPayload} from '../../../core/vendor/listenbrainz/interfaces.ts';
 import { baseFormatPlayObj } from '../../utils/PlayTransformUtils.ts';
-import { ScrobbleSubmitError, SimpleError } from '../errors/MSErrors.ts';
+import { AuthError, ScrobbleSubmitError, SimpleError } from '../errors/MSErrors.ts';
 import pRetry from 'p-retry';
 import { findCauseByFunc } from '../../utils/ErrorUtils.ts';
 import { isSuperAgentResponseError } from '../errors/ErrorUtils.ts';
@@ -183,8 +183,12 @@ export class ListenbrainzApiClient extends AbstractApiClient implements Pageless
         try {
             const resp = await this.callApi(() => request.get(`${joinedUrl(this.url.url,'1/validate-token')}`));
             return true;
-        } catch (e) {
-            throw e;
+        } catch (err) {
+            const cause = findCauseByFunc<request.ResponseError>(err, (e) => isSuperAgentResponseError(e));
+            if(cause !== undefined && [401,403,400].includes(cause.status)) {
+                throw new AuthError('Failed to validate token', {cause: err, unrecoverable: true});
+            }
+            throw new AuthError('Failed to validate token due to non-auth error', {cause: err, unrecoverable: false});
         }
     }
 

src/backend/common/vendor/RockSkyApiClient.ts

@@ -14,7 +14,7 @@ import type {RockskyScrobble} from './rocksky/interfaces.ts';
 import type {Handle} from "@atcute/lexicons";
 import { getATProtoIdentifier, identifierToAtProtoHandle } from './atproto/atUtils.ts';
 import { baseFormatPlayObj } from "../../utils/PlayTransformUtils.ts";
-import { ScrobbleSubmitError } from "../errors/MSErrors.ts";
+import { AuthError, ScrobbleSubmitError } from "../errors/MSErrors.ts";
 import { tryApiCall } from "../../utils/RequestUtils.ts";
 import { type CreateScrobbleInput, RockskyClient } from "@rocksky/sdk";
 import { getRoot } from "../../ioc.ts";
@@ -23,6 +23,8 @@ import type {HandleData} from "../infrastructure/config/client/atproto.ts";
 import { parseRegexSingle } from "@foxxmd/regex-buddy-core";
 import { removeUndefinedKeys } from "../../../core/DataUtils.ts";
 import { isrcNoHyphens } from '../../../core/PlayUtils.ts';
+import { findCauseByFunc } from "../../utils/ErrorUtils.ts";
+import { isSuperAgentResponseError } from "../errors/ErrorUtils.ts";
 
 interface SubmitOptions {
     log?: boolean
@@ -166,7 +168,8 @@ export class RockSkyApiClient extends AbstractApiClient {
                 const resp = await this.callLZApi(() => request.get(`${joinedUrl(this.lzUrl.url,'1/validate-token')}`));
                 return true;
             } catch (e) {
-                throw e;
+                const cause = findCauseByFunc<request.ResponseError>(e, (ee) => isSuperAgentResponseError(ee));
+                throw new AuthError('Failed to validate token', {cause: e, unrecoverable: cause !== undefined && [401,403].includes(cause.status)});
             }
         } else {
             try {
@@ -174,7 +177,9 @@ export class RockSkyApiClient extends AbstractApiClient {
                 await req;
                 return true;
             } catch (e) {
-                throw new UpstreamError('Failed to get /profile with given token', {cause: e});
+                const upstreamErr = new UpstreamError('Failed to get /profile with given token', {cause: e});
+                const cause = findCauseByFunc<request.ResponseError>(e, (ee) => isSuperAgentResponseError(ee));
+                throw new AuthError('Failed to get /profile with given token', {cause: upstreamErr, unrecoverable: cause !== undefined && [401,403].includes(cause.status)});
             }
         }
     }

src/backend/common/vendor/koito/KoitoApiClient.ts

@@ -11,10 +11,12 @@ import { playToListenPayload } from '../listenbrainz/lzUtils.ts';
 import type {SubmitPayload} from '../../../../core/vendor/listenbrainz/interfaces.ts';
 import type {ListenType} from '../../../../core/vendor/listenbrainz/interfaces.ts';
 import { baseFormatPlayObj } from "../../../utils/PlayTransformUtils.ts";
-import { ScrobbleSubmitError } from "../../errors/MSErrors.ts";
+import { AuthError, ScrobbleSubmitError } from "../../errors/MSErrors.ts";
 import { tryApiCall } from "../../../utils/RequestUtils.ts";
 import { parseRegexSingle } from "@foxxmd/regex-buddy-core";
 import { artistNamesToCredits } from "../../../../core/StringUtils.ts";
+import { findCauseByFunc } from "../../../utils/ErrorUtils.ts";
+import { isSuperAgentResponseError } from "../../errors/ErrorUtils.ts";
 
 interface SubmitOptions {
     log?: boolean
@@ -133,7 +135,8 @@ export class KoitoApiClient extends AbstractApiClient implements PaginatedTimeRa
             const resp = await this.callApi(() => request.get(`${joinedUrl(this.url.url, '/apis/listenbrainz/1/validate-token')}`));
             return true;
         } catch (e) {
-            throw new Error('Could not validate Koito API Key', { cause: e });
+            const superagentError = findCauseByFunc<request.ResponseError>(e, (ee) => isSuperAgentResponseError(ee));
+            throw new AuthError('Could not validate Koito API Key', { cause: e, unrecoverable: superagentError !== undefined && [401,403].includes(superagentError.status)});
         }
     }
 

src/backend/common/vendor/maloja/MalojaApiClient.ts

@@ -16,8 +16,9 @@ import { getMalojaResponseError, isMalojaAPIErrorBody, type MalojaResponseV3Comm
 import { getScrobbleTsSOCDate, getScrobbleTsSOCDateWithContext } from '../../../utils/TimeUtils.ts';
 import { artistCreditsToNames, artistNamesToCredits, buildTrackString } from '../../../../core/StringUtils.ts';
 import { baseFormatPlayObj } from '../../../utils/PlayTransformUtils.ts';
-import { ScrobbleSubmitError } from '../../errors/MSErrors.ts';
+import { AuthError, ScrobbleSubmitError } from '../../errors/MSErrors.ts';
 import { NO_RETRY_HTTP_STATUS, tryApiCall } from '../../../utils/RequestUtils.ts';
+import { findCauseByFunc } from '../../../utils/ErrorUtils.ts';
 
 
 
@@ -175,10 +176,11 @@ export class MalojaApiClient extends AbstractApiClient implements PaginatedTimeR
                     body,
                     text: text.slice(0, 50)
                 },'Maloja API Response');
-                throw new Error('Server Response body was malformed -- should have returned "status: ok"...is the URL correct?', { cause: new Error(`Maloja API Response was ${status}: ${text.slice(0, 50)}`) })
+                throw new UpstreamError('Server Response body was malformed -- should have returned "status: ok"...is the URL correct?', { cause: new UpstreamError(`Maloja API Response was ${status}: ${text.slice(0, 50)}`) })
             }
         } catch (e) {
-            throw e;
+            const superagentError = findCauseByFunc<request.ResponseError>(e, (ee) => isSuperAgentResponseError(ee));
+            throw new AuthError('Failed to test Maloja API with apikey', {cause: e, unrecoverable: superagentError !== undefined && [401,403].includes(superagentError.status)});
         }
     }
 

src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts

@@ -1,5 +1,5 @@
 import type { Response } from 'superagent';
-import type {ArtistCredit, PlayObject, PlayObjectMinimal, URLData} from "../../../../core/Atomic.ts";
+import type {ArtistCredit, OptionalCacheUsage, PlayObject, PlayObjectMinimal, URLData} from "../../../../core/Atomic.ts";
 import { UpstreamError } from "../../errors/UpstreamError.ts";
 import { type AbstractApiOptions, type FormatPlayObjectOptions, MUSICBRAINZ_URL, type MusicbrainzApiConfigData } from "../../infrastructure/Atomic.ts";
 import AbstractApiClient from "../AbstractApiClient.ts";
@@ -114,17 +114,18 @@ export class MusicbrainzApiClient extends AbstractApiClient {
         return 'API';
     }
 
-    callApi = async <T = Response>(func: (mb: MusicBrainzApi) => Promise<any>, options?: { timeout?: number, cacheKey?: string }): Promise<T> => {
+    callApi = async <T = Response>(func: (mb: MusicBrainzApi) => Promise<any>, options?: { timeout?: number, cacheKey?: string } & OptionalCacheUsage): Promise<T> => {
 
         let apiConfig = this.rrApis.next().value;
 
         const {
             timeout = 30000,
-            cacheKey
+            cacheKey,
+            useCachedResult = true
         } = options || {};
 
         try {
-            const cachedTransform = await this.cache.get<T>(cacheKey);
+            const cachedTransform = useCachedResult ? await this.cache.get<T>(cacheKey) : undefined;
             if(cachedTransform !== undefined) {
                 const cacheUrl = await this.cache.get<string>(`${cacheKey}-url`);
                 const cacheQs = await this.cache.get<string>(`${cacheKey}-qs`);
@@ -218,13 +219,14 @@ export class MusicbrainzApiClient extends AbstractApiClient {
         }
     }
 
-    searchByRecording = async(play: PlayObject, options?: SearchOptions): Promise<IRecordingMSList | undefined> => {
+    searchByRecording = async(play: PlayObject, options?: SearchOptions & OptionalCacheUsage): Promise<IRecordingMSList | undefined> => {
 
         const {
             escapeCharacters = true,
             removeCharacters = false,
             using = ['album','artist','title'],
-            freetext
+            freetext,
+            useCachedResult
         } = options || {};
 
         const cacheKey = `mb-recSearch-${hashObject({...playContentInvariantTransform(play), using})}`;
@@ -347,7 +349,8 @@ export class MusicbrainzApiClient extends AbstractApiClient {
                 query: q
             });
         }, {
-            cacheKey
+            cacheKey,
+            useCachedResult
         });
 
         if(res === undefined) {

src/backend/common/vendor/teal/TealApiClient.ts

@@ -1,5 +1,5 @@
 import dayjs, { type Dayjs, type ManipulateType } from "dayjs";
-import type {PlayObject, PlayObjectMinimal, BrainzMeta, MBID, ScrobbleActionResult} from "../../../../core/Atomic.ts";
+import {type PlayObject, type PlayObjectMinimal, type BrainzMeta, type MBID, type ScrobbleActionResult, PARSED_FROM} from "../../../../core/Atomic.ts";
 import { getRoot } from "../../../ioc.ts";
 import { removeUndefinedKeys } from '../../../../core/DataUtils.ts';
 import { baseFormatPlayObj } from "../../../utils/PlayTransformUtils.ts";
@@ -159,7 +159,7 @@ export const recordToPlay = (record: TealPlayRecord, options: RecordOptions = {}
         },
         meta: {
             source: 'tealfm',
-            parsedFrom: 'history',
+            parsedFrom: PARSED_FROM.history,
             musicService,
             playId: options.playId,
             url: {

src/backend/index.ts

@@ -209,9 +209,9 @@ const dataDir = getDataDir();
         if(nameColl.length > 0) {
             logger.warn(`Last.FM source and clients have same names [${nameColl.map(x => x.name).join(',')}] -- this may cause issues`);
         }
-        const clientInitOptions = {deadDelay: nonEmptyStringOrDefault(process.env.DEBUG_DEAD_DELAY, undefined) !== undefined ? Number.parseInt(process.env.DEBUG_DEAD_DELAY) : undefined};
+        const initOptions = {deadDelay: nonEmptyStringOrDefault(process.env.DEBUG_DEAD_DELAY, undefined) !== undefined ? Number.parseInt(process.env.DEBUG_DEAD_DELAY) : undefined};
         for(const c of scrobbleClients.clients) {
-            c.initTasks(clientInitOptions);
+            c.initTasks(initOptions);
             const res = await Promise.race([
                 sleep(2200),
                 (async () => {
@@ -227,7 +227,7 @@ const dataDir = getDataDir();
         }
 
         for(const c of scrobbleSources.sources) {
-            c.initTasks();
+            c.initTasks(initOptions);
             const res = await Promise.race([
                 sleep(2200),
                 (async () => {

src/backend/ioc.ts

@@ -35,11 +35,21 @@ const discovered = new prom.Counter({
             help: 'Number of discovered plays for a Source',
             labelNames: ['name', 'type']
 });
+const sourceDead = new prom.Gauge({
+            name: 'multiscrobbler_source_dead',
+            help: 'Number of dead letter plays for a Source',
+            labelNames: ['name', 'type']
+});
 const queuedGauge = new prom.Gauge({
             name: 'multiscrobbler_client_queued',
             help: 'Number of queued plays for a Client',
             labelNames: ['name', 'type']
         });
+const queuedSourceGauge = new prom.Gauge({
+    name: 'multiscrobbler_source_queued',
+    help: 'Number of queued plays for a Source',
+    labelNames: ['name', 'type']
+});
 const deadLetterGauge = new prom.Gauge({
             name: 'multiscrobbler_client_deadletter',
             help: 'Number of deadletter plays for a Client',
@@ -144,6 +154,8 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => {
         loggingConfig,
         sourceMetics: {
             discovered: discovered,
+            queued: queuedSourceGauge,
+            deadLetter: sourceDead
             //issues: sourceIssues
         },
         clientMetrics: {

src/backend/scrobblers/DiscordScrobbler.ts

@@ -1,8 +1,8 @@
 import type {Logger, LogLevel} from "@foxxmd/logging";
 import type EventEmitter from "events";
-import type {PlayMatchResult, PlayObject, SourcePlayerObj} from "../../core/Atomic.ts";
+import type {ComponentAuthType, PlayMatchResult, PlayObject, SourcePlayerObj} from "../../core/Atomic.ts";
 import { type FormatPlayObjectOptions } from "../common/infrastructure/Atomic.ts";
-import { SINGLE_USER_PLATFORM_ID_STR } from '../../core/Atomic.ts';
+import { COMPONENT_AUTH_TYPE, SINGLE_USER_PLATFORM_ID_STR } from '../../core/Atomic.ts';
 import { CALCULATED_PLAYER_STATUSES } from '../../core/Atomic.ts';
 import { REPORTED_PLAYER_STATUSES } from '../../core/Atomic.ts';
 
@@ -18,6 +18,7 @@ import dayjs from "dayjs";
 export default class DiscordScrobbler extends AbstractScrobbleClient {
 
     api: DiscordWSClient | DiscordIPCClient;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = false;
     override nowPlayingIsRealtime: boolean = true;
@@ -71,7 +72,11 @@ export default class DiscordScrobbler extends AbstractScrobbleClient {
                 this.authed = false;
                 this.connectionOK = false;
             }
-            await this.tryStopScrobbling();
+            try {
+                await this.tryStopScrobbling('API emitted stop event');
+            } catch (e) {
+                this.logger.warn(new Error('Failed to stop scrobbling loop after emitter stopped', {cause: e}));
+            }
         });
 
         if(typeof artwork === 'boolean') {
@@ -119,7 +124,7 @@ export default class DiscordScrobbler extends AbstractScrobbleClient {
         }
     }
 
-    queueScrobble = async (data: PlayObject | PlayObject[], source: string) => {
+    queueScrobble = async (data: PlayObject | PlayObject[]) => {
         // discord does not handle scrobbles, only Now Playing
         // so don't bother queueing any scrobbles as we don't want to cache them
         // or give the user the impression they are used (in UI as a number of queued scrobbles)

src/backend/scrobblers/KoitoScrobbler.ts

@@ -1,6 +1,6 @@
 import type {Logger} from "@foxxmd/logging";
 import type EventEmitter from "events";
-import type {PlayObject, SourcePlayerObj} from "../../core/Atomic.ts";
+import {COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, type SourcePlayerObj} from "../../core/Atomic.ts";
 import { buildTrackString, capitalize } from "../../core/StringUtils.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import type {FormatPlayObjectOptions, TimeRangeListensFetcher} from "../common/infrastructure/Atomic.ts";
@@ -15,6 +15,7 @@ import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.ts
 export default class KoitoScrobbler extends AbstractScrobbleClient {
 
     api: KoitoApiClient;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = false;
     getScrobblesForTimeRange: TimeRangeListensFetcher

src/backend/scrobblers/LastfmScrobbler.ts

@@ -1,6 +1,6 @@
 import type {Logger} from "@foxxmd/logging";
 import type EventEmitter from "events";
-import type {PlayObject, SourcePlayerObj} from "../../core/Atomic.ts";
+import {COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, type SourcePlayerObj} from "../../core/Atomic.ts";
 import { buildTrackString, capitalize } from "../../core/StringUtils.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import type {FormatPlayObjectOptions, InternalConfigOptional, TimeRangeListensFetcher} from "../common/infrastructure/Atomic.ts";
@@ -13,6 +13,7 @@ import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.ts
 export default class LastfmScrobbler extends AbstractScrobbleClient {
 
     api: LastfmApiClient;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.interactive;
     requiresAuth = true;
     requiresAuthInteraction = true;
     upstreamType: string = 'Last.fm';

src/backend/scrobblers/ListenbrainzScrobbler.ts

@@ -1,6 +1,6 @@
 import type {Logger} from "@foxxmd/logging";
 import type EventEmitter from "events";
-import type {PlayObject, SourcePlayerObj} from "../../core/Atomic.ts";
+import {COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, type SourcePlayerObj} from "../../core/Atomic.ts";
 import { buildTrackString, capitalize } from "../../core/StringUtils.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import type {FormatPlayObjectOptions, TimeRangeListensFetcher} from "../common/infrastructure/Atomic.ts";
@@ -16,6 +16,7 @@ import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.ts
 export default class ListenbrainzScrobbler extends AbstractScrobbleClient {
 
     api: ListenbrainzApiClient;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = false;
     getScrobblesForTimeRange: TimeRangeListensFetcher

src/backend/scrobblers/MalojaScrobbler.ts

@@ -1,7 +1,7 @@
 import { childLogger, type Logger } from "@foxxmd/logging";
 import type EventEmitter from "events";
 import normalizeUrl from "normalize-url";
-import type {PlayObject} from "../../core/Atomic.ts";
+import {COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject} from "../../core/Atomic.ts";
 import { buildTrackString, capitalize } from "../../core/StringUtils.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import type {FormatPlayObjectOptions, TimeRangeListensFetcher} from "../common/infrastructure/Atomic.ts";
@@ -15,6 +15,7 @@ const feat = ["ft.", "ft", "feat.", "feat", "featuring", "Ft.", "Ft", "Feat.", "
 
 export default class MalojaScrobbler extends AbstractScrobbleClient {
 
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     serverVersion: any;
     webUrl: string;

src/backend/scrobblers/RockskyScrobbler.ts

@@ -1,6 +1,6 @@
 import { childLogger, type Logger } from "@foxxmd/logging";
 import type EventEmitter from "events";
-import type {PlayObject, SourcePlayerObj} from "../../core/Atomic.ts";
+import {COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, type SourcePlayerObj} from "../../core/Atomic.ts";
 import { buildTrackString, capitalize } from "../../core/StringUtils.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import type {FormatPlayObjectOptions, InternalConfigOptional} from "../common/infrastructure/Atomic.ts";
@@ -27,6 +27,7 @@ import { isAbortError } from "abort-controller-x";
 export default class RockskyScrobbler extends AbstractHistoricalScrobbleClient {
 
     api: RockSkyApiClient;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = false;
 

src/backend/scrobblers/ScrobbleClients.ts

@@ -57,9 +57,11 @@ export default class ScrobbleClients {
         }
 
         this.sourceEmitter.on('playerUpdate', async (payload) => {
-            // agressively update Now Playing so scrobblers that display based on duration are mostly synced
-            // but aggressively *stop* updating if state becomes stale/orphaned
-            this.playingNow(payload.data, {...payload.data.options, scrobbleFrom: { type: payload.type, name: payload.name}});
+            if(payload.data.options.wasMonitored) {
+                // agressively update Now Playing so scrobblers that display based on duration are mostly synced
+                // but aggressively *stop* updating if state becomes stale/orphaned
+                this.playingNow(payload.data, {...payload.data.options, scrobbleFrom: { type: payload.type, name: payload.name}});
+            }
         });
 
         this.sourceEmitter.on('discoveredToScrobble', async (payload) => {
@@ -451,7 +453,7 @@ export default class ScrobbleClients {
                 }
             }
             for (const playObj of playObjs) {
-                await client.queueScrobble(clone(playObj), scrobbleFrom);
+                await client.queueScrobble(clone(playObj));
             }
         }
         if(excluded.length > 0) {

src/backend/scrobblers/TealfmScrobbler.ts

@@ -5,7 +5,7 @@ import fs from 'node:fs';
 import path from 'path';
 
 import { Readable } from 'stream';
-import type {PlayObject, SourcePlayerObj} from "../../core/Atomic.ts";
+import {COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, type SourcePlayerObj} from "../../core/Atomic.ts";
 import { buildTrackString, capitalize } from "../../core/StringUtils.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import type {FormatPlayObjectOptions, InternalConfigOptional} from "../common/infrastructure/Atomic.ts";
@@ -25,9 +25,13 @@ import { fromStream } from '@atcute/repo';
 import { playToRepositoryCreatePlayHistoricalOpts, type RepositoryCreatePlayHistoricalOpts } from "../common/database/drizzle/repositories/PlayHistoricalRepository.ts";
 import { isAbortError } from "abort-controller-x";
 import type { FmTealAlphaFeedPlay, FmTealFeedPlay } from "../common/vendor/teal/lexicons/index.ts";
+import { AuthError } from "../common/errors/MSErrors.ts";
+import { findCauseByReference } from "../utils/ErrorUtils.ts";
+import { ClientResponseError } from "@atcute/client";
 
 export default class TealScrobbler extends AbstractHistoricalScrobbleClient {
 
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = false;
     override nowPlayingIsRealtime: boolean = true;
@@ -97,10 +101,14 @@ export default class TealScrobbler extends AbstractHistoricalScrobbleClient {
                 return res;
             }
         } catch (e) {
-            if(isNodeNetworkException(e)) {
+            const nodeNetError = isNodeNetworkException(e);
+            if(nodeNetError) {
                 this.logger.error('Could not communicate with ATProto API');
+                throw new AuthError(`Failed to validate session due to network issues`, {cause: e, unrecoverable: false});
             }
-            throw e;
+            const clientError = findCauseByReference(e, ClientResponseError);
+            const authIssue = clientError !== undefined && [401,403].includes(clientError.status);
+            throw new AuthError(`Failed to validate session${!authIssue ? ' due to network issues' : ''}`, {cause: e, unrecoverable: authIssue});
         }
     }
 

src/backend/server/api.ts

@@ -6,7 +6,7 @@ import { FixedSizeList } from 'fixed-size-list';
 import type { PassThrough } from "node:stream";
 import { Transform } from "stream";
 import {
-    CLIENT_DEAD_QUEUE,
+    DEAD_QUEUE,
     type ClientStatusData,
     type DeadLetterScrobble,
     type LogOutputConfig,
@@ -17,12 +17,13 @@ import {
     type SOURCE_SOT_TYPES,
     type SourcePlayerJson,
     type SourceStatusData,
+    queueContextSchema,
 } from "../../core/Atomic.ts";
 import { capitalize } from "../../core/StringUtils.ts";
 import type {ExpressHandler, LeveledLogData} from "../common/infrastructure/Atomic.ts";
 import { getRoot } from "../ioc.ts";
 import AbstractScrobbleClient from "../scrobblers/AbstractScrobbleClient.ts";
-import type AbstractSource from "../sources/AbstractSource.ts";
+import AbstractSource from "../sources/AbstractSource.ts";
 import MemorySource from "../sources/MemorySource.ts";
 import { parseBool } from "../utils.ts";
 import { sortByNewestPlayDate } from '../../core/PlayUtils.ts';
@@ -30,24 +31,30 @@ import { setupAuthRoutes } from "./auth.ts";
 import { setupDeezerRoutes } from "./deezerRoutes.ts";
 import {setupLZEndpointRoutes} from "./endpointListenbrainzRoutes.ts";
 import {setupLastfmEndpointRoutes} from "./endpointLastfmRoutes.ts";
-import { type ComponentAwareRequest, makeClientCheckMiddle, makeClientNextMiddle, makeComponentMiddle, makeSourceCheckMiddle, makeSourceNextMiddle, type SourceAwareRequest } from "./middleware.ts";
+import { makeClientCheckMiddle, makeClientNextMiddle, makeComponentMiddle, makeSourceCheckMiddle, makeSourceNextMiddle } from "./middleware.ts";
 import { setupWebscrobblerRoutes } from "./webscrobblerRoutes.ts";
 import type ScrobbleSources from "../sources/ScrobbleSources.ts";
 import type ScrobbleClients from "../scrobblers/ScrobbleClients.ts";
 import prom from 'prom-client';
-import { SimpleError } from "../common/errors/MSErrors.ts";
+import { findAuthIssue, SimpleError } from "../common/errors/MSErrors.ts";
 import { DrizzlePlayRepository, type QueryPlaysOpts, type QueryPlaysOptsJson } from "../common/database/drizzle/repositories/PlayRepository.ts";
 import { playSelectToDeadScrobble } from "../common/database/drizzle/entityUtils.ts";
 import AbstractHistoricalScrobbleClient from "../scrobblers/AbstractHistoricalScrobbleClient.ts";
 import { DrizzlePlayHistoricalRepository } from "../common/database/drizzle/repositories/PlayHistoricalRepository.ts";
-import type {ComponentClientApiJson, ComponentSourceApiJson} from "../../core/Api.ts";
+import {componentStateBodySchema, type ComponentClientApiJson, type ComponentSourceApiJson, type PlayApiCommonDetailed} from "../../core/Api.ts";
 import { asDayjsHydratedObject } from "../../core/DataUtils.ts";
 import type {Dayjs} from "dayjs";
 import { asSerializablePlaySelect } from "../../core/PlayMarshalUtils.ts";
+import { serializeError } from "serialize-error";
+import { z } from 'zod';
+import type { createTypedRouter } from "@minisylar/express-typed-router";
+import pEvent from "p-event";
 
 const maxBufferSize = 300;
 const output: Record<number, FixedSizeList<LogDataPretty>> =  {};
 
+const jsonParser = bodyParser.json({type: ['text/*', 'application/json']});
+
 const createAddToLogBuffer = (levelMap:  {[p: number]: string}) => (log: LogDataPretty) => {
     output[log.level].add({...log, levelLabel: levelMap[log.level]});
 }
@@ -65,7 +72,7 @@ const getLogs = (minLevel: number, limit: number = maxBufferSize, sort: 'asc' |
     return allLogs.flat(1).sort((a, b) => a.time - b.time).slice(0, limit);
 }
 
-export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThrough, initialLogOutput: LogDataPretty[] = [], scrobbleSources: ScrobbleSources, scrobbleClients: ScrobbleClients) => {
+export const setupApi = (app: Express, router: ReturnType<typeof createTypedRouter>, logger: Logger, appLoggerStream: PassThrough, initialLogOutput: LogDataPretty[] = [], scrobbleSources: ScrobbleSources, scrobbleClients: ScrobbleClients) => {
     for(const level of Object.keys(logger.levels.labels)) {
         output[level] = new FixedSizeList<LeveledLogData>(maxBufferSize);
     }
@@ -149,7 +156,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         return res.json({data: slicedLog, settings: logConfig});
     });
 
-    app.get('/api/events', async (req, res) => {
+    router.get('/events', async (req, res) => {
         const {
             query: {
                 next: nextQs
@@ -160,7 +167,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
 
         const session = await bsseDef.createSession(req, res);
         scrobbleSources.emitter.onAny((eventName: string, payload: any) => {
-            if(payload.from !== undefined) {
+            if(payload !== undefined && payload.from !== undefined) {
                 if(isNextapi) {
                     session.push({event: eventName, ...payload}, eventName);
                 } else {
@@ -169,7 +176,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
             }
         });
         scrobbleClients.emitter.onAny((eventName: string, payload: any) => {
-            if(payload.from !== undefined) {
+            if(payload !== undefined && payload.from !== undefined) {
                 if(isNextapi) {
                     session.push({event: eventName, ...payload}, eventName);
                 } else {
@@ -183,19 +190,19 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
     setupWebscrobblerRoutes(app, logger, scrobbleSources);
     setupLZEndpointRoutes(app, logger, scrobbleSources, scrobbleClients);
     setupLastfmEndpointRoutes(app, logger, scrobbleSources);
-    setupAuthRoutes(app, logger, sourceRequiredMiddle, clientRequiredMiddle, scrobbleSources, scrobbleClients);
+    setupAuthRoutes(app, router, logger, sourceRequiredMiddle, clientRequiredMiddle, scrobbleSources, scrobbleClients);
 
-    app.put('/api/webscrobbler', bodyParser.json({type: ['text/*', 'application/json']}), async (req, res) => {
+    router.put('webscrobbler', {middleware: [jsonParser]}, async (req, res) => {
         logger.info(req.body);
         res.sendStatus(200);
     });
 
-    app.get('/api/webscrobbler', bodyParser.json({type: ['text/*', 'application/json']}), async (req, res) => {
+    router.get('webscrobbler', {middleware: [jsonParser]}, async (req, res) => {
         logger.info(req.body);
         res.sendStatus(200);
     });
 
-    app.get('/api/components', async (req, res, next) => {
+    router.get('/components', async (req, res, next) => {
 
         const sourceData = scrobbleSources.sources.filter(x => x.databaseOK).map((x) => {
             const {
@@ -255,13 +262,13 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         return res.json([...sourceData, ...clientData]);
     });
 
-    app.get('/api/sources/:componentVal/players', sourceAwareMiddle, async (req: SourceAwareRequest, res, next) => {
+    router.get('/sources/:componentVal/players', {middleware: [sourceAwareMiddle] }, async (req, res, next) => {
         if(req.component instanceof MemorySource) {
             return res.json(req.component.playersToObject());
         }
         return res.json({});
     });
-    app.get('/api/components/:componentVal/players', componentAwareMiddle, async (req: ComponentAwareRequest, res, next) => {
+    router.get('/components/:componentVal/players', {middleware: [componentAwareMiddle]}, async (req, res, next) => {
         if(req.component instanceof MemorySource) {
             return res.json(req.component.playersToObject());
         } else if(req.component instanceof AbstractScrobbleClient && req.component.nowPlayingEnabled) {
@@ -270,7 +277,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         return res.json({});
     });
 
-    app.get('/api/sources/:componentVal/players/:platformId', sourceAwareMiddle, async (req: SourceAwareRequest, res, next) => {
+    router.get('/sources/:componentVal/players/:platformId', {middleware: [sourceAwareMiddle]}, async (req, res, next) => {
         if(req.component instanceof MemorySource) {
             const {
                 params: {
@@ -285,7 +292,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         }
         return res.json({});
     });
-    app.get('/api/components/:componentVal/players/:platformId', componentAwareMiddle, async (req: ComponentAwareRequest, res, next) => {
+    router.get('/components/:componentVal/players/:platformId', {middleware: [componentAwareMiddle]}, async (req, res, next) => {
         const {
             params: {
                 platformId
@@ -308,14 +315,86 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         return res.status(400).json({error: `Component does not support players`});
     });
 
-    app.get('/api/components/:componentVal', componentAwareMiddle, async (req: ComponentAwareRequest, res, next) => {
+    router.get('/components/:componentVal', {middleware: [componentAwareMiddle]}, async (req, res) => {
         const {
             component,
         } = req;
         return res.json(component.getApiData());
     });
+    //type TA = TypedMiddleware<{component: ComponentAwareRequest['component']}>;
+// componentAwareMiddle, bodyParser.json({ type: ['text/*', 'application/json'] })
+    router.post('/components/:componentVal/state',
+   // {bodySchema: componentStateBodySchema},
+   {middleware: [componentAwareMiddle, bodyParser.json({ type: ['text/*', 'application/json'] })], bodySchema: componentStateBodySchema},
+     async (req, res) => {
+        const {
+            component,
+            body: {
+                state,
+                reason = 'invoked by api'
+            }
+        } = req;
+        switch (state) {
+            case 'stop':
+                try {
+                    await component.stop({ reason: new SimpleError(reason, {simple: true, shortStack: true}) })
+                } catch (e) {
+                    return res.status(500).json({ error: serializeError(e) });
+                }
+                break;
+            case 'start':
+                try {
+                    await component.start({ forceInit: true })
+                } catch (e) {
+                    return res.status(500).json({ error: serializeError(e) });
+                }
+                break;
+            case 'restart':
+                try {
+                    await component.restart({ forceInit: true, reason: new SimpleError(reason, {simple: true, shortStack: true}) })
+                } catch (e) {
+                    return res.status(500).json({ error: serializeError(e) });
+                }
+                break;
+            case 'ignore':
+                component.monitoringActivity = component.getSystemMonitoring() === false ? undefined : false;
+                component.emitComponentUpdate({state: component.getRunningState()});
+                break;
+            case 'monitor':
+                component.monitoringActivity = component.getSystemMonitoring() === true ? undefined : true;
+                component.emitComponentUpdate({state: component.getRunningState()});
+                break;
+            default:
+                return res.status(400).json({ error: { message: `'state' type ${state} was not handled` } });
+        }
+        return res.sendStatus(200);
+    });
 
-    app.get('/api/components/:componentVal/plays', componentAwareMiddle, async (req: ComponentAwareRequest, res, next) => {
+    router.post('/components/:componentVal/auth', {middleware: [componentAwareMiddle]}, async (req, res, next) => {
+        const {
+            component,
+        } = req;
+        let didAuth = false;
+        try {
+            logger.verbose('User requested auth test');
+            await component.testAuth(true);
+            component.clearErrors({predicate: x => findAuthIssue(x) !== undefined});
+            didAuth = true;
+            return res.sendStatus(200);
+        } catch (e) {
+            component.replaceErrors(e, {predicate: x => findAuthIssue(x) !== undefined});
+            return res.status(500).json({error: serializeError(e)});
+        } finally {
+            const data = component.getApiData();
+            component.emitComponentUpdate({
+                errors: data.errors,
+                state: data.state,
+                status: didAuth ? 'Authenticated successfully' : data.status
+            });
+        }
+    });
+
+    router.get('/components/:componentVal/plays', {middleware: [componentAwareMiddle]}, async (req, res, next) => {
         const {
             component,
             query
@@ -324,17 +403,16 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         const hydratedQuery = asDayjsHydratedObject<QueryPlaysOptsJson, QueryPlaysOpts<Dayjs>>(query);
         const playRes = await component.getPlaysPaginated(hydratedQuery);
 
-        // @ts-expect-error
+        // @ts-expect-error its fine
         playRes.data = playRes.data.map(x => asSerializablePlaySelect(x))
         //PlayApiCommonDetailed
         // plus paginatioon
         return res.json(playRes);
     });
 
-    app.get('/api/components/:componentVal/plays/:playUid', componentAwareMiddle, async (req: ComponentAwareRequest, res, next) => {
+    router.get('/components/:componentVal/plays/:playUid', {middleware: [componentAwareMiddle]}, async (req, res, next) => {
         const {
             component,
-            query,
             params: {
                 playUid
             }
@@ -346,7 +424,63 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         return res.json(asSerializablePlaySelect(playRes));
     });
 
-    app.delete('/api/cache/:cacheType', async (req, res) => {
+    router.post('/components/:componentVal/plays/:playUid/queue', {middleware: [componentAwareMiddle], bodySchema: queueContextSchema.optional()}, async (req, res, next) => {
+        const {
+            component,
+            params: {
+                playUid
+            }, 
+            body = {}
+        } = req;
+
+        const play = await component.playRepo.findByUid(playUid);
+        if(play === undefined) {
+            return res.sendStatus(404);
+        }
+
+        if(component instanceof AbstractSource) {
+            await component.queuePlay([play], {...body, isRetry: true});
+        } else {
+            await component.queueScrobble([play], {...body, isRetry: true});
+        }
+        return res.sendStatus(200);
+    });
+
+    router.delete('/components/:componentVal/plays/:playUid/queue', {middleware: [componentAwareMiddle]}, async (req, res, next) => {
+        const {
+            component,
+            params: {
+                playUid
+            }
+        } = req;
+
+        const play = await component.playRepo.findByUid(playUid);
+        if(play === undefined) {
+            return res.sendStatus(404);
+        }
+
+        await component.cancelQueuedPlay(play);
+        return res.sendStatus(200);
+    });
+
+    router.delete('/components/:componentVal/plays/:playUid/dead', {middleware: [componentAwareMiddle]}, async (req, res, next) => {
+        const {
+            component,
+            params: {
+                playUid
+            }
+        } = req;
+
+        const play = await component.playRepo.findByUid(playUid);
+        if(play === undefined) {
+            return res.sendStatus(404);
+        }
+
+        await component.removeDeadLetterScrobble(play);
+        return res.sendStatus(200);
+    });
+
+    router.delete('/cache/:cacheType', async (req, res) => {
         const cache = await getRoot().items.cache();
         logger.verbose(`User request cache deletion for ${req.params.cacheType}`);
         switch(req.params.cacheType) {
@@ -371,7 +505,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
      * 
      *  */
 
-    app.get('/api/status', async (req, res, next) => {
+    router.get('/status', async (req, res, next) => {
 
         const sourceData = scrobbleSources.sources.map((x) => {
             const {
@@ -465,9 +599,8 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         return res.json({sources: sourceData, clients: clientData});
     });
 
-    app.get('/api/recent', sourceMiddleFunc(false), async (req, res, next) => {
+    router.get('/recent', {middleware: [sourceMiddleFunc(false)], querySchema: z.any()}, async (req, res, next) => {
         const {
-            // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
             scrobbleSource: source,
             query: {
                 upstream = 'false',
@@ -499,9 +632,8 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         return res.json(result);
     });
 
-    app.get('/api/source/art', sourceMiddleFunc(false), async (req, res, next) => {
+    router.get('/source/art', {middleware: [sourceMiddleFunc(false)]}, async (req, res, next) => {
         const {
-            // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
             scrobbleSource,
             query: {
                 data
@@ -527,9 +659,8 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         }
     });
 
-    app.get('/api/dead', clientMiddleFunc(true), async (req, res, next) => {
+    router.get('/dead', {middleware: [clientMiddleFunc(true)]}, async (req, res, next) => {
         const {
-            // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
             scrobbleClient: client,
             query = {}
         } = req;
@@ -538,21 +669,19 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
             ...query as Partial<QueryPlaysOpts>,
             queues: [
                 {
-                    queueName: CLIENT_DEAD_QUEUE,
+                    queueName: DEAD_QUEUE,
                     queueStatus: ['queued','failed']
                 }
             ]
         }
 
-        // @ts-ignore
         const result: DeadLetterScrobble<PlayObject>[] = (await (client as AbstractScrobbleClient).getPlaysPaginated(deadQuery)).data.map(x => playSelectToDeadScrobble(x, true));
 
         return res.json(result);
     });
 
-    app.put('/api/dead', clientMiddleFunc(true), async (req, res, next) => {
+    router.put('/dead', {middleware: [clientMiddleFunc(true)]}, async (req, res, next) => {
         const {
-            // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
             scrobbleClient: client,
         } = req;
 
@@ -563,9 +692,8 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         await ((client as AbstractScrobbleClient).processDeadLetterQueue(1000));
     });
 
-    app.put('/api/dead/:id', clientMiddleFunc(true), async (req, res, next) => {
+    router.put('/dead/:id', {middleware: [clientMiddleFunc(true)]}, async (req, res, next) => {
         const {
-            // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
             scrobbleClient: client,
             params: {
                 id
@@ -574,14 +702,24 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
 
         const deadId = id as string;
 
-        (client as AbstractScrobbleClient).logger.verbose(`User requested processing of dead letter scrobble ${deadId} via API call`)
+        (client as AbstractScrobbleClient).logger.verbose(`User requested processing of dead letter scrobble ${deadId} via API call`);
 
         try {
-            const [scrobbled, dead] = await (client as AbstractScrobbleClient).processDeadLetterScrobble(deadId);
-            if(scrobbled) {
+            const playEntity = await client.playRepo.findByUid(id);
+            if(playEntity === undefined) {
+                return res.status(404).json({message: `Play ${deadId} does not exist`});
+            }
+            client.queueScrobble(playEntity, {reason: 'user requested processing via API call'}).then(() => null);
+            const event = await pEvent(client.emitter, 'playUpdate', {
+                timeout: 10000,
+                filter: (val: PlayApiCommonDetailed) => val.uid === id
+            }) as PlayApiCommonDetailed;
+            if(event.state === 'scrobbled') {
                 return res.status(200).send();
+            } else {
+                // @ts-expect-error should be fine
+                return res.json(playSelectToDeadScrobble(event, true));
             }
-            return res.json(playSelectToDeadScrobble(dead, true));
         } catch (e) {
             if(e.message.includes(`Play ${deadId} does not exist`)) {
                 logger.warn(e);
@@ -592,22 +730,20 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         }
     });
 
-    app.delete('/api/dead', clientMiddleFunc(true), async (req, res, next) => {
+    router.delete('/dead', {middleware: [clientMiddleFunc(true)]}, async (req, res, next) => {
         const {
-            // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
             scrobbleClient: client,
         } = req;
 
         (client as AbstractScrobbleClient).logger.verbose('User requested deletion of all dead letter scrobbles via API');
 
-        (client as AbstractScrobbleClient).removeDeadLetterScrobbles(['queued', 'failed'], 'failed', false).then(() => null).catch((e) => logger.error(e));
+        (client as AbstractScrobbleClient).removeDeadLetterScrobbles().then(() => null).catch((e) => logger.error(e));
 
         return res.sendStatus(200);
     });
 
-    app.delete('/api/dead/:id', clientMiddleFunc(true), async (req, res, next) => {
+    router.delete('/dead/:id', {middleware: [clientMiddleFunc(true)]}, async (req, res, next) => {
         const {
-            // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
             scrobbleClient: client,
             params: {
                 id
@@ -618,22 +754,22 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
 
         (client as AbstractScrobbleClient).logger.verbose(`User requested removal of dead letter scrobble ${deadId} via API call`)
 
+        const playEntity = await client.playRepo.findByUid(id);
+        if(playEntity === undefined) {
+            return res.status(404).json({message: `Play ${deadId} does not exist`});
+        }
+
         try {
-            await (client as AbstractScrobbleClient).removeDeadLetterScrobble(deadId,'failed', false);
+            await (client as AbstractScrobbleClient).removeDeadLetterScrobble(playEntity);
             return res.status(200).send();
         } catch (e) {
-            if(e.message.includes(`Play ${deadId} does not exist`)) {
-                logger.warn(e);
-                return res.status(404).json({error: e});
-            }
             logger.error(e);
             return res.status(500).json({error: e});
         }
     });
 
-    app.get('/api/scrobbled', clientMiddleFunc(false), async (req, res, next) => {
+    router.get('/scrobbled', {middleware: [clientMiddleFunc(true)]}, async (req, res, next) => {
         const {
-            // @ts-expect-error scrobbleClient not part of req
             scrobbleClient: client,
             query
         } = req;
@@ -650,9 +786,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         return res.json(result);
     });
 
-    app.use('/api/source/init', sourceRequiredMiddle);
-    app.post('/api/source/init', async (req, res) => {
-        // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
+    router.post('/source/init', {middleware: [sourceRequiredMiddle]}, async (req, res) => {
         const source = req.scrobbleSource as AbstractSource;
 
         const {
@@ -678,9 +812,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         }
     });
 
-    app.use('/api/source/listen', sourceRequiredMiddle);
-    app.post('/api/source/listen', async (req, res) => {
-        // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
+    router.post('/source/listen', {middleware: [sourceRequiredMiddle]}, async (req, res) => {
         const source = req.scrobbleSource as AbstractSource;
 
         const {
@@ -700,9 +832,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         res.status(200).json({listening});
     });
 
-    app.use('/api/client/listen', clientRequiredMiddle);
-    app.post('/api/client/listen', async (req, res) => {
-        // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
+    router.post('/client/listen', {middleware: [clientRequiredMiddle]}, async (req, res) => {
         const client = req.scrobbleClient as AbstractScrobbleClient;
 
         const {
@@ -722,9 +852,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         res.status(200).json({listening});
     });
 
-    app.use('/api/client/init', clientRequiredMiddle);
-    app.post('/api/client/init', async (req, res) => {
-        // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
+    router.post('/client/init', {middleware: [clientRequiredMiddle]}, async (req, res) => {
         const client = req.scrobbleClient as AbstractScrobbleClient;
 
         const {
@@ -747,8 +875,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         res.status(200).send('OK');
     });
 
-    app.post('/api/client/historical', clientRequiredMiddle, async (req, res) => {
-        // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
+    router.post('/client/historical', {middleware: [clientRequiredMiddle]}, async (req, res) => {
         const client = req.scrobbleClient as AbstractScrobbleClient;
         if(client instanceof AbstractHistoricalScrobbleClient) {
             client.logger.info('User requested historical play hydration');
@@ -761,7 +888,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
     });
 
     app.get('/health', async (req, res) => res.redirect(307, `/api/${req.url.slice(1)}`));
-    app.get('/api/health', async (req, res) => {
+    router.get('/health', async (req, res) => {
         const {
             type,
             name

src/backend/server/auth.ts

@@ -1,7 +1,6 @@
 import type {Express} from 'express';
 import type {Logger} from "@foxxmd/logging";
 import passport from "passport";
-import type {ExpressHandler} from "../common/infrastructure/Atomic.ts";
 import type LastfmScrobbler from "../scrobblers/LastfmScrobbler.ts";
 import type ScrobbleClients from "../scrobblers/ScrobbleClients.ts";
 import type LastfmSource from "../sources/LastfmSource.ts";
@@ -11,10 +10,33 @@ import type YTMusicSource from "../sources/YTMusicSource.ts";
 import type LibrefmScrobbler from "../scrobblers/LibrefmScrobbler.ts";
 import type LibrefmSource from "../sources/LibrefmSource.ts";
 import AbstractSource from "../sources/AbstractSource.ts";
+import { makeComponentMiddle, type ClientCheckedMiddleTypedMiddleware, type SourceCheckMiddleTypedMiddleware } from './middleware.ts';
+import { findAuthIssue, SimpleError } from '../common/errors/MSErrors.ts';
+import type { createTypedRouter } from '@minisylar/express-typed-router';
+import * as z from 'zod';
 
-export const setupAuthRoutes = (app: Express, logger: Logger, sourceMiddle: ExpressHandler, clientMiddle: ExpressHandler, scrobbleSources: ScrobbleSources, scrobbleClients: ScrobbleClients) => {
-    app.use('/api/client/auth', clientMiddle);
-    app.get('/api/client/auth', async (req, res) => {
+export const setupAuthRoutes = (app: Express, router: ReturnType<typeof createTypedRouter>, logger: Logger, sourceMiddle: SourceCheckMiddleTypedMiddleware, clientMiddle: ClientCheckedMiddleTypedMiddleware, scrobbleSources: ScrobbleSources, scrobbleClients: ScrobbleClients) => {
+    const componentAwareMiddle = makeComponentMiddle(scrobbleSources, scrobbleClients);
+
+    router.get('/components/:componentVal/auth', {middleware: [componentAwareMiddle]}, async (req, res, next) => {
+        switch(req.component.type) {
+            case 'lastfm':
+            case 'librefm':
+                res.status(200).send((req.component as LastfmScrobbler | LastfmSource | LibrefmScrobbler | LibrefmSource ).api.getAuthUrl());
+                break;
+            case 'spotify': {
+                const source = req.component as SpotifySource;
+                if (source.spotifyApi === undefined) {
+                    res.status(400).send('Spotify configuration is not valid');
+                } else {
+                    logger.info('Redirecting to spotify authorization url');
+                    res.status(200).send(source.createAuthUrl());
+                }
+            } break;
+        }
+    });
+
+    router.get('/client/auth', {middleware: [clientMiddle]}, async (req, res) => {
         const {
             scrobbleClient,
         } = req as any;
@@ -25,35 +47,31 @@ export const setupAuthRoutes = (app: Express, logger: Logger, sourceMiddle: Expr
                 res.redirect(scrobbleClient.api.getAuthUrl());
                 break;
             case 'tealfm':
-                const url = await scrobbleClient.getAuthorizeUrl();
-                res.redirect(url);
+                res.redirect(await scrobbleClient.getAuthorizeUrl());
                 break;
             default:
                 return res.status(400).send(`Specified client does not have auth implemented (${scrobbleClient.type})`);
         }
     });
 
-    app.use('/api/source/auth', sourceMiddle);
-    app.get('/api/source/auth', async (req, res, next) => {
+    router.get('/source/auth', {middleware: [sourceMiddle]}, async (req, res, next) => {
         const {
-            // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
             scrobbleSource: source,
-            // @ts-expect-error TS(2339): Property 'sourceName' does not exist on type 'Requ... Remove this comment to see the full error message
             sourceName: name,
         } = req;
 
         switch (source.type) {
             case 'spotify':
-                if (source.spotifyApi === undefined) {
+                if ((source as SpotifySource).spotifyApi === undefined) {
                     res.status(400).send('Spotify configuration is not valid');
                 } else {
                     logger.info('Redirecting to spotify authorization url');
-                    res.redirect(source.createAuthUrl());
+                    res.redirect((source as SpotifySource).createAuthUrl());
                 }
                 break;
             case 'lastfm':
             case 'librefm':
-                res.redirect(source.api.getAuthUrl());
+                res.redirect((source as LastfmSource).api.getAuthUrl());
                 break;
             case 'deezer':
                 // @ts-expect-error TS(2339): Property 'deezerSource' does not exist on type 'Se... Remove this comment to see the full error message
@@ -68,7 +86,7 @@ export const setupAuthRoutes = (app: Express, logger: Logger, sourceMiddle: Expr
         }
     });
 
-    app.get(/.*callback$/, async (req, res, next) => {
+    app.get(/.*callback$/, async (req, res) => {
         if(req.url.indexOf('/api') !== 0) {
             return res.redirect(307, `/api${req.url}`);
         }
@@ -103,7 +121,7 @@ export const setupAuthRoutes = (app: Express, logger: Logger, sourceMiddle: Expr
                     throw error;
                 }
                 await entity.api.authenticate(token);
-                entity.authFailure = false;
+                entity.clearErrors({predicate: x => findAuthIssue(x) !== undefined});
                 if(entity instanceof AbstractSource) {
                     entity.poll().catch((e) => logger.error(e));
                 } else {
@@ -113,10 +131,16 @@ export const setupAuthRoutes = (app: Express, logger: Logger, sourceMiddle: Expr
                         entity.initScrobbleMonitoring().catch((e) => logger.error(e));
                     })
                 }
-                return res.send('OK');
             } catch (e) {
-                return res.send(e.message);
+                if(entity !== undefined) {
+                    entity.errors.push(e);
+                    entity.logger.error(e);
+                } else {
+                    logger.error(e);
+                }
             }
+            res.redirect('/next');
+            return;
         } else if(req.url.includes('ytmusic')) {
             const entity: YTMusicSource | undefined = scrobbleSources.getByName(name) as (YTMusicSource | undefined);
             if(entity === undefined) {
@@ -130,22 +154,36 @@ export const setupAuthRoutes = (app: Express, logger: Logger, sourceMiddle: Expr
             } else {
                 responseContent = result;
             }
-            return res.send(responseContent);
+            res.send(responseContent);
+            return;
         } else {
             // TODO right now all sources requiring source interaction are covered by logic branches (deezer above and spotify here)
             // but eventually should update all source callbacks to url specific URLS to avoid ambiguity...
             // wish we could use state param to identify name/source but not all auth strategies and auth provides may provide access to that
             logger.info({label: 'Spotify'}, 'Received auth code callback from Spotify');
             const source = scrobbleSources.getByNameAndType(state as string, 'spotify', true) as SpotifySource;
-            const tokenResult = await source.handleAuthCodeCallback(req.query);
-            let responseContent = 'OK';
-            if (tokenResult === true) {
-                source.authFailure = false;
-                source.poll().catch((e) => logger.error(e));
-            } else {
-                responseContent = tokenResult;
+            try {
+                const tokenResult = await source.handleAuthCodeCallback(req.query);
+                if (tokenResult === true) {
+                    source.clearErrors({predicate: x => findAuthIssue(x) !== undefined});
+                    source.poll().catch((e) => logger.error(e));
+                } else {
+                    if (tokenResult instanceof Error) {
+                        source.replaceErrors(tokenResult, {predicate: (x) => x.message === tokenResult.message});
+                        source.logger.error(tokenResult);
+                    } else if (typeof tokenResult === 'string') {
+                        const e = new SimpleError(`Token result was unexpected: ${tokenResult}`);
+                        source.replaceErrors(e, {predicate: (x) => x.message === e.message});
+                        source.logger.error(e);
+                    }
+                }
+            } catch (e) {
+                const err = new SimpleError('Unexpected error while trying to authorize code, or save file', { cause: e });
+                source.replaceErrors(err, {predicate: (x) => err.message === x.message});
+                source.logger.error(err);
             }
-            return res.send(responseContent);
+            res.redirect('/next');
+            return;
         }
     });
 }

src/backend/server/deezerRoutes.ts

@@ -27,9 +27,7 @@ export const setupDeezerRoutes = (app: Express, logger: Logger, scrobbleSources:
         // @ts-expect-error TS(2339): Property 'deezerSource' does not exist on type 'Se... Remove this comment to see the full error message
         const entity = scrobbleSources.getByName(req.session.deezerSource as string) as DeezerSource;
         for(let i = 0; i < 3; i++) {
-            if(entity.error !== undefined) {
-                return res.send('Error with deezer credentials storage');
-            } else if(entity.config.data.accessToken !== undefined) {
+            if(entity.config.data.accessToken !== undefined) {
                 // start polling
                 await entity.doAuthentication();
                 entity.poll()

src/backend/server/index.ts

@@ -16,9 +16,13 @@ import type ScrobbleSources from '../sources/ScrobbleSources.ts';
 import type ScrobbleClients from '../scrobblers/ScrobbleClients.ts';
 import { qsOptions } from '../../core/Atomic.ts';
 import { projectRootDir } from "../common/infrastructure/Atomic.ts";
+import { createTypedRouter } from "@minisylar/express-typed-router";
 
 const app = express();
-const router = Router();
+
+const router = createTypedRouter();
+
+//const router = Router();
 app.set('query parser', (str: string) => qs.parse(str, qsOptions));
 
 export const initServer = async (parentLogger: Logger, appLoggerStream: PassThrough, initialOutput: LogDataPretty[] = [], sources: ScrobbleSources, clients: ScrobbleClients) => {
@@ -26,7 +30,7 @@ export const initServer = async (parentLogger: Logger, appLoggerStream: PassThro
     const logger = childLogger(parentLogger, 'API'); // parentLogger.child({labels: ['API']}, mergeArr);
 
     try {
-        app.use(router);
+        app.use('/api', router.getRouter());
         app.use(bodyParser.json());
         app.use(
             bodyParser.urlencoded({
@@ -52,7 +56,7 @@ export const initServer = async (parentLogger: Logger, appLoggerStream: PassThro
         const local = root.get('localUrl');
         const localDefined = root.get('hasDefinedBaseUrl');
 
-        setupApi(app, logger, appLoggerStream, initialOutput, sources, clients);
+        setupApi(app, router, logger, appLoggerStream, initialOutput, sources, clients);
 
         const addy = getAddress();
         const addresses: string[] = [];

src/backend/server/middleware.ts

@@ -5,8 +5,11 @@ import type ScrobbleSources from "../sources/ScrobbleSources.ts";
 import type {Request, Response, NextFunction} from "express";
 import type AbstractSource from "../sources/AbstractSource.ts";
 import type AbstractScrobbleClient from "../scrobblers/AbstractScrobbleClient.ts";
+import type { TypedMiddleware } from "@minisylar/express-typed-router";
 
-export const makeSourceCheckMiddle = (sources: any) => (required: boolean): ExpressHandler => (req: any, res: any, next: any) => {
+export type SourceCheckMiddleTypedMiddleware = TypedMiddleware<{sourceName: string, scrobbleSource: AbstractSource}>;
+
+export const makeSourceCheckMiddle = (sources: any) => (required: boolean): SourceCheckMiddleTypedMiddleware => (req: any, res: any, next: any) => {
     const {
         query: {
             name,
@@ -30,7 +33,9 @@ export const makeSourceCheckMiddle = (sources: any) => (required: boolean): Expr
     next();
 }
 
-export const makeClientCheckMiddle = (clients: any) => (required: boolean): ExpressHandler => (req: any, res: any, next: any) => {
+export type ClientCheckedMiddleTypedMiddleware = TypedMiddleware<{scrobbleClient: AbstractScrobbleClient}>;
+
+export const makeClientCheckMiddle = (clients: any) => (required: boolean): ClientCheckedMiddleTypedMiddleware => (req: any, res: any, next: any) => {
     const {
         query: {
             name
@@ -75,7 +80,7 @@ export interface ComponentAwareRequest extends Request {
     component: AbstractSource | AbstractScrobbleClient
 }
 
-export const makeComponentMiddle = (sources: ScrobbleSources, clients: ScrobbleClients): ExpressHandler => async (req: Request, res: Response, next: NextFunction) => {
+export const makeComponentMiddle = (sources: ScrobbleSources, clients: ScrobbleClients): TypedMiddleware<{component: ComponentAwareRequest['component']}> => (req, res, next) => {
     const {
         params: {
             componentVal
@@ -84,7 +89,8 @@ export const makeComponentMiddle = (sources: ScrobbleSources, clients: ScrobbleC
 
     const componentId = Number.parseInt(componentVal as string);
     if (isNaN(componentId)) {
-        return res.status(400).json({ error: 'Component id must be a number' });
+        res.status(400).json({ error: 'Component id must be a number' });
+        return;
     }
 
     let component: AbstractSource | AbstractScrobbleClient;
@@ -94,7 +100,8 @@ export const makeComponentMiddle = (sources: ScrobbleSources, clients: ScrobbleC
     }
 
     if(component === undefined) {
-        return res.status(404).json({error: `No Component with the Id ${componentId} exists`});
+        res.status(404).json({error: `No Component with the Id ${componentId} exists`});
+        return;
     }
 
     (req as ComponentAwareRequest).component = component;
@@ -109,7 +116,7 @@ export interface ClientAwareRequest extends Request {
     component: AbstractScrobbleClient
 }
 
-export const makeSourceNextMiddle = (sources: ScrobbleSources): ExpressHandler => async (req: Request, res: Response, next: NextFunction) => {
+export const makeSourceNextMiddle = (sources: ScrobbleSources): TypedMiddleware<{component: AbstractSource}> => async (req: Request, res: Response, next: NextFunction) => {
     const {
         params: {
             componentVal
@@ -118,13 +125,14 @@ export const makeSourceNextMiddle = (sources: ScrobbleSources): ExpressHandler =
 
     const componentId = Number.parseInt(componentVal as string);
     if (isNaN(componentId)) {
-        return res.status(400).json({ error: 'Source Id must be a number' });
+       res.status(400).json({ error: 'Source Id must be a number' });
+       return;
     }
 
-    let component: AbstractSource;
-    component = sources.sources.find(x => x.componentId === componentId);
+    const component: AbstractSource = sources.sources.find(x => x.componentId === componentId);
     if(component === undefined) {
-        return res.status(404).json({error: `No Source with the Id ${componentId} exists`});
+        res.status(404).json({error: `No Source with the Id ${componentId} exists`});
+        return;
     }
 
     (req as SourceAwareRequest).component = component;
@@ -132,7 +140,7 @@ export const makeSourceNextMiddle = (sources: ScrobbleSources): ExpressHandler =
     next();
 }
 
-export const makeClientNextMiddle = (clients: ScrobbleClients): ExpressHandler => async (req: Request, res: Response, next: NextFunction) => {
+export const makeClientNextMiddle = (clients: ScrobbleClients): TypedMiddleware<{component: AbstractScrobbleClient}> => async (req: Request, res: Response, next: NextFunction) => {
     const {
         params: {
             componentVal
@@ -141,13 +149,14 @@ export const makeClientNextMiddle = (clients: ScrobbleClients): ExpressHandler =
 
     const componentId = Number.parseInt(componentVal as string);
     if (isNaN(componentId)) {
-        return res.status(400).json({ error: 'Source Id must be a number' });
+        res.status(400).json({ error: 'Source Id must be a number' });
+        return;
     }
 
-    let component: AbstractScrobbleClient;
-    component = clients.clients.find(x => x.componentId === componentId);
+    const component: AbstractScrobbleClient = clients.clients.find(x => x.componentId === componentId);
     if(component === undefined) {
-        return res.status(404).json({error: `No Client with the Id ${componentId} exists`});
+        res.status(404).json({error: `No Client with the Id ${componentId} exists`});
+        return;
     }
 
     (req as ClientAwareRequest).component = component;

src/backend/sources/AppleMusicSource.ts

@@ -1,7 +1,7 @@
 import dayjs, { type Dayjs } from "dayjs";
 import type EventEmitter from "events";
 import { MusicKit, type MeHistoryRecentlyPlayedTracksProps, type Song } from "node-musickit-api";
-import type { PlayObject, PlayObjectMinimal } from "../../core/Atomic.ts";
+import { COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, type PlayObjectMinimal } from "../../core/Atomic.ts";
 import type { InternalConfig } from "../common/infrastructure/Atomic.ts";
 import type { AppleMusicSourceConfig } from "../common/infrastructure/config/source/applemusic.ts";
 import AbstractSource, { type RecentlyPlayedOptions } from "./AbstractSource.ts";
@@ -33,6 +33,7 @@ export default class AppleMusicSource extends AbstractSource {
     private readonly POLL_TRACK_LIMIT = 20;
     private readonly UPSTREAM_TRACK_LIMIT = 30;
 
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = false;
 
@@ -355,13 +356,7 @@ export default class AppleMusicSource extends AbstractSource {
     onPollPostAuthCheck = async () => {
         if(!this.polling) {
             this.logger.verbose('Hydrating initial recently played tracks for reference.');
-            const referencePlays = await this.getRecentlyPlayed();
-            const reversedPlays = [...referencePlays];
-            reversedPlays.reverse();
-
-            for(const refPlay of reversedPlays) {
-                await this.addPlayToDB(refPlay);
-            }
+            await this.getRecentlyPlayed();
         }
         return true;
     }

src/backend/sources/DeezerInternalSource.ts

@@ -2,7 +2,7 @@ import dayjs from "dayjs";
 import type EventEmitter from "events";
 import type { Request } from 'superagent';
 import request from 'superagent';
-import { type PlayObject, type PlayObjectMinimal, SOURCE_SOT, TA_CLOSE, TA_DURING, TA_EXACT, TA_FUZZY, type TemporalAccuracy } from "../../core/Atomic.ts";
+import { COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayMatchResult, type PlayObject, type PlayObjectMinimal, SOURCE_SOT, TA_CLOSE, TA_DURING, TA_EXACT, TA_FUZZY, type TemporalAccuracy } from "../../core/Atomic.ts";
 import { DEFAULT_RETRY_MULTIPLIER, type FormatPlayObjectOptions, type InternalConfig } from "../common/infrastructure/Atomic.ts";
 import type {DeezerInternalSourceConfig, DeezerInternalTrackData} from "../common/infrastructure/config/source/deezer.ts";
 import { TRANSFORM_HOOK } from "../../core/Transform.ts";
@@ -63,6 +63,7 @@ interface DeezerAuthedUserData {
 }
 
 export default class DeezerInternalSource extends MemorySource {
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = false;
     isSubAccount: boolean = false;
@@ -332,15 +333,22 @@ export default class DeezerInternalSource extends MemorySource {
     protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) => await this.getRecentlyPlayed({formatted: true, ...options})
 
 
-    existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<PlayObject | undefined> => {
-        const list: PlayObject[] = await this.getRecentlyDiscoveredPlays();
+    async existingDiscovered(play: PlayObject): Promise<PlayMatchResult | undefined> {
+        const list: PlayObject[] = await this.getRecentPlays();
         const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate);
         const existing = await findAsync(list, async x => {
             const e = await this.transformPlay(x, TRANSFORM_HOOK.existing);
             return genericSourcePlayMatch(e, candidate);
         });
         if(existing) {
-            return existing;
+            return {
+                match: true,
+                score: 1,
+                breakdowns: [],
+                reason: 'Has matching data with very close timestamps',
+                closestMatchedPlay: existing,
+                createdAt: dayjs().toISOString()
+            }
         }
         if(this.config.options?.fuzzyDiscoveryIgnore === true || this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {
             const fuzzyIndex = await findIndexAsync(list, async x => {
@@ -361,19 +369,45 @@ export default class DeezerInternalSource extends MemorySource {
                 if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {
                     // always return fuzzy match as existing
                     // likely will make MS miss scrobbles for repeated plays
-                    return list[fuzzyIndex];
+                    return {
+                        match: true,
+                        score: 1,
+                        breakdowns: [],
+                        reason: 'Has matching data and timestamp is during the duration of a previous play',
+                        closestMatchedPlay: list[fuzzyIndex],
+                        createdAt: dayjs().toISOString()
+                    }
                 }
                 if(fuzzyIndex + 1 === list.length || playObjDataMatch(list[fuzzyIndex], list[fuzzyIndex + 1])) {
                     // last discovered play was this one, or next played play was also this one
                     // so we'll assume this means the play is on repeat, don't count as existing
-                    return undefined;
+                    return {
+                        match: false,
+                        score: 0.5,
+                        breakdowns: [],
+                        reason: 'Has matching data for previous play but assuming its on repeat',
+                        closestMatchedPlay: list[fuzzyIndex],
+                        createdAt: dayjs().toISOString()
+                    }
                 }
                 // next played play was *not* this one (Deezer reports play between candidate TS and fuzzy match)
                 // so this is likely a duplicate deezer should not have reported
-                return list[fuzzyIndex];
+                    return {
+                        match: true,
+                        score: 1,
+                        breakdowns: [],
+                        reason: 'Has matching data and looks like a misreported play',
+                        closestMatchedPlay: list[fuzzyIndex],
+                        createdAt: dayjs().toISOString()
+                    }
             }
         }
-        return undefined;
+        return {
+            match: false,
+            score: 0,
+            breakdowns: [],
+            createdAt: dayjs().toISOString()
+        }
     }
 }
 

src/backend/sources/DeezerSource.ts

@@ -3,7 +3,7 @@ import type EventEmitter from "events";
 import passport from "passport";
 import { Strategy as DeezerStrategy } from 'passport-deezer';
 import request from 'superagent';
-import type {PlayObject, PlayObjectMinimal} from "../../core/Atomic.ts";
+import {COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, type PlayObjectMinimal} from "../../core/Atomic.ts";
 import { DEFAULT_RETRY_MULTIPLIER, type FormatPlayObjectOptions, type InternalConfig } from "../common/infrastructure/Atomic.ts";
 import type {DeezerSourceConfig} from "../common/infrastructure/config/source/deezer.ts";
 import { parseRetryAfterSecsFromObj, sleep, sortByOldestPlayDate, } from "../utils.ts";
@@ -12,10 +12,12 @@ import { readJson } from '../utils/DataUtils.ts';
 import { joinedUrl } from "../utils/NetworkUtils.ts";
 import AbstractSource, { type RecentlyPlayedOptions } from "./AbstractSource.ts";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.ts";
+import { SimpleError } from "../common/errors/MSErrors.ts";
 
 export default class DeezerSource extends AbstractSource {
     workingCredsPath;
 
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = true;
 
@@ -242,7 +244,7 @@ export default class DeezerSource extends AbstractSource {
             return true;
         } else {
             this.logger.warn('Callback contained an error! User may have denied access?')
-            this.error = error;
+            this.errors.push(error);
             this.logger.error(error);
             return error;
         }

src/backend/sources/EndpointLastfmSource.ts

@@ -1,6 +1,6 @@
 import dayjs from "dayjs";
 import type EventEmitter from "events";
-import { type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
+import { PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
 import {
     type ExpressRequest,
     type FormatPlayObjectOptions,
@@ -78,10 +78,11 @@ export class EndpointLastfmSource extends MemorySource {
         }
 
         const discoverable = stateData.filter(x => x.play.meta.nowPlaying === false);
-        const discovered = await this.discover(discoverable.map(x => x.play));
-        if (discovered.length > 0) {
-            await this.scrobble(discovered);
-        }
+        await this.queuePlay(discoverable.map(x => ({...x.play, meta: {...x.play.meta, parsedFrom: PARSED_FROM.ingress}})));
+        // const discovered = await this.discover(discoverable.map(x => x.play));
+        // if (discovered.length > 0) {
+        //     await this.scrobble(discovered);
+        // }
         this.componentRepo.updateById(this.dbComponent.id, {lastActiveAt: dayjs()});
         this.setStatus('Waiting for Plays');
     }

src/backend/sources/EndpointListenbrainzSource.ts

@@ -1,6 +1,6 @@
 import dayjs from "dayjs";
 import type EventEmitter from "events";
-import { type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
+import { PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
 import {
     type ExpressRequest,
     type FormatPlayObjectOptions,
@@ -104,10 +104,11 @@ export class EndpointListenbrainzSource extends MemorySource {
         }
 
         const discoverable = stateData.filter(x => x.play.meta.nowPlaying === false && this.isValidScrobble(x.play));
-        const discovered = await this.discover(discoverable.map(x => x.play));
-        if (discovered.length > 0) {
-            await this.scrobble(discovered);
-        }
+        await this.queuePlay(discoverable.map(x => ({...x.play, meta: {...x.play.meta, parsedFrom: PARSED_FROM.ingress}})))
+        // const discovered = await this.discover(discoverable.map(x => x.play));
+        // if (discovered.length > 0) {
+        //     await this.scrobble(discovered);
+        // }
         this.componentRepo.updateById(this.dbComponent.id, {lastActiveAt: dayjs()});
         this.setStatus('Waiting for Plays');
     }

src/backend/sources/JRiverSource.ts

@@ -2,7 +2,7 @@ import dayjs from "dayjs";
 import type { EventEmitter } from "events";
 import normalizeUrl from 'normalize-url';
 import { URL } from "url";
-import type {PlayObject, PlayObjectMinimal} from "../../core/Atomic.ts";
+import {COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, type PlayObjectMinimal} from "../../core/Atomic.ts";
 import type {FormatPlayObjectOptions, InternalConfig} from "../common/infrastructure/Atomic.ts";
 import type {JRiverSourceConfig} from "../common/infrastructure/config/source/jriver.ts";
 import { type Info, JRiverApiClient, PLAYER_STATE } from "../common/vendor/JRiverApiClient.ts";
@@ -18,6 +18,7 @@ export class JRiverSource extends MemoryPositionalSource {
 
     client: JRiverApiClient;
     clientReady: boolean = false;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
 
     constructor(name: any, config: JRiverSourceConfig, internal: InternalConfig, emitter: EventEmitter) {
         const {

src/backend/sources/JellyfinApiSource.ts

@@ -37,11 +37,11 @@ import {
 import dayjs from "dayjs";
 import type EventEmitter from "events";
 import { FixedSizeList } from "fixed-size-list";
-import type {ArtistCredit, BrainzMeta, PlayObject, PlayObjectMinimal} from "../../core/Atomic.ts";
+import type {ArtistCredit, BrainzMeta, ComponentAuthType, PlayObject, PlayObjectMinimal} from "../../core/Atomic.ts";
 import { genGroupIdStr } from '../../core/PlayUtils.ts';
 import { artistNameToCredit, buildTrackString, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.ts";
 import type {FormatPlayObjectOptions, InternalConfig, PlayerStateDataMaybePlay} from "../common/infrastructure/Atomic.ts";
-import { REPORTED_PLAYER_STATUSES } from '../../core/Atomic.ts';
+import { COMPONENT_AUTH_TYPE, REPORTED_PLAYER_STATUSES } from '../../core/Atomic.ts';
 import type {JellyApiSourceConfig} from "../common/infrastructure/config/source/jellyfin.ts";
 import { getPlatformIdFromData, isDebugMode } from "../utils.ts";
 import { noCasePropObj } from "../utils/DataUtils.ts";
@@ -49,6 +49,8 @@ import { joinedUrl } from "../utils/NetworkUtils.ts";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.ts";
 import { hashObject, parseArrayFromMaybeString } from "../utils/StringUtils.ts";
 import { MemoryPositionalSource } from "./MemoryPositionalSource.ts";
+import * as axios from 'axios';
+import { AuthError } from "../common/errors/MSErrors.ts";
 
 const shortDeviceId = truncateStringToLength(10, '');
 
@@ -81,6 +83,7 @@ export default class JellyfinApiSource extends MemoryPositionalSource {
     libraries: {name: string, paths: string[], collectionType: CollectionType}[] = [];
 
     declare config: JellyApiSourceConfig;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
 
     constructor(name: any, config: JellyApiSourceConfig, internal: InternalConfig, emitter: EventEmitter) {
         super('jellyfin', name, config, internal, emitter);
@@ -251,7 +254,8 @@ export default class JellyfinApiSource extends MemoryPositionalSource {
             }
             return true;
         } catch (e) {
-            throw e;
+            const unrecoverable = axios.isAxiosError(e) && [401,403].includes(e.status);
+            throw new AuthError('API Key failed to authenticate', {cause: e, unrecoverable});
         }
     }
 

src/backend/sources/KodiSource.ts

@@ -1,5 +1,5 @@
 import type { EventEmitter } from "events";
-import type {PlayObject} from "../../core/Atomic.ts";
+import {COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject} from "../../core/Atomic.ts";
 import type {FormatPlayObjectOptions, InternalConfig} from "../common/infrastructure/Atomic.ts";
 import type {KodiSourceConfig} from "../common/infrastructure/config/source/kodi.ts";
 import { KodiApiClient } from "../common/vendor/KodiApiClient.ts";
@@ -11,6 +11,7 @@ export class KodiSource extends MemoryPositionalSource {
 
     client: KodiApiClient;
     clientReady: boolean = false;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
 
     constructor(name: any, config: KodiSourceConfig, internal: InternalConfig, emitter: EventEmitter) {
         const {

src/backend/sources/KoitoSource.ts

@@ -1,5 +1,5 @@
 import type EventEmitter from "events";
-import { type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
+import { COMPONENT_AUTH_TYPE, type ComponentAuthType, PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import type {FormatPlayObjectOptions, InternalConfig, TimeRangeListensFetcher} from "../common/infrastructure/Atomic.ts";
 import type {RecentlyPlayedOptions} from "./AbstractSource.ts";
@@ -11,6 +11,7 @@ import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.ts
 export default class KoitoSource extends MemorySource {
 
     api: KoitoApiClient;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = false;
     getScrobblesForTimeRange: TimeRangeListensFetcher
@@ -63,7 +64,7 @@ export default class KoitoSource extends MemorySource {
         this.setStatus('Checking for new Plays');
         await this.processRecentPlays([]);
         const resp = await this.getScrobblesForTimeRange({limit, cursor: 0 });
-        return resp;
+        return resp.map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.history}}));
     }
 
     getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise<PlayObject[]> => {

src/backend/sources/LastfmSource.ts

@@ -1,7 +1,7 @@
 import type EventEmitter from "events";
-import { type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
+import { COMPONENT_AUTH_TYPE, PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
 import type {FormatPlayObjectOptions, InternalConfig, TimeRangeListensFetcher} from "../common/infrastructure/Atomic.ts";
-import type {PlayPlatformId} from '../../core/Atomic.ts';
+import type {ComponentAuthType, PlayPlatformId} from '../../core/Atomic.ts';
 import type {SourceType} from "../../core/Atomic.ts";
 import type {LastfmSourceConfig} from "../common/infrastructure/config/source/lastfm.ts";
 import LastfmApiClient, { formatPlayObj } from "../common/vendor/LastfmApiClient.ts";
@@ -16,6 +16,7 @@ import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.ts
 export default class LastfmSource extends MemorySource {
 
     api: LastfmApiClient;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.interactive;
     requiresAuth = true;
     requiresAuthInteraction = true;
     upstreamType: string = 'Last.fm';
@@ -101,7 +102,7 @@ export default class LastfmSource extends MemorySource {
     getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise<PlayObject[]> => {
         try {
             const [history, now] = await this.getLastfmRecentTrack(options);
-            return history;
+            return history.map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.history}}));
         } catch (e) {
             throw e;
         }

src/backend/sources/LibrefmSource.ts

@@ -9,7 +9,7 @@ import type {LastfmSourceConfig} from "../common/infrastructure/config/source/la
 
 export default class LibrefmSource extends LastfmSource {
 
-    // @ts-expect-error
+    // @ts-expect-error its fine
     declare config: LibrefmSourceConfig;
 
     constructor(name: any, config: LibrefmSourceConfig, internal: InternalConfig, emitter: EventEmitter) {

src/backend/sources/ListenbrainzSource.ts

@@ -1,8 +1,8 @@
 import type EventEmitter from "events";
-import { type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
+import { COMPONENT_AUTH_TYPE, PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import type {FormatPlayObjectOptions, InternalConfig, TimeRangeListensFetcher} from "../common/infrastructure/Atomic.ts";
-import type {PlayPlatformId} from '../../core/Atomic.ts';
+import type {ComponentAuthType, PlayPlatformId} from '../../core/Atomic.ts';
 import type {ListenBrainzSourceConfig} from "../common/infrastructure/config/source/listenbrainz.ts";
 import { ListenbrainzApiClient } from "../common/vendor/ListenbrainzApiClient.ts";
 import type {RecentlyPlayedOptions} from "./AbstractSource.ts";
@@ -17,6 +17,7 @@ import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.ts
 export default class ListenbrainzSource extends MemorySource {
 
     api: ListenbrainzApiClient;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = false;
     getScrobblesForTimeRange: TimeRangeListensFetcher
@@ -85,7 +86,7 @@ export default class ListenbrainzSource extends MemorySource {
         const {limit = 20} = options;
         const now = await this.api.getPlayingNow();
         await this.processRecentPlays(now.listens.map(x => ListenbrainzSource.formatPlayObj(x)));
-        return await this.getScrobblesForTimeRange({limit});
+        return (await this.getScrobblesForTimeRange({limit})).map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.history}}));
     }
 
     getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise<PlayObject[]> => {

src/backend/sources/MPDSource.ts

@@ -2,13 +2,13 @@ import dayjs from "dayjs";
 import type { EventEmitter } from "events";
 import path from 'path';
 import {MPC, type Status, type Song, type PlaylistItem} from 'mpc-js';
-import type {BrainzMeta, PlayObject, PlayObjectMinimal} from "../../core/Atomic.ts";
+import type {BrainzMeta, ComponentAuthType, PlayObject, PlayObjectMinimal} from "../../core/Atomic.ts";
 import {
     type FormatPlayObjectOptions,
     type InternalConfig,
     type PlayerStateData,
 } from "../common/infrastructure/Atomic.ts";
-import { SINGLE_USER_PLATFORM_ID } from '../../core/Atomic.ts';
+import { COMPONENT_AUTH_TYPE, SINGLE_USER_PLATFORM_ID } from '../../core/Atomic.ts';
 import { REPORTED_PLAYER_STATUSES } from '../../core/Atomic.ts';
 import type {ReportedPlayerStatus} from '../../core/Atomic.ts';
 import type {MPDSourceConfig, PlayerState} from "../common/infrastructure/config/source/mpd.ts";
@@ -18,6 +18,7 @@ import { MemoryPositionalSource } from "./MemoryPositionalSource.ts";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.ts";
 import { isDebugMode, sleep } from "../utils.ts";
 import { artistNamesToCredits } from "../../core/StringUtils.ts";
+import { AuthError } from "../common/errors/MSErrors.ts";
 
 const CLIENT_PLAYER_STATE: Record<PlayerState, ReportedPlayerStatus> = {
     'play': REPORTED_PLAYER_STATUSES.playing,
@@ -36,6 +37,8 @@ export class MPDSource extends MemoryPositionalSource {
     protected currentPlayPath: string;
     protected currentPlaySong?: Song;
 
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
+
     constructor(name: any, config: MPDSourceConfig, internal: InternalConfig, emitter: EventEmitter) {
         const {
             data = {}
@@ -139,7 +142,7 @@ export class MPDSource extends MemoryPositionalSource {
             //             break;
             //     }
             // }
-            throw new Error(`Could not connect to MPD server${friendlyError !== undefined ? ` (Hint: ${friendlyError})` : ''}`, {cause: e});
+            throw new AuthError(`Could not connect to MPD server${friendlyError !== undefined ? ` (Hint: ${friendlyError})` : ''}`, {cause: e, unrecoverable: false});
         }
     }
 

src/backend/sources/MalojaSource.ts

@@ -1,5 +1,5 @@
 import type EventEmitter from "events";
-import { type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
+import { COMPONENT_AUTH_TYPE, type ComponentAuthType, PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import type {InternalConfig} from "../common/infrastructure/Atomic.ts";
 import type {RecentlyPlayedOptions} from "./AbstractSource.ts";
@@ -10,6 +10,7 @@ import type {MalojaSourceConfig} from "../common/infrastructure/config/source/ma
 export default class MalojaSource extends MemorySource {
 
     api: MalojaApiClient;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = false;
 
@@ -58,7 +59,7 @@ export default class MalojaSource extends MemorySource {
         this.setStatus('Checking for new Plays');
         await this.processRecentPlays([]);
         const resp = await this.api.getPaginatedTimeRangeListens({limit, cursor: 0});
-        return resp.data;
+        return resp.data.map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.history}}));
     }
 
     getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise<PlayObject[]> => {

src/backend/sources/MemorySource.ts

@@ -2,7 +2,7 @@ import type {Logger} from "@foxxmd/logging";
 import dayjs, { type Dayjs } from "dayjs";
 import type { EventEmitter } from "events";
 import { AsyncTask, SimpleIntervalJob, ToadScheduler } from "toad-scheduler";
-import { type PlayObject, SOURCE_SOT, type SOURCE_SOT_TYPES, type SourcePlayerJson, type SourcePlayerObj } from "../../core/Atomic.ts";
+import { PARSED_FROM, type PlayObject, SOURCE_SOT, type SOURCE_SOT_TYPES, type SourcePlayerJson, type SourcePlayerObj } from "../../core/Atomic.ts";
 import { buildTrackString } from "../../core/StringUtils.ts";
 import {
     asPlayerStateDataMaybePlay,
@@ -115,6 +115,7 @@ export default class MemorySource extends AbstractSource {
                 this.emitEvent('playerUpdate', {
                     ...state,
                     options: {
+                        wasMonitored: this.isMonitoring(),
                         scrobbleTo: this.clients
                     }
                 });
@@ -331,6 +332,7 @@ export default class MemorySource extends AbstractSource {
                 this.emitEvent('playerUpdate', {
                     ...apiState,
                     options: {
+                        wasMonitored: this.isMonitoring(),
                         scrobbleTo: this.clients
                     }
                 });
@@ -350,7 +352,7 @@ export default class MemorySource extends AbstractSource {
             }
         }
 
-        return newStatefulPlays;
+        return newStatefulPlays.map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.player}}));
     }
 
     protected isListenedPlayDiscoverable = async (candidate: PlayObject): Promise<[boolean, string]> => {
@@ -366,11 +368,11 @@ export default class MemorySource extends AbstractSource {
 
         if (thresholdResults.passes) {
             const matchingRecent = await this.existingDiscovered(candidate); //sRecentlyPlayed.find(x => playObjDataMatch(x, candidate));
-            if (matchingRecent === undefined) {
+            if (matchingRecent.match === false) {
                 return [true,`${stPrefix} added after ${thresholdResultSummary(thresholdResults)} and not matching any prior plays`];
             } else {
                 const {data: {playDate, duration}} = candidate;
-                const {data: {playDate: rplayDate}} = matchingRecent;
+                const {closestMatchedPlay: {data: {playDate: rplayDate}} = {}} = matchingRecent;
                 if (!playDate.isSame(rplayDate)) {
                     if (duration !== undefined) {
                         if (playDate.isAfter(rplayDate.add(duration, 's'))) {
@@ -386,7 +388,7 @@ export default class MemorySource extends AbstractSource {
                         return [false, `${stPrefix} not added because it matched the last discovered play and could not determine time frame of play`];
                     }
                 } else {
-                    return [false, `${stPrefix} ${EXPECTED_NON_DISCOVERED_REASON}`]
+                    return [false, `${stPrefix} ${EXPECTED_NON_DISCOVERED_REASON}`];
                 }
             }
         }

src/backend/sources/MusikcubeSource.ts

@@ -4,14 +4,14 @@ import type { CloseEvent, ErrorEvent, RetryEvent } from 'iso-websocket';
 import { WS } from 'iso-websocket'
 import { randomUUID } from "node:crypto";
 import pEvent from 'p-event';
-import type {PlayObject, PlayObjectMinimal, URLData} from "../../core/Atomic.ts";
+import type {ComponentAuthType, PlayObject, PlayObjectMinimal, URLData} from "../../core/Atomic.ts";
 import { UpstreamError } from "../common/errors/UpstreamError.ts";
 import {
     type FormatPlayObjectOptions,
     type InternalConfig,
     type PlayerStateData,
 } from "../common/infrastructure/Atomic.ts";
-import { SINGLE_USER_PLATFORM_ID } from '../../core/Atomic.ts';
+import { COMPONENT_AUTH_TYPE, SINGLE_USER_PLATFORM_ID } from '../../core/Atomic.ts';
 import type {MCAuthenticateRequest, MCAuthenticateResponse, MCPlaybackOverviewRequest, MCPlaybackOverviewResponse, MusikcubeSourceConfig} from "../common/infrastructure/config/source/musikcube.ts";
 import { sleep } from "../utils.ts";
 import type {RecentlyPlayedOptions} from "./AbstractSource.ts";
@@ -19,6 +19,7 @@ import { MemoryPositionalSource } from "./MemoryPositionalSource.ts";
 import { normalizeWSAddress } from "../utils/NetworkUtils.ts";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.ts";
 import { artistNamesToCredits } from "../../core/StringUtils.ts";
+import { AuthError } from "../common/errors/MSErrors.ts";
 
 const CLIENT_STATE = {
     0: 'connecting',
@@ -29,6 +30,7 @@ const CLIENT_STATE = {
 
 export class MusikcubeSource extends MemoryPositionalSource {
     declare config: MusikcubeSourceConfig;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
 
     url: URLData;
 
@@ -149,9 +151,9 @@ export class MusikcubeSource extends MemoryPositionalSource {
             if(authE === undefined) {
                 throw new Error('Musikcube did not respond to auth message after 2000 ms');
             } else if(isCloseEvent(authE)) {
-                throw new Error(`Password is not correct: ${authE.code} => ${authE.reason}`);
+                throw new AuthError(`Password is not correct: ${authE.code} => ${authE.reason}`, {unrecoverable: true});
             } else if(isErrorEvent(authE)) {
-                throw new Error(`Unexpected error occurred while authenticating: ${authE.message}`, {cause: authE.error});
+                throw new AuthError(`Unexpected error occurred while authenticating: ${authE.message}`, {cause: authE.error, unrecoverable: false});
             }
 
             return true;

src/backend/sources/PlexApiSource.ts

@@ -1,5 +1,5 @@
 import type EventEmitter from "events";
-import type {BrainzMeta, PlayObject, PlayObjectMinimal, URLData} from "../../core/Atomic.ts";
+import type {BrainzMeta, ComponentAuthType, PlayObject, PlayObjectMinimal, URLData} from "../../core/Atomic.ts";
 import { artistNamesToCredits, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.ts";
 import {
     asPlayerStateDataMaybePlay,
@@ -7,7 +7,7 @@ import {
     type InternalConfig,
     MBID_VARIOUS_ARTISTS,
     type PlayerStateDataMaybePlay} from "../common/infrastructure/Atomic.ts";
-import { REPORTED_PLAYER_STATUSES } from '../../core/Atomic.ts';
+import { COMPONENT_AUTH_TYPE, REPORTED_PLAYER_STATUSES } from '../../core/Atomic.ts';
 import type {PlayPlatformId} from '../../core/Atomic.ts';
 import { getFirstNonEmptyString, isDebugMode, } from "../utils.ts";
 import { buildStatePlayerPlayIdententifyingInfo, hashObject, parseArrayFromMaybeString } from "../utils/StringUtils.ts";
@@ -41,6 +41,8 @@ const THUMB_REGEX = new RegExp(/\/library\/metadata\/(?<ratingkey>\d+)\/thumb\/\
 export default class PlexApiSource extends MemoryPositionalSource {
     users: string[] = [];
 
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
+
     plexApi: PlexAPI;
     plexUser: string;
 

src/backend/sources/RockskySource.ts

@@ -1,5 +1,5 @@
 import type EventEmitter from "events";
-import { type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
+import { COMPONENT_AUTH_TYPE, type ComponentAuthType, PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import type {FormatPlayObjectOptions, InternalConfig} from "../common/infrastructure/Atomic.ts";
 import type {RecentlyPlayedOptions} from "./AbstractSource.ts";
@@ -10,6 +10,7 @@ import type {RockskySourceConfig} from "../common/infrastructure/config/source/r
 export default class RockskySource extends MemorySource {
 
     api: RockSkyApiClient;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = false;
 
@@ -70,7 +71,7 @@ export default class RockskySource extends MemorySource {
     getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}) => {
         const {limit = 20} = options;
         this.setStatus('Checking for new Plays');
-        return await this.api.getRecentlyPlayed(limit);
+        return (await this.api.getRecentlyPlayed(limit)).map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.history}}));
     }
 
     getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise<PlayObject[]> => {

src/backend/sources/SpotifySource.ts

@@ -2,7 +2,7 @@ import dayjs, { type Dayjs } from "dayjs";
 import type EventEmitter from "events";
 import SpotifyWebApi from "spotify-web-api-node";
 import request from 'superagent';
-import { type BrainzMeta, type PlayObject, type PlayObjectMinimal, SCROBBLE_TS_SOC_END, SCROBBLE_TS_SOC_START, type ScrobbleTsSOC } from "../../core/Atomic.ts";
+import { type BrainzMeta, COMPONENT_AUTH_TYPE, type PlayObject, type PlayObjectMinimal, SCROBBLE_TS_SOC_END, SCROBBLE_TS_SOC_START, type ScrobbleTsSOC } from "../../core/Atomic.ts";
 import { artistNameToCredit, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import { hasUpstreamError, UpstreamError } from "../common/errors/UpstreamError.ts";
@@ -19,7 +19,7 @@ import {
 } from "../common/infrastructure/Atomic.ts";
 import { NO_USER } from '../../core/Atomic.ts';
 import { NO_DEVICE } from '../../core/Atomic.ts';
-import type {ReportedPlayerStatus} from '../../core/Atomic.ts';
+import type {ComponentAuthType, ReportedPlayerStatus} from '../../core/Atomic.ts';
 import type {SpotifySourceConfig} from "../common/infrastructure/config/source/spotify.ts";
 import {
     parseRetryAfterSecsFromObj,
@@ -40,7 +40,7 @@ import type {RecentlyPlayedOptions} from "./AbstractSource.ts";
 import { MemoryPositionalSource } from "./MemoryPositionalSource.ts";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.ts";
 import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.ts";
-import { SimpleError } from "../common/errors/MSErrors.ts";
+import { AuthError, SimpleError } from "../common/errors/MSErrors.ts";
 
 const scopes = ['user-read-recently-played', 'user-read-currently-playing', 'user-read-playback-state', 'user-read-playback-position'];
 const state = 'random';
@@ -52,6 +52,7 @@ export default class SpotifySource extends MemoryPositionalSource implements Pag
     spotifyApi: SpotifyWebApi;
     workingCredsPath: string;
 
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.interactive;
     requiresAuth = true;
     requiresAuthInteraction = true;
 
@@ -342,16 +343,18 @@ export default class SpotifySource extends MemoryPositionalSource implements Pag
     doAuthentication = async () => {
         try {
             if(undefined === this.spotifyApi.getAccessToken()) {
-                this.logger.warn('Cannot use API until an access token has been received from the authorization flow. See the dashboard.');
-                return false;
+                throw new AuthError('Cannot use API until an access token has been received from the authorization flow.', {unrecoverable: false});
             }
             await this.callApi<ReturnType<typeof this.spotifyApi.getMe>>(((api: any) => api.getMe()));
             return true;
         } catch (e) {
+            if(e instanceof AuthError) {
+                throw e;
+            }
             if(isNodeNetworkException(e)) {
                 this.logger.error('Could not communicate with Spotify API');
             }
-            throw e;
+            throw new AuthError('Failed to authenticate', {cause: e, unrecoverable: 'statusCode' in e && [401,403].includes(e.statusCode)})
         }
     }
 

src/backend/sources/SubsonicSource.ts

@@ -4,11 +4,11 @@ import isSameOrAfter from "dayjs/plugin/isSameOrAfter.js";
 import type EventEmitter from "events";
 import type { Request } from 'superagent';
 import request from 'superagent';
-import { REPORTED_PLAYER_STATUSES, type PlayObject, type PlayObjectMinimal } from "../../core/Atomic.ts";
+import { COMPONENT_AUTH_TYPE, REPORTED_PLAYER_STATUSES, type PlayObject, type PlayObjectMinimal } from "../../core/Atomic.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import { UpstreamError } from "../common/errors/UpstreamError.ts";
 import { DEFAULT_RETRY_MULTIPLIER, type FormatPlayObjectOptions, type InternalConfig, type PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.ts";
-import type {PlayPlatformId} from '../../core/Atomic.ts';
+import type {ComponentAuthType, PlayPlatformId} from '../../core/Atomic.ts';
 import type {SubSonicSourceConfig} from "../common/infrastructure/config/source/subsonic.ts";
 import { getSubsonicResponse, type EntryData, type OpenSubsonicExtensionsResponse, type SubsonicNowPlayingResponse, type SubsonicResponse, type SubsonicResponseCommon } from "../common/vendor/subsonic/interfaces.ts";
 import { removeDuplicates } from "../utils.ts";
@@ -22,7 +22,9 @@ import { baseFormatPlayObj } from '../utils/PlayTransformUtils.ts';
 import { noRetryOnUpstreamError, tryApiCall } from '../utils/RequestUtils.ts';
 import { artistNameToCredit } from '../../core/StringUtils.ts';
 import { timeToHumanTimestamp, todayAwareFormat } from '../../core/TimeUtils.ts';
-import type { ComponentSourceApiJson, SubsonicSourceApiJson } from '../../core/Api.ts';
+import type { SubsonicSourceApiJson } from '../../core/Api.ts';
+import { isSuperAgentResponseError } from '../common/errors/ErrorUtils.ts';
+import { AuthError } from '../common/errors/MSErrors.ts';
 
 dayjs.extend(isSameOrAfter);
 
@@ -39,6 +41,7 @@ interface SourceIdentifierData {
 export class SubsonicSource extends MemoryPositionalSource {
 
     requiresAuth = true;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
 
     multiPlatform: boolean = true;
 
@@ -341,7 +344,8 @@ export class SubsonicSource extends MemoryPositionalSource {
             this.logger.info('Subsonic API Status: ok');
             return true;
         } catch (e) {
-            throw e;
+            const superagentError = findCauseByFunc<request.ResponseError>(e, (ee) => isSuperAgentResponseError(ee));
+            throw new AuthError('Failed to authenticate', {cause: e, unrecoverable: superagentError !== undefined && [403,401].includes(superagentError.status)})
         }
     }
 

src/backend/sources/TealfmSource.ts

@@ -1,5 +1,5 @@
 import type EventEmitter from "events";
-import { type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
+import { COMPONENT_AUTH_TYPE, type ComponentAuthType, PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.ts";
 import type {FormatPlayObjectOptions, InternalConfig} from "../common/infrastructure/Atomic.ts";
 import type {RecentlyPlayedOptions} from "./AbstractSource.ts";
@@ -13,6 +13,7 @@ import { parseArrayFromMaybeString } from "../utils/StringUtils.ts";
 export default class TealfmSource extends MemorySource {
 
     client: TealApiClient;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = false;
 
@@ -119,7 +120,7 @@ export default class TealfmSource extends MemorySource {
                 || (x.meta.musicService === undefined && !this.serviceAllow.includes('unknown'))
             );
         }
-        return plays;
+        return plays.map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.history}}));
     }
 
     getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise<PlayObject[]> => {

src/backend/sources/VLCSource.ts

@@ -2,13 +2,13 @@ import { parseRegexSingle, parseToRegex } from "@foxxmd/regex-buddy-core";
 import type { EventEmitter } from "events";
 import * as VLC from "vlc-client"
 import type {VlcMeta, VlcStatus} from "vlc-client/dist/Types.js";
-import type {PlayObject, PlayObjectMinimal} from "../../core/Atomic.ts";
+import type {ComponentAuthType, PlayObject, PlayObjectMinimal} from "../../core/Atomic.ts";
 import {
     type FormatPlayObjectOptions,
     type InternalConfig,
     type PlayerStateData,
 } from "../common/infrastructure/Atomic.ts";
-import { SINGLE_USER_PLATFORM_ID } from '../../core/Atomic.ts';
+import { COMPONENT_AUTH_TYPE, SINGLE_USER_PLATFORM_ID } from '../../core/Atomic.ts';
 import { REPORTED_PLAYER_STATUSES } from '../../core/Atomic.ts';
 import type {ReportedPlayerStatus} from '../../core/Atomic.ts';
 import type {VlcAudioMeta, VLCSourceConfig, PlayerState} from "../common/infrastructure/config/source/vlc.ts";
@@ -19,6 +19,7 @@ import { MemoryPositionalSource } from "./MemoryPositionalSource.ts";
 import { isDebugMode } from "../utils.ts";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.ts";
 import { artistNamesToCredits } from "../../core/StringUtils.ts";
+import { AuthError } from "../common/errors/MSErrors.ts";
 
 const CLIENT_PLAYER_STATE: Record<PlayerState, ReportedPlayerStatus> = {
     'playing': REPORTED_PLAYER_STATUSES.playing,
@@ -28,6 +29,7 @@ const CLIENT_PLAYER_STATE: Record<PlayerState, ReportedPlayerStatus> = {
 
 export class VLCSource extends MemoryPositionalSource {
     declare config: VLCSourceConfig;
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
 
     host?: string
     port?: number
@@ -117,7 +119,7 @@ export class VLCSource extends MemoryPositionalSource {
             return true;
         } catch (e) {
             let friendlyError: string | undefined;
-            throw new Error(`Could not connect to VLC server${friendlyError !== undefined ? ` (Hint: ${friendlyError})` : ''}`, {cause: e});
+            throw new AuthError(`Could not connect to VLC server${friendlyError !== undefined ? ` (Hint: ${friendlyError})` : ''}`, {cause: e});
         }
     }
 

src/backend/sources/WebScrobblerSource.ts

@@ -1,6 +1,6 @@
 import dayjs, { type Dayjs } from "dayjs";
 import type EventEmitter from "events";
-import { type PlayObjectMinimal, SOURCE_SOT } from "../../core/Atomic.ts";
+import { PARSED_FROM, type PlayObjectMinimal, SOURCE_SOT } from "../../core/Atomic.ts";
 import {
     type FormatPlayObjectOptions,
     type InternalConfig,
@@ -143,7 +143,7 @@ export class WebScrobblerSource extends MemorySource {
             },
             meta: {
                 trackId: uniqueID,
-                parsedFrom: connectorLabel,
+                parsedFrom: PARSED_FROM.ingress,
                 url: {
                     web: trackUrl,
                     origin: originUrl
@@ -166,8 +166,8 @@ export class WebScrobblerSource extends MemorySource {
             return false;
         }
 
-        if (playObj.meta.parsedFrom !== undefined) {
-            const lowerSource = playObj.meta.parsedFrom.toLowerCase();
+        if (playObj.meta.musicService !== undefined) {
+            const lowerSource = playObj.meta.musicService.toLowerCase();
             if (Array.isArray(this.config.data.blacklist) && this.config.data.blacklist.length > 0) {
                 if (this.config.data.blacklist.some(x => x === lowerSource)) {
                     this.logger.debug(`Will not scrobble play because it is from a blacklisted connector '${lowerSource}'`);
@@ -196,10 +196,11 @@ export class WebScrobblerSource extends MemorySource {
                 this.setStatus('Received Play');
             }
             if (stateData.play.meta.nowPlaying === false) {
-                const discovered = await this.discover([stateData.play]);
-                if (discovered.length > 0) {
-                    await this.scrobble(discovered);
-                }
+                await this.queuePlay([stateData.play]);
+                // const discovered = await this.discover([stateData.play]);
+                // if (discovered.length > 0) {
+                //     await this.scrobble(discovered);
+                // }
             }
         }
 

src/backend/sources/YTMusicSource.ts

@@ -1,6 +1,6 @@
 import dayjs, { type Dayjs } from "dayjs";
 import type EventEmitter from "events";
-import type {PlayObject, PlayObjectMinimal} from "../../core/Atomic.ts";
+import {COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, type PlayObjectMinimal} from "../../core/Atomic.ts";
 import type {FormatPlayObjectOptions, InternalConfig} from "../common/infrastructure/Atomic.ts";
 import type {YTMusicSourceConfig} from "../common/infrastructure/config/source/ytmusic.ts";
 import { Innertube, UniversalCache, Parser, YTNodes, type ApiResponse, type IBrowseResponse, Log, type SessionOptions } from 'youtubei.js';
@@ -113,6 +113,7 @@ const getGoogleOauthOpts = (): GenerateAuthUrlOpts => {
 
 export default class YTMusicSource extends AbstractSource {
 
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
     requiresAuth = true;
     requiresAuthInteraction = true;
 
@@ -279,7 +280,11 @@ Redirect URI  : ${this.redirectUri}`);
     }
 
     reauthenticate = async () => {
-        await this.tryStopPolling();
+        try {
+            await this.tryStopPolling('Reauthenticating yt');
+        } catch (e) {
+            this.logger.warn(new Error('Failed to stop polling but will try to reauth anyway', {cause: e}));
+        }
         if(this.authed) {
             await this.clearCredentials();
             this.authed = false;
@@ -690,8 +695,7 @@ ${humanDiff}`;
             if(this.transientDiscovered.data.length === 0) {
                 // and add to discovered since its empty
                 for(const refPlay of reversedPlays) {
-                    //this.transientDiscovered.add(refPlay);
-                    await this.addPlayToDB(refPlay);
+                    this.transientDiscovered.add(refPlay);
                 }
             }
         }

src/backend/tests/component/transformers.test.ts

@@ -34,6 +34,12 @@ class TestComponent extends AbstractComponent {
     protected getIdentifier(): string {
         return 'test';
     }
+    public async start(opts?: { forceInit?: boolean; }): Promise<boolean> {
+        return true;
+    }
+    public async stop(opts?: { reason?: string | Error; }): Promise<void> {
+        return;
+    }
     constructor(config?: Omit<AbstractComponentConfig, 'id'>) {
         super({id: `test-${Date.now()}`, ...(config ?? {})});
     }
@@ -700,14 +706,14 @@ describe('Play Transforms', function () {
 
             component.buildTransformRules();
             const play = generatePlay({ track: 'My cool something track', artists: artistNamesToCredits([str]), playDate: dayjs().subtract(10, 'm') });
-            const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare, 'all');
+            const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare, {log: 'all'});
             expect(transformed.data.track).equal('My cool bar track');
             expect(artistCreditsToNames(transformed.data.artists)).eql(primaries.concat(secondaries));
 
             const cachablePlay = clone(play);
             const laterDate = dayjs().subtract(5, 'm');
             cachablePlay.data.playDate = laterDate;
-            const cacheTransformed = await component.transformPlay(cachablePlay, TRANSFORM_HOOK.preCompare, 'all');
+            const cacheTransformed = await component.transformPlay(cachablePlay, TRANSFORM_HOOK.preCompare, {log: 'all'});
             expect(cacheTransformed.data.track).equal('My cool bar track');
             expect(artistCreditsToNames(cacheTransformed.data.artists)).eql(primaries.concat(secondaries));
             expect(cacheTransformed.data.playDate.isSame(cachablePlay.data.playDate));

src/backend/tests/database/drizzle.test.ts

@@ -125,7 +125,7 @@ describe('Migrations', function () {
             try {
                 await fs.cp(path.resolve(projectRootDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true });
                 const mf = path.resolve('./migrations');
-                const [db, _] = await getMigratedDb(dbPath, {migrationsFolder: mf});
+                const [db, _] = await getMigratedDb(dbPath, {migrationsFolder: mf, migrationsAppFolder: process.cwd()});
                 await migrateDb(db, { migrationsFolder: mf });
                 const res = await x('drizzle-kit', [
                     'generate',
@@ -144,7 +144,7 @@ describe('Migrations', function () {
                 const newMigrationFolder = (await fs.readdir(path.resolve('./migrations/'))).find(x => x.includes('newMigration'));
                 await fs.appendFile(path.resolve('./migrations/',newMigrationFolder, 'migration.sql'),`\nselect count(*) from plays;`);
 
-                await getMigratedDb(dbPath, {migrationsFolder: mf});
+                await getMigratedDb(dbPath, {migrationsFolder: mf, migrationsAppFolder: process.cwd()});
                 const contents = await fs.readdir(path.resolve('./'));
                 const bakPattern = new RegExp(/ms\.db\.\d+\.bak/);
                 expect(contents.some(x => bakPattern.test(x))).is.true;

src/backend/tests/scrobbler/TestScrobbler.ts

@@ -1,6 +1,6 @@
 import EventEmitter from "events";
 import request from "superagent";
-import type {PlayObject} from "../../../core/Atomic.ts";
+import {COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject} from "../../../core/Atomic.ts";
 import AbstractScrobbleClient from "../../scrobblers/AbstractScrobbleClient.ts";
 import type {CommonClientConfig, CommonClientOptions, NowPlayingOptions} from "../../common/infrastructure/config/client/index.ts";
 import clone from "clone";
@@ -12,6 +12,8 @@ import type { DrizzleQueueRepository } from "../../common/database/drizzle/repos
 import type {PlaySelect} from "../../common/database/drizzle/drizzleTypes.ts";
 import dayjs from "dayjs";
 import type { MarkOptional, MarkRequired } from "ts-essentials";
+import { AuthError } from "../../common/errors/MSErrors.ts";
+import { isSuperAgentResponseError } from "../../common/errors/ErrorUtils.ts";
 
 export class TestScrobbler extends AbstractScrobbleClient {
 
@@ -60,8 +62,11 @@ export class TestScrobbler extends AbstractScrobbleClient {
 }
 
 export class TestAuthScrobbler extends TestScrobbler {
+    override authType: ComponentAuthType = COMPONENT_AUTH_TYPE.unattended;
+
     constructor() {
         super();
+
         this.requiresAuth = true;
     }
     doAuthentication = async() => {
@@ -69,7 +74,7 @@ export class TestAuthScrobbler extends TestScrobbler {
             await request.get('http://example.com');
             return true;
         } catch (e) {
-            throw e;
+            throw new AuthError('Failed to auth', {cause: e, unrecoverable: isSuperAgentResponseError(e) && [401,403].includes(e.status)});
         }
     }
 }

src/backend/tests/scrobbler/scrobblers.test.ts

@@ -6,7 +6,7 @@ import dayjs from "dayjs";
 import { after, describe, it } from 'mocha';
 import { http, HttpResponse } from 'msw';
 import pEvent from 'p-event';
-import { CLIENT_INGRESS_QUEUE, type PlayObject, SOURCE_SOT } from "../../../core/Atomic.ts";
+import { INGRESS_QUEUE, type PlayObject, QUEUE_STATUS_FAILED, SOURCE_SOT } from "../../../core/Atomic.ts";
 import { sleep, sortByOldestPlayDate } from "../../utils.ts";
 import { genGroupIdStr } from '../../../core/PlayUtils.ts';
 import mixedDuration from '../plays/mixedDuration.json' with { type: 'json' };
@@ -91,10 +91,10 @@ describe('Networking', function () {
                 async function() {
                     const authScrobbler = new TestAuthScrobbler();
                     try {
-                        await authScrobbler.testAuth();
+                        await authScrobbler.initialize();
                     } catch (e) { /* empty */ }
                     assert.isTrue(authScrobbler.authGated());
-                    assert.isFalse(authScrobbler.authFailure);
+                    assert.isFalse(authScrobbler.hasUnrecoverableAuthFailure());
                 }
             ));
 
@@ -109,10 +109,10 @@ describe('Networking', function () {
                 async function() {
                     const authScrobbler = new TestAuthScrobbler();
                     try {
-                        await authScrobbler.testAuth();
+                        await authScrobbler.initialize();
                     } catch (e) { /* empty */ }
                     assert.isTrue(authScrobbler.authGated());
-                    assert.isTrue(authScrobbler.authFailure);
+                    assert.isTrue(authScrobbler.hasUnrecoverableAuthFailure());
                 }
             ));
     });
@@ -528,7 +528,7 @@ describe('Upstream Scrobbles', function() {
         const sp = spy(scrobbler, 'getScrobblesForTimeRange');
 
         const play = generatePlay({playDate: dayjs().subtract(60, 's')});
-        await scrobbler.queueScrobble(play, 'test');
+        await scrobbler.queueScrobble(play);
         const emptied = pEvent(scrobbler.emitter, 'queueEmptied');
         scrobbler.startScrobbling(new AbortController().signal).then(() => null);
         await emptied;
@@ -548,7 +548,7 @@ describe('Upstream Scrobbles', function() {
 
         const play1 = generatePlay({playDate: dayjs().subtract(3, 'm')});
         const play2 = generatePlay({playDate: dayjs().subtract(1, 'm')});
-        await scrobbler.queueScrobble([play1, play2], 'test');
+        await scrobbler.queueScrobble([play1, play2]);
         const emptied = pEvent(scrobbler.emitter, 'queueEmptied');
         scrobbler.startScrobbling(new AbortController().signal).then(() => null);
         await emptied;
@@ -569,7 +569,7 @@ describe('Upstream Scrobbles', function() {
         const play1 = generatePlay({playDate: dayjs().subtract(3, 'm')});
         const play2 = generatePlay({playDate: dayjs().subtract(1, 'm')});
         const play3 = generatePlay({playDate: dayjs().subtract(DEFAULT_CONSOLIDATE_DURATION.add(4, 'm'))});
-        await scrobbler.queueScrobble([play1, play2, play3], 'test');
+        await scrobbler.queueScrobble([play1, play2, play3]);
         const emptied = pEvent(scrobbler.emitter, 'queueEmptied');
         scrobbler.startScrobbling(new AbortController().signal).then(() => null);
         await emptied;
@@ -589,7 +589,7 @@ describe('Upstream Scrobbles', function() {
 
         const play1 = generatePlay({playDate: dayjs().subtract(3, 'm')});
         const play2 = generatePlay({playDate: dayjs().subtract(1, 'm')});
-        await scrobbler.queueScrobble([play1], 'test');
+        await scrobbler.queueScrobble([play1]);
         const emptied = pEvent(scrobbler.emitter, 'queueEmptied');
         scrobbler.startScrobbling(new AbortController().signal).then(() => null);
         await emptied;
@@ -597,7 +597,7 @@ describe('Upstream Scrobbles', function() {
 
         MockDate.set(dayjs().add(REFRESH_STALE_DEFAULT + 1, 's').toDate());
         const emptied2 = pEvent(scrobbler.emitter, 'queueEmptied');
-        await scrobbler.queueScrobble([play2], 'test');
+        await scrobbler.queueScrobble([play2]);
         await emptied2;
         scrobbler.tryStopScrobbling().then(() => null);
         expect(sp.calledTwice).is.true;
@@ -613,16 +613,16 @@ describe('Dead Scrobbles', function() {
         await testScrobbler.initialize();
         testScrobbler.testRecentScrobbles = [];
 
-        const queuedPlayed = await testScrobbler.playRepoTest.createPlays(generateArray<RepositoryCreatePlayOpts>(3, () => ({ ...fixtureCreatePlay(), state: 'queued', input: {} })))
+        const queuedPlays = await testScrobbler.playRepoTest.createPlays(generateArray<RepositoryCreatePlayOpts>(3, () => ({ ...fixtureCreatePlay(), state: 'failed', input: {} })));
 
-        for(const dead of queuedPlayed) {
-            await testScrobbler.addDeadLetterScrobble(dead);
+        await testScrobbler.queueRepoTest.createMany(queuedPlays.map(x => ({queueName: INGRESS_QUEUE, queueStatus: QUEUE_STATUS_FAILED, retries: 1, playId: x.id, componentId: testScrobbler.componentId})));
+
+        await testScrobbler.processDeadLetterQueue(undefined, undefined, true);
+
+        const updatedPlays = await testScrobbler.playRepoTest.findPlays({uid: queuedPlays.map(x => x.uid)});
+        for(const p of updatedPlays) {
+            expect(p.state === 'scrobbled');
         }
-
-        await testScrobbler.processDeadLetterQueue();
-        await pEvent(testScrobbler.emitter, 'queueState');
-
-        expect(testScrobbler.deadLetterQueued).eq(0);
     });
 
 });
@@ -677,8 +677,8 @@ describe('Scrobble client uses transform plays correctly', function() {
         const newScrobble = generatePlay({
             track: 'my cool track'
         });
-        await testScrobbler.queueScrobble(newScrobble, 'test');
-        const queuedPlayedData = await testScrobbler.playRepoTest.getQueued(CLIENT_INGRESS_QUEUE);
+        await testScrobbler.queueScrobble(newScrobble);
+        const queuedPlayedData = await testScrobbler.playRepoTest.getQueued(INGRESS_QUEUE);
         expect(queuedPlayedData.data[0].play.data.track).is.eq('my cool track');
         testScrobbler.scrobbleSleep = 100;
         testScrobbler.initScrobbleMonitoring().catch(console.error);
@@ -763,7 +763,7 @@ describe('Manages scrobble queue', function() {
         const newScrobble = generatePlay({
             playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds')
         });
-        await testScrobbler.queueScrobble(newScrobble, 'test');
+        await testScrobbler.queueScrobble(newScrobble);
         const res = await Promise.race([pEvent(testScrobbler.emitter, 'scrobble'), sleep(3000)]);
 
         assert.isDefined(res);
@@ -780,13 +780,13 @@ describe('Manages scrobble queue', function() {
         const dupScrobble = clone(newScrobble);
         dupScrobble.data.playDate = newScrobble.data.playDate.add(2, 'seconds');
 
-        await testScrobbler.queueScrobble(newScrobble, 'test');
+        await testScrobbler.queueScrobble(newScrobble);
         const res = await Promise.race([pEvent(testScrobbler.emitter, 'scrobble'), sleep(1500)]);
 
         assert.isDefined(res);
         assert.isDefined(res.data);
 
-        await testScrobbler.queueScrobble(dupScrobble, 'test');
+        await testScrobbler.queueScrobble(dupScrobble);
         const resDup = await Promise.race([pEvent(testScrobbler.emitter, 'scrobble'), sleep(1100)]);
 
         assert.isUndefined(resDup);
@@ -802,8 +802,8 @@ describe('Manages scrobble queue', function() {
         const dupScrobble = clone(newScrobble);
         dupScrobble.data.playDate = newScrobble.data.playDate.add(2, 'seconds');
 
-        await testScrobbler.queueScrobble(newScrobble, 'test');
-        await testScrobbler.queueScrobble(dupScrobble, 'test');
+        await testScrobbler.queueScrobble(newScrobble);
+        await testScrobbler.queueScrobble(dupScrobble);
         const res = await Promise.race([pEvent(testScrobbler.emitter, 'scrobble'), sleep(1500)]);
 
         assert.isDefined(res);
@@ -830,7 +830,7 @@ describe('Manages scrobble queue', function() {
 
         testScrobbler.scrobbleDelay = 600;
 
-        await testScrobbler.queueScrobble([newScrobble1, newScrobble2, newScrobble3], 'test');
+        await testScrobbler.queueScrobble([newScrobble1, newScrobble2, newScrobble3]);
         await pEvent(testScrobbler.emitter, 'scrobble');
         const initial = dayjs();
         await pEvent(testScrobbler.emitter, 'scrobble');
@@ -1187,7 +1187,7 @@ describe('Scrobble Clients Behavior', function() {
                 pEvent(testClient.emitter, 'scrobbleQueued'),
                 sleep(10)
             ]);
-            const queued = await testClient.getQueued(CLIENT_INGRESS_QUEUE);
+            const queued = await testClient.getQueued(INGRESS_QUEUE);
             expect(queued.data).is.empty;
         });
 
@@ -1214,7 +1214,7 @@ describe('Scrobble Clients Behavior', function() {
                 pEvent(testClient.emitter, 'scrobbleQueued'),
                 sleep(100)
             ])
-            const queued = await testClient.getQueued(CLIENT_INGRESS_QUEUE);
+            const queued = await testClient.getQueued(INGRESS_QUEUE);
             expect(queued.data).is.not.empty;
         });
 
@@ -1241,7 +1241,7 @@ describe('Scrobble Clients Behavior', function() {
                 pEvent(testClient.emitter, 'scrobbleQueued'),
                 sleep(50)
             ])
-            const queued = await testClient.getQueued(CLIENT_INGRESS_QUEUE);
+            const queued = await testClient.getQueued(INGRESS_QUEUE);
             expect(queued.data).is.not.empty;
         });
 
@@ -1268,7 +1268,7 @@ describe('Scrobble Clients Behavior', function() {
                 pEvent(testClient.emitter, 'scrobbleQueued'),
                 sleep(50)
             ])
-            const queued = await testClient.getQueued(CLIENT_INGRESS_QUEUE);
+            const queued = await testClient.getQueued(INGRESS_QUEUE);
             expect(queued.data).is.not.empty;
         });
 
@@ -1295,7 +1295,7 @@ describe('Scrobble Clients Behavior', function() {
                 pEvent(testClient.emitter, 'scrobbleQueued'),
                 sleep(50)
             ])
-            const queued = await testClient.getQueued(CLIENT_INGRESS_QUEUE);
+            const queued = await testClient.getQueued(INGRESS_QUEUE);
             expect(queued.data).is.not.empty;
         });
 
@@ -1322,7 +1322,7 @@ describe('Scrobble Clients Behavior', function() {
                 pEvent(testClient.emitter, 'scrobbleQueued'),
                 sleep(50)
             ])
-            const queued = await testClient.getQueued(CLIENT_INGRESS_QUEUE);
+            const queued = await testClient.getQueued(INGRESS_QUEUE);
             expect(queued.data).is.not.empty;
         });
 

src/backend/tests/source/TestSource.ts

@@ -4,6 +4,8 @@ import { MemoryPositionalSource } from "../../sources/MemoryPositionalSource.ts"
 import MemorySource from "../../sources/MemorySource.ts";
 
 export class TestSource extends AbstractSource {
+    override queueIdleMs: number = 1;
+    override queueConcurrency: number = 1;
     handle(plays: PlayObject[]) {
         this.scrobble(plays);
     }

src/backend/tests/source/source.test.ts

@@ -1,7 +1,6 @@
 import { loggerTest } from "@foxxmd/logging";
 import chai, { expect } from 'chai';
 import asPromised from 'chai-as-promised';
-import EventEmitter from "events";
 import { describe, it } from 'mocha';
 import pEvent from "p-event";
 import clone from 'clone';
@@ -31,6 +30,7 @@ const emitter = new WildcardEmitter<MSBackendEventMap>();
 const generateSource = async () => {
     const source = new TestSource('spotify', 'test-basic', {id: `test-${Date.now()}`}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'},  emitter);
     await source.initialize();
+    await source.initTasks();
     return source;
 }
 const generateMemorySource = async (config: MarkOptional<SourceConfig, 'id'> = {}) => {
@@ -70,7 +70,9 @@ describe('Sources use transform plays correctly', function () {
         const newScrobble = generatePlay({
             track: 'my cool track'
         });
-        const discovered = await source.discover([newScrobble])
+        await source.queuePlay([newScrobble]);
+        await sleep(3);
+        const discovered = await source.getRecentlyDiscoveredPlays();
         expect(discovered.length).eq(1);
         expect(discovered[0].data.track).is.eq('my fun track');
     });
@@ -94,73 +96,87 @@ describe('Sources use transform plays correctly', function () {
         const newScrobble = generatePlay({
             track: 'my cool track'
         });
-        const discovered = await source.discover([newScrobble])
+
+        const pAwaiter =  pEvent(source.emitter, 'discoveredToScrobble') as Promise<MSBackendEventMap['discoveredToScrobble'][0]>;
+
+        const promiseRes = await Promise.all([
+            source.queuePlay([newScrobble]),
+            sleep(3),
+            pAwaiter
+        ]);
+        
+        //await source.queuePlay([newScrobble]);
+        //await sleep(3);
+        const discovered = await source.getRecentlyDiscoveredPlays();
         expect(discovered.length).eq(1);
         expect(discovered[0].data.track).is.eq('my cool track');
 
-        const pAwaiter =  pEvent(source.emitter, 'discoveredToScrobble') as Promise<MSBackendEventMap['discoveredToScrobble'][0]>;
-        source.handle(discovered);
-        const e = await pAwaiter;
+        //const pAwaiter =  pEvent(source.emitter, 'discoveredToScrobble') as Promise<MSBackendEventMap['discoveredToScrobble'][0]>;
+        //source.handle(discovered);
+        const e = promiseRes[2];
+        expect(e).is.not.undefined;
         const res: PlayObject[] = !Array.isArray(e.data.data) ? [e.data.data] : e.data.data;
         expect(res.length).is.eq(1);
         expect(res[0].data.track).is.eq('my fun track');
     });
 
-    it('Transforms play existing comparison', async function() {
-        await using source = await generateSource();
-        source.config.options = {
-            playTransform: {
-                compare: {
-                    existing: {
-                        type: 'user',
-                        title: [
-                            {
-                                search: 'hugely cool and very different track',
-                                replace: 'fun'
-                            }
-                        ]
-                    }
-                }
-            }
-        };
-        source.buildTransformRules();
-        const newScrobble = generatePlay({
-            track: 'my hugely cool and very different track title',
-        });
-        const discovered = await source.discover([newScrobble])
-        expect(discovered.length).eq(1);
-        expect(discovered[0].data.track).is.eq('my hugely cool and very different track title');
+    // TODO need to rework these
 
-        expect((await source.discover([newScrobble])).length).is.eq(1);
-    });
+    // it('Transforms play existing comparison', async function() {
+    //     await using source = await generateSource();
+    //     source.config.options = {
+    //         playTransform: {
+    //             compare: {
+    //                 existing: {
+    //                     type: 'user',
+    //                     title: [
+    //                         {
+    //                             search: 'hugely cool and very different track',
+    //                             replace: 'fun'
+    //                         }
+    //                     ]
+    //                 }
+    //             }
+    //         }
+    //     };
+    //     source.buildTransformRules();
+    //     const newScrobble = generatePlay({
+    //         track: 'my hugely cool and very different track title',
+    //     });
+    //     const discovered = await source.discover([newScrobble])
+    //     expect(discovered.length).eq(1);
+    //     expect(discovered[0].data.track).is.eq('my hugely cool and very different track title');
 
-    it('Transforms play candidate comparison', async function() {
-        await using source = await generateSource();
-        source.config.options = {
-            playTransform: {
-                compare: {
-                    candidate: {
-                        type: 'user',
-                        title: [
-                            {
-                                search: 'hugely cool and very different track',
-                                replace: 'fun'
-                            }
-                        ]
-                    }
-                }
-            }
-        };
-        source.buildTransformRules();
-        const newScrobble = generatePlay({
-            track: 'my hugely cool and very different track title',
-        });
-        const discovered = await source.discover([newScrobble])
-        expect(discovered.length).eq(1);
-        expect(discovered[0].data.track).is.eq('my hugely cool and very different track title');
+    //     expect((await source.discover([newScrobble])).length).is.eq(1);
+    // });
 
-        expect((await source.discover([newScrobble])).length).is.eq(1);
-    });
+    // it('Transforms play candidate comparison', async function() {
+    //     await using source = await generateSource();
+    //     source.config.options = {
+    //         playTransform: {
+    //             compare: {
+    //                 candidate: {
+    //                     type: 'user',
+    //                     title: [
+    //                         {
+    //                             search: 'hugely cool and very different track',
+    //                             replace: 'fun'
+    //                         }
+    //                     ]
+    //                 }
+    //             }
+    //         }
+    //     };
+    //     source.buildTransformRules();
+    //     const newScrobble = generatePlay({
+    //         track: 'my hugely cool and very different track title',
+    //     });
+    //     const discovered = await source.discover([newScrobble])
+    //     expect(discovered.length).eq(1);
+    //     expect(discovered[0].data.track).is.eq('my hugely cool and very different track title');
+
+    //     expect((await source.discover([newScrobble])).length).is.eq(1);
+    // });
 })
 
 
@@ -445,7 +461,11 @@ class DeezerTestSource extends DeezerInternalSource {
 
 const generateDeezerSource = async (options: DeezerInternalSourceOptions = {}) => {
     const source = new DeezerTestSource('test', {id: `test-${Date.now()}`,data: {arl: 'test'}, options}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'},  emitter);
+    source.queueIdleMs = 2;
+    source.queueConcurrency = 1;
+    source.stopPollingWaitInterval = 10;
     await source.initialize();
+    await source.startDiscoveryQueue();
     return source;
 }
 const firstPlayDate = dayjs().subtract(1, 'hour');
@@ -462,12 +482,14 @@ describe('Deezer Internal Source', function() {
             const fuzzyPlay = clone(targetPlay);
             fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
 
-            const source = await generateDeezerSource();
-            source.discover([...normalizedPlays, interimPlay]);
-
-            const discovered = await source.discover([fuzzyPlay]);
-
-            expect(discovered.length).to.eq(1);
+            await using source = await generateDeezerSource();
+            const queued = await source.queuePlay([...normalizedPlays, interimPlay]);
+            expect(queued).length(normalizedPlays.length + 1);
+            await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+            expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 1);
+            await source.queuePlay([fuzzyPlay]);
+            await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+            expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 2);
         });
     });
 
@@ -480,11 +502,12 @@ describe('Deezer Internal Source', function() {
             fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
 
             await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true});
-            await source.discover([...normalizedPlays, interimPlay]);
-
-            const discovered = await source.discover([fuzzyPlay]);
-
-            expect(discovered.length).to.eq(0);
+            const queued = await source.queuePlay([...normalizedPlays, interimPlay]);
+            expect(queued).length(normalizedPlays.length + 1);
+            await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+            await source.queuePlay([fuzzyPlay]);
+            await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+            expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 1);
         });
 
         it('discovers fuzzy play when it is the last play ', async function() {
@@ -493,11 +516,12 @@ describe('Deezer Internal Source', function() {
             fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
 
             await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true});
-            await source.discover(normalizedPlays);
+            const queued = await source.queuePlay(normalizedPlays);
+            expect(queued).length(normalizedPlays.length);
 
-            const discovered = await source.discover([fuzzyPlay]);
-
-            expect(discovered.length).to.eq(1);
+            await source.queuePlay([fuzzyPlay]);
+            await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+            expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 1);
         });
 
         it('discovers fuzzy play when it is played consecutively', async function() {
@@ -506,10 +530,11 @@ describe('Deezer Internal Source', function() {
             fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
             const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate});
 
-            await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true});
-            const discovered = await source.discover(morePlays);
-
-            expect(discovered.length).to.eq(morePlays.length);
+            await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: false});
+            const queued = await source.queuePlay(morePlays);
+            expect(queued).length(morePlays.length);
+            await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+            expect(await source.getRecentlyDiscoveredPlays()).length(morePlays.length);
         });
     });
 
@@ -522,11 +547,14 @@ describe('Deezer Internal Source', function() {
                 fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
 
                 await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
-                await source.discover([...normalizedPlays, interimPlay]);
+                const queued = await source.queuePlay([...normalizedPlays, interimPlay]);
+                expect(queued).length(normalizedPlays.length + 1);
+                await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
 
-                const discovered = await source.discover([fuzzyPlay]);
+                await source.queuePlay([fuzzyPlay]);
+                await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
 
-                expect(discovered.length).to.eq(0);
+                expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 1);
             });
 
             it('does not discover play found during duration of previous', async function() {
@@ -536,11 +564,13 @@ describe('Deezer Internal Source', function() {
                 duringPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration * 0.5, 's');
 
                 await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
-                await source.discover([...normalizedPlays, interimPlay]);
+                //await source.discover([...normalizedPlays, interimPlay]);
+                await source.queuePlay([...normalizedPlays, interimPlay]);
+                await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+                await source.queuePlay([duringPlay]);
+                await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
 
-                const discovered = await source.discover([duringPlay]);
-
-                expect(discovered.length).to.eq(0);
+                expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 1)
             });
 
             it('does not discover fuzzy play with delay of up to 40 seconds', async function() {
@@ -550,11 +580,12 @@ describe('Deezer Internal Source', function() {
                 fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration + 39, 's');
 
                 await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
-                await source.discover([...normalizedPlays, interimPlay]);
+                await source.queuePlay([...normalizedPlays, interimPlay]);
+                await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+                await source.queuePlay([fuzzyPlay]);
+                await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
 
-                const discovered = await source.discover([fuzzyPlay]);
-
-                expect(discovered.length).to.eq(0);
+                expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 1)
             });
 
             it('it does not discover fuzzy play when it is the last play ', async function() {
@@ -563,11 +594,11 @@ describe('Deezer Internal Source', function() {
                 fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
 
                 await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
-                await source.discover(normalizedPlays);
-
-                const discovered = await source.discover([fuzzyPlay]);
-
-                expect(discovered.length).to.eq(0);
+                await source.queuePlay(normalizedPlays);
+                await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+                await source.queuePlay([fuzzyPlay]);
+                await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+                expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length);
             });
 
             it('does not discover fuzzy play when it is played consecutively', async function() {
@@ -577,9 +608,9 @@ describe('Deezer Internal Source', function() {
                 const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate});
 
                 await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
-                const discovered = await source.discover(morePlays);
-
-                expect(discovered.length).to.eq(morePlays.length - 1);
+                await source.queuePlay(morePlays);
+                await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+                expect(await source.getRecentlyDiscoveredPlays()).length(morePlays.length - 1);
             });
 
         });

src/backend/tests/ytm/ytm.test.ts

@@ -12,6 +12,7 @@ import { sleep } from '../../utils.ts';
 import dayjs from 'dayjs';
 import type {ApiResponse} from 'youtubei.js';
 import type { MarkOptional } from 'ts-essentials';
+import pEvent from 'p-event';
 
 chai.use(asPromised);
 
@@ -168,7 +169,8 @@ describe('Handles interim tracks', function () {
 
     it(`Does not add skipped plays`, async function () {
 
-        const source = await createYtSource();
+        await using source = await createYtSource();
+        await source.startDiscoveryQueue();
 
         const plays = [...generatePlays(10, {playDate: dayjs().subtract(20, 'seconds')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(20, 'seconds')}, { comment: 'Yesterday' })];
 
@@ -176,7 +178,9 @@ describe('Handles interim tracks', function () {
         expect(source.parseRecentAgainstResponse(plays).plays).length(20);
 
         source.polling = true;
-        await source.discover(plays);
+        await source.queuePlay(plays);
+        await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+        //await source.discover(plays);
 
         // first true poll emulating no new tracks played (should not add new tracks from base truth)
         expect(source.parseRecentAgainstResponse(plays).plays).length(0);
@@ -195,7 +199,8 @@ describe('Handles interim tracks', function () {
 
     it(`Adds interim plays when discover time is plausible`, async function () {
 
-        const source = await createYtSource();
+        await using source = await createYtSource();
+        await source.startDiscoveryQueue();
 
         const plays = [...generatePlays(10, {playDate: dayjs().subtract(2, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(2, 'minutes')}, { comment: 'Yesterday' })];
 
@@ -203,7 +208,9 @@ describe('Handles interim tracks', function () {
         expect(source.parseRecentAgainstResponse(plays).plays).length(20);
 
         source.polling = true;
-        await source.discover(plays);
+        await source.queuePlay(plays);
+        await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]);
+        //await source.discover(plays);
 
         // first true poll emulating no new tracks played (should not add new tracks from base truth)
         expect(source.parseRecentAgainstResponse(plays).plays).length(0);

src/backend/utils.ts

@@ -19,12 +19,11 @@ import { NO_DEVICE } from '../core/Atomic.ts';
 import type {PlayPlatformId} from '../core/Atomic.ts';
 import { genGroupIdStr } from '../core/PlayUtils.ts';
 import { durationToNormalizedTime } from '../core/TimeUtils.ts';
+import { setTimeout as delay } from 'node:timers/promises'
 
 dayjs.extend(utc);
 
-export function sleep(ms: any) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-}
+export const sleep = (ms: number, opts?: Parameters<typeof delay>[2]) => delay(ms, undefined, opts);
 
 /** sorts playObj formatted objects by playDate in ascending (oldest first) order */
 export const sortByOldestPlayDate = (a: PlayObject, b: PlayObject) => {

src/backend/utils/AsyncUtils.ts

@@ -1,5 +1,6 @@
 import type {Mapper} from "p-map";
 import { sleep } from "../utils.ts";
+import { nanoid } from "nanoid";
 
 /** https://stackoverflow.com/a/63795192/1469797 */
 export async function findAsyncSequential<T>(
@@ -70,4 +71,81 @@ export function staggerMapper<Element, NewElement>(options: StaggerOptions) {
     }
     return await mapper(x, index);
   }
+}
+
+export const consumeQueueOnce = async <T>(next: () => Promise<T | undefined>, process: (item: T) => Promise<void>, opts: {
+  concurrency: number;
+  signal: AbortSignal;
+  onError?: (e: Error) => Promise<void>, onSuccess?: () => void
+}): Promise<void> => {
+  const { concurrency, signal, onError } = opts;
+  signal.throwIfAborted();
+  const inFlight = new Set<Promise<void>>();
+  try {
+    while (true) {
+      signal.throwIfAborted();
+      if (inFlight.size >= concurrency) {
+        await Promise.race(inFlight);
+        continue;
+      }
+      const item = await next();
+      if (item === undefined) break;
+      const task = (async () => {
+        try {
+          await process(item);
+        } catch (err) {
+          await onError?.(err); // swallow so one bad item doesn't kill the loop
+        }
+      })();
+      inFlight.add(task);
+      void task.then(() => inFlight.delete(task));
+    }
+  } finally {
+    await Promise.allSettled(inFlight); // drain before sleeping or rethrowing
+  }
+};
+
+export const consumeQueue = async <T>(
+  next: (queueId: string) => Promise<T | undefined>,
+  process: (item: T, queueId: string) => Promise<T | void>,
+  opts: { 
+    concurrency: number;
+    idleMs: number;
+    signal: AbortSignal;
+    onError?: (e: Error, queueId: string) => Promise<void>,
+    onSuccess?: (item: T, queueId: string) => void,
+    onEmpty?: () => void
+  },
+): Promise<never> => {
+  const { concurrency, idleMs, signal, onError, onEmpty, onSuccess } = opts;
+  while (true) {
+    signal.throwIfAborted();
+    const inFlight = new Set<Promise<void>>();
+    try {
+      while (true) {
+        signal.throwIfAborted();
+        if (inFlight.size >= concurrency) {
+          await Promise.race(inFlight);
+          continue;
+        }
+        const qId = nanoid();
+        const item = await next(qId);
+        if (item === undefined) break;
+        const task = (async () => {
+          try {
+            await process(item, qId);
+            onSuccess?.(item, qId);
+          } catch (err) {
+            await onError?.(err, qId); // swallow so one bad item doesn't kill the loop
+          }
+        })();
+        inFlight.add(task);
+        void task.then(() => inFlight.delete(task));
+      }
+    } finally {
+      await Promise.allSettled(inFlight); // drain before sleeping or rethrowing
+    }
+    onEmpty?.();
+    await sleep(idleMs, { signal });
+  }
 }

src/client/AppNext.tsx

@@ -16,6 +16,7 @@ import { ComponentDetailedRoutable } from './components/msComponent/MSComponentD
 import { MSComponentListFetchable } from './components/msComponent/MSComponentList';
 import { Provider } from './components/Provider';
 import { SettingsContainer } from './components/settings/settings';
+import { Toaster } from './components/Toaster';
 
 function NoMatch() {
     const location = useLocation();
@@ -115,6 +116,7 @@ export const sseProviderOptions = {
 function App() {
   return (
       <Provider>
+            <Toaster/>
         {/* <Box px="4" py="2" pb="4"><AppHeader fetchable/></Box> */}
             <SSEProvider<MsSseEvent> options={sseProviderOptions}>
             <RouterProvider router={router}/>

src/client/components/ActivityDetail.tsx

@@ -1,11 +1,11 @@
-import { Accordion, Alert, Box, Code, Collapsible, Flex, HStack, Separator, Skeleton, SkeletonText, Span, Stack, useAccordionItemContext, type BadgeProps } from '@chakra-ui/react';
+import { Accordion, Alert, Box, Group, Portal, Code, Collapsible, Flex, HStack, Separator, Menu, Skeleton, SkeletonText, Span, Stack, useAccordionItemContext, type BadgeProps, type MenuItemProps, type MenuSelectionDetails, useClipboard } from '@chakra-ui/react';
 import { useSSEContext, useSSEEvent } from "@flamefrontend/sse-runtime-react";
-import { useQuery, useQueryClient, type InfiniteData } from '@tanstack/react-query';
-import React, { Fragment, useEffect, useState } from "react";
+import { useMutation, useQuery, useQueryClient, type InfiniteData } from '@tanstack/react-query';
+import React, { Fragment, useCallback, useEffect, useState, type ComponentProps } from "react";
 import { LuChevronRight } from "react-icons/lu";
 import type { MarkOptional } from "ts-essentials";
 import type { ComponentsApiJson, MsSseEvent, PaginatedResponse, PlayApiCommonDetailed, QueryPlaysOptsJson, SortPlaysByProps } from "../../core/Api";
-import { CLIENT_DEAD_QUEUE, type ComponentType, type Second } from "../../core/Atomic";
+import { INGRESS_QUEUE, type ComponentType, type Second } from "../../core/Atomic";
 import { tanQueries, useQueryWatcher } from "../queries";
 import { activityTimelineHasIssue } from "../utils/ComponentUtils";
 import { ActivityTimeline } from "./ActivityTimeline";
@@ -13,9 +13,13 @@ import { EphemeralElement, PlayStateBadge } from "./Badges";
 import { ShortDateDisplay } from "./DateDisplay";
 import { ErrorAlert } from "./ErrorAlert";
 import { ExpandCollapse } from "./ExpandCollapse";
-import { DebugCopy, ExclamationCircleIcon, ExclamationTriangleIcon, InsertedIcon, RetryButton, UpdatedIcon } from "./icons/ChakraIcons";
+import { DebugIcon, EllipsisButton, ExclamationCircleIcon, ExclamationTriangleIcon, FinishIconRaw, InsertedIcon, type PowerOffButton, RetryButton, RetryIcon, StopButton, StopIconRaw, TrashIconRaw, UpdatedIcon } from "./icons/ChakraIcons";
 import { PlayData } from "./PlayData";
 import { TextMuted } from "./TextMuted";
+import { capitalize } from '../../core/StringUtils';
+import ky from 'ky';
+import type { IconType } from 'react-icons/lib';
+import { toaster } from "./Toaster"
 
 type UseActivityQueryOptions = {
     msQuery?: QueryPlaysOptsJson
@@ -128,6 +132,7 @@ export const ActivitySummary = (props: ActivitySummaryProps) => {
             <HStack gap="1">
                 <ShortDateDisplay date={sortBy === 'played' ? play.data.playDate : play.meta?.seenAt} prefix={sortBy === 'played' ? 'Played' : 'Seen'} /><Separator orientation="vertical" height="4" />
                 <TextMuted>{play.meta?.source}</TextMuted>
+                {play.meta?.parsedFrom !== undefined && <><Separator orientation="vertical" height="4" /><TextMuted>{capitalize(play.meta?.parsedFrom)}</TextMuted></>}
             </HStack>
         </Flex>
     );
@@ -253,7 +258,7 @@ export const ActivityDetails = (props: ActivityDetailProps) => {
                 </Accordion.ItemTrigger>
                 <Accordion.ItemContent>
                     <Accordion.ItemBody>
-                        <PlayData play={original ?? activity.play} final={activity.play} />
+                        <PlayData compareDefault='Final' play={original ?? activity.play} final={activity.play} />
                     </Accordion.ItemBody>
                 </Accordion.ItemContent>
             </Accordion.Item>
@@ -304,26 +309,104 @@ export const ActivityDetailFetchable = (props: ActivityDetailFetchableProps) =>
     return <ActivityDetails componentType={props.componentType} componentName={data?.type} key={props.uid} activity={activity}/>
 }
 
+const playStateMenuItem = (Icon: IconType, value: string, name?: string) => (props: Pick<MenuItemProps, 'disabled'> = {}) => {
+    return (<Menu.Item key={value} value={value} {...props}><Box flex="1">{name ?? capitalize(value)}</Box><Icon/></Menu.Item>);
+}
+
+const MenuItemRetry = playStateMenuItem(RetryIcon, 'retry');
+const MenuItemRetryWith = playStateMenuItem(RetryIcon, 'retry', 'Retry With...');
+const MenuItemCancel = playStateMenuItem(StopIconRaw, 'cancel');
+const MenuItemTrash = playStateMenuItem(TrashIconRaw, 'delete');
+const MenuItemDebug = playStateMenuItem(DebugIcon, 'debug');
+const MenuItemFinish = playStateMenuItem(FinishIconRaw, 'finish', 'Mark Completed');
+
+const primaryActionProps: ComponentProps<typeof PowerOffButton> = {
+    margin: "1px",
+    variant: "subtle",
+    size: 'xs'
+}
+
 export const ActivityStateActions = (props: {activity: PlayApiCommonDetailed}) => {
-    let suffix: React.JSX.Element | null;
+    let suffix: React.JSX.Element | undefined;
+    let primaryAction: React.JSX.Element | undefined;
+    let menuElm: React.JSX.Element | undefined;
+    let menuItems: React.JSX.Element[] = [];
     const badgeProps: BadgeProps = {};
+
+    const clipboard = useClipboard({value: JSON.stringify(props.activity)});
+
+    const {mutate, isPending, variables, isSuccess} = useMutation({
+        mutationKey: ['playAction', props.activity.uid],
+        mutationFn: () => ky.post(`/api/components/${props.activity.componentId}/state`,{
+            json: {reason: 'User initiated from UI'}
+        })
+    });
+
+    const menuCb = useCallback((select: MenuSelectionDetails) => {
+        if(select.value === 'debug') {
+            clipboard.copy();
+            toaster.create({
+                title: 'Copied debug data to clipboard',
+                type: 'success'
+            });
+        }
+        //mutate();
+    },[mutate, props.activity]);
+
     const {
         activity: {
             queueStates = []
         } = {}
     } = props;
-    if(props.activity.state === 'failed') {
-        suffix = <RetryButton size="xs" margin="1px" variant="subtle"/>;
-        badgeProps.paddingRight = 0;
+    const hasDeadQueue = queueStates.some(x => x.queueName === INGRESS_QUEUE && x.retries > 0);
+    //badgeProps.paddingRight = 0;
+    switch(props.activity.state) {
+        case 'queued':
+            primaryAction = <StopButton size={{base: '2xs', smTo2xl: 'xs'}} color="red.400" margin="1px" variant="subtle"/>;
+            menuItems = [<MenuItemTrash disabled={isPending}/>,<MenuItemDebug/>];
+            break;
+        case 'failed':
+            primaryAction = <RetryButton size={{base: '2xs', smTo2xl: 'xs'}} margin="1px" variant="subtle"/>;
+            menuItems = [<MenuItemRetryWith disabled />, <MenuItemTrash disabled={isPending}/>,<MenuItemDebug/>];
+            if(!hasDeadQueue) {
+                menuItems.unshift(<MenuItemFinish disabled={isPending}/>)
+            }
+            break
+        default:
+            primaryAction = <RetryButton size={{base: '2xs', smTo2xl: 'xs'}} margin="1px" variant="subtle"/>;
+            menuItems = [<MenuItemRetryWith disabled />, <MenuItemTrash disabled={isPending}/>,<MenuItemDebug/>];
+            break
+
     }
-    const hasDeadQueue = queueStates.some(x => x.queueName === CLIENT_DEAD_QUEUE && x.queueStatus === 'queued');
+    if(menuItems.length > 0) {
+            menuElm = (
+        <Menu.Root positioning={{ placement: "bottom-end" }} onSelect={menuCb}>
+            <Group attached>
+            {primaryAction}
+            <Menu.Trigger asChild>
+                <EllipsisButton hideBelow="sm" disabled={isPending} {...primaryActionProps}/>
+            </Menu.Trigger>
+            </Group>
+            <Portal>
+            <Menu.Positioner>
+                <Menu.Content>
+                {menuItems}
+                </Menu.Content>
+            </Menu.Positioner>
+            </Portal>
+        </Menu.Root>
+            );
+            suffix = menuElm;
+        } else if(primaryAction !== undefined) {
+            suffix = primaryAction;
+        }
+        if(suffix !== undefined || primaryAction !== undefined) {
+            badgeProps.paddingRight = 0;
+        }
     return (
         <Stack>
             <HStack>
-                <PlayStateBadge {...badgeProps} minH="32px" alignItems="anchor-center" size="lg" hasDeadQueue={hasDeadQueue} state={props.activity.state} suffix={suffix} />
-            </HStack>
-            <HStack justifyContent="flex-end">
-                <DebugCopy variant="ghost" value={JSON.stringify(props.activity)}/>
+                <PlayStateBadge {...badgeProps} minH="32px" alignItems="anchor-center" size={{base: 'xs', sm: 'lg', mdTo2xl: 'lg'}} hasDeadQueue={hasDeadQueue} state={props.activity.state} suffix={suffix} />
             </HStack>
         </Stack>
     )

src/client/components/ActivityTimeline.tsx

@@ -1,14 +1,13 @@
 import type { Collapsible } from '@chakra-ui/react';
-import { Card, Icon, SkeletonCircle, SkeletonText, Span, Tabs, Timeline} from '@chakra-ui/react';
-import type { Dayjs } from "dayjs";
+import { Card, Text, Icon, SkeletonCircle, SkeletonText, Span, Tabs, Timeline} from '@chakra-ui/react';
 import dayjs from "dayjs";
 import React from "react";
 import { BiWrench } from "react-icons/bi";
 import { HiMiniMagnifyingGlass } from "react-icons/hi2";
 import { IoMdCodeDownload } from "react-icons/io";
 import { TbDatabaseEdit } from "react-icons/tb";
-import type {PlayApiCommonDetailed, QueueStateApi} from "../../core/Api";
-import { CLIENT_DEAD_QUEUE, CLIENT_INGRESS_QUEUE, QUEUE_STATUS_COMPLETED, QUEUE_STATUS_FAILED, QUEUE_STATUS_QUEUED, type ComponentType, type JsonPlayObject, type LifecycleStep, type PlayMatchResult, type ScrobbleResult } from "../../core/Atomic";
+import type {PlayApiCommonDetailed} from "../../core/Api";
+import { DEAD_QUEUE, QUEUE_STATUS_COMPLETED, QUEUE_STATUS_FAILED, QUEUE_STATUS_QUEUED, type ComponentType, type JsonPlayObject, type LifecycleStep, type PlayMatchResult, type ScrobbleResult } from "../../core/Atomic";
 import { sortByNewestDate } from "../../core/PlayUtils";
 import { capitalizeWords } from "../../core/StringUtils";
 import { shortTodayAwareFormat } from "../../core/TimeUtils";
@@ -24,6 +23,8 @@ import { ScrobbleMatchResult } from "./ScrobbleMatchResult";
 import { TimelineErrorIcon } from "./timeline/TimelineIcon";
 import { diffElements, TransformSteps } from "./TransformSteps";
 import { Muted } from "./Typography";
+import type { PlayEventPlayStateChange, PlayEventQueueStateChange } from '../../core/PlayEvent';
+import { FaInfo } from "react-icons/fa6";
 
 
 interface ActivityTimelineProps {
@@ -56,22 +57,6 @@ const TimelineLoading = () => (
         </Timeline.Root>
     )
 
-const QueuedCreatedItem = (props: { dead?: boolean, datetime: string }) => (
-    <Timeline.Item>
-        <Timeline.Connector>
-            <Timeline.Separator />
-            <Timeline.Indicator>
-                <TimelineIndicatorIconQueued {...timelineIconProps} />
-            </Timeline.Indicator>
-        </Timeline.Connector>
-        <Timeline.Content>
-            <Timeline.Title>
-                <TimelineItemSummaryText>{props.dead ? 'Dead ' : ''}Queued <Muted>at</Muted> {shortTodayAwareFormat(dayjs(props.datetime))}</TimelineItemSummaryText>
-            </Timeline.Title>
-        </Timeline.Content>
-    </Timeline.Item>
-)
-
 const NewItem = (props: Pick<ActivityTimelineProps, 'collapsibleOpen' | 'activity' | 'componentType'>) => {
     const {
         activity: {
@@ -166,6 +151,7 @@ const TransformsItem = (props: Pick<ActivityTimelineProps, 'activity' | 'collaps
                 <MSCollapsible
                     triggerProps={timelineCollapsibleProps}
                     indicator={<TimelineItemSummaryText>{transformVerb} <Muted>using configured Rules</Muted> <Muted>for</Muted> {steps[0].hook} {transformResult}</TimelineItemSummaryText>}
+                    unmountOnExit
                     defaultOpen={collapsibleOpen}
                     timeline>
                     <Card.Root bgColor="bg.muted" size="sm">
@@ -180,24 +166,6 @@ const TransformsItem = (props: Pick<ActivityTimelineProps, 'activity' | 'collaps
     )
 }
 
-const NoTransformsItem = () => (
-    <Timeline.Item>
-        <Timeline.Connector>
-            <Timeline.Separator />
-            <Timeline.Indicator>
-                <Icon {...timelineIconProps}>
-                    <BiWrench />
-                </Icon>
-            </Timeline.Indicator>
-        </Timeline.Connector>
-        <Timeline.Content>
-            <Timeline.Title >
-                <TimelineItemSummaryText>Play <Muted>was</Muted> not transformed <Muted>because no</Muted> Transform Rules <Muted> were used/configured.</Muted></TimelineItemSummaryText>
-            </Timeline.Title>
-        </Timeline.Content>
-    </Timeline.Item>
-)
-
 const ScrobbleMatchItem = (props: Pick<ActivityTimelineProps, 'collapsibleOpen'> & { match: PlayMatchResult<string> }) => {
     const {
         match,
@@ -221,6 +189,7 @@ const ScrobbleMatchItem = (props: Pick<ActivityTimelineProps, 'collapsibleOpen'>
                         indicator={<TimelineItemSummaryText><Muted>Found </Muted>{match.match ? <Span color="orange.solid"> a duplicate Scrobble</Span> : 'no duplicate Scrobbles'}</TimelineItemSummaryText>}
                         defaultOpen={collapsibleOpen}
                         disableUntil="md"
+                        unmountOnExit
                         timeline>
                         <Card.Root bgColor="bg.muted" size="sm">
                             <Card.Body textStyle="sm">
@@ -280,6 +249,7 @@ const ScrobbleResponseItem = (props: Pick<ActivityTimelineProps, 'collapsibleOpe
                         indicator={scrobbleSummary}
                         defaultOpen={collapsibleOpen}
                         timeline
+                        unmountOnExit
                         disableUntil="md">
                         <Card.Root bgColor="bg.muted" size="sm">
                             <Card.Body textStyle="sm">
@@ -293,84 +263,144 @@ const ScrobbleResponseItem = (props: Pick<ActivityTimelineProps, 'collapsibleOpe
     )
 }
 
-const QueueTimelineItem = (props: {queueState: QueueStateApi, collapsibleOpen: boolean}) => {
+const QueueTimelineItem = (props: {queueState: PlayEventQueueStateChange<string>, collapsibleOpen: boolean}) => {
     const {
-        queueState,
+        queueState: {
+            data: {
+                queueStatus,
+                queueName,
+                error,
+                retries
+            },
+            createdAt,
+        } = {},
         collapsibleOpen,
     } = props;
-    if(queueState.queueStatus === QUEUE_STATUS_QUEUED) {
-        return (
-            <Timeline.Item>
-                    <Timeline.Connector>
-                        <Timeline.Separator />
-                        <Timeline.Indicator>
-                            <TimelineIndicatorIconQueued {...timelineIconProps}/>
-                        </Timeline.Indicator>
-                    </Timeline.Connector>
-                    <Timeline.Content gap="4">
-                        <Timeline.Title>
-                            <TimelineItemSummaryText>{queueState.queueName === CLIENT_DEAD_QUEUE ? 'Dead ' : ''}Queued <Muted>at</Muted> {shortTodayAwareFormat(dayjs(queueState.updatedAt))}</TimelineItemSummaryText>
-                        </Timeline.Title>
-                    </Timeline.Content>
-                </Timeline.Item>
-        );
+
+    let indicator: React.JSX.Element,
+    text: React.JSX.Element,
+    title: React.JSX.Element;
+
+    switch(queueStatus) {
+        case QUEUE_STATUS_QUEUED:
+            indicator = <TimelineIndicatorIconQueued {...timelineIconProps} />;
+            text = <TimelineItemSummaryText>{queueName === DEAD_QUEUE ? 'Dead ' : ''}Queued <Muted>at</Muted> {shortTodayAwareFormat(dayjs(createdAt))}</TimelineItemSummaryText>;
+            break;
+        case QUEUE_STATUS_COMPLETED:
+            indicator = <CheckIcon color="green.focusRing" {...timelineIconProps}/>;
+            text = <TimelineItemSummaryText>{queueName === DEAD_QUEUE ? 'Dead ' : ''}Queue finished processing <Muted>at</Muted> {shortTodayAwareFormat(dayjs(createdAt))}</TimelineItemSummaryText>;
+            break;
+        case QUEUE_STATUS_FAILED:
+            indicator = <ExclamationTriangleIcon color="orange.focusRing" {...timelineIconProps}/>;
+            text = <TimelineItemSummaryText>{queueName === DEAD_QUEUE ? 'Dead ' : ''}Queue failed <Muted>at</Muted> {shortTodayAwareFormat(dayjs(createdAt))}</TimelineItemSummaryText>;
     }
 
-    if(queueState.queueStatus === QUEUE_STATUS_COMPLETED) {
-        return (
-            <Timeline.Item>
-                    <Timeline.Connector>
-                        <Timeline.Separator />
-                        <Timeline.Indicator>
-                            <CheckIcon color="green.focusRing" {...timelineIconProps}/>
-                        </Timeline.Indicator>
-                    </Timeline.Connector>
-                    <Timeline.Content gap="4">
-                        <Timeline.Title>
-                            <TimelineItemSummaryText>{queueState.queueName === CLIENT_DEAD_QUEUE ? 'Dead ' : ''}Queue finished processing <Muted>at</Muted> {shortTodayAwareFormat(dayjs(queueState.updatedAt))}</TimelineItemSummaryText>
-                        </Timeline.Title>
-                    </Timeline.Content>
-                </Timeline.Item>
-        );
-    }
-
-    if(queueState.queueStatus === QUEUE_STATUS_FAILED) {
-        let titleContent: React.JSX.Element;
-        if(queueState.error === undefined) {
-            titleContent = <TimelineItemSummaryText>{queueState.queueName === CLIENT_DEAD_QUEUE ? 'Dead ' : ''}Queue failed <Muted>at</Muted> {shortTodayAwareFormat(dayjs(queueState.updatedAt))}</TimelineItemSummaryText>;
-        } else {
-            titleContent = (
-                <MSCollapsible 
+    if(error !== undefined && error !== null) {
+        title = (
+            <MSCollapsible 
                 triggerProps={timelineCollapsibleProps}
-                indicator={<TimelineItemSummaryText>{queueState.queueName === CLIENT_DEAD_QUEUE ? 'Dead ' : ''}Queue failed <Muted>at</Muted> {shortTodayAwareFormat(dayjs(queueState.updatedAt))}</TimelineItemSummaryText>}
-                                            defaultOpen={collapsibleOpen}
-                                            disableUntil="md"
-                                            timeline>
-                                            <ErrorAlert error={queueState.error} />
-                                        </MSCollapsible>
-            )
-        }
-        return (
-            <Timeline.Item>
-                    <Timeline.Connector>
-                        <Timeline.Separator />
-                        <Timeline.Indicator>
-                            <ExclamationTriangleIcon color="orange.focusRing" {...timelineIconProps}/>
-                        </Timeline.Indicator>
-                    </Timeline.Connector>
-                    <Timeline.Content >
-                        <Timeline.Title>
-                            {titleContent}
-                        </Timeline.Title>
-                    </Timeline.Content>
-                </Timeline.Item>
-        );
+                indicator={text}
+                defaultOpen={collapsibleOpen}
+                disableUntil="md"
+                timeline
+                unmountOnExit>
+                <ErrorAlert error={error} />
+            </MSCollapsible>
+        )
+    } else {
+        title = text;
     }
+
+    return (
+        <Timeline.Item>
+                <Timeline.Connector>
+                    <Timeline.Separator />
+                    <Timeline.Indicator>
+                        {indicator}
+                    </Timeline.Indicator>
+                </Timeline.Connector>
+                <Timeline.Content gap="4">
+                    <Timeline.Title>
+                        {title}
+                    </Timeline.Title>
+                </Timeline.Content>
+            </Timeline.Item>
+    );
 }
 
-type TransformStepsTimelineData = {id: 'transform-steps', dt: Dayjs, steps: LifecycleStep[], original?: JsonPlayObject};
-type TimelineDataTypes = 'new' | 'queue-created-ingress' | 'queue-created-dead' | 'queue-updated-ingress' | 'queue-updated-dead' | 'scrobble-match' | 'scrobble-response' | 'transform-steps';
-type TimelineData = {id: TimelineDataTypes, dt: Dayjs};
+const StateChangeItem = (props: {event: PlayEventPlayStateChange<string>, collapsibleOpen: boolean}) => {
+        const {
+        event: {
+            data: {
+                state,
+                error,
+                reason
+            },
+            createdAt,
+        } = {},
+        collapsibleOpen,
+    } = props;
+
+    let color: string;
+    switch (state) {
+        case 'discarded':
+        case 'duped':
+            color = 'orange';
+            break;
+        case 'discovered':
+        case 'scrobbled':
+            color = 'green';
+            break;
+        case 'failed':
+            color = 'red';
+            break;
+        case 'queued':
+            color = 'gray';
+            break;
+    }
+
+    const text = <TimelineItemSummaryText>State <Muted>was changed to</Muted> <Span color={`${color}.solid`}>{capitalizeWords(state)}</Span> <Muted>at</Muted> {shortTodayAwareFormat(dayjs(createdAt))}</TimelineItemSummaryText>;
+
+    let content: React.JSX.Element;
+    if(error !== null && error !== undefined) {
+        content = (
+            <Timeline.Title>
+            <MSCollapsible 
+                triggerProps={timelineCollapsibleProps}
+                indicator={text}
+                defaultOpen={collapsibleOpen}
+                disableUntil="md"
+                timeline
+                unmountOnExit>
+                {reason !== undefined && reason !== null ? <Text my={2}>{reason}</Text> : null}
+                <ErrorAlert error={error} />
+            </MSCollapsible>
+            </Timeline.Title>
+        )
+    } else {
+        content = (<>
+        <Timeline.Title>
+            {text}
+        </Timeline.Title>
+        {reason !== undefined && reason !== null ? <Timeline.Description>{reason}</Timeline.Description> : null}
+        </>);
+    }
+
+    return (
+        <Timeline.Item>
+                <Timeline.Connector>
+                    <Timeline.Separator />
+                    <Timeline.Indicator>
+                        <FaInfo color="blue.focusRing"/>
+                    </Timeline.Indicator>
+                </Timeline.Connector>
+                <Timeline.Content gap="4">
+                    {content}
+                </Timeline.Content>
+            </Timeline.Item>
+    );
+
+}
 
 export const ActivityTimeline = (props: ActivityTimelineProps) => {
 
@@ -380,173 +410,45 @@ export const ActivityTimeline = (props: ActivityTimelineProps) => {
 
     const {
         activity:{
-            play,
             input,
-            seenAt,
-            queueStates,
+            events = [],
         } = {},
         collapsibleOpen,
         componentType,
         componentName
     } = props;
-    const {
-        lifecycle: steps = [],
-        scrobble: {
-            match,
-            payload,
-            createdAt: scrobbleResultCreatedAt
-        } = {},
-        scrobble,
-    } = play;
     const {
         play: original,
     } = input || {};
 
-    const timelineItems: (TimelineData|TransformStepsTimelineData)[] = [
-        {id: 'new', dt: dayjs(seenAt)},
+    events.sort((a, b) => sortByNewestDate(b.createdAt, a.createdAt));
+
+    const timelineElements: React.JSX.Element[] = [
+        <NewItem key="newPlay" activity={props.activity} collapsibleOpen={collapsibleOpen} componentType={componentType}/>
     ];
 
-        // group transforms by hook
-    const transformGroups: Record<string, LifecycleStep[]> = steps.length === 0  ? {} : steps.reduce((acc, curr) => {
-        if(acc[curr.hook] === undefined) {
-            acc[curr.hook] = [];
-        }
-        return {...acc, [curr.hook]: [...acc[curr.hook], curr]};
-    }, {});
-
     let lastTransformedPlay = original;
-    for(const [_,v] of Object.entries(transformGroups)) {
-        const d: TransformStepsTimelineData = {id: 'transform-steps', dt: dayjs(v[0].createdAt), steps: v, original: lastTransformedPlay};
-        const [__, finalPlay] = diffElements(lastTransformedPlay, v);
-        lastTransformedPlay = finalPlay;
-        timelineItems.push(d);
-    }
-
-    const ingressQueue = queueStates.find(x => x.queueName === CLIENT_INGRESS_QUEUE);
-    if(ingressQueue !== undefined) {
-        if(ingressQueue.updatedAt === ingressQueue.createdAt) {
-            // if queue was never updated but contains extra context then only show updated
-            if(ingressQueue.error !== undefined || ingressQueue.queueStatus === QUEUE_STATUS_FAILED) {
-                timelineItems.push({id: 'queue-updated-ingress', dt: dayjs(ingressQueue.updatedAt)});
-            } else {
-                timelineItems.push({id: 'queue-created-ingress', dt: dayjs(ingressQueue.createdAt)});
-            }
-        } else {
-            timelineItems.push({id: 'queue-created-ingress', dt: dayjs(ingressQueue.createdAt)});
-            timelineItems.push({id: 'queue-updated-ingress', dt: dayjs(ingressQueue.updatedAt)});
+    for(const event of events) {
+        switch(event.eventName) {
+            case 'transform': {
+                //const d: TransformStepsTimelineData = {id: 'transform-steps', dt: dayjs(event.data[0].createdAt), steps: event.data, original: lastTransformedPlay};
+                timelineElements.push(<TransformsItem key={event.id} steps={event.data} original={lastTransformedPlay}/>);
+                const [__, finalPlay] = diffElements(lastTransformedPlay, event.data);
+                lastTransformedPlay = finalPlay;
+            } break;
+            case 'queueStateChange': {
+                timelineElements.push(<QueueTimelineItem key={event.id} queueState={event} collapsibleOpen={collapsibleOpen}/>);
+            } break;
+            case 'dupeCheck': {
+                timelineElements.push(<ScrobbleMatchItem key={event.id} match={event.data} collapsibleOpen={collapsibleOpen}/>);
+            } break;
+            case 'scrobbleResult':
+                timelineElements.push(<ScrobbleResponseItem key={event.id} scrobble={event.data} componentName={componentName} collapsibleOpen={collapsibleOpen}/>);
+                break;
+            case 'playStateChange':
+                timelineElements.push(<StateChangeItem key={event.id} event={event} collapsibleOpen={collapsibleOpen}/>)
         }
     }
-    const deadqueue = queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE);
-    if(deadqueue !== undefined) {
-        if(deadqueue.updatedAt === deadqueue.createdAt) {
-            // if queue was never updated but contains extra context then only show updated
-            if(deadqueue.error !== undefined || deadqueue.queueStatus === QUEUE_STATUS_FAILED) {
-                timelineItems.push({id: 'queue-updated-dead', dt: dayjs(deadqueue.updatedAt)});
-            } else {
-                timelineItems.push({id: 'queue-created-dead', dt: dayjs(deadqueue.createdAt)});
-            }
-        } else {
-            timelineItems.push({id: 'queue-created-dead', dt: dayjs(deadqueue.createdAt)});
-            timelineItems.push({id: 'queue-updated-dead', dt: dayjs(deadqueue.updatedAt)});
-        };
-    }
-
-    if(match !== undefined) {
-        timelineItems.push({id: 'scrobble-match', dt: dayjs(match.createdAt)});
-    }
-
-    // since scrobbleResultCreatedAt has just been implemented older play data will not have it
-    // and if match was never run, due to error earlier in lifecycle, we need to fallback to oldest event + 1s
-    timelineItems.sort((a, b) => sortByNewestDate(b.dt, a.dt));
-    if(payload !== undefined) {
-        timelineItems.push({id: 'scrobble-response', dt: dayjs(scrobbleResultCreatedAt ?? match?.createdAt ?? timelineItems[timelineItems.length - 1].dt)});
-    }
-
-    // now we sort by date as well as logical order
-    timelineItems.sort((a, b) => {
-        // new is always sorted to first in order regardless of timestamp
-        if(b.id === 'new') {
-            return 1;
-        }
-        if(a.id === 'new') {
-            return -1;
-        }
-       
-        if(!a.dt.isSame(b.dt)) {
-            return a.dt.isBefore(b.dt) ? -1 : 1;
-        }
-        // if they are the same timestamp then we need to determine the likely logical order
-
-        // queue created always occurs before other actions as the play is queued first, then processed
-        if(b.id.includes('queue-created')) {
-            return 1;
-        }
-        if(a.id.includes('queue-created')) {
-            return -1;
-        }
-        
-        // transform steps always occur before scrobble actions
-        if(a.id.includes('scrobble') && b.id === 'transform-steps') {
-            return 1;
-        }
-        if(b.id.includes('scrobble') && a.id === 'transform-steps') {
-            return -1;
-        }
-
-        // dupe matching always occurs before scrobbling
-        if(b.id === 'scrobble-match' && a.id === 'scrobble-response') {
-            return 1;
-        }
-        if(a.id === 'scrobble-match' && b.id === 'scrobble-response') {
-            return -1;
-        }
-
-        // queue updated (finished) always occurs last
-        if(a.id.includes('queue-updated')) {
-            return 1;
-        }
-        if(b.id.includes('queue-updated')) {
-            return -1;
-        }
-
-        // nothing else matched, keep order
-        return 0;
-    });
-
-    const f = timelineItems;
-    console.log(f);
-
-
-    const timelineElements: React.JSX.Element[] = timelineItems.flatMap((x) => {
-        const timelineKey = `${x.id}-${x.dt.unix()}`;
-        switch(x.id) {
-            case 'new':
-                {
-                    const newElm = <NewItem key={timelineKey} activity={props.activity} collapsibleOpen={collapsibleOpen} componentType={componentType}/>;
-                    if(steps.length === 0) {
-                        return [newElm, <NoTransformsItem key={`${timelineKey}-notransform`}/>];
-                    }
-                    return newElm;
-                }
-            case 'queue-created-ingress':
-            case 'queue-created-dead':
-                return <QueuedCreatedItem key={timelineKey} dead={x.id.includes('dead')} datetime={x.dt.toISOString()}/>;
-            case 'queue-updated-ingress':
-                return <QueueTimelineItem key={timelineKey} queueState={ingressQueue} collapsibleOpen={collapsibleOpen}/>;
-            case 'queue-updated-dead':
-                return <QueueTimelineItem key={timelineKey} queueState={deadqueue} collapsibleOpen={collapsibleOpen}/>;
-            case 'scrobble-match':
-                return <ScrobbleMatchItem key={timelineKey} match={match} collapsibleOpen={collapsibleOpen}/>;
-            case 'scrobble-response':
-                return <ScrobbleResponseItem key={timelineKey} scrobble={scrobble} componentName={componentName} collapsibleOpen={collapsibleOpen}/>;
-            case 'transform-steps':
-                {
-                    const val = x as TransformStepsTimelineData;
-                    return <TransformsItem key={timelineKey} steps={val.steps} original={val.original}/>
-                }
-        }
-        return undefined;
-    });
 
     return (
         <MSErrorBoundary>

src/client/components/ArtistCreditDisplay.tsx

@@ -1,18 +1,18 @@
 import { Fragment } from 'react';
-import type {ArtistCredit as AC} from '../../core/Atomic';
+import type { ArtistCredit as AC } from '../../core/Atomic';
 
 import { HStack, Tag } from "@chakra-ui/react";
-import { SiMusicbrainz } from "react-icons/si";
-import { Tooltip } from './ChakraTooltip';
+import { MusicbrainzInfoIcon } from './musicServices/Musicbrainz';
 
-export const ArtistCredit = (props: { data: AC, showIdLink?: boolean }) => {
+export const ArtistCredit = (props: { data: AC, showLinks?: boolean, showMbid?: boolean }) => {
 
     const {
         data,
-        showIdLink = true
+        showLinks = true,
+        showMbid
     } = props;
 
-    if (!showIdLink || Object.keys(data).length === 1) {
+    if (!showLinks) {
         return data.name;
     }
 
@@ -20,19 +20,19 @@ export const ArtistCredit = (props: { data: AC, showIdLink?: boolean }) => {
         <HStack>
             {data.name}
             <HStack style={{ userSelect: 'none' }}>
-                {data.mbid !== undefined ? <Tooltip content={`MBID ${data.mbid}`} interactive><a target='__blank' href={`https://musicbrainz.org/artist/${data.mbid}`}><SiMusicbrainz /></a></Tooltip> : null}
+                {data.mbid !== undefined ? <MusicbrainzInfoIcon type="artist" mbid={data.mbid} showMbid={showMbid} link tooltip /> : null}
             </HStack>
         </HStack>
     </Fragment>
 
 }
 
-export const ArtistCreditTags = (props: { data: AC[], showIdLink?: boolean }) => {
-    return <HStack>{props.data.map((x, index) => {
-        return (
+export const ArtistCreditTags = (props: { data: AC[], showLinks?: boolean, showMbid?: boolean }) => (
+    <HStack>
+        {props.data.map((x, index) => (
             <Tag.Root key={index}>
-                <Tag.Label userSelect="all"><ArtistCredit data={x} showIdLink={props.showIdLink} /></Tag.Label>
+                <Tag.Label userSelect="all"><ArtistCredit data={x} showLinks={props.showLinks} showMbid={props.showMbid} /></Tag.Label>
             </Tag.Root>
-        );
-    })}</HStack>
-}
+        ))}
+    </HStack>
+)

src/client/components/Badges.tsx

@@ -1,8 +1,8 @@
-import { Badge, HStack, Separator } from "@chakra-ui/react";
+import { Badge, HStack, Separator, Spinner } from "@chakra-ui/react";
 import { useSSEContext, useSSEEvent } from "@flamefrontend/sse-runtime-react";
 import React, { type ComponentProps, type PropsWithChildren, useCallback, useEffect, useState } from "react";
 import { useTimeout } from 'react-use-timeout';
-import { COMPONENT_STATE, type ComponentCommonApiJson, componentStateToFriendly, type MsSseEvent, type PlayApiCommon } from "../../core/Api";
+import { COMPONENT_STATE, type ComponentCommonApiJson, type ComponentsApiJson, componentStateToFriendly, type MsSseEvent, type PlayApiCommon } from "../../core/Api";
 import type {Second} from "../../core/Atomic";
 import { capitalizeWords } from "../../core/StringUtils";
 
@@ -28,10 +28,8 @@ export const PlayStateBadge = (props: PropsWithChildren<ComponentProps<typeof Ba
         badgeText = 'Dead Queued';
       }
       break;
-    case 'discarded':
-      badgeColor = 'grey';
-      break;
     case 'duped':
+    case 'discarded':
       badgeColor = 'orange';
       break;
   }
@@ -74,14 +72,15 @@ export const EphemeralElement = (props: { expires?: Second | boolean, children:
 }
 
 export const ComponentStateBadge = (props: ComponentProps<typeof Badge> & {
-    data: Pick<ComponentCommonApiJson, 'state'>,
+    data: Pick<ComponentsApiJson, 'state' | 'errors' | 'authType'>,
     componentId?: number,
     live?: boolean,
     separator?: boolean | React.JSX.Element,
-    suffix?: React.JSX.Element
+    suffix?: React.JSX.Element,
+    loading?: boolean
 }) => {
 
-    const { data, suffix, separator, ...rest } = props;
+    const { data, suffix, separator, loading = false, ...rest } = props;
 
     const [componentState, setComponentState] = useState(data.state);
 
@@ -113,10 +112,10 @@ export const ComponentStateBadge = (props: ComponentProps<typeof Badge> & {
             badgeColor = 'red';
             break;
         case COMPONENT_STATE.IDLE:
-            badgeColor = 'orange';
+            badgeColor = 'yellow';
             break;
-        case COMPONENT_STATE.MUTED:
-            badgeColor = 'yellow';  
+        case COMPONENT_STATE.IGNORED:
+            badgeColor = 'orange';  
             break;
     }
 
@@ -129,5 +128,7 @@ export const ComponentStateBadge = (props: ComponentProps<typeof Badge> & {
         }
     }
 
-    return <Badge variant="surface" colorPalette={badgeColor} {...rest}><HStack gap="0">{componentStateToFriendly(componentState)}{sep}{suffix}</HStack></Badge>
+    return (<Badge variant="surface" colorPalette={badgeColor} {...rest}>
+        <HStack gap="0">{loading ? <Spinner animationDuration="0.8s" /> : undefined}{componentStateToFriendly(componentState)}{sep}{suffix}</HStack></Badge>
+        );
 }

src/client/components/ErrorAlert.tsx

@@ -4,6 +4,7 @@ import { isErrorIsh, type ErrorIsh } from '../../core/ErrorUtils';
 import { ChakraClip } from './ChakraClipboard';
 import { ChakraCodeBlock } from './CodeBlock';
 import { EllipsisButton, FatArrowRight } from './icons/ChakraIcons';
+import { walkError, type ErrorData } from '../utils/ComponentUtils';
 
 export interface ErrorAlertProps {
     error: ErrorIsh
@@ -86,25 +87,4 @@ export const ErrorAlert = (props: ErrorAlertProps) => {
             <ChakraClip value={props.error}/>
         </Alert.Root>
     )
-}
-
-interface ErrorData {
-    name?: string
-    code?: string
-    message?: string
-    stack?: string
-}
-
-const walkError = (err: ErrorIsh, errors: ErrorData[] = []): ErrorData[] => {
-    const thisErr: ErrorData = {
-        name: err.name,
-        code: 'code' in err ? err.code : undefined,
-        message: err.message,
-        stack: err.stack
-    };
-    errors.push(thisErr);
-    if(isErrorIsh(err.cause)) {
-        return walkError(err.cause, errors);
-    }
-    return errors;
 }

src/client/components/ExternaLinksMenu.tsx

@@ -1,6 +1,6 @@
 import { Box, Button, Menu, Portal } from "@chakra-ui/react"
 import type {ComponentProps} from "react";
-import { ExternalLinkIcon } from "./icons/ChakraIcons"
+import { ExternalLinkIconRaw } from "./icons/ChakraIcons"
 import { EXTERNAL_LINKS } from "./SideNav"
 
 
@@ -18,7 +18,7 @@ export const ExternaLinksMenu = (props: ComponentProps<typeof Box> = {}) => (
             {EXTERNAL_LINKS.items.map((link) => (
               <Menu.Item key={link.url} asChild value={link.title} cursor="pointer">
                 <a href={link.url} target="_blank" rel="noreferrer">
-                  {link.title} <ExternalLinkIcon />
+                  {link.title} <ExternalLinkIconRaw />
                 </a>
               </Menu.Item>
             ))}

src/client/components/PlayData.tsx

@@ -1,7 +1,7 @@
-import React, { useState } from 'react';
-import { EmptyState, DataList, HStack, Tabs, Box, Flex, Stack, Text, Separator, IconButton, Container, Float, Icon, Link, Span, Show } from "@chakra-ui/react"
+import React, { useCallback, useState, type ComponentProps } from 'react';
+import { EmptyState, DataList, HStack, Tabs, Box, Flex, Stack, Text, Separator, IconButton, Container, Float, Icon, Link, Span, Show, Menu, Group, Portal, type MenuItemProps, type MenuSelectionDetails } from "@chakra-ui/react"
 import { LuCode, LuText, LuCheck, LuX } from "react-icons/lu"
-import type {JsonPlayObject, PlayObjectMinimal} from '../../core/Atomic.js';
+import type { JsonPlayObject, PlayObjectMinimal } from '../../core/Atomic.js';
 import { shortTodayAwareFormat, timeToHumanTimestamp } from '../../core/TimeUtils.js';
 import dayjs from 'dayjs';
 import { ChakraCodeBlock } from './CodeBlock.js';
@@ -10,6 +10,10 @@ import { formatNumber } from '../../core/DataUtils.js';
 import { Muted } from './Typography.js';
 import { ArtistCreditTags } from './ArtistCreditDisplay.js';
 import { MSErrorBoundary } from './ErrorBoundary.js';
+import { EllipsisButton, EyeClosedIcon, EyeIcon, getMusicServiceIconElement } from './icons/ChakraIcons.js';
+import { MusicbrainzInfoIcon } from './musicServices/Musicbrainz.js';
+import type { IconType } from 'react-icons/lib';
+import { capitalize } from '../../core/StringUtils.js';
 
 const EmptyPlayData = () => {
     return (
@@ -35,6 +39,17 @@ export interface PlayInfoProps {
     dates?: false | 'all' | 'played' | 'seen'
 }
 
+const primaryActionProps: ComponentProps<typeof EllipsisButton> = {
+    //margin: "1px",
+    variant: "outline",
+    size: 'xs'
+}
+
+const menuItem = (Icon: IconType, value: string, name?: string) => (props: Pick<MenuItemProps, 'disabled'> = {}) => (<Menu.Item key={value} value={value} {...props}><Icon /><Box flex="1">{name ?? capitalize(value)}</Box></Menu.Item>)
+
+const MenuMbidShow = menuItem(EyeIcon, 'mbidShow', 'Show MBIDs');
+const MenuMbidHide = menuItem(EyeClosedIcon, 'mbidHide', 'Hide MBIDs');
+
 export const PlayData = (props?: PlayInfoProps) => {
     const {
         play,
@@ -47,56 +62,90 @@ export const PlayData = (props?: PlayInfoProps) => {
 
 
     const [codeMode, setCodeMode] = useState(false);
+    const [showMbid, setShowMBid] = useState(false);
 
-    let code: React.JSX.Element | null = null;
-
-    const comparable = showCompare && final !== undefined;
+    const menuCb = useCallback((select: MenuSelectionDetails) => {
+        if (select.value === 'mbidShow') {
+            setShowMBid(true);
+        } else if (select.value === 'mbidHide') {
+            setShowMBid(false);
+        }
+    }, [setShowMBid])
 
     if (play === undefined) {
         return <EmptyPlayData />
     }
+    let code: React.JSX.Element | null = null;
+
+    const comparable = showCompare && final !== undefined;
+
+    const menuItems: React.JSX.Element[] = [
+        showMbid ? <MenuMbidHide /> : <MenuMbidShow />
+    ];
 
     if (showCodeToggle) {
         code = (
-            <IconButton variant="outline" size="xs" onClick={() => setCodeMode(!codeMode)}>
+            <IconButton hideBelow="sm" variant="outline" size="xs" {...primaryActionProps} onClick={() => setCodeMode(!codeMode)}>
                 {codeMode ? <LuText /> : <LuCode />}
             </IconButton>
         );
     }
 
+    let content: React.JSX.Element;
+
     if (!comparable) {
-        return (<Box position="relative">
-            <Float placement="top-end" offsetX="4" offsetY="2" hideBelow="sm" zIndex={100}>{code}</Float>
-            {codeMode ? <ChakraCodeBlock code={play} /> : <PlayDataDataList play={play} dates={dates} />}
-        </Box>);
+        content = codeMode ? <ChakraCodeBlock code={play} /> : <PlayDataDataList play={play} dates={dates} showMbid={showMbid} />;
+    } else {
+        content = (
+            <Tabs.Root size="sm" variant="outline" defaultValue={compareDefault}>
+                <Tabs.List>
+                    <Tabs.Trigger value="Initial">Initial</Tabs.Trigger>
+                    <Tabs.Trigger value="Final">Final</Tabs.Trigger>
+                </Tabs.List>
+                <Tabs.Content value="Initial">
+                    {codeMode ? <ChakraCodeBlock code={play} /> : <PlayDataDataList play={play} dates={dates} showMbid={showMbid} />}
+                </Tabs.Content>
+                <Tabs.Content value="Final">
+                    {codeMode ? <ChakraCodeBlock code={final} /> : <PlayDataDataList play={final} dates={dates} showMbid={showMbid} />}
+                </Tabs.Content>
+            </Tabs.Root>
+        )
     }
 
     return (
         <Box position="relative">
             <MSErrorBoundary>
-                <Float placement="top-end" offsetX="4" offsetY="2" hideBelow="sm" zIndex={100}>{code}</Float>
-                <Tabs.Root size="sm" variant="outline" defaultValue={compareDefault}>
-                    <Tabs.List>
-                        <Tabs.Trigger value="Initial">Initial</Tabs.Trigger>
-                        <Tabs.Trigger value="Final">Final</Tabs.Trigger>
-                    </Tabs.List>
-                    <Tabs.Content value="Initial">
-                        {codeMode ? <ChakraCodeBlock code={play} /> : <PlayDataDataList play={play} dates={dates} />}
-                    </Tabs.Content>
-                    <Tabs.Content value="Final">
-                        {codeMode ? <ChakraCodeBlock code={final} /> : <PlayDataDataList play={final} dates={dates} />}
-                    </Tabs.Content>
-                </Tabs.Root>
+                <Float placement="top-end" offsetX="6" offsetY="2" zIndex={100}>
+                    <HStack>
+                        <Menu.Root positioning={{ placement: "bottom-end" }} onSelect={menuCb}>
+                            <Group attached>
+                                {code}
+                                <Menu.Trigger asChild>
+                                    <EllipsisButton {...primaryActionProps} />
+                                </Menu.Trigger>
+                            </Group>
+                            <Portal>
+                                <Menu.Positioner>
+                                    <Menu.Content>
+                                        {menuItems}
+                                    </Menu.Content>
+                                </Menu.Positioner>
+                            </Portal>
+                        </Menu.Root>
+                    </HStack>
+                </Float>
+                {content}
             </MSErrorBoundary>
         </Box>
     );
 }
 
-export const PlayDataDataList = (props: { play: JsonPlayObject, dates: DisplayDates }) => {
+export const PlayDataDataList = (props: { play: JsonPlayObject, dates: DisplayDates, showMbid?: boolean }) => {
 
     const {
         play,
-        dates
+        dates,
+        showMbid = false
     } = props;
 
 
@@ -128,16 +177,23 @@ export const PlayDataDataList = (props: { play: JsonPlayObject, dates: DisplayDa
             url: {
                 web: webUrl,
                 origin: originUrl
-            } = {}
+            } = {},
+            musicService
         } = {}
     } = play;
 
-    let titleElm: React.JSX.Element;
-    if(webUrl !== undefined || originUrl !== undefined) {
-        titleElm = <Link variant="underline" target="_blank" href={webUrl ?? originUrl}>{track}</Link>
-    } else {
-        titleElm = <Span>{track}</Span>;
+    const titleLinks: React.JSX.Element[] = [];
+    if (webUrl !== undefined || originUrl !== undefined) {
+        titleLinks.push(<Link key="weblink" variant="underline" target="_blank" href={webUrl ?? originUrl}><Icon size="sm">{getMusicServiceIconElement(musicService)}</Icon></Link>);
     }
+    if (brainz.track !== undefined) {
+        titleLinks.push(<MusicbrainzInfoIcon type="track" mbid={brainz.track} tooltip link showMbid={showMbid} />)
+    }
+    if (brainz.recording !== undefined) {
+        titleLinks.push(<MusicbrainzInfoIcon type="recording" mbid={brainz.recording} tooltip link showMbid={showMbid} />)
+    }
+
+    const titleElm = <HStack><Span>{track}</Span>{titleLinks}</HStack>
 
     return (
         <Flex flexDirection="column" gap="4">
@@ -150,13 +206,20 @@ export const PlayDataDataList = (props: { play: JsonPlayObject, dates: DisplayDa
                     <DataList.ItemLabel>Artists</DataList.ItemLabel>
                     <DataList.ItemValue>
                         {artists.length === 0 ? <Text color="fg.muted">(No Artists)</Text> :
-                            <ArtistCreditTags data={play.data.artists} />}
+                            <ArtistCreditTags data={play.data.artists} showMbid={showMbid} />}
                     </DataList.ItemValue>
                 </DataList.Item>
                 {albumArtistElm}
                 <DataList.Item flexGrow="1">
                     <DataList.ItemLabel>Album</DataList.ItemLabel>
-                    <DataList.ItemValue>{play.data.album}</DataList.ItemValue>
+                    <DataList.ItemValue>
+                        <HStack>
+                            {play.data.album}
+                            <Show when={brainz.album !== undefined}>
+                                <MusicbrainzInfoIcon type="release" mbid={brainz.album} link tooltip showMbid={showMbid} />
+                            </Show>
+                        </HStack>
+                    </DataList.ItemValue>
                 </DataList.Item>
             </DataList.Root>
             <DataList.Root flexWrap="wrap" flexDirection="row">
@@ -172,23 +235,9 @@ export const PlayDataDataList = (props: { play: JsonPlayObject, dates: DisplayDa
                 </DataList.Item>
                 <DataList.Item flexGrow="1" hideBelow="sm">
                     <DataList.ItemLabel>Repeat?</DataList.ItemLabel>
-                    <DataList.ItemValue><Icon>{repeat ? <LuCheck/> : <LuX/>}</Icon></DataList.ItemValue>
+                    <DataList.ItemValue><Icon>{repeat ? <LuCheck /> : <LuX />}</Icon></DataList.ItemValue>
                 </DataList.Item>
             </DataList.Root>
-            <DataList.Root flexWrap="wrap" flexDirection="row" hideBelow="sm">
-                <Show when={Object.keys(brainz).length > 0}>
-                    <DataList.Item flexGrow="1">
-                        <DataList.ItemLabel>MBIDs</DataList.ItemLabel>
-                        <DataList.ItemValue>
-                            <Stack gap="1">
-                                <Show when={brainz.track !== undefined}><Text textStyle="xs"><Muted>Track:</Muted> {brainz.track}</Text></Show>
-                                <Show when={brainz.recording !== undefined}><Text textStyle="xs"><Muted>Recording</Muted>: {brainz.recording}</Text></Show>
-                                <Show when={brainz.album !== undefined}><Text textStyle="xs"><Muted>Album</Muted>: {brainz.album}</Text></Show>
-                            </Stack>
-                        </DataList.ItemValue>
-                    </DataList.Item>
-                </Show>
-            </DataList.Root>
         </Flex>
     )
 }

src/client/components/SideNav.tsx

@@ -8,7 +8,7 @@ import {
   Badge
 } from "@chakra-ui/react"
 import { Link as RouterLink } from "react-router";
-import { ExternalLinkIcon } from "./icons/ChakraIcons"
+import { ExternalLinkIconRaw } from "./icons/ChakraIcons"
 
 interface SideNavItem {
   title: React.ReactNode
@@ -68,7 +68,7 @@ export const SideNav = (props: SideNavProps) => {
                 aria-current={item.url === currentUrl ? "page" : undefined}
               >
                 {item.title}
-                <ExternalLinkIcon />
+                <ExternalLinkIconRaw />
                 {item.status && <StatusBadge>{item.status}</StatusBadge>}
               </Link>
             ) : (

src/client/components/Toaster.tsx

@@ -0,0 +1,41 @@
+"use client"
+
+import {
+  Toaster as ChakraToaster,
+  Portal,
+  Spinner,
+  Stack,
+  Toast,
+  createToaster,
+} from "@chakra-ui/react"
+
+export const toaster = createToaster({
+  placement: "bottom-end",
+  pauseOnPageIdle: true,
+})
+
+export const Toaster = () => (
+    <Portal>
+      <ChakraToaster toaster={toaster} insetInline={{ mdDown: "4" }}>
+        {(toast) => (
+          <Toast.Root width={{ md: "sm" }}>
+            {toast.type === "loading" ? (
+              <Spinner size="sm" color="blue.solid" />
+            ) : (
+              <Toast.Indicator />
+            )}
+            <Stack gap="1" flex="1" maxWidth="100%">
+              {toast.title && <Toast.Title>{toast.title}</Toast.Title>}
+              {toast.description && (
+                <Toast.Description>{toast.description}</Toast.Description>
+              )}
+            </Stack>
+            {toast.action && (
+              <Toast.ActionTrigger>{toast.action.label}</Toast.ActionTrigger>
+            )}
+            {toast.closable && <Toast.CloseTrigger />}
+          </Toast.Root>
+        )}
+      </ChakraToaster>
+    </Portal>
+  )