test: Add atproto identifier

Allow user-selected max grid width on non-mobile breakpoints

#500

.dockerignore

@@ -31,4 +31,5 @@ tmp-*
 .env
 *.env
 .storybook
-.stories
+.stories
+**/lexicons/**/*.json

.github/workflows/testAndSanity.yml

@@ -60,7 +60,7 @@ jobs:
         run: |
           set +e
           export NODE_ENV=production
-          timeout --preserve-status 10s node node_modules/.bin/tsx src/backend/index.ts
+          timeout --preserve-status 10s node src/backend/index.ts
           exitcode="$?"
           if [[ "$exitcode" -eq 143 ]] || [[ "$exitcode" -eq 137 ]]; then
             echo "App stayed up long enough and exited with expected status"

.gitignore

@@ -153,4 +153,6 @@ tmp-*
 storybook-static
 
 *.db
-*.db*
+*.db*
+
+**/lexicons/**/*.json

.mocharc.json

@@ -1,7 +1,6 @@
 {
   "reporter": "dot",
   "extension": "ts",
-  "import": "tsx/esm",
   "spec": "./src/backend/tests/**/*.test.ts",
   "file": [
     "./src/backend/tests/setup.ts"

.storybook/main.ts

@@ -12,12 +12,14 @@ export default defineMain({
 //    "@storybook/addon-vitest",
     "@storybook/addon-a11y",
     "@storybook/addon-docs",
+    'storybook-addon-remix-react-router'
 //    "@storybook/addon-onboarding"
   ],
   features: {
     // https://github.com/storybookjs/storybook/discussions/33279
     sidebarOnboardingChecklist: false,
   },
+   staticDirs: ['../public'],
   "framework": "@storybook/react-vite",
     viteFinal: async (config) => {
     return mergeConfig(config, {

.storybook/manager.ts

@@ -2,5 +2,5 @@ import { addons } from 'storybook/manager-api';
 import { themes } from 'storybook/theming';
  
 addons.setConfig({
-  theme: themes.dark,
+  theme: themes.normal,
 });

.storybook/preview.ts

@@ -3,8 +3,17 @@ import addonA11y from "@storybook/addon-a11y";
 import { definePreview } from "@storybook/react-vite";
 
 import { themes } from 'storybook/theming';
+import { initialize, mswLoader, getWorker } from 'msw-storybook-addon';
+
+// https://github.com/mswjs/msw-storybook-addon/issues/82#issuecomment-3894302575
+try {
+  getWorker();
+} catch (_) {
+  initialize();
+}
 
 export default definePreview({
+  loaders: [mswLoader],
   parameters: {
     controls: {
       matchers: {
@@ -20,7 +29,7 @@ export default definePreview({
       test: 'todo'
     },
     docs: {
-      theme: themes.dark,
+      theme: themes.normal,
     },
   },
 

.vscode/launch.json

@@ -20,7 +20,7 @@
             "request": "launch",
             // Debug app in VSCode
             "program": "${workspaceFolder}/src/backend/index.ts",
-            "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/tsx",
+            "runtimeExecutable": "node",
             "console": "integratedTerminal",
             "internalConsoleOptions": "neverOpen",
             "skipFiles": [
@@ -35,7 +35,7 @@
             "request": "launch",
             // Debug app in VSCode
             "program": "${workspaceFolder}/src/backend/index.ts",
-            "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/tsx",
+            "runtimeExecutable": "node",
             "console": "integratedTerminal",
             "envFile": "${workspaceFolder}/.env",
             "env": {
@@ -48,28 +48,13 @@
                 "${workspaceFolder}/node_modules/**",
             ],
         },
-        {
-            "name": "tsx",
-            "type": "node",
-            "request": "launch",
-            // Debug current file in VSCode
-            "program": "${file}",
-            "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/tsx",
-            "console": "integratedTerminal",
-            "internalConsoleOptions": "neverOpen",
-            "skipFiles": [
-                "<node_internals>/**",
-                // Ignore all dependencies (optional)
-                "${workspaceFolder}/node_modules/**",
-            ],
-        },
         {
             "name": "schema",
             "type": "node",
             "request": "launch",
             // Debug app in VSCode
             "program": "${workspaceFolder}/src/backend/utils/SchemaStaticUtil.ts",
-            "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/tsx",
+            "runtimeExecutable": "node",
             "console": "integratedTerminal",
             "internalConsoleOptions": "neverOpen",
             "skipFiles": [
@@ -96,7 +81,7 @@
                 "--recursive",
                 "${workspaceFolder}/src/backend/tests/**/*.test.ts"
             ],
-            "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/tsx",
+            "runtimeExecutable": "node",
             "internalConsoleOptions": "openOnSessionStart",
             "name": "Mocha Tests",
             "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha",
@@ -120,7 +105,7 @@
                 "--config", "${workspaceRoot}/.mocharc.json",
                 "${file}"
             ],
-            "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/tsx",
+            "runtimeExecutable": "node",
             "internalConsoleOptions": "openOnSessionStart",
             "name": "Mocha Test on File",
             "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha",

.vscode/settings.json

@@ -1,5 +1,4 @@
 {
-    "mochaExplorer.require": "tsx/esm",
     "mochaExplorer.timeout": 1200000,
     "mochaExplorer.exit": true,
     "search.exclude": {
@@ -11,7 +10,9 @@
         "docsite/**"
     ],
     "files.associations": {
-        "*.css": "tailwindcss"
+        "*.css": "tailwindcss",
+        "**/config/*.json": "jsonc"
     },
-    "tailwindCSS.experimental.configFile": "src/client/index.css"
+    "tailwindCSS.experimental.configFile": "src/client/index.css",
+    
 }

Dockerfile

@@ -118,6 +118,8 @@ RUN npm ci --omit=dev --no-audit \
     && npm uninstall ts-json-schema-generator \
     && npm cache clean --force \
     && chown -R abc:abc node_modules \
+    ## peerDependency of validbot and can(?) be removed for production
+    && rm -r node_modules/typescript \
     && npx @usex/prune-mod -w \
     && rm -rf /root/.cache
 

config/rocksky.json.example

@@ -4,7 +4,7 @@
     "name": "FoxxMD Rocksky Client",
     "configureAs": "client",
     "data": {
-      "key": "7cdr86vis3mpq5b1pi01md0hkm7ykzzxjy81r",
+      "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkaWQiOiJkaWQ6cGxjOng0NjVwZmtyeW5zaDZ...",
       "handle": "alice.bsky.social"
     }
   },
@@ -13,7 +13,7 @@
     "name": "FoxxMD Rocksky Source",
     "configureAs": "source",
     "data": {
-      "key": "e4xwktm7jxm607x8el1fcda30eu14dzb64h3j",
+      "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkaWQiOiJkaWQ6cGxjOng0NjVwZmtyeW5zaDZ...",
       "handle": "alice.bsky.social"
     }
   }

docker/root/etc/s6-overlay/s6-rc.d/svc-node/run

@@ -7,4 +7,4 @@ echo -e "\nmulti-scrobbler is starting!"
 export NODE_ENV=production
 cd /app || exit
 exec \
-	  s6-setuidgid abc /usr/bin/node $NODE_ARGS /app/node_modules/.bin/tsx /app/src/backend/index.ts
+	  s6-setuidgid abc /usr/bin/node $NODE_ARGS /app/src/backend/index.ts

docsite/docs/configuration/clients/rocksky.mdx

@@ -13,7 +13,7 @@ To monitor and re-scrobble activity **from** Rocksky create a [Rocksky (Source)]
 
 :::
 
-[**Rocksky**](https://rocksky.app/) is music tracking and discovery platform built on the AT Protocol (Bluesky).
+[**Rocksky**](https://rocksky.app/) is music tracking and discovery platform built on the AT Protocol and your your [**Atmosphere Account**](https://atmosphereaccount.com/).
 
 <details>
 
@@ -23,13 +23,13 @@ To monitor and re-scrobble activity **from** Rocksky create a [Rocksky (Source)]
 
 </details>
 
-<DetailsAdmo type="warning" summary="Your Data Is Public">
+<DetailsAdmo type="important" summary="Your Data Is Public">
 
 The default Bluesky PDS is **public**, meaning your scrobbles are also public (read-only). This isn't any different than using Last.fm or Listenbrainz, in terms of privacy.
 
 </DetailsAdmo>
 
-<DetailsAdmo type="warning" summary="No Scrobbling Guarantee">
+<DetailsAdmo type="important" summary="No Scrobbling Guarantee">
 
 Rocksky **does not guarantee** that every scrobble you send will be persisted. Rocksky's design goal is to store scrobbles with **metdata-complete, 3rd party verifiable track information** so that it can be a metadata-rich discovery platform, rather than an exhaustive scrobble data source-of-truth.
 
@@ -41,22 +41,16 @@ If you need your scrobbles to be 100% captured then you should scrobble to an *a
 
 ## Setup
 
-You will need a [Bluesky](https://bsky.app/) account to use Rocksky.
+You will need an [**Atmosphere Account**](https://atmosphereaccount.com/) to use Rocksky.
 
-:::note 
-
-Usage is adapted from [Rocksky docs](https://docs.rocksky.app/migrating-to-rocksky-scrobble-api-957839m0) on [docs.rocksky.app](https://docs.rocksky.app)
-
-:::
-
-* Navigate to [rocksky.app](https://rocksky.app/) and sign up/logn using your Bluesky account
-* Then, navigate to the [RockSky developer dashboard](https://rocksky.app/apikeys) and obtain an API Key
+* Navigate to [rocksky.app](https://rocksky.app/) and sign up/logn using your Atmosphere Account
+* Then, navigate to the [Access Tokens](https://rocksky.app/access-tokens) in the Rocksky settings dropdown, and create a new token
 
 ## Configuration
 
 ### Handle
 
-The handle used with multi-scrobbler should be your **full** ATProto handle, including TLD.
+The handle used with multi-scrobbler should be your **full** [**Atmosphere Account**](https://atmosphereaccount.com/) handle, including TLD.
 
 * For regular Bluesky account this will be like: `alice.bsky.social`
 * For Bluesky accounts that [use their website as their account](https://bsky.social/about/blog/4-28-2023-domain-handle-tutorial) this is your domain: `mydomain.com`
@@ -64,11 +58,28 @@ The handle used with multi-scrobbler should be your **full** ATProto handle, inc
 
 If you do not include a TLD then multi-scrobbler will automatically append `.bsky.social` to your handle value.
 
+<DetailsAdmo type="warning" summary="API Applications Deprecated">
+
+In < 0.14.1, Multi-Scrobber used Rocksky's API Appication "API Key" for communication with a Listenbrainz-compatible API from Rocksky.
+
+Going forward, this should be replaced with the official guidance in the [Setup](#setup) section above: use an **Access Token**. This uses the native Rocksky API which has more functionality.
+
+If your configuration is using either
+
+* (File/AIO) `'key': ...`
+* (ENV) `ROCKSKY_KEY`
+
+you should redo the [Setup](#setup) section and modify your configuration to use an Access Token instead.
+
+Keys are now **deprecated** and functionality for them will be removed in a future version of Multi-Scrobbler. 
+
+</DetailsAdmo>
+
 <Config config="RockSkyClientConfig" fileContent={JsonConfig} client name="rocksky">
         | Environmental Variable | Required? | Default | Description                                                                         |
         | :--------------------- | --------- | ------- | ----------------------------------------------------------------------------------- |
-        | `ROCKSKY_ID`           | Yes       |         | A unique ID                                                                         |
+        | `ROCKSKY_ID`           | Yes       |         | A unique ID used in MS                                                              |
         | `ROCKSKY_HANDLE`       | Yes       |         | Your **full** ATProto handle. For Bluesky account this is like `myUser.bsky.social` |
-        | `ROCKSKY_KEY`          | Yes       |         | API Key obtained from Rocksky developer dashboard                                   |
+        | `ROCKSKY_TOKEN`        | Yes       |         | Access Token obtained from Rocksky                                                  |
         | `ROCKSKY_NAME`         | No        |         | A vanity name different than ID                                                     |
 </Config>

docsite/docs/configuration/clients/tealfm.mdx

@@ -16,7 +16,7 @@ To monitor and re-scrobble activity from a ATProto Profile create a [teal.fm (So
 
 :::
 
-[**teal.fm**](https://teal.fm/) is a social discovery and web viewing service for scrobbles stored/available on the ATProto network/your repository in a [PDS (Personal Data Server)](https://github.com/Bluesky-social/pds).
+[**teal.fm**](https://teal.fm/) is a social discovery and browsing app for scrobbles stored in your [**Atmosphere Account**](https://atmosphereaccount.com/).
 
 <details>
 
@@ -46,6 +46,13 @@ To view your teal.fm scrobble data you can:
 
 ## Setup
 
+
+:::note
+
+Multi-scrobbler will work with *any* Atmosphere Account, not just Bluesky. See [Configuration](#configuration) below for provider-agnostic config.
+
+:::
+
 #### Bluesky
 
 * Create a [Bluesky](https://bsky.app) account, if you don't already have one
@@ -57,10 +64,10 @@ To view your teal.fm scrobble data you can:
 
 ### Identifier
 
-The ATPRoto identifier used with multi-scrobbler should be either:
+The ATProto identifier used with multi-scrobbler should be either:
 
 * A valid [DID](https://atproto.com/specs/did#at-protocol-did-identifier-syntax), starting with `did:plc:...` or `did:web:...`
-* Your **full** ATProto handle, including TLD
+* Your **full** [Atmosphere Account](https://atmosphereaccount.com/) ATProto handle, including TLD
 
 If using a handle:
 

docsite/docs/configuration/sources/azuracast.mdx

@@ -16,9 +16,9 @@ Azuracast may not correctly report track length or position. If this is the case
 
 :::
 
-The Azuracast server should have **Use High-Performance Now Playing Updates** enabled in _Administration -> System Settings_
+The Azuracast server should have **Use [High-Performance Now Playing Updates](https://www.azuracast.com/docs/developers/now-playing-data/#high-performance-updates)** enabled in _Administration -> System Settings_
 
-#### URL
+### URL
 
 The URL used by MS to connect to Azuracast has the syntax:
 
@@ -47,6 +47,68 @@ If either is configured then MS will automatically scrobble based on when these
 
 If neither is configured automatic scrobble defaults to **off** and you should either manually initiate it or explicitly set the default behavior.
 
+### Station
+
+#### Station Identifier
+
+The station identifier multi-scrobbler uses (ENV `AZURA_STATION` or `station` in File/AIO config) needs to be the **URL Stub** or **shortcode** of the Azuracast station, **not** its display name. Use of the methods below to find an apprioriate station identifier.
+
+<Tabs groupId="stationId" queryString>
+<TabItem value="stub" label="URL Stub">
+The **URL Stub** is the value used in the URL to access the station EX
+
+```
+https://azura.mydomain.com/public/my-cool-station
+                                  ^^^^ url stub ^^
+```
+
+Use `my-cool-station` as the station identifier.
+</TabItem>
+<TabItem value="shortcode" label="Shortcode">
+The **shortcode** can be found in the raw JSON response for your station. Navigate to this URL in your browser:
+
+```
+https://azura.mydomain.com/api/stations
+```
+
+Find your station in the returned data (by `name`) and then locate the value of the `shortcode` property in that object.
+
+<details>
+
+<summary>Example</summary>
+
+```json
+[
+    {
+        "id": 1,
+        "name": "My Cool Station",
+        "shortcode": "my-cool-station",
+        "description": "Re-broadcasts of my cool music",
+        "frontend": "icecast",
+        "backend": "liquidsoap",
+        ...
+```
+
+Use `my-cool-station` as the station identifier.
+</details>
+</TabItem>
+</Tabs>
+
+#### Station Access
+
+Multi-Scrobbler does not use authentication to communicate with Azuracast. Your station will likely need some form of **Public Pages** in order for you to monitor it.
+
+In your station's **Edit Station Settings** -> **Public Pages** make sure that at least one of these is enabled:
+
+* Enable Public Pages
+* Enable Public APIs
+
+:::note
+
+For users who aren't administrators of Azuracast: if you can access the station using `https://azura.mydomain.com/public/xxxxxx` then it is public.
+
+:::
+
 ## Configuration
 
 <Config config="AzuracastSourceConfig" fileContent={AzuracastConfig} name="azuracast">
@@ -54,7 +116,7 @@ If neither is configured automatic scrobble defaults to **off** and you should e
         | :--------------------- | :-------- | :------ | ---------------------------------------------------------------------------------------------- |
         | `AZURA_ID`             | Yes       |         | A unique ID                                                                                    |
         | `AZURA_URL`            | Yes       |         | Azuracast URL *without station name*                                                           |
-        | `AZURA_STATION`        | Yes       |         | The station name shown on the public page                                                      |
+        | `AZURA_STATION`        | Yes       |         | The station URL Stub or shortcode                                                              |
         | `AZURA_LIVE`           | No        | Yes     | Only scrobble when station status is ONLINE                                                    |
         | `AZURA_LISTENERS_NUM`  | No        | `true`  | Only scrobble if station has any listeners (`true`) or listeners are equal-to/greater-than `X` |
         | `AZURA_NAME`           | No        |         | A vanity name different than ID                                                                |

docsite/docs/configuration/sources/jriver.mdx

@@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem';
 import CodeBlock from '@theme/CodeBlock';
 import JriverConfig from '!!raw-loader!@site/../config/jriver.json.example';
 
-In order for multi-scrobbler to communicate with JRiver you must have [Web Server Interface](https://wiki.jriver.com/index.php/Web_Service_Interface#Documentation_of_Functions) enabled. This can can be in the JRiver GUI:
+In order for multi-scrobbler to communicate with JRiver you must have [Web Server Interface](https://wiki.jriver.com/index.php/Web_Service_Interface#Documentation_of_Functions) enabled. This can be found in the JRiver GUI:
 
 * Tools -> Options -> Media Network
   * Check `Use Media Network to share this library...`

docsite/docs/configuration/transforms/transforms.mdx

@@ -593,39 +593,6 @@ MS uses [caching](/configuration/#caching) to reduce the number of API calls nee
 
 **If you are using any Transform stages you should configure [secondary caching](/configuration#secondary-caching)** to increase the cache size and lifetime of cached items. This will also reduce memory usage in MS.
 
-<details>
-
-<summary>Example</summary>
-
-Add valkey service to your [multi-scrobbler docker compose stack](/installation?runType=docker-compose#docker) and configure MS to use it for metadata:
-
-```yaml title="docker-compose.yml"
-services:
-  multi-scrobbler:
-    image: foxxmd/multi-scrobbler
-    # ...
-    environment:
-      # ...
-      // highlight-start
-      - CACHE_METADATA=valkey
-      - CACHE_METADATA_CONN=redis://valkey:6379
-      // highlight-end
-    # ...
-
-  // highlight-start
-  valkey:
-    image: valkey/valkey
-    volumes:
-      - valkeydata:/data
-
-volumes:
-    valkeydata:
-      driver: local
-  // highlight-end
-```
-
-</details>
-
 ## Examples
 
 See **Examples** sections in specific Stage docs (also in the sidebar):

docsite/docs/updating/upgrade-path/0140.mdx

@@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem';
 import CodeBlock from '@theme/CodeBlock';
 import RequiredUpgrade from "@site/src/components/snippets/_upgrade-required.mdx"
 
-<RequiredUpgrade before='< 0.14.0' current='0.14.0' old='0.13.2' />
+{/* <RequiredUpgrade before='< 0.14.0' current='0.14.0' old='0.13.2' /> */}
 
 ## Source/Client IDs
 
@@ -75,32 +75,34 @@ Add an `id` to the top-level for each Source/Client configuration, next to `data
 
 Add an `id` to the top-level for each Source/Client configuration, next to `data`:
 
-```json title="koito.json"
-[
-  {
-    "name": "koito-source",
-    "configureAs": "source",
-    "type": "koito",
-    // highlight-start
-    "id": "myKoitoID",
-    // highlight-end    
-    "data": {
-      "token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b",
-      "username": "admin",
-      "url": "http://192.168.0.100:4110" 
+```json title="config.json"
+{
+  "sources": [
+    {
+      "name": "koito-source",
+      "configureAs": "source",
+      "type": "koito",
+      // highlight-start
+      "id": "myKoitoID",
+      // highlight-end    
+      "data": {
+        "token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b",
+        "username": "admin",
+        "url": "http://192.168.0.100:4110" 
+      }
     }
-  }
-]
+  ]
+}
 ```
 
 </TabItem>
 </Tabs>
 
-:::note[Default ID]
+:::important[Default ID]
 
-If you do not add an ID then Multi-Scrobbler will automatically use the **name** of the Source/Client as the ID. The name is shown in the Dashboard.
+**If you do not add an ID then Multi-Scrobbler will automatically use the name of the Source/Client as the ID.** The name is shown in the Dashboard.
 
-If you decide to add an ID later you must use the name as the ID in order to keep Plays/Scobbles associated with the same config.
+If you decide to add an ID later, you must use the Source/Client name as the ID in order to keep Plays/Scobbles associated with the same config.
 
 :::
 
@@ -108,13 +110,19 @@ If you decide to add an ID later you must use the name as the ID in order to kee
 
 :::tip
 
-Before upgrading, if your MS dashboard shows 0 queued/failed for all [Scrobble Clients](/configuration/clients) then you can **ignore this step.**
+Before upgrading, if your MS dashboard shows 0 queued/failed for all [Scrobble Clients](/configuration/clients) then you can **ignore this section.**
+
+:::
+
+:::important
+
+The cached scrobbles for a Client/Source were based on its **name**. If you plan on changing the name of a Client/Source with this upgrade you should do it **after** this migration has run or else MS will not be able to find any associated cached scrobbles. 
 
 :::
 
 MS `0.14.0` introduces a database for persisting Plays/Scrobbles, queues, and other data. Prior to `0.14.0`, queued/failed scrobbles were stored in a cache **file** inside your [`CONFIG_DIR`](/installation/?dockerSetting=storage#recommended-settings) named `ms-scrobble.cache`.
 
-When a [Client](/configuration/clients) is created and **enabled** in `0.14.0` it will attempt to automatically migrate scrobbles from this cache into the database.
+When a [Client](/configuration/clients) is created and **enabled** in `0.14.0` it will attempt to **automatically migrate scrobbles** from this cache into the database.
 
 * You can see progress/results for this process by filtering logs for `[Cached Scrobble Migration]`
 * If the process fails MS will attempt to migrate failed scrobbles, again, the next time it is started
@@ -140,7 +148,7 @@ In `0.14.0` [Cache](/configuration#caching) has been simplified with much of the
 
 This only applies to users who have `cache` in their [AIO Config](/configuration/?configType=aio#configuration-types) `config.json` or are using `CACHE_*` [ENV Config](/configuration/?configType=env#configuration-types)
 
-If you do not have any of the above [Cache](/configuration#caching) configuration defined then you can **ignore this step.** 
+If you do not have any of the above [Cache](/configuration#caching) configuration defined then you can **ignore this section.** 
 
 :::
 
@@ -210,9 +218,41 @@ Remove the `connection` property from the `auth` object. If using `file` the ent
 </TabItem>
 </Tabs>
 
-### Renamed ENV Keys
+## Renamed ENV Keys
 
 In an effort to standard ENV prefixes some ENV keys have been renamed:
 
 * `LZENDPOINT_ENABLE` => `LZE_ENABLE`
-* `LFMENDPOINT_ENABLE` => `LFM_ENABLE`
+* `LFMENDPOINT_ENABLE` => `LFM_ENABLE`
+
+## (Optional) ENV Client/Source Names
+
+In the MS Dashboard, prior to 0.14.0, UI Client/Sources configured with [ENV Config](/configuration/?configType=env#configuration-types) always had generated names like `unnamed-koito`.
+
+ENV Config Clients/Sources can now be configured with a name by using the `_NAME` suffix, example:
+
+```yaml title="compose.yaml"
+services:
+  multi-scrobbler:
+  # ...
+  environment:
+    - KOTIO_TOKEN=...
+    # ...
+    - KOITO_ID=myKoitoID
+  // highlight-start
+    - KOITO_NAME=Koito A
+  // highlight-end
+
+    - JELLYFIN_URL=...
+    # ..
+    - JELLYFIN_ID=myJellyfinID
+  // highlight-start
+    - JELLYFIN_NAME=Foxx Jelly
+  // highlight-end    
+```
+
+:::important
+
+If you have **cached scrobbles** make sure to upgrade and run [Cached Scrobble Migration](#cached-scrobble-migration) **before** adding names to Client/Sources.
+
+:::

docsite/src/components/snippets/_bluesky.mdx

@@ -1,14 +1,14 @@
 import CodeInline from '@theme/CodeInline';
 
-The [AT Protocol](https://docs.bsky.app/docs/advanced-guides/atproto) is a standard to allow different applications to share and interpret your social data.
+Your [**Atmosphere Account**](https://atmosphereaccount.com) consists of social data (it's just JSON) stored in a personal [repository](https://atproto.com/guides/data-repos) on a [PDS (Personal Data Server)](https://github.com/Bluesky-social/pds), all of which is standardized under the [**AT Protocol**](https://atproto.com).
 
-Your data is stored in a personal [repository](https://atproto.com/guides/data-repos) in a [PDS (Personal Data Server)](https://github.com/Bluesky-social/pds), which is data-type agnostic (it's just JSON). Each application defines their own datatypes and pulls data from your repository to render to you, your network, etc...
+The AT Protocol defines how your data is stored, how you authenticate to write to it, and how indpendent apps share and consume it. Critically, each layer (data storage, app, communication) of the AT Protocol can be operated by *indpendent* entities so your data and experience not locked behind a walled garden.
 
-Even though the application and the PDS/repository are independent, all of this is done for you automatically when you create a [Bluesky](https://bsky.social/about) account.
+Although each layer can be implemented separately (you can self-host everything!), it is normally a *simple* process for an end-user to create and start using an Atmosphere Account as many apps can do everything for you. The most common example being a [Bluesky](https://bsky.social/about), but there are other [public account providers](https://atmosphereaccount.com/explore).
 
-So...
+When Multi-Scrobbler uses the AT Protocol...
 
-* multi-scrobbler "scrobbles" your plays to your repository
+* multi-scrobbler "scrobbles" (writes) your plays to your repository
   * in a JSON data structure which includes [{props.name}'s data type and structure](https://atproto.com/specs/lexicon)
   * at a [known location for {props.name}'s data](https://atproto.com/guides/glossary#collection) (<CodeInline>{props.lexicon}</CodeInline>), in a list that is appended to as you continue to scrobble
-* when accessing {props.name}, or another application that knows how to read it, your data is pulled from the repository to render the app
+* when accessing {props.name}, or another application that knows how to read it, your data is pulled from the repository to render their app

docsite/src/css/custom.css

@@ -13,7 +13,8 @@
   --ifm-color-primary-light: #33925d;
   --ifm-color-primary-lighter: #359962;
   --ifm-color-primary-lightest: #3cad6e;
-  --ifm-color-important-contrast-background: #bb91ff;
+  --ifm-color-important-contrast-background: #e7d7ff;
+  --ifm-color-important-highlight: #b94be8;
   --ifm-code-font-size: 95%;
   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1);
 }
@@ -28,6 +29,7 @@
   --ifm-color-primary-lighter: #32d8b4;
   --ifm-color-primary-lightest: #4fddbf;
   --ifm-color-important-contrast-background: #5a3795;
+  --ifm-color-important-highlight: #e09cfd;
   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3);
 }
 
@@ -40,7 +42,7 @@ details[class^='details_'].alert--info {
 
   margin-bottom: var(--ifm-paragraph-margin-bottom);
 
-  &:not(.alert--warning,.alert--success,.alert--secondary,.alert--danger) {
+  &:not(.alert--warning,.alert--success,.alert--secondary,.alert--danger,.alert--important) {
     --ifm-alert-background-color: transparent;
 
     > summary {
@@ -98,6 +100,12 @@ details[class^='details_'].alert--info {
 
 }
 
+.alert--info.theme-admonition-important {
+    --ifm-alert-background-color: var(--ifm-color-important-contrast-background);
+    --ifm-alert-foreground-color: var(--ifm-color-secondary-contrast-foreground);
+    --ifm-alert-border-color: var(--ifm-color-important-highlight);
+}
+
 /** https://stackoverflow.com/a/79808430/1469797 */
 img.image-center {
   display: block;

eslint.config.js

@@ -1,39 +1,224 @@
 // For more info, see https://github.com/storybookjs/eslint-plugin-storybook#configuration-flat-config-format
 import storybook from "eslint-plugin-storybook";
+import boundaries from 'eslint-plugin-boundaries';
+import unusedImports from 'eslint-plugin-unused-imports';
+
+import { defineConfig, globalIgnores } from "eslint/config";
 
 // @ts-check
 
-import eslint from '@eslint/js';
+import js from '@eslint/js';
+import globals from 'globals';
 import tsEslint from 'typescript-eslint';
 import arrow from 'eslint-plugin-prefer-arrow-functions';
+import hooks from 'eslint-plugin-react-hooks';
+import mochaPlugin from 'eslint-plugin-mocha';
 
-export default tsEslint.config({
-    files: ['src/backend/**/*.ts'],
-    plugins: {
-        "prefer-arrow-functions": arrow
-    },
-    ignores: [
-        'eslint.config.js',
-        'src/backend/tests/**/*.ts'
+const defaultRules = {
+    'no-useless-catch': 'off',
+    '@typescript-eslint/no-unused-vars': 'warn',
+    'no-unused-vars': [
+        'warn',
+        {
+            args: 'none',
+            caughtErrors: 'none',
+            destructuredArrayIgnorePattern: "^_",
+            "argsIgnorePattern": "^_",
+            ignoreRestSiblings: true,
+            ignoreUsingDeclarations: true
+        }
     ],
-    extends: [
-        eslint.configs.recommended,
-        ...tsEslint.configs.recommended,
+    "@typescript-eslint/no-unused-vars": [
+      "warn",
+      {
+        args: 'none',
+        caughtErrors: 'none',
+        destructuredArrayIgnorePattern: "^_",
+        "argsIgnorePattern": "^_",
+        ignoreRestSiblings: true,
+        ignoreUsingDeclarations: true
+      }
     ],
-    rules: {
-        'no-useless-catch': 'off',
-        '@typescript-eslint/no-unused-vars': 'off',
-        "prefer-arrow-functions/prefer-arrow-functions": [
-            "warn",
-            {
-                "allowNamedFunctions": false,
-                "classPropertiesAllowed": false,
-                "disallowPrototype": false,
-                "returnStyle": "unchanged",
-                "singleReturnOnly": false
-            }
+    "prefer-arrow-functions/prefer-arrow-functions": [
+        "warn",
+        {
+            "allowNamedFunctions": false,
+            "classPropertiesAllowed": false,
+            "disallowPrototype": false,
+            "returnStyle": "unchanged",
+            "singleReturnOnly": false
+        }
+    ],
+    "arrow-body-style": ["warn", "as-needed"],
+    "@typescript-eslint/no-explicit-any": "warn",
+    "@typescript-eslint/no-empty-object-type": [
+        "warn",
+        {
+            "allowInterfaces": 'with-single-extends'
+        }
+    ]
+};
+
+export default defineConfig([
+    globalIgnores([
+        'docsite/build',
+        'docsite/.docusaurus',
+        'public/mockServiceWorker.js',
+        'dist'
+    ]),
+    {
+        plugins: {
+            "prefer-arrow-functions": arrow,
+            js,
+            'unused-imports': unusedImports
+        },
+        rules: {
+            ...defaultRules,
+            // https://typescript-eslint.io/rules/consistent-type-imports/#comparison-with-importsnotusedasvalues--verbatimmodulesyntax
+            //
+            // when using tsconfig compiler verbatimModuleSyntax and EX import {type Foo, type Bar} from 'a';
+            // true => typescript will *still* import a module if all types are inline  
+            // false => typescript will erase the entire module import
+            //
+            // we need to use verbatimModuleSyntax: true for nodejs type stripping compatibility so
+            // its important that we use top-level type imports so we don't accidentally import server-side modules into frontend
+            // but can still use types where necessary
+            //
+            // this rule *should* do this...it does detect imports that are typed but not explicitly declared
+            // but its not moving inline -> top-level
+            "@typescript-eslint/consistent-type-imports": [
+                "error",
+                {
+                    prefer: 'type-imports',
+                    fixStyle: "separate-type-imports"
+                }
+            ],
+            // however, import/consistent-type-specifier-style from eslint-plugin-import *does* move inline => top-level
+            // but it does not yet support eslint10 :(
+            //
+            // TODO eventually add this once plugin-import supports eslint10
+            // https://github.com/import-js/eslint-plugin-import
+            // https://github.com/import-js/eslint-plugin-import/blob/main/docs/rules/consistent-type-specifier-style.md
+            // -- this enforces that type-only imports are fixed to top-level type imports IE
+            // import {type Foo, type Bar} from 'a'; => import type { Foo, Bar} from 'a';
+            // "import/consistent-type-specifier-style": [
+            //     "error", "prefer-top-level-if-only-type-imports"
+            // ]
+        },
+        extends: [
+            tsEslint.configs.recommended,
+            "js/recommended"
         ],
-        "arrow-body-style": ["warn", "as-needed"],
-        "@typescript-eslint/no-explicit-any": "warn"
+        languageOptions: {
+            globals: {
+                ...globals.node,
+            }
+        },
+        files: ['src/**/*.ts','src/**/*.tsx']
+    },
+    {
+        extends: [
+            storybook.configs["flat/recommended"],
+        ],
+        files: ['src/client/stories/**/*.tsx'],
+    },
+    {
+        extends: [
+            hooks.configs.flat.recommended,
+        ],
+        languageOptions: {
+            globals: {
+                ...globals.browser,
+            }
+        },
+        files: ['src/client/**/*.tsx'],
+    },
+    {
+        extends: [
+            mochaPlugin.configs.recommended,
+        ],
+        languageOptions: {
+            globals: {
+                ...globals.node,
+            }
+        },
+        files: ['src/backend/tests/**/*.ts'],
+        rules: {
+            ...defaultRules,
+                "prefer-arrow-functions/prefer-arrow-functions": ["off"],
+                "@typescript-eslint/no-unused-expressions": 'off',
+                'mocha/max-top-level-suites': 'off'
+        },
+    },
+    {
+        // https://typescript-eslint.io/troubleshooting/faqs/eslint#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors
+        files: ['**/*.{ts,tsx,mts,cts}'],
+        plugins: {
+            'unused-imports': unusedImports
+        },
+        rules: {
+            'no-undef': 'off',
+            'unused-imports/no-unused-imports': 'error'
+        }
+    }, 
+    // this ruleset helps keep backend, frontend, and core keep imports isolated
+    // in order to prevent frontend (vite) from accidentally bundling backend files + backend packages when importing directly (backend) or transitively (through core)
+    //
+    // folder (module?) boundaries should be like
+    //
+    // backend <-- all node/server side code
+    // core <-- shared types, utils, and logic between backend and client
+    // client <-- frontend code to be bundled by vite, SHOULD NOT import directly/transitively from backend
+    {
+        files: ['src/**/*.{ts,tsx}'],
+        plugins: { boundaries },
+        settings: {
+        // Define your three architectural layers
+        'boundaries/elements': [
+            { type: 'frontend', mode: 'file', pattern: 'src/client/**/*' },
+            { type: 'config', mode: 'file', pattern: 'config/*.example' },
+            { type: 'core', mode: 'file', pattern: ['src/core/!(tests)/**','src/core/!(tests)'] },
+            { type: 'core-tests', mode: 'file', pattern: ['src/core/tests/**'] },
+            { type: 'backend', mode: 'file', pattern: 'src/backend/**/*' },
+            { type: 'stories', mode: 'file', pattern: ['src/stories/**/*', '.storybook/**'] },
+        ],
+        // So it understands TS path aliases when resolving imports
+        'import/resolver': {
+            typescript: true,
+        },
+        },
+        rules: {
+        'boundaries/element-types': [
+            'error',
+            {
+            default: 'disallow', // deny anything not explicitly allowed
+            rules: [
+                {
+                from: 'frontend',
+                allow: ['frontend', 'core'], // frontend can use itself + core, never backend
+                },
+                {
+                from: 'core',
+                allow: ['core'], // core can only use itself — never backend, never frontend
+                },
+                {
+                from: 'backend',
+                allow: ['backend', 'core', 'config'], // backend can use itself + core
+                },
+                {
+                from: 'stories',
+                allow: ['stories', 'core', 'frontend', 'core-tests'], // backend can use itself + core
+                },
+                {
+                from: 'core-tests',
+                allow: ['core', 'core-tests'], // backend can use itself + core
+                },
+            ],
+            },
+        ],
+        // Optional: flag any file under src/ that doesn't match one of the
+        // three element patterns above (catches stray/misplaced files)
+        'boundaries/no-unknown': 'error'
+        },
     }
-}, storybook.configs["flat/recommended"]);
+]);

next/index.html

@@ -0,0 +1,21 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8" />
+    <meta
+            name="description"
+            content="Dashboard for Multi Scrobbler"
+    />
+    <meta name="theme-color" content="#000000" />
+    <link rel="apple-touch-icon" href="/logo192.png" />
+    <link rel="icon" type="image/svg+xml" href="/icon.svg" />
+    <link rel="manifest" href="/manifest.json" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Multi Scrobbler</title>
+</head>
+<body>
+<noscript>You need to enable JavaScript to run this app. aaaa</noscript>
+<div id="root"></div>
+<script type="module" src="/src/client/index-next.tsx"></script>
+</body>
+</html>

package.json

@@ -1,17 +1,17 @@
 {
   "name": "multi-scrobbler",
-  "version": "0.14.0-beta.3",
+  "version": "0.14.2",
   "type": "module",
   "description": "scrobble plays from multiple sources to multiple clients",
   "scripts": {
     "schema": "npm run -s schema:docs && npm run -s schema:app",
-    "schema:docs": "tsx src/backend/utils/SchemaDocsStaticUtils.ts",
-    "schema:app": "tsx src/backend/utils/SchemaAppStaticUtils.ts",
+    "schema:docs": "node src/backend/utils/SchemaDocsStaticUtils.ts",
+    "schema:app": "node src/backend/utils/SchemaAppStaticUtils.ts",
     "circular": "madge --circular --extensions ts src/index.ts",
     "test": "npm run -s test:backend",
     "test:backend": "mocha --reporter spec",
-    "dev": "nodemon -w src/backend -x tsx src/backend/index.ts",
-    "start": "NODE_ENV=production tsx src/backend/index.ts",
+    "dev": "node src/backend/index.ts",
+    "start": "NODE_ENV=production node src/backend/index.ts",
     "build:frontend": "vite build",
     "build:backend": "tsc -p src/backend && npm run -s schema:app",
     "build": "npm run -s build:backend && npm run -s build:frontend && npm run -s docs:build",
@@ -24,7 +24,9 @@
     "install:parallel": "concurrently --kill-others-on-fail --names app,docs \"npm ci --no-audit\" \"npm run docs:install\"",
     "cliff": "git-cliff --unreleased",
     "storybook": "storybook dev -p 6006",
-    "build-storybook": "storybook build"
+    "build-storybook": "storybook build",
+    "atproto:lex:tealfm": "npm exec -- lex-cli pull -c src/backend/common/vendor/teal/lex.config.ts && npm exec -- lex-cli generate -c src/backend/common/vendor/teal/lex.config.ts",
+    "frontend:analyze": "npx esbuild src/client/index-next.tsx --bundle --platform=node --metafile=meta.json --outfile=/dev/null"
   },
   "exports": {
     ".": {
@@ -49,9 +51,13 @@
   "homepage": ".",
   "dependencies": {
     "@astronautlabs/mdns": "^1.0.7",
-    "@atcute/identity-resolver-node": "^1.0.3",
-    "@atproto/api": "^0.18.0",
-    "@atproto/oauth-client-node": "^0.3.10",
+    "@atcute/bluesky": "^4.0.5",
+    "@atcute/client": "^5.1.0",
+    "@atcute/identity": "^2.0.0",
+    "@atcute/identity-resolver": "^2.0.0",
+    "@atcute/lexicons": "^2.0.0",
+    "@atcute/password-session": "^1.0.0",
+    "@atcute/repo": "^1.0.0",
     "@donedeal0/superdiff": "^1.1.1",
     "@ewanc26/tid": "^1.0.2",
     "@foxxmd/chromecast-client": "^1.0.4",
@@ -64,9 +70,11 @@
     "@kenyip/backoff-strategies": "^1.0.4",
     "@keyv/valkey": "^1.0.8",
     "@lukehagar/plexjs": "^0.39.0",
+    "@pierre/diffs": "^1.2.4",
+    "@rocksky/sdk": "^0.2.2",
     "@supercharge/promise-pool": "^3.0.0",
     "@svrooij/sonos": "^2.5.0",
-    "@xhayper/discord-rpc": "^1.3.0",
+    "@xhayper/discord-rpc": "1.3.0",
     "abort-controller-x": "^0.5.0",
     "address": "^1.2.2",
     "ajv": "^8.18.0",
@@ -74,7 +82,7 @@
     "avahi-browse": "^1.1.4",
     "better-sse": "^0.8.0",
     "body-parser": "^2.2.2",
-    "cacheable": "^1.10.4",
+    "cacheable": "^2.5.0",
     "castv2": "^0.1.10",
     "clone": "^2.1.2",
     "common-tags": "^1.8.2",
@@ -102,7 +110,7 @@
     "json-diff-ts": "^5.0.0-alpha.2",
     "json5": "^2.2.3",
     "jsondiffpatch": "^0.7.3",
-    "keyv": "^5.5.0",
+    "keyv": "^5.6.0",
     "kodi-api": "^0.2.1",
     "lastfm-ts-api": "^2.6.0",
     "merge-error-cause": "^5.0.2",
@@ -121,6 +129,8 @@
     "passport-deezer": "^0.2.0",
     "patch-package": "^8.0.1",
     "prom-client": "^15.1.3",
+    "qs": "^6.15.2",
+    "react-use-timeout": "^1.0.0",
     "round-robin-js": "^3.0.10",
     "serialize-error": "^13.0.1",
     "spotify-web-api-node": "^5.0.2",
@@ -128,8 +138,6 @@
     "superagent": "^8.0.9",
     "toad-scheduler": "^3.0.0",
     "tough-cookie": "^5.1.2",
-    "ts-json-schema-generator": "^2.3.0",
-    "tsx": "^4.21.0",
     "typeson": "^9.0.4",
     "typeson-registry": "^11.1.1",
     "undici": "7.24.6",
@@ -139,22 +147,33 @@
     "youtubei.js": "^15.0.1"
   },
   "devDependencies": {
-    "@chakra-ui/react": "^3.34.0",
+    "@atcute/lex-cli": "^3.1.0",
+    "@chakra-ui/react": "^3.36.0",
     "@chromatic-com/storybook": "^5.1.2",
     "@dbus-types/notifications": "^0.0.5",
     "@emotion/react": "^11.14.0",
+    "@emotion/styled": "^11.14.1",
     "@eslint/js": "^10.0.1",
     "@faker-js/faker": "^9.0.1",
+    "@flamefrontend/sse-runtime-react": "^0.7.0",
     "@fortawesome/fontawesome-svg-core": "^6.7.2",
     "@fortawesome/free-solid-svg-icons": "^6.7.2",
     "@fortawesome/react-fontawesome": "^0.2.6",
     "@gr2m/net-interceptor": "^1.0.0",
+    "@lukemorales/query-key-factory": "^1.3.4",
+    "@mui/material": "^9.1.1",
+    "@react-hook/resize-observer": "^2.0.2",
+    "@react-hook/window-size": "^3.1.1",
     "@react-nano/use-event-source": "^0.16.0",
     "@reduxjs/toolkit": "^2.11.2",
-    "@storybook/addon-a11y": "^10.3.6",
-    "@storybook/addon-docs": "^10.3.6",
-    "@storybook/react-vite": "^10.3.6",
+    "@rolldown/plugin-babel": "^0.2.3",
+    "@storybook/addon-a11y": "^10.4.3",
+    "@storybook/addon-docs": "^10.4.3",
+    "@storybook/react-vite": "^10.4.3",
     "@tailwindcss/vite": "^4.2.2",
+    "@tanstack/react-query": "^5.101.0",
+    "@tanstack/react-query-devtools": "^5.101.0",
+    "@tanstack/react-virtual": "^3.14.3",
     "@tsconfig/node18": "^1.0.1",
     "@types/chai": "^4.3.0",
     "@types/chai-as-promised": "^7.1.5",
@@ -172,6 +191,7 @@
     "@types/node": "^24.12.2",
     "@types/object-hash": "^3.0.0",
     "@types/passport": "^1.0.12",
+    "@types/qs": "^6.15.1",
     "@types/react": "^19.2.14",
     "@types/react-dom": "^19.2.3",
     "@types/react-window": "^1.8.5",
@@ -179,21 +199,30 @@
     "@types/spotify-web-api-node": "^5.0.7",
     "@types/superagent": "^8.1.9",
     "@types/xml2js": "^0.4.11",
-    "@vitejs/plugin-react": "^6.0.1",
+    "@vitejs/plugin-react": "^6.0.2",
+    "@welldone-software/why-did-you-render": "^10.0.1",
     "ansi-to-react": "^6.2.6",
     "autoprefixer": "^10.4.27",
+    "babel-plugin-react-compiler": "^1.0.0",
     "chai": "^4.3.6",
     "chai-as-promised": "^8.0.2",
     "clsx": "^2.1.1",
     "drizzle-kit": "^1.0.0-rc.2-c5a84d1",
-    "eslint": "^10.3.0",
+    "eslint": "^10.6.0",
+    "eslint-import-resolver-typescript": "^4.4.5",
+    "eslint-plugin-boundaries": "^7.0.2",
+    "eslint-plugin-mocha": "^11.3.0",
     "eslint-plugin-prefer-arrow-functions": "^3.2.4",
+    "eslint-plugin-react-hooks": "^7.1.1",
     "eslint-plugin-storybook": "^10.3.6",
+    "eslint-plugin-unused-imports": "^4.4.1",
     "git-cliff": "^2.12.0",
-    "jsondiffpatch-react": "^1.0.8",
+    "globals": "^17.7.0",
+    "ky": "^2.0.2",
     "mocha": "^10.3.0",
     "mockdate": "^3.0.5",
-    "msw": "^2.12.10",
+    "msw": "^2.14.6",
+    "msw-storybook-addon": "^2.0.7",
     "next-themes": "^0.4.6",
     "nodemon": "^3.0.3",
     "playwright": "^1.58.2",
@@ -203,16 +232,19 @@
     "react-icons": "^5.6.0",
     "react-redux": "^9.2.0",
     "react-router-dom": "^7.15.0",
-    "react-virtuoso": "^4.18.4",
+    "react-virtuoso": "^4.18.7",
     "sass": "^1.99.0",
     "shiki": "^4.0.2",
     "sinon": "^21.0.2",
-    "storybook": "^10.3.6",
+    "storybook": "^10.4.3",
+    "storybook-addon-remix-react-router": "^6.1.0",
     "tailwindcss": "^4.2.2",
     "tinyexec": "^1.1.1",
     "ts-essentials": "^10.2.0",
+    "ts-json-schema-generator": "^2.3.0",
     "typescript": "^6.0.3",
     "typescript-eslint": "^8.59.3",
+    "usehooks-ts": "^3.1.1",
     "vite": "^8.0.12",
     "with-local-tmp-dir": "^7.0.1"
   },
@@ -234,10 +266,11 @@
     },
     "musicbrainz-api": {
       "rate-limit-threshold": "0.2.1"
-    },
-    "jsondiffpatch-react": {
-      "react": "$react",
-      "react-dom": "$react-dom"
     }
+  },
+  "msw": {
+    "workerDirectory": [
+      "public"
+    ]
   }
 }

public/mockServiceWorker.js

@@ -0,0 +1,349 @@
+/* eslint-disable */
+/* tslint:disable */
+
+/**
+ * Mock Service Worker.
+ * @see https://github.com/mswjs/msw
+ * - Please do NOT modify this file.
+ */
+
+const PACKAGE_VERSION = '2.14.6'
+const INTEGRITY_CHECKSUM = '4db4a41e972cec1b64cc569c66952d82'
+const IS_MOCKED_RESPONSE = Symbol('isMockedResponse')
+const activeClientIds = new Set()
+
+addEventListener('install', function () {
+  self.skipWaiting()
+})
+
+addEventListener('activate', function (event) {
+  event.waitUntil(self.clients.claim())
+})
+
+addEventListener('message', async function (event) {
+  const clientId = Reflect.get(event.source || {}, 'id')
+
+  if (!clientId || !self.clients) {
+    return
+  }
+
+  const client = await self.clients.get(clientId)
+
+  if (!client) {
+    return
+  }
+
+  const allClients = await self.clients.matchAll({
+    type: 'window',
+  })
+
+  switch (event.data) {
+    case 'KEEPALIVE_REQUEST': {
+      sendToClient(client, {
+        type: 'KEEPALIVE_RESPONSE',
+      })
+      break
+    }
+
+    case 'INTEGRITY_CHECK_REQUEST': {
+      sendToClient(client, {
+        type: 'INTEGRITY_CHECK_RESPONSE',
+        payload: {
+          packageVersion: PACKAGE_VERSION,
+          checksum: INTEGRITY_CHECKSUM,
+        },
+      })
+      break
+    }
+
+    case 'MOCK_ACTIVATE': {
+      activeClientIds.add(clientId)
+
+      sendToClient(client, {
+        type: 'MOCKING_ENABLED',
+        payload: {
+          client: {
+            id: client.id,
+            frameType: client.frameType,
+          },
+        },
+      })
+      break
+    }
+
+    case 'CLIENT_CLOSED': {
+      activeClientIds.delete(clientId)
+
+      const remainingClients = allClients.filter((client) => {
+        return client.id !== clientId
+      })
+
+      // Unregister itself when there are no more clients
+      if (remainingClients.length === 0) {
+        self.registration.unregister()
+      }
+
+      break
+    }
+  }
+})
+
+addEventListener('fetch', function (event) {
+  const requestInterceptedAt = Date.now()
+
+  // Bypass navigation requests.
+  if (event.request.mode === 'navigate') {
+    return
+  }
+
+  // Opening the DevTools triggers the "only-if-cached" request
+  // that cannot be handled by the worker. Bypass such requests.
+  if (
+    event.request.cache === 'only-if-cached' &&
+    event.request.mode !== 'same-origin'
+  ) {
+    return
+  }
+
+  // Bypass all requests when there are no active clients.
+  // Prevents the self-unregistered worked from handling requests
+  // after it's been terminated (still remains active until the next reload).
+  if (activeClientIds.size === 0) {
+    return
+  }
+
+  const requestId = crypto.randomUUID()
+  event.respondWith(handleRequest(event, requestId, requestInterceptedAt))
+})
+
+/**
+ * @param {FetchEvent} event
+ * @param {string} requestId
+ * @param {number} requestInterceptedAt
+ */
+async function handleRequest(event, requestId, requestInterceptedAt) {
+  const client = await resolveMainClient(event)
+  const requestCloneForEvents = event.request.clone()
+  const response = await getResponse(
+    event,
+    client,
+    requestId,
+    requestInterceptedAt,
+  )
+
+  // Send back the response clone for the "response:*" life-cycle events.
+  // Ensure MSW is active and ready to handle the message, otherwise
+  // this message will pend indefinitely.
+  if (client && activeClientIds.has(client.id)) {
+    const serializedRequest = await serializeRequest(requestCloneForEvents)
+
+    // Clone the response so both the client and the library could consume it.
+    const responseClone = response.clone()
+
+    sendToClient(
+      client,
+      {
+        type: 'RESPONSE',
+        payload: {
+          isMockedResponse: IS_MOCKED_RESPONSE in response,
+          request: {
+            id: requestId,
+            ...serializedRequest,
+          },
+          response: {
+            type: responseClone.type,
+            status: responseClone.status,
+            statusText: responseClone.statusText,
+            headers: Object.fromEntries(responseClone.headers.entries()),
+            body: responseClone.body,
+          },
+        },
+      },
+      responseClone.body ? [serializedRequest.body, responseClone.body] : [],
+    )
+  }
+
+  return response
+}
+
+/**
+ * Resolve the main client for the given event.
+ * Client that issues a request doesn't necessarily equal the client
+ * that registered the worker. It's with the latter the worker should
+ * communicate with during the response resolving phase.
+ * @param {FetchEvent} event
+ * @returns {Promise<Client | undefined>}
+ */
+async function resolveMainClient(event) {
+  const client = await self.clients.get(event.clientId)
+
+  if (activeClientIds.has(event.clientId)) {
+    return client
+  }
+
+  if (client?.frameType === 'top-level') {
+    return client
+  }
+
+  const allClients = await self.clients.matchAll({
+    type: 'window',
+  })
+
+  return allClients
+    .filter((client) => {
+      // Get only those clients that are currently visible.
+      return client.visibilityState === 'visible'
+    })
+    .find((client) => {
+      // Find the client ID that's recorded in the
+      // set of clients that have registered the worker.
+      return activeClientIds.has(client.id)
+    })
+}
+
+/**
+ * @param {FetchEvent} event
+ * @param {Client | undefined} client
+ * @param {string} requestId
+ * @param {number} requestInterceptedAt
+ * @returns {Promise<Response>}
+ */
+async function getResponse(event, client, requestId, requestInterceptedAt) {
+  // Clone the request because it might've been already used
+  // (i.e. its body has been read and sent to the client).
+  const requestClone = event.request.clone()
+
+  function passthrough() {
+    // Cast the request headers to a new Headers instance
+    // so the headers can be manipulated with.
+    const headers = new Headers(requestClone.headers)
+
+    // Remove the "accept" header value that marked this request as passthrough.
+    // This prevents request alteration and also keeps it compliant with the
+    // user-defined CORS policies.
+    const acceptHeader = headers.get('accept')
+    if (acceptHeader) {
+      const values = acceptHeader.split(',').map((value) => value.trim())
+      const filteredValues = values.filter(
+        (value) => value !== 'msw/passthrough',
+      )
+
+      if (filteredValues.length > 0) {
+        headers.set('accept', filteredValues.join(', '))
+      } else {
+        headers.delete('accept')
+      }
+    }
+
+    return fetch(requestClone, { headers })
+  }
+
+  // Bypass mocking when the client is not active.
+  if (!client) {
+    return passthrough()
+  }
+
+  // Bypass initial page load requests (i.e. static assets).
+  // The absence of the immediate/parent client in the map of the active clients
+  // means that MSW hasn't dispatched the "MOCK_ACTIVATE" event yet
+  // and is not ready to handle requests.
+  if (!activeClientIds.has(client.id)) {
+    return passthrough()
+  }
+
+  // Notify the client that a request has been intercepted.
+  const serializedRequest = await serializeRequest(event.request)
+  const clientMessage = await sendToClient(
+    client,
+    {
+      type: 'REQUEST',
+      payload: {
+        id: requestId,
+        interceptedAt: requestInterceptedAt,
+        ...serializedRequest,
+      },
+    },
+    [serializedRequest.body],
+  )
+
+  switch (clientMessage.type) {
+    case 'MOCK_RESPONSE': {
+      return respondWithMock(clientMessage.data)
+    }
+
+    case 'PASSTHROUGH': {
+      return passthrough()
+    }
+  }
+
+  return passthrough()
+}
+
+/**
+ * @param {Client} client
+ * @param {any} message
+ * @param {Array<Transferable>} transferrables
+ * @returns {Promise<any>}
+ */
+function sendToClient(client, message, transferrables = []) {
+  return new Promise((resolve, reject) => {
+    const channel = new MessageChannel()
+
+    channel.port1.onmessage = (event) => {
+      if (event.data && event.data.error) {
+        return reject(event.data.error)
+      }
+
+      resolve(event.data)
+    }
+
+    client.postMessage(message, [
+      channel.port2,
+      ...transferrables.filter(Boolean),
+    ])
+  })
+}
+
+/**
+ * @param {Response} response
+ * @returns {Response}
+ */
+function respondWithMock(response) {
+  // Setting response status code to 0 is a no-op.
+  // However, when responding with a "Response.error()", the produced Response
+  // instance will have status code set to 0. Since it's not possible to create
+  // a Response instance with status code 0, handle that use-case separately.
+  if (response.status === 0) {
+    return Response.error()
+  }
+
+  const mockedResponse = new Response(response.body, response)
+
+  Reflect.defineProperty(mockedResponse, IS_MOCKED_RESPONSE, {
+    value: true,
+    enumerable: true,
+  })
+
+  return mockedResponse
+}
+
+/**
+ * @param {Request} request
+ */
+async function serializeRequest(request) {
+  return {
+    url: request.url,
+    mode: request.mode,
+    method: request.method,
+    headers: Object.fromEntries(request.headers.entries()),
+    cache: request.cache,
+    credentials: request.credentials,
+    destination: request.destination,
+    integrity: request.integrity,
+    redirect: request.redirect,
+    referrer: request.referrer,
+    referrerPolicy: request.referrerPolicy,
+    body: await request.arrayBuffer(),
+    keepalive: request.keepalive,
+  }
+}

src/backend/common/AbstractComponent.ts

@@ -1,41 +1,46 @@
-import { childLogger, Logger } from "@foxxmd/logging";
+import { childLogger, type Logger } from "@foxxmd/logging";
 import {
     cacheFunctions,
 } from "@foxxmd/regex-buddy-core";
-import { LifecycleStep, PlayData, PlayObject, TransformResult } from "../../core/Atomic.js";
-import { buildPlayHumanDiffable, buildTrackString } from "../../core/StringUtils.js";
-import { CommonClientConfig } from "./infrastructure/config/client/index.js";
-import { CommonSourceConfig } from "./infrastructure/config/source/index.js";
-import { mergeSimpleError, SimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js";
+import type EventEmitter from "events";
+import type {ComponentType, LifecycleInput, LifecycleStep, PlayData, PlayObject} from "../../core/Atomic.ts";
+import { buildTrackString } from "../../core/StringUtils.ts";
+import type {CommonClientConfig} from "./infrastructure/config/client/index.ts";
+import type {CommonSourceConfig} from "./infrastructure/config/source/index.ts";
+import { mergeSimpleError, SimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.ts";
 import {
     FLOW_CONTROL_TERM,
-    PlayTransformRules,
-    StageConfig,
+    type PlayTransformRules,
+    type StageConfig,
     TRANSFORM_HOOK,
-    TransformHook
-} from "./infrastructure/Transform.js";
-import AbstractInitializable from "./AbstractInitializable.js";
-import TransformerManager from "./transforms/TransformerManager.js";
-import { getRoot } from "../ioc.js";
+    type TransformHook
+} from "../../core/Transform.ts";
+import AbstractInitializable from "./AbstractInitializable.ts";
+import type TransformerManager from "./transforms/TransformerManager.ts";
+import { getRoot } from "../ioc.ts";
 import { nanoid } from "nanoid";
-import { isDebugMode } from "../utils.js";
-import { findCauseByReference } from "../utils/ErrorUtils.js";
-import { hashObject, parseArrayFromMaybeString } from "../utils/StringUtils.js";
-import { metaInvariantTransform, playContentInvariantTransform } from "../utils/PlayComparisonUtils.js";
-import { MSCache } from "./Cache.js";
-import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/DataUtils.js";
+import { isDebugMode } from "../utils.ts";
+import { findCauseByFunc, findCauseByReference } from "../utils/ErrorUtils.ts";
+import { hashObject, parseArrayFromMaybeString } from "../utils/StringUtils.ts";
+import { playContentInvariantTransform } from "../utils/PlayComparisonUtils.ts";
+import type { MSCache } from "./Cache.ts";
+import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/DataUtils.ts";
 import clone from "clone";
-import { loggerNoop } from "./MaybeLogger.js";
-import { objectsEqual } from "../utils/DataUtils.js";
-import { RetentionOptions } from "./infrastructure/config/database.js";
-import { getRetentionCompactAfterFromEnv, getRetentionDeleteAfterFromEnv, isCompactableProperty, parseRetentionOptions, parseRetentionOptionsDurations } from "./database/Database.js";
-import { DbConcrete } from "./database/drizzle/drizzleUtils.js";
-import { ComponentSelect } from "./database/drizzle/drizzleTypes.js";
-import { DrizzlePlayRepository } from "./database/drizzle/repositories/PlayRepository.js";
-import { ClientType } from "./infrastructure/config/client/clients.js";
-import { SourceType } from "./infrastructure/config/source/sources.js";
-import { DrizzleComponentRepository } from "./database/drizzle/repositories/ComponentRepository.js";
+import { loggerNoop } from "./MaybeLogger.ts";
+import { objectsEqual } from "../utils/DataUtils.ts";
+import type {RetentionOptions} from "./infrastructure/config/database.ts";
+import { getRetentionCompactAfterFromEnv, getRetentionDeleteAfterFromEnv, isCompactableProperty, parseRetentionOptions, parseRetentionOptionsDurations } from "./database/Database.ts";
+import type {DbConcrete} from "./database/drizzle/drizzleUtils.ts";
+import type {ComponentSelect} from "./database/drizzle/drizzleTypes.ts";
+import { DrizzlePlayRepository } from "./database/drizzle/repositories/PlayRepository.ts";
+import type {ClientType} from "../../core/Atomic.ts";
+import type {SourceType} from "../../core/Atomic.ts";
+import { DrizzleComponentRepository } from "./database/drizzle/repositories/ComponentRepository.ts";
 import dayjs from "dayjs";
+import { COMPONENT_STATE, type ComponentCommonApi, type ComponentCommonApiJson, type ComponentState, type PlayApiCommonDetailed } from "../../core/Api.ts";
+import type {WebhookPayload} from "./infrastructure/config/health/webhooks.ts";
+import type { MarkRequired } from "ts-essentials";
+import { serializeError } from "serialize-error";
 
 export type AbstractComponentConfig = (CommonClientConfig | CommonSourceConfig) & { transformManager?: TransformerManager };
 
@@ -50,10 +55,14 @@ export default abstract class AbstractComponent extends AbstractInitializable {
     protected db: DbConcrete;
     protected componentRepo!: DrizzleComponentRepository;
     protected dbComponent!: ComponentSelect;
+    componentId!: number;
     protected retentionOpts: RetentionOptions;
+    status: string = 'Waiting to initialize...';
+    emitter: EventEmitter;
 
-    protected componentType: 'source' | 'client';
+    protected componentType: ComponentType;
     type: ClientType | SourceType;
+    name: string;
 
     protected constructor(config: AbstractComponentConfig) {
         super(config);
@@ -80,12 +89,12 @@ export default abstract class AbstractComponent extends AbstractInitializable {
     }
 
     protected async postInitialize(): Promise<void> {
-        super.postInitialize();
+        await super.postInitialize();
         this.componentRepo.updateById(this.dbComponent.id, {lastReadyAt: dayjs()});
     }
 
     protected async doBuildDatabase(): Promise<true | string | undefined> {
-        super.doBuildDatabase();
+        await super.doBuildDatabase();
 
         let name: string;
         if('name' in this) {
@@ -100,6 +109,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             uid: this.config.id ?? this.config.name ?? name,
             name: this.config.name ?? name
         });
+        this.componentId = this.dbComponent.id;
         return true;
     }
 
@@ -208,11 +218,16 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             this.logger.warn(`Cannot run retention cleanup because ${this.componentType} database state is not OK`);
             return;
         }
+        this.setStatus('Running retention cleanup');
         try {
             const repo = new DrizzlePlayRepository(this.db, {logger: this.logger});
             await repo.retentionCleanup(this.componentType, this.retentionOpts);
+            this.setStatus('Retention cleanup finished');
         } catch (e) {
-            this.logger.warn(new Error('Failed to do retention cleanup', {cause: e}));
+            const retentionErr = new Error('Failed to do retention cleanup', {cause: e});
+            this.warning = retentionErr;
+            this.logger.warn(retentionErr);
+            this.setStatus('Retention cleanup failed');
         }
     }
 
@@ -288,7 +303,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
                     }
 
                     for(const s of cachedSteps) {
-                        steps.push({...s, cached: true});
+                        steps.push({...s, cached: true, createdAt: dayjs().toISOString()});
                         if(shouldTransform && s.patch !== undefined) {
                             transformedPlay.data = patchObject(transformedPlay.data, s.patch); // jdiff.patch(clone(transformedPlay.data),s.patch);
                         }
@@ -307,7 +322,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
                         if(step.error !== undefined && step.flowKnownState !== 'skip' && !step.returnPartial) {
                             // revert to original play but keep steps for paper trail
                             transformedPlay = play;
-                            transformedPlay.meta.lifecycle.steps = steps;
+                            transformedPlay.lifecycle = steps;
                         } else {
                             transformedPlay = stepPlay;
                         }
@@ -321,34 +336,30 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             const historyToDiff: {name: string, data?: PlayData}[] = [
                 {name: 'Pre Transform', data: play.data}
             ];
-            if(steps.length > 0 && transformedPlay.meta?.lifecycle?.steps === undefined) {
-                const {
-                    meta: {
-                        lifecycle: {
-                            steps = [],
-                            ...lifecycleRest
-                        },
-                    } = {}
-                } = transformedPlay;
-                transformedPlay.meta.lifecycle = {
-                    ...lifecycleRest,
-                    steps
-                }
+            let isNew = false;
+            if(steps.length > 0 && transformedPlay.lifecycle === undefined) {
+                transformedPlay.lifecycle = steps;
+                isNew = true;
             }
+            const {
+                lifecycle = []
+            } = transformedPlay;
             steps.forEach((s, index) => {
-                const existingStepIndex = transformedPlay.meta.lifecycle.steps.findIndex(x => x.name === 'hookType' && x.source === this.getIdentifier());
-                if(existingStepIndex !== -1) {
-                    transformedPlay.meta.lifecycle.steps[existingStepIndex] = s;
-                } else {
-                    transformedPlay.meta.lifecycle.steps.push(s);
+                if(!isNew) {
+                    const existingStepIndex = lifecycle.findIndex(x => x.stageName === s.stageName && x.stageType === s.stageType && x.hook === s.hook && x.source === this.getIdentifier());
+                    if(existingStepIndex !== -1) {
+                        transformedPlay.lifecycle[existingStepIndex] = s;
+                    } else {
+                        transformedPlay.lifecycle.push(s);
+                    }
                 }
 
                 if(shouldLog === 'all') {
                     if(s.patch === undefined) {
-                        historyToDiff.push({name: `${s.source}-${s.name}`});
+                        historyToDiff.push({name: `${s.source}-${s.hook}-${s.stageType}-${s.stageName}`});
                     } else {
                         const patched = patchObject(historyToDiff[historyToDiff.length - 1].data, s.patch);
-                        historyToDiff.push({name: `${s.source}-${s.name}${s.cached ? ' (Cached)' : ''}`, data: patched});
+                        historyToDiff.push({name: `${s.source}-${s.hook}-${s.stageType}-${s.stageName} ${s.cached ? ' (Cached)' : ''}`, data: patched});
                     }
                 }
             });
@@ -410,11 +421,18 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             asyncId = nanoid(6)
         } = opts;
 
-        const stepName = `${hookType} - ${hookItem.type} - ${hookItem.name}`
-        const existingStepIndex = playTruth.meta.lifecycle.steps.findIndex(x => x.name === stepName && x.source === this.getIdentifier());
-        const step: LifecycleStep = existingStepIndex !== -1 && playTruth.meta.lifecycle.steps[existingStepIndex] !== undefined ? playTruth.meta.lifecycle.steps[existingStepIndex] : {
-            name: stepName,
+        const {
+            lifecycle = []
+        } = playTruth;
+
+        //const stepName = `${hookType} - ${hookItem.type} - ${hookItem.name}`
+        const existingStepIndex = lifecycle.findIndex(x => x.hook === hookType && hookItem.name === x.stageName && x.stageType === hookItem.type && x.source === this.getIdentifier());
+        const step: LifecycleStep = existingStepIndex !== -1 && lifecycle[existingStepIndex] !== undefined ? lifecycle[existingStepIndex] : {
+            stageName: hookItem.name,
+            hook: hookType,
+            stageType: hookItem.type,
             source: this.getIdentifier(),
+            createdAt: dayjs().toString()
         }
 
         let newTransformedPlay: PlayObject,
@@ -430,7 +448,16 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             }
         }
 
+        step.stageName = stageName;
+
         if (err !== undefined) {
+            const lifecycleErrorContext = findCauseByFunc(err, (e) => 'lifecycleInputs' in e && e.lifecycleInputs !== undefined);
+            if (lifecycleErrorContext !== undefined) {
+                // @ts-expect-error it does exist
+                step.inputs = clone(lifecycleErrorContext.lifecycleInputs) as LifecycleInput[];
+                // @ts-expect-error it does exist
+                delete lifecycleErrorContext.lifecycleInputs;
+            }
             const merged = mergeSimpleError(err);
             step.error = merged;
 
@@ -495,4 +522,67 @@ export default abstract class AbstractComponent extends AbstractInitializable {
 
         return [step, newTransformedPlay];
     }
+
+    public abstract getRunningState(): ComponentState
+
+    public getApiData(): Omit<ComponentCommonApiJson, 'type' | 'countLive' | 'players'> & Pick<ComponentCommonApi, 'state' | 'error' | 'warning'>  {
+        let state: ComponentState;
+        if(!this.initializedOnce || this.initializing) {
+            state = COMPONENT_STATE.INITIALIZING;
+        } else if(!this.isReady()) {
+            state = COMPONENT_STATE.NOT_READY;
+        } else {
+            state = this.getRunningState()
+        }
+        return {
+            id: this.dbComponent.id,
+            uid: this.dbComponent.uid,
+            name: this.dbComponent.name,
+            state,
+            mode: this.dbComponent.mode,
+            countNonLive: this.dbComponent.countNonLive,
+            createdAt: this.dbComponent.createdAt?.toISOString(),
+            lastReadyAt: this.dbComponent.lastReadyAt?.toISOString(),
+            lastActiveAt: this.dbComponent.lastActiveAt?.toISOString(),
+            error: this.error !== undefined && this.error instanceof Error ? serializeError(this.error) : this.error,
+            warning: this.warning !== undefined && this.warning instanceof Error ? serializeError(this.warning) : this.warning,
+            ...this.additionalApiData()
+        }
+    }
+
+    public emitEvent = (eventName: string, payload: Record<string, any> = {}) => {
+        this.emitter.emit(eventName, {
+            type: this.type,
+            name: this.name,
+            componentId: this.dbComponent?.id,
+            from: this.componentType,
+            data: payload,
+        });
+    }
+
+    protected emitComponentUpdate = <T extends Partial<ReturnType<typeof this.getApiData>>>(payload: T) => {
+        if('error' in payload && payload.error instanceof Error) {
+            payload.error = serializeError(payload.error);
+        }
+        if('warning' in payload && payload.warning instanceof Error) {
+            payload.warning = serializeError(payload.warning);
+        }
+        this.emitEvent('componentUpdate', payload);
+    }
+    protected emitPlayUpdate = (payload: MarkRequired<Partial<PlayApiCommonDetailed>, 'uid'>) => {
+        this.emitEvent('playUpdate', payload);
+    }
+    protected emitPlayInsert = (payload: PlayApiCommonDetailed) => {
+        this.emitEvent('playInsert', payload);
+    }
+
+    async notify(payload: Omit<WebhookPayload, 'identifier'>) {
+        this.emitEvent('notify', {...payload, identifier: this.getIdentifier()});
+        this.setStatus(payload.title);
+    }
+
+    protected setStatus = (status: string) => {
+        this.status = status;
+        this.emitComponentUpdate({status});
+    }
 }

src/backend/common/AbstractInitializable.ts

@@ -1,10 +1,10 @@
-import { childLogger, Logger } from "@foxxmd/logging";
-import {truncateStringToLength } from "../../core/StringUtils.js";
-import { hasNodeNetworkException } from "./errors/NodeErrors.js";
-import { hasUpstreamError } from "./errors/UpstreamError.js";
-import { WebhookPayload } from "./infrastructure/config/health/webhooks.js";
-import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, StageError, TransformRulesError } from "./errors/MSErrors.js";
-import { messageWithCausesTruncatedDefault } from "../../core/ErrorUtils.js";
+import type { Logger } from "@foxxmd/logging";
+import {truncateStringToLength } from "../../core/StringUtils.ts";
+import { hasNodeNetworkException } from "./errors/NodeErrors.ts";
+import { hasUpstreamError } from "./errors/UpstreamError.ts";
+import type {WebhookPayload} from "./infrastructure/config/health/webhooks.ts";
+import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, StageError } from "./errors/MSErrors.ts";
+import { messageWithCausesTruncatedDefault } from "../../core/ErrorUtils.ts";
 import { spawn } from 'abort-controller-x';
 
 export default abstract class AbstractInitializable {
@@ -19,7 +19,10 @@ export default abstract class AbstractInitializable {
     databaseOK?: boolean | null;
     connectionOK?: boolean | null;
     cacheOK?: boolean | null;
+    error?: Error;
+    warning?: Error;
 
+    protected initializedOnce: boolean = false;
     initializing: boolean = false;
 
     config: Record<string, any>;
@@ -32,17 +35,21 @@ export default abstract class AbstractInitializable {
     }
 
     public abstract notify(payload: WebhookPayload): Promise<void>;
+    protected setStatus(status: string): void {
+        return;
+    }
 
     protected abstract getIdentifier(): string;
 
     initialize = async (options: {force?: boolean, notify?: boolean, notifyTitle?: string, internalOnly?: boolean} = {}) => {
-
+        this.initializedOnce = true;
         if(this.initController !== undefined) {
             throw new Error(`Already trying to initialize, cannot attempt while an existing initialization attempt is running.`);
         }
 
         this.initController = new AbortController();
         return spawn(this.initController.signal, async (signal, {defer, fork}) => {
+            this.setStatus('Initializing...');
 
             defer(async () => {
                 this.initializing = false;
@@ -75,17 +82,22 @@ export default abstract class AbstractInitializable {
                 await this.checkConnection(force);
                 await this.testAuth(force);
                 this.logger.info('Fully Initialized!');
+                this.setStatus('Initialized!');
                 try {
                     await this.postInitialize();
                 } catch (e) {
                     throw new PostInitError('Error occurred during post-initialization hook', {cause: e});
                 }
+                this.error = undefined;
+                this.warning = undefined;
                 return true;
             } catch(e) {
                 if(notify) {
-                    await this.notify({title: `${this.getIdentifier()} - ${notifyTitle}`, message: truncateStringToLength(500)(messageWithCausesTruncatedDefault(e)), priority: 'error'});
+                    await this.notify({identifier: this.getIdentifier(), title: notifyTitle, message: truncateStringToLength(500)(messageWithCausesTruncatedDefault(e)), priority: 'error'});
                 }
-                throw new Error('Initialization failed', {cause: e});
+                const initError = new Error('Initialization failed', {cause: e});
+                this.error = initError;
+                throw initError;
             } finally {
                 this.initializing = false;
             }

src/backend/common/Cache.ts

@@ -1,7 +1,7 @@
-import { Cacheable, createKeyv, Keyv, KeyvStoreAdapter, KeyvOptions, CacheableOptions, KeyvCacheableMemory } from 'cacheable';
-import { FlatCache, FlatCacheOptions } from 'flat-cache';
-import KeyvValkey, { KeyvValkeyOptions } from '@keyv/valkey';
-import dayjs, { Dayjs } from 'dayjs';
+import { Cacheable, createKeyv, Keyv, type KeyvStoreAdapter, type KeyvOptions, type CacheableOptions, KeyvCacheableMemory } from 'cacheable';
+import { FlatCache, type FlatCacheOptions } from 'flat-cache';
+import KeyvValkey, { type KeyvValkeyOptions } from '@keyv/valkey';
+import dayjs, { type Dayjs } from 'dayjs';
 import duration from 'dayjs/plugin/duration.js';
 import isBetween from 'dayjs/plugin/isBetween.js';
 import relativeTime from 'dayjs/plugin/relativeTime.js';
@@ -9,19 +9,19 @@ import isToday from 'dayjs/plugin/isToday.js';
 import timezone from 'dayjs/plugin/timezone.js';
 import utc from 'dayjs/plugin/utc.js';
 import clone from 'clone';
-import { childLogger, Logger } from '@foxxmd/logging';
-import { projectDir } from './index.js';
+import { childLogger, type Logger } from '@foxxmd/logging';
+import { projectDir } from './index.ts';
 import path from 'path';
 import { cacheFunctions } from "@foxxmd/regex-buddy-core";
-import { fileExists, fileOrDirectoryIsWriteable } from '../utils/FSUtils.js';
-import { copyFile } from 'fs/promises';
-import { asCacheConfig, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheConfigUser, CacheScrobbleProvider } from './infrastructure/Atomic.js';
+import { fileOrDirectoryIsWriteable } from '../utils/FSUtils.ts';
+import { asCacheConfig, type CacheAuthProvider, type CacheConfig, type CacheConfigOptions, type CacheConfigUser, type CacheScrobbleProvider } from './infrastructure/Atomic.ts';
 import { Typeson } from 'typeson';
 import { builtin } from 'typeson-registry';
-import { loggerNoop } from './MaybeLogger.js';
+import { loggerNoop } from './MaybeLogger.ts';
 const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
-import prom, { Gauge } from 'prom-client';
-import { nonEmptyStringOrDefault } from '../../core/StringUtils.js';
+import type { Gauge } from 'prom-client';
+import prom from 'prom-client';
+import { nonEmptyStringOrDefault } from '../../core/StringUtils.ts';
 
 dayjs.extend(utc)
 dayjs.extend(isBetween);
@@ -217,7 +217,7 @@ export class MSCache {
 
     protected initCacheable = async (cacheFor: string, primaryConfig: CacheConfig, secondaryConfig?: CacheConfig) => {
 
-        let logger = childLogger(this.logger, cacheFor);
+        const logger = childLogger(this.logger, cacheFor);
         const providerHints = [];
         if(primaryConfig.provider === false) {
             const cache = new Cacheable({primary: noopKeyv});
@@ -329,12 +329,12 @@ export class MSCache {
         if (!this.hasInit) {
             let authConfig: CacheConfig | undefined;
             try {
-                if(asCacheConfig(this.config.scrobble)) {
+                if(asCacheConfig(this.config.auth)) {
                     authConfig = this.config.auth;
                 }
             } catch (e) {
                 this.logger.warn(new Error('Could not validate auth config! will fallback to memory cache only', {cause: e}));
-                this.cacheScrobble = await this.initCacheable('Scrobble', {provider: false});
+                this.cacheAuth = await this.initCacheable('Auth', {provider: false});
             }
 
             this.cacheAuth = await this.initCacheable('Auth', {provider: 'memory', ttl: '3m'}, authConfig);

src/backend/common/MaybeLogger.ts

@@ -1,4 +1,4 @@
-import { Logger } from "@foxxmd/logging";
+import type { Logger } from "@foxxmd/logging";
 
 
 export class MaybeLogger {

src/backend/common/database/Database.ts

@@ -1,18 +1,16 @@
-import { configDir } from '../index.js';
+import { configDir } from '../index.ts';
 import * as path from 'path';
-import { promises as fs } from 'fs'
-import { childLogger, Logger } from '@foxxmd/logging';
-import { loggerNoop } from '../MaybeLogger.js';
-import { fileExists, fileOrDirectoryIsWriteable } from '../../utils/FSUtils.js';
-import { COMPACTABLE, compactableProperties, CompactableProperty, DEFAULT_RETENTION_COMPACT_AFTER, DEFAULT_RETENTION_DELETE_AFTER, RententionGranular, RetentionConfig, RetentionConfigValue, RetentionOption, RetentionValue, RetentionValueUnparsed } from '../infrastructure/config/database.js';
-import { DurationValue } from '../infrastructure/Atomic.js';
-import { Duration } from 'dayjs/plugin/duration.js';
+import { childLogger, type Logger } from '@foxxmd/logging';
+import { loggerNoop } from '../MaybeLogger.ts';
+import { COMPACTABLE, type CompactableProperty, DEFAULT_RETENTION_COMPACT_AFTER, DEFAULT_RETENTION_DELETE_AFTER, type RetentionConfigValue, type RetentionOption, type RetentionValue, type RetentionValueUnparsed } from '../infrastructure/config/database.ts';
+import type {DurationValue} from '../infrastructure/Atomic.ts';
+import type {Duration} from 'dayjs/plugin/duration.js';
 import dayjs from 'dayjs';
-import { parseDurationFromDurationValue } from '../../utils/TimeUtils.js';
-import assert, { AssertionError } from 'node:assert';
+import { parseDurationFromDurationValue } from '../../utils/TimeUtils.ts';
+import assert from 'node:assert';
 import * as sqlite from 'node:sqlite';
-import { parseBoolStrict } from '../../utils.js';
-import { SimpleError } from '../errors/MSErrors.js';
+import { parseBoolStrict } from '../../utils.ts';
+import { SimpleError } from '../errors/MSErrors.ts';
 
 export const MEMORY_DB_NAME = ':memory:';
 export const isMemoryDb = (name: string): boolean => name === MEMORY_DB_NAME;
@@ -34,7 +32,7 @@ export const backupDb = async (db: sqlite.DatabaseSync, dbPath: string, opts: {
 
     const {
         logger: parentLogger = loggerNoop,
-        suffix,
+        suffix = dayjs().unix().toString(),
     } = opts;
 
     const logger = childLogger(parentLogger, 'Migrations');

src/backend/common/database/appMigrations/001_lifecycleLoc.ts

@@ -0,0 +1,66 @@
+import type { SqliteDatabase, Migration } from 'sqlite-up';
+import type {MigrateBaseContext} from '../appMigrator.ts';
+import { plays as drizzlePlays } from '../drizzle/schema/schema.ts';
+import clone from 'clone';
+import { eq } from 'drizzle-orm';
+import type {PlayLifecycle, PlayObject} from '../../../../core/Atomic.ts';
+
+
+export const up: Migration<MigrateBaseContext>['up'] = async (db: SqliteDatabase, ctx: MigrateBaseContext): Promise<void> => {
+
+    ctx.logger.info('Migrating Play lifecycle data to top-level location. This may take some time...');
+
+    let more = true;
+    let offset = 0,
+        processed = 0,
+        updated = 0;
+    while (more) {
+        const playRows = await ctx.db.select().from(drizzlePlays).limit(100).offset(offset);
+        for (const row of playRows) {
+            const play = row.play as PlayObject<{lifecycle?: PlayLifecycle}>;
+            try {
+                const {
+                    meta: {
+                        lifecycle,
+                        lifecycle: {
+                            steps = [],
+                            scrobble
+                        } = {}
+                    } = {}
+                } = play;
+                if (lifecycle === undefined) {
+                    // already migrated or unneeded
+                    processed++;
+                    continue;
+                }
+                if (steps.length > 0) {
+                    row.play.lifecycle = clone(steps);
+                }
+                if (scrobble !== undefined) {
+                    row.play.scrobble = clone(scrobble);
+                }
+                // @ts-expect-error
+                delete row.play.meta.lifecycle;
+                await ctx.db.update(drizzlePlays).set({ play: row.play }).where(eq(drizzlePlays.id, row.id));
+                updated++;
+                processed++;
+            } catch (e) {
+                ctx.logger.warn(new Error(`Failed to migrate Play ${row.id} (${row.uid})`, { cause: e }));
+            }
+        }
+        offset += 100;
+        ctx.logger.verbose(`Migration Progress: Processed ${processed} | Updated ${updated}`);
+        if(playRows.length < 100) {
+            more = false;
+        }
+    }
+
+
+    // Migration code here
+    // context is passed as ctx
+};
+
+export const down: Migration<MigrateBaseContext>['down'] = async (db: SqliteDatabase, ctx: MigrateBaseContext): Promise<void> => {
+    // Rollback code here
+    // context is passed as ctx
+};

src/backend/common/database/appMigrator.ts

@@ -1,16 +1,18 @@
-import { DbConcrete } from "./drizzle/drizzleUtils.js";
-import { loggerNoop } from "../MaybeLogger.js";
+import type {DbConcrete} from "./drizzle/drizzleUtils.ts";
+import { loggerNoop } from "../MaybeLogger.ts";
 import * as path from 'path';
-import { childLogger, Logger } from "@foxxmd/logging";
-import { projectDir } from "../index.js";
+import { childLogger, type Logger } from "@foxxmd/logging";
+import { projectDir } from "../index.ts";
 import { Migrator } from 'sqlite-up';
-import { MigrationStatus } from "../infrastructure/Atomic.js";
+import type {MigrationStatus} from "../infrastructure/Atomic.ts";
 
 export interface MigrateBaseContext {
     db: DbConcrete
     logger: Logger
 }
 
+const migrationsTable = '__app_migrations';
+
 export const getAppMigrationStatus = async (db: DbConcrete, opts: {logger?: Logger, migrationsAppFolder?: string} = {}): Promise<MigrationStatus> => {
     const {
     logger: parentLogger = loggerNoop,
@@ -21,7 +23,7 @@ export const getAppMigrationStatus = async (db: DbConcrete, opts: {logger?: Logg
     const migrator = new Migrator({
         db: db.$client,
         migrationsDir: migrationsAppFolder,
-        migrationsTable: '__app_migrations'
+        migrationsTable: migrationsTable
     });
 
     const status = await migrator.status();
@@ -40,12 +42,12 @@ export const migrateApp = async (db: DbConcrete, opts: {logger?: Logger, migrati
     logger: parentLogger = loggerNoop,
     migrationsAppFolder = path.resolve(projectDir, 'src/backend/common/database/appMigrations')
     } = opts;
-    const logger = childLogger(parentLogger, 'App');
+    const logger = childLogger(parentLogger, ['App', 'Migrations']);
 
     const migrator = new Migrator<MigrateBaseContext>({
         db: db.$client,
         migrationsDir: migrationsAppFolder,
-        migrationsTable: 'appMigrations'
+        migrationsTable: migrationsTable
     });
 
     migrator.on('migration:applied', function (name: string, batch: number): void {

src/backend/common/database/drizzle/drizzleTypes.ts

@@ -1,11 +1,11 @@
-import { DBQueryConfig, DBQueryConfigWith, ExtractTablesFromSchema, KnownKeysOnly, RelationFieldsFilterInternals, Many, InferSelectModel, ExtractTablesWithRelations, type BuildQueryResult, RelationsFilter } from "drizzle-orm";
-import { components, componentMigrations, playInputs, plays, queueStates, relations } from "./schema/schema.js";
-import {TSchema, TableName, Schema } from "./schema/schema.js";
-import { MarkOptional, MarkRequired } from "ts-essentials";
+import type {DBQueryConfig, DBQueryConfigWith, KnownKeysOnly, RelationFieldsFilterInternals, BuildQueryResult, RelationsFilter} from "drizzle-orm";
+import type { components, componentMigrations, playInputs, plays, queueStates, relations, playsHistorical } from "./schema/schema.ts";
+import type {TSchema, TableName} from "./schema/schema.ts";
 
 
 export type ComponentNew = typeof components.$inferInsert;
 export type ComponentSelect = GenericRelationResult<'components', 'migrations'>;
+export type ComponentMinimalSelect = typeof components.$inferSelect;
 
 export type ComponentMigrationNew = typeof componentMigrations.$inferInsert;
 export type ComponentMigrationSelect = typeof componentMigrations.$inferSelect;
@@ -21,6 +21,8 @@ export type PlaySelectWithQueueStates = GenericRelationResult<'plays', 'queueSta
 export type PlayWith<K extends keyof TSchema['plays']["relations"]> = GenericRelationResult<'plays', K>;
 export type PlayNew = typeof plays.$inferInsert;
 
+export type PlayHistoricalSelect = typeof playsHistorical.$inferSelect;
+export type PlayHistoricalNew = typeof playsHistorical.$inferInsert;
 
 // useful references for building types
 // https://github.com/drizzle-team/drizzle-orm/discussions/2596

src/backend/common/database/drizzle/drizzleUtils.ts

@@ -1,19 +1,18 @@
 import { drizzle } from 'drizzle-orm/node-sqlite';
 import { migrate } from 'drizzle-orm/node-sqlite/migrator';
-import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
-import { sql as dsl, LogWriter, Logger as DrizzleLogger } from 'drizzle-orm';
+import type { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
+import { sql as dsl, type Logger as DrizzleLogger } from 'drizzle-orm';
 import * as fs from 'fs/promises';
 import * as path from 'path';
-import { backupDb, getDbPath, getDbBackupPath, MEMORY_DB_NAME } from '../Database.js';
-import { fileExists, fileOrDirectoryIsWriteable } from '../../../utils/FSUtils.js';
-import { childLogger, Logger, LogLevel } from '@foxxmd/logging';
-import { loggerNoop } from '../../MaybeLogger.js';
-import { projectDir } from '../../index.js';
-import { relations } from './schema/schema.js';
-import { addToContext, executeQuery } from './logContext.js';
-import { DatabaseSync } from 'node:sqlite';
-import { migrateApp, getAppMigrationStatus } from '../appMigrator.js';
-import { MigrationStatus } from '../../infrastructure/Atomic.js';
+import { backupDb, getDbBackupPath, MEMORY_DB_NAME } from '../Database.ts';
+import { fileExists, fileOrDirectoryIsWriteable } from '../../../utils/FSUtils.ts';
+import { childLogger, type Logger, type LogLevel } from '@foxxmd/logging';
+import { loggerNoop } from '../../MaybeLogger.ts';
+import { projectDir } from '../../index.ts';
+import { relations } from './schema/schema.ts';
+import { addToContext, executeQuery } from './logContext.ts';
+import { migrateApp, getAppMigrationStatus } from '../appMigrator.ts';
+import type {MigrationStatus} from '../../infrastructure/Atomic.ts';
 
 export async function getDbMigrationStatus(dbVal: string | DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise<MigrationStatus> {
   const {

src/backend/common/database/drizzle/entityUtils.ts

@@ -1,14 +1,13 @@
 import assert from "node:assert";
-import { PlayNew, PlaySelect, PlaySelectWithQueueStates } from "./drizzleTypes.js";
-import { PlayInputNew } from "./drizzleTypes.js";
-import { QueueStateNew } from "./drizzleTypes.js";
-import { ComponentNew } from "./drizzleTypes.js";
-import { MarkOptional, MarkRequired } from "ts-essentials";
-import { CLIENT_DEAD_QUEUE, DeadLetterScrobble, ErrorLike, PlayObject } from "../../../../core/Atomic.js";
-import dayjs, { Dayjs } from "dayjs";
-import { asPlay } from "../../../../core/PlayMarshalUtils.js";
-import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../utils/PlayComparisonUtils.js";
-import { hashObject } from "../../../utils/StringUtils.js";
+import type {PlayHistoricalNew, PlayHistoricalSelect, PlayNew, PlaySelect, PlaySelectWithQueueStates} from "./drizzleTypes.ts";
+import type {PlayInputNew} from "./drizzleTypes.ts";
+import type {QueueStateNew} from "./drizzleTypes.ts";
+import type {ComponentNew} from "./drizzleTypes.ts";
+import type { MarkOptional } from "ts-essentials";
+import { CLIENT_DEAD_QUEUE, type DeadLetterScrobble, type ErrorLike, type PlayObject } from "../../../../core/Atomic.ts";
+import dayjs from "dayjs";
+import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../utils/PlayComparisonUtils.ts";
+import { hashObject } from "../../../utils/StringUtils.ts";
 import { serializeError } from "serialize-error";
 
 export const generateComponentEntity = (data: MarkOptional<ComponentNew, 'uid'>): ComponentNew => {
@@ -20,6 +19,7 @@ export const generateComponentEntity = (data: MarkOptional<ComponentNew, 'uid'>)
 }
 
 export type PlayEntityOpts = Partial<Pick<PlayNew, 'seenAt' | 'playedAt' | 'uid' | 'state' | 'parentId' | 'componentId'>> & { error?: ErrorLike };
+export type PlayHistoricalEntityOpts = Partial<Pick<PlayHistoricalNew, 'seenAt' | 'playedAt' | 'uid' | 'componentId'>>;
 
 export const generatePlayEntity = (play: PlayObject, opts: PlayEntityOpts = {}): PlayNew => {
     const {
@@ -51,12 +51,12 @@ export const generatePlayEntity = (play: PlayObject, opts: PlayEntityOpts = {}):
 
 export type PlayHydateOptions = 'asPlay' | 'id' | 'uid';
 
-export const hydratePlaySelect = (select: PlaySelect, opts: PlayHydateOptions[] = ['id','uid']): PlayObject => {
+export const hydratePlaySelect = <T extends PlaySelect | PlayHistoricalSelect>(select: T, opts: PlayHydateOptions[] = ['id','uid']): PlayObject => {
     if(opts.length === 0) {
         return select.play;
     }
 
-    let res = select.play;
+    const res = select.play;
     // if(opts.includes('asPlay')) {
     //     res = asPlay(res);
     // }

src/backend/common/database/drizzle/logContext.ts

@@ -1,4 +1,4 @@
-import { Logger } from '@foxxmd/logging'
+import type { Logger } from '@foxxmd/logging'
 import { AsyncLocalStorage } from 'async_hooks'
 
 // based on https://numeric.substack.com/p/upgrading-drizzleorm-logging-with

src/backend/common/database/drizzle/repositories/BaseRepository.ts

@@ -1,36 +1,22 @@
-import { childLogger, Logger } from "@foxxmd/logging";
-import { DbConcrete } from "../drizzleUtils.js";
-import { CompareOpKey } from "../drizzleTypes.js";
-import { Dayjs } from "dayjs";
-import { RelationsFieldFilter, eq, inArray } from "drizzle-orm";
-import { loggerNoop } from "../../../MaybeLogger.js";
-import { capitalize } from "../../../../../core/StringUtils.js";
-import { getConfigByTableName, relations, TableName } from "../schema/schema.js";
+import { childLogger, type Logger } from "@foxxmd/logging";
+import type { DbConcrete } from "../drizzleUtils.ts";
+import type { Dayjs } from "dayjs";
+import { type RelationsFieldFilter, eq, inArray } from "drizzle-orm";
+import { loggerNoop } from "../../../MaybeLogger.ts";
+import { capitalize } from "../../../../../core/StringUtils.ts";
+import { getConfigByTableName, type TableName } from "../schema/schema.ts";
+import assert from 'node:assert';
+import type { Cacheable } from "cacheable";
+import type { DateLike } from "../../../../../core/Atomic.ts";
+import type { CompareDateBetween, CompareDateSingle } from "../../../../../core/Api.ts";
 
 export interface DrizzleRepositoryOpts {
     logger?: Logger
+    cache?: Cacheable
     componentId?: number
 }
 
-export type CompareDateOp = {
-    type: CompareOpKey<Dayjs>
-    date: Dayjs
-} | {
-    type: 'between',
-    range: [Dayjs, Dayjs],
-    inclusive?: boolean
-}
-
-export interface PaginatedQueryResponse {
-    limit: number,
-    offset: number
-}
-
-export interface PaginatedResponse<T> {
-    data: T[]
-    meta: PaginatedQueryResponse
-}
-
+export type CompareDateOp<D extends DateLike = Dayjs> = CompareDateSingle<D> | CompareDateBetween<D>;
 export interface ComponentConstrainedRepoOpts {
     componentId?: number
 }
@@ -43,6 +29,7 @@ export abstract class DrizzleBaseRepository<T extends TableName> {
     table: ReturnType<typeof getConfigByTableName<T>>
     db: DbConcrete;
     componentId?: number
+    cache?: Cacheable
 
     constructor(db: DbConcrete, tableName: TableName, displayName: string, opts: DrizzleRepositoryOpts = {}) {
         this.db = db;
@@ -51,6 +38,7 @@ export abstract class DrizzleBaseRepository<T extends TableName> {
         this.table = getConfigByTableName(this.tableName);
         this.logger = childLogger(opts.logger ?? loggerNoop, ['Database', capitalize(displayName)]);
         this.componentId = opts.componentId;
+        this.cache = opts.cache;
     }
 
     async deleteByIds(ids: number[]): Promise<void> {
@@ -58,6 +46,7 @@ export abstract class DrizzleBaseRepository<T extends TableName> {
     }
 
     async updateById(id: number, data: Partial<typeof this.table.$inferInsert>): Promise<void> {
+        assert(id !== null && id !== undefined, `${id === null ? 'null' : 'undefined'} given for entity id`);
         await this.db.update(this.table).set(data).where(eq(this.table.id, id));
     }
 
@@ -90,13 +79,12 @@ export class GenericRepository<T extends TableName> extends DrizzleBaseRepositor
 }
 
 export const buildDateCompare = (data: CompareDateOp): RelationsFieldFilter<Dayjs> => {
-    let q: RelationsFieldFilter<Dayjs> = {};
     if (data.type !== 'between') {
-        q = {
+        return {
             [data.type]: data.date
         }
     } else {
-        q = {
+        return {
             AND: [
                 {
                     [data.inclusive ?? true ? 'gte' : 'gt']: data.range[0]
@@ -107,5 +95,4 @@ export const buildDateCompare = (data: CompareDateOp): RelationsFieldFilter<Dayj
             ]
         }
     }
-    return q;
 }

src/backend/common/database/drizzle/repositories/ComponentRepository.ts

@@ -1,9 +1,9 @@
-import { Logger } from "drizzle-orm";
-import { DrizzleBaseRepository, DrizzleRepositoryOpts } from "./BaseRepository.js";
-import { DbConcrete } from "../drizzleUtils.js";
-import { ComponentNew, ComponentSelect, FindWhere } from "../drizzleTypes.js";
-import { components } from "../schema/schema.js";
-import { generateComponentEntity } from "../entityUtils.js";
+import { DrizzleBaseRepository, type DrizzleRepositoryOpts } from "./BaseRepository.ts";
+import type {DbConcrete} from "../drizzleUtils.ts";
+import type {ComponentSelect, FindWhere} from "../drizzleTypes.ts";
+import { components } from "../schema/schema.ts";
+import { generateComponentEntity } from "../entityUtils.ts";
+import type {ComponentType} from "../../../../../core/Atomic.ts";
 
 export class DrizzleComponentRepository extends DrizzleBaseRepository<'components'> {
 
@@ -11,7 +11,7 @@ export class DrizzleComponentRepository extends DrizzleBaseRepository<'component
         super(db, 'components', 'Component', opts);
     }
 
-    findOrInsert = async (data: { mode: 'source' | 'client', type: string, uid?: string, name?: string }): Promise<ComponentSelect> => {
+    findOrInsert = async (data: { mode: ComponentType, type: string, uid?: string, name?: string }): Promise<ComponentSelect> => {
         const where: FindWhere<'components'> = {
             mode: data.mode,
             type: data.type,

src/backend/common/database/drizzle/repositories/PlayHistoricalRepository.ts

@@ -0,0 +1,318 @@
+import { type DbConcrete, runTransaction } from "../drizzleUtils.ts";
+import { type PlayObject, TA_DEFAULT_ACCURACY, type TemporalAccuracy } from "../../../../../core/Atomic.ts";
+import { generatePlayEntity, hydratePlaySelect, type PlayHydateOptions, type PlayHistoricalEntityOpts } from "../entityUtils.ts";
+import { plays, playsHistorical } from "../schema/schema.ts";
+import type {FindWhere, FindMany, WhereClause, PlayHistoricalSelect, PlayHistoricalNew} from "../drizzleTypes.ts";;
+import type { MarkOptional } from "ts-essentials";
+import { removeUndefinedKeys } from '../../../../../core/DataUtils.ts';
+import type {Dayjs} from "dayjs";
+import { inArray, sql } from "drizzle-orm";
+import { buildDateCompare, type CompareDateOp, type ComponentConstrainedRepoOpts, DrizzleBaseRepository, type DrizzleRepositoryOpts } from "./BaseRepository.ts";
+import type {PaginatedResponse} from "../../../../../core/Api.ts";
+import { hashObject } from "../../../../utils/StringUtils.ts";
+import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../../utils/PlayComparisonUtils.ts";
+import { comparePlayTemporally, getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy } from "../../../../utils/TimeUtils.ts";
+import type {SourceType} from "../../../../../core/Atomic.ts";
+import { getTemporallyCloseDateCompareOp } from "./PlayRepository.ts";
+
+// https://github.com/drizzle-team/drizzle-orm/issues/695 may be useful for typing models with relations?
+
+export interface PlayWhereOpts {
+    componentId?: number
+    seenAt?: CompareDateOp
+    playedAt?: CompareDateOp
+    uid?: string[]
+}
+
+export interface QueryPlaysOpts extends PlayWhereOpts {
+    sort?: 'seenAt' | 'playedAt'
+    order?: 'asc' | 'desc'
+    limit?: number
+    offset?: number
+}
+
+export interface HydrateOpts {
+   hydrate?: PlayHydateOptions[] 
+}
+
+export type RepositoryCreatePlayHistoricalOpts = PlayHistoricalEntityOpts
+    & Pick<PlayHistoricalNew, 'play' | 'componentId'>;
+
+type PlayIdentifierPrimitiveMap = {
+  uid: string;
+  id: number;
+};
+
+const identifierExtractor: { [K in keyof PlayIdentifierPrimitiveMap]: (play: {id: number, uid: string}) => PlayIdentifierPrimitiveMap[K] } = {
+  id: (play) => play.id,
+  uid: (play) => play.uid,
+};
+export class DrizzlePlayHistoricalRepository extends DrizzleBaseRepository<'playsHistorical'> {
+
+    constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
+        super(db, 'playsHistorical', 'Plays Historical', opts);
+    }
+
+    findByUid = async (uid: string, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PlayHistoricalSelect | undefined> => {
+        const res = await this.db.query.playsHistorical.findFirst({
+            where: {
+                uid,
+                componentId: opts.componentId ?? this.componentId
+            }
+        });
+        res.play = hydratePlaySelect(res, opts.hydrate);
+        return res;
+    }
+
+    hasByUid = async (uid: string, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<boolean> => {
+        const res = await this.db.query.playsHistorical.findFirst({
+            columns: {id: true},
+            where: {
+                uid,
+                componentId: opts.componentId ?? this.componentId
+            }
+        });
+        return res !== undefined;
+    }
+
+    createPlays = async (entitiesOpts: RepositoryCreatePlayHistoricalOpts[], opts: HydrateOpts = {}) => {
+
+        const {
+            hydrate
+        } = opts;
+        let playRows: PlayHistoricalSelect[];
+
+        await runTransaction(this.db, async () => {
+
+            const entitiesData = entitiesOpts.map((data) => {
+                const {
+                    play,
+                    ...rest
+                } = data;
+                return generatePlayEntity(play, { componentId: this.componentId, ...rest });
+            });
+
+            playRows = await this.db.insert(playsHistorical).values(entitiesData).returning();
+        });
+
+        return playRows.map(x => ({...x, play: hydratePlaySelect(x, hydrate)}));
+    }
+
+    findPlays = async (args: QueryPlaysOpts, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PlayHistoricalSelect[]> => {
+        const {
+            hydrate,
+            componentId = this.componentId
+        } = opts;
+        // this does not work as type for query variable
+        // it erases the result type for some reason
+        //
+        // Parameters<typeof this.db.query.plays.findMany>[0]
+        
+        // this does work but it is also integrated into FindWith
+        //let withQuery: Parameters<typeof this.db.query.plays.findMany>[0]['with'] = undefined;
+
+        let query: FindMany<'playsHistorical'> = {
+            limit: args.limit,
+            offset: args.offset
+        };
+
+        query.where = buildPlayHistoricalWhere({componentId: componentId,  ...args});
+
+        if (args.sort !== undefined) {
+            query.orderBy = {
+                [args.sort]: args.order ?? 'desc'
+            }
+        } else {
+            query.orderBy = {
+                id: 'asc'
+            }
+        }
+
+        query = removeUndefinedKeys(query);
+        const results = await this.db.query.playsHistorical.findMany(query);
+        return results.map((x) => ({...x, play: hydratePlaySelect(x, hydrate)}));
+    }
+
+    findPlayIds = async (args: QueryPlaysOpts, opts: ComponentConstrainedRepoOpts = {}): Promise<number[]> => {
+        const {
+            componentId = this.componentId
+        } = opts;
+
+        let query: FindMany<'playsHistorical'> = {
+            limit: args.limit,
+            offset: args.offset,
+        };
+
+        query.where = buildPlayHistoricalWhere({componentId: componentId,  ...args});
+
+        if (args.sort !== undefined) {
+            query.orderBy = {
+                [args.sort]: args.order ?? 'desc'
+            }
+        } else {
+            query.orderBy = {
+                id: 'asc'
+            }
+        }
+
+        query = removeUndefinedKeys(query);
+        const results = await this.db.query.playsHistorical.findMany({
+            ...query,
+            limit: args.limit,
+            offset: args.offset,
+            columns: {id: true},
+            orderBy: args.sort !== undefined ? {[args.sort]: args.order ?? 'desc'} : {id: 'asc'},
+        });
+        return results.map((x) => x.id);
+    }
+
+    findPlayIdentifiers = async <T extends keyof PlayIdentifierPrimitiveMap>(args: QueryPlaysOpts, identifier: T, opts: ComponentConstrainedRepoOpts = {}): Promise<PlayIdentifierPrimitiveMap[T][]> => {
+        const {
+            componentId = this.componentId,
+        } = opts;
+
+        const results = await this.db.query.playsHistorical.findMany({
+            limit: args.limit,
+            offset: args.offset,
+            columns: {id: true, uid: true},
+            orderBy: args.sort !== undefined ? {[args.sort]: args.order ?? 'desc'} : {id: 'asc'},
+            where: buildPlayHistoricalWhere({componentId: componentId,  ...args})
+        });
+
+        // we getting fancy now
+        return results.map(identifierExtractor[identifier]);
+    }
+
+    findPlaysPaginated = async (args: QueryPlaysOpts, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PaginatedResponse<PlayHistoricalSelect>> => {
+        const {
+            limit = 100,
+            offset = 0,
+            ...rest
+        } = args;
+        const clampedLimit = Math.min(limit, 100);
+        const res = await this.findPlays({limit: clampedLimit, offset, ...rest}, opts);
+        return {data: res, meta: {limit: clampedLimit, offset}};
+    }
+
+    // async updateById(id: number, data: Partial<PlayNew>): Promise<void> {
+    //     if(data.play !== undefined) {
+    //         data.play = withoutDbAwareness(data.play);
+    //     }
+    //     super.updateById(id, data);
+    // }
+
+    deletePlays = async (playsData: (Pick<PlayHistoricalSelect, 'id'> | number)[]) => {
+        const ids = playsData.map(x => typeof x === 'number' ? x : x.id);
+        await this.db.delete(playsHistorical).where(inArray(plays.id, ids));
+    }
+
+    public checkExisting = async (play: PlayObject, opts: {taAccuracy?: TemporalAccuracy[]} & ComponentConstrainedRepoOpts = {}): Promise<PlayHistoricalSelect | undefined> => {
+        const {
+            componentId = this.componentId,
+            taAccuracy = TA_DEFAULT_ACCURACY,
+        } = opts;
+        const hash = hashObject(playContentBasicInvariantTransform(play).data);
+
+        // we get all plays with a play date between playdate - (source accuracy) AND (playDateCompleted or playDate) + (source accuracy)
+        // which we can then use with temporal comparison to make sure we are comparing the correct dates
+        //
+        // this isn't as fast as just comparing playDate directly but its still much faster/cheaper than paginating plays and doing everything in-memory
+        const dateGranularity = getTemporalAccuracyCloseVal(play.meta.source as SourceType);
+        let endRange: Dayjs;
+        if(play.data.playDateCompleted !== undefined) {
+            // this will be present if source reports it
+            // or we tracked it live with MemorySource
+            endRange = play.data.playDateCompleted.add(dateGranularity, 's');
+        } else {
+            endRange = play.data.playDate.add(dateGranularity, 's');
+        }
+        const where: FindWhere<'playsHistorical'> = {
+            componentId,
+            playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play)),
+        };
+
+        const mbidId = playMbidIdentifier(play);
+        if(mbidId !== undefined) {
+            where.AND = [
+                {
+                    OR: [
+                        {
+                            playHash: hash
+                        },
+                        {
+                            mbidIdentifier: mbidId
+                        }
+                    ]
+                }
+            ]
+        } else {
+            where.playHash = hash;
+        }
+
+        const res = await this.db.query.playsHistorical.findMany({
+            where
+        });
+        if(res.length === 0) {
+            return undefined;
+        }
+        return res.map(x => ({...x, play: hydratePlaySelect(x)})).find(x => {
+            const temporalComparison = comparePlayTemporally(x.play, play);
+            return hasAcceptableTemporalAccuracy(temporalComparison.match, taAccuracy)
+        })
+    }
+
+    public getTemporallyClosePlays = async (play: PlayObject, opts: {bufferTime?: number} & ComponentConstrainedRepoOpts = {}): Promise<PlayHistoricalSelect[]> => {
+        const {
+            componentId = this.componentId,
+            bufferTime
+        } = opts;
+
+        const query: FindMany<'playsHistorical'> = {};
+
+        const where: FindWhere<'playsHistorical'> = {
+            componentId,
+            playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play, {bufferTime})),
+        };
+        query.where = where;
+
+        return ((await this.db.query.playsHistorical.findMany({
+            where,
+        })) as PlayHistoricalSelect[]).map(x => ({...x, play: hydratePlaySelect(x)}));
+    }
+
+    public getPlayCountByComponent = async () => {
+
+        const res = await this.db.all(sql`select componentId, count(*) from plays_historical p
+group by componentId;`);
+        return res;
+    }
+}
+
+export const buildPlayHistoricalWhere = (args: PlayWhereOpts): WhereClause<'playsHistorical'> => {
+    // old way
+    // let where: Parameters<(ReturnType<typeof getDb>)['query']['plays']['findMany']>[0]['where'] = {
+    // };
+    const where: FindWhere<'playsHistorical'> = {
+        componentId: args.componentId
+    };
+    if (args.seenAt !== undefined) {
+        where.seenAt = buildDateCompare(args.seenAt);
+    }
+    if (args.playedAt !== undefined) {
+        where.playedAt = buildDateCompare(args.playedAt);
+    }
+    if(args.uid !== undefined) {
+        where.uid = {
+            in: args.uid
+        }
+    }
+    return where;
+}
+
+export const playToRepositoryCreatePlayHistoricalOpts = (data: MarkOptional<RepositoryCreatePlayHistoricalOpts, 'componentId'>): RepositoryCreatePlayHistoricalOpts => {
+    return {
+        play: data.play,
+        uid: data.play.meta?.playId,
+        ...data
+    }
+}

src/backend/common/database/drizzle/repositories/PlayRepository.ts

@@ -1,43 +1,45 @@
-import { childLogger, Logger, LoggerAppExtras } from "@foxxmd/logging";
-import { DbConcrete, runTransaction } from "../drizzleUtils.js";
-import { loggerNoop } from "../../../MaybeLogger.js";
-import { ErrorLike, PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT, TemporalAccuracy } from "../../../../../core/Atomic.js";
-import { generateInputEntity, generatePlayEntity, PlayEntityOpts, hydratePlaySelect, PlayHydateOptions } from "../entityUtils.js";
-import { playInputs, plays, queueStates, relations } from "../schema/schema.js";
-import { PlayNew, PlaySelect, PlayInputNew, FindWhere, FindMany, QueueStateSelect, FindWith, PlaySelectWithQueueStates, WhereClause, PlayWith } from "../drizzleTypes.js";;
-import { MarkOptional, MarkRequired, PathValue } from "ts-essentials";
-import { genGroupIdStrFromPlay, removeEmptyArrays, removeUndefinedKeys } from "../../../../utils.js";
-import dayjs, { Dayjs } from "dayjs";
-import { RelationsFieldFilter, eq, inArray, ne, notInArray, desc, asc, and, sql, Placeholder } from "drizzle-orm";
-import { CompactableProperty, RetentionOptions, retentionPlayTypes } from "../../../infrastructure/config/database.js";
-import { shortTodayAwareFormat } from "../../../../../core/TimeUtils.js";
-import { buildDateCompare, CompareDateOp, ComponentConstrainedRepoOpts, DrizzleBaseRepository, DrizzleRepositoryOpts, PaginatedQueryResponse, PaginatedResponse } from "./BaseRepository.js";
-import { asPlay } from "../../../../../core/PlayMarshalUtils.js";
-import assert, { Assert } from "node:assert";
-import { hashObject, parseArrayFromMaybeString } from "../../../../utils/StringUtils.js";
-import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../../utils/PlayComparisonUtils.js";
-import { comparePlayTemporally, getScrobbleTsSOCDate, getScrobbleTsSOCDateWithContext, getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy } from "../../../../utils/TimeUtils.js";
-import { SourceType } from "../../../infrastructure/config/source/sources.js";
+import { childLogger } from "@foxxmd/logging";
+import dayjs, { type Dayjs } from "dayjs";
+import { eq, inArray, relationsFilterToSQL, sql } from "drizzle-orm";
+import assert from "node:assert";
+import type { MarkOptional } from "ts-essentials";
+import { type DateLike, type DeepReplaceValue, type PlayObject, type PlayState, type QueueName, TA_DEFAULT_ACCURACY, type TemporalAccuracy } from "../../../../../core/Atomic.ts";
+import { removeUndefinedKeys } from '../../../../../core/DataUtils.ts';
+import { shortTodayAwareFormat } from "../../../../../core/TimeUtils.ts";
+import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../../utils/PlayComparisonUtils.ts";
+import { hashObject } from "../../../../utils/StringUtils.ts";
+import { comparePlayTemporally, getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy } from "../../../../utils/TimeUtils.ts";
+import { type CompactableProperty, type RetentionOptions, retentionPlayTypes } from "../../../infrastructure/config/database.ts";
+import type {SourceType} from "../../../../../core/Atomic.ts";
+import type {FindMany, FindWhere, FindWith, PlayInputNew, PlayNew, PlaySelect, PlaySelectWithQueueStates, PlayWith, QueueStateSelect, WhereClause} from "../drizzleTypes.ts";
+import { type DbConcrete, runTransaction } from "../drizzleUtils.ts";
+import { generateInputEntity, generatePlayEntity, hydratePlaySelect, type PlayEntityOpts, type PlayHydateOptions } from "../entityUtils.ts";
+import { playInputs, plays, relations } from "../schema/schema.ts";
+import { buildDateCompare, type CompareDateOp, type ComponentConstrainedRepoOpts, DrizzleBaseRepository, type DrizzleRepositoryOpts } from "./BaseRepository.ts";
+import type {PaginatedResponse} from "../../../../../core/Api.ts";
+import type {PaginatedQueryResponse} from "../../../../../core/Api.ts";
+;
 
 // https://github.com/drizzle-team/drizzle-orm/issues/695 may be useful for typing models with relations?
 
 export interface QueueCriteria {
-    queueName: string
+    queueName: QueueName
     queueStatus: QueueStateSelect['queueStatus'][] | QueueStateSelect['queueStatus']
 }
 
-export interface PlayWhereOpts {
+export interface PlayWhereOpts<D extends DateLike = Dayjs> {
     state?: PlaySelect['state'][]
     stateNot?: PlaySelect['state'][]
     componentId?: number
-    seenAt?: CompareDateOp
-    playedAt?: CompareDateOp
+    seenAt?: CompareDateOp<D>
+    playedAt?: CompareDateOp<D>
     queues?: QueueCriteria[]
     uid?: string[]
+    text?: string[]
 }
 
 export type WithPlayRelation = 'input' | 'parent' | 'parent-input' | 'queues';
-export interface QueryPlaysOpts extends PlayWhereOpts {
+export interface QueryPlaysOpts<D extends DateLike = Dayjs> extends PlayWhereOpts<D> {
     sort?: 'seenAt' | 'playedAt'
     order?: 'asc' | 'desc'
     with?: WithPlayRelation[]
@@ -45,6 +47,8 @@ export interface QueryPlaysOpts extends PlayWhereOpts {
     offset?: number
 }
 
+export type QueryPlaysOptsJson = DeepReplaceValue<QueryPlaysOpts<Dayjs>, [[Dayjs, string]]>;
+
 export interface HydrateOpts {
    hydrate?: PlayHydateOptions[] 
 }
@@ -73,26 +77,27 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         super(db, 'plays', 'Plays', opts);
     }
 
-    findByUid = async (uid: string, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PlaySelectWithQueueStates | undefined> => {
+    findByUid = async (uid: string, opts: HydrateOpts & Pick<QueryPlaysOpts, 'with'> & ComponentConstrainedRepoOpts = {}): Promise<PlaySelectWithQueueStates | undefined> => {
         const res = await this.db.query.plays.findFirst({
             where: {
                 uid,
                 componentId: opts.componentId ?? this.componentId
             },
-            with: {
-                queueStates: true
-            }
+            with: buildPlayWith([...(opts.with ?? []), 'queues'])
         });
+        if(res === undefined) {
+            return undefined;
+        }
         res.play = hydratePlaySelect(res, opts.hydrate);
-        return res;
+        return res as PlaySelectWithQueueStates;
     }
 
-    createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[], opts: HydrateOpts = {}) => {
+    createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[], opts: HydrateOpts = {}): Promise<PlayWith<'input'>[]> => {
 
         const {
             hydrate
         } = opts;
-        let playRows: PlaySelect[];
+        let playRows: PlayWith<'input'>[];
 
         await runTransaction(this.db, async () => {
 
@@ -105,9 +110,9 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
                 return generatePlayEntity(play, { componentId: this.componentId, ...rest });
             });
 
-            playRows = await this.db.insert(plays).values(entitiesData).returning();
+            const nakedPlays = await this.db.insert(plays).values(entitiesData).returning();
 
-            const inputDatas = playRows.map((x, index) => {
+            const inputDatas = nakedPlays.map((x, index) => {
                 const {
                     play,
                     input,
@@ -121,10 +126,10 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
             });
 
             const inputRow = await this.db.insert(playInputs).values(inputDatas);
-
+            playRows = nakedPlays.map((x, index) => ({...x, play: hydratePlaySelect(x, hydrate), input: inputRow[index]}));
         });
 
-        return playRows.map(x => ({...x, play: hydratePlaySelect(x, hydrate)}));
+        return playRows;
     }
 
     findPlays = async (args: QueryPlaysOpts, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PlaySelect[]> => {
@@ -211,6 +216,7 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
 
         query = removeUndefinedKeys(query);
         const results = await this.db.query.plays.findMany({
+            ...query,
             limit: args.limit,
             offset: args.offset,
             columns: {id: true},
@@ -244,7 +250,20 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         } = args;
         const clampedLimit = Math.min(limit, 100);
         const res = await this.findPlays({limit: clampedLimit, offset, ...rest}, opts) as T[];
-        return {data: res, meta: {limit: clampedLimit, offset}};
+
+        
+        const where = buildPlayWhere({componentId: args.componentId ?? this.componentId,  ...rest});
+  
+        // https://github.com/drizzle-team/drizzle-orm/issues/5218#issuecomment-3854900241
+        const filter = relationsFilterToSQL(plays,
+            // @ts-expect-error don't have a good way to type this yet
+            where,
+            relations.plays.relations,
+            relations);
+        // https://github.com/drizzle-team/drizzle-orm/discussions/3119#discussioncomment-16379557
+        const count = await this.db.$count(plays, filter);
+
+        return {data: res, meta: {limit: clampedLimit, offset, total: count}};
     }
 
     // async updateById(id: number, data: Partial<PlayNew>): Promise<void> {
@@ -274,7 +293,7 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
             dateComparer = 'updatedAt'
         } = opts;
 
-        let where: FindWhere<'plays'> = {
+        const where: FindWhere<'plays'> = {
             component: {
                 id: componentId
             },
@@ -324,8 +343,8 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
 
         const loggerDel = childLogger(this.logger, ['Retention', 'Delete']);
         const loggerCom = childLogger(this.logger, ['Retention', 'Compact']);
-        let summaryDelStates: string[] = [];
-        let summaryCompactStates: string[] = [];
+        const summaryDelStates: string[] = [];
+        const summaryCompactStates: string[] = [];
 
         loggerDel.debug('Starting cleanup...');
         for(const retentionType of retentionPlayTypes) {
@@ -360,7 +379,7 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         }
 
         const compactTypes = retentionOpts.compact;
-        let compactedFlags: CompactableProperty[] = [];
+        const compactedFlags: CompactableProperty[] = [];
         if(compactTypes.includes('input')) {
             compactedFlags.push('input');
         }
@@ -404,12 +423,14 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
                             }
 
                             compactedPlay = playRow.play;
-                            compactedPlay.meta.lifecycle.steps = compactedPlay.meta.lifecycle.steps.map(x => {
-                                if(x.inputs == undefined) {
-                                    return x;
-                                }
-                                return {...x, inputs: x.inputs.map(y => ({type: y.type, input: 'Removed by compaction'}))};
-                            });
+                            if(compactedPlay.lifecycle !== undefined) {
+                                compactedPlay.lifecycle = compactedPlay.lifecycle.map(x => {
+                                    if(x.inputs == undefined) {
+                                        return x;
+                                    }
+                                    return {...x, inputs: x.inputs.map(y => ({type: y.type, input: 'Removed by compaction'}))};
+                                });
+                            }
                         }
 
                         const updater = this.db.update(plays);
@@ -564,7 +585,7 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
             componentId = this.componentId,
             hydrate
         } = opts;
-        let where: FindWhere<'plays'> = {
+        const where: FindWhere<'plays'> = {
             componentId
         }
         if(retries !== undefined) {
@@ -614,7 +635,7 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         } else {
             endRange = play.data.playDate.add(dateGranularity, 's');
         }
-        let where: FindWhere<'plays'> = {
+        const where: FindWhere<'plays'> = {
             componentId,
             playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play)),
         };
@@ -672,9 +693,9 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
             with: qWith
         } = opts;
 
-        let query: FindMany<'plays'> = {};
+        const query: FindMany<'plays'> = {};
 
-        let where: FindWhere<'plays'> = {
+        const where: FindWhere<'plays'> = {
             componentId,
             playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play, {bufferTime})),
         };
@@ -690,6 +711,29 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
             with: buildPlayWith(qWith)
         })) as PlayWith<'queueStates'>[]).map(x => ({...x, play: hydratePlaySelect(x)}));
     }
+
+    public getComponentPlayCountByState = async (componentId?: number): Promise<{state: PlayState, 'count(*)': number}[]> => {
+
+        const res = await this.db.all(sql`select state, count(*) from plays p
+where componentId = ${componentId ?? this.componentId}
+group by state;`);
+        return res as {state: PlayState, 'count(*)': number}[];
+    }
+
+    public getPlayCountByState = async () => {
+
+        const res = await this.db.all(sql`select state,componentId, count(*) from plays p
+group by state,componentId;`);
+        return res;
+    }
+
+    public getCompactedPlayCountByComponent = async () => {
+
+        const res = await this.db.all(sql`select componentId,compacted,count(*) from plays p
+where compacted IS NOT NULL
+group by componentId,compacted;`);
+        return res;
+    }
 }
 
 export const getTemporallyCloseDateCompareOp = (play: PlayObject, opts: {bufferTime?: number, useCompleted?: boolean} = {}): CompareDateOp => {
@@ -747,7 +791,7 @@ export const buildPlayWhere = (args: PlayWhereOpts): WhereClause<'plays'> => {
     // old way
     // let where: Parameters<(ReturnType<typeof getDb>)['query']['plays']['findMany']>[0]['where'] = {
     // };
-    let where: FindWhere<'plays'> = {
+    const where: FindWhere<'plays'> = {
         componentId: args.componentId
     };
     if (args.state !== undefined) {
@@ -783,7 +827,7 @@ export const buildPlayWhere = (args: PlayWhereOpts): WhereClause<'plays'> => {
         }
         // so that we can use this type
         // or else assigning an array to OR using only `typeof where.queueStates` causes a type error
-        let queueWhere: typeof where.queueStates.OR[0][] = [];
+        const queueWhere: typeof where.queueStates.OR[0][] = [];
         for(const q of queues) {
             queueWhere.push(
                 {
@@ -802,20 +846,37 @@ export const buildPlayWhere = (args: PlayWhereOpts): WhereClause<'plays'> => {
             }
         }
     }
+
+    if(args.text !== undefined && args.text.length > 0) {
+        
+        where.AND = [
+            {
+                OR: []
+            }
+        ];
+        const textWhere: typeof where.RAW[] = [];
+        for(const t of args.text) {
+            textWhere.push((p) => sql`lower(json_extract(${p.play}, '$.data.track')) LIKE '%' || ${t.toLocaleLowerCase()} || '%'`)
+            textWhere.push((p) => sql`lower(json_extract(${p.play}, '$.data.artists')) LIKE '%' || ${t.toLocaleLowerCase()} || '%'`)
+            textWhere.push((p) => sql`lower(json_extract(${p.play}, '$.data.album')) LIKE '%' || ${t.toLocaleLowerCase()} || '%'`)
+        }
+        where.AND = [
+            {
+                OR: textWhere.map(x => ({RAW: x}))
+            }
+        ]
+    }
+
     return where;
 }
 
 export const playToRepositoryCreatePlayOpts = (data: MarkOptional<RepositoryCreatePlayOpts, 'input' | 'componentId'>): RepositoryCreatePlayOpts => {
     const {
         play: {
-            meta: {
-                lifecycle: {
-                    input,
-                    original,
-                    ...lifecycleRest
-                } = {},
-                ...metaRest
-            },
+            original: {
+                play: playOriginal,
+                data: playData
+            } = {},
             ...playRest
         },
         ...rest
@@ -824,58 +885,13 @@ export const playToRepositoryCreatePlayOpts = (data: MarkOptional<RepositoryCrea
     return {
         play: {
             ...playRest,
-            meta: {
-                ...metaRest,
-                // @ts-expect-error
-                lifecycle: {
-                    ...lifecycleRest
-                }
-            }
         },
         ...rest,
         input: {
-            play: original,
-            data: input
+            play: playOriginal,
+            data: playData
         }
     }
 }
 
-export type RequestPlayQuery = Partial< Record<keyof Exclude<QueryPlaysOpts, 'componentId'>, string>>;
-
-export const queryArgsFromRequest = (rec: RequestPlayQuery): QueryPlaysOpts => {
-
-    const {
-        state,
-        stateNot,
-        uid,
-        with: withQuery,
-        seenAt,
-        playedAt,
-        limit,
-        sort,
-        order,
-        offset,
-        componentId,
-        queues,
-        ...rest
-    } = rec;
-
-    let queryArgs: QueryPlaysOpts = removeEmptyArrays<QueryPlaysOpts>({
-        state: parseArrayFromMaybeString(state) as PlaySelect['state'][],
-        stateNot: parseArrayFromMaybeString(stateNot) as PlaySelect['state'][],
-        uid: parseArrayFromMaybeString(uid),
-        with: parseArrayFromMaybeString(withQuery) as WithPlayRelation[],
-        sort: sort as 'playedAt' | 'seenAt',
-        order: order as 'asc' | 'desc',
-        ...rest
-    });
-
-    if(limit !== undefined) {
-        queryArgs.limit = Number.parseInt(limit);
-    }
-    if(offset !== undefined) {
-        queryArgs.offset = Number.parseInt(offset);
-    }
-
-    return queryArgs;
-}
+export type RequestPlayQuery = Partial< Record<keyof Exclude<QueryPlaysOpts, 'componentId'>, string>>;

src/backend/common/database/drizzle/repositories/QueueRepository.ts

@@ -1,9 +1,9 @@
 import { eq, and, lte, inArray } from "drizzle-orm";
-import { DrizzleBaseRepository, DrizzleRepositoryOpts } from "./BaseRepository.js";
-import { DbConcrete } from "../drizzleUtils.js";
-import { QueueStateSelect } from "../drizzleTypes.js";
-import { queueStates } from "../schema/schema.js";
-import { CLIENT_DEAD_QUEUE } from "../../../../../core/Atomic.js";
+import { DrizzleBaseRepository, type DrizzleRepositoryOpts } from "./BaseRepository.ts";
+import type {DbConcrete} from "../drizzleUtils.ts";
+import type {QueueStateSelect} from "../drizzleTypes.ts";
+import { queueStates } from "../schema/schema.ts";
+import { CLIENT_DEAD_QUEUE } from "../../../../../core/Atomic.ts";
 export class DrizzleQueueRepository extends DrizzleBaseRepository<'queueStates'> {
 
     constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {

src/backend/common/database/drizzle/schema/schema.ts

@@ -1,12 +1,12 @@
-import { integer, sqliteTable, text, index, uniqueIndex, customType, AnySQLiteColumn } from "drizzle-orm/sqlite-core";
+import { integer, sqliteTable, text, index, uniqueIndex, customType, type AnySQLiteColumn } from "drizzle-orm/sqlite-core";
 import { defineRelations } from 'drizzle-orm';
-import dayjs, { Dayjs } from "dayjs";
-import { nanoid } from "nanoid";
-import { ErrorLike, PlayObject } from "../../../../../core/Atomic.js";
-import { asPlayCheap } from "../../../../../core/PlayMarshalUtils.js";
-import { ExternalMetadataTerm, PlayTransformPartsConfig, SearchAndReplaceTerm } from "../../../infrastructure/Transform.js";
-import { JobRangeCount, JobRangeTime } from "../../../infrastructure/Job.js";
+import dayjs, { type Dayjs } from "dayjs";
+import { COMPONENT_TYPE_CLIENT, COMPONENT_TYPE_SOURCE, type ErrorLike, type PlayObject } from "../../../../../core/Atomic.ts";
+import { asPlayCheap } from "../../../../../core/PlayMarshalUtils.ts";
+import type {ExternalMetadataTerm, PlayTransformPartsConfig, SearchAndReplaceTerm} from "../../../../../core/Transform.ts";
+import type {JobRangeCount, JobRangeTime} from "../../../infrastructure/Job.ts";
 import { serializeError, deserializeError } from "serialize-error";
+import { generatePlayUid } from "../../../../../core/StringUtils.ts";
 
 const DayjsTimestamp = customType<
   {
@@ -67,12 +67,12 @@ const ErrorLikeJson = customType<
 
 export const plays = sqliteTable("plays", {
   id: integer().primaryKey(),
-  uid: text({ length: 30 }).notNull().unique().$defaultFn(() => nanoid(20)),
+  uid: text({ length: 30 }).notNull().unique().$defaultFn(() => generatePlayUid()),
   componentId: integer().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}),
   error: ErrorLikeJson('error'),
   playedAt: DayjsTimestamp('playedAt'),
   seenAt: DayjsTimestamp('seenAt'),
-  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()),
+  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()).$onUpdate(() => dayjs()),
   play: PlayJson('play').notNull(), //  text({ mode: 'json' }).notNull().$type<PlayObject>(),
   state: text({enum: ['queued','discovered','discarded','scrobbled','failed','duped']}).notNull(),
   // https://orm.drizzle.team/docs/indexes-constraints#foreign-key
@@ -144,7 +144,7 @@ export const queueStates = sqliteTable("play_queue_states", {
   retries: integer().notNull().default(0),
   error: ErrorLikeJson('error'),
   createdAt: DayjsTimestamp('createdAt').notNull().$defaultFn(() => dayjs()),
-  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs())
+  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()).$onUpdate(() => dayjs())
 }, (table) => [
   index('play_queue_state_id_idx').on(table.playId)
 ]);
@@ -165,7 +165,7 @@ export const components = sqliteTable("components", {
   id: integer({ mode: 'number' }).primaryKey(),
   // user-provided id
   uid: text({ length: 200 }).notNull(),
-  mode: text({enum: ['source','client']}).notNull(),
+  mode: text({enum: [COMPONENT_TYPE_SOURCE,COMPONENT_TYPE_CLIENT]}).notNull(),
   // spotify, lastfm, etc...
   type: text({length: 50}).notNull(),
   // vanity display name
@@ -207,7 +207,7 @@ export const jobs = sqliteTable("jobs", {
   imported: integer().notNull().default(0),
   scrobbled: integer().notNull().default(0),
   createdAt: DayjsTimestamp('createdAt').notNull().$defaultFn(() => dayjs()),
-  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()),
+  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()).$onUpdate(() => dayjs()),
   completedAt: DayjsTimestamp('completedAt')
 });
 
@@ -289,7 +289,7 @@ export const getConfigByTableName = <T extends TableName>(name: T) => {
   }
 }
 
-const schema = {playInputs, plays, components, componentMigrations, queueStates, jobs};
+export const schema = {playInputs, plays, components, componentMigrations, queueStates, jobs};
 
 export type TSchema = typeof relations;
 export type Schema = typeof schema;

src/backend/common/errors/ErrorUtils.ts

@@ -1,4 +1,4 @@
-import { ResponseError } from "superagent";
+import type {ResponseError} from "superagent";
 
 export const isSuperAgentResponseError = (e: any): e is ResponseError => {
     return typeof e === 'object'

src/backend/common/errors/MSErrors.ts

@@ -1,12 +1,15 @@
 import { parseRegexSingle } from "@foxxmd/regex-buddy-core";
 import mergeErrorCause from 'merge-error-cause';
-import { findCauseByFunc, isAbortReasonErrorLike } from "../../utils/ErrorUtils.js";
-import { UpstreamError, UpstreamErrorOptions } from "./UpstreamError.js";
-import { isAbortError } from "abort-controller-x";
-import {addKnownErrorConstructor} from 'serialize-error';
+import { findCauseByFunc, isAbortReasonErrorLike } from "../../utils/ErrorUtils.ts";
+import { UpstreamError, type UpstreamErrorOptions } from "./UpstreamError.ts";
+import {addKnownErrorConstructor, serializeError} from 'serialize-error';
+import type {LifecycleInput} from "../../../core/Atomic.ts";
 
 export abstract class NamedError extends Error {
     public abstract name: string;
+    public toJSON() {
+        return serializeError(this);
+    }
 }
 
 export abstract class StageError extends NamedError {}
@@ -44,7 +47,10 @@ const STACK_AT_REGEX = new RegExp(/[\n\r]\s*at/);
 
 export class SimpleError extends Error implements HasSimpleError {
     simple: boolean;
-    name = 'SimpleError';
+    override name = 'SimpleError';
+    public toJSON() {
+        return serializeError(this);
+    }
 
     stackShortened: boolean = false;
 
@@ -77,22 +83,47 @@ export class SimpleError extends Error implements HasSimpleError {
 addKnownErrorConstructor(SimpleError, () => new SimpleError(''))
 
 export class StageTransformError extends NamedError {
-    name = 'Stage Transform';
+    override name = 'Stage Transform';
+    lifecycleInputs?: LifecycleInput[]
     stageName: string;
-    constructor(name: string, message: string, options?: ErrorOptions) {
-        super(message, options);
+    constructor(name: string, message: string, options: ErrorOptions & {inputs?: LifecycleInput[]} = {}) {
+        const {
+            inputs,
+            ...restOpts
+        } = options;
+        super(message, restOpts);
+        this.lifecycleInputs = inputs;
         this.stageName = name;
     }
 }
 addKnownErrorConstructor(StageTransformError, () => new StageTransformError('',''))
 
 export class SkipTransformStageError extends SimpleError {
-    name = 'Skip Transform Stage';
+    lifecycleInputs?: LifecycleInput[]
+    override name = 'Skip Transform Stage';
+
+    public constructor(msg: string, options: ErrorOptions & { simple?: boolean, shortStack?: boolean, inputs?: LifecycleInput[]} = {}) {
+        const {
+            inputs,
+            ...restOpts
+        } = options;
+        super(msg, restOpts);
+        this.lifecycleInputs = inputs;
+    }
 }
 addKnownErrorConstructor(SkipTransformStageError, () => new SkipTransformStageError(''))
 
 export class StagePrerequisiteError extends SimpleError {
-    name = 'Stage Prerequisite';
+    lifecycleInputs?: LifecycleInput[]
+    override name = 'Stage Prerequisite';
+    public constructor(msg: string, options: ErrorOptions & { simple?: boolean, shortStack?: boolean, inputs?: LifecycleInput[]} = {}) {
+        const {
+            inputs,
+            ...restOpts
+        } = options;
+        super(msg, restOpts);
+        this.lifecycleInputs = inputs;
+    }
 }
 addKnownErrorConstructor(StagePrerequisiteError, () => new StagePrerequisiteError(''))
 
@@ -113,13 +144,17 @@ export const mergeSimpleError = (err: Error): Error => {
         // mergeErrorCause mutates the argument
         // and we want to be able to do more cause/error parsing after merging for logging
         // so give it a copy instead of the original
-        return mergeErrorCause(structuredClone(err));
+        const mergedErr = mergeErrorCause(structuredClone(err));
+        if(!('toJson' in mergedErr)) {
+            // @ts-expect-error this is used by serialize error with out custom errors
+            mergedErr.toJSON = () => serializeError(mergedErr);
+        }
     }
     return err;
 }
 
 export class ScrobbleSubmitError<T extends (object | string) = object> extends UpstreamError {
-    name = 'Scrobble Submit Error';
+    override name = 'Scrobble Submit Error';
     payload?: T;
     constructor(message: string, options?: UpstreamErrorOptions & {payload?: T}) {
         super(message, options);
@@ -151,7 +186,7 @@ export class InvalidRegexError extends SimpleError {
         const msgParts = [
             message ?? 'Regex(es) did not match the value given.',
         ];
-        let regArr = Array.isArray(regex) ? regex : [regex];
+        const regArr = Array.isArray(regex) ? regex : [regex];
         for(const r of regArr) {
             msgParts.push(`Regex: ${r}`)
         }

src/backend/common/errors/NodeErrors.ts

@@ -1,13 +1,12 @@
-import { findCauseByFunc } from "../../utils/ErrorUtils.js";
-import { isArbitraryObject } from "../infrastructure/Atomic.js";
-import ErrnoException = NodeJS.ErrnoException;
+import { findCauseByFunc } from "../../utils/ErrorUtils.ts";
+import { isArbitraryObject } from "../infrastructure/Atomic.ts";
 
 export type NodeNetworkErrorCode = 'ENOTFOUND' | 'ETIMEDOUT' | 'EAI_AGAIN' | 'ECONNRESET' | 'ECONNREFUSED' | 'ERRADDRINUSE' | 'EADDRNOTAVAIL' | 'ECONNABORTED' | 'EHOSTUNREACH';
 export const NETWORK_ERROR_CODES: NodeNetworkErrorCode[] = ['ENOTFOUND', 'ETIMEDOUT',  'EAI_AGAIN', 'ECONNRESET', 'ECONNREFUSED', 'ERRADDRINUSE', 'EADDRNOTAVAIL', 'ECONNABORTED', 'EHOSTUNREACH'];
 
 export const NETWORK_ERROR_FAILURE_CODES: NodeNetworkErrorCode[] = ['ENOTFOUND', 'EAI_AGAIN', 'ECONNRESET', 'ECONNREFUSED', 'ERRADDRINUSE', 'EADDRNOTAVAIL', 'ECONNABORTED', 'EHOSTUNREACH'];
 
-export const isErrnoException = (error: unknown): error is ErrnoException => {
+export const isErrnoException = (error: unknown): error is NodeJS.ErrnoException => {
     return isArbitraryObject(error) &&
         error instanceof Error &&
         (typeof error.errno === "number" || typeof error.errno === "undefined") &&
@@ -16,7 +15,7 @@ export const isErrnoException = (error: unknown): error is ErrnoException => {
         (typeof error.syscall === "string" || typeof error.syscall === "undefined");
 }
 
-export interface NodeNetworkException extends ErrnoException {
+export interface NodeNetworkException extends NodeJS.ErrnoException {
     code: NodeNetworkErrorCode
 }
 

src/backend/common/errors/UpstreamError.ts

@@ -1,13 +1,13 @@
-import { Response } from 'superagent';
+import type { Response } from 'superagent';
 
-import { findCauseByFunc } from "../../utils/ErrorUtils.js";
+import { findCauseByFunc } from "../../utils/ErrorUtils.ts";
 import { addKnownErrorConstructor } from 'serialize-error';
 
 export type UpstreamErrorOptions = ErrorOptions & { showStopper?: boolean, response?: Response, responseBody?: object | string };
 
 export class UpstreamError extends Error {
 
-    name = 'UpstreamError';
+    override name = 'UpstreamError';
     showStopper: boolean = false;
     response?: Response
     responseBody?: object | string

src/backend/common/infrastructure/Atomic.ts

@@ -1,21 +1,21 @@
-import { Logger } from '@foxxmd/logging';
-import { Dayjs, ManipulateType } from "dayjs";
-import { Request, Response } from "express";
-import { type NextFunction, type ParamsDictionary, type Query } from "express-serve-static-core";
-import { FixedSizeList } from 'fixed-size-list';
-import { ErrorLike, isPlayObject, PlayMetaLifecycleless, PlayObject, PlayObjectLifecycleless, UnixTimestamp } from "../../../core/Atomic.js";
-import TupleMap from "../TupleMap.js";
-import { MusicBrainzApi } from 'musicbrainz-api';
-import { SourceType } from './config/source/sources.js';
-import { ClientType, clientTypes } from './config/client/clients.js';
-import assert, { AssertionError } from 'assert';
+import type { Logger, LogDataPretty } from '@foxxmd/logging';
+import type { Dayjs, ManipulateType } from "dayjs";
+import type { Request, Response } from "express";
+import type { NextFunction, ParamsDictionary, Query } from "express-serve-static-core";
+import type { FixedSizeList } from 'fixed-size-list';
+import { type DeviceId, type ErrorLike, isPlayObject, type PlayMeta, type PlayObject, type PlayObjectMinimal, type PlayPlatformId, type PlayUserId, type UnixTimestamp } from "../../../core/Atomic.ts";
+import type TupleMap from "../TupleMap.ts";
+import type { MusicBrainzApi } from 'musicbrainz-api';
+import type { ReportedPlayerStatus, SourceType } from "../../../core/Atomic.ts";
+import type { ClientType } from "../../../core/Atomic.ts";
+import assert from 'assert';
+
+export interface LeveledLogData extends LogDataPretty {
+    levelLabel: string
+}
 
 export const lowGranularitySources: SourceType[] = ['subsonic', 'ytmusic'];
 
-export const isClientType = (data: string): data is ClientType => {
-    return clientTypes.includes(data as ClientType);
-}
-
 export interface ComponentIdentifier {
     type: SourceType | ClientType
     name: string
@@ -51,21 +51,6 @@ export interface InternalConfig {
 
 export type InternalConfigOptional = Omit<InternalConfig, 'logger'>
 
-export type ReportedPlayerStatus = 'playing' | 'stopped' | 'paused' | 'unknown';
-export const REPORTED_PLAYER_STATUSES = {
-    playing: 'playing' as ReportedPlayerStatus,
-    stopped: 'stopped' as ReportedPlayerStatus,
-    paused: 'paused' as ReportedPlayerStatus,
-    unknown: 'unknown' as ReportedPlayerStatus
-}
-
-export type CalculatedPlayerStatus = ReportedPlayerStatus | 'stale' | 'orphaned';
-export const CALCULATED_PLAYER_STATUSES = {
-    ...REPORTED_PLAYER_STATUSES,
-    stale: 'stale' as CalculatedPlayerStatus,
-    orphaned: 'orphaned' as CalculatedPlayerStatus,
-}
-
 export type ConfigureAsSource = 'source';
 export type ConfigureAsClient = 'client';
 export type ConfigureAs = ConfigureAsSource | ConfigureAsClient;
@@ -109,16 +94,11 @@ export interface FormatPlayObjectOptions {
     [key: string]: any
 }
 
-export interface ProgressAwarePlayObject extends PlayObjectLifecycleless {
-    meta: PlayMetaLifecycleless & {
+export interface ProgressAwarePlayObject extends PlayObjectMinimal {
+    meta: PlayMeta & {
         initialTrackProgressPosition?: number
     }
 }
-export type DeviceId = string;
-export type PlayUserId = string;
-export type PlayPlatformId = [DeviceId, PlayUserId];
-export type PlayPlatformIdStr = string;
-
 export type GroupedPlays = TupleMap<DeviceId,PlayUserId,ProgressAwarePlayObject[]>;
 
 export type GroupedFixedPlays = TupleMap<DeviceId,PlayUserId,FixedSizeList<ProgressAwarePlayObject>>;
@@ -131,7 +111,7 @@ export const SINGLE_USER_PLATFORM_ID_STR = `${NO_DEVICE}-${NO_USER}`;
 
 export interface ScrobbledPlayObject {
     play: PlayObject
-    scrobble: PlayObjectLifecycleless
+    scrobble: PlayObjectMinimal
 }
 
 
@@ -191,10 +171,6 @@ export type ExpressRequest = Request<ParamsDictionary, any, any, Query, Record<s
 export type ExpressResponse = Response<any, Record<string, any>>;
 export type ExpressHandler = (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => Promise<void | Response<any, Record<string, any>>>
 
-export const DELIMITERS = [',','&','/','\\'];
-export const DELIMITERS_NO_AMP = [',','/','\\'];
-export const DELIMETERS_REGEX: RegExp = new RegExp(/[,&\/\\]/);
-
 export const ARTIST_WEIGHT = 0.3;
 export const TITLE_WEIGHT = 0.4;
 export const TIME_WEIGHT = 0.5;
@@ -386,7 +362,7 @@ export interface PaginatedListens {
     getPaginatedListens(params: PaginatedListensOptions<CursorType>): Promise<{data: PlayObject[], meta: PaginatedListensOptions<CursorType> & PaginatedResults<CursorType>}>
 }
 
-export const hasPaginagedListens = (obj: Object): obj is PaginatedListens => {
+export const hasPaginagedListens = (obj: object): obj is PaginatedListens => {
     return 'getPaginatedListens' in obj;
 }
 
@@ -399,7 +375,7 @@ export interface PaginatedTimeRangeListens<T extends CursorType = CursorType> {
     getPaginatedUnitOfTime(): ManipulateType;
 }
 
-export const hasPaginatedTimeRangeListens = (obj: Object): obj is PaginatedTimeRangeListens => {
+export const hasPaginatedTimeRangeListens = (obj: object): obj is PaginatedTimeRangeListens => {
     return 'getPaginatedTimeRangeListens' in obj;
 }
 
@@ -412,7 +388,7 @@ export interface PagelessTimeRangeListens {
     getPaginatedUnitOfTime(): ManipulateType;
 }
 
-export const hasPagelessTimeRangeListens = (obj: Object): obj is PagelessTimeRangeListens => {
+export const hasPagelessTimeRangeListens = (obj: object): obj is PagelessTimeRangeListens => {
     return 'getPagelessTimeRangeListens' in obj;
 }
 

src/backend/common/infrastructure/Job.ts

@@ -1,4 +1,4 @@
-import { UnixTimestamp } from "../../../core/Atomic.js"
+import type {UnixTimestamp} from "../../../core/Atomic.ts";
 
 export interface JobParameters {
     /** maximum number of results to get for the entire job */

src/backend/common/infrastructure/config/aioConfig.ts

@@ -1,13 +1,13 @@
-import { LogOptions } from "@foxxmd/logging";
-import { ClientAIOConfig } from "./client/clients.js";
-import { CommonClientOptions } from "./client/index.js";
-import { RequestRetryOptions } from "./common.js";
-import { WebhookConfig } from "./health/webhooks.js";
-import { CommonSourceOptions, SourceRetryOptions } from "./source/index.js";
-import { SourceAIOConfig } from "./source/sources.js";
-import { CacheConfigOptions, CacheConfigUser, DurationValue } from "../Atomic.js";
-import { TransformerCommonConfig } from "../../../../core/Atomic.js";
-import { RetentionConfig } from "./database.js";
+import type { LogOptions } from "@foxxmd/logging";
+import type { ClientAIOConfig } from "./client/clients.ts";
+import type { CommonClientOptions } from "./client/index.ts";
+import type { RequestRetryOptions } from "./common.ts";
+import type { WebhookConfig } from "./health/webhooks.ts";
+import type { CommonSourceOptions, SourceRetryOptions } from "./source/index.ts";
+import type { SourceAIOConfig } from "./source/sources.ts";
+import type { CacheConfigUser, DurationValue } from "../Atomic.ts";
+import type { TransformerCommonConfig } from "../../../../core/Atomic.ts";
+import type { RetentionConfig } from "./database.ts";
 
 
 export interface SourceDefaults extends CommonSourceOptions {

src/backend/common/infrastructure/config/client/atproto.ts

@@ -0,0 +1,29 @@
+import type {AtprotoDid} from "@atcute/lexicons/syntax";
+
+export interface ATProtoUserIdentifierData {
+    /**
+     * Identify the account to login as
+     * 
+     * * For **App Password** Auth - your email
+     * * For **Oauth** - your handle minus the @
+     */
+    identifier: string
+    did?: string
+}
+
+export interface ATProtoAppData {
+    /**
+     * The [App Password](https://atproto.com/specs/xrpc#app-passwords) you created for your account
+     * 
+     * This is created under https://bsky.app/settings/app-passwords
+     * 
+     * **Use this if you are self-hosting Multi-Scrobbler on localhost or accessed like http://IP:PORT**
+     */
+    appPassword: string
+}
+
+export interface HandleData {
+    did: AtprotoDid
+    pds: string
+    handle: string
+}

src/backend/common/infrastructure/config/client/clients.ts

@@ -1,11 +1,11 @@
-import { KoitoClientAIOConfig, KoitoClientConfig } from "./koito.js";
-import { LastfmClientAIOConfig, LastfmClientConfig } from "./lastfm.js";
-import { ListenBrainzClientAIOConfig, ListenBrainzClientConfig } from "./listenbrainz.js";
-import { MalojaClientAIOConfig, MalojaClientConfig } from "./maloja.js";
-import { TealClientAIOConfig, TealClientConfig } from "./tealfm.js";
-import { RockSkyClientAIOConfig, RockSkyClientConfig } from "./rocksky.js";
-import { LibrefmClientConfig, LibrefmClientAIOConfig } from "./librefm.js";
-import { DiscordClientAIOConfig, DiscordClientConfig } from "./discord.js";
+import type {KoitoClientAIOConfig, KoitoClientConfig} from "./koito.ts";
+import type {LastfmClientAIOConfig, LastfmClientConfig} from "./lastfm.ts";
+import type {ListenBrainzClientAIOConfig, ListenBrainzClientConfig} from "./listenbrainz.ts";
+import type {MalojaClientAIOConfig, MalojaClientConfig} from "./maloja.ts";
+import type {TealClientAIOConfig, TealClientConfig} from "./tealfm.ts";
+import type {RockSkyClientAIOConfig, RockSkyClientConfig} from "./rocksky.ts";
+import type {LibrefmClientConfig, LibrefmClientAIOConfig} from "./librefm.ts";
+import type {DiscordClientAIOConfig, DiscordClientConfig} from "./discord.ts";
 
 export type ClientConfig = 
 MalojaClientConfig 
@@ -57,23 +57,3 @@ export const clientInterfaces = [
     ...atomicClientInterfaces
 ];
 
-export type ClientType =
-    'maloja'
-    | 'lastfm'
-    | 'librefm'
-    | 'listenbrainz'
-    | 'koito'
-    | 'tealfm'
-    | 'rocksky'
-    | 'discord';
-    
-export const clientTypes: ClientType[] = [
-    'maloja',
-    'lastfm',
-    'librefm',
-    'listenbrainz',
-    'koito',
-    'tealfm',
-    'rocksky',
-    'discord'
-];

src/backend/common/infrastructure/config/client/discord.ts

@@ -1,5 +1,5 @@
-import { ActivityType, StatusDisplayType } from "discord.js"
-import { CommonClientConfig, CommonClientData } from "./index.js"
+import type {CommonClientConfig, CommonClientData} from "./index.ts";
+import type {ComponentType} from "../../../../../core/Atomic.ts";
 
 export interface DiscordData {
     token?: string
@@ -20,7 +20,7 @@ export interface DiscordClientConfig extends CommonClientConfig {
      * @default client
      * @examples ["client"]
      * */
-    configureAs?: 'client' | 'source'
+    configureAs?: ComponentType
     data: DiscordClientData
 }
 
@@ -62,38 +62,14 @@ export interface ActivityTimestamps {
     end?: number
 }
 
-export enum ActivityTypeOthers {
-    Hanging = 6
-}
+export type StatusDisplayType = 0 | 1 | 2;
+export const STATUS_DISPLAY_TYPE = {
+    Name: 0,
+    State: 1,
+    Details: 2
+} as const satisfies Record<string, StatusDisplayType>
 
-export declare enum ActivityTypeReal {
-    /**
-     * Playing \{game\}
-     */
-    Playing = 0,
-    /**
-     * Streaming \{details\}
-     */
-    Streaming = 1,
-    /**
-     * Listening to \{name\}
-     */
-    Listening = 2,
-    /**
-     * Watching \{details\}
-     */
-    Watching = 3,
-    /**
-     * \{emoji\} \{state\}
-     */
-    Custom = 4,
-    /**
-     * Competing in \{name\}
-     */
-    Competing = 5,
-
-    Hanging = 6
-}
+export type ActivityType = 0 | 1 | 2 | 3 | 4 | 5 | 6;
 
 export interface ActivityData {
     name: string
@@ -119,7 +95,7 @@ export const ACTIVITY_TYPE = {
     Custom: 4,
     Competing: 5,
     Hanging: 6
-} as const satisfies Record<string, ActivityTypeReal>
+} as const satisfies Record<string, ActivityType>
 
 export const ARTWORK_PLACEHOLDER = 'https://raw.githubusercontent.com/FoxxMD/multi-scrobbler/master/assets/default-artwork.png';
 export const MS_ART = 'https://raw.githubusercontent.com/FoxxMD/multi-scrobbler/master/assets/icon.png';

src/backend/common/infrastructure/config/client/index.ts

@@ -1,7 +1,7 @@
-import { DurationValue } from "../../Atomic.js";
-import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js";
-import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js";
-import { RetentionConfig } from "../database.js";
+import type {DurationValue} from "../../Atomic.ts";
+import type {PlayTransformOptions} from "../../../../../core/Transform.ts";
+import type {CommonConfig, RequestRetryOptions} from "../common.ts";
+import type {RetentionConfig} from "../database.ts";
 
 /**
  * Scrobble matching (between new source track and existing client scrobbles) logging options. Used for debugging.

src/backend/common/infrastructure/config/client/koito.ts

@@ -1,6 +1,6 @@
-import { UnixTimestamp } from "../../../../../core/Atomic.js"
-import { RequestRetryOptions } from "../common.js"
-import { CommonClientConfig, CommonClientData } from "./index.js"
+import type {ComponentType, UnixTimestamp} from "../../../../../core/Atomic.ts";
+import type {RequestRetryOptions} from "../common.ts";
+import type {CommonClientConfig, CommonClientData} from "./index.ts";
 
 export interface ListensResponse {
     items: ListenObjectResponse[]
@@ -74,7 +74,7 @@ export interface KoitoClientConfig extends CommonClientConfig {
      * @default client
      * @examples ["client"]
      * */
-    configureAs?: 'client' | 'source'
+    configureAs?: ComponentType
     data: KoitoClientData
 }
 

src/backend/common/infrastructure/config/client/lastfm.ts

@@ -1,5 +1,5 @@
-import { RequestRetryOptions } from "../common.js";
-import { CommonClientConfig, CommonClientData, CommonClientOptions, NowPlayingOptions } from "./index.js";
+import type {RequestRetryOptions} from "../common.ts";
+import type {CommonClientConfig, CommonClientData, CommonClientOptions, NowPlayingOptions} from "./index.ts";
 
 export interface LastfmData extends CommonClientData, RequestRetryOptions {
     /**

src/backend/common/infrastructure/config/client/librefm.ts

@@ -1,6 +1,6 @@
-import { RequestRetryOptions } from "../common.js";
-import { CommonClientConfig, CommonClientData } from "./index.js";
-import { LastfmClientOptions } from "./lastfm.js";
+import type {RequestRetryOptions} from "../common.ts";
+import type {CommonClientConfig, CommonClientData} from "./index.ts";
+import type {LastfmClientOptions} from "./lastfm.ts";
 
 export interface LibrefmData extends CommonClientData, RequestRetryOptions {
         /**

src/backend/common/infrastructure/config/client/listenbrainz.ts

@@ -1,5 +1,6 @@
-import { RequestRetryOptions } from "../common.js";
-import { CommonClientConfig, CommonClientData } from "./index.js";
+import type {ComponentType} from "../../../../../core/Atomic.ts";
+import type {RequestRetryOptions} from "../common.ts";
+import type {CommonClientConfig, CommonClientData} from "./index.ts";
 
 export interface ListenBrainzData extends RequestRetryOptions{
     /**
@@ -31,7 +32,7 @@ export interface ListenBrainzClientConfig extends CommonClientConfig {
      * @default client
      * @examples ["client"]
      * */
-    configureAs?: 'client' | 'source'
+    configureAs?: ComponentType
     data: ListenBrainzClientData
 }
 

src/backend/common/infrastructure/config/client/maloja.ts

@@ -1,5 +1,6 @@
-import { RequestRetryOptions } from "../common.js";
-import { CommonClientConfig, CommonClientData } from "./index.js";
+import type {ComponentType} from "../../../../../core/Atomic.ts";
+import type {RequestRetryOptions} from "../common.ts";
+import type {CommonClientConfig, CommonClientData} from "./index.ts";
 
 export interface MalojaData extends RequestRetryOptions {
     /**
@@ -27,7 +28,7 @@ export interface MalojaClientConfig extends CommonClientConfig {
      * @default client
      * @examples ["client"]
      * */
-    configureAs?: 'client' | 'source'
+    configureAs?: ComponentType
     data: MalojaClientData
 }
 

src/backend/common/infrastructure/config/client/rocksky.ts

@@ -1,5 +1,5 @@
-import { RequestRetryOptions } from "../common.js";
-import { CommonClientConfig, CommonClientData, CommonClientOptions, NowPlayingOptions } from "./index.js";
+import type {RequestRetryOptions} from "../common.ts";
+import type {CommonClientConfig, CommonClientData, CommonClientOptions, NowPlayingOptions} from "./index.ts";
 
 export interface RockSkyData extends RequestRetryOptions{
 
@@ -8,7 +8,14 @@ export interface RockSkyData extends RequestRetryOptions{
      *
      * @examples ["6794186bf-1157-4de6-80e5-uvb411f3ea2b"]
      * */
-    key: string
+    key?: string
+
+    /**
+     * Access Token generated from https://rocksky.app/access-tokens in Rocksky for your account
+     *
+     * @examples ["eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkaWQ....."]
+     * */
+    token?: string
 
     /**
      * The **fully-qualified** handle for your ATPRoto/Bluesky account, like:

src/backend/common/infrastructure/config/client/tealfm.ts

@@ -1,28 +1,15 @@
-import { RequestRetryOptions } from "../common.js"
-import { CommonClientConfig, CommonClientData, CommonClientOptions } from "./index.js"
+import type {ComponentType} from "../../../../../core/Atomic.ts";
+import type {RequestRetryOptions} from "../common.ts";
+import type {ATProtoAppData, ATProtoUserIdentifierData} from "./atproto.ts";
+import type {CommonClientConfig, CommonClientData, CommonClientOptions} from "./index.ts";
 
-export interface TealData extends RequestRetryOptions {
+export type TealData = RequestRetryOptions & ATProtoUserIdentifierData & Partial<ATProtoAppData> & {
         /**
      * The base URI of the Multi-Scrobbler to use for ATProto OAuth
      * 
      * Only include this if you want to use OAuth. The URI must be a non-IP/non-local domain using https: protocol.
     */
     baseUri?: string
-    /**
-     * Identify the account to login as
-     * 
-     * * For **App Password** Auth - your email
-     * * For **Oauth** - your handle minus the @
-     */
-    identifier: string
-    /**
-     * The [App Password](https://atproto.com/specs/xrpc#app-passwords) you created for your account
-     * 
-     * This is created under https://bsky.app/settings/app-passwords
-     * 
-     * **Use this if you are self-hosting Multi-Scrobbler on localhost or accessed like http://IP:PORT**
-     */
-    appPassword?: string
 }
 
 export interface TealClientData extends TealData, CommonClientData {
@@ -35,7 +22,7 @@ export interface TealClientConfig extends CommonClientConfig {
      * @default client
      * @examples ["client"]
      * */
-    configureAs?: 'client' | 'source'
+    configureAs?: ComponentType
     data: TealClientData
     options?: TealClientOptions
 }
@@ -51,50 +38,6 @@ export interface TealClientOptions extends TealOptions,CommonClientOptions {
 export interface TealClientAIOConfig extends TealClientConfig {
     type: 'tealfm'
 }
-export interface TealArtistCredit {
-    artistName?: string,
-    /** The MusicBrainz artist ID URI, formatted as mbid:<uuid> */
-    artistMbId?: string
-}
-/**
- *  https://github.com/teal-fm/teal/blob/main/lexicons/fm.teal.alpha/feed/play.json
- *  https://github.com/teal-fm/teal/blob/main/lexicons/fm.teal.alpha/feed/defs.json
- * */
-export interface ScrobbleRecord {
-    $type: "fm.teal.alpha.feed.play",
-    trackName: string,
-    playedTime: string,
-    duration?: number
-    artists?: TealArtistCredit[]
-    /** Album name  */
-    releaseName?: string
-    /** A metadata string specifying the user agent where the format is `<app-identifier>/<version> (<kernel/OS-base>; <platform/OS-version>; <device-model>)` */
-    submissionClientAgent: string,
-    musicServiceBaseDomain?: string
-    /** The MusicBrainz ID URI of the track, formatted as mbid:<uuid> */
-    trackMbid?: string
-    /** The MusicBrainz recording ID URI of the track, formatted as mbid:<uuid> */
-    recordingMbId?: string
-    /** The MusicBrainz release ID URI, formatted as mbid:<uuid> */
-    releaseMbId?: string
-    isrc?: string,
-    /** The URL associated with this track */
-    originUrl?: string
-    /** Distinguishing information for track variants (e.g. 'Acoustic Version', 'Live at Wembley', 'Radio Edit', 'Demo'). Used to differentiate between different versions of the same base track while maintaining grouping capabilities. */
-    trackDiscriminant?: string;
-    /** Distinguishing information for release variants (e.g. 'Deluxe Edition', 'Remastered', '2023 Remaster', 'Special Edition'). Used to differentiate between different versions of the same base release while maintaining grouping capabilities. */
-    releaseDiscriminant?: string;
-    [x: string]: unknown
-}
-
-export interface StatusRecord {
-    $type: "fm.teal.alpha.actor.status",
-    /** item is just ScrobbleRecord, but without $type */
-    item: Omit<ScrobbleRecord, '$type'>,
-    time: string,
-    expiry: string,
-    [x: string]: unknown
-}
 
 export interface ListRecord<T> {
   uri: string;

src/backend/common/infrastructure/config/common.ts

@@ -1,5 +1,3 @@
-import { keyOmit } from "../Atomic.js";
-
 export interface CommonConfigPrimitives {
     name?: string
     id?: string

src/backend/common/infrastructure/config/database.ts

@@ -1,5 +1,5 @@
-import { Duration } from "dayjs/plugin/duration.js";
-import { DurationValue } from "../Atomic.js";
+import type {Duration} from "dayjs/plugin/duration.js";
+import type {DurationValue} from "../Atomic.ts";
 
 export type RetentionPlayType = 'failed' | 'completed' | 'duped';
 export const retentionPlayTypes: RetentionPlayType[] = ['failed','completed','duped'];

src/backend/common/infrastructure/config/health/webhooks.ts

@@ -1,5 +1,6 @@
 export interface WebhookPayload {
     title?: string
+    identifier: string
     message: string
     priority: Priority
 }

src/backend/common/infrastructure/config/source/azuracast.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions, ManualListeningOptions } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions, ManualListeningOptions} from "./index.ts";
 
 export interface AzuraStationInfoResponse {
     id: string

src/backend/common/infrastructure/config/source/chromecast.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface ChromecastData extends CommonSourceData {
     /**

src/backend/common/infrastructure/config/source/deezer.ts

@@ -1,6 +1,6 @@
-import { Second } from "../../../../../core/Atomic.js";
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js";
+import type {Second} from "../../../../../core/Atomic.ts";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.ts";
 
 export interface DeezerData extends CommonSourceData, PollingOptions {
     /**

src/backend/common/infrastructure/config/source/endpointlfm.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface LastFMEndpointData extends CommonSourceData {
     /**

src/backend/common/infrastructure/config/source/endpointlz.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface ListenbrainzEndpointData extends CommonSourceData {
     /**

src/backend/common/infrastructure/config/source/icecast.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions, ManualListeningOptions } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions, ManualListeningOptions} from "./index.ts";
 
 
 export interface IcecastMetadata {

src/backend/common/infrastructure/config/source/index.ts

@@ -1,9 +1,9 @@
-import { FileLogOptions, LogLevel } from "@foxxmd/logging";
+import type { FileLogOptions, LogLevel } from "@foxxmd/logging";
 
-import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js";
-import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js";
-import { RetentionConfig } from "../database.js";
-import { DurationValue } from "../../Atomic.js";
+import type { PlayTransformOptions } from "../../../../../core/Transform.ts";
+import type { CommonConfig, RequestRetryOptions } from "../common.ts";
+import type { RetentionConfig } from "../database.ts";
+import type { DurationValue } from "../../Atomic.ts";
 
 export interface SourceRetryOptions extends RequestRetryOptions {
     /**

src/backend/common/infrastructure/config/source/jellyfin.ts

@@ -1,5 +1,5 @@
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js";
-import {
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.ts";
+import type {
     // @ts-expect-error weird typings?
     CollectionType,
         // @ts-expect-error weird typings?

src/backend/common/infrastructure/config/source/jriver.ts

@@ -1,5 +1,5 @@
-import { PollingOptions, RequestRetryOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {PollingOptions, RequestRetryOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface JRiverData extends CommonSourceData, PollingOptions, RequestRetryOptions {
     /**

src/backend/common/infrastructure/config/source/kodi.ts

@@ -1,5 +1,5 @@
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 
 export interface KodiData extends CommonSourceData, PollingOptions {

src/backend/common/infrastructure/config/source/koito.ts

@@ -1,7 +1,6 @@
-import { KoitoData } from "../client/koito.js";
-import { ListenBrainzData } from "../client/listenbrainz.js";
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {KoitoData} from "../client/koito.ts";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface KoitoSourceData extends KoitoData, CommonSourceData, PollingOptions {
 }

src/backend/common/infrastructure/config/source/lastfm.ts

@@ -1,6 +1,6 @@
-import { LastfmData } from "../client/lastfm.js";
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {LastfmData} from "../client/lastfm.ts";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface LastFmSourceData extends CommonSourceData, PollingOptions, LastfmData{}
 

src/backend/common/infrastructure/config/source/librefm.ts

@@ -1,6 +1,6 @@
-import { LibrefmData } from "../client/librefm.js";
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {LibrefmData} from "../client/librefm.ts";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface librefmSourceData extends CommonSourceData, PollingOptions, LibrefmData{}
 

src/backend/common/infrastructure/config/source/listenbrainz.ts

@@ -1,6 +1,6 @@
-import { ListenBrainzData } from "../client/listenbrainz.js";
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {ListenBrainzData} from "../client/listenbrainz.ts";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface ListenBrainzSourceData extends ListenBrainzData, CommonSourceData, PollingOptions {
 }

src/backend/common/infrastructure/config/source/maloja.ts

@@ -1,6 +1,6 @@
-import { MalojaData } from "../client/maloja.js";
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {MalojaData} from "../client/maloja.ts";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface MalojaSourceData extends MalojaData, CommonSourceData, PollingOptions {
 }

src/backend/common/infrastructure/config/source/mopidy.ts

@@ -1,5 +1,5 @@
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface MopidyData extends CommonSourceData, PollingOptions {
     /**

src/backend/common/infrastructure/config/source/mpd.ts

@@ -1,5 +1,5 @@
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.ts";
 
 export interface MPDData extends CommonSourceData, PollingOptions {
     /**

src/backend/common/infrastructure/config/source/mpris.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export const PLAYBACK_STATUS_PLAYING = 'Playing';
 export const PLAYBACK_STATUS_PAUSED = 'Paused';

src/backend/common/infrastructure/config/source/musiccast.ts

@@ -1,5 +1,6 @@
-import { REPORTED_PLAYER_STATUSES, ReportedPlayerStatus } from "../../Atomic.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import { REPORTED_PLAYER_STATUSES } from '../../../../../core/Atomic.ts';
+import type {ReportedPlayerStatus} from '../../../../../core/Atomic.ts';
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export type PlaybackStatus = 'play' | 'stop' | 'pause' | 'fast_reverse' | 'fast_forward'
 

src/backend/common/infrastructure/config/source/musikcube.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export const PLAYBACK_STATUS_PLAYING_MC = 'playing';
 export const PLAYBACK_STATUS_PAUSED_MC = 'paused';

src/backend/common/infrastructure/config/source/plex.ts

@@ -1,5 +1,5 @@
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.ts";
 
 export interface PlexApiData extends CommonSourceData, PollingOptions {
     token?: string

src/backend/common/infrastructure/config/source/rocksky.ts

@@ -1,6 +1,6 @@
-import { RockSkyData, RockSkyOptions } from "../client/rocksky.js";
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js";
+import type {RockSkyData, RockSkyOptions} from "../client/rocksky.ts";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.ts";
 
 export interface RockskySourceData extends RockSkyData, CommonSourceData, PollingOptions {
 }

src/backend/common/infrastructure/config/source/sonos.ts

@@ -1,5 +1,5 @@
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.ts";
 
 export interface SonosData extends CommonSourceData, PollingOptions {
     /**

src/backend/common/infrastructure/config/source/sources.ts

@@ -1,32 +1,32 @@
-import { AzuracastSourceAIOConfig, AzuracastSourceConfig } from "./azuracast.js";
-import { ChromecastSourceAIOConfig, ChromecastSourceConfig } from "./chromecast.js";
-import { ListenbrainzEndpointSourceAIOConfig, ListenbrainzEndpointSourceConfig } from "./endpointlz.js";
-import { LastFMEndpointSourceAIOConfig, LastFMEndpointSourceConfig } from "./endpointlfm.js";
-import { DeezerInternalSourceConfig, DeezerInternalAIOConfig, DeezerSourceAIOConfig, DeezerSourceConfig, DeezerCompatConfig, DeezerAIOCompatConfig } from "./deezer.js";
-import { JellyApiSourceAIOConfig, JellyApiSourceConfig } from "./jellyfin.js";
-import { JRiverSourceAIOConfig, JRiverSourceConfig } from "./jriver.js";
-import { KodiSourceAIOConfig, KodiSourceConfig } from "./kodi.js";
-import { LastFmSouceAIOConfig, LastfmSourceConfig } from "./lastfm.js";
-import { ListenBrainzSourceAIOConfig, ListenBrainzSourceConfig } from "./listenbrainz.js";
-import { MopidySourceAIOConfig, MopidySourceConfig } from "./mopidy.js";
-import { MPDSourceAIOConfig, MPDSourceConfig } from "./mpd.js";
-import { MPRISSourceAIOConfig, MPRISSourceConfig } from "./mpris.js";
-import { MusikcubeSourceAIOConfig, MusikcubeSourceConfig } from "./musikcube.js";
-import { MusicCastSourceConfig, MusicCastSourceAIOConfig } from "./musiccast.js";
-import { PlexApiSourceConfig, PlexApiSourceAIOConfig } from "./plex.js";
-import { SpotifySourceAIOConfig, SpotifySourceConfig } from "./spotify.js";
-import { SubsonicSourceAIOConfig, SubSonicSourceConfig } from "./subsonic.js";
-import { VLCSourceAIOConfig, VLCSourceConfig } from "./vlc.js";
-import { WebScrobblerSourceAIOConfig, WebScrobblerSourceConfig } from "./webscrobbler.js";
-import { YTMusicSourceAIOConfig, YTMusicSourceConfig } from "./ytmusic.js";
-import { YandexMusicBridgeSourceAIOConfig, YandexMusicBridgeSourceConfig } from "./ymbridge.js";
-import { IcecastSourceAIOConfig, IcecastSourceConfig } from "./icecast.js";
-import { KoitoSourceAIOConfig, KoitoSourceConfig } from "./koito.js";
-import { MalojaSourceAIOConfig, MalojaSourceConfig } from "./maloja.js";
-import { TealSourceAIOConfig, TealSourceConfig } from "./tealfm.js";
-import { RockskySourceAIOConfig, RockskySourceConfig } from "./rocksky.js";
-import { LibrefmSouceAIOConfig, LibrefmSourceConfig } from "./librefm.js";
-import { SonosSourceAIOConfig, SonosSourceConfig } from "./sonos.js";
+import type {AzuracastSourceAIOConfig, AzuracastSourceConfig} from "./azuracast.ts";
+import type {ChromecastSourceAIOConfig, ChromecastSourceConfig} from "./chromecast.ts";
+import type {ListenbrainzEndpointSourceAIOConfig, ListenbrainzEndpointSourceConfig} from "./endpointlz.ts";
+import type {LastFMEndpointSourceAIOConfig, LastFMEndpointSourceConfig} from "./endpointlfm.ts";
+import type {DeezerInternalSourceConfig, DeezerSourceConfig, DeezerCompatConfig, DeezerAIOCompatConfig} from "./deezer.ts";
+import type {JellyApiSourceAIOConfig, JellyApiSourceConfig} from "./jellyfin.ts";
+import type {JRiverSourceAIOConfig, JRiverSourceConfig} from "./jriver.ts";
+import type {KodiSourceAIOConfig, KodiSourceConfig} from "./kodi.ts";
+import type {LastFmSouceAIOConfig, LastfmSourceConfig} from "./lastfm.ts";
+import type {ListenBrainzSourceAIOConfig, ListenBrainzSourceConfig} from "./listenbrainz.ts";
+import type {MopidySourceAIOConfig, MopidySourceConfig} from "./mopidy.ts";
+import type {MPDSourceAIOConfig, MPDSourceConfig} from "./mpd.ts";
+import type {MPRISSourceAIOConfig, MPRISSourceConfig} from "./mpris.ts";
+import type {MusikcubeSourceAIOConfig, MusikcubeSourceConfig} from "./musikcube.ts";
+import type {MusicCastSourceConfig, MusicCastSourceAIOConfig} from "./musiccast.ts";
+import type {PlexApiSourceConfig, PlexApiSourceAIOConfig} from "./plex.ts";
+import type {SpotifySourceAIOConfig, SpotifySourceConfig} from "./spotify.ts";
+import type {SubsonicSourceAIOConfig, SubSonicSourceConfig} from "./subsonic.ts";
+import type {VLCSourceAIOConfig, VLCSourceConfig} from "./vlc.ts";
+import type {WebScrobblerSourceAIOConfig, WebScrobblerSourceConfig} from "./webscrobbler.ts";
+import type {YTMusicSourceAIOConfig, YTMusicSourceConfig} from "./ytmusic.ts";
+import type {YandexMusicBridgeSourceAIOConfig, YandexMusicBridgeSourceConfig} from "./ymbridge.ts";
+import type {IcecastSourceAIOConfig, IcecastSourceConfig} from "./icecast.ts";
+import type {KoitoSourceAIOConfig, KoitoSourceConfig} from "./koito.ts";
+import type {MalojaSourceAIOConfig, MalojaSourceConfig} from "./maloja.ts";
+import type {TealSourceAIOConfig, TealSourceConfig} from "./tealfm.ts";
+import type {RockskySourceAIOConfig, RockskySourceConfig} from "./rocksky.ts";
+import type {LibrefmSouceAIOConfig, LibrefmSourceConfig} from "./librefm.ts";
+import type {SonosSourceAIOConfig, SonosSourceConfig} from "./sonos.ts";
 
 
 export type SourceConfig =
@@ -130,69 +130,6 @@ export type RockskySourceConfigs = RockskySourceConfig[];
 export type SonosSourceConfigs = SonosSourceConfig[];
 
 
-export type SourceType =
-    'spotify'
-    | 'plex'
-    | 'subsonic'
-    | 'jellyfin'
-    | 'lastfm'
-    | 'librefm'
-    | 'deezer'
-    | 'endpointlz'
-    | 'endpointlfm'
-    | 'ytmusic'
-    | 'ymbridge'
-    | 'mpris'
-    | 'mopidy'
-    | 'musiccast'
-    | 'listenbrainz'
-    | 'jriver'
-    | 'kodi'
-    | 'webscrobbler'
-    | 'chromecast'
-    | 'maloja'
-    | 'musikcube'
-    | 'mpd'
-    | 'vlc'
-    | 'icecast'
-    | 'azuracast'
-    | 'koito'
-    | 'tealfm'
-    | 'rocksky'
-    | 'sonos';
-    
-export const sourceTypes: SourceType[] = [
-    'spotify',
-    'plex',
-    'subsonic',
-    'jellyfin',
-    'lastfm',
-    'librefm',
-    'deezer',
-    'endpointlz',
-    'endpointlfm',
-    'ytmusic',
-    'ymbridge',
-    'mpris',
-    'mopidy',
-    'musiccast',
-    'listenbrainz',
-    'jriver',
-    'kodi',
-    'webscrobbler',
-    'chromecast',
-    'maloja',
-    'musikcube',
-    'mpd',
-    'vlc',
-    'icecast',
-    'azuracast',
-    'koito',
-    'tealfm',
-    'rocksky',
-    'sonos'
-];
-
 export const atomicSourceInterfaces = [
     'SpotifySourceConfig',
     'PlexApiSourceConfig',
@@ -230,7 +167,4 @@ export const sourceInterfaces = [
     ...atomicSourceInterfaces
 ];
 
-export const isSourceType = (data: string): data is SourceType => {
-    return sourceTypes.includes(data as SourceType);
-};
 

src/backend/common/infrastructure/config/source/spotify.ts

@@ -1,5 +1,5 @@
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface SpotifySourceData extends CommonSourceData, PollingOptions {
     /**

src/backend/common/infrastructure/config/source/subsonic.ts

@@ -1,5 +1,5 @@
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface SubsonicData extends CommonSourceData, PollingOptions {
     /**

src/backend/common/infrastructure/config/source/tealfm.ts

@@ -1,6 +1,6 @@
-import { TealData, TealOptions } from "../client/tealfm.js"
-import { PollingOptions } from "../common.js"
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js"
+import type {TealData, TealOptions} from "../client/tealfm.ts";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.ts";
 
 
 export interface TealSourceData extends TealData, CommonSourceData, PollingOptions {

src/backend/common/infrastructure/config/source/vlc.ts

@@ -1,6 +1,6 @@
-import { VlcMeta } from "vlc-client/dist/Types.js";
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js";
+import type {VlcMeta} from "vlc-client/dist/Types.js";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.ts";
 
 export interface VLCData extends CommonSourceData, PollingOptions {
     /**

src/backend/common/infrastructure/config/source/webscrobbler.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface WebScrobblerData extends CommonSourceData {
     /**

src/backend/common/infrastructure/config/source/ymbridge.ts

@@ -1,5 +1,5 @@
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface YandexMusicBridgeData extends CommonSourceData, PollingOptions {
     /** URL of the local Python bridge, for example http://yandex-music-bridge:9980 */

src/backend/common/infrastructure/config/source/ytmusic.ts

@@ -1,5 +1,5 @@
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.ts";
 
 export interface InnertubeOptions {
     /**

src/backend/common/logging.ts

@@ -1,10 +1,11 @@
-import { FileLogOptions, Logger, loggerAppRolling, LogLevel, LogLevelStreamEntry, LogOptions, parseLogOptions } from '@foxxmd/logging';
+import { type FileLogOptions, type Logger, loggerAppRolling, type LogLevel, type LogLevelStreamEntry, type LogOptions, parseLogOptions } from '@foxxmd/logging';
 import { buildDestinationJsonPrettyStream, buildDestinationRollingFile, buildDestinationStdout, buildLogger } from "@foxxmd/logging/factory";
-import { PassThrough, Transform } from "node:stream";
+import type { Transform } from "node:stream";
+import { PassThrough } from "node:stream";
 import path from "path";
 import process from "process";
-import { projectDir } from "./index.js";
-import { isDebugMode } from '../utils.js';
+import { projectDir } from "./index.ts";
+import { isDebugMode } from '../utils.ts';
 
 export let logPath = path.resolve(projectDir, `./logs`);
 if (typeof process.env.CONFIG_DIR === 'string') {

src/backend/common/transforms/AbstractTransformer.ts

@@ -1,15 +1,16 @@
-import { childLogger, Logger } from "@foxxmd/logging";
-import { PlayObject, TransformerCommon, TransformerCommonConfig } from "../../../core/Atomic.js";
-import { isStageTyped, testWhenConditions } from "../../utils/PlayTransformUtils.js";
-import AbstractInitializable from "../AbstractInitializable.js";
-import { StageConfig } from "../infrastructure/Transform.js";
-import { cacheFunctions,  parseToRegexOrLiteralSearch, testMaybeRegex, searchAndReplace} from "@foxxmd/regex-buddy-core";
-import { Cacheable } from "cacheable";
-import { hashObject } from "../../utils/StringUtils.js";
-import { playContentInvariantTransform } from "../../utils/PlayComparisonUtils.js";
-import { isSimpleError, SkipTransformStageError, StagePrerequisiteError } from "../errors/MSErrors.js";
-import { capitalize } from "../../../core/StringUtils.js";
-import { StaggerOptions } from "../../utils/AsyncUtils.js";
+import { childLogger, type Logger } from "@foxxmd/logging";
+import type { PlayObject, TransformerCommon, TransformerCommonConfig } from "../../../core/Atomic.ts";
+import { isStageTyped, testWhenConditions } from "../../utils/PlayTransformUtils.ts";
+import AbstractInitializable from "../AbstractInitializable.ts";
+import type { StageConfig } from "../../../core/Transform.ts";
+import type { cacheFunctions} from "@foxxmd/regex-buddy-core";
+import {  parseToRegexOrLiteralSearch, testMaybeRegex, searchAndReplace} from "@foxxmd/regex-buddy-core";
+import type { Cacheable } from "cacheable";
+import { hashObject } from "../../utils/StringUtils.ts";
+import { playContentInvariantTransform } from "../../utils/PlayComparisonUtils.ts";
+import { SkipTransformStageError, StagePrerequisiteError } from "../errors/MSErrors.ts";
+import { capitalize } from "../../../core/StringUtils.ts";
+import type { StaggerOptions } from "../../utils/AsyncUtils.ts";
 
 export interface TransformerOptions {
         logger: Logger

src/backend/common/transforms/AtomicPartsTransformer.ts

@@ -1,6 +1,6 @@
-import { ArtistCredit, isPlayObject, ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js";
-import { AtomicStageConfig, StageConfig } from "../infrastructure/Transform.js";
-import AbstractTransformer from "./AbstractTransformer.js";
+import { type ArtistCredit, isPlayObject, type ObjectPlayData, type PlayObject, type TrackMeta } from "../../../core/Atomic.ts";
+import type {AtomicStageConfig, StageConfig} from "../../../core/Transform.ts";
+import AbstractTransformer from "./AbstractTransformer.ts";
 
 //export type GenericAtomicStageConfig<A> = 
 

src/backend/common/transforms/MusicbrainzTransformer.ts

@@ -1,22 +1,24 @@
-import { ArtistCredit, asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, isMBReleasePrimaryGroupType, MBReleaseGroupPrimaryType, MBReleaseGroupSecondaryType, MBReleaseStatus, MissingMbidType, PlayObject, TrackMeta, TransformerCommon, TransformOptions } from "../../../core/Atomic.js";
-import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js";
-import { WebhookPayload } from "../infrastructure/config/health/webhooks.js";
-import { ExternalMetadataTerm, PlayTransformMetadataStage } from "../infrastructure/Transform.js";
-import AtomicPartsTransformer from "./AtomicPartsTransformer.js";
-import { TransformerOptions } from "./AbstractTransformer.js";
-import { ARTIST_WEIGHT, DELIMITERS, MUSICBRAINZ_URL, MusicbrainzApiConfigData, TITLE_WEIGHT } from "../infrastructure/Atomic.js";
-import { MaybeLogger } from '../MaybeLogger.js';
-import { childLogger, Logger } from "@foxxmd/logging";
-import { MusicbrainzApiClient, MusicbrainzApiConfig, recordingToPlay, UsingTypes } from "../vendor/musicbrainz/MusicbrainzApiClient.js";
-import { IRecordingList, IRecordingMatch, IRelease, MusicBrainzApi } from "musicbrainz-api";
-import { intersect, isDebugMode, missingMbidTypes, removeUndefinedKeys } from "../../utils.js";
-import { SimpleError, SkipTransformStageError, StagePrerequisiteError } from "../errors/MSErrors.js";
-import { parseArrayFromMaybeString, scoreNormalizedStringsWeighted } from "../../utils/StringUtils.js";
+import { type ArtistCredit, asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, type LifecycleInput, type MBReleaseGroupPrimaryType, type MBReleaseGroupSecondaryType, type MBReleaseStatus, type MissingMbidType, type PlayObject, type TrackMeta, type TransformerCommon, type TransformOptions } from "../../../core/Atomic.ts";
+import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.ts";
+import type {WebhookPayload} from "../infrastructure/config/health/webhooks.ts";
+import type {ExternalMetadataTerm, PlayTransformMetadataStage} from "../../../core/Transform.ts";
+import AtomicPartsTransformer from "./AtomicPartsTransformer.ts";
+import type {TransformerOptions} from "./AbstractTransformer.ts";
+import { ARTIST_WEIGHT, type MusicbrainzApiConfigData, TITLE_WEIGHT } from "../infrastructure/Atomic.ts";
+import { DELIMITERS } from '../../../core/Atomic.ts';
+import { MaybeLogger } from '../MaybeLogger.ts';
+import { childLogger } from "@foxxmd/logging";
+import { MusicbrainzApiClient, recordingToPlay, type UsingTypes } from "../vendor/musicbrainz/MusicbrainzApiClient.ts";
+import type {IRecordingList, IRecordingMatch} from "musicbrainz-api";
+import { intersect, missingMbidTypes } from "../../utils.ts";
+import { removeUndefinedKeys } from '../../../core/DataUtils.ts';
+import { SimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError } from "../errors/MSErrors.ts";
+import { parseArrayFromMaybeString, scoreNormalizedStringsWeighted } from "../../utils/StringUtils.ts";
 import clone from "clone";
-import { Cacheable } from "cacheable";
-import { splitByFirstRegexFound } from "../../../core/StringUtils.js";
-import { nativeParse } from "./NativeTransformer.js";
-import { comparePlayArtistsNormalized, scoreTrackWeightedAndNormalized } from "../../utils/PlayComparisonUtils.js";
+import type { Cacheable } from "cacheable";
+import { splitByFirstRegexFound } from "../../../core/StringUtils.ts";
+import { nativeParse } from "./NativeTransformer.ts";
+import { comparePlayArtistsNormalized, scoreTrackWeightedAndNormalized } from "../../utils/PlayComparisonUtils.ts";
 
 export const asMissingMbid = (str: string): MissingMbidType => {
     const clean = str.trim().toLocaleLowerCase();
@@ -193,6 +195,7 @@ export interface IRecordingMSList extends IRecordingList {
     recordings: RecordingRankedMatched[]
     freeText?: boolean
     requestQuery: string
+    requestQueries?: LifecycleInput[]
 }
 
 export const parseStageConfig = (data: MusicbrainzTransformerData | undefined = {}, logger: MaybeLogger = new MaybeLogger()): MusicbrainzTransformerDataStrong => {
@@ -246,10 +249,6 @@ export const parseStageConfig = (data: MusicbrainzTransformerData | undefined =
         releaseCountryAllow: releaseCountryAllow !== undefined ? parseArrayFromMaybeString(releaseCountryAllow, {lower: true}) : undefined,
         releaseCountryDeny:  releaseCountryDeny !== undefined ? parseArrayFromMaybeString(releaseCountryDeny, {lower: true}) : undefined,
         releaseCountryPriority:  releaseCountryPriority !== undefined ? parseArrayFromMaybeString(releaseCountryPriority, {lower: true}) : undefined,
-        artistWeight: 0,
-        titleWeight: 0,
-        albumWeight: 0,
-
         ...rest,
     };
 
@@ -438,6 +437,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
         } = stageConfig;
 
         let results: IRecordingMSList;
+        const queries: LifecycleInput[] = [];
 
         for(const searchType of searchOrder) {
             try {
@@ -464,6 +464,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
                         results = await this.searchByRecordingMbid(play, stageConfig);
                         break;
                 }
+                queries.push({type: `mbQuery-${searchType}${results.recordings.length === 0 ? '-empty' : ''}`, input: results.requestQuery});
                 if(results.recordings.length === 0) {
                     this.logger.debug(`'${searchType}' search type returned no matches`);
                 } else {
@@ -476,12 +477,12 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
                     // we should be catching any unrecoverable errors in api calls
                     // so we should only get here if something truly bad has happened
                     // and we probably don't want to try additional api calls
-                    throw e;
+                    throw new StageTransformError('Search Error', 'Unexpected error occurred while getting musicbrainz recordings', {cause: e, inputs: queries});
                 }
             }
         }
 
-        return results;
+        return {...results, requestQueries: queries};
     }
 
     public async searchByBasicFields(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
@@ -601,7 +602,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
 
     public async handlePostFetch(play: PlayObject, transformData: IRecordingMSList, stageConfig: MusicbrainzTransformerDataStage): Promise<PlayObject> {
         if(transformData.recordings.length === 0) {
-            throw new StagePrerequisiteError('No matches returned from Musicbrainz API', {shortStack: true});
+            throw new StagePrerequisiteError('No matches returned from Musicbrainz API', {shortStack: true, inputs: transformData.requestQueries});
         }
 
         const {
@@ -610,7 +611,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
 
         // if brainz meta contains track MBID then we should be able to get the exact release
         let explicitList: IRecordingMatch[];
-        let filtered = false;
+        let filtered: boolean;
         [explicitList, filtered] = filterByExplicitTrackMbid(transformData.recordings, play);
         if(filtered) {
             this.logger.debug(`Found exact release using track MBID`);
@@ -642,7 +643,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
         this.logger.debug(`${filteredList.length} of ${transformData.count} were valid, filtered matches. Using match with best score of ${filteredList[0].score}`);
 
         const recordingPlay = recordingToPlay(filteredList[0], {ignoreVA: stageConfig.ignoreVA});
-        recordingPlay.meta.lifecycleInputs = [...(recordingPlay.meta.lifecycleInputs ?? []), {type: 'mbQuery', input: transformData.requestQuery}, {type: 'mbRecording', input: filteredList[0]}];
+        recordingPlay.meta.lifecycleInputs = [...(recordingPlay.meta.lifecycleInputs ?? []), ...(transformData.requestQueries ?? []), {type: 'mbRecording', input: filteredList[0]}];
         return recordingPlay;
     }
 
@@ -751,8 +752,7 @@ export const filterByValidReleaseStatus = <T extends IRecordingMatch[]>(list: T,
     if(releaseStatusAllow.length === 0 && releaseStatusDeny.length === 0) {
         return list;
     }
-    const releaseFiltered = list.map(x => {
-        return {
+    const releaseFiltered = list.map(x => ({
             ...x,
             releases: x.releases === undefined ? [] : x.releases.filter(y => {
                 if(releaseStatusAllow.length > 0) {
@@ -760,8 +760,7 @@ export const filterByValidReleaseStatus = <T extends IRecordingMatch[]>(list: T,
                 }
                  return !releaseStatusDeny.includes(y.status?.toLocaleLowerCase() as MBReleaseStatus)
             })
-        }
-    });
+        }));
     return releaseFiltered.filter(x => (
         (list.find(y => y.id === x.id).releases ?? []).length === 0 
         && releaseAllowEmpty
@@ -778,8 +777,7 @@ export const filterByValidReleaseGroupPrimary = <T extends IRecordingMatch[]>(li
     if(releaseGroupPrimaryTypeAllow.length === 0 && releaseGroupPrimaryTypeAllow.length === 0) {
         return list;
     }
-    const releaseFiltered = list.map(x => {
-        return {
+    const releaseFiltered = list.map(x => ({
             ...x,
             releases: x.releases === undefined ? [] : x.releases.filter(y => {
                 if(releaseGroupPrimaryTypeAllow.length > 0) {
@@ -787,8 +785,7 @@ export const filterByValidReleaseGroupPrimary = <T extends IRecordingMatch[]>(li
                 }
                  return !releaseGroupPrimaryTypeDeny.includes(y["release-group"]?.["primary-type"]?.toLocaleLowerCase() as MBReleaseGroupPrimaryType)
             })
-        }
-    });
+        }));
     return releaseFiltered.filter(x => (
         (list.find(y => y.id === x.id).releases ?? []).length === 0 
         && releaseAllowEmpty
@@ -805,8 +802,7 @@ export const filterByValidReleaseGroupSecondary = (list: IRecordingMatch[], stag
     if(releaseGroupSecondaryTypeAllow.length === 0 && releaseGroupSecondaryTypeDeny.length === 0) {
         return list;
     }
-    const releaseFiltered = list.map(x => {
-        return {
+    const releaseFiltered = list.map(x => ({
             ...x,
             releases: x.releases === undefined ? [] : x.releases.filter(y => {
                 if(releaseGroupSecondaryTypeAllow.length > 0) {
@@ -814,8 +810,7 @@ export const filterByValidReleaseGroupSecondary = (list: IRecordingMatch[], stag
                 }
                   return intersect(releaseGroupSecondaryTypeDeny, (y["release-group"]?.["secondary-types"] ?? []).map(x => x.toLocaleLowerCase()) as MBReleaseGroupSecondaryType[]).length === 0;
             })
-        }
-    });
+        }));
     return releaseFiltered.filter(x => (
         (list.find(y => y.id === x.id).releases ?? []).length === 0 
         && releaseAllowEmpty
@@ -832,8 +827,7 @@ export const filterByValidReleaseCountry = (list: IRecordingMatch[], stageConfig
     if(releaseCountryAllow.length === 0 && releaseCountryDeny.length === 0) {
         return list;
     }
-    const releaseFiltered = list.map(x => {
-        return {
+    const releaseFiltered = list.map(x => ({
             ...x,
             releases: x.releases === undefined ? [] : x.releases.filter(y => {
                 if(releaseCountryAllow.length > 0) {
@@ -841,8 +835,7 @@ export const filterByValidReleaseCountry = (list: IRecordingMatch[], stageConfig
                 }
                  return !releaseCountryDeny.includes(y.country?.toLocaleLowerCase())
             })
-        }
-    });
+        }));
     return releaseFiltered.filter(x => (
         (list.find(y => y.id === x.id).releases ?? []).length === 0 
         && releaseAllowEmpty
@@ -861,8 +854,7 @@ export const filterByExplicitTrackMbid = (list: IRecordingMatch[], play: PlayObj
             break;
         }
         for (const rel of rec.releases) {
-            // @ts-ignore
-            if (rel.media.some(x => x.track.some(y => y.id === play.data.meta.brainz.track))) {
+            if (rel.media.some(x => x.tracks.some(y => y.id === play.data.meta.brainz.track))) {
                 releaseMatchId = rel.id;
                 recMatch = rec;
                 break;
@@ -908,11 +900,17 @@ export const rankReleasesByPriority = (list: IRecordingMatch[], stageConfig: Mus
         releaseGroupPrimaryTypePriority = [],
         releaseGroupSecondaryTypePriority = [],
         releaseCountryPriority = [],
-        albumWeight,
-        titleWeight,
-        artistWeight
+        albumWeight = 0,
+        titleWeight = 0,
+        artistWeight = 0
     } = stageConfig;
 
+    // reverse order so that "highest" priority (first in user list) ends up with the highest index, that we use as score
+    const groupPrimaryPriority = [...releaseGroupPrimaryTypePriority].reverse(),
+    groupSecPriority = [...releaseGroupSecondaryTypePriority].reverse(),
+    countryPriority = [...releaseCountryPriority].reverse(),
+    statusPriority = [...releaseStatusPriority].reverse();
+
     const cList = clone(list) as RecordingRankedMatched[];
     const rankedList = cList.map((x) => {
         let artistScore = 0;
@@ -921,10 +919,10 @@ export const rankReleasesByPriority = (list: IRecordingMatch[], stageConfig: Mus
             artistScore = artistRes[0] * (artistWeight + (artistRes[1] > 0 ? 0.05 : 0));
         }
         const releases = (x.releases ?? []).map((a) => {
-            const statAScore = releaseStatusPriority.findIndex(x => a.status !== undefined && x === a.status.toLocaleLowerCase()) + 1;
-            const grpPAScore = releaseGroupPrimaryTypePriority.findIndex(x => x === a["release-group"]?.["primary-type"]?.toLocaleLowerCase()) + 1;
-            const grpSAScore = (a["release-group"]?.["secondary-types"] ?? []).reduce((acc: number, curr: MBReleaseGroupSecondaryType) => acc + releaseGroupSecondaryTypePriority.findIndex(x => x === (curr as MBReleaseGroupSecondaryType).toLocaleLowerCase()) + 1,0);
-            const countryAScore = releaseCountryPriority.findIndex(x => a.country === undefined ? false : x === a.country.toLocaleLowerCase()) + 1;
+            const statAScore = statusPriority.findIndex(x => a.status !== undefined && x === a.status.toLocaleLowerCase()) + 1;
+            const grpPAScore = groupPrimaryPriority.findIndex(x => x === a["release-group"]?.["primary-type"]?.toLocaleLowerCase()) + 1;
+            const grpSAScore = (a["release-group"]?.["secondary-types"] ?? []).reduce((acc: number, curr: MBReleaseGroupSecondaryType) => acc + groupSecPriority.findIndex(x => x === (curr as MBReleaseGroupSecondaryType).toLocaleLowerCase()) + 1,0);
+            const countryAScore = countryPriority.findIndex(x => a.country === undefined ? false : x === a.country.toLocaleLowerCase()) + 1;
             const compareScore = scoreNormalizedStringsWeighted(play.data.album, a.title, albumWeight, albumWeight !== 0 ? 0.05 : 0);
             return {
                 ...a,

src/backend/common/transforms/NativeTransformer.ts

@@ -1,16 +1,16 @@
-import { ArtistCredit, PlayObject, TransformerCommon } from "../../../core/Atomic.js";
-import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js";
-import { WebhookPayload } from "../infrastructure/config/health/webhooks.js";
-import { ExternalMetadataTerm, PlayTransformNativeStage, StageConfig } from "../infrastructure/Transform.js";
-import AtomicPartsTransformer from "./AtomicPartsTransformer.js";
-import { parseArtistCredits, parseTrackCredits, uniqueNormalizedStrArr } from "../../utils/StringUtils.js";
+import type {ArtistCredit, PlayObject, TransformerCommon} from "../../../core/Atomic.ts";
+import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.ts";
+import type {WebhookPayload} from "../infrastructure/config/health/webhooks.ts";
+import type {ExternalMetadataTerm, PlayTransformNativeStage} from "../../../core/Transform.ts";
+import AtomicPartsTransformer from "./AtomicPartsTransformer.ts";
+import { parseArtistCredits, parseTrackCredits, uniqueNormalizedStrArr } from "../../utils/StringUtils.ts";
 import { parseRegexSingle, parseToRegexOrLiteralSearch } from "@foxxmd/regex-buddy-core";
-import { TransformerOptions } from "./AbstractTransformer.js";
-import { DELIMITERS_NO_AMP } from "../infrastructure/Atomic.js";
-import { asArray } from "../../utils/DataUtils.js";
-import { MaybeLogger } from '../MaybeLogger.js';
+import type {TransformerOptions} from "./AbstractTransformer.ts";
+import { DELIMITERS_NO_AMP } from '../../../core/Atomic.ts';
+import { asArray } from "../../utils/DataUtils.ts";
+import { MaybeLogger } from '../MaybeLogger.ts';
 import { childLogger } from "@foxxmd/logging";
-import { artistCreditToName, artistNameToCredit } from "../../../core/StringUtils.js";
+import { artistCreditToName, artistNameToCredit } from "../../../core/StringUtils.ts";
 
 export type ArtistParseSource = 'artists' | 'title'
 

src/backend/common/transforms/TransformerManager.ts

@@ -1,16 +1,16 @@
-import { childLogger, Logger } from "@foxxmd/logging";
-import AbstractTransformer from "./AbstractTransformer.js";
-import { TransformerCommonConfig } from "../../../core/Atomic.js";
-import UserTransformer from "./UserTransformer.js";
-import { StageConfig } from "../infrastructure/Transform.js";
-import { PlayObject } from "../../../core/Atomic.js";
-import { isStageTyped } from "../../utils/PlayTransformUtils.js";
-import { MSCache } from "../Cache.js";
-import NativeTransformer from "./NativeTransformer.js";
-import MusicbrainzTransformer, { configFromEnv, MusicbrainzTransformerConfig } from "./MusicbrainzTransformer.js";
+import { childLogger, type Logger } from "@foxxmd/logging";
+import type AbstractTransformer from "./AbstractTransformer.ts";
+import type {TransformerCommonConfig} from "../../../core/Atomic.ts";
+import UserTransformer from "./UserTransformer.ts";
+import type {StageConfig} from "../../../core/Transform.ts";
+import type {PlayObject} from "../../../core/Atomic.ts";
+import { isStageTyped } from "../../utils/PlayTransformUtils.ts";
+import type { MSCache } from "../Cache.ts";
+import NativeTransformer from "./NativeTransformer.ts";
+import MusicbrainzTransformer, { configFromEnv, type MusicbrainzTransformerConfig } from "./MusicbrainzTransformer.ts";
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { nanoid } from "nanoid";
-import { SimpleError, StageTransformError } from "../errors/MSErrors.js";
+import { SimpleError, StageTransformError } from "../errors/MSErrors.ts";
 
 export default class TransformerManager {