fix(rocksky): Fix missing albumArtist field #632

When in core its being pulled into vite bundler and causing build issues. It's only needed by backend anyways so move to backend-scope.

.dockerignore

@@ -31,4 +31,5 @@ tmp-*
 .env
 *.env
 .storybook
-.stories
+.stories
+**/lexicons/**/*.json

.github/workflows/testAndSanity.yml

@@ -60,7 +60,7 @@ jobs:
         run: |
           set +e
           export NODE_ENV=production
-          timeout --preserve-status 10s node node_modules/.bin/tsx src/backend/index.ts
+          timeout --preserve-status 10s node src/backend/index.ts
           exitcode="$?"
           if [[ "$exitcode" -eq 143 ]] || [[ "$exitcode" -eq 137 ]]; then
             echo "App stayed up long enough and exited with expected status"

.gitignore

@@ -153,8 +153,10 @@ tmp-*
 storybook-static
 
 *.db
-*.db.*
-*.db-*lib
-*.db
-*.db.*
-*.db-*lib
+*.db*
+
+**/lexicons/**/*.json
+
+bun.lock
+.DS_Store
+**/.DS_Store

.mocharc.json

@@ -1,7 +1,6 @@
 {
   "reporter": "dot",
   "extension": "ts",
-  "import": "tsx/esm",
   "spec": "./src/backend/tests/**/*.test.ts",
   "file": [
     "./src/backend/tests/setup.ts"

.storybook/main.ts

@@ -12,12 +12,14 @@ export default defineMain({
 //    "@storybook/addon-vitest",
     "@storybook/addon-a11y",
     "@storybook/addon-docs",
+    'storybook-addon-remix-react-router'
 //    "@storybook/addon-onboarding"
   ],
   features: {
     // https://github.com/storybookjs/storybook/discussions/33279
     sidebarOnboardingChecklist: false,
   },
+   staticDirs: ['../public'],
   "framework": "@storybook/react-vite",
     viteFinal: async (config) => {
     return mergeConfig(config, {

.storybook/manager.ts

@@ -2,5 +2,5 @@ import { addons } from 'storybook/manager-api';
 import { themes } from 'storybook/theming';
  
 addons.setConfig({
-  theme: themes.dark,
+  theme: themes.normal,
 });

.storybook/preview.ts

@@ -3,8 +3,17 @@ import addonA11y from "@storybook/addon-a11y";
 import { definePreview } from "@storybook/react-vite";
 
 import { themes } from 'storybook/theming';
+import { initialize, mswLoader, getWorker } from 'msw-storybook-addon';
+
+// https://github.com/mswjs/msw-storybook-addon/issues/82#issuecomment-3894302575
+try {
+  getWorker();
+} catch (_) {
+  initialize();
+}
 
 export default definePreview({
+  loaders: [mswLoader],
   parameters: {
     controls: {
       matchers: {
@@ -20,7 +29,7 @@ export default definePreview({
       test: 'todo'
     },
     docs: {
-      theme: themes.dark,
+      theme: themes.normal,
     },
   },
 

.vscode/launch.json

@@ -20,7 +20,7 @@
             "request": "launch",
             // Debug app in VSCode
             "program": "${workspaceFolder}/src/backend/index.ts",
-            "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/tsx",
+            "runtimeExecutable": "node",
             "console": "integratedTerminal",
             "internalConsoleOptions": "neverOpen",
             "skipFiles": [
@@ -35,7 +35,7 @@
             "request": "launch",
             // Debug app in VSCode
             "program": "${workspaceFolder}/src/backend/index.ts",
-            "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/tsx",
+            "runtimeExecutable": "node",
             "console": "integratedTerminal",
             "envFile": "${workspaceFolder}/.env",
             "env": {
@@ -48,28 +48,13 @@
                 "${workspaceFolder}/node_modules/**",
             ],
         },
-        {
-            "name": "tsx",
-            "type": "node",
-            "request": "launch",
-            // Debug current file in VSCode
-            "program": "${file}",
-            "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/tsx",
-            "console": "integratedTerminal",
-            "internalConsoleOptions": "neverOpen",
-            "skipFiles": [
-                "<node_internals>/**",
-                // Ignore all dependencies (optional)
-                "${workspaceFolder}/node_modules/**",
-            ],
-        },
         {
             "name": "schema",
             "type": "node",
             "request": "launch",
             // Debug app in VSCode
             "program": "${workspaceFolder}/src/backend/utils/SchemaStaticUtil.ts",
-            "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/tsx",
+            "runtimeExecutable": "node",
             "console": "integratedTerminal",
             "internalConsoleOptions": "neverOpen",
             "skipFiles": [
@@ -96,7 +81,7 @@
                 "--recursive",
                 "${workspaceFolder}/src/backend/tests/**/*.test.ts"
             ],
-            "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/tsx",
+            "runtimeExecutable": "node",
             "internalConsoleOptions": "openOnSessionStart",
             "name": "Mocha Tests",
             "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha",
@@ -120,7 +105,7 @@
                 "--config", "${workspaceRoot}/.mocharc.json",
                 "${file}"
             ],
-            "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/tsx",
+            "runtimeExecutable": "node",
             "internalConsoleOptions": "openOnSessionStart",
             "name": "Mocha Test on File",
             "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha",

.vscode/settings.json

@@ -1,5 +1,4 @@
 {
-    "mochaExplorer.require": "tsx/esm",
     "mochaExplorer.timeout": 1200000,
     "mochaExplorer.exit": true,
     "search.exclude": {
@@ -11,7 +10,9 @@
         "docsite/**"
     ],
     "files.associations": {
-        "*.css": "tailwindcss"
+        "*.css": "tailwindcss",
+        "**/config/*.json": "jsonc"
     },
-    "tailwindCSS.experimental.configFile": "src/client/index.css"
+    "tailwindCSS.experimental.configFile": "src/client/index.css",
+    
 }

Dockerfile

@@ -56,6 +56,7 @@ RUN npm install -g concurrently
 ARG data_dir=/config
 VOLUME $data_dir
 ENV CONFIG_DIR=$data_dir
+ENV DATA_DIR=$data_dir
 
 COPY docker/root /
 
@@ -118,6 +119,8 @@ RUN npm ci --omit=dev --no-audit \
     && npm uninstall ts-json-schema-generator \
     && npm cache clean --force \
     && chown -R abc:abc node_modules \
+    ## peerDependency of validbot and can(?) be removed for production
+    && rm -r node_modules/typescript \
     && npx @usex/prune-mod -w \
     && rm -rf /root/.cache
 

README.md

@@ -12,6 +12,7 @@ alt="multi-scrobbler logo" width="180" height="180">
 A dockerized app that monitors your music listening activity from *everywhere* and scrobbles it *anywhere*.
 
 * Supports scrobbling from many [**Sources**](https://docs.multi-scrobbler.app/configuration/sources)
+    * [Apple Music](https://docs.multi-scrobbler.app/configuration/sources/applemusic/?configType=file)
     * [Azuracast](https://docs.multi-scrobbler.app/configuration/sources/azuracast)
     * [Deezer](https://docs.multi-scrobbler.app/configuration/sources/deezer)
     * [Google Cast (Chromecast)](https://docs.multi-scrobbler.app/configuration/sources/google-cast)

config/applemusic.json.example

@@ -0,0 +1,29 @@
+[
+  {
+    "id": "myAppleMusic",
+    "name": "My Apple Music",
+    "enable": true,
+    "clients": [],
+    "data": {
+      // either key or token needs to be provided
+      "key": {
+        "id": "2HPSNJZ88N",
+        "teamId": "SN6YASW8G4",
+        "p8": "-----BEGIN PRIVATE KEY-----\nMIGTAgEAMBMGByqGSM49AgEG...-----END PRIVATE KEY-----"
+      },
+      "token": "your-jwt-token-here",
+      "mediaUserToken": "your-media-user-token-here",
+
+      // optional: Origin header for API requests (required when using a browser token)
+      "origin": "https://music.apple.com",
+
+      // optional: polling interval in seconds (default 60)
+      "interval": 60
+    },
+    "options": {
+      "logDiff": true,
+      "recoverUnchangedTopHistory": true,
+      "normalizeAlbum": true
+    }
+  }
+]

config/rocksky.json.example

@@ -4,7 +4,7 @@
     "name": "FoxxMD Rocksky Client",
     "configureAs": "client",
     "data": {
-      "key": "7cdr86vis3mpq5b1pi01md0hkm7ykzzxjy81r",
+      "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkaWQiOiJkaWQ6cGxjOng0NjVwZmtyeW5zaDZ...",
       "handle": "alice.bsky.social"
     }
   },
@@ -13,7 +13,7 @@
     "name": "FoxxMD Rocksky Source",
     "configureAs": "source",
     "data": {
-      "key": "e4xwktm7jxm607x8el1fcda30eu14dzb64h3j",
+      "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkaWQiOiJkaWQ6cGxjOng0NjVwZmtyeW5zaDZ...",
       "handle": "alice.bsky.social"
     }
   }

docker/root/etc/s6-overlay/s6-rc.d/init-ms-config/run

@@ -1,33 +1,19 @@
 #!/usr/bin/with-contenv bash
 
-# used https://github.com/linuxserver/docker-plex as a template
-
-POPULATE_EXAMPLES=false
-
 echo "-------------------------------------"
-echo -e "Setting up multi-scrobbler config directory based on CONFIG_DIR env: ${CONFIG_DIR}\n"
+echo -e "Verifying multi-scrobbler directory permissions based on CONFIG_DIR env: ${CONFIG_DIR} and DATA_DIR env: ${DATA_DIR}\n"
 
-# make config folder if it does not exist
-if [ ! -d "${CONFIG_DIR}" ]; then
-  echo "Directory does not exist! Creating..."
-  POPULATE_EXAMPLES=true
-  mkdir -p "${CONFIG_DIR}"
-else
-  if [ "$(ls -A ${CONFIG_DIR})" ]; then
-       echo "Directory is not empty, not creating examples."
-    else
-      POPULATE_EXAMPLES=true
-    fi
+if [ -d "${CONFIG_DIR}" ]; then
+  echo "chown'ing config directory to ensure correct permissions."
+  chown -R abc:abc "${CONFIG_DIR}"
+  echo "Done!"
 fi
 
-# add example configs
-if [ "$POPULATE_EXAMPLES" = true ]; then
-  echo "Directory is empty, adding examples..."
-  cp -r /app/config/. "${CONFIG_DIR}"/
+if [ -d "${DATA_DIR}" ]; then
+  echo "chown'ing data directory to ensure correct permissions."
+  chown -R abc:abc "${DATA_DIR}"
+  echo "Done!"
 fi
 
-# permissions
-echo "chown'ing directory to ensure correct permissions."
-chown -R abc:abc "${CONFIG_DIR}"
 echo "Done!"
 echo -e "-------------------------------------\n"

docker/root/etc/s6-overlay/s6-rc.d/svc-node/run

@@ -7,4 +7,4 @@ echo -e "\nmulti-scrobbler is starting!"
 export NODE_ENV=production
 cd /app || exit
 exec \
-	  s6-setuidgid abc /usr/bin/node $NODE_ARGS /app/node_modules/.bin/tsx /app/src/backend/index.ts
+	  s6-setuidgid abc /usr/bin/node $NODE_ARGS /app/src/backend/index.ts

docsite/docs/configuration/clients/librefm.mdx

@@ -15,9 +15,9 @@ import JsonConfig from '!!raw-loader!@site/../config/librefm.json.example';
 
 :::
 
-To use the official [Libre.fm](https://libre.fm/) instance you will need an account. Or, you can run your own Libre.fm instance [using the legacy source code](https://github.com/foocorp/gnu-fm).
+To use the official [Libre.fm](https://libre.fm/) website you will need an account. Or, you can run your own GNU FM instance [using the source code](https://github.com/foocorp/gnu-fm).
 
-No API account, [API Key, or Secret](https://github.com/libre-fm/developer/wiki/Libre.fm-fundamentals#api-keys) is needed to use the Libre.fm API. Instead, use **any values** for API Key/Secret in the configuration. Authentication is done by allowing access from your account on redirect.
+No API account, [API Key, or Secret](https://github.com/libre-fm/developer/wiki/Libre.fm-fundamentals#api-keys) is currently needed to use the Libre.fm API. Instead, use **any values** for API Key/Secret in the configuration. Authentication is done by allowing access from your account on redirect.
 
 ### Callback/Redirect URL
 
@@ -64,4 +64,4 @@ https://my.cool.libre.com/2.0/
         | `LIBREFM_SESSION`      | No        |                                          | Session id. Will be generated by authentication flow if not provided.                            |
         | `LIBREFM_URLBASE`      | No        |                                          | Use for custom Libre.fm instance only. The host + path prefix EX `https://my.libre.example/2.0/` |
         | `LIBREFM_NAME`         | No        |                                          | A vanity name different than ID                                                                  |
-</Config>
+</Config>

docsite/docs/configuration/clients/rocksky.mdx

@@ -13,7 +13,7 @@ To monitor and re-scrobble activity **from** Rocksky create a [Rocksky (Source)]
 
 :::
 
-[**Rocksky**](https://rocksky.app/) is music tracking and discovery platform built on the AT Protocol (Bluesky).
+[**Rocksky**](https://rocksky.app/) is music tracking and discovery platform built on the AT Protocol and your your [**Atmosphere Account**](https://atmosphereaccount.com/).
 
 <details>
 
@@ -23,13 +23,13 @@ To monitor and re-scrobble activity **from** Rocksky create a [Rocksky (Source)]
 
 </details>
 
-<DetailsAdmo type="warning" summary="Your Data Is Public">
+<DetailsAdmo type="important" summary="Your Data Is Public">
 
 The default Bluesky PDS is **public**, meaning your scrobbles are also public (read-only). This isn't any different than using Last.fm or Listenbrainz, in terms of privacy.
 
 </DetailsAdmo>
 
-<DetailsAdmo type="warning" summary="No Scrobbling Guarantee">
+<DetailsAdmo type="important" summary="No Scrobbling Guarantee">
 
 Rocksky **does not guarantee** that every scrobble you send will be persisted. Rocksky's design goal is to store scrobbles with **metdata-complete, 3rd party verifiable track information** so that it can be a metadata-rich discovery platform, rather than an exhaustive scrobble data source-of-truth.
 
@@ -41,22 +41,16 @@ If you need your scrobbles to be 100% captured then you should scrobble to an *a
 
 ## Setup
 
-You will need a [Bluesky](https://bsky.app/) account to use Rocksky.
+You will need an [**Atmosphere Account**](https://atmosphereaccount.com/) to use Rocksky.
 
-:::note 
-
-Usage is adapted from [Rocksky docs](https://docs.rocksky.app/migrating-to-rocksky-scrobble-api-957839m0) on [docs.rocksky.app](https://docs.rocksky.app)
-
-:::
-
-* Navigate to [rocksky.app](https://rocksky.app/) and sign up/logn using your Bluesky account
-* Then, navigate to the [RockSky developer dashboard](https://rocksky.app/apikeys) and obtain an API Key
+* Navigate to [rocksky.app](https://rocksky.app/) and sign up/logn using your Atmosphere Account
+* Then, navigate to the [Access Tokens](https://rocksky.app/access-tokens) in the Rocksky settings dropdown, and create a new token
 
 ## Configuration
 
 ### Handle
 
-The handle used with multi-scrobbler should be your **full** ATProto handle, including TLD.
+The handle used with multi-scrobbler should be your **full** [**Atmosphere Account**](https://atmosphereaccount.com/) handle, including TLD.
 
 * For regular Bluesky account this will be like: `alice.bsky.social`
 * For Bluesky accounts that [use their website as their account](https://bsky.social/about/blog/4-28-2023-domain-handle-tutorial) this is your domain: `mydomain.com`
@@ -64,11 +58,28 @@ The handle used with multi-scrobbler should be your **full** ATProto handle, inc
 
 If you do not include a TLD then multi-scrobbler will automatically append `.bsky.social` to your handle value.
 
+<DetailsAdmo type="warning" summary="API Applications Deprecated">
+
+In < 0.14.1, Multi-Scrobber used Rocksky's API Appication "API Key" for communication with a Listenbrainz-compatible API from Rocksky.
+
+Going forward, this should be replaced with the official guidance in the [Setup](#setup) section above: use an **Access Token**. This uses the native Rocksky API which has more functionality.
+
+If your configuration is using either
+
+* (File/AIO) `'key': ...`
+* (ENV) `ROCKSKY_KEY`
+
+you should redo the [Setup](#setup) section and modify your configuration to use an Access Token instead.
+
+Keys are now **deprecated** and functionality for them will be removed in a future version of Multi-Scrobbler. 
+
+</DetailsAdmo>
+
 <Config config="RockSkyClientConfig" fileContent={JsonConfig} client name="rocksky">
         | Environmental Variable | Required? | Default | Description                                                                         |
         | :--------------------- | --------- | ------- | ----------------------------------------------------------------------------------- |
-        | `ROCKSKY_ID`           | Yes       |         | A unique ID                                                                         |
+        | `ROCKSKY_ID`           | Yes       |         | A unique ID used in MS                                                              |
         | `ROCKSKY_HANDLE`       | Yes       |         | Your **full** ATProto handle. For Bluesky account this is like `myUser.bsky.social` |
-        | `ROCKSKY_KEY`          | Yes       |         | API Key obtained from Rocksky developer dashboard                                   |
+        | `ROCKSKY_TOKEN`        | Yes       |         | Access Token obtained from Rocksky                                                  |
         | `ROCKSKY_NAME`         | No        |         | A vanity name different than ID                                                     |
 </Config>

docsite/docs/configuration/clients/tealfm.mdx

@@ -16,7 +16,7 @@ To monitor and re-scrobble activity from a ATProto Profile create a [teal.fm (So
 
 :::
 
-[**teal.fm**](https://teal.fm/) is a social discovery and web viewing service for scrobbles stored/available on the ATProto network/your repository in a [PDS (Personal Data Server)](https://github.com/Bluesky-social/pds).
+[**teal.fm**](https://teal.fm/) is a social discovery and browsing app for scrobbles stored in your [**Atmosphere Account**](https://atmosphereaccount.com/).
 
 <details>
 
@@ -46,6 +46,13 @@ To view your teal.fm scrobble data you can:
 
 ## Setup
 
+
+:::note
+
+Multi-scrobbler will work with *any* Atmosphere Account, not just Bluesky. See [Configuration](#configuration) below for provider-agnostic config.
+
+:::
+
 #### Bluesky
 
 * Create a [Bluesky](https://bsky.app) account, if you don't already have one
@@ -57,10 +64,10 @@ To view your teal.fm scrobble data you can:
 
 ### Identifier
 
-The ATPRoto identifier used with multi-scrobbler should be either:
+The ATProto identifier used with multi-scrobbler should be either:
 
 * A valid [DID](https://atproto.com/specs/did#at-protocol-did-identifier-syntax), starting with `did:plc:...` or `did:web:...`
-* Your **full** ATProto handle, including TLD
+* Your **full** [Atmosphere Account](https://atmosphereaccount.com/) ATProto handle, including TLD
 
 If using a handle:
 

docsite/docs/configuration/sources/applemusic.mdx

@@ -0,0 +1,262 @@
+---
+title: Apple Music
+toc_min_heading_level: 2
+toc_max_heading_level: 5
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import CodeBlock from '@theme/CodeBlock';
+import AppleMusicConfig from '!!raw-loader!@site/../config/applemusic.json.example';
+
+This Source **monitors your Apple Music listening history** via the official Apple Music API and scrobbles new activity to your configured [Clients](/configuration/clients). 
+
+Because of how the Apple Music API works, Multi-Scrobbler (MS) requires two different tokens to function: a **Media User Token** (identifies your personal account) and an **Authentication Token** (authorizes API access).
+
+### 1. Getting a Media User Token
+
+The `mediaUserToken` is required to access your recently played history. You can extract it directly from the Apple Music web app:
+
+1. Visit [music.apple.com](https://music.apple.com) in your browser and log in.
+2. Open your browser's Developer Tools (usually `F12` or <kbd>⌘</kbd> + <kbd>⌥</kbd> + <kbd>I</kbd>).
+3. Navigate to the **Application** tab (Chrome/Edge) or **Storage** tab (Safari/Firefox).
+4. Expand **Cookies** in the left sidebar and select `https://music.apple.com`.
+5. Find the cookie named `media-user-token` and copy its value (it typically starts with `0.`).
+
+:::caution[Token Expiry]
+The `mediaUserToken` will eventually expire. If Multi-Scrobbler begins throwing authentication errors in the logs, simply repeat these steps to obtain and configure a fresh token.
+:::
+
+### 2. Authentication
+
+To authorize API access, Multi-Scrobbler requires an Authentication Token (JWT). Because the official Apple Music API is designed for developers, there are two ways to provide this token depending on your situation:
+
+* **Browser Token (Free & Most Common):** Best for 99% of users. You can easily extract a temporary token from the Apple Music web player without needing an Apple Developer account.
+* **MusicKit Key (Requires Paid Apple Developer Account):** If you happen to be enrolled in the paid Apple Developer Program ($99/year), you can provide a private key to automatically generate permanent tokens.
+
+Choose your preferred method below:
+
+<Tabs groupId="appleMusicAuth" queryString>
+    <TabItem value="token" label="Browser Token (Free)">
+
+        Since most users don't have a paid developer account, extracting the token from the web player is the standard approach.
+
+        Behind the scenes, the Apple Music website makes a lot of background requests, which can make finding the right token confusing. To filter out the noise and grab the exact token we need, follow the steps for your browser below:
+
+        <details>
+        <summary>Chrome, Edge & Firefox</summary>
+
+        1. Visit [music.apple.com](https://music.apple.com) and log in.
+        2. Open Developer Tools (`F12` or `Ctrl+Shift+I` / <kbd>⌘</kbd> + <kbd>⌥</kbd> + <kbd>I</kbd>) and navigate to the **Network** tab.
+        3. In the filter box at the top left, paste exactly: `https://amp-api.music.apple.com/v1/me/account` (**Label 1**).
+        4. Click the **Fetch/XHR** filter button to hide irrelevant requests (**Label 2**).
+        5. Reload the page (**Label 3**).
+        6. Under the "Name" column, click the request starting with `account?meta=` (**Label 4**).
+        7. In the panel that opens, scroll down to the **Request Headers** section.
+        8. Find the `authorization` header. Copy the long string of text **after** the `Bearer ` prefix (**Label 5**).
+
+        ![Chrome Network Tab Instructions](../applemusic_chrome_devtools.png)
+
+        </details>
+
+        <details>
+        <summary>Safari</summary>
+
+        Before you can extract the token in Safari, you must enable developer tools.
+
+        {/* This weird formatting is because ⌘ is usually incredibly tiny */}
+        1. Open Safari Settings (<kbd>⌘</kbd> + <kbd>,</kbd>), navigate to the **Advanced** tab, and check **"Show features for web developers"** at the very bottom.
+        
+        ![Safari Advanced Settings](../applemusic_safari_settings.png)
+        
+        2. Visit [music.apple.com](https://music.apple.com) and log in.
+        3. Open the Web Inspector (<kbd>⌘</kbd>+<kbd>⌥</kbd>+<kbd>I</kbd> or Develop > Show Web Inspector) and navigate to the **Network** tab.
+        4. In the filter bar on the left side of the inspector, paste exactly: `https://amp-api.music.apple.com/v1/me/account` (**Label 1**).
+        5. Reload the page (**Label 2**).
+        6. Click the `account` request that appears in the Name list (**Label 3**).
+        7. In the details sidebar, look under the **Request** section for the `Authorization` header. Copy the long string of text **after** the `Bearer ` prefix (**Label 4**).
+
+        ![Safari Network Tab Instructions](../applemusic_safari_devtools.png)
+
+        </details>
+        
+        <br />
+
+        Once you have your token, add it to your configuration file along with the origin header:
+
+        ```json title="applemusic.json"
+        {
+            "data": {
+                "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsIm...",
+                "mediaUserToken": "your-media-user-token-here",
+                "origin": "https://music.apple.com"
+            }
+        }
+        ```
+
+        :::info[Origin Header Required]
+        When using a JWT extracted from the browser, Apple requires the request to match the domain it was issued to. You **must** include the `origin` field in your config as shown above.
+        :::
+
+        *Note: Browser-generated JWTs are valid for a maximum of 35 days and must be updated manually when they expire.*
+
+    </TabItem>
+    <TabItem value="key" label="MusicKit Key (Paid Account)">
+
+        :::warning[Apple Developer Account Required]
+        To generate JWTs automatically, you must be enrolled in the paid [Apple Developer Program](https://developer.apple.com/programs/) ($99/year) and create a MusicKit key. If you don't have an account, use the **Browser Token (Free)** tab instead.
+        :::
+
+        To generate JWTs automatically and avoid manual token refreshes, you need an Apple Music API key:
+
+        1. Go to the [Apple Developer portal](https://developer.apple.com/account/resources/authkeys/list) and create a **MusicKit** key.
+        2. Download the `.p8` file — this is your private key.
+        3. Note your **Key ID** and **Team ID** (found in your Apple Developer account Membership details).
+
+        Add these details to your config:
+
+        ```json title="applemusic.json"
+        {
+            "data": {
+                "key": {
+                    "id": "2HPSNJZ88N",
+                    "teamId": "SN6YASW8G4",
+                    "p8": "-----BEGIN PRIVATE KEY-----\nMIGTAgEAMBMGByqGSM49AgEG...-----END PRIVATE KEY-----"
+                },
+                "mediaUserToken": "your-media-user-token-here"
+            }
+        }
+        ```
+
+        :::tip
+        When pasting the contents of your `.p8` file into JSON, make sure to replace physical line breaks with `\n` so it remains a valid, single-line JSON string.
+        :::
+
+    </TabItem>
+</Tabs>
+
+---
+
+### How Multi-Scrobbler handles Apple Music quirks
+
+#### Timestamp Estimation
+
+The Apple Music API **does not provide timestamps** for when tracks were played. Multi-Scrobbler estimates play times by taking the current time and subtracting track durations backwards:
+* The most recent track is assumed to have finished playing **now**.
+* Each older track is estimated to have played `duration` seconds before the previous one.
+
+*For the most accurate scrobble timestamps, it is highly recommended to keep the polling interval (`APPLEMUSIC_INTERVAL`) low (the default is 60 seconds).*
+
+#### Duplicate Play Recovery
+
+The Apple Music history API strictly deduplicates tracks. Instead of showing the same song multiple times in a row, it simply bumps the re-played track back to the #1 spot and removes the older entry. 
+
+For example, if you listen to **Song A** → **Song B**, your history is `[Song B, Song A]`. If you then listen to **Song A** again, the API returns `[Song A, Song B]` (Song A was bumped to the top). 
+
+If Multi-Scrobbler polled when the top track was Song A, and polls again after it's bumped back to the top, the top track appears functionally unchanged. This usually confuses standard scrobblers into ignoring the update entirely. By default, MS correctly detects this "top-rebound" pattern, extracts the intermediate tracks that were squeezed in (Song B), and scrobbles Song A again as a re-listen. 
+
+*Note: Because of this strict deduplication, **"pure loops"** (listening to Song A continuously on repeat without playing anything else in between) cannot be tracked. The history list never changes shape, so MS has no way of knowing it was played again.*
+
+If you ever experience false positives (tracks being scrobbled that you didn't actually re-listen to), you can disable this recovery behavior with `"recoverUnchangedTopHistory": false` in your config options or by setting the `APPLEMUSIC_RECOVER_UNCHANGED_TOP_HISTORY=false` env var.
+
+#### Album name normalization
+
+The Apple Music API appends ` - EP` or ` - Single` to the album name for EPs and singles. These suffixes are *not* part of the official names of the album and can cause issues when trying to scrobble or match metadata.
+
+By default, Multi-Scrobbler strips these suffixes out so only the official name is used.
+
+If you do not want this behavior it can be disabled with ENV `APPLEMUSIC_NORMALIZE_ALBUM=false` or File/AIO option `"normalizeAlbum": false`.
+
+<DetailsAdmo type="tip" summary="Stripping Suffix with User Stage">
+
+If you still want this stripping functionality but with more control over how it is applied you can use a [User Stage](/configuration/transforms/user).
+
+Here is an example of creating a User Stage to only strip suffixes for a specific scrobble client:
+
+<details>
+
+<summary>Example</summary>
+
+```json5 title="config.json"
+{
+  // ...
+  "transformers": [
+    {
+      "type": "user",
+      "name": "AppleAlbumStip",
+      "defaults": {
+        "album": [
+          "/ - (EP|Single)$/i"
+        ]
+      }
+    }
+  ]
+}
+```
+
+```json5 title="applemusic.json"
+{
+  {
+    "id": "myAppleMusic",
+    "name": "My Apple Music",
+    "data": {
+      // ...
+    },
+    // highlight-start
+    "options": {
+      "normalizeAlbum": false
+    }
+    // highlight-end
+  }
+}
+```
+
+```json5 title="lastfm.json"
+[
+  {
+    "name": "Foxx LFM Client",
+    "id": "myLastFmClient",
+    "enable": true,
+    "configureAs": "client",
+    "data": {
+      "apiKey": "a89cba1569901a0671d5a9875fed4be1",
+      "secret": "ec42e09d5ae0ee0f0816ca151008412a",
+      "redirectUri": "http://localhost:9078/lastfm/callback"
+    },
+    // highlight-start
+    "options": {
+      "playTransform": {
+        "preCompare": [
+          {
+            "type": "user",
+            "name": "AppleAlbumStip",
+          }
+        ]
+      }
+    }
+    // highlight-end
+  }
+]
+```
+
+</details>
+
+</DetailsAdmo>
+
+## Configuration Reference
+
+<Config config="AppleMusicSourceConfig" fileContent={AppleMusicConfig} name="applemusic">
+    | Environment Variable                       | Required? | Default | Description                                                                                                                                                                                                                                                                                                                           |
+    | ------------------------------------------ | --------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+    | `APPLEMUSIC_ID`                            | Yes       |         | A unique ID for this source.                                                                                                                                                                                                                                                                                                          |
+    | `APPLEMUSIC_MEDIA_USER_TOKEN`              | Yes       |         | The media-user-token extracted from the browser.                                                                                                                                                                                                                                                                                      |
+    | `APPLEMUSIC_KEY_ID`                        | No        |         | Key ID from your MusicKit key.                                                                                                                                                                                                                                                                                                        |
+    | `APPLEMUSIC_TEAM_ID`                       | No        |         | Team ID from your Apple Developer account.                                                                                                                                                                                                                                                                                            |
+    | `APPLEMUSIC_KEY_P8`                        | No        |         | The contents of your MusicKit `.p8` private key file.                                                                                                                                                                                                                                                                                 |
+    | `APPLEMUSIC_TOKEN`                         | No        |         | The authentication JWT extracted from the browser.                                                                                                                                                                                                                                                                                    |
+    | `APPLEMUSIC_INTERVAL`                      | No        | `60`    | Polling interval in seconds.                                                                                                                                                                                                                                                                                                          |
+    | `APPLEMUSIC_ORIGIN_HEADER`                 | No        |         | Origin header for API requests, e.g. `https://music.apple.com`. Required when using a browser token.                                                                                                                                                                                                                                  |
+    | `APPLEMUSIC_RECOVER_UNCHANGED_TOP_HISTORY` | No        | `true`  | Apple Music deduplicates its history by bumping re-played tracks to the top. If you listen to Song A → Song B → Song A, the history changes from `[A]` to `[A, B]` (bumping A). MS detects this "top-rebound" to recover the intermediate play (B) and the re-listen (A). Disable this only if you see incorrect duplicate scrobbles. |
+    | `APPLEMUSIC_NORMALIZE_ALBUM`               | No        | `true`  | Strips extraneous `- EP` and `- Single` suffixes from album names                                                                                                                                                                                                                                                                                                                                      |
+    | `APPLEMUSIC_NAME`                          | No        |         | A vanity name different than the ID.                                                                                                                                                                                                                                                                                                  |
+</Config>

docsite/docs/configuration/sources/azuracast.mdx

@@ -16,9 +16,9 @@ Azuracast may not correctly report track length or position. If this is the case
 
 :::
 
-The Azuracast server should have **Use High-Performance Now Playing Updates** enabled in _Administration -> System Settings_
+The Azuracast server should have **Use [High-Performance Now Playing Updates](https://www.azuracast.com/docs/developers/now-playing-data/#high-performance-updates)** enabled in _Administration -> System Settings_
 
-#### URL
+### URL
 
 The URL used by MS to connect to Azuracast has the syntax:
 
@@ -47,6 +47,68 @@ If either is configured then MS will automatically scrobble based on when these
 
 If neither is configured automatic scrobble defaults to **off** and you should either manually initiate it or explicitly set the default behavior.
 
+### Station
+
+#### Station Identifier
+
+The station identifier multi-scrobbler uses (ENV `AZURA_STATION` or `station` in File/AIO config) needs to be the **URL Stub** or **shortcode** of the Azuracast station, **not** its display name. Use of the methods below to find an apprioriate station identifier.
+
+<Tabs groupId="stationId" queryString>
+<TabItem value="stub" label="URL Stub">
+The **URL Stub** is the value used in the URL to access the station EX
+
+```
+https://azura.mydomain.com/public/my-cool-station
+                                  ^^^^ url stub ^^
+```
+
+Use `my-cool-station` as the station identifier.
+</TabItem>
+<TabItem value="shortcode" label="Shortcode">
+The **shortcode** can be found in the raw JSON response for your station. Navigate to this URL in your browser:
+
+```
+https://azura.mydomain.com/api/stations
+```
+
+Find your station in the returned data (by `name`) and then locate the value of the `shortcode` property in that object.
+
+<details>
+
+<summary>Example</summary>
+
+```json
+[
+    {
+        "id": 1,
+        "name": "My Cool Station",
+        "shortcode": "my-cool-station",
+        "description": "Re-broadcasts of my cool music",
+        "frontend": "icecast",
+        "backend": "liquidsoap",
+        ...
+```
+
+Use `my-cool-station` as the station identifier.
+</details>
+</TabItem>
+</Tabs>
+
+#### Station Access
+
+Multi-Scrobbler does not use authentication to communicate with Azuracast. Your station will likely need some form of **Public Pages** in order for you to monitor it.
+
+In your station's **Edit Station Settings** -> **Public Pages** make sure that at least one of these is enabled:
+
+* Enable Public Pages
+* Enable Public APIs
+
+:::note
+
+For users who aren't administrators of Azuracast: if you can access the station using `https://azura.mydomain.com/public/xxxxxx` then it is public.
+
+:::
+
 ## Configuration
 
 <Config config="AzuracastSourceConfig" fileContent={AzuracastConfig} name="azuracast">
@@ -54,7 +116,7 @@ If neither is configured automatic scrobble defaults to **off** and you should e
         | :--------------------- | :-------- | :------ | ---------------------------------------------------------------------------------------------- |
         | `AZURA_ID`             | Yes       |         | A unique ID                                                                                    |
         | `AZURA_URL`            | Yes       |         | Azuracast URL *without station name*                                                           |
-        | `AZURA_STATION`        | Yes       |         | The station name shown on the public page                                                      |
+        | `AZURA_STATION`        | Yes       |         | The station URL Stub or shortcode                                                              |
         | `AZURA_LIVE`           | No        | Yes     | Only scrobble when station status is ONLINE                                                    |
         | `AZURA_LISTENERS_NUM`  | No        | `true`  | Only scrobble if station has any listeners (`true`) or listeners are equal-to/greater-than `X` |
         | `AZURA_NAME`           | No        |         | A vanity name different than ID                                                                |

docsite/docs/configuration/sources/jriver.mdx

@@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem';
 import CodeBlock from '@theme/CodeBlock';
 import JriverConfig from '!!raw-loader!@site/../config/jriver.json.example';
 
-In order for multi-scrobbler to communicate with JRiver you must have [Web Server Interface](https://wiki.jriver.com/index.php/Web_Service_Interface#Documentation_of_Functions) enabled. This can can be in the JRiver GUI:
+In order for multi-scrobbler to communicate with JRiver you must have [Web Server Interface](https://wiki.jriver.com/index.php/Web_Service_Interface#Documentation_of_Functions) enabled. This can be found in the JRiver GUI:
 
 * Tools -> Options -> Media Network
   * Check `Use Media Network to share this library...`

docsite/docs/configuration/sources/sources.mdx

@@ -15,6 +15,7 @@ A **Source** is a data source that contains information about tracks you are pla
 
 | Name                                                                    | Networking                                               | Scrobble SOT                                        | [Multi Device/User](#multi-deviceuser) | [Scrobble Destination](#limiting-scrobble-destination) | [Thresholds](#scrobble-thresholds) | [Should Scrobble](#should-scrobble-behavior) |
 | :---------------------------------------------------------------------- | :------------------------------------------------------- | :-------------------------------------------------- | :------------------------------------- | :----------------------------------------------------- | :--------------------------------- | :------------------------------------------- |
+| [Apple Music](/configuration/sources/applemusic)                       | [Active](./?sourceComm=active#by-communication-method)   | [History](./?sot=history#by-data-source-of-truth) | ❌                                     | ✅                                                     | ❌                                 | ❌                                           |
 | [Azuracast](/configuration/sources/azuracast)                           | [Active](./?sourceComm=active#by-communication-method)   | [Activity](./?sot=activity#by-data-source-of-truth) | ❌                                     | ✅                                                     | ✅                                 | ✅                                           |
 | [Deezer](/configuration/sources/deezer)                                 | [Active](./?sourceComm=active#by-communication-method)   | [History](./?sot=history#by-data-source-of-truth)   | ✅                                     | ✅                                                     | ❌                                 | ❌                                           |
 | [Google Cast (Chromecast)](/configuration/sources/google-cast)          | [Active](./?sourceComm=active#by-communication-method)   | [Activity](./?sot=activity#by-data-source-of-truth) | ✅                                     | ✅                                                     | ✅                                 | ❌                                           |

docsite/docs/configuration/transforms/transforms.mdx

@@ -593,39 +593,6 @@ MS uses [caching](/configuration/#caching) to reduce the number of API calls nee
 
 **If you are using any Transform stages you should configure [secondary caching](/configuration#secondary-caching)** to increase the cache size and lifetime of cached items. This will also reduce memory usage in MS.
 
-<details>
-
-<summary>Example</summary>
-
-Add valkey service to your [multi-scrobbler docker compose stack](/installation?runType=docker-compose#docker) and configure MS to use it for metadata:
-
-```yaml title="docker-compose.yml"
-services:
-  multi-scrobbler:
-    image: foxxmd/multi-scrobbler
-    # ...
-    environment:
-      # ...
-      // highlight-start
-      - CACHE_METADATA=valkey
-      - CACHE_METADATA_CONN=redis://valkey:6379
-      // highlight-end
-    # ...
-
-  // highlight-start
-  valkey:
-    image: valkey/valkey
-    volumes:
-      - valkeydata:/data
-
-volumes:
-    valkeydata:
-      driver: local
-  // highlight-end
-```
-
-</details>
-
 ## Examples
 
 See **Examples** sections in specific Stage docs (also in the sidebar):

docsite/docs/index.mdx

@@ -3,6 +3,8 @@ slug: /
 title: Overview
 sidebar_position: 1
 ---
+import Carousel from '@site/src/components/Carousel/NextCarousel';
+import BrowserOnly from '@docusaurus/BrowserOnly';
 
 ![Latest Release](https://img.shields.io/github/v/release/foxxmd/multi-scrobbler)
 ![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)
@@ -13,6 +15,7 @@ sidebar_position: 1
 A dockerized app that monitors your music listening activity from *everywhere* and scrobbles it *anywhere*.
 
 * Supports monitoring activity from many [**Sources**](/configuration/sources)
+    * [Apple Music](/configuration/sources/applemusic)
     * [Azuracast](/configuration/sources/azuracast)
     * [Deezer](/configuration/sources/deezer)
     * [Google Cast (Chromecast)](/configuration/sources/google-cast)
@@ -64,6 +67,18 @@ A dockerized app that monitors your music listening activity from *everywhere* a
 
 [**Quick Start Guide**](/quickstart)
 
+<DetailsAdmo type="tip" summary="New UI Preview">
+
+A **full rewrite** of the UI is now available for preview. The new UI modernizes Multi-Scrobbler and gives you in depth insight into how your scrobbles are handled end-to-end.
+
+<Carousel/>
+
+To try out the stable-release preview navigate to `http://yourMSDomain:9078/next`
+
+Or to try out the **latest preview snapshot** see the docker image pinned [in this comment.](https://github.com/FoxxMD/multi-scrobbler/issues/500#issuecomment-4917525794)
+
+</DetailsAdmo>
+
 <img src={require('/img/status-ui.png').default} width="800"/>
 
 **Why should I use this over a browser extension and/or mobile app scrobbler?**

docsite/docs/installation/installation.mdx

@@ -42,7 +42,7 @@ Cross-platform images are built for x86 (Intel/AMD) and ARM64 (IE Raspberry Pi)
 
 <Tabs groupId="dockerSetting" queryString>
 <TabItem value="storage" label="Storage">
-You **should** bind a host directory into the container for storing configurations and credentials. Otherwise, these will be lost when the container is updated.
+You **should** bind a host directory into the container for storing configurations, credentials, database, and other files. Otherwise, these will be lost when the container is updated.
 
 ```yaml title="docker-compose.yml"
 services:
@@ -54,6 +54,24 @@ services:
   // highlight-end
 ```
 
+Optionally, use a separate directory for data (database, cache, etc...) by mounting an additional directory and setting the `DATA_DIR` ENV to the directory that should be used inside the container:
+
+```yaml title="docker-compose.yml"
+services:
+  multi-scrobbler:
+  # ...
+  environment:
+    # ...
+    // highlight-start
+    - DATA_DIR=/msData
+    // highlight-end
+  volumes:
+    - "./config:/config" # used for config files
+  // highlight-start
+    - "./my/host/folder:/msData" # used for data files
+  // highlight-end
+```
+
 </TabItem>
 <TabItem value="networking" label="Networking">
 If you are using a [bridge network](https://www.appsdeveloperblog.com/docker-networking-bridging-host-and-overlay/) (default docker setup) you must map a port to the container in order to access the dashboard and use MS with some sources (Webscrobbler, LFM/LZ Endpoints). The default container port is `9078`.
@@ -231,6 +249,33 @@ npm run docs:install && npm run build
 npm run start
 ```
 
+#### Config And Data Directories
+
+The directories MS uses for **Configuration** (`config.json`, `jellyfin.json`, etc...) and **Data** (database, cache, etc...) can be specified by environmental variables passed directly to `npm`/`node`, or set in an `.env` file in the working directory.
+
+When parsing these ENVs MS will resolve relative directories but *not* bash expansions like `~` for home. Examples:
+
+* `CONFIG_DIR=/my/absolute/path` => sets directory to use for config files
+* `DATA_DIR=./relative/data` => sets directory relative to working directory for data files
+
+If you do not set these environmental variables then MS will attempt to use OS-standardized directories:
+
+<Tabs>
+    <TabItem value="linux" label="Linux">
+        * Config => `$XDG_CONFIG_HOME/multi-scrobbler` or `~/.config/multi-scrobbler`
+        * Data => `$XDG_DATA_HOME/multi-scrobbler` or `~/.local/share/multi-scrobbler`
+    </TabItem>
+    <TabItem value="mac" label="MacOS">
+        * Config => `~/Library/Preferences/multi-scrobbler`
+        * Data => `~/Library/Application Support/multi-scrobbler`
+    </TabItem>
+    <TabItem value="win" label="Windows">
+        * Config => `%APPDATA%\multi-scrobbler\Config`
+        * Data => `%LOCALAPPDATA%\multi-scrobbler\Data`
+    </TabItem>
+</Tabs>
+
+
 #### Rollup build error
 
 During building if you encounter an error like: `Your current platform "XXX" and architecture "XXX" combination is not yet supported by the native Rollup build.`

docsite/docs/updating/upgrade-path/0140.mdx

@@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem';
 import CodeBlock from '@theme/CodeBlock';
 import RequiredUpgrade from "@site/src/components/snippets/_upgrade-required.mdx"
 
-<RequiredUpgrade before='< 0.14.0' current='0.14.0' old='0.13.2' />
+{/* <RequiredUpgrade before='< 0.14.0' current='0.14.0' old='0.13.2' /> */}
 
 ## Source/Client IDs
 
@@ -75,32 +75,34 @@ Add an `id` to the top-level for each Source/Client configuration, next to `data
 
 Add an `id` to the top-level for each Source/Client configuration, next to `data`:
 
-```json title="koito.json"
-[
-  {
-    "name": "koito-source",
-    "configureAs": "source",
-    "type": "koito",
-    // highlight-start
-    "id": "myKoitoID",
-    // highlight-end    
-    "data": {
-      "token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b",
-      "username": "admin",
-      "url": "http://192.168.0.100:4110" 
+```json title="config.json"
+{
+  "sources": [
+    {
+      "name": "koito-source",
+      "configureAs": "source",
+      "type": "koito",
+      // highlight-start
+      "id": "myKoitoID",
+      // highlight-end    
+      "data": {
+        "token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b",
+        "username": "admin",
+        "url": "http://192.168.0.100:4110" 
+      }
     }
-  }
-]
+  ]
+}
 ```
 
 </TabItem>
 </Tabs>
 
-:::note[Default ID]
+:::important[Default ID]
 
-If you do not add an ID then Multi-Scrobbler will automatically use the **name** of the Source/Client as the ID. The name is shown in the Dashboard.
+**If you do not add an ID then Multi-Scrobbler will automatically use the name of the Source/Client as the ID.** The name is shown in the Dashboard.
 
-If you decide to add an ID later you must use the name as the ID in order to keep Plays/Scobbles associated with the same config.
+If you decide to add an ID later, you must use the Source/Client name as the ID in order to keep Plays/Scobbles associated with the same config.
 
 :::
 
@@ -108,13 +110,19 @@ If you decide to add an ID later you must use the name as the ID in order to kee
 
 :::tip
 
-Before upgrading, if your MS dashboard shows 0 queued/failed for all [Scrobble Clients](/configuration/clients) then you can **ignore this step.**
+Before upgrading, if your MS dashboard shows 0 queued/failed for all [Scrobble Clients](/configuration/clients) then you can **ignore this section.**
+
+:::
+
+:::important
+
+The cached scrobbles for a Client/Source were based on its **name**. If you plan on changing the name of a Client/Source with this upgrade you should do it **after** this migration has run or else MS will not be able to find any associated cached scrobbles. 
 
 :::
 
 MS `0.14.0` introduces a database for persisting Plays/Scrobbles, queues, and other data. Prior to `0.14.0`, queued/failed scrobbles were stored in a cache **file** inside your [`CONFIG_DIR`](/installation/?dockerSetting=storage#recommended-settings) named `ms-scrobble.cache`.
 
-When a [Client](/configuration/clients) is created and **enabled** in `0.14.0` it will attempt to automatically migrate scrobbles from this cache into the database.
+When a [Client](/configuration/clients) is created and **enabled** in `0.14.0` it will attempt to **automatically migrate scrobbles** from this cache into the database.
 
 * You can see progress/results for this process by filtering logs for `[Cached Scrobble Migration]`
 * If the process fails MS will attempt to migrate failed scrobbles, again, the next time it is started
@@ -140,7 +148,7 @@ In `0.14.0` [Cache](/configuration#caching) has been simplified with much of the
 
 This only applies to users who have `cache` in their [AIO Config](/configuration/?configType=aio#configuration-types) `config.json` or are using `CACHE_*` [ENV Config](/configuration/?configType=env#configuration-types)
 
-If you do not have any of the above [Cache](/configuration#caching) configuration defined then you can **ignore this step.** 
+If you do not have any of the above [Cache](/configuration#caching) configuration defined then you can **ignore this section.** 
 
 :::
 
@@ -210,9 +218,41 @@ Remove the `connection` property from the `auth` object. If using `file` the ent
 </TabItem>
 </Tabs>
 
-### Renamed ENV Keys
+## Renamed ENV Keys
 
 In an effort to standard ENV prefixes some ENV keys have been renamed:
 
 * `LZENDPOINT_ENABLE` => `LZE_ENABLE`
-* `LFMENDPOINT_ENABLE` => `LFM_ENABLE`
+* `LFMENDPOINT_ENABLE` => `LFM_ENABLE`
+
+## (Optional) ENV Client/Source Names
+
+In the MS Dashboard, prior to 0.14.0, UI Client/Sources configured with [ENV Config](/configuration/?configType=env#configuration-types) always had generated names like `unnamed-koito`.
+
+ENV Config Clients/Sources can now be configured with a name by using the `_NAME` suffix, example:
+
+```yaml title="compose.yaml"
+services:
+  multi-scrobbler:
+  # ...
+  environment:
+    - KOTIO_TOKEN=...
+    # ...
+    - KOITO_ID=myKoitoID
+  // highlight-start
+    - KOITO_NAME=Koito A
+  // highlight-end
+
+    - JELLYFIN_URL=...
+    # ..
+    - JELLYFIN_ID=myJellyfinID
+  // highlight-start
+    - JELLYFIN_NAME=Foxx Jelly
+  // highlight-end    
+```
+
+:::important
+
+If you have **cached scrobbles** make sure to upgrade and run [Cached Scrobble Migration](#cached-scrobble-migration) **before** adding names to Client/Sources.
+
+:::

docsite/package-lock.json

@@ -22,6 +22,8 @@
         "clsx": "^2.0.0",
         "docusaurus-plugin-sass": "^0.2.6",
         "docusaurus-theme-github-codeblock": "^2.0.2",
+        "embla-carousel": "^9.0.0-rc02",
+        "embla-carousel-react": "^9.0.0-rc02",
         "json5": "^2.2.3",
         "micromark-extension-directive": "^3.0.1",
         "modern-react-json-editor": "^1.0.2",
@@ -10863,6 +10865,34 @@
       "integrity": "sha512-/4t+AS1l4S3ZC0Ja7PHFIWeBIxGA3QGqV8/yKsP36v7NcyUCl+bIcmw6s5zVuMIECWwBrAK/6QLzTmbJChBboQ==",
       "license": "ISC"
     },
+    "node_modules/embla-carousel": {
+      "version": "9.0.0-rc02",
+      "resolved": "https://registry.npmjs.org/embla-carousel/-/embla-carousel-9.0.0-rc02.tgz",
+      "integrity": "sha512-GOQBX6v5cNk/ptfuHUoHhW9rvl5KeSGsBoppaUqtihu2kDmQAyzvnH/95r3TaSScFSJTj68V9AM6E5nq3jUVcQ==",
+      "license": "MIT"
+    },
+    "node_modules/embla-carousel-react": {
+      "version": "9.0.0-rc02",
+      "resolved": "https://registry.npmjs.org/embla-carousel-react/-/embla-carousel-react-9.0.0-rc02.tgz",
+      "integrity": "sha512-DpcD0kTv6L31KvBcrx5d57E6T2rP/YceTGAdq7llkpKFdfM1of8YotHVEQTNIU3I9o5mjE/QxLi1ndJmm1DikA==",
+      "license": "MIT",
+      "dependencies": {
+        "embla-carousel": "9.0.0-rc02",
+        "embla-carousel-reactive-utils": "9.0.0-rc02"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.1 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
+      }
+    },
+    "node_modules/embla-carousel-reactive-utils": {
+      "version": "9.0.0-rc02",
+      "resolved": "https://registry.npmjs.org/embla-carousel-reactive-utils/-/embla-carousel-reactive-utils-9.0.0-rc02.tgz",
+      "integrity": "sha512-H1c+yUbcVST9eJoGU+05L1vSDyBs0hf/Gf1OadHYIJ3asBfyvn6oaGCs2wXq1D8/f09y0wwCxeRxT9UZU/TxJA==",
+      "license": "MIT",
+      "peerDependencies": {
+        "embla-carousel": "9.0.0-rc02"
+      }
+    },
     "node_modules/emoji-regex": {
       "version": "9.2.2",
       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",

docsite/package.json

@@ -30,6 +30,8 @@
     "clsx": "^2.0.0",
     "docusaurus-plugin-sass": "^0.2.6",
     "docusaurus-theme-github-codeblock": "^2.0.2",
+    "embla-carousel": "^9.0.0-rc02",
+    "embla-carousel-react": "^9.0.0-rc02",
     "json5": "^2.2.3",
     "micromark-extension-directive": "^3.0.1",
     "modern-react-json-editor": "^1.0.2",

docsite/src/components/Carousel/Arrows.tsx

@@ -0,0 +1,96 @@
+import React, {
+  ComponentPropsWithRef,
+  useCallback,
+  useEffect,
+  useState
+} from 'react'
+import { EmblaCarouselType } from 'embla-carousel'
+
+type UsePrevNextButtonsType = {
+  prevBtnDisabled: boolean
+  nextBtnDisabled: boolean
+  onPrevButtonClick: () => void
+  onNextButtonClick: () => void
+}
+
+export const usePrevNextButtons = (
+  emblaApi: EmblaCarouselType | undefined
+): UsePrevNextButtonsType => {
+  const [prevBtnDisabled, setPrevBtnDisabled] = useState(true)
+  const [nextBtnDisabled, setNextBtnDisabled] = useState(true)
+
+  const onPrevButtonClick = useCallback(() => {
+    if (!emblaApi) return
+    emblaApi.goToPrev()
+  }, [emblaApi])
+
+  const onNextButtonClick = useCallback(() => {
+    if (!emblaApi) return
+    emblaApi.goToNext()
+  }, [emblaApi])
+
+  const onSelect = useCallback((emblaApi: EmblaCarouselType) => {
+    setPrevBtnDisabled(!emblaApi.canGoToPrev())
+    setNextBtnDisabled(!emblaApi.canGoToNext())
+  }, [])
+
+  useEffect(() => {
+    if (!emblaApi) return
+
+    onSelect(emblaApi)
+    emblaApi.on('reinit', onSelect).on('select', onSelect)
+  }, [emblaApi, onSelect])
+
+  return {
+    prevBtnDisabled,
+    nextBtnDisabled,
+    onPrevButtonClick,
+    onNextButtonClick
+  }
+}
+
+type PropType = ComponentPropsWithRef<'button'>
+
+export const PrevButton = (props: PropType) => {
+  const { children, disabled, ...restProps } = props
+
+  return (
+    <button
+      className={'embla__button embla__button--prev'.concat(
+        disabled ? ' embla__button--disabled' : ''
+      )}
+      type="button"
+      {...restProps}
+    >
+      <svg className="embla__button__svg" viewBox="0 0 532 532">
+        <path
+          fill="currentColor"
+          d="M355.66 11.354c13.793-13.805 36.208-13.805 50.001 0 13.785 13.804 13.785 36.238 0 50.034L201.22 266l204.442 204.61c13.785 13.805 13.785 36.239 0 50.044-13.793 13.796-36.208 13.796-50.002 0a5994246.277 5994246.277 0 0 0-229.332-229.454 35.065 35.065 0 0 1-10.326-25.126c0-9.2 3.393-18.26 10.326-25.2C172.192 194.973 332.731 34.31 355.66 11.354Z"
+        />
+      </svg>
+      {children}
+    </button>
+  )
+}
+
+export const NextButton = (props: PropType) => {
+  const { children, disabled, ...restProps } = props
+
+  return (
+    <button
+      className={'embla__button embla__button--next'.concat(
+        disabled ? ' embla__button--disabled' : ''
+      )}
+      type="button"
+      {...restProps}
+    >
+      <svg className="embla__button__svg" viewBox="0 0 532 532">
+        <path
+          fill="currentColor"
+          d="M176.34 520.646c-13.793 13.805-36.208 13.805-50.001 0-13.785-13.804-13.785-36.238 0-50.034L330.78 266 126.34 61.391c-13.785-13.805-13.785-36.239 0-50.044 13.793-13.796 36.208-13.796 50.002 0 22.928 22.947 206.395 206.507 229.332 229.454a35.065 35.065 0 0 1 10.326 25.126c0 9.2-3.393 18.26-10.326 25.2-45.865 45.901-206.404 206.564-229.332 229.52Z"
+        />
+      </svg>
+      {children}
+    </button>
+  )
+}

docsite/src/components/Carousel/NextCarousel.scss

@@ -0,0 +1,158 @@
+:root[data-theme="light"] {
+  --text-body: rgb(54, 49, 61);
+  --text-comment: rgb(99, 94, 105);
+  --text-high-contrast: rgb(49, 49, 49);
+  --text-medium-contrast: rgb(99, 94, 105);
+  --text-low-contrast: rgb(116, 109, 118);
+  --detail-high-contrast: rgb(192, 192, 192);
+  --detail-medium-contrast: rgb(234, 234, 234);
+  --detail-low-contrast: rgb(240, 240, 242);
+  --text-body-rgb-value: 54, 49, 61;
+  --text-comment-rgb-value: 99, 94, 105;
+  --text-high-contrast-rgb-value: 49, 49, 49;
+  --text-medium-contrast-rgb-value: 99, 94, 105;
+  --text-low-contrast-rgb-value: 116, 109, 118;
+  --detail-high-contrast-rgb-value: 192, 192, 192;
+  --detail-medium-contrast-rgb-value: 234, 234, 234;
+  --detail-low-contrast-rgb-value: 240, 240, 242;
+}
+:root[data-theme="dark"] {
+  --text-body: rgb(222, 222, 222);
+  --text-comment: rgb(170, 170, 170);
+  --text-high-contrast: rgb(230, 230, 230);
+  --text-medium-contrast: rgb(202, 202, 202);
+  --text-low-contrast: rgb(170, 170, 170);
+  --detail-high-contrast: rgb(101, 101, 101);
+  --detail-medium-contrast: rgb(25, 25, 25);
+  --detail-low-contrast: rgb(21, 21, 21);
+  --text-body-rgb-value: 222, 222, 222;
+  --text-comment-rgb-value: 170, 170, 170;
+  --text-high-contrast-rgb-value: 230, 230, 230;
+  --text-medium-contrast-rgb-value: 202, 202, 202;
+  --text-low-contrast-rgb-value: 170, 170, 170;
+  --detail-high-contrast-rgb-value: 101, 101, 101;
+  --detail-medium-contrast-rgb-value: 25, 25, 25;
+  --detail-low-contrast-rgb-value: 21, 21, 21;
+}
+
+.embla {
+  max-width: 48rem;
+  margin: auto;
+
+  --slide-height: 33rem;
+  --slide-spacing: 1rem;
+  --slide-size: 100%;
+  margin-bottom: 1rem;
+}
+
+.embla__viewport {
+  overflow: hidden;
+}
+
+.embla__container {
+  display: flex;
+  touch-action: pan-y pinch-zoom;
+  margin-left: calc(var(--slide-spacing) * -1);
+}
+
+.embla__slide {
+  transform: translate3d(0, 0, 0);
+  flex: 0 0 var(--slide-size);
+  min-width: 0;
+  padding-left: var(--slide-spacing); 
+
+}
+
+.embla__slide__container {
+  border: 0.15rem solid transparent;
+  border-radius: 1.2rem;
+  display: flex;
+  align-items: normal;
+  justify-content: center;
+  height: var(--slide-height);
+  user-select: none;
+  overflow:hidden;
+  flex-wrap: wrap;
+}
+
+.imageCaption {
+  font-style: italic;
+  font-size: 0.8em;
+  text-wrap-style: balance;
+  text-align: center;
+}
+
+.embla__controls {
+  display: grid;
+  grid-template-columns: auto 1fr;
+  justify-content: space-between;
+  gap: 1.2rem;
+  // margin-top: 1.8rem;
+}
+
+.embla__buttons {
+  display: grid;
+  grid-template-columns: repeat(2, 1fr);
+  gap: 0.6rem;
+  align-items: center;
+}
+
+.embla__button {
+  -webkit-tap-highlight-color: rgba(var(--text-high-contrast-rgb-value), 0.5);
+  -webkit-appearance: none;
+  appearance: none;
+  background-color: transparent;
+  touch-action: manipulation;
+  display: inline-flex;
+  text-decoration: none;
+  cursor: pointer;
+  border: 0;
+  padding: 0;
+  margin: 0;
+  border: 0.2rem solid var(--detail-medium-contrast);
+  width: 2.5rem;
+  height: 2.5rem;
+
+  z-index: 1;
+  border-radius: 50%;
+  color: var(--text-body);
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  transform: rotate(0deg);
+}
+
+.embla__button--disabled {
+  color: var(--detail-high-contrast);
+}
+
+.embla__button__svg {
+  width: 35%;
+  height: 35%;
+}
+
+.embla__selected-snap-display {
+  justify-self: flex-end;
+  align-self: center;
+  color: var(--text-low-contrast);
+  font-weight: 600;
+}
+
+.embla__slide:nth-child(1) {
+  flex: 0 0 90%;
+}
+.embla__slide:nth-child(2) {
+  flex: 0 0 60%;
+}
+.embla__slide:nth-child(3) {
+  flex: 0 0 75%;
+}
+.embla__slide:nth-child(4) {
+  flex: 0 0 80%;
+}
+.embla__slide:nth-child(5) {
+  flex: 0 0 100%;
+}
+.embla__slide:nth-child(6) {
+  flex: 0 0 100%;
+}

docsite/src/components/Carousel/NextCarousel.tsx

@@ -0,0 +1,100 @@
+import React, {useEffect} from 'react'
+import BrowserOnly from "@docusaurus/BrowserOnly"
+import './NextCarousel.scss';
+import { EmblaOptionsType } from 'embla-carousel'
+import useEmblaCarousel from 'embla-carousel-react'
+import {
+  NextButton,
+  PrevButton,
+  usePrevNextButtons
+} from './Arrows'
+import {
+  SelectedSnapDisplay,
+  useSelectedSnapDisplay
+} from './SnapDisplay'
+import ComponentList from '@site/static/img/next/ms-componentlist.png';
+import ComponentListDesktop from '@site/static/img/next/ms-componentlistdesktop.png';
+import ComponentDetailed from '@site/static/img/next/ms-componentdetailed.png';
+import PlaysList from '@site/static/img/next/ms-playlist.jpg';
+import Timeline from '@site/static/img/next/ms-mbemptyquery.png';
+import Logs from '@site/static/img/next/ms-logs.jpg';
+
+const EmblaCarousel = (props: {options?: EmblaOptionsType}) => {
+  const { options } = props
+  const [emblaRef, emblaApi] = useEmblaCarousel(options)
+
+  const {
+    prevBtnDisabled,
+    nextBtnDisabled,
+    onPrevButtonClick,
+    onNextButtonClick
+  } = usePrevNextButtons(emblaApi)
+
+  const { selectedSnap, snapCount } = useSelectedSnapDisplay(emblaApi)
+
+  return (
+    <div className="embla">
+      <div className="embla__viewport" ref={emblaRef}>
+        <div className="embla__container">
+          <div className="embla__slide" key="1">
+            <div className="embla__slide__container">
+              <img src={ComponentListDesktop} />
+              <div className="imageCaption">Source/Client List on desktop</div>
+              </div>
+          </div>
+          <div className="embla__slide" key="2">
+            <div className="embla__slide__container">
+              <img src={ComponentList} />
+              <div className="imageCaption">Source/Client List on mobile</div>
+              </div>
+          </div>
+          <div className="embla__slide" key="3">
+            <div className="embla__slide__container">
+              <img src={ComponentDetailed} />
+              <div className="imageCaption">Source/Client Details</div>
+              </div>
+          </div>
+          <div className="embla__slide" key="4">
+            <div className="embla__slide__container">
+              <img src={PlaysList} />
+              <div className="imageCaption">Search for Plays/Scrobbles with live updates</div>
+              </div>
+          </div>
+          <div className="embla__slide" key="5">
+            <div className="embla__slide__container">
+              <img src={Timeline} />
+              <div className="imageCaption">Scrobble timeline audit trail with detailed errors</div>
+              </div>
+          </div>
+          <div className="embla__slide" key="6">
+            <div className="embla__slide__container">
+              <img src={Logs} />
+              <div className="imageCaption">Floating logs for dev-level troubleshooting</div>
+              </div>
+          </div>
+        </div>
+      </div>
+
+      <div className="embla__controls">
+        <div className="embla__buttons">
+          <PrevButton onClick={onPrevButtonClick} disabled={prevBtnDisabled} />
+          <NextButton onClick={onNextButtonClick} disabled={nextBtnDisabled} />
+        </div>
+        <SelectedSnapDisplay
+          selectedSnap={selectedSnap}
+          snapCount={snapCount}
+        />
+      </div>
+    </div>
+  )
+}
+
+const ClientCarousel = () => {
+  return (
+    <BrowserOnly>{() => {
+      return <EmblaCarousel/>
+    }}</BrowserOnly>
+  )
+}
+
+export default ClientCarousel;

docsite/src/components/Carousel/SnapDisplay.tsx

@@ -0,0 +1,47 @@
+import React, { useCallback, useEffect, useState } from 'react'
+import { EmblaCarouselType } from 'embla-carousel'
+
+type UseSelectedSnapDisplayType = {
+  selectedSnap: number
+  snapCount: number
+}
+
+export const useSelectedSnapDisplay = (
+  emblaApi: EmblaCarouselType | undefined
+): UseSelectedSnapDisplayType => {
+  const [selectedSnap, setSelectedSnap] = useState(0)
+  const [snapCount, setSnapCount] = useState(0)
+
+  const updateScrollSnapState = useCallback((emblaApi: EmblaCarouselType) => {
+    setSnapCount(emblaApi.snapList().length)
+    setSelectedSnap(emblaApi.selectedSnap())
+  }, [])
+
+  useEffect(() => {
+    if (!emblaApi) return
+
+    updateScrollSnapState(emblaApi)
+    emblaApi.on('select', updateScrollSnapState)
+    emblaApi.on('reinit', updateScrollSnapState)
+  }, [emblaApi, updateScrollSnapState])
+
+  return {
+    selectedSnap,
+    snapCount
+  }
+}
+
+type PropType = {
+  selectedSnap: number
+  snapCount: number
+}
+
+export const SelectedSnapDisplay = (props: PropType) => {
+  const { selectedSnap, snapCount } = props
+
+  return (
+    <div className="embla__selected-snap-display">
+      {selectedSnap + 1} / {snapCount}
+    </div>
+  )
+}

docsite/src/components/snippets/_bluesky.mdx

@@ -1,14 +1,14 @@
 import CodeInline from '@theme/CodeInline';
 
-The [AT Protocol](https://docs.bsky.app/docs/advanced-guides/atproto) is a standard to allow different applications to share and interpret your social data.
+Your [**Atmosphere Account**](https://atmosphereaccount.com) consists of social data (it's just JSON) stored in a personal [repository](https://atproto.com/guides/data-repos) on a [PDS (Personal Data Server)](https://github.com/Bluesky-social/pds), all of which is standardized under the [**AT Protocol**](https://atproto.com).
 
-Your data is stored in a personal [repository](https://atproto.com/guides/data-repos) in a [PDS (Personal Data Server)](https://github.com/Bluesky-social/pds), which is data-type agnostic (it's just JSON). Each application defines their own datatypes and pulls data from your repository to render to you, your network, etc...
+The AT Protocol defines how your data is stored, how you authenticate to write to it, and how indpendent apps share and consume it. Critically, each layer (data storage, app, communication) of the AT Protocol can be operated by *indpendent* entities so your data and experience not locked behind a walled garden.
 
-Even though the application and the PDS/repository are independent, all of this is done for you automatically when you create a [Bluesky](https://bsky.social/about) account.
+Although each layer can be implemented separately (you can self-host everything!), it is normally a *simple* process for an end-user to create and start using an Atmosphere Account as many apps can do everything for you. The most common example being a [Bluesky](https://bsky.social/about), but there are other [public account providers](https://atmosphereaccount.com/explore).
 
-So...
+When Multi-Scrobbler uses the AT Protocol...
 
-* multi-scrobbler "scrobbles" your plays to your repository
+* multi-scrobbler "scrobbles" (writes) your plays to your repository
   * in a JSON data structure which includes [{props.name}'s data type and structure](https://atproto.com/specs/lexicon)
   * at a [known location for {props.name}'s data](https://atproto.com/guides/glossary#collection) (<CodeInline>{props.lexicon}</CodeInline>), in a list that is appended to as you continue to scrobble
-* when accessing {props.name}, or another application that knows how to read it, your data is pulled from the repository to render the app
+* when accessing {props.name}, or another application that knows how to read it, your data is pulled from the repository to render their app

docsite/src/css/custom.css

@@ -13,7 +13,8 @@
   --ifm-color-primary-light: #33925d;
   --ifm-color-primary-lighter: #359962;
   --ifm-color-primary-lightest: #3cad6e;
-  --ifm-color-important-contrast-background: #bb91ff;
+  --ifm-color-important-contrast-background: #e7d7ff;
+  --ifm-color-important-highlight: #b94be8;
   --ifm-code-font-size: 95%;
   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1);
 }
@@ -28,6 +29,7 @@
   --ifm-color-primary-lighter: #32d8b4;
   --ifm-color-primary-lightest: #4fddbf;
   --ifm-color-important-contrast-background: #5a3795;
+  --ifm-color-important-highlight: #e09cfd;
   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3);
 }
 
@@ -40,7 +42,7 @@ details[class^='details_'].alert--info {
 
   margin-bottom: var(--ifm-paragraph-margin-bottom);
 
-  &:not(.alert--warning,.alert--success,.alert--secondary,.alert--danger) {
+  &:not(.alert--warning,.alert--success,.alert--secondary,.alert--danger,.alert--important) {
     --ifm-alert-background-color: transparent;
 
     > summary {
@@ -98,6 +100,12 @@ details[class^='details_'].alert--info {
 
 }
 
+.alert--info.theme-admonition-important {
+    --ifm-alert-background-color: var(--ifm-color-important-contrast-background);
+    --ifm-alert-foreground-color: var(--ifm-color-secondary-contrast-foreground);
+    --ifm-alert-border-color: var(--ifm-color-important-highlight);
+}
+
 /** https://stackoverflow.com/a/79808430/1469797 */
 img.image-center {
   display: block;

drizzle.config.ts

@@ -1,13 +1,14 @@
 import 'dotenv/config';
 import { defineConfig } from 'drizzle-kit';
-import { configDir, projectDir } from './src/backend/common/index.js';
+import { getDataDir } from './src/backend/common/index.js';
 import * as path from 'path';
+import { projectRootDir } from "./src/backend/common/infrastructure/Atomic.ts";
 
 export default defineConfig({
-  schema: path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'),
-  out: path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations'),
+  schema: path.resolve(projectRootDir, 'src/backend/common/database/drizzle/schema'),
+  out: path.resolve(projectRootDir, 'src/backend/common/database/drizzle/migrations'),
   dialect: 'sqlite',
   dbCredentials: {
-    url: path.resolve(configDir, 'ms.db'),
+    url: path.resolve(getDataDir(), 'ms.db'),
   },
 });

eslint.config.js

@@ -1,39 +1,254 @@
 // For more info, see https://github.com/storybookjs/eslint-plugin-storybook#configuration-flat-config-format
 import storybook from "eslint-plugin-storybook";
+import boundaries from 'eslint-plugin-boundaries';
+import unusedImports from 'eslint-plugin-unused-imports';
+import { importX } from 'eslint-plugin-import-x'
+
+import { defineConfig, globalIgnores } from "eslint/config";
 
 // @ts-check
 
-import eslint from '@eslint/js';
+import js from '@eslint/js';
+import globals from 'globals';
 import tsEslint from 'typescript-eslint';
 import arrow from 'eslint-plugin-prefer-arrow-functions';
+import hooks from 'eslint-plugin-react-hooks';
+import mochaPlugin from 'eslint-plugin-mocha';
+import unicorn from 'eslint-plugin-unicorn';
 
-export default tsEslint.config({
-    files: ['src/backend/**/*.ts'],
-    plugins: {
-        "prefer-arrow-functions": arrow
-    },
-    ignores: [
-        'eslint.config.js',
-        'src/backend/tests/**/*.ts'
+const defaultRules = {
+    'no-useless-catch': 'off',
+    '@typescript-eslint/no-unused-vars': 'warn',
+    'no-unused-vars': [
+        'warn',
+        {
+            args: 'none',
+            caughtErrors: 'none',
+            destructuredArrayIgnorePattern: "^_",
+            "argsIgnorePattern": "^_",
+            ignoreRestSiblings: true,
+            ignoreUsingDeclarations: true
+        }
     ],
-    extends: [
-        eslint.configs.recommended,
-        ...tsEslint.configs.recommended,
+    "@typescript-eslint/no-unused-vars": [
+      "warn",
+      {
+        args: 'none',
+        caughtErrors: 'none',
+        destructuredArrayIgnorePattern: "^_",
+        "argsIgnorePattern": "^_",
+        ignoreRestSiblings: true,
+        ignoreUsingDeclarations: true
+      }
     ],
-    rules: {
-        'no-useless-catch': 'off',
-        '@typescript-eslint/no-unused-vars': 'off',
-        "prefer-arrow-functions/prefer-arrow-functions": [
-            "warn",
-            {
-                "allowNamedFunctions": false,
-                "classPropertiesAllowed": false,
-                "disallowPrototype": false,
-                "returnStyle": "unchanged",
-                "singleReturnOnly": false
-            }
+    "prefer-arrow-functions/prefer-arrow-functions": [
+        "warn",
+        {
+            "allowNamedFunctions": false,
+            "classPropertiesAllowed": false,
+            "disallowPrototype": false,
+            "returnStyle": "unchanged",
+            "singleReturnOnly": false
+        }
+    ],
+    "arrow-body-style": ["warn", "as-needed"],
+    "@typescript-eslint/no-explicit-any": "warn",
+    "@typescript-eslint/no-empty-object-type": [
+        "warn",
+        {
+            "allowInterfaces": 'with-single-extends'
+        }
+    ]
+};
+
+export default defineConfig([
+    globalIgnores([
+        'docsite/build',
+        'docsite/.docusaurus',
+        'public/mockServiceWorker.js',
+        'dist'
+    ]),
+    {
+        plugins: {
+            "prefer-arrow-functions": arrow,
+            js,
+            'unused-imports': unusedImports,
+            unicorn,
+            'import-x': importX,
+        },
+        rules: {
+            ...defaultRules,
+            // https://typescript-eslint.io/rules/consistent-type-imports/#comparison-with-importsnotusedasvalues--verbatimmodulesyntax
+            //
+            // when using tsconfig compiler verbatimModuleSyntax and EX import {type Foo, type Bar} from 'a';
+            // true => typescript will *still* import a module if all types are inline  
+            // false => typescript will erase the entire module import
+            //
+            // we need to use verbatimModuleSyntax: true for nodejs type stripping compatibility so
+            // its important that we use top-level type imports so we don't accidentally import server-side modules into frontend
+            // but can still use types where necessary
+            //
+            // this rule *should* do this...it does detect imports that are typed but not explicitly declared
+            // but its not moving inline -> top-level
+            "@typescript-eslint/consistent-type-imports": [
+                "error",
+                {
+                    prefer: 'type-imports',
+                    fixStyle: "separate-type-imports"
+                }
+            ],
+            // however, import/consistent-type-specifier-style from eslint-plugin-import *does* move inline => top-level
+            // but it does not yet support eslint10 :(
+            //
+            // TODO eventually add this once plugin-import supports eslint10
+            // https://github.com/import-js/eslint-plugin-import
+            // https://github.com/import-js/eslint-plugin-import/blob/main/docs/rules/consistent-type-specifier-style.md
+            // -- this enforces that type-only imports are fixed to top-level type imports IE
+            // import {type Foo, type Bar} from 'a'; => import type { Foo, Bar} from 'a';
+            //
+            // USE eslint-plugin-import-x since its more actively developed?
+            // waiting on the status of this for supporting mixed inline types
+            // https://github.com/un-ts/eslint-plugin-import-x/issues/501
+            // "import-x/consistent-type-specifier-style": [
+            //     "error", "prefer-top-level"
+            // ],
+            "unicorn/prefer-then-catch": "error",
+            "unicorn/consistent-destructuring": "warn",
+            "unicorn/consistent-function-scoping": "warn",
+            "unicorn/consistent-optional-chaining": "error",
+            "unicorn/no-array-callback-reference": "warn",
+            "unicorn/no-accidental-bitwise-operator": "warn",
+            "no-obj-calls": "error",
+            "unicorn/new-for-builtins": "error",
+            "unicorn/no-impossible-length-comparison": "error",
+            "unicorn/no-duplicate-loops": "warn",
+            "unicorn/no-duplicate-logical-operands": "warn",
+            "unicorn/no-declarations-before-early-exit": "warn",
+            "unicorn/prefer-negative-index": "warn",
+            "unicorn/prefer-import-meta-properties": "error",
+            "unicorn/prefer-array-from-async": "warn",
+            "unicorn/prefer-array-flat-map": "warn",
+            "unicorn/no-useless-else": "warn",
+            "unicorn/no-unused-array-method-return": "error",
+            //"unicorn/no-unreadable-object-destructuring": "warn",
+            "unicorn/prefer-object-destructuring-defaults": "warn"
+        },
+        extends: [
+            tsEslint.configs.recommended,
+            "js/recommended"
         ],
-        "arrow-body-style": ["warn", "as-needed"],
-        "@typescript-eslint/no-explicit-any": "warn"
+        languageOptions: {
+            globals: {
+                ...globals.node,
+            },
+            ecmaVersion: 'latest',
+            sourceType: 'module',
+        },
+        files: ['src/**/*.ts','src/**/*.tsx']
+    },
+    {
+        extends: [
+            storybook.configs["flat/recommended"],
+        ],
+        files: ['src/client/stories/**/*.tsx'],
+    },
+    {
+        extends: [
+            hooks.configs.flat.recommended,
+        ],
+        languageOptions: {
+            globals: {
+                ...globals.browser,
+            }
+        },
+        files: ['src/client/**/*.tsx'],
+    },
+    {
+        extends: [
+            mochaPlugin.configs.recommended,
+        ],
+        languageOptions: {
+            globals: {
+                ...globals.node,
+            }
+        },
+        files: ['src/backend/tests/**/*.ts'],
+        rules: {
+            ...defaultRules,
+                "prefer-arrow-functions/prefer-arrow-functions": ["off"],
+                "@typescript-eslint/no-unused-expressions": 'off',
+                'mocha/max-top-level-suites': 'off'
+        },
+    },
+    {
+        // https://typescript-eslint.io/troubleshooting/faqs/eslint#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors
+        files: ['**/*.{ts,tsx,mts,cts}'],
+        plugins: {
+            'unused-imports': unusedImports
+        },
+        rules: {
+            'no-undef': 'off',
+            'unused-imports/no-unused-imports': 'error'
+        }
+    }, 
+    // this ruleset helps keep backend, frontend, and core keep imports isolated
+    // in order to prevent frontend (vite) from accidentally bundling backend files + backend packages when importing directly (backend) or transitively (through core)
+    //
+    // folder (module?) boundaries should be like
+    //
+    // backend <-- all node/server side code
+    // core <-- shared types, utils, and logic between backend and client
+    // client <-- frontend code to be bundled by vite, SHOULD NOT import directly/transitively from backend
+    {
+        files: ['src/**/*.{ts,tsx}'],
+        plugins: { boundaries },
+        settings: {
+        // Define your three architectural layers
+        'boundaries/elements': [
+            { type: 'frontend', mode: 'file', pattern: 'src/client/**/*' },
+            { type: 'config', mode: 'file', pattern: 'config/*.example' },
+            { type: 'core', mode: 'file', pattern: ['src/core/!(tests)/**','src/core/!(tests)'] },
+            { type: 'core-tests', mode: 'file', pattern: ['src/core/tests/**'] },
+            { type: 'backend', mode: 'file', pattern: 'src/backend/**/*' },
+            { type: 'stories', mode: 'file', pattern: ['src/stories/**/*', '.storybook/**'] },
+        ],
+        // So it understands TS path aliases when resolving imports
+        'import/resolver': {
+            typescript: true,
+        },
+        },
+        rules: {
+        'boundaries/element-types': [
+            'warn',
+            {
+            default: 'disallow', // deny anything not explicitly allowed
+            rules: [
+                {
+                from: 'frontend',
+                allow: ['frontend', 'core'], // frontend can use itself + core, never backend
+                },
+                {
+                from: 'core',
+                allow: ['core'], // core can only use itself — never backend, never frontend
+                },
+                {
+                from: 'backend',
+                allow: ['backend', 'core', 'config', 'core-tests'], // backend can use itself + core + tests
+                },
+                {
+                from: 'stories',
+                allow: ['stories', 'core', 'frontend', 'core-tests'], // backend can use itself + core
+                },
+                {
+                from: 'core-tests',
+                allow: ['core', 'core-tests'], // backend can use itself + core
+                },
+            ],
+            },
+        ],
+        // Optional: flag any file under src/ that doesn't match one of the
+        // three element patterns above (catches stray/misplaced files)
+        'boundaries/no-unknown': 'warn'
+        },
     }
-}, storybook.configs["flat/recommended"]);
+]);

next/index.html

@@ -0,0 +1,21 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8" />
+    <meta
+            name="description"
+            content="Dashboard for Multi Scrobbler"
+    />
+    <meta name="theme-color" content="#000000" />
+    <link rel="apple-touch-icon" href="/logo192.png" />
+    <link rel="icon" type="image/svg+xml" href="/icon.svg" />
+    <link rel="manifest" href="/manifest.json" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Multi Scrobbler</title>
+</head>
+<body>
+<noscript>You need to enable JavaScript to run this app. aaaa</noscript>
+<div id="root"></div>
+<script type="module" src="/src/client/index-next.tsx"></script>
+</body>
+</html>

package.json

@@ -1,17 +1,17 @@
 {
   "name": "multi-scrobbler",
-  "version": "0.14.0-beta.2",
+  "version": "0.14.2",
   "type": "module",
   "description": "scrobble plays from multiple sources to multiple clients",
   "scripts": {
     "schema": "npm run -s schema:docs && npm run -s schema:app",
-    "schema:docs": "tsx src/backend/utils/SchemaDocsStaticUtils.ts",
-    "schema:app": "tsx src/backend/utils/SchemaAppStaticUtils.ts",
+    "schema:docs": "node src/backend/utils/SchemaDocsStaticUtils.ts",
+    "schema:app": "node src/backend/utils/SchemaAppStaticUtils.ts",
     "circular": "madge --circular --extensions ts src/index.ts",
     "test": "npm run -s test:backend",
     "test:backend": "mocha --reporter spec",
-    "dev": "nodemon -w src/backend -x tsx src/backend/index.ts",
-    "start": "NODE_ENV=production tsx src/backend/index.ts",
+    "dev": "node src/backend/index.ts",
+    "start": "NODE_ENV=production node src/backend/index.ts",
     "build:frontend": "vite build",
     "build:backend": "tsc -p src/backend && npm run -s schema:app",
     "build": "npm run -s build:backend && npm run -s build:frontend && npm run -s docs:build",
@@ -24,7 +24,9 @@
     "install:parallel": "concurrently --kill-others-on-fail --names app,docs \"npm ci --no-audit\" \"npm run docs:install\"",
     "cliff": "git-cliff --unreleased",
     "storybook": "storybook dev -p 6006",
-    "build-storybook": "storybook build"
+    "build-storybook": "storybook build",
+    "atproto:lex:tealfm": "npm exec -- lex-cli pull -c src/backend/common/vendor/teal/lex.config.ts && npm exec -- lex-cli generate -c src/backend/common/vendor/teal/lex.config.ts",
+    "frontend:analyze": "npx esbuild src/client/index-next.tsx --bundle --format=esm --platform=node --metafile=meta.json --outfile=/dev/null"
   },
   "exports": {
     ".": {
@@ -49,9 +51,13 @@
   "homepage": ".",
   "dependencies": {
     "@astronautlabs/mdns": "^1.0.7",
-    "@atcute/identity-resolver-node": "^1.0.3",
-    "@atproto/api": "^0.18.0",
-    "@atproto/oauth-client-node": "^0.3.10",
+    "@atcute/bluesky": "^4.0.5",
+    "@atcute/client": "^5.1.0",
+    "@atcute/identity": "^2.0.0",
+    "@atcute/identity-resolver": "^2.0.0",
+    "@atcute/lexicons": "^2.0.0",
+    "@atcute/password-session": "^1.0.0",
+    "@atcute/repo": "^1.0.0",
     "@donedeal0/superdiff": "^1.1.1",
     "@ewanc26/tid": "^1.0.2",
     "@foxxmd/chromecast-client": "^1.0.4",
@@ -64,9 +70,11 @@
     "@kenyip/backoff-strategies": "^1.0.4",
     "@keyv/valkey": "^1.0.8",
     "@lukehagar/plexjs": "^0.39.0",
+    "@pierre/diffs": "^1.2.4",
+    "@rocksky/sdk": "^0.3.0",
     "@supercharge/promise-pool": "^3.0.0",
     "@svrooij/sonos": "^2.5.0",
-    "@xhayper/discord-rpc": "^1.3.0",
+    "@xhayper/discord-rpc": "1.3.0",
     "abort-controller-x": "^0.5.0",
     "address": "^1.2.2",
     "ajv": "^8.18.0",
@@ -74,7 +82,7 @@
     "avahi-browse": "^1.1.4",
     "better-sse": "^0.8.0",
     "body-parser": "^2.2.2",
-    "cacheable": "^1.10.4",
+    "cacheable": "^2.5.0",
     "castv2": "^0.1.10",
     "clone": "^2.1.2",
     "common-tags": "^1.8.2",
@@ -86,6 +94,7 @@
     "discord.js": "^14.26.0",
     "dotenv": "^10.0.0",
     "drizzle-orm": "^1.0.0-rc.2-c5a84d1",
+    "env-paths": "^4.0.0",
     "express": "^5.2.1",
     "express-session": "^1.19.0",
     "fast-equals": "^6.0.0",
@@ -102,7 +111,7 @@
     "json-diff-ts": "^5.0.0-alpha.2",
     "json5": "^2.2.3",
     "jsondiffpatch": "^0.7.3",
-    "keyv": "^5.5.0",
+    "keyv": "^5.6.0",
     "kodi-api": "^0.2.1",
     "lastfm-ts-api": "^2.6.0",
     "merge-error-cause": "^5.0.2",
@@ -111,6 +120,7 @@
     "musicbrainz-api": "^0.27.0",
     "nanoid": "^3.3.1",
     "neotraverse": "^0.6.18",
+    "node-musickit-api": "^3.1.0-next.5",
     "node-object-hash": "^3.1.1",
     "normalize-url": "^8.0.1",
     "ntfy": "^1.15.2",
@@ -121,6 +131,8 @@
     "passport-deezer": "^0.2.0",
     "patch-package": "^8.0.1",
     "prom-client": "^15.1.3",
+    "qs": "^6.15.2",
+    "react-use-timeout": "^1.0.0",
     "round-robin-js": "^3.0.10",
     "serialize-error": "^13.0.1",
     "spotify-web-api-node": "^5.0.2",
@@ -128,8 +140,6 @@
     "superagent": "^8.0.9",
     "toad-scheduler": "^3.0.0",
     "tough-cookie": "^5.1.2",
-    "ts-json-schema-generator": "^2.3.0",
-    "tsx": "^4.21.0",
     "typeson": "^9.0.4",
     "typeson-registry": "^11.1.1",
     "undici": "7.24.6",
@@ -139,22 +149,34 @@
     "youtubei.js": "^15.0.1"
   },
   "devDependencies": {
-    "@chakra-ui/react": "^3.34.0",
+    "@atcute/lex-cli": "^3.1.0",
+    "@chakra-ui/cli": "^3.36.0",
+    "@chakra-ui/react": "^3.36.0",
     "@chromatic-com/storybook": "^5.1.2",
     "@dbus-types/notifications": "^0.0.5",
     "@emotion/react": "^11.14.0",
+    "@emotion/styled": "^11.14.1",
     "@eslint/js": "^10.0.1",
     "@faker-js/faker": "^9.0.1",
+    "@flamefrontend/sse-runtime-react": "^0.7.0",
     "@fortawesome/fontawesome-svg-core": "^6.7.2",
     "@fortawesome/free-solid-svg-icons": "^6.7.2",
     "@fortawesome/react-fontawesome": "^0.2.6",
     "@gr2m/net-interceptor": "^1.0.0",
+    "@lukemorales/query-key-factory": "^1.3.4",
+    "@mui/material": "^9.1.1",
+    "@react-hook/resize-observer": "^2.0.2",
+    "@react-hook/window-size": "^3.1.1",
     "@react-nano/use-event-source": "^0.16.0",
     "@reduxjs/toolkit": "^2.11.2",
-    "@storybook/addon-a11y": "^10.3.6",
-    "@storybook/addon-docs": "^10.3.6",
-    "@storybook/react-vite": "^10.3.6",
+    "@rolldown/plugin-babel": "^0.2.3",
+    "@storybook/addon-a11y": "^10.4.3",
+    "@storybook/addon-docs": "^10.4.3",
+    "@storybook/react-vite": "^10.4.3",
     "@tailwindcss/vite": "^4.2.2",
+    "@tanstack/react-query": "^5.101.0",
+    "@tanstack/react-query-devtools": "^5.101.0",
+    "@tanstack/react-virtual": "^3.14.3",
     "@tsconfig/node18": "^1.0.1",
     "@types/chai": "^4.3.0",
     "@types/chai-as-promised": "^7.1.5",
@@ -172,6 +194,7 @@
     "@types/node": "^24.12.2",
     "@types/object-hash": "^3.0.0",
     "@types/passport": "^1.0.12",
+    "@types/qs": "^6.15.1",
     "@types/react": "^19.2.14",
     "@types/react-dom": "^19.2.3",
     "@types/react-window": "^1.8.5",
@@ -179,40 +202,54 @@
     "@types/spotify-web-api-node": "^5.0.7",
     "@types/superagent": "^8.1.9",
     "@types/xml2js": "^0.4.11",
-    "@vitejs/plugin-react": "^6.0.1",
+    "@vitejs/plugin-react": "^6.0.2",
+    "@welldone-software/why-did-you-render": "^10.0.1",
     "ansi-to-react": "^6.2.6",
     "autoprefixer": "^10.4.27",
+    "babel-plugin-react-compiler": "^1.0.0",
     "chai": "^4.3.6",
     "chai-as-promised": "^8.0.2",
     "clsx": "^2.1.1",
     "drizzle-kit": "^1.0.0-rc.2-c5a84d1",
-    "eslint": "^10.3.0",
+    "eslint": "^10.6.0",
+    "eslint-import-resolver-typescript": "^4.4.5",
+    "eslint-plugin-boundaries": "^7.0.2",
+    "eslint-plugin-import-x": "^4.17.1",
+    "eslint-plugin-mocha": "^11.3.0",
     "eslint-plugin-prefer-arrow-functions": "^3.2.4",
+    "eslint-plugin-react-hooks": "^7.1.1",
     "eslint-plugin-storybook": "^10.3.6",
+    "eslint-plugin-unicorn": "^72.0.0",
+    "eslint-plugin-unused-imports": "^4.4.1",
     "git-cliff": "^2.12.0",
-    "jsondiffpatch-react": "^1.0.8",
+    "globals": "^17.7.0",
+    "ky": "^2.0.2",
     "mocha": "^10.3.0",
     "mockdate": "^3.0.5",
-    "msw": "^2.12.10",
+    "msw": "^2.14.6",
+    "msw-storybook-addon": "^2.0.7",
     "next-themes": "^0.4.6",
     "nodemon": "^3.0.3",
     "playwright": "^1.58.2",
     "react": "^19.2.6",
     "react-dom": "^19.2.6",
     "react-error-boundary": "^6.1.1",
-    "react-icons": "^5.6.0",
+    "react-icons": "^5.7.0",
     "react-redux": "^9.2.0",
     "react-router-dom": "^7.15.0",
-    "react-virtuoso": "^4.18.4",
+    "react-virtuoso": "^4.18.7",
     "sass": "^1.99.0",
     "shiki": "^4.0.2",
     "sinon": "^21.0.2",
-    "storybook": "^10.3.6",
+    "storybook": "^10.4.3",
+    "storybook-addon-remix-react-router": "^6.1.0",
     "tailwindcss": "^4.2.2",
     "tinyexec": "^1.1.1",
     "ts-essentials": "^10.2.0",
+    "ts-json-schema-generator": "^2.3.0",
     "typescript": "^6.0.3",
     "typescript-eslint": "^8.59.3",
+    "usehooks-ts": "^3.1.1",
     "vite": "^8.0.12",
     "with-local-tmp-dir": "^7.0.1"
   },
@@ -234,10 +271,11 @@
     },
     "musicbrainz-api": {
       "rate-limit-threshold": "0.2.1"
-    },
-    "jsondiffpatch-react": {
-      "react": "$react",
-      "react-dom": "$react-dom"
     }
+  },
+  "msw": {
+    "workerDirectory": [
+      "public"
+    ]
   }
 }

public/mockServiceWorker.js

@@ -0,0 +1,349 @@
+/* eslint-disable */
+/* tslint:disable */
+
+/**
+ * Mock Service Worker.
+ * @see https://github.com/mswjs/msw
+ * - Please do NOT modify this file.
+ */
+
+const PACKAGE_VERSION = '2.14.6'
+const INTEGRITY_CHECKSUM = '4db4a41e972cec1b64cc569c66952d82'
+const IS_MOCKED_RESPONSE = Symbol('isMockedResponse')
+const activeClientIds = new Set()
+
+addEventListener('install', function () {
+  self.skipWaiting()
+})
+
+addEventListener('activate', function (event) {
+  event.waitUntil(self.clients.claim())
+})
+
+addEventListener('message', async function (event) {
+  const clientId = Reflect.get(event.source || {}, 'id')
+
+  if (!clientId || !self.clients) {
+    return
+  }
+
+  const client = await self.clients.get(clientId)
+
+  if (!client) {
+    return
+  }
+
+  const allClients = await self.clients.matchAll({
+    type: 'window',
+  })
+
+  switch (event.data) {
+    case 'KEEPALIVE_REQUEST': {
+      sendToClient(client, {
+        type: 'KEEPALIVE_RESPONSE',
+      })
+      break
+    }
+
+    case 'INTEGRITY_CHECK_REQUEST': {
+      sendToClient(client, {
+        type: 'INTEGRITY_CHECK_RESPONSE',
+        payload: {
+          packageVersion: PACKAGE_VERSION,
+          checksum: INTEGRITY_CHECKSUM,
+        },
+      })
+      break
+    }
+
+    case 'MOCK_ACTIVATE': {
+      activeClientIds.add(clientId)
+
+      sendToClient(client, {
+        type: 'MOCKING_ENABLED',
+        payload: {
+          client: {
+            id: client.id,
+            frameType: client.frameType,
+          },
+        },
+      })
+      break
+    }
+
+    case 'CLIENT_CLOSED': {
+      activeClientIds.delete(clientId)
+
+      const remainingClients = allClients.filter((client) => {
+        return client.id !== clientId
+      })
+
+      // Unregister itself when there are no more clients
+      if (remainingClients.length === 0) {
+        self.registration.unregister()
+      }
+
+      break
+    }
+  }
+})
+
+addEventListener('fetch', function (event) {
+  const requestInterceptedAt = Date.now()
+
+  // Bypass navigation requests.
+  if (event.request.mode === 'navigate') {
+    return
+  }
+
+  // Opening the DevTools triggers the "only-if-cached" request
+  // that cannot be handled by the worker. Bypass such requests.
+  if (
+    event.request.cache === 'only-if-cached' &&
+    event.request.mode !== 'same-origin'
+  ) {
+    return
+  }
+
+  // Bypass all requests when there are no active clients.
+  // Prevents the self-unregistered worked from handling requests
+  // after it's been terminated (still remains active until the next reload).
+  if (activeClientIds.size === 0) {
+    return
+  }
+
+  const requestId = crypto.randomUUID()
+  event.respondWith(handleRequest(event, requestId, requestInterceptedAt))
+})
+
+/**
+ * @param {FetchEvent} event
+ * @param {string} requestId
+ * @param {number} requestInterceptedAt
+ */
+async function handleRequest(event, requestId, requestInterceptedAt) {
+  const client = await resolveMainClient(event)
+  const requestCloneForEvents = event.request.clone()
+  const response = await getResponse(
+    event,
+    client,
+    requestId,
+    requestInterceptedAt,
+  )
+
+  // Send back the response clone for the "response:*" life-cycle events.
+  // Ensure MSW is active and ready to handle the message, otherwise
+  // this message will pend indefinitely.
+  if (client && activeClientIds.has(client.id)) {
+    const serializedRequest = await serializeRequest(requestCloneForEvents)
+
+    // Clone the response so both the client and the library could consume it.
+    const responseClone = response.clone()
+
+    sendToClient(
+      client,
+      {
+        type: 'RESPONSE',
+        payload: {
+          isMockedResponse: IS_MOCKED_RESPONSE in response,
+          request: {
+            id: requestId,
+            ...serializedRequest,
+          },
+          response: {
+            type: responseClone.type,
+            status: responseClone.status,
+            statusText: responseClone.statusText,
+            headers: Object.fromEntries(responseClone.headers.entries()),
+            body: responseClone.body,
+          },
+        },
+      },
+      responseClone.body ? [serializedRequest.body, responseClone.body] : [],
+    )
+  }
+
+  return response
+}
+
+/**
+ * Resolve the main client for the given event.
+ * Client that issues a request doesn't necessarily equal the client
+ * that registered the worker. It's with the latter the worker should
+ * communicate with during the response resolving phase.
+ * @param {FetchEvent} event
+ * @returns {Promise<Client | undefined>}
+ */
+async function resolveMainClient(event) {
+  const client = await self.clients.get(event.clientId)
+
+  if (activeClientIds.has(event.clientId)) {
+    return client
+  }
+
+  if (client?.frameType === 'top-level') {
+    return client
+  }
+
+  const allClients = await self.clients.matchAll({
+    type: 'window',
+  })
+
+  return allClients
+    .filter((client) => {
+      // Get only those clients that are currently visible.
+      return client.visibilityState === 'visible'
+    })
+    .find((client) => {
+      // Find the client ID that's recorded in the
+      // set of clients that have registered the worker.
+      return activeClientIds.has(client.id)
+    })
+}
+
+/**
+ * @param {FetchEvent} event
+ * @param {Client | undefined} client
+ * @param {string} requestId
+ * @param {number} requestInterceptedAt
+ * @returns {Promise<Response>}
+ */
+async function getResponse(event, client, requestId, requestInterceptedAt) {
+  // Clone the request because it might've been already used
+  // (i.e. its body has been read and sent to the client).
+  const requestClone = event.request.clone()
+
+  function passthrough() {
+    // Cast the request headers to a new Headers instance
+    // so the headers can be manipulated with.
+    const headers = new Headers(requestClone.headers)
+
+    // Remove the "accept" header value that marked this request as passthrough.
+    // This prevents request alteration and also keeps it compliant with the
+    // user-defined CORS policies.
+    const acceptHeader = headers.get('accept')
+    if (acceptHeader) {
+      const values = acceptHeader.split(',').map((value) => value.trim())
+      const filteredValues = values.filter(
+        (value) => value !== 'msw/passthrough',
+      )
+
+      if (filteredValues.length > 0) {
+        headers.set('accept', filteredValues.join(', '))
+      } else {
+        headers.delete('accept')
+      }
+    }
+
+    return fetch(requestClone, { headers })
+  }
+
+  // Bypass mocking when the client is not active.
+  if (!client) {
+    return passthrough()
+  }
+
+  // Bypass initial page load requests (i.e. static assets).
+  // The absence of the immediate/parent client in the map of the active clients
+  // means that MSW hasn't dispatched the "MOCK_ACTIVATE" event yet
+  // and is not ready to handle requests.
+  if (!activeClientIds.has(client.id)) {
+    return passthrough()
+  }
+
+  // Notify the client that a request has been intercepted.
+  const serializedRequest = await serializeRequest(event.request)
+  const clientMessage = await sendToClient(
+    client,
+    {
+      type: 'REQUEST',
+      payload: {
+        id: requestId,
+        interceptedAt: requestInterceptedAt,
+        ...serializedRequest,
+      },
+    },
+    [serializedRequest.body],
+  )
+
+  switch (clientMessage.type) {
+    case 'MOCK_RESPONSE': {
+      return respondWithMock(clientMessage.data)
+    }
+
+    case 'PASSTHROUGH': {
+      return passthrough()
+    }
+  }
+
+  return passthrough()
+}
+
+/**
+ * @param {Client} client
+ * @param {any} message
+ * @param {Array<Transferable>} transferrables
+ * @returns {Promise<any>}
+ */
+function sendToClient(client, message, transferrables = []) {
+  return new Promise((resolve, reject) => {
+    const channel = new MessageChannel()
+
+    channel.port1.onmessage = (event) => {
+      if (event.data && event.data.error) {
+        return reject(event.data.error)
+      }
+
+      resolve(event.data)
+    }
+
+    client.postMessage(message, [
+      channel.port2,
+      ...transferrables.filter(Boolean),
+    ])
+  })
+}
+
+/**
+ * @param {Response} response
+ * @returns {Response}
+ */
+function respondWithMock(response) {
+  // Setting response status code to 0 is a no-op.
+  // However, when responding with a "Response.error()", the produced Response
+  // instance will have status code set to 0. Since it's not possible to create
+  // a Response instance with status code 0, handle that use-case separately.
+  if (response.status === 0) {
+    return Response.error()
+  }
+
+  const mockedResponse = new Response(response.body, response)
+
+  Reflect.defineProperty(mockedResponse, IS_MOCKED_RESPONSE, {
+    value: true,
+    enumerable: true,
+  })
+
+  return mockedResponse
+}
+
+/**
+ * @param {Request} request
+ */
+async function serializeRequest(request) {
+  return {
+    url: request.url,
+    mode: request.mode,
+    method: request.method,
+    headers: Object.fromEntries(request.headers.entries()),
+    cache: request.cache,
+    credentials: request.credentials,
+    destination: request.destination,
+    integrity: request.integrity,
+    redirect: request.redirect,
+    referrer: request.referrer,
+    referrerPolicy: request.referrerPolicy,
+    body: await request.arrayBuffer(),
+    keepalive: request.keepalive,
+  }
+}

src/backend/common/AbstractComponent.ts

@@ -1,41 +1,46 @@
-import { childLogger, Logger } from "@foxxmd/logging";
+import { childLogger, type Logger } from "@foxxmd/logging";
 import {
     cacheFunctions,
 } from "@foxxmd/regex-buddy-core";
-import { LifecycleStep, PlayData, PlayObject, TransformResult } from "../../core/Atomic.js";
-import { buildPlayHumanDiffable, buildTrackString } from "../../core/StringUtils.js";
-import { CommonClientConfig } from "./infrastructure/config/client/index.js";
-import { CommonSourceConfig } from "./infrastructure/config/source/index.js";
-import { mergeSimpleError, SimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js";
+import type EventEmitter from "events";
+import type {ComponentType, LifecycleInput, LifecycleStep, PlayData, PlayObject} from "../../core/Atomic.ts";
+import { buildTrackString } from "../../core/StringUtils.ts";
+import type {CommonClientConfig} from "./infrastructure/config/client/index.ts";
+import type {CommonSourceConfig} from "./infrastructure/config/source/index.ts";
+import { mergeSimpleError, SimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.ts";
 import {
     FLOW_CONTROL_TERM,
-    PlayTransformRules,
-    StageConfig,
+    type PlayTransformRules,
+    type StageConfig,
     TRANSFORM_HOOK,
-    TransformHook
-} from "./infrastructure/Transform.js";
-import AbstractInitializable from "./AbstractInitializable.js";
-import TransformerManager from "./transforms/TransformerManager.js";
-import { getRoot } from "../ioc.js";
+    type TransformHook
+} from "../../core/Transform.ts";
+import AbstractInitializable from "./AbstractInitializable.ts";
+import type TransformerManager from "./transforms/TransformerManager.ts";
+import { getRoot } from "../ioc.ts";
 import { nanoid } from "nanoid";
-import { isDebugMode } from "../utils.js";
-import { findCauseByReference } from "../utils/ErrorUtils.js";
-import { hashObject, parseArrayFromMaybeString } from "../utils/StringUtils.js";
-import { metaInvariantTransform, playContentInvariantTransform } from "../utils/PlayComparisonUtils.js";
-import { MSCache } from "./Cache.js";
-import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/DataUtils.js";
+import { isDebugMode } from "../utils.ts";
+import { findCauseByFunc, findCauseByReference } from "../utils/ErrorUtils.ts";
+import { hashObject, parseArrayFromMaybeString } from "../utils/StringUtils.ts";
+import { playContentInvariantTransform } from "../utils/PlayComparisonUtils.ts";
+import type { MSCache } from "./Cache.ts";
+import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/DataUtils.ts";
 import clone from "clone";
-import { loggerNoop } from "./MaybeLogger.js";
-import { objectsEqual } from "../utils/DataUtils.js";
-import { RetentionOptions } from "./infrastructure/config/database.js";
-import { getRetentionCompactAfterFromEnv, getRetentionDeleteAfterFromEnv, isCompactableProperty, parseRetentionOptions, parseRetentionOptionsDurations } from "./database/Database.js";
-import { DbConcrete } from "./database/drizzle/drizzleUtils.js";
-import { ComponentSelect } from "./database/drizzle/drizzleTypes.js";
-import { DrizzlePlayRepository } from "./database/drizzle/repositories/PlayRepository.js";
-import { ClientType } from "./infrastructure/config/client/clients.js";
-import { SourceType } from "./infrastructure/config/source/sources.js";
-import { DrizzleComponentRepository } from "./database/drizzle/repositories/ComponentRepository.js";
+import { loggerNoop } from "./MaybeLogger.ts";
+import { objectsEqual } from "../utils/DataUtils.ts";
+import type {RetentionOptions} from "./infrastructure/config/database.ts";
+import { getRetentionCompactAfterFromEnv, getRetentionDeleteAfterFromEnv, isCompactableProperty, parseRetentionOptions, parseRetentionOptionsDurations } from "./database/Database.ts";
+import type {DbConcrete} from "./database/drizzle/drizzleUtils.ts";
+import type {ComponentSelect} from "./database/drizzle/drizzleTypes.ts";
+import { DrizzlePlayRepository } from "./database/drizzle/repositories/PlayRepository.ts";
+import type {ClientType} from "../../core/Atomic.ts";
+import type {SourceType} from "../../core/Atomic.ts";
+import { DrizzleComponentRepository } from "./database/drizzle/repositories/ComponentRepository.ts";
 import dayjs from "dayjs";
+import { COMPONENT_STATE, type ComponentCommonApi, type ComponentCommonApiJson, type ComponentState, type PlayApiCommonDetailed } from "../../core/Api.ts";
+import type {WebhookPayload} from "./infrastructure/config/health/webhooks.ts";
+import type { MarkRequired } from "ts-essentials";
+import { serializeError } from "serialize-error";
 
 export type AbstractComponentConfig = (CommonClientConfig | CommonSourceConfig) & { transformManager?: TransformerManager };
 
@@ -50,10 +55,14 @@ export default abstract class AbstractComponent extends AbstractInitializable {
     protected db: DbConcrete;
     protected componentRepo!: DrizzleComponentRepository;
     protected dbComponent!: ComponentSelect;
+    componentId!: number;
     protected retentionOpts: RetentionOptions;
+    status: string = 'Waiting to initialize...';
+    emitter: EventEmitter;
 
-    protected componentType: 'source' | 'client';
+    protected componentType: ComponentType;
     type: ClientType | SourceType;
+    name: string;
 
     protected constructor(config: AbstractComponentConfig) {
         super(config);
@@ -70,6 +79,10 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         };
     }
 
+    public getUid() {
+        return this.config?.id ?? this.config?.name ?? this.name;
+    }
+
     protected postCache(): Promise<void> {
         try {
             this.buildTransformRules();
@@ -80,26 +93,22 @@ export default abstract class AbstractComponent extends AbstractInitializable {
     }
 
     protected async postInitialize(): Promise<void> {
-        super.postInitialize();
+        await super.postInitialize();
         this.componentRepo.updateById(this.dbComponent.id, {lastReadyAt: dayjs()});
     }
 
     protected async doBuildDatabase(): Promise<true | string | undefined> {
-        super.doBuildDatabase();
-
-        let name: string;
-        if('name' in this) {
-            name = this.name as string;
-        }
+        await super.doBuildDatabase();
 
         this.db = await getRoot().items.db();
         this.componentRepo = new DrizzleComponentRepository(this.db, {logger: this.logger});
         this.dbComponent = await this.componentRepo.findOrInsert({
             mode: this.componentType,
             type: this.type,
-            uid: this.config.id ?? this.config.name ?? name,
-            name: this.config.name ?? name
+            uid: this.getUid(),
+            name: this.config?.name ?? this.name
         });
+        this.componentId = this.dbComponent.id;
         return true;
     }
 
@@ -208,11 +217,16 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             this.logger.warn(`Cannot run retention cleanup because ${this.componentType} database state is not OK`);
             return;
         }
+        this.setStatus('Running retention cleanup');
         try {
             const repo = new DrizzlePlayRepository(this.db, {logger: this.logger});
             await repo.retentionCleanup(this.componentType, this.retentionOpts);
+            this.setStatus('Retention cleanup finished');
         } catch (e) {
-            this.logger.warn(new Error('Failed to do retention cleanup', {cause: e}));
+            const retentionErr = new Error('Failed to do retention cleanup', {cause: e});
+            this.warning = retentionErr;
+            this.logger.warn(retentionErr);
+            this.setStatus('Retention cleanup failed');
         }
     }
 
@@ -288,7 +302,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
                     }
 
                     for(const s of cachedSteps) {
-                        steps.push({...s, cached: true});
+                        steps.push({...s, cached: true, createdAt: dayjs().toISOString()});
                         if(shouldTransform && s.patch !== undefined) {
                             transformedPlay.data = patchObject(transformedPlay.data, s.patch); // jdiff.patch(clone(transformedPlay.data),s.patch);
                         }
@@ -307,7 +321,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
                         if(step.error !== undefined && step.flowKnownState !== 'skip' && !step.returnPartial) {
                             // revert to original play but keep steps for paper trail
                             transformedPlay = play;
-                            transformedPlay.meta.lifecycle.steps = steps;
+                            transformedPlay.lifecycle = steps;
                         } else {
                             transformedPlay = stepPlay;
                         }
@@ -321,34 +335,30 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             const historyToDiff: {name: string, data?: PlayData}[] = [
                 {name: 'Pre Transform', data: play.data}
             ];
-            if(steps.length > 0 && transformedPlay.meta?.lifecycle?.steps === undefined) {
-                const {
-                    meta: {
-                        lifecycle: {
-                            steps = [],
-                            ...lifecycleRest
-                        },
-                    } = {}
-                } = transformedPlay;
-                transformedPlay.meta.lifecycle = {
-                    ...lifecycleRest,
-                    steps
-                }
+            let isNew = false;
+            if(steps.length > 0 && transformedPlay.lifecycle === undefined) {
+                transformedPlay.lifecycle = steps;
+                isNew = true;
             }
+            const {
+                lifecycle = []
+            } = transformedPlay;
             steps.forEach((s, index) => {
-                const existingStepIndex = transformedPlay.meta.lifecycle.steps.findIndex(x => x.name === 'hookType' && x.source === this.getIdentifier());
-                if(existingStepIndex !== -1) {
-                    transformedPlay.meta.lifecycle.steps[existingStepIndex] = s;
-                } else {
-                    transformedPlay.meta.lifecycle.steps.push(s);
+                if(!isNew) {
+                    const existingStepIndex = lifecycle.findIndex(x => x.stageName === s.stageName && x.stageType === s.stageType && x.hook === s.hook && x.source === this.getIdentifier());
+                    if(existingStepIndex !== -1) {
+                        transformedPlay.lifecycle[existingStepIndex] = s;
+                    } else {
+                        transformedPlay.lifecycle.push(s);
+                    }
                 }
 
                 if(shouldLog === 'all') {
                     if(s.patch === undefined) {
-                        historyToDiff.push({name: `${s.source}-${s.name}`});
+                        historyToDiff.push({name: `${s.source}-${s.hook}-${s.stageType}-${s.stageName}`});
                     } else {
                         const patched = patchObject(historyToDiff[historyToDiff.length - 1].data, s.patch);
-                        historyToDiff.push({name: `${s.source}-${s.name}${s.cached ? ' (Cached)' : ''}`, data: patched});
+                        historyToDiff.push({name: `${s.source}-${s.hook}-${s.stageType}-${s.stageName} ${s.cached ? ' (Cached)' : ''}`, data: patched});
                     }
                 }
             });
@@ -410,11 +420,18 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             asyncId = nanoid(6)
         } = opts;
 
-        const stepName = `${hookType} - ${hookItem.type} - ${hookItem.name}`
-        const existingStepIndex = playTruth.meta.lifecycle.steps.findIndex(x => x.name === stepName && x.source === this.getIdentifier());
-        const step: LifecycleStep = existingStepIndex !== -1 && playTruth.meta.lifecycle.steps[existingStepIndex] !== undefined ? playTruth.meta.lifecycle.steps[existingStepIndex] : {
-            name: stepName,
+        const {
+            lifecycle = []
+        } = playTruth;
+
+        //const stepName = `${hookType} - ${hookItem.type} - ${hookItem.name}`
+        const existingStepIndex = lifecycle.findIndex(x => x.hook === hookType && hookItem.name === x.stageName && x.stageType === hookItem.type && x.source === this.getIdentifier());
+        const step: LifecycleStep = existingStepIndex !== -1 && lifecycle[existingStepIndex] !== undefined ? lifecycle[existingStepIndex] : {
+            stageName: hookItem.name,
+            hook: hookType,
+            stageType: hookItem.type,
             source: this.getIdentifier(),
+            createdAt: dayjs().toString()
         }
 
         let newTransformedPlay: PlayObject,
@@ -430,7 +447,16 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             }
         }
 
+        step.stageName = stageName;
+
         if (err !== undefined) {
+            const lifecycleErrorContext = findCauseByFunc(err, (e) => 'lifecycleInputs' in e && e.lifecycleInputs !== undefined);
+            if (lifecycleErrorContext !== undefined) {
+                // @ts-expect-error it does exist
+                step.inputs = clone(lifecycleErrorContext.lifecycleInputs) as LifecycleInput[];
+                // @ts-expect-error it does exist
+                delete lifecycleErrorContext.lifecycleInputs;
+            }
             const merged = mergeSimpleError(err);
             step.error = merged;
 
@@ -446,6 +472,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
                 step.flowReason = skipMsg;
 
                 logger.debug(merged, skipMsg);
+                return [step, playTruth];
             } else {
                 step.flowResult = onFailure;
                 let reason: string;
@@ -493,6 +520,69 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             delete newTransformedPlay.meta.lifecycleInputs;
         }
 
-        return [step, newTransformedPlay];
+        return [step, newTransformedPlay ?? playTruth];
+    }
+
+    public abstract getRunningState(): ComponentState
+
+    public getApiData(): Omit<ComponentCommonApiJson, 'type' | 'countLive' | 'players'> & Pick<ComponentCommonApi, 'state' | 'error' | 'warning'>  {
+        let state: ComponentState;
+        if(!this.initializedOnce || this.initializing) {
+            state = COMPONENT_STATE.INITIALIZING;
+        } else if(!this.isReady()) {
+            state = COMPONENT_STATE.NOT_READY;
+        } else {
+            state = this.getRunningState()
+        }
+        return {
+            id: this.dbComponent.id,
+            uid: this.dbComponent.uid,
+            name: this.dbComponent.name,
+            state,
+            mode: this.dbComponent.mode,
+            countNonLive: this.dbComponent.countNonLive,
+            createdAt: this.dbComponent.createdAt?.toISOString(),
+            lastReadyAt: this.dbComponent.lastReadyAt?.toISOString(),
+            lastActiveAt: this.dbComponent.lastActiveAt?.toISOString(),
+            error: this.error !== undefined && this.error instanceof Error ? serializeError(this.error) : this.error,
+            warning: this.warning !== undefined && this.warning instanceof Error ? serializeError(this.warning) : this.warning,
+            ...this.additionalApiData()
+        }
+    }
+
+    public emitEvent = (eventName: string, payload: Record<string, any> = {}) => {
+        this.emitter.emit(eventName, {
+            type: this.type,
+            name: this.name,
+            componentId: this.dbComponent?.id,
+            from: this.componentType,
+            data: payload,
+        });
+    }
+
+    protected emitComponentUpdate = <T extends Partial<ReturnType<typeof this.getApiData>>>(payload: T) => {
+        if('error' in payload && payload.error instanceof Error) {
+            payload.error = serializeError(payload.error);
+        }
+        if('warning' in payload && payload.warning instanceof Error) {
+            payload.warning = serializeError(payload.warning);
+        }
+        this.emitEvent('componentUpdate', payload);
+    }
+    protected emitPlayUpdate = (payload: MarkRequired<Partial<PlayApiCommonDetailed>, 'uid'>) => {
+        this.emitEvent('playUpdate', payload);
+    }
+    protected emitPlayInsert = (payload: PlayApiCommonDetailed) => {
+        this.emitEvent('playInsert', payload);
+    }
+
+    async notify(payload: Omit<WebhookPayload, 'identifier'>) {
+        this.emitEvent('notify', {...payload, identifier: this.getIdentifier()});
+        this.setStatus(payload.title);
+    }
+
+    protected setStatus = (status: string) => {
+        this.status = status;
+        this.emitComponentUpdate({status});
     }
 }

src/backend/common/AbstractInitializable.ts

@@ -1,10 +1,10 @@
-import { childLogger, Logger } from "@foxxmd/logging";
-import {truncateStringToLength } from "../../core/StringUtils.js";
-import { hasNodeNetworkException } from "./errors/NodeErrors.js";
-import { hasUpstreamError } from "./errors/UpstreamError.js";
-import { WebhookPayload } from "./infrastructure/config/health/webhooks.js";
-import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, StageError, TransformRulesError } from "./errors/MSErrors.js";
-import { messageWithCausesTruncatedDefault } from "../../core/ErrorUtils.js";
+import type { Logger } from "@foxxmd/logging";
+import {truncateStringToLength } from "../../core/StringUtils.ts";
+import { hasNodeNetworkException } from "./errors/NodeErrors.ts";
+import { hasUpstreamError } from "./errors/UpstreamError.ts";
+import type {WebhookPayload} from "./infrastructure/config/health/webhooks.ts";
+import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, StageError } from "./errors/MSErrors.ts";
+import { messageWithCausesTruncatedDefault } from "../../core/ErrorUtils.ts";
 import { spawn } from 'abort-controller-x';
 
 export default abstract class AbstractInitializable {
@@ -19,7 +19,10 @@ export default abstract class AbstractInitializable {
     databaseOK?: boolean | null;
     connectionOK?: boolean | null;
     cacheOK?: boolean | null;
+    error?: Error;
+    warning?: Error;
 
+    protected initializedOnce: boolean = false;
     initializing: boolean = false;
 
     config: Record<string, any>;
@@ -32,17 +35,21 @@ export default abstract class AbstractInitializable {
     }
 
     public abstract notify(payload: WebhookPayload): Promise<void>;
+    protected setStatus(status: string): void {
+        return;
+    }
 
     protected abstract getIdentifier(): string;
 
     initialize = async (options: {force?: boolean, notify?: boolean, notifyTitle?: string, internalOnly?: boolean} = {}) => {
-
+        this.initializedOnce = true;
         if(this.initController !== undefined) {
             throw new Error(`Already trying to initialize, cannot attempt while an existing initialization attempt is running.`);
         }
 
         this.initController = new AbortController();
         return spawn(this.initController.signal, async (signal, {defer, fork}) => {
+            this.setStatus('Initializing...');
 
             defer(async () => {
                 this.initializing = false;
@@ -75,17 +82,22 @@ export default abstract class AbstractInitializable {
                 await this.checkConnection(force);
                 await this.testAuth(force);
                 this.logger.info('Fully Initialized!');
+                this.setStatus('Initialized!');
                 try {
                     await this.postInitialize();
                 } catch (e) {
                     throw new PostInitError('Error occurred during post-initialization hook', {cause: e});
                 }
+                this.error = undefined;
+                this.warning = undefined;
                 return true;
             } catch(e) {
                 if(notify) {
-                    await this.notify({title: `${this.getIdentifier()} - ${notifyTitle}`, message: truncateStringToLength(500)(messageWithCausesTruncatedDefault(e)), priority: 'error'});
+                    await this.notify({identifier: this.getIdentifier(), title: notifyTitle, message: truncateStringToLength(500)(messageWithCausesTruncatedDefault(e)), priority: 'error'});
                 }
-                throw new Error('Initialization failed', {cause: e});
+                const initError = new Error('Initialization failed', {cause: e});
+                this.error = initError;
+                throw initError;
             } finally {
                 this.initializing = false;
             }

src/backend/common/Cache.ts

@@ -1,7 +1,7 @@
-import { Cacheable, createKeyv, Keyv, KeyvStoreAdapter, KeyvOptions, CacheableOptions, KeyvCacheableMemory } from 'cacheable';
-import { FlatCache, FlatCacheOptions } from 'flat-cache';
-import KeyvValkey, { KeyvValkeyOptions } from '@keyv/valkey';
-import dayjs, { Dayjs } from 'dayjs';
+import { Cacheable, createKeyv, Keyv, type KeyvStoreAdapter, type KeyvOptions, type CacheableOptions, KeyvCacheableMemory } from 'cacheable';
+import { FlatCache, type FlatCacheOptions } from 'flat-cache';
+import KeyvValkey, { type KeyvValkeyOptions } from '@keyv/valkey';
+import dayjs, { type Dayjs } from 'dayjs';
 import duration from 'dayjs/plugin/duration.js';
 import isBetween from 'dayjs/plugin/isBetween.js';
 import relativeTime from 'dayjs/plugin/relativeTime.js';
@@ -9,19 +9,18 @@ import isToday from 'dayjs/plugin/isToday.js';
 import timezone from 'dayjs/plugin/timezone.js';
 import utc from 'dayjs/plugin/utc.js';
 import clone from 'clone';
-import { childLogger, Logger } from '@foxxmd/logging';
-import { projectDir } from './index.js';
+import { childLogger, type Logger } from '@foxxmd/logging';
+import { getConfigDir } from './index.ts';
 import path from 'path';
 import { cacheFunctions } from "@foxxmd/regex-buddy-core";
-import { fileExists, fileOrDirectoryIsWriteable } from '../utils/FSUtils.js';
-import { copyFile } from 'fs/promises';
-import { asCacheConfig, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheConfigUser, CacheScrobbleProvider } from './infrastructure/Atomic.js';
+import { fileOrDirectoryIsWriteable } from '../utils/FSUtils.ts';
+import { asCacheConfig, type CacheAuthProvider, type CacheConfig, type CacheConfigOptions, type CacheConfigUser, type CacheScrobbleProvider } from './infrastructure/Atomic.ts';
 import { Typeson } from 'typeson';
 import { builtin } from 'typeson-registry';
-import { loggerNoop } from './MaybeLogger.js';
-const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
-import prom, { Gauge } from 'prom-client';
-import { nonEmptyStringOrDefault } from '../../core/StringUtils.js';
+import { loggerNoop } from './MaybeLogger.ts';
+import type { Gauge } from 'prom-client';
+import prom from 'prom-client';
+import { nonEmptyStringOrDefault } from '../../core/StringUtils.ts';
 
 dayjs.extend(utc)
 dayjs.extend(isBetween);
@@ -217,7 +216,7 @@ export class MSCache {
 
     protected initCacheable = async (cacheFor: string, primaryConfig: CacheConfig, secondaryConfig?: CacheConfig) => {
 
-        let logger = childLogger(this.logger, cacheFor);
+        const logger = childLogger(this.logger, cacheFor);
         const providerHints = [];
         if(primaryConfig.provider === false) {
             const cache = new Cacheable({primary: noopKeyv});
@@ -277,11 +276,12 @@ export class MSCache {
                 throw e;
             }
         }
+        const confDir = getConfigDir();
         if (config.provider === 'file') {
-            logger.debug(`Building file cache from ${path.join(config.connection ?? configDir, `${namespace}.cache`)}`);
+            logger.debug(`Building file cache from ${path.join(config.connection ?? confDir, `${namespace}.cache`)}`);
 
             try {
-                const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection ?? configDir, cacheId: `${namespace}.cache` }, {ttl: config.ttl}, logger);
+                const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection ?? confDir, cacheId: `${namespace}.cache` }, {ttl: config.ttl}, logger);
                 return keyvFile;
             } catch (e) {
                 throw e;
@@ -329,12 +329,12 @@ export class MSCache {
         if (!this.hasInit) {
             let authConfig: CacheConfig | undefined;
             try {
-                if(asCacheConfig(this.config.scrobble)) {
+                if(asCacheConfig(this.config.auth)) {
                     authConfig = this.config.auth;
                 }
             } catch (e) {
                 this.logger.warn(new Error('Could not validate auth config! will fallback to memory cache only', {cause: e}));
-                this.cacheScrobble = await this.initCacheable('Scrobble', {provider: false});
+                this.cacheAuth = await this.initCacheable('Auth', {provider: false});
             }
 
             this.cacheAuth = await this.initCacheable('Auth', {provider: 'memory', ttl: '3m'}, authConfig);
@@ -538,7 +538,7 @@ export const parseUserConfig = (config: CacheConfigUser = {}, parentLogger: Logg
             // } = {},
             scrobble: {
                 provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheScrobbleProvider | undefined) ?? 'file'),
-                connection = (process.env.CACHE_SCROBBLE_CONN ?? configDir),
+                connection = (process.env.CACHE_SCROBBLE_CONN ?? getConfigDir()),
                 ...restScrobble
             } = {},
             auth: {
@@ -563,7 +563,7 @@ export const parseUserConfig = (config: CacheConfigUser = {}, parentLogger: Logg
         if(authProvider === 'valkey') {
             if(valkey === undefined) {
                 logger.warn(`Auth Provider set to 'valkey' but not valkey connection string was not provided, falling back to file.`);
-                authConn = configDir;
+                authConn = getConfigDir();
                 authProvider = 'file';
             } else {
                 authConn = valkey;
@@ -572,7 +572,7 @@ export const parseUserConfig = (config: CacheConfigUser = {}, parentLogger: Logg
             if(authProvider !== 'file') {
                 logger.warn(`Unsupported provider given for auth: ${authProvider}`);
             }
-            authConn = configDir;
+            authConn = getConfigDir();
             authProvider = 'file';
         }
 

src/backend/common/MaybeLogger.ts

@@ -1,4 +1,4 @@
-import { Logger } from "@foxxmd/logging";
+import type { Logger } from "@foxxmd/logging";
 
 
 export class MaybeLogger {

src/backend/common/database/Database.ts

@@ -1,18 +1,16 @@
-import { configDir } from '../index.js';
+import { getDataDir } from '../index.ts';
 import * as path from 'path';
-import { promises as fs } from 'fs'
-import { childLogger, Logger } from '@foxxmd/logging';
-import { loggerNoop } from '../MaybeLogger.js';
-import { fileExists, fileOrDirectoryIsWriteable } from '../../utils/FSUtils.js';
-import { COMPACTABLE, compactableProperties, CompactableProperty, DEFAULT_RETENTION_COMPACT_AFTER, DEFAULT_RETENTION_DELETE_AFTER, RententionGranular, RetentionConfig, RetentionConfigValue, RetentionOption, RetentionValue, RetentionValueUnparsed } from '../infrastructure/config/database.js';
-import { DurationValue } from '../infrastructure/Atomic.js';
-import { Duration } from 'dayjs/plugin/duration.js';
+import { childLogger, type Logger } from '@foxxmd/logging';
+import { loggerNoop } from '../MaybeLogger.ts';
+import { COMPACTABLE, type CompactableProperty, DEFAULT_RETENTION_COMPACT_AFTER, DEFAULT_RETENTION_DELETE_AFTER, type RetentionConfigValue, type RetentionOption, type RetentionValue, type RetentionValueUnparsed } from '../infrastructure/config/database.ts';
+import type {DurationValue} from '../infrastructure/Atomic.ts';
+import type {Duration} from 'dayjs/plugin/duration.js';
 import dayjs from 'dayjs';
-import { parseDurationFromDurationValue } from '../../utils/TimeUtils.js';
-import assert, { AssertionError } from 'node:assert';
+import { parseDurationFromDurationValue } from '../../utils/TimeUtils.ts';
+import assert from 'node:assert';
 import * as sqlite from 'node:sqlite';
-import { parseBoolStrict } from '../../utils.js';
-import { SimpleError } from '../errors/MSErrors.js';
+import { parseBoolStrict } from '../../utils.ts';
+import { SimpleError } from '../errors/MSErrors.ts';
 
 export const MEMORY_DB_NAME = ':memory:';
 export const isMemoryDb = (name: string): boolean => name === MEMORY_DB_NAME;
@@ -21,7 +19,7 @@ export const getDbPath = (name: string = 'ms', workingDirectory?: string): strin
     if (isMemoryDb(name)) {
         return MEMORY_DB_NAME;
     }
-    return path.resolve(workingDirectory ?? configDir, `${name}.db`);
+    return path.resolve(workingDirectory ?? getDataDir(), `${name}.db`);
 }
 
 export const getDbBackupPath = (dbPath: string, suffix?: string): string => {
@@ -34,7 +32,7 @@ export const backupDb = async (db: sqlite.DatabaseSync, dbPath: string, opts: {
 
     const {
         logger: parentLogger = loggerNoop,
-        suffix,
+        suffix = dayjs().unix().toString(),
     } = opts;
 
     const logger = childLogger(parentLogger, 'Migrations');

src/backend/common/database/appMigrations/001_lifecycleLoc.ts

@@ -0,0 +1,66 @@
+import type { SqliteDatabase, Migration } from 'sqlite-up';
+import type {MigrateBaseContext} from '../appMigrator.ts';
+import { plays as drizzlePlays } from '../drizzle/schema/schema.ts';
+import clone from 'clone';
+import { eq } from 'drizzle-orm';
+import type {PlayLifecycle, PlayObject} from '../../../../core/Atomic.ts';
+
+
+export const up: Migration<MigrateBaseContext>['up'] = async (db: SqliteDatabase, ctx: MigrateBaseContext): Promise<void> => {
+
+    ctx.logger.info('Migrating Play lifecycle data to top-level location. This may take some time...');
+
+    let more = true;
+    let offset = 0,
+        processed = 0,
+        updated = 0;
+    while (more) {
+        const playRows = await ctx.db.select().from(drizzlePlays).limit(100).offset(offset);
+        for (const row of playRows) {
+            const play = row.play as PlayObject<{lifecycle?: PlayLifecycle}>;
+            try {
+                const {
+                    meta: {
+                        lifecycle,
+                        lifecycle: {
+                            steps = [],
+                            scrobble
+                        } = {}
+                    } = {}
+                } = play;
+                if (lifecycle === undefined) {
+                    // already migrated or unneeded
+                    processed++;
+                    continue;
+                }
+                if (steps.length > 0) {
+                    row.play.lifecycle = clone(steps);
+                }
+                if (scrobble !== undefined) {
+                    row.play.scrobble = clone(scrobble);
+                }
+                // @ts-expect-error
+                delete row.play.meta.lifecycle;
+                await ctx.db.update(drizzlePlays).set({ play: row.play }).where(eq(drizzlePlays.id, row.id));
+                updated++;
+                processed++;
+            } catch (e) {
+                ctx.logger.warn(new Error(`Failed to migrate Play ${row.id} (${row.uid})`, { cause: e }));
+            }
+        }
+        offset += 100;
+        ctx.logger.verbose(`Migration Progress: Processed ${processed} | Updated ${updated}`);
+        if(playRows.length < 100) {
+            more = false;
+        }
+    }
+
+
+    // Migration code here
+    // context is passed as ctx
+};
+
+export const down: Migration<MigrateBaseContext>['down'] = async (db: SqliteDatabase, ctx: MigrateBaseContext): Promise<void> => {
+    // Rollback code here
+    // context is passed as ctx
+};

src/backend/common/database/appMigrator.ts

@@ -1,27 +1,29 @@
-import { DbConcrete } from "./drizzle/drizzleUtils.js";
-import { loggerNoop } from "../MaybeLogger.js";
+import type {DbConcrete} from "./drizzle/drizzleUtils.ts";
+import { loggerNoop } from "../MaybeLogger.ts";
 import * as path from 'path';
-import { childLogger, Logger } from "@foxxmd/logging";
-import { projectDir } from "../index.js";
+import { childLogger, type Logger } from "@foxxmd/logging";
 import { Migrator } from 'sqlite-up';
-import { MigrationStatus } from "../infrastructure/Atomic.js";
+import type {MigrationStatus} from "../infrastructure/Atomic.ts";
+import { projectRootDir } from "../infrastructure/Atomic.ts";
 
 export interface MigrateBaseContext {
     db: DbConcrete
     logger: Logger
 }
 
+const migrationsTable = '__app_migrations';
+
 export const getAppMigrationStatus = async (db: DbConcrete, opts: {logger?: Logger, migrationsAppFolder?: string} = {}): Promise<MigrationStatus> => {
     const {
     logger: parentLogger = loggerNoop,
-    migrationsAppFolder = path.resolve(projectDir, 'src/backend/common/database/appMigrations')
+    migrationsAppFolder = path.resolve(projectRootDir, 'src/backend/common/database/appMigrations')
     } = opts;
     const logger = childLogger(parentLogger, 'App Migrations');
 
     const migrator = new Migrator({
         db: db.$client,
         migrationsDir: migrationsAppFolder,
-        migrationsTable: '__app_migrations'
+        migrationsTable: migrationsTable
     });
 
     const status = await migrator.status();
@@ -38,14 +40,14 @@ export const getAppMigrationStatus = async (db: DbConcrete, opts: {logger?: Logg
 export const migrateApp = async (db: DbConcrete, opts: {logger?: Logger, migrationsAppFolder?: string} = {}): Promise<string[]> => {
     const {
     logger: parentLogger = loggerNoop,
-    migrationsAppFolder = path.resolve(projectDir, 'src/backend/common/database/appMigrations')
+    migrationsAppFolder = path.resolve(projectRootDir, 'src/backend/common/database/appMigrations')
     } = opts;
-    const logger = childLogger(parentLogger, 'App');
+    const logger = childLogger(parentLogger, ['App', 'Migrations']);
 
     const migrator = new Migrator<MigrateBaseContext>({
         db: db.$client,
         migrationsDir: migrationsAppFolder,
-        migrationsTable: 'appMigrations'
+        migrationsTable: migrationsTable
     });
 
     migrator.on('migration:applied', function (name: string, batch: number): void {

src/backend/common/database/drizzle/drizzleTypes.ts

@@ -1,11 +1,11 @@
-import { DBQueryConfig, DBQueryConfigWith, ExtractTablesFromSchema, KnownKeysOnly, RelationFieldsFilterInternals, Many, InferSelectModel, ExtractTablesWithRelations, type BuildQueryResult, RelationsFilter } from "drizzle-orm";
-import { components, componentMigrations, playInputs, plays, queueStates, relations } from "./schema/schema.js";
-import {TSchema, TableName, Schema } from "./schema/schema.js";
-import { MarkOptional, MarkRequired } from "ts-essentials";
+import type {DBQueryConfig, DBQueryConfigWith, KnownKeysOnly, RelationFieldsFilterInternals, BuildQueryResult, RelationsFilter} from "drizzle-orm";
+import type { components, componentMigrations, playInputs, plays, queueStates, relations, playsHistorical } from "./schema/schema.ts";
+import type {TSchema, TableName} from "./schema/schema.ts";
 
 
 export type ComponentNew = typeof components.$inferInsert;
 export type ComponentSelect = GenericRelationResult<'components', 'migrations'>;
+export type ComponentMinimalSelect = typeof components.$inferSelect;
 
 export type ComponentMigrationNew = typeof componentMigrations.$inferInsert;
 export type ComponentMigrationSelect = typeof componentMigrations.$inferSelect;
@@ -21,6 +21,8 @@ export type PlaySelectWithQueueStates = GenericRelationResult<'plays', 'queueSta
 export type PlayWith<K extends keyof TSchema['plays']["relations"]> = GenericRelationResult<'plays', K>;
 export type PlayNew = typeof plays.$inferInsert;
 
+export type PlayHistoricalSelect = typeof playsHistorical.$inferSelect;
+export type PlayHistoricalNew = typeof playsHistorical.$inferInsert;
 
 // useful references for building types
 // https://github.com/drizzle-team/drizzle-orm/discussions/2596

src/backend/common/database/drizzle/drizzleUtils.ts

@@ -1,24 +1,23 @@
 import { drizzle } from 'drizzle-orm/node-sqlite';
 import { migrate } from 'drizzle-orm/node-sqlite/migrator';
-import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
-import { sql as dsl, LogWriter, Logger as DrizzleLogger } from 'drizzle-orm';
+import type { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
+import { sql as dsl, type Logger as DrizzleLogger } from 'drizzle-orm';
 import * as fs from 'fs/promises';
 import * as path from 'path';
-import { backupDb, getDbPath, getDbBackupPath, MEMORY_DB_NAME } from '../Database.js';
-import { fileExists, fileOrDirectoryIsWriteable } from '../../../utils/FSUtils.js';
-import { childLogger, Logger, LogLevel } from '@foxxmd/logging';
-import { loggerNoop } from '../../MaybeLogger.js';
-import { projectDir } from '../../index.js';
-import { relations } from './schema/schema.js';
-import { addToContext, executeQuery } from './logContext.js';
-import { DatabaseSync } from 'node:sqlite';
-import { migrateApp, getAppMigrationStatus } from '../appMigrator.js';
-import { MigrationStatus } from '../../infrastructure/Atomic.js';
+import { backupDb, getDbBackupPath, MEMORY_DB_NAME } from '../Database.ts';
+import { fileExists, fileOrDirectoryIsWriteable } from '../../../utils/FSUtils.ts';
+import { childLogger, type Logger, type LogLevel } from '@foxxmd/logging';
+import { loggerNoop } from '../../MaybeLogger.ts';
+import { relations } from './schema/schema.ts';
+import { addToContext, executeQuery } from './logContext.ts';
+import { migrateApp, getAppMigrationStatus } from '../appMigrator.ts';
+import type {MigrationStatus} from '../../infrastructure/Atomic.ts';
+import { projectRootDir } from "../../infrastructure/Atomic.ts";
 
 export async function getDbMigrationStatus(dbVal: string | DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise<MigrationStatus> {
   const {
     logger: parentLogger = loggerNoop,
-    migrationsFolder = path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')
+    migrationsFolder = path.resolve(projectRootDir, 'src/backend/common/database/drizzle/migrations')
   } = opts;
   const logger = childLogger(parentLogger, 'Migrations');
   
@@ -92,7 +91,7 @@ export const migrateDb = async (db: DbConcrete, opts: {logger?: Logger, migratio
 
   try {
     logger.info('Starting migrations...');
-    await executeQuery('migrations', async () => migrate(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') }), logger, process.env.LOG_MIGRATION === 'true' ? true : 'error');
+    await executeQuery('migrations', async () => migrate(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectRootDir, 'src/backend/common/database/drizzle/migrations') }), logger, process.env.LOG_MIGRATION === 'true' ? true : 'error');
     logger.info('Migrations complete');
   } catch (e) {
     throw new Error('Failed to migrate database', { cause: e });
@@ -108,7 +107,7 @@ export const migrateDbSync = (db: ReturnType<typeof drizzle>, opts: {logger?: Lo
 
   try {
     logger.info('Starting migrations...');
-    migrate(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') });
+    migrate(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectRootDir, 'src/backend/common/database/drizzle/migrations') });
     logger.info('Migrations complete');
   } catch (e) {
     throw new Error('Failed to migrate database', { cause: e });

src/backend/common/database/drizzle/entityUtils.ts

@@ -1,14 +1,13 @@
 import assert from "node:assert";
-import { PlayNew, PlaySelect, PlaySelectWithQueueStates } from "./drizzleTypes.js";
-import { PlayInputNew } from "./drizzleTypes.js";
-import { QueueStateNew } from "./drizzleTypes.js";
-import { ComponentNew } from "./drizzleTypes.js";
-import { MarkOptional, MarkRequired } from "ts-essentials";
-import { CLIENT_DEAD_QUEUE, DeadLetterScrobble, ErrorLike, PlayObject } from "../../../../core/Atomic.js";
-import dayjs, { Dayjs } from "dayjs";
-import { asPlay } from "../../../../core/PlayMarshalUtils.js";
-import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../utils/PlayComparisonUtils.js";
-import { hashObject } from "../../../utils/StringUtils.js";
+import type {PlayHistoricalNew, PlayHistoricalSelect, PlayNew, PlaySelect, PlaySelectWithQueueStates} from "./drizzleTypes.ts";
+import type {PlayInputNew} from "./drizzleTypes.ts";
+import type {QueueStateNew} from "./drizzleTypes.ts";
+import type {ComponentNew} from "./drizzleTypes.ts";
+import type { MarkOptional } from "ts-essentials";
+import { CLIENT_DEAD_QUEUE, type DeadLetterScrobble, type ErrorLike, type PlayObject } from "../../../../core/Atomic.ts";
+import dayjs from "dayjs";
+import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../utils/PlayComparisonUtils.ts";
+import { hashObject } from "../../../utils/StringUtils.ts";
 import { serializeError } from "serialize-error";
 
 export const generateComponentEntity = (data: MarkOptional<ComponentNew, 'uid'>): ComponentNew => {
@@ -20,6 +19,7 @@ export const generateComponentEntity = (data: MarkOptional<ComponentNew, 'uid'>)
 }
 
 export type PlayEntityOpts = Partial<Pick<PlayNew, 'seenAt' | 'playedAt' | 'uid' | 'state' | 'parentId' | 'componentId'>> & { error?: ErrorLike };
+export type PlayHistoricalEntityOpts = Partial<Pick<PlayHistoricalNew, 'seenAt' | 'playedAt' | 'uid' | 'componentId'>>;
 
 export const generatePlayEntity = (play: PlayObject, opts: PlayEntityOpts = {}): PlayNew => {
     const {
@@ -51,12 +51,12 @@ export const generatePlayEntity = (play: PlayObject, opts: PlayEntityOpts = {}):
 
 export type PlayHydateOptions = 'asPlay' | 'id' | 'uid';
 
-export const hydratePlaySelect = (select: PlaySelect, opts: PlayHydateOptions[] = ['id','uid']): PlayObject => {
+export const hydratePlaySelect = <T extends PlaySelect | PlayHistoricalSelect>(select: T, opts: PlayHydateOptions[] = ['id','uid']): PlayObject => {
     if(opts.length === 0) {
         return select.play;
     }
 
-    let res = select.play;
+    const res = select.play;
     // if(opts.includes('asPlay')) {
     //     res = asPlay(res);
     // }

src/backend/common/database/drizzle/logContext.ts

@@ -1,4 +1,4 @@
-import { Logger } from '@foxxmd/logging'
+import type { Logger } from '@foxxmd/logging'
 import { AsyncLocalStorage } from 'async_hooks'
 
 // based on https://numeric.substack.com/p/upgrading-drizzleorm-logging-with

src/backend/common/database/drizzle/repositories/BaseRepository.ts

@@ -1,36 +1,22 @@
-import { childLogger, Logger } from "@foxxmd/logging";
-import { DbConcrete } from "../drizzleUtils.js";
-import { CompareOpKey } from "../drizzleTypes.js";
-import { Dayjs } from "dayjs";
-import { RelationsFieldFilter, eq, inArray } from "drizzle-orm";
-import { loggerNoop } from "../../../MaybeLogger.js";
-import { capitalize } from "../../../../../core/StringUtils.js";
-import { getConfigByTableName, relations, TableName } from "../schema/schema.js";
+import { childLogger, type Logger } from "@foxxmd/logging";
+import type { DbConcrete } from "../drizzleUtils.ts";
+import type { Dayjs } from "dayjs";
+import { type RelationsFieldFilter, eq, inArray } from "drizzle-orm";
+import { loggerNoop } from "../../../MaybeLogger.ts";
+import { capitalize } from "../../../../../core/StringUtils.ts";
+import { getConfigByTableName, type TableName } from "../schema/schema.ts";
+import assert from 'node:assert';
+import type { Cacheable } from "cacheable";
+import type { DateLike } from "../../../../../core/Atomic.ts";
+import type { CompareDateBetween, CompareDateSingle } from "../../../../../core/Api.ts";
 
 export interface DrizzleRepositoryOpts {
     logger?: Logger
+    cache?: Cacheable
     componentId?: number
 }
 
-export type CompareDateOp = {
-    type: CompareOpKey<Dayjs>
-    date: Dayjs
-} | {
-    type: 'between',
-    range: [Dayjs, Dayjs],
-    inclusive?: boolean
-}
-
-export interface PaginatedQueryResponse {
-    limit: number,
-    offset: number
-}
-
-export interface PaginatedResponse<T> {
-    data: T[]
-    meta: PaginatedQueryResponse
-}
-
+export type CompareDateOp<D extends DateLike = Dayjs> = CompareDateSingle<D> | CompareDateBetween<D>;
 export interface ComponentConstrainedRepoOpts {
     componentId?: number
 }
@@ -43,6 +29,7 @@ export abstract class DrizzleBaseRepository<T extends TableName> {
     table: ReturnType<typeof getConfigByTableName<T>>
     db: DbConcrete;
     componentId?: number
+    cache?: Cacheable
 
     constructor(db: DbConcrete, tableName: TableName, displayName: string, opts: DrizzleRepositoryOpts = {}) {
         this.db = db;
@@ -51,6 +38,7 @@ export abstract class DrizzleBaseRepository<T extends TableName> {
         this.table = getConfigByTableName(this.tableName);
         this.logger = childLogger(opts.logger ?? loggerNoop, ['Database', capitalize(displayName)]);
         this.componentId = opts.componentId;
+        this.cache = opts.cache;
     }
 
     async deleteByIds(ids: number[]): Promise<void> {
@@ -58,6 +46,7 @@ export abstract class DrizzleBaseRepository<T extends TableName> {
     }
 
     async updateById(id: number, data: Partial<typeof this.table.$inferInsert>): Promise<void> {
+        assert(id !== null && id !== undefined, `${id === null ? 'null' : 'undefined'} given for entity id`);
         await this.db.update(this.table).set(data).where(eq(this.table.id, id));
     }
 
@@ -90,13 +79,12 @@ export class GenericRepository<T extends TableName> extends DrizzleBaseRepositor
 }
 
 export const buildDateCompare = (data: CompareDateOp): RelationsFieldFilter<Dayjs> => {
-    let q: RelationsFieldFilter<Dayjs> = {};
     if (data.type !== 'between') {
-        q = {
+        return {
             [data.type]: data.date
         }
     } else {
-        q = {
+        return {
             AND: [
                 {
                     [data.inclusive ?? true ? 'gte' : 'gt']: data.range[0]
@@ -107,5 +95,4 @@ export const buildDateCompare = (data: CompareDateOp): RelationsFieldFilter<Dayj
             ]
         }
     }
-    return q;
 }

src/backend/common/database/drizzle/repositories/ComponentRepository.ts

@@ -1,9 +1,9 @@
-import { Logger } from "drizzle-orm";
-import { DrizzleBaseRepository, DrizzleRepositoryOpts } from "./BaseRepository.js";
-import { DbConcrete } from "../drizzleUtils.js";
-import { ComponentNew, ComponentSelect, FindWhere } from "../drizzleTypes.js";
-import { components } from "../schema/schema.js";
-import { generateComponentEntity } from "../entityUtils.js";
+import { DrizzleBaseRepository, type DrizzleRepositoryOpts } from "./BaseRepository.ts";
+import type {DbConcrete} from "../drizzleUtils.ts";
+import type {ComponentSelect, FindWhere} from "../drizzleTypes.ts";
+import { components } from "../schema/schema.ts";
+import { generateComponentEntity } from "../entityUtils.ts";
+import type {ComponentType} from "../../../../../core/Atomic.ts";
 
 export class DrizzleComponentRepository extends DrizzleBaseRepository<'components'> {
 
@@ -11,7 +11,7 @@ export class DrizzleComponentRepository extends DrizzleBaseRepository<'component
         super(db, 'components', 'Component', opts);
     }
 
-    findOrInsert = async (data: { mode: 'source' | 'client', type: string, uid?: string, name?: string }): Promise<ComponentSelect> => {
+    findOrInsert = async (data: { mode: ComponentType, type: string, uid?: string, name?: string }): Promise<ComponentSelect> => {
         const where: FindWhere<'components'> = {
             mode: data.mode,
             type: data.type,

src/backend/common/database/drizzle/repositories/PlayHistoricalRepository.ts

@@ -0,0 +1,318 @@
+import { type DbConcrete, runTransaction } from "../drizzleUtils.ts";
+import { type PlayObject, TA_DEFAULT_ACCURACY, type TemporalAccuracy } from "../../../../../core/Atomic.ts";
+import { generatePlayEntity, hydratePlaySelect, type PlayHydateOptions, type PlayHistoricalEntityOpts } from "../entityUtils.ts";
+import { plays, playsHistorical } from "../schema/schema.ts";
+import type {FindWhere, FindMany, WhereClause, PlayHistoricalSelect, PlayHistoricalNew} from "../drizzleTypes.ts";;
+import type { MarkOptional } from "ts-essentials";
+import { removeUndefinedKeys } from '../../../../../core/DataUtils.ts';
+import type {Dayjs} from "dayjs";
+import { inArray, sql } from "drizzle-orm";
+import { buildDateCompare, type CompareDateOp, type ComponentConstrainedRepoOpts, DrizzleBaseRepository, type DrizzleRepositoryOpts } from "./BaseRepository.ts";
+import type {PaginatedResponse} from "../../../../../core/Api.ts";
+import { hashObject } from "../../../../utils/StringUtils.ts";
+import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../../utils/PlayComparisonUtils.ts";
+import { comparePlayTemporally, getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy } from "../../../../utils/TimeUtils.ts";
+import type {SourceType} from "../../../../../core/Atomic.ts";
+import { getTemporallyCloseDateCompareOp } from "./PlayRepository.ts";
+
+// https://github.com/drizzle-team/drizzle-orm/issues/695 may be useful for typing models with relations?
+
+export interface PlayWhereOpts {
+    componentId?: number
+    seenAt?: CompareDateOp
+    playedAt?: CompareDateOp
+    uid?: string[]
+}
+
+export interface QueryPlaysOpts extends PlayWhereOpts {
+    sort?: 'seenAt' | 'playedAt'
+    order?: 'asc' | 'desc'
+    limit?: number
+    offset?: number
+}
+
+export interface HydrateOpts {
+   hydrate?: PlayHydateOptions[] 
+}
+
+export type RepositoryCreatePlayHistoricalOpts = PlayHistoricalEntityOpts
+    & Pick<PlayHistoricalNew, 'play' | 'componentId'>;
+
+type PlayIdentifierPrimitiveMap = {
+  uid: string;
+  id: number;
+};
+
+const identifierExtractor: { [K in keyof PlayIdentifierPrimitiveMap]: (play: {id: number, uid: string}) => PlayIdentifierPrimitiveMap[K] } = {
+  id: (play) => play.id,
+  uid: (play) => play.uid,
+};
+export class DrizzlePlayHistoricalRepository extends DrizzleBaseRepository<'playsHistorical'> {
+
+    constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
+        super(db, 'playsHistorical', 'Plays Historical', opts);
+    }
+
+    findByUid = async (uid: string, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PlayHistoricalSelect | undefined> => {
+        const res = await this.db.query.playsHistorical.findFirst({
+            where: {
+                uid,
+                componentId: opts.componentId ?? this.componentId
+            }
+        });
+        res.play = hydratePlaySelect(res, opts.hydrate);
+        return res;
+    }
+
+    hasByUid = async (uid: string, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<boolean> => {
+        const res = await this.db.query.playsHistorical.findFirst({
+            columns: {id: true},
+            where: {
+                uid,
+                componentId: opts.componentId ?? this.componentId
+            }
+        });
+        return res !== undefined;
+    }
+
+    createPlays = async (entitiesOpts: RepositoryCreatePlayHistoricalOpts[], opts: HydrateOpts = {}) => {
+
+        const {
+            hydrate
+        } = opts;
+        let playRows: PlayHistoricalSelect[];
+
+        await runTransaction(this.db, async () => {
+
+            const entitiesData = entitiesOpts.map((data) => {
+                const {
+                    play,
+                    ...rest
+                } = data;
+                return generatePlayEntity(play, { componentId: this.componentId, ...rest });
+            });
+
+            playRows = await this.db.insert(playsHistorical).values(entitiesData).returning();
+        });
+
+        return playRows.map(x => ({...x, play: hydratePlaySelect(x, hydrate)}));
+    }
+
+    findPlays = async (args: QueryPlaysOpts, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PlayHistoricalSelect[]> => {
+        const {
+            hydrate,
+            componentId = this.componentId
+        } = opts;
+        // this does not work as type for query variable
+        // it erases the result type for some reason
+        //
+        // Parameters<typeof this.db.query.plays.findMany>[0]
+        
+        // this does work but it is also integrated into FindWith
+        //let withQuery: Parameters<typeof this.db.query.plays.findMany>[0]['with'] = undefined;
+
+        let query: FindMany<'playsHistorical'> = {
+            limit: args.limit,
+            offset: args.offset
+        };
+
+        query.where = buildPlayHistoricalWhere({componentId: componentId,  ...args});
+
+        if (args.sort !== undefined) {
+            query.orderBy = {
+                [args.sort]: args.order ?? 'desc'
+            }
+        } else {
+            query.orderBy = {
+                id: 'asc'
+            }
+        }
+
+        query = removeUndefinedKeys(query);
+        const results = await this.db.query.playsHistorical.findMany(query);
+        return results.map((x) => ({...x, play: hydratePlaySelect(x, hydrate)}));
+    }
+
+    findPlayIds = async (args: QueryPlaysOpts, opts: ComponentConstrainedRepoOpts = {}): Promise<number[]> => {
+        const {
+            componentId = this.componentId
+        } = opts;
+
+        let query: FindMany<'playsHistorical'> = {
+            limit: args.limit,
+            offset: args.offset,
+        };
+
+        query.where = buildPlayHistoricalWhere({componentId: componentId,  ...args});
+
+        if (args.sort !== undefined) {
+            query.orderBy = {
+                [args.sort]: args.order ?? 'desc'
+            }
+        } else {
+            query.orderBy = {
+                id: 'asc'
+            }
+        }
+
+        query = removeUndefinedKeys(query);
+        const results = await this.db.query.playsHistorical.findMany({
+            ...query,
+            limit: args.limit,
+            offset: args.offset,
+            columns: {id: true},
+            orderBy: args.sort !== undefined ? {[args.sort]: args.order ?? 'desc'} : {id: 'asc'},
+        });
+        return results.map((x) => x.id);
+    }
+
+    findPlayIdentifiers = async <T extends keyof PlayIdentifierPrimitiveMap>(args: QueryPlaysOpts, identifier: T, opts: ComponentConstrainedRepoOpts = {}): Promise<PlayIdentifierPrimitiveMap[T][]> => {
+        const {
+            componentId = this.componentId,
+        } = opts;
+
+        const results = await this.db.query.playsHistorical.findMany({
+            limit: args.limit,
+            offset: args.offset,
+            columns: {id: true, uid: true},
+            orderBy: args.sort !== undefined ? {[args.sort]: args.order ?? 'desc'} : {id: 'asc'},
+            where: buildPlayHistoricalWhere({componentId: componentId,  ...args})
+        });
+
+        // we getting fancy now
+        return results.map(identifierExtractor[identifier]);
+    }
+
+    findPlaysPaginated = async (args: QueryPlaysOpts, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PaginatedResponse<PlayHistoricalSelect>> => {
+        const {
+            limit = 100,
+            offset = 0,
+            ...rest
+        } = args;
+        const clampedLimit = Math.min(limit, 100);
+        const res = await this.findPlays({limit: clampedLimit, offset, ...rest}, opts);
+        return {data: res, meta: {limit: clampedLimit, offset}};
+    }
+
+    // async updateById(id: number, data: Partial<PlayNew>): Promise<void> {
+    //     if(data.play !== undefined) {
+    //         data.play = withoutDbAwareness(data.play);
+    //     }
+    //     super.updateById(id, data);
+    // }
+
+    deletePlays = async (playsData: (Pick<PlayHistoricalSelect, 'id'> | number)[]) => {
+        const ids = playsData.map(x => typeof x === 'number' ? x : x.id);
+        await this.db.delete(playsHistorical).where(inArray(plays.id, ids));
+    }
+
+    public checkExisting = async (play: PlayObject, opts: {taAccuracy?: TemporalAccuracy[]} & ComponentConstrainedRepoOpts = {}): Promise<PlayHistoricalSelect | undefined> => {
+        const {
+            componentId = this.componentId,
+            taAccuracy = TA_DEFAULT_ACCURACY,
+        } = opts;
+        const hash = hashObject(playContentBasicInvariantTransform(play).data);
+
+        // we get all plays with a play date between playdate - (source accuracy) AND (playDateCompleted or playDate) + (source accuracy)
+        // which we can then use with temporal comparison to make sure we are comparing the correct dates
+        //
+        // this isn't as fast as just comparing playDate directly but its still much faster/cheaper than paginating plays and doing everything in-memory
+        const dateGranularity = getTemporalAccuracyCloseVal(play.meta.source as SourceType);
+        let endRange: Dayjs;
+        if(play.data.playDateCompleted !== undefined) {
+            // this will be present if source reports it
+            // or we tracked it live with MemorySource
+            endRange = play.data.playDateCompleted.add(dateGranularity, 's');
+        } else {
+            endRange = play.data.playDate.add(dateGranularity, 's');
+        }
+        const where: FindWhere<'playsHistorical'> = {
+            componentId,
+            playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play)),
+        };
+
+        const mbidId = playMbidIdentifier(play);
+        if(mbidId !== undefined) {
+            where.AND = [
+                {
+                    OR: [
+                        {
+                            playHash: hash
+                        },
+                        {
+                            mbidIdentifier: mbidId
+                        }
+                    ]
+                }
+            ]
+        } else {
+            where.playHash = hash;
+        }
+
+        const res = await this.db.query.playsHistorical.findMany({
+            where
+        });
+        if(res.length === 0) {
+            return undefined;
+        }
+        return res.map(x => ({...x, play: hydratePlaySelect(x)})).find(x => {
+            const temporalComparison = comparePlayTemporally(x.play, play);
+            return hasAcceptableTemporalAccuracy(temporalComparison.match, taAccuracy)
+        })
+    }
+
+    public getTemporallyClosePlays = async (play: PlayObject, opts: {bufferTime?: number} & ComponentConstrainedRepoOpts = {}): Promise<PlayHistoricalSelect[]> => {
+        const {
+            componentId = this.componentId,
+            bufferTime
+        } = opts;
+
+        const query: FindMany<'playsHistorical'> = {};
+
+        const where: FindWhere<'playsHistorical'> = {
+            componentId,
+            playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play, {bufferTime})),
+        };
+        query.where = where;
+
+        return ((await this.db.query.playsHistorical.findMany({
+            where,
+        })) as PlayHistoricalSelect[]).map(x => ({...x, play: hydratePlaySelect(x)}));
+    }
+
+    public getPlayCountByComponent = async () => {
+
+        const res = await this.db.all(sql`select componentId, count(*) from plays_historical p
+group by componentId;`);
+        return res;
+    }
+}
+
+export const buildPlayHistoricalWhere = (args: PlayWhereOpts): WhereClause<'playsHistorical'> => {
+    // old way
+    // let where: Parameters<(ReturnType<typeof getDb>)['query']['plays']['findMany']>[0]['where'] = {
+    // };
+    const where: FindWhere<'playsHistorical'> = {
+        componentId: args.componentId
+    };
+    if (args.seenAt !== undefined) {
+        where.seenAt = buildDateCompare(args.seenAt);
+    }
+    if (args.playedAt !== undefined) {
+        where.playedAt = buildDateCompare(args.playedAt);
+    }
+    if(args.uid !== undefined) {
+        where.uid = {
+            in: args.uid
+        }
+    }
+    return where;
+}
+
+export const playToRepositoryCreatePlayHistoricalOpts = (data: MarkOptional<RepositoryCreatePlayHistoricalOpts, 'componentId'>): RepositoryCreatePlayHistoricalOpts => {
+    return {
+        play: data.play,
+        uid: data.play.meta?.playId,
+        ...data
+    }
+}

src/backend/common/database/drizzle/repositories/PlayRepository.ts

@@ -1,43 +1,45 @@
-import { childLogger, Logger, LoggerAppExtras } from "@foxxmd/logging";
-import { DbConcrete, runTransaction } from "../drizzleUtils.js";
-import { loggerNoop } from "../../../MaybeLogger.js";
-import { ErrorLike, PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT, TemporalAccuracy } from "../../../../../core/Atomic.js";
-import { generateInputEntity, generatePlayEntity, PlayEntityOpts, hydratePlaySelect, PlayHydateOptions } from "../entityUtils.js";
-import { playInputs, plays, queueStates, relations } from "../schema/schema.js";
-import { PlayNew, PlaySelect, PlayInputNew, FindWhere, FindMany, QueueStateSelect, FindWith, PlaySelectWithQueueStates, WhereClause, PlayWith } from "../drizzleTypes.js";;
-import { MarkOptional, MarkRequired, PathValue } from "ts-essentials";
-import { genGroupIdStrFromPlay, removeEmptyArrays, removeUndefinedKeys } from "../../../../utils.js";
-import dayjs, { Dayjs } from "dayjs";
-import { RelationsFieldFilter, eq, inArray, ne, notInArray, desc, asc, and, sql, Placeholder } from "drizzle-orm";
-import { CompactableProperty, RetentionOptions, retentionPlayTypes } from "../../../infrastructure/config/database.js";
-import { shortTodayAwareFormat } from "../../../../../core/TimeUtils.js";
-import { buildDateCompare, CompareDateOp, ComponentConstrainedRepoOpts, DrizzleBaseRepository, DrizzleRepositoryOpts, PaginatedQueryResponse, PaginatedResponse } from "./BaseRepository.js";
-import { asPlay } from "../../../../../core/PlayMarshalUtils.js";
-import assert, { Assert } from "node:assert";
-import { hashObject, parseArrayFromMaybeString } from "../../../../utils/StringUtils.js";
-import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../../utils/PlayComparisonUtils.js";
-import { comparePlayTemporally, getScrobbleTsSOCDate, getScrobbleTsSOCDateWithContext, getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy } from "../../../../utils/TimeUtils.js";
-import { SourceType } from "../../../infrastructure/config/source/sources.js";
+import { childLogger } from "@foxxmd/logging";
+import dayjs, { type Dayjs } from "dayjs";
+import { eq, inArray, relationsFilterToSQL, sql } from "drizzle-orm";
+import assert from "node:assert";
+import type { MarkOptional } from "ts-essentials";
+import { type DateLike, type DeepReplaceValue, type PlayObject, type PlayState, type QueueName, TA_DEFAULT_ACCURACY, type TemporalAccuracy } from "../../../../../core/Atomic.ts";
+import { removeUndefinedKeys } from '../../../../../core/DataUtils.ts';
+import { shortTodayAwareFormat } from "../../../../../core/TimeUtils.ts";
+import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../../utils/PlayComparisonUtils.ts";
+import { hashObject } from "../../../../utils/StringUtils.ts";
+import { comparePlayTemporally, getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy } from "../../../../utils/TimeUtils.ts";
+import { type CompactableProperty, type RetentionOptions, retentionPlayTypes } from "../../../infrastructure/config/database.ts";
+import type {SourceType} from "../../../../../core/Atomic.ts";
+import type {FindMany, FindWhere, FindWith, PlayInputNew, PlayNew, PlaySelect, PlaySelectWithQueueStates, PlayWith, QueueStateSelect, WhereClause} from "../drizzleTypes.ts";
+import { type DbConcrete, runTransaction } from "../drizzleUtils.ts";
+import { generateInputEntity, generatePlayEntity, hydratePlaySelect, type PlayEntityOpts, type PlayHydateOptions } from "../entityUtils.ts";
+import { playInputs, plays, relations } from "../schema/schema.ts";
+import { buildDateCompare, type CompareDateOp, type ComponentConstrainedRepoOpts, DrizzleBaseRepository, type DrizzleRepositoryOpts } from "./BaseRepository.ts";
+import type {PaginatedResponse} from "../../../../../core/Api.ts";
+import type {PaginatedQueryResponse} from "../../../../../core/Api.ts";
+;
 
 // https://github.com/drizzle-team/drizzle-orm/issues/695 may be useful for typing models with relations?
 
 export interface QueueCriteria {
-    queueName: string
+    queueName: QueueName
     queueStatus: QueueStateSelect['queueStatus'][] | QueueStateSelect['queueStatus']
 }
 
-export interface PlayWhereOpts {
+export interface PlayWhereOpts<D extends DateLike = Dayjs> {
     state?: PlaySelect['state'][]
     stateNot?: PlaySelect['state'][]
     componentId?: number
-    seenAt?: CompareDateOp
-    playedAt?: CompareDateOp
+    seenAt?: CompareDateOp<D>
+    playedAt?: CompareDateOp<D>
     queues?: QueueCriteria[]
     uid?: string[]
+    text?: string[]
 }
 
 export type WithPlayRelation = 'input' | 'parent' | 'parent-input' | 'queues';
-export interface QueryPlaysOpts extends PlayWhereOpts {
+export interface QueryPlaysOpts<D extends DateLike = Dayjs> extends PlayWhereOpts<D> {
     sort?: 'seenAt' | 'playedAt'
     order?: 'asc' | 'desc'
     with?: WithPlayRelation[]
@@ -45,6 +47,8 @@ export interface QueryPlaysOpts extends PlayWhereOpts {
     offset?: number
 }
 
+export type QueryPlaysOptsJson = DeepReplaceValue<QueryPlaysOpts<Dayjs>, [[Dayjs, string]]>;
+
 export interface HydrateOpts {
    hydrate?: PlayHydateOptions[] 
 }
@@ -73,26 +77,27 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         super(db, 'plays', 'Plays', opts);
     }
 
-    findByUid = async (uid: string, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PlaySelectWithQueueStates | undefined> => {
+    findByUid = async (uid: string, opts: HydrateOpts & Pick<QueryPlaysOpts, 'with'> & ComponentConstrainedRepoOpts = {}): Promise<PlaySelectWithQueueStates | undefined> => {
         const res = await this.db.query.plays.findFirst({
             where: {
                 uid,
                 componentId: opts.componentId ?? this.componentId
             },
-            with: {
-                queueStates: true
-            }
+            with: buildPlayWith([...(opts.with ?? []), 'queues'])
         });
+        if(res === undefined) {
+            return undefined;
+        }
         res.play = hydratePlaySelect(res, opts.hydrate);
-        return res;
+        return res as PlaySelectWithQueueStates;
     }
 
-    createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[], opts: HydrateOpts = {}) => {
+    createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[], opts: HydrateOpts = {}): Promise<PlayWith<'input'>[]> => {
 
         const {
             hydrate
         } = opts;
-        let playRows: PlaySelect[];
+        let playRows: PlayWith<'input'>[];
 
         await runTransaction(this.db, async () => {
 
@@ -105,9 +110,9 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
                 return generatePlayEntity(play, { componentId: this.componentId, ...rest });
             });
 
-            playRows = await this.db.insert(plays).values(entitiesData).returning();
+            const nakedPlays = await this.db.insert(plays).values(entitiesData).returning();
 
-            const inputDatas = playRows.map((x, index) => {
+            const inputDatas = nakedPlays.map((x, index) => {
                 const {
                     play,
                     input,
@@ -121,10 +126,10 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
             });
 
             const inputRow = await this.db.insert(playInputs).values(inputDatas);
-
+            playRows = nakedPlays.map((x, index) => ({...x, play: hydratePlaySelect(x, hydrate), input: inputRow[index]}));
         });
 
-        return playRows.map(x => ({...x, play: hydratePlaySelect(x, hydrate)}));
+        return playRows;
     }
 
     findPlays = async (args: QueryPlaysOpts, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PlaySelect[]> => {
@@ -211,6 +216,7 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
 
         query = removeUndefinedKeys(query);
         const results = await this.db.query.plays.findMany({
+            ...query,
             limit: args.limit,
             offset: args.offset,
             columns: {id: true},
@@ -244,7 +250,20 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         } = args;
         const clampedLimit = Math.min(limit, 100);
         const res = await this.findPlays({limit: clampedLimit, offset, ...rest}, opts) as T[];
-        return {data: res, meta: {limit: clampedLimit, offset}};
+
+        
+        const where = buildPlayWhere({componentId: args.componentId ?? this.componentId,  ...rest});
+  
+        // https://github.com/drizzle-team/drizzle-orm/issues/5218#issuecomment-3854900241
+        const filter = relationsFilterToSQL(plays,
+            // @ts-expect-error don't have a good way to type this yet
+            where,
+            relations.plays.relations,
+            relations);
+        // https://github.com/drizzle-team/drizzle-orm/discussions/3119#discussioncomment-16379557
+        const count = await this.db.$count(plays, filter);
+
+        return {data: res, meta: {limit: clampedLimit, offset, total: count}};
     }
 
     // async updateById(id: number, data: Partial<PlayNew>): Promise<void> {
@@ -274,7 +293,7 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
             dateComparer = 'updatedAt'
         } = opts;
 
-        let where: FindWhere<'plays'> = {
+        const where: FindWhere<'plays'> = {
             component: {
                 id: componentId
             },
@@ -324,8 +343,8 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
 
         const loggerDel = childLogger(this.logger, ['Retention', 'Delete']);
         const loggerCom = childLogger(this.logger, ['Retention', 'Compact']);
-        let summaryDelStates: string[] = [];
-        let summaryCompactStates: string[] = [];
+        const summaryDelStates: string[] = [];
+        const summaryCompactStates: string[] = [];
 
         loggerDel.debug('Starting cleanup...');
         for(const retentionType of retentionPlayTypes) {
@@ -360,7 +379,7 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         }
 
         const compactTypes = retentionOpts.compact;
-        let compactedFlags: CompactableProperty[] = [];
+        const compactedFlags: CompactableProperty[] = [];
         if(compactTypes.includes('input')) {
             compactedFlags.push('input');
         }
@@ -404,12 +423,14 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
                             }
 
                             compactedPlay = playRow.play;
-                            compactedPlay.meta.lifecycle.steps = compactedPlay.meta.lifecycle.steps.map(x => {
-                                if(x.inputs == undefined) {
-                                    return x;
-                                }
-                                return {...x, inputs: x.inputs.map(y => ({type: y.type, input: 'Removed by compaction'}))};
-                            });
+                            if(compactedPlay.lifecycle !== undefined) {
+                                compactedPlay.lifecycle = compactedPlay.lifecycle.map(x => {
+                                    if(x.inputs == undefined) {
+                                        return x;
+                                    }
+                                    return {...x, inputs: x.inputs.map(y => ({type: y.type, input: 'Removed by compaction'}))};
+                                });
+                            }
                         }
 
                         const updater = this.db.update(plays);
@@ -564,7 +585,7 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
             componentId = this.componentId,
             hydrate
         } = opts;
-        let where: FindWhere<'plays'> = {
+        const where: FindWhere<'plays'> = {
             componentId
         }
         if(retries !== undefined) {
@@ -614,7 +635,7 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
         } else {
             endRange = play.data.playDate.add(dateGranularity, 's');
         }
-        let where: FindWhere<'plays'> = {
+        const where: FindWhere<'plays'> = {
             componentId,
             playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play)),
         };
@@ -672,9 +693,9 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
             with: qWith
         } = opts;
 
-        let query: FindMany<'plays'> = {};
+        const query: FindMany<'plays'> = {};
 
-        let where: FindWhere<'plays'> = {
+        const where: FindWhere<'plays'> = {
             componentId,
             playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play, {bufferTime})),
         };
@@ -690,6 +711,29 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
             with: buildPlayWith(qWith)
         })) as PlayWith<'queueStates'>[]).map(x => ({...x, play: hydratePlaySelect(x)}));
     }
+
+    public getComponentPlayCountByState = async (componentId?: number): Promise<{state: PlayState, 'count(*)': number}[]> => {
+
+        const res = await this.db.all(sql`select state, count(*) from plays p
+where componentId = ${componentId ?? this.componentId}
+group by state;`);
+        return res as {state: PlayState, 'count(*)': number}[];
+    }
+
+    public getPlayCountByState = async () => {
+
+        const res = await this.db.all(sql`select state,componentId, count(*) from plays p
+group by state,componentId;`);
+        return res;
+    }
+
+    public getCompactedPlayCountByComponent = async () => {
+
+        const res = await this.db.all(sql`select componentId,compacted,count(*) from plays p
+where compacted IS NOT NULL
+group by componentId,compacted;`);
+        return res;
+    }
 }
 
 export const getTemporallyCloseDateCompareOp = (play: PlayObject, opts: {bufferTime?: number, useCompleted?: boolean} = {}): CompareDateOp => {
@@ -747,7 +791,7 @@ export const buildPlayWhere = (args: PlayWhereOpts): WhereClause<'plays'> => {
     // old way
     // let where: Parameters<(ReturnType<typeof getDb>)['query']['plays']['findMany']>[0]['where'] = {
     // };
-    let where: FindWhere<'plays'> = {
+    const where: FindWhere<'plays'> = {
         componentId: args.componentId
     };
     if (args.state !== undefined) {
@@ -783,7 +827,7 @@ export const buildPlayWhere = (args: PlayWhereOpts): WhereClause<'plays'> => {
         }
         // so that we can use this type
         // or else assigning an array to OR using only `typeof where.queueStates` causes a type error
-        let queueWhere: typeof where.queueStates.OR[0][] = [];
+        const queueWhere: typeof where.queueStates.OR[0][] = [];
         for(const q of queues) {
             queueWhere.push(
                 {
@@ -802,20 +846,37 @@ export const buildPlayWhere = (args: PlayWhereOpts): WhereClause<'plays'> => {
             }
         }
     }
+
+    if(args.text !== undefined && args.text.length > 0) {
+        
+        where.AND = [
+            {
+                OR: []
+            }
+        ];
+        const textWhere: typeof where.RAW[] = [];
+        for(const t of args.text) {
+            textWhere.push((p) => sql`lower(json_extract(${p.play}, '$.data.track')) LIKE '%' || ${t.toLocaleLowerCase()} || '%'`)
+            textWhere.push((p) => sql`lower(json_extract(${p.play}, '$.data.artists')) LIKE '%' || ${t.toLocaleLowerCase()} || '%'`)
+            textWhere.push((p) => sql`lower(json_extract(${p.play}, '$.data.album')) LIKE '%' || ${t.toLocaleLowerCase()} || '%'`)
+        }
+        where.AND = [
+            {
+                OR: textWhere.map(x => ({RAW: x}))
+            }
+        ]
+    }
+
     return where;
 }
 
 export const playToRepositoryCreatePlayOpts = (data: MarkOptional<RepositoryCreatePlayOpts, 'input' | 'componentId'>): RepositoryCreatePlayOpts => {
     const {
         play: {
-            meta: {
-                lifecycle: {
-                    input,
-                    original,
-                    ...lifecycleRest
-                } = {},
-                ...metaRest
-            },
+            original: {
+                play: playOriginal,
+                data: playData
+            } = {},
             ...playRest
         },
         ...rest
@@ -824,58 +885,13 @@ export const playToRepositoryCreatePlayOpts = (data: MarkOptional<RepositoryCrea
     return {
         play: {
             ...playRest,
-            meta: {
-                ...metaRest,
-                // @ts-expect-error
-                lifecycle: {
-                    ...lifecycleRest
-                }
-            }
         },
         ...rest,
         input: {
-            play: original,
-            data: input
+            play: playOriginal,
+            data: playData
         }
     }
 }
 
-export type RequestPlayQuery = Partial< Record<keyof Exclude<QueryPlaysOpts, 'componentId'>, string>>;
-
-export const queryArgsFromRequest = (rec: RequestPlayQuery): QueryPlaysOpts => {
-
-    const {
-        state,
-        stateNot,
-        uid,
-        with: withQuery,
-        seenAt,
-        playedAt,
-        limit,
-        sort,
-        order,
-        offset,
-        componentId,
-        queues,
-        ...rest
-    } = rec;
-
-    let queryArgs: QueryPlaysOpts = removeEmptyArrays<QueryPlaysOpts>({
-        state: parseArrayFromMaybeString(state) as PlaySelect['state'][],
-        stateNot: parseArrayFromMaybeString(stateNot) as PlaySelect['state'][],
-        uid: parseArrayFromMaybeString(uid),
-        with: parseArrayFromMaybeString(withQuery) as WithPlayRelation[],
-        sort: sort as 'playedAt' | 'seenAt',
-        order: order as 'asc' | 'desc',
-        ...rest
-    });
-
-    if(limit !== undefined) {
-        queryArgs.limit = Number.parseInt(limit);
-    }
-    if(offset !== undefined) {
-        queryArgs.offset = Number.parseInt(offset);
-    }
-
-    return queryArgs;
-}
+export type RequestPlayQuery = Partial< Record<keyof Exclude<QueryPlaysOpts, 'componentId'>, string>>;

src/backend/common/database/drizzle/repositories/QueueRepository.ts

@@ -1,9 +1,9 @@
 import { eq, and, lte, inArray } from "drizzle-orm";
-import { DrizzleBaseRepository, DrizzleRepositoryOpts } from "./BaseRepository.js";
-import { DbConcrete } from "../drizzleUtils.js";
-import { QueueStateSelect } from "../drizzleTypes.js";
-import { queueStates } from "../schema/schema.js";
-import { CLIENT_DEAD_QUEUE } from "../../../../../core/Atomic.js";
+import { DrizzleBaseRepository, type DrizzleRepositoryOpts } from "./BaseRepository.ts";
+import type {DbConcrete} from "../drizzleUtils.ts";
+import type {QueueStateSelect} from "../drizzleTypes.ts";
+import { queueStates } from "../schema/schema.ts";
+import { CLIENT_DEAD_QUEUE } from "../../../../../core/Atomic.ts";
 export class DrizzleQueueRepository extends DrizzleBaseRepository<'queueStates'> {
 
     constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {

src/backend/common/database/drizzle/schema/schema.ts

@@ -1,12 +1,12 @@
-import { integer, sqliteTable, text, index, uniqueIndex, customType, AnySQLiteColumn } from "drizzle-orm/sqlite-core";
+import { integer, sqliteTable, text, index, uniqueIndex, customType, type AnySQLiteColumn } from "drizzle-orm/sqlite-core";
 import { defineRelations } from 'drizzle-orm';
-import dayjs, { Dayjs } from "dayjs";
-import { nanoid } from "nanoid";
-import { ErrorLike, PlayObject } from "../../../../../core/Atomic.js";
-import { asPlayCheap } from "../../../../../core/PlayMarshalUtils.js";
-import { ExternalMetadataTerm, PlayTransformPartsConfig, SearchAndReplaceTerm } from "../../../infrastructure/Transform.js";
-import { JobRangeCount, JobRangeTime } from "../../../infrastructure/Job.js";
+import dayjs, { type Dayjs } from "dayjs";
+import { COMPONENT_TYPE_CLIENT, COMPONENT_TYPE_SOURCE, type ErrorLike, type PlayObject } from "../../../../../core/Atomic.ts";
+import { asPlayCheap } from "../../../../../core/PlayMarshalUtils.ts";
+import type {ExternalMetadataTerm, PlayTransformPartsConfig, SearchAndReplaceTerm} from "../../../../../core/Transform.ts";
+import type {JobRangeCount, JobRangeTime} from "../../../infrastructure/Job.ts";
 import { serializeError, deserializeError } from "serialize-error";
+import { generatePlayUid } from "../../../../../core/StringUtils.ts";
 
 const DayjsTimestamp = customType<
   {
@@ -67,12 +67,12 @@ const ErrorLikeJson = customType<
 
 export const plays = sqliteTable("plays", {
   id: integer().primaryKey(),
-  uid: text({ length: 30 }).notNull().unique().$defaultFn(() => nanoid(20)),
+  uid: text({ length: 30 }).notNull().unique().$defaultFn(() => generatePlayUid()),
   componentId: integer().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}),
   error: ErrorLikeJson('error'),
   playedAt: DayjsTimestamp('playedAt'),
   seenAt: DayjsTimestamp('seenAt'),
-  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()),
+  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()).$onUpdate(() => dayjs()),
   play: PlayJson('play').notNull(), //  text({ mode: 'json' }).notNull().$type<PlayObject>(),
   state: text({enum: ['queued','discovered','discarded','scrobbled','failed','duped']}).notNull(),
   // https://orm.drizzle.team/docs/indexes-constraints#foreign-key
@@ -144,7 +144,7 @@ export const queueStates = sqliteTable("play_queue_states", {
   retries: integer().notNull().default(0),
   error: ErrorLikeJson('error'),
   createdAt: DayjsTimestamp('createdAt').notNull().$defaultFn(() => dayjs()),
-  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs())
+  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()).$onUpdate(() => dayjs())
 }, (table) => [
   index('play_queue_state_id_idx').on(table.playId)
 ]);
@@ -165,7 +165,7 @@ export const components = sqliteTable("components", {
   id: integer({ mode: 'number' }).primaryKey(),
   // user-provided id
   uid: text({ length: 200 }).notNull(),
-  mode: text({enum: ['source','client']}).notNull(),
+  mode: text({enum: [COMPONENT_TYPE_SOURCE,COMPONENT_TYPE_CLIENT]}).notNull(),
   // spotify, lastfm, etc...
   type: text({length: 50}).notNull(),
   // vanity display name
@@ -207,7 +207,7 @@ export const jobs = sqliteTable("jobs", {
   imported: integer().notNull().default(0),
   scrobbled: integer().notNull().default(0),
   createdAt: DayjsTimestamp('createdAt').notNull().$defaultFn(() => dayjs()),
-  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()),
+  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()).$onUpdate(() => dayjs()),
   completedAt: DayjsTimestamp('completedAt')
 });
 
@@ -289,7 +289,7 @@ export const getConfigByTableName = <T extends TableName>(name: T) => {
   }
 }
 
-const schema = {playInputs, plays, components, componentMigrations, queueStates, jobs};
+export const schema = {playInputs, plays, components, componentMigrations, queueStates, jobs};
 
 export type TSchema = typeof relations;
 export type Schema = typeof schema;

src/backend/common/errors/ErrorUtils.ts

@@ -1,4 +1,4 @@
-import { ResponseError } from "superagent";
+import type {ResponseError} from "superagent";
 
 export const isSuperAgentResponseError = (e: any): e is ResponseError => {
     return typeof e === 'object'

src/backend/common/errors/MSErrors.ts

@@ -1,12 +1,15 @@
 import { parseRegexSingle } from "@foxxmd/regex-buddy-core";
 import mergeErrorCause from 'merge-error-cause';
-import { findCauseByFunc, isAbortReasonErrorLike } from "../../utils/ErrorUtils.js";
-import { UpstreamError, UpstreamErrorOptions } from "./UpstreamError.js";
-import { isAbortError } from "abort-controller-x";
-import {addKnownErrorConstructor} from 'serialize-error';
+import { findCauseByFunc, isAbortReasonErrorLike } from "../../utils/ErrorUtils.ts";
+import { UpstreamError, type UpstreamErrorOptions } from "./UpstreamError.ts";
+import {addKnownErrorConstructor, serializeError} from 'serialize-error';
+import type {LifecycleInput} from "../../../core/Atomic.ts";
 
 export abstract class NamedError extends Error {
     public abstract name: string;
+    public toJSON() {
+        return serializeError(this);
+    }
 }
 
 export abstract class StageError extends NamedError {}
@@ -44,7 +47,10 @@ const STACK_AT_REGEX = new RegExp(/[\n\r]\s*at/);
 
 export class SimpleError extends Error implements HasSimpleError {
     simple: boolean;
-    name = 'SimpleError';
+    override name = 'SimpleError';
+    public toJSON() {
+        return serializeError(this);
+    }
 
     stackShortened: boolean = false;
 
@@ -77,22 +83,47 @@ export class SimpleError extends Error implements HasSimpleError {
 addKnownErrorConstructor(SimpleError, () => new SimpleError(''))
 
 export class StageTransformError extends NamedError {
-    name = 'Stage Transform';
+    override name = 'Stage Transform';
+    lifecycleInputs?: LifecycleInput[]
     stageName: string;
-    constructor(name: string, message: string, options?: ErrorOptions) {
-        super(message, options);
+    constructor(name: string, message: string, options: ErrorOptions & {inputs?: LifecycleInput[]} = {}) {
+        const {
+            inputs,
+            ...restOpts
+        } = options;
+        super(message, restOpts);
+        this.lifecycleInputs = inputs;
         this.stageName = name;
     }
 }
 addKnownErrorConstructor(StageTransformError, () => new StageTransformError('',''))
 
 export class SkipTransformStageError extends SimpleError {
-    name = 'Skip Transform Stage';
+    lifecycleInputs?: LifecycleInput[]
+    override name = 'Skip Transform Stage';
+
+    public constructor(msg: string, options: ErrorOptions & { simple?: boolean, shortStack?: boolean, inputs?: LifecycleInput[]} = {}) {
+        const {
+            inputs,
+            ...restOpts
+        } = options;
+        super(msg, restOpts);
+        this.lifecycleInputs = inputs;
+    }
 }
 addKnownErrorConstructor(SkipTransformStageError, () => new SkipTransformStageError(''))
 
 export class StagePrerequisiteError extends SimpleError {
-    name = 'Stage Prerequisite';
+    lifecycleInputs?: LifecycleInput[]
+    override name = 'Stage Prerequisite';
+    public constructor(msg: string, options: ErrorOptions & { simple?: boolean, shortStack?: boolean, inputs?: LifecycleInput[]} = {}) {
+        const {
+            inputs,
+            ...restOpts
+        } = options;
+        super(msg, restOpts);
+        this.lifecycleInputs = inputs;
+    }
 }
 addKnownErrorConstructor(StagePrerequisiteError, () => new StagePrerequisiteError(''))
 
@@ -113,13 +144,17 @@ export const mergeSimpleError = (err: Error): Error => {
         // mergeErrorCause mutates the argument
         // and we want to be able to do more cause/error parsing after merging for logging
         // so give it a copy instead of the original
-        return mergeErrorCause(structuredClone(err));
+        const mergedErr = mergeErrorCause(structuredClone(err));
+        if(!('toJson' in mergedErr)) {
+            // @ts-expect-error this is used by serialize error with out custom errors
+            mergedErr.toJSON = () => serializeError(mergedErr);
+        }
     }
     return err;
 }
 
 export class ScrobbleSubmitError<T extends (object | string) = object> extends UpstreamError {
-    name = 'Scrobble Submit Error';
+    override name = 'Scrobble Submit Error';
     payload?: T;
     constructor(message: string, options?: UpstreamErrorOptions & {payload?: T}) {
         super(message, options);
@@ -151,7 +186,7 @@ export class InvalidRegexError extends SimpleError {
         const msgParts = [
             message ?? 'Regex(es) did not match the value given.',
         ];
-        let regArr = Array.isArray(regex) ? regex : [regex];
+        const regArr = Array.isArray(regex) ? regex : [regex];
         for(const r of regArr) {
             msgParts.push(`Regex: ${r}`)
         }

src/backend/common/errors/NodeErrors.ts

@@ -1,13 +1,12 @@
-import { findCauseByFunc } from "../../utils/ErrorUtils.js";
-import { isArbitraryObject } from "../infrastructure/Atomic.js";
-import ErrnoException = NodeJS.ErrnoException;
+import { findCauseByFunc } from "../../utils/ErrorUtils.ts";
+import { isArbitraryObject } from "../infrastructure/Atomic.ts";
 
 export type NodeNetworkErrorCode = 'ENOTFOUND' | 'ETIMEDOUT' | 'EAI_AGAIN' | 'ECONNRESET' | 'ECONNREFUSED' | 'ERRADDRINUSE' | 'EADDRNOTAVAIL' | 'ECONNABORTED' | 'EHOSTUNREACH';
 export const NETWORK_ERROR_CODES: NodeNetworkErrorCode[] = ['ENOTFOUND', 'ETIMEDOUT',  'EAI_AGAIN', 'ECONNRESET', 'ECONNREFUSED', 'ERRADDRINUSE', 'EADDRNOTAVAIL', 'ECONNABORTED', 'EHOSTUNREACH'];
 
 export const NETWORK_ERROR_FAILURE_CODES: NodeNetworkErrorCode[] = ['ENOTFOUND', 'EAI_AGAIN', 'ECONNRESET', 'ECONNREFUSED', 'ERRADDRINUSE', 'EADDRNOTAVAIL', 'ECONNABORTED', 'EHOSTUNREACH'];
 
-export const isErrnoException = (error: unknown): error is ErrnoException => {
+export const isErrnoException = (error: unknown): error is NodeJS.ErrnoException => {
     return isArbitraryObject(error) &&
         error instanceof Error &&
         (typeof error.errno === "number" || typeof error.errno === "undefined") &&
@@ -16,7 +15,7 @@ export const isErrnoException = (error: unknown): error is ErrnoException => {
         (typeof error.syscall === "string" || typeof error.syscall === "undefined");
 }
 
-export interface NodeNetworkException extends ErrnoException {
+export interface NodeNetworkException extends NodeJS.ErrnoException {
     code: NodeNetworkErrorCode
 }
 

src/backend/common/errors/UpstreamError.ts

@@ -1,13 +1,13 @@
-import { Response } from 'superagent';
+import type { Response } from 'superagent';
 
-import { findCauseByFunc } from "../../utils/ErrorUtils.js";
+import { findCauseByFunc } from "../../utils/ErrorUtils.ts";
 import { addKnownErrorConstructor } from 'serialize-error';
 
 export type UpstreamErrorOptions = ErrorOptions & { showStopper?: boolean, response?: Response, responseBody?: object | string };
 
 export class UpstreamError extends Error {
 
-    name = 'UpstreamError';
+    override name = 'UpstreamError';
     showStopper: boolean = false;
     response?: Response
     responseBody?: object | string

src/backend/common/index.ts

@@ -1,8 +1,28 @@
 import * as path from 'path';
-//import {fileURLToPath} from "url";
+import envPaths from 'env-paths';
 
-//const __filename = fileURLToPath(import.meta.url);
-//const __dirname = path.dirname(__filename);
+const osPaths = envPaths('multi-scrobbler', {suffix: ''});
 
-export const projectDir = process.cwd(); //path.resolve(__dirname, '../../../');
-export const configDir: string = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
+export const getConfigDir = (): string => {
+    let configDirVal: string = process.env.CONFIG_DIR ?? osPaths.config;
+    // this shouldn't happen...
+    // but if it does we need to have some known path fallback so things don't explode
+    if(configDirVal === undefined) {
+        configDirVal = getPathFromCWD('./config'); // backwards compatibility
+    }
+    // resolve from relative directory, if one was used
+    return path.resolve(configDirVal);
+}
+
+export const getDataDir = (): string => {
+    let dataDirVal: string = process.env.DATA_DIR ?? osPaths.data;
+    // this shouldn't happen...
+    // but if it does we need to have some known path fallback so things don't explode
+    if(dataDirVal === undefined) {
+        dataDirVal = getPathFromCWD('./config'); // defaulting to same directory for backwards compatibility
+    }
+    // resolve from relative directory, if one was used
+    return path.resolve(dataDirVal);
+}
+
+export const getPathFromCWD = (...relativePaths: string[]) => path.resolve(process.cwd(), ...relativePaths);

src/backend/common/infrastructure/Atomic.ts

@@ -1,21 +1,26 @@
-import { Logger } from '@foxxmd/logging';
-import { Dayjs, ManipulateType } from "dayjs";
-import { Request, Response } from "express";
-import { type NextFunction, type ParamsDictionary, type Query } from "express-serve-static-core";
-import { FixedSizeList } from 'fixed-size-list';
-import { ErrorLike, isPlayObject, PlayMetaLifecycleless, PlayObject, PlayObjectLifecycleless, UnixTimestamp } from "../../../core/Atomic.js";
-import TupleMap from "../TupleMap.js";
-import { MusicBrainzApi } from 'musicbrainz-api';
-import { SourceType } from './config/source/sources.js';
-import { ClientType, clientTypes } from './config/client/clients.js';
-import assert, { AssertionError } from 'assert';
+import type { Logger, LogDataPretty } from '@foxxmd/logging';
+import type { Dayjs, ManipulateType } from "dayjs";
+import type { Request, Response } from "express";
+import type { NextFunction, ParamsDictionary, Query } from "express-serve-static-core";
+import type { FixedSizeList } from 'fixed-size-list';
+import { isPlayObject } from "../../../core/Atomic.ts";
+import type {DeviceId, ErrorLike, PlayMeta, PlayObject, PlayObjectMinimal, PlayPlatformId, PlayUserId, UnixTimestamp} from "../../../core/Atomic.ts";
+import type TupleMap from "../TupleMap.ts";
+import type { MusicBrainzApi } from 'musicbrainz-api';
+import type { ReportedPlayerStatus, SourceType } from "../../../core/Atomic.ts";
+import type { ClientType } from "../../../core/Atomic.ts";
+import assert from 'assert';
+import * as path from 'path';
 
-export const lowGranularitySources: SourceType[] = ['subsonic', 'ytmusic'];
+export const __filename = import.meta.filename;
+export const projectRootDir = path.resolve(__filename, '../../../../../');
 
-export const isClientType = (data: string): data is ClientType => {
-    return clientTypes.includes(data as ClientType);
+export interface LeveledLogData extends LogDataPretty {
+    levelLabel: string
 }
 
+export const lowGranularitySources: SourceType[] = ['subsonic', 'ytmusic', 'applemusic'];
+
 export interface ComponentIdentifier {
     type: SourceType | ClientType
     name: string
@@ -51,21 +56,6 @@ export interface InternalConfig {
 
 export type InternalConfigOptional = Omit<InternalConfig, 'logger'>
 
-export type ReportedPlayerStatus = 'playing' | 'stopped' | 'paused' | 'unknown';
-export const REPORTED_PLAYER_STATUSES = {
-    playing: 'playing' as ReportedPlayerStatus,
-    stopped: 'stopped' as ReportedPlayerStatus,
-    paused: 'paused' as ReportedPlayerStatus,
-    unknown: 'unknown' as ReportedPlayerStatus
-}
-
-export type CalculatedPlayerStatus = ReportedPlayerStatus | 'stale' | 'orphaned';
-export const CALCULATED_PLAYER_STATUSES = {
-    ...REPORTED_PLAYER_STATUSES,
-    stale: 'stale' as CalculatedPlayerStatus,
-    orphaned: 'orphaned' as CalculatedPlayerStatus,
-}
-
 export type ConfigureAsSource = 'source';
 export type ConfigureAsClient = 'client';
 export type ConfigureAs = ConfigureAsSource | ConfigureAsClient;
@@ -109,16 +99,11 @@ export interface FormatPlayObjectOptions {
     [key: string]: any
 }
 
-export interface ProgressAwarePlayObject extends PlayObjectLifecycleless {
-    meta: PlayMetaLifecycleless & {
+export interface ProgressAwarePlayObject extends PlayObjectMinimal {
+    meta: PlayMeta & {
         initialTrackProgressPosition?: number
     }
 }
-export type DeviceId = string;
-export type PlayUserId = string;
-export type PlayPlatformId = [DeviceId, PlayUserId];
-export type PlayPlatformIdStr = string;
-
 export type GroupedPlays = TupleMap<DeviceId,PlayUserId,ProgressAwarePlayObject[]>;
 
 export type GroupedFixedPlays = TupleMap<DeviceId,PlayUserId,FixedSizeList<ProgressAwarePlayObject>>;
@@ -131,7 +116,7 @@ export const SINGLE_USER_PLATFORM_ID_STR = `${NO_DEVICE}-${NO_USER}`;
 
 export interface ScrobbledPlayObject {
     play: PlayObject
-    scrobble: PlayObjectLifecycleless
+    scrobble: PlayObjectMinimal
 }
 
 
@@ -191,10 +176,6 @@ export type ExpressRequest = Request<ParamsDictionary, any, any, Query, Record<s
 export type ExpressResponse = Response<any, Record<string, any>>;
 export type ExpressHandler = (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => Promise<void | Response<any, Record<string, any>>>
 
-export const DELIMITERS = [',','&','/','\\'];
-export const DELIMITERS_NO_AMP = [',','/','\\'];
-export const DELIMETERS_REGEX: RegExp = new RegExp(/[,&\/\\]/);
-
 export const ARTIST_WEIGHT = 0.3;
 export const TITLE_WEIGHT = 0.4;
 export const TIME_WEIGHT = 0.5;
@@ -386,7 +367,7 @@ export interface PaginatedListens {
     getPaginatedListens(params: PaginatedListensOptions<CursorType>): Promise<{data: PlayObject[], meta: PaginatedListensOptions<CursorType> & PaginatedResults<CursorType>}>
 }
 
-export const hasPaginagedListens = (obj: Object): obj is PaginatedListens => {
+export const hasPaginagedListens = (obj: object): obj is PaginatedListens => {
     return 'getPaginatedListens' in obj;
 }
 
@@ -399,7 +380,7 @@ export interface PaginatedTimeRangeListens<T extends CursorType = CursorType> {
     getPaginatedUnitOfTime(): ManipulateType;
 }
 
-export const hasPaginatedTimeRangeListens = (obj: Object): obj is PaginatedTimeRangeListens => {
+export const hasPaginatedTimeRangeListens = (obj: object): obj is PaginatedTimeRangeListens => {
     return 'getPaginatedTimeRangeListens' in obj;
 }
 
@@ -412,7 +393,7 @@ export interface PagelessTimeRangeListens {
     getPaginatedUnitOfTime(): ManipulateType;
 }
 
-export const hasPagelessTimeRangeListens = (obj: Object): obj is PagelessTimeRangeListens => {
+export const hasPagelessTimeRangeListens = (obj: object): obj is PagelessTimeRangeListens => {
     return 'getPagelessTimeRangeListens' in obj;
 }
 

src/backend/common/infrastructure/Job.ts

@@ -1,4 +1,4 @@
-import { UnixTimestamp } from "../../../core/Atomic.js"
+import type {UnixTimestamp} from "../../../core/Atomic.ts";
 
 export interface JobParameters {
     /** maximum number of results to get for the entire job */

src/backend/common/infrastructure/config/aioConfig.ts

@@ -1,13 +1,13 @@
-import { LogOptions } from "@foxxmd/logging";
-import { ClientAIOConfig } from "./client/clients.js";
-import { CommonClientOptions } from "./client/index.js";
-import { RequestRetryOptions } from "./common.js";
-import { WebhookConfig } from "./health/webhooks.js";
-import { CommonSourceOptions, SourceRetryOptions } from "./source/index.js";
-import { SourceAIOConfig } from "./source/sources.js";
-import { CacheConfigOptions, CacheConfigUser, DurationValue } from "../Atomic.js";
-import { TransformerCommonConfig } from "../../../../core/Atomic.js";
-import { RetentionConfig } from "./database.js";
+import type { LogOptions } from "@foxxmd/logging";
+import type { ClientAIOConfig } from "./client/clients.ts";
+import type { CommonClientOptions } from "./client/index.ts";
+import type { RequestRetryOptions } from "./common.ts";
+import type { WebhookConfig } from "./health/webhooks.ts";
+import type { CommonSourceOptions, SourceRetryOptions } from "./source/index.ts";
+import type { SourceAIOConfig } from "./source/sources.ts";
+import type { CacheConfigUser, DurationValue } from "../Atomic.ts";
+import type { TransformerCommonConfig } from "../../../../core/Atomic.ts";
+import type { RetentionConfig } from "./database.ts";
 
 
 export interface SourceDefaults extends CommonSourceOptions {

src/backend/common/infrastructure/config/client/atproto.ts

@@ -0,0 +1,29 @@
+import type {AtprotoDid} from "@atcute/lexicons/syntax";
+
+export interface ATProtoUserIdentifierData {
+    /**
+     * Identify the account to login as
+     * 
+     * * For **App Password** Auth - your email
+     * * For **Oauth** - your handle minus the @
+     */
+    identifier: string
+    did?: string
+}
+
+export interface ATProtoAppData {
+    /**
+     * The [App Password](https://atproto.com/specs/xrpc#app-passwords) you created for your account
+     * 
+     * This is created under https://bsky.app/settings/app-passwords
+     * 
+     * **Use this if you are self-hosting Multi-Scrobbler on localhost or accessed like http://IP:PORT**
+     */
+    appPassword: string
+}
+
+export interface HandleData {
+    did: AtprotoDid
+    pds: string
+    handle: string
+}

src/backend/common/infrastructure/config/client/clients.ts

@@ -1,11 +1,11 @@
-import { KoitoClientAIOConfig, KoitoClientConfig } from "./koito.js";
-import { LastfmClientAIOConfig, LastfmClientConfig } from "./lastfm.js";
-import { ListenBrainzClientAIOConfig, ListenBrainzClientConfig } from "./listenbrainz.js";
-import { MalojaClientAIOConfig, MalojaClientConfig } from "./maloja.js";
-import { TealClientAIOConfig, TealClientConfig } from "./tealfm.js";
-import { RockSkyClientAIOConfig, RockSkyClientConfig } from "./rocksky.js";
-import { LibrefmClientConfig, LibrefmClientAIOConfig } from "./librefm.js";
-import { DiscordClientAIOConfig, DiscordClientConfig } from "./discord.js";
+import type {KoitoClientAIOConfig, KoitoClientConfig} from "./koito.ts";
+import type {LastfmClientAIOConfig, LastfmClientConfig} from "./lastfm.ts";
+import type {ListenBrainzClientAIOConfig, ListenBrainzClientConfig} from "./listenbrainz.ts";
+import type {MalojaClientAIOConfig, MalojaClientConfig} from "./maloja.ts";
+import type {TealClientAIOConfig, TealClientConfig} from "./tealfm.ts";
+import type {RockSkyClientAIOConfig, RockSkyClientConfig} from "./rocksky.ts";
+import type {LibrefmClientConfig, LibrefmClientAIOConfig} from "./librefm.ts";
+import type {DiscordClientAIOConfig, DiscordClientConfig} from "./discord.ts";
 
 export type ClientConfig = 
 MalojaClientConfig 
@@ -57,23 +57,3 @@ export const clientInterfaces = [
     ...atomicClientInterfaces
 ];
 
-export type ClientType =
-    'maloja'
-    | 'lastfm'
-    | 'librefm'
-    | 'listenbrainz'
-    | 'koito'
-    | 'tealfm'
-    | 'rocksky'
-    | 'discord';
-    
-export const clientTypes: ClientType[] = [
-    'maloja',
-    'lastfm',
-    'librefm',
-    'listenbrainz',
-    'koito',
-    'tealfm',
-    'rocksky',
-    'discord'
-];

src/backend/common/infrastructure/config/client/discord.ts

@@ -1,5 +1,5 @@
-import { ActivityType, StatusDisplayType } from "discord.js"
-import { CommonClientConfig, CommonClientData } from "./index.js"
+import type {CommonClientConfig, CommonClientData} from "./index.ts";
+import type {ComponentType} from "../../../../../core/Atomic.ts";
 
 export interface DiscordData {
     token?: string
@@ -20,7 +20,7 @@ export interface DiscordClientConfig extends CommonClientConfig {
      * @default client
      * @examples ["client"]
      * */
-    configureAs?: 'client' | 'source'
+    configureAs?: ComponentType
     data: DiscordClientData
 }
 
@@ -62,38 +62,14 @@ export interface ActivityTimestamps {
     end?: number
 }
 
-export enum ActivityTypeOthers {
-    Hanging = 6
-}
+export type StatusDisplayType = 0 | 1 | 2;
+export const STATUS_DISPLAY_TYPE = {
+    Name: 0,
+    State: 1,
+    Details: 2
+} as const satisfies Record<string, StatusDisplayType>
 
-export declare enum ActivityTypeReal {
-    /**
-     * Playing \{game\}
-     */
-    Playing = 0,
-    /**
-     * Streaming \{details\}
-     */
-    Streaming = 1,
-    /**
-     * Listening to \{name\}
-     */
-    Listening = 2,
-    /**
-     * Watching \{details\}
-     */
-    Watching = 3,
-    /**
-     * \{emoji\} \{state\}
-     */
-    Custom = 4,
-    /**
-     * Competing in \{name\}
-     */
-    Competing = 5,
-
-    Hanging = 6
-}
+export type ActivityType = 0 | 1 | 2 | 3 | 4 | 5 | 6;
 
 export interface ActivityData {
     name: string
@@ -119,7 +95,7 @@ export const ACTIVITY_TYPE = {
     Custom: 4,
     Competing: 5,
     Hanging: 6
-} as const satisfies Record<string, ActivityTypeReal>
+} as const satisfies Record<string, ActivityType>
 
 export const ARTWORK_PLACEHOLDER = 'https://raw.githubusercontent.com/FoxxMD/multi-scrobbler/master/assets/default-artwork.png';
 export const MS_ART = 'https://raw.githubusercontent.com/FoxxMD/multi-scrobbler/master/assets/icon.png';

src/backend/common/infrastructure/config/client/index.ts

@@ -1,7 +1,7 @@
-import { DurationValue } from "../../Atomic.js";
-import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js";
-import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js";
-import { RetentionConfig } from "../database.js";
+import type {DurationValue} from "../../Atomic.ts";
+import type {PlayTransformOptions} from "../../../../../core/Transform.ts";
+import type {CommonConfig, RequestRetryOptions} from "../common.ts";
+import type {RetentionConfig} from "../database.ts";
 
 /**
  * Scrobble matching (between new source track and existing client scrobbles) logging options. Used for debugging.

src/backend/common/infrastructure/config/client/koito.ts

@@ -1,6 +1,6 @@
-import { UnixTimestamp } from "../../../../../core/Atomic.js"
-import { RequestRetryOptions } from "../common.js"
-import { CommonClientConfig, CommonClientData } from "./index.js"
+import type {ComponentType, UnixTimestamp} from "../../../../../core/Atomic.ts";
+import type {RequestRetryOptions} from "../common.ts";
+import type {CommonClientConfig, CommonClientData} from "./index.ts";
 
 export interface ListensResponse {
     items: ListenObjectResponse[]
@@ -74,7 +74,7 @@ export interface KoitoClientConfig extends CommonClientConfig {
      * @default client
      * @examples ["client"]
      * */
-    configureAs?: 'client' | 'source'
+    configureAs?: ComponentType
     data: KoitoClientData
 }
 

src/backend/common/infrastructure/config/client/lastfm.ts

@@ -1,5 +1,5 @@
-import { RequestRetryOptions } from "../common.js";
-import { CommonClientConfig, CommonClientData, CommonClientOptions, NowPlayingOptions } from "./index.js";
+import type {RequestRetryOptions} from "../common.ts";
+import type {CommonClientConfig, CommonClientData, CommonClientOptions, NowPlayingOptions} from "./index.ts";
 
 export interface LastfmData extends CommonClientData, RequestRetryOptions {
     /**

src/backend/common/infrastructure/config/client/librefm.ts

@@ -1,6 +1,6 @@
-import { RequestRetryOptions } from "../common.js";
-import { CommonClientConfig, CommonClientData } from "./index.js";
-import { LastfmClientOptions } from "./lastfm.js";
+import type {RequestRetryOptions} from "../common.ts";
+import type {CommonClientConfig, CommonClientData} from "./index.ts";
+import type {LastfmClientOptions} from "./lastfm.ts";
 
 export interface LibrefmData extends CommonClientData, RequestRetryOptions {
         /**

src/backend/common/infrastructure/config/client/listenbrainz.ts

@@ -1,5 +1,6 @@
-import { RequestRetryOptions } from "../common.js";
-import { CommonClientConfig, CommonClientData } from "./index.js";
+import type {ComponentType} from "../../../../../core/Atomic.ts";
+import type {RequestRetryOptions} from "../common.ts";
+import type {CommonClientConfig, CommonClientData} from "./index.ts";
 
 export interface ListenBrainzData extends RequestRetryOptions{
     /**
@@ -31,7 +32,7 @@ export interface ListenBrainzClientConfig extends CommonClientConfig {
      * @default client
      * @examples ["client"]
      * */
-    configureAs?: 'client' | 'source'
+    configureAs?: ComponentType
     data: ListenBrainzClientData
 }
 

src/backend/common/infrastructure/config/client/maloja.ts

@@ -1,5 +1,6 @@
-import { RequestRetryOptions } from "../common.js";
-import { CommonClientConfig, CommonClientData } from "./index.js";
+import type {ComponentType} from "../../../../../core/Atomic.ts";
+import type {RequestRetryOptions} from "../common.ts";
+import type {CommonClientConfig, CommonClientData} from "./index.ts";
 
 export interface MalojaData extends RequestRetryOptions {
     /**
@@ -27,7 +28,7 @@ export interface MalojaClientConfig extends CommonClientConfig {
      * @default client
      * @examples ["client"]
      * */
-    configureAs?: 'client' | 'source'
+    configureAs?: ComponentType
     data: MalojaClientData
 }
 

src/backend/common/infrastructure/config/client/rocksky.ts

@@ -1,5 +1,5 @@
-import { RequestRetryOptions } from "../common.js";
-import { CommonClientConfig, CommonClientData, CommonClientOptions, NowPlayingOptions } from "./index.js";
+import type {RequestRetryOptions} from "../common.ts";
+import type {CommonClientConfig, CommonClientData, CommonClientOptions, NowPlayingOptions} from "./index.ts";
 
 export interface RockSkyData extends RequestRetryOptions{
 
@@ -8,7 +8,14 @@ export interface RockSkyData extends RequestRetryOptions{
      *
      * @examples ["6794186bf-1157-4de6-80e5-uvb411f3ea2b"]
      * */
-    key: string
+    key?: string
+
+    /**
+     * Access Token generated from https://rocksky.app/access-tokens in Rocksky for your account
+     *
+     * @examples ["eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkaWQ....."]
+     * */
+    token?: string
 
     /**
      * The **fully-qualified** handle for your ATPRoto/Bluesky account, like:

src/backend/common/infrastructure/config/client/tealfm.ts

@@ -1,28 +1,15 @@
-import { RequestRetryOptions } from "../common.js"
-import { CommonClientConfig, CommonClientData, CommonClientOptions } from "./index.js"
+import type {ComponentType} from "../../../../../core/Atomic.ts";
+import type {RequestRetryOptions} from "../common.ts";
+import type {ATProtoAppData, ATProtoUserIdentifierData} from "./atproto.ts";
+import type {CommonClientConfig, CommonClientData, CommonClientOptions} from "./index.ts";
 
-export interface TealData extends RequestRetryOptions {
+export type TealData = RequestRetryOptions & ATProtoUserIdentifierData & Partial<ATProtoAppData> & {
         /**
      * The base URI of the Multi-Scrobbler to use for ATProto OAuth
      * 
      * Only include this if you want to use OAuth. The URI must be a non-IP/non-local domain using https: protocol.
     */
     baseUri?: string
-    /**
-     * Identify the account to login as
-     * 
-     * * For **App Password** Auth - your email
-     * * For **Oauth** - your handle minus the @
-     */
-    identifier: string
-    /**
-     * The [App Password](https://atproto.com/specs/xrpc#app-passwords) you created for your account
-     * 
-     * This is created under https://bsky.app/settings/app-passwords
-     * 
-     * **Use this if you are self-hosting Multi-Scrobbler on localhost or accessed like http://IP:PORT**
-     */
-    appPassword?: string
 }
 
 export interface TealClientData extends TealData, CommonClientData {
@@ -35,7 +22,7 @@ export interface TealClientConfig extends CommonClientConfig {
      * @default client
      * @examples ["client"]
      * */
-    configureAs?: 'client' | 'source'
+    configureAs?: ComponentType
     data: TealClientData
     options?: TealClientOptions
 }
@@ -51,42 +38,6 @@ export interface TealClientOptions extends TealOptions,CommonClientOptions {
 export interface TealClientAIOConfig extends TealClientConfig {
     type: 'tealfm'
 }
-export interface TealArtistCredit {
-    artistName?: string,
-    artistMbId?: string
-}
-/**
- *  https://github.com/teal-fm/teal/blob/main/lexicons/fm.teal.alpha/feed/play.json
- *  https://github.com/teal-fm/teal/blob/main/lexicons/fm.teal.alpha/feed/defs.json
- * */
-export interface ScrobbleRecord {
-    $type: "fm.teal.alpha.feed.play",
-    trackName: string,
-    playedTime: string,
-    duration?: number
-    artists?: TealArtistCredit[]
-    /** Album name  */
-    releaseName?: string
-    /** A metadata string specifying the user agent where the format is `<app-identifier>/<version> (<kernel/OS-base>; <platform/OS-version>; <device-model>)` */
-    submissionClientAgent: string,
-    musicServiceBaseDomain?: string
-    // musicbrainz
-    recordingMbId?: string
-    releaseMbId?: string
-    isrc?: string,
-    /** The URL associated with this track */
-    originUrl?: string
-    [x: string]: unknown
-}
-
-export interface StatusRecord {
-    $type: "fm.teal.alpha.actor.status",
-    /** item is just ScrobbleRecord, but without $type */
-    item: Omit<ScrobbleRecord, '$type'>,
-    time: string,
-    expiry: string,
-    [x: string]: unknown
-}
 
 export interface ListRecord<T> {
   uri: string;

src/backend/common/infrastructure/config/common.ts

@@ -1,5 +1,3 @@
-import { keyOmit } from "../Atomic.js";
-
 export interface CommonConfigPrimitives {
     name?: string
     id?: string

src/backend/common/infrastructure/config/database.ts

@@ -1,5 +1,5 @@
-import { Duration } from "dayjs/plugin/duration.js";
-import { DurationValue } from "../Atomic.js";
+import type {Duration} from "dayjs/plugin/duration.js";
+import type {DurationValue} from "../Atomic.ts";
 
 export type RetentionPlayType = 'failed' | 'completed' | 'duped';
 export const retentionPlayTypes: RetentionPlayType[] = ['failed','completed','duped'];

src/backend/common/infrastructure/config/health/webhooks.ts

@@ -1,5 +1,6 @@
 export interface WebhookPayload {
     title?: string
+    identifier: string
     message: string
     priority: Priority
 }

src/backend/common/infrastructure/config/source/applemusic.ts

@@ -0,0 +1,58 @@
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.ts";
+
+export interface AppleMusicKey {
+    id: string
+    teamId: string
+    p8: string
+}
+
+export interface AppleMusicData extends CommonSourceData, PollingOptions {
+    key?: AppleMusicKey
+    token?: string
+    mediaUserToken?: string
+    /**
+     * Origin header to include in every Apple Music API request.
+     * Required when using a browser token (not a MusicKit key).
+     *
+     * @example "https://music.apple.com"
+     */
+    origin?: string
+}
+
+export interface AppleMusicSourceConfig extends CommonSourceConfig {
+    data?: AppleMusicData
+    options?: CommonSourceOptions & {
+        logAuth?: boolean
+        logDiff?: boolean
+        /**
+         * Fixes a quirk where Apple Music's history API hides duplicate plays.
+         * If you listen to A → B → A, the API returns [A, B, X, Y ...] instead of [A, B, A, X, Y ...].
+         * This can cause MS to skip the second A play.
+         *
+         * When enabled (default), MS detects this pattern, keeps the interim tracks (B),
+         * and re-scrobbles A as a re-listen. Disable only if you notice false positives.
+         *
+         * @default true
+         * @examples [true, false]
+         */
+        recoverUnchangedTopHistory?: boolean
+        /**
+         * Removes extraneous suffixes from album data
+         * 
+         * Apple Music add ' - EP' and ' - Single' to album names for EP's and singles, respectively.
+         * These suffixes are not part of the official names for the album and can cause issues in scrobble services
+         * or when matching metadata (musicbrainz).
+         * 
+         * When this option is true (default), Multi-scrobbler automatically removes these suffixes.
+         * 
+         * @default true
+         * @examples [true, false]
+         */
+        normalizeAlbum?: boolean
+    }
+}
+
+export interface AppleMusicSourceAIOConfig extends AppleMusicSourceConfig {
+    type: 'applemusic'
+}

src/backend/common/infrastructure/config/source/azuracast.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions, ManualListeningOptions } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions, ManualListeningOptions} from "./index.ts";
 
 export interface AzuraStationInfoResponse {
     id: string

src/backend/common/infrastructure/config/source/chromecast.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface ChromecastData extends CommonSourceData {
     /**

src/backend/common/infrastructure/config/source/deezer.ts

@@ -1,6 +1,6 @@
-import { Second } from "../../../../../core/Atomic.js";
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js";
+import type {Second} from "../../../../../core/Atomic.ts";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.ts";
 
 export interface DeezerData extends CommonSourceData, PollingOptions {
     /**

src/backend/common/infrastructure/config/source/endpointlfm.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface LastFMEndpointData extends CommonSourceData {
     /**

src/backend/common/infrastructure/config/source/endpointlz.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface ListenbrainzEndpointData extends CommonSourceData {
     /**

src/backend/common/infrastructure/config/source/icecast.ts

@@ -1,4 +1,4 @@
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions, ManualListeningOptions } from "./index.js";
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions, ManualListeningOptions} from "./index.ts";
 
 
 export interface IcecastMetadata {

src/backend/common/infrastructure/config/source/index.ts

@@ -1,9 +1,9 @@
-import { FileLogOptions, LogLevel } from "@foxxmd/logging";
+import type { FileLogOptions, LogLevel } from "@foxxmd/logging";
 
-import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js";
-import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js";
-import { RetentionConfig } from "../database.js";
-import { DurationValue } from "../../Atomic.js";
+import type { PlayTransformOptions } from "../../../../../core/Transform.ts";
+import type { CommonConfig, RequestRetryOptions } from "../common.ts";
+import type { RetentionConfig } from "../database.ts";
+import type { DurationValue } from "../../Atomic.ts";
 
 export interface SourceRetryOptions extends RequestRetryOptions {
     /**

src/backend/common/infrastructure/config/source/jellyfin.ts

@@ -1,5 +1,5 @@
-import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js";
-import {
+import type {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.ts";
+import type {
     // @ts-expect-error weird typings?
     CollectionType,
         // @ts-expect-error weird typings?

src/backend/common/infrastructure/config/source/jriver.ts

@@ -1,5 +1,5 @@
-import { PollingOptions, RequestRetryOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {PollingOptions, RequestRetryOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface JRiverData extends CommonSourceData, PollingOptions, RequestRetryOptions {
     /**

src/backend/common/infrastructure/config/source/kodi.ts

@@ -1,5 +1,5 @@
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 
 export interface KodiData extends CommonSourceData, PollingOptions {

src/backend/common/infrastructure/config/source/koito.ts

@@ -1,7 +1,6 @@
-import { KoitoData } from "../client/koito.js";
-import { ListenBrainzData } from "../client/listenbrainz.js";
-import { PollingOptions } from "../common.js";
-import { CommonSourceConfig, CommonSourceData } from "./index.js";
+import type {KoitoData} from "../client/koito.ts";
+import type {PollingOptions} from "../common.ts";
+import type {CommonSourceConfig, CommonSourceData} from "./index.ts";
 
 export interface KoitoSourceData extends KoitoData, CommonSourceData, PollingOptions {
 }