trying to determine why gh actions times out...

TODO migration check for db existence and abnormal db

.devcontainer/devcontainer.json

@@ -17,7 +17,7 @@
 	// "features": {},
 
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
-	"forwardPorts": [9078,3000],
+	"forwardPorts": [9078,3000,5433],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
 	"postCreateCommand": ".devcontainer/postCreate.sh",
@@ -30,7 +30,9 @@
 				"dbaeumer.vscode-eslint",
 				"unifiedjs.vscode-mdx",
 				"bradlc.vscode-tailwindcss",
-				"TakumiI.markdowntable"
+				"TakumiI.markdowntable",
+				"YoavBls.pretty-ts-errors",
+				"bowlerr.sqlite-intelliview-vscode"
 			]
 		}
 	},
@@ -40,6 +42,9 @@
 		},
 		"9078": {
 			"label": "App"
+		},
+		"5433": {
+			"label": "PGLite Server"
 		}
 	}
 

.github/pull_request_template.md

@@ -1,6 +1,6 @@
 ## Checklist before requesting a review
 
-- [ ] I have read the [contributing guidelines.](../CONTRIBUTING.md)
+- [ ] I have read the [contributing guidelines.](../blob/master/CONTRIBUTING.md)
 
 ## Type of change
 

.github/workflows/docsDeploy.yml

@@ -39,9 +39,8 @@ jobs:
         working-directory: ./docsite
 
       - name: Replace canonical
-        working-directory: ./docsite/build
-        # https://stackoverflow.com/a/74211242/1469797
-        run: grep '<link data-rh="true" rel="canonical" href="https://foxxmd.github.io' . -lr | xargs sed -i 's/<link data-rh="true" rel="canonical" href="https:\/\/foxxmd.github.io\/docs/<link data-rh="true" rel="canonical" href="https:\/\/docs.multi-scrobbler.app/g'
+        working-directory: ./docsite
+        run: node --experimental-strip-types canonical-replace.ts ./build
 
       # Popular action to deploy to GitHub Pages:
       # Docs: https://github.com/peaceiris/actions-gh-pages#%EF%B8%8F-docusaurus

.gitignore

@@ -122,6 +122,7 @@ config/*.json
 config/mscache
 config/yti-*
 config/*.cache
+config/msDb
 *.txt
 !robots.txt
 .idea/
@@ -138,6 +139,7 @@ docsite/static/schemas/*.json
 
 
 *.bak
+*.bak.used
 *.p8
 .flatpak-builder
 flatpak/generated-sources.json
@@ -151,4 +153,10 @@ tmp-*
 
 *storybook.log
 storybook-static
-lib
+
+*.db
+*.db.*
+*.db-*lib
+*.db
+*.db.*
+*.db-*lib

.mocharc.json

@@ -6,5 +6,6 @@
   "file": [
     "./src/backend/tests/setup.ts"
   ],
-  "exit": true
+  "exit": true,
+  "timeout": 2500
 }

.vscode/launch.json

@@ -1,6 +1,19 @@
 {
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Run Canonical Replace",
+            "program": "${workspaceFolder}/docsite/canonical-replace.ts",
+            "runtimeArgs": ["--experimental-strip-types"],
+            "args": [
+                "${workspaceFolder}/docsite/build"
+            ],
+            "request": "launch",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "type": "node"
+        },
         {
             "name": "dev",
             "type": "node",

.vscode/settings.json

@@ -13,5 +13,6 @@
     "files.associations": {
         "*.css": "tailwindcss"
     },
-    "tailwindCSS.experimental.configFile": "src/client/index.css"
+    "tailwindCSS.experimental.configFile": "src/client/index.css",
+    "pgliteExplorer.databasePaths": []
 }

docsite/.gitignore

@@ -18,3 +18,4 @@
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+lib

docsite/canonical-replace.ts

@@ -0,0 +1,42 @@
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+var args = process.argv.slice(2);
+
+const linkRegex = new RegExp(/<link data-rh="true" rel="\w+"\shref="(https:\/\/foxxmd\.github\.io\/multi-scrobbler)/g,);
+const replacement = process.env.CANONICAL_HREF ?? 'https://docs.multi-scrobbler.app';
+
+/**
+ * Replace canonical and alternative <link> nodes with the "real" domain of the site
+ * so that SEO (google) chooses the correct domain when showing search results
+ * 
+ */
+
+(async function () {
+
+    if(replacement === undefined || replacement === '') {
+        console.warn('No replacement value found in process.env.CANONICAL_HREF');
+        return;
+    }
+
+    const buildDir = path.resolve(args[0]);
+    console.log(`Reading dir recursively ${buildDir}`);
+    const files = (await fs.readdir(buildDir, { recursive: true })).filter(x => x.includes('.html'));
+    console.log(`Found ${files.length} files with .html extensions`);
+    let modifications = 0;
+
+    await Promise.all(files.map(async (x) => {
+            const filePath = path.resolve(path.join(buildDir, x));
+            //console.log(`Replacing at ${filePath}`);
+            await fs.writeFile(filePath, (await fs.readFile(filePath)).toString().replace(linkRegex, (match, capture, offset, string, groups) => {
+                // this may not be fully accurate since we're mutating concurrently/async
+                // but its a good enough signal for logging to tell if replacements happened
+                modifications++;
+                return match.replace(capture, replacement);
+            }));
+    }));
+
+    console.log(`Done with ${modifications} replacements`);
+
+}());

docsite/docs/configuration/configuration.mdx

@@ -245,11 +245,8 @@ WARN : [App] [Sources] [spotify Secrets] Matched: None | Unmatched: SPOTIFY_SECR
 
 </DetailsAdmo>
 
-## Application Options
 
-These options affect multi-scrobbler's behavior and are not specific to any source/client.
-
-### Base URL
+## Base URL
 
 Defines the URL that is used to generate default redirect URLs for authentication on [spotify](/configuration/sources/spotify) and [lastfm](/configuration/clients/lastfm) -- as well as some logging hints.
 
@@ -280,79 +277,51 @@ Useful when running with [docker](../installation/installation.mdx#docker) so th
 
 </details>
 
-### Caching
+## Caching
 
-Multi-scrobbler caches some activities to persist important data across restarts, reduce external API calls, and make some actions faster.
+Multi-scrobbler implements caching to persist important data across restarts, reduce external API calls, and make some actions faster.
 
-All of the activities below are **always** cached **in-memory** with an optional, configurable [**secondary** store](#secondary-caching-configuration) for persistence.
+A default **in-memory** cache store is used so that you always benefit from some caching. An optional, [**secondary** store](#secondary-caching) can be configured for greater caching capabilities.
 
-<Tabs groupId="cachedThings" queryString>
-<TabItem value="scrobbles" label="Scrobbles">
-
-**Queued** and **Failed** Scrobbles are cached so that any un-scrobbled data you have is persisted across restarts of multi-scrobbler.
-
-:::tip
-
-By default, this data use a [Secondary](#secondary-caching-configuration) [File](./?cacheType=file#secondary-caching-configuration) store, configured for you automatically.
-
-If you have configured a [persisted volume/bind mount](/installation?dockerSetting=storage#recommended-settings) for configuration (`/config` is mounted in [docker compose](/quickstart#create-docker-compose-file)) then you are already done. If you are not persisting this directory then you should consider setting up [Valkey Cache](./?cacheType=valkey#secondary-caching-configuration) for this.
-
-:::
-
-##### Configuration
-
-Use any [Secondary Cache](#secondary-caching-configuration), the config examples below show the default values:
-
-<Tabs>
-<TabItem value="env" label="ENV">
-
-| Environmental Variable | Required? | Default   | Description           |
-| :--------------------- | --------- | --------- | --------------------- |
-| `CACHE_SCROBBLE`       | No        | `file`    | The cache type to use |
-| `CACHE_SCROBBLE_CONN`  | No        | `/config` |                       |
+<DetailsAdmo type="note" summary="What is Cached?">
 
+<Tabs groupId="cachedThings">
+<TabItem value="auth" label="Auth Data">
+Authentication sessions/tokens/etc... are cached for quicker requests and for persistence across restarts.
 </TabItem>
-
-<TabItem value="aio" label="AIO">
-
-```json5 title="config.json"
-{
-    "cache": {
-        "scrobble": {
-            "provider": "file",
-            "connection": "/config"
-        }
-    },
-    // ...
-}
-```
+<TabItem value="transform" label="Scrobble Transforms">
+The results of [transform rules](/configuration/transforms) are cached so that if a scrobble with identical data (track/artists/album) is identified and it has the same set of transform rules then the cached transform results can be applied.
+</TabItem>
+<TabItem value="metadata" label="Transform API Calls">
+API Calls to external (metadata) services used to [Enhance Scrobbles](/configuration/transforms), like calls to [Musicbrainz](/configuration/transforms/musicbrainz), can be cached to avoid duplicate calls and speed up scrobble transformations.
 </TabItem>
 </Tabs>
 
-</TabItem>
-<TabItem value="auth" label="Auth Data">
+</DetailsAdmo>
 
-Authentication sessions/tokens/etc... are cached for quicker requests and for persistence across restarts.
+<DetailsAdmo type="important" summary="Auth Cache Configuration">
 
-:::tip
+Auth caching defaults to a **file** that is stored in the `CONFIG_DIR` directory using the pre-defined file name `ms-auth.cache`.
 
-By default, this data use a [Secondary](#secondary-caching-configuration) [File](./?cacheType=file#secondary-caching-configuration) store, configured for you automatically.
+This provides automatic persistence across restarts for long-lived auth data/credentials if you have configured a [persisted volume/bind mount](/installation?dockerSetting=storage#recommended-settings) for configuration (`/config` is mounted in [docker compose](/quickstart#create-docker-compose-file)).
 
-If you have configured a [persisted volume/bind mount](/installation?dockerSetting=storage#recommended-settings) for configuration (`/config` is mounted in [docker compose](/quickstart#create-docker-compose-file)) then you are already done. If you are not persisting this directory then you should consider setting up [Valkey Cache](./?cacheType=valkey#secondary-caching-configuration) for this.
+If you wish to use the [secondary store](#secondary-caching) for caching Auth you must explicitly configure it. This is because valkey can potentially be *ephemeral* if you do not provide a volume for its data directory.
 
-:::
+To explicitly configure auth to use the secondary store:
 
-##### Configuration
-
-Use any [Secondary Cache](#secondary-caching-configuration), the config examples below show the default values:
-
-<Tabs>
+<Tabs groupId="configType" queryString>
 <TabItem value="env" label="ENV">
 
-| Environmental Variable | Required? | Default   | Description           |
-| :--------------------- | --------- | --------- | --------------------- |
-| `CACHE_AUTH`           | No        | `file`    | The cache type to use |
-| `CACHE_AUTH_CONN`      | No        | `/config` |                       |
+```yaml title="compose.yaml"
+services:
+  multi-scrobbler:
+  # ...
+  environment:
+  // highlight-start
+    - CACHE_AUTH=valkey
+  // highlight-end
+    # ...
+```
 
 </TabItem>
 
@@ -362,8 +331,7 @@ Use any [Secondary Cache](#secondary-caching-configuration), the config examples
 {
     "cache": {
         "auth": {
-            "provider": "file",
-            "connection": "/config"
+            "provider": "valkey"
         }
     },
     // ...
@@ -372,93 +340,19 @@ Use any [Secondary Cache](#secondary-caching-configuration), the config examples
 </TabItem>
 </Tabs>
 
-</TabItem>
-<TabItem value="metadata" label="Transform API Calls">
+</DetailsAdmo>
 
-API Calls to external (metadata) services used to [Enhance Scrobbles](/configuration/transforms), like calls to [Musicbrainz](/configuration/transforms/musicbrainz), can be cached to avoid duplicate calls and speed up scrobble transformations.
+### Secondary Caching Store {#secondary-caching}
 
-By default, these calls are only cached in memory. If you wish for cached calls to be persisted across restarts then setup [Valkey Cache](./?cacheType=valkey#secondary-caching-configuration).
+Using a secondary store enables:
 
-##### Configuration
+* persistence of cached data across restarts
+* a larger store (more data is saved)
+* a longer time-to-live in the store (cached data is fetchable for a longer period)
 
-Use any [Secondary Cache](#secondary-caching-configuration), the config examples below show the default values:
+These benefits are particularly beneficial when using [transforms](/configuration/transforms) like Musicbrainz and it is **strongly recommended** for these scenarios.
 
-<Tabs>
-<TabItem value="env" label="ENV">
-
-| Environmental Variable | Required? | Default | Description           |
-| :--------------------- | --------- | ------- | --------------------- |
-| `CACHE_METADATA`       | No        |         | The cache type to use |
-| `CACHE_METADATA_CONN`  | No        |         |                       |
-
-</TabItem>
-
-<TabItem value="aio" label="AIO">
-
-```json5 title="config.json"
-{
-    "cache": {
-        "metadata": {
-            "provider": "valkey",
-            "connection": "yourConnectionStringHere"
-        }
-    },
-    // ...
-}
-```
-</TabItem>
-</Tabs>
-
-</TabItem>
-</Tabs>
-
-#### Secondary Caching Configuration
-
-The type of cache used, and its connection properties, can be configured through **ENV** or **AIO** config.
-
-<Tabs groupId="cacheType" queryString>
-
-<TabItem value="file" label="File">
-
-**File** cache is stored in the `CONFIG_DIR` directory using the pre-defined file name `ms-[cacheName].cache`.
-
-<Tabs groupId="configType" queryString>
-
-<TabItem value="env" label="ENV">
-
-Example
-
-| Environmental Variable | Required? | Default   | Description                                                  |
-| :--------------------- | --------- | --------- | ------------------------------------------------------------ |
-| `CACHE_SCROBBLE`       | No        | `file`    | The cache type to use                                        |
-| `CACHE_SCROBBLE_CONN`  | No        | `/config` | The directory, within the container, to store the cache file |
-
-</TabItem>
-
-<TabItem value="aio" label="AIO">
-
-Example
-
-```json5 title="config.json"
-{
-    "cache": {
-        "scrobble": {
-            "provider": "file",
-            "connection": "/config"
-        }
-    },
-    // ...
-}
-```
-
-</TabItem>
-
-</Tabs>
-</TabItem>
-
-<TabItem value="valkey" label="Valkey">
-
-[**Valkey**](https://valkey.io/) is an open-source fork of Redis.
+Currently, Multi-scrobbler only supports [**Valkey**](https://valkey.io/), an open-source fork of Redis, as a secondary store.
 
 <details>
 
@@ -466,11 +360,13 @@ Example
 
 A valkey container can be added to the [multi-scrobbler docker compose stack](/installation?runType=docker-compose#docker):
 
+<Tabs>
+<TabItem value="named" label="Named Volume">
+
 ```yaml title="docker-compose.yml"
 services:
   multi-scrobbler:
   # ...
-  # adding everything below
   // highlight-start
   valkey:
     image: valkey/valkey
@@ -483,6 +379,25 @@ volumes:
   // highlight-end
 ```
 
+</TabItem>
+<TabItem value="bind" label="Bind Mount">
+
+```yaml title="docker-compose.yml"
+services:
+  multi-scrobbler:
+  # ...
+  // highlight-start
+  valkey:
+    image: valkey/valkey
+    volumes:
+      - ./valkeyData:/data
+  // highlight-end
+```
+
+</TabItem>
+
+</Tabs>
+
 Use `redis://valkey:6379` as the connection string in the configurations below.
 
 </details>
@@ -497,12 +412,16 @@ redis://HOST_IP:HOST_PORT
 
 <TabItem value="env" label="ENV">
 
-Example
-
-| Environmental Variable | Required? | Default  | Description                                                         |
-| :--------------------- | --------- | -------- | ------------------------------------------------------------------- |
-| `CACHE_METADATA`       | Yes       | `valkey` | The cache type to use                                               |
-| `CACHE_METADATA_CONN`  | Yes       |          | The host/IP and port to connect to EX: `redis://192.168.0.120:6379` |
+```yaml title="compose.yaml"
+services:
+  multi-scrobbler:
+  # ...
+  environment:
+  // highlight-start
+    - CACHE_VALKEY=redis://192.168.0.120:6379
+  // highlight-end
+    # ...
+```
 
 </TabItem>
 
@@ -513,10 +432,7 @@ Example
 ```json5 title="config.json"
 {
     "cache": {
-        "metadata": {
-            "provider": "valkey",
-            "connection": "redis://192.168.0.120:6379"
-        }
+        "valkey": "redis://192.168.0.120:6379"
     },
     // ...
 }
@@ -525,11 +441,8 @@ Example
 </TabItem>
 
 </Tabs>
-</TabItem>
 
-</Tabs>
-
-### Debug Mode
+## Debug Mode
 
 Turning on Debug Mode will
 
@@ -550,7 +463,7 @@ To set debug mode either add it to [AIO `config.json`](./?configType=aio#configu
 
 or set the [ENV](./?configType=env#configuration-types) `DEBUG_MODE=true`
 
-### Disable Web
+## Disable Web
 
 If you do not need the dashboard and/or ingress sources, or have security concerns about ingress and cannot control your hosting environment, the web server and API can be disabled.
 

docsite/docs/configuration/sources/lastfm-endpoint.mdx

@@ -41,6 +41,6 @@ http://localhost:9078/api/lastfm/mySlug
 <Config config="LastFMEndpointSourceConfig" fileContent={LastfmEndpointConfig} name="endpointlfm">
         | Environmental Variable | Required? | Default | Description                                                                                                        |
         | :--------------------- | :-------- | ------- | ------------------------------------------------------------------------------------------------------------------ |
-        | `LFMENDPOINT_ENABLE`   | No        |         | Use LFM Endpoint as a Source without any other configuration. Only required if slug/token are not provided as ENVs |
+        | `LFM_ENABLE`           | No        |         | Use LFM Endpoint as a Source without any other configuration. Only required if slug/token are not provided as ENVs |
         | `LFM_SLUG`             | No        |         | (Optional) The URL suffix to use for accepting LFM scrobbles                                                       |
 </Config>

docsite/docs/configuration/sources/listenbrainz-endpoint.mdx

@@ -111,7 +111,7 @@ To troubleshoot any errors, and assuming you are using Home Assistant, view the
 <Config config="ListenbrainzEndpointSourceConfig" fileContent={ListenbrainzEndpointConfig} name="endpointlz">
         | Environmental Variable | Required? | Default | Description                                                                                                                                                |
         | :--------------------- | :-------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |
-        | `LZENDPOINT_ENABLE`    | No        |         | Use LZ Endpoint as a Source without any other configuration. Only required if slug/token are not provided as ENVs                                          |
+        | `LZE_ENABLE`           | No        |         | Use LZ Endpoint as a Source without any other configuration. Only required if slug/token are not provided as ENVs                                          |
         | `LZE_TOKEN`            | No        |         | LZ "Authentication Token" you provided to the scrobbling application                                                                                       |
         | `LZE_SLUG`             | No        |         | (Optional) The URL suffix to use for accepting LZ scrobbles                                                                                                |
         | `LZE_USERNAME`         | No        |         | (Optional) A fake username that will be returned for token validation and on now-playing responses. If none is provided the Source's name is used instead. |

docsite/docs/configuration/transforms/transforms.mdx

@@ -591,7 +591,7 @@ The output shows the diff between the previous stage (or original Play) and the
 
 MS uses [caching](/configuration/#caching) to reduce the number of API calls needed for stages like [Musicbrainz](/configuration/transforms/musicbrainz) and to speed up all transforms by caching steps and results. However, the default caching strategy uses a small cache size and very short [TTLs](https://en.wikipedia.org/wiki/Time_to_live) because it is *in-memory*.
 
-**If you are using any Transform stages you should configure [secondary caching with Valkey for Metadata](/configuration/?cacheType=valkey&cachedThings=metadata#secondary-caching-configuration)** to increase the cache size and lifetime of cached items. This will also reduce memory usage in MS.
+**If you are using any Transform stages you should configure [secondary caching](/configuration#secondary-caching)** to increase the cache size and lifetime of cached items. This will also reduce memory usage in MS.
 
 <details>
 

docsite/docs/installation/installation.mdx

@@ -132,7 +132,7 @@ services:
 
 </TabItem>
 <TabItem value="caching" label="Caching">
-**Optionally**, add a [Valkey](https://valkey.io/) service to your stack for [secondary caching](/configuration/?cacheType=valkey#secondary-caching-configuration) to take advantage of faster performance and reduced memory usage.
+**Optionally**, add a [Valkey](https://valkey.io/) service to your stack for [secondary caching](/configuration#secondary-caching) to take advantage of faster performance and reduced memory usage.
 
 ```yaml title="docker-compose.yml"
 services:
@@ -142,8 +142,7 @@ services:
     environment:
       # ...
       // highlight-start
-      CACHE_METADATA=valkey
-      CACHE_METADATA_CONN=redis://valkey:6379
+      CACHE_VALKEY=redis://valkey:6379
       // highlight-end
     # ...
 

docsite/docs/quickstart.mdx

@@ -336,7 +336,7 @@ Visit `http://192.168.0.100:9078` to see the dashboard where
 ## Next Steps
 
 * See more advanced docker options as well as other install methods in the [**Installation**](/installation#docker) docs
-  * Setup [secondary caching](/configuration/?cacheType=valkey#secondary-caching-configuration) with [valkey](/installation?dockerSetting=caching#recommended-settings) for increased performance and reduced memory usage
+  * Setup [secondary caching](/configuration#secondary-caching) with [valkey](/installation?dockerSetting=caching#recommended-settings) for increased performance and reduced memory usage
 * Review the [**Configuration**](/configuration) docs
   * Learn about how to configure multi-scrobbler using files for more complicated Source/Client scenarios
   * See all available [**Sources**](/configuration/sources) and [**Clients**](/configuration/clients) alongside configuration examples

docsite/docs/updating/updating.mdx

@@ -10,11 +10,37 @@ import CodeBlock from '@theme/CodeBlock';
 
 ## Updating
 
-Currently, multi-scrobbler does not have any databases or dependencies that require additional interaction when updating.
+The majority of Multi-scrobbler updates can be completed without any manual intervention. This is **guaranteed** for [patch version updates](#versioning).
 
-Any **breaking changes** will be related to [configuration](/configuration) that has been deprecated/changed, or tooling that usually only affects [Local Installations.](/installation#local-installation)
+Regardless, it is recommended to consult the [**Github Release Notes**](https://github.com/FoxxMD/multi-scrobbler/releases) before any upgrades. The release notes contain all changelogs as well as most **breaking changes**/notices.
 
-These changes, and how to update configs accordingly, are detailed in [**Github Release Notes**](https://github.com/FoxxMD/multi-scrobbler/releases). It is recommended to check this page before upgrading [minor or major versions.](#versioning)
+The [**Upgrade Path** docs](/updating/upgrade-path) contain information for upgrading through **required versions** or in-depth migration guides. Check this section before upgrading any [minor versions](#versioning).
+
+### Database
+
+Multi-scrobbler depends on a SQLite database (`ms.db`) that is created on first run and stored in the [`CONFIG_DIR`](/installation/?dockerSetting=storage#recommended-settings).
+
+If any database migrations are required for an update then your database is **automatically backed up** before migration occurs. The backup file is created in the same directory.
+
+You can manually make a backup of this database by making a copy of `ms.db` and any similarly named files like `ms.db-journal`.
+
+:::tip
+
+If upgrading a [minor version](#versioning) you may want to make a backup for extra safety.
+
+:::
+
+### Configuration
+
+[Minor versions](#versioning) may have **breaking changes** related to [configuration](/configuration). Consult the [**Github Release Notes**](https://github.com/FoxxMD/multi-scrobbler/releases) and [Upgrade Paths](/updating/upgrade-path) before upgrading.
+
+
+## Update Instructions
+
+Assuming:
+
+* upgrade(s) are only [patch versions](#versioning) and the [Release Notes](https://github.com/FoxxMD/multi-scrobbler/releases) do not contain any other guidance or
+* you have checked the release notes, upgrade paths docs, and completed all migration steps for the specific version upgrade
 
 <Tabs groupId="installType" queryString>
     <TabItem value="docker-compose" label="Docker Compose">

docsite/docs/updating/upgrade-path/0140.mdx

@@ -0,0 +1,212 @@
+---
+sidebar_position: 1
+title: From < 0.14.0
+description: Upgrading to 0.14.0
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import CodeBlock from '@theme/CodeBlock';
+import RequiredUpgrade from "@site/src/components/snippets/_upgrade-required.mdx"
+
+<RequiredUpgrade before='< 0.14.0' current='0.14.0' old='0.13.2' />
+
+## Source/Client IDs
+
+MS `0.14.0` introduces a database for persisting Plays/Scrobbles, queues, and other data **associated with a Source/Client**.
+
+To make this association atomic a Source/Client **ID** is now configurable. This ID is what MS will use to identify the Source/Client in your **config** with the Source/Client in the **database**.
+
+This ID needs to be unique to the client or source type it is used on IE:
+
+* two [Koito Clients](/configuration/clients/koito/) cannot both have the ID `myKoitoID`
+* it is recommended that the ID be *globally* unique among all sources/clients, but it's not required
+
+### Configuring IDs
+
+<Tabs groupId="configType" queryString>
+
+<TabItem value="env" label="ENV">
+
+For each Source/Client in your environmental variables, add a key with the suffix `_ID` and the ID value. EX
+
+```yaml title="compose.yaml"
+services:
+  multi-scrobbler:
+  # ...
+  environment:
+    - KOTIO_TOKEN=...
+    # ...more koito config
+  // highlight-start
+    - KOITO_ID=myKoitoID
+  // highlight-end
+
+    - JELLYFIN_URL=...
+    # ...more jellyin config
+  // highlight-start
+    - JELLYFIN_ID=myJellyfinID
+  // highlight-end    
+```
+
+</TabItem>
+<TabItem value="file" label="File">
+
+Add an `id` to the top-level for each Source/Client configuration, next to `data`:
+
+```json title="koito.json"
+[
+  {
+    "name": "koito-source",
+    "configureAs": "source",
+    // highlight-start
+    "id": "myKoitoID",
+    // highlight-end   
+    "data": {
+      "token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b",
+      "username": "admin",
+      "url": "http://192.168.0.100:4110" 
+    }
+  }
+]
+```
+
+</TabItem>
+<TabItem value="aio" label="AIO">
+
+Add an `id` to the top-level for each Source/Client configuration, next to `data`:
+
+```json title="koito.json"
+[
+  {
+    "name": "koito-source",
+    "configureAs": "source",
+    "type": "koito",
+    // highlight-start
+    "id": "myKoitoID",
+    // highlight-end    
+    "data": {
+      "token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b",
+      "username": "admin",
+      "url": "http://192.168.0.100:4110" 
+    }
+  }
+]
+```
+
+</TabItem>
+</Tabs>
+
+:::note[Default ID]
+
+If you do not add an ID then Multi-Scrobbler will automatically use the **name** of the Source/Client as the ID. The name is shown in the Dashboard.
+
+If you decide to add an ID later you must use the name as the ID in order to keep Plays/Scobbles associated with the same config.
+
+:::
+
+## Cached Scrobble Migration
+
+:::tip
+
+Before upgrading, if your MS dashboard shows 0 queued/failed for all [Scrobble Clients](/configuration/clients) then you **skip this step** and can safely delete `ms-scrobble.cache` before the upgrade occurs.
+
+:::
+
+MS `0.14.0` introduces a database for persisting Plays/Scrobbles, queues, and other data. Prior to `0.14.0`, queued/failed scrobbles were stored in a cache **file** inside your [`CONFIG_DIR`](/installation/?dockerSetting=storage#recommended-settings) named `ms-scrobble.cache`.
+
+On first run of `0.14.0`:
+
+* MS will make a copy of `ms-scrobble.cache` named `ms-scrobble.cache.bak`
+  * You may make a manual copy of this file before upgrading, for additional safety
+* Scrobbles in the cache will automatically be migrated to the new database
+* The now "old" cache data will be cleared so that subsequent application starts don't duplicate migrations
+  * the empty `ms-scrobble.cache` file will remain
+
+You can follow progress of this by looking for log lines starting with `Migrating cached scrobbles to database...`
+
+If MS does not report/log any errors during this time and you see your scrobbles processed normally then it is safe to delete `ms-scrobble.cache` the next time MS is stopped/restarted.
+
+## Cache Configuration
+
+In `0.14.0` [Cache](/configuration#caching) has been simplified with much of the required configuration being removed.
+
+:::tip
+
+This only applies to users who have `cache` in their [AIO Config](/configuration/?configType=aio#configuration-types) `config.json` or are using `CACHE_*` [ENV Config](/configuration/?configType=env#configuration-types)
+
+If you do not have any of the above [Cache](/configuration#caching) configuration defined then you can **skip this step.** 
+
+:::
+
+#### Scrobble Caching Removed
+
+[Scrobble caching has removed and replaced by the new database.](#cached-scrobble-migration)
+
+* Remove any ENV Config starting with `CACHE_SCROBBLE`
+* Remove `cache.scrobble` from the AIO Config
+
+#### Metadata Config Simplified
+
+Metadata caching remains the same but the config has been simplified. If you were using [Valkey for caching](/configuration/#secondary-caching) update your config:
+
+<Tabs groupId="configType" queryString>
+
+<TabItem value="env" label="ENV">
+
+* Remove `CACHE_METADATA`
+* Rename `CACHE_METADATA_CONN` to [`CACHE_VALKEY`](/configuration/#secondary-caching)
+
+</TabItem>
+<TabItem value="aio" label="AIO">
+
+Remove `cache.metadata` and add a new string key `valkey` to the `cache` object, containing your valkey connection string:
+
+```diff
+  "cache": {
++    "valkey": "redis://valkey:6379"
+-    "metadata": {
+-      "provider": "valkey",
+-      "connection": "redis://valkey:6379"
+-    }
+  }
+```
+
+</TabItem>
+</Tabs>
+
+
+#### Auth Config Simplified
+
+Auth caching remains the same but the config has been simplified. The connection option for Auth is no longer configurable. You may specify provider as either `file` (default, uses `CONFIG_DIR`) or `valkey`. If you want to use Valkey for auth it will use the same config as Metadata.
+
+<Tabs groupId="configType" queryString>
+
+<TabItem value="env" label="ENV">
+
+* Remove `CACHE_AUTH_CONN`
+* Add [`CACHE_VALKEY`](/configuration/#secondary-caching) (if using valkey)
+
+</TabItem>
+<TabItem value="aio" label="AIO">
+
+Remove the `connection` property from the `auth` object. If using `file` the entire `auth` object can be removed.
+
+```diff
+  "cache": {
++   "valkey": "redis://valkey:6379",
+    "auth": {
+      "provider": "valkey",
+-     "connection": "redis://valkey:6379"
+    }
+  }
+```
+
+</TabItem>
+</Tabs>
+
+### Renamed ENV Keys
+
+In an effort to standard ENV prefixes some ENV keys have been renamed:
+
+* `LZENDPOINT_ENABLE` => `LZE_ENABLE`
+* `LFMENDPOINT_ENABLE` => `LFM_ENABLE`

docsite/docs/updating/upgrade-path/_category_.json

@@ -0,0 +1,7 @@
+{
+  "label": "Upgrade Path",
+  "link": {
+    "type": "doc",
+    "id": "updating/upgrade-path/upgrade-path"
+  }
+}

docsite/docs/updating/upgrade-path/upgrade-path.mdx

@@ -0,0 +1,39 @@
+---
+sidebar_position: 1
+title: Upgrade Path
+description: Updating Multi-Scrobbler
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import CodeBlock from '@theme/CodeBlock';
+
+The majority of changes are detailed in [**Github Release Notes**](https://github.com/FoxxMD/multi-scrobbler/releases), including *most* breaking changes. You should consult the release notes for each version you will be updating through/to.
+
+Upgrade paths for specific version, detailed here, are reserved for:
+
+<Tabs groupId="upgradeType" queryString>
+
+<TabItem value="required" label="Required Upgrades">
+
+:::warning[Required Upgrade]
+
+Versions you **must** upgrade to **before** upgrading to a newer version. Likely due to the codebase containing functionality to upgrade persistent data that cannot be migrating across multiple versions.
+
+:::
+
+</TabItem>
+
+<TabItem value="depth" label="In Depth Upgrades">
+
+:::note[Optional Upgrade]
+
+(Breaking) Changes to configuration/data that require more detail or formatting than is feasible in Github release notes. These upgrades may be skipped.
+
+:::
+
+</TabItem>
+
+</Tabs>
+
+The type of upgrade is shown at the top of each page with the same colored callout shown above.

docsite/src/components/AdmonitionDetails.tsx

@@ -5,9 +5,10 @@ import IconDanger from '@theme/Admonition/Icon/Danger';
 import IconTip from '@theme/Admonition/Icon/Tip';
 import IconNote from '@theme/Admonition/Icon/Note';
 import { ReactElement } from 'react';
+import AdmonitionIconImportant from './ImportantIcon';
 
 export interface DetailsAdmoProps extends DetailProps {
-    type?: 'warning' | 'danger' | 'note' | 'tip'
+    type?: 'warning' | 'danger' | 'note' | 'tip' | 'important'
 }
 
 const DetailsAdmo = (props: DetailsAdmoProps) => {
@@ -32,6 +33,10 @@ const DetailsAdmo = (props: DetailsAdmoProps) => {
             cn = 'alert--success';
             icon = <IconTip/>
             break;
+        case 'important':
+            cn = 'alert--important';
+            icon = <AdmonitionIconImportant/>;
+            break;            
     }
 
     const iconWrapper = icon === undefined ? null : <span className="admoIcon">{icon}</span>;

docsite/src/components/ImportantIcon.tsx

@@ -0,0 +1,13 @@
+import type { Props } from '@theme/Admonition/Icon/Info';
+
+export default function AdmonitionIconImportant(props: Props) {
+    // based on https://icon-sets.iconify.design/zondicons/exclamation-outline/
+  return (
+    <svg viewBox="0 0 22 22" {...props}>
+      <path
+        fillRule="evenodd"
+        d="M2.93 17.07A10 10 0 1 1 17.07 2.93A10 10 0 0 1 2.93 17.07m12.73-1.41A8 8 0 1 0 4.34 4.34a8 8 0 0 0 11.32 11.32M9 5h2v6H9zm0 8h2v2H9z"
+      />
+    </svg>
+  );
+}

docsite/src/components/snippets/_upgrade-required.mdx

@@ -0,0 +1,15 @@
+:::warning[Required Upgrade]
+
+You **MUST** upgrade to this version **before** upgrading Multi-Scrobbler to a newer version.
+
+This version contains functionality to migrate your data from **{props.before}** that newer versions depend on.
+
+<details>
+
+<summary>Upgrade Process Example</summary>
+
+* Upgrade Multi-Scrobbler {props.old} to {props.current}
+* **After** upgrading, you can optionally upgrade to > {props.current}
+
+</details>
+:::

docsite/src/css/custom.css

@@ -13,6 +13,7 @@
   --ifm-color-primary-light: #33925d;
   --ifm-color-primary-lighter: #359962;
   --ifm-color-primary-lightest: #3cad6e;
+  --ifm-color-important-contrast-background: #bb91ff;
   --ifm-code-font-size: 95%;
   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1);
 }
@@ -26,6 +27,7 @@
   --ifm-color-primary-light: #29d5b0;
   --ifm-color-primary-lighter: #32d8b4;
   --ifm-color-primary-lightest: #4fddbf;
+  --ifm-color-important-contrast-background: #5a3795;
   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3);
 }
 
@@ -59,6 +61,10 @@ details[class^='details_'].alert--info {
     --ifm-alert-background-color: var(--ifm-color-secondary-contrast-background);
     --ifm-alert-foreground-color: var(--ifm-color-secondary-contrast-foreground);
   }
+  &.alert--important {
+    --ifm-alert-background-color: var(--ifm-color-important-contrast-background);
+    --ifm-alert-foreground-color: var(--ifm-color-secondary-contrast-foreground);
+  }
 
   > summary {
     --docusaurus-details-decoration-color: currentColor;

drizzle.config.ts

@@ -0,0 +1,13 @@
+import 'dotenv/config';
+import { defineConfig } from 'drizzle-kit';
+import { configDir, projectDir } from './src/backend/common/index.js';
+import * as path from 'path';
+
+export default defineConfig({
+  schema: path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'),
+  out: path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations'),
+  dialect: 'postgresql',
+  dbCredentials: {
+    url: path.resolve(configDir, 'msDb'),
+  },
+});

package-lock.json

@@ -15,6 +15,8 @@
         "@atproto/api": "^0.18.0",
         "@atproto/oauth-client-node": "^0.3.10",
         "@donedeal0/superdiff": "^1.1.1",
+        "@electric-sql/pglite": "^0.4.5",
+        "@electric-sql/pglite-socket": "^0.1.5",
         "@ewanc26/tid": "^1.0.2",
         "@foxxmd/chromecast-client": "^1.0.4",
         "@foxxmd/get-version": "^0.0.3",
@@ -47,6 +49,7 @@
         "dbus-ts": "^0.0.7",
         "discord.js": "^14.26.0",
         "dotenv": "^10.0.0",
+        "drizzle-orm": "^1.0.0-rc.2-c5a84d1",
         "express": "^5.2.1",
         "express-session": "^1.19.0",
         "fast-equals": "^6.0.0",
@@ -71,6 +74,7 @@
         "mpc-js": "^2.1.1",
         "musicbrainz-api": "^0.27.0",
         "nanoid": "^3.3.1",
+        "neotraverse": "^0.6.18",
         "node-object-hash": "^3.1.1",
         "normalize-url": "^8.0.1",
         "ntfy": "^1.15.2",
@@ -102,6 +106,7 @@
         "@chromatic-com/storybook": "^5.0.1",
         "@curvenote/ansi-to-react": "^7.0.0",
         "@dbus-types/notifications": "^0.0.5",
+        "@electric-sql/pglite-prepopulatedfs": "^0.0.3",
         "@emotion/react": "^11.14.0",
         "@eslint/js": "^8.56.0",
         "@faker-js/faker": "^9.0.1",
@@ -129,7 +134,7 @@
         "@types/jest": "^27.5.2",
         "@types/jscodeshift": "^0.11.6",
         "@types/mocha": "^9.1.0",
-        "@types/node": "^20.19.2",
+        "@types/node": "^24.12.2",
         "@types/object-hash": "^3.0.0",
         "@types/passport": "^1.0.12",
         "@types/react": "^18.2.18",
@@ -144,6 +149,7 @@
         "chai": "^4.3.6",
         "chai-as-promised": "^8.0.2",
         "clsx": "^2.1.1",
+        "drizzle-kit": "^1.0.0-rc.2-c5a84d1",
         "eslint": "^8.56.0",
         "eslint-plugin-prefer-arrow-functions": "^3.2.4",
         "eslint-plugin-storybook": "10.1.11",
@@ -152,7 +158,6 @@
         "mocha": "^10.3.0",
         "mockdate": "^3.0.5",
         "msw": "^2.12.10",
-        "neotraverse": "^0.6.18",
         "next-themes": "^0.4.6",
         "nodemon": "^3.0.3",
         "playwright": "^1.58.2",
@@ -168,11 +173,12 @@
         "sinon": "^21.0.2",
         "storybook": "10.2.0",
         "tailwindcss": "^4.2.2",
-        "ts-essentials": "^10.1.1",
+        "tinyexec": "^1.1.1",
+        "ts-essentials": "^10.2.0",
         "typescript": "^5.9.3",
         "typescript-eslint": "^7.0.1",
         "vite": "^5.4.21",
-        "with-local-tmp-dir": "^6.0.0"
+        "with-local-tmp-dir": "^7.0.1"
       },
       "engines": {
         "node": ">=24.14.0",
@@ -1055,6 +1061,13 @@
       "version": "1.1.3",
       "license": "ISC"
     },
+    "node_modules/@drizzle-team/brocli": {
+      "version": "0.11.0",
+      "resolved": "https://registry.npmjs.org/@drizzle-team/brocli/-/brocli-0.11.0.tgz",
+      "integrity": "sha512-hD3pekGiPg0WPCCGAZmusBBJsDqGUR66Y452YgQsZOnkdQ7ViEPKuyP4huUGEZQefp8g34RRodXYmJ2TbCH+tg==",
+      "dev": true,
+      "license": "Apache-2.0"
+    },
     "node_modules/@dword-design/chdir": {
       "version": "4.0.0",
       "dev": true,
@@ -1066,6 +1079,31 @@
         "url": "https://github.com/sponsors/dword-design"
       }
     },
+    "node_modules/@electric-sql/pglite": {
+      "version": "0.4.5",
+      "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.4.5.tgz",
+      "integrity": "sha512-aGG2zGEyZzGWKy8P+9ZoNUV0jxt1+hgbeTf+bVAYyxVZZLXg3/9aFlfLxb08AYZVAfAkQlQIysmWjhc5hwDG8g==",
+      "license": "Apache-2.0"
+    },
+    "node_modules/@electric-sql/pglite-prepopulatedfs": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/@electric-sql/pglite-prepopulatedfs/-/pglite-prepopulatedfs-0.0.3.tgz",
+      "integrity": "sha512-3MNFt+gR0P22foWi55j/HZ6DvQ82DEVIvmoKVYCdoG/gezMikGR794tO07/15CV4RcR3PnfCPoQjApPfXfD01w==",
+      "dev": true,
+      "license": "Apache-2.0"
+    },
+    "node_modules/@electric-sql/pglite-socket": {
+      "version": "0.1.5",
+      "resolved": "https://registry.npmjs.org/@electric-sql/pglite-socket/-/pglite-socket-0.1.5.tgz",
+      "integrity": "sha512-/RAye+3EPKfO9nY4tljzxXmkT7yIpFDm0L3F+c28b+Z6uxPOjy/Zz/QEHYHXcrfuUC88/a9S72EO0+3E0j97wQ==",
+      "license": "Apache-2.0",
+      "bin": {
+        "pglite-server": "dist/scripts/server.js"
+      },
+      "peerDependencies": {
+        "@electric-sql/pglite": "0.4.5"
+      }
+    },
     "node_modules/@emotion/babel-plugin": {
       "version": "11.13.5",
       "dev": true,
@@ -2174,6 +2212,19 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "node_modules/@js-temporal/polyfill": {
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/@js-temporal/polyfill/-/polyfill-0.5.1.tgz",
+      "integrity": "sha512-hloP58zRVCRSpgDxmqCWJNlizAlUgJFqG2ypq79DCvyv9tHjRYMDOcPFjzfl/A1/YxDvRCZz8wvZvmapQnKwFQ==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "jsbi": "^4.3.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/@kenyip/backoff-strategies": {
       "version": "1.0.4",
       "license": "MIT"
@@ -4288,10 +4339,12 @@
       "license": "MIT"
     },
     "node_modules/@types/node": {
-      "version": "20.19.37",
+      "version": "24.12.2",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.2.tgz",
+      "integrity": "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==",
       "license": "MIT",
       "dependencies": {
-        "undici-types": "~6.21.0"
+        "undici-types": "~7.16.0"
       }
     },
     "node_modules/@types/object-hash": {
@@ -4364,11 +4417,6 @@
         "@types/node": "*"
       }
     },
-    "node_modules/@types/retry": {
-      "version": "0.12.2",
-      "dev": true,
-      "license": "MIT"
-    },
     "node_modules/@types/send": {
       "version": "0.17.4",
       "dev": true,
@@ -7241,6 +7289,665 @@
         "node": ">=10"
       }
     },
+    "node_modules/drizzle-kit": {
+      "version": "1.0.0-rc.2-c5a84d1",
+      "resolved": "https://registry.npmjs.org/drizzle-kit/-/drizzle-kit-1.0.0-rc.2-c5a84d1.tgz",
+      "integrity": "sha512-TjXBd6/Jo8aqC2uBbbgPoIVE5Qr9s0KubVtdj3Ro7bF2lz1TkGOhnDIzsOgMJPMSPn9uyhNG6WBKdGz+XzcL9g==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@drizzle-team/brocli": "^0.11.0",
+        "@js-temporal/polyfill": "^0.5.1",
+        "esbuild": "^0.25.10",
+        "get-tsconfig": "^4.13.6",
+        "jiti": "^2.6.1"
+      },
+      "bin": {
+        "drizzle-kit": "bin.cjs"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/aix-ppc64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz",
+      "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==",
+      "cpu": [
+        "ppc64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "aix"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/android-arm": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz",
+      "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/android-arm64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz",
+      "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/android-x64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz",
+      "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/darwin-arm64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz",
+      "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/darwin-x64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz",
+      "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/freebsd-arm64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz",
+      "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "freebsd"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/freebsd-x64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz",
+      "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "freebsd"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/linux-arm": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz",
+      "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/linux-arm64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz",
+      "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/linux-ia32": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz",
+      "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==",
+      "cpu": [
+        "ia32"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/linux-loong64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz",
+      "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==",
+      "cpu": [
+        "loong64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/linux-mips64el": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz",
+      "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==",
+      "cpu": [
+        "mips64el"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/linux-ppc64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz",
+      "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==",
+      "cpu": [
+        "ppc64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/linux-riscv64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz",
+      "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==",
+      "cpu": [
+        "riscv64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/linux-s390x": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz",
+      "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==",
+      "cpu": [
+        "s390x"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/linux-x64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz",
+      "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/netbsd-arm64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz",
+      "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "netbsd"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/netbsd-x64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz",
+      "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "netbsd"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/openbsd-arm64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz",
+      "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "openbsd"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/openbsd-x64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz",
+      "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "openbsd"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/openharmony-arm64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz",
+      "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "openharmony"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/sunos-x64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz",
+      "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "sunos"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/win32-arm64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz",
+      "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/win32-ia32": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz",
+      "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==",
+      "cpu": [
+        "ia32"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/@esbuild/win32-x64": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz",
+      "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/drizzle-kit/node_modules/esbuild": {
+      "version": "0.25.12",
+      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz",
+      "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "bin": {
+        "esbuild": "bin/esbuild"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "@esbuild/aix-ppc64": "0.25.12",
+        "@esbuild/android-arm": "0.25.12",
+        "@esbuild/android-arm64": "0.25.12",
+        "@esbuild/android-x64": "0.25.12",
+        "@esbuild/darwin-arm64": "0.25.12",
+        "@esbuild/darwin-x64": "0.25.12",
+        "@esbuild/freebsd-arm64": "0.25.12",
+        "@esbuild/freebsd-x64": "0.25.12",
+        "@esbuild/linux-arm": "0.25.12",
+        "@esbuild/linux-arm64": "0.25.12",
+        "@esbuild/linux-ia32": "0.25.12",
+        "@esbuild/linux-loong64": "0.25.12",
+        "@esbuild/linux-mips64el": "0.25.12",
+        "@esbuild/linux-ppc64": "0.25.12",
+        "@esbuild/linux-riscv64": "0.25.12",
+        "@esbuild/linux-s390x": "0.25.12",
+        "@esbuild/linux-x64": "0.25.12",
+        "@esbuild/netbsd-arm64": "0.25.12",
+        "@esbuild/netbsd-x64": "0.25.12",
+        "@esbuild/openbsd-arm64": "0.25.12",
+        "@esbuild/openbsd-x64": "0.25.12",
+        "@esbuild/openharmony-arm64": "0.25.12",
+        "@esbuild/sunos-x64": "0.25.12",
+        "@esbuild/win32-arm64": "0.25.12",
+        "@esbuild/win32-ia32": "0.25.12",
+        "@esbuild/win32-x64": "0.25.12"
+      }
+    },
+    "node_modules/drizzle-orm": {
+      "version": "1.0.0-rc.2-c5a84d1",
+      "resolved": "https://registry.npmjs.org/drizzle-orm/-/drizzle-orm-1.0.0-rc.2-c5a84d1.tgz",
+      "integrity": "sha512-2nw0eVFNcLRU7mhs/f/UodwvRIP78ZzZXMrl/r6LpTpcJ42ZzQZZTH0ObPCR722SaEzn35kT4gvuoiam/OCK9g==",
+      "license": "Apache-2.0",
+      "peerDependencies": {
+        "@aws-sdk/client-rds-data": ">=3",
+        "@cloudflare/workers-types": ">=4",
+        "@effect/sql-pg": ">=4.0.0-beta.58 || >=4.0.0",
+        "@electric-sql/pglite": ">=0.2.0",
+        "@libsql/client": ">=0.10.0",
+        "@libsql/client-wasm": ">=0.10.0",
+        "@neondatabase/serverless": ">=0.10.0",
+        "@op-engineering/op-sqlite": ">=2",
+        "@opentelemetry/api": "^1.4.1",
+        "@planetscale/database": ">=1.13",
+        "@sinclair/typebox": ">=0.34.8",
+        "@sqlitecloud/drivers": ">=1.0.653",
+        "@tidbcloud/serverless": "*",
+        "@tursodatabase/database": ">=0.2.1",
+        "@tursodatabase/database-common": ">=0.2.1",
+        "@tursodatabase/database-wasm": ">=0.2.1",
+        "@types/better-sqlite3": "*",
+        "@types/mssql": "^9.1.4",
+        "@types/pg": "*",
+        "@types/sql.js": "*",
+        "@upstash/redis": ">=1.34.7",
+        "@vercel/postgres": ">=0.8.0",
+        "@xata.io/client": "*",
+        "arktype": ">=2.0.0",
+        "better-sqlite3": ">=9.3.0",
+        "bun-types": "*",
+        "effect": ">=4.0.0-beta.58 || >=4.0.0",
+        "expo-sqlite": ">=14.0.0",
+        "mssql": "^11.0.1",
+        "mysql2": ">=2",
+        "pg": ">=8",
+        "postgres": ">=3",
+        "sql.js": ">=1",
+        "sqlite3": ">=5",
+        "typebox": ">=1.0.0",
+        "valibot": ">=1.0.0-beta.7",
+        "zod": "^3.25.0 || ^4.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@aws-sdk/client-rds-data": {
+          "optional": true
+        },
+        "@cloudflare/workers-types": {
+          "optional": true
+        },
+        "@effect/sql-pg": {
+          "optional": true
+        },
+        "@electric-sql/pglite": {
+          "optional": true
+        },
+        "@libsql/client": {
+          "optional": true
+        },
+        "@libsql/client-wasm": {
+          "optional": true
+        },
+        "@neondatabase/serverless": {
+          "optional": true
+        },
+        "@op-engineering/op-sqlite": {
+          "optional": true
+        },
+        "@opentelemetry/api": {
+          "optional": true
+        },
+        "@planetscale/database": {
+          "optional": true
+        },
+        "@sinclair/typebox": {
+          "optional": true
+        },
+        "@sqlitecloud/drivers": {
+          "optional": true
+        },
+        "@tidbcloud/serverless": {
+          "optional": true
+        },
+        "@tursodatabase/database": {
+          "optional": true
+        },
+        "@tursodatabase/database-common": {
+          "optional": true
+        },
+        "@tursodatabase/database-wasm": {
+          "optional": true
+        },
+        "@types/better-sqlite3": {
+          "optional": true
+        },
+        "@types/mssql": {
+          "optional": true
+        },
+        "@types/pg": {
+          "optional": true
+        },
+        "@types/sql.js": {
+          "optional": true
+        },
+        "@upstash/redis": {
+          "optional": true
+        },
+        "@vercel/postgres": {
+          "optional": true
+        },
+        "@xata.io/client": {
+          "optional": true
+        },
+        "arktype": {
+          "optional": true
+        },
+        "better-sqlite3": {
+          "optional": true
+        },
+        "bun-types": {
+          "optional": true
+        },
+        "effect": {
+          "optional": true
+        },
+        "expo-sqlite": {
+          "optional": true
+        },
+        "mssql": {
+          "optional": true
+        },
+        "mysql2": {
+          "optional": true
+        },
+        "pg": {
+          "optional": true
+        },
+        "postgres": {
+          "optional": true
+        },
+        "sql.js": {
+          "optional": true
+        },
+        "sqlite3": {
+          "optional": true
+        },
+        "typebox": {
+          "optional": true
+        },
+        "valibot": {
+          "optional": true
+        },
+        "zod": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/dunder-proto": {
       "version": "1.0.1",
       "license": "MIT",
@@ -8564,7 +9271,9 @@
       }
     },
     "node_modules/get-tsconfig": {
-      "version": "4.8.1",
+      "version": "4.14.0",
+      "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz",
+      "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==",
       "license": "MIT",
       "dependencies": {
         "resolve-pkg-maps": "^1.0.0"
@@ -9597,6 +10306,13 @@
         "js-yaml": "bin/js-yaml.js"
       }
     },
+    "node_modules/jsbi": {
+      "version": "4.3.2",
+      "resolved": "https://registry.npmjs.org/jsbi/-/jsbi-4.3.2.tgz",
+      "integrity": "sha512-9fqMSQbhJykSeii05nxKl4m6Eqn2P6rOlYiS+C5Dr/HPIU/7yZxu5qzbs40tgaFORiw2Amd0mirjxatXYMkIew==",
+      "dev": true,
+      "license": "Apache-2.0"
+    },
     "node_modules/jsesc": {
       "version": "3.1.0",
       "dev": true,
@@ -10924,7 +11640,8 @@
     },
     "node_modules/neotraverse": {
       "version": "0.6.18",
-      "dev": true,
+      "resolved": "https://registry.npmjs.org/neotraverse/-/neotraverse-0.6.18.tgz",
+      "integrity": "sha512-Z4SmBUweYa09+o6pG+eASabEpP6QkQ70yHj351pQoEXIs8uHbaU2DWVmzBANKgflPa47A50PtB2+NgRpQvr7vA==",
       "license": "MIT",
       "engines": {
         "node": ">= 10"
@@ -13430,6 +14147,16 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/tinyexec": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.1.1.tgz",
+      "integrity": "sha512-VKS/ZaQhhkKFMANmAOhhXVoIfBXblQxGX1myCQ2faQrfmobMftXeJPcZGp0gS07ocvGJWDLZGyOZDadDBqYIJg==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/tinyglobby": {
       "version": "0.2.15",
       "dev": true,
@@ -13594,7 +14321,9 @@
       }
     },
     "node_modules/ts-essentials": {
-      "version": "10.1.1",
+      "version": "10.2.0",
+      "resolved": "https://registry.npmjs.org/ts-essentials/-/ts-essentials-10.2.0.tgz",
+      "integrity": "sha512-z9FlLywg0XEV46Ws1FwYN4NZDMr9qAe38lTTtgVBqzhhyEgwrnCUkFe4MEqnvar1kY1kFEnlkp56bxn2g0V+UA==",
       "dev": true,
       "license": "MIT",
       "peerDependencies": {
@@ -13947,7 +14676,9 @@
       }
     },
     "node_modules/undici-types": {
-      "version": "6.21.0",
+      "version": "7.16.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
+      "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
       "license": "MIT"
     },
     "node_modules/unicorn-magic": {
@@ -14723,37 +15454,23 @@
       }
     },
     "node_modules/with-local-tmp-dir": {
-      "version": "6.0.0",
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/with-local-tmp-dir/-/with-local-tmp-dir-7.0.1.tgz",
+      "integrity": "sha512-ipyC5Q9ZBbpeLnrBi4BEUElnvDxv0QM0I+ptSIGUMegR9/KVz/bd+WMMWm0Q0qnjOSwSRaxqVfqzy0kVPmaVig==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "@dword-design/chdir": "^4.0.0",
         "@types/fs-extra": "^11.0.4",
-        "p-retry": "^6.2.1"
+        "p-retry": "^7.1.1"
       },
       "engines": {
-        "node": ">=20"
+        "node": ">=22"
       },
       "funding": {
         "url": "https://github.com/sponsors/dword-design"
       }
     },
-    "node_modules/with-local-tmp-dir/node_modules/p-retry": {
-      "version": "6.2.1",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@types/retry": "0.12.2",
-        "is-network-error": "^1.0.0",
-        "retry": "^0.13.1"
-      },
-      "engines": {
-        "node": ">=16.17"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/word-wrap": {
       "version": "1.2.5",
       "dev": true,
@@ -15089,7 +15806,9 @@
       }
     },
     "node_modules/zod": {
-      "version": "3.23.8",
+      "version": "3.25.76",
+      "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
+      "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==",
       "license": "MIT",
       "funding": {
         "url": "https://github.com/sponsors/colinhacks"
@@ -15097,6 +15816,8 @@
     },
     "node_modules/zod-validation-error": {
       "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/zod-validation-error/-/zod-validation-error-2.1.0.tgz",
+      "integrity": "sha512-VJh93e2wb4c3tWtGgTa0OF/dTt/zoPCPzXq4V11ZjxmEAFaPi/Zss1xIZdEB5RD8GD00U0/iVXgqkF77RV7pdQ==",
       "license": "MIT",
       "engines": {
         "node": ">=18.0.0"

package.json

@@ -16,6 +16,7 @@
     "build:backend": "tsc -p src/backend && npm run -s schema:app",
     "build": "npm run -s build:backend && npm run -s build:frontend && npm run -s docs:build",
     "build:parallel": "concurrently --kill-others-on-fail --names backend,frontend,docs \"npm run -s build:backend\" \"npm run -s build:frontend\" \"npm run docs:build\"",
+    "db:start": "pglite-server --db=./config/msDb --port=5433 --host=0.0.0.0 -m 10",
     "docs:install": "cd docsite && npm install --no-audit",
     "docs:start": "cd docsite && npm start",
     "docs:build": "npm run -s schema:docs && cd docsite && npm run build",
@@ -53,6 +54,8 @@
     "@atproto/api": "^0.18.0",
     "@atproto/oauth-client-node": "^0.3.10",
     "@donedeal0/superdiff": "^1.1.1",
+    "@electric-sql/pglite": "^0.4.5",
+    "@electric-sql/pglite-socket": "^0.1.5",
     "@ewanc26/tid": "^1.0.2",
     "@foxxmd/chromecast-client": "^1.0.4",
     "@foxxmd/get-version": "^0.0.3",
@@ -85,6 +88,7 @@
     "dbus-ts": "^0.0.7",
     "discord.js": "^14.26.0",
     "dotenv": "^10.0.0",
+    "drizzle-orm": "^1.0.0-rc.2-c5a84d1",
     "express": "^5.2.1",
     "express-session": "^1.19.0",
     "fast-equals": "^6.0.0",
@@ -109,6 +113,7 @@
     "mpc-js": "^2.1.1",
     "musicbrainz-api": "^0.27.0",
     "nanoid": "^3.3.1",
+    "neotraverse": "^0.6.18",
     "node-object-hash": "^3.1.1",
     "normalize-url": "^8.0.1",
     "ntfy": "^1.15.2",
@@ -140,6 +145,7 @@
     "@chromatic-com/storybook": "^5.0.1",
     "@curvenote/ansi-to-react": "^7.0.0",
     "@dbus-types/notifications": "^0.0.5",
+    "@electric-sql/pglite-prepopulatedfs": "^0.0.3",
     "@emotion/react": "^11.14.0",
     "@eslint/js": "^8.56.0",
     "@faker-js/faker": "^9.0.1",
@@ -167,7 +173,7 @@
     "@types/jest": "^27.5.2",
     "@types/jscodeshift": "^0.11.6",
     "@types/mocha": "^9.1.0",
-    "@types/node": "^20.19.2",
+    "@types/node": "^24.12.2",
     "@types/object-hash": "^3.0.0",
     "@types/passport": "^1.0.12",
     "@types/react": "^18.2.18",
@@ -182,6 +188,7 @@
     "chai": "^4.3.6",
     "chai-as-promised": "^8.0.2",
     "clsx": "^2.1.1",
+    "drizzle-kit": "^1.0.0-rc.2-c5a84d1",
     "eslint": "^8.56.0",
     "eslint-plugin-prefer-arrow-functions": "^3.2.4",
     "eslint-plugin-storybook": "10.1.11",
@@ -190,7 +197,6 @@
     "mocha": "^10.3.0",
     "mockdate": "^3.0.5",
     "msw": "^2.12.10",
-    "neotraverse": "^0.6.18",
     "next-themes": "^0.4.6",
     "nodemon": "^3.0.3",
     "playwright": "^1.58.2",
@@ -206,11 +212,12 @@
     "sinon": "^21.0.2",
     "storybook": "10.2.0",
     "tailwindcss": "^4.2.2",
-    "ts-essentials": "^10.1.1",
+    "tinyexec": "^1.1.1",
+    "ts-essentials": "^10.2.0",
     "typescript": "^5.9.3",
     "typescript-eslint": "^7.0.1",
     "vite": "^5.4.21",
-    "with-local-tmp-dir": "^6.0.0"
+    "with-local-tmp-dir": "^7.0.1"
   },
   "browserslist": {
     "production": [

src/backend/common/AbstractComponent.ts

@@ -6,7 +6,7 @@ import { LifecycleStep, PlayData, PlayObject, TransformResult } from "../../core
 import { buildPlayHumanDiffable, buildTrackString } from "../../core/StringUtils.js";
 import { CommonClientConfig } from "./infrastructure/config/client/index.js";
 import { CommonSourceConfig } from "./infrastructure/config/source/index.js";
-import { mergeSimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js";
+import { mergeSimpleError, SimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js";
 import {
     FLOW_CONTROL_TERM,
     PlayTransformRules,
@@ -20,13 +20,21 @@ import { getRoot } from "../ioc.js";
 import { nanoid } from "nanoid";
 import { isDebugMode } from "../utils.js";
 import { findCauseByReference } from "../utils/ErrorUtils.js";
-import { hashObject } from "../utils/StringUtils.js";
+import { hashObject, parseArrayFromMaybeString } from "../utils/StringUtils.js";
 import { metaInvariantTransform, playContentInvariantTransform } from "../utils/PlayComparisonUtils.js";
 import { MSCache } from "./Cache.js";
 import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/DataUtils.js";
 import clone from "clone";
 import { loggerNoop } from "./MaybeLogger.js";
 import { objectsEqual } from "../utils/DataUtils.js";
+import { RetentionOptions } from "./infrastructure/config/database.js";
+import { getRetentionCompactAfterFromEnv, getRetentionDeleteAfterFromEnv, isCompactableProperty, parseRetentionOptions, parseRetentionOptionsDurations } from "./database/Database.js";
+import { DbConcrete } from "./database/drizzle/drizzleUtils.js";
+import { ComponentSelect } from "./database/drizzle/drizzleTypes.js";
+import { DrizzlePlayRepository } from "./database/drizzle/repositories/PlayRepository.js";
+import { ClientType } from "./infrastructure/config/client/clients.js";
+import { SourceType } from "./infrastructure/config/source/sources.js";
+import { DrizzleComponentRepository } from "./database/drizzle/repositories/ComponentRepository.js";
 
 export type AbstractComponentConfig = (CommonClientConfig | CommonSourceConfig) & { transformManager?: TransformerManager };
 
@@ -38,11 +46,27 @@ export default abstract class AbstractComponent extends AbstractInitializable {
     regexCache!: ReturnType<typeof cacheFunctions>;
     protected transformManager: TransformerManager;
     protected cache: MSCache;
+    protected db: DbConcrete;
+    protected componentRepo!: DrizzleComponentRepository;
+    protected dbComponent!: ComponentSelect;
+    protected retentionOpts: RetentionOptions;
+
+    protected componentType: 'source' | 'client';
+    type: ClientType | SourceType;
 
     protected constructor(config: AbstractComponentConfig) {
         super(config);
         this.transformManager = config.transformManager ?? getRoot().items.transformerManager;
         this.cache = getRoot().items.cache();
+        const cProps = config.options?.retention?.compact ?? parseArrayFromMaybeString(process.env.COMPACT_PROPERTIES, {lower: true});
+        if(!cProps.every(isCompactableProperty)) {
+            throw new SimpleError(`Compactable properties must be one of 'transform' or 'input'. Given: ${cProps.join(',')}`);
+        }
+        this.retentionOpts = {
+            deleteAfter: parseRetentionOptionsDurations(config.options?.retention?.deleteAfter, getRetentionDeleteAfterFromEnv()),
+            compactAfter: parseRetentionOptions(config.options?.retention?.compactAfter, getRetentionCompactAfterFromEnv()),
+            compact: cProps
+        };
     }
 
     protected postCache(): Promise<void> {
@@ -54,6 +78,25 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         }
     }
 
+    protected async doBuildDatabase(): Promise<true | string | undefined> {
+        super.doBuildDatabase();
+
+        let name: string;
+        if('name' in this) {
+            name = this.name as string;
+        }
+
+        this.db = await getRoot().items.db();
+        this.componentRepo = new DrizzleComponentRepository(this.db, {logger: this.logger});
+        this.dbComponent = await this.componentRepo.findOrInsert({
+            mode: this.componentType,
+            type: this.type,
+            uid: this.config.id ?? this.config.name ?? name,
+            name: this.config.name ?? name
+        });
+        return true;
+    }
+
     public buildTransformRules() {
         this.logger.debug('Building transformer rules...');
         try {
@@ -154,6 +197,19 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         }
     }
 
+    public retentionCleanup = async () => {
+        if(this.databaseOK !== true) {
+            this.logger.warn(`Cannot run retention cleanup because ${this.componentType} database state is not OK`);
+            return;
+        }
+        try {
+            const repo = new DrizzlePlayRepository(this.db, {logger: this.logger});
+            await repo.retentionCleanup(this.componentType, this.retentionOpts);
+        } catch (e) {
+            this.logger.warn(new Error('Failed to do retention cleanup', {cause: e}));
+        }
+    }
+
     protected transformPartToStrong(data: any) {
         if(data === undefined) {
             return undefined;

src/backend/common/AbstractInitializable.ts

@@ -13,6 +13,7 @@ export default abstract class AbstractInitializable {
     authFailure?: boolean;
 
     buildOK?: boolean | null;
+    databaseOK?: boolean | null;
     connectionOK?: boolean | null;
     cacheOK?: boolean | null;
 
@@ -41,6 +42,7 @@ export default abstract class AbstractInitializable {
             if(this.componentLogger === undefined) {
                 await this.buildComponentLogger();
             }
+            await this.buildDatabase(force);
             await this.buildInitData(force);
             await this.parseCache(force);
             try {
@@ -96,13 +98,13 @@ export default abstract class AbstractInitializable {
             if(!force) {
                 return;
             }
-            this.logger.debug('Cache OK but step was forced');
+            this.logger.verbose('Cache OK but step was forced');
         }
         try {
             const res = await this.doParseCache();
             if(res === undefined) {
                 this.cacheOK = null;
-                this.logger.debug('No cache to parse.');
+                this.logger.trace('No cache to parse.');
                 return;
             }
             if (res === true) {
@@ -139,13 +141,13 @@ export default abstract class AbstractInitializable {
             if(!force) {
                 return;
             }
-            this.logger.debug('Build OK but step was forced');
+            this.logger.verbose('Build OK but step was forced');
         }
         try {
             const res = await this.doBuildInitData();
             if(res === undefined) {
                 this.buildOK = null;
-                this.logger.debug('No required data to build.');
+                this.logger.trace('No required data to build.');
                 return;
             }
             if (res === true) {
@@ -172,6 +174,58 @@ export default abstract class AbstractInitializable {
         return;
     }
 
+    public async buildDatabase(force: boolean = false) {
+        if(this.databaseOK) {
+            if(!force) {
+                return;
+            }
+            this.logger.verbose('Database OK but step was forced');
+        }
+        try {
+            const res = await this.doBuildDatabase();
+            if(res === undefined) {
+                this.databaseOK = null;
+                this.logger.trace('No required database steps.');
+                return;
+            }
+            if (res === true) {
+                this.logger.verbose('Required database init succeeded');
+            } else if (typeof res === 'string') {
+                this.logger.verbose(`Required database init succeeded => ${res}`);
+            }
+            this.databaseOK = true;
+        } catch (e) {
+            this.databaseOK = false;
+            throw new BuildDataError('Required database init failed', {cause: e});
+        }
+
+        try {
+            await this.postDatabase();
+        } catch (e) {
+            if(e instanceof StageError) {
+                throw e;
+            } else {
+                throw new Error('Error occurred during post-database hook', {cause: e});
+            }
+        }
+    }
+
+    protected async postDatabase(): Promise<void> {
+        return;
+    }
+
+    /**
+     * Run/fetch/create any database data needed for this component to operate when ready
+     *
+     * * Return undefined if not possible or not required
+     * * Return TRUE if database steps succeeded
+     * * Return string if database steps succeeded and should log result
+     * * Throw error on failure
+     * */
+    protected async doBuildDatabase(): Promise<true | string | undefined> {
+        return;
+    }
+
     public async checkConnection(force: boolean = false) {
         if(this.connectionOK) {
             if(!force) {
@@ -252,12 +306,14 @@ export default abstract class AbstractInitializable {
 
     public isReady() {
         return (this.buildOK === null || this.buildOK === true) &&
+            (this.databaseOK === null || this.databaseOK === true) &&
             (this.connectionOK === null || this.connectionOK === true)
             && !this.authGated();
     }
 
     public isUsable() {
         return (this.buildOK === null || this.buildOK === true) &&
+            (this.databaseOK === null || this.databaseOK === true) &&
             (this.connectionOK === null || this.connectionOK === true);
     }
 

src/backend/common/Cache.ts

@@ -13,14 +13,15 @@ import { childLogger, Logger } from '@foxxmd/logging';
 import { projectDir } from './index.js';
 import path from 'path';
 import { cacheFunctions } from "@foxxmd/regex-buddy-core";
-import { fileOrDirectoryIsWriteable } from '../utils/FSUtils.js';
-import { asCacheAuthProvider, asCacheConfig, asCacheMetadataProvider, asCacheScrobbleProvider, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheMetadataProvider, CacheProvider, CacheScrobbleProvider } from './infrastructure/Atomic.js';
+import { fileExists, fileOrDirectoryIsWriteable } from '../utils/FSUtils.js';
+import { copyFile } from 'fs/promises';
+import { asCacheConfig, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheConfigUser, CacheScrobbleProvider } from './infrastructure/Atomic.js';
 import { Typeson } from 'typeson';
 import { builtin } from 'typeson-registry';
 import { loggerNoop } from './MaybeLogger.js';
-import { ListenProgressPositional, ListenProgressTS } from '../sources/PlayerState/ListenProgress.js';
 const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
 import prom, { Gauge } from 'prom-client';
+import { nonEmptyStringOrDefault } from '../../core/StringUtils.js';
 
 dayjs.extend(utc)
 dayjs.extend(isBetween);
@@ -37,17 +38,24 @@ typeson.register({
         (x) => dayjs.isDayjs(x),
         (d: Dayjs) => d.toJSON(),
         (date) => dayjs(date)
-    ],
-    ListenProgressTS,
-    ListenProgressPositional
+    ]
 });
 
+const unsupportedEnvKeys = [
+    'CACHE_AUTH_CONN',
+    'CACHE_METADATA',
+    'CACHE_SCROBBLE',
+    'CACHE_SCROBBLE_CONN',
+    'CACHE_AUTH_CONN'
+];
+
 export class MSCache {
 
     config: Required<CacheConfigOptions>
 
     cacheMetadata: Cacheable;
     cacheScrobble: Cacheable;
+    cacheDb: Cacheable;
     cacheAuth: Cacheable;
     regexCache: ReturnType<typeof cacheFunctions>;
     cacheTransform: Cacheable;
@@ -68,19 +76,19 @@ export class MSCache {
 
         const {
             metadata: {
-                provider: mProvider = (process.env.CACHE_METADATA as (CacheMetadataProvider | undefined) ?? false),
-                connection: mConn = process.env.CACHE_METADATA_CONN,
-                ...restMetadata
+                provider: mProvider = false,
+                connection: mConn,
+                //...restMetadata
             } = {},
             scrobble: {
-                provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheScrobbleProvider | undefined) ?? 'file'),
-                connection = (process.env.CACHE_SCROBBLE_CONN ?? configDir),
-                ...restScrobble
+                provider: sProvider = false,
+                connection: sConnection,
+                //...restScrobble
             } = {},
             auth: {
-                provider: aProvider = (process.env.CACHE_AUTH as (CacheAuthProvider | undefined) ?? 'file'),
-                connection: aConn = (process.env.CACHE_AUTH_CONN ?? configDir),
-                ...restAuth
+                provider: aProvider = false,
+                connection: aConn,
+                //...restAuth
             } = {},
             regex = 200,
         } = config;
@@ -89,17 +97,14 @@ export class MSCache {
             metadata: {
                 provider: mProvider,
                 connection: mConn,
-                ...restMetadata,
             },
             scrobble: {
                 provider: sProvider,
-                connection,
-                ...restScrobble
+                connection: sConnection,
             },
             auth: {
                 provider: aProvider,
                 connection: aConn,
-                ...restAuth
             },
             regex
         };
@@ -114,6 +119,7 @@ export class MSCache {
         this.cacheAuth = inMemory;
         this.cacheScrobble = inMemory;
         this.cacheApi = inMemory;
+        this.cacheDb = new Cacheable({primary: initMemoryCache({lruSize: 500, ttl: '1m'})});
     }
 
     init = async (enableCollectors: boolean = false) => {
@@ -133,7 +139,8 @@ export class MSCache {
             { cache: this.cacheScrobble, name: 'queued_scrobbles' },
             { cache: this.cacheTransform, name: 'transformer' },
             { cache: this.cacheClientScrobbles, name: 'historical_scrobbles' },
-            { cache: this.cacheApi, name: 'external_apis' }
+            { cache: this.cacheApi, name: 'external_apis' },
+            { cache: this.cacheDb, name: 'database' }
         ];
 
         this.cacheHits = new prom.Gauge({
@@ -271,10 +278,10 @@ export class MSCache {
             }
         }
         if (config.provider === 'file') {
-            logger.debug(`Building file cache from ${path.join(config.connection, `${namespace}.cache`)}`);
+            logger.debug(`Building file cache from ${path.join(config.connection ?? configDir, `${namespace}.cache`)}`);
 
             try {
-                const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection, cacheId: `${namespace}.cache` }, {ttl: config.ttl}, logger);
+                const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection ?? configDir, cacheId: `${namespace}.cache` }, {ttl: config.ttl}, logger);
                 return keyvFile;
             } catch (e) {
                 throw e;
@@ -364,8 +371,8 @@ export const flatCacheCreate = (opts: FlatCacheOptions) => {
     return new FlatCache({
         ttl: 0,
         lruSize: 2000,
-        cacheDir: opts.cacheDir ?? configDir,
-        cacheId: opts.cacheId ?? 'scrobble.cache',
+        cacheDir: opts.cacheDir,
+        cacheId: opts.cacheId ?? 'ms.cache',
         persistInterval: 1 * 1000 * 10,
         expirationInterval: 1 * 1000 * 10, // 10 seconds
         ...opts
@@ -381,6 +388,12 @@ export const flatCacheLoad = async (flatCache: FlatCache, logger: Logger = logge
         throw new Error(`Unable to use path for file cache at ${cachePath}`, { cause: e })
     }
 
+    if(fileExists(cachePath) && !fileExists(`${cachePath}.bak`)) {
+        logger.info(`Backing up ${cachePath} in preparation for migration to database...`);
+        await copyFile(cachePath, `${cachePath}.bak`);
+        logger.info(`Done! Backed up to ${cachePath}.bak\nAll data has been loaded into cache. It will be deleted (from cache) after migrating to database.\nIf there are migration issues or you wish to downgrade then overwrite ${cachePath} with the .bak backup copy`);
+    }
+
     const streamPromise = new Promise((resolve, reject) => {
         flatCache.loadFileStream(cachePath, (progress: number, total: number) => {
             logger.trace(`Loading ${progress}/${total} chunks...`);
@@ -497,4 +510,86 @@ const noopKeyv: KeyvStoreAdapter = {
         delete: (_) => undefined,
         clear: () => Promise.resolve(),
         on: (_, __) => undefined
+}
+
+export const parseUserConfig = (config: CacheConfigUser = {}, parentLogger: Logger = loggerNoop): CacheConfigOptions => {
+    const logger = childLogger(parentLogger, 'Cache');
+
+        let valkeyEnvVal: string | undefined = nonEmptyStringOrDefault(process.env.CACHE_VALKEY);
+        if(valkeyEnvVal === undefined) {
+            valkeyEnvVal = nonEmptyStringOrDefault(process.env.CACHE_METADATA_CONN);
+            if(valkeyEnvVal !== undefined) {
+                logger.warn('ENV CACHE_METADATA_CONN is deprecated! Replace it with CACHE_VALKEY');
+            }
+        }
+
+        for(const key of unsupportedEnvKeys) {
+            if(nonEmptyStringOrDefault(process.env[key]) !== undefined) {
+                logger.warn(`ENV ${key} is no longer supported. Refer to the Caching docs.`);
+            }
+        }
+
+        const {
+            valkey = valkeyEnvVal,
+            // metadata: {
+            //     provider: mProvider = (process.env.CACHE_METADATA as (CacheMetadataProvider | undefined) ?? false),
+            //     connection: mConn = process.env.CACHE_METADATA_CONN,
+            //     //...restMetadata
+            // } = {},
+            scrobble: {
+                provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheScrobbleProvider | undefined) ?? 'file'),
+                connection = (process.env.CACHE_SCROBBLE_CONN ?? configDir),
+                ...restScrobble
+            } = {},
+            auth: {
+                provider: aProvider = (process.env.CACHE_AUTH as (CacheAuthProvider | undefined) ?? 'file'),
+                //...restAuth
+            } = {},
+            regex = 200,
+        } = config;
+
+        if(config.metadata !== undefined) {
+            logger.warn('Configuring cache.metadata is no longer supported. Refer to the Caching docs.');
+        }
+        if(config.scrobble !== undefined) {
+            logger.warn('Configuring cache.scrobble is no longer supported. Refer to the Caching docs.');
+        }
+        if(config.auth?.connection !== undefined) {
+            logger.warn('Configuring cache.auth.connection is no longer supported. Refer to the Caching docs.');
+        }
+        
+        let authConn: string,
+        authProvider = aProvider;
+        if(authProvider === 'valkey') {
+            if(valkey === undefined) {
+                logger.warn(`Auth Provider set to 'valkey' but not valkey connection string was not provided, falling back to file.`);
+                authConn = configDir;
+                authProvider = 'file';
+            } else {
+                authConn = valkey;
+            }
+        } else {
+            if(authProvider !== 'file') {
+                logger.warn(`Unsupported provider given for auth: ${authProvider}`);
+            }
+            authConn = configDir;
+            authProvider = 'file';
+        }
+
+        return {
+            metadata: {
+                provider: valkey !== undefined ? 'valkey' : false,
+                connection: valkey,
+            },
+            scrobble: {
+                provider: sProvider,
+                connection,
+                ...restScrobble
+            },
+            auth: {
+                provider: authProvider,
+                connection: authConn,
+            },
+            regex
+        };
 }

src/backend/common/database/Database.ts

@@ -0,0 +1,192 @@
+import { configDir } from '../index.js';
+import * as path from 'path';
+import { promises as fs } from 'fs'
+import { childLogger, Logger } from '@foxxmd/logging';
+import { loggerNoop } from '../MaybeLogger.js';
+import { fileExists, fileOrDirectoryIsWriteable } from '../../utils/FSUtils.js';
+import { COMPACTABLE, compactableProperties, CompactableProperty, DEFAULT_RETENTION_DELETE_AFTER, RententionGranular, RetentionConfig, RetentionConfigValue, RetentionOption, RetentionValue, RetentionValueUnparsed } from '../infrastructure/config/database.js';
+import { DurationValue } from '../infrastructure/Atomic.js';
+import { Duration } from 'dayjs/plugin/duration.js';
+import dayjs from 'dayjs';
+import { parseDurationFromDurationValue } from '../../utils/TimeUtils.js';
+import assert, { AssertionError } from 'node:assert';
+import { parseBoolStrict } from '../../utils.js';
+import { SimpleError } from '../errors/MSErrors.js';
+
+export const MEMORY_DB_NAME = ':memory:';
+export const isMemoryDb = (name: string): boolean => name === MEMORY_DB_NAME;
+
+export const getDbPath = (name: string = 'msDb', workingDirectory?: string): string => {
+    if (isMemoryDb(name)) {
+        return MEMORY_DB_NAME;
+    }
+    return path.resolve(workingDirectory ?? configDir, `${name}`);
+}
+
+export const getDbBackupPath = (dbPath: string, suffix?: string): string => {
+    const pathInfo = path.parse(dbPath);
+    const backupPath = `${path.join(pathInfo.dir, pathInfo.name)}.bak${suffix !== undefined ? `.${suffix}` : ''}`;
+    return backupPath;
+}
+
+export const backupDb = async (dbName: string, opts: { logger?: Logger, workingDirectory?: string } = {}): Promise<void> => {
+
+    const {
+        logger: parentLogger = loggerNoop,
+        workingDirectory
+    } = opts;
+
+    const logger = childLogger(parentLogger, 'Migrations');
+
+    const dbPath = getDbPath(dbName, workingDirectory);
+    let newDb = false;
+
+    if (dbPath !== MEMORY_DB_NAME) {
+        if (!fileExists(dbPath)) {
+            logger.info(`Database at ${dbPath} does not exist, will create it.`);
+            newDb = true;
+        }
+        try {
+            fileOrDirectoryIsWriteable(dbPath);
+        } catch (e) {
+            throw new Error('Database path/folder is not writeable, cannot backup database', { cause: e });
+        }
+    }
+
+    if (dbPath !== MEMORY_DB_NAME && !newDb) {
+        const backupPath = `${getDbPath(`${Date.now()}-${dbName}`, workingDirectory)}.bak`;
+        logger.info(`Backing up database before migrating => ${backupPath}`);
+        await fs.copyFile(dbPath, backupPath)
+        logger.info('Backed up!');
+    }
+}
+
+const parseRetentionValue = (val: RetentionValueUnparsed): RetentionValue => {
+    if(typeof val === 'string' || typeof val === 'boolean') {
+        try {
+            const boolVal = parseBoolStrict(val);
+            assert(boolVal === false, 'retention value cannot be true');
+            return boolVal;
+        } catch (e) {
+            // swallow
+        }
+    } else if(dayjs.isDuration(val)) {
+        return val;
+    } else {
+        return parseDurationFromDurationValue(val);
+    }
+    throw new SimpleError('retention value be of one: false, number, or string');
+}
+
+const parseRetentionFromEnv = (): RetentionOption<RetentionValue> => {
+    const deleteAfterEnv = process.env.RETENTION_DELETE_AFTER ?? DEFAULT_RETENTION_DELETE_AFTER,
+        deleteCompletedEnv = process.env.RETENTION_DELETE_COMPLETED_AFTER ?? deleteAfterEnv,
+        deleteFailedEnv = process.env.RETENTION_DELETE_FAILED_AFTER ?? deleteAfterEnv,
+        deleteDupedEnv = process.env.RETENTION_DELETE_DUPED_AFTER ?? deleteAfterEnv;
+
+    return {
+        completed: parseRetentionValue(deleteCompletedEnv),
+        failed: parseRetentionValue(deleteFailedEnv),
+        duped: parseRetentionValue(deleteDupedEnv)
+    }
+}
+
+const isRetentionOptionDurations = (val: RetentionOption<RetentionValue>): val is RetentionOption<Duration> => {
+    return dayjs.isDuration(val.completed)
+    && dayjs.isDuration(val.duped)
+    && dayjs.isDuration(val.failed);
+}
+
+let retentionDeleteAfterFromEnv: RetentionOption<Duration>,
+retentionCompactAfterFromEnv: RetentionOption<RetentionValue>;
+
+export const getRetentionDeleteAfterFromEnv = () => {
+    if (retentionDeleteAfterFromEnv === undefined) {
+        const deleteEnv = parseRetentionFromEnv();
+        if(isRetentionOptionDurations(deleteEnv)) {
+            retentionDeleteAfterFromEnv = deleteEnv;
+        } else {
+            throw new SimpleError('retention deleteAfter values from env must all be one of: number or string');
+        }
+    }
+    return retentionDeleteAfterFromEnv;
+}
+export const getRetentionCompactAfterFromEnv = () => {
+    if (retentionCompactAfterFromEnv === undefined) {
+        const compactEnv = parseRetentionFromEnv();
+        retentionCompactAfterFromEnv = compactEnv;
+    }
+    return retentionCompactAfterFromEnv;
+}
+
+export const parseRetentionOptions = (opts: RetentionConfigValue<DurationValue> = {}, defaults: RetentionOption<RetentionValue>): RetentionOption<RetentionValue> => {
+    if (typeof opts === 'number' || typeof opts === 'string') {
+        const dur = parseDurationFromDurationValue(opts);
+        return {
+            completed: dur,
+            duped: dur,
+            failed: dur
+        }
+    }
+
+    if (opts === undefined) {
+        return defaults;
+    }
+
+    if(dayjs.isDuration(opts)) {
+        return {
+            completed: opts,
+            failed: opts,
+            duped: opts
+        }
+    }
+
+    const {
+        completed = defaults.completed,
+        failed = defaults.failed,
+        duped = defaults.duped
+    } = opts;
+
+    return {
+        completed: parseRetentionValue(completed),
+        failed: parseRetentionValue(failed),
+        duped: parseRetentionValue(duped),
+    }
+}
+
+export const parseRetentionOptionsDurations = (opts: RetentionConfigValue<Exclude<RetentionValueUnparsed, false>> = {}, defaults: RetentionOption<Duration>): RetentionOption<Duration> => {
+    if (typeof opts === 'number' || typeof opts === 'string') {
+        const dur = parseDurationFromDurationValue(opts);
+        return {
+            completed: dur,
+            duped: dur,
+            failed: dur
+        }
+    }
+
+    if(dayjs.isDuration(opts)) {
+        return {
+            completed: opts,
+            failed: opts,
+            duped: opts
+        }
+    }
+
+    if (opts === undefined) {
+        return defaults;
+    }
+
+    const {
+        completed = defaults.completed,
+        failed = defaults.failed,
+        duped = defaults.duped
+    } = opts;
+
+    return {
+        completed: dayjs.isDuration(completed) ? completed : parseDurationFromDurationValue(completed),
+        failed: dayjs.isDuration(failed) ? failed : parseDurationFromDurationValue(failed),
+        duped: dayjs.isDuration(duped) ? duped : parseDurationFromDurationValue(duped),
+    }
+}
+
+export const isCompactableProperty = (val: string): val is CompactableProperty => val === COMPACTABLE.input || val === COMPACTABLE.transform;

src/backend/common/database/drizzle/drizzleTypes.ts

@@ -0,0 +1,100 @@
+import { DBQueryConfig, DBQueryConfigWith, ExtractTablesFromSchema, KnownKeysOnly, RelationFieldsFilterInternals, Many, InferSelectModel, ExtractTablesWithRelations, type BuildQueryResult, RelationsFilter } from "drizzle-orm";
+import { components, playInputs, plays, queueStates, relations } from "./schema/schema.js";
+import {TSchema, TableName, Schema } from "./schema/schema.js";
+import { MarkOptional, MarkRequired } from "ts-essentials";
+
+
+export type ComponentNew = typeof components.$inferInsert;
+export type ComponentSelect = typeof components.$inferSelect;
+
+export type QueueStateNew = typeof queueStates.$inferInsert;
+export type QueueStateSelect = typeof queueStates.$inferSelect;
+
+export type PlayInputNew = typeof playInputs.$inferInsert;
+export type PlayInputSelect = typeof playInputs.$inferSelect;
+
+export type PlaySelect = typeof plays.$inferSelect;
+export type PlaySelectRel = ModelWithRelations<typeof plays>;
+export type PlaySelectWithQueueStates = GenericRelationResult<'plays', 'queueStates'>;
+export type PlayNew = typeof plays.$inferInsert;
+
+
+// useful references for building types
+// https://github.com/drizzle-team/drizzle-orm/discussions/2596
+// https://github.com/drizzle-team/drizzle-orm/discussions/1539
+// https://gist.github.com/ikupenov/10bc89d92d92eaba8cc5569013e04069
+// https://github.com/drizzle-team/drizzle-orm/issues/695 most examples
+// https://github.com/drizzle-team/drizzle-orm/discussions/2316 relation focused
+// https://github.com/drizzle-team/drizzle-orm/issues/1319
+
+//type p = TSchema['plays']['relations'];
+export type FindWith<T extends TableName> = DBQueryConfigWith<TSchema, TSchema[T]['relations']>;
+export type QueryConfig<T extends TableName> = DBQueryConfig<"many", TSchema, TSchema[T]>;
+export type FindMany<T extends TableName> = Pick<KnownKeysOnly<QueryConfig<T>, DBQueryConfig<"many", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> & {with?: FindWith<T>}
+export type FindOne<T extends TableName> = Pick<KnownKeysOnly<QueryConfig<T>, DBQueryConfig<"one", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> & {with?: FindWith<T>}
+export type FindWhere<T extends TableName> = QueryConfig<T>['where'];
+// https://github.com/drizzle-team/drizzle-orm/issues/5218#issuecomment-4154686086
+export type WhereClause<T extends keyof typeof relations> = RelationsFilter<typeof relations[T], typeof relations>
+
+
+export type CompareOp<T> = Pick<RelationFieldsFilterInternals<T>, 'gt' | 'gte' | 'eq' | 'lt' | 'lte' | 'ne'>
+export type CompareOpKey<T> = keyof CompareOp<T>;
+
+
+
+/**
+ * Based on https://github.com/drizzle-team/drizzle-orm/issues/695#issuecomment-3133969178
+ */
+
+// Helper type to find the tsName corresponding to a given dbName in TSchema
+type FindTsNameByDbName<TDbNameToFind extends string> = {
+    [K in keyof TSchema]: TSchema[K] extends {
+        // updated dbName -> name
+        name: TDbNameToFind;
+    }
+        ? K
+        : TDbNameToFind;
+}[keyof TSchema];
+
+// Helper type to find the dbName corresponding to a given tsName in TSchema
+type FindDbNameByTsName<TTable extends Schema[keyof Schema]> = {
+    [K in keyof Schema]: Schema[K] extends TTable ? K : never;
+}[keyof Schema];
+
+/**
+ * Utility type to infer the model type for a given table name from the schema.
+ * Handles nested relations recursively.
+ * Uses referencedTableName (dbName) and FindTsNameByDbName helper.
+ */
+export type ModelWithRelationsFromName<
+    TTableName extends keyof TSchema,
+> = InferSelectModel<Schema[TTableName]> & {
+    [K in keyof TSchema[TTableName]['relations']]?: TSchema[TTableName]['relations'][K] extends infer TRelation
+        // updated referencedTableName -> targetTableName
+        ? TRelation extends { targetTableName: infer TRefDbName extends string }
+            ? FindTsNameByDbName<TRefDbName> extends infer TRefTsName extends
+                  keyof TSchema
+                ? TRelation extends Many<any>
+                    ? ModelWithRelationsFromName<TRefTsName>[]
+                    : ModelWithRelationsFromName<TRefTsName> | null
+                : never
+            : never
+        : never;
+};
+
+/**
+ * Utility type to infer the model type for a given table from the schema.
+ * Handles nested relations recursively.
+ * Uses referencedTableName (dbName) and FindDbNameByTsName helper.
+ */
+export type ModelWithRelations<TTable extends Schema[keyof Schema]> =
+    FindDbNameByTsName<TTable> extends infer TTableName extends keyof TSchema
+        ? ModelWithRelationsFromName<TTableName>
+        : never;
+
+
+// all relations are are now fully typed and optional
+//type FullPlay = ModelWithRelations<typeof plays>;
+
+// https://github.com/drizzle-team/drizzle-orm/issues/695#issuecomment-4389296482
+type GenericRelationResult<T extends keyof TSchema, K extends keyof TSchema[T]['relations']> = BuildQueryResult<TSchema, TSchema[T], { with: Record<K, true> }>;

src/backend/common/database/drizzle/drizzleUtils.ts

@@ -0,0 +1,263 @@
+import { drizzle } from 'drizzle-orm/node-sqlite';
+import { drizzle as drizzlePglite } from 'drizzle-orm/pglite';
+import { migrate } from 'drizzle-orm/node-sqlite/migrator';
+import { migrate as migratePglite } from 'drizzle-orm/pglite/migrator';
+import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
+import { PGlite, PGliteOptions } from '@electric-sql/pglite';
+import { sql as dsl, LogWriter, Logger as DrizzleLogger } from 'drizzle-orm';
+import * as fs from 'fs/promises';
+import * as fsSync from 'fs';
+import * as path from 'path';
+import { backupDb, getDbBackupPath, getDbPath, MEMORY_DB_NAME } from '../Database.js';
+import { fileExists, fileOrDirectoryIsWriteable } from '../../../utils/FSUtils.js';
+import { childLogger, Logger, LogLevel } from '@foxxmd/logging';
+import { loggerNoop } from '../../MaybeLogger.js';
+import { projectDir } from '../../index.js';
+import { relations } from './schema/schema.js';
+import { addToContext, executeQuery } from './logContext.js';
+
+export async function shouldBackupDb(dbVal: string | DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> {
+  const {
+    logger: parentLogger = loggerNoop,
+    migrationsFolder = path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')
+  } = opts;
+  const logger = childLogger(parentLogger, 'Migrations');
+
+  let db: DbConcrete;
+  
+  if(typeof dbVal === 'string') {
+    logger.info(`Checking for database at ${dbVal}`);
+    if (dbVal !== MEMORY_DB_NAME && !fileExists(dbVal)) {
+      logger.info(`No database exists, no backup needed.`);
+      return [false, []];
+    }
+    db = await getDb(dbVal);
+  } else {
+    db = dbVal;
+  }
+
+
+  // const db = drizzlePglite(dbPath);
+
+  try {
+    // Ensure the migrations table exists
+    // https://github.com/drizzle-team/drizzle-orm/issues/1953
+    const res = await db.execute(dsl`
+      SELECT EXISTS (
+    SELECT FROM 
+        pg_tables
+    WHERE 
+        schemaname = 'drizzle' AND 
+        tablename  = '__drizzle_migrations'
+    );
+      `);
+
+    // const res3 = await db.execute(dsl`
+    // SELECT * FROM 
+    //     pg_tables;
+    //   `);
+
+    if (res.rows[0].exists === false) {
+      logger.info(`Database exists but there is no __drizzle_migrations table??`);
+      return [true, []];
+    }
+
+    const dbMigrations = await db.execute(dsl`SELECT id, hash, created_at, name, applied_at FROM drizzle.__drizzle_migrations ORDER BY created_at DESC`);
+    // @ts-ignore
+    const appliedMigrations = new Set(dbMigrations.rows.map((m: any) => m.name));
+
+    const allFiles = await fs.readdir(migrationsFolder);
+    const migrationFiles = allFiles
+      .sort();
+
+    const pendingMigrations = migrationFiles.filter(file => {
+      return !appliedMigrations.has(file);
+    });
+
+    //console.log('Applied migrations:', Array.from(appliedMigrations));
+    if (pendingMigrations.length > 0) {
+      logger.info(`${pendingMigrations.length} pending migrations:\n${pendingMigrations.join('\n')}`);
+      return [true, pendingMigrations];
+    } else {
+      logger.info('No pending migrations.');
+      return [false, []];
+    }
+  } catch (error) {
+    logger.error(new Error('Failed to get pending migrations', { cause: error }));
+    return [true, []];
+  }
+}
+
+export const getDb = async (dbVal: string | PGlite, opts: { logger?: Logger, backupPath?: string, loadDataDir?: Promise<Blob> } = {}) => {
+  const {
+    logger = loggerNoop,
+    backupPath,
+    loadDataDir
+  } = opts;
+  let client: PGlite;
+
+  if(typeof dbVal === 'string') {
+    const opts: PGliteOptions = {};
+    if(dbVal !== MEMORY_DB_NAME) {
+      opts.dataDir = dbVal;
+      if(backupPath !== undefined) {
+        opts.loadDataDir = new Blob([fsSync.readFileSync(backupPath)]);
+      }
+    }
+    // only load one
+    // but this could be for a memory db so don't put it in above if
+    if(loadDataDir !== undefined && backupDb === undefined) {
+      opts.loadDataDir = await loadDataDir
+    }
+    client = await PGlite.create(opts);
+  } else {
+    client = dbVal;
+  }
+  return drizzlePglite({relations: relations, logger: createDrizzleLogger(logger), client});
+}
+
+export type DbConcrete = Awaited<ReturnType<typeof getDb>>;
+
+export const migrateDb = async (db: DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}) => {
+  const {
+    migrationsFolder,
+    logger: parentLogger = loggerNoop
+  } = opts;
+  const logger = childLogger(parentLogger, 'Migrations');
+
+  try {
+    logger.info('Starting migrations...');
+    await executeQuery('migrations', async () => migratePglite(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') }), logger, process.env.LOG_MIGRATION === 'true' ? true : 'error');
+    logger.info('Migrations complete');
+  } catch (e) {
+    throw new Error('Failed to migrate database', { cause: e });
+  }
+}
+
+export const migrateDbSync = (db: ReturnType<typeof drizzle>, opts: {logger?: Logger, migrationsFolder?: string} = {}) => {
+  const {
+    migrationsFolder,
+    logger: parentLogger = loggerNoop
+  } = opts;
+  const logger = childLogger(parentLogger, 'Migrations');
+
+  try {
+    logger.info('Starting migrations...');
+    migrate(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') });
+    logger.info('Migrations complete');
+  } catch (e) {
+    throw new Error('Failed to migrate database', { cause: e });
+  }
+}
+
+export const getMigratedDb = async (dbPath: string, opts: { logger?: Logger, workingDirectory?: string, migrationsFolder?: string, backupPath?: string, loadDataDir?: Promise<Blob> } = {}): Promise<[DbConcrete, boolean]> => {
+  const {
+    logger = loggerNoop
+  } = opts;
+  let db: DbConcrete,
+  isNew = false,
+  hasPendingMigrations: boolean = true;
+  if (dbPath !== MEMORY_DB_NAME) {
+    try {
+      fileOrDirectoryIsWriteable(dbPath);
+    } catch (e) {
+      throw new Error('Database directory is not accessible', { cause: e });
+    }
+
+    const backupPath = getDbBackupPath(dbPath);
+
+    if (fileExists(dbPath)) {
+      db = await getDb(dbPath, opts);
+      const [shouldBackup, pendingMigrations] = await shouldBackupDb(db, opts);
+      if (shouldBackup) {
+        hasPendingMigrations = true;
+        await backupPgDb(db, dbPath, { logger: opts.logger });
+      }
+    } else if(fileExists(backupPath)) {
+      logger.info(`Detected no database, using backup to recreate db. Backup file: ${backupPath}`);
+      db = await getDb(dbPath, {...opts, backupPath});
+      const usedBackedPath = getDbBackupPath(dbPath, 'used');
+      logger.info(`Backup loaded! Renaming backup to indicate it has already been used, new path: ${usedBackedPath}`);
+      await fs.rename(backupPath, usedBackedPath);
+    } else {
+      logger.info('Detected no database, creating a new one...');
+      db = await getDb(dbPath, opts);
+      isNew = true;
+    }
+  } else {
+    logger.info('Detected in-memory database');
+    db = await getDb(dbPath, opts);
+    isNew = true;
+  }
+
+  if(hasPendingMigrations && dbPath !== MEMORY_DB_NAME) {
+    logger.info('TIP: Migrations may take some time, depending on the size of your database');
+  }
+  await migrateDb(db, opts);
+
+  return [db, isNew];
+}
+
+export const backupPgDb = async (db: DbConcrete, dbPath: string, opts: { logger?: Logger } = {}): Promise<void> => {
+
+    const {
+        logger: parentLogger = loggerNoop,
+    } = opts;
+
+    const logger = childLogger(parentLogger, 'Backup');
+
+    const pathInfo = path.parse(dbPath);
+    // being extra sure there isn't a trailing slash
+    const backupPath = `${path.join(pathInfo.dir, pathInfo.name)}-${Date.now()}.bak`;
+    logger.info(`Backing up database before migrating => ${backupPath}`);
+    fs.writeFile(backupPath, Buffer.from(await (await db.$client.dumpDataDir()).arrayBuffer()));
+    //await fs.copyFile(dbPath, backupPath)
+    logger.info('Backed up!');
+}
+
+export const createDrizzleLogger = (parentLogger: Logger, opts: {level?: LogLevel} = {}): DrizzleLogger => {
+  return {
+    logQuery: (query: string, params: unknown[]) => {
+      addToContext({sql: query, params})
+    }
+  }
+}
+
+
+// cannot really use transactions right now because async isn't supporting for sqlite
+// https://github.com/drizzle-team/drizzle-orm/issues/1472
+// https://github.com/drizzle-team/drizzle-orm/issues/2275
+// so use this workaround for now
+// https://github.com/drizzle-team/drizzle-orm/issues/2275#issuecomment-2496503801
+let currentTransaction: null | Promise<void> = null;
+export const runTransaction = async <
+    T,
+    TQueryResult,
+    TSchema extends Record<string, unknown> = Record<string, never>
+>(
+    db: BaseSQLiteDatabase<"sync", TQueryResult, TSchema>,
+    executor: () => Promise<T>
+) => {
+    while (currentTransaction !== null) {
+        await currentTransaction;
+    }
+    let resolve!: () => void;
+    currentTransaction = new Promise<void>(_resolve => {
+        resolve = _resolve;
+    });
+    try {
+        db.run(dsl.raw(`BEGIN`))
+
+        try {
+            const result = await executor();
+            await db.run(dsl.raw(`COMMIT`));
+            return result;
+        } catch (error) {
+            await db.run(dsl.raw(`ROLLBACK`));
+            throw error;
+        }
+    } finally {
+        resolve();
+        currentTransaction = null;
+    }
+};

src/backend/common/database/drizzle/entityUtils.ts

@@ -0,0 +1,93 @@
+import assert from "node:assert";
+import { PlayNew, PlaySelect, PlaySelectWithQueueStates } from "./drizzleTypes.js";
+import { PlayInputNew } from "./drizzleTypes.js";
+import { QueueStateNew } from "./drizzleTypes.js";
+import { ComponentNew } from "./drizzleTypes.js";
+import { MarkOptional, MarkRequired } from "ts-essentials";
+import { CLIENT_DEAD_QUEUE, DeadLetterScrobble, ErrorLike, PlayObject } from "../../../../core/Atomic.js";
+import dayjs, { Dayjs } from "dayjs";
+import { asPlay } from "../../../../core/PlayMarshalUtils.js";
+import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../utils/PlayComparisonUtils.js";
+import { hashObject } from "../../../utils/StringUtils.js";
+import { messageWithCauses } from "../../../utils/ErrorUtils.js";
+
+export const generateComponentEntity = (data: MarkOptional<ComponentNew, 'uid'>): ComponentNew => {
+    assert(data.name !== undefined, 'Must provide name');
+    return {
+        ...data,
+        uid: data.uid ?? data.name
+    };
+}
+
+export type PlayEntityOpts = Partial<Pick<PlayNew, 'seenAt' | 'playedAt' | 'uid' | 'state' | 'parentId' | 'componentId'>> & { error?: ErrorLike };
+
+export const generatePlayEntity = (play: PlayObject, opts: PlayEntityOpts = {}): PlayNew => {
+    const {
+        seenAt = dayjs(),
+        state = 'queued',
+        playedAt = play.data.playDate,
+        ...restOpts
+    } = opts;
+    let playHash: string = undefined;
+    try {
+        playHash = hashObject(playContentBasicInvariantTransform(play).data);
+    } catch (e) {
+        // swallow
+    }
+    const data: PlayNew = {
+        play,
+        playHash,
+        state,
+        playedAt,
+        seenAt: play.meta.seenAt ?? seenAt,
+        ...restOpts
+    };
+    const mbidId = playMbidIdentifier(play);
+    if(mbidId !== undefined) {
+        data.mbidIdentifier = mbidId;
+    }
+    return data;
+}
+
+export type PlayHydateOptions = 'asPlay' | 'id' | 'uid';
+
+export const hydratePlaySelect = (select: PlaySelect, opts: PlayHydateOptions[] = ['id','uid']): PlayObject => {
+    if(opts.length === 0) {
+        return select.play;
+    }
+
+    let res = select.play;
+    // if(opts.includes('asPlay')) {
+    //     res = asPlay(res);
+    // }
+    if(opts.includes('uid')) {
+        res.uid = select.uid;
+        //res.meta.dbUid = select.uid;
+    }
+    if(opts.includes('id')) {
+        res.id = select.id;
+        //res.meta.dbId = select.id;
+    }
+    return res;
+}
+
+export const playSelectToDeadScrobble = (select: PlaySelectWithQueueStates): DeadLetterScrobble<PlayObject> => {
+    const deadQueue = select.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE);
+    return {
+        play: select.play,
+        id: select.uid,
+        source: select.play.meta.source,
+        retries: deadQueue.retries,
+        lastRetry: deadQueue.updatedAt,
+        error: deadQueue.error as unknown as string,
+        status: deadQueue.queueStatus as 'queued' | 'failed'
+    }
+}
+
+export const generateInputEntity = (data: PlayInputNew): PlayInputNew => {
+    return data;
+}
+
+export const generateQueueStateEntity = (data: QueueStateNew): QueueStateNew => {
+    return data;
+}

src/backend/common/database/drizzle/logContext.ts

@@ -0,0 +1,71 @@
+import { Logger } from '@foxxmd/logging'
+import { AsyncLocalStorage } from 'async_hooks'
+
+// based on https://numeric.substack.com/p/upgrading-drizzleorm-logging-with
+interface QueryContext {
+    queryKey: string
+    startTime: number
+    queries: { sql?: string, params?: unknown[] }[]
+}
+
+const queryStorage = new AsyncLocalStorage<QueryContext>()
+
+function wrapQuery<T>(queryKey: string, fn: () => Promise<T>): Promise<T> {
+    return queryStorage.run(
+        {
+            queryKey,
+            startTime: Date.now(),
+            queries: []
+        },
+        fn
+    )
+}
+
+function getContext(): QueryContext | undefined {
+    return queryStorage.getStore()
+}
+
+export function addToContext(data: { sql?: string, params?: unknown[] }): void {
+    const context = getContext()
+    if (context) {
+        context.queries.push(data);
+    }
+}
+
+/**
+ * Log all queries made by drizzle during the execution of a promise
+ * 
+ * use second parameter to configure when logging occurs
+ * * true    => log everything (default)
+ * * false   => log nothing, skips asyncstorage entirely
+ * * 'error' => only log if promise throws
+ * 
+ */
+export async function executeQuery<T = any>(queryKey: string, queryPromise: () => Promise<T>, logger: Logger, when: boolean | 'error' = true) {
+    if(when === false) {
+        try {
+            return await queryPromise();
+        } catch (e) {
+            throw e;
+        }
+    }
+    return wrapQuery(queryKey, async () => {
+        try {
+            const results = await queryPromise()
+
+            if (when !== 'error') {
+                // Query is done - grab everything from context
+                const context = getContext()
+                const executionTime = context ? Date.now() - context.startTime : 0
+                logger.info({ labels: ['DB Query', queryKey], queries: context?.queries }, `Execution Complete in ${executionTime}ms`);
+            }
+
+            return results
+        } catch (error) {
+            const context = getContext()
+            const executionTime = context ? Date.now() - context.startTime : 0;
+            logger.warn({ labels: ['DB Query', queryKey], queries: context?.queries }, `Execution failed in ${executionTime}ms`);
+            throw error
+        }
+    })
+}

src/backend/common/database/drizzle/migrations/20260509011012_dizzy_naoko/migration.sql

@@ -0,0 +1,83 @@
+CREATE TABLE "components" (
+	"id" serial PRIMARY KEY,
+	"uid" varchar(200) NOT NULL,
+	"mode" varchar(15) NOT NULL,
+	"type" varchar(50) NOT NULL,
+	"name" varchar NOT NULL,
+	"countLive" integer DEFAULT 0 NOT NULL,
+	"countNonLive" integer DEFAULT 0 NOT NULL,
+	"createdAt" timestamp
+);
+--> statement-breakpoint
+CREATE TABLE "jobs" (
+	"id" serial PRIMARY KEY,
+	"componentFromId" integer NOT NULL,
+	"componentToId" integer NOT NULL,
+	"name" varchar(200) NOT NULL,
+	"status" varchar(20) DEFAULT 'idle' NOT NULL,
+	"retries" integer DEFAULT 0 NOT NULL,
+	"error" json,
+	"transformOptions" json,
+	"initialParameters" json,
+	"cursor" json,
+	"total" integer,
+	"imported" integer DEFAULT 0 NOT NULL,
+	"scrobbled" integer DEFAULT 0 NOT NULL,
+	"createdAt" timestamp NOT NULL,
+	"updatedAt" timestamp NOT NULL,
+	"completedAt" timestamp
+);
+--> statement-breakpoint
+CREATE TABLE "play_inputs" (
+	"id" serial PRIMARY KEY,
+	"playId" integer NOT NULL,
+	"data" json,
+	"play" jsonb NOT NULL,
+	"createdAt" timestamp
+);
+--> statement-breakpoint
+CREATE TABLE "plays" (
+	"id" serial PRIMARY KEY,
+	"uid" varchar(30) NOT NULL UNIQUE,
+	"componentId" integer,
+	"error" json,
+	"playedAt" timestamp,
+	"seenAt" timestamp,
+	"updatedAt" timestamp NOT NULL,
+	"play" jsonb NOT NULL,
+	"state" varchar(20) NOT NULL,
+	"parentId" integer,
+	"jobId" integer,
+	"playHash" varchar(100),
+	"mbidIdentifier" varchar(100),
+	"compacted" varchar(30)
+);
+--> statement-breakpoint
+CREATE TABLE "play_queue_states" (
+	"id" serial PRIMARY KEY,
+	"playId" integer NOT NULL,
+	"componentId" integer NOT NULL,
+	"queueName" varchar(50) NOT NULL,
+	"queueStatus" varchar(20) DEFAULT 'queued' NOT NULL,
+	"retries" integer DEFAULT 0 NOT NULL,
+	"error" json,
+	"createdAt" timestamp NOT NULL,
+	"updatedAt" timestamp NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX "uid_mode_type_idx" ON "components" ("uid","mode","type");--> statement-breakpoint
+CREATE UNIQUE INDEX "play_input_id_idx" ON "play_inputs" ("playId");--> statement-breakpoint
+CREATE INDEX "play_parent_id_idx" ON "plays" ("parentId");--> statement-breakpoint
+CREATE INDEX "play_component_id_idx" ON "plays" ("componentId");--> statement-breakpoint
+CREATE UNIQUE INDEX "play_uid_idx" ON "plays" ("uid");--> statement-breakpoint
+CREATE INDEX "play_playedAt_idx" ON "plays" ("playedAt");--> statement-breakpoint
+CREATE INDEX "play_seenAt_idx" ON "plays" ("seenAt");--> statement-breakpoint
+CREATE INDEX "play_queue_state_id_idx" ON "play_queue_states" ("playId");--> statement-breakpoint
+ALTER TABLE "jobs" ADD CONSTRAINT "jobs_componentFromId_components_id_fkey" FOREIGN KEY ("componentFromId") REFERENCES "components"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "jobs" ADD CONSTRAINT "jobs_componentToId_components_id_fkey" FOREIGN KEY ("componentToId") REFERENCES "components"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "play_inputs" ADD CONSTRAINT "play_inputs_playId_plays_id_fkey" FOREIGN KEY ("playId") REFERENCES "plays"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "plays" ADD CONSTRAINT "plays_componentId_components_id_fkey" FOREIGN KEY ("componentId") REFERENCES "components"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "plays" ADD CONSTRAINT "plays_parentId_plays_id_fkey" FOREIGN KEY ("parentId") REFERENCES "plays"("id") ON DELETE SET NULL ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "plays" ADD CONSTRAINT "plays_jobId_jobs_id_fkey" FOREIGN KEY ("jobId") REFERENCES "jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "play_queue_states" ADD CONSTRAINT "play_queue_states_playId_plays_id_fkey" FOREIGN KEY ("playId") REFERENCES "plays"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "play_queue_states" ADD CONSTRAINT "play_queue_states_componentId_components_id_fkey" FOREIGN KEY ("componentId") REFERENCES "components"("id") ON DELETE CASCADE ON UPDATE CASCADE;

src/backend/common/database/drizzle/repositories/BaseRepository.ts

@@ -0,0 +1,111 @@
+import { childLogger, Logger } from "@foxxmd/logging";
+import { DbConcrete } from "../drizzleUtils.js";
+import { CompareOpKey } from "../drizzleTypes.js";
+import { Dayjs } from "dayjs";
+import { RelationsFieldFilter, eq, inArray } from "drizzle-orm";
+import { loggerNoop } from "../../../MaybeLogger.js";
+import { capitalize } from "../../../../../core/StringUtils.js";
+import { getConfigByTableName, relations, TableName } from "../schema/schema.js";
+
+export interface DrizzleRepositoryOpts {
+    logger?: Logger
+    componentId?: number
+}
+
+export type CompareDateOp = {
+    type: CompareOpKey<Dayjs>
+    date: Dayjs
+} | {
+    type: 'between',
+    range: [Dayjs, Dayjs],
+    inclusive?: boolean
+}
+
+export interface PaginatedQueryResponse {
+    limit: number,
+    offset: number
+}
+
+export interface PaginatedResponse<T> {
+    data: T[]
+    meta: PaginatedQueryResponse
+}
+
+export interface ComponentConstrainedRepoOpts {
+    componentId?: number
+}
+
+export abstract class DrizzleBaseRepository<T extends TableName> {
+
+    logger: Logger;
+    displayName: string;
+    tableName: TableName;
+    table: ReturnType<typeof getConfigByTableName<T>>
+    db: DbConcrete;
+    componentId?: number
+
+    constructor(db: DbConcrete, tableName: TableName, displayName: string, opts: DrizzleRepositoryOpts = {}) {
+        this.db = db;
+        this.displayName = displayName;
+        this.tableName = tableName;
+        this.table = getConfigByTableName(this.tableName);
+        this.logger = childLogger(opts.logger ?? loggerNoop, ['Database', capitalize(displayName)]);
+        this.componentId = opts.componentId;
+    }
+
+    async deleteByIds(ids: number[]): Promise<void> {
+        await this.db.delete(this.table).where(inArray(this.table.id, ids));
+    }
+
+    async updateById(id: number, data: Partial<typeof this.table.$inferInsert>): Promise<void> {
+        await this.db.update(this.table).set(data).where(eq(this.table.id, id));
+    }
+
+    async create(data: typeof this.table.$inferInsert): Promise<typeof this.table.$inferSelect> {
+        const res = await this.db.insert(this.table).values([data]).returning();
+        return res[0];
+    }
+
+    async createMany(data: typeof this.table.$inferInsert[]): Promise<typeof this.table.$inferSelect[]> {
+        const res = await this.db.insert(this.table).values(data).returning();
+        return res;
+    }
+
+    async findById(id: number): Promise<typeof this.table.$inferSelect | undefined> {
+        // const res = await this.db.query[this.tableName].findFirst({
+        //     where: {
+        //         id
+        //     }
+        // });
+        const res = await this.db.select().from(this.table).where(eq(this.table.id, id));
+        if(res.length === 0) {
+            return undefined;
+        }
+        return res[0];
+    }
+}
+
+export class GenericRepository<T extends TableName> extends DrizzleBaseRepository<T> {
+
+}
+
+export const buildDateCompare = (data: CompareDateOp): RelationsFieldFilter<Dayjs> => {
+    let q: RelationsFieldFilter<Dayjs> = {};
+    if (data.type !== 'between') {
+        q = {
+            [data.type]: data.date
+        }
+    } else {
+        q = {
+            AND: [
+                {
+                    [data.inclusive ?? true ? 'gte' : 'gt']: data.range[0]
+                },
+                {
+                    [data.inclusive ?? true ? 'lte' : 'lt']: data.range[1]
+                },
+            ]
+        }
+    }
+    return q;
+}

src/backend/common/database/drizzle/repositories/ComponentRepository.ts

@@ -0,0 +1,34 @@
+import { Logger } from "drizzle-orm";
+import { DrizzleBaseRepository, DrizzleRepositoryOpts } from "./BaseRepository.js";
+import { DbConcrete } from "../drizzleUtils.js";
+import { ComponentNew, ComponentSelect, FindWhere } from "../drizzleTypes.js";
+import { components } from "../schema/schema.js";
+import { generateComponentEntity } from "../entityUtils.js";
+
+export class DrizzleComponentRepository extends DrizzleBaseRepository<'components'> {
+
+    constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
+        super(db, 'components', 'Component', opts);
+    }
+
+    findOrInsert = async (data: { mode: 'source' | 'client', type: string, uid?: string, name?: string }): Promise<ComponentSelect> => {
+        const where: FindWhere<'components'> = {
+            mode: data.mode,
+            type: data.type,
+            uid: data.uid ?? data.name
+        };
+        const component = await this.db.query.components.findFirst({
+            where
+        });
+        if (component !== undefined) {
+            return component;
+        }
+
+        return (await this.db.insert(components).values(generateComponentEntity({
+            uid: data.uid ?? data.name,
+            mode: data.mode,
+            type: data.type,
+            name: data.name
+        })).returning())[0];
+    }
+}

src/backend/common/database/drizzle/repositories/PlayRepository.ts

@@ -0,0 +1,881 @@
+import { childLogger, Logger, LoggerAppExtras } from "@foxxmd/logging";
+import { DbConcrete } from "../drizzleUtils.js";
+import { loggerNoop } from "../../../MaybeLogger.js";
+import { ErrorLike, PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT, TemporalAccuracy } from "../../../../../core/Atomic.js";
+import { generateInputEntity, generatePlayEntity, PlayEntityOpts, hydratePlaySelect, PlayHydateOptions } from "../entityUtils.js";
+import { playInputs, plays, queueStates, relations } from "../schema/schema.js";
+import { PlayNew, PlaySelect, PlayInputNew, FindWhere, FindMany, CompareOpKey, QueueStateSelect, PlayInputSelect, PlaySelectRel, FindWith, PlaySelectWithQueueStates, WhereClause } from "../drizzleTypes.js";;
+import { MarkOptional, MarkRequired, PathValue } from "ts-essentials";
+import { genGroupIdStrFromPlay, removeEmptyArrays, removeUndefinedKeys } from "../../../../utils.js";
+import dayjs, { Dayjs } from "dayjs";
+import { RelationsFieldFilter, eq, inArray, ne, notInArray, desc, asc, and, sql, Placeholder } from "drizzle-orm";
+import { CompactableProperty, RetentionOptions, retentionPlayTypes } from "../../../infrastructure/config/database.js";
+import { shortTodayAwareFormat } from "../../../../../core/TimeUtils.js";
+import { buildDateCompare, CompareDateOp, ComponentConstrainedRepoOpts, DrizzleBaseRepository, DrizzleRepositoryOpts, PaginatedQueryResponse, PaginatedResponse } from "./BaseRepository.js";
+import { asPlay } from "../../../../../core/PlayMarshalUtils.js";
+import assert, { Assert } from "node:assert";
+import { hashObject, parseArrayFromMaybeString } from "../../../../utils/StringUtils.js";
+import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../../utils/PlayComparisonUtils.js";
+import { comparePlayTemporally, getScrobbleTsSOCDate, getScrobbleTsSOCDateWithContext, getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy } from "../../../../utils/TimeUtils.js";
+import { SourceType } from "../../../infrastructure/config/source/sources.js";
+
+// https://github.com/drizzle-team/drizzle-orm/issues/695 may be useful for typing models with relations?
+
+export interface QueueCriteria {
+    queueName: string
+    queueStatus: QueueStateSelect['queueStatus'][] | QueueStateSelect['queueStatus']
+}
+
+export interface PlayWhereOpts {
+    state?: PlaySelect['state'][]
+    stateNot?: PlaySelect['state'][]
+    componentId?: number
+    seenAt?: CompareDateOp
+    playedAt?: CompareDateOp
+    queues?: QueueCriteria[]
+    uid?: string[]
+}
+
+export type WithPlayRelation = 'input' | 'parent' | 'parent-input' | 'queues';
+export interface QueryPlaysOpts extends PlayWhereOpts {
+    sort?: 'seenAt' | 'playedAt'
+    order?: 'asc' | 'desc'
+    with?: WithPlayRelation[]
+    limit?: number
+    offset?: number
+}
+
+export interface HydrateOpts {
+   hydrate?: PlayHydateOptions[] 
+}
+
+export type RepositoryCreatePlayOpts = PlayEntityOpts
+    & {
+        input: MarkOptional<PlayInputNew, 'playId' | 'play'>
+    }
+    & Pick<PlayNew, 'play' | 'componentId'>;
+
+type PlayIdentifierPrimitiveMap = {
+  uid: string;
+  id: number;
+};
+const identifierExtractor: { [K in keyof PlayIdentifierPrimitiveMap]: (play: {id: number, uid: string}) => PlayIdentifierPrimitiveMap[K] } = {
+  id: (play) => play.id,
+  uid: (play) => play.uid,
+};
+export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> {
+
+    protected hasQueueNextPrepared?: ReturnType<typeof this.prepareHasQueueNext>
+    protected getQueueNextPrepared?: ReturnType<typeof this.prepareGetQueueNext>
+    protected getQueuedScrobbleRangePrepared?: ReturnType<typeof this.prepareGetQueuedScrobbleRange>
+
+    constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
+        super(db, 'plays', 'Plays', opts);
+    }
+
+    findByUid = async (uid: string, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PlaySelectWithQueueStates | undefined> => {
+        const res = await this.db.query.plays.findFirst({
+            where: {
+                uid,
+                componentId: opts.componentId ?? this.componentId
+            },
+            with: {
+                queueStates: true
+            }
+        });
+        res.play = hydratePlaySelect(res, opts.hydrate);
+        return res;
+    }
+
+    createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[], opts: HydrateOpts = {}) => {
+
+        const {
+            hydrate
+        } = opts;
+        let playRows: PlaySelect[];
+
+        await this.db.transaction(async (tx) => {
+
+            const entitiesData = entitiesOpts.map((data) => {
+                const {
+                    play,
+                    input,
+                    ...rest
+                } = data;
+                return generatePlayEntity(play, { componentId: this.componentId, ...rest });
+            });
+
+            playRows = await tx.insert(plays).values(entitiesData).returning();
+
+            const inputDatas = playRows.map((x, index) => {
+                const {
+                    play,
+                    input,
+                } = entitiesOpts[index];
+                const {
+                    play: inputPlay = play,
+                    ...restInput
+                } = input;
+
+                return generateInputEntity({ play: inputPlay, playId: x.id, ...restInput });
+            });
+
+            const inputRow = await tx.insert(playInputs).values(inputDatas);
+
+        });
+
+        return playRows.map(x => ({...x, play: hydratePlaySelect(x, hydrate)}));
+    }
+
+    findPlays = async (args: QueryPlaysOpts, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PlaySelect[]> => {
+        const {
+            hydrate,
+            componentId = this.componentId
+        } = opts;
+        // this does not work as type for query variable
+        // it erases the result type for some reason
+        //
+        // Parameters<typeof this.db.query.plays.findMany>[0]
+        
+        // this does work but it is also integrated into FindWith
+        //let withQuery: Parameters<typeof this.db.query.plays.findMany>[0]['with'] = undefined;
+
+        let query: FindMany<'plays'> = {
+            limit: args.limit,
+            offset: args.offset
+        };
+
+        query.where = buildPlayWhere({componentId: componentId,  ...args});
+
+        if (args.sort !== undefined) {
+            query.orderBy = {
+                [args.sort]: args.order ?? 'desc'
+            }
+        } else {
+            query.orderBy = {
+                id: 'asc'
+            }
+        }
+
+        if(args.with !== undefined) {
+            query.with = {};
+            for(const w of args.with) {
+                switch (w) {
+                    case 'input':
+                        query.with.input = true;
+                        break;
+                    case 'parent':
+                        query.with.parent = true;
+                        break;
+                    case 'parent-input':
+                        query.with.parent = {
+                            with: {
+                                input: true
+                            }
+                        };
+                        break;
+                    case 'queues':
+                        query.with.queueStates = true;
+                        break;
+                    default:
+                        throw new Error(`Unknown relation ${w}`);
+                }
+            }
+        }
+        query = removeUndefinedKeys(query);
+        const results = await this.db.query.plays.findMany(query);
+        return results.map((x) => ({...x, play: hydratePlaySelect(x, hydrate)}));
+    }
+
+    findPlayIds = async (args: QueryPlaysOpts, opts: ComponentConstrainedRepoOpts = {}): Promise<number[]> => {
+        const {
+            componentId = this.componentId
+        } = opts;
+
+        let query: FindMany<'plays'> = {
+            limit: args.limit,
+            offset: args.offset,
+        };
+
+        query.where = buildPlayWhere({componentId: componentId,  ...args});
+
+        if (args.sort !== undefined) {
+            query.orderBy = {
+                [args.sort]: args.order ?? 'desc'
+            }
+        } else {
+            query.orderBy = {
+                id: 'asc'
+            }
+        }
+
+        query = removeUndefinedKeys(query);
+        const results = await this.db.query.plays.findMany({
+            limit: args.limit,
+            offset: args.offset,
+            columns: {id: true},
+            orderBy: args.sort !== undefined ? {[args.sort]: args.order ?? 'desc'} : {id: 'asc'},
+        });
+        return results.map((x) => x.id);
+    }
+
+    findPlayIdentifiers = async <T extends keyof PlayIdentifierPrimitiveMap>(args: QueryPlaysOpts, identifier: T, opts: ComponentConstrainedRepoOpts = {}): Promise<PlayIdentifierPrimitiveMap[T][]> => {
+        const {
+            componentId = this.componentId,
+        } = opts;
+
+        const results = await this.db.query.plays.findMany({
+            limit: args.limit,
+            offset: args.offset,
+            columns: {id: true, uid: true},
+            orderBy: args.sort !== undefined ? {[args.sort]: args.order ?? 'desc'} : {id: 'asc'},
+            where: buildPlayWhere({componentId: componentId,  ...args})
+        });
+
+        // we getting fancy now
+        return results.map(identifierExtractor[identifier]);
+    }
+
+    findPlaysPaginated = async <T = PlaySelectRel>(args: QueryPlaysOpts, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise<PaginatedResponse<T>> => {
+        const {
+            limit = 100,
+            offset = 0,
+            ...rest
+        } = args;
+        const clampedLimit = Math.min(limit, 100);
+        const res = await this.findPlays({limit: clampedLimit, offset, ...rest}, opts) as T[];
+        return {data: res, meta: {limit: clampedLimit, offset}};
+    }
+
+    // async updateById(id: number, data: Partial<PlayNew>): Promise<void> {
+    //     if(data.play !== undefined) {
+    //         data.play = withoutDbAwareness(data.play);
+    //     }
+    //     super.updateById(id, data);
+    // }
+
+    setStateById = async (state: PlayNew['state'], ids: number[]): Promise<void> => {
+        const validIds = ids.filter(x => x !== undefined && x !== null);
+        assert(validIds.length > 0, `Should not pass empty array of ids, after filtering, to update state. Original ids list: ${ids}`);
+        await this.db.update(plays).set({state}).where(inArray(plays.id, ids));
+    }
+
+    deletePlays = async (playsData: (Pick<PlaySelect, 'id'> | number)[]) => {
+        const ids = playsData.map(x => typeof x === 'number' ? x : x.id);
+        await this.db.delete(plays).where(inArray(plays.id, ids));
+    }
+
+    findPurgablePlayIds = async (olderThanDate: Dayjs, opts: { states?: PlaySelect['state'][], compacted?: string, dateComparer?: 'updatedAt' | 'seenAt' } & ComponentConstrainedRepoOpts = {}): Promise<number[]> => {
+
+        const {
+            states,
+            compacted,
+            componentId = this.componentId,
+            dateComparer = 'updatedAt'
+        } = opts;
+
+        let where: FindWhere<'plays'> = {
+            component: {
+                id: componentId
+            },
+            [dateComparer]: {
+                lte: olderThanDate
+            },
+            NOT: {
+                children: {}
+            }
+        };
+
+        if (states !== undefined) {
+            where.state = {
+                in: states
+            }
+        }
+
+        if(compacted !== undefined) {
+            where.compacted = {
+                OR: [
+                    {
+                        isNull: true
+                    },
+                    {
+                        NOT: {
+                            eq: compacted
+                        }
+                    }
+                ]
+            }
+        }
+
+        const rows = await this.db.query.plays.findMany({
+            columns: {
+                id: true
+            },
+            where,
+            orderBy: {
+                id: 'asc'
+            }
+        });
+
+        return rows.map(x => x.id);
+    }
+
+    public retentionCleanup = async (componentType: string, retentionOpts: RetentionOptions & ComponentConstrainedRepoOpts) => {
+
+        const loggerDel = childLogger(this.logger, ['Retention', 'Delete']);
+        const loggerCom = childLogger(this.logger, ['Retention', 'Compact']);
+        let summaryDelStates: string[] = [];
+        let summaryCompactStates: string[] = [];
+
+        loggerDel.debug('Starting cleanup...');
+        for(const retentionType of retentionPlayTypes) {
+            try {
+                const date = dayjs().subtract(retentionOpts.deleteAfter[retentionType].asMilliseconds());
+                let state: PlaySelect['state'];
+                if(retentionType === 'completed') {
+                    state = componentType === 'source' ? 'discovered' : 'scrobbled';
+                } else {
+                    state = retentionType;
+                }
+                loggerDel.trace(`Finding '${retentionType}' plays older than ${shortTodayAwareFormat(date)}...`);
+                const ids = await this.findPurgablePlayIds(date, {states: [state], componentId: retentionOpts.componentId});
+                loggerDel.trace(`Found ${ids.length} '${retentionType}' plays`);
+                if(ids.length === 0) {
+                    summaryDelStates.push(`No '${retentionType}' Plays older than ${shortTodayAwareFormat(date)}`);
+                } else {
+                    loggerDel.trace(`Deleting ${ids.length} '${retentionType}' plays`);
+                    await this.deletePlays(ids);
+                    loggerDel.trace(`'${retentionType}' plays deleted!`);
+                    summaryDelStates.push(`${ids.length} '${retentionType}' Plays older than ${shortTodayAwareFormat(date)}`)
+                }
+            } catch (e) {
+                loggerDel.warn(new Error(`Failed to perform retention cleanup on '${retentionType}' type`, {cause: e}));
+            }
+        }
+        loggerDel.verbose(`Cleanup done! Summary:\n${summaryDelStates.join(' | ')}`);
+
+        if(retentionOpts.compact.length === 0) {
+            loggerCom.debug('Compacting is disabled, skipping cleanup.');
+            return;
+        }
+
+        const compactTypes = retentionOpts.compact;
+        let compactedFlags: CompactableProperty[] = [];
+        if(compactTypes.includes('input')) {
+            compactedFlags.push('input');
+        }
+        if(compactTypes.includes('transform')) {
+            compactedFlags.push('transform');
+        }
+
+        loggerCom.debug('Starting cleanup...');
+        for(const retentionType of retentionPlayTypes) {
+            if(retentionOpts.compactAfter[retentionType] === false) {
+                summaryCompactStates.push(`Skipped ${retentionType}`);
+                continue;
+            }
+            try {
+                const date = dayjs().subtract(retentionOpts.compactAfter[retentionType].asMilliseconds());
+                let state: PlaySelect['state'];
+                if(retentionType === 'completed') {
+                    state = componentType === 'source' ? 'discovered' : 'scrobbled';
+                } else {
+                    state = retentionType;
+                }
+                loggerCom.trace(`Finding '${retentionType}' plays older than ${shortTodayAwareFormat(date)}...`);
+                const ids = await this.findPurgablePlayIds(date, {compacted: compactedFlags.join('-'), states: [state], dateComparer: 'seenAt', componentId: retentionOpts.componentId});
+                loggerCom.trace(`Found ${ids.length} '${retentionType}' plays`);
+                if(ids.length === 0) {
+                    summaryDelStates.push(`No '${retentionType}' Plays older than ${shortTodayAwareFormat(date)}`);
+                } else {
+                    for(const id of ids) {
+                        let compactedPlay: PlayObject;
+                        if(compactTypes.includes('input')) {
+                            await this.db.update(playInputs).set({
+                                data: {removedReason: 'Removed by compaction'}
+                            }).where(eq(playInputs.playId, id));
+                        }
+                        if(compactTypes.includes('transform')) {
+                            const playRow = await this.db.query.plays.findFirst({where: {id: id}});
+                            if(playRow === undefined) {
+                                // uhh shouldn't be
+                                loggerCom.warn(`No Play found with ID ${id}, but it should have been...`);
+                                continue;
+                            }
+
+                            compactedPlay = playRow.play;
+                            compactedPlay.meta.lifecycle.steps = compactedPlay.meta.lifecycle.steps.map(x => {
+                                if(x.inputs == undefined) {
+                                    return x;
+                                }
+                                return {...x, inputs: x.inputs.map(y => ({type: y.type, input: 'Removed by compaction'}))};
+                            });
+                        }
+
+                        const updater = this.db.update(plays);
+                        const vals: Parameters<typeof updater.set>[0] = {
+                            compacted: compactedFlags.join('-')
+                        };
+                        if(compactedPlay !== undefined) {
+                            vals.play = compactedPlay;
+                        }
+                        await this.db.update(plays).set(vals).where(eq(plays.id, id));
+                    }
+                    loggerCom.trace(`Compacted ${ids.length} '${retentionType}' plays`);
+                    summaryCompactStates.push(`${ids.length} '${retentionType}' Plays older than ${shortTodayAwareFormat(date)}`)
+                }
+            } catch (e) {
+                loggerCom.warn(new Error(`Failed to perform retention cleanup on '${retentionType}' type`, {cause: e}));
+            }
+        }
+
+        loggerCom.verbose(`Cleanup done! Summary:\n${summaryDelStates.join(' | ')}`);
+    }
+
+    protected prepareGetQueueNext = () => this.db.query.plays.findFirst({
+        where: {
+            componentId: sql.placeholder('componentId'),
+            queueStates: {
+                queueName: sql.placeholder('queueName'),
+                queueStatus: 'queued',
+                retries: {
+                    lte: sql.placeholder('retries')
+                }
+            },
+        },
+        with: {
+            queueStates: true
+        },
+        orderBy: {
+            seenAt: 'asc'
+        },
+    }).prepare()
+
+    public getQueueNext = async (queueName: string, opts: {order?: 'asc' | 'desc', retries?: number} & ComponentConstrainedRepoOpts = {}): Promise<PlaySelectWithQueueStates | undefined> => {
+        const {
+            retries = 0,
+            order = 'asc',
+            componentId = this.componentId
+        } = opts;
+
+        // let where: FindWhere<'plays'> = {
+        //     componentId
+        // }
+
+        // if(retries !== undefined) {
+        //     where.queueStates = {
+        //         queueName,
+        //         queueStatus: 'queued',
+        //         retries: {
+        //             lte: retries
+        //         }
+        //     }
+        // } else {
+        //     where.queueStates = {
+        //         queueName,
+        //         queueStatus: 'queued'
+        //     }
+        // }
+
+        // const res = await this.db.query.plays.findFirst({
+        //         where: where,
+        //         orderBy: {
+        //             seenAt: order
+        //         },
+        //         with: {
+        //             queueStates: true
+        //         }
+        // });
+
+        if(this.getQueueNextPrepared === undefined) {
+            this.getQueueNextPrepared = this.prepareGetQueueNext();
+        }
+
+        const res = await this.getQueueNextPrepared.execute({queueName, retries, componentId});
+ 
+        if(res === undefined) {
+            return undefined;
+        }
+        res.play = hydratePlaySelect(res); // asPlay(res.play);
+        return res;
+    }
+
+    protected prepareHasQueueNext = () => this.db.query.plays.findFirst({
+        columns: {
+                    id: true
+        },
+        where: {
+            componentId: this.componentId,
+            queueStates: {
+                queueName: sql.placeholder('queueName'),
+                queueStatus: 'queued',
+                retries: {
+                    lte: sql.placeholder('retries')
+                }
+            }
+        }
+    }).prepare()
+
+    public hasQueueNext = async (queueName: string, retries: number = 0): Promise<boolean> => {
+        if(this.hasQueueNextPrepared === undefined) {
+            this.hasQueueNextPrepared = this.prepareHasQueueNext();
+        }
+        const nextId = await this.hasQueueNextPrepared.execute({queueName, retries});
+        return nextId !== undefined;
+    }
+
+    protected prepareGetQueuedScrobbleRange = () => this.db.query.plays.findMany({
+        where: {
+            componentId: this.componentId,
+            queueStates: {
+                queueName: sql.placeholder('queueName'),
+                queueStatus: 'queued',
+                retries: {
+                    lte: sql.placeholder('retries')
+                }
+            },
+        },
+        orderBy: {
+            seenAt: 'asc',
+        },
+        limit: sql.placeholder('limit')
+    }).prepare()
+
+    public getQueuedScrobbleRange = async (queueName: string, opts: {retries?: number, limit?: number} = {}): Promise<PlayObject[]> => {
+        if(this.getQueuedScrobbleRangePrepared === undefined) {
+            this.getQueuedScrobbleRangePrepared = this.prepareGetQueuedScrobbleRange();
+        }
+        const res = await this.getQueuedScrobbleRangePrepared.execute({queueName, retries: opts.retries ?? 0, limit: opts.limit ?? 30});
+        return res.map(x => x.play);
+    }
+
+    public getQueued = async (queueName: string, opts: {
+        order?: 'asc' | 'desc',
+        limit?: number,
+        offset?: number,
+        retries?: number,
+    } & ComponentConstrainedRepoOpts & HydrateOpts = {}
+    ): Promise<{data: PlaySelect[], meta: PaginatedQueryResponse}> => {
+        const {
+            order = 'asc',
+            limit = 100,
+            offset = 0,
+            retries,
+            componentId = this.componentId,
+            hydrate
+        } = opts;
+        let where: FindWhere<'plays'> = {
+            componentId
+        }
+        if(retries !== undefined) {
+            where.queueStates = {
+                queueName,
+                queueStatus: 'queued',
+                retries: {
+                    lte: retries
+                }
+            }
+        } else {
+            where.queueStates = {
+                queueName,
+                queueStatus: 'queued'
+            }
+        }
+        const res = await this.db.query.plays.findMany({
+            where,
+            orderBy: {
+                seenAt: order
+            },
+            limit,
+            offset
+        });
+        return {data: res.map(x => ({...x, play: hydratePlaySelect(x, hydrate)})), meta: {limit, offset}};
+    }
+
+    public checkExisting = async (play: PlayObject, opts: {queueName?: string, states?: PlaySelect['state'][], taAccuracy?: TemporalAccuracy[]} & ComponentConstrainedRepoOpts = {}): Promise<PlaySelectWithQueueStates | undefined> => {
+        const {
+            queueName,
+            componentId = this.componentId,
+            taAccuracy = TA_DEFAULT_ACCURACY,
+            states
+        } = opts;
+        const hash = hashObject(playContentBasicInvariantTransform(play).data);
+
+        // we get all plays with a play date between playdate - (source accuracy) AND (playDateCompleted or playDate) + (source accuracy)
+        // which we can then use with temporal comparison to make sure we are comparing the correct dates
+        //
+        // this isn't as fast as just comparing playDate directly but its still much faster/cheaper than paginating plays and doing everything in-memory
+        const dateGranularity = getTemporalAccuracyCloseVal(play.meta.source as SourceType);
+        let endRange: Dayjs;
+        if(play.data.playDateCompleted !== undefined) {
+            // this will be present if source reports it
+            // or we tracked it live with MemorySource
+            endRange = play.data.playDateCompleted.add(dateGranularity, 's');
+        } else {
+            endRange = play.data.playDate.add(dateGranularity, 's');
+        }
+        let where: FindWhere<'plays'> = {
+            componentId,
+            playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play)),
+        };
+        
+        if(queueName !== undefined) {
+            where.queueStates = {
+                queueName,
+                queueStatus: 'queued'
+            }
+        }
+        if(states !== undefined) {
+            where.state = {
+                in: states
+            }
+        }
+
+        const mbidId = playMbidIdentifier(play);
+        if(mbidId !== undefined) {
+            where.AND = [
+                {
+                    OR: [
+                        {
+                            playHash: hash
+                        },
+                        {
+                            mbidIdentifier: mbidId
+                        }
+                    ]
+                }
+            ]
+        } else {
+            where.playHash = hash;
+        }
+
+        const res = await this.db.query.plays.findMany({
+            where,
+            with: {
+                queueStates: true
+            }
+        });
+        if(res.length === 0) {
+            return undefined;
+        }
+        return res.map(x => ({...x, play: hydratePlaySelect(x)})).find(x => {
+            const temporalComparison = comparePlayTemporally(x.play, play);
+            return hasAcceptableTemporalAccuracy(temporalComparison.match, taAccuracy)
+        })
+    }
+
+    public getTemporallyClosePlays = async (play: PlayObject, opts: {states?: PlaySelect['state'][], bufferTime?: number} & { with?: WithPlayRelation[] } & ComponentConstrainedRepoOpts = {}): Promise<PlaySelectRel[]> => {
+        const {
+            componentId = this.componentId,
+            bufferTime,
+            states,
+            with: qWith
+        } = opts;
+
+        let query: FindMany<'plays'> = {};
+
+        let where: FindWhere<'plays'> = {
+            componentId,
+            playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play, {bufferTime})),
+        };
+        if(states !== undefined) {
+            where.state = {
+                in: states
+            }
+        }
+        query.where = where;
+
+        return ((await this.db.query.plays.findMany({
+            where,
+            with: buildPlayWith(qWith)
+        })) as PlaySelectRel[]).map(x => ({...x, play: hydratePlaySelect(x)}));
+    }
+}
+
+export const getTemporallyCloseDateCompareOp = (play: PlayObject, opts: {bufferTime?: number, useCompleted?: boolean} = {}): CompareDateOp => {
+    const {
+        // use either provided arg or default to using source granularity
+        bufferTime = getTemporalAccuracyCloseVal(play.meta.source as SourceType),
+        useCompleted = true
+    } = opts;
+        // we get all plays with a play date between playdate - (buffer) AND (playDateCompleted or playDate) + (buffer)
+        let endRange: Dayjs;
+        if(play.data.playDateCompleted !== undefined && useCompleted) {
+            // this will be present if source reports it
+            // or we tracked it live with MemorySource
+            endRange = play.data.playDateCompleted.add(bufferTime, 's');
+        } else {
+            endRange = play.data.playDate.add(bufferTime, 's');
+        }
+        return {
+            type: 'between',
+            range: [play.data.playDate.subtract(bufferTime, 's'), endRange]
+        }
+}
+
+export const buildPlayWith = (args: WithPlayRelation[] | undefined): FindWith<'plays'> | undefined => {
+    if(args === undefined) {
+        return undefined;
+    }
+    const qWith: FindWith<'plays'> = {};
+    for(const w of args) {
+        switch (w) {
+            case 'input':
+                qWith.input = true;
+                break;
+            case 'parent':
+                qWith.parent = true;
+                break;
+            case 'parent-input':
+                qWith.parent = {
+                    with: {
+                        input: true
+                    }
+                };
+                break;
+            case 'queues':
+                qWith.queueStates = true;
+                break;
+            default:
+                throw new Error(`Unknown relation ${w}`);
+        }
+    }
+    return qWith;
+}
+
+export const buildPlayWhere = (args: PlayWhereOpts): WhereClause<'plays'> => {
+    // old way
+    // let where: Parameters<(ReturnType<typeof getDb>)['query']['plays']['findMany']>[0]['where'] = {
+    // };
+    let where: FindWhere<'plays'> = {
+        componentId: args.componentId
+    };
+    if (args.state !== undefined) {
+        where.state = {
+            in: args.state
+        }
+    }
+    if(args.stateNot !== undefined) {
+        where.state = {
+            NOT: {
+                in: args.stateNot
+            }
+        }
+    }
+    if (args.seenAt !== undefined) {
+        where.seenAt = buildDateCompare(args.seenAt);
+    }
+    if (args.playedAt !== undefined) {
+        where.playedAt = buildDateCompare(args.playedAt);
+    }
+    if(args.uid !== undefined) {
+        where.uid = {
+            in: args.uid
+        }
+    }
+    const {
+        queues = []
+    } = args;
+    if(queues.length > 0) {
+        // need to do this optimistically even if we overwrite with only 1 condition later
+        where.queueStates = {
+            OR: []
+        }
+        // so that we can use this type
+        // or else assigning an array to OR using only `typeof where.queueStates` causes a type error
+        let queueWhere: typeof where.queueStates.OR[0][] = [];
+        for(const q of queues) {
+            queueWhere.push(
+                {
+                    queueName: q.queueName,
+                    queueStatus: typeof q.queueStatus === 'string' ? q.queueStatus : {
+                        in: q.queueStatus
+                    }
+                }
+            )
+        }
+        if(queueWhere.length === 1) {
+            where.queueStates = queueWhere[0];
+        } else {
+            where.queueStates = {
+                OR: queueWhere
+            }
+        }
+    }
+    return where;
+}
+
+export const playToRepositoryCreatePlayOpts = (data: MarkOptional<RepositoryCreatePlayOpts, 'input' | 'componentId'>): RepositoryCreatePlayOpts => {
+    const {
+        play: {
+            meta: {
+                lifecycle: {
+                    input,
+                    original,
+                    ...lifecycleRest
+                } = {},
+                ...metaRest
+            },
+            ...playRest
+        },
+        ...rest
+    } = data;
+
+    return {
+        play: {
+            ...playRest,
+            meta: {
+                ...metaRest,
+                // @ts-expect-error
+                lifecycle: {
+                    ...lifecycleRest
+                }
+            }
+        },
+        ...rest,
+        input: {
+            play: original,
+            data: input
+        }
+    }
+}
+
+export type RequestPlayQuery = Partial< Record<keyof Exclude<QueryPlaysOpts, 'componentId'>, string>>;
+
+export const queryArgsFromRequest = (rec: RequestPlayQuery): QueryPlaysOpts => {
+
+    const {
+        state,
+        stateNot,
+        uid,
+        with: withQuery,
+        seenAt,
+        playedAt,
+        limit,
+        sort,
+        order,
+        offset,
+        componentId,
+        queues,
+        ...rest
+    } = rec;
+
+    let queryArgs: QueryPlaysOpts = removeEmptyArrays<QueryPlaysOpts>({
+        state: parseArrayFromMaybeString(state) as PlaySelect['state'][],
+        stateNot: parseArrayFromMaybeString(stateNot) as PlaySelect['state'][],
+        uid: parseArrayFromMaybeString(uid),
+        with: parseArrayFromMaybeString(withQuery) as WithPlayRelation[],
+        sort: sort as 'playedAt' | 'seenAt',
+        order: order as 'asc' | 'desc',
+        ...rest
+    });
+
+    if(limit !== undefined) {
+        queryArgs.limit = Number.parseInt(limit);
+    }
+    if(offset !== undefined) {
+        queryArgs.offset = Number.parseInt(offset);
+    }
+
+    return queryArgs;
+}

src/backend/common/database/drizzle/repositories/QueueRepository.ts

@@ -0,0 +1,41 @@
+import { eq, and, lte, inArray } from "drizzle-orm";
+import { DrizzleBaseRepository, DrizzleRepositoryOpts } from "./BaseRepository.js";
+import { DbConcrete } from "../drizzleUtils.js";
+import { QueueStateSelect } from "../drizzleTypes.js";
+import { queueStates } from "../schema/schema.js";
+import { CLIENT_DEAD_QUEUE } from "../../../../../core/Atomic.js";
+export class DrizzleQueueRepository extends DrizzleBaseRepository<'queueStates'> {
+
+    constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
+        super(db, 'queueStates', 'Queue', opts);
+    }
+
+    public deadFailedToQueue = async (componentId: number, retries: number): Promise<void> => {
+        await this.db.update(queueStates).set({
+            queueStatus: 'queued',
+        }).where(and(
+            eq(queueStates.componentId, componentId),
+            lte(queueStates.retries, retries),
+            eq(queueStates.queueStatus, 'failed'),
+            eq(queueStates.queueName, CLIENT_DEAD_QUEUE)
+        ));
+    }
+
+    public failedQueueToCompleted = async (componentId: number): Promise<void> => {
+        await this.db.update(queueStates).set({
+            queueStatus: 'completed',
+        }).where(and(
+            eq(queueStates.componentId, componentId),
+            eq(queueStates.queueStatus, 'queued'),
+            eq(queueStates.queueName, CLIENT_DEAD_QUEUE)
+        ));
+    }
+
+    public getQueueCount = async (componentId: number, queueNames: string[], queueStatus: QueueStateSelect['queueStatus'][] = ['queued']): Promise<number> => {
+        return await this.db.$count(queueStates, and(
+            eq(queueStates.componentId, componentId),
+            inArray(queueStates.queueName, queueNames),
+            inArray(queueStates.queueStatus, queueStatus)
+        ));
+    }
+}

src/backend/common/database/drizzle/schema/schema.ts

@@ -0,0 +1,239 @@
+import { integer, serial as primaryInt, pgTable as table, text, varchar, json, index, uniqueIndex, customType, AnyPgColumn, timestamp } from "drizzle-orm/pg-core";
+import { defineRelations } from 'drizzle-orm';
+import dayjs, { Dayjs } from "dayjs";
+import { nanoid } from "nanoid";
+import { ErrorLike, PlayObject } from "../../../../../core/Atomic.js";
+import { asPlayCheap } from "../../../../../core/PlayMarshalUtils.js";
+import { ExternalMetadataTerm, PlayTransformPartsConfig, SearchAndReplaceTerm } from "../../../infrastructure/Transform.js";
+import { JobRangeCount, JobRangeTime } from "../../../infrastructure/Job.js";
+
+const DayjsTimestamp = customType<
+  {
+    data: Dayjs;
+    driverData: string;
+  }
+>({
+  dataType() {
+    return 'timestamp'
+  },
+  toDriver(value: Dayjs): string {
+    return value.toISOString();
+  },
+  fromDriver(value: string): Dayjs {
+    return dayjs(value);
+  },
+});
+
+const PlayJson = customType<
+  {
+    data: PlayObject;
+    driverData: string;
+  }
+>({
+  dataType() {
+    return 'jsonb'
+  },
+  toDriver(value: PlayObject): string {
+    const {
+      // meta: {
+      //   // dbId,
+      //   // dbUid,
+      //   ...metaRest
+      // },
+      id,
+      uid,
+      ...rest
+    } = value;
+    return JSON.stringify(rest);
+  },
+  fromDriver(value: any): PlayObject {
+    return asPlayCheap(value);
+  },
+});
+
+
+export const plays = table("plays", {
+  id: primaryInt().primaryKey(),
+  uid: varchar({ length: 30 }).notNull().unique().$defaultFn(() => nanoid(20)),
+  componentId: integer().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}),
+  error: json().$type<ErrorLike>(),
+  playedAt: DayjsTimestamp('playedAt'),
+  seenAt: DayjsTimestamp('seenAt'),
+  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()),
+  play: PlayJson('play').notNull(), //  text({ mode: 'json' }).notNull().$type<PlayObject>(),
+  state: varchar({enum: ['queued','discovered','discarded','scrobbled','failed','duped'], length: 20}).notNull(),
+  // https://orm.drizzle.team/docs/indexes-constraints#foreign-key
+  parentId: integer().references((): AnyPgColumn => plays.id, {onDelete: 'set null', onUpdate: 'cascade'}),
+  jobId: integer().references(() => jobs.id, {onDelete: 'cascade', onUpdate: 'cascade'}),
+  playHash: varchar({length: 100}),
+  mbidIdentifier: varchar({length: 100}),
+  compacted: varchar({length: 30})
+}, (table) => [
+  index("play_parent_id_idx").on(table.parentId),
+  index("play_component_id_idx").on(table.componentId),
+  uniqueIndex("play_uid_idx").on(table.uid),
+  index("play_playedAt_idx").on(table.playedAt),
+  index("play_seenAt_idx").on(table.seenAt)
+]);
+
+export const playInputs = table("play_inputs", {
+  id: primaryInt().primaryKey(),
+  playId: integer().notNull().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}),
+  data: json().$type<object>(),
+  play: PlayJson('play').notNull(),//text({ mode: 'json' }).notNull().$type<PlayObject>(),
+  createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs())
+}, (table) => [
+  uniqueIndex('play_input_id_idx').on(table.playId)
+]);
+
+// export const playParentRelations = defineRelations({plays}, (r) => ({
+//   plays: {
+//     parent: r.one.plays({
+//       from: r.plays.parentId,
+//       to: r.plays.id
+//     }),
+//     children: r.many.plays()
+//   }
+// }))
+
+
+// export const playInputRelations = defineRelations({ plays, playInputs }, (r) => ({
+//   plays: {
+//     input: r.one.playInputs({
+//       from: r.plays.id,
+//       to: r.playInputs.playId,
+//       optional: false,
+//     })
+//   }
+// }));
+
+export const queueStates = table("play_queue_states", {
+  id: primaryInt().primaryKey(),
+  playId: integer().notNull().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}),
+  componentId: integer().notNull().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}),
+  queueName: varchar({length: 50}).notNull(),
+  queueStatus: varchar({enum: ['queued','completed','failed'], length: 20}).notNull().default('queued'),
+  retries: integer().notNull().default(0),
+  error: json().$type<ErrorLike>(),
+  createdAt: DayjsTimestamp('createdAt').notNull().$defaultFn(() => dayjs()),
+  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs())
+}, (table) => [
+  index('play_queue_state_id_idx').on(table.playId)
+]);
+
+// export const playQueueRelations = defineRelations({ plays, queueStates }, (r) => ({
+//   plays: {
+//     queueStates: r.many.queueStates()
+//   },
+//   queueStates: {
+//     play: r.one.plays({
+//       from: r.queueStates.playId,
+//       to: r.plays.id
+//     })
+//   }
+// }));
+
+export const components = table("components", {
+  id: primaryInt().primaryKey(),
+  // user-provided id
+  uid: varchar({ length: 200 }).notNull(),
+  mode: varchar({enum: ['source','client'], length: 15}).notNull(),
+  // spotify, lastfm, etc...
+  type: varchar({length: 50}).notNull(),
+  // vanity display name
+  // used as uid if no user-provided id
+  name: varchar().notNull(),
+  // number of discovered/scrobbled plays found in real time
+  countLive: integer().notNull().default(0),
+  // number of discovered/scrobbled plays from backlog/jobs
+  countNonLive: integer().notNull().default(0),
+  createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs())
+},
+(table) => [
+  uniqueIndex('uid_mode_type_idx').on(table.uid,table.mode,table.type)
+]);
+
+export const jobs = table("jobs", {
+  id: primaryInt().primaryKey(),
+  componentFromId: integer().notNull().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}),
+  componentToId: integer().notNull().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}),
+  name: varchar({length: 200}).notNull(),
+  status: varchar({enum: ['idle','completed','failed','processing'], length: 20}).notNull().default('idle'),
+  retries: integer().notNull().default(0),
+  error: json().$type<ErrorLike>(),
+  transformOptions: json().$type<PlayTransformPartsConfig<SearchAndReplaceTerm[] | ExternalMetadataTerm>>(),
+  initialParameters: json().$type<JobRangeCount | JobRangeTime>(),
+  cursor: json(),
+  total: integer(),
+  imported: integer().notNull().default(0),
+  scrobbled: integer().notNull().default(0),
+  createdAt: DayjsTimestamp('createdAt').notNull().$defaultFn(() => dayjs()),
+  updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()),
+  completedAt: DayjsTimestamp('completedAt')
+});
+
+const playRelations = defineRelations({ plays, queueStates, playInputs, components, jobs }, (r) => ({
+  plays: {
+    queueStates: r.many.queueStates(),
+    input: r.one.playInputs({
+      from: r.plays.id,
+      to: r.playInputs.playId,
+      optional: false,
+    }),
+    parent: r.one.plays({
+      from: r.plays.parentId,
+      to: r.plays.id
+    }),
+    children: r.many.plays(),
+    component: r.one.components({
+      from: r.plays.componentId,
+      to: r.components.id,
+      optional: true
+    }),
+    job: r.one.jobs({
+      from: r.plays.jobId,
+      to: r.jobs.id,
+      optional: true
+    })
+  },
+  queueStates: {
+    play: r.one.plays({
+      from: r.queueStates.playId,
+      to: r.plays.id
+    }),
+    component: r.one.components({
+      from: r.queueStates.componentId,
+      to: r.components.id
+    })
+  },
+  components: {
+    plays: r.many.plays(),
+    queueStates: r.many.queueStates(),
+  },
+  jobs: {
+    plays: r.many.plays()
+  }
+}));
+
+export const relations = playRelations;
+
+export const getConfigByTableName = <T extends TableName>(name: T) => {
+  switch(name) {
+    case 'plays':
+      return plays;
+    case 'components':
+      return components;
+    case 'playInputs':
+      return playInputs;
+    case 'queueStates':
+      return queueStates;
+    case 'jobs':
+      return jobs;
+  }
+}
+
+const schema = {playInputs, plays, components, queueStates, jobs};
+
+export type TSchema = typeof relations;
+export type Schema = typeof schema;
+export type TableName = keyof TSchema;

src/backend/common/errors/MSErrors.ts

@@ -43,12 +43,14 @@ export class SimpleError extends Error implements HasSimpleError {
     stackShortened: boolean = false;
 
     shortenStack() {
-        const atIndex = parseRegexSingle(STACK_AT_REGEX,this.stack);
-        if(atIndex !== undefined) {
-            const firstn = this.stack.indexOf('\n', atIndex.index + atIndex.match.length);
-            if(firstn !== -1) {
-                this.stack = this.stack.slice(0, firstn);
-                this.stackShortened = true;
+        if(this.stack !== undefined) {
+            const atIndex = parseRegexSingle(STACK_AT_REGEX, this.stack);
+            if(atIndex !== undefined) {
+                const firstn = this.stack.indexOf('\n', atIndex.index + atIndex.match.length);
+                if(firstn !== -1) {
+                    this.stack = this.stack.slice(0, firstn);
+                    this.stackShortened = true;
+                }
             }
         }
     }
@@ -128,4 +130,23 @@ export const generateLoggableAbortReason = (msg: string, signal: AbortSignal): A
     }
     Error.captureStackTrace(err, generateLoggableAbortReason);
     return err;
+}
+
+export class InvalidRegexError extends SimpleError {
+    constructor(regex: RegExp | RegExp[], val?: string, url?: string, message?: string) {
+        const msgParts = [
+            message ?? 'Regex(es) did not match the value given.',
+        ];
+        let regArr = Array.isArray(regex) ? regex : [regex];
+        for(const r of regArr) {
+            msgParts.push(`Regex: ${r}`)
+        }
+        if (val !== undefined) {
+            msgParts.push(`Value: ${val}`);
+        }
+        if (url !== undefined) {
+            msgParts.push(`Sample regex: ${url}`);
+        }
+        super(msgParts.join('\r\n'));
+    }
 }

src/backend/common/index.ts

@@ -5,4 +5,4 @@ import * as path from 'path';
 //const __dirname = path.dirname(__filename);
 
 export const projectDir = process.cwd(); //path.resolve(__dirname, '../../../');
-export const configDir: string = path.resolve(projectDir, './config');
+export const configDir: string = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);

src/backend/common/infrastructure/Atomic.ts

@@ -141,9 +141,6 @@ export interface RemoteIdentityParts {
     agent: string | undefined
 }
 
-// https://stackoverflow.com/questions/40510611/typescript-interface-require-one-of-two-properties-to-exist#comment116238286_49725198
-export type RequireAtLeastOne<T, R extends keyof T = keyof T> = Omit<T, R> & {   [ P in R ] : Required<Pick<T, P>> & Partial<Omit<T, P>> }[R];
-
 /**
  * https://www.last.fm/api/scrobbling (When is a scrobble a scrobble?)
  * https://github.com/krateng/maloja/blob/master/API.md#scrobbling-guideline
@@ -299,6 +296,21 @@ export interface CacheConfigOptions {
     regex?: number
 }
 
+export interface CacheConfigUser {
+    auth?: {
+        provider: 'valkey' | 'file',
+        [key: string]: any
+    };
+    valkey?: string
+    /** Number of regex functions to cache (LRU)
+     * 
+     * @default 200
+     */
+    regex?: number
+    // to allow deprecated scrobble config without having it show up in schema docs
+    [key: string]: any
+}
+
 export interface MusicbrainzApiConfigData {
     url?: string
     contact: string,
@@ -414,4 +426,18 @@ export interface ScrobbleRangeResult {
     fetchedAt: Dayjs
 }
 
-export const REFRESH_STALE_DEFAULT = 60;
+export const REFRESH_STALE_DEFAULT = 60;
+
+/**
+ * A duration of time
+ * 
+ * May be either:
+ * 
+ * * a `number` of seconds
+ * * a `string` containing a number and a unit of time compatible with dayjs
+ * 
+ * @example [60, 3600, "1 hour", "4 days"]
+ */
+export type DurationValue = number | string;
+
+export type DbExternalMode = 'none' | 'live' | 'standalone';

src/backend/common/infrastructure/Job.ts

@@ -0,0 +1,33 @@
+import { UnixTimestamp } from "../../../core/Atomic.js"
+
+export interface JobParameters {
+    /** maximum number of results to get for the entire job */
+    fetchMax?: number
+    order?: 'asc' | 'desc'
+    /** Whether to increment targeted scrobbler client scrobble count */
+    countInScrobbler?: boolean
+    maxRetries?: number
+    /** Whether to keep non-failed plays in db after scrobbling has occurred
+     * 
+     * * `true` => keep all plays
+     * * `false` => keep no plays
+     * * number => max number of play to keep based on import date
+     */
+    keepPlays?: boolean | number
+}
+
+export interface JobRangeCount extends JobParameters {
+    order: 'asc' | 'desc'
+}
+
+export interface JobRangeTime extends JobParameters {
+    /** Unix timestamp */
+    from: UnixTimestamp
+    /** Unix timestamp */
+    to: UnixTimestamp
+}
+
+// TODO
+export interface JobCursor {
+
+}

src/backend/common/infrastructure/config/aioConfig.ts

@@ -5,8 +5,9 @@ import { RequestRetryOptions } from "./common.js";
 import { WebhookConfig } from "./health/webhooks.js";
 import { CommonSourceOptions, SourceRetryOptions } from "./source/index.js";
 import { SourceAIOConfig } from "./source/sources.js";
-import { CacheConfigOptions } from "../Atomic.js";
+import { CacheConfigOptions, CacheConfigUser, DurationValue } from "../Atomic.js";
 import { TransformerCommonConfig } from "../../../../core/Atomic.js";
+import { RetentionConfig } from "./database.js";
 
 
 export interface SourceDefaults extends CommonSourceOptions {
@@ -66,9 +67,13 @@ export interface AIOConfig {
      * */
     debugMode?: boolean
 
-    cache?: CacheConfigOptions
+    cache?: CacheConfigUser
 
     transformers?: TransformerCommonConfig[]
+
+    database?: {
+        retention?: RetentionConfig<DurationValue>
+    }
 }
 
 export interface AIOClientConfig {

src/backend/common/infrastructure/config/client/index.ts

@@ -1,5 +1,7 @@
+import { DurationValue } from "../../Atomic.js";
 import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js";
 import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js";
+import { RetentionConfig } from "../database.js";
 
 /**
  * Scrobble matching (between new source track and existing client scrobbles) logging options. Used for debugging.
@@ -105,6 +107,8 @@ export interface CommonClientOptions extends RequestRetryOptions, UpstreamRefres
     deadLetterRetries?: number
 
     playTransform?: PlayTransformOptions
+
+    retention?: RetentionConfig<DurationValue>
 }
 
 export interface CommonClientConfig extends CommonConfig {

src/backend/common/infrastructure/config/common.ts

@@ -1,7 +1,20 @@
 import { keyOmit } from "../Atomic.js";
 
+export interface CommonConfigPrimitives {
+    name?: string
+    id?: string
+    enable?: boolean
+}
+
 export interface CommonConfig {
     name?: string
+    /** A UNIQUE identifier for this Source/Client
+     * 
+     * It should be unique for the given Source/Client type. No other Source/Client of the same type should have this ID. This ID will be used to register this Source/Client in the database so that it can be identified even if you change the name of the component.
+     * 
+     * If no id is given the name of this component will be used.
+     */
+    id?: string
     data?: CommonData
     /**
      * Should MS use this client/source? Defaults to true

src/backend/common/infrastructure/config/database.ts

@@ -0,0 +1,36 @@
+import { Duration } from "dayjs/plugin/duration.js";
+import { DurationValue } from "../Atomic.js";
+
+export type RetentionPlayType = 'failed' | 'completed' | 'duped';
+export const retentionPlayTypes: RetentionPlayType[] = ['failed','completed','duped'];
+
+export type RetentionValueUnparsed = DurationValue | Duration | false;
+export type RetentionValue = Duration | false;
+export interface RententionGranular<T extends RetentionValueUnparsed> {
+    failed?: T
+    completed?: T
+    duped?: T
+}
+
+export type RetentionConfigValue<T extends RetentionValueUnparsed> =  T | RententionGranular<T>;
+export type RetentionOption<T extends RetentionValue> = Required<RententionGranular<T>>;
+
+export type CompactableProperty = 'transform' | 'input';
+export const COMPACTABLE = {
+    transform: 'transform',
+    input: 'input'
+} as const satisfies Record<string, CompactableProperty>;
+export const compactableProperties: CompactableProperty[] = [COMPACTABLE.transform, COMPACTABLE.input];
+export interface RetentionConfig<T extends RetentionValueUnparsed> {
+    deleteAfter?: RetentionConfigValue<T>
+    compactAfter?: RetentionConfigValue<T>
+    compact?: CompactableProperty[]
+}
+
+export interface RetentionOptions {
+    deleteAfter: RetentionOption<Duration>
+    compactAfter: RetentionOption<RetentionValue>
+    compact: CompactableProperty[]
+}
+
+export const DEFAULT_RETENTION_DELETE_AFTER = 604800; // 7 days

src/backend/common/infrastructure/config/source/index.ts

@@ -2,6 +2,8 @@ import { FileLogOptions, LogLevel } from "@foxxmd/logging";
 
 import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js";
 import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js";
+import { RetentionConfig } from "../database.js";
+import { DurationValue } from "../../Atomic.js";
 
 export interface SourceRetryOptions extends RequestRetryOptions {
     /**
@@ -108,6 +110,8 @@ export interface CommonSourceOptions extends SourceRetryOptions {
     scrobbleBacklogCount?: number
 
     playTransform?: PlayTransformOptions
+
+    retention?: RetentionConfig<DurationValue>
 }
 
 export interface ManualListeningOptions {

src/backend/common/transforms/AbstractTransformer.ts

@@ -61,12 +61,13 @@ export default abstract class AbstractTransformer<T = any, Y extends StageConfig
     protected abstract doParseConfig(data: StageConfig): Y;
 
     public async handle(data: Y, play: PlayObject): Promise<PlayObject> {
-        const cacheKey = `${this.configHash}-${hashObject(data)}-${hashObject(playContentInvariantTransform(play))}`
+        const cacheKey = `transformResult-${this.configHash}-${hashObject(data)}-${hashObject(playContentInvariantTransform(play))}`
         try {
-            const cachedTransform = await this.cache.get<PlayObject>(cacheKey);
-            if(cachedTransform !== undefined) {
+            const cachedTransformData = await this.cache.get<T>(cacheKey);
+            if(cachedTransformData !== undefined) {
                 this.logger.debug('Transform cache hit');
-                return cachedTransform;
+                const transformed = await this.doHandle(data, play, cachedTransformData);
+                return transformed;
             }
         } catch (e) {
             this.logger.warn(new Error(`Could not fetch cache key ${cacheKey}`, {cause: e}));
@@ -106,7 +107,7 @@ export default abstract class AbstractTransformer<T = any, Y extends StageConfig
         }
 
         const transformed = await this.doHandle(data, play, transformData);
-        await this.cache.set(cacheKey, transformed, this.config.options?.ttl ?? '15s');
+        await this.cache.set(cacheKey, transformData, this.config.options?.ttl ?? '15s');
         return transformed;
     }
 

src/backend/common/transforms/AtomicPartsTransformer.ts

@@ -1,4 +1,4 @@
-import { isPlayObject, ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js";
+import { ArtistCredit, isPlayObject, ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js";
 import { AtomicStageConfig, StageConfig } from "../infrastructure/Transform.js";
 import AbstractTransformer from "./AbstractTransformer.js";
 
@@ -135,8 +135,8 @@ export default abstract class AtomicPartsTransformer<Y, T = any, Z extends Atomi
         }
     
         protected abstract handleTitle(play: PlayObject, parts: Y, transformData: T): Promise<string | undefined>;
-        protected abstract handleArtists(play: PlayObject, parts: Y, transformData: T): Promise<string[] | undefined>;
-        protected abstract handleAlbumArtists(play: PlayObject, parts: Y, transformData: T): Promise<string[] | undefined>;
+        protected abstract handleArtists(play: PlayObject, parts: Y, transformData: T): Promise<ArtistCredit[] | undefined>;
+        protected abstract handleAlbumArtists(play: PlayObject, parts: Y, transformData: T): Promise<ArtistCredit[] | undefined>;
         protected abstract handleAlbum(play: PlayObject, parts: Y, transformData: T): Promise<string | undefined>;
         protected async handleDuration(play: PlayObject, parts: Y, transformData: T): Promise<number | undefined> {
             return play.data.duration;

src/backend/common/transforms/MusicbrainzTransformer.ts

@@ -1,4 +1,4 @@
-import { asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, isMBReleasePrimaryGroupType, MBReleaseGroupPrimaryType, MBReleaseGroupSecondaryType, MBReleaseStatus, MissingMbidType, PlayObject, TrackMeta, TransformerCommon, TransformOptions } from "../../../core/Atomic.js";
+import { ArtistCredit, asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, isMBReleasePrimaryGroupType, MBReleaseGroupPrimaryType, MBReleaseGroupSecondaryType, MBReleaseStatus, MissingMbidType, PlayObject, TrackMeta, TransformerCommon, TransformOptions } from "../../../core/Atomic.js";
 import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js";
 import { WebhookPayload } from "../infrastructure/config/health/webhooks.js";
 import { ExternalMetadataTerm, PlayTransformMetadataStage } from "../infrastructure/Transform.js";
@@ -661,7 +661,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
 
         return transformData.data.track;
     }
-    protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise<string[] | undefined> {
+    protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise<ArtistCredit[] | undefined> {
         if (parts === false) {
             return play.data.artists;
         }
@@ -676,7 +676,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
 
         return transformData.data.artists;
     }
-    protected async handleAlbumArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise<string[] | undefined> {
+    protected async handleAlbumArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise<ArtistCredit[] | undefined> {
         if (parts === false) {
             return play.data.albumArtists;
         }

src/backend/common/transforms/NativeTransformer.ts

@@ -1,4 +1,4 @@
-import { PlayObject, TransformerCommon } from "../../../core/Atomic.js";
+import { ArtistCredit, PlayObject, TransformerCommon } from "../../../core/Atomic.js";
 import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js";
 import { WebhookPayload } from "../infrastructure/config/health/webhooks.js";
 import { ExternalMetadataTerm, PlayTransformNativeStage, StageConfig } from "../infrastructure/Transform.js";
@@ -10,6 +10,7 @@ import { DELIMITERS_NO_AMP } from "../infrastructure/Atomic.js";
 import { asArray } from "../../utils/DataUtils.js";
 import { MaybeLogger } from '../MaybeLogger.js';
 import { childLogger } from "@foxxmd/logging";
+import { artistCreditToName, artistNameToCredit } from "../../../core/StringUtils.js";
 
 export type ArtistParseSource = 'artists' | 'title'
 
@@ -167,7 +168,7 @@ export default class NativeTransformer extends AtomicPartsTransformer<ExternalMe
     protected async handleTitle(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise<string | undefined> {
         return play.data.track;
     }
-    protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise<string[] | undefined> {
+    protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise<ArtistCredit[] | undefined> {
         if (parts === false) {
             return play.data.artists;
         }
@@ -182,7 +183,7 @@ export default class NativeTransformer extends AtomicPartsTransformer<ExternalMe
 
         return transformData.data.artists;
     }
-    protected async handleAlbumArtists(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise<string[] | undefined> {
+    protected async handleAlbumArtists(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise<ArtistCredit[] | undefined> {
         return play.data.albumArtists;
     }
     protected async handleAlbum(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise<string | undefined> {
@@ -205,7 +206,7 @@ export const nativeParse = (play: PlayObject, options?: NativeTransformerDataStr
             logger = new MaybeLogger()
         } = options || {};
 
-        let artists = [];
+        let artists: ArtistCredit[] = [];
         let track = play.data.track;
 
         if(artistsParseFrom.includes('artists')) {
@@ -214,18 +215,18 @@ export const nativeParse = (play: PlayObject, options?: NativeTransformerDataStr
 
                 for(const artist of play.data.artists) {
                 
-                    const matchedIgnoreArtists = ignoreArtistsRegex.map(x => ({reg: x.toString(), res: parseRegexSingle(x, artist)})).filter(x => x.res !== undefined);
+                    const matchedIgnoreArtists = ignoreArtistsRegex.map(x => ({reg: x.toString(), res: parseRegexSingle(x, artist.name)})).filter(x => x.res !== undefined);
                     if(matchedIgnoreArtists.length > 0) {
                         logger.debug(`Will not parse artist because it matched an ignore regex:\n${matchedIgnoreArtists.map(x => `Reg: ${x.reg} => ${x.res.match}`).join('\n')}`);
                         artists.push(artist);
                     } else {
-                        const artistCredits = parseArtistCredits(artist, delimiters);
+                        const artistCredits = parseArtistCredits(artist.name, delimiters);
                         if (artistCredits !== undefined) {
                             if (artistCredits.primary !== undefined) {
-                                artists.push(artistCredits.primary);
+                                artists.push({name: artistCredits.primary});
                             }
                             if (artistCredits.secondary !== undefined) {
-                                artists = artists.concat(artistCredits.secondary);
+                                artists = artists.concat(artistCredits.secondary.map(artistNameToCredit));
                             }
                         } else {
                             // couldn't parse anything from artist string, use as-is
@@ -246,14 +247,14 @@ export const nativeParse = (play: PlayObject, options?: NativeTransformerDataStr
         if(artistsParseFrom.includes('title')) {
             const trackArtists = parseTrackCredits(play.data.track, delimiters);
             if (trackArtists !== undefined && trackArtists.secondary !== undefined) {
-                artists = artists.concat(trackArtists.secondary);
+                artists = artists.concat(trackArtists.secondary.map(artistNameToCredit));
                 if(titleClean) {
                     track = trackArtists.primary;
                 }
             }
         }
 
-        artists = uniqueNormalizedStrArr([...artists]);
+        artists = (uniqueNormalizedStrArr([...artists.map(artistCreditToName)])).map(artistNameToCredit);
 
         return {
             ...play,

src/backend/common/transforms/UserTransformer.ts

@@ -1,5 +1,5 @@
 import { searchAndReplace } from "@foxxmd/regex-buddy-core";
-import { PlayObject } from "../../../core/Atomic.js";
+import { ArtistCredit, PlayObject } from "../../../core/Atomic.js";
 import { configValToSearchReplace, isSearchAndReplaceTerm, isUserStage, testWhenConditions } from "../../utils/PlayTransformUtils.js";
 import { WebhookPayload } from "../infrastructure/config/health/webhooks.js";
 import { ConditionalSearchAndReplaceRegExp, PlayTransformUserStage, StageConfig } from "../infrastructure/Transform.js";
@@ -74,30 +74,30 @@ export default class UserTransformer extends AtomicPartsTransformer<ConditionalS
         }
         return result.trim();
     }
-    protected async handleArtists(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise<string[] | undefined> {
+    protected async handleArtists(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise<ArtistCredit[] | undefined> {
         if(play.data.artists === undefined || play.data.artists.length === 0) {
             return play.data.artists;
         }
         const mapper = this.generateMapper(play);
         const transformedArtists = [];
         for(const artist of play.data.artists) {
-            const a = searchAndReplace(artist, parts.map(mapper));
+            const a = searchAndReplace(artist.name, parts.map(mapper));
             if(a.trim() !== '') {
-                transformedArtists.push(a);
+                transformedArtists.push({...artist, name: a});
             }
         }
         return transformedArtists;
     }
-    protected async handleAlbumArtists(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise<string[] | undefined> {
+    protected async handleAlbumArtists(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise<ArtistCredit[] | undefined> {
         if(play.data.albumArtists === undefined || play.data.albumArtists.length === 0) {
             return play.data.albumArtists;
         }
         const mapper = this.generateMapper(play);
         const transformedArtists = [];
         for(const artist of play.data.albumArtists) {
-            const a = searchAndReplace(artist, parts.map(mapper));
+            const a = searchAndReplace(artist.name, parts.map(mapper));
             if(a.trim() !== '') {
-                transformedArtists.push(a);
+                transformedArtists.push({...artist, name: a});
             }
         }
         return transformedArtists;

src/backend/common/vendor/KodiApiClient.ts

@@ -8,6 +8,7 @@ import { AbstractApiOptions, FormatPlayObjectOptions } from "../infrastructure/A
 import { KodiData } from "../infrastructure/config/source/kodi.js";
 import AbstractApiClient from "./AbstractApiClient.js";
 import { baseFormatPlayObj } from "../../utils/PlayTransformUtils.js";
+import { artistNamesToCredits } from "../../../core/StringUtils.js";
 
 interface KodiDuration {
     hours: number
@@ -109,8 +110,8 @@ export class KodiApiClient extends AbstractApiClient {
             data: {
                 track: title,
                 album: album,
-                albumArtists: albumartist,
-                artists,
+                albumArtists: artistNamesToCredits(albumartist),
+                artists: artistNamesToCredits(artistVal),
                 duration,
                 playDate: dayjs()
             },

src/backend/common/vendor/LastfmApiClient.ts

@@ -1,6 +1,6 @@
 import dayjs, { Dayjs, ManipulateType } from "dayjs";
 import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, UnixTimestamp, URLData, Writeable } from "../../../core/Atomic.js";
-import { nonEmptyStringOrDefault, splitByFirstFound } from "../../../core/StringUtils.js";
+import { artistNamesToCredits, artistNameToCredit, nonEmptyStringOrDefault, splitByFirstFound } from "../../../core/StringUtils.js";
 import { removeUndefinedKeys, sleep } from "../../utils.js";
 import { writeFile } from '../../utils/FSUtils.js';
 import { objectIsEmpty, readJson } from '../../utils/DataUtils.js';
@@ -18,6 +18,7 @@ import { IncomingMessage } from "http";
 import { baseFormatPlayObj } from "../../utils/PlayTransformUtils.js";
 import { ScrobbleSubmitError, SimpleError } from "../errors/MSErrors.js";
 import { redactString } from "@foxxmd/redact-string";
+import dns from 'node:dns/promises';
 
 const badErrors = [
     'api key suspended',
@@ -226,7 +227,11 @@ export default class LastfmApiClient extends AbstractApiClient implements Pagina
 
     testConnection = async() => {
         try {
-            await isPortReachableConnect(this.url.port, { host: this.url.url.hostname });
+            this.logger.trace(`Looking up IP for ${this.url.url.hostname}`);
+            const resolved = await dns.lookup(this.url.url.hostname);
+            this.logger.trace(`${this.url.url.hostname} resolved to ${resolved.address}`);
+            this.logger.trace(`Checking if ${this.url.url.hostname}:${this.url.port} is reachable...`);
+            await isPortReachableConnect(this.url.port, { host: this.url.url.hostname, timeout: 2000 });
             this.logger.verbose(`${this.url.url.hostname}:${this.url.port} is reachable.`);
             return true;
         } catch (e) {
@@ -565,10 +570,10 @@ export const scrobblePayloadToPlay = (obj: LastFMScrobbleRequestPayload): PlayOb
         data: {
             track,
             album: nonEmptyStringOrDefault(album),
-            albumArtists: nonEmptyStringOrDefault(albumArtist) !== undefined ? [albumArtist] : undefined,
+            albumArtists: nonEmptyStringOrDefault(albumArtist) !== undefined ? [artistNameToCredit(albumArtist)] : undefined,
             duration: typeof duration === 'string' ? parseInt(duration, 10) : duration,
             playDate: ts,
-            artists
+            artists: artistNamesToCredits(artists)
         },
         meta: {
             source: 'lastfm',
@@ -610,7 +615,7 @@ export const playToClientPayload = (playObj: PlayObject): LastFMScrobblePayload
         if (artists.length === 0) {
             artist = "";
         } else {
-            artist = artists[0];
+            artist = artists[0].name;
         }
 
         const additionalRichPayload: Partial<LastFMScrobblePayload> = {};
@@ -629,11 +634,11 @@ export const playToClientPayload = (playObj: PlayObject): LastFMScrobblePayload
 
         // LFM ignores scrobbles where album artist is VA
         // https://github.com/FoxxMD/multi-scrobbler/issues/340#issuecomment-3220774257
-        const nonVaAlbumArtists = albumArtists.filter(x => x.trim().toLocaleLowerCase() !== 'va');
+        const nonVaAlbumArtists = albumArtists.filter(x => x.name.trim().toLocaleLowerCase() !== 'va');
         // LFM does not support multiple artists in scrobble payload
         // https://www.last.fm/api/show/track.scrobble
         if (nonVaAlbumArtists.length > 0) {
-            rawPayload.albumArtist = nonVaAlbumArtists[0];
+            rawPayload.albumArtist = nonVaAlbumArtists[0].name;
         }
 
         // I don't know if its lastfm-node-client building the request params incorrectly
@@ -689,7 +694,7 @@ export const formatPlayObj = (obj: LastFMTrackObject, options: FormatPlayObjectO
 
     const play: PlayObjectLifecycleless = {
         data: {
-            artists: [...new Set(artistStrings)] as string[],
+            artists: artistNamesToCredits([...new Set(artistStrings)] as string[]),
             track: title,
             album: al,
             duration,

src/backend/common/vendor/ListenbrainzApiClient.ts

@@ -2,7 +2,7 @@ import { stringSameness } from '@foxxmd/string-sameness';
 import dayjs from "dayjs";
 import request, { Request, Response } from 'superagent';
 import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, UnixTimestamp, URLData } from "../../../core/Atomic.js";
-import { combinePartsToString, slice } from "../../../core/StringUtils.js";
+import { artistNamesToCredits, combinePartsToString, slice } from "../../../core/StringUtils.js";
 import {
     normalizeListenbrainzUrl,
     normalizeStr,
@@ -548,7 +548,13 @@ export const listenResponseToPlay = (listen: ListenResponse): PlayObject => {
                     ...naivePlay,
                     data: {
                         ...naivePlay.data,
-                        artists: derivedArtists
+                        artists: derivedArtists.map(x => {
+                            const mappedArtist = artistMappings.find(y => y.artist_credit_name === x);
+                            if(mappedArtist !== undefined) {
+                                return {name: mappedArtist.artist_credit_name, mbid: mappedArtist.artist_mbid}
+                            }
+                            return {name: x};
+                        })
                     }
                 };
             }
@@ -579,7 +585,13 @@ export const listenResponseToPlay = (listen: ListenResponse): PlayObject => {
             data: {
                 ...naivePlay.data,
                 track: normalTrackName,
-                artists: derivedArtists
+                artists: derivedArtists.map(x => {
+                    const mappedArtist = artistMappings.find(y => y.artist_credit_name === x);
+                    if(mappedArtist !== undefined) {
+                        return {name: mappedArtist.artist_credit_name, mbid: mappedArtist.artist_mbid}
+                    }
+                    return {name: x};
+                })
             },
             meta: naivePlay.meta
         }
@@ -641,7 +653,7 @@ export const listenToNaivePlay = (listen: ListenResponse): PlayObject => {
 
         if(artist_names.length > 0) {
             artists = artist_names;
-        } else {
+        } else if(artist_name !== null) {
             artists = [artist_name];
 
             // since we aren't using MB mappings we should be conservative and assume artist string with & are proper names (not joiner)
@@ -675,9 +687,9 @@ export const listenToNaivePlay = (listen: ListenResponse): PlayObject => {
             data: {
                 playDate: dayjs.unix(listened_at),
                 track: normalTrackName,
-                artists: artists,
+                artists: artistNamesToCredits(artists),
                 album: release_name,
-                albumArtists,
+                albumArtists: artistNamesToCredits(albumArtists),
                 duration: dur,
                 isrc: isrc !== undefined ? isrc : undefined,
                 meta: {

src/backend/common/vendor/RockSkyApiClient.ts

@@ -1,7 +1,7 @@
 import dayjs from "dayjs";
 import request, { Request, Response } from 'superagent';
 import { PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, URLData } from "../../../core/Atomic.js";
-import { nonEmptyStringOrDefault } from "../../../core/StringUtils.js";
+import { artistNamesToCredits, nonEmptyStringOrDefault } from "../../../core/StringUtils.js";
 import { UpstreamError } from "../errors/UpstreamError.js";
 import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions } from "../infrastructure/Atomic.js";
 import { RockSkyClientData, RockSkyData, RockSkyOptions } from "../infrastructure/config/client/rocksky.js";
@@ -221,8 +221,8 @@ export const rockskyScrobbleToPlay = (obj: RockskyScrobble): PlayObject => {
     const play: PlayObjectLifecycleless = {
         data: {
             track: obj.title,
-            artists: nonEmptyStringOrDefault(obj.artist) ? [obj.artist] : [],
-            albumArtists: nonEmptyStringOrDefault(obj.albumArtist) ? [obj.albumArtist] : [],
+            artists: artistNamesToCredits(nonEmptyStringOrDefault(obj.artist) ? [obj.artist] : []),
+            albumArtists: artistNamesToCredits(nonEmptyStringOrDefault(obj.albumArtist) ? [obj.albumArtist] : []),
             album: nonEmptyStringOrDefault(obj.album),
             playDate: dayjs.utc(obj.createdAt).local()
         },

src/backend/common/vendor/bluesky/AbstractBlueSkyApiClient.ts

@@ -94,7 +94,7 @@ export const playToRecord = (play: PlayObject): ScrobbleRecord => {
     const record: ScrobbleRecord = {
         $type: "fm.teal.alpha.feed.play",
         trackName: play.data.track,
-        artists: play.data.artists.map(x => ({ artistName: x })),
+        artists: play.data.artists.map(x => ({ artistName: x.name })),
         duration: Math.round(play.data.duration),
         playedTime: getScrobbleTsSOCDateWithContext(play)[0].toISOString(),
         releaseName: play.data.album,
@@ -124,7 +124,7 @@ export const recordToPlay = (record: ScrobbleRecord, options: RecordOptions = {}
     const play: PlayObjectLifecycleless = {
         data: {
             track: record.trackName,
-            artists: record.artists.filter(x => x.artistName !== undefined).map(x => x.artistName),
+            artists: record.artists.filter(x => x.artistName !== undefined).map(x => ({name: x.artistName, mbid: x.artistMbId})),
             duration: record.duration,
             playDate: dayjs(record.playedTime),
             album: record.releaseName,

src/backend/common/vendor/discord/DiscordUtils.ts

@@ -46,7 +46,7 @@ export const playStateToActivityData = (data: SourcePlayerObj, opts: { useArt?:
         name: activityName,
         
         details: play.data.track.padEnd(2,'\u200B'),
-        state: play.data.artists !== undefined && play.data.artists.length > 0 ? play.data.artists.map(x => x.padEnd(2, '\u200B')).join(' / ') : undefined,
+        state: play.data.artists !== undefined && play.data.artists.length > 0 ? play.data.artists.map(x => x.name.padEnd(2, '\u200B')).join(' / ') : undefined,
         // https://docs.discord.com/developers/events/gateway-events#activity-object-activity-assets
         // https://docs.discord.com/developers/events/gateway-events#activity-object-activity-asset-image
         assets: {

src/backend/common/vendor/koito/KoitoApiClient.ts

@@ -9,10 +9,11 @@ import { UpstreamError } from "../../errors/UpstreamError.js";
 import { playToListenPayload } from '../listenbrainz/lzUtils.js';
 import { SubmitPayload } from '../listenbrainz/interfaces.js';
 import { ListenType } from '../listenbrainz/interfaces.js';
-import { parseRegexSingleOrFail } from "../../../utils.js";
 import { baseFormatPlayObj } from "../../../utils/PlayTransformUtils.js";
 import { ScrobbleSubmitError } from "../../errors/MSErrors.js";
 import { tryApiCall } from "../../../utils/RequestUtils.js";
+import { parseRegexSingle } from "@foxxmd/regex-buddy-core";
+import { artistNamesToCredits } from "../../../../core/StringUtils.js";
 
 interface SubmitOptions {
     log?: boolean
@@ -36,7 +37,7 @@ export class KoitoApiClient extends AbstractApiClient implements PaginatedTimeRa
         const u = normalizeWebAddress(url);
         if(u.url.pathname === '/') {
             this.url = u;
-        } else if(parseRegexSingleOrFail(KOITO_LZ_PATH, u.url.pathname) !== undefined) {
+        } else if(parseRegexSingle(KOITO_LZ_PATH, u.url.pathname) !== undefined) {
            this.logger.verbose('Detected Koito Server URL path only contains listenbrainz prefix. Removing this for API calls so non-listenbrainz paths work correctly.');
            this.url = normalizeWebAddress(getBaseFromUrl(u.url).toString());
         } else {
@@ -218,7 +219,7 @@ export const listenObjectResponseToPlay = (obj: ListenObjectResponse, options: {
     const play: PlayObjectLifecycleless = {
         data: {
             track: obj.track.title,
-            artists: (obj.track.artists ?? []).map(x => x.name),
+            artists: artistNamesToCredits((obj.track.artists ?? []).map(x => x.name)),
             duration: obj.track.duration,
             playDate: dayjs(obj.time)
         },

src/backend/common/vendor/listenbrainz/lzUtils.ts

@@ -4,6 +4,7 @@ import { getScrobbleTsSOCDate } from "../../../utils/TimeUtils.js";
 import { SubmitOptions } from "../ListenbrainzApiClient.js";
 import { ListenPayload, MinimumTrack, SubmitListenAdditionalTrackInfo, SubmitPayload } from "./interfaces.js";
 import {version as appVersion } from '../../../version.js';
+import { artistCreditsToNames, artistCreditToName } from "../../../../core/StringUtils.js";
 
 export const playToListenPayload = (play: PlayObject, version?: string): ListenPayload => {
     const {
@@ -25,10 +26,10 @@ export const playToListenPayload = (play: PlayObject, version?: string): ListenP
 
     let addInfo: SubmitListenAdditionalTrackInfo = {
         // primary artists
-        artist_names: Array.from(new Set([...artists])),
+        artist_names: Array.from(new Set([...artists.map(artistCreditToName)])),
         // primary artist
-        release_artist_name: albumArtists.length === 1 ? albumArtists[0] : undefined,
-        release_artist_names: albumArtists.length > 0 ? albumArtists : undefined,
+        release_artist_name: albumArtists.length === 1 ? albumArtists[0].name : undefined,
+        release_artist_names: albumArtists.length > 0 ? artistCreditsToNames(albumArtists) : undefined,
         // use data from LZ response, if this Play was originally from LZ Source
         media_player: mediaPlayerName ?? msAdditionalInfo.media_player,
         media_player_version: mediaPlayerVersion ?? msAdditionalInfo.media_player_version,

src/backend/common/vendor/maloja/MalojaApiClient.ts

@@ -12,7 +12,7 @@ import { getNonEmptyVal, parseRetryAfterSecsFromObj, removeUndefinedKeys, sleep
 import { UpstreamError } from "../../errors/UpstreamError.js";
 import { getMalojaResponseError, isMalojaAPIErrorBody, MalojaResponseV3CommonData, MalojaScrobbleData, MalojaScrobbleRequestData, MalojaScrobbleV3RequestData, MalojaScrobbleV3ResponseData, MalojaScrobbleWarning } from "./interfaces.js";
 import { getScrobbleTsSOCDate, getScrobbleTsSOCDateWithContext } from '../../../utils/TimeUtils.js';
-import { buildTrackString } from '../../../../core/StringUtils.js';
+import { artistCreditsToNames, artistNamesToCredits, buildTrackString } from '../../../../core/StringUtils.js';
 import { baseFormatPlayObj } from '../../../utils/PlayTransformUtils.js';
 import { ScrobbleSubmitError } from '../../errors/MSErrors.js';
 import { NO_RETRY_HTTP_STATUS, tryApiCall } from '../../../utils/RequestUtils.js';
@@ -272,7 +272,7 @@ export class MalojaApiClient extends AbstractApiClient implements PaginatedTimeR
                         } = track;
                         scrobbleResponse.track.album = {
                             name: album,
-                            artists: albumArtists,
+                            artists: artistCreditsToNames(albumArtists),
                             ...malojaAlbum,
                         }
                     }
@@ -412,7 +412,7 @@ export const formatPlayObj = (obj: MalojaScrobbleData, options: FormatPlayObject
     const urlParams = new URLSearchParams([['artist', artists[0]], ['title', title]]);
     const play: PlayObjectLifecycleless = {
         data: removeUndefinedKeys({
-            artists: [...new Set(artistStrings)] as string[],
+            artists: artistNamesToCredits([...new Set(artistStrings)] as string[]),
             track: title,
             album,
             duration,
@@ -446,7 +446,7 @@ export const playToScrobblePayload = (playObj: PlayObject, apiKey?: string): Mal
 
     const scrobbleData: MalojaScrobbleV3RequestData = {
         title: track,
-        artists,
+        artists: artistCreditsToNames(artists),
         album,
         key: apiKey,
         time: pd.unix(),
@@ -463,7 +463,7 @@ export const playToScrobblePayload = (playObj: PlayObject, apiKey?: string): Mal
     // https://github.com/krateng/maloja/blob/master/maloja/web/static/js/manualscrobble.js#L136
     // BUT this is not actually working!
     if (albumArtists.length > 0) {
-        scrobbleData.albumartists = albumArtists;
+        scrobbleData.albumartists = artistCreditsToNames(albumArtists);
     }
     // see also https://github.com/krateng/maloja/issues/96#issuecomment-1490562761
     // https://github.com/FoxxMD/multi-scrobbler/issues/454#issuecomment-3806367420

src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts

@@ -1,5 +1,5 @@
 import { Response } from 'superagent';
-import { PlayObject, PlayObjectLifecycleless, URLData } from "../../../../core/Atomic.js";
+import { ArtistCredit, PlayObject, PlayObjectLifecycleless, URLData } from "../../../../core/Atomic.js";
 import { UpstreamError } from "../../errors/UpstreamError.js";
 import { AbstractApiOptions, FormatPlayObjectOptions, MUSICBRAINZ_URL, MusicbrainzApiConfigData } from "../../infrastructure/Atomic.js";
 import AbstractApiClient from "../AbstractApiClient.js";
@@ -21,6 +21,7 @@ import { SimpleError } from '../../errors/MSErrors.js';
 import { baseFormatPlayObj } from '../../../utils/PlayTransformUtils.js';
 import { IRecordingMSList } from '../../transforms/MusicbrainzTransformer.js';
 import dayjs, { Dayjs } from 'dayjs';
+import { artistCreditsToNames } from '../../../../core/StringUtils.js';
 
 export interface SubmitResponse {
     payload?: {
@@ -235,7 +236,16 @@ export class MusicbrainzApiClient extends AbstractApiClient {
         // https://wiki.musicbrainz.org/MusicBrainz_API/Search#Recording
         // https://beta.musicbrainz.org/doc/MusicBrainz_API/Search
         const res = await this.callApi<IRecordingList>((mb) => {
-            const query: Record<string, any> = {
+            const query: {
+                recording_mbid?: string
+                track_mbid?: string
+                release_mbid?: string
+                artist_mbids?: string[]
+                isrc?: string
+                recording?: string
+                artist?: string[]
+                release?: string
+            } = {
             };
 
             if(play.data?.meta?.brainz?.recording !== undefined && using.includes('mbidrecording')) {
@@ -257,7 +267,7 @@ export class MusicbrainzApiClient extends AbstractApiClient {
                 query.recording = play.data.track;
             }
             if(play.data.artists !== undefined && play.data.artists.length > 0 && using.includes('artist')) {
-                query.artist = play.data.artists;
+                query.artist = artistCreditsToNames(play.data.artists);
             }
             if(play.data.album !== undefined && using.includes('album')) {
                 query.release = play.data.album;
@@ -380,17 +390,17 @@ export const recordingToPlay = (data: IRecording, options?: {ignoreVA?: boolean}
 
     let album: IRelease;
 
-    let albumArtists: string[];
+    let albumArtists: ArtistCredit[];
     let albumArtistIds: string[];
-    const artists = (data["artist-credit"] ?? []).map(x => x.name);
+    const artists = (data["artist-credit"] ?? []).map(x => ({ name: x.name, mbid: x.artist.id}));
     if(data.releases !== undefined && data.releases.length > 0) {
         album = data.releases[0];
         if(album["artist-credit"] !== undefined) {
             if(difference(album["artist-credit"].map(x => x.artist.id), (data["artist-credit"] ?? []).map(x => x.artist.id)).length > 0) {
-                albumArtists = album["artist-credit"].map(x => x.artist.name);
+                albumArtists = album["artist-credit"].map(x => ({name: x.artist.name, mbid: x.artist.id}));
                 albumArtistIds = album["artist-credit"].map(x => x.artist.id);
             }
-            if(albumArtists !== undefined && ignoreVA && albumArtists.includes('Various Artists')) {
+            if(albumArtists !== undefined && ignoreVA && albumArtists.map(x => x.name).includes('Various Artists')) {
                 albumArtists = undefined;
                 albumArtistIds = undefined;
             }

src/backend/index.ts

@@ -16,14 +16,18 @@ import { appLogger, initLogger as getInitLogger } from "./common/logging.js";
 import { getRoot } from "./ioc.js";
 import { parseVersion } from "./version.js";
 import { initServer } from "./server/index.js";
-import { createHeartbeatClientsTask } from "./tasks/heartbeatClients.js";
-import { createHeartbeatSourcesTask } from "./tasks/heartbeatSources.js";
-import { isDebugMode, parseBool, retry } from "./utils.js";
+import { isDebugMode, parseBool, parseBoolStrict, retry, sleep } from "./utils.js";
 import { readJson } from './utils/DataUtils.js';
-//import { createVegaGenerator } from './utils/SchemaUtils.js';
 import ScrobbleClients from './scrobblers/ScrobbleClients.js';
 import ScrobbleSources from './sources/ScrobbleSources.js';
 import { Notifiers } from './notifier/Notifiers.js';
+import { DbConcrete, getMigratedDb } from './common/database/drizzle/drizzleUtils.js';
+import { getDbPath } from './common/database/Database.js';
+import { createRetentionCleanupTask } from './tasks/retentionCleanup.js';
+import { parseUserConfig } from './common/Cache.js';
+import { nonEmptyStringOrDefault } from '../core/StringUtils.js';
+import { DbExternalMode } from './common/infrastructure/Atomic.js';
+import { PGLiteSocketServer } from '@electric-sql/pglite-socket';
 
 dayjs.extend(utc)
 dayjs.extend(isBetween);
@@ -49,13 +53,48 @@ output = output.slice(0, 301);
 
 let logger: FoxLogger;
 
-process.on('uncaughtExceptionMonitor', (err, origin) => {
+let server: PGLiteSocketServer;
+let db: DbConcrete;
+let dbConnectionsClosed = false;
+
+process.on('uncaughtExceptionMonitor', async (err, origin) => {
     const appError = new Error(`Uncaught exception is crashing the app! :( Type: ${origin}`, {cause: err});
     if(logger !== undefined) {
         logger.error(appError)
     } else {
         initLogger.error(appError);
     }
+    if(!dbConnectionsClosed) {
+        const parts = [];
+        if(server !== undefined) {
+            await server.stop();
+            parts.push('PGLite Socket Server');
+        }
+        if(db !== undefined && !db.$client.closed) {
+            await db.$client.close();
+            parts.push('Database');
+        }
+        if(parts.length > 0 && logger !== undefined) {
+            logger.info(`Closed ${parts.join(' and ')}`);
+        }
+    }
+});
+process.on('SIGINT', async () => {
+    if(!dbConnectionsClosed) {
+        const parts = [];
+        if(server !== undefined) {
+            await server.stop();
+            parts.push('PGLite Socket Server');
+        }
+        if(db !== undefined && !db.$client.closed) {
+            await db.$client.close();
+            parts.push('Database');
+        }
+        if(parts.length > 0 && logger !== undefined) {
+            logger.info(`Closed ${parts.join(' and ')}`);
+        }
+    }
+    process.exit(0);
 })
 
 const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
@@ -75,6 +114,7 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`)
             webhooks = [],
             logging = {},
             debugMode,
+            cache,
         } = (config || {}) as AIOConfig;
 
         if (process.env.DEBUG_MODE === undefined && debugMode !== undefined) {
@@ -88,90 +128,164 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`)
 
         initLogger.info(`Debug Mode: ${isDebugMode() ? 'YES' : 'NO'}`);
 
-        await parseVersion();
+        const version = await parseVersion();
+
+        initLogger.info(`Version: ${version}`);
 
         const [aLogger, appLoggerStream] = await appLogger(logging)
         logger = childLogger(aLogger, 'App');
 
-        const root = getRoot({...config, logger, loggingConfig: logging, loggerStream: appLoggerStream});
-        initLogger.info(`Version: ${root.get('version')}`);
+        const dbModeVal: string = nonEmptyStringOrDefault(process.env.DB_MODE, undefined);
+        let dbMode: DbExternalMode;
+        if(dbModeVal !== undefined) {
+            if(['none','live','standalone'].includes(dbModeVal.toLocaleLowerCase())) {
+                dbMode = dbModeVal as typeof dbMode;
+            } else {
+                throw new Error(`DB_MODE env must be one of 'none' 'live' 'standalone', found ${dbModeVal}`);
+            }
+        } else {
+            dbMode = 'none';
+        }
+        logger.info(`DB External Mode: ${dbMode}`);
 
-        //initLogger.info('Generating schema definitions...');
-        //createVegaGenerator()
-        //initLogger.info('Schema definitions generated');
+        
+        const dbPath = getDbPath('msDb');
+        logger.info(`Using database at ${getDbPath('msDb')}`);
+        const [migratedDb, isNew] = await getMigratedDb(dbPath, {logger: childLogger(logger, 'DB')});
+        db = migratedDb;
 
-        const internalConfigOptional = {
-             localUrl: root.get('localUrl'),
-            configDir: root.get('configDir'),
-             version: root.get('version')
-             };
+        
 
-        const scrobbleClients = new ScrobbleClients(root.get('clientEmitter'), root.get('sourceEmitter'), internalConfigOptional, root.get('logger'));
-        const scrobbleSources = new ScrobbleSources(root.get('sourceEmitter'), internalConfigOptional, root.get('logger'));
-
-        await root.items.cache().init(true);
-
-        initServer(logger, appLoggerStream, output, scrobbleSources, scrobbleClients);
-
-        if(process.env.IS_LOCAL === 'true') {
-            logger.info('multi-scrobbler can be run as a background service! See: https://docs.multi-scrobbler.app/installation/service');
+        if(['live','standalone'].includes(dbMode)) {
+            server = new PGLiteSocketServer({
+                host: '0.0.0.0',
+                port: 5433,
+                db: db.$client,
+                maxConnections: 10,
+                debug: parseBoolStrict(nonEmptyStringOrDefault(process.env.DB_DEBUG, false))
+            });
+            await server.start();
+            logger.info('Started PGLite Socket Server');
         }
 
-        if(appConfigFail !== undefined) {
-            logger.warn('App config file exists but could not be parsed!');
-            logger.warn(appConfigFail);
-        }
+        if (dbMode === 'standalone') {
+            logger.info('MS App startup stopped early due to Standalone DB Mode.');
+        } else {
 
-        const notifiers = new Notifiers(root.get('notifierEmitter'), root.get('clientEmitter'), root.get('sourceEmitter'), root.get('logger')); //root.get('notifiers');
-        await notifiers.buildWebhooks(webhooks);
+            const root = getRoot({
+                ...config,
+                cache: parseUserConfig(cache, logger),
+                logger,
+                loggingConfig: logging,
+                loggerStream: appLoggerStream,
+                db
+            });
 
-        await root.items.transformerManager.registerFromEnv();
-        await root.items.transformerManager.registeryDefaults();
-        await root.items.transformerManager.initTransformers();
+            const internalConfigOptional = {
+                localUrl: root.get('localUrl'),
+                configDir: root.get('configDir'),
+                version: root.get('version')
+            };
 
-        /*
-        * setup clients
-        * */
-        await scrobbleClients.buildClientsFromConfig(notifiers);
-        /*
-        * setup sources
-        * */
-        await scrobbleSources.buildSourcesFromConfig([]);
+            const scrobbleClients = new ScrobbleClients(root.get('clientEmitter'), root.get('sourceEmitter'), internalConfigOptional, root.get('logger'));
+            const scrobbleSources = new ScrobbleSources(root.get('sourceEmitter'), internalConfigOptional, root.get('logger'));
 
-        // check ambiguous client/source types like this for now
-        const lastfmSources = scrobbleSources.getByType('lastfm');
-        const lastfmScrobbles = scrobbleClients.getByType('lastfm');
+            await root.items.cache().init(true);
 
-        const scrobblerNames = lastfmScrobbles.map(x => x.name);
-        const nameColl = lastfmSources.filter(x => scrobblerNames.includes(x.name));
-        if(nameColl.length > 0) {
-            logger.warn(`Last.FM source and clients have same names [${nameColl.map(x => x.name).join(',')}] -- this may cause issues`);
-        }
+            initServer(logger, appLoggerStream, output, scrobbleSources, scrobbleClients);
 
-        const clientTask = createHeartbeatClientsTask(scrobbleClients, logger);
-        clientTask.execute();
-        try {
-            await retry(() => {
-                if(clientTask.isExecuting) {
-                    throw new Error('Waiting')
+            if (process.env.IS_LOCAL === 'true') {
+                logger.info('multi-scrobbler can be run as a background service! See: https://docs.multi-scrobbler.app/installation/service');
+            }
+
+            if (appConfigFail !== undefined) {
+                logger.warn('App config file exists but could not be parsed!');
+                logger.warn(appConfigFail);
+            }
+
+            const notifiers = new Notifiers(root.get('notifierEmitter'), root.get('clientEmitter'), root.get('sourceEmitter'), root.get('logger')); //root.get('notifiers');
+            await notifiers.buildWebhooks(webhooks);
+
+            await root.items.transformerManager.registerFromEnv();
+            await root.items.transformerManager.registeryDefaults();
+            await root.items.transformerManager.initTransformers();
+
+            /*
+            * setup clients
+            * */
+            await scrobbleClients.buildClientsFromConfig(notifiers);
+            /*
+            * setup sources
+            * */
+            await scrobbleSources.buildSourcesFromConfig([]);
+
+            // check ambiguous client/source types like this for now
+            const lastfmSources = scrobbleSources.getByType('lastfm');
+            const lastfmScrobbles = scrobbleClients.getByType('lastfm');
+
+            const scrobblerNames = lastfmScrobbles.map(x => x.name);
+            const nameColl = lastfmSources.filter(x => scrobblerNames.includes(x.name));
+            if (nameColl.length > 0) {
+                logger.warn(`Last.FM source and clients have same names [${nameColl.map(x => x.name).join(',')}] -- this may cause issues`);
+            }
+            const clientInitOptions = { deadDelay: nonEmptyStringOrDefault(process.env.DEBUG_DEAD_DELAY, undefined) !== undefined ? Number.parseInt(process.env.DEBUG_DEAD_DELAY) : undefined }; for (const c of scrobbleClients.clients) {
+                c.initTasks(clientInitOptions);
+                const res = await Promise.race([
+                    sleep(2200),
+                    (async () => {
+                        while (!c.isReady()) {
+                            await sleep(400)
+                        }
+                        return true;
+                    })()
+                ]);
+                if (res === undefined) {
+                    logger.debug(`Not waiting for Client ${c.name} to finish init, moving on to the next Client...`);
                 }
-                return true;
-            },{retries: scrobbleClients.clients.length + 1, retryIntervalMs: 2000});
-        } catch (e) {
-            logger.warn('Waited too long for clients to start! Moving ahead with sources init...');
+            }
+
+            for (const c of scrobbleSources.sources) {
+                c.initTasks();
+                const res = await Promise.race([
+                    sleep(2200),
+                    (async () => {
+                        while (!c.isReady()) {
+                            await sleep(400)
+                        }
+                        return true;
+                    })()
+                ]);
+                if (res === undefined) {
+                    logger.debug(`Not waiting for Source ${c.name} to finish init, moving on to the next Source...`);
+                }
+            }
+
+            let runRetentionNow = parseBool(process.env.RETENTION_IMMEDIATE, false);
+
+            const retentionTask = createRetentionCleanupTask(scrobbleSources, scrobbleClients, logger);
+            let retentionJobAdded = false;
+            const addJob = () => {
+                retentionJobAdded = true;
+                scheduler.addSimpleIntervalJob(new SimpleIntervalJob({
+                    minutes: 60,
+                    runImmediately: runRetentionNow
+                }, retentionTask, { id: 'retention', preventOverrun: true }));
+                logger.debug('Added Retention Cleanup task to scheduler');
+            };
+            logger.debug('Added Client Heartbeat task to scheduler');
+
+            if (runRetentionNow === false || (scrobbleClients.clients.every(x => x.isReady()) && scrobbleSources.sources.every(x => x.isReady()))) {
+                addJob();
+            }
+
+            logger.info('Scheduler started.');
+
+            if (runRetentionNow === true && !retentionJobAdded) {
+                logger.info('Detected that Retention Cleanup should run immediately but all sources/clients have not started yet! Delaying retention cleanup by 1 minute to allow all sources/clients to finish starting.');
+                await sleep(60 * 1000);
+                addJob();
+            }
         }
-        scheduler.addSimpleIntervalJob(new SimpleIntervalJob({
-            minutes: 20,
-            runImmediately: false
-        }, clientTask, {id: 'clients_heart'}));
-
-        const sourceTask = createHeartbeatSourcesTask(scrobbleSources, logger);
-        scheduler.addSimpleIntervalJob(new SimpleIntervalJob({
-            minutes: 20,
-            runImmediately: true
-        }, sourceTask, {id: 'sources_heart'}));
-
-        logger.info('Scheduler started.');
 
     } catch (e) {
         const appError = new Error('Exited with uncaught error', {cause: e});
@@ -180,6 +294,20 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`)
         } else {
             initLogger.error(appError);
         }
+        if(!dbConnectionsClosed) {
+            const parts = [];
+            if(server !== undefined) {
+                await server.stop();
+                parts.push('PGLite Socket Server');
+            }
+            if(db !== undefined && !db.$client.closed) {
+                await db.$client.close();
+                parts.push('Database');
+            }
+            if(parts.length > 0 && logger !== undefined) {
+                logger.info(`Closed ${parts.join(' and ')}`);
+            }
+        }
         process.exit(1);
     }
 }());

src/backend/ioc.ts

@@ -15,6 +15,7 @@ import prom, { Counter, Gauge } from 'prom-client';
 import { CoverArtApiClient } from "./common/vendor/musicbrainz/CoverArtApiClient.js";
 import { version } from "./version.js";
 import { StaggerOptions } from "./utils/AsyncUtils.js";
+import { DbConcrete } from "./common/database/drizzle/drizzleUtils.js";
 
 let root: ReturnType<typeof createRoot>;
 export interface RootOptions {
@@ -27,6 +28,7 @@ export interface RootOptions {
     cache?: CacheConfigOptions | MSCache | (() => MSCache)
     mbMap?: MusicBrainzSingletonMap | (() => MusicBrainzSingletonMap)
     transformers?: TransformerCommonConfig[]
+    db?: DbConcrete | (() => Promise<DbConcrete>)
 }
 
 const discovered = new prom.Counter({
@@ -60,6 +62,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => {
         logger,
         cache,
         mbMap,
+        db,
         transformers = []
     } = options || {};
     const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
@@ -89,6 +92,12 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => {
         maybeSingletonMb = new Map();
     }
 
+    let dbFunc: () => Promise<DbConcrete>;
+    if(typeof db === 'function') {
+        dbFunc = db;
+    } else {
+        dbFunc = async () => db;
+    }
 
     const cEmitter = new WildcardEmitter();
     // do nothing, just catch
@@ -148,6 +157,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => {
         cache: () => maybeSingletonCache !== undefined ? () => maybeSingletonCache : cacheFunc,
         mbMap: () => maybeSingletonMb !== undefined ? () => maybeSingletonMb : mbFunc,
         coverArtApi,
+        db: () => dbFunc
     }).add((items) => {
         const localUrl = generateBaseURL(baseUrl, items.port)
         return {

src/backend/scrobblers/DiscordScrobbler.ts

@@ -1,4 +1,4 @@
-import { Logger } from "@foxxmd/logging";
+import { Logger, LogLevel } from "@foxxmd/logging";
 import EventEmitter from "events";
 import { PlayMatchResult, PlayObject, SourcePlayerObj } from "../../core/Atomic.js";
 import { CALCULATED_PLAYER_STATUSES, FormatPlayObjectOptions, REPORTED_PLAYER_STATUSES, ReportedPlayerStatus, SINGLE_USER_PLATFORM_ID_STR, TimeRangeListensFetcher } from "../common/infrastructure/Atomic.js";
@@ -120,7 +120,7 @@ export default class DiscordScrobbler extends AbstractScrobbleClient {
         // discord does not handle scrobbles, only Now Playing
         // so don't bother queueing any scrobbles as we don't want to cache them
         // or give the user the impression they are used (in UI as a number of queued scrobbles)
-        return;
+        return [];
     }
 
     alreadyScrobbled = async (playObj: PlayObject, log = false): Promise<[boolean, PlayMatchResult]> => ([false, {match: false, breakdowns: [], score: 0}])
@@ -156,34 +156,33 @@ export default class DiscordScrobbler extends AbstractScrobbleClient {
         }
     }
 
-    shouldUpdatePlayingNowPlatformSpecific = async (data: SourcePlayerObj) => {
+    shouldUpdatePlayingNowPlatformSpecific = async (data: SourcePlayerObj): Promise<[boolean, string?, LogLevel?]> => {
         if ([CALCULATED_PLAYER_STATUSES.stopped, CALCULATED_PLAYER_STATUSES.paused, CALCULATED_PLAYER_STATUSES.playing].includes(data.status.calculated as ReportedPlayerStatus)
             || (data.nowPlayingMode && !CALCULATED_PLAYER_STATUSES.stopped)
             || data.status.stale) {
 
             const [sendOk, reasons, level = 'warn'] = await this.api.checkOkToSend();
             if (!sendOk) {
-                this.npLogger[level](`Cannot update playing now because api client is ${reasons}`);
-                return false;
+                return [false, `Cannot update playing now because api client is ${reasons}`, level as LogLevel];
             }
 
             if(this.api instanceof DiscordWSClient) {
                 const [allowed, reason] = this.api.presenceIsAllowed();
                 if(!allowed) {
                     this.npLogger.debug(reason);
+                    return [false, reason];
                 }
-
-                return true;
             }
 
-            return true;
+            return [true];
         } else {
             if(!data.nowPlayingMode && ![CALCULATED_PLAYER_STATUSES.stopped, CALCULATED_PLAYER_STATUSES.paused, CALCULATED_PLAYER_STATUSES.playing].includes(data.status.calculated as ReportedPlayerStatus)) {
-                this.npLogger.trace(`Will not update because player is not in state: stopped | paused | playing => Found '${data.status.calculated }'`);
+                return [false,`player is not in state: stopped | paused | playing => Found '${data.status.calculated }'`];
             } else if(data.nowPlayingMode && CALCULATED_PLAYER_STATUSES.stopped) {
                 this.npLogger.trace(`Will not update because now playing player is stopped => Found ${data.status.calculated}`);
+                return [false,`playing player is stopped => Found ${data.status.calculated}` ]
             } else {
-                this.npLogger.trace('Will not update because now player is in an unexpected state for discord usage');
+                return [false, 'player is in an unexpected state for discord usage']
             }
         }
     }

src/backend/scrobblers/ScrobbleClients.ts

@@ -11,8 +11,8 @@ import { ListenBrainzClientConfig, ListenBrainzData } from "../common/infrastruc
 import { MalojaClientConfig, MalojaData } from "../common/infrastructure/config/client/maloja.js";
 import { WildcardEmitter } from "../common/WildcardEmitter.js";
 import { Notifiers } from "../notifier/Notifiers.js";
-import { isDebugMode, parseBool } from "../utils.js";
-import { readJson } from '../utils/DataUtils.js';
+import { isDebugMode, nonEmptyObj, parseBool, removeUndefinedKeys } from "../utils.js";
+import { getCommonComponentEnvConfig, readJson } from '../utils/DataUtils.js';
 import { validateJson } from "../utils/ValidationUtils.js";
 import AbstractScrobbleClient from "./AbstractScrobbleClient.js";
 import { KoitoClientConfig, KoitoData } from '../common/infrastructure/config/client/koito.js';
@@ -130,8 +130,11 @@ export default class ScrobbleClients {
             const {
                 clients: mainConfigClientConfigs = [],
                 clientDefaults: cd = {},
+                database: {
+                    retention
+                } = {},
             } = aioConfig;
-            clientDefaults = cd;
+            clientDefaults = {retention, ...cd};
             for (const [index, c] of mainConfigClientConfigs.entries()) {
                 const {name = 'unnamed'} = c;
                 if(c.type === undefined) {
@@ -168,138 +171,149 @@ export default class ScrobbleClients {
         for (const clientType of clientTypes) {
             const defaultConfigureAs = 'client';
             switch (clientType) {
-                case 'maloja':
+                case 'maloja': {
                     // env builder for single user mode
-                    const url = process.env.MALOJA_URL;
-                    const apiKey = process.env.MALOJA_API_KEY;
-                    if (url !== undefined || apiKey !== undefined) {
-                        const malojaData: MalojaData = {
-                            url,
-                            apiKey
-                        }
+                    const data = removeUndefinedKeys<MalojaData>({
+                        url: process.env.MALOJA_URL,
+                        apiKey: process.env.MALOJA_API_KEY
+                    }, false);
+                    const p = getCommonComponentEnvConfig('MALOJA');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'maloja',
                             name: 'unnamed-mlj',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'client',
-                            data: malojaData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('MALOJA')
                         })
                     }
-                    break;
-                case 'lastfm':
-                    const lfm: LastfmData = {
+                }  break;
+                case 'lastfm': {
+                    const data = removeUndefinedKeys<LastfmData>({
                         apiKey: process.env.LASTFM_API_KEY,
                         secret: process.env.LASTFM_SECRET,
                         redirectUri: process.env.LASTFM_REDIRECT_URI,
                         session: process.env.LASTFM_SESSION
-                    };
-                    if (!Object.values(lfm).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('LASTFM');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'lastfm',
                             name: 'unnamed-lfm',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'client',
-                            data: lfm,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('LASTFM')
                         })
                     }
-                    break;
+                }    break;
                 case 'librefm': {
-                    const shouldUse = parseBool(process.env.LIBRFM_ENABLE)
-                    const libre: LibrefmData = {
+                    const data = removeUndefinedKeys<LibrefmData>({
                         apiKey: process.env.LIBREFM_API_KEY,
                         secret: process.env.LIBREFM_SECRET,
                         redirectUri: process.env.LIBREFM_REDIRECT_URI,
                         session: process.env.LIBREFM_SESSION,
                         urlBase: process.env.LIBREFM_URLBASE,
-                    };
-                    if (!Object.values(libre).every(x => x === undefined) || shouldUse) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('LIBREFM');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'librefm',
                             name: 'unnamed-librefm',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'client',
-                            data: libre,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('LIBREFM')
                         })
                     }
                 }    break;
-                case 'listenbrainz':
-                    const lz: ListenBrainzData = {
+                case 'listenbrainz': {
+                    const data = removeUndefinedKeys<ListenBrainzData>({
                         url: process.env.LZ_URL,
                         token: process.env.LZ_TOKEN,
                         username: process.env.LZ_USER
-                    };
-                    if (!Object.values(lz).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('LZ');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'listenbrainz',
                             name: 'unnamed-lz',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'client',
-                            data: lz,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('LZ')
                         })
                     }
-                    break;
-                case 'koito':
-                    const koit: KoitoData = {
+                }    break;
+                case 'koito': {
+                    const data = removeUndefinedKeys<KoitoData>({
                         url: process.env.KOITO_URL,
                         token: process.env.KOITO_TOKEN,
                         username: process.env.KOITO_USER
-                    };
-                    if (!Object.values(koit).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('KOITO');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'koito',
                             name: 'unnamed-koito',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'client',
-                            data: koit,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('KOITO')
                         })
                     }
-                    break;
-                case 'tealfm':
-                    const teal: TealData = {
+                }    break;
+                case 'tealfm': {
+                    const data: TealData = removeUndefinedKeys<TealData>({
                         identifier: process.env.TEALFM_IDENTIFIER,
                         appPassword: process.env.TEALFM_APP_PW,
-                    };
-                    if (!Object.values(teal).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('TEALFM');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'tealfm',
                             name: 'unnamed-tealfm',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'client',
-                            data: teal,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('TEALFM')
                         })
                     }
-                    break;
-                case 'rocksky':
-                    const rocksky: RockSkyData = {
+                }    break;
+                case 'rocksky': {
+                    const data: RockSkyData = removeUndefinedKeys<RockSkyData>({
                         key: process.env.ROCKSKY_KEY,
                         handle: process.env.ROCKSKY_HANDLE
-                    };
-                    if (!Object.values(rocksky).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('ROCKSKY');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'rocksky',
                             name: 'unnamed-rocksky',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'client',
-                            data: rocksky,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('ROCKSKY')
                         })
                     }
-                    break;
+                }   break;
                 case 'discord': {
-                    const discord: DiscordData = {
+                    const data: DiscordData = removeUndefinedKeys<DiscordData>({
                         token: process.env.DISCORD_TOKEN,
                         artwork: process.env.DISCORD_ARTWORK,
                         applicationId: process.env.DISCORD_APPLICATION_ID,
@@ -307,15 +321,17 @@ export default class ScrobbleClients {
                         artworkDefaultUrl: process.env.DISCORD_ARTWORK_DEFAULT_URL,
                         statusOverrideAllow: process.env.DISCORD_STATUS_OVERRIDE_ALLOW,
                         listeningActivityAllow: process.env.DISCORD_LISTENING_ACTIVITY_ALLOW
-                    }
-                    if (!Object.values(discord).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('DISCORD');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'discord',
                             name: 'unnamed-discord',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'client',
-                            data: discord,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('DISCORD')
                         })
                     }
@@ -414,7 +430,7 @@ ${sources.join('\n')}`);
         if (isValidConfig !== true) {
             throw new Error(`Config object from ${clientConfig.source || 'unknown'} with name [${clientConfig.name || 'unnamed'}] of type [${clientConfig.type || 'unknown'}] has errors: ${isValidConfig.join(' | ')}`)
         }*/
-        const {type, name, enable = true, source, data: d = {}} = clientConfig;
+        const {type, name, enable = true, source, data: d = {}, options = {}} = clientConfig;
 
         if(enable === false) {
             this.logger.warn({labels: [`${type} - ${name}`]}, `Client from ${source} was disabled by config`);
@@ -422,41 +438,41 @@ ${sources.join('\n')}`);
         }
 
         // add defaults
-        const data = {...defaults, ...d};
+        const compositeOptions = {...defaults, ...options};
         let newClient;
         this.logger.debug({labels: [`${type} - ${name}`]}, `Constructing Client from ${source}`);
         switch (type) {
             case 'maloja':
                 const MalojaScrobbler = (await import('./MalojaScrobbler.js')).default;
-                newClient = new MalojaScrobbler(name, ({...clientConfig, data} as unknown as MalojaClientConfig), notifier, this.emitter, this.logger);
+                newClient = new MalojaScrobbler(name, ({...clientConfig, data: d, options: compositeOptions} as unknown as MalojaClientConfig), notifier, this.emitter, this.logger);
                 break;
             case 'lastfm':
                 const LastfmScrobbler = (await import('./LastfmScrobbler.js')).default;
-                newClient = new LastfmScrobbler(name, {...clientConfig, data } as unknown as LastfmClientConfig, this.internalConfig, notifier, this.emitter, this.logger);
+                newClient = new LastfmScrobbler(name, {...clientConfig, data: d, options: compositeOptions } as unknown as LastfmClientConfig, this.internalConfig, notifier, this.emitter, this.logger);
                 break;
             case 'librefm':
                 const LibrefmScrobbler = (await import('./LibrefmScrobbler.js')).default;
-                newClient = new LibrefmScrobbler(name, {...clientConfig, data } as unknown as LibrefmClientConfig, this.internalConfig, notifier, this.emitter, this.logger);
+                newClient = new LibrefmScrobbler(name, {...clientConfig, data: d, options: compositeOptions } as unknown as LibrefmClientConfig, this.internalConfig, notifier, this.emitter, this.logger);
                 break;
             case 'listenbrainz':
                 const ListenbrainzScrobbler = (await import('./ListenbrainzScrobbler.js')).default;
-                newClient = new ListenbrainzScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...data} } as unknown as ListenBrainzClientConfig, {}, notifier, this.emitter, this.logger);
+                newClient = new ListenbrainzScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...d}, options: compositeOptions } as unknown as ListenBrainzClientConfig, {}, notifier, this.emitter, this.logger);
                 break;
             case 'koito':
                 const KoitoScrobbler = (await import('./KoitoScrobbler.js')).default;
-                newClient = new KoitoScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...data} } as unknown as KoitoClientConfig, {}, notifier, this.emitter, this.logger);
+                newClient = new KoitoScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...d}, options: compositeOptions } as unknown as KoitoClientConfig, {}, notifier, this.emitter, this.logger);
                 break;
             case 'tealfm':
                 const TealScrobbler = (await import('./TealfmScrobbler.js')).default;
-                newClient = new TealScrobbler(name, {...clientConfig, data: {...data}} as unknown as TealClientConfig, {}, notifier, this.emitter, this.logger);
+                newClient = new TealScrobbler(name, {...clientConfig, data: d, options: compositeOptions} as unknown as TealClientConfig, {}, notifier, this.emitter, this.logger);
                 break;
             case 'rocksky':
                 const RockskyScrobbler = (await import('./RockskyScrobbler.js')).default;
-                newClient = new RockskyScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...data} } as unknown as RockSkyClientConfig, {}, notifier, this.emitter, this.logger);
+                newClient = new RockskyScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...d}, options: compositeOptions } as unknown as RockSkyClientConfig, {}, notifier, this.emitter, this.logger);
                 break;
             case 'discord':
                 const DiscordScrobbler = (await import('./DiscordScrobbler.js')).default;
-                newClient = new DiscordScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...data} } as unknown as DiscordClientConfig, {}, notifier, this.emitter, this.logger);
+                newClient = new DiscordScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...d}, options: compositeOptions } as unknown as DiscordClientConfig, {}, notifier, this.emitter, this.logger);
                 break;                
             default:
                 break;

src/backend/server/api.ts

@@ -6,6 +6,7 @@ import { FixedSizeList } from 'fixed-size-list';
 import { PassThrough } from "node:stream";
 import { Transform } from "stream";
 import {
+    CLIENT_DEAD_QUEUE,
     ClientStatusData,
     DeadLetterScrobble,
     LeveledLogData,
@@ -34,6 +35,8 @@ import ScrobbleSources from "../sources/ScrobbleSources.js";
 import ScrobbleClients from "../scrobblers/ScrobbleClients.js";
 import prom from 'prom-client';
 import { SimpleError } from "../common/errors/MSErrors.js";
+import { QueryPlaysOpts } from "../common/database/drizzle/repositories/PlayRepository.js";
+import { playSelectToDeadScrobble } from "../common/database/drizzle/entityUtils.js";
 
 const maxBufferSize = 300;
 const output: Record<number, FixedSizeList<LogDataPretty>> =  {};
@@ -236,8 +239,9 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
                 hasAuthInteraction: requiresAuthInteraction,
                 authed,
                 initialized: x.isReady(),
-                deadLetterScrobbles: x.deadLetterScrobbles.length,
-                queued: x.queuedScrobbles.length
+                deadLetterScrobbles: x.deadLetterQueued, // x.deadLetterScrobbles.length,
+                deadLetterScrobblesTotal: x.deadLetterLength,
+                queued: x.queuedLength // x.queuedScrobbles.length
             };
             if (!base.initialized) {
                 if(x.buildOK === false) {
@@ -262,7 +266,9 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
             // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
             scrobbleSource: source,
             query: {
-                upstream = 'false'
+                upstream = 'false',
+                next: queryNext = 'false',
+                ...rest
             }
         } = req;
 
@@ -278,7 +284,11 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
                     return res.status(500).json({message: e.message});
                 }
             } else {
-                result = (source as AbstractSource).getFlatRecentlyDiscoveredPlays();
+                if(queryNext === 'true') {
+                    return res.json(await (source as AbstractSource).getRecentPlaysApi(rest));
+                }
+                result = await (source as AbstractSource).getFlatRecentlyDiscoveredPlays();
+                
             }
         }
 
@@ -317,9 +327,21 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
         const {
             // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
             scrobbleClient: client,
+            query = {}
         } = req;
 
-        const result: DeadLetterScrobble<PlayObject>[] = (client as AbstractScrobbleClient).deadLetterScrobbles;
+        const deadQuery: QueryPlaysOpts = {
+            ...query as Partial<QueryPlaysOpts>,
+            queues: [
+                {
+                    queueName: CLIENT_DEAD_QUEUE,
+                    queueStatus: ['queued','failed']
+                }
+            ]
+        }
+
+        // @ts-ignore
+        const result: DeadLetterScrobble<PlayObject>[] = (await (client as AbstractScrobbleClient).getPlaysPaginated(deadQuery)).data.map(playSelectToDeadScrobble);
 
         return res.json(result);
     });
@@ -350,20 +372,20 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
 
         (client as AbstractScrobbleClient).logger.verbose(`User requested processing of dead letter scrobble ${deadId} via API call`)
 
-        const deadScrobble = (client as AbstractScrobbleClient).deadLetterScrobbles.find(x => x.id === deadId);
-
-        if(deadScrobble === undefined) {
-            (client as AbstractScrobbleClient).logger.debug(`No dead letter scrobble with ID ${deadId}`)
-            return res.status(404).send();
+        try {
+            const [scrobbled, dead] = await (client as AbstractScrobbleClient).processDeadLetterScrobble(deadId);
+            if(scrobbled) {
+                return res.status(200).send();
+            }
+            return res.json(playSelectToDeadScrobble(dead));
+        } catch (e) {
+            if(e.message.includes(`Play ${deadId} does not exist`)) {
+                logger.warn(e);
+                return res.status(404).json({error: e});
+            }
+            logger.error(e);
+            return res.status(500).json({error: e});
         }
-
-        const [scrobbled, dead] = await ((client as AbstractScrobbleClient).processDeadLetterScrobble(deadId));
-
-        if(scrobbled) {
-            return res.status(200).send();
-        }
-
-        return res.json(dead);
     });
 
     app.delete('/api/dead', clientMiddleFunc(true), async (req, res, next) => {
@@ -374,9 +396,9 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
 
         (client as AbstractScrobbleClient).logger.verbose('User requested deletion of all dead letter scrobbles via API');
 
-        (client as AbstractScrobbleClient).removeDeadLetterScrobbles();
+        (client as AbstractScrobbleClient).removeDeadLetterScrobbles(['queued', 'failed'], 'failed', false).then(() => null).catch((e) => logger.error(e));
 
-        return res.json([]);
+        return res.sendStatus(200);
     });
 
     app.delete('/api/dead/:id', clientMiddleFunc(true), async (req, res, next) => {
@@ -392,26 +414,33 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
 
         (client as AbstractScrobbleClient).logger.verbose(`User requested removal of dead letter scrobble ${deadId} via API call`)
 
-        const deadScrobble = (client as AbstractScrobbleClient).deadLetterScrobbles.find(x => x.id === deadId);
-
-        if(deadScrobble === undefined) {
-            (client as AbstractScrobbleClient).logger.verbose(`No dead letter scrobble with ID ${deadId}`)
-            return res.status(404).send();
+        try {
+            await (client as AbstractScrobbleClient).removeDeadLetterScrobble(deadId,'failed', false);
+            return res.status(200).send();
+        } catch (e) {
+            if(e.message.includes(`Play ${deadId} does not exist`)) {
+                logger.warn(e);
+                return res.status(404).json({error: e});
+            }
+            logger.error(e);
+            return res.status(500).json({error: e});
         }
-
-        (client as AbstractScrobbleClient).removeDeadLetterScrobble(deadId);
-        return res.status(200).send();
     });
 
     app.get('/api/scrobbled', clientMiddleFunc(false), async (req, res, next) => {
         const {
             // @ts-expect-error scrobbleClient not part of req
             scrobbleClient: client,
+            query
         } = req;
 
         let result: PlayObject[] = [];
         if (client !== undefined) {
-            result = [...(client as AbstractScrobbleClient).getScrobbledPlays()].sort(sortByNewestPlayDate);
+            const q: Partial<QueryPlaysOpts> = {
+                ...query as Partial<QueryPlaysOpts>,
+                state: ['scrobbled']
+            }
+            result = [...(await (client as AbstractScrobbleClient).getPlaysPaginated(q)).data.map(x => x.play)].sort(sortByNewestPlayDate);
         }
 
         return res.json(result);

src/backend/sources/AbstractSource.ts

@@ -2,7 +2,7 @@ import { childLogger, LogDataPretty, LogLevel } from '@foxxmd/logging';
 import dayjs, { Dayjs } from "dayjs";
 import { EventEmitter } from "events";
 import { FixedSizeList } from "fixed-size-list";
-import { PlayObject } from "../../core/Atomic.js";
+import { JsonPlayObject, PlayMatchResult, PlayObject } from "../../core/Atomic.js";
 import { buildTrackString, capitalize, truncateStringToLength } from "../../core/StringUtils.js";
 import AbstractComponent from "../common/AbstractComponent.js";
 import {
@@ -31,7 +31,7 @@ import {
     sleep,
     sortByOldestPlayDate,
 } from "../utils.js";
-import { sortByNewestPlayDate } from '../../core/PlayUtils.js';
+import { genGroupIdStr, sortByNewestPlayDate } from '../../core/PlayUtils.js';
 import { formatNumber } from '../../core/DataUtils.js';
 import { timeToHumanTimestamp } from "../../core/TimeUtils.js";
 import { todayAwareFormat } from "../../core/TimeUtils.js";
@@ -39,13 +39,16 @@ import { getRoot } from '../ioc.js';
 import { componentFileLogger } from '../common/logging.js';
 import { WebhookPayload } from '../common/infrastructure/config/health/webhooks.js';
 import { isAbortReasonErrorLike, messageWithCauses, messageWithCausesTruncatedDefault } from '../utils/ErrorUtils.js';
-import { genericSourcePlayMatch } from '../utils/PlayComparisonUtils.js';
+import { existingScrobble, ExistingScrobbleOpts, genericSourcePlayMatch } from '../utils/PlayComparisonUtils.js';
 import { findAsync, staggerMapper, StaggerOptions } from '../utils/AsyncUtils.js';
 import pMap, {pMapIterable} from 'p-map';
 import prom, { Counter, Gauge } from 'prom-client';
 import { normalizeStr } from '../utils/StringUtils.js';
 import { spawn, catchAbortError, isAbortError, rethrowAbortError, delay, forever, AbortError, throwIfAborted } from 'abort-controller-x';
 import { AbortedError, generateLoggableAbortReason } from '../common/errors/MSErrors.js';
+import { DrizzlePlayRepository, playToRepositoryCreatePlayOpts, queryArgsFromRequest, QueryPlaysOpts, RequestPlayQuery } from '../common/database/drizzle/repositories/PlayRepository.js';
+import { asPlay } from '../../core/PlayMarshalUtils.js';
+import { AsyncTask, SimpleIntervalJob, ToadScheduler } from 'toad-scheduler';
 
 export interface RecentlyPlayedOptions {
     limit?: number
@@ -57,7 +60,7 @@ export interface RecentlyPlayedOptions {
 export default abstract class AbstractSource extends AbstractComponent implements Authenticatable {
 
     name: string;
-    type: SourceType;
+    declare type: SourceType;
 
     declare config: SourceConfig;
     clients: string[];
@@ -88,6 +91,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
 
     manualListening?: boolean
 
+    scheduler: ToadScheduler = new ToadScheduler();
     emitter: EventEmitter;
 
     protected SCROBBLE_BACKLOG_COUNT: number = 30;
@@ -103,8 +107,15 @@ export default abstract class AbstractSource extends AbstractComponent implement
         postCompare: staggerMapper<PlayObject, PlayObject>({concurrency: 2})
     }
 
+    declare protected componentType: 'source';
+
+    protected playRepo!: DrizzlePlayRepository;
+
+    existingDiscoveredPlay: (playObjPre: PlayObject, existingScrobbles: PlayObject[], log?: boolean) => Promise<PlayMatchResult>
+
     constructor(type: SourceType, name: string, config: SourceConfig, internal: InternalConfig, emitter: EventEmitter) {
         super(config);
+        this.componentType = 'source';
         const {clients = [] } = config;
         this.type = type;
         this.name = name;
@@ -119,6 +130,15 @@ export default abstract class AbstractSource extends AbstractComponent implement
         this.emitter = emitter;
         
         this.discoveredCounter = getRoot().items.sourceMetics.discovered;
+
+        const existingScrobbleOpts: ExistingScrobbleOpts = {
+            logger: this.logger,
+            transformRules: this.transformRules,
+            transformPlay: this.transformPlay,
+            existingSubmitted: async (_) => [undefined, undefined]
+        }
+        this.existingDiscoveredPlay = (playObjPre: PlayObject, existingScrobbles: PlayObject[], log?: boolean) => existingScrobble(playObjPre, existingScrobbles, existingScrobbleOpts, log);
+            
     }
 
     async [Symbol.asyncDispose]() {
@@ -127,11 +147,70 @@ export default abstract class AbstractSource extends AbstractComponent implement
         }
     }
 
+    public initTasks() {
+        if(this.scheduler.existsById('heartbeat') === false) {
+            this.logger.info('Adding Heartbeat Task and running immediately');
+            this.scheduler.addSimpleIntervalJob(new SimpleIntervalJob({
+                minutes: 20,
+                runImmediately: true
+            }, new AsyncTask(
+                'Heartbeat',
+                (): Promise<any> => {
+                    return this.heartbeatTask().then(() => null).catch((err) => {
+                        this.logger.error(err);
+                    });
+                },
+                (err: Error) => {
+                    this.logger.error(err);
+                }
+            ), {id: 'heartbeat'}));
+        } else {
+            this.logger.warn('Heartbeat task is already added to scheduler.');
+        }
+    }
+
+    protected async heartbeatTask(): Promise<boolean> {
+        if(!this.isReady()) {
+            if(!this.canAuthUnattended()) {
+                this.logger.warn({labels: 'Heartbeat'}, 'Source is not ready but will not try to initialize because auth state is not good and cannot be corrected unattended.')
+                return false;
+            }
+            try {
+                await this.tryInitialize({force: false, notify: true, notifyTitle: 'Could not initialize automatically'});
+            } catch (e) {
+                this.logger.error(new Error('Could not initialize automatically', {cause: e}));
+                return false;
+            }
+
+            if('discoverDevices' in this && typeof this.discoverDevices === 'function') {
+                this.discoverDevices();
+            }
+
+            if (this.canPoll && !this.polling) {
+                if(!this.canAuthUnattended()) {
+                    this.logger.warn({labels: 'Heartbeat'}, 'Should be polling but will not attempt to start because auth state is not good and cannot be correct unattended.');
+                    return false;
+                } else {
+                    this.logger.info({labels: 'Heartbeat'}, 'Should be polling, attempting to start polling...');
+                    this.poll({force: false, notify: true}).catch(e => this.logger.error(e));
+                }
+                return true;
+            }
+        }
+        return true;
+    }
+
     protected async postCache(): Promise<void> {
         await super.postCache();
         this.generateStaggerMappers();
     }
 
+    protected async postDatabase(): Promise<void> {
+        this.playRepo = new DrizzlePlayRepository(this.db, {logger: this.logger});
+        this.tracksDiscovered = this.dbComponent.countLive;
+        this.playRepo.componentId = this.dbComponent.id;
+    }
+
     protected generateStaggerMappers() {
         const {
             preCompare = [],
@@ -200,83 +279,91 @@ export default abstract class AbstractSource extends AbstractComponent implement
     // TODO make this more descriptive? or move it elsewhere
     recentlyPlayedTrackIsValid = (playObj: PlayObject) => true
 
-    protected addPlayToDiscovered = (play: PlayObject) => {
-        const platformId = this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID;
-        const list = this.recentDiscoveredPlays.get(platformId) ?? new FixedSizeList<ProgressAwarePlayObject>(200);
-        list.add(play);
-        this.recentDiscoveredPlays.set(platformId, list);
+    protected addPlayToDiscovered = async (play: PlayObject): Promise<PlayObject> => {
+        const playRow = await this.playRepo.createPlays([(playToRepositoryCreatePlayOpts({play, state: 'discovered'}))]);
+        const recentPlays = await this.getRecentlyDiscoveredPlays(false);
+        // only need to update if its already in memory,
+        // and better to update in-memory than clear cache so we aren't refetching from db on every discover
+        if(recentPlays !== undefined) {
+            recentPlays.push(play);
+            recentPlays.sort(sortByOldestPlayDate);
+            this.cache.cacheDb.set(this.recentDiscoveredCacheKey(), recentPlays, '2m');
+        }
         this.tracksDiscovered++;
         this.logger.info(`Discovered => ${buildTrackString(play)}`);
         this.emitEvent('discovered', {play});
         this.discoveredCounter.labels(this.getPrometheusLabels()).inc();
+        play.id = playRow[0].id;
+        play.uid = playRow[0].uid;
+        return play;
     }
 
-    getFlatRecentlyDiscoveredPlays = (): PlayObject[] =>
-         Array.from(this.recentDiscoveredPlays.values()).map(x => x.data).flat(3).sort(sortByNewestPlayDate)
-    
+    getFlatRecentlyDiscoveredPlays = async (): Promise<PlayObject[]> => {
+        const list: PlayObject[] = await this.getRecentlyDiscoveredPlays();
+        return list.sort(sortByNewestPlayDate);
+    }
 
-    getRecentlyDiscoveredPlaysByPlatform = (platformId: PlayPlatformId): PlayObject[] => {
-        const list = this.recentDiscoveredPlays.get(platformId);
-        if (list !== undefined) {
-            const data = [...list.data];
-            data.sort(sortByOldestPlayDate);
-            return data;
+    getRecentPlaysApi = async (query: RequestPlayQuery) => {
+        const res = await this.playRepo.findPlays({
+            limit: 100,
+            ...queryArgsFromRequest(query)
+        });
+        return res.map((x) => {
+            const {id, ...rest} = x;
+            return rest;
+        })
+    }
+
+    protected recentDiscoveredCacheKey = () => {
+        return `recent-${this.dbComponent.id}`;
+    }
+
+    getRecentlyDiscoveredPlays = async (hydrate: boolean = true): Promise<PlayObject[]> => {
+        const cacheKey = this.recentDiscoveredCacheKey();
+        let list = await this.cache.cacheDb.get<PlayObject[]>(cacheKey);
+        if(list === undefined && hydrate) {
+            list = (await this.playRepo.findPlays({
+                stateNot: ['queued'],
+                order: 'desc',
+                sort: 'playedAt',
+                limit: 200
+            })).map(x => asPlay(x.play))
+            list.sort(sortByOldestPlayDate);
+            await this.cache.cacheDb.set<PlayObject[]>(cacheKey, list, '2m');
         }
-        return [];
+        return list;
     }
 
-    protected getExistingDiscoveredLists = (play: PlayObject, opts: {checkAll?: boolean} = {}): PlayObject[][] => {
-        const lists: PlayObject[][] = [];
-        if(opts.checkAll !== true) {
-            lists.push(this.getRecentlyDiscoveredPlaysByPlatform(this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID));
-        } else {
-            // get as many as we can, optionally filtering by user
-            this.recentDiscoveredPlays.forEach((list, platformId) => {
-                if(play.meta.user !== undefined) {
-                    if(platformId[1] === NO_USER || platformId[1] === play.meta.user) {
-                        lists.push(this.getRecentlyDiscoveredPlaysByPlatform(platformId));
-                    }
-                } else {
-                    lists.push(this.getRecentlyDiscoveredPlaysByPlatform(platformId));
-                }
-            });
-        }
-        return lists;
-    }
-
-    existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<PlayObject | undefined> => {
-        const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts);
-        const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate);
-        for(const list of lists) {
-            
-            const existing = await findAsync(list,async x => {
-                const e = await this.transformPlay(x, TRANSFORM_HOOK.existing);
-                return genericSourcePlayMatch(e, candidate);
-            });
-            if(existing) {
-                return existing;
-            }
+    existingDiscovered = async (play: PlayObject): Promise<PlayObject | undefined> => {
+        const list: PlayObject[] = await this.getRecentlyDiscoveredPlays();
+        const matchResults = await this.existingDiscoveredPlay(play, list);
+        if(matchResults.match) {
+            return matchResults.closestMatchedPlay;
         }
         return undefined;
     }
 
-    alreadyDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<boolean> => {
-        const existing = await this.existingDiscovered(play, opts);
-        return existing !== undefined;
-    }
-
     discover = async (plays: PlayObject[], options: { checkAll?: boolean, signal?: AbortSignal, [key: string]: any } = {}): Promise<PlayObject[]> => {
         const newDiscoveredPlays: PlayObject[] = [];
 
         for await(const play of pMapIterable(plays, this.staggerMappers.preCompare(async x => await this.transformPlay(x, TRANSFORM_HOOK.preCompare)), {concurrency: 3})) {
             options.signal?.throwIfAborted();
-            if(!(await this.alreadyDiscovered(play, options))) {
+            const existing = await this.existingDiscovered(play);
+            if(existing === undefined) {
                 options.signal?.throwIfAborted()
-                this.addPlayToDiscovered(play);
-                newDiscoveredPlays.push(play);
+                const hydratedPlay = await this.addPlayToDiscovered(play);
+                newDiscoveredPlays.push(hydratedPlay);
+            } else {
+                this.playRepo.updateById(existing.id, {updatedAt: dayjs()});
+            }
+        }
+        if(newDiscoveredPlays.length > 0) {
+            try {
+                await this.componentRepo.updateById(this.dbComponent.id, {countLive: this.dbComponent.countLive + newDiscoveredPlays.length});
+            } catch (e) {
+                this.logger.warn(new Error('Unable to update discovered count', {cause: e}));
             }
         }
-
         newDiscoveredPlays.sort(sortByOldestPlayDate);
 
         return newDiscoveredPlays;
@@ -298,6 +385,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
 
         if(newDiscoveredPlays.length > 0) {
             if(!this.shouldScrobble(options.discoverLocation)) {
+                await this.playRepo.setStateById('discarded', newDiscoveredPlays.map(x => x.id));
                 return;
             }
             newDiscoveredPlays.sort(sortByOldestPlayDate);
@@ -500,7 +588,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
         this.abortController.abort(reason);
         let elapsed = 0;
         let lastlog: Dayjs;
-        while(this.polling && elapsed < 10) {
+        while(this.polling && elapsed < (10 * this.stopPollingWaitInterval)) {
             if(lastlog === undefined || dayjs().diff(lastlog, 's') >= 2) {
                 this.logger.verbose(`Waiting for polling stop signal to be acknowledged (waited ${formatNumber(elapsed/1000)}s)`);
             }
@@ -548,7 +636,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
                 }
             
 
-                const interval = this.getInterval();
+                const interval = this.getInterval(true);
                 const maxBackoff = this.getMaxBackoff();
                 let sleepTime = interval;
 
@@ -652,7 +740,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
         return this.wakeAt;
     }
 
-    protected getInterval() {
+    protected getInterval(log?: boolean) {
         let interval = DEFAULT_POLLING_INTERVAL;
 
         if('interval' in this.config.data) {
@@ -670,6 +758,18 @@ export default abstract class AbstractSource extends AbstractComponent implement
         return maxInterval - this.getInterval();
     }
 
+    public getPlays = (args: QueryPlaysOpts) => {
+        const {
+            limit,
+            offset,
+            with: withQuery = ['input','parent-input','queues'],
+            ...rest
+        } = args;
+        let parsedLimit = limit !== undefined ? Number.parseInt(limit as unknown as string) : undefined;
+        let parsedOffset = offset !== undefined ? Number.parseInt(offset as unknown as string) : undefined;
+        return this.playRepo.findPlaysPaginated({limit: parsedLimit, offset: parsedOffset, with: withQuery, ...rest});
+    }
+
     public emitEvent = (eventName: string, payload: object = {}) => {
         this.emitter.emit(eventName, {
             type: this.type,
@@ -684,7 +784,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
     }
 
     protected async doBuildComponentLogger(): Promise<void> {
-        if(this.config.options.logToFile) {
+        if(this.config?.options?.logToFile) {
             this.logger.debug('Enabling component logger...');
             const root = getRoot();
             const stream = root.get('loggerStream');

src/backend/sources/AzuracastSource.ts

@@ -17,6 +17,7 @@ import {
 import { AzuracastSourceConfig, AzuraNowPlayingResponse, AzuraStationResponse } from "../common/infrastructure/config/source/azuracast.js";
 import { isPortReachable, normalizeWSAddress } from "../utils/NetworkUtils.js";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
+import { artistNamesToCredits } from "../../core/StringUtils.js";
 
 
 export class AzuracastSource extends MemorySource {
@@ -250,7 +251,7 @@ const formatPlayObj = (obj: AzuraNowPlayingResponse, options: FormatPlayObjectOp
 
     const play: PlayObjectLifecycleless = {
         data: {
-            artists: artist !== undefined && artist !== '' ? [artist] : [],
+            artists: artistNamesToCredits(artist !== undefined && artist !== '' ? [artist] : []),
             album: album !== '' ? album : undefined,
             track,
             duration

src/backend/sources/ChromecastSource.ts

@@ -6,7 +6,7 @@ import dayjs from "dayjs";
 import { EventEmitter } from "events";
 import e from "express";
 import { PlayObject, PlayObjectLifecycleless } from "../../core/Atomic.js";
-import { buildTrackString } from "../../core/StringUtils.js";
+import { artistNamesToCredits, buildTrackString } from "../../core/StringUtils.js";
 import { NETWORK_ERROR_FAILURE_CODES } from "../common/errors/NodeErrors.js";
 import {
     FormatPlayObjectOptions,
@@ -722,8 +722,8 @@ export class ChromecastSource extends MemoryPositionalSource {
             data: {
                 track,
                 album,
-                albumArtists,
-                artists,
+                albumArtists: artistNamesToCredits(albumArtists),
+                artists: artistNamesToCredits(artists),
                 duration,
                 playDate: dayjs()
             },

src/backend/sources/DeezerInternalSource.ts

@@ -15,6 +15,7 @@ import { TemporalPlayComparisonOptions } from "../utils/TimeUtils.js";
 import { findAsync, findIndexAsync } from "../utils/AsyncUtils.js";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
 import { UpstreamError } from "../common/errors/UpstreamError.js";
+import { artistNamesToCredits } from "../../core/StringUtils.js";
 
 interface DeezerHistoryResponse {
     errors: []
@@ -113,7 +114,7 @@ export default class DeezerInternalSource extends MemorySource {
         const {newFromSource = false} = options;
         const play: PlayObjectLifecycleless = {
             data: {
-                artists: [obj.ART_NAME],
+                artists: artistNamesToCredits([obj.ART_NAME]),
                 album: obj.ALB_TITLE,
                 track: obj.SNG_TITLE,
                 duration: obj.DURATION,
@@ -330,45 +331,43 @@ export default class DeezerInternalSource extends MemorySource {
 
 
     existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<PlayObject | undefined> => {
-        const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts);
+        const list: PlayObject[] = await this.getRecentlyDiscoveredPlays();
         const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate);
-        for(const list of lists) {
-            const existing = await findAsync(list, async x => {
+        const existing = await findAsync(list, async x => {
+            const e = await this.transformPlay(x, TRANSFORM_HOOK.existing);
+            return genericSourcePlayMatch(e, candidate);
+        });
+        if(existing) {
+            return existing;
+        }
+        if(this.config.options?.fuzzyDiscoveryIgnore === true || this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {
+            const fuzzyIndex = await findIndexAsync(list, async x => {
                 const e = await this.transformPlay(x, TRANSFORM_HOOK.existing);
-                return genericSourcePlayMatch(e, candidate);
+                let temporalOptions: TemporalPlayComparisonOptions = {};
+                const temporalAccuracy: TemporalAccuracy[] = [TA_EXACT, TA_CLOSE, TA_FUZZY];
+                if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {
+                    temporalOptions = {
+                        fuzzyDiffThreshold: Math.max(100, x.data.duration * 0.5),
+                        duringReferences: ['duration', 'listenedFor', 'range']
+                    }
+                    temporalAccuracy.push(TA_DURING);
+                }
+                return genericSourcePlayMatch(e, candidate, temporalAccuracy, temporalOptions);
             });
-            if(existing) {
-                return existing;
-            }
-            if(this.config.options?.fuzzyDiscoveryIgnore === true || this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {
-                const fuzzyIndex = await findIndexAsync(list, async x => {
-                    const e = await this.transformPlay(x, TRANSFORM_HOOK.existing);
-                    let temporalOptions: TemporalPlayComparisonOptions = {};
-                    const temporalAccuracy: TemporalAccuracy[] = [TA_EXACT, TA_CLOSE, TA_FUZZY];
-                    if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {
-                        temporalOptions = {
-                            fuzzyDiffThreshold: Math.max(100, x.data.duration * 0.5),
-                            duringReferences: ['duration', 'listenedFor', 'range']
-                        }
-                        temporalAccuracy.push(TA_DURING);
-                    }
-                    return genericSourcePlayMatch(e, candidate, temporalAccuracy, temporalOptions);
-                });
-                if(fuzzyIndex !== -1) {
-                    if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {
-                        // always return fuzzy match as existing
-                        // likely will make MS miss scrobbles for repeated plays
-                        return list[fuzzyIndex];
-                    }
-                    if(fuzzyIndex + 1 === list.length || playObjDataMatch(list[fuzzyIndex], list[fuzzyIndex + 1])) {
-                        // last discovered play was this one, or next played play was also this one
-                        // so we'll assume this means the play is on repeat, don't count as existing
-                        return undefined;
-                    }
-                    // next played play was *not* this one (Deezer reports play between candidate TS and fuzzy match)
-                    // so this is likely a duplicate deezer should not have reported
+            if(fuzzyIndex !== -1) {
+                if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {
+                    // always return fuzzy match as existing
+                    // likely will make MS miss scrobbles for repeated plays
                     return list[fuzzyIndex];
                 }
+                if(fuzzyIndex + 1 === list.length || playObjDataMatch(list[fuzzyIndex], list[fuzzyIndex + 1])) {
+                    // last discovered play was this one, or next played play was also this one
+                    // so we'll assume this means the play is on repeat, don't count as existing
+                    return undefined;
+                }
+                // next played play was *not* this one (Deezer reports play between candidate TS and fuzzy match)
+                // so this is likely a duplicate deezer should not have reported
+                return list[fuzzyIndex];
             }
         }
         return undefined;

src/backend/sources/EndpointLastfmSource.ts

@@ -11,13 +11,13 @@ import {
     REPORTED_PLAYER_STATUSES,
     ReportedPlayerStatus
 } from "../common/infrastructure/Atomic.js";
-import { parseRegexSingleOrFail } from "../utils.js";
 import MemorySource from "./MemorySource.js";
 import { LastFMEndpointSourceConfig } from "../common/infrastructure/config/source/endpointlfm.js";
 import { LastFMScrobbleRequestPayload, scrobblePayloadToPlay } from "../common/vendor/LastfmApiClient.js";
 import { Logger } from "@foxxmd/logging";
 import { PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js";
 import { NowPlayingPlayerState } from "./PlayerState/NowPlayingPlayerState.js";
+import { parseRegexSingle } from "@foxxmd/regex-buddy-core";
 
 const noSlugMatch = new RegExp(/(?:\/api\/lastfm\/?)$|(?:\/1\/?|\/2.0\/?)$/i);
 const slugMatch = new RegExp(/\/api\/lastfm\/([^\/]+)$/i);
@@ -62,7 +62,7 @@ export class EndpointLastfmSource extends MemorySource {
     }
 
     getRecentlyPlayed = async (options = {}) => {
-        return this.getFlatRecentlyDiscoveredPlays();
+        return await this.getFlatRecentlyDiscoveredPlays();
     }
 
     isValidScrobble = (playObj: PlayObject) => {
@@ -97,11 +97,11 @@ export const playStateFromRequest = (obj: LastFMScrobbleRequestPayload): PlayerS
 }
 
 export const parseSlugFromString = (path: string): string | false | undefined => {
-    const noSlug = parseRegexSingleOrFail(noSlugMatch, path);
+    const noSlug = parseRegexSingle(noSlugMatch, path);
     if (noSlug !== undefined) {
         return undefined;
     }
-    const slugResult = parseRegexSingleOrFail(slugMatch, path);
+    const slugResult = parseRegexSingle(slugMatch, path);
     if (slugResult !== undefined) {
         return slugResult.groups[0];
     }

src/backend/sources/EndpointListenbrainzSource.ts

@@ -15,11 +15,11 @@ import { ListenbrainzEndpointSourceConfig } from "../common/infrastructure/confi
 import { ListenbrainzApiClient, listenPayloadToPlay } from "../common/vendor/ListenbrainzApiClient.js";
 import { SubmitPayload } from '../common/vendor/listenbrainz/interfaces.js';
 import { ListenPayload } from '../common/vendor/listenbrainz/interfaces.js';
-import { parseRegexSingleOrFail } from "../utils.js";
 import MemorySource from "./MemorySource.js";
 import { NowPlayingPlayerState } from "./PlayerState/NowPlayingPlayerState.js";
 import { Logger } from "@foxxmd/logging";
 import { PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js";
+import { parseRegexSingle } from "@foxxmd/regex-buddy-core";
 
 const noSlugMatch = new RegExp(/(?:\/api\/listenbrainz\/?)$|(?:\/1\/?|\/1\/submit-listens\/?\/1\/validate-token\/|)$/i);
 const slugMatch = new RegExp(/\/api\/listenbrainz\/([^\/]+)$/i);
@@ -82,7 +82,7 @@ export class EndpointListenbrainzSource extends MemorySource {
     }
 
     getRecentlyPlayed = async (options = {}) => {
-        return this.getFlatRecentlyDiscoveredPlays();
+        return await this.getFlatRecentlyDiscoveredPlays();
     }
 
     isValidScrobble = (playObj: PlayObject) => {
@@ -131,7 +131,7 @@ export const listenTypeAsPlayerStatus = (event: string): ReportedPlayerStatus =>
 }
 
 export const parseTokenFromString = (str: string): string | undefined => {
-    const tokenMatch = parseRegexSingleOrFail(authHeaderRegex, str);
+    const tokenMatch = parseRegexSingle(authHeaderRegex, str);
     if(tokenMatch !== undefined) {
         return tokenMatch.groups[0];
     }
@@ -151,11 +151,11 @@ export const parseTokenFromRequest = (req: ExpressRequest): string | false | und
 }
 
 export const parseSlugFromString = (path: string): string | false | undefined => {
-    const noSlug = parseRegexSingleOrFail(noSlugMatch, path);
+    const noSlug = parseRegexSingle(noSlugMatch, path);
     if (noSlug !== undefined) {
         return undefined;
     }
-    const slugResult = parseRegexSingleOrFail(slugMatch, path);
+    const slugResult = parseRegexSingle(slugMatch, path);
     if (slugResult !== undefined) {
         return slugResult.groups[0];
     }

src/backend/sources/IcecastSource.ts

@@ -15,6 +15,7 @@ import IcecastMetadataStats from "icecast-metadata-stats";
 import { parseArtistCredits, parseTrackCredits } from "../utils/StringUtils.js";
 import { isDebugMode, sleep } from "../utils.js";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
+import { artistNamesToCredits } from "../../core/StringUtils.js";
 
 
 export class IcecastSource extends MemorySource {
@@ -211,7 +212,7 @@ const formatPlayObj = (obj: IcecastMetadata, options: FormatPlayObjectOptions =
     const play: PlayObjectLifecycleless = {
         data: {
             track,
-            artists
+            artists: artistNamesToCredits(artists)
         },
         meta: {
             source: 'icecast',

src/backend/sources/JRiverSource.ts

@@ -9,6 +9,7 @@ import { Info, JRiverApiClient, PLAYER_STATE } from "../common/vendor/JRiverApiC
 import { RecentlyPlayedOptions } from "./AbstractSource.js";
 import { MemoryPositionalSource } from "./MemoryPositionalSource.js";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
+import { artistNamesToCredits } from "../../core/StringUtils.js";
 
 export class JRiverSource extends MemoryPositionalSource {
     declare config: JRiverSourceConfig;
@@ -104,7 +105,7 @@ export class JRiverSource extends MemoryPositionalSource {
             data: {
                 track: Name,
                 album: album,
-                artists,
+                artists: artistNamesToCredits(artists),
                 duration: Math.round(length),
                 playDate: dayjs()
             },

src/backend/sources/JellyfinApiSource.ts

@@ -42,8 +42,8 @@ import {
 from "@jellyfin/sdk/lib/index.js";
 import dayjs from "dayjs";
 import EventEmitter from "events";
-import { BrainzMeta, PlayObject, PlayObjectLifecycleless } from "../../core/Atomic.js";
-import { buildTrackString, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.js";
+import { ArtistCredit, BrainzMeta, PlayObject, PlayObjectLifecycleless } from "../../core/Atomic.js";
+import { artistNamesToCredits, buildTrackString, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.js";
 import {
     FormatPlayObjectOptions,
     InternalConfig,
@@ -466,9 +466,16 @@ export default class JellyfinApiSource extends MemoryPositionalSource {
             meta.albumArtist = [ProviderIds.MusicBrainzAlbumArtist];
         }
 
+        let playArtists: ArtistCredit[] = [];
+        if(Artists.length === 1 && meta.artist !== undefined) {
+            playArtists.push({name: Artists[0], mbid: meta.artist[0]});
+        } else {
+            playArtists = artistNamesToCredits(Artists);
+        }
+
         const play: PlayObjectLifecycleless = {
             data: {
-                artists: Artists,
+                artists: playArtists,
                 album: Album,
                 track: Name,
                 albumArtists: AlbumArtists !== undefined ? AlbumArtists.map(x => x.Name) : undefined,

src/backend/sources/MPDSource.ts

@@ -20,6 +20,7 @@ import { RecentlyPlayedOptions } from "./AbstractSource.js";
 import { MemoryPositionalSource } from "./MemoryPositionalSource.js";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
 import { isDebugMode, sleep } from "../utils.js";
+import { artistNamesToCredits } from "../../core/StringUtils.js";
 
 const CLIENT_PLAYER_STATE: Record<PlayerState, ReportedPlayerStatus> = {
     'play': REPORTED_PLAYER_STATUSES.playing,
@@ -256,8 +257,8 @@ export class MPDSource extends MemoryPositionalSource {
 
         const play: PlayObjectLifecycleless = {
             data: {
-                artists: artists,
-                albumArtists,
+                artists: artistNamesToCredits(artists),
+                albumArtists: artistNamesToCredits(albumArtists),
                 album,
                 track: trackName,
                 duration

src/backend/sources/MPRISSource.ts

@@ -21,6 +21,7 @@ import { Readable, Writable } from 'stream';
 import net from 'net';
 import pEvent from 'p-event';
 import { baseFormatPlayObj } from '../utils/PlayTransformUtils.js';
+import { artistNamesToCredits } from '../../core/StringUtils.js';
 
 
 export class MPRISSource extends MemorySource {
@@ -76,8 +77,8 @@ export class MPRISSource extends MemorySource {
             data: {
                 track: title,
                 album,
-                artists: artist,
-                albumArtists: actualAlbumArtists,
+                artists: artistNamesToCredits(artist),
+                albumArtists:artistNamesToCredits(actualAlbumArtists),
                 duration: length,
                 playDate: dayjs()
             },

src/backend/sources/MemorySource.ts

@@ -368,7 +368,7 @@ export default class MemorySource extends AbstractSource {
                         }
                         return [false, `${stPrefix} ${EXPECTED_NON_DISCOVERED_REASON}`]
                     } else {
-                        const discoveredPlays = this.getRecentlyDiscoveredPlaysByPlatform(genGroupId(candidate));
+                        const discoveredPlays = await this.getRecentlyDiscoveredPlays();
                         if (discoveredPlays.length === 0 || !playObjDataMatch(discoveredPlays[0], candidate)) {
                             // if most recent stateful play is not this track we'll add it
                             return [true,`${stPrefix} added after ${thresholdResultSummary(thresholdResults)}. Matched other recent play but could not determine time frame due to missing duration. Allowed due to not being last played track.`];
@@ -385,7 +385,7 @@ export default class MemorySource extends AbstractSource {
 
     recentlyPlayedTrackIsValid = (playObj: any) => playObj.data.playDate.isBefore(dayjs().subtract(30, 's'))
 
-    protected getInterval(): number {
+    protected getInterval(log?: boolean): number {
         /**
          * If any player is progressing, reports position, and play has duration
          * then we can modify polling interval so that we check source data just before track is supposed to end
@@ -415,7 +415,7 @@ export default class MemorySource extends AbstractSource {
                 }
             }
         }
-        if(logDecrease !== undefined) {
+        if(logDecrease !== undefined && log) {
             this.logger.debug(logDecrease);
         }
         return interval;

src/backend/sources/MopidySource.ts

@@ -6,7 +6,7 @@ import normalizeUrl from 'normalize-url';
 import pEvent from 'p-event';
 import { URL } from "url";
 import { PlayObject, PlayObjectLifecycleless } from "../../core/Atomic.js";
-import { buildTrackString } from "../../core/StringUtils.js";
+import { artistNamesToCredits, buildTrackString } from "../../core/StringUtils.js";
 import {
     FormatPlayObjectOptions,
     InternalConfig,
@@ -164,8 +164,8 @@ export class MopidySource extends MemoryPositionalSource {
             data: {
                 track: name,
                 album: albumName,
-                albumArtists: actualAlbumArtists.length > 0 ? actualAlbumArtists.map(x => x.name) : [],
-                artists: artists.length > 0 ? artists.map(x => x.name) : [],
+                albumArtists: artistNamesToCredits(actualAlbumArtists.length > 0 ? actualAlbumArtists.map(x => x.name) : []),
+                artists: artistNamesToCredits(artists.length > 0 ? artists.map(x => x.name) : []),
                 duration: Math.round(length / 1000),
                 playDate: dayjs()
             },

src/backend/sources/MusicCastSource.ts

@@ -12,6 +12,7 @@ import { isPortReachable, isPortReachableConnect, joinedUrl, normalizeWebAddress
 import { DeviceInfoResponse, DeviceStatusResponse, MusicCastResponseCodes, MusicCastSourceConfig, playbackToReportedStatus, PlayInfoCDResponse, PlayInfoNetResponse } from "../common/infrastructure/config/source/musiccast.js";
 import request, { Request, Response } from 'superagent';
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
+import { artistNamesToCredits } from "../../core/StringUtils.js";
 
 
 export class MusicCastSource extends MemoryPositionalSource {
@@ -149,7 +150,7 @@ const formatPlayObj = (obj: PlayInfoCDResponse | PlayInfoNetResponse, options: F
 
     const play: PlayObjectLifecycleless = {
         data: {
-            artists: artist !== undefined && artist !== '' ? [artist] : [],
+            artists: artistNamesToCredits(artist !== undefined && artist !== '' ? [artist] : []),
             album: album !== '' ? album : undefined,
             track,
             // we should treat 0 time as the same as not being provided

src/backend/sources/PlexApiSource.ts

@@ -1,6 +1,6 @@
 import EventEmitter from "events";
 import { PlayObject, PlayObjectLifecycleless, URLData } from "../../core/Atomic.js";
-import { buildTrackString, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.js";
+import { artistNamesToCredits, buildTrackString, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.js";
 import {
     asPlayerStateDataMaybePlay,
     FormatPlayObjectOptions,
@@ -376,8 +376,8 @@ export default class PlexApiSource extends MemoryPositionalSource {
 
         const play: PlayObjectLifecycleless = {
             data: {
-                artists: realArtists,
-                albumArtists,
+                artists: artistNamesToCredits(realArtists),
+                albumArtists: artistNamesToCredits(albumArtists),
                 album,
                 track,
                 // albumArtists: AlbumArtists !== undefined ? AlbumArtists.map(x => x.Name) : undefined,

src/backend/sources/ScrobbleSources.ts

@@ -36,8 +36,8 @@ import { YTMusicData, YTMusicSourceConfig } from "../common/infrastructure/confi
 import { YandexMusicBridgeData, YandexMusicBridgeSourceConfig } from "../common/infrastructure/config/source/ymbridge.js";
 import { SonosData, SonosSourceConfig } from "../common/infrastructure/config/source/sonos.js";
 import { WildcardEmitter } from "../common/WildcardEmitter.js";
-import { parseBool } from "../utils.js";
-import { readJson } from '../utils/DataUtils.js';
+import { nonEmptyObj, parseBool, removeUndefinedKeys } from "../utils.js";
+import { getCommonComponentEnvConfig, readJson } from '../utils/DataUtils.js';
 import { validateJson } from "../utils/ValidationUtils.js";
 import AbstractSource from "./AbstractSource.js";
 import { nonEmptyStringOrDefault } from '../../core/StringUtils.js';
@@ -54,6 +54,7 @@ import { ListenBrainzData } from '../common/infrastructure/config/client/listenb
 import { KoitoData } from '../common/infrastructure/config/client/koito.js';
 import { TealData } from '../common/infrastructure/config/client/tealfm.js';
 import { RockSkyData } from '../common/infrastructure/config/client/rocksky.js';
+import { DEFAULT_RETENTION_DELETE_AFTER } from '../common/infrastructure/config/database.js';
 
 type groupedNamedConfigs = {[key: string]: ParsedConfig[]};
 
@@ -230,8 +231,11 @@ export default class ScrobbleSources {
             const {
                 sources: mainConfigSourcesConfigs = [],
                 sourceDefaults: sd = {},
+                database: {
+                    retention
+                } = {},
             } = aioConfig;
-            sourceDefaults = this.buildSourceDefaults(sd);
+            sourceDefaults = this.buildSourceDefaults({retention, ...sd});
             for (const [index, c] of mainConfigSourcesConfigs.entries()) {
                 const {name = 'unnamed'} = c;
                 if(c.type === undefined) {
@@ -273,26 +277,28 @@ export default class ScrobbleSources {
             let defaultConfigureAs: ConfigureAsSource = 'source';
             // env builder for single user mode
             switch (sourceType) {
-                case 'spotify':
-                    const s: SpotifySourceData = {
+                case 'spotify': {
+                    const data: SpotifySourceData = removeUndefinedKeys<SpotifySourceData>({
                         clientId: process.env.SPOTIFY_CLIENT_ID as string,
                         clientSecret: process.env.SPOTIFY_CLIENT_SECRET as string,
                         redirectUri: process.env.SPOTIFY_REDIRECT_URI,
-                    };
-                    if (!Object.values(s).every(x => x === undefined && x !== null)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('SPOTIFY');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'spotify',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: s,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('SPOTIFY')
                         })
                     }
-                    break;
-                case 'plex':
-                    const p: PlexApiData = {
+                }    break;
+                case 'plex': {
+                    const data: PlexApiData =  removeUndefinedKeys<PlexApiData>({
                         url: process.env.PLEX_URL,
                         token: process.env.PLEX_TOKEN,
                         usersAllow: process.env.PLEX_USERS_ALLOW,
@@ -301,39 +307,43 @@ export default class ScrobbleSources {
                         devicesBlock: process.env.PLEX_DEVICES_BLOCK,
                         librariesAllow: process.env.PLEX_LIBRARIES_ALLOW,
                         librariesBlock: process.env.PLEX_LIBRARIES_BLOCK
-                    };
-                    if (!Object.values(p).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('PLEX');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'plex',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: p,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('PLEX')
                         })
                     }
-                    break;
-                case 'subsonic':
-                    const sub: SubsonicData = {
+                }    break;
+                case 'subsonic': {
+                    const data: SubsonicData = removeUndefinedKeys<SubsonicData>({
                         user: process.env.SUBSONIC_USER,
                         password: process.env.SUBSONIC_PASSWORD,
                         url: process.env.SUBSONIC_URL,
-                    };
-                    if (!Object.values(sub).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('SUBSONIC');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'subsonic',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: sub as SubsonicData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('SUBSONIC')
                         })
                     }
-                    break;
-                case 'jellyfin':
-                    const j: JellyApiData = {
+                }    break;
+                case 'jellyfin': {
+                    const data: JellyApiData = removeUndefinedKeys<JellyApiData>({
                         user: process.env.JELLYFIN_USER,
                         password: process.env.JELLYFIN_PASSWORD,
                         apiKey: process.env.JELLYFIN_APIKEY,
@@ -346,294 +356,317 @@ export default class ScrobbleSources {
                         librariesBlock: process.env.JELLYFIN_LIBRARIES_BLOCK,
                         frontendUrlOverride: process.env.JELLYFIN_FRONTEND_URL_OVERRIDE,
                         allowMediaTypes: process.env.JELLYFIN_MEDIATYPES_ALLOW
-                    };
-                    if (!Object.values(j).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('JELLYFIN');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'jellyfin',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: j,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('JELLYFIN')
                         })
                     }
-                    break;
+                }    break;
                 case 'lastfm':
                         {
-                        const lfm: LastfmData = {
+                        const data: LastfmData = removeUndefinedKeys<LastfmData>({
                             apiKey: process.env.SOURCE_LASTFM_API_KEY,
                             secret: process.env.SOURCE_LASTFM_SECRET,
                             redirectUri: process.env.SOURCE_LASTFM_REDIRECT_URI,
                             session: process.env.SOURCE_LASTFM_SESSION,
-                        };
-                        if (!Object.values(lfm).every(x => x === undefined)) {
+                        }, false);
+                        const p = getCommonComponentEnvConfig('SOURCE_LASTFM');
+                        if (nonEmptyObj(data) || nonEmptyObj(p)) {
                             configs.push({
                                 type: 'lastfm',
                                 name: 'unnamed-lfm-source',
                                 source: 'ENV',
                                 mode: 'single',
                                 configureAs: 'source',
-                                data: lfm,
+                                data: data,
+                                ...p,
                                 options: transformPresetEnv('SOURCE_LASTFM')
                             })
                         }
                     }
                     break;
-                case 'deezer':
-                    const d = {
+                case 'deezer': {
+                    const data = removeUndefinedKeys({
                         clientId: process.env.DEEZER_CLIENT_ID,
                         clientSecret: process.env.DEEZER_CLIENT_SECRET,
                         redirectUri: process.env.DEEZER_REDIRECT_URI,
                         accessToken: process.env.DEEZER_ACCESS_TOKEN,
                         arl: process.env.DEEZER_ARL,
                         accountId: process.env.DEEZER_ACCOUNT_ID
-                    };
-                    if (!Object.values(d).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('DEEZER');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'deezer',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: d,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('DEEZER')
                         });
                     }
-                    break;
-                case 'mpris':
-                    const shouldUse = parseBool(process.env.MPRIS_ENABLE);
-                    const mp: MPRISData = {
+                }   break;
+                case 'mpris': {
+                    const data: MPRISData = removeUndefinedKeys<MPRISData>({
                         blacklist: process.env.MPRIS_BLACKLIST,
                         whitelist: process.env.MPRIS_WHITELIST
-                    }
-                    if (!Object.values(mp).every(x => x === undefined) || shouldUse) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('MPRIS');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'mpris',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: mp as MPRISData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('MPRIS')
                         });
                     }
-                    break;
+                }    break;
                 case 'maloja': {
-                    // env builder for single user mode
-                    const url = process.env.SOURCE_MALOJA_URL;
-                    const apiKey = process.env.SOURCE_MALOJA_API_KEY;
-                    if (url !== undefined || apiKey !== undefined) {
-                        const malojaData: MalojaData = {
-                            url,
-                            apiKey
-                        }
+                    const data = removeUndefinedKeys<MalojaData>({
+                        url: process.env.MALOJA_URL,
+                        apiKey: process.env.MALOJA_API_KEY
+                    }, false);
+                    const p = getCommonComponentEnvConfig('SOURCE_MALOJA');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'maloja',
                             name: 'unnamed-mlj-source',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'source',
-                            data: malojaData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('SOURCE_MALOJA')
                         })
                     }
                 }
                     break;
                 case 'librefm':{
-                    const shouldUse = parseBool(process.env.LIBRFM_ENABLE)
-                    const libre: LibrefmData = {
+                    const data: LibrefmData = removeUndefinedKeys<LibrefmData>({
                         apiKey: process.env.SOURCE_LIBREFM_API_KEY,
                         secret: process.env.SOURCE_LIBREFM_SECRET,
                         redirectUri: process.env.SOURCE_LIBREFM_REDIRECT_URI,
                         session: process.env.SOURCE_LIBREFM_SESSION,
                         urlBase: process.env.SOURCE_LIBREFM_URLBASE,
-                    };
-                    if (!Object.values(libre).every(x => x === undefined) || shouldUse) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('SOURCE_LIBREFM');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'librefm',
                             name: 'unnamed-librefm-source',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'source',
-                            data: libre,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('SOURCE_LIBREFM')
                         })
                     }
                 }
                     break;
                 case 'listenbrainz': {
-                    const lz: ListenBrainzData = {
+                    const data: ListenBrainzData = removeUndefinedKeys<ListenBrainzData>({
                         url: process.env.SOURCE_LZ_URL,
                         token: process.env.SOURCE_LZ_TOKEN,
                         username: process.env.SOURCE_LZ_USER
-                    };
-                    if (!Object.values(lz).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('SOURCE_LZ');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'listenbrainz',
                             name: 'unnamed-lz-source',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'source',
-                            data: lz,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('SOURCE_LZ')
                         })
                     }
                 }
                     break;
                 case 'koito': {
-                    const koit: KoitoData = {
+                    const data: KoitoData = removeUndefinedKeys<KoitoData>({
                         url: process.env.SOURCE_KOITO_URL,
                         token: process.env.SOURCE_KOITO_TOKEN,
                         username: process.env.SOURCE_KOITO_USER
-                    };
-                    if (!Object.values(koit).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('SOURCE_KOITO');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'koito',
                             name: 'unnamed-koito-source',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'source',
-                            data: koit,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('SOURCE_KOITO')
                         })
                     }
                 }
                     break;
                 case 'tealfm': {
-                    const teal: TealData = {
+                    const data: TealData = removeUndefinedKeys<TealData>({
                         identifier: process.env.SOURCE_TEALFM_IDENTIFIER,
                         appPassword: process.env.SOURCE_TEALFM_APP_PW,
-                    };
-                    if (!Object.values(teal).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('SOURCE_TEALFM');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'tealfm',
                             name: 'unnamed-tealfm-source',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'source',
-                            data: teal,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('SOURCE_TEALFM')
                         })
                     }
                 }
                     break;
                 case 'rocksky': {
-                    const rocksky: RockSkyData = {
+                    const data: RockSkyData = removeUndefinedKeys<RockSkyData>({
                         key: process.env.SOURCE_ROCKSKY_KEY,
                         handle: process.env.SOURCE_ROCKSKY_HANDLE
-                    };
-                    if (!Object.values(rocksky).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('SOURCE_ROCKSKY');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'rocksky',
                             name: 'unnamed-rocksky-source',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: 'source',
-                            data: rocksky,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('SOURCE_ROCKSKY')
                         })
                     }
                 }
                     break;
-                case 'endpointlz':
-                    const lzShouldUse = parseBool(process.env.LZENDPOINT_ENABLE);
-                    const lze: ListenbrainzEndpointData = {
+                case 'endpointlz': {
+                    const data: ListenbrainzEndpointData = removeUndefinedKeys<ListenbrainzEndpointData>({
                         slug: process.env.LZE_SLUG,
                         token: process.env.LZE_TOKEN,
                         username: process.env.LZE_USERNAME
-                    }
-                    if (!Object.values(lze).every(x => x === undefined) || lzShouldUse) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('LZE');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'endpointlz',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: lze as ListenbrainzEndpointData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('LZE')
                         });
                     }
-                    break;
-                case 'endpointlfm':
-                    const lfmShouldUse = parseBool(process.env.LFMENDPOINT_ENABLE);
-                    const lfme: LastFMEndpointData = {
+                }    break;
+                case 'endpointlfm': {
+                    const data: LastFMEndpointData = removeUndefinedKeys<LastFMEndpointData>({
                         slug: process.env.LFM_SLUG,
-                    }
-                    if (!Object.values(lfme).every(x => x === undefined) || lfmShouldUse) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('LFM');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'endpointlfm',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: lfme as LastFMEndpointData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('LFM')
                         });
                     }
-                    break;
+                }    break;
                 case 'icecast': {
                     const scrobbleStart = parseBool(process.env.ICECAST_SCROBBLE_START);
-                    const icecast: IcecastData = {
+                    const data: IcecastData = removeUndefinedKeys<IcecastData>({
                         url: process.env.ICECAST_URL,
-                    }
-                    if (!Object.values(icecast).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('ICECAST');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'icecast',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: icecast as IcecastData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv<IcecastSourceOptions>('ICECAST', {
                                 systemScrobble: scrobbleStart
                             })
                         });
                     }
                     }    break;
-                case 'jriver':
-                    const jr: JRiverData = {
+                case 'jriver': {
+                    const data: JRiverData = removeUndefinedKeys<JRiverData>({
                         url: process.env.JRIVER_URL,
                         username: process.env.JRIVER_USER,
                         password: process.env.JRIVER_PASSWORD
-                    }
-                    if (!Object.values(jr).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('JRIVER');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'jriver',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: jr as JRiverData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('JRIVER')
                         });
                     }
-                    break;
-                case 'kodi':
-                    const ko: KodiData = {
+                }   break;
+                case 'kodi': {
+                    const data: KodiData = removeUndefinedKeys<KodiData>({
                         url: process.env.KODI_URL,
                         username: process.env.KODI_USER,
                         password: process.env.KODI_PASSWORD
-                    }
-                    if (!Object.values(ko).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('KODI');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'kodi',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: ko as KodiData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('KODI')
                         });
                     }
-                    break;
-                case 'webscrobbler':
-                    const wsShouldUse = parseBool(process.env.WS_ENABLE);
-                    const ws: WebScrobblerData = {
+                }    break;
+                case 'webscrobbler': {
+                    const data: WebScrobblerData = removeUndefinedKeys<WebScrobblerData>({
                         blacklist: process.env.WS_BLACKLIST,
                         whitelist: process.env.WS_WHITELIST
-                    }
-                    if (!Object.values(ws).every(x => x === undefined) || wsShouldUse) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('WS');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'webscrobbler',
                             name: 'unnamed',
@@ -641,22 +674,23 @@ export default class ScrobbleSources {
                             mode: 'single',
                             configureAs: defaultConfigureAs,
                             data: {
-                                blacklist: ws.blacklist !== undefined ? (ws.blacklist as string).split(',') : [],
-                                whitelist: ws.whitelist !== undefined ? (ws.whitelist as string).split(',') : [],
+                                blacklist: data.blacklist !== undefined ? (data.blacklist as string).split(',') : [],
+                                whitelist: data.whitelist !== undefined ? (data.whitelist as string).split(',') : [],
                             },
+                            ...p,
                             options: transformPresetEnv('WS')
                         });
                     }
-                    break;
-                case 'chromecast':
-                    const ccShouldUse = parseBool(process.env.CC_ENABLE);
-                    const cc: ChromecastData = {
+                }   break;
+                case 'chromecast': {
+                    const data: ChromecastData = removeUndefinedKeys<ChromecastData>({
                         blacklistDevices: process.env.CC_BLACKLIST_DEVICES,
                         whitelistDevices: process.env.CC_WHITELIST_DEVICES,
                         blacklistApps: process.env.CC_BLACKLIST_APPS,
                         whitelistApps: process.env.CC_WHITELIST_APPS
-                    }
-                    if (!Object.values(cc).every(x => x === undefined) || ccShouldUse) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('CC');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'chromecast',
                             name: 'unnamed',
@@ -664,160 +698,177 @@ export default class ScrobbleSources {
                             mode: 'single',
                             configureAs: defaultConfigureAs,
                             data: {
-                                blacklistDevices: cc.blacklistDevices !== undefined ? (cc.blacklistDevices as string).split(',') : [],
-                                whitelistDevices: cc.whitelistDevices !== undefined ? (cc.whitelistDevices as string).split(',') : [],
-                                blacklistApps: cc.blacklistApps !== undefined ? (cc.blacklistApps as string).split(',') : [],
-                                whitelistApps: cc.whitelistApps !== undefined ? (cc.whitelistApps as string).split(',') : [],
+                                blacklistDevices: data.blacklistDevices !== undefined ? (data.blacklistDevices as string).split(',') : [],
+                                whitelistDevices: data.whitelistDevices !== undefined ? (data.whitelistDevices as string).split(',') : [],
+                                blacklistApps: data.blacklistApps !== undefined ? (data.blacklistApps as string).split(',') : [],
+                                whitelistApps: data.whitelistApps !== undefined ? (data.whitelistApps as string).split(',') : [],
                             },
+                            ...p,
                             options: transformPresetEnv('CC')
                         });
                     }
-                    break;
-                case 'musiccast':
-                    const musecase: MusicCastData = {
+                }    break;
+                case 'musiccast': {
+                    const data: MusicCastData = removeUndefinedKeys<MusicCastData>({
                         url: process.env.MCAST_URL,
-                    }
-                    if (!Object.values(musecase).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('MCAST');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'musiccast',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: musecase as MusicCastData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('MCAST')
                         });
                     }
-                    break;
-                case 'musikcube':
-                    const mc: MusikcubeData = {
+                }    break;
+                case 'musikcube': {
+                    const data: MusikcubeData = removeUndefinedKeys<MusikcubeData>({
                         url: process.env.MC_URL,
                         password: process.env.MC_PASSWORD
-                    }
-                    if (!Object.values(mc).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('MC');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'musikcube',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: mc as MusikcubeData,
+                            data: data as MusikcubeData,
+                            ...p,
                             options: transformPresetEnv('MC')
                         });
                     }
-                    break;
-                case 'mpd':
-                    const mpd: MPDData = {
+                }    break;
+                case 'mpd': {
+                    const data: MPDData = removeUndefinedKeys<MPDData>({
                         url: process.env.MPD_URL,
                         password: process.env.MPD_PASSWORD
-                    }
-                    if (!Object.values(mpd).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('MPD');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'mpd',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: mpd as MPDData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('MPD')
                         });
                     }
-                    break;
-                case 'vlc':
-                    const vlc: VLCData = {
+                }   break;
+                case 'vlc': {
+                    const data: VLCData = removeUndefinedKeys<VLCData>({
                         url: process.env.VLC_URL,
                         password: process.env.VLC_PASSWORD
-                    }
-                    if (!Object.values(vlc).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('VLC');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'vlc',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: vlc as VLCData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('VLC')
                         });
                     }
-                    break;
-                case 'ytmusic':
-                    const ytm: YTMusicData = {
+                }    break;
+                case 'ytmusic': {
+                    const data: YTMusicData = removeUndefinedKeys<YTMusicData>({
                         redirectUri: process.env.YTM_REDIRECT_URI,
                         clientId: process.env.YTM_CLIENT_ID,
                         clientSecret: process.env.YTM_CLIENT_SECRET,
                         cookie: process.env.YTM_COOKIE
-                    }
-                    if (!Object.values(ytm).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('YTM');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'ytmusic',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: ytm as YTMusicData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('YTM')
                         });
                     }
-                    break;
+                }    break;
                 case 'azuracast': {
-                    const azura: AzuracastData = {
+                    const data: AzuracastData = removeUndefinedKeys<AzuracastData>({
                         station: process.env.AZURA_STATION,
                         url: process.env.AZURA_URL,
                         apiKey: process.env.AZURA_KEY
-                    }
+                    }, false);
                     const listenerNum = process.env.AZURA_LISTENERS_NUM ?? '';
                     if(listenerNum.trim() !== '') {
-                        azura.monitorWhenListeners = !isNaN(Number.parseInt(listenerNum)) ? Number.parseInt(listenerNum) : parseBool(listenerNum);
+                        data.monitorWhenListeners = !isNaN(Number.parseInt(listenerNum)) ? Number.parseInt(listenerNum) : parseBool(listenerNum);
                     }
                     const live = process.env.AZURA_LIVE ?? '';
                     if(live.trim() !== '') {
-                        azura.monitorWhenLive = parseBool(live);
+                        data.monitorWhenLive = parseBool(live);
                     }
-                    if (!Object.values(azura).every(x => x === undefined)) {
+                    const p = getCommonComponentEnvConfig('AZURA');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'azuracast',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: azura as unknown as AzuracastData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('AZURA')
                         });
                     }
-                    }    break;
+                }    break;
                 case 'sonos': {
-                    const sonos: SonosData = {
+                    const data: SonosData = removeUndefinedKeys<SonosData>({
                         host: process.env.SONOS_HOST,
                         devicesAllow: process.env.SONOS_DEVICES_ALLOW,
                         devicesBlock: process.env.SONOS_DEVICES_BLOCK,
                         groupsAllow: process.env.SONOS_GROUPS_ALLOW,
                         groupsBlock: process.env.SONOS_GROUPS_BLOCK
-                    }
-                    if (!Object.values(sonos).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('SONOS');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'sonos',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: sonos as unknown as SonosData,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('SONOS')
                         });
                     }
                 }    break;
                 case 'ymbridge': {
-                    const yandex: YandexMusicBridgeData = {
+                    const data: YandexMusicBridgeData = removeUndefinedKeys<YandexMusicBridgeData>({
                         url: process.env.YMBRIDGE_URL,
                         apiKey: process.env.YMBRIDGE_API_KEY,
-                    }
-                    if (!Object.values(yandex).every(x => x === undefined)) {
+                    }, false);
+                    const p = getCommonComponentEnvConfig('YMBRIDGE');
+                    if (nonEmptyObj(data) || nonEmptyObj(p)) {
                         configs.push({
                             type: 'ymbridge',
                             name: 'unnamed',
                             source: 'ENV',
                             mode: 'single',
                             configureAs: defaultConfigureAs,
-                            data: yandex,
+                            data: data,
+                            ...p,
                             options: transformPresetEnv('YMBRIDGE')
                         });
                     }

src/backend/sources/SonosSource.ts

@@ -22,6 +22,7 @@ import { buildStatePlayerPlayIdententifyingInfo, hashObject, parseArrayFromMaybe
 import { isDebugMode, playObjDataMatch, sleep } from "../utils.js";
 import dayjs, { Dayjs } from "dayjs";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
+import { artistNamesToCredits } from "../../core/StringUtils.js";
 
 export interface DeviceState {
     device: SonosDevice
@@ -377,7 +378,7 @@ export const formatPlayObj = (obj: SonosState, options: FormatPlayObjectOptions
         data: {
             track: titleStr,
             album: Album,
-            artists: Artist === undefined ? undefined : [Artist],
+            artists: Artist === undefined ? undefined : artistNamesToCredits([Artist]),
             duration: dur,
         },
         meta: {

src/backend/sources/SpotifySource.ts

@@ -3,7 +3,7 @@ import EventEmitter from "events";
 import SpotifyWebApi from "spotify-web-api-node";
 import request from 'superagent';
 import { BrainzMeta, PlayObject, PlayObjectLifecycleless, SCROBBLE_TS_SOC_END, SCROBBLE_TS_SOC_START, ScrobbleTsSOC, SpotifyMeta } from "../../core/Atomic.js";
-import { combinePartsToString, truncateStringToLength } from "../../core/StringUtils.js";
+import { artistNamesToCredits, artistNameToCredit, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.js";
 import { isNodeNetworkException } from "../common/errors/NodeErrors.js";
 import { hasUpstreamError, UpstreamError } from "../common/errors/UpstreamError.js";
 import {
@@ -214,8 +214,8 @@ export default class SpotifySource extends MemoryPositionalSource implements Pag
 
         const play: PlayObjectLifecycleless = {
             data: {
-                artists: artists.map(x => x.name),
-                albumArtists: actualAlbumArtists.map(x => x.name),
+                artists: artists.map(x => artistNameToCredit(x.name)),
+                albumArtists: actualAlbumArtists.map(x => artistNameToCredit(x.name)),
                 album: albumName,
                 track: name,
                 duration: duration_ms / 1000,
@@ -232,7 +232,7 @@ export default class SpotifySource extends MemoryPositionalSource implements Pag
                 }
             },
             meta: {
-                deviceId: deviceId ?? `${NO_DEVICE}-${NO_USER}`,
+                deviceId: deviceId ?? `${NO_DEVICE}`,
                 source: 'Spotify',
                 musicService: 'Spotify',
                 trackId: id,
@@ -520,7 +520,7 @@ export default class SpotifySource extends MemoryPositionalSource implements Pag
                         platformId: [combinePartsToString([shortDeviceId(device.id), device.name]), NO_USER],
                         status,
                         play: item !== null && item !== undefined ? SpotifySource.formatPlayObj(res.body, {newFromSource: true}) : undefined,
-                        stateUpdatedAt: dayjs(timestamp),
+                        stateUpdatedAt: dayjs(),
                         position: progress_ms !== null && progress_ms !== undefined ? progress_ms / 1000 : undefined,
                     }
                 }

src/backend/sources/WebScrobblerSource.ts

@@ -23,6 +23,7 @@ import { Logger } from "@foxxmd/logging";
 import { PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js";
 import { NowPlayingPlayerState } from "./PlayerState/NowPlayingPlayerState.js";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
+import { artistCreditToName, artistNameToCredit } from "../../core/StringUtils.js";
 
 export class WebScrobblerSource extends MemorySource {
 
@@ -127,9 +128,9 @@ export class WebScrobblerSource extends MemorySource {
         const play: PlayObjectLifecycleless = {
             data: {
                 track,
-                artists: [artist],
+                artists: [artistNameToCredit(artist)],
                 album: album === null ? undefined : album,
-                albumArtists: albumArtist === null ? undefined : [albumArtist],
+                albumArtists: albumArtist === null ? undefined : [artistNameToCredit(albumArtist)],
                 playDate: dayjs.unix(startTimestamp),
                 duration: duration === null ? undefined : duration,
                 meta: {
@@ -155,7 +156,7 @@ export class WebScrobblerSource extends MemorySource {
         return baseFormatPlayObj(obj, play);
     }
 
-    getRecentlyPlayed = async (options = {}) => this.getFlatRecentlyDiscoveredPlays()
+    getRecentlyPlayed = async (options = {}) => await this.getFlatRecentlyDiscoveredPlays()
 
     isValidScrobble = (playObj: PlayObject) => {
         if (playObj.meta?.scrobbleAllowed === false) {

src/backend/sources/YTMusicSource.ts

@@ -18,11 +18,12 @@ import {
     playsAreSortConsistent
 } from "../utils/PlayComparisonUtils.js";
 import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js";
-import { buildTrackString, truncateStringToLength } from "../../core/StringUtils.js";
+import { artistNamesToCredits, buildTrackString, truncateStringToLength } from "../../core/StringUtils.js";
 import { joinedUrl } from "../utils/NetworkUtils.js";
 import { todayAwareFormat } from "../../core/TimeUtils.js";
 import { parseArrayFromMaybeString, parseArtistCredits, parseCredits } from "../utils/StringUtils.js";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
+import { FixedSizeList } from "fixed-size-list";
 
 export interface HistoryIngressResult {
     plays: PlayObject[], 
@@ -118,6 +119,7 @@ export default class YTMusicSource extends AbstractSource {
     declare config: YTMusicSourceConfig
 
     recentlyPlayed: PlayObject[] = [];
+    transientDiscovered: FixedSizeList<PlayObject> = new FixedSizeList<PlayObject>(200);
 
     yti: Innertube;
     userCode?: string;
@@ -149,6 +151,10 @@ export default class YTMusicSource extends AbstractSource {
                 this.config.options = {...rest, logDiff: diffVal};
             }
         }
+
+        this.emitter.on('discovered', (play) => {
+            this.transientDiscovered.add(play);
+        })
     }
 
     public additionalApiData(): Record<string, any> {
@@ -371,7 +377,7 @@ Redirect URI  : ${this.redirectUri}`);
             duration: dur, // string timestamp
         } = obj;
 
-        let artists = [],
+        let artists: string[] = [],
             album = undefined,
         duration = undefined;
         if(artistsData !== undefined) {
@@ -415,8 +421,8 @@ Redirect URI  : ${this.redirectUri}`);
         }
         const play: PlayObjectLifecycleless = {
             data: {
-                artists,
-                albumArtists,
+                artists: artistNamesToCredits(artists),
+                albumArtists: artistNamesToCredits(albumArtists),
                 album,
                 track: title,
                 duration,
@@ -588,7 +594,7 @@ Redirect URI  : ${this.redirectUri}`);
             if(consistent && newPlays.length > 1) {
                 const interimPlays = newPlays.slice(0, newPlays.length - 1);
                 // check enough time has passed since last discovery
-                const discovered = this.getFlatRecentlyDiscoveredPlays();
+                const discovered = this.transientDiscovered.data;
                 if(discovered.length > 0) {
                     const lastDiscovered = discovered[0].data.playDate;
                     // the assumption in behavior is that user skips 1 or more tracks which then get recorded to YTM history
@@ -681,10 +687,11 @@ ${humanDiff}`;
             const reversedPlays = [...referencePlays];
             // actual order they were discovered in (oldest to newest)
             reversedPlays.reverse();
-            if(this.getFlatRecentlyDiscoveredPlays().length === 0) {
+            if(this.transientDiscovered.data.length === 0) {
                 // and add to discovered since its empty
                 for(const refPlay of reversedPlays) {
-                    this.addPlayToDiscovered(refPlay);
+                    //this.transientDiscovered.add(refPlay);
+                    await this.addPlayToDiscovered(refPlay);
                 }
             }
         }

src/backend/sources/YandexMusicBridgeSource.ts

@@ -15,6 +15,7 @@ import { YandexMusicBridgeSourceConfig } from "../common/infrastructure/config/s
 import { isPortReachableConnect, joinedUrl, normalizeWebAddress } from "../utils/NetworkUtils.js";
 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
 import { UpstreamError } from "../common/errors/UpstreamError.js";
+import { artistNamesToCredits } from "../../core/StringUtils.js";
 
 interface BridgeTrackData {
     title?: string
@@ -437,7 +438,7 @@ const formatPlayObj = (obj: BridgeTrackData, playerId: string): PlayObject => {
 
     const play: PlayObjectLifecycleless = {
         data: {
-            artists,
+            artists: artistNamesToCredits(artists),
             album: obj.album ?? undefined,
             track: obj.title ?? undefined,
             duration: obj.duration_ms !== undefined && obj.duration_ms !== null

src/backend/tasks/heartbeatClients.ts

@@ -1,59 +0,0 @@
-import { childLogger, Logger } from '@foxxmd/logging';
-import { PromisePool } from "@supercharge/promise-pool";
-import { AsyncTask } from "toad-scheduler";
-import ScrobbleClients from "../scrobblers/ScrobbleClients.js";
-
-export const createHeartbeatClientsTask = (clients: ScrobbleClients, parentLogger: Logger) => {
-    const logger = childLogger(parentLogger, ['Heartbeat', 'Clients']);
-
-    return new AsyncTask(
-        'Heartbeat',
-        (): Promise<any> => {
-            logger.verbose('Starting check...');
-            return PromisePool
-                .withConcurrency(1)
-                .for(clients.clients)
-                .process(async (client) => {
-                    if(!client.isReady()) {
-                        if(!client.canAuthUnattended()) {
-                            client.logger.warn({labels: 'Heartbeat'}, 'Client is not ready but will not try to initialize because auth state is not good and cannot be correct unattended.')
-                            return 0;
-                        }
-                        try {
-                            await client.tryInitialize({force: false, notify: true, notifyTitle: 'Could not initialize automatically'});
-                        } catch (e) {
-                            client.logger.error(new Error('Could not initialize automatically', {cause: e}));
-                            return 1;
-                        }
-                    }
-
-                    if(!client.canAuthUnattended()) {
-                        client.logger.warn({label: 'Heartbeat'}, 'Should be monitoring scrobbles but will not attempt to start because auth state is not good and cannot be correct unattended.');
-                        return 0;
-                    }
-
-                    await client.processDeadLetterQueue();
-                    if(!client.scrobbling) {
-                        client.logger.info({labels: 'Heartbeat'}, 'Should be processing scrobbles! Attempting to restart scrobbling...');
-                        client.initScrobbleMonitoring();
-                        return 1;
-                    }
-                }).then(({results, errors}) => {
-                    logger.verbose(`Checked Dead letter queue for ${clients.clients.length} clients.`);
-                    const restarted = results.reduce((acc, curr) => acc += curr, 0);
-                    if (restarted > 0) {
-                        logger.info(`Attempted to start ${restarted} clients that were not processing scrobbles.`);
-                    }
-                    if (errors.length > 0) {
-                        logger.error(`Encountered errors!`);
-                        for (const err of errors) {
-                            logger.error(err);
-                        }
-                    }
-                });
-        },
-        (err: Error) => {
-            logger.error(err);
-        }
-    );
-}

src/backend/tasks/heartbeatSources.ts

@@ -1,65 +0,0 @@
-import { childLogger, Logger } from '@foxxmd/logging';
-import { PromisePool } from "@supercharge/promise-pool";
-import { AsyncTask } from "toad-scheduler";
-import { ChromecastSource } from "../sources/ChromecastSource.js";
-import ScrobbleSources from "../sources/ScrobbleSources.js";
-
-export const createHeartbeatSourcesTask = (sources: ScrobbleSources, parentLogger: Logger) => {
-    const logger = childLogger(parentLogger, ['Heartbeat', 'Sources']);
-
-    return new AsyncTask(
-        'Heartbeat',
-        (): Promise<any> => {
-            logger.verbose('Starting check...');
-            return PromisePool
-                .withConcurrency(1)
-                .for(sources.sources)
-                .process(async (source) => {
-                    if(!source.isReady()) {
-                        if(!source.canAuthUnattended()) {
-                            source.logger.warn({label: 'Heartbeat'}, 'Source is not ready but will not try to initialize because auth state is not good and cannot be correct unattended.');
-                            return 0;
-                        }
-                        try {
-                            await source.tryInitialize({force: false, notify: true, notifyTitle: 'Could not initialize automatically'});
-                        } catch (e) {
-                            source.logger.error(new Error('Could not initialize source automatically', {cause: e}));
-                            return 1;
-                        }
-                    }
-
-                    if(source.type === 'chromecast') {
-                        (source as ChromecastSource).discoverDevices();
-                    }
-
-                    if (source.canPoll && !source.polling) {
-                        if(!source.canAuthUnattended()) {
-                            source.logger.warn({label: 'Heartbeat'}, 'Should be polling but will not attempt to start because auth state is not good and cannot be correct unattended.');
-                            return 0;
-                        } else {
-                            source.logger.info({label: 'Heartbeat'}, 'Should be polling, attempting to start polling...');
-                            source.poll({force: false, notify: true}).catch(e => source.logger.error(e));
-                        }
-                        return 1;
-                    }
-
-                    return 0;
-                }).then(({results, errors}) => {
-                    logger.verbose(`Checked ${sources.sources.length} sources for start signals.`);
-                    const restarted = results.reduce((acc, curr) => acc += curr, 0);
-                    if (restarted > 0) {
-                        logger.info(`Attempted to start ${restarted} sources.`);
-                    }
-                    if (errors.length > 0) {
-                        logger.error(`Encountered errors!`);
-                        for (const err of errors) {
-                            logger.error(err);
-                        }
-                    }
-                });
-        },
-        (err: Error) => {
-            logger.error(err);
-        }
-    );
-}

src/backend/tasks/retentionCleanup.ts

@@ -0,0 +1,44 @@
+import { childLogger, Logger } from '@foxxmd/logging';
+import { AsyncTask } from "toad-scheduler";
+import ScrobbleSources from "../sources/ScrobbleSources.js";
+import ScrobbleClients from '../scrobblers/ScrobbleClients.js';
+
+export const createRetentionCleanupTask = (sources: ScrobbleSources, clients: ScrobbleClients, parentLogger: Logger) => {
+    const logger = childLogger(parentLogger, ['Schedule', 'Retention Cleanup']);
+
+    return new AsyncTask(
+        'Retention',
+        (): Promise<any> => {
+            return retentionTask(sources, clients, logger).then(() => null).catch((err) => {
+                logger.error(err);
+            });
+        },
+        (err: Error) => {
+            logger.error(err);
+        }
+    );
+}
+
+const retentionTask = async (sources: ScrobbleSources, clients: ScrobbleClients, logger: Logger): Promise<void> => {
+    // todo may want to implement abort controllers for these in case they don't finish
+
+    logger.verbose('Starting client cleanup...');
+    const validClients = clients.clients.filter(x => x.databaseOK);
+    const invalidClients = clients.clients.filter(x => x.databaseOK !== true);
+    if(invalidClients.length > 0) {
+       logger.debug(`Not running cleanup for ${invalidClients.length} clients because their database state is not OK: ${invalidClients.map(x => x.getSafeExternalName()).join(',')}`);
+    }    
+    const clientCleanupPromises = validClients.map(x => x.retentionCleanup().catch((err) => x.logger.warn(new Error('Failed to catch retention cleanup error!', {cause: err}))));
+    await Promise.all(clientCleanupPromises);
+    logger.verbose('Client cleanup done!');
+
+    logger.verbose('Starting source cleanup...');
+    const validSources = sources.sources.filter(x => x.databaseOK);
+    const invalidSources= sources.sources.filter(x => x.databaseOK !== true);
+    if(invalidClients.length > 0) {
+       logger.debug(`Not running cleanup for ${invalidSources.length} sources because their database state is not OK: ${invalidSources.map(x => x.getSafeExternalName()).join(',')}`);
+    }    
+    const sourceCleanupPromises = validSources.map(x => x.retentionCleanup().catch((err) => x.logger.warn(new Error('Failed to catch retention cleanup error!', {cause: err}))));
+    await Promise.all(sourceCleanupPromises);
+    logger.verbose('Source cleanup done!');
+}

src/backend/tests/cache/cache.test.ts

@@ -4,15 +4,10 @@ import asPromised from 'chai-as-promised';
 import { after, before, describe, it } from 'mocha';
 import dayjs from "dayjs";
 import withLocalTmpDir from 'with-local-tmp-dir';
-import { initFileCache, initMemoryCache, initValkeyCache, MSCache } from "../../common/Cache.js";
-import { generatePlays } from "../../../core/PlayTestUtils.js";
+import { initFileCache, initMemoryCache, initValkeyCache } from "../../common/Cache.js";
 import { ListenProgressPositional, ListenProgressTS } from "../../sources/PlayerState/ListenProgress.js";
 import { isPortReachableConnect } from "../../utils/NetworkUtils.js";
-import { getRoot } from "../../ioc.js";
-import { transientCache } from "../utils/CacheTestUtils.js";
-import { TestScrobbler } from "../scrobbler/TestScrobbler.js";
 import { sleep } from "../../utils.js";
-import {promises} from 'node:fs';
 
 chai.use(asPromised);
 
@@ -56,16 +51,24 @@ describe('#Caching', function () {
 
         it('File cache serializes and deserializes dayjs', async function () {
 
-            withLocalTmpDir(async () => {
+            await withLocalTmpDir(async () => {
+                // for some reason this *recreates* an empty cache after the test has finished (when running the full suite)
+                // i think its because of long persist interval compared to test time?
+                // 
+                // so:
+                // make intervals very small
+                // call destroy on both cache instances (shouldn't be necessary)
+                // sleep for longer than persist interal so tmp dir callback can (hopefully) properly delete any files
 
-                const [keyv, flat] = await initFileCache({ cacheDir: process.cwd() });
+                const [keyv, flat] = await initFileCache({ cacheDir: process.cwd(), persistInterval: 5, expirationInterval: 4 });
 
                 const now = dayjs();
 
                 await keyv.set('foo', now);
-                flat.save();
+                flat.save(true);
+                keyv.disconnect();
 
-                const [cleanKeyv, cleanFlat] = await initFileCache({ cacheDir: process.cwd() });
+                const [cleanKeyv, cleanFlat] = await initFileCache({ cacheDir: process.cwd(), persistInterval: 5, expirationInterval: 4 });
 
                 const time = await cleanKeyv.get('foo');
 
@@ -73,31 +76,9 @@ describe('#Caching', function () {
                 expect(time instanceof dayjs).is.true;
                 expect(now.toJSON()).eq((time as any).toJSON());
                 flat.destroy();
-
-            }, { unsafeCleanup: true });
-        });
-
-        it('File cache serializes and deserializes ListenProgress', async function () {
-
-            withLocalTmpDir(async () => {
-
-                const [keyv, flat] = await initFileCache({ cacheDir: process.cwd() });
-
-                const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 });
-
-                await keyv.set('foo', prog);
-                await flat.save();
-
-                const [cleanKeyv, cleanFlat] = await initFileCache({ cacheDir: process.cwd() });
-
-                const cachedProg = await cleanKeyv.get('foo');
-
-                expect(cachedProg).to.not.be.undefined;
-                expect(cachedProg instanceof ListenProgressTS).is.true;
-                expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON());
-                flat.destroy();
-
-            }, { unsafeCleanup: true });
+                cleanFlat.destroy();
+                await sleep(10);
+            }, { unsafeCleanup: true, postfix: 'fileCacheDajys' });
         });
     });
 
@@ -127,62 +108,5 @@ describe('#Caching', function () {
             expect(now.toJSON()).eq((time as any).toJSON());
 
         });
-
-        it('Valkey cache serializes and deserializes ListenProgress', async function () {
-
-            const keyv = await initValkeyCache('test', 'redis://valkey:6379');
-            await keyv.clear();
-
-            const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 });
-
-            await keyv.set('foo', prog);
-
-            const cachedProg = await keyv.get('foo');
-
-            expect(cachedProg).to.not.be.undefined;
-            expect(cachedProg instanceof ListenProgressTS).is.true;
-            expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON());
-
-        });
     });
-
-    describe('#ScrobbleCache', function () {
-
-        afterEach(function () {
-            const root = getRoot();
-            root.upsert({ cache: () => transientCache });
-            root.items.cache().init();
-        });
-
-        it('Preserves scrobbles', async function () {
-
-           this.timeout(100000);
-
-           // why does this take so long?
-           await withLocalTmpDir(async () => {
-
-                const root = getRoot();
-                root.upsert({ cache: () => () => new MSCache(loggerTest, { scrobble: { provider: 'file', connection: process.cwd(), persistInterval: 100 }, auth: {provider: 'memory'}, metadata: {provider: 'memory'} }) });
-
-                await using test = new TestScrobbler();
-                await test.initialize();
-                const plays = generatePlays(100, {}, {}, {listenRanges: true});
-                await test.queueScrobble(plays, 'testSource');
-                const queued = test.queuedScrobbles.map(x => x.play);
-                await sleep(101);
-                const dirContents = await promises.readdir('.');
-                const hasCache = dirContents.some(x => x === 'ms-scrobble.cache');
-                expect(hasCache).is.true;
-
-                await using newTest = new TestScrobbler();
-                await newTest.initialize();
-                expect(newTest.queuedScrobbles.length).to.eq(plays.length);
-                expect(newTest.queuedScrobbles[0].play.data.track).to.eq(queued[0].data.track);
-
-            }, { unsafeCleanup: true });
-
-        });
-
-    });
-
 });

src/backend/tests/component/transformers.test.ts

@@ -15,9 +15,10 @@ import { initMemoryCache } from "../../common/Cache.js";
 import { Cacheable } from "cacheable";
 import { TransformerCommonConfig } from "../../../core/Atomic.js";
 import TransformerManager from "../../common/transforms/TransformerManager.js";
-import { transientCache } from "../utils/CacheTestUtils.js";
+import { transientCache } from "../utils/TransientTestUtils.js";
 import dayjs from "dayjs";
 import clone from "clone";
+import { artistCreditsToNames, artistNamesToCredits } from "../../../core/StringUtils.js";
 
 chai.use(asPromised);
 
@@ -350,10 +351,10 @@ describe('Play Transforms', function () {
                     }
                     component.buildTransformRules();
 
-                    const play = generatePlay({ artists: ['My Artist One / My Artist Two / Another Guy'] });
+                    const play = generatePlay({ artists: artistNamesToCredits(['My Artist One / My Artist Two / Another Guy']) });
                     const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
                     expect(transformed.data.artists).length(1)
-                    expect(transformed.data.artists[0]).equal('My Artist One');
+                    expect(transformed.data.artists[0].name).equal('My Artist One');
                 });
 
                 it('Removes title when transform replaces with empty string', async function () {
@@ -402,10 +403,10 @@ describe('Play Transforms', function () {
                     }
                     component.buildTransformRules();
 
-                    const play = generatePlay({ artists: ['something', 'big'] });
+                    const play = generatePlay({ artists: artistNamesToCredits(['something', 'big']) });
                     const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
                     expect(transformed.data.artists!.length).is.eq(1)
-                    expect(transformed.data.artists![0]).is.eq('big')
+                    expect(transformed.data.artists![0].name).is.eq('big')
                 });
 
             });
@@ -421,10 +422,10 @@ describe('Play Transforms', function () {
             await t.tryInitialize();
 
             const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}});
-            const play = generatePlay({artists: [str]});
+            const play = generatePlay({artists: artistNamesToCredits([str])});
 
             const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play);
-            expect(transformedPlay.data.artists).eql(primaries.concat(secondaries));
+            expect(artistCreditsToNames(transformedPlay.data.artists)).eql(primaries.concat(secondaries));
         });
 
         it('Ignores artists', async function() {
@@ -435,10 +436,10 @@ describe('Play Transforms', function () {
 
             await t.tryInitialize();
 
-            const play = generatePlay({artists: [str], track: 'My Test'});
+            const play = generatePlay({artists: artistNamesToCredits([str]), track: 'My Test'});
 
             const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play);
-            expect(transformedPlay.data.artists).eql([str]);
+            expect(artistCreditsToNames(transformedPlay.data.artists)).eql([str]);
         });
 
         it('Uses custom delimiters artists', async function() {
@@ -456,10 +457,10 @@ describe('Play Transforms', function () {
 
             await t.tryInitialize();
 
-            const play = generatePlay({artists: [str], track: 'My Test'});
+            const play = generatePlay({artists: artistNamesToCredits([str]), track: 'My Test'});
 
             const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play);
-            expect(transformedPlay.data.artists).eql(primaries.concat(secondaries));
+            expect(artistCreditsToNames(transformedPlay.data.artists)).eql(primaries.concat(secondaries));
         });
 
     });
@@ -484,10 +485,10 @@ describe('Play Transforms', function () {
                 }
                 component.buildTransformRules();
 
-                const play = generatePlay({ artists: ['something', 'big'], album: 'It Has No Match' });
+                const play = generatePlay({ artists: artistNamesToCredits(['something', 'big']), album: 'It Has No Match' });
                 const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
                 expect(transformed.data.artists!.length).is.eq(2)
-                expect(transformed.data.artists![0]).is.eq('something')
+                expect(transformed.data.artists![0].name).is.eq('something')
             });
 
             it('Does run hook if when conditions matches', async function () {
@@ -507,10 +508,10 @@ describe('Play Transforms', function () {
                 }
                 component.buildTransformRules();
 
-                const play = generatePlay({ artists: ['something', 'big'], album: 'It Has This Match' });
+                const play = generatePlay({ artists: artistNamesToCredits(['something', 'big']), album: 'It Has This Match' });
                 const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
                 expect(transformed.data.artists!.length).is.eq(1)
-                expect(transformed.data.artists![0]).is.eq('big')
+                expect(transformed.data.artists![0].name).is.eq('big')
             });
         });
 
@@ -537,10 +538,10 @@ describe('Play Transforms', function () {
                 }
                 component.buildTransformRules();
 
-                const play = generatePlay({ artists: ['something', 'big'], album: 'It Has No Match' });
+                const play = generatePlay({ artists: artistNamesToCredits(['something', 'big']), album: 'It Has No Match' });
                 const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
                 expect(transformed.data.artists!.length).is.eq(2)
-                expect(transformed.data.artists![0]).is.eq('something')
+                expect(transformed.data.artists![0].name).is.eq('something')
             });
 
             it('Does run hook if when conditions matches', async function () {
@@ -565,10 +566,10 @@ describe('Play Transforms', function () {
                 }
                 component.buildTransformRules();
 
-                const play = generatePlay({ artists: ['something', 'big'], album: 'It Has This Match' });
+                const play = generatePlay({ artists: artistNamesToCredits(['something', 'big']), album: 'It Has This Match' });
                 const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
                 expect(transformed.data.artists!.length).is.eq(1)
-                expect(transformed.data.artists![0]).is.eq('big')
+                expect(transformed.data.artists![0].name).is.eq('big')
             });
         });
 
@@ -632,10 +633,10 @@ describe('Play Transforms', function () {
             const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}});
 
             component.buildTransformRules();
-            const play = generatePlay({ track: 'My cool something track', artists: [str] });
+            const play = generatePlay({ track: 'My cool something track', artists: artistNamesToCredits([str]) });
             const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
             expect(transformed.data.track).equal('My cool bar track');
-            expect(transformed.data.artists).eql(primaries.concat(secondaries));
+            expect(artistCreditsToNames(transformed.data.artists)).eql(primaries.concat(secondaries));
         });
 
     });
@@ -667,17 +668,17 @@ describe('Play Transforms', function () {
             const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}});
 
             component.buildTransformRules();
-            const play = generatePlay({ track: 'My cool something track', artists: [str], playDate: dayjs().subtract(10, 'm') });
+            const play = generatePlay({ track: 'My cool something track', artists: artistNamesToCredits([str]), playDate: dayjs().subtract(10, 'm') });
             const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare, 'all');
             expect(transformed.data.track).equal('My cool bar track');
-            expect(transformed.data.artists).eql(primaries.concat(secondaries));
+            expect(artistCreditsToNames(transformed.data.artists)).eql(primaries.concat(secondaries));
 
             const cachablePlay = clone(play);
             const laterDate = dayjs().subtract(5, 'm');
             cachablePlay.data.playDate = laterDate;
             const cacheTransformed = await component.transformPlay(cachablePlay, TRANSFORM_HOOK.preCompare, 'all');
             expect(cacheTransformed.data.track).equal('My cool bar track');
-            expect(cacheTransformed.data.artists).eql(primaries.concat(secondaries));
+            expect(artistCreditsToNames(cacheTransformed.data.artists)).eql(primaries.concat(secondaries));
             expect(cacheTransformed.data.playDate.isSame(cachablePlay.data.playDate));
         });
 

src/backend/tests/config/config.test.ts

@@ -44,7 +44,7 @@ describe('Sample Configs', function () {
             let reset: any;
 
             beforeEach(async function() {
-                reset = await withLocalTmpDir({unsafeCleanup: true});
+                reset = await withLocalTmpDir({unsafeCleanup: true, postfix: 'sourceConfigParse'});
             });
 
             afterEach(async function() {
@@ -78,7 +78,7 @@ describe('Sample Configs', function () {
             let reset: any;
 
             beforeEach(async function() {
-                reset = await withLocalTmpDir({unsafeCleanup: true});
+                reset = await withLocalTmpDir({unsafeCleanup: true, postfix: 'clientConfigParse'});
             });
 
             afterEach(async function() {

src/backend/tests/database/drizzle.test.ts

@@ -0,0 +1,689 @@
+import chai, { assert, expect } from 'chai';
+import asPromised from 'chai-as-promised';
+import { getDb, getMigratedDb, migrateDb, shouldBackupDb } from '../../common/database/drizzle/drizzleUtils.js';
+import withLocalTmpDir from 'with-local-tmp-dir';
+import { components, playInputs, plays, queueStates } from '../../common/database/drizzle/schema/schema.js';
+import dayjs from 'dayjs';
+import { generatePlay } from '../../../core/PlayTestUtils.js';
+import { getDbPath } from '../../common/database/Database.js';
+import { x } from 'tinyexec';
+import * as path from 'path';
+import * as fs from 'fs/promises';
+import { projectDir } from '../../common/index.js';
+import { DatabaseSync } from 'node:sqlite';
+import { fixtureCreateComponent, fixtureCreateInput, fixtureCreatePlay, getPrepopulatedFSPGlite, getPrepopulatedMemoryPGlite } from '../utils/databaseFixtures.js';
+import { DrizzlePlayRepository, RepositoryCreatePlayOpts } from '../../common/database/drizzle/repositories/PlayRepository.js';
+import { generatePlayWithLifecycle, generateRandomObj } from '../../../core/tests/utils/fixtures.js';
+import { formatNumber, generateArray } from '../../../core/DataUtils.js';
+import { objectsEqual } from '../../utils/DataUtils.js';
+import { eq, sql } from 'drizzle-orm';
+import { PlaySelect } from '../../common/database/drizzle/drizzleTypes.js';
+import { loggerDebug } from '@foxxmd/logging';
+import { transientDb } from '../utils/TransientTestUtils.js';
+import { dataDir } from '@electric-sql/pglite-prepopulatedfs'
+
+// would be great to push migrations directly from schema but doesn't seem supported in newest beta
+// https://github.com/drizzle-team/drizzle-orm/discussions/4373
+
+describe('Migrations', function () {
+
+    it('Detects non-existent db', async function () {
+
+        this.timeout(5000);
+
+        await withLocalTmpDir(async () => {
+            const [db, isNew] = await getMigratedDb(getDbPath('notreal', process.cwd()), { loadDataDir: dataDir() });
+            expect(isNew).is.true;
+            db.$client.close();
+        }, {postfix: 'noDb'});
+
+    });
+
+    it('Detects abnormal db', async function () {
+        this.timeout(5000);
+
+        // database exists but there is no __drizzle_migrations table
+        const db = await getDb(':memory:', { loadDataDir: dataDir() });
+        const [shouldBackup, pending] = await shouldBackupDb(db);
+        expect(shouldBackup).is.true;
+        db.$client.close();
+        expect(pending).length(0);
+
+    });
+
+    it('Detects pending migrations', async function () {
+
+        this.timeout(5000);
+
+        const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations'));
+        const migrationFiles = allFiles
+            .sort();
+
+        await withLocalTmpDir(async () => {
+
+            // copy first migration
+            await fs.mkdir('migrations');
+            try {
+                await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true });
+                const mf = path.resolve('./migrations');
+                const db = await getDb(':memory:', {loadDataDir: dataDir()});
+                await migrateDb(db, { migrationsFolder: mf });
+                const res = await x('drizzle-kit', [
+                    'generate',
+                    '--name',
+                    'newMigration',
+                    '--out',
+                    `${mf}`,
+                    '--custom',
+                    '--schema',
+                    path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'),
+                    '--dialect',
+                    'postgresql'
+                ], {throwOnError: true});
+                const [shouldBackup, pending] = await shouldBackupDb(db, { migrationsFolder: mf });
+                expect(shouldBackup).is.true;
+                expect(pending).length(1);
+                expect(pending[0]).includes('newMigration');
+                db.$client.close();
+            } catch (e) {
+                throw e;
+            }
+        }, { unsafeCleanup: true, postfix: 'pendingMigrations' });
+    });
+
+    it('Detects no pending migrations correctly', async function () {
+
+        this.timeout(5000);
+
+        const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations'));
+        const migrationFiles = allFiles
+            .sort();
+
+        await withLocalTmpDir(async () => {
+
+            // copy first migration
+            await fs.mkdir('migrations');
+            try {
+                await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true });
+                const mf = path.resolve('./migrations');
+                const db = await getDb(':memory:', {loadDataDir: dataDir()});
+                await migrateDb(db, { migrationsFolder: mf });
+                const [shouldBackup, pending] = await shouldBackupDb(db, { migrationsFolder: mf });
+                expect(shouldBackup).is.false;
+                expect(pending).length(0);
+                db.$client.close();
+            } catch (e) {
+                throw e;
+            }
+        }, { unsafeCleanup: true, postfix: 'noMigrations' });
+    });
+
+    it('Backs up database when migrations are pending', async function () {
+
+        // this can be slow due to all the io
+        this.timeout(5000);
+
+        const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations'));
+        const migrationFiles = allFiles
+            .sort();
+
+        await withLocalTmpDir(async () => {
+
+            const dbPath = getDbPath('msDb', process.cwd());
+            // copy first migration
+            await fs.mkdir('migrations');
+            try {
+                await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true });
+                const mf = path.resolve('./migrations');
+                const [db, _] = await getMigratedDb(dbPath, {migrationsFolder: mf, loadDataDir: dataDir()});
+                await db.$client.close();
+                const res = await x('drizzle-kit', [
+                    'generate',
+                    '--name',
+                    'newMigration',
+                    '--out',
+                    `${mf}`,
+                    '--custom',
+                    '--schema',
+                    path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'),
+                    '--dialect',
+                    'postgresql'
+                ], {throwOnError: true});
+
+                // add dummy data to migration so migrate() doesn't fail
+                const newMigrationFolder = (await fs.readdir(path.resolve('./migrations/'))).find(x => x.includes('newMigration'));
+                await fs.appendFile(path.resolve('./migrations/',newMigrationFolder, 'migration.sql'),`\nselect count(*) from plays;`);
+
+                await getMigratedDb(dbPath, {migrationsFolder: mf});
+                const contents = await fs.readdir(path.resolve('./'));
+                const backupPattern = new RegExp(/msDb-\d+\.bak/)
+                expect(contents.some(x => backupPattern.test(x))).is.true;
+            } catch (e) {
+                throw e;
+            }
+        }, { unsafeCleanup: true, postfix: 'dbBackup' });
+    });
+
+});
+
+describe('Basic DB Operations', function () {
+
+    it('Should create a play', async function () {
+
+        const db = await transientDb();
+
+        const component = await db.insert(components).values(fixtureCreateComponent()).returning();
+
+        const playRow = await db.insert(plays).values({
+            componentId: component[0].id,
+            state: 'queued',
+            playedAt: dayjs(),
+            seenAt: dayjs(),
+            play: generatePlay()
+        }).returning();
+
+        expect(playRow.length).eq(1);
+        db.$client.close();
+    });
+
+    it('Should create a play with relations', async function () {
+
+        const db = await transientDb();
+
+        try {
+
+            const component = await db.insert(components).values(fixtureCreateComponent()).returning();
+
+            const playRow = await db.insert(plays).values(fixtureCreatePlay({ componentId: component[0].id })).returning();
+
+            const input = await db.insert(playInputs).values(fixtureCreateInput({
+                playId: playRow[0].id,
+                play: playRow[0].play
+            })).returning();
+
+            const twoQueues = await db.insert(queueStates).values([
+                {
+                    playId: playRow[0].id,
+                    componentId: component[0].id,
+                    queueName: 'foo'
+                },
+                {
+                    playId: playRow[0].id,
+                    componentId: component[0].id,
+                    queueName: 'bar',
+                    queueStatus: 'completed'
+                }
+            ]);
+
+            const fullPlay = await db.query.plays.findFirst({
+                with: {
+                    input: true,
+                    queueStates: true,
+                },
+            });
+
+
+            expect(fullPlay.queueStates).to.not.be.undefined;
+            expect(fullPlay.queueStates).length(2);
+
+            expect(fullPlay.input).to.not.be.undefined;
+
+        } catch (e) {
+            throw e;
+        }
+        db.$client.close();
+    });
+
+    it('deletes all dependent relations when a Play is deleted', async function () {
+
+        const db = await transientDb();
+
+        try {
+
+            const component = await db.insert(components).values(fixtureCreateComponent()).returning();
+
+            const playRow = await db.insert(plays).values(fixtureCreatePlay({ componentId: component[0].id })).returning();
+
+            const input = await db.insert(playInputs).values(fixtureCreateInput({
+                playId: playRow[0].id,
+                play: playRow[0].play
+            })).returning();
+
+            const twoQueues = await db.insert(queueStates).values([
+                {
+                    playId: playRow[0].id,
+                    componentId: component[0].id,
+                    queueName: 'foo'
+                },
+                {
+                    playId: playRow[0].id,
+                    componentId: component[0].id,
+                    queueName: 'bar',
+                    queueStatus: 'completed'
+                }
+            ]).returning();
+
+            const fullPlay = await db.query.plays.findFirst({
+                with: {
+                    input: true,
+                    queueStates: true,
+                },
+            });
+
+
+            expect(fullPlay.queueStates).to.not.be.undefined;
+            expect(fullPlay.queueStates).length(2);
+            expect(fullPlay.input).to.not.be.undefined;
+
+            await db.delete(plays).where(eq(plays.id, fullPlay.id));
+            const deletedPlay = await db.query.plays.findFirst({
+                where: {
+                    id: fullPlay.id
+                }
+            });
+            expect(deletedPlay).to.be.undefined;
+
+            const deletedInput = await db.query.playInputs.findFirst({
+                where: {
+                    id: input[0].id
+                }
+            });
+            expect(deletedInput).to.be.undefined;
+
+            const deletedQueues = await db.query.queueStates.findMany({
+                where: {
+                    id: {
+                        in: [twoQueues[0].id, twoQueues[1].id]
+                    }
+                }
+            });
+            expect(deletedQueues).length(0);
+        } catch (e) {
+            throw e;
+        }
+        db.$client.close();
+    });
+
+});
+
+describe('Repository Operations', function () {
+
+    it('creates Plays and inputs', async function () {
+
+        const db = await transientDb();
+
+        const component = await db.insert(components).values(fixtureCreateComponent()).returning();
+
+        const repo = new DrizzlePlayRepository(db, {componentId: component[0].id});
+
+        const numPlays = 3;
+
+        const playData = generateArray<RepositoryCreatePlayOpts>(numPlays, () => ({ ...fixtureCreatePlay(), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } }))
+
+        const rows = await repo.createPlays(playData);
+        expect(rows).length(numPlays);
+        const fullPlays = await db.query.plays.findMany({
+            with: {
+                input: true
+            }
+        });
+        fullPlays.forEach((play, index) => {
+            const ref = playData[index];
+
+            expect(play.play.data.track).eq(ref.play.data.track);
+            expect(play.input).to.not.undefined;
+            expect(objectsEqual(play.input.data, ref.input.data)).is.true;
+        })
+
+    });
+
+    it('finds Plays by state', async function () {
+
+       const db = await transientDb();
+        await migrateDb(db);
+
+        const component = await db.insert(components).values(fixtureCreateComponent()).returning();
+
+        const repo = new DrizzlePlayRepository(db, {componentId: component[0].id});
+
+        const numPlays = 3;
+
+        const playData = generateArray<RepositoryCreatePlayOpts>(numPlays, () => ({
+            ...fixtureCreatePlay(),
+            state: 'queued',
+            input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+        }));
+        const discovered = {
+            ...fixtureCreatePlay(),
+            state: 'discovered' as 'discovered',
+            input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+        };
+        playData.push(discovered)
+
+        await repo.createPlays(playData);
+
+        const plays = await repo.findPlays({ state: ['discovered'] });
+        expect(plays).length(1);
+        expect(plays[0].play.data.track).eq(discovered.play.data.track);
+    });
+
+    it('finds Plays by date range', async function () {
+
+        const db = await transientDb();
+        await migrateDb(db);
+
+        const component = await db.insert(components).values(fixtureCreateComponent()).returning();
+
+        const repo = new DrizzlePlayRepository(db, {componentId: component[0].id});
+
+        const playData: RepositoryCreatePlayOpts[] = [
+            {
+                ...fixtureCreatePlay({ play: generatePlay({ playDate: dayjs().subtract(2, 'm') }) }),
+                state: 'queued' as 'queued',
+                input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+            },
+            {
+                ...fixtureCreatePlay({ play: generatePlay({ playDate: dayjs().subtract(6, 'm') }) }),
+                state: 'queued' as 'queued',
+                input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+            },
+            {
+                ...fixtureCreatePlay({ play: generatePlay({ playDate: dayjs().subtract(8, 'm') }) }),
+                state: 'queued' as 'queued',
+                input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+            },
+            {
+                ...fixtureCreatePlay({ play: generatePlay({ playDate: dayjs().subtract(10, 'm') }) }),
+                state: 'queued' as 'queued',
+                input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+            },
+        ]
+
+        await repo.createPlays(playData);
+
+        const newerPlays = await repo.findPlays({ playedAt: { type: 'gt', date: dayjs().subtract(3, 'm') } });
+        expect(newerPlays).length(1);
+        expect(newerPlays[0].play.data.track).eq(playData[0].play.data.track);
+
+        const olderPlays = await repo.findPlays({ playedAt: { type: 'lt', date: dayjs().subtract(6, 'm').subtract(5, 's') } });
+        expect(olderPlays).length(2);
+        expect(olderPlays[0].play.data.track).eq(playData[2].play.data.track);
+        expect(olderPlays[1].play.data.track).eq(playData[3].play.data.track);
+
+        const bwPlays = await repo.findPlays({ playedAt: { type: 'between', range: [dayjs().subtract(9, 'm'), dayjs().subtract(3, 'm')] } });
+        expect(bwPlays).length(2);
+        expect(bwPlays[0].play.data.track).eq(playData[1].play.data.track);
+        expect(bwPlays[1].play.data.track).eq(playData[2].play.data.track);
+    });
+
+    it('finds Plays by component', async function () {
+
+        const db = await transientDb();
+        await migrateDb(db);
+
+        const component1 = await db.insert(components).values(fixtureCreateComponent()).returning();
+        const component2 = await db.insert(components).values(fixtureCreateComponent({ uid: 'test2', name: 'jelly2' })).returning();
+        const component3 = await db.insert(components).values(fixtureCreateComponent({ uid: 'test3', name: 'jelly3' })).returning();
+
+        const repo = new DrizzlePlayRepository(db);
+
+        const playData: RepositoryCreatePlayOpts[] = [
+            {
+                ...fixtureCreatePlay(),
+                componentId: component1[0].id,
+                state: 'queued' as 'queued',
+                input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+            },
+            {
+                ...fixtureCreatePlay(),
+                componentId: component3[0].id,
+                state: 'queued' as 'queued',
+                input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+            },
+            {
+                ...fixtureCreatePlay(),
+                componentId: component3[0].id,
+                state: 'queued' as 'queued',
+                input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+            }
+        ]
+
+        await repo.createPlays(playData);
+
+        const plays = await repo.findPlays({ componentId: component3[0].id });
+        expect(plays).length(2);
+        expect(plays[0].play.data.track).eq(playData[1].play.data.track);
+        expect(plays[1].play.data.track).eq(playData[2].play.data.track);
+
+        const plays1 = await repo.findPlays({ componentId: component1[0].id });
+        expect(plays1).length(1);
+        expect(plays1[0].play.data.track).eq(playData[0].play.data.track);
+
+        const noPlays = await repo.findPlays({ componentId: component2[0].id });
+        expect(noPlays).length(0);
+    });
+
+    it('finds purgable Plays', async function () {
+
+        const db = await transientDb();
+        await migrateDb(db);
+
+        const component1 = await db.insert(components).values(fixtureCreateComponent()).returning();
+        const component2 = await db.insert(components).values(fixtureCreateComponent({ uid: 'test2', name: 'jelly2' })).returning();
+
+        const repo = new DrizzlePlayRepository(db);
+
+        const playData: RepositoryCreatePlayOpts[] = [
+            {
+                ...fixtureCreatePlay({play: generatePlay({},{seenAt: dayjs().subtract(25, 'h')})}),
+                componentId: component1[0].id,
+                state: 'queued' as 'queued',
+                input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+            },
+            {
+                ...fixtureCreatePlay({play: generatePlay({},{seenAt: dayjs().subtract(26, 'h')})}),
+                componentId: component1[0].id,
+                state: 'queued' as 'queued',
+                input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+            },
+            {
+                ...fixtureCreatePlay({play: generatePlay({},{seenAt: dayjs().subtract(26, 'h')})}),
+                componentId: component2[0].id,
+                state: 'queued' as 'queued',
+                input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+            },
+            {
+                ...fixtureCreatePlay({play: generatePlay({},{seenAt: dayjs().subtract(25, 'h').subtract(1, 'm')})}),
+                componentId: component1[0].id,
+                state: 'queued' as 'queued',
+                input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+            },
+        ]
+
+        const initialPlays = await repo.createPlays(playData);
+
+        const childPlays = await repo.createPlays([
+            {
+                ...fixtureCreatePlay({play: generatePlay({},{seenAt: dayjs().subtract(25, 'h')})}),
+                componentId: component2[0].id,
+                parentId: initialPlays[1].id,
+                state: 'queued' as 'queued',
+                input: { data: generateRandomObj(undefined, { allowUndefined: false }) }
+            },
+        ])
+
+        // does not return newer plays
+        expect((await repo.findPurgablePlayIds(dayjs().subtract(27, 'h'), {componentId: 1}))).length(0);
+
+        const pPlays = await repo.findPurgablePlayIds(dayjs().subtract(24, 'h'), {componentId: 1});
+        // only returns plays that do not have children
+        expect(pPlays).length(2);
+        expect(pPlays[0]).to.eq(initialPlays[0].id);
+        expect(pPlays[1]).to.eq(initialPlays[3].id);
+
+        // only finds plays by component
+        // and allows purging if they have parent id
+        const p2Plays = await repo.findPurgablePlayIds(dayjs().subtract(23, 'h'), {componentId: 2});
+        expect(p2Plays).length(2);
+        expect(p2Plays[0]).to.eq(initialPlays[2].id);
+        expect(p2Plays[1]).to.eq(childPlays[0].id);
+    });
+
+    //     it('Get json property from play', async function () {
+
+    //     const db = getDb(':memory:', { workingDirectory: process.cwd() });
+    //     await migrateDb(db);
+
+    //     try {
+
+    //         const component = await db.insert(components).values(fixtureCreateComponent()).returning();
+
+    //         const playRows = await db.insert(plays).values([
+    //             fixtureCreatePlay({ componentId: component[0].id, play: generatePlay({}, {source: 'test1'}) }),
+    //             fixtureCreatePlay({ componentId: component[0].id, play: generatePlay({}, {source: 'test2'}) })
+    //         ]).returning();
+
+    //         let result: PlaySelect[];
+    //         // https://github.com/drizzle-team/drizzle-orm/discussions/938#discussioncomment-6542336
+    //         result = await db.select().from(plays).where(
+    //             sql`json_extract(${plays.play}, '$.meta.source') = 'test1'`
+    //         );
+
+    //         expect(result).length(1);
+    //         expect(result[0].play.meta.source).eq('test1');
+
+    //     } catch (e) {
+    //         throw e;
+    //     }
+    //     db.$client.close();
+    // });
+
+});
+
+describe('DB Size Stats', function () {
+
+
+    before(function () {
+        if (process.env.DB_SIZE_TEST !== 'true') {
+            this.skip();
+        }
+    });
+
+    it('get empty db size stats', async function () {
+
+        await withLocalTmpDir(async () => {
+            try {
+                let db = await getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
+                await migrateDb(db);
+                const play100Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]);
+                loggerDebug.debug(`100 Plays => ${formatNumber((play100Component / 1024) / 1024, {toFixed: 2})}mb`);
+            } catch (e) {
+                throw e;
+            }
+        }, { unsafeCleanup: true, postfix: 'dbStatEmpty' });
+    });
+
+    it('get db plays size stats', async function () {
+
+        this.timeout(10000);
+
+        await withLocalTmpDir(async () => {
+            try {
+                let db = await getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
+                await migrateDb(db);
+                const component = await db.insert(components).values(fixtureCreateComponent()).returning();
+
+                const playRepo = new DrizzlePlayRepository(db);
+                const playData = generateArray<RepositoryCreatePlayOpts>(100, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlay() }), state: 'queued', input: { data: undefined } }));
+                await playRepo.createPlays(playData);
+
+                const play100Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]);
+                loggerDebug.debug(`100 Plays => ${formatNumber((play100Component / 1024) / 1024, {toFixed: 2})}mb`);
+
+                const morePlayData = generateArray<RepositoryCreatePlayOpts>(900, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlay() }), state: 'queued', input: { data: undefined }}));
+                await playRepo.createPlays(morePlayData);
+                const play1000Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]);
+                loggerDebug.debug(`1000 Plays => ${formatNumber((play1000Component / 1024) / 1024, {toFixed: 2})}mb`);
+
+                for(let i = 0; i < 9; i++) {
+                    const evenMorePlayData = generateArray<RepositoryCreatePlayOpts>(1000, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlay() }), state: 'queued', input: { data: undefined }}));
+                    await playRepo.createPlays(evenMorePlayData);
+                }
+
+                const play10000Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]);
+                loggerDebug.debug(`10000 Plays => ${formatNumber((play10000Component / 1024) / 1024, {toFixed: 2})}mb`);
+            } catch (e) {
+                throw e;
+            }
+        }, { unsafeCleanup: true, postfix: 'dbStatPlain' });
+    });
+    
+    it('get db plays size stats with input', async function () {
+
+        this.timeout(10000);
+
+        await withLocalTmpDir(async () => {
+            try {
+                let db = await getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
+                await migrateDb(db);
+                const component = await db.insert(components).values(fixtureCreateComponent()).returning();
+
+                const playRepo = new DrizzlePlayRepository(db);
+                const playData = generateArray<RepositoryCreatePlayOpts>(100, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlay() }), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } }));
+                await playRepo.createPlays(playData);
+
+                const play100Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]);
+                loggerDebug.debug(`100 Plays => ${formatNumber((play100Component / 1024) / 1024, {toFixed: 2})}mb`);
+
+                const morePlayData = generateArray<RepositoryCreatePlayOpts>(900, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlay() }), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } }));
+                await playRepo.createPlays(morePlayData);
+                const play1000Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]);
+                loggerDebug.debug(`1000 Plays => ${formatNumber((play1000Component / 1024) / 1024, {toFixed: 2})}mb`);
+
+                for(let i = 0; i < 9; i++) {
+                    const evenMorePlayData = generateArray<RepositoryCreatePlayOpts>(1000, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlay() }), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } }));
+                    await playRepo.createPlays(evenMorePlayData);
+                }
+                const play10000Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]);
+                loggerDebug.debug(`10000 Plays => ${formatNumber((play10000Component / 1024) / 1024, {toFixed: 2})}mb`);
+            } catch (e) {
+                throw e;
+            }
+        }, { unsafeCleanup: true, postfix: 'dbStatInput' });
+    });
+
+    it('get db plays size stats with input and lifecycle', async function () {
+
+        this.timeout(10000);
+
+        await withLocalTmpDir(async () => {
+            try {
+                let db = await getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
+                await migrateDb(db);
+                const component = await db.insert(components).values(fixtureCreateComponent()).returning();
+
+                const playRepo = new DrizzlePlayRepository(db);
+                const playData = generateArray<RepositoryCreatePlayOpts>(100, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlayWithLifecycle({lifecycleSteps: {preCompare: 1}}) }), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } }));
+                await playRepo.createPlays(playData);
+
+                const play100Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]);
+                loggerDebug.debug(`100 Plays => ${formatNumber((play100Component / 1024) / 1024, {toFixed: 2})}mb`);
+
+                const morePlayData = generateArray<RepositoryCreatePlayOpts>(900, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlayWithLifecycle({lifecycleSteps: {preCompare: 1}}) }), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } }));
+                await playRepo.createPlays(morePlayData);
+                const play1000Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]);
+                loggerDebug.debug(`1000 Plays => ${formatNumber((play1000Component / 1024) / 1024, {toFixed: 2})}mb`);
+
+                for(let i = 0; i < 9; i++) {
+                    const evenMorePlayData = generateArray<RepositoryCreatePlayOpts>(1000, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlayWithLifecycle({lifecycleSteps: {preCompare: 1}}) }), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } }));
+                    await playRepo.createPlays(evenMorePlayData);
+                }
+
+                const play10000Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]);
+                loggerDebug.debug(`10000 Plays => ${formatNumber((play10000Component / 1024) / 1024, {toFixed: 2})}mb`);
+            } catch (e) {
+                throw e;
+            }
+        }, { unsafeCleanup: true, postfix: 'dbStatAll' });
+    });
+})