trying to determine why gh actions times out...

TODO migration check for db existence and abnormal db

.devcontainer/devcontainer.json

@@ -17,7 +17,7 @@
 	// "features": {},
 
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
-	"forwardPorts": [9078,3000],
+	"forwardPorts": [9078,3000,5433],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
 	"postCreateCommand": ".devcontainer/postCreate.sh",
@@ -30,7 +30,9 @@
 				"dbaeumer.vscode-eslint",
 				"unifiedjs.vscode-mdx",
 				"bradlc.vscode-tailwindcss",
-				"TakumiI.markdowntable"
+				"TakumiI.markdowntable",
+				"YoavBls.pretty-ts-errors",
+				"bowlerr.sqlite-intelliview-vscode"
 			]
 		}
 	},
@@ -40,6 +42,9 @@
 		},
 		"9078": {
 			"label": "App"
+		},
+		"5433": {
+			"label": "PGLite Server"
 		}
 	}
 

.dockerignore

@@ -13,6 +13,7 @@ config/*.json
 config/*.bak
 config/*.p8
 /docs
+dist
 /logs
 .flatpak-builder
 **/generated-sources.*

.github/dependabot.yml

@@ -6,6 +6,11 @@
 version: 2
 updates:
   - package-ecosystem: "npm" # See documentation for possible values
+    groups:
+      sec-updates:
+        applies-to: "security-updates"
+        patterns:
+          - "*"
     directories:
       - "/"
       - "/docsite"
@@ -17,3 +22,16 @@ updates:
     # Check for updates once a week
     schedule:
       interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    groups:
+      sec-updates:
+        applies-to: "security-updates"
+        patterns:
+          - "*"
+      version-updates:
+        applies-to: "version-updates"
+        patterns:
+          - "*"
+    schedule:
+      interval: "weekly"

.github/pull_request_template.md

@@ -1,6 +1,6 @@
 ## Checklist before requesting a review
 
-- [ ] I have read the [contributing guidelines.](../CONTRIBUTING.md)
+- [ ] I have read the [contributing guidelines.](../blob/master/CONTRIBUTING.md)
 
 ## Type of change
 

.github/workflows/docsDeploy.yml

@@ -12,12 +12,12 @@ jobs:
     name: Deploy to GitHub Pages
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 #v4.3.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
         with:
           persist-credentials: false
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 #4.4.0
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f #6.3.0
         with:
-          node-version: 20
+          node-version: 24
           cache: npm
 
       - name: Install app dev dependencies
@@ -28,22 +28,24 @@ jobs:
       - name: Install docusaurus dependencies
         run: npm ci
         working-directory: ./docsite
+
       - name: Build website
         env:
-          ANALYTICS: ${{ vars.ANALYTICS }}
-          ANALYTICS_DOMAIN: ${{ vars.ANALYTICS_DOMAIN }}
           RY_ANALYTICS: ${{ vars.RY_ANALYTICS }}
           RY_ANALYTICS_SITEID: ${{ vars.RY_ANALYTICS_SITEID }}
-          RY_ANALYTICS_REPLAY: ${{ vars.RY_ANALYTICS_REPLAY }}
           GSITEVERIFICATION: ${{ vars.GSITEVERIFICATION }}
           DOCS_BASE: '/multi-scrobbler'
         run: npm run build
         working-directory: ./docsite
 
+      - name: Replace canonical
+        working-directory: ./docsite
+        run: node --experimental-strip-types canonical-replace.ts ./build
+
       # Popular action to deploy to GitHub Pages:
       # Docs: https://github.com/peaceiris/actions-gh-pages#%EF%B8%8F-docusaurus
       - name: Deploy to GitHub Pages
-        uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 #3.9.3
+        uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e #4.0.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           # Build output to publish to the `gh-pages` branch:
@@ -55,3 +57,24 @@ jobs:
           # You can swap them out with your own user credentials.
           user_name: github-actions[bot]
           user_email: 41898282+github-actions[bot]@users.noreply.github.com
+
+      - name: Trigger dependent build
+        if: ${{ vars.DOCS_REPO  != ''}}
+        run: |
+          # Trigger builds in dependent repositories
+          curl -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${{ secrets.DOCS_DISPATCH_TOKEN }}" \
+            "https://api.github.com/repos/${{ vars.DOCS_REPO }}/dispatches" \
+            -d "{
+              \"event_type\": \"release\",
+              \"client_payload\": {
+                \"commit\": \"${{ github.sha }}\"
+              }
+            }"
+      - name: Trigger CF Pages build
+        if: ${{ vars.CF_PAGES == 'true' }}
+        run: |
+          # trigger CF pages hook deployment
+          curl -d "" \
+            "${{ secrets.CF_PAGES_HOOK }}"

.github/workflows/pr.yml

@@ -51,7 +51,9 @@ jobs:
           strings=("${VARS_DOCKERHUB_SLUG}" "${VARS_GHCR_SLUG}")
           for i in ${!strings[@]}; do [[ -z ${strings[i]} ]] && unset strings[i]; done
           joined_string=$(IFS=, ; echo "${strings[*]}")
-          echo "REGISTRIES_JOINED=$joined_string" >> $GITHUB_ENV 
+          echo "REGISTRIES_JOINED=$joined_string" >> $GITHUB_ENV
+          date=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+          echo "BUILD_DATE=$date" >> $GITHUB_ENV
         env:
           VARS_DOCKERHUB_SLUG: ${{ vars.DOCKERHUB_SLUG }}
           VARS_GHCR_SLUG: ${{vars.GHCR_SLUG}}
@@ -104,6 +106,7 @@ jobs:
           context: ${{github.event.pull_request.head.repo.clone_url}}#${{github.event.pull_request.head.sha}}
           build-args: |
             APP_BUILD_VERSION=${{steps.appversion.outputs.appversion}}
+            BUILD_DATE=${{ env.BUILD_DATE }}
           platforms: ${{ matrix.platform }}
           labels: ${{ steps.meta.outputs.labels }}
           outputs: type=image,"name=${{ env.REGISTRIES_JOINED }}",push-by-digest=true,name-canonical=true,push=true
@@ -117,7 +120,7 @@ jobs:
           touch "/tmp/digests/${digest#sha256:}"          
       
       - name: Upload digest
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 #4.6.2
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #7.0.0
         with:
           name: digests-${{ env.PLATFORM_PAIR }}
           path: /tmp/digests/*
@@ -136,7 +139,7 @@ jobs:
       - test
     steps:
       - name: Download digests
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 #4.3.0
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #8.0.1
         with:
           path: /tmp/digests
           pattern: digests-*
@@ -162,7 +165,7 @@ jobs:
       
       - name: Extract metadata (tags, labels)
         id: meta
-        uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 #5.9.0
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf #6.0.0
         with:
           images: |
             ${{ vars.DOCKERHUB_SLUG }}
@@ -220,7 +223,7 @@ jobs:
     if: contains(github.event.pull_request.labels.*.name, 'safe to test')
     steps:
       - name: Create comment
-        uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 #2.9.4
+        uses: marocchino/sticky-pull-request-comment@d4d6b0936434b21bc8345ad45a440c5f7d2c40ff #3.0.3
         with:
           recreate: true
           header: "pr-release"

.github/workflows/publishImage.yml

@@ -66,13 +66,15 @@ jobs:
           for i in ${!strings[@]}; do [[ -z ${strings[i]} ]] && unset strings[i]; done
           joined_string=$(IFS=, ; echo "${strings[*]}")
           echo "REGISTRIES_JOINED=$joined_string" >> $GITHUB_ENV 
+          date=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+          echo "BUILD_DATE=$date" >> $GITHUB_ENV
         env:
           VARS_DOCKERHUB_SLUG: ${{ vars.DOCKERHUB_SLUG }}
           VARS_GHCR_SLUG: ${{vars.GHCR_SLUG}}
           
 
       - name: Check out the repo
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 #v4.3.0
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
         with:
           persist-credentials: false
 
@@ -126,6 +128,7 @@ jobs:
         with:
           build-args: |
             APP_BUILD_VERSION=${{steps.appversion.outputs.appversion}}
+            BUILD_DATE=${{ env.BUILD_DATE }}
           platforms: ${{ matrix.platform }}
           labels: ${{ steps.meta.outputs.labels }}
           outputs: type=image,"name=${{ env.REGISTRIES_JOINED }}",push-by-digest=true,name-canonical=true,push=true
@@ -140,7 +143,7 @@ jobs:
         env:
             VARS_DIGEST: ${{ steps.build.outputs.digest }}      
       - name: Upload digest
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 #4.6.2
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #7.0.0
         with:
           name: digests-${{ env.PLATFORM_PAIR }}
           path: /tmp/digests/*
@@ -159,7 +162,7 @@ jobs:
       - test
     steps:
       - name: Download digests
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 #4.3.0
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #8.0.1
         with:
           path: /tmp/digests
           pattern: digests-*
@@ -185,7 +188,7 @@ jobs:
       
       - name: Extract metadata (tags, labels)
         id: meta
-        uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 #5.9.0
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf #6.0.0
         with:
           images: |
             ${{ vars.DOCKERHUB_SLUG }}

.github/workflows/testAndSanity.yml

@@ -6,7 +6,7 @@ on:
       node-version:
         description: "Node version"
         required: false
-        default: '20.x'
+        default: '24.x'
         type: string
       ref:
         description: "Ref to checkout"
@@ -19,12 +19,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 #v4.3.0
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
         with:
           ref: ${{ inputs.ref }}
           persist-credentials: false
       - name: Use Node.js
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 #4.4.0
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f #6.3.0
         with:
           node-version: ${{ inputs.node-version }}
           cache: 'npm'

.gitignore

@@ -122,6 +122,7 @@ config/*.json
 config/mscache
 config/yti-*
 config/*.cache
+config/msDb
 *.txt
 !robots.txt
 .idea/
@@ -138,6 +139,7 @@ docsite/static/schemas/*.json
 
 
 *.bak
+*.bak.used
 *.p8
 .flatpak-builder
 flatpak/generated-sources.json
@@ -151,3 +153,10 @@ tmp-*
 
 *storybook.log
 storybook-static
+
+*.db
+*.db.*
+*.db-*lib
+*.db
+*.db.*
+*.db-*lib

.mocharc.json

@@ -6,5 +6,6 @@
   "file": [
     "./src/backend/tests/setup.ts"
   ],
-  "exit": true
+  "exit": true,
+  "timeout": 2500
 }

.vscode/launch.json

@@ -1,6 +1,19 @@
 {
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Run Canonical Replace",
+            "program": "${workspaceFolder}/docsite/canonical-replace.ts",
+            "runtimeArgs": ["--experimental-strip-types"],
+            "args": [
+                "${workspaceFolder}/docsite/build"
+            ],
+            "request": "launch",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "type": "node"
+        },
         {
             "name": "dev",
             "type": "node",

.vscode/settings.json

@@ -13,5 +13,6 @@
     "files.associations": {
         "*.css": "tailwindcss"
     },
-    "tailwindCSS.experimental.configFile": "src/client/index.css"
+    "tailwindCSS.experimental.configFile": "src/client/index.css",
+    "pgliteExplorer.databasePaths": []
 }

Dockerfile

@@ -65,7 +65,8 @@ FROM base AS build
 
 COPY --chown=abc:abc package*.json tsconfig.json ./
 COPY --chown=abc:abc patches ./patches
-COPY --chown=abc:abc docsite/package*.json tsconfig.json ./docsite/
+COPY --chown=abc:abc docsite/patches ./docsite/patches
+COPY --chown=abc:abc docsite/package*.json docsite/tsconfig* ./docsite/
 
 # for debugging, so the build fails faster when timing out (arm64)
 #RUN npm config set fetch-retries 1 && \
@@ -92,18 +93,21 @@ RUN npm run build:parallel && rm -rf node_modules && rm -rf docsite/node_modules
 
 FROM base AS app
 
-LABEL org.opencontainers.image.source="https://github.com/FoxxMD/multi-scrobbler"
-
 COPY --chown=abc:abc *.json *.js *.ts index.html ./
 COPY --chown=abc:abc patches ./patches
+# frontend build from vite/esbuild
 COPY --from=build --chown=abc:abc /app/dist /app/dist
-COPY --from=build --chown=abc:abc /app/src /app/src
+# backend
+COPY --from=build --chown=abc:abc /app/src/backend /app/src/backend
+COPY --from=build --chown=abc:abc /app/src/core /app/src/core
+# docusaurus docs
 COPY --from=build --chown=abc:abc /app/docsite /app/docsite
 COPY --from=base /usr/bin /usr/bin
 COPY --from=base /usr/lib /usr/lib
 
 ENV NODE_ENV=production
 ENV IS_DOCKER=true
+ENV COLORED_STD=true
 
 # https://stackoverflow.com/a/63640896/1469797
 ARG APP_BUILD_VERSION
@@ -114,8 +118,22 @@ RUN npm ci --omit=dev --no-audit \
     && npm uninstall ts-json-schema-generator \
     && npm cache clean --force \
     && chown -R abc:abc node_modules \
+    && npx @usex/prune-mod -w \
     && rm -rf /root/.cache
 
+ARG BUILD_DATE=0
+
+LABEL org.opencontainers.image.version="$APP_BUILD_VERSION" \
+      org.opencontainers.image.source="https://github.com/FoxxMD/multi-scrobbler" \
+      org.opencontainers.image.documentation="https://docs.multi-scrobbler.app" \
+      org.opencontainers.image.description="Scrobble from multiple sources to multiple clients" \
+      org.opencontainers.image.title="Multi-Scrobbler" \
+      org.opencontainers.image.licenses="MIT" \
+      org.opencontainers.image.authors="FoxxMD" \
+      org.opencontainers.image.created="$BUILD_DATE" \
+      org.opencontainers.image.url="https://docs.multi-scrobbler.app" \
+      maintainer="FoxxMD" \
+      build_version="$APP_BUILD_VERSION"
 
 ARG webPort=9078
 ENV PORT=$webPort

README.md

@@ -3,7 +3,7 @@
 [![Latest Release](https://img.shields.io/github/v/release/foxxmd/multi-scrobbler)](https://github.com/FoxxMD/multi-scrobbler/releases)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Docker Pulls](https://img.shields.io/docker/pulls/foxxmd/multi-scrobbler)](https://hub.docker.com/r/foxxmd/multi-scrobbler)
-[![Docs](https://img.shields.io/badge/Read%20The%20Docs-1082c2)](https://foxxmd.github.io/multi-scrobbler/)
+[![Docs](https://img.shields.io/badge/Read%20The%20Docs-1082c2)](https://docs.multi-scrobbler.app/)
 
 
 <img src="/assets/icon.png" align="right"
@@ -11,56 +11,56 @@ alt="multi-scrobbler logo" width="180" height="180">
 
 A dockerized app that monitors your music listening activity from *everywhere* and scrobbles it *anywhere*.
 
-* Supports scrobbling from many [**Sources**](https://foxxmd.github.io/multi-scrobbler/configuration/sources)
-    * [Azuracast](https://foxxmd.github.io/multi-scrobbler/configuration/sources/azuracast)
-    * [Deezer](https://foxxmd.github.io/multi-scrobbler/configuration/sources/deezer)
-    * [Google Cast (Chromecast)](https://foxxmd.github.io/multi-scrobbler/configuration/sources/google-cast)
-    * [Icecast](https://foxxmd.github.io/multi-scrobbler/configuration/sources/icecast)
-    * [Jellyfin](https://foxxmd.github.io/multi-scrobbler/configuration/sources/jellyfin)
-    * [JRiver](https://foxxmd.github.io/multi-scrobbler/configuration/sources/jriver)
-    * [Kodi](https://foxxmd.github.io/multi-scrobbler/configuration/sources/kodi)
-    * [Koito](https://foxxmd.github.io/multi-scrobbler/configuration/sources/koito-source)
-    * [Last.fm](https://foxxmd.github.io/multi-scrobbler/configuration/sources/lastfm-source)
-    * [Last.fm (Endpoint)](https://foxxmd.github.io/multi-scrobbler/configuration/sources/lastfm-endpoint)
-    * [Libre.fm](https://foxxmd.github.io/multi-scrobbler/configuration/sources/librefm-source)
-    * [ListenBrainz](https://foxxmd.github.io/multi-scrobbler/configuration/sources/listenbrainz-source)
-    * [ListenBrainz (Endpoint)](https://foxxmd.github.io/multi-scrobbler/configuration/sources/listenbrainz-endpoint)
-    * [Maloja](https://foxxmd.github.io/multi-scrobbler/configuration/sources/maloja)
-    * [Mopidy](https://foxxmd.github.io/multi-scrobbler/configuration/sources/mopidy)
-    * [MPD (Music Player Daemon)](https://foxxmd.github.io/multi-scrobbler/configuration/sources/mpd)
-    * [MPRIS (Linux Desktop)](https://foxxmd.github.io/multi-scrobbler/configuration/sources/mpris)
-    * [Musikcube](https://foxxmd.github.io/multi-scrobbler/configuration/sources/musikcube)
-    * [Music Assistant](https://foxxmd.github.io/multi-scrobbler/configuration/sources/listenbrainz-endpoint#music-assistant)
-    * [Plex](https://foxxmd.github.io/multi-scrobbler/configuration/sources/plex)
-    * [Rocksky](https://foxxmd.github.io/multi-scrobbler/configuration/sources/rocksky-source)
-    * [Sonos](https://foxxmd.github.io/multi-scrobbler/configuration/sources/sonos)
-    * [Spotify](https://foxxmd.github.io/multi-scrobbler/configuration/sources/spotify)
-    * [Subsonic-compatible APIs](https://foxxmd.github.io/multi-scrobbler/configuration/sources/subsonic) (like [Airsonic](https://airsonic.github.io/) and [Navidrome](https://www.navidrome.org/))
-    * [teal.fm](https://foxxmd.github.io/multi-scrobbler/configuration/sources/tealfm-source)
-    * [WebScrobbler](https://foxxmd.github.io/multi-scrobbler/configuration/sources/webscrobbler)
-    * [VLC](https://foxxmd.github.io/multi-scrobbler/configuration/sources/vlc)
-    * [Yamaha MusicCast](https://foxxmd.github.io/multi-scrobbler/configuration/sources/yamaha-musiccast)
-    * [Youtube Music](https://foxxmd.github.io/multi-scrobbler/configuration/sources/youtube-music)
-* Supports scrobbling to many [**Clients**](https://foxxmd.github.io/multi-scrobbler/configuration/clients)
-    * [Discord](https://foxxmd.github.io/multi-scrobbler/configuration/clients/discord) (Now Playing)
-    * [Koito](https://foxxmd.github.io/multi-scrobbler/configuration/clients/koito)
-    * [Last.fm](https://foxxmd.github.io/multi-scrobbler/configuration/clients/lastfm)
-    * [Libre.fm](https://foxxmd.github.io/multi-scrobbler/configuration/clients/librefm)
-    * [ListenBrainz](https://foxxmd.github.io/multi-scrobbler/configuration/clients/listenbrainz)
-    * [Maloja](https://foxxmd.github.io/multi-scrobbler/configuration/clients/maloja)
-    * [Rocksky](https://foxxmd.github.io/multi-scrobbler/configuration/clients/rocksky)
-    * [teal.fm](https://foxxmd.github.io/multi-scrobbler/configuration/clients/tealfm)
-* Monitor status of Sources and Clients using [webhooks (Gotify, Ntfy, Apprise)](https://foxxmd.github.io/multi-scrobbler/configuration#webhook-configurations), [healthcheck endpoints](https://foxxmd.github.io/multi-scrobbler/configuration#health-endpoint), or [prometheus metrics](https://foxxmd.github.io/multi-scrobbler/configuration/#prometheus).
-* Supports [Now Playing](https://foxxmd.github.io/multi-scrobbler/configuration/clients#now-playing) for scrobble Clients
+* Supports scrobbling from many [**Sources**](https://docs.multi-scrobbler.app/configuration/sources)
+    * [Azuracast](https://docs.multi-scrobbler.app/configuration/sources/azuracast)
+    * [Deezer](https://docs.multi-scrobbler.app/configuration/sources/deezer)
+    * [Google Cast (Chromecast)](https://docs.multi-scrobbler.app/configuration/sources/google-cast)
+    * [Icecast](https://docs.multi-scrobbler.app/configuration/sources/icecast)
+    * [Jellyfin](https://docs.multi-scrobbler.app/configuration/sources/jellyfin)
+    * [JRiver](https://docs.multi-scrobbler.app/configuration/sources/jriver)
+    * [Kodi](https://docs.multi-scrobbler.app/configuration/sources/kodi)
+    * [Koito](https://docs.multi-scrobbler.app/configuration/sources/koito-source)
+    * [Last.fm](https://docs.multi-scrobbler.app/configuration/sources/lastfm-source)
+    * [Last.fm (Endpoint)](https://docs.multi-scrobbler.app/configuration/sources/lastfm-endpoint)
+    * [Libre.fm](https://docs.multi-scrobbler.app/configuration/sources/librefm-source)
+    * [ListenBrainz](https://docs.multi-scrobbler.app/configuration/sources/listenbrainz-source)
+    * [ListenBrainz (Endpoint)](https://docs.multi-scrobbler.app/configuration/sources/listenbrainz-endpoint)
+    * [Maloja](https://docs.multi-scrobbler.app/configuration/sources/maloja)
+    * [Mopidy](https://docs.multi-scrobbler.app/configuration/sources/mopidy)
+    * [MPD (Music Player Daemon)](https://docs.multi-scrobbler.app/configuration/sources/mpd)
+    * [MPRIS (Linux Desktop)](https://docs.multi-scrobbler.app/configuration/sources/mpris)
+    * [Musikcube](https://docs.multi-scrobbler.app/configuration/sources/musikcube)
+    * [Music Assistant](https://docs.multi-scrobbler.app/configuration/sources/listenbrainz-endpoint#music-assistant)
+    * [Plex](https://docs.multi-scrobbler.app/configuration/sources/plex)
+    * [Rocksky](https://docs.multi-scrobbler.app/configuration/sources/rocksky-source)
+    * [Sonos](https://docs.multi-scrobbler.app/configuration/sources/sonos)
+    * [Spotify](https://docs.multi-scrobbler.app/configuration/sources/spotify)
+    * [Subsonic-compatible APIs](https://docs.multi-scrobbler.app/configuration/sources/subsonic) (like [Airsonic](https://airsonic.github.io/) and [Navidrome](https://www.navidrome.org/))
+    * [teal.fm](https://docs.multi-scrobbler.app/configuration/sources/tealfm-source)
+    * [WebScrobbler](https://docs.multi-scrobbler.app/configuration/sources/webscrobbler)
+    * [VLC](https://docs.multi-scrobbler.app/configuration/sources/vlc)
+    * [Yamaha MusicCast](https://docs.multi-scrobbler.app/configuration/sources/yamaha-musiccast)
+    * [Youtube Music](https://docs.multi-scrobbler.app/configuration/sources/youtube-music)
+* Supports scrobbling to many [**Clients**](https://docs.multi-scrobbler.app/configuration/clients)
+    * [Discord](https://docs.multi-scrobbler.app/configuration/clients/discord) (Now Playing)
+    * [Koito](https://docs.multi-scrobbler.app/configuration/clients/koito)
+    * [Last.fm](https://docs.multi-scrobbler.app/configuration/clients/lastfm)
+    * [Libre.fm](https://docs.multi-scrobbler.app/configuration/clients/librefm)
+    * [ListenBrainz](https://docs.multi-scrobbler.app/configuration/clients/listenbrainz)
+    * [Maloja](https://docs.multi-scrobbler.app/configuration/clients/maloja)
+    * [Rocksky](https://docs.multi-scrobbler.app/configuration/clients/rocksky)
+    * [teal.fm](https://docs.multi-scrobbler.app/configuration/clients/tealfm)
+* Monitor status of Sources and Clients using [webhooks (Gotify, Ntfy, Apprise)](https://docs.multi-scrobbler.app/configuration#webhook-configurations), [healthcheck endpoints](https://docs.multi-scrobbler.app/configuration#health-endpoint), or [prometheus metrics](https://docs.multi-scrobbler.app/configuration/#prometheus).
+* Supports [Now Playing](https://docs.multi-scrobbler.app/configuration/clients#now-playing) for scrobble Clients
 * Supports configuring for single or multiple users (scrobbling for your friends and family!)
 * Web server interface for stats, basic control, and detailed logs
 * Graceful network and client failure handling (queued scrobbles that auto-retry)
 * Smart handling of credentials (persistent, authorization through app)
 * Easy configuration through [ENVs or JSON](ttps://foxxmd.github.io/multi-scrobbler/configuration#configuration-types)
-* Modify data before scrobbling with [regular expression or search patterns](https://foxxmd.github.io/multi-scrobbler/configuration/transforms)
-* Install using [Docker images for x86/ARM](https://foxxmd.github.io/multi-scrobbler/installation#docker) or [locally with NodeJS](https://foxxmd.github.io/multi-scrobbler/installation#nodejs)
+* Modify data before scrobbling with [regular expression or search patterns](https://docs.multi-scrobbler.app/configuration/transforms)
+* Install using [Docker images for x86/ARM](https://docs.multi-scrobbler.app/installation#docker) or [locally with NodeJS](https://docs.multi-scrobbler.app/installation#nodejs)
 
-[**Quick Start Guide**](https://foxxmd.github.io/multi-scrobbler//quickstart)
+[**Quick Start Guide**](https://docs.multi-scrobbler.app//quickstart)
 
 <img src="/docsite/static/img/status-ui.png" width="800">
 
@@ -73,15 +73,15 @@ A dockerized app that monitors your music listening activity from *everywhere* a
 
 **But I already scrobble my music to Last.fm/ListenBrainz, is multi-scrobbler for me?**
 
-Yes! You can use [Last.fm as a **Source**](https://foxxmd.github.io/multi-scrobbler/configuration/sources/lastfm-source) or [Listenbrainz as a **Source**](https://foxxmd.github.io/multi-scrobbler/configuration/sources/listenbrainz-source) to forward scrobbles from your profile to any other Client! That way you can keep your current scrobble setup as-is but still get the benefit of capturing your data to a self-hosted location.
+Yes! You can use [Last.fm as a **Source**](https://docs.multi-scrobbler.app/configuration/sources/lastfm-source) or [Listenbrainz as a **Source**](https://docs.multi-scrobbler.app/configuration/sources/listenbrainz-source) to forward scrobbles from your profile to any other Client! That way you can keep your current scrobble setup as-is but still get the benefit of capturing your data to a self-hosted location.
 
 ## How Does multi-scrobbler (MS) Work?
 
-You set up [configurations](https://foxxmd.github.io/multi-scrobbler/configuration) for one or more [**Sources**](https://foxxmd.github.io/multi-scrobbler/configuration/sources) and one or more [**Clients**](https://foxxmd.github.io/multi-scrobbler/configuration/clients). MS monitors all of your configured **Sources**. When new tracks are played by a Source MS grabs that information and then sends it (scrobbles it) to all **Clients** that Source is configured to forward to.
+You set up [configurations](https://docs.multi-scrobbler.app/configuration) for one or more [**Sources**](https://docs.multi-scrobbler.app/configuration/sources) and one or more [**Clients**](https://docs.multi-scrobbler.app/configuration/clients). MS monitors all of your configured **Sources**. When new tracks are played by a Source MS grabs that information and then sends it (scrobbles it) to all **Clients** that Source is configured to forward to.
 
 ### Source
 
-A [**Source**](https://foxxmd.github.io/multi-scrobbler/configuration/sources) is a data source that contains information about music you are playing or have listened to, such as: a desktop player, web music player, or cloud music service. Examples are **Spotify, Jellyfin, Plex, Youtube Music, Navidrome**, etc...
+A [**Source**](https://docs.multi-scrobbler.app/configuration/sources) is a data source that contains information about music you are playing or have listened to, such as: a desktop player, web music player, or cloud music service. Examples are **Spotify, Jellyfin, Plex, Youtube Music, Navidrome**, etc...
 
 Source configurations consist of:
 
@@ -91,7 +91,7 @@ Source configurations consist of:
 
 ### Client
 
-A [**Client**](https://foxxmd.github.io/multi-scrobbler/configuration/clients) is an application that stores the historical information about what music you have played (scrobbles). Examples are **Koito, Last.fm, Listenbrainz**...
+A [**Client**](https://docs.multi-scrobbler.app/configuration/clients) is an application that stores the historical information about what music you have played (scrobbles). Examples are **Koito, Last.fm, Listenbrainz**...
 
 Client configurations consist of:
 
@@ -100,15 +100,15 @@ Client configurations consist of:
 
 ## Quick Start
 
-[See the **Quick Start Guide**](https://foxxmd.github.io/multi-scrobbler/quickstart)
+[See the **Quick Start Guide**](https://docs.multi-scrobbler.app/quickstart)
 
 ## Installation
 
-[See the **Installation** documentation](https://foxxmd.github.io/multi-scrobbler/installation)
+[See the **Installation** documentation](https://docs.multi-scrobbler.app/installation)
 
 ## Configuration
 
-[See the **Configuration** documentation](https://foxxmd.github.io/multi-scrobbler/configuration)
+[See the **Configuration** documentation](https://docs.multi-scrobbler.app/configuration)
 
 ## Usage
 
@@ -123,11 +123,11 @@ On first startup you may need to authorize Spotify and/or Last.fm by visiting th
 
 ## Help/FAQ
 
-Having issues with connections or configuration? Check the [FAQ](https://foxxmd.github.io/multi-scrobbler/FAQ) before creating an issue!
+Having issues with connections or configuration? Check the [FAQ](https://docs.multi-scrobbler.app/FAQ) before creating an issue!
 
 ## Development
 
-[Detailed architecture and development guides for Sources/Clients](https://foxxmd.github.io/multi-scrobbler/development/dev-common)
+[Detailed architecture and development guides for Sources/Clients](https://docs.multi-scrobbler.app/development/dev-common)
 
 ## License
 

config/ymbridge.json.example

@@ -0,0 +1,11 @@
+[
+  {
+    "name": "Yandex Music",
+    "type": "ymbridge",
+    "data": {
+      "url": "http://192.168.1.161:9980",
+      "apiKey": "change-me",
+      "interval": 3
+    }
+  }
+]

docker-compose.dev.yml

@@ -4,6 +4,8 @@ services:
   multi-scrobbler:
     build:
       context: .
+      args:
+        BUILD_DATE: "${BUILD_DATE}"
     env_file:
       - path: ./.env
         required: false

docsite/.gitignore

@@ -18,3 +18,4 @@
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+lib

docsite/canonical-replace.ts

@@ -0,0 +1,42 @@
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+var args = process.argv.slice(2);
+
+const linkRegex = new RegExp(/<link data-rh="true" rel="\w+"\shref="(https:\/\/foxxmd\.github\.io\/multi-scrobbler)/g,);
+const replacement = process.env.CANONICAL_HREF ?? 'https://docs.multi-scrobbler.app';
+
+/**
+ * Replace canonical and alternative <link> nodes with the "real" domain of the site
+ * so that SEO (google) chooses the correct domain when showing search results
+ * 
+ */
+
+(async function () {
+
+    if(replacement === undefined || replacement === '') {
+        console.warn('No replacement value found in process.env.CANONICAL_HREF');
+        return;
+    }
+
+    const buildDir = path.resolve(args[0]);
+    console.log(`Reading dir recursively ${buildDir}`);
+    const files = (await fs.readdir(buildDir, { recursive: true })).filter(x => x.includes('.html'));
+    console.log(`Found ${files.length} files with .html extensions`);
+    let modifications = 0;
+
+    await Promise.all(files.map(async (x) => {
+            const filePath = path.resolve(path.join(buildDir, x));
+            //console.log(`Replacing at ${filePath}`);
+            await fs.writeFile(filePath, (await fs.readFile(filePath)).toString().replace(linkRegex, (match, capture, offset, string, groups) => {
+                // this may not be fully accurate since we're mutating concurrently/async
+                // but its a good enough signal for logging to tell if replacements happened
+                modifications++;
+                return match.replace(capture, replacement);
+            }));
+    }));
+
+    console.log(`Done with ${modifications} replacements`);
+
+}());

docsite/docs/FAQ.md

@@ -1,6 +1,7 @@
 ---
 toc_min_heading_level: 2
 toc_max_heading_level: 5
+title: FAQ
 ---
 
 ## Connection Issues
@@ -113,6 +114,12 @@ It means the JSON in your configuration file is not valid. Copy and paste your c
 
 ## Scrobbling Issues
 
+### How do I prevent duplicate scrobbles?
+
+Multi-scrobbler has a [robust duplicate scrobble detection system](/configuration/duplicates#how-duplicates-are-detected) that should handle this for you in the majority of usecases. If you have a [simple setup without any circular dependencies](/configuration/duplicates#valid-configs) then you are already protected against duplicates.
+
+If your scenario is more complex or you have a service(s) acting as **both** a **Client** and **Source** you should refer to the [**Duplicate Scrobble Guidance**](/configuration/duplicates#valid-configs) page to determine if you can avoid a configuration that may result in duplicate scrobbles.
+
 ### Last.fm does not scrobble tracks with multiple artists correctly
 
 This is a limitation of the [Last.fm API](https://www.last.fm/api/show/track.scrobble) where the **artist** field is only one string and Last.fm does not recognize (play well) with "combined" artists.

docsite/docs/configuration/clients/clients.mdx

@@ -1,6 +1,7 @@
 ---
 sidebar_position: 2
 title: Overview
+description: Scrobble Clients
 toc_max_heading_level: 4
 ---
 
@@ -33,60 +34,9 @@ For each Play, MS fetches (cached) scrobbles from the Client in a time range inc
 * Temporal closeness of the Play's timestamp to the existing scrobble's timestamp
 * Whether MS detected the Play as a repeat (applicable to Sources that report realtime Player data)
 
-Detailed scoring breakdowns against each existing scrobble are logged at the `TRACE` level.
+Detailed scoring breakdowns against each existing scrobble are logged at the `TRACE` logging level or in the [debug data](/help#copy-play-debug-data) for a scrobble.
 
-<details>
-
-<summary>Detailed Explanation</summary>
-
-A **candidate** (to be scrobbled) Play is first transformed using the configured [`compare.candidate` Hook](/configuration/transforms/#hook), if any exists.
-
-Next, MS checks (up to) the last 100 scrobbles *in-memory* scrobbles that it has made. These are *not* from the Client but the actual scrobbles MS made while it has been running. The data from these scrobbles is much richer than what is usually parsed from the Client which makes it easier to detect duplicates from.
-
-If no in-memory scrobble matches then MS starts comparing the candidate against historical scrobbles fetched from an inclusive time range of the candidate's timestamp.
-
-<h3>Matching Title/Artist/Album</h3>
-
-For these string-based values MS uses an [token-order-invariant](https://github.com/FoxxMD/multi-scrobbler/blob/0dfa4c7aad6df98e13aee6d395827665c2414adb/src/backend/utils/StringUtils.ts#L299) method that scores string similarity based on a (token count) weighted average of two [similarity](https://foxxmd.github.io/string-sameness/#md:strategies) algorithms, [Levenshtien Distance](https://en.wikipedia.org/wiki/Levenshtein_distance) and [Dice's Coefficient](https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient). The weighted average ensures that very long strings (Track titles) are scored with a confidence proportional to their length.
-
-<h3>Matching Timestamp</h3>
-
-Timestamps are scored on how temporally close they are. There are four possible scores with decreasing value:
-
-* **Exact** - Timestamps are within 1 second of each other
-* **Close** - Timestamps are within `threshold` seconds of each other
-  * This is determined by the smallest update interval of Source
-    * Some Sources (subsonic) only update every 60 seconds so this is the smallest "close" value possible. Most are 10 seconds. Signified by `(Needed <10s)` in the breakdown example below.
-* **Fuzzy** - One timestamp is within `threshold` seconds of the *end* of the other timestamp
-  * Sources can set the scrobble timestamp at different times. Some do it when the track is started listening to, some when it ends, some when the *player* stops.
-  * Where possible, MS knows and keeps track of when this timestamp *should* be, for each Source. If it's not possible then Fuzzy may be allowed.
-* **None** - There is no correlation between timestamps
-
-<h3>Scoring and Breakdows</h3>
-
-A candidate Play must score >= 1 to be detected as a duplicate of an existing scrobble.
-
-Each score and a breakdown of the scores for its individual components can be see at the `TRACE` logging level or in the [debug data](/help#copy-play-debug-data) for a scrobble. An example:
-
-```
-* Artist: 0.06 * 0.3 = 0.02
-* Title: 0.04 * 0.4 = 0.02
-* Time: (Exact) 1 * 0.5 = 0.50
-  * Existing: 19:13:33-04:00 - Candidate: 19:13:33-04:00
-  * Temporal Sameness: Exact
-  * Play Diff: 0s (Needed <10s)
-  * Range Comparison N/A
-Score 0.54 => No Match
-```
-
-In each component equation, the first number is the similarity (or temporal closeness).
-
-* 0 = no correlation
-* 1 = exactly the same
-
-The second number is the *weight* of that component in the final score.
-
-</details>
+See the [**Duplicate Scrobble Guidance**](/configuration/duplicates#how-duplicates-are-detected) page to learn how duplicates are detected and how you can ensure your configuration avoids possible duplicates.
 
 ### Dead Scrobbles
 

docsite/docs/configuration/clients/discord/discord.mdx

@@ -1,6 +1,7 @@
 ---
 sidebar_position: 2
 title: Discord
+description: Discord Client Overview
 toc_max_heading_level: 4
 ---
 

docsite/docs/configuration/clients/koito.mdx

@@ -45,7 +45,7 @@ Using the URL path `/apis/listenbrainz` [Koito docs describe](https://koito.io/g
 
 :::tip
 
-Ensure that Koito is configured to allow requests from multi-scrobbler! In Koito config set [`KOITO_ALLOWED_HOSTS`](https://koito.io/reference/configuration/#koito_allowed_hosts) to the IP the multi-scrobbler dashboard is accessible from.
+For Koito versions `<=0.1.7`, you must have the environment variable `KOITO_ALLOWED_HOSTS` set correctly in your Koito environment/container. Ensure it is set to the IP address or domain name that you use to access Koito with e.g. `koito.mydomain.com`. 
 
 :::
 
@@ -57,4 +57,4 @@ Ensure that Koito is configured to allow requests from multi-scrobbler! In Koito
         | `KOITO_TOKEN`          | Yes       |         | API Key from your Koito Account   |
         | `KOITO_USER`           | Yes       |         | Your Koito username               |
         | `KOITO_URL`            | No        |         | The base URL for the Koito server |
-</Config>
+</Config>

docsite/docs/configuration/configuration.mdx

@@ -1,6 +1,7 @@
 ---
 sidebar_position: 2
 title: Overview
+description: Configuration
 toc_max_heading_level: 3
 ---
 import Tabs from '@theme/Tabs';
@@ -244,11 +245,8 @@ WARN : [App] [Sources] [spotify Secrets] Matched: None | Unmatched: SPOTIFY_SECR
 
 </DetailsAdmo>
 
-## Application Options
 
-These options affect multi-scrobbler's behavior and are not specific to any source/client.
-
-### Base URL
+## Base URL
 
 Defines the URL that is used to generate default redirect URLs for authentication on [spotify](/configuration/sources/spotify) and [lastfm](/configuration/clients/lastfm) -- as well as some logging hints.
 
@@ -279,79 +277,51 @@ Useful when running with [docker](../installation/installation.mdx#docker) so th
 
 </details>
 
-### Caching
+## Caching
 
-Multi-scrobbler caches some activities to persist important data across restarts, reduce external API calls, and make some actions faster.
+Multi-scrobbler implements caching to persist important data across restarts, reduce external API calls, and make some actions faster.
 
-All of the activities below are **always** cached **in-memory** with an optional, configurable [**secondary** store](#secondary-caching-configuration) for persistence.
+A default **in-memory** cache store is used so that you always benefit from some caching. An optional, [**secondary** store](#secondary-caching) can be configured for greater caching capabilities.
 
-<Tabs groupId="cachedThings" queryString>
-<TabItem value="scrobbles" label="Scrobbles">
-
-**Queued** and **Failed** Scrobbles are cached so that any un-scrobbled data you have is persisted across restarts of multi-scrobbler.
-
-:::tip
-
-By default, this data use a [Secondary](#secondary-caching-configuration) [File](./?cacheType=file#secondary-caching-configuration) store, configured for you automatically.
-
-If you have configured a [persisted volume/bind mount](/installation?dockerSetting=storage#recommended-settings) for configuration (`/config` is mounted in [docker compose](/quickstart#create-docker-compose-file)) then you are already done. If you are not persisting this directory then you should consider setting up [Valkey Cache](./?cacheType=valkey#secondary-caching-configuration) for this.
-
-:::
-
-##### Configuration
-
-Use any [Secondary Cache](#secondary-caching-configuration), the config examples below show the default values:
-
-<Tabs>
-<TabItem value="env" label="ENV">
-
-| Environmental Variable | Required? | Default   | Description           |
-| :--------------------- | --------- | --------- | --------------------- |
-| `CACHE_SCROBBLE`       | No        | `file`    | The cache type to use |
-| `CACHE_SCROBBLE_CONN`  | No        | `/config` |                       |
+<DetailsAdmo type="note" summary="What is Cached?">
 
+<Tabs groupId="cachedThings">
+<TabItem value="auth" label="Auth Data">
+Authentication sessions/tokens/etc... are cached for quicker requests and for persistence across restarts.
 </TabItem>
-
-<TabItem value="aio" label="AIO">
-
-```json5 title="config.json"
-{
-    "cache": {
-        "scrobble": {
-            "provider": "file",
-            "connection": "/config"
-        }
-    },
-    // ...
-}
-```
+<TabItem value="transform" label="Scrobble Transforms">
+The results of [transform rules](/configuration/transforms) are cached so that if a scrobble with identical data (track/artists/album) is identified and it has the same set of transform rules then the cached transform results can be applied.
+</TabItem>
+<TabItem value="metadata" label="Transform API Calls">
+API Calls to external (metadata) services used to [Enhance Scrobbles](/configuration/transforms), like calls to [Musicbrainz](/configuration/transforms/musicbrainz), can be cached to avoid duplicate calls and speed up scrobble transformations.
 </TabItem>
 </Tabs>
 
-</TabItem>
-<TabItem value="auth" label="Auth Data">
+</DetailsAdmo>
 
-Authentication sessions/tokens/etc... are cached for quicker requests and for persistence across restarts.
+<DetailsAdmo type="important" summary="Auth Cache Configuration">
 
-:::tip
+Auth caching defaults to a **file** that is stored in the `CONFIG_DIR` directory using the pre-defined file name `ms-auth.cache`.
 
-By default, this data use a [Secondary](#secondary-caching-configuration) [File](./?cacheType=file#secondary-caching-configuration) store, configured for you automatically.
+This provides automatic persistence across restarts for long-lived auth data/credentials if you have configured a [persisted volume/bind mount](/installation?dockerSetting=storage#recommended-settings) for configuration (`/config` is mounted in [docker compose](/quickstart#create-docker-compose-file)).
 
-If you have configured a [persisted volume/bind mount](/installation?dockerSetting=storage#recommended-settings) for configuration (`/config` is mounted in [docker compose](/quickstart#create-docker-compose-file)) then you are already done. If you are not persisting this directory then you should consider setting up [Valkey Cache](./?cacheType=valkey#secondary-caching-configuration) for this.
+If you wish to use the [secondary store](#secondary-caching) for caching Auth you must explicitly configure it. This is because valkey can potentially be *ephemeral* if you do not provide a volume for its data directory.
 
-:::
+To explicitly configure auth to use the secondary store:
 
-##### Configuration
-
-Use any [Secondary Cache](#secondary-caching-configuration), the config examples below show the default values:
-
-<Tabs>
+<Tabs groupId="configType" queryString>
 <TabItem value="env" label="ENV">
 
-| Environmental Variable | Required? | Default   | Description           |
-| :--------------------- | --------- | --------- | --------------------- |
-| `CACHE_AUTH`           | No        | `file`    | The cache type to use |
-| `CACHE_AUTH_CONN`      | No        | `/config` |                       |
+```yaml title="compose.yaml"
+services:
+  multi-scrobbler:
+  # ...
+  environment:
+  // highlight-start
+    - CACHE_AUTH=valkey
+  // highlight-end
+    # ...
+```
 
 </TabItem>
 
@@ -361,8 +331,7 @@ Use any [Secondary Cache](#secondary-caching-configuration), the config examples
 {
     "cache": {
         "auth": {
-            "provider": "file",
-            "connection": "/config"
+            "provider": "valkey"
         }
     },
     // ...
@@ -371,93 +340,19 @@ Use any [Secondary Cache](#secondary-caching-configuration), the config examples
 </TabItem>
 </Tabs>
 
-</TabItem>
-<TabItem value="metadata" label="Transform API Calls">
+</DetailsAdmo>
 
-API Calls to external (metadata) services used to [Enhance Scrobbles](/configuration/transforms), like calls to [Musicbrainz](/configuration/transforms/musicbrainz), can be cached to avoid duplicate calls and speed up scrobble transformations.
+### Secondary Caching Store {#secondary-caching}
 
-By default, these calls are only cached in memory. If you wish for cached calls to be persisted across restarts then setup [Valkey Cache](./?cacheType=valkey#secondary-caching-configuration).
+Using a secondary store enables:
 
-##### Configuration
+* persistence of cached data across restarts
+* a larger store (more data is saved)
+* a longer time-to-live in the store (cached data is fetchable for a longer period)
 
-Use any [Secondary Cache](#secondary-caching-configuration), the config examples below show the default values:
+These benefits are particularly beneficial when using [transforms](/configuration/transforms) like Musicbrainz and it is **strongly recommended** for these scenarios.
 
-<Tabs>
-<TabItem value="env" label="ENV">
-
-| Environmental Variable | Required? | Default | Description           |
-| :--------------------- | --------- | ------- | --------------------- |
-| `CACHE_METADATA`       | No        |         | The cache type to use |
-| `CACHE_METADATA_CONN`  | No        |         |                       |
-
-</TabItem>
-
-<TabItem value="aio" label="AIO">
-
-```json5 title="config.json"
-{
-    "cache": {
-        "metadata": {
-            "provider": "valkey",
-            "connection": "yourConnectionStringHere"
-        }
-    },
-    // ...
-}
-```
-</TabItem>
-</Tabs>
-
-</TabItem>
-</Tabs>
-
-#### Secondary Caching Configuration
-
-The type of cache used, and its connection properties, can be configured through **ENV** or **AIO** config.
-
-<Tabs groupId="cacheType" queryString>
-
-<TabItem value="file" label="File">
-
-**File** cache is stored in the `CONFIG_DIR` directory using the pre-defined file name `ms-[cacheName].cache`.
-
-<Tabs groupId="configType" queryString>
-
-<TabItem value="env" label="ENV">
-
-Example
-
-| Environmental Variable | Required? | Default   | Description                                                  |
-| :--------------------- | --------- | --------- | ------------------------------------------------------------ |
-| `CACHE_SCROBBLE`       | No        | `file`    | The cache type to use                                        |
-| `CACHE_SCROBBLE_CONN`  | No        | `/config` | The directory, within the container, to store the cache file |
-
-</TabItem>
-
-<TabItem value="aio" label="AIO">
-
-Example
-
-```json5 title="config.json"
-{
-    "cache": {
-        "scrobble": {
-            "provider": "file",
-            "connection": "/config"
-        }
-    },
-    // ...
-}
-```
-
-</TabItem>
-
-</Tabs>
-</TabItem>
-
-<TabItem value="valkey" label="Valkey">
-
-[**Valkey**](https://valkey.io/) is an open-source fork of Redis.
+Currently, Multi-scrobbler only supports [**Valkey**](https://valkey.io/), an open-source fork of Redis, as a secondary store.
 
 <details>
 
@@ -465,11 +360,13 @@ Example
 
 A valkey container can be added to the [multi-scrobbler docker compose stack](/installation?runType=docker-compose#docker):
 
+<Tabs>
+<TabItem value="named" label="Named Volume">
+
 ```yaml title="docker-compose.yml"
 services:
   multi-scrobbler:
   # ...
-  # adding everything below
   // highlight-start
   valkey:
     image: valkey/valkey
@@ -482,6 +379,25 @@ volumes:
   // highlight-end
 ```
 
+</TabItem>
+<TabItem value="bind" label="Bind Mount">
+
+```yaml title="docker-compose.yml"
+services:
+  multi-scrobbler:
+  # ...
+  // highlight-start
+  valkey:
+    image: valkey/valkey
+    volumes:
+      - ./valkeyData:/data
+  // highlight-end
+```
+
+</TabItem>
+
+</Tabs>
+
 Use `redis://valkey:6379` as the connection string in the configurations below.
 
 </details>
@@ -496,12 +412,16 @@ redis://HOST_IP:HOST_PORT
 
 <TabItem value="env" label="ENV">
 
-Example
-
-| Environmental Variable | Required? | Default  | Description                                                         |
-| :--------------------- | --------- | -------- | ------------------------------------------------------------------- |
-| `CACHE_METADATA`       | Yes       | `valkey` | The cache type to use                                               |
-| `CACHE_METADATA_CONN`  | Yes       |          | The host/IP and port to connect to EX: `redis://192.168.0.120:6379` |
+```yaml title="compose.yaml"
+services:
+  multi-scrobbler:
+  # ...
+  environment:
+  // highlight-start
+    - CACHE_VALKEY=redis://192.168.0.120:6379
+  // highlight-end
+    # ...
+```
 
 </TabItem>
 
@@ -512,10 +432,7 @@ Example
 ```json5 title="config.json"
 {
     "cache": {
-        "metadata": {
-            "provider": "valkey",
-            "connection": "redis://192.168.0.120:6379"
-        }
+        "valkey": "redis://192.168.0.120:6379"
     },
     // ...
 }
@@ -524,11 +441,8 @@ Example
 </TabItem>
 
 </Tabs>
-</TabItem>
 
-</Tabs>
-
-### Debug Mode
+## Debug Mode
 
 Turning on Debug Mode will
 
@@ -549,7 +463,7 @@ To set debug mode either add it to [AIO `config.json`](./?configType=aio#configu
 
 or set the [ENV](./?configType=env#configuration-types) `DEBUG_MODE=true`
 
-### Disable Web
+## Disable Web
 
 If you do not need the dashboard and/or ingress sources, or have security concerns about ingress and cannot control your hosting environment, the web server and API can be disabled.
 

docsite/docs/configuration/duplicates.mdx

@@ -0,0 +1,267 @@
+---
+sidebar_position: 4
+title: Duplicate Scrobble Guidance
+description: Common Scenarios and Solutions for Duplicate Scrobbles
+---
+
+Multi-scrobbler's flexibility in configuration allows you to build some complex pipelines for scrobbling but it can also be a footgun for duplicating scrobbles.
+
+## How Duplicates Are Detected
+
+Multi-scrobbler uses multiple techniques for evaluating the **similarity** of text in scrobbles against existing data.
+
+These techniques are used on all text from a scrobble (title, artist, album...):
+
+<details>
+
+<summary>Character and string normalization</summary>
+
+Symbols and multiple whitespaces are removed. All characters are converted to lowercase. Non ascii characters are converted to their ascii counterparts.
+
+```
+this string! is the. same    => this string is the same
+this string   is the    same => this string is the same
+ThIs STRING iS THe SAMe      => this string is the same
+
+Dina Ögon => Dina Ogon
+Nanã      => Nana
+```
+
+</details>
+
+<details>
+
+<summary>Word-order invariant text similarity</summary>
+
+Strings with multiple words (tokens) are re-arranged so that tokens are ordered to be as close to other, similar tokens are possible:
+
+Example 1
+
+```
+String A: there change is  => there change is
+String B: there is change  => there change is
+```
+
+Example 2
+
+```
+String A: you, doin hwat are  => you doin hwat are
+String B: what are you doing  => you doing what are
+```
+
+Then, the strings are independently compared using [Levenshtein Distance](https://en.wikipedia.org/wiki/Levenshtein_distance) and [Dice's Coefficient](https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient). The algorithm that gives the highest score for similarity is used so that matching is aggressive.
+
+
+</details>
+
+For **Artists**, Multi-scrobbler attempts to extract unique Artists from common patterns in **single-string** artist values and track **titles**.
+
+<details>
+
+<summary>Matching Common Artist Patterns</summary>
+
+Common joiners and patterns are detected in artist/track strings and the artists are extracted. Example of common joiners are: `ft.` `feat` `vs.` etc...
+
+Example 1
+
+```
+Melendi, Ryan Lewis, The Righteous Brothers (featuring Joan Jett & The Blackhearts, Robin Schulz)
+
+becomes
+
+Melendi, Ryan Lewis, The Righteous Brothers, Joan Jett, The Blackhearts, Robin Schulz
+```
+
+Example 2
+
+```
+Childish Gambino - 12.38 (feat. 21 Savage, Ink & Kadhja Bonet)
+
+becomes
+
+Childish Gambino, 21 Savage, Ink, Kadhja Bonet - 12.38
+```
+
+:::note
+
+* MS uses "smart" heuristics to prevent artists with actual punctuation (`Tyler, The Creator`) from accidentally being separated
+* The extraction examples above are done **only for matching** other scrobble data (which is also parsed like the above), the resulting scrobble data is never transformed unless the user explicitly configures it via a [native transform](/configuration/transforms/native)
+
+:::
+
+</details>
+
+The **timestamps** for Scrobbles are compared at different levels of granularity based on where they originate from.
+
+<details>
+
+<summary>Temporally Comparing Scrobbles</summary>
+
+Timestamp closeness for two scrobbles can be classified in one of these states:
+
+**Exact**
+
+Timestamps differ by 1 seconds or less.
+
+**Close**
+
+Timestamps differ by less than the known granularity of the Source they are from. For most Sources (and all Clients) this is 10 seconds or less. For some sources (subsonic) the granularity is 60 seconds because they only report "now playing" changes every 60 seconds.
+
+**During**
+
+If the Play was monitoring in real-time by Multi-scrobbler then it knows the time range during which the Play was listened to. If an existing scrobble with similar artist/track/album has a timestamp that occurred *during* the known listening time then it is classified as during.
+
+**Fuzzy**
+
+If the timestamp occurs within a few seconds +/- of (existing scrobble + track duration) then, potentially, the Source timestamped the existing scrobble at the *end* of when the user listened to it, rather than when the user *started* listening to it. This is common for Spotify plays.
+
+___
+
+Each of the above classification results in a different score that is *roughly* descending from the order given above. Exact score is based on the Souce of the scrobble and if it was monitored in real-time by Multi-Scrobbler.
+
+</details>
+
+Finally, all of the above comparison/normalization techniques are combined when comparing a "new" Play/Scrobble against existing data. Each comparison is given a weighted score. If the score is above the match threshold then it is marked as a duplicated.
+
+<details>
+
+<summary>Match Scoring Example</summary>
+
+```
+Frédéric Chopin - Ballade No. 4 in F Minor, Op. 52 @ 2026-05-04T13:01:51-04:00 <-- New scrobble
+Frédéric Chopin / Krystian Zimerman - Ballade No. 4 in F Minor, Op. 52 @ 2026-05-04T13:01:48-04:00 <-- Existing scrobble from client
+```
+
+After comparing against all existing scrobbles from the client, in a timerange around the new scrobble, MS finds a match and logs the match along with the score breakdown of the consistuent parts:
+
+```
+Source: 
+Frédéric Chopin - Ballade No. 4 in F Minor, Op. 52 @ 2026-05-04T13:01:51-04:00 (S) 
+
+Closest Scrobble: 
+Frédéric Chopin / Krystian Zimerman - Ballade No. 4 in F Minor, Op. 52 @ 2026-05-04T13:01:48-04:00 (S)
+
+Artist: (0.30 + Whole Match Bonus 0.52) * (0.3 + Whole Match Bonus 0.05) = 0.29
+Title: 1.00 * 0.4 = 0.40
+Time: (Close) 1 * 0.5 = 0.50
+Time Detail => Existing: 13:01:48-04:00 - Candidate: 13:01:51-04:00 | Temporal Sameness: Close | Play Diff: 3s (Needed <10s) | Range Comparison N/A
+Score 1.19 => Matched!
+```
+
+</details>
+
+All of the above techniques and scoring gives Multi-scrobbler a *very good chance* of detecting duplicate scrobbles, regardless of where they were originally from. Of Multi-Scrobbler's [400+ unit tests](https://github.com/FoxxMD/multi-scrobbler/tree/master/src/backend/tests), 70+ are related to testing scoring, artist extraction, and [fuzzing](https://en.wikipedia.org/wiki/Fuzzing) string normalization/simiarlity.
+
+The dupe detection is so good that it is virtually gauranteed for [most configurations](#valid-configs) and is used for *backlogging* missed scrobbles for Sources that support fetching history.
+
+**However**, it's not possible to gaurantee this for *all* possible Multi-scrobbler configurations. See the below sections to learn more.
+
+## Configurations Where Duplicates DO NOT Occur {#valid-configs}
+
+* Any Source/Client configuration where there is only one Source and one Client and they are not the same service/account
+* Any Source/Client configuration where scrobbles only flow in **one direction**
+  * IE you *do not* have a Last.fm **Client** Account A that is *also* a Last.fm **Source** Account A
+
+:::tip
+
+**The majority of simple Multi-scrobbler configurations fall into these categories. If your workflow is straightforward you do not need to worry about duplicates at all.**
+
+:::
+
+Multi-scrobbler can handle recieving the same data from a Source multiple times. This is fine because the data is *always the same* and *always unique* to each Source. Since MS scrobbled the data to your clients you can be pretty confident that the *same* cleanup/matching process for that data will result in an indentical/similar match to the existing scrobble.
+
+IE Same inputs in -> same inputs out.
+
+**Even this scenario with multiple, mixed Sources is OK:**
+
+<img className='image-center' src={require('/img/diagrams/unidirectional-flow.png').default}/>
+
+Because the scrobbles coming from each **Source** are **independent** of each other. And they end up in Clients that are not *also* used as Sources.
+
+:::tip
+
+This is a common scenario when scrobbles are *backlogged* from a Source IE on startup MS fetches the last 50 Plays from Spotify/Lastfm/Listenbrainz.
+
+:::
+
+## Configurations Where Duplicates CAN Occur {#invalid-configs}
+
+### Upstream Scrobbles Sent to Multiple Sources
+
+In this scenario your device using Spotify is configured in MS as a Spotify **Source** but it **also** is scrobbling to Last.fm (via Pano Scrobbler, as an example).
+
+<img className='image-center' src={require('/img/diagrams/dupe sources.png').default}/>
+
+
+❌ **Time Race Condition**: Potentially, the scrobbles arrive in Multi-scrobbler at exactly the same time. There are no existing scrobbles in Koito to check against, yet. So Multi-scrobbler scrobbles both to Koito, resulting in duplicate scrobbles.
+
+❌ **Different Scrobble Data**: Pano Scrobbler, or Last.fm, [may modify the scrobble data.](#lastfm) When MS recieves the scrobble from Last.fm it may be *just different enough* that it does not get detected as a duplicate.
+
+### Sources and Clients are Dependent {#dependent-source-client}
+
+In this scenario, you have a **Client** that is also a **Source** and that Source is feeding scrobbles to a different Client that receives scrobbles from everywhere (or has some overlap with the original Client).
+
+<img className='image-center' src={require('/img/diagrams/duped client source.png').default}/>
+
+❌ **Different Scrobble Data**: [Last.fm may modify the scrobble data.](#lastfm) When MS recieves the scrobble from Last.fm it may be *just different enough* that it does not get detected as a duplicate.
+
+❌ **Consistency Issues**: If the Koito Client has a backed up queue or is not currently scrobbling (imagine it was Last.fm or Listenbrainz, and they had upstream availability issues), then there is no existing scrobble yet and nothing to match against.
+
+## Configuration Solutions to Avoid Duplicates
+
+### Independent Sources
+
+Use a configuration mentioned in [Configurations Where Duplicates DO NOT Occur](#valid-configs): make sure all of your Sources are **independent**. The activity/scrobbles from each Source do not have any upstream duplicates of activity.
+
+<img className='image-center' src={require('/img/diagrams/independent source.png').default}/>
+
+### Alternative Sources
+
+Imagine you have a scenario where a device/player can **only** scrobble to Last.fm. But you want to use MS for all other devices/sources and still have all your scrobbles end up in your "main" Last.fm account for social reasons/stats.
+
+Instead of using the same Last.fm account for all activity and potentially having [dependent client/sources](#dependent-source-client), create an *additional* last.fm account that is used *only for the device that needs it*. This ensures the scrobbles from that device are isolated from the main account and do not prevent duplicates, so you can continue to use the main account as an MS Client:
+
+<img className='image-center' src={require('/img/diagrams/alt source.png').default}/>
+
+### Mirror-Only Clients or Limiting Scrobble Destination {#isolated-sources}
+
+In Multi-scrobbler, by default, all Sources scrobble to all Clients. Using [Limit Scrobble Destinations](/configuration/sources/#limiting-scrobble-destination) configuration can ensure that your dependent Source/Clients only scrobble to destinations where the scrobbles won't be duplicated. And/or limit other Sources to *not* scrobble to dependent Sources/Clients.
+
+Here is an example where all Sources only scrobble to a *dependent* Last.fm Client, and then the *dependent* Last.fm Source only scrobbles to a "mirror-only" (backup) Koito Client:
+
+<img className='image-center' src={require('/img/diagrams/mirror source.png').default}/>
+
+### Normalize All Scrobbles
+
+If using one of the above solutions is not possible you can also try to *aggressively normalize all scrobble data* using [the Musicbrainz transform stage](/configuration/transforms/musicbrainz).
+
+In this scenario you still have [dependent Source/Clients](#dependent-source-client) that are [not isolated](#isolated-sources) but you can *potentially* improve duplicate detection by first using a transform stage to match scrobbles against the Musibrainz database and normalize their track/artist/album data.
+
+:::warning
+
+**There is no gaurantee this will work** but it is better than nothing. You should attempt to re-configure MS to use any of the other solutions before using this one.
+
+:::
+
+<img className='image-center' src={require('/img/diagrams/normalized dupes.png').default}/>
+
+## The Last.fm Problem {#lastfm}
+
+Last.fm will happily accept almost any scrobble data you throw at it.
+
+It can then decide, opaquely, to match or modify that scrobble to some more "official" data it has server-side. This could be modifying the artists, title, or album sent in the scrobble. It can also mean modifying the [`mbid` (unique identifier)](https://musicbrainz.org/doc/MusicBrainz_Identifier) or adding what it _thinks_ is the correct one.
+
+There is nothing in the scrobble history API response from Last.fm that identifies it has done any of this. And [it's not super great at being right about these modifications](https://github.com/FoxxMD/multi-scrobbler/issues/333#issuecomment-3145161339).
+
+What this means for you, the user, is that this is very plausible scenario:
+
+* You scrobble `Artist Foo, Artist Bar - My Cool Track` to Last.fm via Pano Scrobbler (outside of Multi-scrobbler)
+  * Last.fm modifies the data to  `The Artist Foo - My Cool Track ft. Artist Bar (Album Version)` 
+* You _also_ scrobble  `Artist Foo, Artist Bar - My Cool Track` to Multi-scrobbler via Pano LFM endpoint **Source**
+  * MS forwards this to Koito Client as-is
+* Soon after, MS polls your Last.fm **Source** for new scrobbles and finds `The Artist Foo - My Cool Track ft. Artist Bar (Album Version)` 
+  * It queries Koito to see if this was scrobbled
+  * It sees that a scrobble did happen at almost the same time BUT the track titles and artist strings are _just different enough_ that it decides it's not a duplicate
+  * MS forwards the scrobble to Koito and you now have a duplicate
+
+This behavior is unique to Last.fm. It's truly the **worst** out of all the scrobbler services in terms of preserving your data as-is, or providing a unqiue identifier for your data.

docsite/docs/configuration/kitchensink.mdx

@@ -1,6 +1,7 @@
 ---
-sidebar_position: 3
+sidebar_position: 5
 title: Kitchen Sink
+description: Kitchen Sink Example
 ---
 
 # Example Complex Scenario

docsite/docs/configuration/sources/icecast.mdx

@@ -2,6 +2,7 @@
 title: Icecast
 toc_min_heading_level: 2
 toc_max_heading_level: 5
+description: Icecast Source Configuration
 ---
 
 import Tabs from '@theme/Tabs';

docsite/docs/configuration/sources/koito-source.mdx

@@ -1,5 +1,6 @@
 ---
 title: Koito
+description: Koito (Source)
 toc_min_heading_level: 2
 toc_max_heading_level: 5
 ---

docsite/docs/configuration/sources/lastfm-endpoint.mdx

@@ -41,6 +41,6 @@ http://localhost:9078/api/lastfm/mySlug
 <Config config="LastFMEndpointSourceConfig" fileContent={LastfmEndpointConfig} name="endpointlfm">
         | Environmental Variable | Required? | Default | Description                                                                                                        |
         | :--------------------- | :-------- | ------- | ------------------------------------------------------------------------------------------------------------------ |
-        | `LFMENDPOINT_ENABLE`   | No        |         | Use LFM Endpoint as a Source without any other configuration. Only required if slug/token are not provided as ENVs |
+        | `LFM_ENABLE`           | No        |         | Use LFM Endpoint as a Source without any other configuration. Only required if slug/token are not provided as ENVs |
         | `LFM_SLUG`             | No        |         | (Optional) The URL suffix to use for accepting LFM scrobbles                                                       |
 </Config>

docsite/docs/configuration/sources/lastfm-source.mdx

@@ -1,5 +1,6 @@
 ---
 title: Last.fm
+description: Last.fm (Source)
 toc_min_heading_level: 2
 toc_max_heading_level: 5
 ---

docsite/docs/configuration/sources/librefm-source.mdx

@@ -1,5 +1,6 @@
 ---
 title: Libre.fm
+description: Libre.fm (Source)
 toc_min_heading_level: 2
 toc_max_heading_level: 5
 ---

docsite/docs/configuration/sources/listenbrainz-endpoint.mdx

@@ -109,9 +109,10 @@ To troubleshoot any errors, and assuming you are using Home Assistant, view the
 ## Configuration
 
 <Config config="ListenbrainzEndpointSourceConfig" fileContent={ListenbrainzEndpointConfig} name="endpointlz">
-        | Environmental Variable | Required? | Default | Description                                                                                                       |
-        | :--------------------- | :-------- | ------- | ----------------------------------------------------------------------------------------------------------------- |
-        | `LZENDPOINT_ENABLE`    | No        |         | Use LZ Endpoint as a Source without any other configuration. Only required if slug/token are not provided as ENVs |
-        | `LZE_TOKEN`            | No        |         | LZ "Authentication Token" you provided to the scrobbling application                                              |
-        | `LZE_SLUG`             | No        |         | (Optional) The URL suffix to use for accepting LZ scrobbles                                                       |
+        | Environmental Variable | Required? | Default | Description                                                                                                                                                |
+        | :--------------------- | :-------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |
+        | `LZE_ENABLE`           | No        |         | Use LZ Endpoint as a Source without any other configuration. Only required if slug/token are not provided as ENVs                                          |
+        | `LZE_TOKEN`            | No        |         | LZ "Authentication Token" you provided to the scrobbling application                                                                                       |
+        | `LZE_SLUG`             | No        |         | (Optional) The URL suffix to use for accepting LZ scrobbles                                                                                                |
+        | `LZE_USERNAME`         | No        |         | (Optional) A fake username that will be returned for token validation and on now-playing responses. If none is provided the Source's name is used instead. |
 </Config>

docsite/docs/configuration/sources/listenbrainz-source.mdx

@@ -1,5 +1,6 @@
 ---
 title: Listenbrainz
+description: Listenbrainz (Source)
 toc_min_heading_level: 2
 toc_max_heading_level: 5
 ---

docsite/docs/configuration/sources/mpd.mdx

@@ -18,6 +18,8 @@ bind_to_address	"any"   # or a specific ipv4/v6 address
 port            "6600"
 ```
 
+Multi-scrobbler requires only the [`read`](https://mpd.readthedocs.io/en/stable/user.html#permissions-and-passwords) permission in order to monitor your activity.
+
 ## Configuration
 
 <Config config="MPDSourceConfig" fileContent={MPDConfig} name="mpd">

docsite/docs/configuration/sources/rocksky-source.mdx

@@ -1,5 +1,6 @@
 ---
 title: Rocksky
+description: Rocksky (Source)
 toc_min_heading_level: 2
 toc_max_heading_level: 5
 ---

docsite/docs/configuration/sources/sources.mdx

@@ -1,6 +1,7 @@
 ---
 sidebar_position: 2
 title: Overview
+description: Scrobble Sources
 toc_max_heading_level: 4
 ---
 
@@ -40,6 +41,7 @@ A **Source** is a data source that contains information about tracks you are pla
 | [WebScrobbler](/configuration/sources/webscrobbler)                     | [Ingress](./?sourceComm=active#by-communication-method)  | [History](./?sot=history#by-data-source-of-truth)   | ❌                                     | ✅                                                     | ❌                                 | ❌                                           |
 | [VLC](/configuration/sources/vlc)                                       | [Active](./?sourceComm=active#by-communication-method)   | [Activity](./?sot=activity#by-data-source-of-truth) | ❌                                     | ✅                                                     | ✅                                 | ❌                                           |
 | [Yamaha MusicCast](/configuration/sources/yamaha-musiccast)             | [Active](./?sourceComm=active#by-communication-method)   | [Activity](./?sot=activity#by-data-source-of-truth) | ❌                                     | ✅                                                     | ✅                                 | ❌                                           |
+| [Yandex Music](/configuration/sources/yandex-music)                     | [Active](./?sourceComm=active#by-communication-method)   | [Activity](./?sot=activity#by-data-source-of-truth) |                                        | ✅                                                     | ✅                                 | ❌                                           |
 | [Youtube Music](/configuration/sources/youtube-music)                   | [Active](./?sourceComm=active#by-communication-method)   | [History](./?sot=history#by-data-source-of-truth)   | ❌                                     | ✅                                                     | ❌                                 | ❌                                           |
 
 ## Features

docsite/docs/configuration/sources/tealfm-source.mdx

@@ -1,5 +1,6 @@
 ---
 title: Teal.fm
+description: Teal.fm (Source)
 toc_min_heading_level: 2
 toc_max_heading_level: 5
 ---

docsite/docs/configuration/sources/yandex-music.mdx

@@ -0,0 +1,79 @@
+---
+title: Yandex Music
+toc_min_heading_level: 2
+toc_max_heading_level: 5
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import CodeBlock from '@theme/CodeBlock';
+import JsonConfig from '!!raw-loader!@site/../config/ymbridge.json.example';
+
+Monitor your [**Yandex Music**](https://music.yandex.com) listening activity in real-time.
+
+This Source requires a third party docker container to communicate with Yandex Music, [`yandex-music-bridge`](https://github.com/Druidblack/yandex-music-bridge).
+
+<details>
+
+<summary>Example Docker Compose with `yandex-music-bridge`</summary>
+
+```yaml title="~/msData/docker-compose.yml"
+services:
+  multi-scrobbler:
+    image: foxxmd/multi-scrobbler
+    container_name: multi-scrobbler
+    environment:
+      - TZ=Etc/GMT
+      # ...your other sources/clients here
+
+      # add for yandex music
+      // highlight-start
+      - YMBRIDGE_URL=http://yandex-music-bridge:9980
+      - YMBRIDGE_API_KEY=change-me
+      // highlight-end
+
+    volumes:
+      - "./config:/config"
+    ports:
+      - "9078:9078"
+    restart: unless-stopped
+
+  // highlight-start
+  yandex-music-bridge:
+    image: ghcr.io/druidblack/yandex-music-bridge:latest
+    environment:
+      - TZ=Europe/Moscow
+      - YM_TOKEN=AgAAAAACO3_345345
+      - YM_API_KEY=change-me
+      - YM_PORT=9980
+      - YM_LANGUAGE=ru
+      - YM_ENABLE_YNISON=true
+      - YM_PUSH_TTL=45
+      - YM_QUEUE_CACHE_TTL=15
+      - YM_LOG_LEVEL=INFO
+    ports:
+      - "9980:9980"
+    restart: unless-stopped
+   // highlight-end
+```
+
+You will need to acquire your own token for `YM_TOKEN` used in [`yandex-music-bridge`](https://github.com/Druidblack/yandex-music-bridge) using the instructions provided there.
+
+</details>
+
+<DetailsAdmo type="tip" summary="Need Help?">
+
+If your issue is specifically related to [`yandex-music-bridge`](https://github.com/Druidblack/yandex-music-bridge) (errors in the container, usage instructions, general Yandex setup like token etc...) please open an issue on **that repository** instead of Multi-Scrobbler's.
+
+If you have issues specifically with the MS Source please mention [**Druidblack**](https://github.com/Druidblack) (`@Druidblack`) when opening an [issue](https://github.com/FoxxMD/multi-scrobbler/issues/new/choose) or [discussion](https://github.com/FoxxMD/multi-scrobbler/discussions/new/choose). Yandex Music is [not available](https://en.wikipedia.org/wiki/Yandex_Music) in the country of the main Multi-Scrobbler developer (FoxxMD) so they can only help troubleshooting general MS issues related to Yandex Music.
+
+</DetailsAdmo>
+
+## Configuration
+
+<Config config="YandexMusicBridgeSourceConfig" fileContent={JsonConfig} name="ymbridge">
+        | Environmental Variable | Required? | Default | Description                                                               |
+        | :--------------------- | :-------- | ------- | :------------------------------------------------------------------------ |
+        | `YMBRIDGE_URL`         | Yes       |         | `http://URL:PORT` for the `yandex-music-bridge` container                 |
+        | `YMBRIDGE_API_KEY`     | No        |         | The same key used for `YM_API_KEY` on the `yandex-music-bridge` container |
+</Config>

docsite/docs/configuration/transforms/musicbrainz.mdx

@@ -1,5 +1,6 @@
 ---
 title: Musicbrainz Stage
+description: Enhancing Scrobbles with Musicbrainz
 toc_min_heading_level: 2
 toc_max_heading_level: 5
 ---
@@ -139,7 +140,7 @@ Example:
 
 ### Rules and Hooks
 
-[Add your Stage](https://foxxmd.github.io/multi-scrobbler/configuration/transforms/#stage) to a Source or Client by specifying it in a [Hook](/configuration/transforms/#hook):
+[Add your Stage](/configuration/transforms/#stage) to a Source or Client by specifying it in a [Hook](/configuration/transforms/#hook):
 
 ```json5 title="subsonic.json"
 [

docsite/docs/configuration/transforms/native.mdx

@@ -1,5 +1,6 @@
 ---
 title: Native Stage
+description: Enhancing Scrobbles with Built-in Parsing
 toc_min_heading_level: 2
 toc_max_heading_level: 5
 ---

docsite/docs/configuration/transforms/transforms.mdx

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 4
+sidebar_position: 3
 title: Enhancing Scrobbles
 toc_max_heading_level: 5
 ---
@@ -591,7 +591,7 @@ The output shows the diff between the previous stage (or original Play) and the
 
 MS uses [caching](/configuration/#caching) to reduce the number of API calls needed for stages like [Musicbrainz](/configuration/transforms/musicbrainz) and to speed up all transforms by caching steps and results. However, the default caching strategy uses a small cache size and very short [TTLs](https://en.wikipedia.org/wiki/Time_to_live) because it is *in-memory*.
 
-**If you are using any Transform stages you should configure [secondary caching with Valkey for Metadata](/configuration/?cacheType=valkey&cachedThings=metadata#secondary-caching-configuration)** to increase the cache size and lifetime of cached items. This will also reduce memory usage in MS.
+**If you are using any Transform stages you should configure [secondary caching](/configuration#secondary-caching)** to increase the cache size and lifetime of cached items. This will also reduce memory usage in MS.
 
 <details>
 

docsite/docs/configuration/transforms/user.mdx

@@ -1,5 +1,6 @@
 ---
 title: User Stage
+description: Enhancing Scrobbles with Regex Parsing
 toc_min_heading_level: 2
 toc_max_heading_level: 5
 ---

docsite/docs/development/dev-client.md

@@ -1,8 +0,0 @@
----
-toc_min_heading_level: 2
-toc_max_heading_level: 5
-sidebar_position: 3
-title: Client Development/Tutorial
----
-
-To do...

docsite/docs/development/dev-common.md

@@ -3,7 +3,7 @@ toc_min_heading_level: 2
 toc_max_heading_level: 5
 sidebar_position: 1
 title: Common Development
-description: Start here for MS development
+description: Common Development Knowledge
 ---
 
 # Development
@@ -177,7 +177,7 @@ For more refer to the TS documentation for `PlayObject` or [`AmbPlayObject`](htt
 ## Creating Clients and Sources
 
 * [Source Development and Tutorial](dev-source.md)
-* [Client Development and Tutorial](dev-client.md)
+* Client Development and Tutorial (TBD)
 
 ## Profiling
 

docsite/docs/development/dev-source.md

@@ -3,6 +3,7 @@ toc_min_heading_level: 2
 toc_max_heading_level: 5
 sidebar_position: 2
 title: Source Development/Tutorial
+description: Developing a new Source
 ---
 
 This document will provide a step-by-step guide for creating a (trivial) new Source in MS alongside describing what aspects of the Source need to be implemented based on the service you use. Before using this document you should review [Common Development](dev-common.md#common-development).

docsite/docs/development/flatpak.md

@@ -3,7 +3,7 @@ toc_min_heading_level: 2
 toc_max_heading_level: 5
 sidebar_position: 4
 title: Flatpak
-description: Building Flatpak App locally
+description: Building Flatpak App
 ---
 
 ## Install Requirements

docsite/docs/help.mdx

@@ -1,5 +1,5 @@
 ---
-title: 'Help'
+title: Help
 ---
 
 ## Getting Useful Data

docsite/docs/index.mdx

@@ -41,6 +41,7 @@ A dockerized app that monitors your music listening activity from *everywhere* a
     * [WebScrobbler](/configuration/sources/webscrobbler)
     * [VLC](/configuration/sources/vlc)
     * [Yamaha MusicCast](/configuration/sources/yamaha-musiccast)
+    * [Yandex Music](/configuration/sources/yandex-music)
     * [Youtube Music](/configuration/sources/youtube-music)
 * Supports scrobbling to many [**Clients**](/configuration/clients)
     * [Discord](/configuration/clients/discord) (Now Playing)
@@ -83,6 +84,14 @@ Use the **Last.fm** tab in the [Quickstart Sources](/quickstart/#setup-sources)
 
 You set up [configurations](/configuration) for one or more [**Sources**](/configuration/sources) and one or more [**Clients**](/configuration/clients). MS monitors all of your configured **Sources**. When new tracks are played by a Source MS grabs that information and then sends it (scrobbles it) to all **Clients** that Source is configured to forward to.
 
+<details>
+
+<summary>Diagram</summary>
+
+<img className='image-center' src={require('/img/diagrams/ms-visual.png').default}/>
+
+</details>
+
 ### Source
 
 A [**Source**](/configuration/sources) is a data source that contains information about music you are playing or have listened to, such as: a desktop player, web music player, or cloud music service. Examples are **Spotify, Jellyfin, Plex, Youtube Music, Navidrome**, etc...

docsite/docs/installation/installation.mdx

@@ -1,6 +1,7 @@
 ---
 sidebar_position: 1
-title: 'Overview'
+title: Overview
+description: Installation
 ---
 
 import Tabs from '@theme/Tabs';
@@ -131,7 +132,7 @@ services:
 
 </TabItem>
 <TabItem value="caching" label="Caching">
-**Optionally**, add a [Valkey](https://valkey.io/) service to your stack for [secondary caching](/configuration/?cacheType=valkey#secondary-caching-configuration) to take advantage of faster performance and reduced memory usage.
+**Optionally**, add a [Valkey](https://valkey.io/) service to your stack for [secondary caching](/configuration#secondary-caching) to take advantage of faster performance and reduced memory usage.
 
 ```yaml title="docker-compose.yml"
 services:
@@ -141,8 +142,7 @@ services:
     environment:
       # ...
       // highlight-start
-      CACHE_METADATA=valkey
-      CACHE_METADATA_CONN=redis://valkey:6379
+      CACHE_VALKEY=redis://valkey:6379
       // highlight-end
     # ...
 
@@ -220,10 +220,10 @@ After installation see [service.md](service.md) to configure multi-scrobbler to
 
 ### Nodejs
 
-Clone this repository somewhere and then install from the working directory
+Clone this repository somewhere using the latest [git tag](/updating#git) and then install from the working directory.
 
 ```shell
-git clone https://github.com/FoxxMD/multi-scrobbler.git .
+git clone --branch <LATEST_RELEASE_TAG> https://github.com/FoxxMD/multi-scrobbler.git
 cd multi-scrobbler
 nvm use # optional, to set correct Node version
 npm install

docsite/docs/installation/service.md

@@ -1,6 +1,7 @@
 ---
 sidebar_position: 2
-title: 'As a Service'
+title: As a Service
+description: Local Install as a Service
 ---
 
 If you have multi-scrobbler installed [locally](installation.mdx#nodejs) you can enable it to run as a background service when you login.

docsite/docs/quickstart.mdx

@@ -1,5 +1,5 @@
 ---
-title: 'Quickstart'
+title: Quickstart
 sidebar_position: 2
 ---
 
@@ -254,9 +254,9 @@ Modify the `lastfm.json` file you created in the [Setup Sources](#setup-sources)
         ```
 
         :::tip
-
-        Ensure that Koito is configured to allow requests from multi-scrobbler! In Koito config set [`KOITO_ALLOWED_HOSTS`](https://koito.io/reference/configuration/#koito_allowed_hosts) to the IP the multi-scrobbler dashboard is accessible from.
-
+        
+        For Koito versions `<=0.1.7`, you must have the environment variable `KOITO_ALLOWED_HOSTS` set correctly in your Koito environment/container. Ensure it is set to the IP address or domain name that you use to access Koito with e.g. `koito.mydomain.com`. 
+        
         :::
 
     </TabItem>
@@ -336,7 +336,7 @@ Visit `http://192.168.0.100:9078` to see the dashboard where
 ## Next Steps
 
 * See more advanced docker options as well as other install methods in the [**Installation**](/installation#docker) docs
-  * Setup [secondary caching](/configuration/?cacheType=valkey#secondary-caching-configuration) with [valkey](/installation?dockerSetting=caching#recommended-settings) for increased performance and reduced memory usage
+  * Setup [secondary caching](/configuration#secondary-caching) with [valkey](/installation?dockerSetting=caching#recommended-settings) for increased performance and reduced memory usage
 * Review the [**Configuration**](/configuration) docs
   * Learn about how to configure multi-scrobbler using files for more complicated Source/Client scenarios
   * See all available [**Sources**](/configuration/sources) and [**Clients**](/configuration/clients) alongside configuration examples

docsite/docs/updating/_category_.json

@@ -2,7 +2,7 @@
   "label": "Updating",
   "position": 4,
   "link": {
-    "type": "generated-index",
-    "description": "Way to update Multi-Scrobbler"
+    "type": "doc",
+    "id": "updating/updating"
   }
 }

docsite/docs/updating/updating.mdx

@@ -1,6 +1,7 @@
 ---
 sidebar_position: 1
-title: 'Overview'
+title: Overview
+description: Updating Multi-Scrobbler
 ---
 
 import Tabs from '@theme/Tabs';
@@ -9,38 +10,104 @@ import CodeBlock from '@theme/CodeBlock';
 
 ## Updating
 
-Currently, multi-scrobbler does not have any databases or dependencies that require additional interaction/consideration when updating.
+The majority of Multi-scrobbler updates can be completed without any manual intervention. This is **guaranteed** for [patch version updates](#versioning).
 
-The majority of major/minor version releases introduce only *new* functionality but occasionally may also include breaking changes in behavior or configuration. These changes, and how to update configs accordingly, are detailed in [Github Release Notes](https://github.com/FoxxMD/multi-scrobbler/releases). It is recommended to check this page before upgrading.
+Regardless, it is recommended to consult the [**Github Release Notes**](https://github.com/FoxxMD/multi-scrobbler/releases) before any upgrades. The release notes contain all changelogs as well as most **breaking changes**/notices.
 
-Alternatively, multi-scrobbler version can be pinned using [git tags](https://github.com/FoxxMD/multi-scrobbler/tags) or docker image tags that match release versions IE `foxxmd/multi-scrobbler:0.9.7`
+The [**Upgrade Path** docs](/updating/upgrade-path) contain information for upgrading through **required versions** or in-depth migration guides. Check this section before upgrading any [minor versions](#versioning).
 
-### Docker
+### Database
 
-<Tabs groupId="runType" queryString>
-    <TabItem value="docker" label="Docker">
-        ```shell
-        docker image pull foxxmd/multi-scrobbler:latest
-        docker container stop multi-scrobbler
-        docker container rm multi-scrobbler
-        docker run --name multi-scrobbler ...
-        ```
-        The last command is the [same as the command used to initialy run multi-scrobbler.](../installation/installation.mdx?runType=docker#docker-usage-example)
-    </TabItem>
+Multi-scrobbler depends on a SQLite database (`ms.db`) that is created on first run and stored in the [`CONFIG_DIR`](/installation/?dockerSetting=storage#recommended-settings).
+
+If any database migrations are required for an update then your database is **automatically backed up** before migration occurs. The backup file is created in the same directory.
+
+You can manually make a backup of this database by making a copy of `ms.db` and any similarly named files like `ms.db-journal`.
+
+:::tip
+
+If upgrading a [minor version](#versioning) you may want to make a backup for extra safety.
+
+:::
+
+### Configuration
+
+[Minor versions](#versioning) may have **breaking changes** related to [configuration](/configuration). Consult the [**Github Release Notes**](https://github.com/FoxxMD/multi-scrobbler/releases) and [Upgrade Paths](/updating/upgrade-path) before upgrading.
+
+
+## Update Instructions
+
+Assuming:
+
+* upgrade(s) are only [patch versions](#versioning) and the [Release Notes](https://github.com/FoxxMD/multi-scrobbler/releases) do not contain any other guidance or
+* you have checked the release notes, upgrade paths docs, and completed all migration steps for the specific version upgrade
+
+<Tabs groupId="installType" queryString>
     <TabItem value="docker-compose" label="Docker Compose">
         ```shell
         docker compose pull
         docker compose up -d
         ```
     </TabItem>
+    <TabItem value="local" query="Local (Nodejs)">
+        Assuming the [repository was used for initial installation with `git clone`](/installation#local-installation), update by pulling the latest commit:
+
+        ```
+        git pull
+        ```
+
+        and restart multi-scrobbler.
+    </TabItem>
 </Tabs>
 
-### Nodejs
+## Versioning
 
-Assuming the [repository was used for initial installation with `git clone`](../installation/installation.mdx#nodejs), update by pulling the latest commit:
+Multi-scrobbler versioning uses **semver** without a major version (for now):
 
-```
-git pull
+* **Minor** version bumps (`0.12.2` => `0.13.0`) **may have breaking changes** or include major new behavior
+* **Patch** version bumps (`0.12.2` => `0.12.3`) may have new behavior or fix bugs, but have no breaking changes. These can be safely upgraded unattended.
+
+Check the [**Github Release Notes**](https://github.com/FoxxMD/multi-scrobbler/releases) for breaking changes, changelogs, and any migration steps required between minor versions.
+
+### Git
+
+The default [`master`](https://github.com/FoxxMD/multi-scrobbler/tree/master/) branch is **not stable**. The goal is to have it be *functional* bleeding-edge development with non-functioning development isolated to pull requests and other branches.
+
+For a stable release use the latest [git tag](https://github.com/FoxxMD/multi-scrobbler/tags) EX:
+
+```shell
+git clone --branch 0.13.0 https://github.com/FoxxMD/multi-scrobbler.git
 ```
 
-and restart multi-scrobbler.
+### Docker
+
+Docker image tags use this pattern:
+
+* `:latest` is the latest **stable** [release](https://github.com/FoxxMD/multi-scrobbler/releases)
+    * IE If the latest released version was `0.13.0` then `:latest` is also `0.13.0`
+    * `:latest` will *always* be the latest release. Before pulling a new image you should check if the version bump has breaking changes.
+* `:edge` is built automatically from the [`master` branch](#git)
+* Pinned semver tags are published alongside [releases](https://github.com/FoxxMD/multi-scrobbler/releases) EX `:0.12.2`
+
+Additionally, [Pull Requests](https://github.com/FoxxMD/multi-scrobbler/pulls) may have docker images published with tags like `:pr-521`. These are not guaranteed to be stable and should only be used for testing.
+
+## What Version Am I Using?
+
+The current version of Multi-Scrobbler can be found in several places:
+
+* Open the Web Dashboard, the version is displayed next the Multi-Scrobbler logo in the top left corner
+* On startup the version is immediately printed to logs EX:
+    * `[2026-04-16 09:17:32.428 -0400] INFO   : [Init] Version: 0.13.0`
+* Checking the running docker container or image for the `org.opencontainers.image.version` label
+
+When running MS from a non-pinned source (like `:edge` tag, a pull request, or git branch) the version will be shown as the **branch** name and **short commit hash** EX
+
+```
+master-1c78da79
+```
+
+:::tip
+
+Using the above information you can verify if you have pulled and successfully recreated a docker container with a new version. If the version string does not change from the previous version running then you are still running the old container
+
+:::

docsite/docs/updating/upgrade-path/0140.mdx

@@ -0,0 +1,212 @@
+---
+sidebar_position: 1
+title: From < 0.14.0
+description: Upgrading to 0.14.0
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import CodeBlock from '@theme/CodeBlock';
+import RequiredUpgrade from "@site/src/components/snippets/_upgrade-required.mdx"
+
+<RequiredUpgrade before='< 0.14.0' current='0.14.0' old='0.13.2' />
+
+## Source/Client IDs
+
+MS `0.14.0` introduces a database for persisting Plays/Scrobbles, queues, and other data **associated with a Source/Client**.
+
+To make this association atomic a Source/Client **ID** is now configurable. This ID is what MS will use to identify the Source/Client in your **config** with the Source/Client in the **database**.
+
+This ID needs to be unique to the client or source type it is used on IE:
+
+* two [Koito Clients](/configuration/clients/koito/) cannot both have the ID `myKoitoID`
+* it is recommended that the ID be *globally* unique among all sources/clients, but it's not required
+
+### Configuring IDs
+
+<Tabs groupId="configType" queryString>
+
+<TabItem value="env" label="ENV">
+
+For each Source/Client in your environmental variables, add a key with the suffix `_ID` and the ID value. EX
+
+```yaml title="compose.yaml"
+services:
+  multi-scrobbler:
+  # ...
+  environment:
+    - KOTIO_TOKEN=...
+    # ...more koito config
+  // highlight-start
+    - KOITO_ID=myKoitoID
+  // highlight-end
+
+    - JELLYFIN_URL=...
+    # ...more jellyin config
+  // highlight-start
+    - JELLYFIN_ID=myJellyfinID
+  // highlight-end    
+```
+
+</TabItem>
+<TabItem value="file" label="File">
+
+Add an `id` to the top-level for each Source/Client configuration, next to `data`:
+
+```json title="koito.json"
+[
+  {
+    "name": "koito-source",
+    "configureAs": "source",
+    // highlight-start
+    "id": "myKoitoID",
+    // highlight-end   
+    "data": {
+      "token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b",
+      "username": "admin",
+      "url": "http://192.168.0.100:4110" 
+    }
+  }
+]
+```
+
+</TabItem>
+<TabItem value="aio" label="AIO">
+
+Add an `id` to the top-level for each Source/Client configuration, next to `data`:
+
+```json title="koito.json"
+[
+  {
+    "name": "koito-source",
+    "configureAs": "source",
+    "type": "koito",
+    // highlight-start
+    "id": "myKoitoID",
+    // highlight-end    
+    "data": {
+      "token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b",
+      "username": "admin",
+      "url": "http://192.168.0.100:4110" 
+    }
+  }
+]
+```
+
+</TabItem>
+</Tabs>
+
+:::note[Default ID]
+
+If you do not add an ID then Multi-Scrobbler will automatically use the **name** of the Source/Client as the ID. The name is shown in the Dashboard.
+
+If you decide to add an ID later you must use the name as the ID in order to keep Plays/Scobbles associated with the same config.
+
+:::
+
+## Cached Scrobble Migration
+
+:::tip
+
+Before upgrading, if your MS dashboard shows 0 queued/failed for all [Scrobble Clients](/configuration/clients) then you **skip this step** and can safely delete `ms-scrobble.cache` before the upgrade occurs.
+
+:::
+
+MS `0.14.0` introduces a database for persisting Plays/Scrobbles, queues, and other data. Prior to `0.14.0`, queued/failed scrobbles were stored in a cache **file** inside your [`CONFIG_DIR`](/installation/?dockerSetting=storage#recommended-settings) named `ms-scrobble.cache`.
+
+On first run of `0.14.0`:
+
+* MS will make a copy of `ms-scrobble.cache` named `ms-scrobble.cache.bak`
+  * You may make a manual copy of this file before upgrading, for additional safety
+* Scrobbles in the cache will automatically be migrated to the new database
+* The now "old" cache data will be cleared so that subsequent application starts don't duplicate migrations
+  * the empty `ms-scrobble.cache` file will remain
+
+You can follow progress of this by looking for log lines starting with `Migrating cached scrobbles to database...`
+
+If MS does not report/log any errors during this time and you see your scrobbles processed normally then it is safe to delete `ms-scrobble.cache` the next time MS is stopped/restarted.
+
+## Cache Configuration
+
+In `0.14.0` [Cache](/configuration#caching) has been simplified with much of the required configuration being removed.
+
+:::tip
+
+This only applies to users who have `cache` in their [AIO Config](/configuration/?configType=aio#configuration-types) `config.json` or are using `CACHE_*` [ENV Config](/configuration/?configType=env#configuration-types)
+
+If you do not have any of the above [Cache](/configuration#caching) configuration defined then you can **skip this step.** 
+
+:::
+
+#### Scrobble Caching Removed
+
+[Scrobble caching has removed and replaced by the new database.](#cached-scrobble-migration)
+
+* Remove any ENV Config starting with `CACHE_SCROBBLE`
+* Remove `cache.scrobble` from the AIO Config
+
+#### Metadata Config Simplified
+
+Metadata caching remains the same but the config has been simplified. If you were using [Valkey for caching](/configuration/#secondary-caching) update your config:
+
+<Tabs groupId="configType" queryString>
+
+<TabItem value="env" label="ENV">
+
+* Remove `CACHE_METADATA`
+* Rename `CACHE_METADATA_CONN` to [`CACHE_VALKEY`](/configuration/#secondary-caching)
+
+</TabItem>
+<TabItem value="aio" label="AIO">
+
+Remove `cache.metadata` and add a new string key `valkey` to the `cache` object, containing your valkey connection string:
+
+```diff
+  "cache": {
++    "valkey": "redis://valkey:6379"
+-    "metadata": {
+-      "provider": "valkey",
+-      "connection": "redis://valkey:6379"
+-    }
+  }
+```
+
+</TabItem>
+</Tabs>
+
+
+#### Auth Config Simplified
+
+Auth caching remains the same but the config has been simplified. The connection option for Auth is no longer configurable. You may specify provider as either `file` (default, uses `CONFIG_DIR`) or `valkey`. If you want to use Valkey for auth it will use the same config as Metadata.
+
+<Tabs groupId="configType" queryString>
+
+<TabItem value="env" label="ENV">
+
+* Remove `CACHE_AUTH_CONN`
+* Add [`CACHE_VALKEY`](/configuration/#secondary-caching) (if using valkey)
+
+</TabItem>
+<TabItem value="aio" label="AIO">
+
+Remove the `connection` property from the `auth` object. If using `file` the entire `auth` object can be removed.
+
+```diff
+  "cache": {
++   "valkey": "redis://valkey:6379",
+    "auth": {
+      "provider": "valkey",
+-     "connection": "redis://valkey:6379"
+    }
+  }
+```
+
+</TabItem>
+</Tabs>
+
+### Renamed ENV Keys
+
+In an effort to standard ENV prefixes some ENV keys have been renamed:
+
+* `LZENDPOINT_ENABLE` => `LZE_ENABLE`
+* `LFMENDPOINT_ENABLE` => `LFM_ENABLE`

docsite/docs/updating/upgrade-path/_category_.json

@@ -0,0 +1,7 @@
+{
+  "label": "Upgrade Path",
+  "link": {
+    "type": "doc",
+    "id": "updating/upgrade-path/upgrade-path"
+  }
+}

docsite/docs/updating/upgrade-path/upgrade-path.mdx

@@ -0,0 +1,39 @@
+---
+sidebar_position: 1
+title: Upgrade Path
+description: Updating Multi-Scrobbler
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import CodeBlock from '@theme/CodeBlock';
+
+The majority of changes are detailed in [**Github Release Notes**](https://github.com/FoxxMD/multi-scrobbler/releases), including *most* breaking changes. You should consult the release notes for each version you will be updating through/to.
+
+Upgrade paths for specific version, detailed here, are reserved for:
+
+<Tabs groupId="upgradeType" queryString>
+
+<TabItem value="required" label="Required Upgrades">
+
+:::warning[Required Upgrade]
+
+Versions you **must** upgrade to **before** upgrading to a newer version. Likely due to the codebase containing functionality to upgrade persistent data that cannot be migrating across multiple versions.
+
+:::
+
+</TabItem>
+
+<TabItem value="depth" label="In Depth Upgrades">
+
+:::note[Optional Upgrade]
+
+(Breaking) Changes to configuration/data that require more detail or formatting than is feasible in Github release notes. These upgrades may be skipped.
+
+:::
+
+</TabItem>
+
+</Tabs>
+
+The type of upgrade is shown at the top of each page with the same colored callout shown above.

docsite/docusaurus.config.ts

@@ -1,8 +1,32 @@
-import type * as Preset from '@docusaurus/preset-classic';
+import * as Preset from '@docusaurus/preset-classic';
 import type { Config } from '@docusaurus/types';
 import * as themes from 'prism-react-renderer';
+import * as Renderers from './lib/socialCard/ImageRenderers.js';
+import path from 'node:path';
 //import sidebars from './sidebars';
 
+let baseSite: string = 'https://foxxmd.github.io';
+
+const baseUrlEnvs = [
+  // To build with base URL via normal docker build with build-args/env
+  // or if specified, to override the subsequent set ENVs
+  process.env.BASE_SITE,
+  // To build with base URL set via netlify CI
+  // https://docs.netlify.com/build/configure-builds/environment-variables/#deploy-urls-and-metadata
+  process.env.DEPLOY_PRIME_URL,
+  // To build with base URL set via cloudflare pages CI
+  // https://developers.cloudflare.com/pages/configuration/build-configuration/#environment-variables  
+  process.env.CF_PAGES_URL
+];
+
+for(const baseUrlVal of baseUrlEnvs) {
+  if(baseUrlVal !== undefined && baseUrlVal !== null && baseUrlVal.trim() !== '') {
+    baseSite = baseUrlVal;
+    break;
+  }
+}
+console.log(`Base URL => ${baseSite}`);
+
 const config: Config = {
   future: {
     v4: {
@@ -19,7 +43,7 @@ const config: Config = {
   favicon: 'img/favicon.ico',
 
   // Set the production url of your site here
-  url: 'https://foxxmd.github.io',
+  url: baseSite,
   // Set the /<baseUrl>/ pathname under which your site is served
   // For GitHub pages deployment, it is often '/<projectName>/'
   baseUrl: process.env.DOCS_BASE !== undefined && process.env.DOCS_BASE !== '' ? process.env.DOCS_BASE : '/docs',
@@ -31,8 +55,12 @@ const config: Config = {
 
   trailingSlash: true,
 
-  onBrokenLinks: 'warn',
-  onBrokenMarkdownLinks: 'warn',
+
+  markdown: {
+    hooks: {
+      onBrokenMarkdownLinks: 'warn',
+    }
+  },
 
   // Even if you don't use internalization, you can use this field to set useful
   // metadata like html lang. For example, if your site is Chinese, you may want
@@ -52,8 +80,15 @@ const config: Config = {
           routeBasePath: '/',
           // Please change this to your repo.
           // Remove this to remove the "edit this page" links.
-          editUrl:
-            'https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/',
+          editUrl: (params: {
+              version: string;
+              versionDocsDirPath: string;
+              docPath: string;
+              permalink: string;
+              locale: string;
+            }) => {
+              return `${path.join('https://github.com/FoxxMD/multi-scrobbler/blob/master/docsite', params.versionDocsDirPath, params.docPath)}?plain=1`;
+            }
         },
         // blog: {
         //   showReadingTime: true,
@@ -122,6 +157,16 @@ const config: Config = {
           },
         ]
       }
+    ],
+    [
+      '@bony_chops/docusaurus-og',
+      {
+        path: './preview-images', // relative to the build directory
+        imageRenderers: {
+            'docusaurus-plugin-content-docs': Renderers.docs,
+            'docusaurus-plugin-content-pages': Renderers.docs,
+        },
+      },
     ]
   ],
   themeConfig:
@@ -154,7 +199,7 @@ const config: Config = {
             position: 'right',
           },
           {
-            href: 'https://foxxmd.github.io/multi-scrobbler/',
+            href: 'https://multi-scrobbler.app',
             label: 'Website',
             position: 'right',
           },
@@ -193,7 +238,7 @@ const config: Config = {
               },
               {
                 label: 'Website',
-                href: 'https://foxxmd.github.io/multi-scrobbler/',
+                href: 'https://multi-scrobbler.app',
               },
             ],
           },
@@ -216,28 +261,22 @@ const config: Config = {
         showRunmeLink: false,
         runmeLinkLabel: 'Checkout via Runme'
       }
-    } satisfies Preset.ThemeConfig,
-    headTags: [
-      {
-        tagName: 'meta',
-        attributes: {
-          name: 'google-site-verification',
-          content: process.env.GSITEVERIFICATION ?? 'none'
-        }
-      }
-    ]
+    } satisfies Preset.ThemeConfig
 };
 
-if (process.env.ANALYTICS !== undefined && process.env.ANALYTICS !== '') {
-  const script = {
-    src: process.env.ANALYTICS,
-    defer: true
-  }
-  if (process.env.ANALYTICS_DOMAIN !== undefined && process.env.ANALYTICS_DOMAIN !== '') {
-    script['data-domain'] = process.env.ANALYTICS_DOMAIN;
-  }
-  config.scripts.push(script)
+const gverify = process.env.GSITEVERIFICATION;
+if(gverify !== undefined && gverify !== null && gverify.trim() !== '') {
+  config.headTags = [
+    {
+      tagName: 'meta',
+      attributes: {
+        name: 'google-site-verification',
+        content: gverify
+      }
+    }
+  ]
 }
+
 if (process.env.RY_ANALYTICS !== undefined && process.env.RY_ANALYTICS !== '') {
   const script = {
     src: process.env.RY_ANALYTICS,
@@ -246,9 +285,6 @@ if (process.env.RY_ANALYTICS !== undefined && process.env.RY_ANALYTICS !== '') {
   if (process.env.RY_ANALYTICS_SITEID !== undefined && process.env.RY_ANALYTICS_SITEID !== '') {
     script['data-site-id'] = process.env.RY_ANALYTICS_SITEID;
   }
-  if (process.env.RY_ANALYTICS_REPLAY !== undefined && process.env.RY_ANALYTICS_REPLAY !== '') {
-    script['data-session-replay'] = process.env.RY_ANALYTICS_REPLAY;
-  }
   config.scripts.push(script)
 }
 

docsite/package.json

@@ -1,10 +1,10 @@
 {
-  "name": "my-website",
-  "version": "0.0.0",
+  "name": "multi-scrobbler-docs",
+  "version": "0.12.1",
   "private": true,
   "scripts": {
     "start": "docusaurus start --host 0.0.0.0",
-    "build": "docusaurus build",
+    "build": "npm run -s prebuild && docusaurus build",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
     "clear": "docusaurus clear",
@@ -12,13 +12,16 @@
     "write-translations": "docusaurus write-translations",
     "write-heading-ids": "docusaurus write-heading-ids",
     "typecheck": "tsc",
-    "postinstall": "patch-package"
+    "postinstall": "patch-package",
+    "previews:build-renderer": "tsc --project tsconfig.client.json || exit 0",
+    "prebuild": "npm run -s previews:build-renderer"
   },
   "dependencies": {
-    "@docusaurus/core": "3.8.1",
-    "@docusaurus/faster": "3.8.1",
-    "@docusaurus/plugin-client-redirects": "^3.8.1",
-    "@docusaurus/preset-classic": "3.8.1",
+    "@bony_chops/docusaurus-og": "^1.0.3-beta.3",
+    "@docusaurus/core": "3.9.2",
+    "@docusaurus/faster": "3.9.2",
+    "@docusaurus/plugin-client-redirects": "^3.9.2",
+    "@docusaurus/preset-classic": "3.9.2",
     "@easyops-cn/docusaurus-search-local": "0.51.1",
     "@mdx-js/react": "^3.0.0",
     "ajv": "^8.18.0",
@@ -39,10 +42,10 @@
     "sass": "^1.97.0"
   },
   "devDependencies": {
-    "@docusaurus/module-type-aliases": "^3.6.3",
-    "@docusaurus/tsconfig": "^3.6.3",
-    "@docusaurus/types": "^3.6.3",
-    "@types/react": "^18.2.29",
+    "@docusaurus/module-type-aliases": "^3.9.2",
+    "@docusaurus/tsconfig": "^3.9.2",
+    "@docusaurus/types": "^3.9.2",
+    "@types/react": "^19.2.4",
     "typescript": "~5.6.2"
   },
   "browserslist": {
@@ -60,5 +63,9 @@
   "engines": {
     "node": ">=24.14.0",
     "npm": ">=11.12.1"
+  },
+  "overrides": {
+    "lodash": "4.18.1",
+    "satori": "0.25.0"
   }
 }

docsite/patches/@bony_chops+docusaurus-og+1.0.3-beta.3.patch

@@ -0,0 +1,42 @@
+diff --git a/node_modules/@bony_chops/docusaurus-og/lib/server/docs.plugin.js b/node_modules/@bony_chops/docusaurus-og/lib/server/docs.plugin.js
+index b308c02..e0d291a 100644
+--- a/node_modules/@bony_chops/docusaurus-og/lib/server/docs.plugin.js
++++ b/node_modules/@bony_chops/docusaurus-og/lib/server/docs.plugin.js
+@@ -93,7 +93,16 @@ class DocsPlugin {
+             await document.write();
+         }
+     };
+-    getHtmlPath = (doc) => doc.metadata?.permalink &&
+-        path.join(this.context.outDir, doc.metadata.permalink, 'index.html');
++    getHtmlPath = (doc) => {
++        if(doc.metadata?.permalink !== undefined) {
++            let relativePerma = doc.metadata.permalink;
++            if(this.context.baseUrl !== undefined && this.context.baseUrl !== '/') {
++               relativePerma = path.join('/', relativePerma.replace(this.context.baseUrl, ''));
++            }
++            const p = path.join(this.context.outDir, relativePerma, 'index.html');
++            return p;
++        }
++        return undefined;
++    }
+ }
+ exports.DocsPlugin = DocsPlugin;
+diff --git a/node_modules/@bony_chops/docusaurus-og/lib/server/pages.plugin.js b/node_modules/@bony_chops/docusaurus-og/lib/server/pages.plugin.js
+index 4676f56..03f03a5 100644
+--- a/node_modules/@bony_chops/docusaurus-og/lib/server/pages.plugin.js
++++ b/node_modules/@bony_chops/docusaurus-og/lib/server/pages.plugin.js
+@@ -101,6 +101,13 @@ class PagesPlugin {
+             await document.write();
+         }
+     };
+-    getHtmlPath = (permalink) => path.join(this.context.outDir, permalink, 'index.html');
++    getHtmlPath = (permalink) => {
++        let relativePerma = permalink;
++        if(this.context.baseUrl !== undefined && this.context.baseUrl !== '/') {
++            relativePerma = path.join('/', relativePerma.replace(this.context.baseUrl, ''));
++        }
++        const p = path.join(this.context.outDir, relativePerma, 'index.html');
++        return p;
++    }
+ }
+ exports.PagesPlugin = PagesPlugin;

docsite/src/components/AIOExample.tsx

@@ -2,9 +2,6 @@ import React, { Fragment } from "react"
 import CodeBlock from '@theme/CodeBlock';
 import Admonition from '@theme/Admonition';
 import ErrorBoundary from "@docusaurus/ErrorBoundary"
-import Error from "@theme/Error"
-import { Simulate } from "react-dom/test-utils";
-import error = Simulate.error;
 import json5 from 'json5';
 
 export interface AIOProps {

docsite/src/components/AdmonitionDetails.tsx

@@ -5,9 +5,10 @@ import IconDanger from '@theme/Admonition/Icon/Danger';
 import IconTip from '@theme/Admonition/Icon/Tip';
 import IconNote from '@theme/Admonition/Icon/Note';
 import { ReactElement } from 'react';
+import AdmonitionIconImportant from './ImportantIcon';
 
 export interface DetailsAdmoProps extends DetailProps {
-    type?: 'warning' | 'danger' | 'note' | 'tip'
+    type?: 'warning' | 'danger' | 'note' | 'tip' | 'important'
 }
 
 const DetailsAdmo = (props: DetailsAdmoProps) => {
@@ -32,6 +33,10 @@ const DetailsAdmo = (props: DetailsAdmoProps) => {
             cn = 'alert--success';
             icon = <IconTip/>
             break;
+        case 'important':
+            cn = 'alert--important';
+            icon = <AdmonitionIconImportant/>;
+            break;            
     }
 
     const iconWrapper = icon === undefined ? null : <span className="admoIcon">{icon}</span>;

docsite/src/components/FileConfigExample.tsx

@@ -2,9 +2,6 @@ import React, { Fragment, useMemo } from "react"
 import CodeBlock, {Props as CodeBlockProps} from '@theme/CodeBlock';
 import Admonition from '@theme/Admonition';
 import ErrorBoundary from "@docusaurus/ErrorBoundary"
-import Error from "@theme/Error"
-import { Simulate } from "react-dom/test-utils";
-import error = Simulate.error;
 import json5 from 'json5';
 import FileExample, { FileProps } from "./FileExample";
 

docsite/src/components/FileExample.tsx

@@ -1,9 +1,6 @@
 import React, { Fragment, useMemo } from "react"
 import CodeBlock, { Props as CodeBlockProps } from '@theme/CodeBlock';
-import Admonition from '@theme/Admonition';
 import ErrorBoundary from "@docusaurus/ErrorBoundary"
-import Error from "@theme/Error"
-import { Simulate } from "react-dom/test-utils";
 import { useTypedLocalStorage } from "./useLocalStorage";
 import ButtonGroup from "./ButtonGroup";
 import EditorFetch from '@site/src/components/Schema/SchemaEditorFetch';

docsite/src/components/ImportantIcon.tsx

@@ -0,0 +1,13 @@
+import type { Props } from '@theme/Admonition/Icon/Info';
+
+export default function AdmonitionIconImportant(props: Props) {
+    // based on https://icon-sets.iconify.design/zondicons/exclamation-outline/
+  return (
+    <svg viewBox="0 0 22 22" {...props}>
+      <path
+        fillRule="evenodd"
+        d="M2.93 17.07A10 10 0 1 1 17.07 2.93A10 10 0 0 1 2.93 17.07m12.73-1.41A8 8 0 1 0 4.34 4.34a8 8 0 0 0 11.32 11.32M9 5h2v6H9zm0 8h2v2H9z"
+      />
+    </svg>
+  );
+}

docsite/src/components/PlaygroundInner.tsx

@@ -6,10 +6,10 @@ import CodeBlock from '@theme/CodeBlock';
 import Schema from "@site/static/schemas/aio.json";
 import ConfigExample from "@site/static/configExample.json";
 import './Playground.scss';
-import { JsonSchemaViewer } from 'cf-json-schema-viz';
+import { JSONSchema, JsonSchemaViewer } from 'cf-json-schema-viz';
 import "./json-schema-viewer-styles.css";
 import "./modern-json-react-styles.css";
-import { ReactJsonEditor, createAjvValidator, type Content  } from 'modern-react-json-editor';
+import { Mode, ReactJsonEditor, createAjvValidator, type Content  } from 'modern-react-json-editor';
 import f from "ajv-formats"
 import DetailsAdmo from "./AdmonitionDetails";
 
@@ -47,10 +47,9 @@ function PlaygroundInner(): JSX.Element {
                     <p>Use this to understand how to write a valid config.</p>
                 </DetailsAdmo>
                 <JsonSchemaViewer
-                name="Todos Model"
-                schema={Schema}
-                expanded={false}
-                hideTopBar={false}
+                schema={Schema as JSONSchema}
+                // expanded={false}
+                // hideTopBar={false}
                 renderRootTreeLines={true}
                 emptyText="No schema defined"
                 defaultExpandedDepth={0}
@@ -93,7 +92,7 @@ function PlaygroundInner(): JSX.Element {
     validator={validator} 
     theme={colorMode} 
     content={data}
-    mode="text"
+    mode={"text" as Mode.text}
     style={{height: 'initial'}}
      />
                 )}

docsite/src/components/Schema/ExplorerSSR.tsx

@@ -1,7 +0,0 @@
-import Schema from "@site/static/aio.json";
-import Explorer from './Explorer';
-
-export default function ExplorerSSR(): JSX.Element {
-
-    return <Explorer schema={Schema}/>
-}

docsite/src/components/Schema/SchemaEditor.tsx

@@ -4,7 +4,7 @@ import { useColorMode } from '@docusaurus/theme-common';
 
 const STRINGIFY_JSON = (json: unknown) => JSON.stringify(json, null, "\t");
 import "../modern-json-react-styles.css";
-import { JSONSchema, ReactJsonEditor, createAjvValidator, type Content, type Validator } from 'modern-react-json-editor';
+import { JSONSchema, Mode, ReactJsonEditor, createAjvValidator, type Content, type Validator } from 'modern-react-json-editor';
 import f from "ajv-formats"
 import json5 from 'json5';
 
@@ -58,7 +58,7 @@ function SchemaEditorInner(props: SchemaEditorProps): JSX.Element {
             <ReactJsonEditor
                 validator={validator}
                 theme={colorMode}
-                mode="text"
+                mode={"text" as Mode.text}
                 content={initialData}
                 style={{height: 'initial'}}
             />

docsite/src/components/snippets/_upgrade-required.mdx

@@ -0,0 +1,15 @@
+:::warning[Required Upgrade]
+
+You **MUST** upgrade to this version **before** upgrading Multi-Scrobbler to a newer version.
+
+This version contains functionality to migrate your data from **{props.before}** that newer versions depend on.
+
+<details>
+
+<summary>Upgrade Process Example</summary>
+
+* Upgrade Multi-Scrobbler {props.old} to {props.current}
+* **After** upgrading, you can optionally upgrade to > {props.current}
+
+</details>
+:::

docsite/src/contexts/PlaygroundContext.tsx

@@ -1,50 +0,0 @@
-import { useContext, createContext } from "react"
-
-// Type I need for useRef
-import type { MonacoEditorTypes } from "@theme/MonacoEditor"
-
-export type State = {
-    // The full schema
-    // We might need to scope it with a JSON Pointer
-    fullSchema: unknown
-    // The current schema displayed (after the json pointer)
-    userSchema: unknown
-    // The current json pointer
-    jsonPointer: string
-    // Schema editor ref
-    schemaRef: undefined | MonacoEditorTypes.IStandaloneCodeEditor
-    // Data editor ref
-    editorRef: undefined | MonacoEditorTypes.IStandaloneCodeEditor
-    // Data on the data editor
-    // When using "Infer schema from data", it would be annoying to lose our written
-    data: string
-    // Errors in schema
-    schemaErrors: MonacoEditorTypes.IMarkerData[]
-    // Errors in data
-    dataErrors: MonacoEditorTypes.IMarkerData[]
-}
-
-export type Playground = {
-    // state
-    state: State
-    // update function
-    updateState: (_: Partial<State>) => void
-}
-
-export const PlaygroundContext = createContext<Playground>({
-    state: {
-        fullSchema: {},
-        userSchema: {},
-        jsonPointer: "",
-        data: "{}",
-        schemaRef: undefined,
-        editorRef: undefined,
-        schemaErrors: [],
-        dataErrors: [],
-    },
-    updateState: () => {},
-})
-
-export const usePlaygroundContext = () => useContext(PlaygroundContext)
-
-export const PlaygroundContextProvider = PlaygroundContext.Provider

docsite/src/css/custom.css

@@ -13,6 +13,7 @@
   --ifm-color-primary-light: #33925d;
   --ifm-color-primary-lighter: #359962;
   --ifm-color-primary-lightest: #3cad6e;
+  --ifm-color-important-contrast-background: #bb91ff;
   --ifm-code-font-size: 95%;
   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1);
 }
@@ -26,6 +27,7 @@
   --ifm-color-primary-light: #29d5b0;
   --ifm-color-primary-lighter: #32d8b4;
   --ifm-color-primary-lightest: #4fddbf;
+  --ifm-color-important-contrast-background: #5a3795;
   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3);
 }
 
@@ -59,6 +61,10 @@ details[class^='details_'].alert--info {
     --ifm-alert-background-color: var(--ifm-color-secondary-contrast-background);
     --ifm-alert-foreground-color: var(--ifm-color-secondary-contrast-foreground);
   }
+  &.alert--important {
+    --ifm-alert-background-color: var(--ifm-color-important-contrast-background);
+    --ifm-alert-foreground-color: var(--ifm-color-secondary-contrast-foreground);
+  }
 
   > summary {
     --docusaurus-details-decoration-color: currentColor;
@@ -90,4 +96,11 @@ details[class^='details_'].alert--info {
     --ifm-spacing-vertical: 0.5rem;
   }
 
+}
+
+/** https://stackoverflow.com/a/79808430/1469797 */
+img.image-center {
+  display: block;
+  margin-left: auto;
+  margin-right: auto;
 }

docsite/src/pages/playground.tsx

@@ -32,7 +32,7 @@ export default function Playground(): JSX.Element {
     return (
         <Layout
             title={`Playground`}
-            description="Config Playground"
+            description="Playground"
         >
             <PlaygroundComponent />
         </Layout>

docsite/src/socialCard/ImageRenderers.tsx

@@ -0,0 +1,92 @@
+import type { DocsPageData, ImageRenderer } from '@bony_chops/docusaurus-og'
+import { readFileSync } from 'fs'
+import { join } from 'path'
+
+// const logo = (<svg
+//   id="Layer_1"
+//   data-name="Layer 1"
+//   xmlns="http://www.w3.org/2000/svg"
+//   width="128"
+//   height="128"
+//   viewBox="0 0 128 128"
+//   fill="#c34a36"
+// >
+//   <path className="cls-1" d="M76.94,65.49c.87-7.85-5.27-14.51-12.94-14.51-4.56,0-8.57,2.36-10.9,5.91h-4.55c2.36-5.11,7.17-8.87,12.95-9.71V20c0-11.05-8.95-20-20-20H5C2.24,0,0,2.24,0,5V123c0,2.76,2.24,5,5,5H41.5c11.05,0,20-8.95,20-20v-27.17c-5.78-.84-10.59-4.6-12.95-9.72h4.55c2.59,3.96,7.27,6.43,12.46,5.82,6.08-.71,10.71-5.37,11.39-11.45ZM15.3,94.39c0,1.1-.9,2-2,2s-2-.9-2-2V33.61c0-1.1,.9-2,2-2s2,.9,2,2v60.78Zm11.12,14.55c0,1.1-.9,2-2,2s-2-.9-2-2V19.06c0-1.1,.9-2,2-2s2,.9,2,2V108.94Zm37.58-39.53c-2.28,0-4.23-1.41-5.02-3.41h-21.43v28.39c0,1.1-.9,2-2,2s-2-.9-2-2V33.61c0-1.1,.9-2,2-2s2,.9,2,2v28.39h21.43c.93-2.33,3.42-3.86,6.18-3.29,2.05,.42,3.7,2.08,4.13,4.12,.73,3.5-1.92,6.58-5.29,6.58Z" />
+//   <path className="cls-1" d="M123,0h-36.5c-11.05,0-20,8.95-20,20v27.18c7.55,1.12,13.5,7.2,14.4,14.82h9.55v-28.28c0-1,.68-1.92,1.66-2.08,1.25-.2,2.34,.76,2.34,1.97v60.67c0,1-.68,1.92-1.66,2.08-1.25,.2-2.34-.76-2.34-1.97v-28.39h-9.55c-.9,7.62-6.84,13.7-14.4,14.82v27.18c0,11.05,8.95,20,20,20h36.5c2.76,0,5-2.24,5-5V5c0-2.76-2.24-5-5-5Zm-17.42,108.94c0,1.1-.9,2-2,2s-2-.9-2-2V19.06c0-1.1,.9-2,2-2s2,.9,2,2V108.94Zm11.12-14.55c0,1.1-.9,2-2,2s-2-.9-2-2V33.61c0-1.1,.9-2,2-2s2,.9,2,2v60.78Z" />
+// </svg>);
+
+const DocContent = (props: { data: DocsPageData }) => {
+  const { data } = props;
+  return (
+    <div style={{
+      height: '100%',
+      width: '100%',
+      display: 'flex',
+      textAlign: 'center',
+      alignItems: 'flex-start',
+      backgroundColor: "#1b1b1d",
+      color: 'white',
+      justifyContent: 'flex-start',
+      flexDirection: 'column',
+      flexWrap: 'wrap',
+      padding: '3rem 3rem 0 3rem',
+    }}>
+
+      <div style={{
+        display: 'flex',
+        alignItems: 'center',
+        justifyContent: 'space-between',
+        gap: '30px'
+      }}>
+        <img src="https://docs.multi-scrobbler.app/img/icon.png" width={120} height={120} />
+        <div style={{
+          fontSize: 125, 
+          //fontWeight: 600
+          }}>Multi-Scrobbler</div>
+      </div>
+      <div style={{
+        fontSize: 50,
+        color: "#c8c8c8",
+        marginTop: '1rem'
+      }}
+      >Listen everywhere, Scrobble anywhere</div>
+      <div style={{
+        marginTop: '4rem',
+        display: 'flex',
+        alignContent: 'flex-start',
+        justifyContent: 'flex-start',
+        flexDirection: 'column',
+        gap: '10px',
+        paddingLeft: '0.5em',
+        fontSize: 75
+      }}>
+      <div style={{ 
+        display: 'flex',
+        textWrap: 'balance',
+        textAlign: 'left'
+         }}>{data.metadata.frontMatter?.description ?? data.metadata.title}</div>
+      </div>
+    </div>
+  );
+}
+
+export const docs: ImageRenderer<DocsPageData> = (data, context) => [
+  <DocContent data={data} />,
+  {
+    width: 1200,
+    height: 630,
+    fonts: [
+      {
+        name: 'Inter',
+        data: readFileSync(
+          // must be a STATIC (not variable) font due to a bug in satori
+          // https://github.com/vercel/satori/issues/162
+          // use static-font.sh to convert a variable font to static
+          join(__dirname, '../../src/socialCard/AdwaitaSans-Regular-StaticSubset.ttf'),
+        ),
+        weight: 400,
+        style: 'normal',
+      },
+    ],
+  },
+]

docsite/src/socialCard/static-font.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# based on https://github.com/Kikobeats/og.kikobeats.com/blob/master/static-font.sh
+
+# 1) Freeze SF Pro to Bold (700) and make it fully static
+fonttools varLib.instancer AdwaitaSans-Regular.ttf \
+  wght=400 \
+  --static \
+  -o AdwaitaSans-Regular-Static.ttf
+
+# 2) Subset to only the glyphs you actually need (Satori-friendly)
+fonttools subset AdwaitaSans-Regular-Static.ttf \
+  --text="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#\$%^&*()-_=+[]{};:'\",.<>/? " \
+  --no-hinting \
+  --name-legacy \
+  --output-file=AdwaitaSans-Regular-StaticSubset.ttf
+

docsite/static/robots.txt

@@ -1,4 +1,4 @@
-Sitemap: https://foxxmd.github.io/multi-scrobbler/sitemap.xml
+Sitemap: https://docs.multi-scrobbler.app/sitemap.xml
 
 User-agent: *
 Allow: /

docsite/tsconfig.client.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "noEmit": false,
+    "composite": true,
+    "incremental": true,
+    "esModuleInterop": true,
+    "tsBuildInfoFile": "./lib/.tsbuildinfo-client",
+    "rootDir": "src",
+    "outDir": "lib",
+    "module": "CommonJS",
+    "target": "esnext",
+    "jsx": "react-jsx",
+    "types": ["node"],
+    "baseUrl": "./docs",
+    "lib": ["DOM"]
+  },
+  "include": ["src/socialCard/ImageRenderers.tsx"]
+}

docsite/tsconfig.json

@@ -2,6 +2,7 @@
   // This file is not used in compilation. It is here just for a nice editor experience.
   "extends": "@docusaurus/tsconfig",
   "compilerOptions": {
+    "jsx": "react-jsx",
     "baseUrl": ".",
     "resolveJsonModule": true,
     "types": ["node", "@docusaurus/module-type-aliases", "@docusaurus/theme-classic","docusaurus-plugin-sass"]

drizzle.config.ts

@@ -0,0 +1,13 @@
+import 'dotenv/config';
+import { defineConfig } from 'drizzle-kit';
+import { configDir, projectDir } from './src/backend/common/index.js';
+import * as path from 'path';
+
+export default defineConfig({
+  schema: path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'),
+  out: path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations'),
+  dialect: 'postgresql',
+  dbCredentials: {
+    url: path.resolve(configDir, 'msDb'),
+  },
+});

package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-scrobbler",
-  "version": "0.12.1",
+  "version": "0.13.2",
   "type": "module",
   "description": "scrobble plays from multiple sources to multiple clients",
   "scripts": {
@@ -16,7 +16,8 @@
     "build:backend": "tsc -p src/backend && npm run -s schema:app",
     "build": "npm run -s build:backend && npm run -s build:frontend && npm run -s docs:build",
     "build:parallel": "concurrently --kill-others-on-fail --names backend,frontend,docs \"npm run -s build:backend\" \"npm run -s build:frontend\" \"npm run docs:build\"",
-    "docs:install": "cd docsite && npm ci --no-audit",
+    "db:start": "pglite-server --db=./config/msDb --port=5433 --host=0.0.0.0 -m 10",
+    "docs:install": "cd docsite && npm install --no-audit",
     "docs:start": "cd docsite && npm start",
     "docs:build": "npm run -s schema:docs && cd docsite && npm run build",
     "docs:netlify": "npm run -s docs:install && npm run -s docs:build",
@@ -34,7 +35,7 @@
   },
   "engines": {
     "node": ">=24.14.0",
-    "npm": ">=11.12.1"
+    "npm": "^11.12.1"
   },
   "repository": {
     "type": "git",
@@ -52,15 +53,10 @@
     "@atcute/identity-resolver-node": "^1.0.3",
     "@atproto/api": "^0.18.0",
     "@atproto/oauth-client-node": "^0.3.10",
-    "@awaitjs/express": "^0.6.3",
-    "@chakra-ui/react": "^3.34.0",
-    "@curvenote/ansi-to-react": "^7.0.0",
     "@donedeal0/superdiff": "^1.1.1",
-    "@emotion/react": "^11.14.0",
+    "@electric-sql/pglite": "^0.4.5",
+    "@electric-sql/pglite-socket": "^0.1.5",
     "@ewanc26/tid": "^1.0.2",
-    "@fortawesome/fontawesome-svg-core": "^6.4.2",
-    "@fortawesome/free-solid-svg-icons": "^6.4.2",
-    "@fortawesome/react-fontawesome": "^0.2.0",
     "@foxxmd/chromecast-client": "^1.0.4",
     "@foxxmd/get-version": "^0.0.3",
     "@foxxmd/logging": "^0.2.6",
@@ -71,24 +67,19 @@
     "@kenyip/backoff-strategies": "^1.0.4",
     "@keyv/valkey": "^1.0.8",
     "@lukehagar/plexjs": "^0.39.0",
-    "@react-nano/use-event-source": "^0.13.0",
-    "@reduxjs/toolkit": "^1.9.5",
     "@supercharge/promise-pool": "^3.0.0",
     "@svrooij/sonos": "^2.5.0",
-    "@tailwindcss/vite": "^4.1.16",
-    "@types/express-serve-static-core": "^4.19.6",
     "@xhayper/discord-rpc": "^1.3.0",
+    "abort-controller-x": "^0.5.0",
     "address": "^1.2.2",
     "ajv": "^8.18.0",
     "ajv-formats": "^3.0.1",
-    "autoprefixer": "^10.4.17",
     "avahi-browse": "^1.1.4",
     "better-sse": "^0.8.0",
     "body-parser": "^2.2.2",
     "cacheable": "^1.10.4",
     "castv2": "^0.1.10",
     "clone": "^2.1.2",
-    "clsx": "^2.0.0",
     "common-tags": "^1.8.2",
     "compare-versions": "^4.1.2",
     "concat-stream": "^2.0.0",
@@ -97,9 +88,10 @@
     "dbus-ts": "^0.0.7",
     "discord.js": "^14.26.0",
     "dotenv": "^10.0.0",
-    "express": "^4.22.1",
+    "drizzle-orm": "^1.0.0-rc.2-c5a84d1",
+    "express": "^5.2.1",
     "express-session": "^1.19.0",
-    "fast-deep-equal": "^3.1.3",
+    "fast-equals": "^6.0.0",
     "fixed-size-list": "^0.3.0",
     "flat-cache": "^6.1.13",
     "formidable": "^3.5",
@@ -113,16 +105,15 @@
     "json-diff-ts": "^5.0.0-alpha.2",
     "json5": "^2.2.3",
     "jsondiffpatch": "^0.7.3",
-    "jsondiffpatch-react": "^1.0.8",
     "keyv": "^5.5.0",
     "kodi-api": "^0.2.1",
     "lastfm-ts-api": "^2.6.0",
     "merge-error-cause": "^5.0.2",
     "mopidy": "^1.3.0",
-    "mpd-api": "^1.1.2",
+    "mpc-js": "^2.1.1",
     "musicbrainz-api": "^0.27.0",
     "nanoid": "^3.3.1",
-    "next-themes": "^0.4.6",
+    "neotraverse": "^0.6.18",
     "node-object-hash": "^3.1.1",
     "normalize-url": "^8.0.1",
     "ntfy": "^1.15.2",
@@ -133,41 +124,41 @@
     "passport-deezer": "^0.2.0",
     "patch-package": "^8.0.1",
     "prom-client": "^15.1.3",
-    "react": "^18.2.0",
-    "react-dom": "^18.2.0",
-    "react-error-boundary": "^6.1.1",
-    "react-icons": "^5.6.0",
-    "react-redux": "^8.1.2",
-    "react-router-dom": "^6.30.3",
-    "react-virtuoso": "^4.18.3",
     "round-robin-js": "^3.0.10",
-    "sass": "^1.93.3",
     "serialize-error": "^13.0.1",
-    "shiki": "^4.0.2",
     "spotify-web-api-node": "^5.0.2",
     "superagent": "^8.0.9",
-    "tailwindcss": "^4.1.16",
     "toad-scheduler": "^3.0.0",
     "tough-cookie": "^5.1.2",
     "ts-json-schema-generator": "^2.3.0",
-    "tsx": "^4.7.0",
+    "tsx": "^4.21.0",
     "typeson": "^9.0.4",
     "typeson-registry": "^11.1.1",
     "undici": "7.24.6",
-    "vite-express": "^0.16.0",
+    "vite-express": "^0.22.1",
     "vlc-client": "^1.1.1",
     "xml2js": "0.6.2",
     "youtubei.js": "^15.0.1"
   },
   "devDependencies": {
+    "@chakra-ui/react": "^3.34.0",
     "@chromatic-com/storybook": "^5.0.1",
+    "@curvenote/ansi-to-react": "^7.0.0",
     "@dbus-types/notifications": "^0.0.5",
+    "@electric-sql/pglite-prepopulatedfs": "^0.0.3",
+    "@emotion/react": "^11.14.0",
     "@eslint/js": "^8.56.0",
     "@faker-js/faker": "^9.0.1",
+    "@fortawesome/fontawesome-svg-core": "^6.7.2",
+    "@fortawesome/free-solid-svg-icons": "^6.7.2",
+    "@fortawesome/react-fontawesome": "^0.2.6",
     "@gr2m/net-interceptor": "^1.0.0",
+    "@react-nano/use-event-source": "^0.13.0",
+    "@reduxjs/toolkit": "^1.9.7",
     "@storybook/addon-a11y": "10.1.11",
     "@storybook/addon-docs": "10.1.11",
     "@storybook/react-vite": "10.1.11",
+    "@tailwindcss/vite": "^4.2.2",
     "@tsconfig/node18": "^1.0.1",
     "@types/chai": "^4.3.0",
     "@types/chai-as-promised": "^7.1.5",
@@ -175,13 +166,14 @@
     "@types/common-tags": "^1.8.1",
     "@types/concat-stream": "^2.0.0",
     "@types/cors": "^2.8.17",
-    "@types/express": "^4.17.13",
-    "@types/express-session": "^1.17.4",
+    "@types/express": "^5.0.6",
+    "@types/express-serve-static-core": "^5.1.1",
+    "@types/express-session": "^1.18.2",
     "@types/formidable": "^3.4.5",
     "@types/jest": "^27.5.2",
     "@types/jscodeshift": "^0.11.6",
     "@types/mocha": "^9.1.0",
-    "@types/node": "^20.19.2",
+    "@types/node": "^24.12.2",
     "@types/object-hash": "^3.0.0",
     "@types/passport": "^1.0.12",
     "@types/react": "^18.2.18",
@@ -192,25 +184,40 @@
     "@types/superagent": "^8.1.9",
     "@types/xml2js": "^0.4.11",
     "@vitejs/plugin-react": "^4.2.1",
+    "autoprefixer": "^10.4.27",
     "chai": "^4.3.6",
-    "chai-as-promised": "^7.1.1",
+    "chai-as-promised": "^8.0.2",
+    "clsx": "^2.1.1",
+    "drizzle-kit": "^1.0.0-rc.2-c5a84d1",
     "eslint": "^8.56.0",
     "eslint-plugin-prefer-arrow-functions": "^3.2.4",
     "eslint-plugin-storybook": "10.1.11",
     "git-cliff": "^2.12.0",
+    "jsondiffpatch-react": "^1.0.8",
     "mocha": "^10.3.0",
     "mockdate": "^3.0.5",
     "msw": "^2.12.10",
-    "neotraverse": "^0.6.18",
+    "next-themes": "^0.4.6",
     "nodemon": "^3.0.3",
     "playwright": "^1.58.2",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "react-error-boundary": "^6.1.1",
+    "react-icons": "^5.6.0",
+    "react-redux": "^8.1.3",
+    "react-router-dom": "^6.30.3",
+    "react-virtuoso": "^4.18.4",
+    "sass": "^1.99.0",
+    "shiki": "^4.0.2",
     "sinon": "^21.0.2",
     "storybook": "10.2.0",
-    "ts-essentials": "^10.1.1",
+    "tailwindcss": "^4.2.2",
+    "tinyexec": "^1.1.1",
+    "ts-essentials": "^10.2.0",
     "typescript": "^5.9.3",
     "typescript-eslint": "^7.0.1",
     "vite": "^5.4.21",
-    "with-local-tmp-dir": "^6.0.0"
+    "with-local-tmp-dir": "^7.0.1"
   },
   "browserslist": {
     "production": [
@@ -227,6 +234,9 @@
   "overrides": {
     "spotify-web-api-node": {
       "superagent": "$superagent"
+    },
+    "musicbrainz-api": {
+      "rate-limit-threshold": "0.2.1"
     }
   }
 }

src/backend/common/AbstractComponent.ts

@@ -2,13 +2,11 @@ import { childLogger, Logger } from "@foxxmd/logging";
 import {
     cacheFunctions,
 } from "@foxxmd/regex-buddy-core";
-import deepEqual from 'fast-deep-equal';
-import { Simulate } from "react-dom/test-utils";
 import { LifecycleStep, PlayData, PlayObject, TransformResult } from "../../core/Atomic.js";
 import { buildPlayHumanDiffable, buildTrackString } from "../../core/StringUtils.js";
 import { CommonClientConfig } from "./infrastructure/config/client/index.js";
 import { CommonSourceConfig } from "./infrastructure/config/source/index.js";
-import { mergeSimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js";
+import { mergeSimpleError, SimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js";
 import {
     FLOW_CONTROL_TERM,
     PlayTransformRules,
@@ -22,12 +20,21 @@ import { getRoot } from "../ioc.js";
 import { nanoid } from "nanoid";
 import { isDebugMode } from "../utils.js";
 import { findCauseByReference } from "../utils/ErrorUtils.js";
-import { hashObject } from "../utils/StringUtils.js";
+import { hashObject, parseArrayFromMaybeString } from "../utils/StringUtils.js";
 import { metaInvariantTransform, playContentInvariantTransform } from "../utils/PlayComparisonUtils.js";
 import { MSCache } from "./Cache.js";
 import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/DataUtils.js";
 import clone from "clone";
 import { loggerNoop } from "./MaybeLogger.js";
+import { objectsEqual } from "../utils/DataUtils.js";
+import { RetentionOptions } from "./infrastructure/config/database.js";
+import { getRetentionCompactAfterFromEnv, getRetentionDeleteAfterFromEnv, isCompactableProperty, parseRetentionOptions, parseRetentionOptionsDurations } from "./database/Database.js";
+import { DbConcrete } from "./database/drizzle/drizzleUtils.js";
+import { ComponentSelect } from "./database/drizzle/drizzleTypes.js";
+import { DrizzlePlayRepository } from "./database/drizzle/repositories/PlayRepository.js";
+import { ClientType } from "./infrastructure/config/client/clients.js";
+import { SourceType } from "./infrastructure/config/source/sources.js";
+import { DrizzleComponentRepository } from "./database/drizzle/repositories/ComponentRepository.js";
 
 export type AbstractComponentConfig = (CommonClientConfig | CommonSourceConfig) & { transformManager?: TransformerManager };
 
@@ -39,11 +46,27 @@ export default abstract class AbstractComponent extends AbstractInitializable {
     regexCache!: ReturnType<typeof cacheFunctions>;
     protected transformManager: TransformerManager;
     protected cache: MSCache;
+    protected db: DbConcrete;
+    protected componentRepo!: DrizzleComponentRepository;
+    protected dbComponent!: ComponentSelect;
+    protected retentionOpts: RetentionOptions;
+
+    protected componentType: 'source' | 'client';
+    type: ClientType | SourceType;
 
     protected constructor(config: AbstractComponentConfig) {
         super(config);
         this.transformManager = config.transformManager ?? getRoot().items.transformerManager;
         this.cache = getRoot().items.cache();
+        const cProps = config.options?.retention?.compact ?? parseArrayFromMaybeString(process.env.COMPACT_PROPERTIES, {lower: true});
+        if(!cProps.every(isCompactableProperty)) {
+            throw new SimpleError(`Compactable properties must be one of 'transform' or 'input'. Given: ${cProps.join(',')}`);
+        }
+        this.retentionOpts = {
+            deleteAfter: parseRetentionOptionsDurations(config.options?.retention?.deleteAfter, getRetentionDeleteAfterFromEnv()),
+            compactAfter: parseRetentionOptions(config.options?.retention?.compactAfter, getRetentionCompactAfterFromEnv()),
+            compact: cProps
+        };
     }
 
     protected postCache(): Promise<void> {
@@ -55,6 +78,25 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         }
     }
 
+    protected async doBuildDatabase(): Promise<true | string | undefined> {
+        super.doBuildDatabase();
+
+        let name: string;
+        if('name' in this) {
+            name = this.name as string;
+        }
+
+        this.db = await getRoot().items.db();
+        this.componentRepo = new DrizzleComponentRepository(this.db, {logger: this.logger});
+        this.dbComponent = await this.componentRepo.findOrInsert({
+            mode: this.componentType,
+            type: this.type,
+            uid: this.config.id ?? this.config.name ?? name,
+            name: this.config.name ?? name
+        });
+        return true;
+    }
+
     public buildTransformRules() {
         this.logger.debug('Building transformer rules...');
         try {
@@ -155,6 +197,19 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         }
     }
 
+    public retentionCleanup = async () => {
+        if(this.databaseOK !== true) {
+            this.logger.warn(`Cannot run retention cleanup because ${this.componentType} database state is not OK`);
+            return;
+        }
+        try {
+            const repo = new DrizzlePlayRepository(this.db, {logger: this.logger});
+            await repo.retentionCleanup(this.componentType, this.retentionOpts);
+        } catch (e) {
+            this.logger.warn(new Error('Failed to do retention cleanup', {cause: e}));
+        }
+    }
+
     protected transformPartToStrong(data: any) {
         if(data === undefined) {
             return undefined;
@@ -167,9 +222,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
 
     public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean | 'all') => {
 
-
-        const asyncId = nanoid(6);
-        let logger = childLogger(this.logger, ['Play Transform', hookType, asyncId]);
+        let logger: Logger;
 
         try {
             let hook: StageConfig[];
@@ -193,6 +246,9 @@ export default abstract class AbstractComponent extends AbstractInitializable {
                 return play;
             }
 
+            const asyncId = nanoid(6);
+            logger = childLogger(this.logger, ['Play Transform', hookType, asyncId]);
+
             const shouldLog = log ?? this.config.options?.playTransform?.log ?? isDebugMode();
 
             const transformHash = `playTransform-${hashObject(hook)}-${hashObject(playContentInvariantTransform(play))}`;
@@ -204,7 +260,8 @@ export default abstract class AbstractComponent extends AbstractInitializable {
 
             logger.debug(`Transform start for => ${buildTrackString(play)}`);
             let transformedPlay: PlayObject = clone(play);
-            let cacheOk: boolean = true;
+            let cacheOk: boolean = true,
+            generateSteps = true;
             const steps: LifecycleStep[] = [];
 
             const opts = {
@@ -213,22 +270,29 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             }
 
             if(cachedSteps !== undefined) {
+                generateSteps = false;
                 // don't re-cache cached steps
                 cacheOk = false;
-                // only patch play if steps didn't end in a failure that we are okay with returning partial from
-                let shouldTransform = true;
-                const lastCachedStep = cachedSteps[cachedSteps.length - 1];
-                if(lastCachedStep.error !== undefined && lastCachedStep.flowKnownState !== 'skip' && !lastCachedStep.returnPartial) {
-                    shouldTransform = false;
-                }
-
-                for(const s of cachedSteps) {
-                    steps.push({...s, cached: true});
-                    if(shouldTransform && s.patch !== undefined) {
-                        transformedPlay.data = patchObject(transformedPlay.data, s.patch); // jdiff.patch(clone(transformedPlay.data),s.patch);
+                try {
+                    // only patch play if steps didn't end in a failure that we are okay with returning partial from
+                    let shouldTransform = true;
+                    const lastCachedStep = cachedSteps[cachedSteps.length - 1];
+                    if(lastCachedStep.error !== undefined && lastCachedStep.flowKnownState !== 'skip' && !lastCachedStep.returnPartial) {
+                        shouldTransform = false;
                     }
+
+                    for(const s of cachedSteps) {
+                        steps.push({...s, cached: true});
+                        if(shouldTransform && s.patch !== undefined) {
+                            transformedPlay.data = patchObject(transformedPlay.data, s.patch); // jdiff.patch(clone(transformedPlay.data),s.patch);
+                        }
+                    } 
+                } catch (e) {
+                    logger.warn(new Error('Error occurred while trying to use cached steps. Falling back to generating fresh transform steps', {cause: e}));
+                    generateSteps = true;
                 }
-            } else {
+            }
+            if(generateSteps) {
                 for(const hookItem of hook) {
                     const [step, stepPlay] = await this.generateStepFromStage(transformedPlay, hookItem, hookType, opts);
                     steps.push(step);
@@ -317,7 +381,12 @@ export default abstract class AbstractComponent extends AbstractInitializable {
 
             return transformedPlay;
         } catch (e) {
-            logger.warn(new Error(`Unexpected error occurred, returning original play.`, {cause: e}));
+            const err = new Error(`Unexpected error occurred, returning original play.`, {cause: e});
+            if(logger === undefined) {
+                this.logger.warn(err);
+            } else {
+                logger.warn(err);
+            }
             return play;
         }
     }
@@ -337,7 +406,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
 
         const stepName = `${hookType} - ${hookItem.type} - ${hookItem.name}`
         const existingStepIndex = playTruth.meta.lifecycle.steps.findIndex(x => x.name === stepName && x.source === this.getIdentifier());
-        const step: LifecycleStep = existingStepIndex !== -1 ? playTruth.meta.lifecycle.steps[existingStepIndex] : {
+        const step: LifecycleStep = existingStepIndex !== -1 && playTruth.meta.lifecycle.steps[existingStepIndex] !== undefined ? playTruth.meta.lifecycle.steps[existingStepIndex] : {
             name: stepName,
             source: this.getIdentifier(),
         }
@@ -401,7 +470,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         } else {
             step.flowResult = onSuccess;
 
-            if (!deepEqual(playTruth.data, newTransformedPlay.data)) {
+            if (!objectsEqual(playTruth.data, newTransformedPlay.data)) {
                 const o = JSON.parse(JSON.stringify(playTruth.data));
                 const t = JSON.parse(JSON.stringify(newTransformedPlay.data));
                 const patch = diffObjects(o, t); // jdiff.diff(o, t);

src/backend/common/AbstractInitializable.ts

@@ -1,9 +1,7 @@
 import { childLogger, Logger } from "@foxxmd/logging";
-import { Simulate } from "react-dom/test-utils";
 import {truncateStringToLength } from "../../core/StringUtils.js";
 import { hasNodeNetworkException } from "./errors/NodeErrors.js";
 import { hasUpstreamError } from "./errors/UpstreamError.js";
-import play = Simulate.play;
 import { WebhookPayload } from "./infrastructure/config/health/webhooks.js";
 import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, StageError, TransformRulesError } from "./errors/MSErrors.js";
 import { messageWithCauses, messageWithCausesTruncatedDefault } from "../utils/ErrorUtils.js";
@@ -15,6 +13,7 @@ export default abstract class AbstractInitializable {
     authFailure?: boolean;
 
     buildOK?: boolean | null;
+    databaseOK?: boolean | null;
     connectionOK?: boolean | null;
     cacheOK?: boolean | null;
 
@@ -43,6 +42,7 @@ export default abstract class AbstractInitializable {
             if(this.componentLogger === undefined) {
                 await this.buildComponentLogger();
             }
+            await this.buildDatabase(force);
             await this.buildInitData(force);
             await this.parseCache(force);
             try {
@@ -98,13 +98,13 @@ export default abstract class AbstractInitializable {
             if(!force) {
                 return;
             }
-            this.logger.debug('Cache OK but step was forced');
+            this.logger.verbose('Cache OK but step was forced');
         }
         try {
             const res = await this.doParseCache();
             if(res === undefined) {
                 this.cacheOK = null;
-                this.logger.debug('No cache to parse.');
+                this.logger.trace('No cache to parse.');
                 return;
             }
             if (res === true) {
@@ -141,13 +141,13 @@ export default abstract class AbstractInitializable {
             if(!force) {
                 return;
             }
-            this.logger.debug('Build OK but step was forced');
+            this.logger.verbose('Build OK but step was forced');
         }
         try {
             const res = await this.doBuildInitData();
             if(res === undefined) {
                 this.buildOK = null;
-                this.logger.debug('No required data to build.');
+                this.logger.trace('No required data to build.');
                 return;
             }
             if (res === true) {
@@ -174,6 +174,58 @@ export default abstract class AbstractInitializable {
         return;
     }
 
+    public async buildDatabase(force: boolean = false) {
+        if(this.databaseOK) {
+            if(!force) {
+                return;
+            }
+            this.logger.verbose('Database OK but step was forced');
+        }
+        try {
+            const res = await this.doBuildDatabase();
+            if(res === undefined) {
+                this.databaseOK = null;
+                this.logger.trace('No required database steps.');
+                return;
+            }
+            if (res === true) {
+                this.logger.verbose('Required database init succeeded');
+            } else if (typeof res === 'string') {
+                this.logger.verbose(`Required database init succeeded => ${res}`);
+            }
+            this.databaseOK = true;
+        } catch (e) {
+            this.databaseOK = false;
+            throw new BuildDataError('Required database init failed', {cause: e});
+        }
+
+        try {
+            await this.postDatabase();
+        } catch (e) {
+            if(e instanceof StageError) {
+                throw e;
+            } else {
+                throw new Error('Error occurred during post-database hook', {cause: e});
+            }
+        }
+    }
+
+    protected async postDatabase(): Promise<void> {
+        return;
+    }
+
+    /**
+     * Run/fetch/create any database data needed for this component to operate when ready
+     *
+     * * Return undefined if not possible or not required
+     * * Return TRUE if database steps succeeded
+     * * Return string if database steps succeeded and should log result
+     * * Throw error on failure
+     * */
+    protected async doBuildDatabase(): Promise<true | string | undefined> {
+        return;
+    }
+
     public async checkConnection(force: boolean = false) {
         if(this.connectionOK) {
             if(!force) {
@@ -254,12 +306,14 @@ export default abstract class AbstractInitializable {
 
     public isReady() {
         return (this.buildOK === null || this.buildOK === true) &&
+            (this.databaseOK === null || this.databaseOK === true) &&
             (this.connectionOK === null || this.connectionOK === true)
             && !this.authGated();
     }
 
     public isUsable() {
         return (this.buildOK === null || this.buildOK === true) &&
+            (this.databaseOK === null || this.databaseOK === true) &&
             (this.connectionOK === null || this.connectionOK === true);
     }
 

src/backend/common/Cache.ts

@@ -13,14 +13,15 @@ import { childLogger, Logger } from '@foxxmd/logging';
 import { projectDir } from './index.js';
 import path from 'path';
 import { cacheFunctions } from "@foxxmd/regex-buddy-core";
-import { fileOrDirectoryIsWriteable } from '../utils/FSUtils.js';
-import { asCacheAuthProvider, asCacheConfig, asCacheMetadataProvider, asCacheScrobbleProvider, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheMetadataProvider, CacheProvider, CacheScrobbleProvider } from './infrastructure/Atomic.js';
+import { fileExists, fileOrDirectoryIsWriteable } from '../utils/FSUtils.js';
+import { copyFile } from 'fs/promises';
+import { asCacheConfig, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheConfigUser, CacheScrobbleProvider } from './infrastructure/Atomic.js';
 import { Typeson } from 'typeson';
 import { builtin } from 'typeson-registry';
 import { loggerNoop } from './MaybeLogger.js';
-import { ListenProgressPositional, ListenProgressTS } from '../sources/PlayerState/ListenProgress.js';
 const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
 import prom, { Gauge } from 'prom-client';
+import { nonEmptyStringOrDefault } from '../../core/StringUtils.js';
 
 dayjs.extend(utc)
 dayjs.extend(isBetween);
@@ -37,17 +38,24 @@ typeson.register({
         (x) => dayjs.isDayjs(x),
         (d: Dayjs) => d.toJSON(),
         (date) => dayjs(date)
-    ],
-    ListenProgressTS,
-    ListenProgressPositional
+    ]
 });
 
+const unsupportedEnvKeys = [
+    'CACHE_AUTH_CONN',
+    'CACHE_METADATA',
+    'CACHE_SCROBBLE',
+    'CACHE_SCROBBLE_CONN',
+    'CACHE_AUTH_CONN'
+];
+
 export class MSCache {
 
     config: Required<CacheConfigOptions>
 
     cacheMetadata: Cacheable;
     cacheScrobble: Cacheable;
+    cacheDb: Cacheable;
     cacheAuth: Cacheable;
     regexCache: ReturnType<typeof cacheFunctions>;
     cacheTransform: Cacheable;
@@ -68,19 +76,19 @@ export class MSCache {
 
         const {
             metadata: {
-                provider: mProvider = (process.env.CACHE_METADATA as (CacheMetadataProvider | undefined) ?? false),
-                connection: mConn = process.env.CACHE_METADATA_CONN,
-                ...restMetadata
+                provider: mProvider = false,
+                connection: mConn,
+                //...restMetadata
             } = {},
             scrobble: {
-                provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheScrobbleProvider | undefined) ?? 'file'),
-                connection = (process.env.CACHE_SCROBBLE_CONN ?? configDir),
-                ...restScrobble
+                provider: sProvider = false,
+                connection: sConnection,
+                //...restScrobble
             } = {},
             auth: {
-                provider: aProvider = (process.env.CACHE_AUTH as (CacheAuthProvider | undefined) ?? 'file'),
-                connection: aConn = (process.env.CACHE_AUTH_CONN ?? configDir),
-                ...restAuth
+                provider: aProvider = false,
+                connection: aConn,
+                //...restAuth
             } = {},
             regex = 200,
         } = config;
@@ -89,17 +97,14 @@ export class MSCache {
             metadata: {
                 provider: mProvider,
                 connection: mConn,
-                ...restMetadata,
             },
             scrobble: {
                 provider: sProvider,
-                connection,
-                ...restScrobble
+                connection: sConnection,
             },
             auth: {
                 provider: aProvider,
                 connection: aConn,
-                ...restAuth
             },
             regex
         };
@@ -114,6 +119,7 @@ export class MSCache {
         this.cacheAuth = inMemory;
         this.cacheScrobble = inMemory;
         this.cacheApi = inMemory;
+        this.cacheDb = new Cacheable({primary: initMemoryCache({lruSize: 500, ttl: '1m'})});
     }
 
     init = async (enableCollectors: boolean = false) => {
@@ -133,7 +139,8 @@ export class MSCache {
             { cache: this.cacheScrobble, name: 'queued_scrobbles' },
             { cache: this.cacheTransform, name: 'transformer' },
             { cache: this.cacheClientScrobbles, name: 'historical_scrobbles' },
-            { cache: this.cacheApi, name: 'external_apis' }
+            { cache: this.cacheApi, name: 'external_apis' },
+            { cache: this.cacheDb, name: 'database' }
         ];
 
         this.cacheHits = new prom.Gauge({
@@ -271,10 +278,10 @@ export class MSCache {
             }
         }
         if (config.provider === 'file') {
-            logger.debug(`Building file cache from ${path.join(config.connection, `${namespace}.cache`)}`);
+            logger.debug(`Building file cache from ${path.join(config.connection ?? configDir, `${namespace}.cache`)}`);
 
             try {
-                const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection, cacheId: `${namespace}.cache` }, {ttl: config.ttl}, logger);
+                const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection ?? configDir, cacheId: `${namespace}.cache` }, {ttl: config.ttl}, logger);
                 return keyvFile;
             } catch (e) {
                 throw e;
@@ -364,8 +371,8 @@ export const flatCacheCreate = (opts: FlatCacheOptions) => {
     return new FlatCache({
         ttl: 0,
         lruSize: 2000,
-        cacheDir: opts.cacheDir ?? configDir,
-        cacheId: opts.cacheId ?? 'scrobble.cache',
+        cacheDir: opts.cacheDir,
+        cacheId: opts.cacheId ?? 'ms.cache',
         persistInterval: 1 * 1000 * 10,
         expirationInterval: 1 * 1000 * 10, // 10 seconds
         ...opts
@@ -381,6 +388,12 @@ export const flatCacheLoad = async (flatCache: FlatCache, logger: Logger = logge
         throw new Error(`Unable to use path for file cache at ${cachePath}`, { cause: e })
     }
 
+    if(fileExists(cachePath) && !fileExists(`${cachePath}.bak`)) {
+        logger.info(`Backing up ${cachePath} in preparation for migration to database...`);
+        await copyFile(cachePath, `${cachePath}.bak`);
+        logger.info(`Done! Backed up to ${cachePath}.bak\nAll data has been loaded into cache. It will be deleted (from cache) after migrating to database.\nIf there are migration issues or you wish to downgrade then overwrite ${cachePath} with the .bak backup copy`);
+    }
+
     const streamPromise = new Promise((resolve, reject) => {
         flatCache.loadFileStream(cachePath, (progress: number, total: number) => {
             logger.trace(`Loading ${progress}/${total} chunks...`);
@@ -497,4 +510,86 @@ const noopKeyv: KeyvStoreAdapter = {
         delete: (_) => undefined,
         clear: () => Promise.resolve(),
         on: (_, __) => undefined
+}
+
+export const parseUserConfig = (config: CacheConfigUser = {}, parentLogger: Logger = loggerNoop): CacheConfigOptions => {
+    const logger = childLogger(parentLogger, 'Cache');
+
+        let valkeyEnvVal: string | undefined = nonEmptyStringOrDefault(process.env.CACHE_VALKEY);
+        if(valkeyEnvVal === undefined) {
+            valkeyEnvVal = nonEmptyStringOrDefault(process.env.CACHE_METADATA_CONN);
+            if(valkeyEnvVal !== undefined) {
+                logger.warn('ENV CACHE_METADATA_CONN is deprecated! Replace it with CACHE_VALKEY');
+            }
+        }
+
+        for(const key of unsupportedEnvKeys) {
+            if(nonEmptyStringOrDefault(process.env[key]) !== undefined) {
+                logger.warn(`ENV ${key} is no longer supported. Refer to the Caching docs.`);
+            }
+        }
+
+        const {
+            valkey = valkeyEnvVal,
+            // metadata: {
+            //     provider: mProvider = (process.env.CACHE_METADATA as (CacheMetadataProvider | undefined) ?? false),
+            //     connection: mConn = process.env.CACHE_METADATA_CONN,
+            //     //...restMetadata
+            // } = {},
+            scrobble: {
+                provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheScrobbleProvider | undefined) ?? 'file'),
+                connection = (process.env.CACHE_SCROBBLE_CONN ?? configDir),
+                ...restScrobble
+            } = {},
+            auth: {
+                provider: aProvider = (process.env.CACHE_AUTH as (CacheAuthProvider | undefined) ?? 'file'),
+                //...restAuth
+            } = {},
+            regex = 200,
+        } = config;
+
+        if(config.metadata !== undefined) {
+            logger.warn('Configuring cache.metadata is no longer supported. Refer to the Caching docs.');
+        }
+        if(config.scrobble !== undefined) {
+            logger.warn('Configuring cache.scrobble is no longer supported. Refer to the Caching docs.');
+        }
+        if(config.auth?.connection !== undefined) {
+            logger.warn('Configuring cache.auth.connection is no longer supported. Refer to the Caching docs.');
+        }
+        
+        let authConn: string,
+        authProvider = aProvider;
+        if(authProvider === 'valkey') {
+            if(valkey === undefined) {
+                logger.warn(`Auth Provider set to 'valkey' but not valkey connection string was not provided, falling back to file.`);
+                authConn = configDir;
+                authProvider = 'file';
+            } else {
+                authConn = valkey;
+            }
+        } else {
+            if(authProvider !== 'file') {
+                logger.warn(`Unsupported provider given for auth: ${authProvider}`);
+            }
+            authConn = configDir;
+            authProvider = 'file';
+        }
+
+        return {
+            metadata: {
+                provider: valkey !== undefined ? 'valkey' : false,
+                connection: valkey,
+            },
+            scrobble: {
+                provider: sProvider,
+                connection,
+                ...restScrobble
+            },
+            auth: {
+                provider: authProvider,
+                connection: authConn,
+            },
+            regex
+        };
 }

src/backend/common/database/Database.ts

@@ -0,0 +1,192 @@
+import { configDir } from '../index.js';
+import * as path from 'path';
+import { promises as fs } from 'fs'
+import { childLogger, Logger } from '@foxxmd/logging';
+import { loggerNoop } from '../MaybeLogger.js';
+import { fileExists, fileOrDirectoryIsWriteable } from '../../utils/FSUtils.js';
+import { COMPACTABLE, compactableProperties, CompactableProperty, DEFAULT_RETENTION_DELETE_AFTER, RententionGranular, RetentionConfig, RetentionConfigValue, RetentionOption, RetentionValue, RetentionValueUnparsed } from '../infrastructure/config/database.js';
+import { DurationValue } from '../infrastructure/Atomic.js';
+import { Duration } from 'dayjs/plugin/duration.js';
+import dayjs from 'dayjs';
+import { parseDurationFromDurationValue } from '../../utils/TimeUtils.js';
+import assert, { AssertionError } from 'node:assert';
+import { parseBoolStrict } from '../../utils.js';
+import { SimpleError } from '../errors/MSErrors.js';
+
+export const MEMORY_DB_NAME = ':memory:';
+export const isMemoryDb = (name: string): boolean => name === MEMORY_DB_NAME;
+
+export const getDbPath = (name: string = 'msDb', workingDirectory?: string): string => {
+    if (isMemoryDb(name)) {
+        return MEMORY_DB_NAME;
+    }
+    return path.resolve(workingDirectory ?? configDir, `${name}`);
+}
+
+export const getDbBackupPath = (dbPath: string, suffix?: string): string => {
+    const pathInfo = path.parse(dbPath);
+    const backupPath = `${path.join(pathInfo.dir, pathInfo.name)}.bak${suffix !== undefined ? `.${suffix}` : ''}`;
+    return backupPath;
+}
+
+export const backupDb = async (dbName: string, opts: { logger?: Logger, workingDirectory?: string } = {}): Promise<void> => {
+
+    const {
+        logger: parentLogger = loggerNoop,
+        workingDirectory
+    } = opts;
+
+    const logger = childLogger(parentLogger, 'Migrations');
+
+    const dbPath = getDbPath(dbName, workingDirectory);
+    let newDb = false;
+
+    if (dbPath !== MEMORY_DB_NAME) {
+        if (!fileExists(dbPath)) {
+            logger.info(`Database at ${dbPath} does not exist, will create it.`);
+            newDb = true;
+        }
+        try {
+            fileOrDirectoryIsWriteable(dbPath);
+        } catch (e) {
+            throw new Error('Database path/folder is not writeable, cannot backup database', { cause: e });
+        }
+    }
+
+    if (dbPath !== MEMORY_DB_NAME && !newDb) {
+        const backupPath = `${getDbPath(`${Date.now()}-${dbName}`, workingDirectory)}.bak`;
+        logger.info(`Backing up database before migrating => ${backupPath}`);
+        await fs.copyFile(dbPath, backupPath)
+        logger.info('Backed up!');
+    }
+}
+
+const parseRetentionValue = (val: RetentionValueUnparsed): RetentionValue => {
+    if(typeof val === 'string' || typeof val === 'boolean') {
+        try {
+            const boolVal = parseBoolStrict(val);
+            assert(boolVal === false, 'retention value cannot be true');
+            return boolVal;
+        } catch (e) {
+            // swallow
+        }
+    } else if(dayjs.isDuration(val)) {
+        return val;
+    } else {
+        return parseDurationFromDurationValue(val);
+    }
+    throw new SimpleError('retention value be of one: false, number, or string');
+}
+
+const parseRetentionFromEnv = (): RetentionOption<RetentionValue> => {
+    const deleteAfterEnv = process.env.RETENTION_DELETE_AFTER ?? DEFAULT_RETENTION_DELETE_AFTER,
+        deleteCompletedEnv = process.env.RETENTION_DELETE_COMPLETED_AFTER ?? deleteAfterEnv,
+        deleteFailedEnv = process.env.RETENTION_DELETE_FAILED_AFTER ?? deleteAfterEnv,
+        deleteDupedEnv = process.env.RETENTION_DELETE_DUPED_AFTER ?? deleteAfterEnv;
+
+    return {
+        completed: parseRetentionValue(deleteCompletedEnv),
+        failed: parseRetentionValue(deleteFailedEnv),
+        duped: parseRetentionValue(deleteDupedEnv)
+    }
+}
+
+const isRetentionOptionDurations = (val: RetentionOption<RetentionValue>): val is RetentionOption<Duration> => {
+    return dayjs.isDuration(val.completed)
+    && dayjs.isDuration(val.duped)
+    && dayjs.isDuration(val.failed);
+}
+
+let retentionDeleteAfterFromEnv: RetentionOption<Duration>,
+retentionCompactAfterFromEnv: RetentionOption<RetentionValue>;
+
+export const getRetentionDeleteAfterFromEnv = () => {
+    if (retentionDeleteAfterFromEnv === undefined) {
+        const deleteEnv = parseRetentionFromEnv();
+        if(isRetentionOptionDurations(deleteEnv)) {
+            retentionDeleteAfterFromEnv = deleteEnv;
+        } else {
+            throw new SimpleError('retention deleteAfter values from env must all be one of: number or string');
+        }
+    }
+    return retentionDeleteAfterFromEnv;
+}
+export const getRetentionCompactAfterFromEnv = () => {
+    if (retentionCompactAfterFromEnv === undefined) {
+        const compactEnv = parseRetentionFromEnv();
+        retentionCompactAfterFromEnv = compactEnv;
+    }
+    return retentionCompactAfterFromEnv;
+}
+
+export const parseRetentionOptions = (opts: RetentionConfigValue<DurationValue> = {}, defaults: RetentionOption<RetentionValue>): RetentionOption<RetentionValue> => {
+    if (typeof opts === 'number' || typeof opts === 'string') {
+        const dur = parseDurationFromDurationValue(opts);
+        return {
+            completed: dur,
+            duped: dur,
+            failed: dur
+        }
+    }
+
+    if (opts === undefined) {
+        return defaults;
+    }
+
+    if(dayjs.isDuration(opts)) {
+        return {
+            completed: opts,
+            failed: opts,
+            duped: opts
+        }
+    }
+
+    const {
+        completed = defaults.completed,
+        failed = defaults.failed,
+        duped = defaults.duped
+    } = opts;
+
+    return {
+        completed: parseRetentionValue(completed),
+        failed: parseRetentionValue(failed),
+        duped: parseRetentionValue(duped),
+    }
+}
+
+export const parseRetentionOptionsDurations = (opts: RetentionConfigValue<Exclude<RetentionValueUnparsed, false>> = {}, defaults: RetentionOption<Duration>): RetentionOption<Duration> => {
+    if (typeof opts === 'number' || typeof opts === 'string') {
+        const dur = parseDurationFromDurationValue(opts);
+        return {
+            completed: dur,
+            duped: dur,
+            failed: dur
+        }
+    }
+
+    if(dayjs.isDuration(opts)) {
+        return {
+            completed: opts,
+            failed: opts,
+            duped: opts
+        }
+    }
+
+    if (opts === undefined) {
+        return defaults;
+    }
+
+    const {
+        completed = defaults.completed,
+        failed = defaults.failed,
+        duped = defaults.duped
+    } = opts;
+
+    return {
+        completed: dayjs.isDuration(completed) ? completed : parseDurationFromDurationValue(completed),
+        failed: dayjs.isDuration(failed) ? failed : parseDurationFromDurationValue(failed),
+        duped: dayjs.isDuration(duped) ? duped : parseDurationFromDurationValue(duped),
+    }
+}
+
+export const isCompactableProperty = (val: string): val is CompactableProperty => val === COMPACTABLE.input || val === COMPACTABLE.transform;

src/backend/common/database/drizzle/drizzleTypes.ts

@@ -0,0 +1,100 @@
+import { DBQueryConfig, DBQueryConfigWith, ExtractTablesFromSchema, KnownKeysOnly, RelationFieldsFilterInternals, Many, InferSelectModel, ExtractTablesWithRelations, type BuildQueryResult, RelationsFilter } from "drizzle-orm";
+import { components, playInputs, plays, queueStates, relations } from "./schema/schema.js";
+import {TSchema, TableName, Schema } from "./schema/schema.js";
+import { MarkOptional, MarkRequired } from "ts-essentials";
+
+
+export type ComponentNew = typeof components.$inferInsert;
+export type ComponentSelect = typeof components.$inferSelect;
+
+export type QueueStateNew = typeof queueStates.$inferInsert;
+export type QueueStateSelect = typeof queueStates.$inferSelect;
+
+export type PlayInputNew = typeof playInputs.$inferInsert;
+export type PlayInputSelect = typeof playInputs.$inferSelect;
+
+export type PlaySelect = typeof plays.$inferSelect;
+export type PlaySelectRel = ModelWithRelations<typeof plays>;
+export type PlaySelectWithQueueStates = GenericRelationResult<'plays', 'queueStates'>;
+export type PlayNew = typeof plays.$inferInsert;
+
+
+// useful references for building types
+// https://github.com/drizzle-team/drizzle-orm/discussions/2596
+// https://github.com/drizzle-team/drizzle-orm/discussions/1539
+// https://gist.github.com/ikupenov/10bc89d92d92eaba8cc5569013e04069
+// https://github.com/drizzle-team/drizzle-orm/issues/695 most examples
+// https://github.com/drizzle-team/drizzle-orm/discussions/2316 relation focused
+// https://github.com/drizzle-team/drizzle-orm/issues/1319
+
+//type p = TSchema['plays']['relations'];
+export type FindWith<T extends TableName> = DBQueryConfigWith<TSchema, TSchema[T]['relations']>;
+export type QueryConfig<T extends TableName> = DBQueryConfig<"many", TSchema, TSchema[T]>;
+export type FindMany<T extends TableName> = Pick<KnownKeysOnly<QueryConfig<T>, DBQueryConfig<"many", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> & {with?: FindWith<T>}
+export type FindOne<T extends TableName> = Pick<KnownKeysOnly<QueryConfig<T>, DBQueryConfig<"one", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> & {with?: FindWith<T>}
+export type FindWhere<T extends TableName> = QueryConfig<T>['where'];
+// https://github.com/drizzle-team/drizzle-orm/issues/5218#issuecomment-4154686086
+export type WhereClause<T extends keyof typeof relations> = RelationsFilter<typeof relations[T], typeof relations>
+
+
+export type CompareOp<T> = Pick<RelationFieldsFilterInternals<T>, 'gt' | 'gte' | 'eq' | 'lt' | 'lte' | 'ne'>
+export type CompareOpKey<T> = keyof CompareOp<T>;
+
+
+
+/**
+ * Based on https://github.com/drizzle-team/drizzle-orm/issues/695#issuecomment-3133969178
+ */
+
+// Helper type to find the tsName corresponding to a given dbName in TSchema
+type FindTsNameByDbName<TDbNameToFind extends string> = {
+    [K in keyof TSchema]: TSchema[K] extends {
+        // updated dbName -> name
+        name: TDbNameToFind;
+    }
+        ? K
+        : TDbNameToFind;
+}[keyof TSchema];
+
+// Helper type to find the dbName corresponding to a given tsName in TSchema
+type FindDbNameByTsName<TTable extends Schema[keyof Schema]> = {
+    [K in keyof Schema]: Schema[K] extends TTable ? K : never;
+}[keyof Schema];
+
+/**
+ * Utility type to infer the model type for a given table name from the schema.
+ * Handles nested relations recursively.
+ * Uses referencedTableName (dbName) and FindTsNameByDbName helper.
+ */
+export type ModelWithRelationsFromName<
+    TTableName extends keyof TSchema,
+> = InferSelectModel<Schema[TTableName]> & {
+    [K in keyof TSchema[TTableName]['relations']]?: TSchema[TTableName]['relations'][K] extends infer TRelation
+        // updated referencedTableName -> targetTableName
+        ? TRelation extends { targetTableName: infer TRefDbName extends string }
+            ? FindTsNameByDbName<TRefDbName> extends infer TRefTsName extends
+                  keyof TSchema
+                ? TRelation extends Many<any>
+                    ? ModelWithRelationsFromName<TRefTsName>[]
+                    : ModelWithRelationsFromName<TRefTsName> | null
+                : never
+            : never
+        : never;
+};
+
+/**
+ * Utility type to infer the model type for a given table from the schema.
+ * Handles nested relations recursively.
+ * Uses referencedTableName (dbName) and FindDbNameByTsName helper.
+ */
+export type ModelWithRelations<TTable extends Schema[keyof Schema]> =
+    FindDbNameByTsName<TTable> extends infer TTableName extends keyof TSchema
+        ? ModelWithRelationsFromName<TTableName>
+        : never;
+
+
+// all relations are are now fully typed and optional
+//type FullPlay = ModelWithRelations<typeof plays>;
+
+// https://github.com/drizzle-team/drizzle-orm/issues/695#issuecomment-4389296482
+type GenericRelationResult<T extends keyof TSchema, K extends keyof TSchema[T]['relations']> = BuildQueryResult<TSchema, TSchema[T], { with: Record<K, true> }>;

src/backend/common/database/drizzle/drizzleUtils.ts

@@ -0,0 +1,263 @@
+import { drizzle } from 'drizzle-orm/node-sqlite';
+import { drizzle as drizzlePglite } from 'drizzle-orm/pglite';
+import { migrate } from 'drizzle-orm/node-sqlite/migrator';
+import { migrate as migratePglite } from 'drizzle-orm/pglite/migrator';
+import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
+import { PGlite, PGliteOptions } from '@electric-sql/pglite';
+import { sql as dsl, LogWriter, Logger as DrizzleLogger } from 'drizzle-orm';
+import * as fs from 'fs/promises';
+import * as fsSync from 'fs';
+import * as path from 'path';
+import { backupDb, getDbBackupPath, getDbPath, MEMORY_DB_NAME } from '../Database.js';
+import { fileExists, fileOrDirectoryIsWriteable } from '../../../utils/FSUtils.js';
+import { childLogger, Logger, LogLevel } from '@foxxmd/logging';
+import { loggerNoop } from '../../MaybeLogger.js';
+import { projectDir } from '../../index.js';
+import { relations } from './schema/schema.js';
+import { addToContext, executeQuery } from './logContext.js';
+
+export async function shouldBackupDb(dbVal: string | DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> {
+  const {
+    logger: parentLogger = loggerNoop,
+    migrationsFolder = path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')
+  } = opts;
+  const logger = childLogger(parentLogger, 'Migrations');
+
+  let db: DbConcrete;
+  
+  if(typeof dbVal === 'string') {
+    logger.info(`Checking for database at ${dbVal}`);
+    if (dbVal !== MEMORY_DB_NAME && !fileExists(dbVal)) {
+      logger.info(`No database exists, no backup needed.`);
+      return [false, []];
+    }
+    db = await getDb(dbVal);
+  } else {
+    db = dbVal;
+  }
+
+
+  // const db = drizzlePglite(dbPath);
+
+  try {
+    // Ensure the migrations table exists
+    // https://github.com/drizzle-team/drizzle-orm/issues/1953
+    const res = await db.execute(dsl`
+      SELECT EXISTS (
+    SELECT FROM 
+        pg_tables
+    WHERE 
+        schemaname = 'drizzle' AND 
+        tablename  = '__drizzle_migrations'
+    );
+      `);
+
+    // const res3 = await db.execute(dsl`
+    // SELECT * FROM 
+    //     pg_tables;
+    //   `);
+
+    if (res.rows[0].exists === false) {
+      logger.info(`Database exists but there is no __drizzle_migrations table??`);
+      return [true, []];
+    }
+
+    const dbMigrations = await db.execute(dsl`SELECT id, hash, created_at, name, applied_at FROM drizzle.__drizzle_migrations ORDER BY created_at DESC`);
+    // @ts-ignore
+    const appliedMigrations = new Set(dbMigrations.rows.map((m: any) => m.name));
+
+    const allFiles = await fs.readdir(migrationsFolder);
+    const migrationFiles = allFiles
+      .sort();
+
+    const pendingMigrations = migrationFiles.filter(file => {
+      return !appliedMigrations.has(file);
+    });
+
+    //console.log('Applied migrations:', Array.from(appliedMigrations));
+    if (pendingMigrations.length > 0) {
+      logger.info(`${pendingMigrations.length} pending migrations:\n${pendingMigrations.join('\n')}`);
+      return [true, pendingMigrations];
+    } else {
+      logger.info('No pending migrations.');
+      return [false, []];
+    }
+  } catch (error) {
+    logger.error(new Error('Failed to get pending migrations', { cause: error }));
+    return [true, []];
+  }
+}
+
+export const getDb = async (dbVal: string | PGlite, opts: { logger?: Logger, backupPath?: string, loadDataDir?: Promise<Blob> } = {}) => {
+  const {
+    logger = loggerNoop,
+    backupPath,
+    loadDataDir
+  } = opts;
+  let client: PGlite;
+
+  if(typeof dbVal === 'string') {
+    const opts: PGliteOptions = {};
+    if(dbVal !== MEMORY_DB_NAME) {
+      opts.dataDir = dbVal;
+      if(backupPath !== undefined) {
+        opts.loadDataDir = new Blob([fsSync.readFileSync(backupPath)]);
+      }
+    }
+    // only load one
+    // but this could be for a memory db so don't put it in above if
+    if(loadDataDir !== undefined && backupDb === undefined) {
+      opts.loadDataDir = await loadDataDir
+    }
+    client = await PGlite.create(opts);
+  } else {
+    client = dbVal;
+  }
+  return drizzlePglite({relations: relations, logger: createDrizzleLogger(logger), client});
+}
+
+export type DbConcrete = Awaited<ReturnType<typeof getDb>>;
+
+export const migrateDb = async (db: DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}) => {
+  const {
+    migrationsFolder,
+    logger: parentLogger = loggerNoop
+  } = opts;
+  const logger = childLogger(parentLogger, 'Migrations');
+
+  try {
+    logger.info('Starting migrations...');
+    await executeQuery('migrations', async () => migratePglite(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') }), logger, process.env.LOG_MIGRATION === 'true' ? true : 'error');
+    logger.info('Migrations complete');
+  } catch (e) {
+    throw new Error('Failed to migrate database', { cause: e });
+  }
+}
+
+export const migrateDbSync = (db: ReturnType<typeof drizzle>, opts: {logger?: Logger, migrationsFolder?: string} = {}) => {
+  const {
+    migrationsFolder,
+    logger: parentLogger = loggerNoop
+  } = opts;
+  const logger = childLogger(parentLogger, 'Migrations');
+
+  try {
+    logger.info('Starting migrations...');
+    migrate(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') });
+    logger.info('Migrations complete');
+  } catch (e) {
+    throw new Error('Failed to migrate database', { cause: e });
+  }
+}
+
+export const getMigratedDb = async (dbPath: string, opts: { logger?: Logger, workingDirectory?: string, migrationsFolder?: string, backupPath?: string, loadDataDir?: Promise<Blob> } = {}): Promise<[DbConcrete, boolean]> => {
+  const {
+    logger = loggerNoop
+  } = opts;
+  let db: DbConcrete,
+  isNew = false,
+  hasPendingMigrations: boolean = true;
+  if (dbPath !== MEMORY_DB_NAME) {
+    try {
+      fileOrDirectoryIsWriteable(dbPath);
+    } catch (e) {
+      throw new Error('Database directory is not accessible', { cause: e });
+    }
+
+    const backupPath = getDbBackupPath(dbPath);
+
+    if (fileExists(dbPath)) {
+      db = await getDb(dbPath, opts);
+      const [shouldBackup, pendingMigrations] = await shouldBackupDb(db, opts);
+      if (shouldBackup) {
+        hasPendingMigrations = true;
+        await backupPgDb(db, dbPath, { logger: opts.logger });
+      }
+    } else if(fileExists(backupPath)) {
+      logger.info(`Detected no database, using backup to recreate db. Backup file: ${backupPath}`);
+      db = await getDb(dbPath, {...opts, backupPath});
+      const usedBackedPath = getDbBackupPath(dbPath, 'used');
+      logger.info(`Backup loaded! Renaming backup to indicate it has already been used, new path: ${usedBackedPath}`);
+      await fs.rename(backupPath, usedBackedPath);
+    } else {
+      logger.info('Detected no database, creating a new one...');
+      db = await getDb(dbPath, opts);
+      isNew = true;
+    }
+  } else {
+    logger.info('Detected in-memory database');
+    db = await getDb(dbPath, opts);
+    isNew = true;
+  }
+
+  if(hasPendingMigrations && dbPath !== MEMORY_DB_NAME) {
+    logger.info('TIP: Migrations may take some time, depending on the size of your database');
+  }
+  await migrateDb(db, opts);
+
+  return [db, isNew];
+}
+
+export const backupPgDb = async (db: DbConcrete, dbPath: string, opts: { logger?: Logger } = {}): Promise<void> => {
+
+    const {
+        logger: parentLogger = loggerNoop,
+    } = opts;
+
+    const logger = childLogger(parentLogger, 'Backup');
+
+    const pathInfo = path.parse(dbPath);
+    // being extra sure there isn't a trailing slash
+    const backupPath = `${path.join(pathInfo.dir, pathInfo.name)}-${Date.now()}.bak`;
+    logger.info(`Backing up database before migrating => ${backupPath}`);
+    fs.writeFile(backupPath, Buffer.from(await (await db.$client.dumpDataDir()).arrayBuffer()));
+    //await fs.copyFile(dbPath, backupPath)
+    logger.info('Backed up!');
+}
+
+export const createDrizzleLogger = (parentLogger: Logger, opts: {level?: LogLevel} = {}): DrizzleLogger => {
+  return {
+    logQuery: (query: string, params: unknown[]) => {
+      addToContext({sql: query, params})
+    }
+  }
+}
+
+
+// cannot really use transactions right now because async isn't supporting for sqlite
+// https://github.com/drizzle-team/drizzle-orm/issues/1472
+// https://github.com/drizzle-team/drizzle-orm/issues/2275
+// so use this workaround for now
+// https://github.com/drizzle-team/drizzle-orm/issues/2275#issuecomment-2496503801
+let currentTransaction: null | Promise<void> = null;
+export const runTransaction = async <
+    T,
+    TQueryResult,
+    TSchema extends Record<string, unknown> = Record<string, never>
+>(
+    db: BaseSQLiteDatabase<"sync", TQueryResult, TSchema>,
+    executor: () => Promise<T>
+) => {
+    while (currentTransaction !== null) {
+        await currentTransaction;
+    }
+    let resolve!: () => void;
+    currentTransaction = new Promise<void>(_resolve => {
+        resolve = _resolve;
+    });
+    try {
+        db.run(dsl.raw(`BEGIN`))
+
+        try {
+            const result = await executor();
+            await db.run(dsl.raw(`COMMIT`));
+            return result;
+        } catch (error) {
+            await db.run(dsl.raw(`ROLLBACK`));
+            throw error;
+        }
+    } finally {
+        resolve();
+        currentTransaction = null;
+    }
+};

src/backend/common/database/drizzle/entityUtils.ts

@@ -0,0 +1,93 @@
+import assert from "node:assert";
+import { PlayNew, PlaySelect, PlaySelectWithQueueStates } from "./drizzleTypes.js";
+import { PlayInputNew } from "./drizzleTypes.js";
+import { QueueStateNew } from "./drizzleTypes.js";
+import { ComponentNew } from "./drizzleTypes.js";
+import { MarkOptional, MarkRequired } from "ts-essentials";
+import { CLIENT_DEAD_QUEUE, DeadLetterScrobble, ErrorLike, PlayObject } from "../../../../core/Atomic.js";
+import dayjs, { Dayjs } from "dayjs";
+import { asPlay } from "../../../../core/PlayMarshalUtils.js";
+import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../utils/PlayComparisonUtils.js";
+import { hashObject } from "../../../utils/StringUtils.js";
+import { messageWithCauses } from "../../../utils/ErrorUtils.js";
+
+export const generateComponentEntity = (data: MarkOptional<ComponentNew, 'uid'>): ComponentNew => {
+    assert(data.name !== undefined, 'Must provide name');
+    return {
+        ...data,
+        uid: data.uid ?? data.name
+    };
+}
+
+export type PlayEntityOpts = Partial<Pick<PlayNew, 'seenAt' | 'playedAt' | 'uid' | 'state' | 'parentId' | 'componentId'>> & { error?: ErrorLike };
+
+export const generatePlayEntity = (play: PlayObject, opts: PlayEntityOpts = {}): PlayNew => {
+    const {
+        seenAt = dayjs(),
+        state = 'queued',
+        playedAt = play.data.playDate,
+        ...restOpts
+    } = opts;
+    let playHash: string = undefined;
+    try {
+        playHash = hashObject(playContentBasicInvariantTransform(play).data);
+    } catch (e) {
+        // swallow
+    }
+    const data: PlayNew = {
+        play,
+        playHash,
+        state,
+        playedAt,
+        seenAt: play.meta.seenAt ?? seenAt,
+        ...restOpts
+    };
+    const mbidId = playMbidIdentifier(play);
+    if(mbidId !== undefined) {
+        data.mbidIdentifier = mbidId;
+    }
+    return data;
+}
+
+export type PlayHydateOptions = 'asPlay' | 'id' | 'uid';
+
+export const hydratePlaySelect = (select: PlaySelect, opts: PlayHydateOptions[] = ['id','uid']): PlayObject => {
+    if(opts.length === 0) {
+        return select.play;
+    }
+
+    let res = select.play;
+    // if(opts.includes('asPlay')) {
+    //     res = asPlay(res);
+    // }
+    if(opts.includes('uid')) {
+        res.uid = select.uid;
+        //res.meta.dbUid = select.uid;
+    }
+    if(opts.includes('id')) {
+        res.id = select.id;
+        //res.meta.dbId = select.id;
+    }
+    return res;
+}
+
+export const playSelectToDeadScrobble = (select: PlaySelectWithQueueStates): DeadLetterScrobble<PlayObject> => {
+    const deadQueue = select.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE);
+    return {
+        play: select.play,
+        id: select.uid,
+        source: select.play.meta.source,
+        retries: deadQueue.retries,
+        lastRetry: deadQueue.updatedAt,
+        error: deadQueue.error as unknown as string,
+        status: deadQueue.queueStatus as 'queued' | 'failed'
+    }
+}
+
+export const generateInputEntity = (data: PlayInputNew): PlayInputNew => {
+    return data;
+}
+
+export const generateQueueStateEntity = (data: QueueStateNew): QueueStateNew => {
+    return data;
+}

src/backend/common/database/drizzle/logContext.ts

@@ -0,0 +1,71 @@
+import { Logger } from '@foxxmd/logging'
+import { AsyncLocalStorage } from 'async_hooks'
+
+// based on https://numeric.substack.com/p/upgrading-drizzleorm-logging-with
+interface QueryContext {
+    queryKey: string
+    startTime: number
+    queries: { sql?: string, params?: unknown[] }[]
+}
+
+const queryStorage = new AsyncLocalStorage<QueryContext>()
+
+function wrapQuery<T>(queryKey: string, fn: () => Promise<T>): Promise<T> {
+    return queryStorage.run(
+        {
+            queryKey,
+            startTime: Date.now(),
+            queries: []
+        },
+        fn
+    )
+}
+
+function getContext(): QueryContext | undefined {
+    return queryStorage.getStore()
+}
+
+export function addToContext(data: { sql?: string, params?: unknown[] }): void {
+    const context = getContext()
+    if (context) {
+        context.queries.push(data);
+    }
+}
+
+/**
+ * Log all queries made by drizzle during the execution of a promise
+ * 
+ * use second parameter to configure when logging occurs
+ * * true    => log everything (default)
+ * * false   => log nothing, skips asyncstorage entirely
+ * * 'error' => only log if promise throws
+ * 
+ */
+export async function executeQuery<T = any>(queryKey: string, queryPromise: () => Promise<T>, logger: Logger, when: boolean | 'error' = true) {
+    if(when === false) {
+        try {
+            return await queryPromise();
+        } catch (e) {
+            throw e;
+        }
+    }
+    return wrapQuery(queryKey, async () => {
+        try {
+            const results = await queryPromise()
+
+            if (when !== 'error') {
+                // Query is done - grab everything from context
+                const context = getContext()
+                const executionTime = context ? Date.now() - context.startTime : 0
+                logger.info({ labels: ['DB Query', queryKey], queries: context?.queries }, `Execution Complete in ${executionTime}ms`);
+            }
+
+            return results
+        } catch (error) {
+            const context = getContext()
+            const executionTime = context ? Date.now() - context.startTime : 0;
+            logger.warn({ labels: ['DB Query', queryKey], queries: context?.queries }, `Execution failed in ${executionTime}ms`);
+            throw error
+        }
+    })
+}

src/backend/common/database/drizzle/migrations/20260509011012_dizzy_naoko/migration.sql

@@ -0,0 +1,83 @@
+CREATE TABLE "components" (
+	"id" serial PRIMARY KEY,
+	"uid" varchar(200) NOT NULL,
+	"mode" varchar(15) NOT NULL,
+	"type" varchar(50) NOT NULL,
+	"name" varchar NOT NULL,
+	"countLive" integer DEFAULT 0 NOT NULL,
+	"countNonLive" integer DEFAULT 0 NOT NULL,
+	"createdAt" timestamp
+);
+--> statement-breakpoint
+CREATE TABLE "jobs" (
+	"id" serial PRIMARY KEY,
+	"componentFromId" integer NOT NULL,
+	"componentToId" integer NOT NULL,
+	"name" varchar(200) NOT NULL,
+	"status" varchar(20) DEFAULT 'idle' NOT NULL,
+	"retries" integer DEFAULT 0 NOT NULL,
+	"error" json,
+	"transformOptions" json,
+	"initialParameters" json,
+	"cursor" json,
+	"total" integer,
+	"imported" integer DEFAULT 0 NOT NULL,
+	"scrobbled" integer DEFAULT 0 NOT NULL,
+	"createdAt" timestamp NOT NULL,
+	"updatedAt" timestamp NOT NULL,
+	"completedAt" timestamp
+);
+--> statement-breakpoint
+CREATE TABLE "play_inputs" (
+	"id" serial PRIMARY KEY,
+	"playId" integer NOT NULL,
+	"data" json,
+	"play" jsonb NOT NULL,
+	"createdAt" timestamp
+);
+--> statement-breakpoint
+CREATE TABLE "plays" (
+	"id" serial PRIMARY KEY,
+	"uid" varchar(30) NOT NULL UNIQUE,
+	"componentId" integer,
+	"error" json,
+	"playedAt" timestamp,
+	"seenAt" timestamp,
+	"updatedAt" timestamp NOT NULL,
+	"play" jsonb NOT NULL,
+	"state" varchar(20) NOT NULL,
+	"parentId" integer,
+	"jobId" integer,
+	"playHash" varchar(100),
+	"mbidIdentifier" varchar(100),
+	"compacted" varchar(30)
+);
+--> statement-breakpoint
+CREATE TABLE "play_queue_states" (
+	"id" serial PRIMARY KEY,
+	"playId" integer NOT NULL,
+	"componentId" integer NOT NULL,
+	"queueName" varchar(50) NOT NULL,
+	"queueStatus" varchar(20) DEFAULT 'queued' NOT NULL,
+	"retries" integer DEFAULT 0 NOT NULL,
+	"error" json,
+	"createdAt" timestamp NOT NULL,
+	"updatedAt" timestamp NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX "uid_mode_type_idx" ON "components" ("uid","mode","type");--> statement-breakpoint
+CREATE UNIQUE INDEX "play_input_id_idx" ON "play_inputs" ("playId");--> statement-breakpoint
+CREATE INDEX "play_parent_id_idx" ON "plays" ("parentId");--> statement-breakpoint
+CREATE INDEX "play_component_id_idx" ON "plays" ("componentId");--> statement-breakpoint
+CREATE UNIQUE INDEX "play_uid_idx" ON "plays" ("uid");--> statement-breakpoint
+CREATE INDEX "play_playedAt_idx" ON "plays" ("playedAt");--> statement-breakpoint
+CREATE INDEX "play_seenAt_idx" ON "plays" ("seenAt");--> statement-breakpoint
+CREATE INDEX "play_queue_state_id_idx" ON "play_queue_states" ("playId");--> statement-breakpoint
+ALTER TABLE "jobs" ADD CONSTRAINT "jobs_componentFromId_components_id_fkey" FOREIGN KEY ("componentFromId") REFERENCES "components"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "jobs" ADD CONSTRAINT "jobs_componentToId_components_id_fkey" FOREIGN KEY ("componentToId") REFERENCES "components"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "play_inputs" ADD CONSTRAINT "play_inputs_playId_plays_id_fkey" FOREIGN KEY ("playId") REFERENCES "plays"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "plays" ADD CONSTRAINT "plays_componentId_components_id_fkey" FOREIGN KEY ("componentId") REFERENCES "components"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "plays" ADD CONSTRAINT "plays_parentId_plays_id_fkey" FOREIGN KEY ("parentId") REFERENCES "plays"("id") ON DELETE SET NULL ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "plays" ADD CONSTRAINT "plays_jobId_jobs_id_fkey" FOREIGN KEY ("jobId") REFERENCES "jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "play_queue_states" ADD CONSTRAINT "play_queue_states_playId_plays_id_fkey" FOREIGN KEY ("playId") REFERENCES "plays"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint
+ALTER TABLE "play_queue_states" ADD CONSTRAINT "play_queue_states_componentId_components_id_fkey" FOREIGN KEY ("componentId") REFERENCES "components"("id") ON DELETE CASCADE ON UPDATE CASCADE;