POC for sqlite-based queue

.dockerignore

@@ -30,4 +30,6 @@ tmp-*
 .env
 *.env
 .storybook
-.stories
+.stories
+*.db
+*.db-*

.gitignore

@@ -151,3 +151,6 @@ tmp-*
 
 *storybook.log
 storybook-static
+
+*.db
+*.db-*

package-lock.json

@@ -78,6 +78,7 @@
         "jsondiffpatch-react": "^1.0.8",
         "keyv": "^5.5.0",
         "kodi-api": "^0.2.1",
+        "kysely-node-native-sqlite": "^1.1.0",
         "lastfm-ts-api": "^2.6.0",
         "merge-error-cause": "^5.0.2",
         "mopidy": "^1.3.0",
@@ -118,6 +119,7 @@
         "undici": "7.24.6",
         "vite-express": "^0.16.0",
         "vlc-client": "^1.1.1",
+        "workmatic": "github:FoxxMD/workmatic#testDist",
         "xml2js": "0.6.2",
         "youtubei.js": "^15.0.1"
       },
@@ -143,7 +145,7 @@
         "@types/jest": "^27.5.2",
         "@types/jscodeshift": "^0.11.6",
         "@types/mocha": "^9.1.0",
-        "@types/node": "^20.19.2",
+        "@types/node": "^24.1.0",
         "@types/object-hash": "^3.0.0",
         "@types/passport": "^1.0.12",
         "@types/react": "^18.2.18",
@@ -175,7 +177,7 @@
         "with-local-tmp-dir": "^6.0.0"
       },
       "engines": {
-        "node": ">=20.19.2",
+        "node": ">=24.14.0",
         "npm": ">=11.12.1"
       }
     },
@@ -1016,6 +1018,16 @@
         "url": "https://github.com/chalk/strip-ansi?sponsor=1"
       }
     },
+    "node_modules/@colors/colors": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz",
+      "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==",
+      "license": "MIT",
+      "optional": true,
+      "engines": {
+        "node": ">=0.1.90"
+      }
+    },
     "node_modules/@curvenote/ansi-to-react": {
       "version": "7.0.0",
       "license": "BSD-3-Clause",
@@ -4274,10 +4286,12 @@
       "license": "MIT"
     },
     "node_modules/@types/node": {
-      "version": "20.19.37",
+      "version": "24.12.0",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz",
+      "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==",
       "license": "MIT",
       "dependencies": {
-        "undici-types": "~6.21.0"
+        "undici-types": "~7.16.0"
       }
     },
     "node_modules/@types/object-hash": {
@@ -6495,6 +6509,41 @@
         "node": ">=8"
       }
     },
+    "node_modules/cli-table3": {
+      "version": "0.6.5",
+      "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.5.tgz",
+      "integrity": "sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==",
+      "license": "MIT",
+      "dependencies": {
+        "string-width": "^4.2.0"
+      },
+      "engines": {
+        "node": "10.* || >= 12.*"
+      },
+      "optionalDependencies": {
+        "@colors/colors": "1.5.0"
+      }
+    },
+    "node_modules/cli-table3/node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+      "license": "MIT"
+    },
+    "node_modules/cli-table3/node_modules/string-width": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "license": "MIT",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/cli-width": {
       "version": "4.1.0",
       "dev": true,
@@ -7978,8 +8027,9 @@
       }
     },
     "node_modules/fastq": {
-      "version": "1.17.1",
-      "dev": true,
+      "version": "1.20.1",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz",
+      "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==",
       "license": "ISC",
       "dependencies": {
         "reusify": "^1.0.4"
@@ -9657,6 +9707,27 @@
         "uuid": "^8.3.2"
       }
     },
+    "node_modules/kysely": {
+      "version": "0.27.6",
+      "resolved": "https://registry.npmjs.org/kysely/-/kysely-0.27.6.tgz",
+      "integrity": "sha512-FIyV/64EkKhJmjgC0g2hygpBv5RNWVPyNCqSAD7eTCv6eFWNIi4PN1UvdSJGicN/o35bnevgis4Y0UDC0qi8jQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
+    "node_modules/kysely-node-native-sqlite": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/kysely-node-native-sqlite/-/kysely-node-native-sqlite-1.1.0.tgz",
+      "integrity": "sha512-RUEslfInufxA0XN3bLBWwgBNUYC20Ac8RC4aijnYTqQq22ABAH5bVoDtLMr5Re/m94PA+d+XRcJBYB+sM+scxQ==",
+      "license": "ISC",
+      "engines": {
+        "node": "^22.5.0"
+      },
+      "peerDependencies": {
+        "kysely": ">=0.27.0"
+      }
+    },
     "node_modules/lastfm-ts-api": {
       "version": "2.6.0",
       "license": "MIT",
@@ -12118,7 +12189,6 @@
     },
     "node_modules/reusify": {
       "version": "1.0.4",
-      "dev": true,
       "license": "MIT",
       "engines": {
         "iojs": ">=1.0.0",
@@ -13648,7 +13718,9 @@
       }
     },
     "node_modules/undici-types": {
-      "version": "6.21.0",
+      "version": "7.16.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
+      "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
       "license": "MIT"
     },
     "node_modules/unicorn-magic": {
@@ -14428,6 +14500,42 @@
       "dev": true,
       "license": "Apache-2.0"
     },
+    "node_modules/workmatic": {
+      "version": "1.0.5",
+      "resolved": "git+ssh://git@github.com/FoxxMD/workmatic.git#e8b5cecd3b925bf8b4434c7ca554773dfeeeca06",
+      "license": "MIT",
+      "dependencies": {
+        "cli-table3": "^0.6.5",
+        "fastq": "^1.18.0",
+        "kysely": "^0.27.5",
+        "kysely-node-native-sqlite": "^1.1.0",
+        "nanoid": "^5.0.9"
+      },
+      "bin": {
+        "workmatic": "dist/cli.js"
+      },
+      "engines": {
+        "node": ">=22.10.7"
+      }
+    },
+    "node_modules/workmatic/node_modules/nanoid": {
+      "version": "5.1.7",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.7.tgz",
+      "integrity": "sha512-ua3NDgISf6jdwezAheMOk4mbE1LXjm1DfMUDMuJf4AqxLFK3ccGpgWizwa5YV7Yz9EpXwEaWoRXSb/BnV0t5dQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/ai"
+        }
+      ],
+      "license": "MIT",
+      "bin": {
+        "nanoid": "bin/nanoid.js"
+      },
+      "engines": {
+        "node": "^18 || >=20"
+      }
+    },
     "node_modules/wrap-ansi": {
       "version": "8.1.0",
       "license": "MIT",

package.json

@@ -116,6 +116,7 @@
     "jsondiffpatch-react": "^1.0.8",
     "keyv": "^5.5.0",
     "kodi-api": "^0.2.1",
+    "kysely-node-native-sqlite": "^1.1.0",
     "lastfm-ts-api": "^2.6.0",
     "merge-error-cause": "^5.0.2",
     "mopidy": "^1.3.0",
@@ -156,6 +157,7 @@
     "undici": "7.24.6",
     "vite-express": "^0.16.0",
     "vlc-client": "^1.1.1",
+    "workmatic": "github:FoxxMD/workmatic#testDist",
     "xml2js": "0.6.2",
     "youtubei.js": "^15.0.1"
   },
@@ -181,7 +183,7 @@
     "@types/jest": "^27.5.2",
     "@types/jscodeshift": "^0.11.6",
     "@types/mocha": "^9.1.0",
-    "@types/node": "^20.19.2",
+    "@types/node": "^24.1.0",
     "@types/object-hash": "^3.0.0",
     "@types/passport": "^1.0.12",
     "@types/react": "^18.2.18",

patches/kysely-node-native-sqlite+1.1.0.patch

@@ -0,0 +1,60 @@
+diff --git a/node_modules/kysely-node-native-sqlite/dist/index.cjs b/node_modules/kysely-node-native-sqlite/dist/index.cjs
+index 1015da7..0679a95 100644
+--- a/node_modules/kysely-node-native-sqlite/dist/index.cjs
++++ b/node_modules/kysely-node-native-sqlite/dist/index.cjs
+@@ -55,7 +55,11 @@ var import_node_sqlite = require("node:sqlite");
+ var NodeNativeSqliteConnection = class {
+   #db;
+   constructor(...args) {
+-    this.#db = new import_node_sqlite.DatabaseSync(...args);
++    if (args[0] instanceof import_node_sqlite.DatabaseSync) {
++      this.#db = args[0];
++    } else {
++      this.#db = new import_node_sqlite.DatabaseSync(...args);
++    }
+   }
+   [Symbol.dispose]() {
+     this.#db.close();
+diff --git a/node_modules/kysely-node-native-sqlite/dist/index.d.cts b/node_modules/kysely-node-native-sqlite/dist/index.d.cts
+index 72366bb..32e8464 100644
+--- a/node_modules/kysely-node-native-sqlite/dist/index.d.cts
++++ b/node_modules/kysely-node-native-sqlite/dist/index.d.cts
+@@ -3,7 +3,7 @@ import { Dialect, SqliteAdapter, Driver, Kysely, DatabaseIntrospector, SqliteQue
+ 
+ declare class NodeNativeSqliteDialect implements Dialect {
+     #private;
+-    constructor(...args: ConstructorParameters<typeof DatabaseSync>);
++    constructor(...args: ConstructorParameters<typeof DatabaseSync> | [DatabaseSync]);
+     createAdapter(): SqliteAdapter;
+     createDriver(): Driver;
+     createIntrospector(db: Kysely<unknown>): DatabaseIntrospector;
+diff --git a/node_modules/kysely-node-native-sqlite/dist/index.d.ts b/node_modules/kysely-node-native-sqlite/dist/index.d.ts
+index 72366bb..32e8464 100644
+--- a/node_modules/kysely-node-native-sqlite/dist/index.d.ts
++++ b/node_modules/kysely-node-native-sqlite/dist/index.d.ts
+@@ -3,7 +3,7 @@ import { Dialect, SqliteAdapter, Driver, Kysely, DatabaseIntrospector, SqliteQue
+ 
+ declare class NodeNativeSqliteDialect implements Dialect {
+     #private;
+-    constructor(...args: ConstructorParameters<typeof DatabaseSync>);
++    constructor(...args: ConstructorParameters<typeof DatabaseSync> | [DatabaseSync]);
+     createAdapter(): SqliteAdapter;
+     createDriver(): Driver;
+     createIntrospector(db: Kysely<unknown>): DatabaseIntrospector;
+diff --git a/node_modules/kysely-node-native-sqlite/dist/index.js b/node_modules/kysely-node-native-sqlite/dist/index.js
+index 8c2b8cf..17b7eb1 100644
+--- a/node_modules/kysely-node-native-sqlite/dist/index.js
++++ b/node_modules/kysely-node-native-sqlite/dist/index.js
+@@ -32,7 +32,11 @@ import { DatabaseSync } from "node:sqlite";
+ var NodeNativeSqliteConnection = class {
+   #db;
+   constructor(...args) {
+-    this.#db = new DatabaseSync(...args);
++    if (args[0] instanceof DatabaseSync) {
++      this.#db = args[0];
++    } else {
++      this.#db = new DatabaseSync(...args);
++    }
+   }
+   [Symbol.dispose]() {
+     this.#db.close();

src/backend/common/database/JobDb.ts

@@ -0,0 +1,20 @@
+import { createDatabase, WorkmaticDb } from 'workmatic';
+import { DatabaseSync } from 'node:sqlite';
+import { configDir } from '../index.js';
+import * as path from 'path';
+
+export const getJobsDb = (dbName: string = ':memory:'): [WorkmaticDb, DatabaseSync] => {
+    let dbPath: string;
+    if(dbName === ':memory') {
+        dbPath = dbName;
+    } else {
+        dbPath = path.resolve(configDir, `${dbName}.db`);
+    }
+
+    const database = new DatabaseSync(dbPath);
+    const db = createDatabase({ db: database });
+    database.exec(`CREATE INDEX IF NOT EXISTS playdate ON workmatic_jobs((payload->>'$.data.playDate'))`);
+    database.exec(`CREATE INDEX IF NOT EXISTS seenAt ON workmatic_jobs((payload->>'$.meta.seenAt'))`);
+
+    return [db, database];
+}

src/backend/tests/queue/queue.test.ts

@@ -0,0 +1,19 @@
+import chai, { assert, expect } from 'chai';
+import asPromised from 'chai-as-promised';
+import { after, before, describe, it } from 'mocha';
+import { createDatabase, createClient, createWorker } from 'workmatic';
+import { DatabaseSync } from 'node:sqlite';
+import { getJobsDb } from '../../common/database/JobDb.js';
+import { generatePlay } from '../../../core/PlayTestUtils.js';
+
+chai.use(asPromised);
+
+it('does stuff', async function () {
+
+    const [db, nodeDb] = getJobsDb(':memory:');
+    const client = createClient({ db, queue: 'scrobble' });
+
+    const { id } = await client.add(generatePlay());
+    console.log(`Job created: ${id}`);
+    expect(true).to.eq(true);
+});