feat(database): Add id awareness to plays and implement basic api GET for sources

src/backend/common/database/drizzle/drizzleTypes.ts

@@ -1,4 +1,4 @@
-import { DBQueryConfig, ExtractTablesFromSchema, KnownKeysOnly, RelationFieldsFilterInternals } from "drizzle-orm";
+import { DBQueryConfig, DBQueryConfigWith, ExtractTablesFromSchema, KnownKeysOnly, RelationFieldsFilterInternals } from "drizzle-orm";
 import { components, playInputs, plays, queueStates, relations } from "./schema/schema.js";
 import {TSchema, TableName, Schema } from "./schema/schema.js";
 
@@ -23,9 +23,12 @@ export type PlayNew = typeof plays.$inferInsert;
 // https://github.com/drizzle-team/drizzle-orm/issues/695 most examples
 // https://github.com/drizzle-team/drizzle-orm/discussions/2316 relation focused
 // https://github.com/drizzle-team/drizzle-orm/issues/1319
+
+//type p = TSchema['plays']['relations'];
+export type FindWith<T extends TableName> = DBQueryConfigWith<TSchema, TSchema[T]['relations']>;
 export type QueryConfig<T extends TableName> = DBQueryConfig<"many", TSchema, TSchema[T]>;
-export type FindMany<T extends TableName> = Pick<KnownKeysOnly<QueryConfig<T>, DBQueryConfig<"many", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'>
-export type FindOne<T extends TableName> = Pick<KnownKeysOnly<QueryConfig<T>, DBQueryConfig<"one", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'>
+export type FindMany<T extends TableName> = Pick<KnownKeysOnly<QueryConfig<T>, DBQueryConfig<"many", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> & {with?: FindWith<T>}
+export type FindOne<T extends TableName> = Pick<KnownKeysOnly<QueryConfig<T>, DBQueryConfig<"one", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> & {with?: FindWith<T>}
 export type FindWhere<T extends TableName> = QueryConfig<T>['where'];
 
 export type CompareOp<T> = Pick<RelationFieldsFilterInternals<T>, 'gt' | 'gte' | 'eq' | 'lt' | 'lte' | 'ne'>

src/backend/common/database/drizzle/entityUtils.ts

@@ -1,11 +1,12 @@
 import assert from "node:assert";
-import { PlayNew } from "./drizzleTypes.js";
+import { PlayNew, PlaySelect } from "./drizzleTypes.js";
 import { PlayInputNew } from "./drizzleTypes.js";
 import { QueueStateNew } from "./drizzleTypes.js";
 import { ComponentNew } from "./drizzleTypes.js";
 import { MarkOptional } from "ts-essentials";
 import { ErrorLike, PlayObject } from "../../../../core/Atomic.js";
 import dayjs, { Dayjs } from "dayjs";
+import { asPlay } from "../../../../core/PlayMarshalUtils.js";
 
 export const generateComponentEntity = (data: MarkOptional<ComponentNew, 'uid'>): ComponentNew => {
     assert(data.name !== undefined, 'Must provide name');
@@ -33,6 +34,26 @@ export const generatePlayEntity = (play: PlayObject, opts: PlayEntityOpts = {}):
     }
 }
 
+export type PlayHydateOptions = 'asPlay' | 'id' | 'uid';
+
+export const hydratePlaySelect = (select: PlaySelect, opts: PlayHydateOptions[]): PlayObject => {
+    if(opts.length === 0) {
+        return select.play;
+    }
+
+    let res = select.play;
+    if(opts.includes('asPlay')) {
+        res = asPlay(res);
+    }
+    if(opts.includes('uid')) {
+        res.meta.dbUid = select.uid;
+    }
+    if(opts.includes('id')) {
+        res.meta.dbId = select.id;
+    }
+    return res;
+}
+
 export const generateInputEntity = (data: PlayInputNew): PlayInputNew => {
     return data;
 }

src/backend/common/database/drizzle/repositories/PlayRepository.ts

@@ -2,16 +2,19 @@ import { childLogger, Logger, LoggerAppExtras } from "@foxxmd/logging";
 import { DbConcrete, getDb, runTransaction } from "../drizzleUtils.js";
 import { loggerNoop } from "../../../MaybeLogger.js";
 import { PlayObject } from "../../../../../core/Atomic.js";
-import { generateInputEntity, generatePlayEntity, PlayEntityOpts } from "../entityUtils.js";
+import { generateInputEntity, generatePlayEntity, PlayEntityOpts, hydratePlaySelect, PlayHydateOptions } from "../entityUtils.js";
 import { playInputs, plays, relations } from "../schema/schema.js";
 import { PlayNew, PlaySelect, PlayInputNew, FindWhere, FindMany, CompareOpKey } from "../drizzleTypes.js";;
 import { MarkOptional, MarkRequired, PathValue } from "ts-essentials";
-import { genGroupIdStrFromPlay, removeUndefinedKeys } from "../../../../utils.js";
+import { genGroupIdStrFromPlay, removeEmptyArrays, removeUndefinedKeys } from "../../../../utils.js";
 import dayjs, { Dayjs } from "dayjs";
 import { RelationsFieldFilter, eq, inArray, ne, notInArray, desc, asc, and } from "drizzle-orm";
 import { CompactableProperty, RetentionOptions, retentionPlayTypes } from "../../../infrastructure/config/database.js";
 import { shortTodayAwareFormat } from "../../../../../core/TimeUtils.js";
 import { buildDateCompare, CompareDateOp, DrizzleBaseRepository } from "./BaseRepository.js";
+import { asPlay } from "../../../../../core/PlayMarshalUtils.js";
+import assert from "node:assert";
+import { parseArrayFromMaybeString } from "../../../../utils/StringUtils.js";
 
 // https://github.com/drizzle-team/drizzle-orm/issues/695 may be useful for typing models with relations?
 
@@ -25,11 +28,14 @@ export interface PlayWhereOpts {
     platformId?: string
     seenAt?: CompareDateOp
     playedAt?: CompareDateOp
+    uid?: string[]
 }
 
+export type WithPlayRelation = 'input' | 'parent' | 'parent-input';
 export interface QueryPlaysOpts extends PlayWhereOpts {
     sort?: 'seenAt' | 'playedAt'
     order?: 'asc' | 'desc'
+    with?: WithPlayRelation[]
     limit?: number
     offset?: number
 }
@@ -45,8 +51,11 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository {
         super(db, 'plays', 'Plays', opts);
     }
 
-    createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[]) => {
+    createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[], opts: {hydrate?: PlayHydateOptions[]} = {}) => {
 
+        const {
+            hydrate = []
+        } = opts;
         let playRows: PlaySelect[];
 
         await runTransaction(this.db, async () => {
@@ -79,11 +88,21 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository {
 
         });
 
-        return playRows;
+        return playRows.map(x => ({...x, play: hydratePlaySelect(x, hydrate)}));
     }
 
-    findPlays = async (args: QueryPlaysOpts): Promise<PlaySelect[]> => {
-        //let oldQuery: Parameters<typeof this.db.query.plays.findMany>[0] = {};
+    findPlays = async (args: QueryPlaysOpts, opts: {hydrate?: PlayHydateOptions[]} = {}): Promise<PlaySelect[]> => {
+        const {
+            hydrate = []
+        } = opts;
+        // this does not work as type for query variable
+        // it erases the result type for some reason
+        //
+        // Parameters<typeof this.db.query.plays.findMany>[0]
+        
+        // this does work but it is also integrated into FindWith
+        //let withQuery: Parameters<typeof this.db.query.plays.findMany>[0]['with'] = undefined;
+
         let query: FindMany<'plays'> = {
             limit: args.limit,
             offset: args.offset
@@ -100,11 +119,43 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository {
                 id: 'asc'
             }
         }
+
+        if(args.with !== undefined) {
+            query.with = {};
+            for(const w of args.with) {
+                switch (w) {
+                    case 'input':
+                        query.with.input = true;
+                        break;
+                    case 'parent':
+                        query.with.parent = true;
+                        break;
+                    case 'parent-input':
+                        query.with.parent = {
+                            with: {
+                                input: true
+                            }
+                        };
+                        break;
+                    default:
+                        throw new Error(`Unknown relation ${w}`);
+                }
+            }
+        }
         query = removeUndefinedKeys(query);
         const results = await this.db.query.plays.findMany(query);
+        if(hydrate.length > 0) {
+            return results.map((x) => ({...x, play: hydratePlaySelect(x, hydrate)}));
+        }
         return results;
     }
 
+    setStateById = async (state: PlayNew['state'], ids: number[]): Promise<void> => {
+        const validIds = ids.filter(x => x !== undefined && x !== null);
+        assert(validIds.length > 0, `Should not pass empty array of ids, after filtering, to update state. Original ids list: ${ids}`);
+        await this.db.update(plays).set({state}).where(inArray(plays.id, ids));
+    }
+
     deletePlays = async (playsData: (Pick<PlaySelect, 'id'> | number)[]) => {
         const ids = playsData.map(x => typeof x === 'number' ? x : x.id);
         await this.db.delete(plays).where(inArray(plays.id, ids));
@@ -319,6 +370,11 @@ export const buildPlayWhere = (args: PlayWhereOpts): FindWhere<'plays'> => {
     if(args.platformId !== undefined) {
         where.platformId = args.platformId
     }
+    if(args.uid !== undefined) {
+        where.uid = {
+            in: args.uid
+        }
+    }
     return where;
 }
 
@@ -356,4 +412,43 @@ export const playToRepositoryCreatePlayOpts = (data: MarkOptional<RepositoryCrea
         },
         platformId: genGroupIdStrFromPlay(data.play)
     }
+}
+
+export type RequestPlayQuery = Partial< Record<keyof Exclude<QueryPlaysOpts, 'componentId' | 'platformId'>, string>>;
+
+export const queryArgsFromRequest = (rec: RequestPlayQuery): QueryPlaysOpts => {
+
+    const {
+        state,
+        stateNot,
+        uid,
+        with: withQuery,
+        seenAt,
+        playedAt,
+        limit,
+        sort,
+        order,
+        offset,
+        componentId,
+        ...rest
+    } = rec;
+
+    let queryArgs: QueryPlaysOpts = removeEmptyArrays<QueryPlaysOpts>({
+        state: parseArrayFromMaybeString(state) as PlaySelect['state'][],
+        stateNot: parseArrayFromMaybeString(stateNot) as PlaySelect['state'][],
+        uid: parseArrayFromMaybeString(uid),
+        with: parseArrayFromMaybeString(withQuery) as WithPlayRelation[],
+        sort: sort as 'playedAt' | 'seenAt',
+        order: order as 'asc' | 'desc',
+        ...rest
+    });
+
+    if(limit !== undefined) {
+        queryArgs.limit = Number.parseInt(limit);
+    }
+    if(offset !== undefined) {
+        queryArgs.offset = Number.parseInt(offset);
+    }
+
+    return queryArgs;
 }

src/backend/server/api.ts

@@ -262,7 +262,9 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
             // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
             scrobbleSource: source,
             query: {
-                upstream = 'false'
+                upstream = 'false',
+                next: queryNext = 'false',
+                ...rest
             }
         } = req;
 
@@ -278,7 +280,11 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
                     return res.status(500).json({message: e.message});
                 }
             } else {
+                if(queryNext === 'true') {
+                    return res.json(await (source as AbstractSource).getRecentPlaysApi(rest));
+                }
                 result = await (source as AbstractSource).getFlatRecentlyDiscoveredPlays();
+                
             }
         }
 

src/backend/sources/AbstractSource.ts

@@ -46,7 +46,7 @@ import prom, { Counter, Gauge } from 'prom-client';
 import { normalizeStr } from '../utils/StringUtils.js';
 import { spawn, catchAbortError, isAbortError, rethrowAbortError, delay, forever, AbortError, throwIfAborted } from 'abort-controller-x';
 import { AbortedError, generateLoggableAbortReason } from '../common/errors/MSErrors.js';
-import { DrizzlePlayRepository, playToRepositoryCreatePlayOpts } from '../common/database/drizzle/repositories/PlayRepository.js';
+import { DrizzlePlayRepository, playToRepositoryCreatePlayOpts, queryArgsFromRequest, QueryPlaysOpts, RequestPlayQuery } from '../common/database/drizzle/repositories/PlayRepository.js';
 import { asPlay } from '../../core/PlayMarshalUtils.js';
 
 export interface RecentlyPlayedOptions {
@@ -212,7 +212,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
     // TODO make this more descriptive? or move it elsewhere
     recentlyPlayedTrackIsValid = (playObj: PlayObject) => true
 
-    protected addPlayToDiscovered = async (play: PlayObject) => {
+    protected addPlayToDiscovered = async (play: PlayObject): Promise<PlayObject> => {
         const platformId = this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID;
         const playRow = await this.playRepo.createPlays([(playToRepositoryCreatePlayOpts({play, componentId: this.dbComponent.id, state: 'discovered'}))]);
         const recentPlays = await this.getRecentlyDiscoveredPlaysByPlatform(platformId, false);
@@ -231,6 +231,8 @@ export default abstract class AbstractSource extends AbstractComponent implement
         this.logger.info(`Discovered => ${buildTrackString(play)}`);
         this.emitEvent('discovered', {play});
         this.discoveredCounter.labels(this.getPrometheusLabels()).inc();
+        play.meta.dbId = playRow[0].id;
+        return play;
     }
 
     getFlatRecentlyDiscoveredPlays = async (): Promise<PlayObject[]> => {
@@ -243,6 +245,18 @@ export default abstract class AbstractSource extends AbstractComponent implement
         //Array.from(this.recentDiscoveredPlays.values()).map(x => x.data).flat(3).sort(sortByNewestPlayDate)
     }
 
+    getRecentPlaysApi = async (query: RequestPlayQuery) => {
+        const res = await this.playRepo.findPlays({
+            componentId: this.dbComponent.id,
+            limit: 100,
+            ...queryArgsFromRequest(query)
+        });
+        return res.map((x) => {
+            const {id, ...rest} = x;
+            return rest;
+        })
+    }
+
     protected recentDiscoveredCacheKey = (platformId: PlayPlatformId | string) => {
         const platformStr = typeof platformId === 'string' ? platformId : genGroupIdStr(platformId);
         return `recent-${this.dbComponent.id}-${platformStr}`;
@@ -330,8 +344,8 @@ export default abstract class AbstractSource extends AbstractComponent implement
             options.signal?.throwIfAborted();
             if(!(await this.alreadyDiscovered(play, options))) {
                 options.signal?.throwIfAborted()
-                await this.addPlayToDiscovered(play);
-                newDiscoveredPlays.push(play);
+                const hydratedPlay = await this.addPlayToDiscovered(play);
+                newDiscoveredPlays.push(hydratedPlay);
             }
         }
         if(newDiscoveredPlays.length > 0) {
@@ -362,6 +376,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
 
         if(newDiscoveredPlays.length > 0) {
             if(!this.shouldScrobble(options.discoverLocation)) {
+                await this.playRepo.setStateById('discarded', newDiscoveredPlays.map(x => x.meta.dbId));
                 return;
             }
             newDiscoveredPlays.sort(sortByOldestPlayDate);

src/backend/utils.ts

@@ -225,6 +225,20 @@ export const removeUndefinedKeys = <T extends Record<string, any>>(obj: T): T |
     return newObj;
 }
 
+export const removeEmptyArrays = <T extends Record<string, any>>(obj: T): T => {
+    const newObj: any = {};
+    Object.keys(obj).forEach((key) => {
+        if(Array.isArray(obj[key])) {
+            if(obj[key].length !== 0) {
+                newObj[key] = obj[key];
+            }
+        } else {
+            newObj[key] = obj[key];
+        }
+    });
+    return newObj;
+}
+
 export const remoteHostIdentifiers = (req: Request): RemoteIdentityParts => {
     const remote = req.connection.remoteAddress;
     const proxyRemote = Array.isArray(req.headers["x-forwarded-for"]) ? req.headers["x-forwarded-for"][0] : req.headers["x-forwarded-for"];

src/core/Atomic.ts

@@ -202,6 +202,9 @@ export interface PlayMeta<D extends DateLike = Dayjs> {
 
     seenAt?: D
 
+    dbUid?: string
+    dbId?: number
+
     /*
     * If applicable, the name of the Service providing the track (Spotify, Tidal, etc...)
     */