multi-scrobbler/multi-scrobbler
1
0
Fork 0
mirror of https://github.com/FoxxMD/multi-scrobbler.git synced 2026-09-03 05:10:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat(database): Initial Souce database usage implementation

Browse Source
This commit is contained in:
FoxxMD
2026-05-06 02:56:34 +00:00
parent 2a52d1e050
commit 047de49ee6
21 changed files with 293 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backend/common/AbstractComponent.ts
+7 -1
View File
@@ -83,10 +83,16 @@ export default abstract class AbstractComponent extends AbstractInitializable {
protected async doBuildDatabase(): Promise<true | string | undefined> {
super.doBuildDatabase();
let name: string;
if('name' in this) {
name = this.name as string;
}
this.dbComponent = await this.componentRepo.findOrInsert({
mode: this.componentType,
type: this.type,
uid: this.config.id ?? this.config.name
uid: this.config.id ?? this.config.name ?? name,
name: this.config.name ?? name
});
return true;
}
src/backend/common/AbstractInitializable.ts
+14
View File
@@ -198,6 +198,20 @@ export default abstract class AbstractInitializable {
this.databaseOK = false;
throw new BuildDataError('Required database init failed', {cause: e});
}
try {
await this.postDatabase();
} catch (e) {
if(e instanceof StageError) {
throw e;
} else {
throw new Error('Error occurred during post-database hook', {cause: e});
}
}
}
protected async postDatabase(): Promise<void> {
return;
}
/**
src/backend/common/Cache.ts
+4 -1
View File
@@ -48,6 +48,7 @@ export class MSCache {
cacheMetadata: Cacheable;
cacheScrobble: Cacheable;
cacheDb: Cacheable;
cacheAuth: Cacheable;
regexCache: ReturnType<typeof cacheFunctions>;
cacheTransform: Cacheable;
@@ -114,6 +115,7 @@ export class MSCache {
this.cacheAuth = inMemory;
this.cacheScrobble = inMemory;
this.cacheApi = inMemory;
this.cacheDb = new Cacheable({primary: initMemoryCache({lruSize: 500, ttl: '1m'})});
}
init = async (enableCollectors: boolean = false) => {
@@ -133,7 +135,8 @@ export class MSCache {
{ cache: this.cacheScrobble, name: 'queued_scrobbles' },
{ cache: this.cacheTransform, name: 'transformer' },
{ cache: this.cacheClientScrobbles, name: 'historical_scrobbles' },
{ cache: this.cacheApi, name: 'external_apis' }
{ cache: this.cacheApi, name: 'external_apis' },
{ cache: this.cacheDb, name: 'database' }
];
this.cacheHits = new prom.Gauge({
src/backend/common/database/drizzle/entityUtils.ts
+1 -1
View File
@@ -15,7 +15,7 @@ export const generateComponentEntity = (data: MarkOptional<ComponentNew, 'uid'>)
};
}
export type PlayEntityOpts = Partial<Pick<PlayNew, 'seenAt' | 'playedAt' | 'uid' | 'state' | 'parentId' | 'componentId'>> & { error?: ErrorLike };
export type PlayEntityOpts = Partial<Pick<PlayNew, 'seenAt' | 'playedAt' | 'uid' | 'state' | 'parentId' | 'componentId' | 'platformId'>> & { error?: ErrorLike };
export const generatePlayEntity = (play: PlayObject, opts: PlayEntityOpts = {}): PlayNew => {
const {
src/backend/common/database/drizzle/migrations/20260426001747_jittery_demogoblin/migration.sql → src/backend/common/database/drizzle/migrations/20260427134808_far_thundra/migration.sql
+2
View File
@@ -28,6 +28,7 @@ CREATE TABLE `plays` (
`play` text NOT NULL,
`state` text NOT NULL,
`parentId` integer,
`platformId` text,
`compacted` text,
CONSTRAINT `fk_plays_componentId_components_id_fk` FOREIGN KEY (`componentId`) REFERENCES `components`(`id`) ON UPDATE CASCADE ON DELETE CASCADE,
CONSTRAINT `fk_plays_parentId_plays_id_fk` FOREIGN KEY (`parentId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE SET NULL
@@ -54,4 +55,5 @@ CREATE INDEX `play_component_id_idx` ON `plays` (`componentId`);--> statement-br
CREATE UNIQUE INDEX `play_uid_idx` ON `plays` (`uid`);--> statement-breakpoint
CREATE INDEX `play_playedAt_idx` ON `plays` (`playedAt`);--> statement-breakpoint
CREATE INDEX `play_seenAt_idx` ON `plays` (`seenAt`);--> statement-breakpoint
CREATE INDEX `play_platform_idx` ON `plays` (`platformId`);--> statement-breakpoint
CREATE INDEX `play_queue_state_id_idx` ON `play_queue_states` (`playId`);
src/backend/common/database/drizzle/migrations/20260426001747_jittery_demogoblin/snapshot.json → src/backend/common/database/drizzle/migrations/20260427134808_far_thundra/snapshot.json
+25 -1
View File
@@ -1,7 +1,7 @@
{
"version": "7",
"dialect": "sqlite",
"id": "c9d77fe1-7dce-4014-bd6e-10174b7ebccc",
"id": "2a0aea39-bc8a-436a-a84c-6041c2b63c3f",
"prevIds": [
"00000000-0000-0000-0000-000000000000"
],
@@ -242,6 +242,16 @@
"entityType": "columns",
"table": "plays"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "platformId",
"entityType": "columns",
"table": "plays"
},
{
"type": "text",
"notNull": false,
@@ -559,6 +569,20 @@
"entityType": "indexes",
"table": "plays"
},
{
"columns": [
{
"value": "platformId",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "play_platform_idx",
"entityType": "indexes",
"table": "plays"
},
{
"columns": [
{
src/backend/common/database/drizzle/repositories/PlayRepository.ts
+62 -2
View File
@@ -6,9 +6,9 @@ import { generateInputEntity, generatePlayEntity, PlayEntityOpts } from "../enti
import { playInputs, plays, relations } from "../schema/schema.js";
import { PlayNew, PlaySelect, PlayInputNew, FindWhere, FindMany, CompareOpKey } from "../drizzleTypes.js";;
import { MarkOptional, MarkRequired, PathValue } from "ts-essentials";
import { removeUndefinedKeys } from "../../../../utils.js";
import { genGroupIdStrFromPlay, removeUndefinedKeys } from "../../../../utils.js";
import dayjs, { Dayjs } from "dayjs";
import { RelationsFieldFilter, eq, inArray } from "drizzle-orm";
import { RelationsFieldFilter, eq, inArray, ne, notInArray, desc, asc, and } from "drizzle-orm";
import { CompactableProperty, RetentionOptions, retentionPlayTypes } from "../../../infrastructure/config/database.js";
import { shortTodayAwareFormat } from "../../../../../core/TimeUtils.js";
import { buildDateCompare, CompareDateOp, DrizzleBaseRepository } from "./BaseRepository.js";
@@ -20,7 +20,9 @@ export interface DrizzleRepositoryOpts {
}
export interface PlayWhereOpts {
state?: PlaySelect['state'][]
stateNot?: PlaySelect['state'][]
componentId?: number
platformId?: string
seenAt?: CompareDateOp
playedAt?: CompareDateOp
}
@@ -274,6 +276,19 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository {
loggerCom.verbose(`Cleanup done! Summary:\n${summaryDelStates.join(' | ')}`);
}
public selectRecentDistinctPlatforms = async (componentId: number, limitPlays: number = 500): Promise<string[]> => {
const recentPlatformIds = await this.db.selectDistinct({platformId: plays.platformId}).from(plays)
.where(
and(
eq(plays.componentId, componentId),
ne(plays.state, 'queued'))
)
.orderBy(desc(plays.playedAt))
.limit(limitPlays);
return recentPlatformIds.map(x => x.platformId);
}
}
export const buildPlayWhere = (args: PlayWhereOpts): FindWhere<'plays'> => {
@@ -288,12 +303,57 @@ export const buildPlayWhere = (args: PlayWhereOpts): FindWhere<'plays'> => {
in: args.state
}
}
if(args.stateNot !== undefined) {
where.state = {
NOT: {
in: args.stateNot
}
}
}
if (args.seenAt !== undefined) {
where.seenAt = buildDateCompare(args.seenAt);
}
if (args.playedAt !== undefined) {
where.playedAt = buildDateCompare(args.playedAt);
}
if(args.platformId !== undefined) {
where.platformId = args.platformId
}
return where;
}
export const playToRepositoryCreatePlayOpts = (data: MarkOptional<RepositoryCreatePlayOpts, 'input'>): RepositoryCreatePlayOpts => {
const {
play: {
meta: {
lifecycle: {
input,
original,
...lifecycleRest
} = {},
...metaRest
},
...playRest
},
...rest
} = data;
return {
play: {
...playRest,
meta: {
...metaRest,
// @ts-expect-error
lifecycle: {
...lifecycleRest
}
}
},
...rest,
input: {
play: original,
data: input
},
platformId: genGroupIdStrFromPlay(data.play)
}
}
src/backend/common/database/drizzle/schema/schema.ts
+3 -1
View File
@@ -29,9 +29,10 @@ export const plays = sqliteTable("plays", {
playedAt: DayjsTimestamp('playedAt'),
seenAt: DayjsTimestamp('seenAt'),
play: text({ mode: 'json' }).notNull().$type<PlayObject>(),
state: text({enum: ['queued','discovered','scrobbled','failed','duped']}).notNull(),
state: text({enum: ['queued','discovered','discarded','scrobbled','failed','duped']}).notNull(),
// https://orm.drizzle.team/docs/indexes-constraints#foreign-key
parentId: integer().references((): AnySQLiteColumn => plays.id, {onDelete: 'set null', onUpdate: 'cascade'}),
platformId: text(),
compacted: text()
}, (table) => [
index("play_parent_id_idx").on(table.parentId),
@@ -39,6 +40,7 @@ export const plays = sqliteTable("plays", {
uniqueIndex("play_uid_idx").on(table.uid),
index("play_playedAt_idx").on(table.playedAt),
index("play_seenAt_idx").on(table.seenAt),
index("play_platform_idx").on(table.platformId)
]);
export const playInputs = sqliteTable("play_inputs", {
src/backend/server/api.ts
+1 -1
View File
@@ -278,7 +278,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
return res.status(500).json({message: e.message});
}
} else {
result = (source as AbstractSource).getFlatRecentlyDiscoveredPlays();
result = await (source as AbstractSource).getFlatRecentlyDiscoveredPlays();
}
}
src/backend/sources/AbstractSource.ts
+81 -26
View File
@@ -2,7 +2,7 @@ import { childLogger, LogDataPretty, LogLevel } from '@foxxmd/logging';
import dayjs, { Dayjs } from "dayjs";
import { EventEmitter } from "events";
import { FixedSizeList } from "fixed-size-list";
import { PlayObject } from "../../core/Atomic.js";
import { JsonPlayObject, PlayObject } from "../../core/Atomic.js";
import { buildTrackString, capitalize, truncateStringToLength } from "../../core/StringUtils.js";
import AbstractComponent from "../common/AbstractComponent.js";
import {
@@ -31,7 +31,7 @@ import {
sleep,
sortByOldestPlayDate,
} from "../utils.js";
import { sortByNewestPlayDate } from '../../core/PlayUtils.js';
import { genGroupIdStr, sortByNewestPlayDate } from '../../core/PlayUtils.js';
import { formatNumber } from '../../core/DataUtils.js';
import { timeToHumanTimestamp } from "../../core/TimeUtils.js";
import { todayAwareFormat } from "../../core/TimeUtils.js";
@@ -46,6 +46,8 @@ import prom, { Counter, Gauge } from 'prom-client';
import { normalizeStr } from '../utils/StringUtils.js';
import { spawn, catchAbortError, isAbortError, rethrowAbortError, delay, forever, AbortError, throwIfAborted } from 'abort-controller-x';
import { AbortedError, generateLoggableAbortReason } from '../common/errors/MSErrors.js';
import { DrizzlePlayRepository, playToRepositoryCreatePlayOpts } from '../common/database/drizzle/repositories/PlayRepository.js';
import { asPlay } from '../../core/PlayMarshalUtils.js';
export interface RecentlyPlayedOptions {
limit?: number
@@ -105,6 +107,8 @@ export default abstract class AbstractSource extends AbstractComponent implement
declare protected componentType: 'source';
protected playRepo: DrizzlePlayRepository;
constructor(type: SourceType, name: string, config: SourceConfig, internal: InternalConfig, emitter: EventEmitter) {
super(config);
this.componentType = 'source';
@@ -122,6 +126,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
this.emitter = emitter;
this.discoveredCounter = getRoot().items.sourceMetics.discovered;
this.playRepo = new DrizzlePlayRepository(this.db, {logger: this.logger});
}
async [Symbol.asyncDispose]() {
@@ -135,6 +140,10 @@ export default abstract class AbstractSource extends AbstractComponent implement
this.generateStaggerMappers();
}
protected async postDatabase(): Promise<void> {
this.tracksDiscovered = this.dbComponent.countLive;
}
protected generateStaggerMappers() {
const {
preCompare = [],
@@ -203,52 +212,98 @@ export default abstract class AbstractSource extends AbstractComponent implement
// TODO make this more descriptive? or move it elsewhere
recentlyPlayedTrackIsValid = (playObj: PlayObject) => true
protected addPlayToDiscovered = (play: PlayObject) => {
protected addPlayToDiscovered = async (play: PlayObject) => {
const platformId = this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID;
const list = this.recentDiscoveredPlays.get(platformId) ?? new FixedSizeList<ProgressAwarePlayObject>(200);
list.add(play);
this.recentDiscoveredPlays.set(platformId, list);
const playRow = await this.playRepo.createPlays([(playToRepositoryCreatePlayOpts({play, componentId: this.dbComponent.id, state: 'discovered'}))]);
const recentPlays = await this.getRecentlyDiscoveredPlaysByPlatform(platformId, false);
// only need to update if its already in memory,
// and better to update in-memory than clear cache so we aren't refetching from db on every discover
if(recentPlays !== undefined) {
recentPlays.push(play);
recentPlays.sort(sortByOldestPlayDate);
this.cache.cacheDb.set(this.recentDiscoveredCacheKey(platformId), recentPlays, '2m');
}
const platformIds = await this.getRecentPlatformIds(false);
if(platformIds !== undefined && !platformIds.includes(genGroupIdStr(platformId))) {
this.cache.cacheDb.set(this.recentPlatformsCacheKey(), platformIds, '10m');
}
this.tracksDiscovered++;
this.logger.info(`Discovered => ${buildTrackString(play)}`);
this.emitEvent('discovered', {play});
this.discoveredCounter.labels(this.getPrometheusLabels()).inc();
}
getFlatRecentlyDiscoveredPlays = (): PlayObject[] =>
Array.from(this.recentDiscoveredPlays.values()).map(x => x.data).flat(3).sort(sortByNewestPlayDate)
getRecentlyDiscoveredPlaysByPlatform = (platformId: PlayPlatformId): PlayObject[] => {
const list = this.recentDiscoveredPlays.get(platformId);
if (list !== undefined) {
const data = [...list.data];
data.sort(sortByOldestPlayDate);
return data;
getFlatRecentlyDiscoveredPlays = async (): Promise<PlayObject[]> => {
const platforms = await this.getRecentPlatformIds();
const list: PlayObject[][] = [];
for(const platformId of platforms) {
list.push(await this.getRecentlyDiscoveredPlaysByPlatform(platformId));
}
return [];
return list.flat().sort(sortByNewestPlayDate);
//Array.from(this.recentDiscoveredPlays.values()).map(x => x.data).flat(3).sort(sortByNewestPlayDate)
}
protected getExistingDiscoveredLists = (play: PlayObject, opts: {checkAll?: boolean} = {}): PlayObject[][] => {
protected recentDiscoveredCacheKey = (platformId: PlayPlatformId | string) => {
const platformStr = typeof platformId === 'string' ? platformId : genGroupIdStr(platformId);
return `recent-${this.dbComponent.id}-${platformStr}`;
}
protected recentPlatformsCacheKey = () => {
return `recentPlatformIds-${this.dbComponent.id}`;
}
getRecentlyDiscoveredPlaysByPlatform = async (platformId: PlayPlatformId | string, hydrate: boolean = true): Promise<PlayObject[]> => {
const platformStr = typeof platformId === 'string' ? platformId : genGroupIdStr(platformId);
const cacheKey = this.recentDiscoveredCacheKey(platformId);
let list = await this.cache.cacheDb.get<PlayObject[]>(cacheKey);
if(list === undefined && hydrate) {
list = (await this.playRepo.findPlays({
platformId: platformStr,
componentId: this.dbComponent.id,
stateNot: ['queued'],
order: 'desc',
sort: 'playedAt',
limit: 200
})).map(x => asPlay(x.play))
list.sort(sortByOldestPlayDate);
await this.cache.cacheDb.set<PlayObject[]>(cacheKey, list, '2m');
}
return list;
}
protected getRecentPlatformIds = async (hydrate: boolean = true) => {
const cacheKey = this.recentPlatformsCacheKey();
let list = await this.cache.cacheDb.get<string[]>(cacheKey);
if(list === undefined && hydrate) {
list = await this.playRepo.selectRecentDistinctPlatforms(this.dbComponent.id);
await this.cache.cacheDb.set<string[]>(cacheKey, list, '10m');
}
return list;
}
protected getExistingDiscoveredLists = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<PlayObject[][]> => {
const lists: PlayObject[][] = [];
if(opts.checkAll !== true) {
lists.push(this.getRecentlyDiscoveredPlaysByPlatform(this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID));
lists.push(await this.getRecentlyDiscoveredPlaysByPlatform(this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID));
} else {
const platforms = await this.getRecentPlatformIds();
// get as many as we can, optionally filtering by user
this.recentDiscoveredPlays.forEach((list, platformId) => {
for(const platformId of platforms) {
if(play.meta.user !== undefined) {
if(platformId[1] === NO_USER || platformId[1] === play.meta.user) {
lists.push(this.getRecentlyDiscoveredPlaysByPlatform(platformId));
lists.push(await this.getRecentlyDiscoveredPlaysByPlatform(platformId));
}
} else {
lists.push(this.getRecentlyDiscoveredPlaysByPlatform(platformId));
lists.push(await this.getRecentlyDiscoveredPlaysByPlatform(platformId));
}
});
}
}
return lists;
}
existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<PlayObject | undefined> => {
const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts);
const lists: PlayObject[][] = await this.getExistingDiscoveredLists(play, opts);
const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate);
for(const list of lists) {
@@ -275,7 +330,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
options.signal?.throwIfAborted();
if(!(await this.alreadyDiscovered(play, options))) {
options.signal?.throwIfAborted()
this.addPlayToDiscovered(play);
await this.addPlayToDiscovered(play);
newDiscoveredPlays.push(play);
}
}
@@ -693,7 +748,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
}
protected async doBuildComponentLogger(): Promise<void> {
if(this.config.options.logToFile) {
if(this.config?.options?.logToFile) {
this.logger.debug('Enabling component logger...');
const root = getRoot();
const stream = root.get('loggerStream');
src/backend/sources/DeezerInternalSource.ts
+1 -1
View File
@@ -330,7 +330,7 @@ export default class DeezerInternalSource extends MemorySource {
existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<PlayObject | undefined> => {
const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts);
const lists: PlayObject[][] = await this.getExistingDiscoveredLists(play, opts);
const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate);
for(const list of lists) {
const existing = await findAsync(list, async x => {
src/backend/sources/EndpointLastfmSource.ts
+1 -1
View File
@@ -62,7 +62,7 @@ export class EndpointLastfmSource extends MemorySource {
}
getRecentlyPlayed = async (options = {}) => {
return this.getFlatRecentlyDiscoveredPlays();
return await this.getFlatRecentlyDiscoveredPlays();
}
isValidScrobble = (playObj: PlayObject) => {
src/backend/sources/EndpointListenbrainzSource.ts
+1 -1
View File
@@ -82,7 +82,7 @@ export class EndpointListenbrainzSource extends MemorySource {
}
getRecentlyPlayed = async (options = {}) => {
return this.getFlatRecentlyDiscoveredPlays();
return await this.getFlatRecentlyDiscoveredPlays();
}
isValidScrobble = (playObj: PlayObject) => {
src/backend/sources/MemorySource.ts
+1 -1
View File
@@ -368,7 +368,7 @@ export default class MemorySource extends AbstractSource {
}
return [false, `${stPrefix} ${EXPECTED_NON_DISCOVERED_REASON}`]
} else {
const discoveredPlays = this.getRecentlyDiscoveredPlaysByPlatform(genGroupId(candidate));
const discoveredPlays = await this.getRecentlyDiscoveredPlaysByPlatform(genGroupId(candidate));
if (discoveredPlays.length === 0 || !playObjDataMatch(discoveredPlays[0], candidate)) {
// if most recent stateful play is not this track we'll add it
return [true,`${stPrefix} added after ${thresholdResultSummary(thresholdResults)}. Matched other recent play but could not determine time frame due to missing duration. Allowed due to not being last played track.`];
src/backend/sources/WebScrobblerSource.ts
+1 -1
View File
@@ -155,7 +155,7 @@ export class WebScrobblerSource extends MemorySource {
return baseFormatPlayObj(obj, play);
}
getRecentlyPlayed = async (options = {}) => this.getFlatRecentlyDiscoveredPlays()
getRecentlyPlayed = async (options = {}) => await this.getFlatRecentlyDiscoveredPlays()
isValidScrobble = (playObj: PlayObject) => {
if (playObj.meta?.scrobbleAllowed === false) {
src/backend/sources/YTMusicSource.ts
+10 -3
View File
@@ -23,6 +23,7 @@ import { joinedUrl } from "../utils/NetworkUtils.js";
import { todayAwareFormat } from "../../core/TimeUtils.js";
import { parseArrayFromMaybeString, parseArtistCredits, parseCredits } from "../utils/StringUtils.js";
import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
import { FixedSizeList } from "fixed-size-list";
export interface HistoryIngressResult {
plays: PlayObject[],
@@ -118,6 +119,7 @@ export default class YTMusicSource extends AbstractSource {
declare config: YTMusicSourceConfig
recentlyPlayed: PlayObject[] = [];
transientDiscovered: FixedSizeList<PlayObject> = new FixedSizeList<PlayObject>(200);
yti: Innertube;
userCode?: string;
@@ -149,6 +151,10 @@ export default class YTMusicSource extends AbstractSource {
this.config.options = {...rest, logDiff: diffVal};
}
}
this.emitter.on('discovered', (play) => {
this.transientDiscovered.add(play);
})
}
public additionalApiData(): Record<string, any> {
@@ -588,7 +594,7 @@ Redirect URI : ${this.redirectUri}`);
if(consistent && newPlays.length > 1) {
const interimPlays = newPlays.slice(0, newPlays.length - 1);
// check enough time has passed since last discovery
const discovered = this.getFlatRecentlyDiscoveredPlays();
const discovered = this.transientDiscovered.data;
if(discovered.length > 0) {
const lastDiscovered = discovered[0].data.playDate;
// the assumption in behavior is that user skips 1 or more tracks which then get recorded to YTM history
@@ -681,10 +687,11 @@ ${humanDiff}`;
const reversedPlays = [...referencePlays];
// actual order they were discovered in (oldest to newest)
reversedPlays.reverse();
if(this.getFlatRecentlyDiscoveredPlays().length === 0) {
if(this.transientDiscovered.data.length === 0) {
// and add to discovered since its empty
for(const refPlay of reversedPlays) {
this.addPlayToDiscovered(refPlay);
//this.transientDiscovered.add(refPlay);
await this.addPlayToDiscovered(refPlay);
}
}
}
src/backend/tests/database/drizzle.test.ts
+31 -1
View File
@@ -16,7 +16,8 @@ import { DrizzlePlayRepository, RepositoryCreatePlayOpts } from '../../common/da
import { generateRandomObj } from '../../../core/tests/utils/fixtures.js';
import { generateArray } from '../../../core/DataUtils.js';
import { objectsEqual } from '../../utils/DataUtils.js';
import { eq } from 'drizzle-orm';
import { eq, sql } from 'drizzle-orm';
import { PlaySelect } from '../../common/database/drizzle/drizzleTypes.js';
// would be great to push migrations directly from schema but doesn't seem supported in newest beta
// https://github.com/drizzle-team/drizzle-orm/discussions/4373
@@ -530,5 +531,34 @@ describe('Repository Operations', function () {
expect(p2Plays[1]).to.eq(childPlays[0].id);
});
it('Get json property from play', async function () {
const db = getDb(':memory:', { workingDirectory: process.cwd() });
await migrateDb(db);
try {
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
const playRows = await db.insert(plays).values([
fixtureCreatePlay({ componentId: component[0].id, play: generatePlay({}, {source: 'test1'}) }),
fixtureCreatePlay({ componentId: component[0].id, play: generatePlay({}, {source: 'test2'}) })
]).returning();
let result: PlaySelect[];
// https://github.com/drizzle-team/drizzle-orm/discussions/938#discussioncomment-6542336
result = await db.select().from(plays).where(
sql`json_extract(${plays.play}, '$.meta.source') = 'test1'`
);
expect(result).length(1);
expect(result[0].play.meta.source).eq('test1');
} catch (e) {
throw e;
}
db.$client.close();
});
});
src/backend/tests/source/source.test.ts
+37 -31
View File
@@ -24,19 +24,23 @@ chai.use(asPromised);
const emitter = new EventEmitter();
const generateSource = () => {
return new TestSource('spotify', 'test', {}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
const generateSource = async () => {
const source = new TestSource('spotify', 'test-basic', {}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
await source.tryInitialize();
return source;
}
const generateMemorySource = (config: SourceConfig = {}) => {
const s = new TestMemorySource('spotify', 'test', config, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
s.buildTransformRules();
const generateMemorySource = async (config: SourceConfig = {}) => {
const s = new TestMemorySource('spotify', 'test-memory', config, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
await s.tryInitialize();
// s.buildTransformRules();
s.scheduler.stop();
return s;
}
const generateMemoryPositionalSource = (config: SourceConfig = {}) => {
const s = new TestMemoryPositionalSource('spotify', 'test', config, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
s.buildTransformRules();
const generateMemoryPositionalSource = async (config: SourceConfig = {}) => {
const s = new TestMemoryPositionalSource('spotify', 'test-positional', config, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
await s.tryInitialize();
//s.buildTransformRules();
s.scheduler.stop();
return s;
}
@@ -44,7 +48,7 @@ const generateMemoryPositionalSource = (config: SourceConfig = {}) => {
describe('Sources use transform plays correctly', function () {
it('Transforms play on preCompare', async function() {
await using source = generateSource();
await using source = await generateSource();
source.config.options = {
playTransform: {
preCompare: {
@@ -67,7 +71,7 @@ describe('Sources use transform plays correctly', function () {
});
it('Transforms play on postCompare', async function() {
await using source = generateSource();
await using source = await generateSource();
source.config.options = {
playTransform: {
postCompare: {
@@ -96,7 +100,7 @@ describe('Sources use transform plays correctly', function () {
});
it('Transforms play existing comparison', async function() {
await using source = generateSource();
await using source = await generateSource();
source.config.options = {
playTransform: {
compare: {
@@ -123,7 +127,7 @@ describe('Sources use transform plays correctly', function () {
});
it('Transforms play candidate comparison', async function() {
await using source = generateSource();
await using source = await generateSource();
source.config.options = {
playTransform: {
compare: {
@@ -192,8 +196,8 @@ describe('Player Cleanup', function () {
setRtTick(1);
});
const cleanedUpDuration = async (generateSource: (config: SourceConfig) => MemorySource) => {
await using source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
const cleanedUpDuration = async (generateSource: (config: SourceConfig) => Promise<MemorySource>) => {
await using source = await generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
const initialDate = dayjs();
const initialState = generatePlayerStateData({position: 0, playData: {duration: 50}, stateUpdatedAt: initialDate, status: REPORTED_PLAYER_STATUSES.playing});
expect((await source.processRecentPlays([initialState])).length).to.be.eq(0);
@@ -235,9 +239,9 @@ describe('Player Cleanup', function () {
await cleanedUpDuration(generateMemoryPositionalSource);
});
const noScrobbleRediscoveryOnActive = async (generateSource: (config: SourceConfig) => MemorySource) => {
const noScrobbleRediscoveryOnActive = async (generateSource: (config: SourceConfig) => Promise<MemorySource>) => {
await using source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
await using source = await generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
const initialDate = dayjs();
const initialState = generatePlayerStateData({position: 0, playData: {duration: 50}, stateUpdatedAt: initialDate, status: REPORTED_PLAYER_STATUSES.playing});
expect((await source.processRecentPlays([initialState])).length).to.be.eq(0);
@@ -303,9 +307,9 @@ describe('Player Cleanup', function () {
await noScrobbleRediscoveryOnActive(generateMemoryPositionalSource);
});
const noScrobbleStale = async (generateSource: (config: SourceConfig) => MemorySource) => {
const noScrobbleStale = async (generateSource: (config: SourceConfig) => Promise<MemorySource>) => {
await using source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
await using source = await generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
const initialDate = dayjs();
// if player incorrectly counted stale time then 30s of actual play + 20s of stale time > scrobble threshold of 50% of 90s
@@ -348,9 +352,9 @@ describe('Player Cleanup', function () {
await noScrobbleStale(generateMemoryPositionalSource);
});
const scrobbleRediscoveryOnActive = async (generateSource: (config: SourceConfig) => MemorySource) => {
const scrobbleRediscoveryOnActive = async (generateSource: (config: SourceConfig) => Promise<MemorySource>) => {
await using source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
await using source = await generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
const initialDate = dayjs();
// if player incorrectly counted stale time then 30s of actual play + 20s of stale time > scrobble threshold of 50% of 90s
@@ -421,8 +425,10 @@ describe('Player Cleanup', function () {
});
});
const generateDeezerSource = (options: DeezerInternalSourceOptions = {}) => {
return new DeezerInternalSource('test', {data: {arl: 'test'}, options}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
const generateDeezerSource = async (options: DeezerInternalSourceOptions = {}) => {
const source = new DeezerInternalSource('test', {data: {arl: 'test'}, options}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
await source.tryInitialize();
return source;
}
const firstPlayDate = dayjs().subtract(1, 'hour');
const normalizedPlays = normalizePlays(generatePlays(6), {initialDate: firstPlayDate});
@@ -438,7 +444,7 @@ describe('Deezer Internal Source', function() {
const fuzzyPlay = clone(targetPlay);
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
const source = generateDeezerSource();
const source = await generateDeezerSource();
source.discover([...normalizedPlays, interimPlay]);
const discovered = await source.discover([fuzzyPlay]);
@@ -455,7 +461,7 @@ describe('Deezer Internal Source', function() {
const fuzzyPlay = clone(targetPlay);
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: true});
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true});
await source.discover([...normalizedPlays, interimPlay]);
const discovered = await source.discover([fuzzyPlay]);
@@ -468,7 +474,7 @@ describe('Deezer Internal Source', function() {
const fuzzyPlay = clone(targetPlay);
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: true});
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true});
await source.discover(normalizedPlays);
const discovered = await source.discover([fuzzyPlay]);
@@ -482,7 +488,7 @@ describe('Deezer Internal Source', function() {
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate});
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: true});
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true});
const discovered = await source.discover(morePlays);
expect(discovered.length).to.eq(morePlays.length);
@@ -497,7 +503,7 @@ describe('Deezer Internal Source', function() {
const fuzzyPlay = clone(targetPlay);
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
await source.discover([...normalizedPlays, interimPlay]);
const discovered = await source.discover([fuzzyPlay]);
@@ -511,7 +517,7 @@ describe('Deezer Internal Source', function() {
const duringPlay = clone(targetPlay);
duringPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration * 0.5, 's');
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
await source.discover([...normalizedPlays, interimPlay]);
const discovered = await source.discover([duringPlay]);
@@ -525,7 +531,7 @@ describe('Deezer Internal Source', function() {
const fuzzyPlay = clone(targetPlay);
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration + 39, 's');
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
await source.discover([...normalizedPlays, interimPlay]);
const discovered = await source.discover([fuzzyPlay]);
@@ -538,7 +544,7 @@ describe('Deezer Internal Source', function() {
const fuzzyPlay = clone(targetPlay);
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
await source.discover(normalizedPlays);
const discovered = await source.discover([fuzzyPlay]);
@@ -552,7 +558,7 @@ describe('Deezer Internal Source', function() {
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate});
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
const discovered = await source.discover(morePlays);
expect(discovered.length).to.eq(morePlays.length - 1);
src/backend/tests/ytm/ytm.test.ts
+8 -7
View File
@@ -14,7 +14,7 @@ import { ApiResponse } from 'youtubei.js';
chai.use(asPromised);
const createYtSource = (opts?: {
const createYtSource = async (opts?: {
config?: YTMusicSourceConfig
emitter?: EventEmitter
}) => {
@@ -27,6 +27,7 @@ const createYtSource = (opts?: {
emitter = new EventEmitter
} = opts || {};
const source = new YTMusicSource('test', config, { localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test' }, emitter);
await source.buildDatabase();
source.buildTransformRules();
return source;
}
@@ -48,7 +49,7 @@ describe('Handles temporal inconsistency in history', function () {
it(`Adds new, prepended track`, async function () {
const source = createYtSource();
const source = await createYtSource();
const plays = [...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Yesterday' })];
@@ -72,7 +73,7 @@ describe('Handles temporal inconsistency in history', function () {
it(`Adds bumped, prepended track`, async function () {
const source = createYtSource();
const source = await createYtSource();
const plays = [...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Yesterday' })];
@@ -98,7 +99,7 @@ describe('Handles temporal inconsistency in history', function () {
it(`Does not add appended track`, async function () {
const source = createYtSource();
const source = await createYtSource();
const plays = [...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Yesterday' })];
@@ -122,7 +123,7 @@ describe('Handles temporal inconsistency in history', function () {
this.timeout(3700);
const source = createYtSource();
const source = await createYtSource();
const plays = [...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Yesterday' })];
@@ -166,7 +167,7 @@ describe('Handles interim tracks', function () {
it(`Does not add skipped plays`, async function () {
const source = createYtSource();
const source = await createYtSource();
const plays = [...generatePlays(10, {playDate: dayjs().subtract(20, 'seconds')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(20, 'seconds')}, { comment: 'Yesterday' })];
@@ -193,7 +194,7 @@ describe('Handles interim tracks', function () {
it(`Adds interim plays when discover time is plausible`, async function () {
const source = createYtSource();
const source = await createYtSource();
const plays = [...generatePlays(10, {playDate: dayjs().subtract(2, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(2, 'minutes')}, { comment: 'Yesterday' })];
src/core/Atomic.ts
+1 -1
View File
@@ -304,7 +304,7 @@ export interface ScrobbleResult<D extends DateLike = Dayjs> {
export interface PlayLifecycle<D extends DateLike = Dayjs> {
input?: object
original: PlayObjectLifecycleless<D>
original?: PlayObjectLifecycleless<D>
steps: LifecycleStep[]
scrobble?: ScrobbleResult<D>
}
src/core/PlayMarshalUtils.ts
+1 -1
View File
@@ -56,7 +56,7 @@ export const asJsonPlayObject = (play: AmbPlayObject): JsonPlayObject => {
return cloned as unknown as JsonPlayObject;
};
export const asPlay = (data: JsonPlayObject): PlayObject => {
export const asPlay = (data: JsonPlayObject | PlayObject): PlayObject => {
const cloned = clone(data);
new Traverse(cloned).forEach((ctx, x) => {
if (shouldBlock(ctx)) {