feat(transform): Implement path to control cache usage #634

src/backend/common/AbstractComponent.ts

@@ -33,7 +33,7 @@ import { getRetentionCompactAfterFromEnv, getRetentionDeleteAfterFromEnv, isComp
 import type {DbConcrete} from "./database/drizzle/drizzleUtils.ts";
 import type {ComponentSelect} from "./database/drizzle/drizzleTypes.ts";
 import { DrizzlePlayRepository } from "./database/drizzle/repositories/PlayRepository.ts";
-import type {ClientType, MonitoringStatus} from "../../core/Atomic.ts";
+import type {ClientType, MonitoringStatus, OptionalCacheUsage} from "../../core/Atomic.ts";
 import type {SourceType} from "../../core/Atomic.ts";
 import { DrizzleComponentRepository } from "./database/drizzle/repositories/ComponentRepository.ts";
 import dayjs, { type Dayjs } from "dayjs";
@@ -248,7 +248,12 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         return partArr.map(x => this.transformManager.parseTransformerConfig(x));
     }
 
-    public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean | 'all') => {
+    public transformPlay = async (play: PlayObject, hookType: TransformHook, transformOpts: {log?: boolean | 'all'} & OptionalCacheUsage = {}) => {
+
+        const {
+            log,
+            useCachedResult = true
+        } = transformOpts;
 
         let logger: Logger;
 
@@ -280,7 +285,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             const shouldLog = log ?? this.config.options?.playTransform?.log ?? isDebugMode();
 
             const transformHash = `playTransform-${hashObject(hook)}-${hashObject(playContentInvariantTransform(play))}`;
-            const cachedSteps = await this.cache.cacheTransform.get<LifecycleStep[]>(transformHash);
+            const cachedSteps = useCachedResult ?  await this.cache.cacheTransform.get<LifecycleStep[]>(transformHash) : undefined;
             if(cachedSteps !== undefined) {
                 logger.trace(`Cache hit for Steps => ${transformHash}`);
                 //return cachedTransformPlay;
@@ -294,7 +299,8 @@ export default abstract class AbstractComponent extends AbstractInitializable {
 
             const opts = {
                 logger,
-                asyncId
+                asyncId,
+                useCachedResult
             }
 
             if(cachedSteps !== undefined) {
@@ -415,17 +421,18 @@ export default abstract class AbstractComponent extends AbstractInitializable {
         }
     }
 
-    protected generateStepFromStage = async (playTruth: PlayObject, hookItem: StageConfig, hookType: TransformHook, opts: { logger?: Logger, asyncId?: string } = {}): Promise<[LifecycleStep, PlayObject]> => {
+    protected generateStepFromStage = async (playTruth: PlayObject, hookItem: StageConfig, hookType: TransformHook, opts: { logger?: Logger, asyncId?: string } & OptionalCacheUsage = {}): Promise<[LifecycleStep, PlayObject]> => {
         const {
             onSuccess = 'continue',
             onFailure = 'stop',
             onSkip = 'continue',
-            failureReturnPartial = false
+            failureReturnPartial = false,
         } = hookItem;
 
         const {
             logger = loggerNoop,
-            asyncId = nanoid(6)
+            asyncId = nanoid(6),
+            useCachedResult
         } = opts;
 
         const {
@@ -446,7 +453,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
             stageName: string = 'Unnamed',
             err: Error;
         try {
-            [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, playTruth, asyncId);
+            [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, playTruth, {asyncId, useCachedResult});
             newTransformedPlay = clone(newTransformedPlay);
         } catch (e) {
             err = e;

src/backend/common/transforms/AbstractTransformer.ts

@@ -1,5 +1,5 @@
 import { childLogger, type Logger } from "@foxxmd/logging";
-import type { PlayObject, TransformerCommon, TransformerCommonConfig } from "../../../core/Atomic.ts";
+import type { OptionalCacheUsage, PlayObject, TransformerCommon, TransformerCommonConfig } from "../../../core/Atomic.ts";
 import { isStageTyped, testWhenConditions } from "../../utils/PlayTransformUtils.ts";
 import AbstractInitializable from "../AbstractInitializable.ts";
 import type { StageConfig } from "../../../core/Transform.ts";
@@ -61,13 +61,16 @@ export default abstract class AbstractTransformer<T = any, Y extends StageConfig
 
     protected abstract doParseConfig(data: StageConfig): Y;
 
-    public async handle(data: Y, play: PlayObject): Promise<PlayObject> {
+    public async handle(data: Y, play: PlayObject, opts?: OptionalCacheUsage): Promise<PlayObject> {
+        const {
+            useCachedResult = true,
+        } = (opts ?? {});
         const cacheKey = `transformResult-${this.configHash}-${hashObject(data)}-${hashObject(playContentInvariantTransform(play))}`
         try {
-            const cachedTransformData = await this.cache.get<T>(cacheKey);
+            const cachedTransformData = useCachedResult ? await this.cache.get<T>(cacheKey) : undefined;
             if(cachedTransformData !== undefined) {
                 this.logger.debug('Transform cache hit');
-                const transformed = await this.doHandle(data, play, cachedTransformData);
+                const transformed = await this.doHandle(data, play, cachedTransformData, opts);
                 return transformed;
             }
         } catch (e) {
@@ -82,7 +85,7 @@ export default abstract class AbstractTransformer<T = any, Y extends StageConfig
         }
 
         try {
-            await this.handlePreFetch(play, data);
+            await this.handlePreFetch(play, data, opts);
         } catch (e) {
             if(e instanceof SkipTransformStageError) {
                 await this.cache.set(cacheKey, play, this.config.options?.ttl ?? '15s');
@@ -93,13 +96,13 @@ export default abstract class AbstractTransformer<T = any, Y extends StageConfig
         let transformData: T;
         let fetchedTransformData: any;
         try {
-            fetchedTransformData = await this.getTransformerData(play, data);
+            fetchedTransformData = await this.getTransformerData(play, data, opts);
         } catch (e) {
             throw new Error(`Could not fetch transformer data`, { cause: e });
         }
 
         try {
-            transformData = await this.handlePostFetch(play, fetchedTransformData, data);
+            transformData = await this.handlePostFetch(play, fetchedTransformData, data, opts);
         } catch (e) {
             if(e instanceof StagePrerequisiteError) {
                 await this.cache.set(cacheKey, play, this.config.options?.ttl ?? '15s');
@@ -107,22 +110,22 @@ export default abstract class AbstractTransformer<T = any, Y extends StageConfig
             throw new Error('postFetch did not pass', { cause: e });
         }
 
-        const transformed = await this.doHandle(data, play, transformData);
+        const transformed = await this.doHandle(data, play, transformData, opts);
         await this.cache.set(cacheKey, transformData, this.config.options?.ttl ?? '15s');
         return transformed;
     }
 
-    protected abstract doHandle(data: StageConfig, play: PlayObject, transformData: T): Promise<PlayObject>;
+    protected abstract doHandle(data: StageConfig, play: PlayObject, transformData: T, opts?: OptionalCacheUsage): Promise<PlayObject>;
 
-    public async getTransformerData(play: PlayObject, stageConfig: Y): Promise<any> {
+    public async getTransformerData(play: PlayObject, stageConfig: Y, opts?: OptionalCacheUsage): Promise<any> {
         return undefined;
     }
 
-    public async handlePostFetch(play: PlayObject, transformData: any, stageConfig: Y): Promise<T> {
+    public async handlePostFetch(play: PlayObject, transformData: any, stageConfig: Y, opts?: OptionalCacheUsage): Promise<T> {
         return transformData;
     }
 
-    public async handlePreFetch(play: PlayObject, stageConfig: Y): Promise<void> {
+    public async handlePreFetch(play: PlayObject, stageConfig: Y, opts?: OptionalCacheUsage): Promise<void> {
         return
     }
 }

src/backend/common/transforms/MusicbrainzTransformer.ts

@@ -1,4 +1,4 @@
-import { type ArtistCredit, asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, type LifecycleInput, type MBReleaseGroupPrimaryType, type MBReleaseGroupSecondaryType, type MBReleaseStatus, type MissingMbidType, type PlayObject, type TrackMeta, type TransformerCommon, type TransformOptions } from "../../../core/Atomic.ts";
+import { type ArtistCredit, asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, type LifecycleInput, type MBReleaseGroupPrimaryType, type MBReleaseGroupSecondaryType, type MBReleaseStatus, type MissingMbidType, type OptionalCacheUsage, type PlayObject, type TrackMeta, type TransformerCommon, type TransformOptions } from "../../../core/Atomic.ts";
 import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.ts";
 import type {WebhookPayload} from "../infrastructure/config/health/webhooks.ts";
 import type {ExternalMetadataTerm, PlayTransformMetadataStage} from "../../../core/Transform.ts";
@@ -429,7 +429,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
         }
     }
 
-    public async getTransformerData(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async getTransformerData(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts?: OptionalCacheUsage): Promise<IRecordingMSList> {
         
         const {
             // preserve order of search from before searchOrder
@@ -443,25 +443,25 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
             try {
                 switch(searchType) {
                     case 'isrc':
-                        results = await this.searchByIsrc(play, stageConfig);
+                        results = await this.searchByIsrc(play, stageConfig, opts);
                         break;
                     case 'album':
-                        results = await this.searchByAlbum(play, stageConfig);
+                        results = await this.searchByAlbum(play, stageConfig, opts);
                         break;
                     case 'artist':
-                        results = await this.searchByArtist(play, stageConfig);
+                        results = await this.searchByArtist(play, stageConfig, opts);
                         break;
                     case 'basic':
-                        results = await this.searchByBasicFields(play, stageConfig);
+                        results = await this.searchByBasicFields(play, stageConfig, opts);
                         break;
                     case 'freetext':
-                        results = await this.searchByFreeText(play, stageConfig);
+                        results = await this.searchByFreeText(play, stageConfig, opts);
                         break;
                     case 'basicorids':
-                        results = await this.searchByBasicFieldsOrMBIDs(play, stageConfig);
+                        results = await this.searchByBasicFieldsOrMBIDs(play, stageConfig, opts);
                         break;
                     case 'mbidrecording':
-                        results = await this.searchByRecordingMbid(play, stageConfig);
+                        results = await this.searchByRecordingMbid(play, stageConfig, opts);
                         break;
                 }
                 queries.push({type: `mbQuery-${searchType}${results.recordings.length === 0 ? '-empty' : ''}`, input: results.requestQuery});
@@ -486,12 +486,12 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
         return {...results, requestQueries: queries};
     }
 
-    public async searchByBasicFields(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByBasicFields(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         this.logger.debug({labels: ['Basic Search']}, 'Searching by artist/album/track');
-        return await this.api.searchByRecording(play);
+        return await this.api.searchByRecording(play, opts);
     }
 
-    public async searchByBasicFieldsOrMBIDs(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByBasicFieldsOrMBIDs(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         const using: UsingTypes[] = [];
         const {
             data: {
@@ -512,31 +512,31 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
         using.push((brainz.artist ?? []).length > 0 ? 'mbidartist' : 'artist');
 
         this.logger.debug({labels: ['Basic Or MBID Search']}, `Searching using ${using.join(', ')}}`);
-        return await this.api.searchByRecording(play, {using});
+        return await this.api.searchByRecording(play, {using, ...opts});
     }
 
-    public async searchByIsrc(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByIsrc(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         if(play.data.isrc !== undefined) {
             this.logger.debug({labels: ['ISRC Search']},'Searching with ISRC');
-            return await this.api.searchByRecording(play, {using: ['isrc']});
+            return await this.api.searchByRecording(play, {using: ['isrc'], ...opts});
         }
         throw new SearchPrerequisiteError('Play does not have ISRC');
     }
 
-    public async searchByRecordingMbid(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByRecordingMbid(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         if(play.data.meta?.brainz?.recording !== undefined) {
             this.logger.debug({labels: ['MBID Search']},'Searching with Recording MBID');
-            return await this.api.searchByRecording(play, {using: ['mbidrecording']});
+            return await this.api.searchByRecording(play, {using: ['mbidrecording'], ...opts});
         }
         throw new SearchPrerequisiteError('Play does not have recording MBID');
     }
 
-    public async searchByAlbum(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByAlbum(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         // possibly the artist is incorrect (may be combined as one string)
         // if we have an album we can likely still get a decent hit w/o using artist
         if(play.data.album !== undefined && play.data.artists !== undefined && play.data.artists.length > 0) {
             this.logger.debug({labels: ['Album Search']},'Searching with only track+album');
-            return await this.api.searchByRecording(play, {using: ['title','album']});
+            return await this.api.searchByRecording(play, {using: ['title','album'], ...opts});
         }
         if(play.data.album === undefined) {
             throw new SearchPrerequisiteError('Play does not have an album');
@@ -549,7 +549,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
         }
     }
 
-    public async searchByArtist(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByArtist(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         const {
             searchArtistMethod = this.defaults.searchArtistMethod,
         } = stageConfig;
@@ -582,7 +582,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
                     // so we split out all artists by all found delimiters
                     const nativePlay = nativeParse(play, {titleClean: true, delimiters: DELIMITERS});
                     this.logger.debug({labels: ['Parsed Artist Search']},'Searching with aggressive native parsing');
-                    return await this.api.searchByRecording(nativePlay, {using: ['title','artist']});
+                    return await this.api.searchByRecording(nativePlay, {using: ['title','artist'], ...opts});
                 }
             }
 
@@ -594,9 +594,9 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer<Exter
             }
     }
 
-    public async searchByFreeText(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise<IRecordingMSList> {
+    public async searchByFreeText(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise<IRecordingMSList> {
         this.logger.debug({labels: ['Freetext Search']},'Trying freetext search');
-        const results = await this.api.searchByRecording(play, {freetext: true}) as IRecordingMSList;
+        const results = await this.api.searchByRecording(play, {freetext: true, ...opts}) as IRecordingMSList;
         results.freeText = true;
         return results;
     }

src/backend/common/transforms/TransformerManager.ts

@@ -1,6 +1,6 @@
 import { childLogger, type Logger } from "@foxxmd/logging";
 import type AbstractTransformer from "./AbstractTransformer.ts";
-import type {TransformerCommonConfig} from "../../../core/Atomic.ts";
+import type {OptionalCacheUsage, TransformerCommonConfig} from "../../../core/Atomic.ts";
 import UserTransformer from "./UserTransformer.ts";
 import type {StageConfig} from "../../../core/Transform.ts";
 import type {PlayObject} from "../../../core/Atomic.ts";
@@ -146,10 +146,10 @@ export default class TransformerManager {
         return config;
     }
 
-    public async handleStage(data: StageConfig, play: PlayObject, asyncId: string = nanoid(6)): Promise<[PlayObject, string]> {
+    public async handleStage(data: StageConfig, play: PlayObject, opts: {asyncId?: string} & OptionalCacheUsage): Promise<[PlayObject, string]> {
         const t: AbstractTransformer = this.getTransformerByStage(data);
         try {
-            const transformedPlay = await this.asyncStore.run(asyncId, async () => {
+            const transformedPlay = await this.asyncStore.run(opts.asyncId ?? nanoid(6), async () => {
                 return await t.handle(data, play);
             });
             return [transformedPlay, t.name];

src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts

@@ -1,5 +1,5 @@
 import type { Response } from 'superagent';
-import type {ArtistCredit, PlayObject, PlayObjectMinimal, URLData} from "../../../../core/Atomic.ts";
+import type {ArtistCredit, OptionalCacheUsage, PlayObject, PlayObjectMinimal, URLData} from "../../../../core/Atomic.ts";
 import { UpstreamError } from "../../errors/UpstreamError.ts";
 import { type AbstractApiOptions, type FormatPlayObjectOptions, MUSICBRAINZ_URL, type MusicbrainzApiConfigData } from "../../infrastructure/Atomic.ts";
 import AbstractApiClient from "../AbstractApiClient.ts";
@@ -114,17 +114,18 @@ export class MusicbrainzApiClient extends AbstractApiClient {
         return 'API';
     }
 
-    callApi = async <T = Response>(func: (mb: MusicBrainzApi) => Promise<any>, options?: { timeout?: number, cacheKey?: string }): Promise<T> => {
+    callApi = async <T = Response>(func: (mb: MusicBrainzApi) => Promise<any>, options?: { timeout?: number, cacheKey?: string } & OptionalCacheUsage): Promise<T> => {
 
         let apiConfig = this.rrApis.next().value;
 
         const {
             timeout = 30000,
-            cacheKey
+            cacheKey,
+            useCachedResult = true
         } = options || {};
 
         try {
-            const cachedTransform = await this.cache.get<T>(cacheKey);
+            const cachedTransform = useCachedResult ? await this.cache.get<T>(cacheKey) : undefined;
             if(cachedTransform !== undefined) {
                 const cacheUrl = await this.cache.get<string>(`${cacheKey}-url`);
                 const cacheQs = await this.cache.get<string>(`${cacheKey}-qs`);
@@ -218,13 +219,14 @@ export class MusicbrainzApiClient extends AbstractApiClient {
         }
     }
 
-    searchByRecording = async(play: PlayObject, options?: SearchOptions): Promise<IRecordingMSList | undefined> => {
+    searchByRecording = async(play: PlayObject, options?: SearchOptions & OptionalCacheUsage): Promise<IRecordingMSList | undefined> => {
 
         const {
             escapeCharacters = true,
             removeCharacters = false,
             using = ['album','artist','title'],
-            freetext
+            freetext,
+            useCachedResult
         } = options || {};
 
         const cacheKey = `mb-recSearch-${hashObject({...playContentInvariantTransform(play), using})}`;
@@ -347,7 +349,8 @@ export class MusicbrainzApiClient extends AbstractApiClient {
                 query: q
             });
         }, {
-            cacheKey
+            cacheKey,
+            useCachedResult
         });
 
         if(res === undefined) {

src/backend/tests/component/transformers.test.ts

@@ -700,14 +700,14 @@ describe('Play Transforms', function () {
 
             component.buildTransformRules();
             const play = generatePlay({ track: 'My cool something track', artists: artistNamesToCredits([str]), playDate: dayjs().subtract(10, 'm') });
-            const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare, 'all');
+            const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare, {log: 'all'});
             expect(transformed.data.track).equal('My cool bar track');
             expect(artistCreditsToNames(transformed.data.artists)).eql(primaries.concat(secondaries));
 
             const cachablePlay = clone(play);
             const laterDate = dayjs().subtract(5, 'm');
             cachablePlay.data.playDate = laterDate;
-            const cacheTransformed = await component.transformPlay(cachablePlay, TRANSFORM_HOOK.preCompare, 'all');
+            const cacheTransformed = await component.transformPlay(cachablePlay, TRANSFORM_HOOK.preCompare, {log: 'all'});
             expect(cacheTransformed.data.track).equal('My cool bar track');
             expect(artistCreditsToNames(cacheTransformed.data.artists)).eql(primaries.concat(secondaries));
             expect(cacheTransformed.data.playDate.isSame(cachablePlay.data.playDate));

src/core/Atomic.ts

@@ -806,3 +806,6 @@ export const NO_DEVICE = 'NoDevice';export const NO_USER = 'SingleUser';
 export const SINGLE_USER_PLATFORM_ID: PlayPlatformId = [NO_DEVICE, NO_USER];
 export const SINGLE_USER_PLATFORM_ID_STR = `${NO_DEVICE}-${NO_USER}`;
 
+export interface OptionalCacheUsage {
+    useCachedResult?: boolean
+}