mirror of
https://github.com/FoxxMD/multi-scrobbler.git
synced 2026-09-03 05:10:00 +03:00
fix: Refactor play tranforming to dogfood steps and fix caching #495
* Re-use lifecycle steps/patch to build diffs * Break out step generation into own function to simplify logic * Cache steps instead of play to avoid returning a cached play when data is the same #495 * Replace patch object/patching logic with json-diff-ts as its more robust and actually maintained * Reduces need to stringify everywhere due to lack of cloning in jsonpatchdiff
This commit is contained in:
FoxxMD
Generated
+7
-84
@@ -72,7 +72,7 @@
|
||||
"icecast-metadata-stats": "^0.1.12",
|
||||
"iso-websocket": "^0.3.0",
|
||||
"iti": "^0.7.0",
|
||||
"jest-diff": "^30.2.0",
|
||||
"json-diff-ts": "^5.0.0-alpha.2",
|
||||
"json5": "^2.2.3",
|
||||
"jsondiffpatch": "^0.7.3",
|
||||
"jsondiffpatch-react": "^1.0.8",
|
||||
@@ -2568,36 +2568,6 @@
|
||||
"axios": "^1.3.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@jest/diff-sequences": {
|
||||
"version": "30.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.0.1.tgz",
|
||||
"integrity": "sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@jest/get-type": {
|
||||
"version": "30.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz",
|
||||
"integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@jest/schemas": {
|
||||
"version": "30.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.5.tgz",
|
||||
"integrity": "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@sinclair/typebox": "^0.34.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@joshwooding/vite-plugin-react-docgen-typescript": {
|
||||
"version": "0.6.4",
|
||||
"resolved": "https://registry.npmjs.org/@joshwooding/vite-plugin-react-docgen-typescript/-/vite-plugin-react-docgen-typescript-0.6.4.tgz",
|
||||
@@ -3928,12 +3898,6 @@
|
||||
"integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@sinclair/typebox": {
|
||||
"version": "0.34.41",
|
||||
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.41.tgz",
|
||||
"integrity": "sha512-6gS8pZzSXdyRHTIqoqSVknxolr1kzfy4/CeDnrzsVz8TTIWUbOBr6gnzOmTYJ3eXQNh4IYHIGi5aIL7sOZ2G/g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@sindresorhus/is": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.1.tgz",
|
||||
@@ -13525,53 +13489,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/jest-diff": {
|
||||
"version": "30.2.0",
|
||||
"resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.2.0.tgz",
|
||||
"integrity": "sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@jest/diff-sequences": "30.0.1",
|
||||
"@jest/get-type": "30.1.0",
|
||||
"chalk": "^4.1.2",
|
||||
"pretty-format": "30.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/jest-diff/node_modules/ansi-styles": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz",
|
||||
"integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/jest-diff/node_modules/pretty-format": {
|
||||
"version": "30.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz",
|
||||
"integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@jest/schemas": "30.0.5",
|
||||
"ansi-styles": "^5.2.0",
|
||||
"react-is": "^18.3.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/jest-diff/node_modules/react-is": {
|
||||
"version": "18.3.1",
|
||||
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
|
||||
"integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/jest-get-type": {
|
||||
"version": "27.5.1",
|
||||
"resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-27.5.1.tgz",
|
||||
@@ -13684,6 +13601,12 @@
|
||||
"integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/json-diff-ts": {
|
||||
"version": "5.0.0-alpha.2",
|
||||
"resolved": "https://registry.npmjs.org/json-diff-ts/-/json-diff-ts-5.0.0-alpha.2.tgz",
|
||||
"integrity": "sha512-7AI6rZNrvqW3Bkmm9Wsq6985MeC8KkK68mgtW7z3aq39QBGMcoyClrbAXlUr6yQ0VbafVtAM+f5b4bAwlxVSAg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/json-parse-even-better-errors": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
|
||||
|
||||
+1
-1
@@ -110,7 +110,7 @@
|
||||
"icecast-metadata-stats": "^0.1.12",
|
||||
"iso-websocket": "^0.3.0",
|
||||
"iti": "^0.7.0",
|
||||
"jest-diff": "^30.2.0",
|
||||
"json-diff-ts": "^5.0.0-alpha.2",
|
||||
"json5": "^2.2.3",
|
||||
"jsondiffpatch": "^0.7.3",
|
||||
"jsondiffpatch-react": "^1.0.8",
|
||||
|
||||
@@ -10,28 +10,24 @@ import { CommonClientConfig } from "./infrastructure/config/client/index.js";
|
||||
import { CommonSourceConfig } from "./infrastructure/config/source/index.js";
|
||||
import { mergeSimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js";
|
||||
import {
|
||||
FLOW_CONTROL_TERM,
|
||||
PlayTransformRules,
|
||||
StageConfig,
|
||||
TRANSFORM_HOOK,
|
||||
TransformHook
|
||||
} from "./infrastructure/Transform.js";
|
||||
import AbstractInitializable from "./AbstractInitializable.js";
|
||||
import play = Simulate.play;
|
||||
import TransformerManager from "./transforms/TransformerManager.js";
|
||||
import { getRoot } from "../ioc.js";
|
||||
import { nanoid } from "nanoid";
|
||||
import {diffStringsUnified, DiffOptionsColor} from 'jest-diff';
|
||||
import chalk from 'chalk';
|
||||
import { isDebugMode } from "../utils.js";
|
||||
import { findCauseByReference } from "../utils/ErrorUtils.js";
|
||||
import { hashObject } from "../utils/StringUtils.js";
|
||||
import { metaInvariantTransform, playContentInvariantTransform } from "../utils/PlayComparisonUtils.js";
|
||||
import { MSCache } from "./Cache.js";
|
||||
import { jdiff } from "../../core/DataUtils.js";
|
||||
import ConsoleFormatter from "jsondiffpatch/formatters/console";
|
||||
import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/DataUtils.js";
|
||||
import clone from "clone";
|
||||
|
||||
const console = new ConsoleFormatter();
|
||||
import { loggerNoop } from "./MaybeLogger.js";
|
||||
|
||||
export type AbstractComponentConfig = (CommonClientConfig | CommonSourceConfig) & { transformManager?: TransformerManager };
|
||||
|
||||
@@ -169,7 +165,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
|
||||
return partArr.map(x => this.transformManager.parseTransformerConfig(x));
|
||||
}
|
||||
|
||||
public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean) => {
|
||||
public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean | 'all') => {
|
||||
|
||||
|
||||
const asyncId = nanoid(6);
|
||||
@@ -200,177 +196,123 @@ export default abstract class AbstractComponent extends AbstractInitializable {
|
||||
const shouldLog = log ?? this.config.options?.playTransform?.log ?? isDebugMode();
|
||||
|
||||
const transformHash = `playTransform-${hashObject(hook)}-${hashObject(playContentInvariantTransform(play))}`;
|
||||
const cachedTransformPlay = await this.cache.cacheTransform.get<PlayObject>(transformHash);
|
||||
if(cachedTransformPlay !== undefined) {
|
||||
logger.trace(`Cache hit ${transformHash}`);
|
||||
return cachedTransformPlay;
|
||||
const cachedSteps = await this.cache.cacheTransform.get<LifecycleStep[]>(transformHash);
|
||||
if(cachedSteps !== undefined) {
|
||||
logger.trace(`Cache hit for Steps => ${transformHash}`);
|
||||
//return cachedTransformPlay;
|
||||
}
|
||||
|
||||
logger.debug(`Transform start for => ${buildTrackString(play)}`);
|
||||
let transformedPlay: PlayObject = play;
|
||||
let transformHistory: TransformResult[] = [];
|
||||
let shouldBreak = false;
|
||||
let transformedPlay: PlayObject = clone(play);
|
||||
let cacheOk: boolean = true;
|
||||
for(const hookItem of hook) {
|
||||
const steps: LifecycleStep[] = [];
|
||||
|
||||
const {
|
||||
onSuccess = 'continue',
|
||||
onFailure = 'stop',
|
||||
onSkip = 'continue',
|
||||
failureReturnPartial = false
|
||||
} = hookItem;
|
||||
const opts = {
|
||||
logger,
|
||||
asyncId
|
||||
}
|
||||
|
||||
const existingStepIndex = transformedPlay.meta.lifecycle.steps.findIndex(x => x.name === 'hookType' && x.source === this.getIdentifier());
|
||||
const step: LifecycleStep = existingStepIndex !== -1 ? transformedPlay.meta.lifecycle.steps[existingStepIndex] : {
|
||||
name: hookType,
|
||||
source: this.getIdentifier(),
|
||||
if(cachedSteps !== undefined) {
|
||||
// don't re-cache cached steps
|
||||
cacheOk = false;
|
||||
// only patch play if steps didn't end in a failure that we are okay with returning partial from
|
||||
let shouldTransform = true;
|
||||
const lastCachedStep = cachedSteps[cachedSteps.length - 1];
|
||||
if(lastCachedStep.error !== undefined && lastCachedStep.flowKnownState !== 'skip' && !lastCachedStep.returnPartial) {
|
||||
shouldTransform = false;
|
||||
}
|
||||
|
||||
let newTransformedPlay: PlayObject,
|
||||
stageName: string = 'Unnamed',
|
||||
err: Error;
|
||||
try {
|
||||
[newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, transformedPlay, asyncId);
|
||||
} catch (e) {
|
||||
err = e;
|
||||
if(e instanceof StageTransformError) {
|
||||
stageName = e.stageName;
|
||||
for(const s of cachedSteps) {
|
||||
steps.push({...s, cached: true});
|
||||
if(shouldTransform && s.patch !== undefined) {
|
||||
transformedPlay.data = patchObject(transformedPlay.data, s.patch); // jdiff.patch(clone(transformedPlay.data),s.patch);
|
||||
}
|
||||
}
|
||||
|
||||
if(err !== undefined) {
|
||||
const merged = mergeSimpleError(err);
|
||||
step.error = merged;
|
||||
|
||||
const skipError = findCauseByReference(err, SkipTransformStageError);
|
||||
if(skipError !== undefined) {
|
||||
let skipMsg = `Stage '${stageName}' was skipped`;
|
||||
step.flowResult = onSkip;
|
||||
step.flowKnownState = 'skip';
|
||||
|
||||
if(onSkip === 'stop') {
|
||||
skipMsg += ' and will stop transform due to onSkip: stop';
|
||||
}
|
||||
step.flowReason = skipMsg;
|
||||
|
||||
logger.debug(merged, skipMsg);
|
||||
if(onSkip === 'stop') {
|
||||
shouldBreak = true;
|
||||
} else {
|
||||
for(const hookItem of hook) {
|
||||
const [step, stepPlay] = await this.generateStepFromStage(transformedPlay, hookItem, hookType, opts);
|
||||
steps.push(step);
|
||||
cacheOk = step.error === undefined || (step.error !== undefined && step.flowKnownState !== undefined);
|
||||
if(step.flowResult === FLOW_CONTROL_TERM.stop) {
|
||||
if(step.error !== undefined && step.flowKnownState !== 'skip' && !step.returnPartial) {
|
||||
// revert to original play but keep steps for paper trail
|
||||
transformedPlay = play;
|
||||
transformedPlay.meta.lifecycle.steps = steps;
|
||||
} else {
|
||||
transformedPlay = stepPlay;
|
||||
}
|
||||
break;
|
||||
} else {
|
||||
step.flowResult = onFailure;
|
||||
let reason: string;
|
||||
|
||||
const reqError = findCauseByReference(err, StagePrerequisiteError);
|
||||
if(reqError !== undefined) {
|
||||
reason = 'Transform could not be completed due to prerequisite failure';
|
||||
step.flowKnownState = 'prereq';
|
||||
} else {
|
||||
reason = 'Transform encountered an error';
|
||||
cacheOk = false;
|
||||
}
|
||||
|
||||
if(onFailure === 'continue') {
|
||||
reason += ' but will continue due to onFailure: continue';
|
||||
logger.warn(merged, reason);
|
||||
} else {
|
||||
logger[reqError !== undefined ? 'warn' : 'error'](merged, reason);
|
||||
|
||||
if(!failureReturnPartial) {
|
||||
// rewind to original play so we don't return partial transform
|
||||
transformedPlay = play;
|
||||
transformHistory = [];
|
||||
} else if(transformHistory.length > 0) {
|
||||
cacheOk = true;
|
||||
step.flowReason += ' | Preserving play transformations up to this point due to failureReturnPartial=true'
|
||||
}
|
||||
shouldBreak = true;
|
||||
}
|
||||
|
||||
step.flowReason = reason;
|
||||
transformedPlay = stepPlay;
|
||||
}
|
||||
} else {
|
||||
// reset cache state
|
||||
// in the event it was false due to previous step error but onFailure: continue
|
||||
cacheOk = true;
|
||||
step.flowResult = onSuccess;
|
||||
|
||||
if(!deepEqual(play, newTransformedPlay)) {
|
||||
const o = JSON.parse(JSON.stringify(play));
|
||||
const t = JSON.parse(JSON.stringify(newTransformedPlay));
|
||||
const patch = jdiff.diff(o,t);
|
||||
step.patch = patch;
|
||||
}
|
||||
|
||||
if(newTransformedPlay.meta.lifecycleInputs?.length > 0) {
|
||||
step.inputs = clone(newTransformedPlay.meta.lifecycleInputs)
|
||||
} else if(transformedPlay.meta.lifecycleInputs?.length > 0) {
|
||||
step.inputs = clone(transformedPlay.meta.lifecycleInputs)
|
||||
}
|
||||
|
||||
delete transformedPlay.meta.lifecycleInputs;
|
||||
|
||||
// for logging only
|
||||
transformHistory.push({
|
||||
type: hookItem.type,
|
||||
name: stageName,
|
||||
play: newTransformedPlay.data
|
||||
});
|
||||
transformedPlay = newTransformedPlay;
|
||||
}
|
||||
|
||||
if(err === undefined && onSuccess === 'stop') {
|
||||
logger.debug(`Stopping transform due to onSuccess: stop`);
|
||||
shouldBreak = true;
|
||||
}
|
||||
|
||||
if(existingStepIndex !== -1) {
|
||||
transformedPlay.meta.lifecycle.steps[existingStepIndex] = step;
|
||||
} else {
|
||||
transformedPlay.meta.lifecycle.steps.push(step);
|
||||
}
|
||||
|
||||
if(shouldBreak) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const historyToDiff: {name: string, data?: PlayData}[] = [
|
||||
{name: 'Pre Transform', data: play.data}
|
||||
];
|
||||
if(steps.length > 0 && transformedPlay.meta?.lifecycle?.steps === undefined) {
|
||||
const {
|
||||
meta: {
|
||||
lifecycle: {
|
||||
steps = [],
|
||||
...lifecycleRest
|
||||
},
|
||||
} = {}
|
||||
} = transformedPlay;
|
||||
transformedPlay.meta.lifecycle = {
|
||||
...lifecycleRest,
|
||||
steps
|
||||
}
|
||||
}
|
||||
steps.forEach((s, index) => {
|
||||
const existingStepIndex = transformedPlay.meta.lifecycle.steps.findIndex(x => x.name === 'hookType' && x.source === this.getIdentifier());
|
||||
if(existingStepIndex !== -1) {
|
||||
transformedPlay.meta.lifecycle.steps[existingStepIndex] = s;
|
||||
} else {
|
||||
transformedPlay.meta.lifecycle.steps.push(s);
|
||||
}
|
||||
|
||||
if(shouldLog === 'all') {
|
||||
if(s.patch === undefined) {
|
||||
historyToDiff.push({name: `${s.source}-${s.name}`});
|
||||
} else {
|
||||
const patched = patchObject(historyToDiff[historyToDiff.length - 1].data, s.patch);
|
||||
historyToDiff.push({name: `${s.source}-${s.name}${s.cached ? ' (Cached)' : ''}`, data: patched});
|
||||
}
|
||||
}
|
||||
});
|
||||
if(shouldLog !== false) {
|
||||
if(transformHistory.length === 0) {
|
||||
if(steps.filter(x => x.patch !== undefined).length === 0) {
|
||||
logger.debug('Transform Diff: No Change');
|
||||
} else {
|
||||
const historyToDiff: {name: string, data: PlayData}[] = [
|
||||
{name: 'Original', data: play.data}
|
||||
];
|
||||
if(shouldLog === true) {
|
||||
const last = transformHistory[transformHistory.length - 1];
|
||||
historyToDiff.push({name: `${last.type}-${last.name}`, data: last.play});
|
||||
} else {
|
||||
for(const t of transformHistory) {
|
||||
historyToDiff.push({name: `${t.type}-${t.name}`, data: t.play})
|
||||
}
|
||||
}
|
||||
const diffs: string[] = [];
|
||||
|
||||
historyToDiff.forEach((curr, index) => {
|
||||
if(index === 0) {
|
||||
return;
|
||||
}
|
||||
const last = historyToDiff[index - 1];
|
||||
if(deepEqual(last.data, curr.data)) {
|
||||
if(curr.data === undefined) {
|
||||
diffs.push(`${last.name} => ${curr.name} -- No Change`);
|
||||
} else {
|
||||
const unchanged = index - 1 === 0 ? last.data : undefined;
|
||||
|
||||
const formattedDiff = console.format(jdiff.diff(last.data, curr.data), unchanged);
|
||||
const formattedDiff = diffObjectsConsoleOutput(last.data, curr.data);
|
||||
diffs.push(`${last.name} => ${curr.name}\n${formattedDiff}`);
|
||||
}
|
||||
});
|
||||
|
||||
if(shouldLog === true || steps.filter(x => x.patch !== undefined).length > 2) {
|
||||
const finalData = transformedPlay.data;
|
||||
const formattedDiff = diffObjectsConsoleOutput(play.data, finalData, true);
|
||||
diffs.push(`Original => Final\n${formattedDiff}`);
|
||||
}
|
||||
|
||||
logger.debug(`Transform Diff\n${diffs.join('\n')}`);
|
||||
}
|
||||
}
|
||||
|
||||
if(cacheOk) {
|
||||
await this.cache.cacheTransform.set<PlayObject>(transformHash, transformedPlay, '10m');
|
||||
await this.cache.cacheTransform.set<LifecycleStep[]>(transformHash, steps, '10m');
|
||||
}
|
||||
|
||||
return transformedPlay;
|
||||
@@ -379,4 +321,103 @@ export default abstract class AbstractComponent extends AbstractInitializable {
|
||||
return play;
|
||||
}
|
||||
}
|
||||
|
||||
protected generateStepFromStage = async (playTruth: PlayObject, hookItem: StageConfig, hookType: TransformHook, opts: { logger?: Logger, asyncId?: string } = {}): Promise<[LifecycleStep, PlayObject]> => {
|
||||
const {
|
||||
onSuccess = 'continue',
|
||||
onFailure = 'stop',
|
||||
onSkip = 'continue',
|
||||
failureReturnPartial = false
|
||||
} = hookItem;
|
||||
|
||||
const {
|
||||
logger = loggerNoop,
|
||||
asyncId = nanoid(6)
|
||||
} = opts;
|
||||
|
||||
const stepName = `${hookType} - ${hookItem.type} - ${hookItem.name}`
|
||||
const existingStepIndex = playTruth.meta.lifecycle.steps.findIndex(x => x.name === stepName && x.source === this.getIdentifier());
|
||||
const step: LifecycleStep = existingStepIndex !== -1 ? playTruth.meta.lifecycle.steps[existingStepIndex] : {
|
||||
name: stepName,
|
||||
source: this.getIdentifier(),
|
||||
}
|
||||
|
||||
let newTransformedPlay: PlayObject,
|
||||
stageName: string = 'Unnamed',
|
||||
err: Error;
|
||||
try {
|
||||
[newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, playTruth, asyncId);
|
||||
newTransformedPlay = clone(newTransformedPlay);
|
||||
} catch (e) {
|
||||
err = e;
|
||||
if (e instanceof StageTransformError) {
|
||||
stageName = e.stageName;
|
||||
}
|
||||
}
|
||||
|
||||
if (err !== undefined) {
|
||||
const merged = mergeSimpleError(err);
|
||||
step.error = merged;
|
||||
|
||||
const skipError = findCauseByReference(err, SkipTransformStageError);
|
||||
if (skipError !== undefined) {
|
||||
let skipMsg = `Stage '${stageName}' was skipped`;
|
||||
step.flowResult = onSkip;
|
||||
step.flowKnownState = 'skip';
|
||||
|
||||
if (onSkip === 'stop') {
|
||||
skipMsg += ' and will stop transform due to onSkip: stop';
|
||||
}
|
||||
step.flowReason = skipMsg;
|
||||
|
||||
logger.debug(merged, skipMsg);
|
||||
} else {
|
||||
step.flowResult = onFailure;
|
||||
let reason: string;
|
||||
|
||||
const reqError = findCauseByReference(err, StagePrerequisiteError);
|
||||
if (reqError !== undefined) {
|
||||
reason = 'Transform could not be completed due to prerequisite failure';
|
||||
step.flowKnownState = 'prereq';
|
||||
} else {
|
||||
reason = 'Transform encountered an error';
|
||||
}
|
||||
|
||||
if (onFailure === 'continue') {
|
||||
reason += ' but will continue due to onFailure: continue';
|
||||
logger.warn(merged, reason);
|
||||
} else {
|
||||
if (failureReturnPartial) {
|
||||
reason += ' | Preserving play transformations up to this point due to failureReturnPartial=true';
|
||||
step.returnPartial = true;
|
||||
}
|
||||
|
||||
logger[reqError !== undefined ? 'warn' : 'error'](merged, reason);
|
||||
}
|
||||
step.flowReason = reason;
|
||||
|
||||
return [step, playTruth];
|
||||
}
|
||||
} else {
|
||||
step.flowResult = onSuccess;
|
||||
|
||||
if (!deepEqual(playTruth.data, newTransformedPlay.data)) {
|
||||
const o = JSON.parse(JSON.stringify(playTruth.data));
|
||||
const t = JSON.parse(JSON.stringify(newTransformedPlay.data));
|
||||
const patch = diffObjects(o, t); // jdiff.diff(o, t);
|
||||
step.patch = patch;
|
||||
}
|
||||
|
||||
if (newTransformedPlay.meta.lifecycleInputs?.length > 0) {
|
||||
step.inputs = clone(newTransformedPlay.meta.lifecycleInputs)
|
||||
} else if (playTruth.meta.lifecycleInputs?.length > 0) {
|
||||
logger.warn({ label: `${hookItem.type} - ${hookItem.name}` }, `Should only be adding inputs to transformed play!`);
|
||||
step.inputs = clone(playTruth.meta.lifecycleInputs)
|
||||
}
|
||||
|
||||
delete newTransformedPlay.meta.lifecycleInputs;
|
||||
}
|
||||
|
||||
return [step, newTransformedPlay];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +50,10 @@ export interface Whennable {
|
||||
}
|
||||
|
||||
export type FlowControlTerm = 'continue' | 'stop'
|
||||
export const FLOW_CONTROL_TERM = {
|
||||
continue: 'continue',
|
||||
stop: 'stop'
|
||||
} as const satisfies Record<string, FlowControlTerm>;
|
||||
|
||||
export interface FlowControl {
|
||||
onSuccess: FlowControlTerm
|
||||
|
||||
@@ -16,6 +16,8 @@ import { Cacheable } from "cacheable";
|
||||
import { TransformerCommonConfig } from "../../../core/Atomic.js";
|
||||
import TransformerManager from "../../common/transforms/TransformerManager.js";
|
||||
import { transientCache } from "../utils/CacheTestUtils.js";
|
||||
import dayjs from "dayjs";
|
||||
import clone from "clone";
|
||||
|
||||
chai.use(asPromised);
|
||||
|
||||
@@ -638,6 +640,49 @@ describe('Play Transforms', function () {
|
||||
|
||||
});
|
||||
|
||||
describe("Stage Caching", function () {
|
||||
|
||||
it('Re-uses steps without modifying other Play properties', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: [
|
||||
{
|
||||
name: "barChange",
|
||||
title: [
|
||||
{
|
||||
search: "something",
|
||||
replace: "bar"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
type: 'native'
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}});
|
||||
|
||||
component.buildTransformRules();
|
||||
const play = generatePlay({ track: 'My cool something track', artists: [str], playDate: dayjs().subtract(10, 'm') });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare, 'all');
|
||||
expect(transformed.data.track).equal('My cool bar track');
|
||||
expect(transformed.data.artists).eql(primaries.concat(secondaries));
|
||||
|
||||
const cachablePlay = clone(play);
|
||||
const laterDate = dayjs().subtract(5, 'm');
|
||||
cachablePlay.data.playDate = laterDate;
|
||||
const cacheTransformed = await component.transformPlay(cachablePlay, TRANSFORM_HOOK.preCompare, 'all');
|
||||
expect(cacheTransformed.data.track).equal('My cool bar track');
|
||||
expect(cacheTransformed.data.artists).eql(primaries.concat(secondaries));
|
||||
expect(cacheTransformed.data.playDate.isSame(cachablePlay.data.playDate));
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
describe('Transform Manager', function() {
|
||||
|
||||
it('Uses user transforms in the order supplied within component', async function() {
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import React, { ComponentProps } from 'react';
|
||||
import JsonDiffReact from 'jsondiffpatch-react';
|
||||
import { MSErrorBoundary } from './ErrorBoundary';
|
||||
import { Delta } from 'jsondiffpatch';
|
||||
import { MarkOptional } from 'ts-essentials';
|
||||
import { jdiff } from '../../core/DataUtils';
|
||||
import { patchObject } from '../../core/DataUtils';
|
||||
import './JsonDiff.css';
|
||||
import { IJsonDelta } from 'json-diff-ts';
|
||||
|
||||
type DiffableVal = [] | object | string;
|
||||
|
||||
@@ -18,7 +18,7 @@ export interface JsonDiffReactProps {
|
||||
}
|
||||
|
||||
export type JsonDiffPatchProps = MarkOptional<JsonDiffReactProps, 'right'> & {
|
||||
diff?: Delta | object
|
||||
diff?: IJsonDelta
|
||||
}
|
||||
|
||||
export const JsonDiffPatch = (props: JsonDiffPatchProps) => {
|
||||
@@ -31,9 +31,9 @@ export const JsonDiffPatch = (props: JsonDiffPatchProps) => {
|
||||
const detachedLeft = JSON.parse(JSON.stringify(left));
|
||||
let realRight: DiffableVal;
|
||||
if (right !== undefined) {
|
||||
realRight = JSON.parse(JSON.stringify(right));
|
||||
realRight = structuredClone(right);
|
||||
} else if (diff !== undefined) {
|
||||
realRight = jdiff.patch(JSON.parse(JSON.stringify(left)), diff as Delta) as DiffableVal;
|
||||
realRight = patchObject(left, diff);
|
||||
} else {
|
||||
throw new Error(`must provide either 'right' or 'diff'`);
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import { FaStop } from "react-icons/fa";
|
||||
import { MdMusicNote } from "react-icons/md";
|
||||
import { ChakraCodeBlockShort, ChakraPlainBlockShort } from "./CodeBlock";
|
||||
import { JsonDiffPatch } from "./JsonDiff";
|
||||
import { jdiff } from "../../core/DataUtils";
|
||||
import { jdiff, patchObject } from "../../core/DataUtils";
|
||||
import { MSCollapsible, MSCollapsibleExternalProps } from "./MSCollapsible";
|
||||
import { Muted } from "./Typography";
|
||||
|
||||
@@ -23,7 +23,7 @@ export interface LifeycleStepsTimelineProps extends MSCollapsibleExternalProps {
|
||||
|
||||
const diffElements = (original: JsonPlayObject, steps: LifecycleStep[]): [JSX.Element[], JsonPlayObject?] => {
|
||||
|
||||
let currentPlay: JsonPlayObject = JSON.parse(JSON.stringify(original));
|
||||
let currentPlay: JsonPlayObject = structuredClone(original); // JSON.parse(JSON.stringify(original));
|
||||
let patchFailed = false;
|
||||
|
||||
const diffElements: JSX.Element[] | null = [];
|
||||
@@ -49,7 +49,7 @@ const diffElements = (original: JsonPlayObject, steps: LifecycleStep[]): [JSX.El
|
||||
diffElements.push(<ChakraCodeBlockShort key={`diffblockfallback-${index}`} title="Diff Patch" code={patch} />);
|
||||
continue;
|
||||
}
|
||||
let left: JsonPlayObject = JSON.parse(JSON.stringify(currentPlay));
|
||||
let left: JsonPlayObject = structuredClone(currentPlay); // JSON.parse(JSON.stringify(currentPlay));
|
||||
left.data.meta = {
|
||||
...(left.data.meta ?? {}),
|
||||
brainz: {
|
||||
@@ -64,10 +64,10 @@ const diffElements = (original: JsonPlayObject, steps: LifecycleStep[]): [JSX.El
|
||||
}
|
||||
|
||||
try {
|
||||
currentPlay = jdiff.patch(currentPlay, patch) as JsonPlayObject;
|
||||
currentPlay.data = patchObject(currentPlay.data, patch)// jdiff.patch(currentPlay, patch) as JsonPlayObject;
|
||||
diffElements.push(
|
||||
<ChakraPlainBlockShort title="Play Diff" key={`diffblock-${index}`} code={left}>
|
||||
<JsonDiffPatch key={`diff-${index}`} left={left} right={JSON.parse(JSON.stringify(currentPlay))} />
|
||||
<ChakraPlainBlockShort title="Play Diff" key={`diffblock-${index}`} code={left.data}>
|
||||
<JsonDiffPatch key={`diff-${index}`} left={left.data} right={currentPlay.data} />
|
||||
</ChakraPlainBlockShort>
|
||||
)
|
||||
} catch (e) {
|
||||
|
||||
+3
-1
@@ -7,6 +7,7 @@ import { MarkOptional } from "ts-essentials";
|
||||
import { ErrorObject } from "serialize-error";
|
||||
import { PlayPlatformIdStr } from "../backend/common/infrastructure/Atomic.js";
|
||||
import { FlowControlTerm } from "../backend/common/infrastructure/Transform.js";
|
||||
import { IJsonDelta } from "json-diff-ts";
|
||||
|
||||
export interface SourceStatusData {
|
||||
status: string;
|
||||
@@ -311,11 +312,12 @@ export interface LifecycleStep {
|
||||
name: string
|
||||
source: string
|
||||
cached?: boolean
|
||||
returnPartial?: boolean
|
||||
flowResult?: FlowControlTerm
|
||||
flowReason?: string
|
||||
flowKnownState?: 'skip' | 'prereq'
|
||||
error?: ErrorLike
|
||||
patch?: Delta
|
||||
patch?: IJsonDelta
|
||||
inputs?: LifecycleInput[]
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,14 @@
|
||||
import { create as diffCreate } from "jsondiffpatch";
|
||||
import { numberFormatOptions } from './Atomic.js';
|
||||
import { diffDelta, applyDelta, IJsonDelta, DeltaOptions } from 'json-diff-ts';
|
||||
// may want to return to this one day
|
||||
// but currently the jsondiffpatch formatter is the best console/ansi diff output for humans :(
|
||||
//import {DiffOptions, DiffOptionsColor, diff as jestDiff} from 'jest-diff';
|
||||
import chalk from 'chalk';
|
||||
import clone from "clone";
|
||||
import ConsoleFormatter from "jsondiffpatch/formatters/console";
|
||||
|
||||
const console = new ConsoleFormatter();
|
||||
|
||||
export const jdiff = diffCreate({
|
||||
propertyFilter(name, context) {
|
||||
@@ -10,6 +18,34 @@ export const jdiff = diffCreate({
|
||||
//omitRemovedValues: true
|
||||
});
|
||||
|
||||
const diffOptions: DeltaOptions = {
|
||||
/*arrayIdentityKeys: {artists: '$value'},*/
|
||||
reversible: false,
|
||||
keysToSkip: ['playDate','playDateCompleted','listenRanges']
|
||||
};
|
||||
|
||||
export const diffObjects = (a: object, b: object) => {
|
||||
return diffDelta(a, b, diffOptions);
|
||||
}
|
||||
|
||||
export const patchObject = <T>(a: T, b: IJsonDelta): T => {
|
||||
return applyDelta(clone(a), b);
|
||||
}
|
||||
|
||||
// const jestDiffOptions: DiffOptions = {
|
||||
// aAnnotation: 'Old',
|
||||
// bAnnotation: 'New',
|
||||
// aColor: chalk.red,
|
||||
// bColor: chalk.green
|
||||
// }
|
||||
|
||||
export const diffObjectsConsoleOutput = (a: object, b: object, showUnchanged: boolean = false) => {
|
||||
//return jestDiff(a, b, jestDiffOptions);
|
||||
|
||||
const left = JSON.parse(JSON.stringify(a));
|
||||
return console.format(jdiff.diff(left, JSON.parse(JSON.stringify(b))), showUnchanged ? left : undefined);
|
||||
}
|
||||
|
||||
export const formatNumber = (val: number | string, options?: numberFormatOptions) => {
|
||||
const {
|
||||
toFixed = 2, defaultVal = null, prefix = '', suffix = '', round,
|
||||
|
||||
@@ -8,7 +8,7 @@ import { MarkOptional } from 'ts-essentials';
|
||||
import { generateBrainz, generateMbid, generatePlay, GeneratePlayOpts, generatePlays } from '../../PlayTestUtils.js';
|
||||
import { lifecyclelessInvariantTransform } from '../../PlayUtils.js';
|
||||
import clone from 'clone';
|
||||
import { jdiff } from '../../DataUtils.js';
|
||||
import { diffObjects } from '../../DataUtils.js';
|
||||
import { existingScrobble } from '../../../backend/utils/PlayComparisonUtils.js';
|
||||
import { UpstreamError } from '../../../backend/common/errors/UpstreamError.js';
|
||||
import { playToListenPayload } from '../../../backend/common/vendor/listenbrainz/lzUtils.js';
|
||||
@@ -312,7 +312,7 @@ export const generateLifecycleStep = (play: PlayObject, opts: GenerateLifecycleO
|
||||
});
|
||||
}
|
||||
|
||||
step.patch = jdiff.diff(play, modifiedPlay);
|
||||
step.patch = diffObjects(play.data, modifiedPlay.data);// jdiff.diff(play, modifiedPlay);
|
||||
|
||||
return [step, modifiedPlay];
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import {Provider} from "../client/components/Provider";
|
||||
import { generateJsonPlays } from "../core/PlayTestUtils.js";
|
||||
import { ErrorLike, JsonPlayObject, PlayLifecycle } from "../core/Atomic.js";
|
||||
import { examplePlay, lastfmErrorExample } from "./storyUtils.js";
|
||||
import { generatePlayWithLifecycle, playWithLifecycleScrobble } from "../core/tests/utils/fixtures.js";
|
||||
import { asJsonPlayObject, generatePlayWithLifecycle, playWithLifecycleScrobble } from "../core/tests/utils/fixtures.js";
|
||||
|
||||
// More on how to set up stories at: https://storybook.js.org/docs/writing-stories#default-export
|
||||
const meta = preview.meta({
|
||||
@@ -35,14 +35,14 @@ decorators: [
|
||||
export const ActivityTimelineStory = meta.story({
|
||||
loaders: [
|
||||
async () => {
|
||||
const scrobbleError = await playWithLifecycleScrobble(generatePlayWithLifecycle(
|
||||
const scrobbleError = asJsonPlayObject(await playWithLifecycleScrobble(generatePlayWithLifecycle(
|
||||
{
|
||||
lifecycleSteps: {
|
||||
preCompare: 1,
|
||||
postCompare: 1,
|
||||
}
|
||||
}
|
||||
))
|
||||
)))
|
||||
return {play: scrobbleError};
|
||||
}
|
||||
],
|
||||
@@ -51,7 +51,7 @@ export const ActivityTimelineStory = meta.story({
|
||||
export const ScrobbleError = meta.story({
|
||||
loaders: [
|
||||
async () => {
|
||||
const scrobbleError = await playWithLifecycleScrobble(generatePlayWithLifecycle(), {error: true});
|
||||
const scrobbleError = asJsonPlayObject(await playWithLifecycleScrobble(generatePlayWithLifecycle(), {error: true}));
|
||||
return {play: scrobbleError};
|
||||
}
|
||||
],
|
||||
@@ -60,12 +60,12 @@ export const ScrobbleError = meta.story({
|
||||
export const TransformError = meta.story({
|
||||
loaders: [
|
||||
async () => {
|
||||
const scrobbleError = await generatePlayWithLifecycle({
|
||||
const scrobbleError = asJsonPlayObject(await generatePlayWithLifecycle({
|
||||
lifecycleSteps: {
|
||||
preCompare: 2,
|
||||
postCompare: [false],
|
||||
}
|
||||
});
|
||||
}));
|
||||
return {play: scrobbleError};
|
||||
}
|
||||
],
|
||||
@@ -74,11 +74,11 @@ export const TransformError = meta.story({
|
||||
export const TransformSkip = meta.story({
|
||||
loaders: [
|
||||
async () => {
|
||||
const play = await generatePlayWithLifecycle({
|
||||
const play = asJsonPlayObject(await generatePlayWithLifecycle({
|
||||
lifecycleSteps: {
|
||||
preCompare: [true, 'skipped', true],
|
||||
}
|
||||
});
|
||||
}));
|
||||
return {play};
|
||||
}
|
||||
],
|
||||
@@ -87,11 +87,11 @@ export const TransformSkip = meta.story({
|
||||
export const TransformPrereq = meta.story({
|
||||
loaders: [
|
||||
async () => {
|
||||
const play = await generatePlayWithLifecycle({
|
||||
const play = asJsonPlayObject(await generatePlayWithLifecycle({
|
||||
lifecycleSteps: {
|
||||
preCompare: [true, 'prereq'],
|
||||
}
|
||||
});
|
||||
}));
|
||||
return {play};
|
||||
}
|
||||
],
|
||||
@@ -100,11 +100,11 @@ export const TransformPrereq = meta.story({
|
||||
export const TransformStop = meta.story({
|
||||
loaders: [
|
||||
async () => {
|
||||
const play = await generatePlayWithLifecycle({
|
||||
const play = asJsonPlayObject(await generatePlayWithLifecycle({
|
||||
lifecycleSteps: {
|
||||
preCompare: [true, 'stop'],
|
||||
}
|
||||
});
|
||||
}));
|
||||
return {play};
|
||||
}
|
||||
],
|
||||
|
||||
@@ -8,7 +8,7 @@ import {Provider} from "../client/components/Provider";
|
||||
import { generateJsonPlays, normalizePlays } from "../core/PlayTestUtils.js";
|
||||
import { ErrorLike, JsonPlayObject } from "../core/Atomic.js";
|
||||
import {examplePlay, lastfmErrorExample} from './storyUtils.js';
|
||||
import {playWithLifecycleScrobble, generatePlayWithLifecycle} from '../core/tests/utils/fixtures'
|
||||
import {playWithLifecycleScrobble, generatePlayWithLifecycle, asJsonPlayObject} from '../core/tests/utils/fixtures'
|
||||
import { generateArray } from "../core/DataUtils.js";
|
||||
import dayjs from "dayjs";
|
||||
|
||||
@@ -55,14 +55,14 @@ decorators: [
|
||||
export const List = meta.story({
|
||||
loaders: [
|
||||
async () => {
|
||||
const queued = normalizePlays(generateArray(7,() => generatePlayWithLifecycle()), {endDate: dayjs()}).map(x => ({play: x, status: 'queued'}));
|
||||
const queued = normalizePlays(generateArray(7,() => generatePlayWithLifecycle()), {endDate: dayjs()}).map(x => ({play: asJsonPlayObject(x), status: 'queued'}));
|
||||
|
||||
const scrobbled = await playWithLifecycleScrobble(generatePlayWithLifecycle({lifecycleSteps: {preCompare: [true, 'skipped', true]}}));
|
||||
const scrobbleError = await playWithLifecycleScrobble(generatePlayWithLifecycle(), {error: true});
|
||||
const scrobbled = asJsonPlayObject(await playWithLifecycleScrobble(generatePlayWithLifecycle({lifecycleSteps: {preCompare: [true, 'skipped', true]}})));
|
||||
const scrobbleError = asJsonPlayObject(await playWithLifecycleScrobble(generatePlayWithLifecycle(), {error: true}));
|
||||
|
||||
const promisedScrobbled = generateArray(10,() => playWithLifecycleScrobble(generatePlayWithLifecycle({lifecycleSteps: {preCompare: [true, 'skipped', true]}})));
|
||||
const promised = await Promise.all(promisedScrobbled);
|
||||
const yesterdayScrobbled = normalizePlays(promised, {endDate: dayjs().subtract(1, 'd').subtract(100, 'm')}).map((x) => ({play: x, status: 'scrobbled'}));
|
||||
const yesterdayScrobbled = normalizePlays(promised, {endDate: dayjs().subtract(1, 'd').subtract(100, 'm')}).map((x) => ({play: asJsonPlayObject(x), status: 'scrobbled'}));
|
||||
return {data: [
|
||||
...queued,
|
||||
{play: scrobbled, status: 'scrobbled'},
|
||||
|
||||
@@ -8,7 +8,7 @@ import {Provider} from "../client/components/Provider";
|
||||
import { generateJsonPlays } from "../core/PlayTestUtils.js";
|
||||
import { ErrorLike, JsonPlayObject, PlayLifecycle } from "../core/Atomic.js";
|
||||
import { examplePlay, lastfmErrorExample } from "./storyUtils.js";
|
||||
import {generatePlayWithLifecycle} from '../core/tests/utils/fixtures'
|
||||
import {asJsonPlayObject, generatePlayWithLifecycle} from '../core/tests/utils/fixtures'
|
||||
|
||||
// More on how to set up stories at: https://storybook.js.org/docs/writing-stories#default-export
|
||||
const meta = preview.meta({
|
||||
@@ -27,7 +27,7 @@ decorators: [
|
||||
// Use `fn` to spy on the onClick arg, which will appear in the actions panel once invoked: https://storybook.js.org/docs/essentials/actions#story-args
|
||||
});
|
||||
|
||||
const multiPlay = generatePlayWithLifecycle({lifecycleSteps: { preCompare: 2}});
|
||||
const multiPlay = asJsonPlayObject(generatePlayWithLifecycle({lifecycleSteps: { preCompare: 2}}));
|
||||
// More on writing stories with args: https://storybook.js.org/docs/writing-stories/args
|
||||
export const Multiple = meta.story({
|
||||
args: {
|
||||
|
||||
Reference in New Issue
Block a user