mirror of
https://github.com/FoxxMD/multi-scrobbler.git
synced 2026-09-03 05:10:00 +03:00
Compare commits
24
Commits
0.10.4
...
glossasryDocs
| Author | SHA1 | Date | |
|---|---|---|---|
 FoxxMD
|
4ed0a06c9e | ||
|
FoxxMD
|
672283b798 | ||
|
Matt Foxx
|
ca8d397653 | ||
|
FoxxMD
|
6ac44397d0 | ||
|
FoxxMD
|
d6c4d7cf25 | ||
|
FoxxMD
|
04ae651b15 | ||
|
FoxxMD
|
ef6e2b0f88 | ||
|
FoxxMD
|
b7271ea4c2 | ||
|
FoxxMD
|
b5e835a0c6 | ||
|
FoxxMD
|
a7df5a8d30 | ||
|
FoxxMD
|
883050e0b4 | ||
|
FoxxMD
|
b6417ed780 | ||
|
FoxxMD
|
affb16fc0e | ||
|
FoxxMD
|
56409dd1d7 | ||
|
FoxxMD
|
1af0193636 | ||
|
FoxxMD
|
53e766318c | ||
|
FoxxMD
|
c5269791af | ||
|
FoxxMD
|
fb5ffd053d | ||
|
Matt Foxx
|
0c04e7b79f | ||
|
FoxxMD
|
dc5fab6f9a | ||
|
FoxxMD
|
128c48947e | ||
|
FoxxMD
|
4cab02e88d | ||
|
FoxxMD
|
a360687264 | ||
|
FoxxMD
|
6a677b6fe1 |
@@ -96,7 +96,7 @@ Check the [**FAQ**](../FAQ.md) if you have any issues after configuration!
|
||||
:::tip
|
||||
* There are **example configurations** for all Source/Client types and AIO config located in the [`/config`](https://github.com/FoxxMD/multi-scrobbler/tree/master/config) directory of this project. These can be used as-is by renaming them to `.json`.
|
||||
* For docker installations these examples are copied to your configuration directory on first-time use.
|
||||
* There is also a [**kitchensink example**](/kitchensink) that provides examples of using all sources/clients in a complex configuration.
|
||||
* There is also a [**kitchensink example**](/configuration/kitchensink) that provides examples of using all sources/clients in a complex configuration.
|
||||
:::
|
||||
|
||||
Each file is named by the **type** of the Client/Source found in below sections. Each file as an **array** of that type of Client/Source.
|
||||
@@ -142,7 +142,7 @@ Check the [**FAQ**](../FAQ.md) if you have any issues after configuration!
|
||||
:::tip
|
||||
* An example AIO config files can be found in the project directory at [`/config/config.json.example`](https://github.com/FoxxMD/multi-scrobbler/tree/master/config/config.json.example)
|
||||
* For docker installations this example is copied to your configuration directory on first-time use.
|
||||
* There is also a [**kitchensink example**](/kitchensink) that provides examples of using all sources/clients in a complex configuration.
|
||||
* There is also a [**kitchensink example**](/configuration/kitchensink) that provides examples of using all sources/clients in a complex configuration.
|
||||
:::
|
||||
|
||||
[**Explore the schema for this configuration, along with an example generator and validator, here**](https://json-schema.app/view/%23?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fmulti-scrobbler%2Fmaster%2Fsrc%2Fbackend%2Fcommon%2Fschema%2Faio.json)
|
||||
|
||||
@@ -1,523 +0,0 @@
|
||||
---
|
||||
sidebar_position: 4
|
||||
title: Scrobble Modification
|
||||
toc_max_heading_level: 4
|
||||
---
|
||||
|
||||
Multi-scrobbler configs support the ability to modify scrobble data in an automated fashion by matching and replacing strings in **title, artists, and album** at many different times in multi-scrobbler's lifecycle.
|
||||
|
||||
### Why?
|
||||
|
||||
You may need to "clean up" data from a Source or before sending to a scrobble Client due to any number of reasons:
|
||||
|
||||
* ID3 tags in your music collection are dirty or have repeating garbage IE `[YourMusicSource.com] My Artist - My Title`
|
||||
* A Source's service often incorrectly adds data to some field IE `My Artist - My Title (Album Version)` when the title should just be `My Title`
|
||||
* An Artist you listen to often is spelled different between a Source and a Client which causes duplicate scrobbles
|
||||
|
||||
In any scenario where a repeating pattern can be found in the data it would be nice to be able to fix it before the data gets downstream or to help prevent duplicate scrobbling. Multi-scrobbler can help you do this.
|
||||
|
||||
## Overview
|
||||
|
||||
### Journey of a Scrobble
|
||||
|
||||
First, let's recap the lifecycle of a scrobble in multi-scrobbler:
|
||||
|
||||
**Sources** are the beginning of the journey for a **Play** (song you've listened to long enough to be scrobblable)
|
||||
|
||||
* A Source finds a new valid **Play**
|
||||
* The Source **compares** this new Play to all the other Plays it has already seen, if the Play is unique (title/artist/album/listened datetime) then...
|
||||
* The Source **discovers** the Play, adds it to Plays it has seen already, and broadcasts the Play should be scrobbled to all Clients
|
||||
|
||||
Scrobble **Clients** listen for discovered Plays from Sources, then...
|
||||
|
||||
* A Client receives a **Play** from a Source
|
||||
* The Client **compares** this Play to all the other scrobbles it has already seen, if the Play is unique (title/artist/album/listened datetime) then...
|
||||
* The Client **scrobbles** the Play downstream to the scrobble service and adds it as a Scrobble it has seen already
|
||||
|
||||
### Lifecyle Hooks
|
||||
|
||||
You'll notice there is a pattern above that looks like this:
|
||||
|
||||
* **Before** data is compared
|
||||
* Data is **compared**
|
||||
* **After** data is compared
|
||||
|
||||
These points, during both Source and Client processes, are when you can hook into the scrobble lifecycle and modify it.
|
||||
|
||||
#### TLDR
|
||||
|
||||
In more concrete terms this is the structure of hooks within a configuration (can be used in any **Source** or **Client**):
|
||||
|
||||
```json5 title="lastfm.json" {10-14}
|
||||
[
|
||||
{
|
||||
"name": "myLastFm",
|
||||
"enable": true,
|
||||
"configureAs": "source",
|
||||
"data": {
|
||||
// ...
|
||||
},
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"preCompare": {/* ... */},
|
||||
"compare": {/* ... */},
|
||||
"postCompare": {/* ... */}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
##### Hook
|
||||
|
||||
For **Sources**:
|
||||
|
||||
* `preCompare` - modify Play data immediately when received
|
||||
* `compare` - temporarily modify Play data when it is being compared to see if Play was already discovered
|
||||
* `postCompare` - modify Play data before sending to scrobble **Clients**
|
||||
|
||||
For **Clients**:
|
||||
|
||||
* `preCompare` - modify Play data immediately when received
|
||||
* `compare` - temporarily modify Play data when it is being compared to see if it was already scrobbled
|
||||
* `postCompare` - modify Play data before scrobbling it to downstream service and adding to already seen scrobbles
|
||||
|
||||
:::tip
|
||||
|
||||
Keep in mind that modifying Scrobble/Play data earlier in the lifecycle will affect that data at all times later in the lifecycle.
|
||||
|
||||
For example, to modify the track so it's the same anywhere it is processed in multi-scrobbler you only need to modify it in the **Source's** `preCompare` hook because all later processes will receive the data with the modified track.
|
||||
|
||||
:::
|
||||
|
||||
### Modification Parts
|
||||
|
||||
|
||||
Each [**hook**](#hook) (`preCompare` etc...) is an object that specifies what part of the **Play** to modify:
|
||||
|
||||
```json5
|
||||
{
|
||||
"title": [/* ... */],
|
||||
"artists": [/* ... */],
|
||||
"album": [/* ... */]
|
||||
}
|
||||
```
|
||||
|
||||
##### Expression
|
||||
|
||||
and then a **list** what pattern/replacements (expressions) to use for the modification by using either simple strings or `search-replace` objects:
|
||||
|
||||
```json5
|
||||
[
|
||||
"badTerm", // remove all instances of 'badTerm'
|
||||
{
|
||||
"search": "anotherBadTerm", // and also match all instances of 'anotherBadTerm'
|
||||
"replace": "goodTerm" // replace with the string 'goodTerm'
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Putting it all together:
|
||||
|
||||
```json5 title="lastfm.json"
|
||||
[
|
||||
{
|
||||
"name": "myLastFm",
|
||||
"enable": true,
|
||||
"configureAs": "source",
|
||||
"data": {
|
||||
// ...
|
||||
},
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"preCompare": {
|
||||
"title": [
|
||||
"badTerm",
|
||||
{
|
||||
"search": "badTerm",
|
||||
"replace": "goodTerm"
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
:::note
|
||||
|
||||
If the value of the field (title, an artist, album) is an empty string after transforming then the field is **removed.**
|
||||
|
||||
:::
|
||||
|
||||
:::tip
|
||||
|
||||
Modifications can also be applied to **all Sources** or **all Clients** when using the [AIO Config](./configuration.mdx?configType=aio#configuration-types) `config.json` by setting `playTransform` in `sourceDefaults` or `clientDefaults`:
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
```json5 title="config.json"
|
||||
{
|
||||
"sourceDefaults": { // will apply playTransform to all sources
|
||||
"playTransform": {
|
||||
"preCompare": {
|
||||
"title": [
|
||||
"(Album Version)"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"sources": [/* ... */],
|
||||
"clients": [/* ... */]
|
||||
}
|
||||
```
|
||||
</details>
|
||||
|
||||
:::
|
||||
|
||||
#### Compare Hook
|
||||
|
||||
The `compare` [hook](#hook) is slightly different than `preCompare` and `postCompare`. It consists of an object where you define which side(s) of the comparison should be modified. It also **does not modify downstream data!** Instead, the modifications are made only for use in the comparison.
|
||||
|
||||
```json5 title="lastfm.json"
|
||||
[
|
||||
{
|
||||
"name": "myLastFm",
|
||||
// ...
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"compare": {
|
||||
"candidate": {/* ... */}, // modify the "new" Play being compared
|
||||
"existing": {/* ... */}, // modify all "existing" Play/Scrobbles the new Play is being compared against
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
#### Regular Expressions
|
||||
|
||||
In addition to plain strings [expressions](#expression) that are matched and removed you can also use Regular Expressions. Write your regex like you normally would, but as a string, and it'll automatically be parsed:
|
||||
|
||||
```json5
|
||||
[
|
||||
"/^\(\w+.com)/i", // matches any string that starts with '(YourMusic.com)' and removes it
|
||||
{
|
||||
"search": "/^\(\w+.com)/i", // matches any string that starts with '(YourMusic.com)'
|
||||
"replace": "[MySite.com]" // replace with the string '[MySite.com]'
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
The `replace` property uses javascript's [`replace()` function and so can use any special string characters.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_the_replacement)
|
||||
|
||||
### Conditional Modification
|
||||
|
||||
#### "When" Condition
|
||||
|
||||
Top-level hooks **and** individual rules also support a `when` key for testing **if they should be run.**
|
||||
|
||||
The `when` key is similar to a normal [modification](#modification-parts) except:
|
||||
|
||||
* the keys accept a single string instead of an array
|
||||
* the `when` key data is an array instead of a single object
|
||||
|
||||
All parts of an individual `when` clause must test true to "pass" but if **any** `when` clauses pass the hook/rule is processed. Example `when` data:
|
||||
|
||||
```json5
|
||||
{
|
||||
"when": [
|
||||
{
|
||||
"artist": "Elephant Gym", // both of these must match the Play object (AND)
|
||||
"album": "Dreams" // both of these must match the Play object (AND)
|
||||
},
|
||||
// OR
|
||||
{
|
||||
"title": "/(Remastered)$/", // both of these must match the Play object (AND)
|
||||
"album": "Various Artists" // both of these must match the Play object (AND)
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
More succinctly:
|
||||
|
||||
* All parts (`artist` `album` `title`) of a `when` are `AND` conditions
|
||||
* All part-objects in the `when` array are `OR` conditions
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example of top-level hook with when condition</summary>
|
||||
|
||||
```json5
|
||||
{
|
||||
// IF the artist is Elephant Gym
|
||||
// THEN Run preCompare hook ELSE skip this hook
|
||||
//
|
||||
// Run search-replace on album
|
||||
// Run regex title remove
|
||||
"sourceDefaults": {
|
||||
"playTransform": {
|
||||
"preCompare": {
|
||||
"when": [
|
||||
{
|
||||
"artist": "/Elephant Gym/"
|
||||
}
|
||||
],
|
||||
"album": [
|
||||
{
|
||||
"search": "Dreams",
|
||||
"replace": "夢境"
|
||||
}
|
||||
],
|
||||
"title": ["/\s\-\s滾石40\s滾石撞樂隊\s40團拚經典(.+)$/i"]
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example of individual rule with when condition</summary>
|
||||
|
||||
```json5
|
||||
{
|
||||
// Always run preCompare
|
||||
//
|
||||
// On search-replace in title...
|
||||
// IF artist matches "Elephant Gym"
|
||||
// THEN Run regex search-replace ELSE skip this rule
|
||||
//
|
||||
// Run live|remastered regex remove
|
||||
"sourceDefaults": {
|
||||
"playTransform": {
|
||||
"preCompare": {
|
||||
"title": [
|
||||
{
|
||||
"search": "/\\s\\-\\s滾石40\\s滾石撞樂隊\\s40團拚經典(.+)$/i",
|
||||
"replace": "",
|
||||
"when": [
|
||||
{
|
||||
"artist": "/Elephant Gym/"
|
||||
}
|
||||
]
|
||||
},
|
||||
"/(\\s\\-\\s|\\s)(feat\\.(.+)|live|remastered(.+))$/i"
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
#### Top-level Hook array
|
||||
|
||||
Top-level hooks can also be an array of hooks. This makes creating multiple scenarios for top-level `when`-gated hooks easier. All hooks in the array will be run (assuming their `when`'s pass, if they exist) and their **input will be the Play object output of the previous hook in the array.**
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
|
||||
```json5
|
||||
{
|
||||
"sourceDefaults": {
|
||||
"playTransform": {
|
||||
"preCompare": [
|
||||
// first lifecycle hook of preCompare to run
|
||||
{
|
||||
"title": [
|
||||
{
|
||||
"search": "something",
|
||||
"replace": "else unique"
|
||||
}
|
||||
]
|
||||
},
|
||||
// second lifecycle hook of preCompare to run
|
||||
{
|
||||
"title": [
|
||||
{
|
||||
"search": "else unique",
|
||||
"replace": "very demure"
|
||||
}
|
||||
]
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### Logging
|
||||
|
||||
MS can log the output of hook transformations if/when they occur. In the `playTransform` object of a Source/Client config use `log`:
|
||||
|
||||
* `"log": true` => Output original play + final transformed output of last hook in the array
|
||||
* `"log": "all"` => Output original play + final transformed output of **each** hook in the array
|
||||
|
||||
```json5
|
||||
{
|
||||
"name": "myThing",
|
||||
"data": {/*...*/},
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"preCompare": {/*...*/},
|
||||
"log": true
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
### Remove phrase from Title in all new Plays
|
||||
|
||||
Removes the phrase `(Album Version)` from the Title of a Play
|
||||
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
```json5 title="config.json"
|
||||
{
|
||||
"sourceDefaults": {
|
||||
"playTransform": {
|
||||
"preCompare": {
|
||||
"title": [
|
||||
"(Album Version)"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
</details>
|
||||
|
||||
### Remove all parenthesized content from the end of a title
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
```json5 title="lastfm.json"
|
||||
[
|
||||
{
|
||||
"name": "myLastFm",
|
||||
// ...
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"compare": {
|
||||
"candidate": {
|
||||
"title": [
|
||||
"/(\(.+\))\s*$/"
|
||||
]
|
||||
},
|
||||
"existing": {
|
||||
"title": [
|
||||
"/(\(.+\))\s*$/"
|
||||
]
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
</details>
|
||||
|
||||
### Rename misspelled artist in all new Plays
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
```json5 title="config.json"
|
||||
{
|
||||
"sourceDefaults": {
|
||||
"playTransform": {
|
||||
"preCompare": {
|
||||
"artists": [
|
||||
{
|
||||
"search": "Boz Skaggs",
|
||||
"replace": "Boz Scaggs"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
</details>
|
||||
|
||||
### Remove "Various Artists" albums in all new Plays
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
```json5 title="config.json"
|
||||
{
|
||||
"sourceDefaults": {
|
||||
"playTransform": {
|
||||
"preCompare": {
|
||||
"album": [
|
||||
{
|
||||
"search": "Various Artists",
|
||||
"replace": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
</details>
|
||||
|
||||
### Extract primary Artist from delimited, multi-Artist string
|
||||
|
||||
<details>
|
||||
|
||||
When the Artist string is actually a multi-artist, delimited string, this search-and-replace will replace the string with just the first artist found.
|
||||
|
||||
Ex
|
||||
|
||||
```
|
||||
My Artist One / My Artist Two / Another Guy
|
||||
My Artist One
|
||||
```
|
||||
|
||||
Artists are delimited with a spaced forward slash (`/`) in the regex below. Replace the contents of the `delim` capture group with the delimiter for your use case. Some more common scenarios:
|
||||
|
||||
* `(?<delim>\\/)` No spaces between slash IE `My Artist One/My Artist Two/Another Guy`
|
||||
* `(?<delim>\\s*\\\\\s*)` Backslash instead of forward slash IE `My Artist One \ My Artist Two \ Another Guy`
|
||||
* `(?<delim>,)` Comma IE `My Artist One, My Artist Two, Another Guy`
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
```json5 title="config.json"
|
||||
{
|
||||
"sourceDefaults": {
|
||||
"playTransform": {
|
||||
"preCompare": {
|
||||
"artists": [
|
||||
{
|
||||
"search": "(.*?)(?<delim>\\s*\\/\\s*)(.*$)",
|
||||
"replace": "$1"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
</details>
|
||||
|
||||
</details>
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"label": "Enhance Scrobbles",
|
||||
"link": {
|
||||
"type": "doc",
|
||||
"id": "configuration/transforms/transforms"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
---
|
||||
title: Native Stage
|
||||
toc_min_heading_level: 2
|
||||
toc_max_heading_level: 5
|
||||
---
|
||||
|
||||
The **Native** [Stage](/configuration/transforms#stage) uses [built-in heuristics](https://github.com/FoxxMD/multi-scrobbler/blob/master/src/backend/tests/plays/playParsing.test.ts) to try to extract Artists from Play artist/track data.
|
||||
|
||||
This Stage is most useful for Sources that report limited data such as:
|
||||
|
||||
* [Subsonic](/configuration/sources/subsonic) - Reports Artists as a single string
|
||||
|
||||
A non-exhaustive list of heuristics:
|
||||
|
||||
* Splits artists in artist string using common delimiters EX `Foo Artist, Bar Guy, Baz Band - My Cool Song`
|
||||
* Does not split artists with `&` in name, if other delimiters are present
|
||||
* Does not split artist name when only one delimiter is present
|
||||
* Splits artists on common joiner phrases (ft. feat. vs. etc...)
|
||||
* Extracts artists from Play title using joiner phrases EX `My Cool Song (feat. SomeGuy)`
|
||||
|
||||
|
||||
## Configuration
|
||||
|
||||
Available properties for [Stage Configuration](/configuration/transforms#configuring-stages):
|
||||
|
||||
|
||||
* `delimitersExtra` - A list of string characters that should be considered delimiters for artists **in addition to** multi-scrobbler's default list (`, / \ `)
|
||||
* `delimiters` - A list of string characters that should be considered delimiters for artists.
|
||||
* **Replaces** all delimiters (MS not use any defaults, only what you give it)
|
||||
* `artistsIgnore` - a list of strings and/or regular expressions. Any monolothic artist string that matches from the list _will not be modified._
|
||||
* `artistsParseFrom` a list of the properties that should be used to try to extract artists. Can be `artists` `title` or both. Defaults to both when not provided in options ( `["artists", "title"]` )
|
||||
* When `artists` is present it tries to extract additional artists from artist strings
|
||||
* When `title` is present it tries to extract artists from ft. feat. vs. etc... found in the track title
|
||||
* Importantly, if `artists` is _not_ present in `artistsParseFrom` then _no artists_ are used at all (only those from `title`, if present)
|
||||
* `artistsParseMonolithicOnly` - boolean value, defaults to `true`. When `true` native tranformer will only attempt to extract artists if the scrobble data has _only one string for artist_ (pre-transform)
|
||||
* This means that, for Sources like Spotify/Jellyfin/etc. that provide proper lists of artists in their data _and the list has more than one string_, it will not try to extract artists from their strings
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example of the behavior for `artistsParseFrom`</summary>
|
||||
|
||||
```
|
||||
The Foos, The Bars - My Cool Track (ft. Frank)
|
||||
```
|
||||
|
||||
Config => extracted artists
|
||||
|
||||
```
|
||||
"artistsParseFrom": ["artists", "title"] => The Foos, The Bars, Frank
|
||||
"artistsParseFrom": ["artists"] => The Foos, The Bars
|
||||
"artistsParseFrom": ["title"] => Frank
|
||||
"artistsParseFrom": [] =>
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### Rules
|
||||
|
||||
Each [Rule](/configuration/transforms#stage-rules) should be either a boolean, specifying if the transformed data should be used for this field, or a [`when` condition.](/configuration/transforms#conditional-moditication):
|
||||
|
||||
```json5
|
||||
{
|
||||
"type": "native",
|
||||
// ...
|
||||
"title": false, // will not apply any changes to Play title
|
||||
"artists": {
|
||||
"when": {/* ... */}, // will only apply changes to Play artists if "when" is satisfied
|
||||
/* ... */
|
||||
},
|
||||
"album": true // will always apply changes to Play album
|
||||
}
|
||||
```
|
||||
|
||||
If a rule is not present then multi-scrobbler defaults it to `true`.
|
||||
|
||||
## Examples
|
||||
|
||||
### Parse only artists string using a custom delimiter
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
|
||||
Your [AIO Config](/configuration?configType=aio#configuration-types):
|
||||
|
||||
```json5 title="config.json"
|
||||
{
|
||||
// ...
|
||||
"transformers": [
|
||||
{
|
||||
"type": "native",
|
||||
"name": "MyNativeTransformer",
|
||||
"defaults": {
|
||||
// default delimiters when this Stage is used in a hook
|
||||
"delimiters": [
|
||||
"•"
|
||||
],
|
||||
// default delimiters when this Stage is used in a hook
|
||||
"artistsParseFrom": ["artists"]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types):
|
||||
|
||||
```json5 title="subsonic.json"
|
||||
[
|
||||
{
|
||||
"name": "MySubsonic",
|
||||
"data": { /* ... */},
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"preCompare": [
|
||||
{
|
||||
"type": "native",
|
||||
"name": "MyNativeTransformer"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
</details>
|
||||
|
||||
### Don't parse specific artists
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
|
||||
Your [AIO Config](/configuration?configType=aio#configuration-types):
|
||||
|
||||
```json5 title="config.json"
|
||||
{
|
||||
// ...
|
||||
"transformers": [
|
||||
{
|
||||
"type": "native",
|
||||
"name": "MyNativeTransformer",
|
||||
"defaults": {
|
||||
"artistsIgnore": [
|
||||
"Crosby, Stills, Nash & Young",
|
||||
"Polo & Pan"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types):
|
||||
|
||||
```json5 title="subsonic.json"
|
||||
[
|
||||
{
|
||||
"name": "MySubsonic",
|
||||
"data": { /* ... */},
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"preCompare": [
|
||||
{
|
||||
"type": "native",
|
||||
"name": "MyNativeTransformer"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
</details>
|
||||
@@ -0,0 +1,475 @@
|
||||
---
|
||||
sidebar_position: 4
|
||||
title: Enhancing Scrobbles
|
||||
toc_max_heading_level: 5
|
||||
---
|
||||
|
||||
Multi-scrobbler configs support the ability to enhance scrobble data in an automated fashion by matching and replacing strings in **title, artists, and album** at many different times in multi-scrobbler's lifecycle.
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Why Would I Do This?</summary>
|
||||
|
||||
You may need to "clean up" data from a Source or before sending to a scrobble Client due to any number of reasons:
|
||||
|
||||
* ID3 tags in your music collection are dirty or have repeating garbage IE `[YourMusicSource.com] My Artist - My Title`
|
||||
* A Source's service often incorrectly adds data to some field IE `My Artist - My Title (Album Version)` when the title should just be `My Title`
|
||||
* An Artist you listen to often is spelled different between a Source and a Client which causes duplicate scrobbles
|
||||
|
||||
In any scenario where a repeating pattern can be found in the data it would be nice to be able to fix it before the data gets downstream or to help prevent duplicate scrobbling. Multi-scrobbler can help you do this.
|
||||
|
||||
</details>
|
||||
|
||||
## Journey of a Scrobble
|
||||
|
||||
First, let's recap the lifecycle of a scrobble in multi-scrobbler:
|
||||
|
||||
**Sources** are the beginning of the journey for a **Play** (song you've listened to long enough to be scrobblable)
|
||||
|
||||
* A Source finds a new valid **Play**
|
||||
* The Source **compares** this new Play to all the other Plays it has already seen, if the Play is unique (title/artist/album/listened datetime) then...
|
||||
* The Source **discovers** the Play, adds it to Plays it has seen already, and broadcasts the Play should be scrobbled to all Clients
|
||||
|
||||
Scrobble **Clients** listen for discovered Plays from Sources, then...
|
||||
|
||||
* A Client receives a **Play** from a Source
|
||||
* The Client **compares** this Play to all the other scrobbles it has already seen, if the Play is unique (title/artist/album/listened datetime) then...
|
||||
* The Client **scrobbles** the Play downstream to the scrobble service and adds it as a Scrobble it has seen already
|
||||
|
||||
## Lifecyle Hooks
|
||||
|
||||
You'll notice there is a pattern above that looks like this:
|
||||
|
||||
* **Before** data is compared
|
||||
* Data is **compared**
|
||||
* **After** data is compared
|
||||
|
||||
These points, during both Source and Client processes, are when you can hook into the scrobble lifecycle and modify it.
|
||||
|
||||
##### TLDR
|
||||
|
||||
In more concrete terms this is the structure of hooks within a configuration (can be used in any **Source** or **Client**):
|
||||
|
||||
```json5 title="lastfm.json" {10-14}
|
||||
[
|
||||
{
|
||||
"name": "myLastFm",
|
||||
"enable": true,
|
||||
"configureAs": "source",
|
||||
"data": {
|
||||
// ...
|
||||
},
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"preCompare": [/* ... */],
|
||||
"compare": [/* ... */],
|
||||
"postCompare": [/* ... */]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### Hook
|
||||
|
||||
For **Sources**:
|
||||
|
||||
* `preCompare` - modify Play data immediately when received
|
||||
* `compare` - temporarily modify Play data when it is being compared to see if Play was already discovered
|
||||
* `postCompare` - modify Play data before sending to scrobble **Clients**
|
||||
|
||||
For **Clients**:
|
||||
|
||||
* `preCompare` - modify Play data immediately when received
|
||||
* `compare` - temporarily modify Play data when it is being compared to see if it was already scrobbled
|
||||
* `postCompare` - modify Play data before scrobbling it to downstream service and adding to already seen scrobbles
|
||||
|
||||
:::tip
|
||||
|
||||
Keep in mind that modifying Scrobble/Play data earlier in the lifecycle will affect that data at all times later in the lifecycle (except when using the **compare** hook).
|
||||
|
||||
For example, to modify the track so it's the same anywhere it is processed in multi-scrobbler you only need to modify it in the **Source's** `preCompare` hook because all later processes will receive the data with the modified track.
|
||||
|
||||
:::
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Using `compare` hook</summary>
|
||||
|
||||
The `compare` [hook](#hook) is slightly different than `preCompare` and `postCompare`. It consists of an object where you define which side(s) of the comparison should be modified. It also **does not modify downstream data!** Instead, the modifications are made only for use in the comparison.
|
||||
|
||||
```json5 title="lastfm.json"
|
||||
[
|
||||
{
|
||||
"name": "myLastFm",
|
||||
// ...
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"compare": [
|
||||
{
|
||||
"candidate": {/* ... */}, // modify the "new" Play being compared
|
||||
"existing": {/* ... */}, // modify all "existing" Play/Scrobbles the new Play is being compared against
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
## Modification Stage {#stage}
|
||||
|
||||
Each [**hook**](#hook) is made up of one or more **Stages**. A Stage is a self-contained, unique way of enhancing or modifying the Play data. Some examples of a Stage:
|
||||
|
||||
* The [User](/configuration/transforms/user) Stage allows a user to define search-and-replace terms for Artist/Title/Album
|
||||
* The [Native](/configuration/transforms/native) Stage uses MS's built-in heuristics to extract Artists from a single Artist string
|
||||
* The Musicbrainz Stage tries to match Play data with the Musicbrainz database and to standardize the Artist/Title/Album data
|
||||
|
||||
Each Stage in a Hook receives Play data from the previous Stage.
|
||||
|
||||
Within a hook, each Stage minimally consists of a `type` to identify what Stage it is along with any other data specific to that stage:
|
||||
|
||||
```json5
|
||||
{
|
||||
"type": "native"
|
||||
// optional, stage specific data here...
|
||||
}
|
||||
```
|
||||
|
||||
### Configuring Stages {#configuring-stages}
|
||||
|
||||
Stages may be globally configured using [AIO Config](/configuration?configType=aio#configuration-types) `config.json` file in the top-level `transformers` block.
|
||||
|
||||
Each Stage consists of:
|
||||
|
||||
* `type` the type of Stage
|
||||
* `name` a unique name for the Stage, to be (potentially) used with hooks
|
||||
* `defaults` - An object defining default configuration for this stage, when used in a Hook.
|
||||
* `data` - An object containing any data required to initially configure the stage itself (Example: API URL, username, password, etc...)
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
|
||||
Your [AIO Config](/configuration?configType=aio#configuration-types):
|
||||
|
||||
```json5 title="config.json"
|
||||
{
|
||||
// ...
|
||||
"transformers": [
|
||||
{
|
||||
"type": "native",
|
||||
"name": "MyNativeTransformer",
|
||||
"defaults": {
|
||||
// default delimiters when this Stage is used in a hook
|
||||
"delimiters": [
|
||||
"•"
|
||||
],
|
||||
// default delimiters when this Stage is used in a hook
|
||||
"artistsParseFrom": ["artists"]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types):
|
||||
|
||||
```json5 title="subsonic.json"
|
||||
[
|
||||
{
|
||||
"name": "MySubsonic",
|
||||
"data": { /* ... */},
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"preCompare": [
|
||||
{
|
||||
"type": "native"
|
||||
// when "name" is not defined, uses first found "native" transformer
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
Multiple stages of the same type may be configured, allowing you to define several sets of default behavior.
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
|
||||
Your [AIO Config](/configuration?configType=aio#configuration-types):
|
||||
|
||||
```json5 title="config.json"
|
||||
{
|
||||
// ...
|
||||
"transformers": [
|
||||
{
|
||||
"type": "native",
|
||||
"name": "DotTransformer",
|
||||
"defaults": {
|
||||
"delimiters": [
|
||||
"•"
|
||||
],
|
||||
"artistsParseFrom": ["artists"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "native",
|
||||
"name": "TitleOnly",
|
||||
"defaults": {
|
||||
// extracts and uses *only* artists found in title string
|
||||
"artistsParseFrom": ["title"]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types):
|
||||
|
||||
```json5 title="subsonic.json"
|
||||
[
|
||||
{
|
||||
"name": "MySubsonic",
|
||||
"data": { /* ... */},
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"preCompare": [
|
||||
{
|
||||
"type": "native"
|
||||
"name": "DotTransformer"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
In a [VLC](/configuration/sources/vlc) [File Config](/configuration?configType=file#configuration-types):
|
||||
|
||||
```json5 title="vlc.json"
|
||||
[
|
||||
{
|
||||
"name": "MyVLC",
|
||||
"data": { /* ... */},
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"preCompare": [
|
||||
{
|
||||
"type": "native"
|
||||
"name": "TitleOnly"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
#### Overriding Configuration
|
||||
|
||||
The default configuration you set for your Stage may be overridden in any usage of the Stage within a Hook.
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
|
||||
Your [AIO Config](/configuration?configType=aio#configuration-types):
|
||||
|
||||
```json5 title="config.json"
|
||||
{
|
||||
// ...
|
||||
"transformers": [
|
||||
{
|
||||
"type": "native",
|
||||
"name": "MyNativeTransformer",
|
||||
"defaults": {
|
||||
// default delimiters when this Stage is used in a hook
|
||||
"delimiters": [
|
||||
"•"
|
||||
],
|
||||
// default delimiters when this Stage is used in a hook
|
||||
"artistsParseFrom": ["artists"]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types):
|
||||
|
||||
```json5 title="subsonic.json"
|
||||
[
|
||||
{
|
||||
"name": "MySubsonic",
|
||||
"data": { /* ... */},
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"preCompare": [
|
||||
{
|
||||
"type": "native"
|
||||
"name": "MyNativeTransformer",
|
||||
// overrides property from "defaults"
|
||||
"artistsParseFrom": ["artists", "title"]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
### Rules for Play Data {#stage-rules}
|
||||
|
||||
Each [Stage](#stage) may specify whether it should apply the resulting transformation to different parts of the Play data by specifying `title`, `artists` and/or `album` in the Stage object.
|
||||
|
||||
```json5
|
||||
{
|
||||
"type": "native",
|
||||
// ...
|
||||
"title": false, // will not apply any changes to Play title
|
||||
"artists": {
|
||||
"when": {/* ... */}, // will only apply changes to Play artists if "when" is satisfied
|
||||
/* ... */
|
||||
},
|
||||
"album": true // will always apply changes to Play album
|
||||
}
|
||||
```
|
||||
|
||||
The actual value of each property may be different for each Stage. Check the docs for the Stage you want to use to see its usage of `title`, `artists`, and `album`.
|
||||
|
||||
Generically, though, each property may be some value **or** an object combining a [`when` condition](#conditional-modification) and that value.
|
||||
|
||||
If none of the properties are specified in the stage then it's assumed all transformed data should be used.
|
||||
|
||||
:::note
|
||||
|
||||
Specifying these Rules is **not** the same as [configuring the Stage](#configuring-stages). Rules only determine if the *result* of the transformation should be used (replace) the existing Play Data.
|
||||
|
||||
:::
|
||||
|
||||
## Conditional Modification
|
||||
|
||||
[Stages](#stage) within a [Hook](#hook), and [Rules](#stage-rules) within each Stage, support a `when` object for testing **if they should be run.**
|
||||
|
||||
The `when` object may have propertes for `artist`, `title` and/or `album`. Each property may be a string or regular expression. The value of the property is used to match the **pre-transformation** values from Play data.
|
||||
|
||||
All parts of an individual `when` clause must test true to "pass" but if **any** `when` clauses pass the Stage/Rule is processed.
|
||||
|
||||
```json5
|
||||
{
|
||||
"when":
|
||||
{
|
||||
"artist": "Elephant Gym", // Play must have an artist matching "Elephant Gym" (AND)
|
||||
"album": "Dreams" // Play object must have an album matching "Dreams" (AND)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```json5
|
||||
{
|
||||
"when": [
|
||||
{
|
||||
"artist": "Elephant Gym", // Play must have an artist matching "Elephant Gym" (AND)
|
||||
"album": "Dreams" // Play object must have an album matching "Dreams" (AND)
|
||||
},
|
||||
// OR
|
||||
{
|
||||
"title": "/(Remastered)$/", // Play title must match regular expression (AND)
|
||||
"album": "Various Artists" // Play album must match "Various Artists" (AND)
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
More succinctly:
|
||||
|
||||
* All parts (`artist` `album` `title`) of a `when` are `AND` conditions
|
||||
* All part-objects in the `when` array are `OR` conditions
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example of Stage with `when` condition</summary>
|
||||
|
||||
```json5
|
||||
{
|
||||
// IF the artist is Elephant Gym
|
||||
// THEN Run native stage
|
||||
"playTransform": {
|
||||
"preCompare": [
|
||||
{
|
||||
"type": "native",
|
||||
"when": [
|
||||
{
|
||||
"artist": "/Elephant Gym/"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example of individual rule with when condition</summary>
|
||||
|
||||
```json5
|
||||
{
|
||||
// Always run native Stage
|
||||
//
|
||||
// IF artist matches "Elephant Gym"
|
||||
// THEN use result of native stage for "artists" of Play data
|
||||
"playTransform": {
|
||||
"preCompare": {
|
||||
"type": "native",
|
||||
"artists":
|
||||
{
|
||||
"when": [
|
||||
{
|
||||
"artist": "/Elephant Gym/"
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
## Logging
|
||||
|
||||
MS can log the output of Stage transformations if/when they occur. In the `playTransform` object of a Source/Client config use `log`:
|
||||
|
||||
* `"log": true` => Output original play + final transformed output of last Stage in the array
|
||||
* `"log": "all"` => Output original play + final transformed output of **each** Stage in the array
|
||||
|
||||
```json5
|
||||
{
|
||||
"name": "myThing",
|
||||
"data": {/*...*/},
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"preCompare": {/*...*/},
|
||||
"log": true
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,275 @@
|
||||
---
|
||||
title: User Stage
|
||||
toc_min_heading_level: 2
|
||||
toc_max_heading_level: 5
|
||||
---
|
||||
|
||||
The **User** [Stage](/configuration/transforms#stage) uses [**search-and-replace** expressions](#search-and-replace-expression), provided by you, to modify/replace parts of Play data.
|
||||
|
||||
This Stage is most useful for correcting individual instances of bad data, or patterns, in your known data. Example scenarios:
|
||||
|
||||
* Removing `(Album Version)` from all Titles
|
||||
* Removing `Various Artists` from Artist data
|
||||
* Correcting spelling mistakes for individual Artist names
|
||||
|
||||
The user [Stage `type`](/configuration/transforms#stage) is `user`.
|
||||
|
||||
## Configuration
|
||||
|
||||
All [Stage Configuration](/configuration/transforms#configuring-stage) is done using [**search-and-replace** expressions](#search-and-replace-expression) inside individual [rules](#rules). Default configuration for all Rules can still be done using Stage `defaults`.
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
|
||||
```json5 title="config.json"
|
||||
{
|
||||
// ...
|
||||
"transformers": [
|
||||
{
|
||||
"type": "user",
|
||||
"name": "Normalizer",
|
||||
"defaults": {
|
||||
"title": [
|
||||
"(Album Version)"
|
||||
],
|
||||
"artists": [
|
||||
"Various Artists"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### Search-And-Replace Expression
|
||||
|
||||
A Search-And-Replace Expression can be a plain string that matches a literal, then removes it:
|
||||
|
||||
```
|
||||
Expression: "badTerm"
|
||||
|
||||
"this is badTerm cool string" => "this is a cool string"
|
||||
```
|
||||
|
||||
or a regular expression that matches and removes the match:
|
||||
|
||||
```
|
||||
Expression: "/bad\w+/i"
|
||||
|
||||
"this is badSomething cool string" => "this is a cool string"
|
||||
```
|
||||
|
||||
Or it may be an object that specifies what to match (using either plain string or regular expression) and what to replace it with:
|
||||
|
||||
```json5
|
||||
{
|
||||
"search": "anotherBadTerm", // match all instances of 'anotherBadTerm'
|
||||
"replace": "goodTerm" // replace with the string 'goodTerm'
|
||||
}
|
||||
```
|
||||
|
||||
```
|
||||
"this is anotherBadTerm cool string" => "this is goodTerm cool string"
|
||||
```
|
||||
|
||||
```json5
|
||||
{
|
||||
"search": "/^\(\w+.com)/i", // matches any string that starts with EX '(YourMusic.com)'
|
||||
"replace": "[MySite.com]" // replace with the string '[MySite.com]'
|
||||
}
|
||||
```
|
||||
|
||||
```
|
||||
"(Foo.com) this is a cool string" => "[MySite.com] this is a cool string"
|
||||
```
|
||||
|
||||
The `replace` property uses javascript's [`replace()` function and so can use any special string characters.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_the_replacement)
|
||||
|
||||
### Rules
|
||||
|
||||
Each [Rule](/configuration/transforms#stage-rules) must be an array of [search-and-replace expressions](#search-and-replace-expressions):
|
||||
|
||||
```json5 title="lastfm.json"
|
||||
[
|
||||
{
|
||||
"name": "myLastFm",
|
||||
"configureAs": "source",
|
||||
"data": {
|
||||
// ...
|
||||
},
|
||||
"options": {
|
||||
"playTransform": {
|
||||
"type": "user",
|
||||
"preCompare": [
|
||||
{
|
||||
"title": [
|
||||
// removes "badTerm" from title
|
||||
"badTerm",
|
||||
{
|
||||
// removes "fooTerm" from title and replaces it with "barTerm"
|
||||
"search": "fooTerm",
|
||||
"replace": "barTerm"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
:::note
|
||||
|
||||
If the value of the field (title, an artist, album) is an empty string after transforming then the field is **removed.**
|
||||
|
||||
:::
|
||||
|
||||
|
||||
## Examples
|
||||
|
||||
### Usage with `when` condition
|
||||
|
||||
Using `when` for [Conditional Modification](/configuration/transforms#conditional-modification)
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
|
||||
```json5
|
||||
// On search-replace in title...
|
||||
// IF artist matches "Elephant Gym"
|
||||
// THEN Run regex search-replace ELSE skip this rule
|
||||
//
|
||||
// Run live|remastered regex remove on title
|
||||
{
|
||||
"title": [
|
||||
{
|
||||
"search": "/\\s\\-\\s滾石40\\s滾石撞樂隊\\s40團拚經典(.+)$/i",
|
||||
"replace": "",
|
||||
"when": [
|
||||
{
|
||||
"artist": "/Elephant Gym/"
|
||||
}
|
||||
]
|
||||
},
|
||||
"/(\\s\\-\\s|\\s)(feat\\.(.+)|live|remastered(.+))$/i"
|
||||
],
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### Remove phrase from Title in all new Plays
|
||||
|
||||
Removes the phrase `(Album Version)` from the Title of a Play
|
||||
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
|
||||
```json5
|
||||
{
|
||||
"title": [
|
||||
"(Album Version)"
|
||||
]
|
||||
}
|
||||
|
||||
```
|
||||
</details>
|
||||
|
||||
### Remove all parenthesized content from the end of a title
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
```json5
|
||||
{
|
||||
"compare": {
|
||||
"candidate": {
|
||||
"title": [
|
||||
"/(\(.+\))\s*$/"
|
||||
]
|
||||
},
|
||||
"existing": {
|
||||
"title": [
|
||||
"/(\(.+\))\s*$/"
|
||||
]
|
||||
},
|
||||
},
|
||||
}
|
||||
```
|
||||
</details>
|
||||
|
||||
### Rename misspelled artist in all new Plays
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
```json5
|
||||
{
|
||||
"artists": [
|
||||
{
|
||||
"search": "Boz Skaggs",
|
||||
"replace": "Boz Scaggs"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
</details>
|
||||
|
||||
### Remove "Various Artists" albums in all new Plays
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
```json5
|
||||
{
|
||||
"album": [
|
||||
{
|
||||
"search": "Various Artists",
|
||||
"replace": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
</details>
|
||||
|
||||
### Extract primary Artist from delimited, multi-Artist string
|
||||
|
||||
<details>
|
||||
|
||||
When the Artist string is actually a multi-artist, delimited string, this search-and-replace will replace the string with just the first artist found.
|
||||
|
||||
Ex
|
||||
|
||||
```
|
||||
My Artist One / My Artist Two / Another Guy
|
||||
My Artist One
|
||||
```
|
||||
|
||||
Artists are delimited with a spaced forward slash (`/`) in the regex below. Replace the contents of the `delim` capture group with the delimiter for your use case. Some more common scenarios:
|
||||
|
||||
* `(?<delim>\\/)` No spaces between slash IE `My Artist One/My Artist Two/Another Guy`
|
||||
* `(?<delim>\\s*\\\\\s*)` Backslash instead of forward slash IE `My Artist One \ My Artist Two \ Another Guy`
|
||||
* `(?<delim>,)` Comma IE `My Artist One, My Artist Two, Another Guy`
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Example</summary>
|
||||
```json
|
||||
{
|
||||
"artists": [
|
||||
{
|
||||
"search": "(.*?)(?<delim>\\s*\\/\\s*)(.*$)",
|
||||
"replace": "$1"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
</details>
|
||||
|
||||
</details>
|
||||
@@ -1,6 +1,7 @@
|
||||
import type * as Preset from '@docusaurus/preset-classic';
|
||||
import type { Config } from '@docusaurus/types';
|
||||
import * as themes from 'prism-react-renderer';
|
||||
import glossaryPlugin from 'docusaurus-plugin-glossary';
|
||||
//import sidebars from './sidebars';
|
||||
|
||||
const config: Config = {
|
||||
@@ -50,10 +51,15 @@ const config: Config = {
|
||||
docs: {
|
||||
sidebarPath: './sidebars.ts',
|
||||
routeBasePath: '/',
|
||||
// Please change this to your repo.
|
||||
// Remove this to remove the "edit this page" links.
|
||||
editUrl:
|
||||
'https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/',
|
||||
remarkPlugins: [
|
||||
glossaryPlugin.getRemarkPlugin(
|
||||
{
|
||||
glossaryPath: 'glossary/glossary.json',
|
||||
routePath: '/glossary',
|
||||
},
|
||||
{ siteDir: __dirname }
|
||||
),
|
||||
],
|
||||
},
|
||||
// blog: {
|
||||
// showReadingTime: true,
|
||||
@@ -121,7 +127,14 @@ const config: Config = {
|
||||
},
|
||||
]
|
||||
}
|
||||
]
|
||||
],
|
||||
[
|
||||
'docusaurus-plugin-glossary',
|
||||
{
|
||||
glossaryPath: 'glossary/glossary.json',
|
||||
routePath: '/glossary',
|
||||
},
|
||||
],
|
||||
],
|
||||
themeConfig:
|
||||
{
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"description": "A collection of technical terms and their definitions",
|
||||
"terms": [
|
||||
{
|
||||
"term": "AIO",
|
||||
"abbreviation": "All-In-One",
|
||||
"definition": "Configuration type that uses the config.json file to configure Sources, Clients, and all application configuration together",
|
||||
"relatedTerms": ["File Config", "ENV Config"]
|
||||
},
|
||||
{
|
||||
"term": "Client",
|
||||
"definition": "Services that recieve/store Scrobbles sent by Multi-Scrobbler.",
|
||||
"relatedTerms": ["Source"]
|
||||
},
|
||||
{
|
||||
"term": "ENV Config",
|
||||
"definition": "Configuration type that uses environmental variales to configure Sources and Clients",
|
||||
"relatedTerms": ["AIO", "File Config"]
|
||||
},
|
||||
{
|
||||
"term": "ENV",
|
||||
"abbreviation": "Environmental Variable",
|
||||
"definition": "A variable passed to a docker container or present in the operating environment.",
|
||||
"relatedTerms": ["Env Config"]
|
||||
},
|
||||
{
|
||||
"term": "File Config",
|
||||
"definition": "Configuration type that uses individual JSON files to configure Sources and Clients",
|
||||
"relatedTerms": ["AIO", "ENV Config"]
|
||||
},
|
||||
{
|
||||
"term": "MS",
|
||||
"abbreviation": "Multi-Scrobbler",
|
||||
"definition": "Multi-Scrobbler"
|
||||
},
|
||||
{
|
||||
"term": "Play",
|
||||
"definition": "The data related to a specific piece of audio you listened to, without a timestamp. EX: Artist, Title, Album...",
|
||||
"relatedTerms": ["Scrobble"]
|
||||
},
|
||||
{
|
||||
"term": "Source",
|
||||
"definition": "A service/app that Multi-Scrobbler can get listening activity/history from.",
|
||||
"relatedTerms": ["Client"]
|
||||
},
|
||||
{
|
||||
"term": "Scrobble",
|
||||
"definition": "The data (Play) and timestamp related to a certain song you listened to at a specific time"
|
||||
},
|
||||
{
|
||||
"term": "Scrobbling",
|
||||
"definition": "The verb for sending a Scrobble to one or more Clients",
|
||||
"relatedTerms": ["Scrobble", "Client"]
|
||||
}
|
||||
]
|
||||
}
|
||||
Generated
+2911
-1791
File diff suppressed because it is too large
Load Diff
+12
-11
@@ -14,26 +14,27 @@
|
||||
"typecheck": "tsc"
|
||||
},
|
||||
"dependencies": {
|
||||
"@docusaurus/core": "3.8.1",
|
||||
"@docusaurus/faster": "3.8.1",
|
||||
"@docusaurus/plugin-client-redirects": "^3.8.1",
|
||||
"@docusaurus/preset-classic": "3.8.1",
|
||||
"@easyops-cn/docusaurus-search-local": "0.51.1",
|
||||
"@docusaurus/core": "3.9.2",
|
||||
"@docusaurus/faster": "3.9.2",
|
||||
"@docusaurus/plugin-client-redirects": "^3.9.2",
|
||||
"@docusaurus/preset-classic": "3.9.2",
|
||||
"@easyops-cn/docusaurus-search-local": "0.52.2",
|
||||
"@mdx-js/react": "^3.0.0",
|
||||
"clsx": "^2.0.0",
|
||||
"docusaurus-json-schema-plugin": "^1.14.0",
|
||||
"docusaurus-json-schema-plugin": "^1.15.0",
|
||||
"docusaurus-plugin-glossary": "^3.0.0",
|
||||
"docusaurus-theme-github-codeblock": "^2.0.2",
|
||||
"json5": "^2.2.3",
|
||||
"micromark-extension-directive": "^3.0.1",
|
||||
"prism-react-renderer": "^2.3.0",
|
||||
"raw-loader": "^4.0.2",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0"
|
||||
"react": "^18.0.0",
|
||||
"react-dom": "^18.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@docusaurus/module-type-aliases": "^3.6.3",
|
||||
"@docusaurus/tsconfig": "^3.6.3",
|
||||
"@docusaurus/types": "^3.6.3",
|
||||
"@docusaurus/module-type-aliases": "^3.9.2",
|
||||
"@docusaurus/tsconfig": "^3.9.2",
|
||||
"@docusaurus/types": "^3.9.2",
|
||||
"@types/react": "^18.2.29",
|
||||
"typescript": "~5.6.2"
|
||||
},
|
||||
|
||||
@@ -5,186 +5,45 @@ import {
|
||||
import deepEqual from 'fast-deep-equal';
|
||||
import { Simulate } from "react-dom/test-utils";
|
||||
import { PlayObject } from "../../core/Atomic.js";
|
||||
import { buildTrackString, truncateStringToLength } from "../../core/StringUtils.js";
|
||||
|
||||
import {
|
||||
configPartsToStrongParts, countRegexes,
|
||||
transformPlayUsingParts
|
||||
} from "../utils/PlayTransformUtils.js";
|
||||
import { hasNodeNetworkException } from "./errors/NodeErrors.js";
|
||||
import { hasUpstreamError } from "./errors/UpstreamError.js";
|
||||
import {
|
||||
ConditionalSearchAndReplaceRegExp,
|
||||
PlayTransformParts, PlayTransformPartsArray,
|
||||
PlayTransformRules,
|
||||
TRANSFORM_HOOK,
|
||||
TransformHook
|
||||
} from "./infrastructure/Atomic.js";
|
||||
import { buildTrackString } from "../../core/StringUtils.js";
|
||||
import { CommonClientConfig } from "./infrastructure/config/client/index.js";
|
||||
import { CommonSourceConfig } from "./infrastructure/config/source/index.js";
|
||||
import { TransformRulesError } from "./errors/MSErrors.js";
|
||||
import {
|
||||
PlayTransformRules,
|
||||
StageConfig,
|
||||
TRANSFORM_HOOK,
|
||||
TransformHook
|
||||
} from "./infrastructure/Transform.js";
|
||||
import AbstractInitializable from "./AbstractInitializable.js";
|
||||
import play = Simulate.play;
|
||||
import { WebhookPayload } from "./infrastructure/config/health/webhooks.js";
|
||||
import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, TransformRulesError } from "./errors/MSErrors.js";
|
||||
import { messageWithCauses, messageWithCausesTruncatedDefault } from "../utils/ErrorUtils.js";
|
||||
import TransformerManager from "./transforms/TransformerManager.js";
|
||||
import { getRoot } from "../ioc.js";
|
||||
|
||||
export default abstract class AbstractComponent {
|
||||
requiresAuth: boolean = false;
|
||||
requiresAuthInteraction: boolean = false;
|
||||
authed: boolean = false;
|
||||
authFailure?: boolean;
|
||||
export default abstract class AbstractComponent extends AbstractInitializable {
|
||||
|
||||
buildOK?: boolean | null;
|
||||
connectionOK?: boolean | null;
|
||||
cacheOK?: boolean | null;
|
||||
|
||||
initializing: boolean = false;
|
||||
|
||||
config: CommonClientConfig | CommonSourceConfig;
|
||||
declare config: CommonClientConfig | CommonSourceConfig;
|
||||
|
||||
transformRules: PlayTransformRules = {};
|
||||
regexCache!: ReturnType<typeof cacheFunctions>;
|
||||
|
||||
logger: Logger;
|
||||
componentLogger?: Logger;
|
||||
protected transformManager: TransformerManager;
|
||||
|
||||
protected constructor(config: CommonClientConfig | CommonSourceConfig) {
|
||||
this.config = config;
|
||||
super(config);
|
||||
this.transformManager = getRoot().items.transformerManager;
|
||||
}
|
||||
|
||||
public abstract notify(payload: WebhookPayload): Promise<void>;
|
||||
|
||||
protected abstract getIdentifier(): string;
|
||||
|
||||
initialize = async (options: {force?: boolean, notify?: boolean, notifyTitle?: string} = {}) => {
|
||||
|
||||
const {force = false, notify = false, notifyTitle = 'Init Error'} = options;
|
||||
|
||||
this.logger.debug('Attempting to initialize...');
|
||||
protected postCache(): Promise<void> {
|
||||
try {
|
||||
this.initializing = true;
|
||||
if(this.componentLogger === undefined) {
|
||||
await this.buildComponentLogger();
|
||||
}
|
||||
await this.buildInitData(force);
|
||||
await this.parseCache(force);
|
||||
this.buildTransformRules();
|
||||
await this.checkConnection(force);
|
||||
await this.testAuth(force);
|
||||
this.logger.info('Fully Initialized!');
|
||||
try {
|
||||
await this.postInitialize();
|
||||
} catch (e) {
|
||||
throw new PostInitError('Error occurred during post-initialization hook', {cause: e});
|
||||
}
|
||||
return true;
|
||||
} catch(e) {
|
||||
if(notify) {
|
||||
await this.notify({title: `${this.getIdentifier()} - ${notifyTitle}`, message: truncateStringToLength(500)(messageWithCausesTruncatedDefault(e)), priority: 'error'});
|
||||
}
|
||||
throw new Error('Initialization failed', {cause: e});
|
||||
} finally {
|
||||
this.initializing = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async buildComponentLogger() {
|
||||
await this.doBuildComponentLogger();
|
||||
return;
|
||||
}
|
||||
|
||||
protected async doBuildComponentLogger() {
|
||||
return;
|
||||
}
|
||||
|
||||
tryInitialize = async (options: {force?: boolean, notify?: boolean, notifyTitle?: string} = {}) => {
|
||||
if(this.initializing) {
|
||||
throw new Error(`Already trying to initialize, cannot attempt while an existing initialization attempt is running.`)
|
||||
}
|
||||
try {
|
||||
return await this.initialize(options);
|
||||
return;
|
||||
} catch (e) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
public async parseCache(force: boolean = false) {
|
||||
if(this.cacheOK) {
|
||||
if(!force) {
|
||||
return;
|
||||
}
|
||||
this.logger.debug('Cache OK but step was forced');
|
||||
}
|
||||
try {
|
||||
const res = await this.doParseCache();
|
||||
if(res === undefined) {
|
||||
this.cacheOK = null;
|
||||
this.logger.debug('No cache to parse.');
|
||||
return;
|
||||
}
|
||||
if (res === true) {
|
||||
this.logger.verbose('Parsing caching succeeded');
|
||||
} else if (typeof res === 'string') {
|
||||
this.logger.verbose(`Parsing caching succeeded => ${res}`);
|
||||
}
|
||||
this.cacheOK = true;
|
||||
} catch (e) {
|
||||
this.cacheOK = false;
|
||||
throw new ParseCacheError('Parsing cache for initialization failed', {cause: e});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build or parse any cache required for this Component
|
||||
*
|
||||
* * Return undefined if not possible or not required
|
||||
* * Return TRUE if build succeeded
|
||||
* * Return string if build succeeded and should log result
|
||||
* * Throw error on failure
|
||||
* */
|
||||
protected async doParseCache(): Promise<true | string | undefined> {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
public async buildInitData(force: boolean = false) {
|
||||
if(this.buildOK) {
|
||||
if(!force) {
|
||||
return;
|
||||
}
|
||||
this.logger.debug('Build OK but step was forced');
|
||||
}
|
||||
try {
|
||||
const res = await this.doBuildInitData();
|
||||
if(res === undefined) {
|
||||
this.buildOK = null;
|
||||
this.logger.debug('No required data to build.');
|
||||
return;
|
||||
}
|
||||
if (res === true) {
|
||||
this.logger.verbose('Building required data init succeeded');
|
||||
} else if (typeof res === 'string') {
|
||||
this.logger.verbose(`Building required data init succeeded => ${res}`);
|
||||
}
|
||||
this.buildOK = true;
|
||||
} catch (e) {
|
||||
this.buildOK = false;
|
||||
throw new BuildDataError('Building required data for initialization failed', {cause: e});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build any data/config/objects required for this Source to communicate with upstream service
|
||||
*
|
||||
* * Return undefined if not possible or not required
|
||||
* * Return TRUE if build succeeded
|
||||
* * Return string if build succeeded and should log result
|
||||
* * Throw error on failure
|
||||
* */
|
||||
protected async doBuildInitData(): Promise<true | string | undefined> {
|
||||
return;
|
||||
}
|
||||
|
||||
public buildTransformRules() {
|
||||
this.logger.debug('Building transformer rules...');
|
||||
try {
|
||||
this.doBuildTransformRules();
|
||||
} catch (e) {
|
||||
@@ -192,8 +51,8 @@ export default abstract class AbstractComponent {
|
||||
throw new TransformRulesError('Could not build playTransform rules. Check your configuration is valid.', {cause: e});
|
||||
}
|
||||
try {
|
||||
const ruleCount = countRegexes(this.transformRules);
|
||||
this.regexCache = cacheFunctions(ruleCount);
|
||||
//const ruleCount = countRegexes(this.transformRules);
|
||||
this.regexCache = cacheFunctions(200);
|
||||
} catch (e) {
|
||||
this.logger.warn(new TransformRulesError('Failed to count number of rule regexes for caching but will continue will fallback to 100', {cause: e}));
|
||||
}
|
||||
@@ -207,6 +66,7 @@ export default abstract class AbstractComponent {
|
||||
} = this.config;
|
||||
|
||||
if (playTransform === undefined) {
|
||||
this.logger.debug(`No rules found under property 'playTransform'`);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -224,30 +84,54 @@ export default abstract class AbstractComponent {
|
||||
existing,
|
||||
postCompare;
|
||||
|
||||
const builtHooks: string[] = [];
|
||||
const emptyHooks: string[] = [];
|
||||
try {
|
||||
preCompare = configPartsToStrongParts(preConfig)
|
||||
preCompare = this.transformPartToStrong(preConfig);
|
||||
if(preCompare === undefined) {
|
||||
emptyHooks.push('preCompare')
|
||||
} else {
|
||||
builtHooks.push('preCompare');
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error('preCompare was not valid', {cause: e});
|
||||
}
|
||||
|
||||
try {
|
||||
candidate = configPartsToStrongParts(candidateConfig)
|
||||
candidate = this.transformPartToStrong(candidateConfig);
|
||||
if(candidate === undefined) {
|
||||
emptyHooks.push('candidate')
|
||||
} else {
|
||||
builtHooks.push('candidate');
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error('candidate was not valid', {cause: e});
|
||||
}
|
||||
|
||||
try {
|
||||
existing = configPartsToStrongParts(existingConfig)
|
||||
existing = this.transformPartToStrong(existingConfig);
|
||||
if(existing === undefined) {
|
||||
emptyHooks.push('existing')
|
||||
} else {
|
||||
builtHooks.push('existing');
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error('existing was not valid', {cause: e});
|
||||
}
|
||||
|
||||
try {
|
||||
postCompare = configPartsToStrongParts(postConfig)
|
||||
postCompare = this.transformPartToStrong(postConfig);
|
||||
if(postCompare === undefined) {
|
||||
emptyHooks.push('postCompare')
|
||||
} else {
|
||||
builtHooks.push('postCompare');
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error('postCompare was not valid', {cause: e});
|
||||
}
|
||||
|
||||
this.logger.debug(`Hooks built. Configured: ${builtHooks.join(', ')} | Empty: ${emptyHooks.join(', ')}`);
|
||||
|
||||
this.transformRules = {
|
||||
preCompare,
|
||||
compare: {
|
||||
@@ -258,112 +142,24 @@ export default abstract class AbstractComponent {
|
||||
}
|
||||
}
|
||||
|
||||
public async checkConnection(force: boolean = false) {
|
||||
if(this.connectionOK) {
|
||||
if(!force) {
|
||||
return;
|
||||
}
|
||||
this.logger.debug('Connection OK but step was forced')
|
||||
}
|
||||
try {
|
||||
const res = await this.doCheckConnection();
|
||||
if (res === undefined) {
|
||||
this.logger.debug('Connection check was not required.');
|
||||
this.connectionOK = null;
|
||||
return;
|
||||
} else if (res === true) {
|
||||
this.logger.verbose('Connection check succeeded');
|
||||
} else {
|
||||
this.logger.verbose(`Connection check succeeded => ${res}`);
|
||||
}
|
||||
this.connectionOK = true;
|
||||
} catch (e) {
|
||||
this.connectionOK = false;
|
||||
throw new ConnectionCheckError('Communicating with upstream service failed', {cause: e});
|
||||
protected transformPartToStrong(data: any) {
|
||||
if(data === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
// default to user transform type for backward compatibility
|
||||
const partArr = (Array.isArray(data) ? data : [data]).map(x => ({type: 'user', ...x}));
|
||||
|
||||
return partArr.map(x => this.transformManager.parseTransformerConfig(x));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check Scrobbler upstream API/connection to ensure we can communicate
|
||||
*
|
||||
* * Return undefined if not possible or not required to check
|
||||
* * Return TRUE if communication succeeded
|
||||
* * Return string if communication succeeded and should log result
|
||||
* * Throw error if communication failed
|
||||
* */
|
||||
protected async doCheckConnection(): Promise<true | string | undefined> {
|
||||
return;
|
||||
}
|
||||
|
||||
authGated = () => this.requiresAuth && !this.authed
|
||||
|
||||
canTryAuth = () => this.isUsable() && this.authGated() && this.authFailure !== true
|
||||
|
||||
canAuthUnattended = () => !this.authGated || !this.requiresAuthInteraction || (this.requiresAuthInteraction && !this.authFailure);
|
||||
|
||||
protected doAuthentication = async (): Promise<boolean> => this.authed
|
||||
|
||||
// default init function, should be overridden if auth stage is required
|
||||
testAuth = async (force: boolean = false) => {
|
||||
if(!this.requiresAuth) {
|
||||
return;
|
||||
}
|
||||
if(this.authed) {
|
||||
if(!force) {
|
||||
return;
|
||||
}
|
||||
this.logger.debug('Auth OK but step was forced');
|
||||
}
|
||||
|
||||
if(this.authFailure) {
|
||||
if(!force) {
|
||||
if(this.requiresAuthInteraction) {
|
||||
throw new AuthCheckError('Authentication failure: Will not retry auth because user interaction is required for authentication');
|
||||
}
|
||||
throw new AuthCheckError('Authentication failure: Will not retry auth because authentication previously failed and must be reauthenticated');
|
||||
}
|
||||
this.logger.debug('Auth previously failed for non upstream/network reasons but retry is being forced');
|
||||
}
|
||||
|
||||
try {
|
||||
this.authed = await this.doAuthentication();
|
||||
this.authFailure = !this.authed;
|
||||
} catch (e) {
|
||||
// only signal as auth failure if error was NOT either a node network error or a non-showstopping upstream error
|
||||
this.authFailure = !(hasNodeNetworkException(e) || hasUpstreamError(e, false));
|
||||
this.authed = false;
|
||||
throw new AuthCheckError(`Authentication test failed!${this.authFailure === false ? ' Due to a network issue. Will retry authentication on next heartbeat.' : ''}`, {cause: e});
|
||||
}
|
||||
}
|
||||
|
||||
public isReady() {
|
||||
return (this.buildOK === null || this.buildOK === true) &&
|
||||
(this.connectionOK === null || this.connectionOK === true)
|
||||
&& !this.authGated();
|
||||
}
|
||||
|
||||
public isUsable() {
|
||||
return (this.buildOK === null || this.buildOK === true) &&
|
||||
(this.connectionOK === null || this.connectionOK === true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Override to perform some action after successfully initializing
|
||||
*
|
||||
* Results will be try-catched and swallowed/logged if an error is thrown. This will not affect initialized state.
|
||||
* */
|
||||
protected async postInitialize(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
public transformPlay = (play: PlayObject, hookType: TransformHook, log?: boolean) => {
|
||||
public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean) => {
|
||||
|
||||
let logger: Logger;
|
||||
const labels = ['Play Transform', hookType];
|
||||
const getLogger = () => logger !== undefined ? logger : childLogger(this.logger, labels);
|
||||
|
||||
try {
|
||||
let hook: PlayTransformPartsArray<ConditionalSearchAndReplaceRegExp> | undefined;
|
||||
let hook: StageConfig[];
|
||||
|
||||
switch (hookType) {
|
||||
case TRANSFORM_HOOK.preCompare:
|
||||
@@ -385,19 +181,46 @@ export default abstract class AbstractComponent {
|
||||
}
|
||||
|
||||
let transformedPlay: PlayObject = play;
|
||||
const transformDetails: string[] = [];
|
||||
let transformDetails: string[] = [];
|
||||
for(const hookItem of hook) {
|
||||
const newTransformedPlay = transformPlayUsingParts(transformedPlay, hookItem, {
|
||||
logger: getLogger,
|
||||
regex: {
|
||||
searchAndReplace: this.regexCache.searchAndReplace,
|
||||
testMaybeRegex: this.regexCache.testMaybeRegex,
|
||||
|
||||
const {
|
||||
onSuccess = 'continue',
|
||||
onFailure = 'stop',
|
||||
failureReturnPartial = false
|
||||
} = hookItem;
|
||||
|
||||
let newTransformedPlay: PlayObject;
|
||||
let err: Error;
|
||||
try {
|
||||
newTransformedPlay = await this.transformManager.handleStage(hookItem, transformedPlay);
|
||||
} catch (e) {
|
||||
err = e;
|
||||
}
|
||||
|
||||
if(err !== undefined) {
|
||||
if(onFailure === 'continue') {
|
||||
this.logger.warn(new Error('A transform encountered an error but continuing due to onFailure: continue', {cause: err}));
|
||||
} else {
|
||||
this.logger.error(new Error('Transform encountered an error', {cause: err}));
|
||||
if(!failureReturnPartial) {
|
||||
// rewind to original play so we don't return partial transform
|
||||
transformedPlay = play;
|
||||
transformDetails = [];
|
||||
}
|
||||
break;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if(!deepEqual(newTransformedPlay, transformedPlay)) {
|
||||
transformDetails.push(buildTrackString(transformedPlay, {include: ['artist', 'track', 'album']}));
|
||||
transformDetails.push(`${hookItem.type} - ${buildTrackString(transformedPlay, {include: ['artist', 'track', 'album']})}`);
|
||||
}
|
||||
transformedPlay = newTransformedPlay;
|
||||
|
||||
if(err === undefined && onSuccess === 'stop') {
|
||||
this.logger.debug('Stopping transform due to onSuccess: stop');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if(transformDetails.length > 0) {
|
||||
@@ -418,8 +241,4 @@ export default abstract class AbstractComponent {
|
||||
return play;
|
||||
}
|
||||
}
|
||||
|
||||
public additionalApiData(): Record<string, any> {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
import { childLogger, Logger } from "@foxxmd/logging";
|
||||
import { Simulate } from "react-dom/test-utils";
|
||||
import {truncateStringToLength } from "../../core/StringUtils.js";
|
||||
import { hasNodeNetworkException } from "./errors/NodeErrors.js";
|
||||
import { hasUpstreamError } from "./errors/UpstreamError.js";
|
||||
import play = Simulate.play;
|
||||
import { WebhookPayload } from "./infrastructure/config/health/webhooks.js";
|
||||
import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, StageError, TransformRulesError } from "./errors/MSErrors.js";
|
||||
import { messageWithCauses, messageWithCausesTruncatedDefault } from "../utils/ErrorUtils.js";
|
||||
|
||||
export default abstract class AbstractInitializable {
|
||||
requiresAuth: boolean = false;
|
||||
requiresAuthInteraction: boolean = false;
|
||||
authed: boolean = false;
|
||||
authFailure?: boolean;
|
||||
|
||||
buildOK?: boolean | null;
|
||||
connectionOK?: boolean | null;
|
||||
cacheOK?: boolean | null;
|
||||
|
||||
initializing: boolean = false;
|
||||
|
||||
config: Record<string, any>;
|
||||
|
||||
logger: Logger;
|
||||
componentLogger?: Logger;
|
||||
|
||||
protected constructor(config: Record<string, any>) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public abstract notify(payload: WebhookPayload): Promise<void>;
|
||||
|
||||
protected abstract getIdentifier(): string;
|
||||
|
||||
initialize = async (options: {force?: boolean, notify?: boolean, notifyTitle?: string} = {}) => {
|
||||
|
||||
const {force = false, notify = false, notifyTitle = 'Init Error'} = options;
|
||||
|
||||
this.logger.debug('Attempting to initialize...');
|
||||
try {
|
||||
this.initializing = true;
|
||||
if(this.componentLogger === undefined) {
|
||||
await this.buildComponentLogger();
|
||||
}
|
||||
await this.buildInitData(force);
|
||||
await this.parseCache(force);
|
||||
try {
|
||||
await this.postCache();
|
||||
} catch (e) {
|
||||
if(e instanceof StageError) {
|
||||
throw e;
|
||||
} else {
|
||||
throw new Error('Error occurred during post-cache hook', {cause: e});
|
||||
}
|
||||
}
|
||||
await this.checkConnection(force);
|
||||
await this.testAuth(force);
|
||||
this.logger.info('Fully Initialized!');
|
||||
try {
|
||||
await this.postInitialize();
|
||||
} catch (e) {
|
||||
throw new PostInitError('Error occurred during post-initialization hook', {cause: e});
|
||||
}
|
||||
return true;
|
||||
} catch(e) {
|
||||
if(notify) {
|
||||
await this.notify({title: `${this.getIdentifier()} - ${notifyTitle}`, message: truncateStringToLength(500)(messageWithCausesTruncatedDefault(e)), priority: 'error'});
|
||||
}
|
||||
throw new Error('Initialization failed', {cause: e});
|
||||
} finally {
|
||||
this.initializing = false;
|
||||
}
|
||||
}
|
||||
|
||||
protected async buildComponentLogger() {
|
||||
await this.doBuildComponentLogger();
|
||||
return;
|
||||
}
|
||||
|
||||
protected async doBuildComponentLogger() {
|
||||
return;
|
||||
}
|
||||
|
||||
tryInitialize = async (options: {force?: boolean, notify?: boolean, notifyTitle?: string} = {}) => {
|
||||
if(this.initializing) {
|
||||
throw new Error(`Already trying to initialize, cannot attempt while an existing initialization attempt is running.`)
|
||||
}
|
||||
try {
|
||||
return await this.initialize(options);
|
||||
} catch (e) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
public async parseCache(force: boolean = false) {
|
||||
if(this.cacheOK) {
|
||||
if(!force) {
|
||||
return;
|
||||
}
|
||||
this.logger.debug('Cache OK but step was forced');
|
||||
}
|
||||
try {
|
||||
const res = await this.doParseCache();
|
||||
if(res === undefined) {
|
||||
this.cacheOK = null;
|
||||
this.logger.debug('No cache to parse.');
|
||||
return;
|
||||
}
|
||||
if (res === true) {
|
||||
this.logger.verbose('Parsing caching succeeded');
|
||||
} else if (typeof res === 'string') {
|
||||
this.logger.verbose(`Parsing caching succeeded => ${res}`);
|
||||
}
|
||||
this.cacheOK = true;
|
||||
} catch (e) {
|
||||
this.cacheOK = false;
|
||||
throw new ParseCacheError('Parsing cache for initialization failed', {cause: e});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build or parse any cache required for this Component
|
||||
*
|
||||
* * Return undefined if not possible or not required
|
||||
* * Return TRUE if build succeeded
|
||||
* * Return string if build succeeded and should log result
|
||||
* * Throw error on failure
|
||||
* */
|
||||
protected async doParseCache(): Promise<true | string | undefined> {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
protected async postCache(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
public async buildInitData(force: boolean = false) {
|
||||
if(this.buildOK) {
|
||||
if(!force) {
|
||||
return;
|
||||
}
|
||||
this.logger.debug('Build OK but step was forced');
|
||||
}
|
||||
try {
|
||||
const res = await this.doBuildInitData();
|
||||
if(res === undefined) {
|
||||
this.buildOK = null;
|
||||
this.logger.debug('No required data to build.');
|
||||
return;
|
||||
}
|
||||
if (res === true) {
|
||||
this.logger.verbose('Building required data init succeeded');
|
||||
} else if (typeof res === 'string') {
|
||||
this.logger.verbose(`Building required data init succeeded => ${res}`);
|
||||
}
|
||||
this.buildOK = true;
|
||||
} catch (e) {
|
||||
this.buildOK = false;
|
||||
throw new BuildDataError('Building required data for initialization failed', {cause: e});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build any data/config/objects required for this Source to communicate with upstream service
|
||||
*
|
||||
* * Return undefined if not possible or not required
|
||||
* * Return TRUE if build succeeded
|
||||
* * Return string if build succeeded and should log result
|
||||
* * Throw error on failure
|
||||
* */
|
||||
protected async doBuildInitData(): Promise<true | string | undefined> {
|
||||
return;
|
||||
}
|
||||
|
||||
public async checkConnection(force: boolean = false) {
|
||||
if(this.connectionOK) {
|
||||
if(!force) {
|
||||
return;
|
||||
}
|
||||
this.logger.debug('Connection OK but step was forced')
|
||||
}
|
||||
try {
|
||||
const res = await this.doCheckConnection();
|
||||
if (res === undefined) {
|
||||
this.logger.debug('Connection check was not required.');
|
||||
this.connectionOK = null;
|
||||
return;
|
||||
} else if (res === true) {
|
||||
this.logger.verbose('Connection check succeeded');
|
||||
} else {
|
||||
this.logger.verbose(`Connection check succeeded => ${res}`);
|
||||
}
|
||||
this.connectionOK = true;
|
||||
} catch (e) {
|
||||
this.connectionOK = false;
|
||||
throw new ConnectionCheckError('Communicating with upstream service failed', {cause: e});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check Scrobbler upstream API/connection to ensure we can communicate
|
||||
*
|
||||
* * Return undefined if not possible or not required to check
|
||||
* * Return TRUE if communication succeeded
|
||||
* * Return string if communication succeeded and should log result
|
||||
* * Throw error if communication failed
|
||||
* */
|
||||
protected async doCheckConnection(): Promise<true | string | undefined> {
|
||||
return;
|
||||
}
|
||||
|
||||
authGated = () => this.requiresAuth && !this.authed
|
||||
|
||||
canTryAuth = () => this.isUsable() && this.authGated() && this.authFailure !== true
|
||||
|
||||
canAuthUnattended = () => !this.authGated || !this.requiresAuthInteraction || (this.requiresAuthInteraction && !this.authFailure);
|
||||
|
||||
protected doAuthentication = async (): Promise<boolean> => this.authed
|
||||
|
||||
// default init function, should be overridden if auth stage is required
|
||||
testAuth = async (force: boolean = false) => {
|
||||
if(!this.requiresAuth) {
|
||||
return;
|
||||
}
|
||||
if(this.authed) {
|
||||
if(!force) {
|
||||
return;
|
||||
}
|
||||
this.logger.debug('Auth OK but step was forced');
|
||||
}
|
||||
|
||||
if(this.authFailure) {
|
||||
if(!force) {
|
||||
if(this.requiresAuthInteraction) {
|
||||
throw new AuthCheckError('Authentication failure: Will not retry auth because user interaction is required for authentication');
|
||||
}
|
||||
throw new AuthCheckError('Authentication failure: Will not retry auth because authentication previously failed and must be reauthenticated');
|
||||
}
|
||||
this.logger.debug('Auth previously failed for non upstream/network reasons but retry is being forced');
|
||||
}
|
||||
|
||||
try {
|
||||
this.authed = await this.doAuthentication();
|
||||
this.authFailure = !this.authed;
|
||||
} catch (e) {
|
||||
// only signal as auth failure if error was NOT either a node network error or a non-showstopping upstream error
|
||||
this.authFailure = !(hasNodeNetworkException(e) || hasUpstreamError(e, false));
|
||||
this.authed = false;
|
||||
throw new AuthCheckError(`Authentication test failed!${this.authFailure === false ? ' Due to a network issue. Will retry authentication on next heartbeat.' : ''}`, {cause: e});
|
||||
}
|
||||
}
|
||||
|
||||
public isReady() {
|
||||
return (this.buildOK === null || this.buildOK === true) &&
|
||||
(this.connectionOK === null || this.connectionOK === true)
|
||||
&& !this.authGated();
|
||||
}
|
||||
|
||||
public isUsable() {
|
||||
return (this.buildOK === null || this.buildOK === true) &&
|
||||
(this.connectionOK === null || this.connectionOK === true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Override to perform some action after successfully initializing
|
||||
*
|
||||
* Results will be try-catched and swallowed/logged if an error is thrown. This will not affect initialized state.
|
||||
* */
|
||||
protected async postInitialize(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
public additionalApiData(): Record<string, any> {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import clone from 'clone';
|
||||
import { childLogger, Logger } from '@foxxmd/logging';
|
||||
import { projectDir } from './index.js';
|
||||
import path from 'path';
|
||||
import { cacheFunctions } from "@foxxmd/regex-buddy-core";
|
||||
import { fileOrDirectoryIsWriteable } from '../utils.js';
|
||||
import { asCacheAuthProvider, asCacheMetadataProvider, asCacheScrobbleProvider, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheMetadataProvider, CacheProvider, CacheScrobbleProvider } from './infrastructure/Atomic.js';
|
||||
import { Typeson } from 'typeson';
|
||||
@@ -47,6 +48,8 @@ export class MSCache {
|
||||
cacheMetadata: Cacheable;
|
||||
cacheScrobble: Cacheable;
|
||||
cacheAuth: Cacheable;
|
||||
regexCache: ReturnType<typeof cacheFunctions>;
|
||||
cacheTransform: Cacheable;
|
||||
|
||||
logger: Logger;
|
||||
|
||||
@@ -69,6 +72,7 @@ export class MSCache {
|
||||
connection: aConn = (process.env.CACHE_AUTH_CONN ?? configDir),
|
||||
...restAuth
|
||||
} = {},
|
||||
regex = 200,
|
||||
} = config;
|
||||
|
||||
this.config = {
|
||||
@@ -86,8 +90,12 @@ export class MSCache {
|
||||
provider: aProvider,
|
||||
connection: aConn,
|
||||
...restAuth
|
||||
}
|
||||
},
|
||||
regex
|
||||
};
|
||||
|
||||
this.regexCache = cacheFunctions(this.config.regex);
|
||||
this.cacheTransform = new Cacheable({primary: initMemoryCache({lruSize: 500})});
|
||||
}
|
||||
|
||||
init = async () => {
|
||||
@@ -95,6 +103,7 @@ export class MSCache {
|
||||
//await this.initMetadataCache();
|
||||
await this.initScrobbleCache();
|
||||
await this.initAuthCache();
|
||||
//this.cacheTransform = await this.initCacheable({provider: false, memory: {lruSize: 500}}, 'transform');
|
||||
}
|
||||
|
||||
protected initCacheable = async (config: CacheConfig, cacheFor: string) => {
|
||||
@@ -109,7 +118,7 @@ export class MSCache {
|
||||
const ns = `ms-${cacheFor.toLocaleLowerCase()}`;
|
||||
|
||||
const cacheOpts: CacheableOptions = {
|
||||
primary: initMemoryCache({ namespace: ns })
|
||||
primary: initMemoryCache({ namespace: ns, lruSize: config.memory?.lruSize, ttl: config.memory?.ttl })
|
||||
}
|
||||
|
||||
let secondaryCache: Keyv | KeyvStoreAdapter | undefined;
|
||||
@@ -174,10 +183,15 @@ export class MSCache {
|
||||
|
||||
|
||||
export const initMemoryCache = (opts: Parameters<typeof createKeyv>[0] = {}): Keyv | KeyvStoreAdapter => {
|
||||
const {
|
||||
ttl = '1h',
|
||||
lruSize = 200,
|
||||
...restOpts
|
||||
} = opts;
|
||||
const memory = createKeyv({
|
||||
ttl: '1h',
|
||||
lruSize: 200,
|
||||
...opts,
|
||||
ttl,
|
||||
lruSize,
|
||||
...restOpts,
|
||||
useClone: false,
|
||||
});
|
||||
// structuredClone does not work well with dayjs https://github.com/iamkun/dayjs/issues/2236
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { Logger } from '@foxxmd/logging';
|
||||
import { SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core";
|
||||
import { Dayjs } from "dayjs";
|
||||
import { Request, Response } from "express";
|
||||
import { NextFunction, ParamsDictionary, Query } from "express-serve-static-core";
|
||||
@@ -268,6 +267,7 @@ export interface numberFormatOptions {
|
||||
}
|
||||
|
||||
export const DELIMITERS = [',','&','/','\\'];
|
||||
export const DELIMITERS_NO_AMP = [',','/','\\'];
|
||||
|
||||
export const ARTIST_WEIGHT = 0.3;
|
||||
export const TITLE_WEIGHT = 0.4;
|
||||
@@ -299,61 +299,6 @@ export type AbstractApiOptions = Record<any, any> & { logger: Logger }
|
||||
|
||||
export type keyOmit<T, U extends keyof any> = T & { [P in U]?: never }
|
||||
|
||||
export interface ConditionalSearchAndReplaceRegExp extends SearchAndReplaceRegExp{
|
||||
when?: WhenConditionsConfig
|
||||
}
|
||||
|
||||
export type ConditionalSearchAndReplaceTerm = Omit<ConditionalSearchAndReplaceRegExp, 'test'>
|
||||
|
||||
export type SearchAndReplaceTerm = string | ConditionalSearchAndReplaceTerm;
|
||||
|
||||
export type PlayTransformParts<T> = PlayTransformPartsAtomic<T[]> & { when?: WhenConditionsConfig };
|
||||
|
||||
export type PlayTransformPartsArray<T> = PlayTransformParts<T>[];
|
||||
|
||||
export type PlayTransformPartsConfig<T> = PlayTransformPartsArray<T> | PlayTransformParts<T>;
|
||||
|
||||
export interface PlayTransformPartsAtomic<T> {
|
||||
title?: T
|
||||
artists?: T
|
||||
album?: T
|
||||
}
|
||||
|
||||
export interface PlayTransformHooksConfig<T> {
|
||||
preCompare?: PlayTransformPartsConfig<T>
|
||||
compare?: {
|
||||
candidate?: PlayTransformPartsConfig<T>
|
||||
existing?: PlayTransformPartsConfig<T>
|
||||
}
|
||||
postCompare?: PlayTransformPartsConfig<T>
|
||||
}
|
||||
|
||||
export interface PlayTransformHooks<T> extends PlayTransformHooksConfig<T> {
|
||||
preCompare?: PlayTransformPartsArray<T>
|
||||
compare?: {
|
||||
candidate?: PlayTransformPartsArray<T>
|
||||
existing?: PlayTransformPartsArray<T>
|
||||
}
|
||||
postCompare?: PlayTransformPartsArray<T>
|
||||
}
|
||||
|
||||
export type PlayTransformRules = PlayTransformHooks<ConditionalSearchAndReplaceRegExp>
|
||||
|
||||
export type TransformHook = 'preCompare' | 'compare' | 'candidate' | 'existing' | 'postCompare';
|
||||
export const TRANSFORM_HOOK = {
|
||||
preCompare: 'preCompare' as TransformHook,
|
||||
candidate: 'candidate' as TransformHook,
|
||||
existing: 'existing' as TransformHook,
|
||||
postCompare: 'postCompare' as TransformHook,
|
||||
}
|
||||
export type PlayTransformConfig = PlayTransformHooksConfig<SearchAndReplaceTerm>;
|
||||
export type PlayTransformOptions = PlayTransformConfig & { log?: boolean | 'all' }
|
||||
|
||||
export type WhenParts<T> = PlayTransformPartsAtomic<T>;
|
||||
|
||||
export type WhenConditions<T> = WhenParts<T>[];
|
||||
export type WhenConditionsConfig = WhenConditions<string>;
|
||||
|
||||
export type WithRequiredProperty<Type, Key extends keyof Type> = Type & {
|
||||
[Property in Key]-?: Type[Property];
|
||||
};
|
||||
@@ -383,5 +328,10 @@ export interface CacheConfigOptions {
|
||||
metadata?: CacheMetadataConfig;
|
||||
scrobble?: CacheScrobbleConfig;
|
||||
auth?: CacheAuthConfig;
|
||||
/** Number of regex functions to cache (LRU)
|
||||
*
|
||||
* @default 200
|
||||
*/
|
||||
regex?: number
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core";
|
||||
|
||||
export interface ConditionalSearchAndReplaceRegExp extends SearchAndReplaceRegExp, Whennable {
|
||||
}
|
||||
|
||||
export type ConditionalSearchAndReplaceTerm = Omit<ConditionalSearchAndReplaceRegExp, 'test'>
|
||||
export type SearchAndReplaceTerm = string | ConditionalSearchAndReplaceTerm;
|
||||
export type ExternalMetadataTerm = boolean | undefined | Whennable;
|
||||
|
||||
export type PlayTransformParts<T, Y = MaybeStageTyped> = Extract<PlayTransformStage<T>, Y> & Whennable;
|
||||
//export type PlayTransformUserParts<T> = PlayTransformUserStage<T[]> & { when?: WhenConditionsConfig };
|
||||
//export type PlayTransformMetaParts<T = ExternalMetadataTerm> = PlayTransformMetadataStage<T> & { when?: WhenConditionsConfig };
|
||||
export type PlayTransformPartsArray<T, Y = MaybeStageTyped> = PlayTransformParts<T, Y>[];
|
||||
|
||||
/** Represents the weakly-defined user config. May be an array of parts or one parts object */
|
||||
export type PlayTransformPartsConfig<T, Y = MaybeStageTyped> = PlayTransformPartsArray<T, Y> | PlayTransformParts<T, Y>;
|
||||
|
||||
export interface PlayTransformPartsAtomic<T> {
|
||||
title?: T
|
||||
artists?: T
|
||||
album?: T
|
||||
}
|
||||
|
||||
export type StageTypeMetadata = 'spotify' | 'listenbrainz' | 'native';
|
||||
export type StageTypeUser = 'user';
|
||||
export type StageType = StageTypeMetadata | StageTypeUser | string;
|
||||
export const STAGE_TYPES_USER: StageTypeUser[] = ['user'];
|
||||
export const STAGE_TYPES_METADATA: StageTypeMetadata[] = ['spotify','listenbrainz','native'];
|
||||
export const STAGE_TYPES: StageType[] = [...STAGE_TYPES_METADATA, ...STAGE_TYPES_USER];
|
||||
|
||||
export interface StageTyped {
|
||||
type: StageType
|
||||
}
|
||||
|
||||
export interface NotStageTyped {
|
||||
type?: never
|
||||
}
|
||||
|
||||
export type MaybeStageTyped = StageTyped | NotStageTyped;
|
||||
|
||||
export interface StageTypedConfig {
|
||||
type: StageType
|
||||
}
|
||||
|
||||
export interface Whennable {
|
||||
when?: WhenConditionsConfig
|
||||
}
|
||||
|
||||
export type FlowControlTerm = 'continue' | 'stop'
|
||||
|
||||
export interface FlowControl {
|
||||
onSuccess: FlowControlTerm
|
||||
onFailure: FlowControlTerm
|
||||
failureReturnPartial: boolean
|
||||
}
|
||||
|
||||
export interface StageConfig extends StageTypedConfig, Whennable, Partial<FlowControl> {}
|
||||
|
||||
export interface AtomicStageConfig<T> extends StageConfig, PlayTransformPartsAtomic<T> {}
|
||||
|
||||
export interface PlayTransformStageTyped<T> extends PlayTransformPartsAtomic<T> {
|
||||
type: StageType
|
||||
}
|
||||
|
||||
export interface PlayTransformMetadataStage extends StageConfig, PlayTransformPartsAtomic<ExternalMetadataTerm> {
|
||||
score?: number
|
||||
// all?: ExternalMetadataTerm
|
||||
type: StageTypeMetadata
|
||||
}
|
||||
|
||||
export interface PlayTransformUserStage<T> extends StageConfig, PlayTransformPartsAtomic<T> {
|
||||
type: StageTypeUser
|
||||
}
|
||||
|
||||
export interface PlayTransformNativeStage extends StageConfig, PlayTransformPartsAtomic<ExternalMetadataTerm> {
|
||||
type: 'native'
|
||||
}
|
||||
|
||||
export interface PlayTransformGenericStage<T> extends StageConfig, PlayTransformPartsAtomic<T> {
|
||||
type: string
|
||||
}
|
||||
|
||||
export type UntypedPlayTransformUserStage<T> = Omit<PlayTransformUserStage<T>, 'type'> & {type?: never};
|
||||
|
||||
export type PlayTransformStage<T> = PlayTransformMetadataStage | PlayTransformUserStage<T> | PlayTransformNativeStage | UntypedPlayTransformUserStage<T> | PlayTransformGenericStage<any>;
|
||||
|
||||
/** Represents the plain json user-configured structure (input) */
|
||||
export interface PlayTransformHooksConfig<T> {
|
||||
preCompare?: PlayTransformPartsConfig<T>
|
||||
compare?: {
|
||||
candidate?: PlayTransformPartsConfig<T>
|
||||
existing?: PlayTransformPartsConfig<T>
|
||||
}
|
||||
postCompare?: PlayTransformPartsConfig<T>
|
||||
}
|
||||
|
||||
/** Represents the final, strongly-typed transform configuration used during runtime */
|
||||
export interface PlayTransformHooks<T> extends PlayTransformHooksConfig<T> {
|
||||
preCompare?: PlayTransformPartsArray<T, StageTyped>
|
||||
compare?: {
|
||||
candidate?: PlayTransformPartsArray<T, StageTyped>
|
||||
existing?: PlayTransformPartsArray<T, StageTyped>
|
||||
}
|
||||
postCompare?: PlayTransformPartsArray<T, StageTyped>
|
||||
}
|
||||
|
||||
export type PlayTransformRules = PlayTransformHooks<ConditionalSearchAndReplaceRegExp[] | ExternalMetadataTerm>
|
||||
export type TransformHook = 'preCompare' | 'compare' | 'candidate' | 'existing' | 'postCompare';
|
||||
export const TRANSFORM_HOOK = {
|
||||
preCompare: 'preCompare' as TransformHook,
|
||||
candidate: 'candidate' as TransformHook,
|
||||
existing: 'existing' as TransformHook,
|
||||
postCompare: 'postCompare' as TransformHook,
|
||||
}
|
||||
export type PlayTransformConfig = PlayTransformHooksConfig<SearchAndReplaceTerm[] | ExternalMetadataTerm>;
|
||||
export type PlayTransformOptions = PlayTransformConfig & { log?: boolean | 'all' }
|
||||
export type WhenParts<T> = PlayTransformPartsAtomic<T>;
|
||||
export type WhenConditions<T> = WhenParts<T>[];
|
||||
export type WhenConditionsConfig = WhenConditions<string>;
|
||||
@@ -6,6 +6,7 @@ import { WebhookConfig } from "./health/webhooks.js";
|
||||
import { CommonSourceOptions, SourceRetryOptions } from "./source/index.js";
|
||||
import { SourceAIOConfig } from "./source/sources.js";
|
||||
import { CacheConfigOptions, ClientType, SourceType } from "../Atomic.js";
|
||||
import { TransformerCommonConfig } from "../../../../core/Atomic.js";
|
||||
|
||||
|
||||
export interface SourceDefaults extends CommonSourceOptions {
|
||||
@@ -66,6 +67,8 @@ export interface AIOConfig {
|
||||
debugMode?: boolean
|
||||
|
||||
cache?: CacheConfigOptions
|
||||
|
||||
transformers?: TransformerCommonConfig[]
|
||||
}
|
||||
|
||||
export interface AIOClientConfig {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { PlayTransformConfig, PlayTransformOptions } from "../../Atomic.js";
|
||||
import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js";
|
||||
import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js";
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { FileLogOptions, LogLevel } from "@foxxmd/logging";
|
||||
import { PlayTransformConfig, PlayTransformOptions } from "../../Atomic.js";
|
||||
|
||||
import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js";
|
||||
import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js";
|
||||
|
||||
export interface SourceRetryOptions extends RequestRetryOptions {
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import { childLogger, Logger } from "@foxxmd/logging";
|
||||
import { PlayObject, TransformerCommon, TransformerCommonConfig } from "../../../core/Atomic.js";
|
||||
import { getRoot } from "../../ioc.js";
|
||||
import { isStageTyped, testWhenConditions } from "../../utils/PlayTransformUtils.js";
|
||||
import AbstractInitializable from "../AbstractInitializable.js";
|
||||
import { StageConfig } from "../infrastructure/Transform.js";
|
||||
import { cacheFunctions, parseToRegexOrLiteralSearch, testMaybeRegex, searchAndReplace} from "@foxxmd/regex-buddy-core";
|
||||
import { Cacheable } from "cacheable";
|
||||
import { hashObject } from "../../utils/StringUtils.js";
|
||||
import { playContentInvariantTransform } from "../../utils/PlayComparisonUtils.js";
|
||||
|
||||
export interface TransformerOptions {
|
||||
logger: Logger
|
||||
regexCache?: ReturnType<typeof cacheFunctions>
|
||||
cache: Cacheable
|
||||
}
|
||||
|
||||
export interface RegexObject {
|
||||
parseToRegexOrLiteralSearch: typeof parseToRegexOrLiteralSearch
|
||||
testMaybeRegex: typeof testMaybeRegex,
|
||||
searchAndReplace: typeof searchAndReplace
|
||||
}
|
||||
|
||||
export default abstract class AbstractTransformer<T = any, Y extends StageConfig = StageConfig> extends AbstractInitializable {
|
||||
|
||||
declare config: TransformerCommonConfig;
|
||||
configHash: string;
|
||||
|
||||
transformType: string
|
||||
|
||||
regex: RegexObject
|
||||
cache: Cacheable;
|
||||
|
||||
public constructor(config: TransformerCommon, options: TransformerOptions) {
|
||||
super(config);
|
||||
this.logger = childLogger(options.logger, ['Transformer', this.config.type, this.config.name]);
|
||||
this.transformType = config.type;
|
||||
this.regex = options.regexCache ?? { searchAndReplace, testMaybeRegex, parseToRegexOrLiteralSearch };
|
||||
this.cache = options.cache;
|
||||
this.configHash = hashObject(this.config);
|
||||
}
|
||||
|
||||
public parseConfig(data: any): Y {
|
||||
if (!isStageTyped(data)) {
|
||||
throw new Error(`Must be an object with a 'type' property.`);
|
||||
}
|
||||
return this.doParseConfig(data);
|
||||
}
|
||||
|
||||
protected abstract doParseConfig(data: StageConfig): Y;
|
||||
|
||||
public async handle(data: Y, play: PlayObject): Promise<PlayObject> {
|
||||
|
||||
const cacheKey = `${this.configHash}-${hashObject(data)}-${hashObject(playContentInvariantTransform(play))}`
|
||||
const cachedTransform = await this.cache.get<PlayObject>(cacheKey);
|
||||
if(cachedTransform !== undefined) {
|
||||
this.logger.debug('Cache hit');
|
||||
return cachedTransform;
|
||||
}
|
||||
|
||||
if (data.when !== undefined) {
|
||||
if (!testWhenConditions(data.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) {
|
||||
this.logger.debug('When condition not met, returning original Play');
|
||||
await this.cache.set(cacheKey, play, '15s');
|
||||
return play;
|
||||
}
|
||||
}
|
||||
|
||||
let transformData: T;
|
||||
try {
|
||||
transformData = await this.getTransformerData(play, data);
|
||||
} catch (e) {
|
||||
throw new Error(`Could not fetch transformer data`, { cause: e });
|
||||
}
|
||||
|
||||
try {
|
||||
await this.checkShouldTransform(play, transformData, data);
|
||||
} catch (e) {
|
||||
this.logger.debug(new Error('checkShouldTransform did not pass, returning original Play', { cause: e }));
|
||||
return play;
|
||||
}
|
||||
|
||||
const transformed = await this.doHandle(data, play, transformData);
|
||||
await this.cache.set(cacheKey, transformed, '15s');
|
||||
return transformed;
|
||||
}
|
||||
|
||||
protected abstract doHandle(data: StageConfig, play: PlayObject, transformData: T): Promise<PlayObject>;
|
||||
|
||||
public async getTransformerData(play: PlayObject, stageConfig: Y): Promise<T> {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
public async checkShouldTransform(play: PlayObject, transformData: T, stageConfig: Y): Promise<void> {
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import { ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js";
|
||||
import { AtomicStageConfig, StageConfig } from "../infrastructure/Transform.js";
|
||||
import AbstractTransformer from "./AbstractTransformer.js";
|
||||
|
||||
//export type GenericAtomicStageConfig<A> =
|
||||
|
||||
export default abstract class AtomicPartsTransformer<Y, T = any, Z extends AtomicStageConfig<Y> = StageConfig> extends AbstractTransformer<T, Z> {
|
||||
|
||||
protected async doHandle(parts: Z, play: PlayObject, transformData: T): Promise<PlayObject> {
|
||||
|
||||
const {
|
||||
throwOnFailure = false,
|
||||
} = this.config.options || {};
|
||||
|
||||
try {
|
||||
await this.checkShouldTransform(play, transformData, parts);
|
||||
} catch (e) {
|
||||
this.logger.debug(new Error('checkShouldTransform did not pass, returning original Play', { cause: e }));
|
||||
return play;
|
||||
}
|
||||
|
||||
const transformedPlayData: Partial<ObjectPlayData> = {};
|
||||
|
||||
if (parts.title !== undefined) {
|
||||
try {
|
||||
const title = await this.handleTitle(play, parts.title, transformData);
|
||||
transformedPlayData.track = title;
|
||||
} catch (e) {
|
||||
const err = new Error(`Failed to transform title: ${play.data.track}`, { cause: e });
|
||||
if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('title'))) {
|
||||
throw err;
|
||||
} else {
|
||||
this.logger.warn(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (parts.artists !== undefined) {
|
||||
try {
|
||||
const artists = await this.handleArtists(play, parts.artists, transformData);
|
||||
transformedPlayData.artists = artists;
|
||||
} catch (e) {
|
||||
const err = new Error(`Failed to transform artists`, { cause: e });
|
||||
if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('artists'))) {
|
||||
throw err;
|
||||
} else {
|
||||
this.logger.warn(err);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const albumArtists = await this.handleAlbumArtists(play, parts.artists, transformData);
|
||||
transformedPlayData.albumArtists = albumArtists;
|
||||
} catch (e) {
|
||||
const err = new Error(`Failed to transform album artists`, { cause: e });
|
||||
if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('albumArtists'))) {
|
||||
throw err;
|
||||
} else {
|
||||
this.logger.warn(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (parts.album !== undefined) {
|
||||
try {
|
||||
const album = await this.handleAlbum(play, parts.album, transformData);
|
||||
transformedPlayData.album = album;
|
||||
} catch (e) {
|
||||
const err = new Error(`Failed to transform album: ${play.data.album}`, { cause: e });
|
||||
if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('album'))) {
|
||||
throw err;
|
||||
} else {
|
||||
this.logger.warn(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const transformedPlay = {
|
||||
...play,
|
||||
data: {
|
||||
...play.data,
|
||||
...transformedPlayData
|
||||
}
|
||||
}
|
||||
|
||||
return transformedPlay;
|
||||
}
|
||||
|
||||
public async getTransformerData(play: PlayObject, stageConfig: Z): Promise<T> {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
public async checkShouldTransform(play: PlayObject, transformData: T, stageConfig: Z): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
protected abstract handleTitle(play: PlayObject, parts: Y, transformData: T): Promise<string | undefined>;
|
||||
protected abstract handleArtists(play: PlayObject, parts: Y, transformData: T): Promise<string[] | undefined>;
|
||||
protected abstract handleAlbumArtists(play: PlayObject, parts: Y, transformData: T): Promise<string[] | undefined>;
|
||||
protected abstract handleAlbum(play: PlayObject, parts: Y, transformData: T): Promise<string | undefined>;
|
||||
|
||||
protected async handleMeta(play: PlayObject, transformData: T): Promise<TrackMeta | undefined> {
|
||||
return play.data.meta;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
import { PlayObject, TransformerCommon } from "../../../core/Atomic.js";
|
||||
import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js";
|
||||
import { WebhookPayload } from "../infrastructure/config/health/webhooks.js";
|
||||
import { ExternalMetadataTerm, PlayTransformNativeStage, StageConfig } from "../infrastructure/Transform.js";
|
||||
import AtomicPartsTransformer from "./AtomicPartsTransformer.js";
|
||||
import { parseArtistCredits, parseTrackCredits, uniqueNormalizedStrArr } from "../../utils/StringUtils.js";
|
||||
import { parseRegexSingle, parseToRegexOrLiteralSearch } from "@foxxmd/regex-buddy-core";
|
||||
import { TransformerOptions } from "./AbstractTransformer.js";
|
||||
import { DELIMITERS_NO_AMP } from "../infrastructure/Atomic.js";
|
||||
import { asArray } from "../../utils/DataUtils.js";
|
||||
import { MaybeLogger } from "../logging.js";
|
||||
import { childLogger } from "@foxxmd/logging";
|
||||
|
||||
export type ArtistParseSource = 'artists' | 'title'
|
||||
|
||||
export const asArtistParseSource = (str: string): ArtistParseSource => {
|
||||
const clean = str.trim().toLocaleLowerCase();
|
||||
switch(clean) {
|
||||
case 'track':
|
||||
case 'title':
|
||||
return 'title';
|
||||
case 'artist':
|
||||
case 'artists':
|
||||
return 'artists';
|
||||
}
|
||||
throw new Error(`ArtistParseSource must be one of 'artist' or 'title', given: ${clean}`);
|
||||
}
|
||||
|
||||
export interface NativeTransformerData {
|
||||
delimiters?: string[]
|
||||
delimitersExtra?: string[]
|
||||
artistsIgnore?: string[]
|
||||
artistsParseFrom?: ArtistParseSource[]
|
||||
artistsParseMonolithicOnly?: boolean
|
||||
}
|
||||
|
||||
export interface NativeTransformerDataStrong {
|
||||
artistsParseFrom?: ArtistParseSource[]
|
||||
artistsParseMonolithicOnly?: boolean
|
||||
ignoreArtistsRegex?: RegExp[]
|
||||
delimiters?: string[]
|
||||
}
|
||||
|
||||
export interface NativeTransformerDataStage extends NativeTransformerDataStrong,PlayTransformNativeStage {
|
||||
}
|
||||
|
||||
export type NativeTransformerConfig = TransformerCommon<NativeTransformerData>;
|
||||
|
||||
export const parseStageConfig = (data: NativeTransformerData | undefined, logger: MaybeLogger = new MaybeLogger()): NativeTransformerDataStrong => {
|
||||
|
||||
if (data === undefined) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const config: NativeTransformerDataStrong = {
|
||||
};
|
||||
|
||||
if (data === null || typeof data !== 'object') {
|
||||
throw new Error('Native Transformer data should be an object or not defined.');
|
||||
}
|
||||
if (data.artistsIgnore !== undefined) {
|
||||
data.artistsIgnore = asArray(data.artistsIgnore);
|
||||
const nonStr = data.artistsIgnore.filter(x => typeof x !== 'string');
|
||||
if (nonStr.length > 0) {
|
||||
throw new Error(`ignoreArtists must be an array of strings but non-strings found: ${nonStr.map(x => (x as unknown).toString()).join(' | ')}`)
|
||||
}
|
||||
config.ignoreArtistsRegex = [];
|
||||
for (const i of data.artistsIgnore) {
|
||||
try {
|
||||
config.ignoreArtistsRegex.push(parseToRegexOrLiteralSearch(i));
|
||||
} catch (e) {
|
||||
throw new Error(`Could not convert ignoreArtist string to regex (or literal): ${i}`);
|
||||
}
|
||||
}
|
||||
logger.debug(`Defaults - Ignoring artists using ${data.artistsIgnore.length} rules`);
|
||||
}
|
||||
|
||||
if (data.delimiters !== undefined) {
|
||||
config.delimiters = asArray(data.delimiters);
|
||||
logger.debug(`Defaults - Using user-defined delimiters '${data.delimitersExtra.join(' ')}' instead of built-ins`);
|
||||
} else if (data.delimitersExtra !== undefined) {
|
||||
config.delimiters = [...DELIMITERS_NO_AMP, ...(asArray(data.delimitersExtra))];
|
||||
logger.debug(`Defaults - Using extra delimiters '${data.delimitersExtra.join(' ')}' with built-in delimiters '${DELIMITERS_NO_AMP.join(' ')}'`);
|
||||
}
|
||||
|
||||
if (config.delimiters !== undefined) {
|
||||
config.delimiters.map(x => x.trim());
|
||||
}
|
||||
|
||||
if (data.artistsParseFrom !== undefined) {
|
||||
const arr = asArray(data.artistsParseFrom);
|
||||
config.artistsParseFrom = arr.map(asArtistParseSource);
|
||||
logger.debug(`Defaults - Will try to parse artists from ${config.artistsParseFrom.join(' and ')} string`);
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
export default class NativeTransformer extends AtomicPartsTransformer<ExternalMetadataTerm, PlayObject | undefined, NativeTransformerDataStage> {
|
||||
|
||||
declare config: NativeTransformerConfig;
|
||||
|
||||
protected defaults: NativeTransformerDataStrong = {};
|
||||
|
||||
ignoreArtistsRegex: RegExp[] = [];
|
||||
delimiters?: string[]
|
||||
parseArtistsFrom: ArtistParseSource[]
|
||||
|
||||
public constructor(config: NativeTransformerConfig, options: TransformerOptions) {
|
||||
super(config, options);
|
||||
}
|
||||
|
||||
protected async doBuildInitData(): Promise<true | string | undefined> {
|
||||
this.defaults = parseStageConfig(this.config.defaults, childLogger(this.logger, 'Defaults'));
|
||||
return true;
|
||||
}
|
||||
|
||||
protected doParseConfig(data: NativeTransformerDataStage) {
|
||||
if (data.type !== 'native') {
|
||||
throw new Error(`NativeTransformer is only usable with 'native' type stages`);
|
||||
}
|
||||
|
||||
const stage: NativeTransformerDataStage = {
|
||||
...data,
|
||||
...parseStageConfig(data),
|
||||
type: 'native'
|
||||
}
|
||||
|
||||
for (const k of ['artists', 'title', 'album']) {
|
||||
if (!(k in stage)) {
|
||||
stage[k] = true;
|
||||
continue;
|
||||
}
|
||||
if (Array.isArray(stage[k])) {
|
||||
throw new Error(`${k} must be a boolean or when object`);
|
||||
}
|
||||
if (typeof stage[k] === 'boolean') {
|
||||
continue;
|
||||
}
|
||||
if (typeof stage[k] === 'object' && !isWhenCondition(stage[k])) {
|
||||
throw new Error(`${k} is not a valid when object`);
|
||||
}
|
||||
}
|
||||
return stage;
|
||||
}
|
||||
|
||||
public async getTransformerData(play: PlayObject, stageConfig: NativeTransformerDataStage): Promise<PlayObject> {
|
||||
let artists = [];
|
||||
const {
|
||||
artistsParseFrom: parseArtistsFrom = this.defaults.artistsParseFrom ?? ['artists', 'title'],
|
||||
artistsParseMonolithicOnly = this.defaults.artistsParseMonolithicOnly ?? true,
|
||||
ignoreArtistsRegex = this.defaults.ignoreArtistsRegex ?? [],
|
||||
delimiters = this.defaults.delimiters
|
||||
} = stageConfig || {};
|
||||
|
||||
if(parseArtistsFrom.includes('artists')) {
|
||||
|
||||
if(play.data.artists.length === 1 || (play.data.artists.length > 1 && artistsParseMonolithicOnly === false)) {
|
||||
|
||||
for(const artist of play.data.artists) {
|
||||
|
||||
const matchedIgnoreArtists = ignoreArtistsRegex.map(x => ({reg: x.toString(), res: parseRegexSingle(x, artist)})).filter(x => x !== undefined);
|
||||
if(matchedIgnoreArtists.length > 0) {
|
||||
this.logger.debug(`Will not parse artist because it matched an ignore regex:\n${matchedIgnoreArtists.map(x => `Reg: ${x.reg} => ${x.res.match}`).join('\n')}`);
|
||||
artists.push(artist);
|
||||
} else {
|
||||
const artistCredits = parseArtistCredits(artist, delimiters);
|
||||
if (artistCredits !== undefined) {
|
||||
if (artistCredits.primary !== undefined) {
|
||||
artists.push(artistCredits.primary);
|
||||
}
|
||||
if (artistCredits.secondary !== undefined) {
|
||||
artists = artists.concat(artistCredits.secondary);
|
||||
}
|
||||
} else {
|
||||
// couldn't parse anything from artist string, use as-is
|
||||
artists.push(artist);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
} else {
|
||||
// user does not want to try to parse artists when we already have more than one artist string
|
||||
// -- likely this is because the user knows the artist data is already good and shouldn't be modified
|
||||
artists = play.data.artists;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(parseArtistsFrom.includes('title')) {
|
||||
const trackArtists = parseTrackCredits(play.data.track, delimiters);
|
||||
if (trackArtists !== undefined && trackArtists.secondary !== undefined) {
|
||||
artists = artists.concat(trackArtists.secondary);
|
||||
}
|
||||
}
|
||||
|
||||
artists = uniqueNormalizedStrArr([...artists]);
|
||||
|
||||
return {
|
||||
...play,
|
||||
data: {
|
||||
...play.data,
|
||||
artists
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
protected async handleTitle(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise<string | undefined> {
|
||||
return play.data.track;
|
||||
}
|
||||
protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise<string[] | undefined> {
|
||||
if (parts === false) {
|
||||
return play.data.artists;
|
||||
}
|
||||
if (typeof parts === 'object') {
|
||||
if (parts.when !== undefined) {
|
||||
if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) {
|
||||
this.logger.debug('When condition for artists not met, returning original artists');
|
||||
return play.data.artists;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return transformData.data.artists;
|
||||
}
|
||||
protected async handleAlbumArtists(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise<string[] | undefined> {
|
||||
return play.data.albumArtists;
|
||||
}
|
||||
protected async handleAlbum(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise<string | undefined> {
|
||||
return play.data.album;
|
||||
}
|
||||
|
||||
public notify(payload: WebhookPayload): Promise<void> {
|
||||
return;
|
||||
}
|
||||
protected getIdentifier(): string {
|
||||
return 'Native Transformer';
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import { childLogger, Logger } from "@foxxmd/logging";
|
||||
import AbstractTransformer from "./AbstractTransformer.js";
|
||||
import { TransformerCommonConfig } from "../../../core/Atomic.js";
|
||||
import UserTransformer from "./UserTransformer.js";
|
||||
import { StageConfig } from "../infrastructure/Transform.js";
|
||||
import { PlayObject } from "../../../core/Atomic.js";
|
||||
import { isStageTyped } from "../../utils/PlayTransformUtils.js";
|
||||
import { MSCache } from "../Cache.js";
|
||||
import NativeTransformer from "./NativeTransformer.js";
|
||||
|
||||
export default class TransformerManager {
|
||||
|
||||
protected logger: Logger;
|
||||
protected parentLogger: Logger;
|
||||
protected transformers: Map<string, AbstractTransformer[]> = new Map();
|
||||
protected cache: MSCache;
|
||||
|
||||
public constructor(logger: Logger, cache: MSCache) {
|
||||
this.logger = childLogger(logger, 'Transformer Manager');
|
||||
this.parentLogger = logger;
|
||||
this.cache = cache;
|
||||
}
|
||||
|
||||
public register(config: TransformerCommonConfig): void {
|
||||
let transformers: AbstractTransformer[] = [];
|
||||
if (!this.transformers.has(config.type)) {
|
||||
this.transformers.set(config.type, []);
|
||||
} else {
|
||||
transformers = this.transformers.get(config.type);
|
||||
}
|
||||
|
||||
if (config.name !== undefined && transformers.some(x => x.config.name === config.name)) {
|
||||
throw new Error(`Cannot register ${config.type} transformer with name '${config.name}' because an existing transformer already has that name`);
|
||||
}
|
||||
const tName = config.name ?? `unnamed-${transformers.length + 1}`;
|
||||
|
||||
this.logger.verbose(`Registering ${config.type} transformer with name '${tName}'`);
|
||||
|
||||
let t: AbstractTransformer;
|
||||
switch (config.type) {
|
||||
case 'user':
|
||||
t = new UserTransformer({ name: tName, ...config }, {logger: this.parentLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform});
|
||||
break;
|
||||
case 'native':
|
||||
t = new NativeTransformer({ name: tName, ...config }, {logger: this.parentLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform});
|
||||
break;
|
||||
default:
|
||||
throw new Error(`No transformer of type '${config.type}' exists.`);
|
||||
}
|
||||
this.transformers.set(config.type, [...transformers, t]);
|
||||
this.logger.verbose(`${config.type} transformer with name '${tName}' registered`);
|
||||
}
|
||||
|
||||
public async initTransformers() {
|
||||
this.logger.verbose('Initializing transformers...');
|
||||
for (const list of this.transformers.values()) {
|
||||
for (const transformer of list) {
|
||||
if (!transformer.isReady()) {
|
||||
if (!transformer.canAuthUnattended()) {
|
||||
transformer.logger.warn({ label: 'Heartbeat' }, 'Transformer is not ready but will not try to initialize because auth state is not good and cannot be correct unattended.');
|
||||
}
|
||||
try {
|
||||
await transformer.tryInitialize({ force: false, notify: true, notifyTitle: 'Could not initialize automatically' });
|
||||
} catch (e) {
|
||||
transformer.logger.error(new Error('Could not initialize source automatically', { cause: e }));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
this.logger.verbose('Done initializing transformers');
|
||||
}
|
||||
|
||||
public hasTransformerType(type: string): boolean {
|
||||
return this.transformers.has(type);
|
||||
}
|
||||
|
||||
protected getTransformerByStage(data: StageConfig): AbstractTransformer {
|
||||
const list = this.transformers.get(data.type);
|
||||
if (list === undefined || list.length === 0) {
|
||||
throw new Error(`No transformer of type '${data.type}' is registered.`);
|
||||
}
|
||||
|
||||
if (list.length > 1 && (data as any).name === undefined) {
|
||||
this.logger.warn(`More than one '${data.type}' transformer but name was not specified, using first registered`);
|
||||
return list[0];
|
||||
} else {
|
||||
return list[0]
|
||||
}
|
||||
}
|
||||
|
||||
public parseTransformerConfig(data: any) {
|
||||
if (!isStageTyped(data)) {
|
||||
throw new Error(`Must be an object with a 'type' property.`);
|
||||
}
|
||||
const t = this.getTransformerByStage(data);
|
||||
return t.parseConfig(data);
|
||||
}
|
||||
|
||||
public async handleStage(data: StageConfig, play: PlayObject): Promise<PlayObject> {
|
||||
const list = this.transformers.get(data.type);
|
||||
if (list === undefined || list.length === 0) {
|
||||
throw new Error(`No transformer of type '${data.type}' is registered.`);
|
||||
}
|
||||
|
||||
let t: AbstractTransformer;
|
||||
if (list.length > 0 && (data as any).name === undefined) {
|
||||
this.logger.warn(`More than one '${data.type}' transformer but name was not specified, using first registered`);
|
||||
t = list[0];
|
||||
} else {
|
||||
t = list[0];
|
||||
}
|
||||
|
||||
return await t.handle(data, play);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import { searchAndReplace } from "@foxxmd/regex-buddy-core";
|
||||
import { PlayObject } from "../../../core/Atomic.js";
|
||||
import { configValToSearchReplace, isSearchAndReplaceTerm, isUserStage, testWhenConditions } from "../../utils/PlayTransformUtils.js";
|
||||
import { WebhookPayload } from "../infrastructure/config/health/webhooks.js";
|
||||
import { ConditionalSearchAndReplaceRegExp, PlayTransformUserStage, StageConfig } from "../infrastructure/Transform.js";
|
||||
import AtomicPartsTransformer from "./AtomicPartsTransformer.js";
|
||||
|
||||
export default class UserTransformer extends AtomicPartsTransformer<ConditionalSearchAndReplaceRegExp[], undefined> {
|
||||
|
||||
// protected constructor(config: TransformerCommon) {
|
||||
// super(name, config);
|
||||
// }
|
||||
|
||||
protected doParseConfig(data: StageConfig) {
|
||||
if (!isUserStage(data)) {
|
||||
throw new Error(`UserTransformer is only usable with 'user' type stages`);
|
||||
}
|
||||
|
||||
const stage: PlayTransformUserStage<ConditionalSearchAndReplaceRegExp[]> = {
|
||||
...data,
|
||||
type: 'user'
|
||||
}
|
||||
|
||||
for (const k of ['artists', 'title', 'album']) {
|
||||
if (!(k in data)) {
|
||||
continue;
|
||||
}
|
||||
if (!Array.isArray(data[k])) {
|
||||
throw new Error(`${k} must be an array`);
|
||||
}
|
||||
try {
|
||||
isSearchAndReplaceTerm(data[k]);
|
||||
stage[k] = data[k].map(configValToSearchReplace);
|
||||
} catch (e) {
|
||||
throw new Error(`Property '${k}' was not a valid type`, { cause: e });
|
||||
}
|
||||
}
|
||||
return stage;
|
||||
}
|
||||
|
||||
protected generateMapper(play: PlayObject) {
|
||||
return (x: ConditionalSearchAndReplaceRegExp): ConditionalSearchAndReplaceRegExp => ({ ...x, test: (x.when !== undefined ? () => testWhenConditions(x.when, play, { testMaybeRegex: this.regex.testMaybeRegex }) : undefined) });
|
||||
}
|
||||
|
||||
protected async handleTitle(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise<string | undefined> {
|
||||
if (play.data.track === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const mapper = this.generateMapper(play);
|
||||
const result = searchAndReplace(play.data.track, parts.map(mapper));
|
||||
if(result.trim() === '') {
|
||||
return undefined;
|
||||
}
|
||||
return result.trim();
|
||||
}
|
||||
protected async handleArtists(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise<string[] | undefined> {
|
||||
if(play.data.artists === undefined || play.data.artists.length === 0) {
|
||||
return play.data.artists;
|
||||
}
|
||||
const mapper = this.generateMapper(play);
|
||||
const transformedArtists = [];
|
||||
for(const artist of play.data.artists) {
|
||||
const a = searchAndReplace(artist, parts.map(mapper));
|
||||
if(a.trim() !== '') {
|
||||
transformedArtists.push(a);
|
||||
}
|
||||
}
|
||||
return transformedArtists;
|
||||
}
|
||||
protected async handleAlbumArtists(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise<string[] | undefined> {
|
||||
if(play.data.albumArtists === undefined || play.data.albumArtists.length === 0) {
|
||||
return play.data.albumArtists;
|
||||
}
|
||||
const mapper = this.generateMapper(play);
|
||||
const transformedArtists = [];
|
||||
for(const artist of play.data.albumArtists) {
|
||||
const a = searchAndReplace(artist, parts.map(mapper));
|
||||
if(a.trim() !== '') {
|
||||
transformedArtists.push(a);
|
||||
}
|
||||
}
|
||||
return transformedArtists;
|
||||
}
|
||||
protected async handleAlbum(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise<string | undefined> {
|
||||
if (play.data.album === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const mapper = this.generateMapper(play);
|
||||
const result = searchAndReplace(play.data.album, parts.map(mapper));
|
||||
if(result.trim() === '') {
|
||||
return undefined;
|
||||
}
|
||||
return result.trim();
|
||||
}
|
||||
|
||||
public notify(payload: WebhookPayload): Promise<void> {
|
||||
return;
|
||||
}
|
||||
protected getIdentifier(): string {
|
||||
return 'User Transformer';
|
||||
}
|
||||
|
||||
}
|
||||
+3
-2
@@ -21,7 +21,8 @@ const badErrors = [
|
||||
'api key suspended',
|
||||
'invalid session key',
|
||||
'invalid api key',
|
||||
'authentication failed'
|
||||
'authentication failed',
|
||||
'invalid parameters'
|
||||
];
|
||||
|
||||
const retryErrors = [
|
||||
@@ -120,7 +121,7 @@ export default class LastfmApiClient extends AbstractApiClient {
|
||||
} = e;
|
||||
// for now check for exceptional errors by matching error code text
|
||||
const retryError = retryErrors.find(x => message.toLocaleLowerCase().includes(x));
|
||||
let networkError = null;
|
||||
let networkError = undefined;
|
||||
if(retryError === undefined) {
|
||||
const nError = getNodeNetworkException(e);
|
||||
if(nError !== undefined) {
|
||||
|
||||
@@ -119,6 +119,8 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`)
|
||||
const notifiers = new Notifiers(root.get('notifierEmitter'), root.get('clientEmitter'), root.get('sourceEmitter'), root.get('logger')); //root.get('notifiers');
|
||||
await notifiers.buildWebhooks(webhooks);
|
||||
|
||||
await root.items.transformerManager.initTransformers();
|
||||
|
||||
/*
|
||||
* setup clients
|
||||
* */
|
||||
|
||||
+24
-1
@@ -10,6 +10,8 @@ import { generateBaseURL } from "./utils/NetworkUtils.js";
|
||||
import { PassThrough } from "stream";
|
||||
import { CacheConfigOptions } from "./common/infrastructure/Atomic.js";
|
||||
import { MSCache } from "./common/Cache.js";
|
||||
import TransformerManager from "./common/transforms/TransformerManager.js";
|
||||
import { TransformerCommonConfig } from "../core/Atomic.js";
|
||||
|
||||
export let version: string = 'unknown';
|
||||
|
||||
@@ -27,6 +29,7 @@ export interface RootOptions {
|
||||
loggerStream?: PassThrough
|
||||
loggingConfig?: LogOptions
|
||||
cache?: CacheConfigOptions | MSCache | (() => MSCache)
|
||||
transformers?: TransformerCommonConfig[]
|
||||
}
|
||||
|
||||
const createRoot = (options: RootOptions = {logger: loggerDebug}) => {
|
||||
@@ -37,7 +40,8 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => {
|
||||
loggerStream,
|
||||
loggingConfig,
|
||||
logger,
|
||||
cache
|
||||
cache,
|
||||
transformers = [],
|
||||
} = options || {};
|
||||
const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
|
||||
let disableWeb = dw;
|
||||
@@ -65,6 +69,24 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => {
|
||||
const f = e;
|
||||
});
|
||||
|
||||
const transformerManager = new TransformerManager(logger, maybeSingletonCache !== undefined ? maybeSingletonCache : cacheFunc());
|
||||
for(const c of transformers) {
|
||||
try {
|
||||
transformerManager.register(c);
|
||||
} catch (e) {
|
||||
logger.warn(new Error('Could not register a transformer', {cause: e}));
|
||||
}
|
||||
}
|
||||
if(transformers.length === 0) {
|
||||
logger.debug('No user-supplied transformer configs were found.');
|
||||
}
|
||||
if(!transformerManager.hasTransformerType('user')) {
|
||||
transformerManager.register({type: 'user', name: 'MSDefault'});
|
||||
}
|
||||
if(!transformerManager.hasTransformerType('native')) {
|
||||
transformerManager.register({type: 'native', name: 'MSDefault'});
|
||||
}
|
||||
|
||||
const portVal: number | string = process.env.PORT ?? port;
|
||||
|
||||
return createContainer().add({
|
||||
@@ -80,6 +102,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => {
|
||||
loggerStream,
|
||||
loggingConfig,
|
||||
logger: logger,
|
||||
transformerManager,
|
||||
cache: () => maybeSingletonCache !== undefined ? () => maybeSingletonCache : cacheFunc
|
||||
}).add((items) => {
|
||||
const localUrl = generateBaseURL(baseUrl, items.port)
|
||||
|
||||
@@ -25,9 +25,10 @@ import {
|
||||
ScrobbledPlayObject,
|
||||
SourceIdentifier,
|
||||
TIME_WEIGHT,
|
||||
TITLE_WEIGHT, TRANSFORM_HOOK,
|
||||
TITLE_WEIGHT,
|
||||
} from "../common/infrastructure/Atomic.js";
|
||||
import { CommonClientConfig, NowPlayingOptions, UpstreamRefreshOptions } from "../common/infrastructure/config/client/index.js";
|
||||
import { TRANSFORM_HOOK } from "../common/infrastructure/Transform.js";
|
||||
import { Notifiers } from "../notifier/Notifiers.js";
|
||||
import {
|
||||
comparingMultipleArtists,
|
||||
@@ -55,6 +56,7 @@ import { AsyncTask, SimpleIntervalJob, Task, ToadScheduler } from "toad-schedule
|
||||
import { MSCache } from "../common/Cache.js";
|
||||
import { getRoot } from "../ioc.js";
|
||||
import { rehydratePlay } from "../utils/CacheUtils.js";
|
||||
import { findAsyncSequential } from "../utils/AsyncUtils.js";
|
||||
|
||||
type PlatformMappedPlays = Map<string, {play: PlayObject, source: SourceIdentifier}>;
|
||||
type NowPlayingQueue = Map<string, PlatformMappedPlays>;
|
||||
@@ -468,12 +470,12 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
|
||||
|
||||
getScrobbledPlays = () => this.scrobbledPlayObjs.data.map(x => x.scrobble)
|
||||
|
||||
findExistingSubmittedPlayObj = (playObjPre: PlayObject): ([undefined, undefined] | [ScrobbledPlayObject, ScrobbledPlayObject[]]) => {
|
||||
findExistingSubmittedPlayObj = async (playObjPre: PlayObject): Promise<([undefined, undefined] | [ScrobbledPlayObject, ScrobbledPlayObject[]])> => {
|
||||
|
||||
const playObj = this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate);
|
||||
const playObj = await this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate);
|
||||
|
||||
const dtInvariantMatches = this.scrobbledPlayObjs.data
|
||||
.map(x => ({...x, play: this.transformPlay(x.play, TRANSFORM_HOOK.existing)}))
|
||||
const dtInvariantMatches = (await Promise.all(this.scrobbledPlayObjs.data
|
||||
.map(async x => ({...x, play: await this.transformPlay(x.play, TRANSFORM_HOOK.existing)}))))
|
||||
.filter(x => playObjDataMatch(playObj, x.play));
|
||||
|
||||
if (dtInvariantMatches.length === 0) {
|
||||
@@ -513,7 +515,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
|
||||
|
||||
existingScrobble = async (playObjPre: PlayObject) => {
|
||||
|
||||
const playObj = this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate);
|
||||
const playObj = await this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate);
|
||||
|
||||
const tr = truncateStringToLength(27);
|
||||
const scoreTrackOpts: TrackStringOptions = {include: ['track', 'artist', 'time'], transformers: {track: (t: any, data, existing) => `${existing ? '- ': ''}${tr(t)}`}};
|
||||
@@ -530,7 +532,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
|
||||
let closestMatch: {score: number, breakdowns: string[], confidence: string, scrobble?: PlayObject} = {score: 0, breakdowns: [], confidence: 'No existing scrobble matched with a score higher than 0'};
|
||||
|
||||
// then check if we have already recorded this
|
||||
const [existingExactSubmitted, existingDataSubmitted = []] = this.findExistingSubmittedPlayObj(playObjPre);
|
||||
const [existingExactSubmitted, existingDataSubmitted = []] = await this.findExistingSubmittedPlayObj(playObjPre);
|
||||
|
||||
// if we have an submitted play with matching data and play date then we can just return the response from the original scrobble
|
||||
if (existingExactSubmitted !== undefined) {
|
||||
@@ -566,9 +568,10 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
|
||||
// -- this is info we only know if play was generated from MS player so we can be reasonably sure
|
||||
const looseTimeAccuracy = playObj.data.repeat ? [TA_DURING] : [TA_FUZZY, TA_DURING];
|
||||
|
||||
existingScrobble = this.recentScrobbles.find((xPre) => {
|
||||
|
||||
existingScrobble = findAsyncSequential(this.recentScrobbles, async (xPre) => {
|
||||
|
||||
const x = this.transformPlay(xPre, TRANSFORM_HOOK.existing);
|
||||
const x = await this.transformPlay(xPre, TRANSFORM_HOOK.existing);
|
||||
|
||||
//const referenceMatch = referenceApiScrobbleResponse !== undefined && playObjDataMatch(x, referenceApiScrobbleResponse);
|
||||
|
||||
@@ -794,8 +797,8 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']});
|
||||
const currQueuedPlay = this.queuedScrobbles.shift();
|
||||
|
||||
const [timeFrameValid, timeFrameValidLog] = this.timeFrameIsValid(currQueuedPlay.play);
|
||||
if (timeFrameValid && !(await this.alreadyScrobbled(this.transformPlay(currQueuedPlay.play, TRANSFORM_HOOK.preCompare)))) {
|
||||
const transformedScrobble = this.transformPlay(currQueuedPlay.play, TRANSFORM_HOOK.postCompare);
|
||||
if (timeFrameValid && !(await this.alreadyScrobbled((await this.transformPlay(currQueuedPlay.play, TRANSFORM_HOOK.preCompare))))) {
|
||||
const transformedScrobble = await this.transformPlay(currQueuedPlay.play, TRANSFORM_HOOK.postCompare);
|
||||
try {
|
||||
const scrobbledPlay = await this.scrobble(transformedScrobble);
|
||||
this.emitEvent('scrobble', {play: transformedScrobble});
|
||||
@@ -879,8 +882,8 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']});
|
||||
await this.refreshScrobbles();
|
||||
}
|
||||
const [timeFrameValid, timeFrameValidLog] = this.timeFrameIsValid(deadScrobble.play);
|
||||
if (timeFrameValid && !(await this.alreadyScrobbled(this.transformPlay(deadScrobble.play, TRANSFORM_HOOK.preCompare)))) {
|
||||
const transformedScrobble = this.transformPlay(deadScrobble.play, TRANSFORM_HOOK.postCompare);
|
||||
if (timeFrameValid && !(await this.alreadyScrobbled((await this.transformPlay(deadScrobble.play, TRANSFORM_HOOK.preCompare))))) {
|
||||
const transformedScrobble = await this.transformPlay(deadScrobble.play, TRANSFORM_HOOK.postCompare);
|
||||
try {
|
||||
const scrobbledPlay = await this.scrobble(transformedScrobble);
|
||||
this.emitEvent('scrobble', {play: transformedScrobble});
|
||||
|
||||
@@ -18,9 +18,10 @@ import {
|
||||
PlayUserId,
|
||||
ProgressAwarePlayObject,
|
||||
SINGLE_USER_PLATFORM_ID,
|
||||
SourceType, TRANSFORM_HOOK,
|
||||
SourceType,
|
||||
} from "../common/infrastructure/Atomic.js";
|
||||
import { SourceConfig } from "../common/infrastructure/config/source/sources.js";
|
||||
import { TRANSFORM_HOOK } from "../common/infrastructure/Transform.js";
|
||||
import TupleMap from "../common/TupleMap.js";
|
||||
import {
|
||||
difference,
|
||||
@@ -39,6 +40,7 @@ import { componentFileLogger } from '../common/logging.js';
|
||||
import { WebhookPayload } from '../common/infrastructure/config/health/webhooks.js';
|
||||
import { messageWithCauses, messageWithCausesTruncatedDefault } from '../utils/ErrorUtils.js';
|
||||
import { genericSourcePlayMatch } from '../utils/PlayComparisonUtils.js';
|
||||
import { findAsync } from '../utils/AsyncUtils.js';
|
||||
|
||||
export interface RecentlyPlayedOptions {
|
||||
limit?: number
|
||||
@@ -172,12 +174,13 @@ export default abstract class AbstractSource extends AbstractComponent implement
|
||||
return lists;
|
||||
}
|
||||
|
||||
existingDiscovered = (play: PlayObject, opts: {checkAll?: boolean} = {}): PlayObject | undefined => {
|
||||
existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<PlayObject | undefined> => {
|
||||
const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts);
|
||||
const candidate = this.transformPlay(play, TRANSFORM_HOOK.candidate);
|
||||
const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate);
|
||||
for(const list of lists) {
|
||||
const existing = list.find(x => {
|
||||
const e = this.transformPlay(x, TRANSFORM_HOOK.existing);
|
||||
|
||||
const existing = await findAsync(list,async x => {
|
||||
const e = await this.transformPlay(x, TRANSFORM_HOOK.existing);
|
||||
return genericSourcePlayMatch(e, candidate);
|
||||
});
|
||||
if(existing) {
|
||||
@@ -187,18 +190,18 @@ export default abstract class AbstractSource extends AbstractComponent implement
|
||||
return undefined;
|
||||
}
|
||||
|
||||
alreadyDiscovered = (play: PlayObject, opts: {checkAll?: boolean} = {}): boolean => {
|
||||
const existing = this.existingDiscovered(play, opts);
|
||||
alreadyDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<boolean> => {
|
||||
const existing = await this.existingDiscovered(play, opts);
|
||||
return existing !== undefined;
|
||||
}
|
||||
|
||||
discover = (plays: PlayObject[], options: { checkAll?: boolean, [key: string]: any } = {}): PlayObject[] => {
|
||||
discover = async (plays: PlayObject[], options: { checkAll?: boolean, [key: string]: any } = {}): Promise<PlayObject[]> => {
|
||||
const newDiscoveredPlays: PlayObject[] = [];
|
||||
|
||||
const transformedPlayed = plays.map(x => this.transformPlay(x, TRANSFORM_HOOK.preCompare));
|
||||
const transformedPlayed = await Promise.all(plays.map(x => this.transformPlay(x, TRANSFORM_HOOK.preCompare)));
|
||||
|
||||
for(const play of transformedPlayed) {
|
||||
if(!this.alreadyDiscovered(play, options)) {
|
||||
if(!(await this.alreadyDiscovered(play, options))) {
|
||||
this.addPlayToDiscovered(play);
|
||||
newDiscoveredPlays.push(play);
|
||||
}
|
||||
@@ -221,7 +224,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
|
||||
}
|
||||
|
||||
|
||||
protected scrobble = (newDiscoveredPlays: PlayObject[], options: { forceRefresh?: boolean, [key: string]: any, discoverLocation?: 'backlog' | [key: string] } = {}) => {
|
||||
protected scrobble = async (newDiscoveredPlays: PlayObject[], options: { forceRefresh?: boolean, [key: string]: any, discoverLocation?: 'backlog' | [key: string] } = {}) => {
|
||||
|
||||
if(newDiscoveredPlays.length > 0) {
|
||||
if(!this.shouldScrobble(options.discoverLocation)) {
|
||||
@@ -229,7 +232,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
|
||||
}
|
||||
newDiscoveredPlays.sort(sortByOldestPlayDate);
|
||||
this.emitter.emit('discoveredToScrobble', {
|
||||
data: newDiscoveredPlays.map(x => this.transformPlay(x, TRANSFORM_HOOK.postCompare)),
|
||||
data: await Promise.all(newDiscoveredPlays.map(x => this.transformPlay(x, TRANSFORM_HOOK.postCompare))),
|
||||
options: {
|
||||
...options,
|
||||
checkTime: newDiscoveredPlays[newDiscoveredPlays.length-1].data.playDate.add(2, 'second'),
|
||||
@@ -258,7 +261,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
|
||||
} catch (e) {
|
||||
throw new Error('Error occurred while fetching backlogged plays', {cause: e});
|
||||
}
|
||||
const discovered = this.discover(backlogPlays, {discoverLocation: 'backlog'});
|
||||
const discovered = await this.discover(backlogPlays, {discoverLocation: 'backlog'});
|
||||
|
||||
const {
|
||||
options: {
|
||||
@@ -269,7 +272,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
|
||||
if (scrobbleBacklog) {
|
||||
if (discovered.length > 0) {
|
||||
this.logger.info('Scrobbling backlogged tracks...');
|
||||
this.scrobble(discovered);
|
||||
await this.scrobble(discovered);
|
||||
this.logger.info('Backlog scrobbling complete.');
|
||||
} else {
|
||||
this.logger.info('All tracks already discovered!');
|
||||
@@ -450,7 +453,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
|
||||
this.logger.info(`Potential plays were discovered close to polling interval! Delaying scrobble clients refresh by ${maxDelay} seconds so other clients have time to scrobble first`);
|
||||
await sleep(maxDelay * 1000);
|
||||
}
|
||||
newDiscovered = this.discover(playObjs);
|
||||
newDiscovered = await this.discover(playObjs);
|
||||
this.scrobble(newDiscovered,
|
||||
{
|
||||
forceRefresh: closeToInterval
|
||||
|
||||
@@ -224,7 +224,7 @@ export class AzuracastSource extends MemorySource {
|
||||
position: online && play !== undefined ? play.meta.trackProgressPosition : undefined
|
||||
}
|
||||
|
||||
return this.processRecentPlays([playerState]);
|
||||
return await this.processRecentPlays([playerState]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -641,7 +641,7 @@ export class ChromecastSource extends MemoryPositionalSource {
|
||||
}
|
||||
}
|
||||
|
||||
const playsToReturn = this.processRecentPlays(plays);
|
||||
const playsToReturn = await this.processRecentPlays(plays);
|
||||
|
||||
this.pruneApplications();
|
||||
|
||||
|
||||
@@ -2,8 +2,9 @@ import dayjs from "dayjs";
|
||||
import EventEmitter from "events";
|
||||
import request, { Request, Response, SuperAgent } from 'superagent';
|
||||
import { PlayObject, SOURCE_SOT, TA_CLOSE, TA_DURING, TA_EXACT, TA_FUZZY, TemporalAccuracy } from "../../core/Atomic.js";
|
||||
import { DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig, TRANSFORM_HOOK } from "../common/infrastructure/Atomic.js";
|
||||
import { DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.js";
|
||||
import { DeezerInternalSourceConfig, DeezerInternalTrackData, DeezerSourceConfig } from "../common/infrastructure/config/source/deezer.js";
|
||||
import { TRANSFORM_HOOK } from "../common/infrastructure/Transform.js";
|
||||
import { parseRetryAfterSecsFromObj, playObjDataMatch, readJson, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js";
|
||||
import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js";
|
||||
import { CookieJar, Cookie } from 'tough-cookie';
|
||||
@@ -11,6 +12,7 @@ import { MixedCookieAgent } from 'http-cookie-agent/http';
|
||||
import MemorySource from "./MemorySource.js";
|
||||
import { genericSourcePlayMatch } from "../utils/PlayComparisonUtils.js";
|
||||
import { TemporalPlayComparisonOptions } from "../utils/TimeUtils.js";
|
||||
import { findAsync, findIndexAsync } from "../utils/AsyncUtils.js";
|
||||
|
||||
interface DeezerHistoryResponse {
|
||||
errors: []
|
||||
@@ -201,20 +203,20 @@ export default class DeezerInternalSource extends MemorySource {
|
||||
protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) => await this.getRecentlyPlayed({formatted: true, ...options})
|
||||
|
||||
|
||||
existingDiscovered = (play: PlayObject, opts: {checkAll?: boolean} = {}): PlayObject | undefined => {
|
||||
existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<PlayObject | undefined> => {
|
||||
const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts);
|
||||
const candidate = this.transformPlay(play, TRANSFORM_HOOK.candidate);
|
||||
const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate);
|
||||
for(const list of lists) {
|
||||
const existing = list.find(x => {
|
||||
const e = this.transformPlay(x, TRANSFORM_HOOK.existing);
|
||||
const existing = await findAsync(list, async x => {
|
||||
const e = await this.transformPlay(x, TRANSFORM_HOOK.existing);
|
||||
return genericSourcePlayMatch(e, candidate);
|
||||
});
|
||||
if(existing) {
|
||||
return existing;
|
||||
}
|
||||
if(this.config.options?.fuzzyDiscoveryIgnore === true || this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {
|
||||
const fuzzyIndex = list.findIndex(x => {
|
||||
const e = this.transformPlay(x, TRANSFORM_HOOK.existing);
|
||||
const fuzzyIndex = await findIndexAsync(list, async x => {
|
||||
const e = await this.transformPlay(x, TRANSFORM_HOOK.existing);
|
||||
let temporalOptions: TemporalPlayComparisonOptions = {};
|
||||
const temporalAccuracy: TemporalAccuracy[] = [TA_EXACT, TA_CLOSE, TA_FUZZY];
|
||||
if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {
|
||||
@@ -278,4 +280,4 @@ const buildInternalUrl = (method: string, token: string = ''): URL => {
|
||||
const u = new URL(`https://www.deezer.com/ajax/gw-light.php?${params.toString()}`);
|
||||
|
||||
return u;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,12 +72,12 @@ export class EndpointLastfmSource extends MemorySource {
|
||||
|
||||
handle = async (stateData: PlayerStateData) => {
|
||||
|
||||
this.processRecentPlays([stateData]);
|
||||
await this.processRecentPlays([stateData]);
|
||||
|
||||
if (stateData.play.meta.nowPlaying === false && this.isValidScrobble(stateData.play)) {
|
||||
const discovered = this.discover([stateData.play]);
|
||||
const discovered = await this.discover([stateData.play]);
|
||||
if (discovered.length > 0) {
|
||||
this.scrobble(discovered);
|
||||
await this.scrobble(discovered);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,12 +89,12 @@ export class EndpointListenbrainzSource extends MemorySource {
|
||||
|
||||
handle = async (stateData: PlayerStateData) => {
|
||||
|
||||
this.processRecentPlays([stateData]);
|
||||
await this.processRecentPlays([stateData]);
|
||||
|
||||
if (stateData.play.meta.nowPlaying === false && this.isValidScrobble(stateData.play)) {
|
||||
const discovered = this.discover([stateData.play]);
|
||||
const discovered = await this.discover([stateData.play]);
|
||||
if (discovered.length > 0) {
|
||||
this.scrobble(discovered);
|
||||
await this.scrobble(discovered);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,7 +122,7 @@ export class IcecastSource extends MemorySource {
|
||||
}
|
||||
|
||||
if (this.currentMetadata === undefined) {
|
||||
return this.processRecentPlays([]);
|
||||
return await this.processRecentPlays([]);
|
||||
}
|
||||
|
||||
// if (this.manualListening === false || (this.config.options.scrobbleOnStart === false && this.manualListening === undefined)) {
|
||||
@@ -146,7 +146,7 @@ export class IcecastSource extends MemorySource {
|
||||
play
|
||||
}
|
||||
|
||||
return this.processRecentPlays([playerState]);
|
||||
return await this.processRecentPlays([playerState]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -142,7 +142,7 @@ export class JRiverSource extends MemoryPositionalSource {
|
||||
}
|
||||
}
|
||||
|
||||
return this.processRecentPlays(play);
|
||||
return await this.processRecentPlays(play);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -497,7 +497,7 @@ export default class JellyfinApiSource extends MemoryPositionalSource {
|
||||
this.logger[this.logFilterFailure](dropReason);
|
||||
}
|
||||
}
|
||||
return this.processRecentPlays(validSessions);
|
||||
return await this.processRecentPlays(validSessions);
|
||||
}
|
||||
|
||||
sessionToPlayerState = (obj: SessionInfo): PlayerStateDataMaybePlay => {
|
||||
|
||||
@@ -354,13 +354,13 @@ export default class JellyfinSource extends MemorySource {
|
||||
scrobbleOpts.checkAll = true;
|
||||
|
||||
} else {
|
||||
newPlays = this.processRecentPlays([playObj]);
|
||||
newPlays = await this.processRecentPlays([playObj]);
|
||||
}
|
||||
|
||||
if(newPlays.length > 0) {
|
||||
try {
|
||||
const discovered = this.discover(newPlays, scrobbleOpts);
|
||||
this.scrobble(discovered);
|
||||
const discovered = await this.discover(newPlays, scrobbleOpts);
|
||||
await this.scrobble(discovered);
|
||||
} catch (e) {
|
||||
this.logger.error('Encountered error while scrobbling')
|
||||
this.logger.error(e)
|
||||
|
||||
@@ -59,7 +59,7 @@ export class KodiSource extends MemoryPositionalSource {
|
||||
|
||||
const play = await this.client.getRecentlyPlayed(options);
|
||||
|
||||
return this.processRecentPlays(play);
|
||||
return await this.processRecentPlays(play);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ export default class KoitoSource extends MemorySource {
|
||||
|
||||
getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}) => {
|
||||
const {limit = 20} = options;
|
||||
this.processRecentPlays([]);
|
||||
await this.processRecentPlays([]);
|
||||
return await this.api.getRecentlyPlayed(limit);
|
||||
}
|
||||
|
||||
|
||||
@@ -131,7 +131,7 @@ export default class LastfmSource extends MemorySource {
|
||||
getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}): Promise<PlayObject[]> => {
|
||||
try {
|
||||
const [history, now] = await this.getLastfmRecentTrack(options);
|
||||
this.processRecentPlays(now);
|
||||
await this.processRecentPlays(now);
|
||||
return history;
|
||||
} catch (e) {
|
||||
throw e;
|
||||
|
||||
@@ -79,7 +79,7 @@ export default class ListenbrainzSource extends MemorySource {
|
||||
return await this.api.getRecentlyPlayedKoito(limit);
|
||||
}
|
||||
const now = await this.api.getPlayingNow();
|
||||
this.processRecentPlays(now.listens.map(x => ListenbrainzSource.formatPlayObj(x)));
|
||||
await this.processRecentPlays(now.listens.map(x => ListenbrainzSource.formatPlayObj(x)));
|
||||
return await this.api.getRecentlyPlayed(limit);
|
||||
}
|
||||
|
||||
|
||||
@@ -228,7 +228,7 @@ export class MPDSource extends MemoryPositionalSource {
|
||||
position: state.elapsed
|
||||
}
|
||||
|
||||
return this.processRecentPlays([playerState]);
|
||||
return await this.processRecentPlays([playerState]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -243,7 +243,7 @@ export class MPRISSource extends MemorySource {
|
||||
if(options.display === true) {
|
||||
return deduped;
|
||||
}
|
||||
return this.processRecentPlays(deduped);
|
||||
return await this.processRecentPlays(deduped);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ export default class MalojaSource extends MemorySource {
|
||||
|
||||
getRecentlyPlayed = async (options: RecentlyPlayedOptions = {}) => {
|
||||
const { limit = 20 } = options;
|
||||
this.processRecentPlays([]);
|
||||
await this.processRecentPlays([]);
|
||||
return await this.api.getRecentScrobbles(limit);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Logger } from "@foxxmd/logging";
|
||||
import dayjs, { Dayjs } from "dayjs";
|
||||
import { EventEmitter } from "events";
|
||||
import { SimpleIntervalJob, Task, ToadScheduler } from "toad-scheduler";
|
||||
import { AsyncTask, SimpleIntervalJob, Task, ToadScheduler } from "toad-scheduler";
|
||||
import { PlayObject, SOURCE_SOT, SOURCE_SOT_TYPES, SourcePlayerObj } from "../../core/Atomic.js";
|
||||
import { buildTrackString } from "../../core/StringUtils.js";
|
||||
import {
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
thresholdResultSummary,
|
||||
} from "../utils.js";
|
||||
import { timePassesScrobbleThreshold, timeToHumanTimestamp } from "../utils/TimeUtils.js";
|
||||
import { PromisePool } from "@supercharge/promise-pool";
|
||||
import AbstractSource from "./AbstractSource.js";
|
||||
import { AbstractPlayerState, createPlayerOptions, PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js";
|
||||
import { GenericPlayerState } from "./PlayerState/GenericPlayerState.js";
|
||||
@@ -56,10 +57,17 @@ export default class MemorySource extends AbstractSource {
|
||||
|
||||
// player cleanup on *schedule* is needed when the Source is non-polling (ingress)
|
||||
// because if the source stops sending updates then processRecentPlays() was never called so we never remove old players
|
||||
this.scheduler.addSimpleIntervalJob(new SimpleIntervalJob({seconds: 15}, new Task('Player Cleanup', () => {
|
||||
if(!this.canPoll) {
|
||||
this.cleanupPlayers();
|
||||
this.scheduler.addSimpleIntervalJob(new SimpleIntervalJob({ seconds: 15 }, new AsyncTask('Player Cleanup', (): Promise<any> => {
|
||||
if (!this.canPoll) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
return PromisePool
|
||||
.withConcurrency(1)
|
||||
.for(this.players.keys())
|
||||
.process(async (key) => {
|
||||
|
||||
await this.cleanupPlayer(key);
|
||||
});
|
||||
})));
|
||||
}
|
||||
|
||||
@@ -69,7 +77,7 @@ export default class MemorySource extends AbstractSource {
|
||||
}
|
||||
}
|
||||
|
||||
cleanupPlayer = (key: string): PlayObject | undefined => {
|
||||
cleanupPlayer = async (key: string): Promise<PlayObject | undefined> => {
|
||||
const player = this.players.get(key);
|
||||
if(player === undefined) {
|
||||
this.logger.warn({labels: 'Player Cleanup'},`No Player with ID ${key} exists! Cannot cleanup.`);
|
||||
@@ -113,7 +121,7 @@ export default class MemorySource extends AbstractSource {
|
||||
const cleanupPlay = player.getPlayedObject(true);
|
||||
let discoverablePlay: boolean;
|
||||
if(cleanupPlay !== undefined) {
|
||||
const [discoverable, discoverableReason] = this.isListenedPlayDiscoverable(cleanupPlay);
|
||||
const [discoverable, discoverableReason] = await this.isListenedPlayDiscoverable(cleanupPlay);
|
||||
discoverablePlay = discoverable;
|
||||
if(this.playerSourceOfTruth === SOURCE_SOT.PLAYER) {
|
||||
player.logger.verbose({labels: label}, discoverableReason);
|
||||
@@ -188,7 +196,7 @@ export default class MemorySource extends AbstractSource {
|
||||
return sessions[0];
|
||||
}
|
||||
|
||||
processRecentPlays = (datas: (PlayObject | PlayerStateDataMaybePlay)[], reportedTS?: Dayjs) => {
|
||||
processRecentPlays = async (datas: (PlayObject | PlayerStateDataMaybePlay)[], reportedTS?: Dayjs) => {
|
||||
|
||||
const {
|
||||
options: {
|
||||
@@ -250,7 +258,7 @@ export default class MemorySource extends AbstractSource {
|
||||
// wait to discover play until it is stale or current play has changed
|
||||
// so that our discovered track has an accurate "listenedFor" count
|
||||
if (candidate !== undefined && (playChanged || player.isUpdateStale())) {
|
||||
const [discoverable, discoverableReason] = this.isListenedPlayDiscoverable(candidate);
|
||||
const [discoverable, discoverableReason] = await this.isListenedPlayDiscoverable(candidate);
|
||||
if(discoverable) {
|
||||
if(this.playerSourceOfTruth === SOURCE_SOT.PLAYER) {
|
||||
player.logger.verbose(discoverableReason);
|
||||
@@ -273,7 +281,7 @@ export default class MemorySource extends AbstractSource {
|
||||
}
|
||||
});
|
||||
} else {
|
||||
const playFromCleanup = this.cleanupPlayer(key);
|
||||
const playFromCleanup = await this.cleanupPlayer(key);
|
||||
if(playFromCleanup !== undefined) {
|
||||
newStatefulPlays.push(playFromCleanup);
|
||||
}
|
||||
@@ -283,7 +291,7 @@ export default class MemorySource extends AbstractSource {
|
||||
return newStatefulPlays;
|
||||
}
|
||||
|
||||
protected isListenedPlayDiscoverable = (candidate: PlayObject): [boolean, string] => {
|
||||
protected isListenedPlayDiscoverable = async (candidate: PlayObject): Promise<[boolean, string]> => {
|
||||
|
||||
const {
|
||||
options: {
|
||||
@@ -295,7 +303,7 @@ export default class MemorySource extends AbstractSource {
|
||||
const thresholdResults = timePassesScrobbleThreshold(scrobbleThresholds, candidate.data.listenedFor, candidate.data.duration);
|
||||
|
||||
if (thresholdResults.passes) {
|
||||
const matchingRecent = this.existingDiscovered(candidate); //sRecentlyPlayed.find(x => playObjDataMatch(x, candidate));
|
||||
const matchingRecent = await this.existingDiscovered(candidate); //sRecentlyPlayed.find(x => playObjDataMatch(x, candidate));
|
||||
if (matchingRecent === undefined) {
|
||||
return [true,`${stPrefix} added after ${thresholdResultSummary(thresholdResults)} and not matching any prior plays`];
|
||||
} else {
|
||||
|
||||
@@ -213,7 +213,7 @@ export class MopidySource extends MemoryPositionalSource {
|
||||
play
|
||||
}
|
||||
|
||||
return this.processRecentPlays([playerState]);
|
||||
return await this.processRecentPlays([playerState]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -112,12 +112,12 @@ export class MusicCastSource extends MemoryPositionalSource {
|
||||
}
|
||||
if ((statusResp.body as DeviceStatusResponse).power !== 'on') {
|
||||
this.logger.debug('MusicCast device is offline');
|
||||
return this.processRecentPlays([]);
|
||||
return await this.processRecentPlays([]);
|
||||
}
|
||||
|
||||
const playInfo = await this.getAnyPlayInfo();
|
||||
if(playInfo === undefined) {
|
||||
return this.processRecentPlays([]);
|
||||
return await this.processRecentPlays([]);
|
||||
}
|
||||
|
||||
const play = formatPlayObj(playInfo);
|
||||
@@ -130,7 +130,7 @@ export class MusicCastSource extends MemoryPositionalSource {
|
||||
position: play.meta.trackProgressPosition
|
||||
}
|
||||
|
||||
return this.processRecentPlays([playerState]);
|
||||
return await this.processRecentPlays([playerState]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -240,7 +240,7 @@ export class MusikcubeSource extends MemoryPositionalSource {
|
||||
position: playbackOverview.options.playing_current_time
|
||||
}
|
||||
|
||||
return this.processRecentPlays([playerState]);
|
||||
return await this.processRecentPlays([playerState]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -270,14 +270,14 @@ export default class PlexApiSource extends MemoryPositionalSource {
|
||||
|
||||
if(state.play !== undefined) {
|
||||
const allowedLibraries = this.getAllowedLibraries();
|
||||
if(allowedLibraries.length > 0 && !allowedLibraries.some(x => state.play.meta.library.toLocaleLowerCase() === x.name.toLocaleLowerCase())) {
|
||||
if(allowedLibraries.length > 0 && !allowedLibraries.some(x => (state.play.meta.library ?? '').toLocaleLowerCase() === x.name.toLocaleLowerCase())) {
|
||||
return `media not included in librariesAllow`;
|
||||
}
|
||||
|
||||
if(allowedLibraries.length === 0) {
|
||||
const blockedLibraries = this.getBlockedLibraries();
|
||||
if(blockedLibraries.length > 0) {
|
||||
const blockedLibrary = blockedLibraries.find(x => state.play.meta.library.toLocaleLowerCase() === x.name.toLocaleLowerCase());
|
||||
const blockedLibrary = blockedLibraries.find(x => (state.play.meta.library ?? '').toLocaleLowerCase() === x.name.toLocaleLowerCase());
|
||||
if(blockedLibrary !== undefined) {
|
||||
return `media included in librariesBlock '${blockedLibrary.name}'`;
|
||||
}
|
||||
@@ -286,7 +286,7 @@ export default class PlexApiSource extends MemoryPositionalSource {
|
||||
// this is inside this block because we SHOULD allow non-music libraries if
|
||||
// user specified name in librariesAllow
|
||||
// -- so only check for this if nothing is specified
|
||||
if(!this.getValidLibraries().some(x => state.play.meta.library === x.name)) {
|
||||
if(!this.getValidLibraries().some(x => (state.play.meta.library ?? '') === x.name)) {
|
||||
return `media not included in a valid library`;
|
||||
}
|
||||
}
|
||||
@@ -406,7 +406,7 @@ export default class PlexApiSource extends MemoryPositionalSource {
|
||||
}
|
||||
}
|
||||
}
|
||||
return this.processRecentPlays(validSessions);
|
||||
return await this.processRecentPlays(validSessions);
|
||||
}
|
||||
|
||||
getSourceArt = async (data: string): Promise<[Readable, string]> => {
|
||||
|
||||
@@ -223,8 +223,8 @@ export default class PlexSource extends AbstractSource {
|
||||
}
|
||||
|
||||
try {
|
||||
const discovered = this.discover([playObj]);
|
||||
this.scrobble(discovered);
|
||||
const discovered = await this.discover([playObj]);
|
||||
await this.scrobble(discovered);
|
||||
} catch (e) {
|
||||
this.logger.error('Encountered error while scrobbling')
|
||||
this.logger.error(e)
|
||||
|
||||
@@ -356,7 +356,7 @@ export default class SpotifySource extends MemoryPositionalSource {
|
||||
plays.push(currPlay);
|
||||
}
|
||||
}
|
||||
const newPlays = this.processRecentPlays(plays);
|
||||
const newPlays = await this.processRecentPlays(plays);
|
||||
// hint that scrobble timestamp source of truth should be when the track ended (player changed tracks)
|
||||
// rather than when we first saw the track
|
||||
//
|
||||
|
||||
@@ -292,7 +292,7 @@ export class SubsonicSource extends MemorySource {
|
||||
// sometimes subsonic sources will return the same track as being played twice on the same player, need to remove this so we don't duplicate plays
|
||||
const deduped = removeDuplicates(entry.map(x => SubsonicSource.formatPlayObj(x, {sourceData: this.sourceData})));
|
||||
const userFiltered = this.usersAllow.length == 0 ? deduped : deduped.filter(x => x.meta.user === undefined || this.usersAllow.map(x => x.toLocaleLowerCase()).includes(x.meta.user.toLocaleLowerCase()));
|
||||
return this.processRecentPlays(userFiltered);
|
||||
return await this.processRecentPlays(userFiltered);
|
||||
}
|
||||
|
||||
getNewPlayer = (logger: Logger, id: PlayPlatformId, opts: PlayerStateOptions) => new SubsonicPlayerState(logger, id, opts);
|
||||
|
||||
@@ -95,7 +95,7 @@ export default class TealfmSource extends MemorySource {
|
||||
} catch (e) {
|
||||
throw new Error('Error occurred while trying to fetch records', {cause: e});
|
||||
}
|
||||
this.processRecentPlays([]);
|
||||
await this.processRecentPlays([]);
|
||||
const plays = list.map(x => listRecordToPlay(x));
|
||||
return plays;
|
||||
}
|
||||
|
||||
@@ -259,7 +259,7 @@ export class VLCSource extends MemoryPositionalSource {
|
||||
position: state.time
|
||||
}
|
||||
|
||||
return this.processRecentPlays([playerState]);
|
||||
return await this.processRecentPlays([playerState]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -180,12 +180,12 @@ export class WebScrobblerSource extends MemorySource {
|
||||
|
||||
handle = async (stateData: PlayerStateData) => {
|
||||
|
||||
this.processRecentPlays([stateData]);
|
||||
await this.processRecentPlays([stateData]);
|
||||
|
||||
if (stateData.play.meta.nowPlaying === false && this.isValidScrobble(stateData.play)) {
|
||||
const discovered = this.discover([stateData.play]);
|
||||
const discovered = await this.discover([stateData.play]);
|
||||
if (discovered.length > 0) {
|
||||
this.scrobble(discovered);
|
||||
await this.scrobble(discovered);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,441 +0,0 @@
|
||||
import { loggerTest, loggerDebug, childLogger } from "@foxxmd/logging";
|
||||
import chai, { assert, expect } from 'chai';
|
||||
import asPromised from 'chai-as-promised';
|
||||
import { after, before, describe, it } from 'mocha';
|
||||
import AbstractComponent from "../../common/AbstractComponent.js";
|
||||
import { TRANSFORM_HOOK } from "../../common/infrastructure/Atomic.js";
|
||||
|
||||
import { isConditionalSearchAndReplace } from "../../utils/PlayTransformUtils.js";
|
||||
import { asPlays, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js";
|
||||
import { WebhookPayload } from "../../common/infrastructure/config/health/webhooks.js";
|
||||
|
||||
chai.use(asPromised);
|
||||
|
||||
class TestComponent extends AbstractComponent {
|
||||
public notify(payload: WebhookPayload): Promise<void> {
|
||||
throw new Error("Method not implemented.");
|
||||
}
|
||||
protected getIdentifier(): string {
|
||||
return 'test';
|
||||
}
|
||||
constructor() {
|
||||
super({});
|
||||
}
|
||||
}
|
||||
|
||||
const component = new TestComponent();
|
||||
component.logger = childLogger(loggerTest, 'App');
|
||||
|
||||
describe('Play Transforms', function () {
|
||||
|
||||
beforeEach(function() {
|
||||
component.config = {};
|
||||
component.transformRules = {};
|
||||
});
|
||||
|
||||
describe('Transform Config Parsing', function() {
|
||||
|
||||
it('Sets transform rules as empty object if config is not present', function() {
|
||||
component.buildTransformRules();
|
||||
expect(component.transformRules).exist;
|
||||
expect(Object.keys(component.transformRules).length).eq(0);
|
||||
});
|
||||
|
||||
it('Converts single object hook into hook array', function() {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
|
||||
expect(component.transformRules.preCompare).to.be.an('array');
|
||||
expect(component.transformRules.preCompare).to.be.length(1);
|
||||
expect(component.transformRules.preCompare).to.have.nested.property('0.title');
|
||||
});
|
||||
|
||||
it('Accepts hook array', function() {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: [
|
||||
{
|
||||
title: ['something']
|
||||
},
|
||||
{
|
||||
title: ['something else']
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
|
||||
expect(component.transformRules.preCompare).to.be.an('array');
|
||||
expect(component.transformRules.preCompare).to.be.length(2);
|
||||
expect(component.transformRules.preCompare).to.have.nested.property('0.title');
|
||||
expect(component.transformRules.preCompare).to.have.nested.property('1.title')
|
||||
});
|
||||
|
||||
it('Converts transform config into real S&P data', function() {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
|
||||
expect(component.transformRules.preCompare![0]).to.exist;
|
||||
expect(component.transformRules.preCompare![0].title).to.exist;
|
||||
expect(Array.isArray(component.transformRules.preCompare![0].title)).is.true;
|
||||
expect( isConditionalSearchAndReplace(component.transformRules.preCompare![0].title![0])).is.true
|
||||
});
|
||||
|
||||
it('Converts transform config into real S&P data with default being empty string', function() {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
|
||||
expect(component.transformRules.preCompare![0]).to.exist;
|
||||
expect(component.transformRules.preCompare![0].title).to.exist;
|
||||
expect(Array.isArray(component.transformRules.preCompare![0].title)).is.true;
|
||||
expect( isConditionalSearchAndReplace(component.transformRules.preCompare![0].title![0])).is.true
|
||||
expect( component.transformRules.preCompare![0].title![0].search).is.eq('something');
|
||||
expect( component.transformRules.preCompare![0].title![0].replace).is.eq('');
|
||||
});
|
||||
|
||||
it('Respects transform config when it is already S&P data', function() {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: [
|
||||
{
|
||||
search: 'nothing',
|
||||
replace: 'anything'
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
|
||||
expect(component.transformRules.preCompare![0]).to.exist;
|
||||
expect(component.transformRules.preCompare![0].title).to.exist;
|
||||
expect(Array.isArray(component.transformRules.preCompare![0].title)).is.true;
|
||||
expect( isConditionalSearchAndReplace(component.transformRules.preCompare![0].title![0])).is.true
|
||||
expect( component.transformRules.preCompare![0].title![0].search).is.eq('nothing');
|
||||
expect( component.transformRules.preCompare![0].title![0].replace).is.eq('anything');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Play Transforming', function() {
|
||||
|
||||
it('Returns original play if no hooks are defined', function () {
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay();
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(JSON.stringify(play)).equal(JSON.stringify(transformed));
|
||||
});
|
||||
|
||||
it('Transforms when hook is present', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({track: 'My coolsomething track'});
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.track).equal('My cool track');
|
||||
});
|
||||
|
||||
it('Transforms consecutively when hook is present with multiple values', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: ['something', 'cool']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({track: 'My coolsomething track'});
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.track).equal('My track');
|
||||
});
|
||||
|
||||
it('Transforms using parsed regex', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: [
|
||||
{
|
||||
search: '/(cool )(some)(thing)/i',
|
||||
replace: '$1$3'
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({track: 'My cool something track'});
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.track).equal('My cool thing track');
|
||||
});
|
||||
|
||||
|
||||
it('Transforms using parsed regex to get primary artist from delimited artist string', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
artists: [
|
||||
{
|
||||
search: '/(.*?)(\\s*\\/\\s*)(.*$)/i',
|
||||
replace: '$1'
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({artists: ['My Artist One / My Artist Two / Another Guy']});
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.artists).length(1)
|
||||
expect(transformed.data.artists[0]).equal('My Artist One');
|
||||
});
|
||||
|
||||
it('Removes title when transform replaces with empty string', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({track: 'something'});
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.track).is.undefined;
|
||||
});
|
||||
|
||||
it('Removes album when transform replaces with empty string', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
album: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({album: 'something'});
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.album).is.undefined;
|
||||
});
|
||||
|
||||
it('Removes an artist when transform replaces with empty string', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
artists: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({artists: ['something', 'big']});
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.artists!.length).is.eq(1)
|
||||
expect(transformed.data.artists![0]).is.eq('big')
|
||||
});
|
||||
});
|
||||
|
||||
describe('Conditional Transforming', function() {
|
||||
|
||||
describe('On Hook', function () {
|
||||
it('Does not run hook if when conditions do not match', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
when: [
|
||||
{
|
||||
album: "Has This"
|
||||
}
|
||||
],
|
||||
artists: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({artists: ['something', 'big'], album: 'It Has No Match'});
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.artists!.length).is.eq(2)
|
||||
expect(transformed.data.artists![0]).is.eq('something')
|
||||
});
|
||||
|
||||
it('Does run hook if when conditions matches', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
when: [
|
||||
{
|
||||
album: "Has This"
|
||||
}
|
||||
],
|
||||
artists: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({artists: ['something', 'big'], album: 'It Has This Match'});
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.artists!.length).is.eq(1)
|
||||
expect(transformed.data.artists![0]).is.eq('big')
|
||||
});
|
||||
});
|
||||
|
||||
describe('On Search-And-Replace', function() {
|
||||
it('Does not run hook if when conditions do not match', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
artists: [
|
||||
{
|
||||
search: "something",
|
||||
replace: "",
|
||||
when: [
|
||||
{
|
||||
album: "Has This"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({artists: ['something', 'big'], album: 'It Has No Match'});
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.artists!.length).is.eq(2)
|
||||
expect(transformed.data.artists![0]).is.eq('something')
|
||||
});
|
||||
|
||||
it('Does run hook if when conditions matches', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
artists: [
|
||||
{
|
||||
search: "something",
|
||||
replace: "",
|
||||
when: [
|
||||
{
|
||||
album: "Has This"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({artists: ['something', 'big'], album: 'It Has This Match'});
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.artists!.length).is.eq(1)
|
||||
expect(transformed.data.artists![0]).is.eq('big')
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
describe('Multiple hook transforms', function() {
|
||||
|
||||
it('Accumulates transforms', function() {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: [
|
||||
{
|
||||
title: [
|
||||
{
|
||||
search: "something",
|
||||
replace: "another else"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
title: [
|
||||
{
|
||||
search: "another else",
|
||||
replace: "final thing"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
const play = generatePlay({track: 'My cool something track'});
|
||||
const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.track).equal('My cool final thing track');
|
||||
});
|
||||
|
||||
});
|
||||
})
|
||||
@@ -0,0 +1,631 @@
|
||||
import { loggerTest, loggerDebug, childLogger } from "@foxxmd/logging";
|
||||
import chai, { assert, expect } from 'chai';
|
||||
import asPromised from 'chai-as-promised';
|
||||
import { after, before, describe, it } from 'mocha';
|
||||
import AbstractComponent from "../../common/AbstractComponent.js";
|
||||
|
||||
import { ConditionalSearchAndReplaceRegExp, STAGE_TYPES, STAGE_TYPES_METADATA, STAGE_TYPES_USER, TRANSFORM_HOOK } from "../../common/infrastructure/Transform.js";
|
||||
|
||||
import { isConditionalSearchAndReplace } from "../../utils/PlayTransformUtils.js";
|
||||
import { asPlays, generateArtistsStr, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js";
|
||||
import { WebhookPayload } from "../../common/infrastructure/config/health/webhooks.js";
|
||||
import { findCauseByMessage } from "../../utils/ErrorUtils.js";
|
||||
import NativeTransformer from "../../common/transforms/NativeTransformer.js";
|
||||
import { initMemoryCache } from "../../common/Cache.js";
|
||||
import { Cacheable } from "cacheable";
|
||||
|
||||
chai.use(asPromised);
|
||||
|
||||
class TestComponent extends AbstractComponent {
|
||||
public notify(payload: WebhookPayload): Promise<void> {
|
||||
throw new Error("Method not implemented.");
|
||||
}
|
||||
protected getIdentifier(): string {
|
||||
return 'test';
|
||||
}
|
||||
constructor() {
|
||||
super({});
|
||||
}
|
||||
}
|
||||
|
||||
const component = new TestComponent();
|
||||
component.logger = childLogger(loggerTest, 'App');
|
||||
|
||||
const memorycache = () => new Cacheable({primary: initMemoryCache()})
|
||||
|
||||
describe('Play Transforms', function () {
|
||||
|
||||
beforeEach(function () {
|
||||
component.config = {};
|
||||
component.transformRules = {};
|
||||
});
|
||||
describe('Transform Config Parsing', function () {
|
||||
|
||||
it('Sets transform rules as empty object if config is not present', function () {
|
||||
component.buildTransformRules();
|
||||
expect(component.transformRules).exist;
|
||||
expect(Object.keys(component.transformRules).length).eq(0);
|
||||
});
|
||||
|
||||
it('Converts single object hook into hook array', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
type: "user",
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
|
||||
expect(component.transformRules.preCompare).to.be.an('array');
|
||||
expect(component.transformRules.preCompare).to.be.length(1);
|
||||
expect(component.transformRules.preCompare).to.have.nested.property('0.title');
|
||||
});
|
||||
|
||||
describe('Stage Parsing', function () {
|
||||
|
||||
it(`Throws an error if stage is an unexpected value`, function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
type: "test",
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// https://github.com/chaijs/chai/issues/655#issuecomment-204386414
|
||||
expect(() => component.buildTransformRules()).to.throw(Error).that.satisfies((e) => {
|
||||
return findCauseByMessage(e, `No transformer of type 'test'`);
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
describe('User Stage Parsing', function () {
|
||||
|
||||
it(`Assumes user 'type' if no type is present`, function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
|
||||
expect(component.transformRules.preCompare).to.be.an('array');
|
||||
expect(component.transformRules.preCompare).to.be.length(1);
|
||||
expect(component.transformRules.preCompare).to.have.nested.property('0.type');
|
||||
expect(component.transformRules.preCompare[0].type).eq('user');
|
||||
});
|
||||
|
||||
it(`Allows user 'type'`, function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
type: "user",
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
|
||||
expect(component.transformRules.preCompare).to.be.an('array');
|
||||
expect(component.transformRules.preCompare).to.be.length(1);
|
||||
expect(component.transformRules.preCompare).to.have.nested.property('0.type');
|
||||
expect(component.transformRules.preCompare[0].type).eq('user');
|
||||
});
|
||||
|
||||
it('Accepts hook array', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: [
|
||||
{
|
||||
title: ['something']
|
||||
},
|
||||
{
|
||||
title: ['something else']
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
|
||||
expect(component.transformRules.preCompare).to.be.an('array');
|
||||
expect(component.transformRules.preCompare).to.be.length(2);
|
||||
expect(component.transformRules.preCompare).to.have.nested.property('0.title');
|
||||
expect(component.transformRules.preCompare).to.have.nested.property('1.title')
|
||||
});
|
||||
|
||||
it('Converts transform config into real S&P data', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
|
||||
expect(component.transformRules.preCompare![0]).to.exist;
|
||||
expect(component.transformRules.preCompare![0].title).to.exist;
|
||||
expect(Array.isArray(component.transformRules.preCompare![0].title)).is.true;
|
||||
expect(isConditionalSearchAndReplace(component.transformRules.preCompare![0].title![0])).is.true
|
||||
});
|
||||
|
||||
it('Converts transform config into real S&P data with default being empty string', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
|
||||
expect(component.transformRules.preCompare![0]).to.exist;
|
||||
expect(component.transformRules.preCompare![0].title).to.exist;
|
||||
expect(Array.isArray(component.transformRules.preCompare![0].title)).is.true;
|
||||
expect(isConditionalSearchAndReplace(component.transformRules.preCompare![0].title![0])).is.true
|
||||
const title = component.transformRules.preCompare![0].title![0] as ConditionalSearchAndReplaceRegExp;
|
||||
expect(title.search).is.eq('something');
|
||||
expect(title.replace).is.eq('');
|
||||
});
|
||||
|
||||
it('Respects transform config when it is already S&P data', function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: [
|
||||
{
|
||||
|
||||
search: 'nothing',
|
||||
replace: 'anything'
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
|
||||
expect(component.transformRules.preCompare![0]).to.exist;
|
||||
expect(component.transformRules.preCompare![0].title).to.exist;
|
||||
expect(Array.isArray(component.transformRules.preCompare![0].title)).is.true;
|
||||
expect(isConditionalSearchAndReplace(component.transformRules.preCompare![0].title![0])).is.true
|
||||
const title = component.transformRules.preCompare![0].title![0] as ConditionalSearchAndReplaceRegExp;
|
||||
expect(title.search).is.eq('nothing');
|
||||
expect(title.replace).is.eq('anything');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Non-User Stage Parsing', function () {
|
||||
|
||||
describe('Non-User Stage Types', function () {
|
||||
|
||||
for(const t of ['native']) {
|
||||
|
||||
it(`Allows non-user Stage Type ${t}`, function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
type: t,
|
||||
title: true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(() => component.buildTransformRules()).to.not.throw();
|
||||
expect(component.transformRules.preCompare).to.be.an('array');
|
||||
expect(component.transformRules.preCompare).to.be.length(1);
|
||||
expect(component.transformRules.preCompare).to.have.nested.property('0.type');
|
||||
expect(component.transformRules.preCompare[0].type).eq(t);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
describe('Play Transforming', function () {
|
||||
|
||||
it('Returns original play if no hooks are defined', async function () {
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay();
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(JSON.stringify(play)).equal(JSON.stringify(transformed));
|
||||
});
|
||||
|
||||
describe('User Play Transforming', function () {
|
||||
it('Transforms when hook is present', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({ track: 'My coolsomething track' });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.track).equal('My cool track');
|
||||
});
|
||||
|
||||
it('Transforms consecutively when hook is present with multiple values', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: ['something', 'cool']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({ track: 'My coolsomething track' });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.track).equal('My track');
|
||||
});
|
||||
|
||||
it('Transforms using parsed regex', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: [
|
||||
{
|
||||
search: '/(cool )(some)(thing)/i',
|
||||
replace: '$1$3'
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({ track: 'My cool something track' });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.track).equal('My cool thing track');
|
||||
});
|
||||
|
||||
|
||||
it('Transforms using parsed regex to get primary artist from delimited artist string', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
artists: [
|
||||
{
|
||||
search: '/(.*?)(\\s*\\/\\s*)(.*$)/i',
|
||||
replace: '$1'
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({ artists: ['My Artist One / My Artist Two / Another Guy'] });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.artists).length(1)
|
||||
expect(transformed.data.artists[0]).equal('My Artist One');
|
||||
});
|
||||
|
||||
it('Removes title when transform replaces with empty string', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
title: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({ track: 'something' });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.track).is.undefined;
|
||||
});
|
||||
|
||||
it('Removes album when transform replaces with empty string', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
album: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({ album: 'something' });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.album).is.undefined;
|
||||
});
|
||||
|
||||
it('Removes an artist when transform replaces with empty string', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
artists: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({ artists: ['something', 'big'] });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.artists!.length).is.eq(1)
|
||||
expect(transformed.data.artists![0]).is.eq('big')
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
});
|
||||
});
|
||||
|
||||
describe('Native Transformer', function () {
|
||||
|
||||
it('Uses artist parsing functions', async function() {
|
||||
|
||||
const t = new NativeTransformer({name: 'test', type: 'native'}, {logger: loggerTest, cache: memorycache()});
|
||||
await t.tryInitialize();
|
||||
|
||||
const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}});
|
||||
const play = generatePlay({artists: [str]});
|
||||
|
||||
const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play);
|
||||
expect(transformedPlay.data.artists).eql(primaries.concat(secondaries));
|
||||
});
|
||||
|
||||
it('Ignores artists', async function() {
|
||||
|
||||
const [str, primaries, secondaries] = generateArtistsStr({primary: {num: 2, ambiguousJoinedNames: false, trailingAmpersand: false, finalJoiner: false}, secondary: 0});
|
||||
|
||||
const t = new NativeTransformer({name: 'test', type: 'native', defaults: {artistsIgnore: [str]}}, {logger: loggerTest, cache: memorycache()});
|
||||
|
||||
await t.tryInitialize();
|
||||
|
||||
const play = generatePlay({artists: [str], track: 'My Test'});
|
||||
|
||||
const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play);
|
||||
expect(transformedPlay.data.artists).eql([str]);
|
||||
});
|
||||
|
||||
it('Uses custom delimiters artists', async function() {
|
||||
|
||||
const [str, primaries, secondaries] = generateArtistsStr({primary: {
|
||||
max: 3,
|
||||
joiner: '•',
|
||||
spacedJoiners: true,
|
||||
ambiguousJoinedNames: false,
|
||||
trailingAmpersand: false,
|
||||
finalJoiner: false
|
||||
}});
|
||||
|
||||
const t = new NativeTransformer({name: 'test', type: 'native', defaults: {delimitersExtra: ['•']}}, {logger: loggerTest, cache: memorycache()});
|
||||
|
||||
await t.tryInitialize();
|
||||
|
||||
const play = generatePlay({artists: [str], track: 'My Test'});
|
||||
|
||||
const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play);
|
||||
expect(transformedPlay.data.artists).eql(primaries.concat(secondaries));
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
describe('Conditional Transforming', function () {
|
||||
|
||||
describe('On Hook', function () {
|
||||
it('Does not run hook if when conditions do not match', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
when: [
|
||||
{
|
||||
album: "Has This"
|
||||
}
|
||||
],
|
||||
artists: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({ artists: ['something', 'big'], album: 'It Has No Match' });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.artists!.length).is.eq(2)
|
||||
expect(transformed.data.artists![0]).is.eq('something')
|
||||
});
|
||||
|
||||
it('Does run hook if when conditions matches', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
when: [
|
||||
{
|
||||
album: "Has This"
|
||||
}
|
||||
],
|
||||
artists: ['something']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({ artists: ['something', 'big'], album: 'It Has This Match' });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.artists!.length).is.eq(1)
|
||||
expect(transformed.data.artists![0]).is.eq('big')
|
||||
});
|
||||
});
|
||||
|
||||
describe('On Search-And-Replace', function () {
|
||||
it('Does not run hook if when conditions do not match', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
artists: [
|
||||
{
|
||||
search: "something",
|
||||
replace: "",
|
||||
when: [
|
||||
{
|
||||
album: "Has This"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({ artists: ['something', 'big'], album: 'It Has No Match' });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.artists!.length).is.eq(2)
|
||||
expect(transformed.data.artists![0]).is.eq('something')
|
||||
});
|
||||
|
||||
it('Does run hook if when conditions matches', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
artists: [
|
||||
{
|
||||
search: "something",
|
||||
replace: "",
|
||||
when: [
|
||||
{
|
||||
album: "Has This"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
component.buildTransformRules();
|
||||
|
||||
const play = generatePlay({ artists: ['something', 'big'], album: 'It Has This Match' });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.artists!.length).is.eq(1)
|
||||
expect(transformed.data.artists![0]).is.eq('big')
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
describe('Multiple hook transforms', function () {
|
||||
|
||||
it('Accumulates transforms within a single stage', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: [
|
||||
{
|
||||
title: [
|
||||
{
|
||||
search: "something",
|
||||
replace: "another else"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
title: [
|
||||
{
|
||||
search: "another else",
|
||||
replace: "final thing"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
component.buildTransformRules();
|
||||
const play = generatePlay({ track: 'My cool something track' });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.track).equal('My cool final thing track');
|
||||
});
|
||||
|
||||
it('Accumulates transforms across multiple stages', async function () {
|
||||
component.config = {
|
||||
options: {
|
||||
playTransform: {
|
||||
preCompare: [
|
||||
{
|
||||
title: [
|
||||
{
|
||||
search: "something",
|
||||
replace: "bar"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
type: 'native'
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}});
|
||||
|
||||
component.buildTransformRules();
|
||||
const play = generatePlay({ track: 'My cool something track', artists: [str] });
|
||||
const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare);
|
||||
expect(transformed.data.track).equal('My cool bar track');
|
||||
expect(transformed.data.artists).eql(primaries.concat(secondaries));
|
||||
});
|
||||
|
||||
});
|
||||
})
|
||||
@@ -4,7 +4,20 @@ import asPromised from 'chai-as-promised';
|
||||
import { after, before, describe, it } from 'mocha';
|
||||
|
||||
import { asPlays, generateArtistsStr, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js";
|
||||
import { parseArtistCredits, parseContextAwareStringList, parseCredits } from "../../utils/StringUtils.js";
|
||||
import { parseArtistCredits, parseContextAwareStringList, parseCredits, parseTrackCredits, uniqueNormalizedStrArr } from "../../utils/StringUtils.js";
|
||||
import testData from '../utils/playTestData.json' with { type: "json" };
|
||||
import { intersect } from "../../utils.js";
|
||||
import { ExpectedResults } from "../utils/interfaces.js";
|
||||
|
||||
interface PlayTestFixture {
|
||||
caseHints: string[]
|
||||
data: {
|
||||
track: string
|
||||
artists: string[]
|
||||
album?: string
|
||||
}
|
||||
expected: ExpectedResults
|
||||
}
|
||||
|
||||
describe('#PlayParse Parsing Artists from String', function() {
|
||||
|
||||
@@ -137,4 +150,22 @@ Found => ${parsed.join(' || ')}`)
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
|
||||
describe('Play Track Strings',function () {
|
||||
|
||||
const testFixtures = testData as unknown as PlayTestFixture[];
|
||||
const joinerData = testFixtures.filter(x => intersect(['joiner','track'], x.caseHints).length === 2);
|
||||
|
||||
it('should parse joiners from track title', function() {
|
||||
for(const test of joinerData) {
|
||||
const res = parseTrackCredits(test.data.track);
|
||||
let artists: string[] = [...test.data.artists];
|
||||
if(res.secondary !== undefined) {
|
||||
artists = uniqueNormalizedStrArr([...artists, ...res.secondary]);
|
||||
}
|
||||
assert.equal(res.primaryComposite, test.expected.track);
|
||||
assert.sameDeepMembers(artists, test.expected.artists);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -453,7 +453,7 @@ describe('Detects duplicate and unique scrobbles using actively tracked scrobble
|
||||
playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds')
|
||||
});
|
||||
|
||||
const [matchedPlay, matchedData] = testScrobbler.findExistingSubmittedPlayObj(newScrobble);
|
||||
const [matchedPlay, matchedData] = await testScrobbler.findExistingSubmittedPlayObj(newScrobble);
|
||||
|
||||
assert.isUndefined(matchedPlay);
|
||||
assert.isEmpty(matchedData);
|
||||
@@ -465,7 +465,7 @@ describe('Detects duplicate and unique scrobbles using actively tracked scrobble
|
||||
});
|
||||
testScrobbler.addScrobbledTrack(newScrobble, newScrobble);
|
||||
|
||||
const [matchedPlay, matchedData] = testScrobbler.findExistingSubmittedPlayObj(newScrobble);
|
||||
const [matchedPlay, matchedData] = await testScrobbler.findExistingSubmittedPlayObj(newScrobble);
|
||||
|
||||
assert.isDefined(matchedPlay);
|
||||
assert.isNotEmpty(matchedData);
|
||||
@@ -480,7 +480,7 @@ describe('Detects duplicate and unique scrobbles using actively tracked scrobble
|
||||
const dupScrobble = clone(newScrobble);
|
||||
dupScrobble.data.playDate = newScrobble.data.playDate.add(2, 'seconds');
|
||||
|
||||
const [matchedPlay, matchedData] = testScrobbler.findExistingSubmittedPlayObj(dupScrobble);
|
||||
const [matchedPlay, matchedData] = await testScrobbler.findExistingSubmittedPlayObj(dupScrobble);
|
||||
|
||||
assert.isDefined(matchedPlay);
|
||||
assert.isNotEmpty(matchedData);
|
||||
|
||||
@@ -51,7 +51,7 @@ describe('Sources use transform plays correctly', function () {
|
||||
source = generateSource();
|
||||
});
|
||||
|
||||
it('Transforms play on preCompare', function() {
|
||||
it('Transforms play on preCompare', async function() {
|
||||
source.config.options = {
|
||||
playTransform: {
|
||||
preCompare: {
|
||||
@@ -68,7 +68,7 @@ describe('Sources use transform plays correctly', function () {
|
||||
const newScrobble = generatePlay({
|
||||
track: 'my cool track'
|
||||
});
|
||||
const discovered = source.discover([newScrobble])
|
||||
const discovered = await source.discover([newScrobble])
|
||||
expect(discovered.length).eq(1);
|
||||
expect(discovered[0].data.track).is.eq('my fun track');
|
||||
});
|
||||
@@ -90,7 +90,7 @@ describe('Sources use transform plays correctly', function () {
|
||||
const newScrobble = generatePlay({
|
||||
track: 'my cool track'
|
||||
});
|
||||
const discovered = source.discover([newScrobble])
|
||||
const discovered = await source.discover([newScrobble])
|
||||
expect(discovered.length).eq(1);
|
||||
expect(discovered[0].data.track).is.eq('my cool track');
|
||||
|
||||
@@ -101,7 +101,7 @@ describe('Sources use transform plays correctly', function () {
|
||||
expect(e.data[0].data.track).is.eq('my fun track');
|
||||
});
|
||||
|
||||
it('Transforms play existing comparison', function() {
|
||||
it('Transforms play existing comparison', async function() {
|
||||
source.config.options = {
|
||||
playTransform: {
|
||||
compare: {
|
||||
@@ -120,14 +120,14 @@ describe('Sources use transform plays correctly', function () {
|
||||
const newScrobble = generatePlay({
|
||||
track: 'my hugely cool and very different track title',
|
||||
});
|
||||
const discovered = source.discover([newScrobble])
|
||||
const discovered = await source.discover([newScrobble])
|
||||
expect(discovered.length).eq(1);
|
||||
expect(discovered[0].data.track).is.eq('my hugely cool and very different track title');
|
||||
|
||||
expect(source.discover([newScrobble]).length).is.eq(1);
|
||||
expect((await source.discover([newScrobble])).length).is.eq(1);
|
||||
});
|
||||
|
||||
it('Transforms play candidate comparison', function() {
|
||||
it('Transforms play candidate comparison', async function() {
|
||||
source.config.options = {
|
||||
playTransform: {
|
||||
compare: {
|
||||
@@ -146,11 +146,11 @@ describe('Sources use transform plays correctly', function () {
|
||||
const newScrobble = generatePlay({
|
||||
track: 'my hugely cool and very different track title',
|
||||
});
|
||||
const discovered = source.discover([newScrobble])
|
||||
const discovered = await source.discover([newScrobble])
|
||||
expect(discovered.length).eq(1);
|
||||
expect(discovered[0].data.track).is.eq('my hugely cool and very different track title');
|
||||
|
||||
expect(source.discover([newScrobble]).length).is.eq(1);
|
||||
expect((await source.discover([newScrobble])).length).is.eq(1);
|
||||
});
|
||||
})
|
||||
|
||||
@@ -200,7 +200,7 @@ describe('Player Cleanup', function () {
|
||||
const source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
|
||||
const initialDate = dayjs();
|
||||
const initialState = generatePlayerStateData({position: 0, playData: {duration: 50}, timestamp: initialDate, status: REPORTED_PLAYER_STATUSES.playing});
|
||||
expect(source.processRecentPlays([initialState]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([initialState])).length).to.be.eq(0);
|
||||
|
||||
let position = 0;
|
||||
let timeSince = 0;
|
||||
@@ -212,7 +212,7 @@ describe('Player Cleanup', function () {
|
||||
MockDate.set(initialDate.add(position, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
const advancedState = generatePlayerStateData({play: initialState.play, timestamp: dayjs(), position, status: REPORTED_PLAYER_STATUSES.playing});
|
||||
expect(source.processRecentPlays([advancedState]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([advancedState])).length).to.be.eq(0);
|
||||
}
|
||||
|
||||
// simulate polling another 20 seconds without any updates from the Source
|
||||
@@ -220,12 +220,12 @@ describe('Player Cleanup', function () {
|
||||
timeSince += 10;
|
||||
MockDate.set(initialDate.add(timeSince, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
expect(source.processRecentPlays([]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([])).length).to.be.eq(0);
|
||||
}
|
||||
|
||||
MockDate.set(initialDate.add(timeSince + 2, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
const discoveredPlays = source.processRecentPlays([]);
|
||||
const discoveredPlays = await source.processRecentPlays([]);
|
||||
// cleanup should discover stale play
|
||||
expect(discoveredPlays.length).to.be.eq(1);
|
||||
expect(discoveredPlays[0].data.listenedFor).closeTo(30, 2);
|
||||
@@ -244,7 +244,7 @@ describe('Player Cleanup', function () {
|
||||
const source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
|
||||
const initialDate = dayjs();
|
||||
const initialState = generatePlayerStateData({position: 0, playData: {duration: 50}, timestamp: initialDate, status: REPORTED_PLAYER_STATUSES.playing});
|
||||
expect(source.processRecentPlays([initialState]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([initialState])).length).to.be.eq(0);
|
||||
|
||||
let position = 0;
|
||||
let timeSince = 0;
|
||||
@@ -256,7 +256,7 @@ describe('Player Cleanup', function () {
|
||||
MockDate.set(initialDate.add(position, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
const advancedState = generatePlayerStateData({play: initialState.play, timestamp: dayjs(), position, status: REPORTED_PLAYER_STATUSES.playing});
|
||||
expect(source.processRecentPlays([advancedState]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([advancedState])).length).to.be.eq(0);
|
||||
}
|
||||
|
||||
// simulate polling another 20 seconds without any updates from the Source
|
||||
@@ -264,14 +264,14 @@ describe('Player Cleanup', function () {
|
||||
timeSince += 10;
|
||||
MockDate.set(initialDate.add(timeSince, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
expect(source.processRecentPlays([]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([])).length).to.be.eq(0);
|
||||
}
|
||||
|
||||
timeSince += 2;
|
||||
|
||||
MockDate.set(initialDate.add(timeSince, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
const discoveredPlays = source.processRecentPlays([]);
|
||||
const discoveredPlays = await source.processRecentPlays([]);
|
||||
// cleanup should discover stale play
|
||||
expect(discoveredPlays.length).to.be.eq(1);
|
||||
expect(discoveredPlays[0].data.listenedFor).closeTo(30, 2);
|
||||
@@ -285,7 +285,7 @@ describe('Player Cleanup', function () {
|
||||
MockDate.set(initialDate.add(timeSince, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
const advancedState = generatePlayerStateData({play: initialState.play, timestamp: dayjs(), position, status: REPORTED_PLAYER_STATUSES.playing});
|
||||
expect(source.processRecentPlays([advancedState]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([advancedState])).length).to.be.eq(0);
|
||||
}
|
||||
|
||||
timeSince += 10;
|
||||
@@ -294,7 +294,7 @@ describe('Player Cleanup', function () {
|
||||
// new Play
|
||||
const advancedState = generatePlayerStateData({timestamp: dayjs(), position: 0, status: REPORTED_PLAYER_STATUSES.playing});
|
||||
// should not return play because it has only been played for ~20 seconds, less than 50% of duration
|
||||
const plays = source.processRecentPlays([advancedState])
|
||||
const plays = await source.processRecentPlays([advancedState])
|
||||
expect(plays.length).to.be.eq(0);
|
||||
}
|
||||
|
||||
@@ -314,7 +314,7 @@ describe('Player Cleanup', function () {
|
||||
|
||||
// if player incorrectly counted stale time then 30s of actual play + 20s of stale time > scrobble threshold of 50% of 90s
|
||||
const initialState = generatePlayerStateData({position: 0, playData: {duration: 90}, timestamp: initialDate, status: REPORTED_PLAYER_STATUSES.playing});
|
||||
expect(source.processRecentPlays([initialState]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([initialState])).length).to.be.eq(0);
|
||||
|
||||
let position = 0;
|
||||
let timeSince = 0;
|
||||
@@ -326,7 +326,7 @@ describe('Player Cleanup', function () {
|
||||
MockDate.set(initialDate.add(position, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
const advancedState = generatePlayerStateData({play: initialState.play, timestamp: initialDate, position, status: REPORTED_PLAYER_STATUSES.playing});
|
||||
expect(source.processRecentPlays([advancedState]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([advancedState])).length).to.be.eq(0);
|
||||
}
|
||||
|
||||
// simulate polling another 20 seconds without any updates from the Source
|
||||
@@ -334,11 +334,11 @@ describe('Player Cleanup', function () {
|
||||
timeSince += 10;
|
||||
MockDate.set(initialDate.add(timeSince, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
expect(source.processRecentPlays([]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([])).length).to.be.eq(0);
|
||||
}
|
||||
|
||||
MockDate.set(initialDate.add(timeSince + 2, 'seconds').toDate());
|
||||
const discoveredPlays = source.processRecentPlays([]);
|
||||
const discoveredPlays = await source.processRecentPlays([]);
|
||||
// cleanup should not discover stale play
|
||||
expect(discoveredPlays.length).to.be.eq(0);
|
||||
|
||||
@@ -359,7 +359,7 @@ describe('Player Cleanup', function () {
|
||||
|
||||
// if player incorrectly counted stale time then 30s of actual play + 20s of stale time > scrobble threshold of 50% of 90s
|
||||
const initialState = generatePlayerStateData({position: 0, playData: {duration: 90}, timestamp: initialDate, status: REPORTED_PLAYER_STATUSES.playing});
|
||||
expect(source.processRecentPlays([initialState]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([initialState])).length).to.be.eq(0);
|
||||
|
||||
let position = 0;
|
||||
let timeSince = 0;
|
||||
@@ -371,7 +371,7 @@ describe('Player Cleanup', function () {
|
||||
MockDate.set(initialDate.add(position, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
const advancedState = generatePlayerStateData({play: initialState.play, timestamp: dayjs(), position, status: REPORTED_PLAYER_STATUSES.playing});
|
||||
expect(source.processRecentPlays([advancedState]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([advancedState])).length).to.be.eq(0);
|
||||
}
|
||||
|
||||
// simulate polling another 20 seconds without any updates from the Source
|
||||
@@ -379,14 +379,14 @@ describe('Player Cleanup', function () {
|
||||
timeSince += 10;
|
||||
MockDate.set(initialDate.add(timeSince, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
expect(source.processRecentPlays([]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([])).length).to.be.eq(0);
|
||||
}
|
||||
|
||||
timeSince += 2;
|
||||
|
||||
MockDate.set(initialDate.add(timeSince, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
const discoveredPlays = source.processRecentPlays([]);
|
||||
const discoveredPlays = await source.processRecentPlays([]);
|
||||
// cleanup should not discover stale play
|
||||
expect(discoveredPlays.length).to.be.eq(0);
|
||||
|
||||
@@ -401,7 +401,7 @@ describe('Player Cleanup', function () {
|
||||
MockDate.set(initialDate.add(position, 'seconds').toDate());
|
||||
await sleep(1);
|
||||
const advancedState = generatePlayerStateData({play: initialState.play, timestamp: dayjs(), position, status: REPORTED_PLAYER_STATUSES.playing});
|
||||
expect(source.processRecentPlays([advancedState]).length).to.be.eq(0);
|
||||
expect((await source.processRecentPlays([advancedState])).length).to.be.eq(0);
|
||||
}
|
||||
|
||||
timeSince += 10;
|
||||
@@ -410,7 +410,7 @@ describe('Player Cleanup', function () {
|
||||
// new Play
|
||||
const advancedState = generatePlayerStateData({timestamp: dayjs(), position: 0, status: REPORTED_PLAYER_STATUSES.playing});
|
||||
// should return discovered play with ~90 seconds of duration
|
||||
const plays = source.processRecentPlays([advancedState])
|
||||
const plays = await source.processRecentPlays([advancedState])
|
||||
expect(plays.length).to.be.eq(1);
|
||||
expect(plays[0].data.duration).to.be.closeTo(90, 2);
|
||||
|
||||
@@ -436,7 +436,7 @@ describe('Deezer Internal Source', function() {
|
||||
|
||||
describe('When fuzzyDiscoveryIgnore is not defined or false', function () {
|
||||
|
||||
it('discovers fuzzy play', function() {
|
||||
it('discovers fuzzy play', async function() {
|
||||
const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80});
|
||||
const targetPlay = normalizedPlays[normalizedPlays.length - 2]
|
||||
const fuzzyPlay = clone(targetPlay);
|
||||
@@ -445,7 +445,7 @@ describe('Deezer Internal Source', function() {
|
||||
const source = generateDeezerSource();
|
||||
source.discover([...normalizedPlays, interimPlay]);
|
||||
|
||||
const discovered = source.discover([fuzzyPlay]);
|
||||
const discovered = await source.discover([fuzzyPlay]);
|
||||
|
||||
expect(discovered.length).to.eq(1);
|
||||
});
|
||||
@@ -453,41 +453,41 @@ describe('Deezer Internal Source', function() {
|
||||
|
||||
describe('When fuzzyDiscoveryIgnore is true', function () {
|
||||
|
||||
it('does not discover fuzzy play with interim plays', function() {
|
||||
it('does not discover fuzzy play with interim plays', async function() {
|
||||
const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80});
|
||||
const targetPlay = normalizedPlays[normalizedPlays.length - 2]
|
||||
const fuzzyPlay = clone(targetPlay);
|
||||
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
|
||||
|
||||
const source = generateDeezerSource({fuzzyDiscoveryIgnore: true});
|
||||
source.discover([...normalizedPlays, interimPlay]);
|
||||
await source.discover([...normalizedPlays, interimPlay]);
|
||||
|
||||
const discovered = source.discover([fuzzyPlay]);
|
||||
const discovered = await source.discover([fuzzyPlay]);
|
||||
|
||||
expect(discovered.length).to.eq(0);
|
||||
});
|
||||
|
||||
it('discovers fuzzy play when it is the last play ', function() {
|
||||
it('discovers fuzzy play when it is the last play ', async function() {
|
||||
const targetPlay = normalizedPlays[normalizedPlays.length - 1]
|
||||
const fuzzyPlay = clone(targetPlay);
|
||||
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
|
||||
|
||||
const source = generateDeezerSource({fuzzyDiscoveryIgnore: true});
|
||||
source.discover(normalizedPlays);
|
||||
await source.discover(normalizedPlays);
|
||||
|
||||
const discovered = source.discover([fuzzyPlay]);
|
||||
const discovered = await source.discover([fuzzyPlay]);
|
||||
|
||||
expect(discovered.length).to.eq(1);
|
||||
});
|
||||
|
||||
it('discovers fuzzy play when it is played consecutively', function() {
|
||||
it('discovers fuzzy play when it is played consecutively', async function() {
|
||||
const targetPlay = normalizedPlays[normalizedPlays.length - 1]
|
||||
const fuzzyPlay = clone(targetPlay);
|
||||
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
|
||||
const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate});
|
||||
|
||||
const source = generateDeezerSource({fuzzyDiscoveryIgnore: true});
|
||||
const discovered = source.discover(morePlays);
|
||||
const discovered = await source.discover(morePlays);
|
||||
|
||||
expect(discovered.length).to.eq(morePlays.length);
|
||||
});
|
||||
@@ -495,69 +495,69 @@ describe('Deezer Internal Source', function() {
|
||||
|
||||
describe('When fuzzyDiscoveryIgnore is aggressive', function () {
|
||||
|
||||
it('does not discover fuzzy play with interim plays', function() {
|
||||
it('does not discover fuzzy play with interim plays', async function() {
|
||||
const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80});
|
||||
const targetPlay = normalizedPlays[normalizedPlays.length - 2]
|
||||
const fuzzyPlay = clone(targetPlay);
|
||||
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
|
||||
|
||||
const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
|
||||
source.discover([...normalizedPlays, interimPlay]);
|
||||
await source.discover([...normalizedPlays, interimPlay]);
|
||||
|
||||
const discovered = source.discover([fuzzyPlay]);
|
||||
const discovered = await source.discover([fuzzyPlay]);
|
||||
|
||||
expect(discovered.length).to.eq(0);
|
||||
});
|
||||
|
||||
it('does not discover play found during duration of previous', function() {
|
||||
it('does not discover play found during duration of previous', async function() {
|
||||
const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80});
|
||||
const targetPlay = normalizedPlays[normalizedPlays.length - 2]
|
||||
const duringPlay = clone(targetPlay);
|
||||
duringPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration * 0.5, 's');
|
||||
|
||||
const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
|
||||
source.discover([...normalizedPlays, interimPlay]);
|
||||
await source.discover([...normalizedPlays, interimPlay]);
|
||||
|
||||
const discovered = source.discover([duringPlay]);
|
||||
const discovered = await source.discover([duringPlay]);
|
||||
|
||||
expect(discovered.length).to.eq(0);
|
||||
});
|
||||
|
||||
it('does not discover fuzzy play with delay of up to 40 seconds', function() {
|
||||
it('does not discover fuzzy play with delay of up to 40 seconds', async function() {
|
||||
const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80});
|
||||
const targetPlay = normalizedPlays[normalizedPlays.length - 2]
|
||||
const fuzzyPlay = clone(targetPlay);
|
||||
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration + 39, 's');
|
||||
|
||||
const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
|
||||
source.discover([...normalizedPlays, interimPlay]);
|
||||
await source.discover([...normalizedPlays, interimPlay]);
|
||||
|
||||
const discovered = source.discover([fuzzyPlay]);
|
||||
const discovered = await source.discover([fuzzyPlay]);
|
||||
|
||||
expect(discovered.length).to.eq(0);
|
||||
});
|
||||
|
||||
it('it does not discover fuzzy play when it is the last play ', function() {
|
||||
it('it does not discover fuzzy play when it is the last play ', async function() {
|
||||
const targetPlay = normalizedPlays[normalizedPlays.length - 1]
|
||||
const fuzzyPlay = clone(targetPlay);
|
||||
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
|
||||
|
||||
const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
|
||||
source.discover(normalizedPlays);
|
||||
await source.discover(normalizedPlays);
|
||||
|
||||
const discovered = source.discover([fuzzyPlay]);
|
||||
const discovered = await source.discover([fuzzyPlay]);
|
||||
|
||||
expect(discovered.length).to.eq(0);
|
||||
});
|
||||
|
||||
it('does not discover fuzzy play when it is played consecutively', function() {
|
||||
it('does not discover fuzzy play when it is played consecutively', async function() {
|
||||
const targetPlay = normalizedPlays[normalizedPlays.length - 1]
|
||||
const fuzzyPlay = clone(targetPlay);
|
||||
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
|
||||
const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate});
|
||||
|
||||
const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
|
||||
const discovered = source.discover(morePlays);
|
||||
const discovered = await source.discover(morePlays);
|
||||
|
||||
expect(discovered.length).to.eq(morePlays.length - 1);
|
||||
});
|
||||
|
||||
@@ -1,26 +1,11 @@
|
||||
import { assert, expect } from 'chai';
|
||||
import { describe, it } from 'mocha';
|
||||
import { intersect } from "../../utils.js";
|
||||
import {
|
||||
compareNormalizedStrings,
|
||||
normalizeStr,
|
||||
parseTrackCredits,
|
||||
uniqueNormalizedStrArr
|
||||
} from "../../utils/StringUtils.js";
|
||||
import { ExpectedResults } from "./interfaces.js";
|
||||
import testData from './playTestData.json' with { type: "json" };
|
||||
import { splitByFirstFound } from '../../../core/StringUtils.js';
|
||||
|
||||
interface PlayTestFixture {
|
||||
caseHints: string[]
|
||||
data: {
|
||||
track: string
|
||||
artists: string[]
|
||||
album?: string
|
||||
}
|
||||
expected: ExpectedResults
|
||||
}
|
||||
|
||||
describe('String Comparisons', function () {
|
||||
|
||||
it('should ignore symbols', async function () {
|
||||
@@ -127,26 +112,6 @@ describe('String Comparisons', function () {
|
||||
});
|
||||
});
|
||||
|
||||
describe('Play Strings',function () {
|
||||
|
||||
const testFixtures = testData as unknown as PlayTestFixture[];
|
||||
const joinerData = testFixtures.filter(x => intersect(['joiner','track'], x.caseHints).length === 2);
|
||||
|
||||
it('should parse joiners from track title', function() {
|
||||
for(const test of joinerData) {
|
||||
const res = parseTrackCredits(test.data.track);
|
||||
let artists: string[] = [...test.data.artists];
|
||||
if(res.secondary !== undefined) {
|
||||
artists = uniqueNormalizedStrArr([...artists, ...res.secondary]);
|
||||
}
|
||||
assert.equal(res.primaryComposite, test.expected.track);
|
||||
assert.sameDeepMembers(artists, test.expected.artists);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
|
||||
describe('String Splitting', function() {
|
||||
|
||||
it('should not split string with no delimiter', function() {
|
||||
|
||||
@@ -174,7 +174,7 @@ describe('Handles interim tracks', function () {
|
||||
expect(source.parseRecentAgainstResponse(plays).plays).length(20);
|
||||
|
||||
source.polling = true;
|
||||
source.discover(plays);
|
||||
await source.discover(plays);
|
||||
|
||||
// first true poll emulating no new tracks played (should not add new tracks from base truth)
|
||||
expect(source.parseRecentAgainstResponse(plays).plays).length(0);
|
||||
@@ -201,7 +201,7 @@ describe('Handles interim tracks', function () {
|
||||
expect(source.parseRecentAgainstResponse(plays).plays).length(20);
|
||||
|
||||
source.polling = true;
|
||||
source.discover(plays);
|
||||
await source.discover(plays);
|
||||
|
||||
// first true poll emulating no new tracks played (should not add new tracks from base truth)
|
||||
expect(source.parseRecentAgainstResponse(plays).plays).length(0);
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
/** https://stackoverflow.com/a/63795192/1469797 */
|
||||
export async function findAsyncSequential<T>(
|
||||
array: T[],
|
||||
predicate: (t: T) => Promise<boolean>,
|
||||
): Promise<T | undefined> {
|
||||
const i = await findIndexAsyncSequential(array, predicate);
|
||||
if(i === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
return array[i];
|
||||
}
|
||||
|
||||
export async function findIndexAsyncSequential<T>(
|
||||
array: T[],
|
||||
predicate: (t: T) => Promise<boolean>,
|
||||
): Promise<number | undefined> {
|
||||
let index = 0;
|
||||
for (const t of array) {
|
||||
if (await predicate(t)) {
|
||||
return index;
|
||||
}
|
||||
index++;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** https://stackoverflow.com/a/55601090/1469797 */
|
||||
export async function findAsync<T>(
|
||||
array: T[],
|
||||
predicate: (t: T) => Promise<boolean>): Promise<T | undefined> {
|
||||
const i = await findIndexAsync(array, predicate);
|
||||
if(i === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
return array[i];
|
||||
}
|
||||
|
||||
export async function findIndexAsync<T>(
|
||||
array: T[],
|
||||
predicate: (t: T) => Promise<boolean>): Promise<number | undefined> {
|
||||
const promises = array.map(predicate);
|
||||
const results = await Promise.all(promises);
|
||||
const index = results.findIndex(result => result);
|
||||
return index;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export const asArray = <T>(data: T | T[]): T[] => {
|
||||
if(Array.isArray(data)) {
|
||||
return data;
|
||||
}
|
||||
return [data];
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT, TemporalAccuracy }
|
||||
import { buildTrackString } from "../../core/StringUtils.js";
|
||||
import { playObjDataMatch } from "../utils.js";
|
||||
import { comparePlayTemporally, hasAcceptableTemporalAccuracy, TemporalPlayComparisonOptions } from "./TimeUtils.js";
|
||||
import { RestType } from "ts-json-schema-generator";
|
||||
|
||||
|
||||
export const metaInvariantTransform = (play: PlayObject): PlayObject => {
|
||||
@@ -34,6 +35,22 @@ export const playDateInvariantTransform = (play: PlayObject): PlayObject => {
|
||||
}
|
||||
}
|
||||
|
||||
export const playContentInvariantTransform = (play: PlayObject): PlayObject => {
|
||||
const {
|
||||
data: {
|
||||
playDate,
|
||||
playDateCompleted,
|
||||
...rest
|
||||
}
|
||||
} = play;
|
||||
return {
|
||||
data: {
|
||||
...rest
|
||||
},
|
||||
meta: {}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export type PlayTransformer = (play: PlayObject) => PlayObject;
|
||||
export type ListTransformers = PlayTransformer[];
|
||||
|
||||
@@ -2,13 +2,20 @@ import { Logger, loggerTest } from "@foxxmd/logging";
|
||||
import { searchAndReplace as searchAndReplaceFunc, testMaybeRegex as testMaybeRegexFunc } from "@foxxmd/regex-buddy-core";
|
||||
import { ObjectPlayData, PlayObject } from "../../core/Atomic.js";
|
||||
import { buildTrackString } from "../../core/StringUtils.js";
|
||||
|
||||
import {
|
||||
ConditionalSearchAndReplaceRegExp,
|
||||
PlayTransformParts, PlayTransformPartsArray, PlayTransformPartsConfig, PlayTransformRules,
|
||||
ConditionalSearchAndReplaceTerm,
|
||||
ExternalMetadataTerm,
|
||||
PlayTransformParts,
|
||||
PlayTransformStage,
|
||||
SearchAndReplaceTerm,
|
||||
STAGE_TYPES,
|
||||
StageType,
|
||||
StageTypedConfig,
|
||||
WhenConditionsConfig,
|
||||
WhenParts
|
||||
} from "../common/infrastructure/Atomic.js";
|
||||
} from "../common/infrastructure/Transform.js";
|
||||
|
||||
export const isWhenCondition = (val: unknown): val is WhenParts<string> => {
|
||||
if (val !== null && typeof val === 'object') {
|
||||
@@ -49,57 +56,84 @@ export const isConditionalSearchAndReplace = (val: unknown): val is ConditionalS
|
||||
&& ('replace' in val && typeof val.replace === 'string')
|
||||
&& (!('when' in val) || isWhenConditionConfig(val.when));
|
||||
}
|
||||
export const configPartsToStrongParts = (val: PlayTransformPartsConfig<SearchAndReplaceTerm> | undefined): PlayTransformPartsArray<ConditionalSearchAndReplaceRegExp> => {
|
||||
if (val === undefined) {
|
||||
return []
|
||||
|
||||
export const isSearchAndReplaceTerm = (val: unknown | string | ConditionalSearchAndReplaceTerm): val is SearchAndReplaceTerm => {
|
||||
const tf = typeof val;
|
||||
if(tf === 'string') {
|
||||
return true;
|
||||
}
|
||||
const arr = Array.isArray(val) ? val : [val];
|
||||
if(!(tf == 'object')) {
|
||||
throw new Error(`Must be a string or an object, but found ${tf}`);
|
||||
}
|
||||
if(tf === null) {
|
||||
throw new Error('Cannot be null');
|
||||
}
|
||||
return isConditionalSearchAndReplace(val);
|
||||
}
|
||||
|
||||
return arr.map((x) => {
|
||||
const {
|
||||
title: titleConfig,
|
||||
artists: artistConfig,
|
||||
album: albumConfig,
|
||||
when: whenConfig
|
||||
} = x;
|
||||
let title,
|
||||
artists,
|
||||
album,
|
||||
when;
|
||||
export const isExternalMetadataTerm = (val: unknown): val is ExternalMetadataTerm => {
|
||||
if(val === undefined) {
|
||||
return true;
|
||||
}
|
||||
const tf = typeof val;
|
||||
if(tf === 'boolean') {
|
||||
return true;
|
||||
}
|
||||
if(tf === null) {
|
||||
throw new Error(`Value is null but must be one of: true, undefined, or object with 'when'`);
|
||||
}
|
||||
if(tf === 'object') {
|
||||
if(isWhenConditionConfig(val)) {
|
||||
return true;
|
||||
}
|
||||
throw new Error(`Value is not a proper 'when' object`);
|
||||
}
|
||||
throw new Error(`Value is type of ${tf} but must be one of: boolean, undefined, or object with 'when'`);
|
||||
}
|
||||
|
||||
if (titleConfig !== undefined) {
|
||||
if (!Array.isArray(titleConfig)) {
|
||||
throw new Error('title must be an array');
|
||||
}
|
||||
title = titleConfig.map(configValToSearchReplace);
|
||||
}
|
||||
if (artistConfig !== undefined) {
|
||||
if (!Array.isArray(artistConfig)) {
|
||||
throw new Error('arist must be an array');
|
||||
}
|
||||
artists = artistConfig.map(configValToSearchReplace);
|
||||
}
|
||||
if (albumConfig !== undefined) {
|
||||
if (!Array.isArray(albumConfig)) {
|
||||
throw new Error('album must be an array');
|
||||
}
|
||||
album = albumConfig.map(configValToSearchReplace);
|
||||
}
|
||||
if (whenConfig !== undefined) {
|
||||
if (!isWhenConditionConfig(whenConfig)) {
|
||||
throw new Error('when must be an array of artist/title/album objects and each object\'s property must be a string');
|
||||
}
|
||||
when = whenConfig;
|
||||
}
|
||||
export const isStageTyped = (val: unknown): val is StageTypedConfig => {
|
||||
if(typeof val !== 'object' || val === null) {
|
||||
return false;
|
||||
}
|
||||
return 'type' in val;
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
artists,
|
||||
album,
|
||||
when
|
||||
}
|
||||
});
|
||||
export const isPlayTransformStage = (val: object | Partial<PlayTransformStage<SearchAndReplaceTerm[]>>): val is PlayTransformStage<SearchAndReplaceTerm[]> => {
|
||||
if (!('type' in val)) {
|
||||
throw new Error(`Stage is missing 'type'. Must be one of: ${STAGE_TYPES.join(', ')}`);
|
||||
}
|
||||
if (!STAGE_TYPES.includes(val.type)) {
|
||||
throw new Error(`Stage has invalid 'type'. Must be one of: ${STAGE_TYPES.join(', ')}`);
|
||||
}
|
||||
|
||||
for (const k of ['artist', 'title', 'album']) {
|
||||
if (!(k in val)) {
|
||||
continue;
|
||||
}
|
||||
if (val.type === 'user') {
|
||||
if (!Array.isArray(val[k])) {
|
||||
throw new Error(`${k} must be an array`);
|
||||
}
|
||||
try {
|
||||
isSearchAndReplaceTerm(val[k]);
|
||||
} catch (e) {
|
||||
throw new Error(`Property '${k}' was not a valid type`, { cause: e });
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
isExternalMetadataTerm(val[k]);
|
||||
|
||||
} catch (e) {
|
||||
throw new Error(`Property '${k}' was not a valid type`, { cause: e });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
export const isUserStage = <T>(val: StageTypedConfig): val is StageTypedConfig => {
|
||||
return val.type === 'user';
|
||||
}
|
||||
|
||||
export const testWhen = (parts: WhenParts<string>, play: PlayObject, options?: SuppliedRegex): boolean => {
|
||||
@@ -137,173 +171,4 @@ export interface SuppliedRegex {
|
||||
export interface TransformPlayPartsOptions {
|
||||
logger?: () => Logger,
|
||||
regex?: SuppliedRegex
|
||||
}
|
||||
|
||||
export const transformPlayUsingParts = (play: PlayObject, parts: PlayTransformParts<ConditionalSearchAndReplaceRegExp>, options?: TransformPlayPartsOptions): PlayObject => {
|
||||
const {
|
||||
data: {
|
||||
track,
|
||||
artists,
|
||||
albumArtists,
|
||||
album
|
||||
} = {}
|
||||
} = play;
|
||||
|
||||
const {
|
||||
logger = () => loggerTest,
|
||||
regex: {
|
||||
searchAndReplace = searchAndReplaceFunc,
|
||||
testMaybeRegex = testMaybeRegexFunc,
|
||||
} = {},
|
||||
} = options || {};
|
||||
|
||||
const transformedPlayData: Partial<ObjectPlayData> = {};
|
||||
|
||||
let isTransformed = false;
|
||||
|
||||
if(parts.when !== undefined) {
|
||||
if(!testWhenConditions(parts.when, play, {testMaybeRegex})) {
|
||||
return play;
|
||||
}
|
||||
}
|
||||
|
||||
const searchAndReplaceMapper = (x: ConditionalSearchAndReplaceRegExp): ConditionalSearchAndReplaceRegExp => ({...x, test: (x.when !== undefined ? () => testWhenConditions(x.when, play, {testMaybeRegex}) : undefined)})
|
||||
|
||||
if (parts.title !== undefined && track !== undefined) {
|
||||
try {
|
||||
const t = searchAndReplace(track, parts.title.map(x => ({...x, test: (x.when !== undefined ? () => testWhenConditions(x.when, play, {testMaybeRegex}) : undefined)})));
|
||||
if (t !== track) {
|
||||
transformedPlayData.track = t.trim() === '' ? undefined : t;
|
||||
isTransformed = true;
|
||||
}
|
||||
} catch (e) {
|
||||
logger().warn(new Error(`Failed to transform title: ${track}`, {cause: e}));
|
||||
}
|
||||
}
|
||||
|
||||
if (parts.artists !== undefined && artists !== undefined && artists.length > 0) {
|
||||
const transformedArtists: string[] = [];
|
||||
let anyArtistTransformed = false;
|
||||
for (const artist of artists) {
|
||||
try {
|
||||
const t = searchAndReplace(artist, parts.artists.map(searchAndReplaceMapper));
|
||||
if (t !== artist) {
|
||||
anyArtistTransformed = true;
|
||||
isTransformed = true;
|
||||
}
|
||||
if (t.trim() !== '') {
|
||||
transformedArtists.push(t);
|
||||
}
|
||||
} catch (e) {
|
||||
logger().warn(new Error(`Failed to transform artist: ${artist}`, {cause: e}));
|
||||
transformedArtists.push(artist);
|
||||
}
|
||||
}
|
||||
if (anyArtistTransformed) {
|
||||
transformedPlayData.artists = transformedArtists;
|
||||
}
|
||||
}
|
||||
|
||||
if (parts.artists !== undefined && albumArtists !== undefined && albumArtists.length > 0) {
|
||||
const transformedArtists: string[] = [];
|
||||
let anyArtistTransformed = false;
|
||||
for (const artist of albumArtists) {
|
||||
try {
|
||||
const t = searchAndReplace(artist, parts.artists.map(searchAndReplaceMapper));
|
||||
if (t !== artist) {
|
||||
anyArtistTransformed = true;
|
||||
isTransformed = true;
|
||||
}
|
||||
if (t.trim() !== '') {
|
||||
transformedArtists.push(t);
|
||||
}
|
||||
} catch (e) {
|
||||
logger().warn(new Error(`Failed to transform albumArtist: ${artist}`, {cause: e}));
|
||||
transformedArtists.push(artist);
|
||||
}
|
||||
}
|
||||
if (anyArtistTransformed) {
|
||||
transformedPlayData.albumArtists = transformedArtists;
|
||||
}
|
||||
}
|
||||
|
||||
if (parts.album !== undefined && album !== undefined) {
|
||||
try {
|
||||
const t = searchAndReplace(album, parts.album.map(searchAndReplaceMapper));
|
||||
if (t !== album) {
|
||||
isTransformed = true;
|
||||
transformedPlayData.album = t.trim() === '' ? undefined : t;
|
||||
}
|
||||
} catch (e) {
|
||||
logger().warn(new Error(`Failed to transform album: ${album}`, {cause: e}));
|
||||
}
|
||||
}
|
||||
|
||||
if (isTransformed) {
|
||||
|
||||
const transformedPlay = {
|
||||
...play,
|
||||
data: {
|
||||
...play.data,
|
||||
...transformedPlayData
|
||||
}
|
||||
}
|
||||
|
||||
return transformedPlay;
|
||||
}
|
||||
|
||||
return play;
|
||||
}
|
||||
|
||||
export const countRegexes = (rules: PlayTransformRules): number => {
|
||||
let rulesCount = 0;
|
||||
if(rules.preCompare !== undefined) {
|
||||
for(const hookItem of rules.preCompare) {
|
||||
rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when);
|
||||
}
|
||||
|
||||
}
|
||||
if(rules.postCompare !== undefined) {
|
||||
for(const hookItem of rules.postCompare) {
|
||||
rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when);
|
||||
}
|
||||
}
|
||||
if(rules.compare !== undefined) {
|
||||
if(rules.compare.existing !== undefined) {
|
||||
for(const hookItem of rules.compare.existing) {
|
||||
rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when);
|
||||
}
|
||||
}
|
||||
if(rules.compare.candidate !== undefined) {
|
||||
for(const hookItem of rules.compare.candidate) {
|
||||
rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when);
|
||||
}
|
||||
}
|
||||
}
|
||||
return rulesCount;
|
||||
}
|
||||
|
||||
const countWhens = (when: WhenConditionsConfig | undefined): number => {
|
||||
if(when === undefined) {
|
||||
return 0;
|
||||
}
|
||||
return when.reduce((acc, curr) => {
|
||||
return acc + Object.keys(curr).length;
|
||||
},0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts all rules within title/artist/album + whens WITHIN those rules
|
||||
* */
|
||||
const countRulesInParts = (parts: PlayTransformParts<ConditionalSearchAndReplaceRegExp>): number => {
|
||||
return Object.entries(parts).reduce((acc: number, entries: [string, ConditionalSearchAndReplaceRegExp[]]) => {
|
||||
let curr = acc;
|
||||
for(const rule of (entries[1] ?? [])) {
|
||||
curr++;
|
||||
if(typeof rule !== 'string' && rule.when !== undefined) {
|
||||
curr += countWhens(rule.when);
|
||||
}
|
||||
}
|
||||
return curr;
|
||||
}, 0)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import { strategies, stringSameness, StringSamenessResult } from "@foxxmd/string-sameness";
|
||||
import { hasher } from 'node-object-hash';
|
||||
import { PlayObject } from "../../core/Atomic.js";
|
||||
import { asPlayerStateData, DELIMITERS, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js";
|
||||
import { asPlayerStateData, DELIMITERS, DELIMITERS_NO_AMP, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js";
|
||||
import { genGroupIdStr, getPlatformIdFromData, intersect, parseRegexSingleOrFail } from "../utils.js";
|
||||
import { buildTrackString } from "../../core/StringUtils.js";
|
||||
|
||||
@@ -183,7 +183,7 @@ export const parseStringList = (str: string, delimiters: string[] = DELIMITERS):
|
||||
return explodedStrings.flat(1);
|
||||
}, [str]).map(x => x.trim());
|
||||
}
|
||||
export const parseContextAwareStringList = (str: string, delimiters: string[] = [',', '/', '\\'], opts: {ignoreGlobalAmpersand?: boolean} = {}): string[] => {
|
||||
export const parseContextAwareStringList = (str: string, delimiters: string[] = DELIMITERS_NO_AMP, opts: {ignoreGlobalAmpersand?: boolean} = {}): string[] => {
|
||||
if (delimiters.length === 0) {
|
||||
return [str];
|
||||
}
|
||||
|
||||
+21
-5
@@ -136,6 +136,11 @@ export interface SpotifyMeta {
|
||||
track?: string
|
||||
}
|
||||
|
||||
export interface TrackMeta {
|
||||
brainz?: BrainzMeta
|
||||
spotify?: SpotifyMeta
|
||||
}
|
||||
|
||||
export interface TrackData {
|
||||
artists?: string[]
|
||||
albumArtists?: string[]
|
||||
@@ -146,10 +151,7 @@ export interface TrackData {
|
||||
* */
|
||||
duration?: number
|
||||
|
||||
meta?: {
|
||||
brainz?: BrainzMeta
|
||||
spotify?: SpotifyMeta
|
||||
}
|
||||
meta?: TrackMeta
|
||||
}
|
||||
|
||||
export interface PlayData extends TrackData {
|
||||
@@ -387,4 +389,18 @@ export type FinalJoiners = '&';
|
||||
export const JOINERS_FINAL: FinalJoiners[] = ['&'];
|
||||
|
||||
export type Feat = 'ft' | 'feat' | 'vs' | 'ft.' | 'feat.' | 'vs.' | 'featuring'
|
||||
export const FEAT: Feat[] = ['ft','feat','vs','ft.','feat.','vs.','featuring'];
|
||||
export const FEAT: Feat[] = ['ft','feat','vs','ft.','feat.','vs.','featuring'];
|
||||
export interface TransformerCommonConfig<T = Record<string, any>, Y = Record<string, any>> {
|
||||
defaults?: T;
|
||||
data?: Y
|
||||
type: string;
|
||||
name?: string;
|
||||
options?: {
|
||||
failOnFetch?: boolean;
|
||||
throwOnFailure?: boolean | ('artists' | 'title' | 'albumArtists' | 'album')[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface TransformerCommon<T = Record<string, any>> extends TransformerCommonConfig<T> {
|
||||
name: string
|
||||
}
|
||||
Reference in New Issue
Block a user