Merge pull request #347 from FoxxMD/ytm400

Youtube Music updates

docsite/docs/configuration/configuration.mdx

@@ -2358,17 +2358,12 @@ Due to monitoring being unofficial, listening history from YTM can be inconsiste
 
 :::
 
-#### Authentication
+#### Authentication {#ytm-auth}
 
-Only one of these methods needs to be used. **Cookies** are easier but **OAuth Client**  may be more stable.
+Only one of these methods needs to be used.
 
 <Tabs groupId="ytmAuth" queryString>
     <TabItem value="cookie" label="Cookies">
-        :::info
-
-        If cookies stop working for you or are being invalidated often try switching to **OAuth Client** authentication.
-
-        :::
 
         Use instructions from 
         
@@ -2400,10 +2395,14 @@ Only one of these methods needs to be used. **Cookies** are easier but **OAuth C
 
     </TabItem>
     <TabItem value="oauth" label="OAuth Client">
-        :::note
+        :::warning
+
+        As of Sept 25' OAuth clients no longer seem to have permission to access the YTM service, as reported by the upstream YTM library developer and [several other users.](https://github.com/FoxxMD/multi-scrobbler/issues/345#issuecomment-3258518769)
+
+        If you have existing, working OAuth credentials keep using them but there is no gaurantee they will continue to work.
+
+        If you are setting up the YTM Source for the first time you should first try to use [Cookies](./?ytmAuth=cookie#ytm-auth).
 
-        This is likely to be the most stable method and least likely to be blocked or have authentication invalidated after an extended period. It requires more setup but is worth the effort.
-        
         :::
 
         [Based on the instructions from here...](https://github.com/LuanRT/YouTube.js/issues/803#issuecomment-2479689924)

package-lock.json

@@ -19,7 +19,7 @@
         "@fortawesome/react-fontawesome": "^0.2.0",
         "@foxxmd/chromecast-client": "^1.0.4",
         "@foxxmd/get-version": "^0.0.3",
-        "@foxxmd/logging": "^0.2.2",
+        "@foxxmd/logging": "^0.2.3",
         "@foxxmd/regex-buddy-core": "^0.1.2",
         "@foxxmd/string-sameness": "^0.4.0",
         "@gr2m/net-interceptor": "^1.0.0",
@@ -86,7 +86,7 @@
         "vite-express": "^0.16.0",
         "vlc-client": "^1.1.1",
         "xml2js": "0.6.1",
-        "youtubei.js": "^13.1.0"
+        "youtubei.js": "^15.0.1"
       },
       "devDependencies": {
         "@dbus-types/notifications": "^0.0.5",
@@ -1049,15 +1049,6 @@
         "npm": ">=9.0.0"
       }
     },
-    "node_modules/@fastify/busboy": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz",
-      "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=14"
-      }
-    },
     "node_modules/@fortawesome/fontawesome-common-types": {
       "version": "6.6.0",
       "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.6.0.tgz",
@@ -1126,9 +1117,10 @@
       }
     },
     "node_modules/@foxxmd/logging": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/@foxxmd/logging/-/logging-0.2.2.tgz",
-      "integrity": "sha512-aEbLh6kDqr5UJ1yBl/h/31bE2XyTfQob6EkgYBVaJb/T+qaVeB/T7T3nfKLZQhS31Y53/LxQKrwHU30ZkkBv4Q==",
+      "version": "0.2.3",
+      "resolved": "https://registry.npmjs.org/@foxxmd/logging/-/logging-0.2.3.tgz",
+      "integrity": "sha512-KMqWy42niMLMyz8YmQ09XVay5szhU2Y8lWzM3Fbm+zyfFRTiaYCSal5tQSeWmVdjps6LqtQNGDH4KG5a3Wgnrw==",
+      "license": "MIT",
       "dependencies": {
         "pino": "^9.2.0",
         "pino-abstract-transport": "^1.2.0",
@@ -6914,12 +6906,13 @@
       }
     },
     "node_modules/jintr": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/jintr/-/jintr-3.2.1.tgz",
-      "integrity": "sha512-yjKUBuwTTg4nc4izMysxuIk0BKh45hnbc1KnXE6LxagIGZn5od+I2elpuRY9IIm3EiKiUZxhxV89a0iX+xoEZg==",
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/jintr/-/jintr-3.3.1.tgz",
+      "integrity": "sha512-nnOzyhf0SLpbWuZ270Omwbj5LcXUkTcZkVnK8/veJXtSZOiATM5gMZMdmzN75FmTyj+NVgrGaPdH12zIJ24oIA==",
       "funding": [
         "https://github.com/sponsors/LuanRT"
       ],
+      "license": "MIT",
       "dependencies": {
         "acorn": "^8.8.0"
       }
@@ -12048,29 +12041,26 @@
       }
     },
     "node_modules/youtubei.js": {
-      "version": "13.1.0",
-      "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-13.1.0.tgz",
-      "integrity": "sha512-uL4TyojAYET0c5NGFD7+ScCod/k8Pc/B+D5tLrunFcz1GaBjRMOGRPcNGaRmnhwisegU7ibtw0iUxCN+BZ0ang==",
+      "version": "15.0.1",
+      "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-15.0.1.tgz",
+      "integrity": "sha512-2slapqJS5NuXKHvcACEknyVz0AjH/TrXaOhDM0q2twQKa54kCmfj+7B/2nGfd20uzAe29zW1ejk2qOc4ABuGkg==",
       "funding": [
         "https://github.com/sponsors/LuanRT"
       ],
+      "license": "MIT",
       "dependencies": {
         "@bufbuild/protobuf": "^2.0.0",
-        "jintr": "^3.2.1",
-        "tslib": "^2.5.0",
-        "undici": "^5.19.1"
+        "jintr": "^3.3.1",
+        "undici": "^6.21.3"
       }
     },
     "node_modules/youtubei.js/node_modules/undici": {
-      "version": "5.29.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz",
-      "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==",
+      "version": "6.21.3",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.3.tgz",
+      "integrity": "sha512-gBLkYIlEnSp8pFbT64yFgGE6UIB9tAkhukC23PmMDCe5Nd+cRqKxSjw5y54MK2AZMgZfJWMaNE4nYUHgi1XEOw==",
       "license": "MIT",
-      "dependencies": {
-        "@fastify/busboy": "^2.0.0"
-      },
       "engines": {
-        "node": ">=14.0"
+        "node": ">=18.17"
       }
     },
     "node_modules/zod": {

package.json

@@ -51,7 +51,7 @@
     "@fortawesome/react-fontawesome": "^0.2.0",
     "@foxxmd/chromecast-client": "^1.0.4",
     "@foxxmd/get-version": "^0.0.3",
-    "@foxxmd/logging": "^0.2.2",
+    "@foxxmd/logging": "^0.2.3",
     "@foxxmd/regex-buddy-core": "^0.1.2",
     "@foxxmd/string-sameness": "^0.4.0",
     "@gr2m/net-interceptor": "^1.0.0",
@@ -118,7 +118,7 @@
     "vite-express": "^0.16.0",
     "vlc-client": "^1.1.1",
     "xml2js": "0.6.1",
-    "youtubei.js": "^13.1.0"
+    "youtubei.js": "^15.0.1"
   },
   "devDependencies": {
     "@dbus-types/notifications": "^0.0.5",

src/backend/common/vendor/ListenbrainzApiClient.ts

@@ -14,11 +14,11 @@ import {
 } from "../../utils/StringUtils.js";
 import { getScrobbleTsSOCDate } from "../../utils/TimeUtils.js";
 import { UpstreamError } from "../errors/UpstreamError.js";
-import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions } from "../infrastructure/Atomic.js";
+import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, DELIMITERS, FormatPlayObjectOptions } from "../infrastructure/Atomic.js";
 import { ListenBrainzClientData } from "../infrastructure/config/client/listenbrainz.js";
 import AbstractApiClient from "./AbstractApiClient.js";
 import { getBaseFromUrl, isPortReachableConnect, joinedUrl, normalizeWebAddress } from '../../utils/NetworkUtils.js';
-import { removeUndefinedKeys } from '../../utils.js';
+import { removeUndefinedKeys, unique } from '../../utils.js';
 import {ListensResponse as KoitoListensResponse} from '../infrastructure/config/client/koito.js'
 import { listenObjectResponseToPlay } from './koito/KoitoApiClient.js';
 import { version } from '../../ioc.js';
@@ -395,7 +395,15 @@ export class ListenbrainzApiClient extends AbstractApiClient {
             }
 
             // now try to extract any remaining artists from filtered artist/name values
-            const parsedArtists = parseArtistCredits(filteredSubmittedArtistName);
+            const splitAmpersand = artistsWithJoiners.length === 0 && artistMappings.some(x => x.join_phrase.includes('&'));
+            let nonProperJoinedDelims = undefined;
+            if(artistsWithJoiners.length === 0) {
+                nonProperJoinedDelims = unique(artistMappings.filter(x => DELIMITERS.includes(x.join_phrase.trim())).map(x => x.join_phrase.trim()));
+                if(nonProperJoinedDelims.length === 0) {
+                    nonProperJoinedDelims = undefined;
+                }
+            }
+            const parsedArtists = parseArtistCredits(filteredSubmittedArtistName, nonProperJoinedDelims);
             if (parsedArtists !== undefined) {
                 if (parsedArtists.primary !== undefined) {
                     artistsFromUserValues.push(parsedArtists.primary);

src/backend/sources/YTMusicSource.ts

@@ -19,10 +19,8 @@ import {
 import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js";
 import { buildTrackString, truncateStringToLength } from "../../core/StringUtils.js";
 import { joinedUrl } from "../utils/NetworkUtils.js";
-import { FixedSizeList } from "fixed-size-list";
 import { todayAwareFormat } from "../utils/TimeUtils.js";
-import { RestType } from "ts-json-schema-generator";
-import { parseArtistCredits, parseCredits } from "../utils/StringUtils.js";
+import { parseArrayFromMaybeString, parseArtistCredits, parseCredits } from "../utils/StringUtils.js";
 
 export interface HistoryIngressResult {
     plays: PlayObject[], 
@@ -72,18 +70,43 @@ export const ytiHistoryResponseFromShelfToPlays = (res: ApiResponse, options: {n
     return items;
 }
 
-const GOOGLE_OAUTH_OPTS: GenerateAuthUrlOpts = {
-    access_type: 'offline',
-    scope: [
-        "http://gdata.youtube.com",
-        "https://www.googleapis.com/auth/youtube",
-        "https://www.googleapis.com/auth/youtube.force-ssl",
-        "https://www.googleapis.com/auth/youtube-paid-content",
-        "https://www.googleapis.com/auth/accounts.reauth",
-    ],
-    include_granted_scopes: true,
-    prompt: 'consent',
-};
+const DEFAULT_SCOPES = [
+    "http://gdata.youtube.com",
+    "https://www.googleapis.com/auth/youtube",
+    "https://www.googleapis.com/auth/youtube.force-ssl",
+    "https://www.googleapis.com/auth/youtube-paid-content",
+    "https://www.googleapis.com/auth/accounts.reauth",
+];
+
+const VALID_SCOPES = [
+    "https://www.googleapis.com/auth/youtube",
+    "https://www.googleapis.com/auth/youtube.force-ssl",
+    "https://www.googleapis.com/auth/youtube-paid-content",
+]
+
+const getGoogleOauthOpts = (): GenerateAuthUrlOpts => {
+    let scopes: string[];
+    const userInput = parseArrayFromMaybeString(process.env.YTM_SCOPES);
+    if (userInput.length > 0) {
+        scopes = userInput.map(x => {
+            if (x.toLocaleLowerCase() === 'default') {
+                return DEFAULT_SCOPES;
+            } else if (x.toLocaleLowerCase() === 'valid') {
+                return VALID_SCOPES;
+            }
+            return x;
+        }).flat(1);
+    } else {
+        scopes = DEFAULT_SCOPES;
+    }
+
+    return {
+        access_type: 'offline',
+        scope: scopes,
+        include_granted_scopes: true,
+        prompt: 'consent',
+    };
+}
 
 export default class YTMusicSource extends AbstractSource {
 
@@ -162,6 +185,9 @@ export default class YTMusicSource extends AbstractSource {
                 this.authed = true;
                 await this.yti.session.oauth.cacheCredentials();
             });
+            if(isDebugMode()) {
+                Log.setLevel(Log.Level.DEBUG);
+            }
     }
 
     protected configureCustomOauth() {
@@ -203,7 +229,9 @@ export default class YTMusicSource extends AbstractSource {
             redirectUri: this.redirectUri,
         });
 
-        const authorizationUrl = this.oauthClient.generateAuthUrl(GOOGLE_OAUTH_OPTS);
+        const scopeOpts = getGoogleOauthOpts();
+        this.logger.debug(`Using scopes:\n${(scopeOpts.scope as string[]).join('\n')}`)
+        const authorizationUrl = this.oauthClient.generateAuthUrl(getGoogleOauthOpts());
         this.verificationUrl = authorizationUrl;
     }
 
@@ -420,6 +448,10 @@ Redirect URI  : ${this.redirectUri}`);
             });
             return res;
         } catch (e) {
+            const info = loggedErrorExtra(e);
+            if(info !== undefined) {
+                this.logger.error(info, 'Additional API response details')
+            }
             throw e;
         }
     }

src/backend/tests/listenbrainz/correctlyMapped/multiArtistInArtistName.json

@@ -74,7 +74,7 @@
             {
               "artist_credit_name": "Metro Boomin",
               "artist_mbid": "59db3d82-86ea-451f-881f-dffc8ec387c9",
-              "join_phrase": " & "
+              "join_phrase": " , "
             },
             {
               "artist_credit_name": "James Blake",
@@ -116,7 +116,7 @@
             {
               "artist_credit_name": "Childish Gambino",
               "artist_mbid": "7fb57fba-a6ef-44c2-abab-2fa3bdee607e",
-              "join_phrase": " & "
+              "join_phrase": " , "
             },
             {
               "artist_credit_name": "Ariana Grande",

src/backend/tests/listenbrainz/listenbrainz.test.ts

@@ -10,24 +10,24 @@ import { ListenbrainzApiClient } from "../../common/vendor/ListenbrainzApiClient
 import { ListenResponse } from '../../common/vendor/listenbrainz/interfaces.js';
 import { ExpectedResults } from "../utils/interfaces.js";
 import { withRequestInterception } from "../utils/networking.js";
-import artistWithProperJoiner from './correctlyMapped/artistProperHasJoinerInName.json';
+import artistWithProperJoiner from './correctlyMapped/artistProperHasJoinerInName.json' with { type: "json" };
 // correct mappings
-import multiArtistInArtistName from './correctlyMapped/multiArtistInArtistName.json';
-import multiArtistsInTrackName from './correctlyMapped/multiArtistInTrackName.json';
-import multiMappedArtistsWithSingleUserArtist from './correctlyMapped/multiArtistMappingWithSingleRecordedArtist.json';
-import noArtistMapping from './correctlyMapped/noArtistMapping.json';
-import normalizedValues from './correctlyMapped/normalizedName.json';
-import slightlyDifferentNames from './correctlyMapped/trackNameSlightlyDifferent.json';
+import multiArtistInArtistName from './correctlyMapped/multiArtistInArtistName.json' with { type: "json" };
+import multiArtistsInTrackName from './correctlyMapped/multiArtistInTrackName.json' with { type: "json" };
+import multiMappedArtistsWithSingleUserArtist from './correctlyMapped/multiArtistMappingWithSingleRecordedArtist.json' with { type: "json" };
+import noArtistMapping from './correctlyMapped/noArtistMapping.json' with { type: "json" };
+import normalizedValues from './correctlyMapped/normalizedName.json' with { type: "json" };
+import slightlyDifferentNames from './correctlyMapped/trackNameSlightlyDifferent.json' with { type: "json" };
 
 // incorrect mappings
-import incorrectMultiArtistsTrackName from './incorrectlyMapped/multiArtistsInTrackName.json';
-import veryWrong from './incorrectlyMapped/veryWrong.json';
+import incorrectMultiArtistsTrackName from './incorrectlyMapped/multiArtistsInTrackName.json' with { type: "json" };
+import veryWrong from './incorrectlyMapped/veryWrong.json' with { type: "json" };
 
 interface LZTestFixture {
     data: ListenResponse
     expected: ExpectedResults
 }
-describe('Listenbrainz Listen Parsing', function () {
+describe('#PlayParse Listenbrainz Listen Parsing', function () {
 
     describe('When user-submitted artist/track do NOT match MB mappings', function() {
         it('Uses user submitted values when no artist mappings', async function () {
@@ -56,7 +56,7 @@ describe('Listenbrainz Listen Parsing', function () {
     })
 
 
-    describe('When user-submitted artist/track matches a MB mapped value', function() {
+    describe('#PlayParse When user-submitted artist/track matches a MB mapped value', function() {
 
         it('Detects slightly different track names as equal', async function () {
             for(const test of slightlyDifferentNames as unknown as LZTestFixture[]) {

src/backend/tests/plays/playParsing.test.ts

@@ -0,0 +1,140 @@
+import { loggerTest, loggerDebug, childLogger } from "@foxxmd/logging";
+import chai, { assert, expect } from 'chai';
+import asPromised from 'chai-as-promised';
+import { after, before, describe, it } from 'mocha';
+
+import { asPlays, generateArtistsStr, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js";
+import { parseArtistCredits, parseContextAwareStringList, parseCredits } from "../../utils/StringUtils.js";
+
+describe('#PlayParse Parsing Artists from String', function() {
+
+    it('Parses Artists from an Artist-like string', function () {
+        for(const i of Array(40)) {
+            const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}});
+            const credits = parseArtistCredits(str);
+            const allArtists = primaries.concat(secondaries);
+            const parsed = [credits.primary].concat(credits.secondary ?? []);
+            expect(primaries.concat(secondaries),`
+'${str}'
+Expected => ${allArtists.join(' || ')}
+Found    => ${parsed.join(' || ')}`)
+
+.eql(parsed)
+        }
+    });
+
+    it('Parses & as "local" joiner when other delimiters present', function () {
+
+        const data = [
+        {
+            str: `Melendi \\ Ryan Lewis \\ The Righteous Brothers (featuring Joan Jett & The Blackhearts \\ Robin Schulz)`,
+            expected: ['Melendi', 'Ryan Lewis', 'The Righteous Brothers', 'Joan Jett & The Blackhearts', 'Robin Schulz']
+        }, 
+        {
+            str: `Gigi D'Agostino \\ YOASOBI (vs Sam Hunt, Lisa Loeb & Booba)`,
+            expected: [`Gigi D'Agostino`, 'YOASOBI', 'Sam Hunt', 'Lisa Loeb', 'Booba']
+        },
+        {
+            str: `Wham!, Hillsong Worship & Bruce Channel feat. I Prevail`,
+            expected: ['Wham!', 'Hillsong Worship & Bruce Channel', 'I Prevail']
+        }
+    ];
+
+        for(const d of data) {
+            const credits = parseArtistCredits(d.str);
+            const parsed = [credits.primary].concat(credits.secondary ?? [])
+            expect(d.expected).eql(parsed)
+        }
+    
+    });
+
+    it('Only parses & as "global" joiner when no other delimiters present', function () {
+
+        const data = [{
+            str: `Melendi & Ryan Lewis & The Righteous Brothers (featuring The Blackhearts \\ Robin Schulz)`,
+            expected: ['Melendi', 'Ryan Lewis', 'The Righteous Brothers', 'The Blackhearts', 'Robin Schulz']
+        }];
+
+        for(const d of data) {
+            const credits = parseArtistCredits(d.str);
+            const parsed = [credits.primary].concat(credits.secondary ?? [])
+            expect(d.expected).eql(parsed)
+        }
+    });
+
+    it('Does not split artist name when only one joiner is present', function () {
+
+        const data = [
+            {
+            str: `Melendi & Ryan Lewis`,
+            expected: ['Melendi & Ryan Lewis']
+        },{
+            str: `Melendi and Ryan Lewis`,
+            expected: ['Melendi and Ryan Lewis']
+        },
+    ];
+
+        for(const d of data) {
+            const credits = parseArtistCredits(d.str);
+            const parsed = [credits.primary].concat(credits.secondary ?? [])
+            expect(d.expected).eql(parsed)
+        }
+    });
+
+    it('Parses secondary free regex', function () {
+
+        const data = [{
+            str: `Diddy & Grand Funk Railroad feat. Daya & (G)I-DLE`,
+            expected: ['Diddy', 'Grand Funk Railroad', 'Daya', '(G)I-DLE']
+        }];
+
+        for(const d of data) {
+            const credits = parseArtistCredits(d.str);
+            const parsed = [credits.primary].concat(credits.secondary ?? [])
+            expect(d.expected).eql(parsed)
+        }
+    });
+
+    it('Parses singlar Artist with wrapped vs multiple', function () {
+        const [str, primaries, secondaries] = generateArtistsStr({primary: 1, secondary: {num: 2, ft: 'vs', joiner: '/', ftWrap: true}});
+        const credits = parseArtistCredits(str);
+        const moreCredits = parseCredits(str);
+        expect(true).eq(true);
+    });
+
+    describe('When joiner is known', function () {
+
+        it('Parses many primary artists', function () {
+            for(const i of Array(10)) {
+                const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, joiner: '/'}, secondary: 0});
+                const credits = parseArtistCredits(str, ['/']);
+                const allArtists = primaries.concat(secondaries);
+                const parsed = [credits.primary].concat(credits.secondary ?? [])
+                expect(primaries.concat(secondaries),`
+'${str}'
+Expected => ${allArtists.join(' || ')}
+Found    => ${parsed.join(' || ')}`)
+        .eql(parsed)
+            }
+        });
+
+        it('Parses many secondary artists', function () {
+            // fails on -- Peso Pluma / Lil Baby / R. Kelly (featuring TOMORROW X TOGETHER / AC/DC / DaVido)
+            for(const i of Array(10)) {
+                const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, joiner: '/'}, secondary: {joiner: '/', finalJoiner: false}});
+                const credits = parseArtistCredits(str, ['/']);
+                const allArtists = primaries.concat(secondaries);
+                const parsed = [credits.primary].concat(credits.secondary ?? [])
+                expect(primaries.concat(secondaries),`
+'${str}'
+Expected => ${allArtists.join(' || ')}
+Found    => ${parsed.join(' || ')}`)
+        .eql(parsed)
+            }
+        });
+
+
+    });
+
+
+});

src/backend/tests/utils/PlayTestUtils.ts

@@ -5,9 +5,11 @@ import isBetween from "dayjs/plugin/isBetween.js";
 import relativeTime from "dayjs/plugin/relativeTime.js";
 import timezone from "dayjs/plugin/timezone.js";
 import utc from "dayjs/plugin/utc.js";
-import { JsonPlayObject, ObjectPlayData, PlayMeta, PlayObject } from "../../../core/Atomic.js";
+import { FEAT, JOINERS, JOINERS_FINAL, JsonPlayObject, ObjectPlayData, PlayMeta, PlayObject } from "../../../core/Atomic.js";
 import { sortByNewestPlayDate } from "../../utils.js";
 import { NO_DEVICE, NO_USER, PlayerStateDataMaybePlay, PlayPlatformId, ReportedPlayerStatus } from '../../common/infrastructure/Atomic.js';
+import { arrayListAnd } from '../../../core/StringUtils.js';
+import { findDelimiters } from '../../utils/StringUtils.js';
 
 dayjs.extend(utc)
 dayjs.extend(isBetween);
@@ -176,3 +178,121 @@ export const generatePlayPlatformId = (deviceId?: string, userId?: string): Play
 export const generatePlays = (numberOfPlays: number, data: ObjectPlayData = {}, meta: PlayMeta = {}): PlayObject[] => {
     return Array.from(Array(numberOfPlays), () => generatePlay(data, meta));
 }
+
+export const generateArtist = () => faker.music.artist;
+
+export interface ArtistGenerationOptions
+{
+    ambiguousJoinedNames?: boolean, 
+    trailingAmpersand?: boolean
+}
+
+export const generateArtists = (num?: number, max: number = 3, opts: ArtistGenerationOptions = {}) => {
+    if(num === 0 || max === 0) {
+        return [];
+    }
+    let artists = faker.helpers.multiple(faker.music.artist, {count: {min: num ?? 1, max: num ?? max}});
+
+    const {
+        trailingAmpersand = false,
+        ambiguousJoinedNames = false
+    } = opts;
+
+    if(!trailingAmpersand) {
+        // its really hard to parse an artist name that contains an '&' when it comes at the end of a list
+        // because its ambigious if the list is joining the list with & or if & is part of the artist name
+        // so by default don't generate these (we test for specific scenarios in playParsing.test.ts)
+        while(artists[artists.length - 1].includes('&')) {
+            artists = artists.slice(0, artists.length - 1).concat(faker.music.artist());
+        }
+    }
+    if(!ambiguousJoinedNames) {
+        artists = artists.map(x => {
+            let a = x;
+            let foundDelims = findDelimiters(a);
+            while(foundDelims !== undefined && foundDelims.length > 0 && !(foundDelims.length === 1 && foundDelims[0] === '&')) {
+                a = faker.music.artist();
+                foundDelims = findDelimiters(a);
+            }
+            return a;
+        });
+    }
+    return artists;
+}
+
+export interface ArtistGenerateOptions extends ArtistGenerationOptions {
+    num?: number
+    max?: number
+    joiner?: string
+    finalJoiner?: false | string
+    spacedJoiners?: boolean
+}
+
+export interface SecondaryArtistGenerateOptions extends ArtistGenerateOptions {
+    ft?: string
+    ftWrap?: boolean
+}
+
+export interface CompoundArtistGenerateOptions {
+    primary?: number | ArtistGenerateOptions
+    secondary?: number | SecondaryArtistGenerateOptions
+}
+
+export const generateArtistsStr = (options: CompoundArtistGenerateOptions = {}): [string, string[], string[]] => {
+
+    const {primary = {}, secondary = {}} = options;
+
+    const primaryOpts: ArtistGenerateOptions = typeof primary === 'number' ? {num: primary} : primary;
+    const secondaryOpts: SecondaryArtistGenerateOptions = typeof secondary === 'number' ? {num: secondary} : secondary;
+
+    const primaryArt = generateArtists(primaryOpts.num, primaryOpts.max)
+    const secondaryArt = generateArtists(secondaryOpts.num, secondaryOpts.max);
+
+
+    const joinerPrimary: string = primaryOpts.joiner ?? faker.helpers.arrayElement(JOINERS);
+    let finalJoinerPrimary: string = joinerPrimary;
+    if(primaryOpts.finalJoiner !== false) {
+        if(primaryOpts.finalJoiner === undefined) {
+            if(joinerPrimary === ',' && !primaryArt.some(x => x.includes('&'))) {
+                finalJoinerPrimary = faker.helpers.arrayElement(JOINERS_FINAL);
+            }
+        
+        } else {
+            finalJoinerPrimary = primaryOpts.finalJoiner;
+        }
+    }
+
+    const primaryStr = arrayListAnd(primaryArt, joinerPrimary, finalJoinerPrimary, primaryOpts.spacedJoiners);
+
+    if(secondaryArt.length === 0) {
+        return [primaryStr, primaryArt, []];
+    }
+
+    const joinerSecondary: string = secondaryOpts.joiner ?? faker.helpers.arrayElement(JOINERS);
+    let finalJoinerSecondary: string = joinerSecondary;
+    if(secondaryOpts.finalJoiner !== false) {
+        if(secondaryOpts.finalJoiner === undefined) {
+            if(joinerSecondary === ',' && !secondaryArt.some(x => x.includes('&'))) {
+                finalJoinerSecondary = faker.helpers.arrayElement(JOINERS_FINAL);
+            }
+        } else {
+            finalJoinerSecondary = secondaryOpts.finalJoiner;
+        }
+    }
+
+    const secondaryStr = arrayListAnd(secondaryArt, joinerSecondary, finalJoinerSecondary, secondaryOpts.spacedJoiners);
+    const ft = secondaryOpts.ft ?? faker.helpers.arrayElement(FEAT);
+    let sec = `${ft} ${secondaryStr}`;
+    let wrap: boolean;
+    if(secondaryOpts.ftWrap !== undefined) {
+        wrap = secondaryOpts.ftWrap;
+    } else {
+        wrap = faker.datatype.boolean();
+    }
+    if(wrap) {
+        sec = `(${sec})`;
+    }
+    const artistStr = `${primaryStr} ${sec}`;
+
+    return [artistStr, primaryArt, secondaryArt];
+}

src/backend/tests/utils/strings.test.ts

@@ -9,7 +9,7 @@ import {
     uniqueNormalizedStrArr
 } from "../../utils/StringUtils.js";
 import { ExpectedResults } from "./interfaces.js";
-import testData from './playTestData.json';
+import testData from './playTestData.json' with { type: "json" };
 import { splitByFirstFound } from '../../../core/StringUtils.js';
 
 interface PlayTestFixture {

src/backend/tests/ytm/ytm.test.ts

@@ -4,12 +4,13 @@ import chai, { assert, expect } from 'chai';
 import asPromised from 'chai-as-promised';
 import clone from "clone";
 import YTMusicSource, { ytiHistoryResponseFromShelfToPlays, ytiHistoryResponseToListItems } from "../../sources/YTMusicSource.js";
-import ytHistoryRes from './ytres.json' assert {type: 'json'};
+import ytHistoryRes from './ytres.json' with {type: 'json'};
 import EventEmitter from "events";
 import { generatePlay, generatePlays, normalizePlays } from '../utils/PlayTestUtils.js';
 import { YTMusicSourceConfig } from '../../common/infrastructure/config/source/ytmusic.js';
 import { sleep } from '../../utils.js';
 import dayjs from 'dayjs';
+import { ApiResponse } from 'youtubei.js';
 
 chai.use(asPromised);
 
@@ -33,12 +34,12 @@ const createYtSource = (opts?: {
 describe('Parses History', function () {
 
     it(`Parses a history response to tracks`, async function () {
-        const items = ytiHistoryResponseToListItems(ytHistoryRes);
+        const items = ytiHistoryResponseToListItems(ytHistoryRes as ApiResponse);
         expect(items).length(10);
     });
 
     it(`Parses a history response plays with shelf name`, async function () {
-        const items = ytiHistoryResponseFromShelfToPlays(ytHistoryRes);
+        const items = ytiHistoryResponseFromShelfToPlays(ytHistoryRes as ApiResponse);
         expect(items[0]?.meta?.comment).to.eq('March 2023');
     });
 });

src/backend/utils/StringUtils.ts

@@ -1,7 +1,7 @@
 import { strategies, stringSameness, StringSamenessResult } from "@foxxmd/string-sameness";
 import { PlayObject } from "../../core/Atomic.js";
 import { asPlayerStateData, DELIMITERS, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js";
-import { genGroupIdStr, getPlatformIdFromData, parseRegexSingleOrFail } from "../utils.js";
+import { genGroupIdStr, getPlatformIdFromData, intersect, parseRegexSingleOrFail } from "../utils.js";
 import { buildTrackString } from "../../core/StringUtils.js";
 
 const {levenStrategy, diceStrategy} = strategies;
@@ -61,7 +61,7 @@ export const SECONDARY_CAPTURED_REGEX = new RegExp(/[([]\s*(?<joiner>ft\.?\W|fea
  *    !!!!  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ *******
  *
  * */
-export const SECONDARY_FREE_REGEX = new RegExp(/^\s*(?<joiner>ft\.?\W|feat\.?\W|featuring|vs\.?\W)\s*(?<credits>(?:.+?(?= - |\s*[([]))|(?:.*))(?<creditsSuffix>.*)/i);
+export const SECONDARY_FREE_REGEX = new RegExp(/^\s*(?<joiner>ft\.?\W|feat\.?\W|featuring|vs\.?\W)\s*(?<credits>(?:.+?(?= - |\s*[([].+[)\]]$))|(?:.*))(?<creditsSuffix>.*)/i);
 
 const SECONDARY_REGEX_STRATS: RegExp[] = [SECONDARY_CAPTURED_REGEX, SECONDARY_FREE_REGEX];
 
@@ -116,7 +116,7 @@ export const parseCredits = (str: string, delimiters?: boolean | string[]): Play
         for(const strat of SECONDARY_REGEX_STRATS) {
             const secCredits = parseRegexSingleOrFail(strat, results.named.secondary);
             if(secCredits !== undefined) {
-                secondary = parseStringList(secCredits.named.credits as string, delims)
+                secondary = parseContextAwareStringList(secCredits.named.credits as string, delims)
                 suffix = secCredits.named.creditsSuffix;
                 break;
             }
@@ -134,7 +134,7 @@ export const parseCredits = (str: string, delimiters?: boolean | string[]): Play
     }
     return undefined;
 }
-export const parseArtistCredits = (str: string, delimiters?: boolean | string[]): PlayCredits | undefined => {
+export const parseArtistCredits = (str: string, delimiters?: boolean | string[], ignoreGlobalAmpersand?: boolean): PlayCredits | undefined => {
     if (str.trim() === '') {
         return undefined;
     }
@@ -148,7 +148,7 @@ export const parseArtistCredits = (str: string, delimiters?: boolean | string[])
     if (withJoiner !== undefined) {
         // all this does is make sure and "ft" or parenthesis/brackets are separated --
         // it doesn't also separate primary artists so do that now
-        const primaries = parseStringList(withJoiner.primary, delims);
+        const primaries = parseContextAwareStringList(withJoiner.primary, delims, {ignoreGlobalAmpersand: ignoreGlobalAmpersand ?? false});
         if (primaries.length > 1) {
             return {
                 primary: primaries[0],
@@ -159,7 +159,7 @@ export const parseArtistCredits = (str: string, delimiters?: boolean | string[])
         return withJoiner;
     }
     // likely this is a plain string with just delims
-    const artists = parseStringList(str, delims);
+    const artists = parseContextAwareStringList(str, delims, {ignoreGlobalAmpersand: ignoreGlobalAmpersand ?? true});
     if (artists.length > 1) {
         return {
             primary: artists[0],
@@ -173,7 +173,7 @@ export const parseArtistCredits = (str: string, delimiters?: boolean | string[])
     }
 }
 export const parseTrackCredits = (str: string, delimiters?: boolean | string[]): PlayCredits | undefined => parseCredits(str, delimiters);
-export const parseStringList = (str: string, delimiters: string[] = [',', '&', '/', '\\']): string[] => {
+export const parseStringList = (str: string, delimiters: string[] = DELIMITERS): string[] => {
     if (delimiters.length === 0) {
         return [str];
     }
@@ -182,10 +182,54 @@ export const parseStringList = (str: string, delimiters: string[] = [',', '&', '
         return explodedStrings.flat(1);
     }, [str]).map(x => x.trim());
 }
+export const parseContextAwareStringList = (str: string, delimiters: string[] = [',', '/', '\\'], opts: {ignoreGlobalAmpersand?: boolean} = {}): string[] => {
+    if (delimiters.length === 0) {
+        return [str];
+    }
+    // bypass tokens using slashes without spaces
+    const cleanStr = bypassJoiners(str);
+    const nonAmpersandDelims = delimiters.some(x => cleanStr.includes(x));
+    const shouldIgnoreGlobalAmpersand = opts.ignoreGlobalAmpersand ?? nonAmpersandDelims;
+
+    let awareList: string[] = [];
+
+    const list = parseStringList(cleanStr, nonAmpersandDelims === false && shouldIgnoreGlobalAmpersand === false ? ['&'] : delimiters);
+    if(shouldIgnoreGlobalAmpersand && list.length > 1 && list[list.length - 1].includes('&') && nonAmpersandDelims) { //&& !list[list.length - 1].includes('& the')
+        awareList = list.slice(0, list.length - 1).concat(list[list.length - 1].split('&') );
+    } else {
+        awareList = list;
+    }
+    return awareList.map(x =>rejoinBypassed(x.trim()));
+}
+
+const bypassJoinerMap = [
+    {
+        rejoin: str => str.replaceAll(/(.*?\S)(\^\^\^)(\S.*?)/g, '$1/$3'),
+        bypass: str => str.replaceAll(/(.*?\S)(\/)(\S.*?)/g, '$1^^^$3')
+    },
+    {
+        rejoin: str => str.replaceAll(/(.*)(###)(.*)/g, '$1\\$3'),
+        bypass: str => str.replaceAll(/(.*\S)(\\)(.*\S)/g, '$1###$3')
+    }
+];
+export const bypassJoiners = (str: string): string => {
+    let bypassed: string = str;
+    for(const b of bypassJoinerMap) {
+        bypassed = b.bypass(bypassed)
+    }
+    return bypassed;
+}
+export const rejoinBypassed = (str: string): string => {
+    let bypassed: string = str;
+    for(const b of bypassJoinerMap) {
+        bypassed = b.rejoin(bypassed)
+    }
+    return bypassed;
+}
 export const containsDelimiters = (str: string) => null !== str.match(/[,&/\\]+/i)
-export const findDelimiters = (str: string) => {
+export const findDelimiters = (str: string, delimiters = DELIMITERS) => {
     const found: string[] = [];
-    for (const d of DELIMITERS) {
+    for (const d of delimiters) {
         if (str.indexOf(d) !== -1) {
             found.push(d);
         }

src/core/Atomic.ts

@@ -343,4 +343,13 @@ export interface URLData {
     url: URL
     normal: string
     port: number
-}
+}
+
+export type Joiner = ',' | '&' | '/' | '\\' | string;
+export const JOINERS: Joiner[] = [',','/','\\'];
+
+export type FinalJoiners = '&';
+export const JOINERS_FINAL: FinalJoiners[] = ['&'];
+
+export type Feat = 'ft' | 'feat' | 'vs' | 'ft.' | 'feat.' | 'vs.' | 'featuring'
+export const FEAT: Feat[] = ['ft','feat','vs','ft.','feat.','vs.','featuring'];

src/core/StringUtils.ts

@@ -201,3 +201,29 @@ export const combinePartsToString = (parts: any[], glue: string = '-'): string |
     }
     return undefined;
 }
+
+export const arrayListOxfordAnd = (list: string[], joiner: string, finalJoiner: string, spaced: boolean = true): string => {
+    if(list.length === 1) {
+        return list[0];
+    }
+    const start = list.slice(0, list.length - 1);
+    const end = list.slice(list.length - 1);
+
+    const joinerProper = joiner === ',' ? ', ' : (spaced ? ` ${joiner} ` : joiner);
+    const finalProper = spaced ? ` ${finalJoiner} ` : finalJoiner;
+
+    return [start.join(joinerProper), end].join(joiner === ',' && spaced ? `,${finalProper}` : finalProper);
+}
+
+export const arrayListAnd = (list: string[], joiner: string, finalJoiner: string, spaced: boolean = true): string => {
+    if(list.length === 1) {
+        return list[0];
+    }
+    const start = list.slice(0, list.length - 1);
+    const end = list.slice(list.length - 1);
+
+    const joinerProper = joiner === ',' ? ', ' : (spaced ? ` ${joiner} ` : joiner);
+    const finalProper = spaced ? ` ${finalJoiner} ` : finalJoiner;
+
+    return [start.join(joinerProper), end].join(finalProper);
+}