POC apple base source

package-lock.json

@@ -78,6 +78,7 @@
         "mpd-api": "^1.1.2",
         "musicbrainz-api": "^0.27.0",
         "nanoid": "^3.3.1",
+        "node-musickit-api": "^2.1.1",
         "node-object-hash": "^3.1.1",
         "normalize-url": "^8.0.1",
         "ntfy": "^1.7.6",
@@ -8359,6 +8360,58 @@
         "node": ">=0.2.0"
       }
     },
+    "node_modules/jsonwebtoken": {
+      "version": "8.5.1",
+      "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz",
+      "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==",
+      "license": "MIT",
+      "dependencies": {
+        "jws": "^3.2.2",
+        "lodash.includes": "^4.3.0",
+        "lodash.isboolean": "^3.0.3",
+        "lodash.isinteger": "^4.0.4",
+        "lodash.isnumber": "^3.0.3",
+        "lodash.isplainobject": "^4.0.6",
+        "lodash.isstring": "^4.0.1",
+        "lodash.once": "^4.0.0",
+        "ms": "^2.1.1",
+        "semver": "^5.6.0"
+      },
+      "engines": {
+        "node": ">=4",
+        "npm": ">=1.4.28"
+      }
+    },
+    "node_modules/jsonwebtoken/node_modules/jwa": {
+      "version": "1.4.2",
+      "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.2.tgz",
+      "integrity": "sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==",
+      "license": "MIT",
+      "dependencies": {
+        "buffer-equal-constant-time": "^1.0.1",
+        "ecdsa-sig-formatter": "1.0.11",
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "node_modules/jsonwebtoken/node_modules/jws": {
+      "version": "3.2.3",
+      "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.3.tgz",
+      "integrity": "sha512-byiJ0FLRdLdSVSReO/U4E7RoEyOCKnEnEPMjq3HxWtvzLsV08/i5RQKsFVNkCldrCaPr2vDNAOMsfs8T/Hze7g==",
+      "license": "MIT",
+      "dependencies": {
+        "jwa": "^1.4.2",
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "node_modules/jsonwebtoken/node_modules/semver": {
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+      "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
+      "license": "ISC",
+      "bin": {
+        "semver": "bin/semver"
+      }
+    },
     "node_modules/jwa": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz",
@@ -8727,18 +8780,60 @@
       "dev": true,
       "peer": true
     },
+    "node_modules/lodash.includes": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
+      "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==",
+      "license": "MIT"
+    },
     "node_modules/lodash.isarguments": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
       "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
       "license": "MIT"
     },
+    "node_modules/lodash.isboolean": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
+      "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==",
+      "license": "MIT"
+    },
+    "node_modules/lodash.isinteger": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
+      "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==",
+      "license": "MIT"
+    },
+    "node_modules/lodash.isnumber": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
+      "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==",
+      "license": "MIT"
+    },
+    "node_modules/lodash.isplainobject": {
+      "version": "4.0.6",
+      "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
+      "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==",
+      "license": "MIT"
+    },
+    "node_modules/lodash.isstring": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
+      "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==",
+      "license": "MIT"
+    },
     "node_modules/lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
       "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
       "dev": true
     },
+    "node_modules/lodash.once": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
+      "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==",
+      "license": "MIT"
+    },
     "node_modules/log-symbols": {
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz",
@@ -9389,6 +9484,28 @@
         }
       }
     },
+    "node_modules/node-musickit-api": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/node-musickit-api/-/node-musickit-api-2.1.1.tgz",
+      "integrity": "sha512-6dl6KbvSSXZyabxvFEYhW3yQbrqM0pn/8b0BNk2wHJfjIxWkgjMl+Uh0q/NkxdWIq/pY1jTsGuVEfraRDv5nQQ==",
+      "license": "MIT",
+      "dependencies": {
+        "axios": "^0.21.4",
+        "jsonwebtoken": "^8.5.1"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/node-musickit-api/node_modules/axios": {
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+      "license": "MIT",
+      "dependencies": {
+        "follow-redirects": "^1.14.0"
+      }
+    },
     "node_modules/node-object-hash": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/node-object-hash/-/node-object-hash-3.1.1.tgz",

package.json

@@ -112,6 +112,7 @@
     "mpd-api": "^1.1.2",
     "musicbrainz-api": "^0.27.0",
     "nanoid": "^3.3.1",
+    "node-musickit-api": "^2.1.1",
     "node-object-hash": "^3.1.1",
     "normalize-url": "^8.0.1",
     "ntfy": "^1.7.6",

src/backend/common/infrastructure/Atomic.ts

@@ -35,7 +35,8 @@ export type SourceType =
     | 'koito'
     | 'tealfm'
     | 'rocksky'
-    | 'sonos';
+    | 'sonos'
+    | 'apple';
 
 export const sourceTypes: SourceType[] = [
     'spotify',
@@ -65,7 +66,8 @@ export const sourceTypes: SourceType[] = [
     'koito',
     'tealfm',
     'rocksky',
-    'sonos'
+    'sonos',
+    'apple'
 ];
 
 export const sourceInterfaces = [
@@ -97,7 +99,8 @@ export const sourceInterfaces = [
     'KoitoSourceConfig',
     'TealSourceConfig',
     'RockskySourceConfig',
-    'SonosSourceConfig'
+    'SonosSourceConfig',
+    'AppleMusicSourceConfig'
 ]
 
 export const isSourceType = (data: string): data is SourceType => {

src/backend/common/infrastructure/config/source/apple.ts

@@ -0,0 +1,48 @@
+import {CommonSourceConfig, CommonSourceData, CommonSourceOptions} from "./index.js";
+import {PollingOptions} from "../common.js";
+
+// See issue for more documentation
+// https://github.com/FoxxMD/multi-scrobbler/issues/9#issuecomment-946871774
+export interface AppleMusicData extends CommonSourceData, PollingOptions {
+    /**
+     * Contents of MusicKit private key or a path to the key file
+     *
+     * https://help.apple.com/developer-account/#/devcdfbb56a3
+     * */
+    key: string
+
+    /**
+     * Team Id (tid) from Developer account
+     * */
+    teamId: string
+    /**
+     * Key identifier (kid) from Developer account
+     * */
+    keyId: string
+
+    /**
+     *  URL to another MS instance running the apple music source *server*
+     * */
+    endpoint: string
+}
+
+export interface AppleMusicOptions extends CommonSourceOptions {
+        /**
+         * Always log history diff
+         * 
+         * By default MS will log to `WARN` if history diff is inconsistent but does not log if diff is expected (on new tracks found)
+         * Set this to `true` to ALWAYS log diff on new tracks. Expected diffs will log to `DEBUG` and inconsistent diffs will continue to log to `WARN`
+         * 
+         * @default false
+         */
+        logDiff?: boolean
+}
+
+export interface AppleMusicSourceConfig extends CommonSourceConfig {
+    data: AppleMusicData
+    options?: AppleMusicOptions
+}
+
+export interface AppleMusicAIOSourceConfig extends AppleMusicSourceConfig {
+    type: 'apple'
+}

src/backend/common/infrastructure/config/source/sources.ts

@@ -26,6 +26,7 @@ import { TealSourceAIOConfig, TealSourceConfig } from "./tealfm.js";
 import { RockskySourceAIOConfig, RockskySourceConfig } from "./rocksky.js";
 import { LibrefmSouceAIOConfig, LibrefmSourceConfig } from "./librefm.js";
 import { SonosSourceAIOConfig, SonosSourceConfig } from "./sonos.js";
+import { AppleMusicAIOSourceConfig, AppleMusicSourceConfig } from "./apple.js";
 
 
 export type SourceConfig =
@@ -57,7 +58,8 @@ export type SourceConfig =
     | KoitoSourceConfig
     | TealSourceConfig
     | RockskySourceConfig
-    | SonosSourceConfig;
+    | SonosSourceConfig
+    | AppleMusicSourceConfig;
 
 export type SourceAIOConfig =
     SpotifySourceAIOConfig
@@ -88,4 +90,5 @@ export type SourceAIOConfig =
     | KoitoSourceAIOConfig
     | TealSourceAIOConfig
     | RockskySourceAIOConfig
-    | SonosSourceAIOConfig;
+    | SonosSourceAIOConfig
+    | AppleMusicAIOSourceConfig;

src/backend/server/auth.ts

@@ -17,6 +17,7 @@ import LibrefmScrobbler from "../scrobblers/LibrefmScrobbler.js";
 import LibrefmSource from "../sources/LibrefmSource.js";
 import e from "express";
 import AbstractSource from "../sources/AbstractSource.js";
+import { AppleSource } from "../sources/AppleSource.js";
 
 export const setupAuthRoutes = (app: ExpressWithAsync, logger: Logger, sourceMiddle: ExpressHandler, clientMiddle: ExpressHandler, scrobbleSources: ScrobbleSources, scrobbleClients: ScrobbleClients) => {
     app.use('/api/client/auth', clientMiddle);
@@ -65,6 +66,8 @@ export const setupAuthRoutes = (app: ExpressWithAsync, logger: Logger, sourceMid
                 // @ts-expect-error TS(2339): Property 'deezerSource' does not exist on type 'Se... Remove this comment to see the full error message
                 req.session.deezerSource = name;
                 return passport.authenticate(`deezer-${source.name}`)(req,res,next);
+            case 'apple':
+                return res.render('apple', {token: source.generateDeveloperToken()});
             case 'ytmusic':
                 await (source as YTMusicSource).reauthenticate();
                 res.redirect((source as YTMusicSource).verificationUrl);
@@ -137,6 +140,14 @@ export const setupAuthRoutes = (app: ExpressWithAsync, logger: Logger, sourceMid
                 responseContent = result;
             }
             return res.send(responseContent);
+        } else if(req.url.includes('apple')) {
+            const {
+                query: {
+                    token
+                } = {}
+            } = req;
+            const entity = scrobbleSources.getByName(state) as AppleSource;
+            await entity.handleAuthCodeCallback({token});
         } else {
             // TODO right now all sources requiring source interaction are covered by logic branches (deezer above and spotify here)
             // but eventually should update all source callbacks to url specific URLS to avoid ambiguity...

src/backend/sources/AppleSource.ts

@@ -0,0 +1,168 @@
+import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js";
+import {AppleMusicSourceConfig} from "../common/infrastructure/config/source/apple.js";
+import {FormatPlayObjectOptions, InternalConfig} from "../common/infrastructure/Atomic.js";
+import EventEmitter from "events";
+import {parseBool, readText, writeFile} from "../utils.js";
+import MusicKit from 'node-musickit-api/personalized/index.js';
+import {createJWT} from 'node-musickit-api/modules/createJWT.js';
+import { readJson } from "../utils/DataUtils.js";
+import {resolve} from 'path';
+import { PlayObject, PlayObjectLifecycleless } from "../../core/Atomic.js";
+import dayjs from "dayjs";
+import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
+
+
+export class AppleSource extends AbstractSource {
+    requiresAuth = true;
+    requiresAuthInteraction = true;
+
+    declare config: AppleMusicSourceConfig;
+
+    recentlyPlayed: PlayObject[] = [];
+
+    workingCredsPath: string;
+
+    keyContents?: string
+    userToken?: string
+
+    apiClient: MusicKit;
+
+    constructor(name: any, config: AppleMusicSourceConfig, internal: InternalConfig, emitter: EventEmitter) {
+        super('apple', name, config, internal, emitter);
+
+        this.canPoll = true;
+        this.supportsUpstreamRecentlyPlayed = true;
+
+        this.workingCredsPath = resolve(this.configDir, `apple-${this.name}.json`);
+
+        const {
+            logDiff,
+            ...rest
+        } = this.config.options || {};
+
+        let diffVal = logDiff;
+
+        if(diffVal === undefined) {
+            const diffEnv = process.env.YTM_LOG_DIFF;
+            if(diffEnv !== undefined) {
+                diffVal = parseBool(diffEnv);
+                this.config.options = {...rest, logDiff: diffVal};
+            }
+        }
+    }
+
+    protected async doBuildInitData(): Promise<true | string | undefined> {
+        if (this.config.data.key !== undefined || this.config.data.teamId !== undefined || this.config.data.keyId !== undefined) {
+            if (this.config.data.key === undefined) {
+                throw new Error(`For apple source as SERVER the property 'key' must be defined`);
+            } else {
+                // try to parse as file
+                try {
+                    this.keyContents = await readText(this.config.data.key, {throwOnNotFound: false})
+                    if (this.keyContents === undefined) {
+                        // could not find as file, that's fine
+                        this.keyContents = this.config.data.key;
+                    }
+                } catch (e) {
+                    throw new Error(`Apple config 'key' property seems to be a valid file path but could not be read: ${e.message}`);
+                }
+            }
+            if (this.config.data.teamId === undefined) {
+                throw new Error(`For apple source as SERVER the property 'teamId' must be defined`);
+            }
+            if (this.config.data.keyId === undefined) {
+                throw new Error(`For apple source as SERVER the property 'keyId' must be defined`);
+            }
+        }
+
+        return true;
+    }
+
+    doAuthentication = async () => {
+        try {
+            const credFile = await readJson(this.workingCredsPath, {throwOnNotFound: false});
+            if(credFile !== undefined) {
+                this.userToken = credFile.userToken;
+                // temp
+                this.apiClient = new MusicKit({
+                    key: this.keyContents,
+                    teamId: this.config.data.teamId,
+                    keyId: this.config.data.keyId,
+                    userToken: this.userToken,
+                });
+                return true;
+            } else {
+                throw new Error('No user token has been written yet, Authenticate from the dashboard to complete setup.');
+            }
+        } catch (e) {
+            throw new Error('Current apple credentials file exists but could not be parsed');
+        }
+    }
+
+    getRecentlyPlayed = async (options: RecentlyPlayedOptions = {}) => {
+
+        // https://musickit.js.org/#/personalized/recentlyPlayed?id=get-recently-played-tracks
+        const songs = this.apiClient.getRecentlyPlayed(options.limit ?? 20, 0, "songs");
+
+        // TODO fully implement formatPlayObj
+        const rawPlays = songs.map(x => formatPlayObj(x));
+
+        /**
+         * TODO implement comparing latest list of plays against previously seen/processed plays
+         * 
+         * See YTMusicSource@parseRecentAgainstResponse -- this is going to be basically the same process
+         * because both YTMusic and Apple do not return timestamps of when tracks were played
+         * so we do a state-machine-esque comparison of tracks we *know* we have already seen
+         * and then "discover" tracks that appear in the latest list that weren't in the previous list,
+         * with their timestamp set to an approximation of when polling for the latest list happened
+         * 
+         * It might be better to refactor the YTMusic parseRecentAgainstResponse etc. functions into indepedent functions that can be re-used by both sources
+         */
+
+        throw new Error('Not implemented');
+    }
+
+    generateDeveloperToken = () => {
+        return createJWT({
+            key: this.keyContents,
+            teamId: this.config.data.teamId,
+            keyId: this.config.data.keyId,
+        });
+    }
+
+    handleAuthCodeCallback = async ({token}) => {
+        await writeFile(this.workingCredsPath, JSON.stringify({
+            userToken: token
+        }));
+        this.userToken = token;
+        this.logger.info('Got apple user music token callback!');
+        // temp
+        this.apiClient = new MusicKit({
+            key: this.keyContents,
+            teamId: this.config.data.teamId,
+            keyId: this.config.data.keyId,
+            userToken: this.userToken,
+        })
+        return true;
+    }
+}
+
+export const formatPlayObj = (obj: object, options: FormatPlayObjectOptions = {}): PlayObjectLifecycleless => {
+    // node-musickit-api does not have typings
+    // need to make an interface from response reference here https://developer.apple.com/documentation/applemusicapi/get-v1-me-recent-played-tracks
+
+    const {newFromSource = false} = options;
+
+    const play: PlayObjectLifecycleless = {
+        data: {
+            // if object is new from source then we know we've picked it up AFTER we started polling so it can't be older than 1 minute (default polling interval)
+            playDate: newFromSource ? dayjs().startOf('minute') : undefined,
+        },
+        meta: {
+
+        }
+    }
+
+    throw new Error('Not implemented yet');
+    return baseFormatPlayObj(obj, play);
+}

src/backend/sources/ScrobbleSources.ts

@@ -43,6 +43,7 @@ import { RockskySourceConfig } from '../common/infrastructure/config/source/rock
 import { CommonSourceOptions } from '../common/infrastructure/config/source/index.js';
 import { ExternalMetadataTerm, PlayTransformHooks, PlayTransformOptions } from '../common/infrastructure/Transform.js';
 import { LibrefmSourceConfig } from '../common/infrastructure/config/source/librefm.js';
+import { AppleMusicSourceConfig } from '../common/infrastructure/config/source/apple.js';
 
 type groupedNamedConfigs = {[key: string]: ParsedConfig[]};
 
@@ -159,6 +160,8 @@ export default class ScrobbleSources {
                     return "RockskySourceConfig";
                 case 'sonos':
                     return 'SonosSourceConfig';
+                case 'apple':
+                    return 'AppleMusicSourceConfig';
             }
     }
 
@@ -1019,6 +1022,10 @@ export default class ScrobbleSources {
                 const {SonosSource} = (await import('./SonosSource.js'));
                 newSource = await new SonosSource(name, compositeConfig as SonosSourceConfig, this.internalConfig, this.emitter);
                 break;
+            case 'apple':
+                const {AppleSource} = (await import('./AppleSource.js'));
+                newSource = await new AppleSource(name, compositeConfig as AppleMusicSourceConfig, this.internalConfig, this.emitter);
+                break;
             default:
                 break;
         }

src/backend/utils.ts

@@ -25,19 +25,22 @@ import {
 //const { default: Ajv } = AjvNS;
 dayjs.extend(utc);
 
-export async function readText(path: any) {
-    await promises.access(path, constants.R_OK);
-    const data = await promises.readFile(path);
-    return data.toString();
-
-    // return new Promise((resolve, reject) => {
-    //     fs.readFile(path, 'utf8', function (err, data) {
-    //         if (err) {
-    //             reject(err);
-    //         }
-    //         resolve(JSON.parse(data));
-    //     });
-    // });
+export async function readText(path: any, {throwOnNotFound = true} = {}) {
+    try {
+        await promises.access(path, constants.R_OK);
+        const data = await promises.readFile(path);
+        return data.toString();
+    }  catch (e) {
+        const {code} = e;
+        if (code === 'ENOENT') {
+            if (throwOnNotFound) {
+                throw new Error(`No file found at given path: ${path}`, {cause: e});
+            } else {
+                return;
+            }
+        }
+        throw new Error(`Encountered error while parsing file: ${path}`, {cause: e})
+    }
 }
 
 export async function writeFile(path: any, text: any) {

src/core/Atomic.ts

@@ -36,7 +36,8 @@ export interface SourceStatusData {
     | 'koito'
     | 'tealfm'
     | 'rocksky'
-    | 'sonos';
+    | 'sonos'
+    | 'apple';
     display: string;
     tracksDiscovered: number;
     name: string;