fix(app): stop the service worker serving the unrendered index template (#1680)

fluxer_app/scripts/build/utils/ServiceWorker.tsx

@@ -2,15 +2,11 @@
 
 import {promises as fs} from 'node:fs';
 import * as path from 'node:path';
+import type {PrecacheEntry} from '@app/features/platform/service_worker/WorkerAppShell';
 import {DIST_DIR, SRC_DIR} from '@app_scripts/build/Config';
 import * as esbuild from 'esbuild';
 
-interface PrecacheEntry {
-	url: string;
-	revision: string;
-}
-
-const PRECACHE_ROOT_FILES = ['index.html', 'manifest.json', 'browserconfig.xml', 'robots.txt', 'version.json'];
+const PRECACHE_ROOT_FILES = ['manifest.json', 'browserconfig.xml', 'robots.txt', 'version.json'];
 
 const NEVER_PRECACHED_EXTENSIONS = ['.woff', '.woff2', '.ttf', '.otf', '.eot'];
 
@@ -38,11 +34,7 @@ async function collectPrecacheManifest(): Promise<Array<PrecacheEntry>> {
 	for (const file of PRECACHE_ROOT_FILES) {
 		const filePath = path.join(DIST_DIR, file);
 		try {
-			const revision = await fileRevision(filePath);
-			entries.set(`/${file}`, revision);
-			if (file === 'index.html') {
-				entries.set('/', revision);
-			}
+			entries.set(`/${file}`, await fileRevision(filePath));
 		} catch {}
 	}
 	try {

fluxer_app/src/features/platform/service_worker/Worker.ts

@@ -5,6 +5,14 @@ import {
 	getNotificationAlertOptions,
 	isMobileOrTabletUserAgent,
 } from '@app/features/platform/notifications/NotificationAlertOptions';
+import {
+	type AppShellRuntime,
+	fetchAppShellNavigation,
+	isCacheableResponse,
+	type PrecacheEntry,
+	precacheAssets,
+	seedAppShell,
+} from '@app/features/platform/service_worker/WorkerAppShell';
 import {shouldDeleteWorkerCache, WORKER_CACHE_PREFIX} from '@app/features/platform/service_worker/WorkerCacheCleanup';
 import {getWorkerFetchRoute} from '@app/features/platform/service_worker/WorkerFetchRouting';
 import {
@@ -27,11 +35,6 @@ declare const self: ServiceWorkerGlobalScope &
 		console: Console;
 	};
 
-interface PrecacheEntry {
-	url: string;
-	revision: string;
-}
-
 declare const __FLUXER_PRECACHE_MANIFEST__: ReadonlyArray<PrecacheEntry>;
 declare const __FLUXER_SW_VERSION__: string;
 const workerNavigator = self.navigator as {readonly userAgent: string; readonly maxTouchPoints?: number};
@@ -66,8 +69,16 @@ const log = async (level: SwLogLevel, message: string, data?: unknown): Promise<
 		}
 	} catch {}
 };
-const isCacheableResponse = (response: Response): boolean => {
-	return response.ok || response.type === 'opaque';
+const appShellRuntime: AppShellRuntime = {
+	caches: serviceWorkerCaches,
+	fetch: (request) => fetch(request),
+	origin: self.location.origin,
+	precacheName: PRECACHE_CACHE,
+	navigationCacheName: NAVIGATION_CACHE,
+	networkTimeoutMs: NAVIGATION_NETWORK_TIMEOUT_MS,
+	onCacheWriteError: (error) => {
+		void log('warn', 'app shell cache put failed', {error: describeError(error)});
+	},
 };
 const pruneCacheEntries = async (cache: Cache, maxEntries: number): Promise<void> => {
 	const keys = await cache.keys();
@@ -99,21 +110,6 @@ const cacheRequest = async (
 		await log('warn', 'cache put failed', {cacheName, error: describeError(error)});
 	}
 };
-const precacheAppShell = async (): Promise<void> => {
-	if (!serviceWorkerCaches) {
-		return;
-	}
-	const cache = await serviceWorkerCaches.open(PRECACHE_CACHE);
-	await Promise.allSettled(
-		PRECACHE_MANIFEST.map(async (entry) => {
-			const request = new Request(new URL(entry.url, self.location.origin).toString(), {cache: 'reload'});
-			const response = await fetch(request);
-			if (isCacheableResponse(response)) {
-				await cache.put(entry.url, response);
-			}
-		}),
-	);
-};
 const cleanupOldCaches = async (): Promise<void> => {
 	if (!serviceWorkerCaches) {
 		return;
@@ -128,42 +124,6 @@ const cleanupOldCaches = async (): Promise<void> => {
 		}),
 	);
 };
-const getCachedAppShell = async (): Promise<Response | undefined> => {
-	if (!serviceWorkerCaches) {
-		return undefined;
-	}
-	return (
-		(await serviceWorkerCaches.match('/index.html')) ??
-		(await serviceWorkerCaches.match('/')) ??
-		(await serviceWorkerCaches.match(new Request('/index.html', {cache: 'reload'}))) ??
-		undefined
-	);
-};
-const fetchNavigation = async (request: Request): Promise<Response> => {
-	const timeout = new Promise<Response | undefined>((resolve) => {
-		setTimeout(() => resolve(undefined), NAVIGATION_NETWORK_TIMEOUT_MS);
-	});
-	const network = fetch(request).then(async (response) => {
-		await cacheRequest(NAVIGATION_CACHE, '/index.html', response);
-		return response;
-	});
-	let networkResponse: Response | undefined;
-	try {
-		networkResponse = await Promise.race([network, timeout]);
-		if (networkResponse && isCacheableResponse(networkResponse)) {
-			return networkResponse;
-		}
-	} catch {}
-	const cached = await getCachedAppShell();
-	if (cached) {
-		network.catch(() => undefined);
-		return cached;
-	}
-	if (networkResponse) {
-		return networkResponse;
-	}
-	return network;
-};
 const fetchCacheFirst = async (request: Request): Promise<Response> => {
 	const cached = await serviceWorkerCaches?.match(request);
 	if (cached) {
@@ -191,7 +151,7 @@ self.addEventListener('install', (event: ExtendableEvent) => {
 	event.waitUntil(
 		(async () => {
 			await ensureServiceWorkerReady;
-			await precacheAppShell();
+			await Promise.allSettled([precacheAssets(appShellRuntime, PRECACHE_MANIFEST), seedAppShell(appShellRuntime)]);
 			await log('info', 'install');
 			self.skipWaiting();
 		})(),
@@ -214,7 +174,7 @@ self.addEventListener('fetch', (event: FetchEvent) => {
 	const request = event.request;
 	const route = getWorkerFetchRoute(request, self.location.origin);
 	if (route === 'navigation') {
-		event.respondWith(fetchNavigation(request));
+		event.respondWith(fetchAppShellNavigation(appShellRuntime, request));
 		return;
 	}
 	if (route === 'static-asset') {

fluxer_app/src/features/platform/service_worker/WorkerAppShell.test.ts

@@ -0,0 +1,196 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {
+	type AppShellRuntime,
+	fetchAppShellNavigation,
+	type PrecacheEntry,
+	precacheAssets,
+	seedAppShell,
+} from '@app/features/platform/service_worker/WorkerAppShell';
+import {beforeEach, describe, expect, it} from 'vitest';
+
+const WORKER_ORIGIN = 'https://self-hosted.fluxer.test';
+const PRECACHE_NAME = 'fluxer-precache-test';
+const NAVIGATION_CACHE_NAME = 'fluxer-navigation-test';
+const NETWORK_TIMEOUT_MS = 10;
+const SLOW_NETWORK_MS = 80;
+
+const UNRENDERED_INDEX_TEMPLATE = [
+	'<!doctype html><html lang="en"><head>',
+	'<link rel="preconnect" href="{{STATIC_CDN_ENDPOINT}}">',
+	'<link rel="apple-touch-icon" href="{{STATIC_CDN_ENDPOINT}}/web/apple-touch-icon.png">',
+	'<script nonce="{{CSP_NONCE_PLACEHOLDER}}"></script>',
+	'</head><body><div id="root"></div></body></html>',
+].join('');
+
+const RENDERED_INDEX_DOCUMENT = [
+	'<!doctype html><html lang="en"><head>',
+	'<link rel="preconnect" href="https://cdn.fluxer.test">',
+	'<script nonce="abc123">window.__FLUXER_BOOTSTRAP__={"instance":{}};</script>',
+	'</head><body><div id="root"></div></body></html>',
+].join('');
+
+const DEPLOYED_PRECACHE_MANIFEST: ReadonlyArray<PrecacheEntry> = [
+	{url: '/index.html', revision: '2757:1'},
+	{url: '/', revision: '2757:1'},
+	{url: '/assets/app.js', revision: '10:1'},
+];
+
+class FakeCache {
+	private readonly entries = new Map<string, Response>();
+
+	async put(request: Request | string, response: Response): Promise<void> {
+		this.entries.set(cacheKey(request), response);
+	}
+
+	async match(request: Request | string): Promise<Response | undefined> {
+		return this.entries.get(cacheKey(request))?.clone();
+	}
+
+	async delete(request: Request | string): Promise<boolean> {
+		return this.entries.delete(cacheKey(request));
+	}
+}
+
+class FakeCacheStorage {
+	private readonly caches = new Map<string, FakeCache>();
+
+	async open(name: string): Promise<FakeCache> {
+		const existing = this.caches.get(name);
+		if (existing) {
+			return existing;
+		}
+		const created = new FakeCache();
+		this.caches.set(name, created);
+		return created;
+	}
+
+	async match(request: Request | string): Promise<Response | undefined> {
+		for (const cache of this.caches.values()) {
+			const hit = await cache.match(request);
+			if (hit) {
+				return hit;
+			}
+		}
+		return undefined;
+	}
+
+	async keys(): Promise<Array<string>> {
+		return Array.from(this.caches.keys());
+	}
+}
+
+function cacheKey(request: Request | string): string {
+	const url = typeof request === 'string' ? request : request.url;
+	return new URL(url, WORKER_ORIGIN).toString();
+}
+
+function delay(milliseconds: number): Promise<void> {
+	return new Promise((resolve) => {
+		setTimeout(resolve, milliseconds);
+	});
+}
+
+function createAppProxyFetch(navigationDelayMs: number): (request: Request) => Promise<Response> {
+	return async (request: Request) => {
+		const {pathname} = new URL(request.url);
+		if (pathname === '/index.html') {
+			return new Response(UNRENDERED_INDEX_TEMPLATE, {headers: {'content-type': 'text/html; charset=utf-8'}});
+		}
+		if (pathname === '/assets/app.js') {
+			return new Response('console.log(1)', {headers: {'content-type': 'text/javascript'}});
+		}
+		await delay(navigationDelayMs);
+		return new Response(RENDERED_INDEX_DOCUMENT, {headers: {'content-type': 'text/html; charset=utf-8'}});
+	};
+}
+
+function navigationRequest(pathname: string): Request {
+	return new Request(`${WORKER_ORIGIN}${pathname}`, {headers: {accept: 'text/html'}});
+}
+
+describe('WorkerAppShell', () => {
+	let cacheStorage: FakeCacheStorage;
+
+	function createRuntime(navigationDelayMs: number): AppShellRuntime {
+		return {
+			caches: cacheStorage as unknown as CacheStorage,
+			fetch: createAppProxyFetch(navigationDelayMs),
+			origin: WORKER_ORIGIN,
+			precacheName: PRECACHE_NAME,
+			navigationCacheName: NAVIGATION_CACHE_NAME,
+			networkTimeoutMs: NETWORK_TIMEOUT_MS,
+			onCacheWriteError: (error) => {
+				throw error;
+			},
+		};
+	}
+
+	beforeEach(() => {
+		cacheStorage = new FakeCacheStorage();
+	});
+
+	it('never serves the app-proxy index template that was fetched outside the bootstrap render', async () => {
+		const runtime = createRuntime(SLOW_NETWORK_MS);
+		await precacheAssets(runtime, DEPLOYED_PRECACHE_MANIFEST);
+
+		const response = await fetchAppShellNavigation(runtime, navigationRequest('/channels/1234567890/9876543210'));
+		const html = await response.text();
+
+		expect(html).toContain('window.__FLUXER_BOOTSTRAP__');
+		expect(html).not.toContain('{{STATIC_CDN_ENDPOINT}}');
+	});
+
+	it('replays the previously rendered document when the network loses the navigation race', async () => {
+		const runtime = createRuntime(0);
+		await precacheAssets(runtime, DEPLOYED_PRECACHE_MANIFEST);
+		await fetchAppShellNavigation(runtime, navigationRequest('/channels/@me'));
+
+		const slowRuntime = createRuntime(SLOW_NETWORK_MS);
+		const response = await fetchAppShellNavigation(slowRuntime, navigationRequest('/channels/1234567890/9876543210'));
+		const html = await response.text();
+
+		expect(html).toBe(RENDERED_INDEX_DOCUMENT);
+	});
+
+	it('keeps documents out of the precache', async () => {
+		const runtime = createRuntime(0);
+		await precacheAssets(runtime, DEPLOYED_PRECACHE_MANIFEST);
+
+		const precache = await cacheStorage.open(PRECACHE_NAME);
+
+		expect(await precache.match('/index.html')).toBeUndefined();
+		expect(await precache.match('/')).toBeUndefined();
+		expect(await precache.match('/assets/app.js')).toBeDefined();
+	});
+	it('serves the seeded app shell when the network is unavailable', async () => {
+		await seedAppShell(createRuntime(0));
+
+		const offlineRuntime: AppShellRuntime = {
+			...createRuntime(0),
+			fetch: () => Promise.reject(new Error('Failed to fetch')),
+		};
+		const response = await fetchAppShellNavigation(offlineRuntime, navigationRequest('/channels/@me'));
+
+		expect(await response.text()).toBe(RENDERED_INDEX_DOCUMENT);
+	});
+
+	it('ignores an unrendered template left in the precache by an older worker', async () => {
+		const precache = await cacheStorage.open(PRECACHE_NAME);
+		await precache.put(
+			'/index.html',
+			new Response(UNRENDERED_INDEX_TEMPLATE, {headers: {'content-type': 'text/html; charset=utf-8'}}),
+		);
+		await precache.put(
+			'/',
+			new Response(UNRENDERED_INDEX_TEMPLATE, {headers: {'content-type': 'text/html; charset=utf-8'}}),
+		);
+		await seedAppShell(createRuntime(0));
+
+		const response = await fetchAppShellNavigation(createRuntime(SLOW_NETWORK_MS), navigationRequest('/channels/1/2'));
+		const html = await response.text();
+
+		expect(html).toContain('window.__FLUXER_BOOTSTRAP__');
+		expect(html).not.toContain('{{STATIC_CDN_ENDPOINT}}');
+	});
+});

fluxer_app/src/features/platform/service_worker/WorkerAppShell.ts

@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+const APP_SHELL_CACHE_KEY = '/app-shell';
+const APP_SHELL_SEED_PATH = '/';
+
+export interface PrecacheEntry {
+	readonly url: string;
+	readonly revision: string;
+}
+
+export interface AppShellRuntime {
+	readonly caches: CacheStorage | undefined;
+	readonly fetch: (request: Request) => Promise<Response>;
+	readonly origin: string;
+	readonly precacheName: string;
+	readonly navigationCacheName: string;
+	readonly networkTimeoutMs: number;
+	readonly onCacheWriteError: (error: unknown) => void;
+}
+
+export function isCacheableResponse(response: Response): boolean {
+	return response.ok || response.type === 'opaque';
+}
+
+export function isPrecacheableAssetUrl(url: string): boolean {
+	const pathname = url.split(/[?#]/, 1)[0].toLowerCase();
+	return pathname !== '/' && !pathname.endsWith('.html');
+}
+
+export async function precacheAssets(runtime: AppShellRuntime, manifest: ReadonlyArray<PrecacheEntry>): Promise<void> {
+	if (!runtime.caches) {
+		return;
+	}
+	const cache = await runtime.caches.open(runtime.precacheName);
+	await Promise.allSettled(
+		manifest
+			.filter((entry) => isPrecacheableAssetUrl(entry.url))
+			.map(async (entry) => {
+				const request = new Request(new URL(entry.url, runtime.origin).toString(), {cache: 'reload'});
+				const response = await runtime.fetch(request);
+				if (isCacheableResponse(response)) {
+					await cache.put(entry.url, response);
+				}
+			}),
+	);
+}
+
+async function readAppShell(runtime: AppShellRuntime): Promise<Response | undefined> {
+	if (!runtime.caches) {
+		return undefined;
+	}
+	const cache = await runtime.caches.open(runtime.navigationCacheName);
+	return (await cache.match(APP_SHELL_CACHE_KEY)) ?? undefined;
+}
+
+async function storeAppShell(runtime: AppShellRuntime, response: Response): Promise<void> {
+	if (!runtime.caches || !isCacheableResponse(response)) {
+		return;
+	}
+	try {
+		const cache = await runtime.caches.open(runtime.navigationCacheName);
+		await cache.put(APP_SHELL_CACHE_KEY, response.clone());
+	} catch (error) {
+		runtime.onCacheWriteError(error);
+	}
+}
+
+export async function seedAppShell(runtime: AppShellRuntime): Promise<void> {
+	if (!runtime.caches) {
+		return;
+	}
+	const request = new Request(new URL(APP_SHELL_SEED_PATH, runtime.origin).toString(), {cache: 'reload'});
+	const response = await runtime.fetch(request);
+	await storeAppShell(runtime, response);
+}
+
+export async function fetchAppShellNavigation(runtime: AppShellRuntime, request: Request): Promise<Response> {
+	const timeout = new Promise<Response | undefined>((resolve) => {
+		setTimeout(() => resolve(undefined), runtime.networkTimeoutMs);
+	});
+	const network = runtime.fetch(request).then(async (response) => {
+		await storeAppShell(runtime, response);
+		return response;
+	});
+	let networkResponse: Response | undefined;
+	try {
+		networkResponse = await Promise.race([network, timeout]);
+		if (networkResponse && isCacheableResponse(networkResponse)) {
+			return networkResponse;
+		}
+	} catch {}
+	const cached = await readAppShell(runtime);
+	if (cached) {
+		network.catch(() => undefined);
+		return cached;
+	}
+	if (networkResponse) {
+		return networkResponse;
+	}
+	return network;
+}

fluxer_app_proxy/src/routes/spa_index.rs

@@ -19,9 +19,7 @@ use axum::{
 use std::path::Path;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
-use super::spa_static::{
-    CORS_ALLOW_ANY_VALUE, guess_mime, is_font_mime, is_hashed_asset, is_static_asset,
-};
+use super::spa_static::{CORS_ALLOW_ANY_VALUE, guess_mime, is_font_mime, is_hashed_asset};
 
 const ACCEPT_CH_VALUE: &str = "DPR, Sec-CH-DPR, Sec-CH-Width, Save-Data, ECT, Downlink";
 const CRITICAL_CH_VALUE: &str = "Sec-CH-DPR, Sec-CH-Width, Save-Data";
@@ -34,13 +32,21 @@ pub async fn spa_catch_all(
 ) -> Response {
     let request_path = request.uri().path();
 
-    if is_static_asset(request_path) {
+    if is_static_root_file(request_path) {
         return serve_static_file(&state.config.static_dir, request_path).await;
     }
 
     serve_spa_index(&state, &headers, request_path).await
 }
 
+const STATIC_ROOT_FILES: &[&str] = &["/robots.txt"];
+
+fn is_static_root_file(request_path: &str) -> bool {
+    STATIC_ROOT_FILES
+        .iter()
+        .any(|candidate| request_path.eq_ignore_ascii_case(candidate))
+}
+
 async fn serve_static_file(static_dir: &str, request_path: &str) -> Response {
     let file_path = Path::new(static_dir).join(request_path.trim_start_matches('/'));
 
@@ -503,6 +509,20 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn only_declared_static_root_files_bypass_the_spa_document() {
+        assert!(is_static_root_file("/robots.txt"));
+        assert!(!is_static_root_file("/index.html"));
+        assert!(!is_static_root_file("/channels/@me"));
+    }
+
+    #[test]
+    fn spa_routes_containing_a_dot_still_render_the_document() {
+        assert!(!is_static_root_file("/theme/my.custom.theme"));
+        assert!(!is_static_root_file("/invite/abc.def"));
+        assert!(!is_static_root_file("/users/1.2.3"));
+    }
+
     #[test]
     fn font_mime_types_are_cors_enabled() {
         assert!(is_font_mime("font/woff2"));

fluxer_app_proxy/src/routes/spa_static.rs

@@ -240,11 +240,6 @@ pub fn is_font_mime(mime_type: &str) -> bool {
     )
 }
 
-pub fn is_static_asset(path: &str) -> bool {
-    let filename = path.rsplit('/').next().unwrap_or(path);
-    filename.contains('.')
-}
-
 pub fn is_hashed_asset(path: &str) -> bool {
     let filename = path.rsplit('/').next().unwrap_or(path);
     let Some(last_dot) = filename.rfind('.') else {
@@ -337,18 +332,6 @@ mod tests {
         assert_eq!(guess_mime("F.JS"), "application/javascript; charset=utf-8");
     }
 
-    #[test]
-    fn static_asset_with_ext() {
-        assert!(is_static_asset("/assets/app.js"));
-        assert!(is_static_asset("style.css"));
-    }
-
-    #[test]
-    fn static_asset_without_ext() {
-        assert!(!is_static_asset("/channels/me"));
-        assert!(!is_static_asset("/login"));
-    }
-
     #[test]
     fn hashed_asset_positive() {
         assert!(is_hashed_asset("app.a1b2c3d4.js"));