diff --git a/.github/workflows/build-desktop.yaml b/.github/workflows/build-desktop.yaml index 60364144d..034da42e7 100644 --- a/.github/workflows/build-desktop.yaml +++ b/.github/workflows/build-desktop.yaml @@ -560,6 +560,12 @@ jobs: cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop --step prepare_release_assets + - name: Publish GitHub release descriptor + if: needs.meta.outputs.test_build != 'true' + run: >- + cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop + --step publish_release_descriptor + - name: Upload payload to S3 run: >- cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop @@ -647,3 +653,10 @@ jobs: release_args+=(--prerelease) fi cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- "${release_args[@]}" + + - name: Publish GitHub release readiness marker + env: + SOURCE_SHA: ${{ needs.meta.outputs.source_sha }} + run: >- + cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop + --step publish_release_marker diff --git a/fluxer_api/pkgs/geoip/src/GeoipLookup.ts b/fluxer_api/pkgs/geoip/src/GeoipLookup.ts index 6560dfc35..ea1553b04 100644 --- a/fluxer_api/pkgs/geoip/src/GeoipLookup.ts +++ b/fluxer_api/pkgs/geoip/src/GeoipLookup.ts @@ -23,6 +23,7 @@ type CacheEntry = { }; const CACHE_TTL_MS = 10 * 60 * 1000; +const CACHE_MAX_ENTRIES = 10_000; const geoipCache = new Map(); let maxmindReader: Reader | null = null; @@ -85,6 +86,32 @@ function isAsciiUpperAlpha2(value: string): boolean { ); } +function getCachedGeoipResult(cacheKey: string, normalizedIp: string): GeoipResult | null { + const cached = geoipCache.get(cacheKey); + if (!cached) { + return null; + } + if (Date.now() >= cached.expiresAt) { + geoipCache.delete(cacheKey); + return null; + } + geoipCache.delete(cacheKey); + geoipCache.set(cacheKey, cached); + return {...cached.result, normalizedIp}; +} + +function setCachedGeoipResult(cacheKey: string, result: GeoipResult): void { + geoipCache.delete(cacheKey); + if (geoipCache.size >= CACHE_MAX_ENTRIES) { + const oldestKey = geoipCache.keys().next().value; + if (oldestKey === undefined) { + throw new Error('GeoIP cache reached capacity without an entry to evict'); + } + geoipCache.delete(oldestKey); + } + geoipCache.set(cacheKey, {result, expiresAt: Date.now() + CACHE_TTL_MS}); +} + async function lookupMaxmind(clean: string, dbPath: string): Promise { try { const reader = await ensureReader(dbPath); @@ -108,14 +135,13 @@ async function lookupMaxmind(clean: string, dbPath: string): Promise { - const now = Date.now(); const cacheKey = getSameIpDecisionKey(clean) ?? clean; - const cached = geoipCache.get(cacheKey); - if (cached && now < cached.expiresAt) { - return {...cached.result, normalizedIp: clean}; + const cached = getCachedGeoipResult(cacheKey, clean); + if (cached) { + return cached; } const result = await lookupMaxmind(clean, dbPath); - geoipCache.set(cacheKey, {result, expiresAt: now + CACHE_TTL_MS}); + setCachedGeoipResult(cacheKey, result); return result; } diff --git a/fluxer_api/src/api/Config.ts b/fluxer_api/src/api/Config.ts index 8f6a8faae..5111087ff 100644 --- a/fluxer_api/src/api/Config.ts +++ b/fluxer_api/src/api/Config.ts @@ -85,6 +85,18 @@ function normalizeIpBanExemptIps(values: Array): Array { return Array.from(normalized); } +function normalizeCountryCodes(values: Array, configName: string): ReadonlySet { + const normalized = new Set(); + for (const value of values) { + const countryCode = value.trim().toUpperCase(); + if (!/^[A-Z]{2}$/u.test(countryCode)) { + throw new Error(`${configName} contains an invalid ISO 3166-1 alpha-2 country code: ${value}`); + } + normalized.add(countryCode); + } + return normalized; +} + function mapPushProviderApps( apps: | Array<{ @@ -142,6 +154,10 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { nodeEnv: master.env === 'test' ? 'development' : master.env, port: master.services.api.port, ipBanExemptIps: normalizeIpBanExemptIps(master.services.api.ip_ban_exempt_ips), + desktopGitHubRedirectCountries: normalizeCountryCodes( + master.services.api.desktop_github_redirect_countries, + 'FLUXER_API_DESKTOP_GITHUB_REDIRECT_COUNTRIES', + ), cassandra: { hosts: cassandraSource?.hosts.join(',') ?? '', port: cassandraSource?.port ?? 9042, diff --git a/fluxer_api/src/api/config/APIConfig.ts b/fluxer_api/src/api/config/APIConfig.ts index 487aea7f8..10184a097 100644 --- a/fluxer_api/src/api/config/APIConfig.ts +++ b/fluxer_api/src/api/config/APIConfig.ts @@ -35,6 +35,7 @@ export interface APIConfig { nodeEnv: 'development' | 'production'; port: number; ipBanExemptIps: Array; + desktopGitHubRedirectCountries: ReadonlySet; cassandra: { hosts: string; port: number; diff --git a/fluxer_api/src/api/download/DesktopReleaseContract.ts b/fluxer_api/src/api/download/DesktopReleaseContract.ts new file mode 100644 index 000000000..ca3941e83 --- /dev/null +++ b/fluxer_api/src/api/download/DesktopReleaseContract.ts @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {DesktopArch, DesktopChannel, DesktopPlatform} from '@fluxer/schema/src/domains/download/DownloadSchemas'; +import {isJsonRecord} from '../utils/JsonBoundaryUtils'; + +const DESKTOP_BUCKET_PREFIX = 'desktop'; +const MIN_RELEASE_ROUTE_COUNT = 28; +const MAX_RELEASE_ROUTE_COUNT = 128; +const MIN_RELEASE_ASSET_COUNT = 26; + +interface DesktopReleaseAsset { + storage_key: string; + release_asset: string; + sha256: string; + size: number; +} + +interface DesktopReleaseDescriptor { + schema_version: 1; + channel: DesktopChannel; + version: string; + release_tag: string; + source_sha: string; + assets: Array; +} + +interface DesktopReleaseReadiness { + schema_version: 1; + channel: DesktopChannel; + version: string; + release_tag: string; + source_sha: string; + descriptor_sha256: string; +} + +interface DesktopArtifactScope { + channel: DesktopChannel; + plat: DesktopPlatform; + arch: DesktopArch; +} + +export function parseDesktopArtifactScope(key: string): DesktopArtifactScope | null { + const segments = key.split('/'); + if (segments.length !== 5 || segments[0] !== DESKTOP_BUCKET_PREFIX || segments[4].length === 0) { + return null; + } + const [, channel, plat, arch] = segments; + if ( + (channel !== 'stable' && channel !== 'canary') || + (plat !== 'win32' && plat !== 'darwin' && plat !== 'linux') || + (arch !== 'x64' && arch !== 'arm64') + ) { + return null; + } + return {channel, plat, arch}; +} + +function parseDesktopReleaseAsset(value: unknown): DesktopReleaseAsset | null { + if ( + !isJsonRecord(value) || + typeof value.storage_key !== 'string' || + typeof value.release_asset !== 'string' || + typeof value.sha256 !== 'string' || + typeof value.size !== 'number' + ) { + return null; + } + if ( + !/^desktop\/(?:stable|canary)\/(?:win32|darwin|linux)\/(?:x64|arm64)\/[A-Za-z0-9._-]+$/u.test(value.storage_key) || + !/^[A-Za-z0-9._-]+$/u.test(value.release_asset) || + !/^[a-f0-9]{64}$/u.test(value.sha256) || + !Number.isSafeInteger(value.size) || + value.size <= 0 + ) { + return null; + } + return { + storage_key: value.storage_key, + release_asset: value.release_asset, + sha256: value.sha256, + size: value.size, + }; +} + +export function parseDesktopReleaseDescriptor(value: unknown): DesktopReleaseDescriptor | null { + if ( + !isJsonRecord(value) || + value.schema_version !== 1 || + (value.channel !== 'stable' && value.channel !== 'canary') || + typeof value.version !== 'string' || + !/^\d+\.\d+\.\d+$/u.test(value.version) || + typeof value.release_tag !== 'string' || + typeof value.source_sha !== 'string' || + !/^[a-f0-9]{40}$/u.test(value.source_sha) || + !Array.isArray(value.assets) || + value.assets.length < MIN_RELEASE_ROUTE_COUNT || + value.assets.length > MAX_RELEASE_ROUTE_COUNT + ) { + return null; + } + const expectedTag = `fluxer-desktop-${value.channel}@${value.version}`; + const expectedStoragePrefix = `desktop/${value.channel}/`; + const expectedReleasePrefix = `${value.channel === 'canary' ? 'Fluxer-Canary' : 'Fluxer'}-${value.version}-`; + if (value.release_tag !== expectedTag) { + return null; + } + const storageKeys = new Set(); + const routeCounts = new Map(); + const releaseAssets = new Map(); + const assets: Array = []; + for (const rawAsset of value.assets) { + const asset = parseDesktopReleaseAsset(rawAsset); + if ( + !asset || + !asset.storage_key.startsWith(expectedStoragePrefix) || + !asset.release_asset.startsWith(expectedReleasePrefix) || + storageKeys.has(asset.storage_key) + ) { + return null; + } + storageKeys.add(asset.storage_key); + const [, , platform, arch, filename] = asset.storage_key.split('/'); + const platformToken = platform === 'win32' ? 'win' : platform === 'darwin' ? 'mac' : 'linux'; + const expectedReleaseAsset = filename.startsWith(expectedReleasePrefix) + ? filename + : `${expectedReleasePrefix}${platformToken}-${arch}-${filename}`; + if (asset.release_asset !== expectedReleaseAsset) { + return null; + } + const scope = `${platform}/${arch}`; + routeCounts.set(scope, (routeCounts.get(scope) ?? 0) + 1); + const existing = releaseAssets.get(asset.release_asset); + if (existing && (existing.sha256 !== asset.sha256 || existing.size !== asset.size)) { + return null; + } + releaseAssets.set(asset.release_asset, {sha256: asset.sha256, size: asset.size}); + assets.push(asset); + } + if (releaseAssets.size < MIN_RELEASE_ASSET_COUNT || releaseAssets.size > MAX_RELEASE_ROUTE_COUNT) { + return null; + } + const expectedRouteCounts = new Map([ + ['darwin/arm64', 4], + ['darwin/x64', 4], + ['linux/arm64', 4], + ['linux/x64', 4], + ['win32/arm64', 6], + ['win32/x64', 6], + ]); + if ( + routeCounts.size !== expectedRouteCounts.size || + Array.from(expectedRouteCounts).some(([scope, count]) => (routeCounts.get(scope) ?? 0) < count) + ) { + return null; + } + return { + schema_version: 1, + channel: value.channel, + version: value.version, + release_tag: value.release_tag, + source_sha: value.source_sha, + assets, + }; +} + +export function parseDesktopReleaseReadiness(value: unknown): DesktopReleaseReadiness | null { + if ( + !isJsonRecord(value) || + value.schema_version !== 1 || + (value.channel !== 'stable' && value.channel !== 'canary') || + typeof value.version !== 'string' || + !/^\d+\.\d+\.\d+$/u.test(value.version) || + typeof value.release_tag !== 'string' || + typeof value.source_sha !== 'string' || + !/^[a-f0-9]{40}$/u.test(value.source_sha) || + typeof value.descriptor_sha256 !== 'string' || + !/^[a-f0-9]{64}$/u.test(value.descriptor_sha256) + ) { + return null; + } + return { + schema_version: 1, + channel: value.channel, + version: value.version, + release_tag: value.release_tag, + source_sha: value.source_sha, + descriptor_sha256: value.descriptor_sha256, + }; +} diff --git a/fluxer_api/src/api/download/DownloadController.ts b/fluxer_api/src/api/download/DownloadController.ts index 7429e3a6b..61a8f0448 100644 --- a/fluxer_api/src/api/download/DownloadController.ts +++ b/fluxer_api/src/api/download/DownloadController.ts @@ -17,6 +17,7 @@ import {Config} from '../Config'; import {OpenAPI} from '../middleware/ResponseTypeMiddleware'; import type {HonoEnv} from '../types/HonoEnv'; import {Validator} from '../Validator'; +import {resolveArtifactRoute} from './DownloadRouting'; import type {DesktopChecksumFile, DownloadService, DownloadStreamResult} from './DownloadService'; import { DESKTOP_REDIRECT_PREFIX, @@ -29,6 +30,17 @@ function artifactFilename(key: string, filenameOverride?: string): string { return filenameOverride ?? key.split('/').pop() ?? 'download'; } +function artifactRedirectResponse(location: string, cacheControl = 'no-store'): Response { + return new Response(null, { + status: 302, + headers: new Headers({ + Location: location, + 'Cache-Control': cacheControl, + 'Accept-Ranges': 'bytes', + }), + }); +} + function setCommonArtifactHeaders( headers: Headers, key: string, @@ -87,8 +99,12 @@ async function streamArtifactResponse( cacheControl: string, filenameOverride?: string, ): Promise { + const route = await resolveArtifactRoute({request: ctx.req.raw, downloadService, key, cacheControl}); + if (route.kind === 'redirect') { + return artifactRedirectResponse(route.location, route.cacheControl); + } if (ctx.req.method === 'HEAD') { - return headArtifactResponse(ctx, downloadService, key, cacheControl, filenameOverride); + return headArtifactResponse(ctx, downloadService, key, route.cacheControl, filenameOverride); } if (downloadService.isPresignedDownloadEnabled()) { const location = await downloadService.getPresignedDownloadRedirect({ @@ -99,14 +115,7 @@ async function streamArtifactResponse( if (!location) { return ctx.text('Not Found', 404); } - return new Response(null, { - status: 302, - headers: new Headers({ - Location: location, - 'Cache-Control': 'no-store', - 'Accept-Ranges': 'bytes', - }), - }); + return artifactRedirectResponse(location); } const range = ctx.req.header('range') ?? undefined; let result: DownloadStreamResult | null; @@ -117,7 +126,7 @@ async function streamArtifactResponse( const headers = new Headers(); headers.set('Accept-Ranges', 'bytes'); headers.set('Content-Range', `bytes */${error.totalSize}`); - headers.set('Cache-Control', cacheControl); + headers.set('Cache-Control', route.cacheControl); return new Response(null, {status: 416, headers}); } throw error; @@ -129,7 +138,7 @@ async function streamArtifactResponse( setCommonArtifactHeaders( headers, key, - cacheControl, + route.cacheControl, filenameOverride, result.contentType, result.contentDisposition, diff --git a/fluxer_api/src/api/download/DownloadRouting.ts b/fluxer_api/src/api/download/DownloadRouting.ts new file mode 100644 index 000000000..e2e933c55 --- /dev/null +++ b/fluxer_api/src/api/download/DownloadRouting.ts @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {Config} from '../Config'; +import {Logger} from '../Logger'; +import {lookupGeoip} from '../utils/IpUtils'; +import {parseDesktopArtifactScope} from './DesktopReleaseContract'; +import type {DownloadService, GitHubDesktopReleaseResolution} from './DownloadService'; + +const COUNTRY_DEPENDENT_CACHE_CONTROL = 'private, no-store'; + +type ArtifactRoute = + | {kind: 'storage'; cacheControl: string} + | {kind: 'redirect'; cacheControl: string; location: string}; + +export async function resolveArtifactRoute(params: { + request: Request; + downloadService: DownloadService; + key: string; + cacheControl: string; +}): Promise { + if (Config.instance.selfHosted) { + return {kind: 'storage', cacheControl: params.cacheControl}; + } + if (Config.desktopGitHubRedirectCountries.size === 0) { + return {kind: 'storage', cacheControl: params.cacheControl}; + } + if (!parseDesktopArtifactScope(params.key)) { + return {kind: 'storage', cacheControl: params.cacheControl}; + } + const geoip = await lookupGeoip(params.request); + const countryCode = geoip.countryCode?.trim().toUpperCase(); + if (!countryCode || !Config.desktopGitHubRedirectCountries.has(countryCode)) { + return {kind: 'storage', cacheControl: COUNTRY_DEPENDENT_CACHE_CONTROL}; + } + let release: GitHubDesktopReleaseResolution; + try { + release = await params.downloadService.resolveGitHubDesktopRelease(params.key); + } catch (error) { + Logger.error({error, key: params.key}, 'Failed to resolve GitHub desktop download route'); + return {kind: 'storage', cacheControl: COUNTRY_DEPENDENT_CACHE_CONTROL}; + } + if (release.kind === 'not_current') { + return {kind: 'storage', cacheControl: COUNTRY_DEPENDENT_CACHE_CONTROL}; + } + if (release.kind === 'ready') { + return { + kind: 'redirect', + cacheControl: COUNTRY_DEPENDENT_CACHE_CONTROL, + location: release.location, + }; + } + return {kind: 'storage', cacheControl: COUNTRY_DEPENDENT_CACHE_CONTROL}; +} diff --git a/fluxer_api/src/api/download/DownloadService.ts b/fluxer_api/src/api/download/DownloadService.ts index 59ad385b6..10a138642 100644 --- a/fluxer_api/src/api/download/DownloadService.ts +++ b/fluxer_api/src/api/download/DownloadService.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {createHash} from 'node:crypto'; import {posix} from 'node:path'; import {Readable} from 'node:stream'; import {S3ServiceException} from '@aws-sdk/client-s3'; @@ -12,6 +13,11 @@ import type { import {Config} from '../Config'; import type {IStorageService} from '../infrastructure/IStorageService'; import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils'; +import { + parseDesktopArtifactScope, + parseDesktopReleaseDescriptor, + parseDesktopReleaseReadiness, +} from './DesktopReleaseContract'; export const DOWNLOAD_PREFIX = '/dl'; export const DESKTOP_REDIRECT_PREFIX = `${DOWNLOAD_PREFIX}/desktop`; @@ -46,6 +52,8 @@ function isUnsatisfiableRangeError(error: unknown): boolean { const DESKTOP_BUCKET_PREFIX = 'desktop'; const DESKTOP_TEST_BUCKET_PREFIX = 'desktop-test'; const DEFAULT_API_CLIENT_BASE_URL = 'https://api.fluxer.app'; +const GITHUB_RELEASE_DOWNLOAD_BASE_URL = 'https://github.com/fluxerapp/fluxer/releases/download'; +const GITHUB_RELEASE_MARKER_DIRECTORY = 'github-releases'; function desktopBucketPrefix(test?: boolean): string { return test ? DESKTOP_TEST_BUCKET_PREFIX : DESKTOP_BUCKET_PREFIX; @@ -176,9 +184,70 @@ interface ManifestFilenameResolutionParams extends LatestFilenameLookupParams { filename: string; } +export type GitHubDesktopReleaseResolution = + | {kind: 'not_current'} + | {kind: 'awaiting_release'} + | {kind: 'ready'; location: string}; + export class DownloadService { constructor(private readonly storageService: IStorageService) {} + async resolveGitHubDesktopRelease(key: string): Promise { + const scope = parseDesktopArtifactScope(key); + if (!scope) { + return {kind: 'not_current'}; + } + const manifestKey = `${DESKTOP_BUCKET_PREFIX}/${scope.channel}/${scope.plat}/${scope.arch}/manifest.json`; + const manifest = await this.readOptionalJsonObjectFromStorage(manifestKey); + if ( + !isDesktopManifest(manifest) || + manifest.channel !== scope.channel || + manifest.platform !== scope.plat || + manifest.arch !== scope.arch + ) { + return {kind: 'not_current'}; + } + const descriptorKey = `${DESKTOP_BUCKET_PREFIX}/${scope.channel}/${GITHUB_RELEASE_MARKER_DIRECTORY}/${manifest.version}.json`; + const descriptorText = await this.readOptionalTextFromStorage(descriptorKey); + if (descriptorText == null) { + return {kind: 'not_current'}; + } + const descriptor = parseDesktopReleaseDescriptor(parseJsonUnknown(descriptorText)); + if ( + !descriptor || + descriptor.channel !== scope.channel || + descriptor.version !== manifest.version || + descriptor.release_tag !== `fluxer-desktop-${scope.channel}@${manifest.version}` + ) { + throw new Error(`Invalid GitHub desktop release descriptor: ${descriptorKey}`); + } + const releaseAsset = descriptor.assets.find((asset) => asset.storage_key === key); + if (!releaseAsset) { + return {kind: 'not_current'}; + } + const markerKey = `${DESKTOP_BUCKET_PREFIX}/${scope.channel}/${GITHUB_RELEASE_MARKER_DIRECTORY}/${manifest.version}.ready.json`; + const marker = await this.readOptionalJsonObjectFromStorage(markerKey); + if (marker == null) { + return {kind: 'awaiting_release'}; + } + const readiness = parseDesktopReleaseReadiness(marker); + const descriptorSha256 = createHash('sha256').update(descriptorText).digest('hex'); + if ( + !readiness || + readiness.channel !== descriptor.channel || + readiness.version !== descriptor.version || + readiness.release_tag !== descriptor.release_tag || + readiness.source_sha !== descriptor.source_sha || + readiness.descriptor_sha256 !== descriptorSha256 + ) { + throw new Error(`Invalid GitHub desktop release readiness marker: ${markerKey}`); + } + return { + kind: 'ready', + location: `${GITHUB_RELEASE_DOWNLOAD_BASE_URL}/${encodeURIComponent(descriptor.release_tag)}/${encodeURIComponent(releaseAsset.release_asset)}`, + }; + } + async resolveLatestDesktopKey(params: { channel: DesktopChannel; plat: DesktopPlatform; @@ -659,6 +728,11 @@ export class DownloadService { } private async readJsonObjectFromStorage(key: string): Promise { + const text = await this.readTextFromStorage(key); + return text == null ? null : parseJsonUnknown(text); + } + + private async readTextFromStorage(key: string): Promise { const streamResult = await this.storageService.streamObject({ bucket: Config.s3.buckets.downloads, key, @@ -667,8 +741,29 @@ export class DownloadService { return null; } const body = Readable.toWeb(streamResult.body); - const text = await new Response(body as ReadableStream).text(); - return parseJsonUnknown(text); + return new Response(body as ReadableStream).text(); + } + + private async readOptionalJsonObjectFromStorage(key: string): Promise { + try { + return await this.readJsonObjectFromStorage(key); + } catch (error) { + if (isStorageNotFoundError(error)) { + return null; + } + throw error; + } + } + + private async readOptionalTextFromStorage(key: string): Promise { + try { + return await this.readTextFromStorage(key); + } catch (error) { + if (isStorageNotFoundError(error)) { + return null; + } + throw error; + } } private isValidSha256(value: string): boolean { @@ -774,9 +869,11 @@ export class DownloadService { const {ext, arch: archMap} = mapping; const filenames = new Set(); for (const archSuffix of this.getArchTokens(archMap[arch as 'x64' | 'arm64'])) { - const modernFilename = this.buildModernArtifactFilename(channel, version, plat, archSuffix, ext); - if (modernFilename) { - filenames.add(modernFilename); + for (const productName of this.getModernProductNames(channel)) { + filenames.add(`${productName}-${version}-${MODERN_PLATFORM_TOKENS[plat]}-${archSuffix}${ext}`); + if (format === 'portable') { + filenames.add(`${productName}-${version}-portable-${MODERN_PLATFORM_TOKENS[plat]}-${archSuffix}${ext}`); + } } if (format === 'setup') { filenames.add(`fluxer-${channel}-${version}-${archSuffix}-setup${ext}`); @@ -784,9 +881,6 @@ export class DownloadService { filenames.add(`fluxer-${version}-${archSuffix}-setup${ext}`); filenames.add(`Fluxer-${version}-${archSuffix}-Setup${ext}`); } else if (format === 'portable') { - filenames.add( - `${this.getModernProductName(channel)}-${version}-portable-${MODERN_PLATFORM_TOKENS[plat]}-${archSuffix}${ext}`, - ); filenames.add(`fluxer-${channel}-${version}-portable-${archSuffix}${ext}`); filenames.add(`Fluxer-${version}-portable-${archSuffix}${ext}`); } else { @@ -816,7 +910,6 @@ export class DownloadService { } const {ext, arch: archMap} = mapping; const escapedExt = this.escapeRegex(ext); - const escapedModernFilenamePrefix = this.escapeRegex(this.getModernProductName(channel)); const modernPlatformToken = MODERN_PLATFORM_TOKENS[plat]; for (const archSuffix of this.getArchTokens(archMap[arch as 'x64' | 'arm64'])) { const patterns = [ @@ -828,18 +921,23 @@ export class DownloadService { `^[Ff]luxer-(\\d+\\.\\d+\\.\\d+)-${this.escapeRegex(archSuffix)}(?:-[Ss]etup)?${escapedExt}$`, 'u', ), - new RegExp( - `^${escapedModernFilenamePrefix}-(\\d+\\.\\d+\\.\\d+)-${this.escapeRegex(modernPlatformToken)}-${this.escapeRegex(archSuffix)}${escapedExt}$`, - 'iu', - ), ]; - if (format === 'portable') { + for (const productName of this.getModernProductNames(channel)) { + const escapedProductName = this.escapeRegex(productName); patterns.push( new RegExp( - `^${escapedModernFilenamePrefix}-(\\d+\\.\\d+\\.\\d+)-portable-${this.escapeRegex(modernPlatformToken)}-${this.escapeRegex(archSuffix)}${escapedExt}$`, + `^${escapedProductName}-(\\d+\\.\\d+\\.\\d+)-${this.escapeRegex(modernPlatformToken)}-${this.escapeRegex(archSuffix)}${escapedExt}$`, 'iu', ), ); + if (format === 'portable') { + patterns.push( + new RegExp( + `^${escapedProductName}-(\\d+\\.\\d+\\.\\d+)-portable-${this.escapeRegex(modernPlatformToken)}-${this.escapeRegex(archSuffix)}${escapedExt}$`, + 'iu', + ), + ); + } } for (const pattern of patterns) { const match = filename.match(pattern); @@ -852,18 +950,8 @@ export class DownloadService { return null; } - private getModernProductName(channel: DesktopChannel): string { - return channel === 'canary' ? 'Fluxer Canary' : 'Fluxer'; - } - - private buildModernArtifactFilename( - channel: DesktopChannel, - version: string, - plat: DesktopPlatform, - archToken: string, - ext: string, - ): string { - return `${this.getModernProductName(channel)}-${version}-${MODERN_PLATFORM_TOKENS[plat]}-${archToken}${ext}`; + private getModernProductNames(channel: DesktopChannel): Array { + return channel === 'canary' ? ['Fluxer-Canary', 'Fluxer Canary'] : ['Fluxer']; } private getArchTokens(archToken: string | Array): Array { diff --git a/fluxer_desktop/electron-builder.config.cjs b/fluxer_desktop/electron-builder.config.cjs index 5732f9b63..c00a19302 100644 --- a/fluxer_desktop/electron-builder.config.cjs +++ b/fluxer_desktop/electron-builder.config.cjs @@ -8,6 +8,7 @@ const path = require('node:path'); const {promisify} = require('node:util'); const execFileAsync = promisify(execFile); const productName = isCanary ? 'Fluxer Canary' : 'Fluxer'; +const artifactProductName = isCanary ? 'Fluxer-Canary' : 'Fluxer'; const appId = isCanary ? 'app.fluxer.canary' : 'app.fluxer'; const iconDir = isCanary ? 'icons-canary' : 'icons-stable'; const packageName = isCanary ? 'fluxer_desktop_canary' : 'fluxer_desktop'; @@ -1269,8 +1270,7 @@ module.exports = { appId, productName, copyright: 'Copyright © 2026 Fluxer Platform AB', - // biome-ignore lint/suspicious/noTemplateCurlyInString: electron-builder placeholders, not JS template literals. - artifactName: '${productName}-${version}-${os}-${arch}.${ext}', + artifactName: `${artifactProductName}-\${version}-\${os}-\${arch}.\${ext}`, directories: { buildResources: 'build_resources', output: 'dist-electron', @@ -1440,8 +1440,7 @@ module.exports = { target: winTargets, }, portable: { - // biome-ignore lint/suspicious/noTemplateCurlyInString: electron-builder expands these placeholders. - artifactName: '${productName}-${version}-portable-${os}-${arch}.${ext}', + artifactName: `${artifactProductName}-\${version}-portable-\${os}-\${arch}.\${ext}`, }, linux: { icon: `build_resources/${iconDir}/1024x1024.png`, diff --git a/fluxer_desktop/src/main/Updater.ts b/fluxer_desktop/src/main/Updater.ts index 670c0bd88..05b00057b 100644 --- a/fluxer_desktop/src/main/Updater.ts +++ b/fluxer_desktop/src/main/Updater.ts @@ -473,14 +473,14 @@ function buildManualLatestDownloadUrl(format: ManualDesktopFormat): string { return `${UPDATE_BASE_URL}/latest/${format}`; } -function getModernProductName(): string { - return BUILD_CHANNEL === 'canary' ? 'Fluxer Canary' : 'Fluxer'; +function getArtifactProductName(): string { + return BUILD_CHANNEL === 'canary' ? 'Fluxer-Canary' : 'Fluxer'; } function getManualUpdateSuggestedName(format: LinuxManualDesktopFormat, version: string): string { const archToken = LINUX_MANUAL_ARCH_TOKENS[format][DESKTOP_DOWNLOAD_ARCH]; const extension = LINUX_MANUAL_FORMAT_EXTENSIONS[format]; - return `${getModernProductName()}-${version}-linux-${archToken}${extension}`; + return `${getArtifactProductName()}-${version}-linux-${archToken}${extension}`; } function getManualDownloadOptions(info: ManualLatestInfo): Array { diff --git a/packages/config/src/ConfigLoader.ts b/packages/config/src/ConfigLoader.ts index 7098be534..447f934dd 100644 --- a/packages/config/src/ConfigLoader.ts +++ b/packages/config/src/ConfigLoader.ts @@ -92,6 +92,7 @@ function defaultConfig(): MasterConfig { api: { port: 8080, ip_ban_exempt_ips: [], + desktop_github_redirect_countries: [], presigned_attachment_uploads_enabled: false, presigned_downloads_enabled: false, presigned_harvest_downloads_enabled: true, diff --git a/packages/config/src/MasterConfig.ts b/packages/config/src/MasterConfig.ts index 1d7d4cadf..7a3fe40de 100644 --- a/packages/config/src/MasterConfig.ts +++ b/packages/config/src/MasterConfig.ts @@ -91,6 +91,7 @@ export interface MasterConfig { api: { port: number; ip_ban_exempt_ips: Array; + desktop_github_redirect_countries: Array; presigned_attachment_uploads_enabled: boolean; presigned_downloads_enabled: boolean; presigned_harvest_downloads_enabled: boolean; diff --git a/packages/config/src/config_loader/EnvironmentOverrides.ts b/packages/config/src/config_loader/EnvironmentOverrides.ts index c32f72496..de0e3370b 100644 --- a/packages/config/src/config_loader/EnvironmentOverrides.ts +++ b/packages/config/src/config_loader/EnvironmentOverrides.ts @@ -77,6 +77,10 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { FLUXER_NATS_AUTH_TOKEN: {path: ['services', 'nats', 'auth_token']}, FLUXER_API_PORT: {path: ['services', 'api', 'port'], parse: parseEnvValue}, FLUXER_API_IP_BAN_EXEMPT_IPS: {path: ['services', 'api', 'ip_ban_exempt_ips'], parse: parseCsv}, + FLUXER_API_DESKTOP_GITHUB_REDIRECT_COUNTRIES: { + path: ['services', 'api', 'desktop_github_redirect_countries'], + parse: parseCsv, + }, FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED: { path: ['services', 'api', 'presigned_attachment_uploads_enabled'], parse: parseEnvValue, diff --git a/tools/ci/src/desktop.rs b/tools/ci/src/desktop.rs index 666aaa420..a26ddf1bd 100644 --- a/tools/ci/src/desktop.rs +++ b/tools/ci/src/desktop.rs @@ -10,7 +10,11 @@ use crate::common::{ upload_directory_to_s3, upload_s3_plan_append_only, upload_s3_plan_overwrite, }; use crate::functions::write_json_pretty; -use crate::release::DESKTOP_RELEASE_ASSET_SUFFIXES; +use crate::release::{ + DESKTOP_RELEASE_DESCRIPTOR_SCHEMA_VERSION, DesktopReleaseAsset, DesktopReleaseDescriptor, + desktop_release_descriptor_filename, desktop_release_product, + validate_desktop_release_descriptor, +}; use anyhow::{Context, Result, anyhow, bail, ensure}; use aws_sdk_s3::Client as S3Client; use chrono::Utc; @@ -115,6 +119,8 @@ enum DesktopStep { UploadReleaseAssets, DownloadReleaseAssets, UploadPayload, + PublishReleaseDescriptor, + PublishReleaseMarker, BuildSummary, } @@ -243,6 +249,8 @@ pub async fn run(args: BuildDesktopArgs) -> Result<()> { DesktopStep::UploadReleaseAssets => upload_release_assets_step().await, DesktopStep::DownloadReleaseAssets => download_release_assets_step().await, DesktopStep::UploadPayload => upload_payload_step().await, + DesktopStep::PublishReleaseDescriptor => publish_release_descriptor_step().await, + DesktopStep::PublishReleaseMarker => publish_release_marker_step().await, DesktopStep::BuildSummary => build_summary_step(), } } @@ -1901,12 +1909,23 @@ fn resolve_windows_unpacked_dir(arch: &str, main_exe: &str) -> Result { struct WindowsPackageConfig { pack_id: &'static str, pack_title: &'static str, + artifact_prefix: &'static str, icon_dir: &'static str, runtime: &'static str, main_exe: String, output_dir: PathBuf, } +#[derive(Debug, Serialize, Deserialize)] +struct VelopackAssetIndexEntry { + #[serde(rename = "RelativeFileName")] + relative_file_name: String, + #[serde(rename = "Type")] + asset_type: String, + #[serde(flatten)] + extra: BTreeMap, +} + fn windows_package_config(build_channel: &str, arch: &str) -> WindowsPackageConfig { let canary = build_channel == "canary"; let pack_title = if canary { "Fluxer Canary" } else { "Fluxer" }; @@ -1917,6 +1936,7 @@ fn windows_package_config(build_channel: &str, arch: &str) -> WindowsPackageConf "fluxer_desktop" }, pack_title, + artifact_prefix: if canary { "Fluxer-Canary" } else { "Fluxer" }, icon_dir: if canary { "icons-canary" } else { @@ -2081,6 +2101,7 @@ fn validate_velopack_output( let full_nupkg = full_nupkg.ok_or_else(|| { anyhow!("Velopack did not produce a full nupkg payload for Windows updates.") })?; + let full_nupkg = rename_windows_update_package(config, version, arch, &full_nupkg)?; let release_feed = fs::read_to_string(&legacy_releases) .with_context(|| format!("Failed to read {}", legacy_releases.display()))?; let nupkg_name = file_name_string(&full_nupkg)?; @@ -2096,7 +2117,7 @@ fn validate_velopack_output( config.output_dir.display() ) })?; - let desired_setup_name = format!("{}-{version}-win-{arch}.exe", config.pack_title); + let desired_setup_name = format!("{}-{version}-win-{arch}.exe", config.artifact_prefix); if file_name_string(&setup_exe)? != desired_setup_name { fs::rename(&setup_exe, config.output_dir.join(desired_setup_name)) .with_context(|| format!("Failed to rename {}", setup_exe.display()))?; @@ -2104,6 +2125,122 @@ fn validate_velopack_output( Ok(()) } +fn rename_windows_update_package( + config: &WindowsPackageConfig, + version: &str, + arch: &str, + source: &Path, +) -> Result { + let source_name = file_name_string(source)?; + let target_name = format!("{}-{version}-win-{arch}-full.nupkg", config.artifact_prefix); + let setup_name = format!("{}-{version}-win-{arch}.exe", config.artifact_prefix); + let portable_name = format!( + "{}-{version}-portable-win-{arch}.zip", + config.artifact_prefix + ); + ensure!( + source_name != target_name, + "Velopack unexpectedly emitted the canonical package name {target_name:?} before feed normalization" + ); + let legacy_path = config.output_dir.join("RELEASES"); + let legacy = fs::read_to_string(&legacy_path) + .with_context(|| format!("Failed to read {}", legacy_path.display()))?; + ensure!( + legacy.matches(&source_name).count() == 1, + "{} must reference Velopack package {source_name:?} exactly once", + legacy_path.display() + ); + fs::write(&legacy_path, legacy.replace(&source_name, &target_name)) + .with_context(|| format!("Failed to rewrite {}", legacy_path.display()))?; + + let releases_path = config.output_dir.join("releases.win.json"); + let mut releases: Value = serde_json::from_slice( + &fs::read(&releases_path) + .with_context(|| format!("Failed to read {}", releases_path.display()))?, + ) + .with_context(|| format!("Failed to parse {}", releases_path.display()))?; + let replacements = replace_json_string(&mut releases, &source_name, &target_name); + ensure!( + replacements == 1, + "{} must reference Velopack package {source_name:?} exactly once, found {replacements}", + releases_path.display() + ); + write_json_pretty(&releases_path, &releases)?; + + let assets_path = config.output_dir.join("assets.win.json"); + let mut assets: Vec = serde_json::from_slice( + &fs::read(&assets_path) + .with_context(|| format!("Failed to read {}", assets_path.display()))?, + ) + .with_context(|| format!("Failed to parse {}", assets_path.display()))?; + ensure!( + assets.len() == 3, + "{} must contain exactly three Velopack assets, found {}", + assets_path.display(), + assets.len() + ); + let mut asset_types = BTreeSet::new(); + for asset in &mut assets { + ensure!( + asset_types.insert(asset.asset_type.as_str()), + "{} contains duplicate asset type {:?}", + assets_path.display(), + asset.asset_type + ); + asset.relative_file_name = match asset.asset_type.as_str() { + "Installer" => setup_name.clone(), + "Portable" => portable_name.clone(), + "Full" => { + ensure!( + asset.relative_file_name == source_name, + "{} Full asset references {:?}, expected {source_name:?}", + assets_path.display(), + asset.relative_file_name + ); + target_name.clone() + } + other => bail!( + "{} contains unsupported Velopack asset type {other:?}", + assets_path.display() + ), + }; + } + ensure!( + asset_types == BTreeSet::from(["Full", "Installer", "Portable"]), + "{} contains an incomplete Velopack asset inventory", + assets_path.display() + ); + write_json_pretty(&assets_path, &assets)?; + + let target = config.output_dir.join(&target_name); + fs::rename(source, &target).with_context(|| { + format!( + "Failed to rename {} to {}", + source.display(), + target.display() + ) + })?; + Ok(target) +} + +fn replace_json_string(value: &mut Value, source: &str, target: &str) -> usize { + match value { + Value::String(current) if current == source => { + *current = target.to_string(); + 1 + } + Value::Array(values) => values + .iter_mut() + .map(|value| replace_json_string(value, source, target)) + .sum(), + Value::Object(values) => values + .values_mut() + .map(|value| replace_json_string(value, source, target)) + .sum(), + _ => 0, + } +} + fn find_windows_unpacked_app(arch: &str, main_exe: &str) -> Option { windows_unpacked_candidates(arch) .into_iter() @@ -2225,7 +2362,10 @@ fn create_portable_zip_windows_step() -> Result<()> { let portable_marker = pack_dir.join(".portable"); fs::write(&portable_marker, "") .with_context(|| format!("Failed to write {}", portable_marker.display()))?; - let zip_name = format!("{}-{version}-portable-win-{arch}.zip", config.pack_title); + let zip_name = format!( + "{}-{version}-portable-win-{arch}.zip", + config.artifact_prefix + ); let zip_path = PathBuf::from("dist-electron").join(zip_name); create_zip_from_dir(&pack_dir, &zip_path)?; remove_file_if_exists(&portable_marker)?; @@ -3159,9 +3299,10 @@ fn verify_windows_signed_artifacts_step() -> Result<()> { config.output_dir.display() ) })?; - let setup_exe = config - .output_dir - .join(format!("{}-{version}-win-{arch}.exe", config.pack_title)); + let setup_exe = config.output_dir.join(format!( + "{}-{version}-win-{arch}.exe", + config.artifact_prefix + )); ensure!( setup_exe.is_file(), "Velopack Setup.exe not found: {}", @@ -3169,7 +3310,7 @@ fn verify_windows_signed_artifacts_step() -> Result<()> { ); let portable_zip = PathBuf::from("dist-electron").join(format!( "{}-{version}-portable-win-{arch}.zip", - config.pack_title + config.artifact_prefix )); ensure!( portable_zip.is_file(), @@ -3507,100 +3648,243 @@ fn build_payload_step() -> Result<()> { fn prepare_release_assets_step() -> Result<()> { let channel = require_env("CHANNEL")?; let version = require_env("VERSION")?; - let product = match channel.as_str() { - "stable" => "Fluxer", - "canary" => "Fluxer.Canary", - other => bail!("Unsupported desktop release channel {other:?}"), - }; - let expected_asset_names = DESKTOP_RELEASE_ASSET_SUFFIXES - .iter() - .map(|suffix| format!("{product}-{version}-{suffix}")) - .collect::>(); - let artifacts = Path::new("artifacts"); + let source_sha = require_env("SOURCE_SHA")?; + let s3_prefix = require_env("S3_DESKTOP_PREFIX")?; + ensure!( + s3_prefix == "desktop", + "GitHub desktop releases require S3_DESKTOP_PREFIX=desktop, received {s3_prefix:?}" + ); + let product = desktop_release_product(&channel)?; + let payload_root = Path::new("s3_payload").join(&s3_prefix).join(&channel); let release_assets = Path::new("release_assets"); remove_dir_if_exists(release_assets)?; fs::create_dir_all(release_assets)?; - let mut asset_names = BTreeSet::new(); - for (dir, identity) in payload_artifact_dirs(artifacts, &channel)? { + let mut release_builder = + DesktopReleaseAssetBuilder::new(&s3_prefix, &channel, &version, product, release_assets); + for (platform, arch) in [ + ("win32", "x64"), + ("win32", "arm64"), + ("darwin", "x64"), + ("darwin", "arm64"), + ("linux", "x64"), + ("linux", "arm64"), + ] { + let dir = payload_root.join(platform).join(arch); ensure!( - identity.desktop_variant == DEFAULT_DESKTOP_VARIANT, - "GitHub release asset naming is undefined for desktop variant {:?}", - identity.desktop_variant + dir.is_dir(), + "Desktop release payload directory is missing: {}", + dir.display() ); - let platform = match identity.platform.as_str() { - "windows" => "win32", - "macos" => "darwin", + let manifest_path = dir.join("manifest.json"); + let manifest: DesktopManifest = serde_json::from_slice( + &fs::read(&manifest_path) + .with_context(|| format!("Failed to read {}", manifest_path.display()))?, + ) + .with_context(|| format!("Failed to parse {}", manifest_path.display()))?; + ensure!( + manifest.channel == channel + && manifest.platform == platform + && manifest.arch == arch + && manifest.version == version + && manifest.variant.is_none(), + "Desktop release manifest identity mismatch in {}", + manifest_path.display() + ); + let expected_kinds = match platform { + "win32" => BTreeSet::from(["portable", "setup"]), + "darwin" => BTreeSet::from(["dmg", "zip"]), + "linux" => BTreeSet::from(["appimage", "deb", "rpm", "tar_gz"]), + _ => unreachable!(), + }; + let actual_kinds = manifest + .files + .keys() + .map(String::as_str) + .collect::>(); + ensure!( + actual_kinds == expected_kinds, + "Incomplete shipped artifact set for {platform}/{arch}: expected {:?}, found {:?}", + expected_kinds, + actual_kinds + ); + for entry in manifest.files.values() { + release_builder.add(platform, arch, &dir.join(entry.filename()), false)?; + } + let updater_files = desktop_updater_release_files(&dir, platform)?; + for updater_file in updater_files { + release_builder.add(platform, arch, &updater_file, true)?; + } + } + let mut descriptor_assets = release_builder.finish(); + descriptor_assets.sort_by(|left, right| left.storage_key.cmp(&right.storage_key)); + let descriptor = DesktopReleaseDescriptor { + schema_version: DESKTOP_RELEASE_DESCRIPTOR_SCHEMA_VERSION, + channel: channel.clone(), + version: version.clone(), + release_tag: format!("fluxer-desktop-{channel}@{version}"), + source_sha, + assets: descriptor_assets, + }; + validate_desktop_release_descriptor(&descriptor, &channel, &version, &descriptor.source_sha)?; + let descriptor_path = + release_assets.join(desktop_release_descriptor_filename(&channel, &version)?); + write_json_pretty(&descriptor_path, &descriptor)?; + println!("GitHub release asset tree:"); + print_tree(release_assets, 2) +} + +fn desktop_updater_release_files(dir: &Path, platform: &str) -> Result> { + let mut files = match platform { + "win32" => vec![ + dir.join("RELEASES"), + dir.join("releases.win.json"), + dir.join("assets.win.json"), + ], + "darwin" => vec![dir.join("RELEASES.json"), dir.join("releases.json")], + "linux" => Vec::new(), + other => bail!("Unsupported desktop release platform {other:?}"), + }; + if platform == "win32" { + let nupkgs = collect_files(dir)? + .into_iter() + .filter(|path| { + path.file_name() + .and_then(OsStr::to_str) + .is_some_and(|name| name.ends_with("-full.nupkg")) + }) + .collect::>(); + ensure!( + nupkgs.len() == 1, + "Expected one Windows full update package in {}, found {}", + dir.display(), + nupkgs.len() + ); + files.push(nupkgs[0].clone()); + } + for path in &files { + ensure!( + path.is_file(), + "Desktop updater release file is missing: {}", + path.display() + ); + } + Ok(files) +} + +struct DesktopReleaseAssetBuilder<'a> { + s3_prefix: &'a str, + channel: &'a str, + version: &'a str, + product: &'a str, + release_assets: &'a Path, + descriptor_assets: Vec, + storage_keys: BTreeSet, + release_asset_content: BTreeMap, +} + +impl<'a> DesktopReleaseAssetBuilder<'a> { + fn new( + s3_prefix: &'a str, + channel: &'a str, + version: &'a str, + product: &'a str, + release_assets: &'a Path, + ) -> Self { + Self { + s3_prefix, + channel, + version, + product, + release_assets, + descriptor_assets: Vec::new(), + storage_keys: BTreeSet::new(), + release_asset_content: BTreeMap::new(), + } + } + + fn add(&mut self, platform: &str, arch: &str, source: &Path, qualify_name: bool) -> Result<()> { + ensure!( + source.is_file(), + "Release source is missing: {}", + source.display() + ); + let source_name = file_name_string(source)?; + let platform_token = match platform { + "win32" => "win", + "darwin" => "mac", "linux" => "linux", other => bail!("Unsupported desktop release platform {other:?}"), }; - let candidates = manifest_candidates(&dir, platform, &identity.arch)?; - let candidate_kinds = candidates - .iter() - .map(|(kind, _)| kind.as_str()) - .collect::>(); - let expected_kinds = match platform { - "win32" => vec!["setup", "portable"], - "darwin" => vec!["dmg", "zip"], - "linux" => vec!["appimage", "deb", "rpm", "tar_gz"], - _ => unreachable!(), + let canonical_prefix = format!("{}-{}-", self.product, self.version); + let release_asset = if qualify_name && !source_name.starts_with(&canonical_prefix) { + format!( + "{}-{}-{platform_token}-{arch}-{source_name}", + self.product, self.version + ) + } else { + source_name.clone() }; ensure!( - candidate_kinds == expected_kinds, - "Incomplete shipped artifact set for {}/{:?}: expected {:?}, found {:?}", - identity.platform, - identity.arch, - expected_kinds, - candidate_kinds + release_asset.starts_with(&canonical_prefix) + && release_asset.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'_') + }), + "Desktop release asset name is not canonical and URL-safe: {release_asset:?}" ); - - for (_, source) in candidates { - let source_name = file_name_string(&source)?; - let asset_name = clean_release_asset_name(&source_name)?; + let storage_key = format!( + "{}/{}/{platform}/{arch}/{source_name}", + self.s3_prefix, self.channel + ); + ensure!( + self.storage_keys.insert(storage_key.clone()), + "Duplicate desktop release storage key {storage_key:?}" + ); + let size = fs::metadata(source) + .with_context(|| format!("Failed to inspect {}", source.display()))? + .len(); + ensure!( + size > 0, + "Desktop release source is empty: {}", + source.display() + ); + let sha256 = sha256_file(source)?; + if let Some((existing_sha256, existing_size)) = + self.release_asset_content.get(&release_asset) + { ensure!( - asset_names.insert(asset_name.clone()), - "Duplicate GitHub release asset name {asset_name:?}" + existing_sha256 == &sha256 && *existing_size == size, + "Desktop release asset {release_asset:?} has conflicting source content" ); - let destination = release_assets.join(&asset_name); - let copied = fs::copy(&source, &destination).with_context(|| { + } else { + let destination = self.release_assets.join(&release_asset); + let copied = fs::copy(source, &destination).with_context(|| { format!( - "Failed to copy shipped artifact {} to {}", + "Failed to copy desktop release asset {} to {}", source.display(), destination.display() ) })?; ensure!( - copied > 0, - "Copied empty GitHub release asset {}", + copied == size, + "Desktop release asset copy size mismatch for {}", destination.display() ); + self.release_asset_content + .insert(release_asset.clone(), (sha256.clone(), size)); } + self.descriptor_assets.push(DesktopReleaseAsset { + storage_key, + release_asset, + sha256, + size, + }); + Ok(()) } - ensure!( - asset_names == expected_asset_names, - "GitHub release asset inventory mismatch: expected {expected_asset_names:?}, found {asset_names:?}" - ); - println!("GitHub release asset tree:"); - print_tree(release_assets, 2) -} -fn clean_release_asset_name(source_name: &str) -> Result { - let name = source_name - .chars() - .map(|character| { - if character.is_ascii_whitespace() { - '.' - } else { - character - } - }) - .collect::(); - ensure!( - name.bytes() - .all(|byte| { byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'_') }), - "Cannot produce a clean GitHub release asset name from {source_name:?}" - ); - Ok(name) + fn finish(self) -> Vec { + self.descriptor_assets + } } async fn upload_release_assets_step() -> Result<()> { @@ -3922,6 +4206,70 @@ async fn upload_payload_step() -> Result<()> { .await } +fn read_release_descriptor_from_assets() -> Result<(PathBuf, DesktopReleaseDescriptor)> { + let channel = require_env("CHANNEL")?; + let version = require_env("VERSION")?; + let source_sha = require_env("SOURCE_SHA")?; + let path = + Path::new("release_assets").join(desktop_release_descriptor_filename(&channel, &version)?); + let descriptor: DesktopReleaseDescriptor = serde_json::from_slice( + &fs::read(&path).with_context(|| format!("Failed to read {}", path.display()))?, + ) + .with_context(|| format!("Failed to parse {}", path.display()))?; + validate_desktop_release_descriptor(&descriptor, &channel, &version, &source_sha)?; + Ok((path, descriptor)) +} + +async fn publish_release_descriptor_step() -> Result<()> { + let (descriptor_path, descriptor) = read_release_descriptor_from_assets()?; + let client = s3_client(None).await?; + let bucket = require_env("S3_BUCKET")?; + let key = format!( + "desktop/{}/github-releases/{}.json", + descriptor.channel, descriptor.version + ); + let plan = vec![ + S3UploadPlanItem::new(descriptor_path, key) + .with_content_type("application/json; charset=utf-8") + .with_cache_control(VERSIONED_ARTIFACT_CACHE_CONTROL), + ]; + upload_s3_plan_append_only(&client, &bucket, plan).await?; + Ok(()) +} + +async fn publish_release_marker_step() -> Result<()> { + let (descriptor_path, descriptor) = read_release_descriptor_from_assets()?; + let descriptor_sha256 = sha256_file(&descriptor_path)?; + let temp = TempDir::new().context("Failed to create desktop release marker temp directory")?; + let marker_path = temp + .path() + .join(format!("{}.ready.json", descriptor.version)); + write_json_pretty( + &marker_path, + &json!({ + "schema_version": 1, + "channel": descriptor.channel, + "version": descriptor.version, + "release_tag": descriptor.release_tag, + "source_sha": descriptor.source_sha, + "descriptor_sha256": descriptor_sha256, + }), + )?; + let client = s3_client(None).await?; + let bucket = require_env("S3_BUCKET")?; + let key = format!( + "desktop/{}/github-releases/{}.ready.json", + descriptor.channel, descriptor.version + ); + let plan = vec![ + S3UploadPlanItem::new(marker_path, key) + .with_content_type("application/json; charset=utf-8") + .with_cache_control(VERSIONED_ARTIFACT_CACHE_CONTROL), + ]; + upload_s3_plan_append_only(&client, &bucket, plan).await?; + Ok(()) +} + async fn upload_payload_directory( client: &S3Client, bucket: &str, diff --git a/tools/ci/src/release.rs b/tools/ci/src/release.rs index 04dc323fa..292d6da11 100644 --- a/tools/ci/src/release.rs +++ b/tools/ci/src/release.rs @@ -4,7 +4,7 @@ use crate::common::{CommandSpec, output_text, parse_version_instant, run_command use anyhow::{Context, Result, bail, ensure}; use chrono::{DateTime, Utc}; use clap::{Args, Subcommand}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; use std::fs::{self, File}; @@ -13,22 +13,188 @@ use std::path::{Path, PathBuf}; const RELEASE_REPOSITORY: &str = "fluxerapp/fluxer"; const RELEASE_COMPARE_URL: &str = "https://github.com/fluxerapp/fluxer/compare"; -pub(crate) const DESKTOP_RELEASE_ASSET_SUFFIXES: &[&str] = &[ - "linux-aarch64.rpm", - "linux-amd64.deb", - "linux-arm64.AppImage", - "linux-arm64.deb", - "linux-arm64.tar.gz", - "linux-x64.tar.gz", - "linux-x86_64.AppImage", - "linux-x86_64.rpm", - "mac-universal.dmg", - "mac-universal.zip", - "portable-win-arm64.zip", - "portable-win-x64.zip", - "win-arm64.exe", - "win-x64.exe", -]; +pub(crate) const DESKTOP_RELEASE_DESCRIPTOR_SCHEMA_VERSION: u8 = 1; +pub(crate) const DESKTOP_RELEASE_ROUTE_COUNT: usize = 28; +pub(crate) const DESKTOP_RELEASE_ASSET_COUNT: usize = 26; + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub(crate) struct DesktopReleaseAsset { + pub(crate) storage_key: String, + pub(crate) release_asset: String, + pub(crate) sha256: String, + pub(crate) size: u64, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub(crate) struct DesktopReleaseDescriptor { + pub(crate) schema_version: u8, + pub(crate) channel: String, + pub(crate) version: String, + pub(crate) release_tag: String, + pub(crate) source_sha: String, + pub(crate) assets: Vec, +} + +pub(crate) fn desktop_release_product(channel: &str) -> Result<&'static str> { + match channel { + "stable" => Ok("Fluxer"), + "canary" => Ok("Fluxer-Canary"), + other => bail!("Unsupported desktop release channel {other:?}"), + } +} + +pub(crate) fn desktop_release_descriptor_filename(channel: &str, version: &str) -> Result { + Ok(format!( + "{}-{version}-release-manifest.json", + desktop_release_product(channel)? + )) +} + +pub(crate) fn validate_desktop_release_descriptor( + descriptor: &DesktopReleaseDescriptor, + channel: &str, + version: &str, + source_sha: &str, +) -> Result<()> { + ensure!( + descriptor.schema_version == DESKTOP_RELEASE_DESCRIPTOR_SCHEMA_VERSION, + "Unsupported desktop release descriptor schema version {}", + descriptor.schema_version + ); + ensure!( + descriptor.channel == channel, + "Desktop release descriptor channel {:?} does not match {channel:?}", + descriptor.channel + ); + ensure!( + descriptor.version == version, + "Desktop release descriptor version {:?} does not match {version:?}", + descriptor.version + ); + ensure!( + descriptor.release_tag == format!("fluxer-desktop-{channel}@{version}"), + "Desktop release descriptor tag {:?} is invalid", + descriptor.release_tag + ); + ensure!( + descriptor.source_sha == source_sha, + "Desktop release descriptor source SHA {:?} does not match {source_sha:?}", + descriptor.source_sha + ); + ensure!( + source_sha.len() == 40 + && source_sha + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()), + "Invalid desktop release source SHA {source_sha:?}" + ); + parse_version_instant(version) + .with_context(|| format!("Invalid desktop release descriptor version {version:?}"))?; + ensure!( + descriptor.assets.len() == DESKTOP_RELEASE_ROUTE_COUNT, + "Desktop release descriptor must contain {DESKTOP_RELEASE_ROUTE_COUNT} routes, found {}", + descriptor.assets.len() + ); + let storage_prefix = format!("desktop/{channel}/"); + let release_prefix = format!("{}-{version}-", desktop_release_product(channel)?); + let descriptor_name = desktop_release_descriptor_filename(channel, version)?; + let mut storage_keys = BTreeSet::new(); + let mut route_counts = BTreeMap::::new(); + let mut release_assets = BTreeMap::<&str, (&str, u64)>::new(); + for asset in &descriptor.assets { + ensure!( + storage_keys.insert(asset.storage_key.as_str()), + "Desktop release descriptor contains duplicate storage key {:?}", + asset.storage_key + ); + let key_segments = asset.storage_key.split('/').collect::>(); + ensure!( + key_segments.len() == 5 + && key_segments[0] == "desktop" + && key_segments[1] == channel + && matches!(key_segments[2], "win32" | "darwin" | "linux") + && matches!(key_segments[3], "x64" | "arm64") + && !key_segments[4].is_empty() + && key_segments[4].bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'_') + }) + && asset.storage_key.starts_with(&storage_prefix), + "Desktop release descriptor contains invalid storage key {:?}", + asset.storage_key + ); + *route_counts + .entry(format!("{}/{}", key_segments[2], key_segments[3])) + .or_default() += 1; + let platform_token = match key_segments[2] { + "win32" => "win", + "darwin" => "mac", + "linux" => "linux", + _ => unreachable!(), + }; + let expected_release_asset = if key_segments[4].starts_with(&release_prefix) { + key_segments[4].to_string() + } else { + format!( + "{release_prefix}{platform_token}-{}-{}", + key_segments[3], key_segments[4] + ) + }; + ensure!( + asset.release_asset.starts_with(&release_prefix) + && asset.release_asset != descriptor_name + && asset.release_asset == expected_release_asset + && asset.release_asset.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'_') + }), + "Desktop release descriptor contains invalid release asset {:?}", + asset.release_asset + ); + ensure!( + asset.sha256.len() == 64 + && asset + .sha256 + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()), + "Desktop release descriptor contains invalid SHA-256 for {:?}", + asset.release_asset + ); + ensure!( + asset.size > 0, + "Desktop release descriptor contains an empty asset {:?}", + asset.release_asset + ); + if let Some((sha256, size)) = release_assets.get(asset.release_asset.as_str()) { + ensure!( + *sha256 == asset.sha256 && *size == asset.size, + "Desktop release descriptor maps conflicting content to {:?}", + asset.release_asset + ); + } else { + release_assets.insert( + asset.release_asset.as_str(), + (asset.sha256.as_str(), asset.size), + ); + } + } + ensure!( + release_assets.len() == DESKTOP_RELEASE_ASSET_COUNT, + "Desktop release descriptor must contain {DESKTOP_RELEASE_ASSET_COUNT} unique release assets, found {}", + release_assets.len() + ); + let expected_route_counts = BTreeMap::from([ + ("darwin/arm64".to_string(), 4usize), + ("darwin/x64".to_string(), 4usize), + ("linux/arm64".to_string(), 4usize), + ("linux/x64".to_string(), 4usize), + ("win32/arm64".to_string(), 6usize), + ("win32/x64".to_string(), 6usize), + ]); + ensure!( + route_counts == expected_route_counts, + "Desktop release descriptor route inventory mismatch: expected {expected_route_counts:?}, found {route_counts:?}" + ); + Ok(()) +} #[derive(Debug, Args, Clone)] pub struct ReleaseArgs { @@ -202,6 +368,7 @@ fn publish(args: PublishArgs) -> Result<()> { let assets = local_release_assets( &args.component, &args.build_version, + &source_sha, args.asset_dir.as_deref(), )?; if let Some(existing) = existing_summary.filter(|release| !release.is_draft) { @@ -404,6 +571,7 @@ fn tag_exists(tag: &str) -> Result { fn local_release_assets( component: &str, version: &str, + source_sha: &str, asset_dir: Option<&Path>, ) -> Result> { let Some(channel) = desktop_channel(component) else { @@ -419,12 +587,16 @@ fn local_release_assets( "Desktop release asset directory does not exist: {}", asset_dir.display() ); - let product = match channel { - "stable" => "Fluxer", - "canary" => "Fluxer.Canary", - other => bail!("Unsupported desktop release channel {other:?}"), - }; + let product = desktop_release_product(channel)?; let prefix = format!("{product}-{version}-"); + let descriptor_name = desktop_release_descriptor_filename(channel, version)?; + let descriptor_path = asset_dir.join(&descriptor_name); + let descriptor: DesktopReleaseDescriptor = serde_json::from_slice( + &fs::read(&descriptor_path) + .with_context(|| format!("Failed to read {}", descriptor_path.display()))?, + ) + .with_context(|| format!("Failed to parse {}", descriptor_path.display()))?; + validate_desktop_release_descriptor(&descriptor, channel, version, source_sha)?; let mut entries = fs::read_dir(asset_dir) .with_context(|| { format!( @@ -464,12 +636,9 @@ fn local_release_assets( .all(|byte| { byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'_') }), "Release asset name is not clean and URL-safe: {name:?}" ); - let suffix = name - .strip_prefix(&prefix) - .with_context(|| format!("Release asset {name:?} must start with {prefix:?}"))?; ensure!( - DESKTOP_RELEASE_ASSET_SUFFIXES.contains(&suffix), - "Release asset {name:?} is not a supported shipped desktop artifact" + name.starts_with(&prefix), + "Release asset {name:?} must start with {prefix:?}" ); assets.push(LocalReleaseAsset { digest: sha256_file(&path)?, @@ -478,9 +647,11 @@ fn local_release_assets( size: metadata.len(), }); } - let expected_names = DESKTOP_RELEASE_ASSET_SUFFIXES + let expected_names = descriptor + .assets .iter() - .map(|suffix| format!("{prefix}{suffix}")) + .map(|asset| asset.release_asset.clone()) + .chain(std::iter::once(descriptor_name)) .collect::>(); let actual_names = assets .iter() @@ -490,6 +661,25 @@ fn local_release_assets( actual_names == expected_names, "Desktop release asset inventory mismatch: expected {expected_names:?}, found {actual_names:?}" ); + let local_by_name = assets + .iter() + .map(|asset| (asset.name.as_str(), asset)) + .collect::>(); + for descriptor_asset in &descriptor.assets { + let local = local_by_name + .get(descriptor_asset.release_asset.as_str()) + .with_context(|| { + format!( + "Desktop release descriptor references missing asset {:?}", + descriptor_asset.release_asset + ) + })?; + ensure!( + local.digest == descriptor_asset.sha256 && local.size == descriptor_asset.size, + "Desktop release descriptor metadata does not match {:?}", + descriptor_asset.release_asset + ); + } Ok(assets) }