From 8f4f9a8601820ac388d70cb9c4cc0537d30e5296 Mon Sep 17 00:00:00 2001 From: Hampus Date: Thu, 20 Aug 2026 19:55:40 +0200 Subject: [PATCH] refactor(media): share one external proxy url codec across every service (#1794) --- Cargo.lock | 9 + fluxer_common/Cargo.toml | 4 + fluxer_common/src/external_media_path.rs | 276 +++++++++++++++++++++++ fluxer_common/src/lib.rs | 1 + fluxer_gifs/Cargo.toml | 1 + fluxer_gifs/src/media_proxy.rs | 31 +-- fluxer_media_proxy/Cargo.toml | 1 + fluxer_media_proxy/src/external_path.rs | 259 +-------------------- fluxer_messages/Cargo.toml | 1 + fluxer_messages/src/shard_impl.rs | 22 +- fluxer_unfurl/Cargo.toml | 1 + fluxer_unfurl/src/media_proxy.rs | 44 ++-- tools/dev/Cargo.toml | 1 + tools/dev/src/media_external.rs | 35 ++- 14 files changed, 341 insertions(+), 345 deletions(-) create mode 100644 fluxer_common/src/external_media_path.rs diff --git a/Cargo.lock b/Cargo.lock index c3f4b8df3..b19cffeb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1777,6 +1777,7 @@ dependencies = [ "axum", "base64", "clap", + "fluxer_common", "hmac 0.13.0", "hyper 1.10.1", "hyper-util", @@ -1800,6 +1801,7 @@ dependencies = [ "anyhow", "base64", "fluxer-svc", + "fluxer_common", "hmac 0.13.0", "moka", "reqwest", @@ -1836,6 +1838,7 @@ dependencies = [ "cc", "clap", "criterion", + "fluxer_common", "hex", "hmac 0.13.0", "http 1.4.2", @@ -1875,6 +1878,7 @@ dependencies = [ "chrono", "criterion", "fluxer-svc", + "fluxer_common", "fluxer_markdown_parser", "futures", "hmac 0.13.0", @@ -1943,6 +1947,7 @@ dependencies = [ "encoding_rs", "entities", "fluxer-svc", + "fluxer_common", "hmac 0.13.0", "infer", "moka", @@ -2038,10 +2043,14 @@ dependencies = [ "aws-credential-types", "aws-sigv4", "axum", + "base64", + "hmac 0.13.0", "maxminddb", "moka", "reqwest", "serde_json", + "sha2 0.11.0", + "thiserror", "time", "tracing", "urlencoding", diff --git a/fluxer_common/Cargo.toml b/fluxer_common/Cargo.toml index 48504c61a..13fc762a3 100644 --- a/fluxer_common/Cargo.toml +++ b/fluxer_common/Cargo.toml @@ -15,4 +15,8 @@ reqwest = { version = "0.13.4", default-features = false, features = ["blocking" serde_json = "1.0.150" time = { version = "0.3.47", features = ["formatting", "macros", "parsing"] } tracing = "0.1.44" +base64 = "0.22" +hmac = "0.13.0" +sha2 = "0.11.0" +thiserror = "2" urlencoding = "2.1.3" diff --git a/fluxer_common/src/external_media_path.rs b/fluxer_common/src/external_media_path.rs new file mode 100644 index 000000000..59707e311 --- /dev/null +++ b/fluxer_common/src/external_media_path.rs @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use base64::prelude::*; +use thiserror::Error; + +const V2_PREFIX: &str = "v2/"; + +#[derive(Debug, Error, Eq, PartialEq)] +pub enum ExternalPathError { + #[error("invalid external path")] + InvalidExternalPath, + #[error("invalid base64")] + InvalidBase64, + #[error("invalid utf-8")] + InvalidUtf8, +} + +pub fn build_v2_external_media_proxy_path(input_url: &str) -> String { + format!( + "{V2_PREFIX}{}", + BASE64_URL_SAFE_NO_PAD.encode(input_url.as_bytes()) + ) +} + +fn percent_encode_component(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for byte in value.bytes() { + match byte { + b'A'..=b'Z' + | b'a'..=b'z' + | b'0'..=b'9' + | b'-' + | b'_' + | b'.' + | b'!' + | b'~' + | b'*' + | b'\'' + | b'(' + | b')' => out.push(byte as char), + _ => out.push_str(&format!("%{byte:02X}")), + } + } + out +} + +pub fn build_external_media_proxy_path(input_url: &str) -> Result { + let (scheme, remainder) = input_url + .split_once("://") + .ok_or(ExternalPathError::InvalidExternalPath)?; + if scheme.is_empty() || remainder.is_empty() { + return Err(ExternalPathError::InvalidExternalPath); + } + let (authority_and_path, query) = match remainder.split_once('?') { + Some((head, tail)) => (head, Some(tail)), + None => (remainder, None), + }; + let (host, path) = match authority_and_path.split_once('/') { + Some((host, path)) => (host, path), + None => (authority_and_path, ""), + }; + if host.is_empty() { + return Err(ExternalPathError::InvalidExternalPath); + } + let mut segments: Vec = Vec::new(); + if let Some(query) = query { + segments.push(percent_encode_component(&format!("?{query}"))); + } + segments.push(scheme.to_owned()); + segments.push(host.to_owned()); + if !path.is_empty() { + segments.push( + path.split('/') + .map(percent_encode_component) + .collect::>() + .join("/"), + ); + } + Ok(segments.join("/")) +} + +fn decode_v2(proxy_path: &str) -> Result { + let encoded = proxy_path + .strip_prefix(V2_PREFIX) + .ok_or(ExternalPathError::InvalidExternalPath)?; + if encoded.is_empty() { + return Err(ExternalPathError::InvalidExternalPath); + } + let bytes = BASE64_URL_SAFE_NO_PAD + .decode(encoded) + .map_err(|_| ExternalPathError::InvalidBase64)?; + String::from_utf8(bytes).map_err(|_| ExternalPathError::InvalidUtf8) +} + +fn hex_val(c: u8) -> Option { + match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + b'A'..=b'F' => Some(c - b'A' + 10), + _ => None, + } +} + +pub fn percent_decode(input: &str, plus_as_space: bool) -> Vec { + let bytes = input.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + let ch = bytes[i]; + if ch == b'%' + && i + 2 < bytes.len() + && let (Some(hi), Some(lo)) = (hex_val(bytes[i + 1]), hex_val(bytes[i + 2])) + { + out.push((hi << 4) | lo); + i += 3; + continue; + } + out.push(if plus_as_space && ch == b'+' { + b' ' + } else { + ch + }); + i += 1; + } + out +} + +pub fn percent_decode_string(input: &str, plus_as_space: bool) -> String { + String::from_utf8_lossy(&percent_decode(input, plus_as_space)).into_owned() +} + +fn legacy_protocol_index(parts: &[&str]) -> Option { + for (index, part) in parts.iter().enumerate() { + if part.is_empty() { + continue; + } + if index == 0 && part.contains("%3D") { + continue; + } + let mut chars = part.bytes(); + let first = chars.next()?; + if !first.is_ascii_alphabetic() { + continue; + } + if chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, b'+' | b'.' | b'-')) { + return Some(index); + } + } + None +} + +fn reconstruct_legacy(proxy_path: &str) -> Result { + let parts: Vec<&str> = proxy_path.split('/').collect(); + let protocol_index = + legacy_protocol_index(&parts).ok_or(ExternalPathError::InvalidExternalPath)?; + if protocol_index + 1 >= parts.len() { + return Err(ExternalPathError::InvalidExternalPath); + } + let protocol = parts[protocol_index]; + let host_port = percent_decode_string(parts[protocol_index + 1], false); + let query_raw = parts[..protocol_index].join("/"); + let path_raw = parts[protocol_index + 2..].join("/"); + let query = percent_decode_string(&query_raw, false); + let path = percent_decode_string(&path_raw, false); + let normalized_query = query.strip_prefix('?').unwrap_or(&query); + Ok(format!( + "{protocol}://{host_port}/{path}{}{normalized_query}", + if normalized_query.is_empty() { "" } else { "?" } + )) +} + +pub fn reconstruct_original_url(proxy_path: &str) -> Result { + if proxy_path.starts_with(V2_PREFIX) { + decode_v2(proxy_path) + } else { + reconstruct_legacy(proxy_path) + } +} + +type HmacSha256 = hmac::Hmac; + +pub fn create_signature(input: &str, secret: &[u8]) -> String { + use hmac::{KeyInit, Mac}; + let mut mac = HmacSha256::new_from_slice(secret).expect("HMAC accepts any key length"); + mac.update(input.as_bytes()); + BASE64_URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) +} + +pub fn build_external_media_proxy_url( + public_endpoint: &str, + input_url: &str, + secret: &[u8], +) -> Option { + let path = build_external_media_proxy_path(input_url).ok()?; + let signature = create_signature(&path, secret); + Some(format!("{public_endpoint}/external/{signature}/{path}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builds_the_plain_path_shape() { + assert_eq!( + "https/static.klipy.com/ii/c8/28/HkAKKCzZ.webp", + build_external_media_proxy_path("https://static.klipy.com/ii/c8/28/HkAKKCzZ.webp") + .unwrap() + ); + } + + #[test] + fn builds_an_encoded_query_ahead_of_the_scheme() { + assert_eq!( + "%3Fv%3Dquery_param%26goes%3Dhere/https/static.klipy.com/ii/HkAKKCzZ.webp", + build_external_media_proxy_path( + "https://static.klipy.com/ii/HkAKKCzZ.webp?v=query_param&goes=here" + ) + .unwrap() + ); + } + + #[test] + fn keeps_a_non_default_port() { + assert_eq!( + "https/example.com:8443/a.png", + build_external_media_proxy_path("https://example.com:8443/a.png").unwrap() + ); + } + + #[test] + fn round_trips_every_shape() { + for url in [ + "https://static.klipy.com/ii/c8/28/HkAKKCzZ.webp", + "https://static.klipy.com/ii/HkAKKCzZ.webp?v=query_param&goes=here", + "https://example.com:8443/a.png", + "https://avatars.githubusercontent.com/u/241303489?v=4", + "http://example.com/plain.gif", + "https://example.com/file.png?a=1&b=2&c=3", + ] { + let path = build_external_media_proxy_path(url).unwrap(); + assert_eq!( + url, + reconstruct_original_url(&path).unwrap(), + "round trip {url}" + ); + } + } + + #[test] + fn does_not_double_the_question_mark() { + let decoded = reconstruct_original_url("%3Fa%3D1/https/example.com/x.png").unwrap(); + assert_eq!("https://example.com/x.png?a=1", decoded); + assert!(!decoded.contains("??")); + } + + #[test] + fn still_accepts_a_query_without_a_leading_question_mark() { + assert_eq!( + "https://example.com/x.png?a=1", + reconstruct_original_url("a%3D1/https/example.com/x.png").unwrap() + ); + } + + #[test] + fn rejects_a_url_without_a_scheme() { + assert!(build_external_media_proxy_path("example.com/a.png").is_err()); + } + + #[test] + fn v2_path_roundtrip() { + let path = build_v2_external_media_proxy_path("https://example.com/a b.png?x=1"); + let decoded = reconstruct_original_url(&path).unwrap(); + assert_eq!("https://example.com/a b.png?x=1", decoded); + } +} diff --git a/fluxer_common/src/lib.rs b/fluxer_common/src/lib.rs index 6726f75d8..324fff742 100644 --- a/fluxer_common/src/lib.rs +++ b/fluxer_common/src/lib.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: AGPL-3.0-or-later pub mod config; +pub mod external_media_path; pub mod geoip; diff --git a/fluxer_gifs/Cargo.toml b/fluxer_gifs/Cargo.toml index 7b6585829..599f82745 100644 --- a/fluxer_gifs/Cargo.toml +++ b/fluxer_gifs/Cargo.toml @@ -7,6 +7,7 @@ edition.workspace = true license.workspace = true [dependencies] +fluxer_common = { path = "../fluxer_common" } anyhow = "1.0.102" base64 = "0.22.1" fluxer-svc = { path = "../fluxer_svc", default-features = false } diff --git a/fluxer_gifs/src/media_proxy.rs b/fluxer_gifs/src/media_proxy.rs index f57d5ae3c..4b463de7e 100644 --- a/fluxer_gifs/src/media_proxy.rs +++ b/fluxer_gifs/src/media_proxy.rs @@ -1,12 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use base64::prelude::*; -use hmac::{Hmac, KeyInit, Mac}; -use sha2::Sha256; use url::Url; -const V2_PATH_PREFIX: &str = "v2/"; - #[derive(Clone)] pub struct MediaProxyUrlBuilder { endpoint: String, @@ -66,28 +61,14 @@ impl MediaProxyUrlBuilder { return Some(input_url.to_owned()); } - let proxy_path = build_external_media_proxy_path(parsed.as_str()); - let signature = create_signature(&proxy_path, &self.secret_key); - Some(format!( - "{}/external/{signature}/{proxy_path}", - self.endpoint - )) + fluxer_common::external_media_path::build_external_media_proxy_url( + &self.endpoint, + parsed.as_str(), + self.secret_key.as_bytes(), + ) } } -fn build_external_media_proxy_path(input_url: &str) -> String { - format!( - "{V2_PATH_PREFIX}{}", - BASE64_URL_SAFE_NO_PAD.encode(input_url) - ) -} - -fn create_signature(input: &str, secret: &str) -> String { - let mut mac = Hmac::::new_from_slice(secret.as_bytes()).expect("HMAC accepts any key"); - mac.update(input.as_bytes()); - BASE64_URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) -} - #[cfg(test)] mod tests { use super::*; @@ -153,7 +134,7 @@ mod tests { .expect("proxy url"); assert!(url.starts_with("https://media.example.test/external/")); - assert!(url.contains("/v2/")); + assert!(url.contains("/https/")); assert_eq!( builder.external_proxy_url("https://media.example.test/external/existing"), Some("https://media.example.test/external/existing".to_owned()) diff --git a/fluxer_media_proxy/Cargo.toml b/fluxer_media_proxy/Cargo.toml index 185d86371..322c3a5a4 100644 --- a/fluxer_media_proxy/Cargo.toml +++ b/fluxer_media_proxy/Cargo.toml @@ -18,6 +18,7 @@ name = "core" harness = false [dependencies] +fluxer_common = { path = "../fluxer_common" } anyhow = "1.0.102" axum = {version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio"]} base64 = "0.22.1" diff --git a/fluxer_media_proxy/src/external_path.rs b/fluxer_media_proxy/src/external_path.rs index f211aa4d8..86775cc03 100644 --- a/fluxer_media_proxy/src/external_path.rs +++ b/fluxer_media_proxy/src/external_path.rs @@ -1,257 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use base64::prelude::*; -use thiserror::Error; - -const V2_PREFIX: &str = "v2/"; - -#[derive(Debug, Error, Eq, PartialEq)] -pub enum ExternalPathError { - #[error("invalid external path")] - InvalidExternalPath, - #[error("invalid base64")] - InvalidBase64, - #[error("invalid utf-8")] - InvalidUtf8, -} - -pub fn build_v2_external_media_proxy_path(input_url: &str) -> String { - format!( - "{V2_PREFIX}{}", - BASE64_URL_SAFE_NO_PAD.encode(input_url.as_bytes()) - ) -} - -fn percent_encode_component(value: &str) -> String { - let mut out = String::with_capacity(value.len()); - for byte in value.bytes() { - match byte { - b'A'..=b'Z' - | b'a'..=b'z' - | b'0'..=b'9' - | b'-' - | b'_' - | b'.' - | b'!' - | b'~' - | b'*' - | b'\'' - | b'(' - | b')' => out.push(byte as char), - _ => out.push_str(&format!("%{byte:02X}")), - } - } - out -} - -pub fn build_external_media_proxy_path(input_url: &str) -> Result { - let (scheme, remainder) = input_url - .split_once("://") - .ok_or(ExternalPathError::InvalidExternalPath)?; - if scheme.is_empty() || remainder.is_empty() { - return Err(ExternalPathError::InvalidExternalPath); - } - let (authority_and_path, query) = match remainder.split_once('?') { - Some((head, tail)) => (head, Some(tail)), - None => (remainder, None), - }; - let (host, path) = match authority_and_path.split_once('/') { - Some((host, path)) => (host, path), - None => (authority_and_path, ""), - }; - if host.is_empty() { - return Err(ExternalPathError::InvalidExternalPath); - } - let mut segments: Vec = Vec::new(); - if let Some(query) = query { - segments.push(percent_encode_component(&format!("?{query}"))); - } - segments.push(scheme.to_owned()); - segments.push(host.to_owned()); - if !path.is_empty() { - segments.push( - path.split('/') - .map(percent_encode_component) - .collect::>() - .join("/"), - ); - } - Ok(segments.join("/")) -} - -fn decode_v2(proxy_path: &str) -> Result { - let encoded = proxy_path - .strip_prefix(V2_PREFIX) - .ok_or(ExternalPathError::InvalidExternalPath)?; - if encoded.is_empty() { - return Err(ExternalPathError::InvalidExternalPath); - } - let bytes = BASE64_URL_SAFE_NO_PAD - .decode(encoded) - .map_err(|_| ExternalPathError::InvalidBase64)?; - String::from_utf8(bytes).map_err(|_| ExternalPathError::InvalidUtf8) -} - -fn hex_val(c: u8) -> Option { - match c { - b'0'..=b'9' => Some(c - b'0'), - b'a'..=b'f' => Some(c - b'a' + 10), - b'A'..=b'F' => Some(c - b'A' + 10), - _ => None, - } -} - -pub fn percent_decode(input: &str, plus_as_space: bool) -> Vec { - let bytes = input.as_bytes(); - let mut out = Vec::with_capacity(bytes.len()); - let mut i = 0; - while i < bytes.len() { - let ch = bytes[i]; - if ch == b'%' - && i + 2 < bytes.len() - && let (Some(hi), Some(lo)) = (hex_val(bytes[i + 1]), hex_val(bytes[i + 2])) - { - out.push((hi << 4) | lo); - i += 3; - continue; - } - out.push(if plus_as_space && ch == b'+' { - b' ' - } else { - ch - }); - i += 1; - } - out -} - -pub fn percent_decode_string(input: &str, plus_as_space: bool) -> String { - String::from_utf8_lossy(&percent_decode(input, plus_as_space)).into_owned() -} - -fn legacy_protocol_index(parts: &[&str]) -> Option { - for (index, part) in parts.iter().enumerate() { - if part.is_empty() { - continue; - } - if index == 0 && part.contains("%3D") { - continue; - } - let mut chars = part.bytes(); - let first = chars.next()?; - if !first.is_ascii_alphabetic() { - continue; - } - if chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, b'+' | b'.' | b'-')) { - return Some(index); - } - } - None -} - -fn reconstruct_legacy(proxy_path: &str) -> Result { - let parts: Vec<&str> = proxy_path.split('/').collect(); - let protocol_index = - legacy_protocol_index(&parts).ok_or(ExternalPathError::InvalidExternalPath)?; - if protocol_index + 1 >= parts.len() { - return Err(ExternalPathError::InvalidExternalPath); - } - let protocol = parts[protocol_index]; - let host_port = percent_decode_string(parts[protocol_index + 1], false); - let query_raw = parts[..protocol_index].join("/"); - let path_raw = parts[protocol_index + 2..].join("/"); - let query = percent_decode_string(&query_raw, false); - let path = percent_decode_string(&path_raw, false); - let normalized_query = query.strip_prefix('?').unwrap_or(&query); - Ok(format!( - "{protocol}://{host_port}/{path}{}{normalized_query}", - if normalized_query.is_empty() { "" } else { "?" } - )) -} - -pub fn reconstruct_original_url(proxy_path: &str) -> Result { - if proxy_path.starts_with(V2_PREFIX) { - decode_v2(proxy_path) - } else { - reconstruct_legacy(proxy_path) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn builds_the_plain_path_shape() { - assert_eq!( - "https/static.klipy.com/ii/c8/28/HkAKKCzZ.webp", - build_external_media_proxy_path("https://static.klipy.com/ii/c8/28/HkAKKCzZ.webp") - .unwrap() - ); - } - - #[test] - fn builds_an_encoded_query_ahead_of_the_scheme() { - assert_eq!( - "%3Fv%3Dquery_param%26goes%3Dhere/https/static.klipy.com/ii/HkAKKCzZ.webp", - build_external_media_proxy_path( - "https://static.klipy.com/ii/HkAKKCzZ.webp?v=query_param&goes=here" - ) - .unwrap() - ); - } - - #[test] - fn keeps_a_non_default_port() { - assert_eq!( - "https/example.com:8443/a.png", - build_external_media_proxy_path("https://example.com:8443/a.png").unwrap() - ); - } - - #[test] - fn round_trips_every_shape() { - for url in [ - "https://static.klipy.com/ii/c8/28/HkAKKCzZ.webp", - "https://static.klipy.com/ii/HkAKKCzZ.webp?v=query_param&goes=here", - "https://example.com:8443/a.png", - "https://avatars.githubusercontent.com/u/241303489?v=4", - "http://example.com/plain.gif", - "https://example.com/file.png?a=1&b=2&c=3", - ] { - let path = build_external_media_proxy_path(url).unwrap(); - assert_eq!( - url, - reconstruct_original_url(&path).unwrap(), - "round trip {url}" - ); - } - } - - #[test] - fn does_not_double_the_question_mark() { - let decoded = reconstruct_original_url("%3Fa%3D1/https/example.com/x.png").unwrap(); - assert_eq!("https://example.com/x.png?a=1", decoded); - assert!(!decoded.contains("??")); - } - - #[test] - fn still_accepts_a_query_without_a_leading_question_mark() { - assert_eq!( - "https://example.com/x.png?a=1", - reconstruct_original_url("a%3D1/https/example.com/x.png").unwrap() - ); - } - - #[test] - fn rejects_a_url_without_a_scheme() { - assert!(build_external_media_proxy_path("example.com/a.png").is_err()); - } - - #[test] - fn v2_path_roundtrip() { - let path = build_v2_external_media_proxy_path("https://example.com/a b.png?x=1"); - let decoded = reconstruct_original_url(&path).unwrap(); - assert_eq!("https://example.com/a b.png?x=1", decoded); - } -} +pub use fluxer_common::external_media_path::{ + ExternalPathError, build_external_media_proxy_path, build_v2_external_media_proxy_path, + percent_decode, percent_decode_string, reconstruct_original_url, +}; diff --git a/fluxer_messages/Cargo.toml b/fluxer_messages/Cargo.toml index afca51a2c..662d1ee4f 100644 --- a/fluxer_messages/Cargo.toml +++ b/fluxer_messages/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true license.workspace = true [dependencies] +fluxer_common = { path = "../fluxer_common" } anyhow = "1.0.102" base64 = "0.22.1" chrono = { version = "0.4.45", default-features = false, features = ["serde"] } diff --git a/fluxer_messages/src/shard_impl.rs b/fluxer_messages/src/shard_impl.rs index 98a32e72e..5d63f4a04 100644 --- a/fluxer_messages/src/shard_impl.rs +++ b/fluxer_messages/src/shard_impl.rs @@ -15,13 +15,11 @@ use crate::types::{ MessageSnapshot, MessageStickerItem, }; use crate::udt; -use base64::Engine; use chrono::{DateTime, Utc}; use fluxer_svc::shard::ShardService; use fluxer_svc::transport::NatsTransport; use fluxer_svc::{postgres, postgres::BigIntBound, postgres::KeyPart}; use futures::stream::{self, StreamExt}; -use hmac::{Hmac, KeyInit, Mac}; #[cfg(feature = "scylla")] use scylla::DeserializeRow; #[cfg(feature = "scylla")] @@ -31,7 +29,6 @@ use scylla::response::query_result::QueryRowsResult; #[cfg(feature = "scylla")] use scylla::statement::prepared::PreparedStatement; use serde::Deserialize; -use sha2::Sha256; use std::collections::{HashMap, HashSet}; #[cfg(feature = "scylla")] use std::sync::Arc; @@ -2796,23 +2793,12 @@ fn external_media_proxy_url(input_url: &str, options: &ResponseBuildOptions) -> Ok(url) => url, Err(_) => return input_url.to_owned(), }; - let encoded = - base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(parsed_url.as_str().as_bytes()); - let path = format!("v2/{encoded}"); - let signature = create_signature(&path, &options.media_proxy_secret_key); - format!( - "{}/external/{}/{}", + fluxer_common::external_media_path::build_external_media_proxy_url( options.media_endpoint.trim_end_matches('/'), - signature, - path + parsed_url.as_str(), + options.media_proxy_secret_key.as_bytes(), ) -} - -fn create_signature(input: &str, secret: &str) -> String { - let mut mac = - Hmac::::new_from_slice(secret.as_bytes()).expect("hmac accepts any key size"); - mac.update(input.as_bytes()); - base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) + .unwrap_or_else(|| input_url.to_owned()) } fn dt_to_epoch_millis(dt: &DateTime) -> i64 { diff --git a/fluxer_unfurl/Cargo.toml b/fluxer_unfurl/Cargo.toml index d65fe9480..eabb3aae1 100644 --- a/fluxer_unfurl/Cargo.toml +++ b/fluxer_unfurl/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true license.workspace = true [dependencies] +fluxer_common = { path = "../fluxer_common" } ammonia = "4.1.2" anyhow = "1.0.102" base64 = "0.22.1" diff --git a/fluxer_unfurl/src/media_proxy.rs b/fluxer_unfurl/src/media_proxy.rs index 435851fa9..839bdc990 100644 --- a/fluxer_unfurl/src/media_proxy.rs +++ b/fluxer_unfurl/src/media_proxy.rs @@ -5,7 +5,6 @@ use std::time::Duration; use url::Url; const METADATA_TIMEOUT: Duration = Duration::from_secs(5); -const EXTERNAL_PROXY_VERSION_PREFIX: &str = "v2/"; pub struct MediaProxyClient { http_client: reqwest::Client, @@ -120,30 +119,14 @@ impl MediaProxyClient { return Some(input_url.to_owned()); } let parsed = Url::parse(input_url).ok()?; - let encoded = base64::Engine::encode( - &base64::engine::general_purpose::URL_SAFE_NO_PAD, + fluxer_common::external_media_path::build_external_media_proxy_url( + &self.public_endpoint, parsed.as_str(), - ); - let path = format!("{EXTERNAL_PROXY_VERSION_PREFIX}{encoded}"); - let signature = create_signature(&path, &self.secret_key); - Some(format!( - "{}/external/{}/{}", - self.public_endpoint, signature, path - )) + self.secret_key.as_bytes(), + ) } } -fn create_signature(input: &str, secret: &str) -> String { - use hmac::{Hmac, KeyInit, Mac}; - let mut mac = - Hmac::::new_from_slice(secret.as_bytes()).expect("hmac accepts any key size"); - mac.update(input.as_bytes()); - base64::Engine::encode( - &base64::engine::general_purpose::URL_SAFE_NO_PAD, - mac.finalize().into_bytes(), - ) -} - #[cfg(test)] mod tests { use super::*; @@ -178,7 +161,7 @@ mod tests { } #[test] - fn external_proxy_url_uses_public_endpoint_and_v2_path() { + fn external_proxy_url_uses_public_endpoint_and_plain_path() { let client = reqwest::Client::new(); let mp = MediaProxyClient::new_with_public_endpoint( "http://media-proxy:8080/", @@ -191,15 +174,18 @@ mod tests { .expect("proxy url"); assert!(proxy.starts_with("https://media.example.test/external/")); - assert!(proxy.contains("/v2/")); + assert!(proxy.contains("/https/")); - let encoded = proxy.rsplit('/').next().expect("encoded path segment"); - let decoded = - base64::Engine::decode(&base64::engine::general_purpose::URL_SAFE_NO_PAD, encoded) - .expect("valid base64"); + let path = proxy + .split_once("/external/") + .expect("external segment") + .1 + .split_once('/') + .expect("signature segment") + .1; assert_eq!( - String::from_utf8(decoded).expect("utf8"), - "https://pbs.twimg.com/media/a.jpg?name=orig" + "https://pbs.twimg.com/media/a.jpg?name=orig", + fluxer_common::external_media_path::reconstruct_original_url(path).expect("decodes") ); assert_eq!( mp.external_proxy_url("https://media.example.test/external/already"), diff --git a/tools/dev/Cargo.toml b/tools/dev/Cargo.toml index 0536c60a8..6c03cba23 100644 --- a/tools/dev/Cargo.toml +++ b/tools/dev/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true license.workspace = true [dependencies] +fluxer_common = { path = "../../fluxer_common" } anyhow = "1.0.102" axum = "0.8.9" base64 = "0.22.1" diff --git a/tools/dev/src/media_external.rs b/tools/dev/src/media_external.rs index 37b91daea..e7ca75204 100644 --- a/tools/dev/src/media_external.rs +++ b/tools/dev/src/media_external.rs @@ -1,10 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later use anyhow::{Context, Result}; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use clap::Args; -use hmac::{Hmac, KeyInit, Mac}; -use sha2::Sha256; #[derive(Debug, Clone, Args)] pub struct SignExternalUrlArgs { @@ -16,17 +13,12 @@ pub struct SignExternalUrlArgs { } pub fn sign_external_url(secret_key: &str, server_url: &str, upstream: &str) -> Result { - let path = format!("v2/{}", URL_SAFE_NO_PAD.encode(upstream.as_bytes())); - let mut mac = Hmac::::new_from_slice(secret_key.as_bytes()) - .context("failed to create HMAC signer")?; - mac.update(path.as_bytes()); - let signature = URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()); - Ok(format!( - "{}/external/{}/{}", + fluxer_common::external_media_path::build_external_media_proxy_url( server_url.trim_end_matches('/'), - signature, - path - )) + upstream, + secret_key.as_bytes(), + ) + .context("failed to build the external media proxy url") } #[cfg(test)] @@ -41,13 +33,20 @@ mod tests { "https://example.test/a b.jpg", ) .unwrap(); - let parts = signed.split('/').collect::>(); assert!(signed.starts_with("http://127.0.0.1:19110/external/")); - assert_eq!(parts[5], "v2"); + assert!(signed.ends_with("/https/example.test/a%20b.jpg")); assert_eq!( - URL_SAFE_NO_PAD.decode(parts[6]).unwrap(), - b"https://example.test/a b.jpg" + "https://example.test/a b.jpg", + fluxer_common::external_media_path::reconstruct_original_url( + signed + .split_once("/external/") + .unwrap() + .1 + .split_once('/') + .unwrap() + .1 + ) + .unwrap() ); - assert_eq!(URL_SAFE_NO_PAD.decode(parts[4]).unwrap().len(), 32); } }