From c8ff177f85dbe7881ba4ed30caf93d9f8566ef96 Mon Sep 17 00:00:00 2001 From: Hampus Date: Mon, 24 Aug 2026 12:28:32 +0200 Subject: [PATCH] fix(unfurl): cache empty results briefly and key entries by provider (#1838) --- fluxer_unfurl/src/cache_policy.rs | 229 +++++++++++++++++++++- fluxer_unfurl/src/main.rs | 4 +- fluxer_unfurl/src/router_impl.rs | 212 ++++++++++++++++---- fluxer_unfurl/src/shard_impl.rs | 310 +++++++++++++++++++++++++++--- 4 files changed, 687 insertions(+), 68 deletions(-) diff --git a/fluxer_unfurl/src/cache_policy.rs b/fluxer_unfurl/src/cache_policy.rs index b0e61f238..d30c3d109 100644 --- a/fluxer_unfurl/src/cache_policy.rs +++ b/fluxer_unfurl/src/cache_policy.rs @@ -1,3 +1,230 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -pub const FIXED_UNFURL_CACHE_TTL_SECS: u64 = 30 * 60; +use crate::types::{NsfwMode, UnfurlResponse, UnfurlResult}; +use moka::Expiry; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +pub const EMPTY_UNFURL_CACHE_TTL_SECS: u64 = 60; + +const SCAN_TOKENS: [&str; 2] = ["scanned", "unscanned"]; + +const PROVIDER_CAPABILITIES: [&str; 4] = ["none", "youtube", "klipy", "youtube+klipy"]; + +pub fn nsfw_scan_token(nsfw_mode: NsfwMode) -> &'static str { + match nsfw_mode { + NsfwMode::Block | NsfwMode::Flag => SCAN_TOKENS[0], + NsfwMode::Allow => SCAN_TOKENS[1], + } +} + +pub fn provider_capability(has_youtube_key: bool, has_klipy_key: bool) -> &'static str { + match (has_youtube_key, has_klipy_key) { + (true, true) => "youtube+klipy", + (true, false) => "youtube", + (false, true) => "klipy", + (false, false) => "none", + } +} + +pub fn unfurl_cache_key(url: &str, nsfw_mode: NsfwMode, capability: &str) -> String { + let scan = nsfw_scan_token(nsfw_mode); + format!("{scan}:{capability}:{url}") +} + +pub fn every_unfurl_cache_key(url: &str) -> Vec { + let mut keys = Vec::with_capacity(SCAN_TOKENS.len() * PROVIDER_CAPABILITIES.len()); + for scan in SCAN_TOKENS { + for capability in PROVIDER_CAPABILITIES { + keys.push(format!("{scan}:{capability}:{url}")); + } + } + keys +} + +pub fn default_empty_entry_lifetime() -> Duration { + Duration::from_secs(EMPTY_UNFURL_CACHE_TTL_SECS) +} + +pub struct UnfurlEntryExpiry { + empty_entry_lifetime: Duration, +} + +impl UnfurlEntryExpiry { + pub fn new(empty_entry_lifetime: Duration) -> Self { + Self { + empty_entry_lifetime, + } + } + + fn lifetime(&self, has_embeds: bool) -> Option { + if has_embeds { + None + } else { + Some(self.empty_entry_lifetime) + } + } +} + +impl Expiry> for UnfurlEntryExpiry { + fn expire_after_create( + &self, + _key: &String, + value: &Arc, + _created_at: Instant, + ) -> Option { + self.lifetime(!value.embeds.is_empty()) + } + + fn expire_after_update( + &self, + _key: &String, + value: &Arc, + _updated_at: Instant, + _duration_until_expiry: Option, + ) -> Option { + self.lifetime(!value.embeds.is_empty()) + } +} + +impl Expiry for UnfurlEntryExpiry { + fn expire_after_create( + &self, + _key: &String, + value: &UnfurlResponse, + _created_at: Instant, + ) -> Option { + self.lifetime(response_has_embeds(value)) + } + + fn expire_after_update( + &self, + _key: &String, + value: &UnfurlResponse, + _updated_at: Instant, + _duration_until_expiry: Option, + ) -> Option { + self.lifetime(response_has_embeds(value)) + } +} + +fn response_has_embeds(response: &UnfurlResponse) -> bool { + match response { + UnfurlResponse::Resolved(result) => !result.embeds.is_empty(), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::MessageEmbed; + + const URL: &str = "https://example.com/a"; + + const EMPTY_UNFURL_CACHE_TTL_CEILING_SECS: u64 = 5 * 60; + + fn resolved(embed_count: usize) -> UnfurlResponse { + UnfurlResponse::Resolved(Arc::new(UnfurlResult { + embeds: (0..embed_count) + .map(|_| MessageEmbed::new("link")) + .collect(), + cache_ttl_seconds: None, + })) + } + + #[test] + fn every_key_covers_the_full_scan_and_capability_matrix() { + let keys = every_unfurl_cache_key(URL); + assert_eq!(keys.len(), 8); + let unique: std::collections::HashSet<&String> = keys.iter().collect(); + assert_eq!(unique.len(), keys.len(), "invalidation keys must be unique"); + for nsfw_mode in [NsfwMode::Block, NsfwMode::Flag, NsfwMode::Allow] { + for capability in PROVIDER_CAPABILITIES { + assert!( + keys.contains(&unfurl_cache_key(URL, nsfw_mode, capability)), + "invalidation must cover {nsfw_mode:?}/{capability}" + ); + } + } + } + + #[test] + fn scanning_and_unscanned_requests_do_not_share_an_entry() { + assert_ne!( + unfurl_cache_key(URL, NsfwMode::Block, "none"), + unfurl_cache_key(URL, NsfwMode::Allow, "none"), + "an unscanned result carries no nsfw media flag and must not serve a scanning request" + ); + } + + #[test] + fn the_two_scanning_modes_share_a_single_entry() { + assert_eq!( + unfurl_cache_key(URL, NsfwMode::Block, "none"), + unfurl_cache_key(URL, NsfwMode::Flag, "none") + ); + } + + #[test] + fn capability_distinguishes_every_key_combination() { + let all = [ + provider_capability(false, false), + provider_capability(true, false), + provider_capability(false, true), + provider_capability(true, true), + ]; + let unique: std::collections::HashSet<&str> = all.iter().copied().collect(); + assert_eq!(unique.len(), all.len()); + for capability in all { + assert_ne!( + unfurl_cache_key(URL, NsfwMode::Block, capability), + unfurl_cache_key(URL, NsfwMode::Block, "unrelated") + ); + } + } + + #[test] + fn the_url_distinguishes_entries() { + assert_ne!( + unfurl_cache_key(URL, NsfwMode::Block, "none"), + unfurl_cache_key("https://example.com/b", NsfwMode::Block, "none") + ); + } + + #[test] + fn the_empty_entry_lifetime_stays_a_short_retry_window() { + assert!( + default_empty_entry_lifetime() + <= Duration::from_secs(EMPTY_UNFURL_CACHE_TTL_CEILING_SECS), + "a site that was merely down must become retryable again quickly" + ); + } + + #[test] + fn the_expiry_reads_the_short_lifetime_off_an_empty_response() { + let expiry = UnfurlEntryExpiry::new(default_empty_entry_lifetime()); + let now = Instant::now(); + let key = unfurl_cache_key(URL, NsfwMode::Block, "none"); + assert_eq!( + Some(default_empty_entry_lifetime()), + Expiry::::expire_after_create(&expiry, &key, &resolved(0), now) + ); + assert_eq!( + None, + Expiry::::expire_after_create(&expiry, &key, &resolved(1), now) + ); + assert_eq!( + Some(default_empty_entry_lifetime()), + Expiry::>::expire_after_create( + &expiry, + &key, + &Arc::new(UnfurlResult { + embeds: Vec::new(), + cache_ttl_seconds: None, + }), + now + ) + ); + } +} diff --git a/fluxer_unfurl/src/main.rs b/fluxer_unfurl/src/main.rs index af03caa29..84771ccf9 100644 --- a/fluxer_unfurl/src/main.rs +++ b/fluxer_unfurl/src/main.rs @@ -40,11 +40,11 @@ async fn main() -> anyhow::Result<()> { match config.mode { Mode::Router => { - let router = UnfurlRouter::new(config.cache_max_entries); + let router = UnfurlRouter::new(config.cache_max_entries, config.cache_ttl); fluxer_svc::router::run_router(&config, router, transport).await } Mode::Shard => { - let shard = UnfurlShard::new(); + let shard = UnfurlShard::new(config.cache_ttl); fluxer_svc::shard::run_shard(&config, shard, transport).await } } diff --git a/fluxer_unfurl/src/router_impl.rs b/fluxer_unfurl/src/router_impl.rs index b61739717..629633855 100644 --- a/fluxer_unfurl/src/router_impl.rs +++ b/fluxer_unfurl/src/router_impl.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use crate::cache_policy::FIXED_UNFURL_CACHE_TTL_SECS; -use crate::types::{NsfwMode, UnfurlRequest, UnfurlResponse}; +use crate::cache_policy::{ + UnfurlEntryExpiry, default_empty_entry_lifetime, every_unfurl_cache_key, provider_capability, + unfurl_cache_key, +}; +use crate::types::{UnfurlRequest, UnfurlResponse}; use fluxer_svc::router::RouterService; use moka::sync::Cache; use std::time::Duration; @@ -11,20 +14,27 @@ pub struct UnfurlRouter { } impl UnfurlRouter { - pub fn new(max_entries: u64) -> Self { + pub fn new(max_entries: u64, lifetime: Duration) -> Self { + Self::with_empty_entry_lifetime(max_entries, lifetime, default_empty_entry_lifetime()) + } + + fn with_empty_entry_lifetime( + max_entries: u64, + lifetime: Duration, + empty_entry_lifetime: Duration, + ) -> Self { Self { l1: Cache::builder() .max_capacity(max_entries) - .time_to_live(Duration::from_secs(FIXED_UNFURL_CACHE_TTL_SECS)) + .time_to_live(lifetime) + .expire_after(UnfurlEntryExpiry::new(empty_entry_lifetime)) .build(), } } fn invalidate_all_variants(&self, url: &str) { - for mode in [NsfwMode::Block, NsfwMode::Flag, NsfwMode::Allow] { - for capability in PROVIDER_CAPABILITIES { - self.l1.invalidate(&unfurl_cache_key(url, mode, capability)); - } + for key in every_unfurl_cache_key(url) { + self.l1.invalidate(&key); } } } @@ -63,7 +73,7 @@ impl RouterService for UnfurlRouter { unfurl_cache_key( url, nsfw_mode.unwrap_or_default(), - provider_capability(youtube_api_key, klipy_api_key) + provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()) ) )) } @@ -87,7 +97,7 @@ impl RouterService for UnfurlRouter { let key = unfurl_cache_key( url, nsfw_mode.unwrap_or_default(), - provider_capability(youtube_api_key, klipy_api_key), + provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()), ); self.l1.get(&key) } @@ -101,18 +111,19 @@ impl RouterService for UnfurlRouter { UnfurlRequest::Unfurl { url, nsfw_mode, + cache_only, youtube_api_key, klipy_api_key, .. } => { if let UnfurlResponse::Resolved(result) = resp - && !result.embeds.is_empty() + && (!cache_only || !result.embeds.is_empty()) { self.l1.insert( unfurl_cache_key( url, nsfw_mode.unwrap_or_default(), - provider_capability(youtube_api_key, klipy_api_key), + provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()), ), resp.clone(), ); @@ -129,32 +140,14 @@ impl RouterService for UnfurlRouter { } } -const PROVIDER_CAPABILITIES: &[&str] = &["none", "youtube", "klipy", "youtube+klipy"]; - -fn provider_capability( - youtube_api_key: &Option, - klipy_api_key: &Option, -) -> &'static str { - match (youtube_api_key.is_some(), klipy_api_key.is_some()) { - (true, true) => "youtube+klipy", - (true, false) => "youtube", - (false, true) => "klipy", - (false, false) => "none", - } -} - -fn unfurl_cache_key(url: &str, nsfw_mode: NsfwMode, capability: &str) -> String { - let mode = match nsfw_mode { - NsfwMode::Block => "block", - NsfwMode::Flag => "flag", - NsfwMode::Allow => "allow", - }; - format!("{mode}:{capability}:{url}") -} - #[cfg(test)] mod tests { use super::*; + use crate::types::NsfwMode; + + fn test_lifetime() -> Duration { + Duration::from_secs(30 * 60) + } fn request(cache_only: bool) -> UnfurlRequest { UnfurlRequest::Unfurl { @@ -174,6 +167,13 @@ mod tests { })) } + fn empty_response() -> UnfurlResponse { + UnfurlResponse::Resolved(std::sync::Arc::new(crate::types::UnfurlResult { + embeds: Vec::new(), + cache_ttl_seconds: None, + })) + } + fn unfurl(url: &str, youtube_api_key: Option<&str>) -> UnfurlRequest { UnfurlRequest::Unfurl { url: url.to_owned(), @@ -185,9 +185,42 @@ mod tests { } } + fn unfurl_with_klipy(url: &str, klipy_api_key: Option<&str>) -> UnfurlRequest { + UnfurlRequest::Unfurl { + url: url.to_owned(), + nsfw_mode: Some(NsfwMode::Block), + bypass_cache: false, + cache_only: false, + youtube_api_key: None, + klipy_api_key: klipy_api_key.map(str::to_owned), + } + } + + fn probe(url: &str) -> UnfurlRequest { + UnfurlRequest::Unfurl { + url: url.to_owned(), + nsfw_mode: Some(NsfwMode::Block), + bypass_cache: false, + cache_only: true, + youtube_api_key: None, + klipy_api_key: None, + } + } + + fn unfurl_allowing_nsfw(url: &str) -> UnfurlRequest { + UnfurlRequest::Unfurl { + url: url.to_owned(), + nsfw_mode: Some(NsfwMode::Allow), + bypass_cache: false, + cache_only: false, + youtube_api_key: None, + klipy_api_key: None, + } + } + #[test] fn configuring_a_youtube_key_does_not_reuse_the_keyless_cache_entry() { - let router = UnfurlRouter::new(16); + let router = UnfurlRouter::new(16, test_lifetime()); let url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"; router.l1_insert(&unfurl(url, None), &resolved_response()); assert!( @@ -202,7 +235,7 @@ mod tests { #[test] fn invalidate_clears_every_provider_capability_variant() { - let router = UnfurlRouter::new(16); + let router = UnfurlRouter::new(16, test_lifetime()); let url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"; router.l1_insert(&unfurl(url, None), &resolved_response()); router.l1_insert(&unfurl(url, Some("configured")), &resolved_response()); @@ -223,4 +256,109 @@ mod tests { UnfurlRouter::coalesce_key(&request(false)) ); } + #[test] + fn a_url_that_produced_no_embed_is_remembered_instead_of_being_resolved_again() { + let router = UnfurlRouter::new(16, test_lifetime()); + let url = "https://example.com/no-metadata"; + router.l1_insert(&unfurl(url, None), &empty_response()); + assert!( + router.l1_lookup(&unfurl(url, None)).is_some(), + "an empty resolution must be served from the upper cache, not re-resolved" + ); + } + + #[test] + fn an_empty_cache_only_probe_is_never_written_to_the_upper_cache() { + let router = UnfurlRouter::new(16, test_lifetime()); + let url = "https://example.com/never-probed-into-cache"; + router.l1_insert(&probe(url), &empty_response()); + assert!( + router.l1_lookup(&unfurl(url, None)).is_none(), + "a cache-only miss means unknown, not resolved-to-nothing" + ); + } + + #[test] + fn a_cache_only_probe_that_found_embeds_still_warms_the_upper_cache() { + let router = UnfurlRouter::new(16, test_lifetime()); + let url = "https://example.com/warm"; + router.l1_insert(&probe(url), &resolved_response()); + assert!(router.l1_lookup(&unfurl(url, None)).is_some()); + } + + #[test] + fn configuring_a_klipy_key_does_not_reuse_the_keyless_cache_entry() { + let router = UnfurlRouter::new(16, test_lifetime()); + let url = "https://klipy.com/gifs/example"; + router.l1_insert(&unfurl_with_klipy(url, None), &resolved_response()); + assert!( + router + .l1_lookup(&unfurl_with_klipy(url, Some("configured"))) + .is_none(), + "a result resolved without a KLIPY key must not be served once a key is configured" + ); + assert!( + router.l1_lookup(&unfurl_with_klipy(url, None)).is_some(), + "the keyless entry must still serve keyless requests" + ); + } + + #[test] + fn an_unscanned_entry_is_not_served_to_a_scanning_request() { + let router = UnfurlRouter::new(16, test_lifetime()); + let url = "https://example.com/media"; + router.l1_insert(&unfurl_allowing_nsfw(url), &resolved_response()); + assert!( + router.l1_lookup(&unfurl(url, None)).is_none(), + "a result resolved with nsfw scanning off carries no nsfw flags and must not be reused" + ); + assert!(router.l1_lookup(&unfurl_allowing_nsfw(url)).is_some()); + } + + #[test] + fn invalidate_clears_both_scanning_and_unscanned_entries() { + let router = UnfurlRouter::new(16, test_lifetime()); + let url = "https://example.com/both"; + router.l1_insert(&unfurl(url, None), &resolved_response()); + router.l1_insert(&unfurl_allowing_nsfw(url), &resolved_response()); + router.l1_insert( + &UnfurlRequest::Invalidate { + url: url.to_owned(), + }, + &resolved_response(), + ); + assert!(router.l1_lookup(&unfurl(url, None)).is_none()); + assert!(router.l1_lookup(&unfurl_allowing_nsfw(url)).is_none()); + } + #[tokio::test] + async fn an_empty_upper_cache_entry_lapses_long_before_a_populated_one() { + let router = + UnfurlRouter::with_empty_entry_lifetime(16, test_lifetime(), Duration::from_millis(50)); + let empty_url = "https://example.com/transient"; + let populated_url = "https://example.com/stable"; + router.l1_insert(&unfurl(empty_url, None), &empty_response()); + router.l1_insert(&unfurl(populated_url, None), &resolved_response()); + + tokio::time::sleep(Duration::from_millis(200)).await; + + assert!( + router.l1_lookup(&unfurl(empty_url, None)).is_none(), + "a link that failed to unfurl must become retryable again quickly" + ); + assert!( + router.l1_lookup(&unfurl(populated_url, None)).is_some(), + "a real embed must keep the full lifetime" + ); + } + #[tokio::test] + async fn the_configured_lifetime_governs_the_upper_cache() { + let router = UnfurlRouter::new(16, Duration::from_millis(80)); + let url = "https://example.com/configured"; + router.l1_insert(&unfurl(url, None), &resolved_response()); + tokio::time::sleep(Duration::from_millis(220)).await; + assert!( + router.l1_lookup(&unfurl(url, None)).is_none(), + "the configured lifetime must govern the upper cache" + ); + } } diff --git a/fluxer_unfurl/src/shard_impl.rs b/fluxer_unfurl/src/shard_impl.rs index 2ed8bf838..8333f3aa0 100644 --- a/fluxer_unfurl/src/shard_impl.rs +++ b/fluxer_unfurl/src/shard_impl.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use crate::cache_policy::FIXED_UNFURL_CACHE_TTL_SECS; +use crate::cache_policy::{ + UnfurlEntryExpiry, default_empty_entry_lifetime, every_unfurl_cache_key, provider_capability, + unfurl_cache_key, +}; use crate::embed_normalizer::normalize_embeds; use crate::media_proxy::MediaProxyClient; use crate::resolvers::{self, ResolveContext, ResolverResult}; @@ -12,10 +15,10 @@ use std::time::Duration; use url::Url; const L2_MAX_ENTRIES: u64 = 50_000; -const L2_TTL: Duration = Duration::from_secs(FIXED_UNFURL_CACHE_TTL_SECS); pub struct UnfurlShard { cache: Cache>, + lifetime: Duration, http_client: reqwest::Client, resolvers: Vec>, media_proxy: MediaProxyClient, @@ -23,7 +26,7 @@ pub struct UnfurlShard { } impl UnfurlShard { - pub fn new() -> Self { + pub fn new(lifetime: Duration) -> Self { let http_client = external_http_client(); let media_proxy_http_client = internal_http_client(); @@ -75,10 +78,8 @@ impl UnfurlShard { ); Self { - cache: Cache::builder() - .max_capacity(L2_MAX_ENTRIES) - .time_to_live(L2_TTL) - .build(), + cache: build_l2_cache(lifetime, default_empty_entry_lifetime()), + lifetime, http_client, resolvers, media_proxy, @@ -86,6 +87,28 @@ impl UnfurlShard { } } + #[cfg(test)] + fn for_test() -> Self { + Self::for_test_with_lifetimes(long_lifetime(), default_empty_entry_lifetime()) + } + + #[cfg(test)] + fn for_test_with_lifetimes(lifetime: Duration, empty_entry_lifetime: Duration) -> Self { + Self { + cache: build_l2_cache(lifetime, empty_entry_lifetime), + lifetime, + http_client: external_http_client(), + resolvers: resolvers::build_resolver_chain(), + media_proxy: MediaProxyClient::new_with_public_endpoint( + "http://media-proxy.invalid", + "test-secret", + None, + internal_http_client(), + ), + static_cdn_endpoint: String::new(), + } + } + async fn resolve_url( &self, url_str: &str, @@ -163,11 +186,27 @@ impl UnfurlShard { fn finalize_result(&self, result: ResolverResult) -> UnfurlResult { UnfurlResult { embeds: normalize_embeds(result.embeds, &self.media_proxy), - cache_ttl_seconds: Some(FIXED_UNFURL_CACHE_TTL_SECS), + cache_ttl_seconds: Some(self.lifetime.as_secs()), } } } +#[cfg(test)] +fn long_lifetime() -> Duration { + Duration::from_secs(30 * 60) +} + +fn build_l2_cache( + lifetime: Duration, + empty_entry_lifetime: Duration, +) -> Cache> { + Cache::builder() + .max_capacity(L2_MAX_ENTRIES) + .time_to_live(lifetime) + .expire_after(UnfurlEntryExpiry::new(empty_entry_lifetime)) + .build() +} + fn base_http_client_builder() -> reqwest::ClientBuilder { reqwest::Client::builder() .user_agent("Mozilla/5.0 (compatible; Fluxerbot/1.0; +https://fluxer.app)") @@ -209,7 +248,11 @@ impl ShardService for UnfurlShard { ref klipy_api_key, } => { let nsfw = nsfw_mode.unwrap_or_default(); - let cache_key = unfurl_cache_key(url, nsfw); + let cache_key = unfurl_cache_key( + url, + nsfw, + provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()), + ); if !bypass_cache && let Some(cached) = self.cache.get(&cache_key).await { return Ok(UnfurlResponse::Resolved(cached)); @@ -239,21 +282,13 @@ impl ShardService for UnfurlShard { } }; - if !result.embeds.is_empty() { - self.cache.insert(cache_key, result.clone()).await; - } + self.cache.insert(cache_key, result.clone()).await; Ok(UnfurlResponse::Resolved(result)) } UnfurlRequest::Invalidate { ref url } => { - self.cache - .invalidate(&unfurl_cache_key(url, NsfwMode::Block)) - .await; - self.cache - .invalidate(&unfurl_cache_key(url, NsfwMode::Flag)) - .await; - self.cache - .invalidate(&unfurl_cache_key(url, NsfwMode::Allow)) - .await; + for key in every_unfurl_cache_key(url) { + self.cache.invalidate(&key).await; + } Ok(UnfurlResponse::Invalidated(InvalidatedResponse { invalidated: true, })) @@ -262,11 +297,230 @@ impl ShardService for UnfurlShard { } } -fn unfurl_cache_key(url: &str, nsfw_mode: NsfwMode) -> String { - let mode = match nsfw_mode { - NsfwMode::Block => "block", - NsfwMode::Flag => "flag", - NsfwMode::Allow => "allow", - }; - format!("{mode}:{url}") +#[cfg(test)] +mod tests { + use super::*; + use crate::types::MessageEmbed; + + const BLOCKED_URL: &str = "http://127.0.0.1/unreachable"; + + fn unfurl(url: &str, youtube_api_key: Option<&str>) -> UnfurlRequest { + UnfurlRequest::Unfurl { + url: url.to_owned(), + nsfw_mode: Some(NsfwMode::Block), + bypass_cache: false, + cache_only: false, + youtube_api_key: youtube_api_key.map(str::to_owned), + klipy_api_key: None, + } + } + + fn probe(url: &str) -> UnfurlRequest { + UnfurlRequest::Unfurl { + url: url.to_owned(), + nsfw_mode: Some(NsfwMode::Block), + bypass_cache: false, + cache_only: true, + youtube_api_key: None, + klipy_api_key: None, + } + } + + fn keyless_block_key(url: &str) -> String { + unfurl_cache_key(url, NsfwMode::Block, provider_capability(false, false)) + } + + fn embed_count(response: &UnfurlResponse) -> usize { + match response { + UnfurlResponse::Resolved(result) => result.embeds.len(), + _ => panic!("expected a resolved response"), + } + } + + async fn seed_keyless_entry(shard: &UnfurlShard) { + shard + .cache + .insert( + unfurl_cache_key( + BLOCKED_URL, + NsfwMode::Block, + provider_capability(false, false), + ), + Arc::new(UnfurlResult { + embeds: vec![MessageEmbed::new("link")], + cache_ttl_seconds: Some(long_lifetime().as_secs()), + }), + ) + .await; + } + + #[tokio::test] + async fn a_shard_entry_resolved_without_a_youtube_key_is_not_served_once_one_is_configured() { + let shard = UnfurlShard::for_test(); + seed_keyless_entry(&shard).await; + + assert_eq!( + 1, + embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap()), + "the keyless entry must still serve keyless requests" + ); + assert_eq!( + 0, + embed_count( + &shard + .handle(unfurl(BLOCKED_URL, Some("configured"))) + .await + .unwrap() + ), + "a result resolved without a YouTube key must not be served once a key is configured" + ); + } + + #[tokio::test] + async fn invalidate_clears_the_shard_entry_for_every_provider_capability() { + let shard = UnfurlShard::for_test(); + seed_keyless_entry(&shard).await; + + shard + .handle(UnfurlRequest::Invalidate { + url: BLOCKED_URL.to_owned(), + }) + .await + .unwrap(); + + assert_eq!( + 0, + embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap()) + ); + } + #[tokio::test] + async fn a_url_that_produced_no_embed_is_written_to_the_shard_cache() { + let shard = UnfurlShard::for_test(); + + assert_eq!( + 0, + embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap()) + ); + assert!( + shard + .cache + .get(&keyless_block_key(BLOCKED_URL)) + .await + .is_some(), + "an empty resolution must be remembered so the next message does not refetch the page" + ); + } + + #[tokio::test] + async fn a_cache_only_probe_never_writes_an_entry() { + let shard = UnfurlShard::for_test(); + + assert_eq!( + 0, + embed_count(&shard.handle(probe(BLOCKED_URL)).await.unwrap()) + ); + assert!( + shard + .cache + .get(&keyless_block_key(BLOCKED_URL)) + .await + .is_none(), + "a cache-only miss means unknown, not resolved-to-nothing" + ); + } + + #[tokio::test] + async fn an_unscanned_entry_is_not_served_to_a_scanning_request() { + let shard = UnfurlShard::for_test(); + shard + .cache + .insert( + unfurl_cache_key( + BLOCKED_URL, + NsfwMode::Allow, + provider_capability(false, false), + ), + Arc::new(UnfurlResult { + embeds: vec![MessageEmbed::new("link")], + cache_ttl_seconds: Some(long_lifetime().as_secs()), + }), + ) + .await; + + assert_eq!( + 0, + embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap()), + "a result resolved with nsfw scanning off must not be served to a scanning request" + ); + } + #[tokio::test] + async fn an_empty_shard_entry_lapses_long_before_a_populated_one() { + let shard = + UnfurlShard::for_test_with_lifetimes(long_lifetime(), Duration::from_millis(50)); + let empty_key = keyless_block_key("https://example.com/transient"); + let populated_key = keyless_block_key("https://example.com/stable"); + shard + .cache + .insert( + empty_key.clone(), + Arc::new(UnfurlResult { + embeds: Vec::new(), + cache_ttl_seconds: None, + }), + ) + .await; + shard + .cache + .insert( + populated_key.clone(), + Arc::new(UnfurlResult { + embeds: vec![MessageEmbed::new("link")], + cache_ttl_seconds: Some(long_lifetime().as_secs()), + }), + ) + .await; + + tokio::time::sleep(Duration::from_millis(200)).await; + + assert!( + shard.cache.get(&empty_key).await.is_none(), + "a link that failed to unfurl must become retryable again quickly" + ); + assert!( + shard.cache.get(&populated_key).await.is_some(), + "a real embed must keep the full lifetime" + ); + } + #[tokio::test] + async fn the_configured_lifetime_governs_the_shard_cache_and_the_reported_ttl() { + let shard = UnfurlShard::for_test_with_lifetimes( + Duration::from_millis(80), + default_empty_entry_lifetime(), + ); + let finalized = shard.finalize_result(crate::resolvers::ResolverResult { + embeds: vec![MessageEmbed::new("link")], + }); + assert_eq!( + Some(Duration::from_millis(80).as_secs()), + finalized.cache_ttl_seconds, + "the reported ttl must be the configured lifetime, not a compiled-in constant" + ); + + let key = keyless_block_key("https://example.com/configured"); + shard + .cache + .insert( + key.clone(), + Arc::new(UnfurlResult { + embeds: vec![MessageEmbed::new("link")], + cache_ttl_seconds: None, + }), + ) + .await; + tokio::time::sleep(Duration::from_millis(220)).await; + assert!( + shard.cache.get(&key).await.is_none(), + "the configured lifetime must govern the shard cache" + ); + } }