fix(unfurl): cache empty results briefly and key entries by provider (#1838)

fluxer_unfurl/src/cache_policy.rs

@@ -1,3 +1,230 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-pub const FIXED_UNFURL_CACHE_TTL_SECS: u64 = 30 * 60;
+use crate::types::{NsfwMode, UnfurlResponse, UnfurlResult};
+use moka::Expiry;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+pub const EMPTY_UNFURL_CACHE_TTL_SECS: u64 = 60;
+
+const SCAN_TOKENS: [&str; 2] = ["scanned", "unscanned"];
+
+const PROVIDER_CAPABILITIES: [&str; 4] = ["none", "youtube", "klipy", "youtube+klipy"];
+
+pub fn nsfw_scan_token(nsfw_mode: NsfwMode) -> &'static str {
+    match nsfw_mode {
+        NsfwMode::Block | NsfwMode::Flag => SCAN_TOKENS[0],
+        NsfwMode::Allow => SCAN_TOKENS[1],
+    }
+}
+
+pub fn provider_capability(has_youtube_key: bool, has_klipy_key: bool) -> &'static str {
+    match (has_youtube_key, has_klipy_key) {
+        (true, true) => "youtube+klipy",
+        (true, false) => "youtube",
+        (false, true) => "klipy",
+        (false, false) => "none",
+    }
+}
+
+pub fn unfurl_cache_key(url: &str, nsfw_mode: NsfwMode, capability: &str) -> String {
+    let scan = nsfw_scan_token(nsfw_mode);
+    format!("{scan}:{capability}:{url}")
+}
+
+pub fn every_unfurl_cache_key(url: &str) -> Vec<String> {
+    let mut keys = Vec::with_capacity(SCAN_TOKENS.len() * PROVIDER_CAPABILITIES.len());
+    for scan in SCAN_TOKENS {
+        for capability in PROVIDER_CAPABILITIES {
+            keys.push(format!("{scan}:{capability}:{url}"));
+        }
+    }
+    keys
+}
+
+pub fn default_empty_entry_lifetime() -> Duration {
+    Duration::from_secs(EMPTY_UNFURL_CACHE_TTL_SECS)
+}
+
+pub struct UnfurlEntryExpiry {
+    empty_entry_lifetime: Duration,
+}
+
+impl UnfurlEntryExpiry {
+    pub fn new(empty_entry_lifetime: Duration) -> Self {
+        Self {
+            empty_entry_lifetime,
+        }
+    }
+
+    fn lifetime(&self, has_embeds: bool) -> Option<Duration> {
+        if has_embeds {
+            None
+        } else {
+            Some(self.empty_entry_lifetime)
+        }
+    }
+}
+
+impl Expiry<String, Arc<UnfurlResult>> for UnfurlEntryExpiry {
+    fn expire_after_create(
+        &self,
+        _key: &String,
+        value: &Arc<UnfurlResult>,
+        _created_at: Instant,
+    ) -> Option<Duration> {
+        self.lifetime(!value.embeds.is_empty())
+    }
+
+    fn expire_after_update(
+        &self,
+        _key: &String,
+        value: &Arc<UnfurlResult>,
+        _updated_at: Instant,
+        _duration_until_expiry: Option<Duration>,
+    ) -> Option<Duration> {
+        self.lifetime(!value.embeds.is_empty())
+    }
+}
+
+impl Expiry<String, UnfurlResponse> for UnfurlEntryExpiry {
+    fn expire_after_create(
+        &self,
+        _key: &String,
+        value: &UnfurlResponse,
+        _created_at: Instant,
+    ) -> Option<Duration> {
+        self.lifetime(response_has_embeds(value))
+    }
+
+    fn expire_after_update(
+        &self,
+        _key: &String,
+        value: &UnfurlResponse,
+        _updated_at: Instant,
+        _duration_until_expiry: Option<Duration>,
+    ) -> Option<Duration> {
+        self.lifetime(response_has_embeds(value))
+    }
+}
+
+fn response_has_embeds(response: &UnfurlResponse) -> bool {
+    match response {
+        UnfurlResponse::Resolved(result) => !result.embeds.is_empty(),
+        _ => false,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::types::MessageEmbed;
+
+    const URL: &str = "https://example.com/a";
+
+    const EMPTY_UNFURL_CACHE_TTL_CEILING_SECS: u64 = 5 * 60;
+
+    fn resolved(embed_count: usize) -> UnfurlResponse {
+        UnfurlResponse::Resolved(Arc::new(UnfurlResult {
+            embeds: (0..embed_count)
+                .map(|_| MessageEmbed::new("link"))
+                .collect(),
+            cache_ttl_seconds: None,
+        }))
+    }
+
+    #[test]
+    fn every_key_covers_the_full_scan_and_capability_matrix() {
+        let keys = every_unfurl_cache_key(URL);
+        assert_eq!(keys.len(), 8);
+        let unique: std::collections::HashSet<&String> = keys.iter().collect();
+        assert_eq!(unique.len(), keys.len(), "invalidation keys must be unique");
+        for nsfw_mode in [NsfwMode::Block, NsfwMode::Flag, NsfwMode::Allow] {
+            for capability in PROVIDER_CAPABILITIES {
+                assert!(
+                    keys.contains(&unfurl_cache_key(URL, nsfw_mode, capability)),
+                    "invalidation must cover {nsfw_mode:?}/{capability}"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn scanning_and_unscanned_requests_do_not_share_an_entry() {
+        assert_ne!(
+            unfurl_cache_key(URL, NsfwMode::Block, "none"),
+            unfurl_cache_key(URL, NsfwMode::Allow, "none"),
+            "an unscanned result carries no nsfw media flag and must not serve a scanning request"
+        );
+    }
+
+    #[test]
+    fn the_two_scanning_modes_share_a_single_entry() {
+        assert_eq!(
+            unfurl_cache_key(URL, NsfwMode::Block, "none"),
+            unfurl_cache_key(URL, NsfwMode::Flag, "none")
+        );
+    }
+
+    #[test]
+    fn capability_distinguishes_every_key_combination() {
+        let all = [
+            provider_capability(false, false),
+            provider_capability(true, false),
+            provider_capability(false, true),
+            provider_capability(true, true),
+        ];
+        let unique: std::collections::HashSet<&str> = all.iter().copied().collect();
+        assert_eq!(unique.len(), all.len());
+        for capability in all {
+            assert_ne!(
+                unfurl_cache_key(URL, NsfwMode::Block, capability),
+                unfurl_cache_key(URL, NsfwMode::Block, "unrelated")
+            );
+        }
+    }
+
+    #[test]
+    fn the_url_distinguishes_entries() {
+        assert_ne!(
+            unfurl_cache_key(URL, NsfwMode::Block, "none"),
+            unfurl_cache_key("https://example.com/b", NsfwMode::Block, "none")
+        );
+    }
+
+    #[test]
+    fn the_empty_entry_lifetime_stays_a_short_retry_window() {
+        assert!(
+            default_empty_entry_lifetime()
+                <= Duration::from_secs(EMPTY_UNFURL_CACHE_TTL_CEILING_SECS),
+            "a site that was merely down must become retryable again quickly"
+        );
+    }
+
+    #[test]
+    fn the_expiry_reads_the_short_lifetime_off_an_empty_response() {
+        let expiry = UnfurlEntryExpiry::new(default_empty_entry_lifetime());
+        let now = Instant::now();
+        let key = unfurl_cache_key(URL, NsfwMode::Block, "none");
+        assert_eq!(
+            Some(default_empty_entry_lifetime()),
+            Expiry::<String, UnfurlResponse>::expire_after_create(&expiry, &key, &resolved(0), now)
+        );
+        assert_eq!(
+            None,
+            Expiry::<String, UnfurlResponse>::expire_after_create(&expiry, &key, &resolved(1), now)
+        );
+        assert_eq!(
+            Some(default_empty_entry_lifetime()),
+            Expiry::<String, Arc<UnfurlResult>>::expire_after_create(
+                &expiry,
+                &key,
+                &Arc::new(UnfurlResult {
+                    embeds: Vec::new(),
+                    cache_ttl_seconds: None,
+                }),
+                now
+            )
+        );
+    }
+}

fluxer_unfurl/src/main.rs

@@ -40,11 +40,11 @@ async fn main() -> anyhow::Result<()> {
 
     match config.mode {
         Mode::Router => {
-            let router = UnfurlRouter::new(config.cache_max_entries);
+            let router = UnfurlRouter::new(config.cache_max_entries, config.cache_ttl);
             fluxer_svc::router::run_router(&config, router, transport).await
         }
         Mode::Shard => {
-            let shard = UnfurlShard::new();
+            let shard = UnfurlShard::new(config.cache_ttl);
             fluxer_svc::shard::run_shard(&config, shard, transport).await
         }
     }

fluxer_unfurl/src/router_impl.rs

@@ -1,7 +1,10 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-use crate::cache_policy::FIXED_UNFURL_CACHE_TTL_SECS;
-use crate::types::{NsfwMode, UnfurlRequest, UnfurlResponse};
+use crate::cache_policy::{
+    UnfurlEntryExpiry, default_empty_entry_lifetime, every_unfurl_cache_key, provider_capability,
+    unfurl_cache_key,
+};
+use crate::types::{UnfurlRequest, UnfurlResponse};
 use fluxer_svc::router::RouterService;
 use moka::sync::Cache;
 use std::time::Duration;
@@ -11,20 +14,27 @@ pub struct UnfurlRouter {
 }
 
 impl UnfurlRouter {
-    pub fn new(max_entries: u64) -> Self {
+    pub fn new(max_entries: u64, lifetime: Duration) -> Self {
+        Self::with_empty_entry_lifetime(max_entries, lifetime, default_empty_entry_lifetime())
+    }
+
+    fn with_empty_entry_lifetime(
+        max_entries: u64,
+        lifetime: Duration,
+        empty_entry_lifetime: Duration,
+    ) -> Self {
         Self {
             l1: Cache::builder()
                 .max_capacity(max_entries)
-                .time_to_live(Duration::from_secs(FIXED_UNFURL_CACHE_TTL_SECS))
+                .time_to_live(lifetime)
+                .expire_after(UnfurlEntryExpiry::new(empty_entry_lifetime))
                 .build(),
         }
     }
 
     fn invalidate_all_variants(&self, url: &str) {
-        for mode in [NsfwMode::Block, NsfwMode::Flag, NsfwMode::Allow] {
-            for capability in PROVIDER_CAPABILITIES {
-                self.l1.invalidate(&unfurl_cache_key(url, mode, capability));
-            }
+        for key in every_unfurl_cache_key(url) {
+            self.l1.invalidate(&key);
         }
     }
 }
@@ -63,7 +73,7 @@ impl RouterService for UnfurlRouter {
                     unfurl_cache_key(
                         url,
                         nsfw_mode.unwrap_or_default(),
-                        provider_capability(youtube_api_key, klipy_api_key)
+                        provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some())
                     )
                 ))
             }
@@ -87,7 +97,7 @@ impl RouterService for UnfurlRouter {
                     let key = unfurl_cache_key(
                         url,
                         nsfw_mode.unwrap_or_default(),
-                        provider_capability(youtube_api_key, klipy_api_key),
+                        provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()),
                     );
                     self.l1.get(&key)
                 }
@@ -101,18 +111,19 @@ impl RouterService for UnfurlRouter {
             UnfurlRequest::Unfurl {
                 url,
                 nsfw_mode,
+                cache_only,
                 youtube_api_key,
                 klipy_api_key,
                 ..
             } => {
                 if let UnfurlResponse::Resolved(result) = resp
-                    && !result.embeds.is_empty()
+                    && (!cache_only || !result.embeds.is_empty())
                 {
                     self.l1.insert(
                         unfurl_cache_key(
                             url,
                             nsfw_mode.unwrap_or_default(),
-                            provider_capability(youtube_api_key, klipy_api_key),
+                            provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()),
                         ),
                         resp.clone(),
                     );
@@ -129,32 +140,14 @@ impl RouterService for UnfurlRouter {
     }
 }
 
-const PROVIDER_CAPABILITIES: &[&str] = &["none", "youtube", "klipy", "youtube+klipy"];
-
-fn provider_capability(
-    youtube_api_key: &Option<String>,
-    klipy_api_key: &Option<String>,
-) -> &'static str {
-    match (youtube_api_key.is_some(), klipy_api_key.is_some()) {
-        (true, true) => "youtube+klipy",
-        (true, false) => "youtube",
-        (false, true) => "klipy",
-        (false, false) => "none",
-    }
-}
-
-fn unfurl_cache_key(url: &str, nsfw_mode: NsfwMode, capability: &str) -> String {
-    let mode = match nsfw_mode {
-        NsfwMode::Block => "block",
-        NsfwMode::Flag => "flag",
-        NsfwMode::Allow => "allow",
-    };
-    format!("{mode}:{capability}:{url}")
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::types::NsfwMode;
+
+    fn test_lifetime() -> Duration {
+        Duration::from_secs(30 * 60)
+    }
 
     fn request(cache_only: bool) -> UnfurlRequest {
         UnfurlRequest::Unfurl {
@@ -174,6 +167,13 @@ mod tests {
         }))
     }
 
+    fn empty_response() -> UnfurlResponse {
+        UnfurlResponse::Resolved(std::sync::Arc::new(crate::types::UnfurlResult {
+            embeds: Vec::new(),
+            cache_ttl_seconds: None,
+        }))
+    }
+
     fn unfurl(url: &str, youtube_api_key: Option<&str>) -> UnfurlRequest {
         UnfurlRequest::Unfurl {
             url: url.to_owned(),
@@ -185,9 +185,42 @@ mod tests {
         }
     }
 
+    fn unfurl_with_klipy(url: &str, klipy_api_key: Option<&str>) -> UnfurlRequest {
+        UnfurlRequest::Unfurl {
+            url: url.to_owned(),
+            nsfw_mode: Some(NsfwMode::Block),
+            bypass_cache: false,
+            cache_only: false,
+            youtube_api_key: None,
+            klipy_api_key: klipy_api_key.map(str::to_owned),
+        }
+    }
+
+    fn probe(url: &str) -> UnfurlRequest {
+        UnfurlRequest::Unfurl {
+            url: url.to_owned(),
+            nsfw_mode: Some(NsfwMode::Block),
+            bypass_cache: false,
+            cache_only: true,
+            youtube_api_key: None,
+            klipy_api_key: None,
+        }
+    }
+
+    fn unfurl_allowing_nsfw(url: &str) -> UnfurlRequest {
+        UnfurlRequest::Unfurl {
+            url: url.to_owned(),
+            nsfw_mode: Some(NsfwMode::Allow),
+            bypass_cache: false,
+            cache_only: false,
+            youtube_api_key: None,
+            klipy_api_key: None,
+        }
+    }
+
     #[test]
     fn configuring_a_youtube_key_does_not_reuse_the_keyless_cache_entry() {
-        let router = UnfurlRouter::new(16);
+        let router = UnfurlRouter::new(16, test_lifetime());
         let url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ";
         router.l1_insert(&unfurl(url, None), &resolved_response());
         assert!(
@@ -202,7 +235,7 @@ mod tests {
 
     #[test]
     fn invalidate_clears_every_provider_capability_variant() {
-        let router = UnfurlRouter::new(16);
+        let router = UnfurlRouter::new(16, test_lifetime());
         let url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ";
         router.l1_insert(&unfurl(url, None), &resolved_response());
         router.l1_insert(&unfurl(url, Some("configured")), &resolved_response());
@@ -223,4 +256,109 @@ mod tests {
             UnfurlRouter::coalesce_key(&request(false))
         );
     }
+    #[test]
+    fn a_url_that_produced_no_embed_is_remembered_instead_of_being_resolved_again() {
+        let router = UnfurlRouter::new(16, test_lifetime());
+        let url = "https://example.com/no-metadata";
+        router.l1_insert(&unfurl(url, None), &empty_response());
+        assert!(
+            router.l1_lookup(&unfurl(url, None)).is_some(),
+            "an empty resolution must be served from the upper cache, not re-resolved"
+        );
+    }
+
+    #[test]
+    fn an_empty_cache_only_probe_is_never_written_to_the_upper_cache() {
+        let router = UnfurlRouter::new(16, test_lifetime());
+        let url = "https://example.com/never-probed-into-cache";
+        router.l1_insert(&probe(url), &empty_response());
+        assert!(
+            router.l1_lookup(&unfurl(url, None)).is_none(),
+            "a cache-only miss means unknown, not resolved-to-nothing"
+        );
+    }
+
+    #[test]
+    fn a_cache_only_probe_that_found_embeds_still_warms_the_upper_cache() {
+        let router = UnfurlRouter::new(16, test_lifetime());
+        let url = "https://example.com/warm";
+        router.l1_insert(&probe(url), &resolved_response());
+        assert!(router.l1_lookup(&unfurl(url, None)).is_some());
+    }
+
+    #[test]
+    fn configuring_a_klipy_key_does_not_reuse_the_keyless_cache_entry() {
+        let router = UnfurlRouter::new(16, test_lifetime());
+        let url = "https://klipy.com/gifs/example";
+        router.l1_insert(&unfurl_with_klipy(url, None), &resolved_response());
+        assert!(
+            router
+                .l1_lookup(&unfurl_with_klipy(url, Some("configured")))
+                .is_none(),
+            "a result resolved without a KLIPY key must not be served once a key is configured"
+        );
+        assert!(
+            router.l1_lookup(&unfurl_with_klipy(url, None)).is_some(),
+            "the keyless entry must still serve keyless requests"
+        );
+    }
+
+    #[test]
+    fn an_unscanned_entry_is_not_served_to_a_scanning_request() {
+        let router = UnfurlRouter::new(16, test_lifetime());
+        let url = "https://example.com/media";
+        router.l1_insert(&unfurl_allowing_nsfw(url), &resolved_response());
+        assert!(
+            router.l1_lookup(&unfurl(url, None)).is_none(),
+            "a result resolved with nsfw scanning off carries no nsfw flags and must not be reused"
+        );
+        assert!(router.l1_lookup(&unfurl_allowing_nsfw(url)).is_some());
+    }
+
+    #[test]
+    fn invalidate_clears_both_scanning_and_unscanned_entries() {
+        let router = UnfurlRouter::new(16, test_lifetime());
+        let url = "https://example.com/both";
+        router.l1_insert(&unfurl(url, None), &resolved_response());
+        router.l1_insert(&unfurl_allowing_nsfw(url), &resolved_response());
+        router.l1_insert(
+            &UnfurlRequest::Invalidate {
+                url: url.to_owned(),
+            },
+            &resolved_response(),
+        );
+        assert!(router.l1_lookup(&unfurl(url, None)).is_none());
+        assert!(router.l1_lookup(&unfurl_allowing_nsfw(url)).is_none());
+    }
+    #[tokio::test]
+    async fn an_empty_upper_cache_entry_lapses_long_before_a_populated_one() {
+        let router =
+            UnfurlRouter::with_empty_entry_lifetime(16, test_lifetime(), Duration::from_millis(50));
+        let empty_url = "https://example.com/transient";
+        let populated_url = "https://example.com/stable";
+        router.l1_insert(&unfurl(empty_url, None), &empty_response());
+        router.l1_insert(&unfurl(populated_url, None), &resolved_response());
+
+        tokio::time::sleep(Duration::from_millis(200)).await;
+
+        assert!(
+            router.l1_lookup(&unfurl(empty_url, None)).is_none(),
+            "a link that failed to unfurl must become retryable again quickly"
+        );
+        assert!(
+            router.l1_lookup(&unfurl(populated_url, None)).is_some(),
+            "a real embed must keep the full lifetime"
+        );
+    }
+    #[tokio::test]
+    async fn the_configured_lifetime_governs_the_upper_cache() {
+        let router = UnfurlRouter::new(16, Duration::from_millis(80));
+        let url = "https://example.com/configured";
+        router.l1_insert(&unfurl(url, None), &resolved_response());
+        tokio::time::sleep(Duration::from_millis(220)).await;
+        assert!(
+            router.l1_lookup(&unfurl(url, None)).is_none(),
+            "the configured lifetime must govern the upper cache"
+        );
+    }
 }

fluxer_unfurl/src/shard_impl.rs

@@ -1,6 +1,9 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-use crate::cache_policy::FIXED_UNFURL_CACHE_TTL_SECS;
+use crate::cache_policy::{
+    UnfurlEntryExpiry, default_empty_entry_lifetime, every_unfurl_cache_key, provider_capability,
+    unfurl_cache_key,
+};
 use crate::embed_normalizer::normalize_embeds;
 use crate::media_proxy::MediaProxyClient;
 use crate::resolvers::{self, ResolveContext, ResolverResult};
@@ -12,10 +15,10 @@ use std::time::Duration;
 use url::Url;
 
 const L2_MAX_ENTRIES: u64 = 50_000;
-const L2_TTL: Duration = Duration::from_secs(FIXED_UNFURL_CACHE_TTL_SECS);
 
 pub struct UnfurlShard {
     cache: Cache<String, Arc<UnfurlResult>>,
+    lifetime: Duration,
     http_client: reqwest::Client,
     resolvers: Vec<Box<dyn crate::resolvers::Resolver>>,
     media_proxy: MediaProxyClient,
@@ -23,7 +26,7 @@ pub struct UnfurlShard {
 }
 
 impl UnfurlShard {
-    pub fn new() -> Self {
+    pub fn new(lifetime: Duration) -> Self {
         let http_client = external_http_client();
         let media_proxy_http_client = internal_http_client();
 
@@ -75,10 +78,8 @@ impl UnfurlShard {
         );
 
         Self {
-            cache: Cache::builder()
-                .max_capacity(L2_MAX_ENTRIES)
-                .time_to_live(L2_TTL)
-                .build(),
+            cache: build_l2_cache(lifetime, default_empty_entry_lifetime()),
+            lifetime,
             http_client,
             resolvers,
             media_proxy,
@@ -86,6 +87,28 @@ impl UnfurlShard {
         }
     }
 
+    #[cfg(test)]
+    fn for_test() -> Self {
+        Self::for_test_with_lifetimes(long_lifetime(), default_empty_entry_lifetime())
+    }
+
+    #[cfg(test)]
+    fn for_test_with_lifetimes(lifetime: Duration, empty_entry_lifetime: Duration) -> Self {
+        Self {
+            cache: build_l2_cache(lifetime, empty_entry_lifetime),
+            lifetime,
+            http_client: external_http_client(),
+            resolvers: resolvers::build_resolver_chain(),
+            media_proxy: MediaProxyClient::new_with_public_endpoint(
+                "http://media-proxy.invalid",
+                "test-secret",
+                None,
+                internal_http_client(),
+            ),
+            static_cdn_endpoint: String::new(),
+        }
+    }
+
     async fn resolve_url(
         &self,
         url_str: &str,
@@ -163,11 +186,27 @@ impl UnfurlShard {
     fn finalize_result(&self, result: ResolverResult) -> UnfurlResult {
         UnfurlResult {
             embeds: normalize_embeds(result.embeds, &self.media_proxy),
-            cache_ttl_seconds: Some(FIXED_UNFURL_CACHE_TTL_SECS),
+            cache_ttl_seconds: Some(self.lifetime.as_secs()),
         }
     }
 }
 
+#[cfg(test)]
+fn long_lifetime() -> Duration {
+    Duration::from_secs(30 * 60)
+}
+
+fn build_l2_cache(
+    lifetime: Duration,
+    empty_entry_lifetime: Duration,
+) -> Cache<String, Arc<UnfurlResult>> {
+    Cache::builder()
+        .max_capacity(L2_MAX_ENTRIES)
+        .time_to_live(lifetime)
+        .expire_after(UnfurlEntryExpiry::new(empty_entry_lifetime))
+        .build()
+}
+
 fn base_http_client_builder() -> reqwest::ClientBuilder {
     reqwest::Client::builder()
         .user_agent("Mozilla/5.0 (compatible; Fluxerbot/1.0; +https://fluxer.app)")
@@ -209,7 +248,11 @@ impl ShardService for UnfurlShard {
                 ref klipy_api_key,
             } => {
                 let nsfw = nsfw_mode.unwrap_or_default();
-                let cache_key = unfurl_cache_key(url, nsfw);
+                let cache_key = unfurl_cache_key(
+                    url,
+                    nsfw,
+                    provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()),
+                );
 
                 if !bypass_cache && let Some(cached) = self.cache.get(&cache_key).await {
                     return Ok(UnfurlResponse::Resolved(cached));
@@ -239,21 +282,13 @@ impl ShardService for UnfurlShard {
                     }
                 };
 
-                if !result.embeds.is_empty() {
-                    self.cache.insert(cache_key, result.clone()).await;
-                }
+                self.cache.insert(cache_key, result.clone()).await;
                 Ok(UnfurlResponse::Resolved(result))
             }
             UnfurlRequest::Invalidate { ref url } => {
-                self.cache
-                    .invalidate(&unfurl_cache_key(url, NsfwMode::Block))
-                    .await;
-                self.cache
-                    .invalidate(&unfurl_cache_key(url, NsfwMode::Flag))
-                    .await;
-                self.cache
-                    .invalidate(&unfurl_cache_key(url, NsfwMode::Allow))
-                    .await;
+                for key in every_unfurl_cache_key(url) {
+                    self.cache.invalidate(&key).await;
+                }
                 Ok(UnfurlResponse::Invalidated(InvalidatedResponse {
                     invalidated: true,
                 }))
@@ -262,11 +297,230 @@ impl ShardService for UnfurlShard {
     }
 }
 
-fn unfurl_cache_key(url: &str, nsfw_mode: NsfwMode) -> String {
-    let mode = match nsfw_mode {
-        NsfwMode::Block => "block",
-        NsfwMode::Flag => "flag",
-        NsfwMode::Allow => "allow",
-    };
-    format!("{mode}:{url}")
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::types::MessageEmbed;
+
+    const BLOCKED_URL: &str = "http://127.0.0.1/unreachable";
+
+    fn unfurl(url: &str, youtube_api_key: Option<&str>) -> UnfurlRequest {
+        UnfurlRequest::Unfurl {
+            url: url.to_owned(),
+            nsfw_mode: Some(NsfwMode::Block),
+            bypass_cache: false,
+            cache_only: false,
+            youtube_api_key: youtube_api_key.map(str::to_owned),
+            klipy_api_key: None,
+        }
+    }
+
+    fn probe(url: &str) -> UnfurlRequest {
+        UnfurlRequest::Unfurl {
+            url: url.to_owned(),
+            nsfw_mode: Some(NsfwMode::Block),
+            bypass_cache: false,
+            cache_only: true,
+            youtube_api_key: None,
+            klipy_api_key: None,
+        }
+    }
+
+    fn keyless_block_key(url: &str) -> String {
+        unfurl_cache_key(url, NsfwMode::Block, provider_capability(false, false))
+    }
+
+    fn embed_count(response: &UnfurlResponse) -> usize {
+        match response {
+            UnfurlResponse::Resolved(result) => result.embeds.len(),
+            _ => panic!("expected a resolved response"),
+        }
+    }
+
+    async fn seed_keyless_entry(shard: &UnfurlShard) {
+        shard
+            .cache
+            .insert(
+                unfurl_cache_key(
+                    BLOCKED_URL,
+                    NsfwMode::Block,
+                    provider_capability(false, false),
+                ),
+                Arc::new(UnfurlResult {
+                    embeds: vec![MessageEmbed::new("link")],
+                    cache_ttl_seconds: Some(long_lifetime().as_secs()),
+                }),
+            )
+            .await;
+    }
+
+    #[tokio::test]
+    async fn a_shard_entry_resolved_without_a_youtube_key_is_not_served_once_one_is_configured() {
+        let shard = UnfurlShard::for_test();
+        seed_keyless_entry(&shard).await;
+
+        assert_eq!(
+            1,
+            embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap()),
+            "the keyless entry must still serve keyless requests"
+        );
+        assert_eq!(
+            0,
+            embed_count(
+                &shard
+                    .handle(unfurl(BLOCKED_URL, Some("configured")))
+                    .await
+                    .unwrap()
+            ),
+            "a result resolved without a YouTube key must not be served once a key is configured"
+        );
+    }
+
+    #[tokio::test]
+    async fn invalidate_clears_the_shard_entry_for_every_provider_capability() {
+        let shard = UnfurlShard::for_test();
+        seed_keyless_entry(&shard).await;
+
+        shard
+            .handle(UnfurlRequest::Invalidate {
+                url: BLOCKED_URL.to_owned(),
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(
+            0,
+            embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap())
+        );
+    }
+    #[tokio::test]
+    async fn a_url_that_produced_no_embed_is_written_to_the_shard_cache() {
+        let shard = UnfurlShard::for_test();
+
+        assert_eq!(
+            0,
+            embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap())
+        );
+        assert!(
+            shard
+                .cache
+                .get(&keyless_block_key(BLOCKED_URL))
+                .await
+                .is_some(),
+            "an empty resolution must be remembered so the next message does not refetch the page"
+        );
+    }
+
+    #[tokio::test]
+    async fn a_cache_only_probe_never_writes_an_entry() {
+        let shard = UnfurlShard::for_test();
+
+        assert_eq!(
+            0,
+            embed_count(&shard.handle(probe(BLOCKED_URL)).await.unwrap())
+        );
+        assert!(
+            shard
+                .cache
+                .get(&keyless_block_key(BLOCKED_URL))
+                .await
+                .is_none(),
+            "a cache-only miss means unknown, not resolved-to-nothing"
+        );
+    }
+
+    #[tokio::test]
+    async fn an_unscanned_entry_is_not_served_to_a_scanning_request() {
+        let shard = UnfurlShard::for_test();
+        shard
+            .cache
+            .insert(
+                unfurl_cache_key(
+                    BLOCKED_URL,
+                    NsfwMode::Allow,
+                    provider_capability(false, false),
+                ),
+                Arc::new(UnfurlResult {
+                    embeds: vec![MessageEmbed::new("link")],
+                    cache_ttl_seconds: Some(long_lifetime().as_secs()),
+                }),
+            )
+            .await;
+
+        assert_eq!(
+            0,
+            embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap()),
+            "a result resolved with nsfw scanning off must not be served to a scanning request"
+        );
+    }
+    #[tokio::test]
+    async fn an_empty_shard_entry_lapses_long_before_a_populated_one() {
+        let shard =
+            UnfurlShard::for_test_with_lifetimes(long_lifetime(), Duration::from_millis(50));
+        let empty_key = keyless_block_key("https://example.com/transient");
+        let populated_key = keyless_block_key("https://example.com/stable");
+        shard
+            .cache
+            .insert(
+                empty_key.clone(),
+                Arc::new(UnfurlResult {
+                    embeds: Vec::new(),
+                    cache_ttl_seconds: None,
+                }),
+            )
+            .await;
+        shard
+            .cache
+            .insert(
+                populated_key.clone(),
+                Arc::new(UnfurlResult {
+                    embeds: vec![MessageEmbed::new("link")],
+                    cache_ttl_seconds: Some(long_lifetime().as_secs()),
+                }),
+            )
+            .await;
+
+        tokio::time::sleep(Duration::from_millis(200)).await;
+
+        assert!(
+            shard.cache.get(&empty_key).await.is_none(),
+            "a link that failed to unfurl must become retryable again quickly"
+        );
+        assert!(
+            shard.cache.get(&populated_key).await.is_some(),
+            "a real embed must keep the full lifetime"
+        );
+    }
+    #[tokio::test]
+    async fn the_configured_lifetime_governs_the_shard_cache_and_the_reported_ttl() {
+        let shard = UnfurlShard::for_test_with_lifetimes(
+            Duration::from_millis(80),
+            default_empty_entry_lifetime(),
+        );
+        let finalized = shard.finalize_result(crate::resolvers::ResolverResult {
+            embeds: vec![MessageEmbed::new("link")],
+        });
+        assert_eq!(
+            Some(Duration::from_millis(80).as_secs()),
+            finalized.cache_ttl_seconds,
+            "the reported ttl must be the configured lifetime, not a compiled-in constant"
+        );
+
+        let key = keyless_block_key("https://example.com/configured");
+        shard
+            .cache
+            .insert(
+                key.clone(),
+                Arc::new(UnfurlResult {
+                    embeds: vec![MessageEmbed::new("link")],
+                    cache_ttl_seconds: None,
+                }),
+            )
+            .await;
+        tokio::time::sleep(Duration::from_millis(220)).await;
+        assert!(
+            shard.cache.get(&key).await.is_none(),
+            "the configured lifetime must govern the shard cache"
+        );
+    }
 }