Add SimilarityLFU cache admission policy

tests/unit/AdmissionPolicyStrategy/test_admission_policy.py

@@ -0,0 +1,114 @@
+import unittest
+from unittest.mock import MagicMock
+
+from vcache.vcache_core.cache.admission_policy.strategies.always_admit import (
+    AlwaysAdmitPolicy,
+)
+from vcache.vcache_core.cache.admission_policy.strategies.similarity_lfu import (
+    SimilarityLFUAdmissionPolicy,
+)
+from vcache.vcache_core.cache.cache import Cache
+
+
+class TestAlwaysAdmitPolicy(unittest.TestCase):
+    def test_always_admits(self):
+        """AlwaysAdmitPolicy should admit every embedding, unconditionally."""
+        policy = AlwaysAdmitPolicy()
+        self.assertTrue(policy.should_admit([1.0, 0.0, 0.0]))
+        self.assertTrue(policy.should_admit([0.0, 0.0, 0.0]))
+
+
+class TestSimilarityLFUAdmissionPolicy(unittest.TestCase):
+    def test_first_sighting_is_not_admitted(self):
+        """A never-seen-before embedding should be put on probation, not admitted."""
+        policy = SimilarityLFUAdmissionPolicy(admission_similarity_threshold=0.9)
+        self.assertFalse(policy.should_admit([1.0, 0.0, 0.0]))
+
+    def test_similar_second_sighting_is_admitted(self):
+        """A repeat of a probated embedding (above the similarity threshold) is admitted."""
+        policy = SimilarityLFUAdmissionPolicy(admission_similarity_threshold=0.9)
+        policy.should_admit([1.0, 0.0, 0.0])
+        self.assertTrue(policy.should_admit([1.0, 0.0001, 0.0]))
+
+    def test_dissimilar_second_sighting_is_not_admitted(self):
+        """An unrelated embedding should not be promoted by an unrelated probation entry."""
+        policy = SimilarityLFUAdmissionPolicy(admission_similarity_threshold=0.9)
+        policy.should_admit([1.0, 0.0, 0.0])
+        self.assertFalse(policy.should_admit([0.0, 1.0, 0.0]))
+
+    def test_promoted_entry_is_removed_from_probation(self):
+        """Once promoted, an entry should not still be sitting on probation."""
+        policy = SimilarityLFUAdmissionPolicy(admission_similarity_threshold=0.9)
+        policy.should_admit([1.0, 0.0, 0.0])
+        policy.should_admit([1.0, 0.0001, 0.0])  # promotes and removes the entry
+        self.assertEqual(len(policy._probation), 0)
+
+    def test_probation_capacity_evicts_oldest(self):
+        """When probation is full, the oldest entry is dropped to make room."""
+        policy = SimilarityLFUAdmissionPolicy(
+            admission_similarity_threshold=0.9, probation_capacity=2
+        )
+        policy.should_admit([1.0, 0.0, 0.0])
+        policy.should_admit([0.0, 1.0, 0.0])
+        policy.should_admit([0.0, 0.0, 1.0])  # probation was full -> oldest dropped
+
+        self.assertEqual(len(policy._probation), 2)
+        # The first entry should have been evicted, so its repeat is not admitted.
+        self.assertFalse(policy.should_admit([1.0, 0.0, 0.0]))
+
+    def test_zero_vector_is_never_admitted(self):
+        """A zero-norm embedding can't be compared by cosine similarity, so it's rejected."""
+        policy = SimilarityLFUAdmissionPolicy()
+        self.assertFalse(policy.should_admit([0.0, 0.0, 0.0]))
+
+
+class TestCacheAdmissionPolicyWiring(unittest.TestCase):
+    def setUp(self):
+        self.embedding_store = MagicMock()
+        self.embedding_store.add_embedding.return_value = 42
+        self.embedding_engine = MagicMock()
+        self.embedding_engine.get_embedding.return_value = [1.0, 0.0, 0.0]
+        self.eviction_policy = MagicMock()
+
+    def test_defaults_to_always_admit(self):
+        """With no admission_policy given, Cache.add() should behave as before (always admit)."""
+        cache = Cache(
+            embedding_store=self.embedding_store,
+            embedding_engine=self.embedding_engine,
+            eviction_policy=self.eviction_policy,
+        )
+        result = cache.add(prompt="hi", response="hello", id_set=1)
+        self.assertEqual(result, 42)
+        self.embedding_store.add_embedding.assert_called_once()
+
+    def test_rejecting_admission_policy_skips_insertion(self):
+        """When the admission policy declines, Cache.add() should not insert anything."""
+        rejecting_policy = MagicMock()
+        rejecting_policy.should_admit.return_value = False
+        cache = Cache(
+            embedding_store=self.embedding_store,
+            embedding_engine=self.embedding_engine,
+            eviction_policy=self.eviction_policy,
+            admission_policy=rejecting_policy,
+        )
+        result = cache.add(prompt="hi", response="hello", id_set=1)
+        self.assertEqual(result, -1)
+        self.embedding_store.add_embedding.assert_not_called()
+
+    def test_accepting_admission_policy_allows_insertion(self):
+        """When the admission policy admits, Cache.add() should behave normally."""
+        accepting_policy = MagicMock()
+        accepting_policy.should_admit.return_value = True
+        cache = Cache(
+            embedding_store=self.embedding_store,
+            embedding_engine=self.embedding_engine,
+            eviction_policy=self.eviction_policy,
+            admission_policy=accepting_policy,
+        )
+        result = cache.add(prompt="hi", response="hello", id_set=1)
+        self.assertEqual(result, 42)
+        self.embedding_store.add_embedding.assert_called_once()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/VCachePolicyStrategy/test_vcache_policy.py

@@ -54,6 +54,7 @@ def update_metadata(embedding_id, embedding_metadata):
         mock_config.embedding_metadata_storage = MagicMock()
         mock_config.vector_db = MagicMock()
         mock_config.eviction_policy = MagicMock()
+        mock_config.admission_policy = MagicMock()
 
         self.policy = VerifiedDecisionPolicy()
         self.policy.setup(mock_config)

vcache/config.py

@@ -2,6 +2,10 @@
 
 from vcache.inference_engine.inference_engine import InferenceEngine
 from vcache.inference_engine.strategies.open_ai import OpenAIInferenceEngine
+from vcache.vcache_core.cache.admission_policy.admission_policy import AdmissionPolicy
+from vcache.vcache_core.cache.admission_policy.strategies.always_admit import (
+    AlwaysAdmitPolicy,
+)
 from vcache.vcache_core.cache.embedding_engine import OpenAIEmbeddingEngine
 from vcache.vcache_core.cache.embedding_engine.embedding_engine import EmbeddingEngine
 from vcache.vcache_core.cache.embedding_store.embedding_metadata_storage.embedding_metadata_storage import (
@@ -38,6 +42,7 @@ def __init__(
         vector_db: VectorDB = HNSWLibVectorDB(),
         embedding_metadata_storage: EmbeddingMetadataStorage = InMemoryEmbeddingMetadataStorage(),
         eviction_policy: EvictionPolicy = NoEvictionPolicy(),
+        admission_policy: AdmissionPolicy = AlwaysAdmitPolicy(),
         similarity_evaluator: SimilarityEvaluator = StringComparisonSimilarityEvaluator(),
         system_prompt: Optional[str] = None,
     ):
@@ -50,13 +55,18 @@ def __init__(
             vector_db: Vector database for storing and retrieving embeddings.
             embedding_metadata_storage: Storage for embedding metadata.
             eviction_policy: Policy for removing items from cache when full.
+            admission_policy: Policy for deciding whether a cache-miss item is
+                worth caching before it takes up a slot. Defaults to
+                `AlwaysAdmitPolicy`, which admits every miss (vCache's original
+                behavior).
             similarity_evaluator: Evaluator for determining similarity between prompts.
             system_prompt: Optional system prompt to use for all inferences.
         """
         self.inference_engine = inference_engine
         self.embedding_engine = embedding_engine
         self.vector_db = vector_db
         self.eviction_policy = eviction_policy
+        self.admission_policy = admission_policy
         self.embedding_metadata_storage = embedding_metadata_storage
         self.similarity_evaluator = similarity_evaluator
         self.system_prompt = system_prompt

vcache/vcache_core/cache/admission_policy/README.md

@@ -0,0 +1,24 @@
+# Cache Admission Policies
+
+An `EvictionPolicy` decides what gets removed once the cache is full. An `AdmissionPolicy` answers a different question: is a cache-miss item even worth caching in the first place? By default, `vCache` admits every miss (`AlwaysAdmitPolicy`), so plugging in an `AdmissionPolicy` is entirely opt-in and does not change existing behavior unless explicitly configured via `VCacheConfig(admission_policy=...)`.
+
+## Always Admit
+
+`AlwaysAdmitPolicy` is the default: every cache-miss item is cached. This preserves vCache's original behavior.
+
+## Similarity-LFU
+
+`SimilarityLFUAdmissionPolicy` is adapted from the idea behind TinyLFU (used e.g. by the Caffeine cache library): don't admit something until it has shown some sign of recurring. TinyLFU itself checks recurrence with a hashed frequency sketch over *exact* keys, which doesn't transfer to vCache directly -- two semantically identical queries with different wording hash completely differently, so an exact-key sketch can't see the relationship vCache's whole caching strategy is built around. This policy checks recurrence by semantic similarity instead: a cache-miss item is *not* admitted the first time it's seen. Its embedding is kept on a small, bounded probation list. If a similar embedding is seen again while the original is still on probation, the item is promoted (admitted) on that second sighting and removed from probation. If probation is full when a new, non-matching item arrives, the oldest entry is dropped to make room (a FIFO doorkeeper, in TinyLFU's terms).
+
+The probation list is separate from the main cache -- it never touches the eviction policy or the vector index, so it doesn't consume any of the main cache's capacity.
+
+### Empirical Results
+
+Tested against a real 1591-row workload (real LmArena duplicate-query clusters, 80% ordinary / 10% expensive / 10% unique-noise mix) across all 6 eviction policies (LRU, FIFO, SCU, CostAware, CostAwareSCU, GPCA) and 7 MB-calibrated cache sizes, `SimilarityLFUAdmissionPolicy` shows a sharp, consistent split by cache pressure:
+
+- **Tight caches (20-40MB, real eviction churn):** admission control helps every eviction policy substantially. E.g. at 40MB, average hit ratio across all 6 policies goes from 2.13% (AlwaysAdmit) to 5.82% (SimilarityLFU), and average precision (`expected_hit_ratio`) from 76.3% to 82.2%.
+- **Generous caches (80-200MB, little eviction pressure):** it consistently *hurts* every policy instead. E.g. at 100MB, average hit ratio drops from 13.26% to 8.10%, precision from 97.7% to 87.0%.
+
+This is expected from the mechanism itself: `SimilarityLFUAdmissionPolicy` never admits an item on its first sighting, so under real eviction pressure it stops one-off noise from ever burning a cache slot, letting genuine repeats survive eviction. But once the cache is large enough to hold most of the working set anyway, that same one-sighting delay just costs free hits, since there was no eviction pressure for it to protect against in the first place.
+
+**Practical takeaway:** `SimilarityLFUAdmissionPolicy` is not a universal improvement -- it is a tight-cache-pressure tool. It should be enabled when the cache is small relative to the working set, and left as `AlwaysAdmitPolicy` (the default) when the cache is generously sized.

vcache/vcache_core/cache/admission_policy/__init__.py

@@ -0,0 +1,5 @@
+from vcache.vcache_core.cache.admission_policy.admission_policy import AdmissionPolicy
+
+__all__ = [
+    "AdmissionPolicy",
+]

vcache/vcache_core/cache/admission_policy/admission_policy.py

@@ -0,0 +1,29 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+
+class AdmissionPolicy(ABC):
+    """
+    Abstract base class defining the interface for cache admission policies.
+
+    This class provides a standardized framework for deciding whether a
+    cache-miss item is actually worth caching, before it takes up a slot.
+    Unlike an `EvictionPolicy` (which decides what to remove once the cache
+    is full), an `AdmissionPolicy` decides what gets let in, in the first
+    place.
+    """
+
+    @abstractmethod
+    def should_admit(self, embedding: List[float]) -> bool:
+        """
+        Decides whether a cache-miss item should actually be added to the
+        cache.
+
+        Args:
+            embedding: The embedding vector of the prompt that just missed.
+
+        Returns:
+            True if the item should be cached, False if it should be
+            skipped this time.
+        """
+        pass

vcache/vcache_core/cache/admission_policy/strategies/__init__.py

@@ -0,0 +1,11 @@
+from vcache.vcache_core.cache.admission_policy.strategies.always_admit import (
+    AlwaysAdmitPolicy,
+)
+from vcache.vcache_core.cache.admission_policy.strategies.similarity_lfu import (
+    SimilarityLFUAdmissionPolicy,
+)
+
+__all__ = [
+    "AlwaysAdmitPolicy",
+    "SimilarityLFUAdmissionPolicy",
+]

vcache/vcache_core/cache/admission_policy/strategies/always_admit.py

@@ -0,0 +1,24 @@
+from typing import List
+
+from vcache.vcache_core.cache.admission_policy.admission_policy import AdmissionPolicy
+
+
+class AlwaysAdmitPolicy(AdmissionPolicy):
+    """
+    The default admission policy: every cache-miss item is admitted.
+
+    This preserves vCache's original behavior (no admission filtering at
+    all), so plugging in an `AdmissionPolicy` is fully opt-in.
+    """
+
+    def should_admit(self, embedding: List[float]) -> bool:
+        """Always admits the item.
+
+        Args:
+            embedding: The embedding vector of the prompt that just missed
+                (unused).
+
+        Returns:
+            True, unconditionally.
+        """
+        return True

vcache/vcache_core/cache/admission_policy/strategies/similarity_lfu.py

@@ -0,0 +1,111 @@
+from collections import deque
+from typing import Deque, List, Tuple
+
+import numpy as np
+
+from vcache.vcache_core.cache.admission_policy.admission_policy import AdmissionPolicy
+
+
+class SimilarityLFUAdmissionPolicy(AdmissionPolicy):
+    """
+    A similarity-aware admission policy, adapted from the idea behind
+    TinyLFU (used e.g. by the Caffeine cache library).
+
+    TinyLFU decides whether to admit a new key by checking how often that
+    *exact* key has recently been probed, using a hashed frequency sketch.
+    That doesn't transfer to vCache directly: two semantically identical
+    queries with different wording hash completely differently, so an
+    exact-key frequency sketch can't see the relationship that vCache's
+    whole caching strategy is built around.
+
+    This policy keeps the same underlying idea -- don't admit something
+    until it has shown some sign of recurring -- but checks recurrence by
+    semantic similarity instead of exact-key hashing, reusing the same
+    kind of embedding comparison vCache already relies on elsewhere.
+
+    How it works: cache-miss items are not admitted the first time they're
+    seen. Instead, their embedding is kept on a small, bounded "probation"
+    list. If a *similar* embedding is seen again while the original is
+    still on probation, the item is promoted (admitted) on this second
+    sighting and removed from probation. If the probation list is full
+    when a new, non-matching item arrives, the oldest probation entry is
+    dropped to make room (a FIFO doorkeeper, in TinyLFU's terms).
+
+    Note:
+        The probation list is a separate, bounded, in-memory structure
+        distinct from the main cache -- it never touches the eviction
+        policy or the main vector index, so it does not consume any of the
+        main cache's capacity.
+    """
+
+    def __init__(
+        self,
+        admission_similarity_threshold: float = 0.9,
+        probation_capacity: int = 1000,
+    ):
+        """
+        Initializes the similarity-aware admission policy.
+
+        Args:
+            admission_similarity_threshold: Cosine similarity, in [0, 1],
+                above which a new embedding is considered a repeat of an
+                existing probation entry and gets promoted (admitted).
+                Higher values require a closer match before admitting.
+            probation_capacity: The maximum number of not-yet-admitted
+                embeddings tracked at once. When full, the oldest entry is
+                dropped to make room for a new one.
+        """
+        if not (0.0 <= admission_similarity_threshold <= 1.0):
+            admission_similarity_threshold = 0.9
+        if probation_capacity <= 0:
+            probation_capacity = 1000
+
+        self.admission_similarity_threshold: float = admission_similarity_threshold
+        self.probation_capacity: int = probation_capacity
+        self._probation: Deque[Tuple[np.ndarray, np.ndarray]] = deque()
+
+    def should_admit(self, embedding: List[float]) -> bool:
+        """
+        Decides whether a cache-miss item should actually be added to the
+        cache, based on whether a similar item was recently seen while on
+        probation.
+
+        Args:
+            embedding: The embedding vector of the prompt that just missed.
+
+        Returns:
+            True if a similar embedding was already on probation (this is
+            treated as a repeat and promoted); False if this is the first
+            sighting (the embedding is added to probation instead).
+        """
+        candidate = np.asarray(embedding, dtype=np.float64)
+        candidate_norm = np.linalg.norm(candidate)
+        if candidate_norm == 0:
+            return False
+
+        for index, (probation_vector, probation_norm) in enumerate(self._probation):
+            if probation_norm == 0:
+                continue
+            similarity = float(
+                np.dot(candidate, probation_vector) / (candidate_norm * probation_norm)
+            )
+            if similarity >= self.admission_similarity_threshold:
+                del self._probation[index]
+                return True
+
+        if len(self._probation) >= self.probation_capacity:
+            self._probation.popleft()
+        self._probation.append((candidate, candidate_norm))
+        return False
+
+    def __str__(self) -> str:
+        """Returns a string representation of the SimilarityLFUAdmissionPolicy.
+
+        Returns:
+            str: A string representation of the instance.
+        """
+        return (
+            f"SimilarityLFUAdmissionPolicy("
+            f"admission_similarity_threshold={self.admission_similarity_threshold}, "
+            f"probation_capacity={self.probation_capacity})"
+        )