From 58f903aec0335d4f1052f2fe1c9bb68875871d49 Mon Sep 17 00:00:00 2001
From: jenstandstad <jens.tandstad@gmail.com>
Date: Mon, 20 Apr 2026 18:21:23 +0200
Subject: [PATCH] Adding updates to Festinger

---
 plugins/festinger/festinger/main.py         | 32 ++++++++++++++-------
 plugins/festinger/festinger/recollection.py | 24 +++++++++++++---
 2 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/plugins/festinger/festinger/main.py b/plugins/festinger/festinger/main.py
index 119651f..a473440 100644
--- a/plugins/festinger/festinger/main.py
+++ b/plugins/festinger/festinger/main.py
@@ -495,17 +495,19 @@ async def process_prompt(body: dict, path: str, pool, cfg: dict) -> dict:
     salient_for_read: list[int] = []
     salient_for_write: list[str] = []
 
+    # Novel words found this prompt that aren't in the cache yet.
+    # We cap at MAX_NOVEL_PER_PROMPT to avoid flooding on large system prompts.
+    MAX_NOVEL_PER_PROMPT = 5
+    novel_this_prompt: list[str] = []
+
     for token in tokens:
         soas_row = cache.soas_by_token.get(token)
 
         if soas_row is None:
-            # Token is absent from the dictionary entirely → novel domain word.
-            # Give it an initial high saliency so recollection fires immediately
-            # and instructs the model to ask the user what it is.
-            soas_row = await create_novel_soas(pool, token)
-            salient_for_read.append(soas_row.id)
-            # Do NOT add to salient_for_write: we have no basis for LLM-inferred
-            # relationships yet — let the conversation teach us via cue scanner.
+            # Token absent from dictionary → candidate novel domain word.
+            # Collect for batch processing; apply a per-prompt cap.
+            if len(novel_this_prompt) < MAX_NOVEL_PER_PROMPT:
+                novel_this_prompt.append(token)
             continue
 
         if soas_row.saliency == 0.0 and soas_row.novelty == 0.0:
@@ -517,11 +519,21 @@ async def process_prompt(body: dict, path: str, pool, cfg: dict) -> dict:
         if soas_row.saliency >= read_threshold:
             salient_for_read.append(soas_row.id)
 
-        if soas_row.saliency >= write_threshold and soas_row.novelty > 0.0:
-            # Only enqueue domain-specific words for LLM relationship extraction,
-            # not freshly-created novel words (novelty=1.0 but just inserted).
+        # Only enqueue for LLM write if the concept already has URD edges —
+        # i.e. we know *something* about it and may want to expand that knowledge.
+        # Never enqueue freshly-novel words: let the conversation teach us instead.
+        if (
+            soas_row.saliency >= write_threshold
+            and soas_row.novelty > 0.0
+            and cache.urd_by_concept.get(soas_row.id)
+        ):
             salient_for_write.append(token)
 
+    # Create SOAS entries for novel words and add them to the read list.
+    for token in novel_this_prompt:
+        soas_row = await create_novel_soas(pool, token)
+        salient_for_read.append(soas_row.id)
+
     for token in salient_for_write:
         await enqueue_concept(token)
 
diff --git a/plugins/festinger/festinger/recollection.py b/plugins/festinger/festinger/recollection.py
index d0cfd57..346de13 100644
--- a/plugins/festinger/festinger/recollection.py
+++ b/plugins/festinger/festinger/recollection.py
@@ -77,21 +77,37 @@ def build_recollection_block(
     salient_concept_ids: list[int],
     confidence_floor: float,
     recency_days: int,
+    max_lines: int = 12,
 ) -> Optional[str]:
     """
     Build the full <recollection> block for a list of above-threshold concept IDs.
     Returns None if there is nothing to say.
-    """
-    lines: list[str] = []
 
+    Concepts with known edges are shown first (most informative); zero-hit
+    "don't know" lines fill the remaining budget up to max_lines.
+    """
+    hit_lines: list[str] = []
+    zero_lines: list[str] = []
+
+    # Deduplicate — same concept may appear twice if novel + known
+    seen: set[int] = set()
     for cid in salient_concept_ids:
+        if cid in seen:
+            continue
+        seen.add(cid)
+
         token = cache.soas_by_id.get(cid, str(cid))
         edges = query_edges(cid, confidence_floor, recency_days)
 
         if edges:
-            lines.append(render_hit(token, edges, cid))
+            hit_lines.append(render_hit(token, edges, cid))
         else:
-            lines.append(render_zero_hit(token))
+            zero_lines.append(render_zero_hit(token))
+
+    # Hits always included (up to max_lines); zero-hits fill the remainder
+    lines = hit_lines[:max_lines]
+    remaining = max_lines - len(lines)
+    lines += zero_lines[:remaining]
 
     if not lines:
         return None