From ccbb5b2d45227c50838a4b0e88eea91d6e29daca Mon Sep 17 00:00:00 2001 From: jenstandstad Date: Tue, 21 Apr 2026 19:09:25 +0200 Subject: [PATCH] Adding sanity to the recollection decider --- plugins/festinger/festinger/db.py | 28 +++++- plugins/festinger/festinger/llm_client.py | 23 +++-- plugins/festinger/festinger/main.py | 86 ++++++++++------ plugins/festinger/festinger/write_queue.py | 111 +++++++++++++-------- 4 files changed, 158 insertions(+), 90 deletions(-) diff --git a/plugins/festinger/festinger/db.py b/plugins/festinger/festinger/db.py index 40fd55f..b91df73 100644 --- a/plugins/festinger/festinger/db.py +++ b/plugins/festinger/festinger/db.py @@ -250,10 +250,16 @@ async def get_or_create_soas(pool: asyncpg.Pool, token: str) -> SoasRow: # Novel-word bootstrap # --------------------------------------------------------------------------- -# Saliency assigned to a word seen for the first time that is not in the -# standard English dictionary. Must be above saliency_read_threshold (0.5) -# so the recollection engine immediately picks it up. -NOVEL_INITIAL_SALIENCY = 2.0 +# Saliency for a word first seen that is not in the standard English dictionary. +# Kept BELOW saliency_read_threshold (default 0.5) so the concept doesn't +# surface in recollection until the LLM has confirmed it is meaningful. +# Once the LLM inserts a URD edge, saliency is raised to NOVEL_CONFIRMED_SALIENCY. +NOVEL_INITIAL_SALIENCY = 0.1 + +# Saliency set after the LLM confirms a concept by generating a URD triple. +# Must be well above saliency_read_threshold so the concept becomes a +# recollection attractor on subsequent turns. +NOVEL_CONFIRMED_SALIENCY = 2.0 async def create_novel_soas(pool: asyncpg.Pool, token: str, context: str = "") -> SoasRow: @@ -375,7 +381,19 @@ async def flush_encounter_deltas(pool: asyncpg.Pool) -> None: new_count = (row.encounter_count if row else 0) # novelty = 0 for common English words (pre-seeded) is_common = (row.novelty == 0.0 and row.saliency == 0.0) if row else False - new_saliency = recalculate_saliency(new_count, is_common) + + # Unconfirmed novel concepts (novelty > 0, no URD edges yet) must not + # be promoted above the read threshold by encounter-count alone. + # Their saliency is raised explicitly when the LLM confirms them. + is_unconfirmed_novel = ( + row is not None + and row.novelty > 0.0 + and not cache.urd_by_concept.get(soas_id) + ) + if is_unconfirmed_novel: + new_saliency = row.saliency # preserve low saliency until LLM confirms + else: + new_saliency = recalculate_saliency(new_count, is_common) await conn.execute( """ diff --git a/plugins/festinger/festinger/llm_client.py b/plugins/festinger/festinger/llm_client.py index b79e411..b00272f 100644 --- a/plugins/festinger/festinger/llm_client.py +++ b/plugins/festinger/festinger/llm_client.py @@ -84,24 +84,25 @@ async def _call_openai(model: ModelConfig, prompt: str) -> str: # Structured prompts # --------------------------------------------------------------------------- -CONTEXT_EXTRACT_PROMPT_TEMPLATE = """You are a knowledge extraction assistant helping build a semantic memory graph. +CONTEXT_DISCOVER_PROMPT_TEMPLATE = """You are building a semantic memory graph for an AI agent. -The following words appear in the conversation below and are NOT in the standard English dictionary — they are likely domain-specific terms that should be remembered: -{concepts} +The following words appeared in the user message and are NOT in the standard English dictionary. They may be domain-specific terms worth remembering, or they may be typos/noise: +{candidates} -Read the conversation excerpt and, for each concept above, generate relationship triples IF the excerpt provides enough context to make a confident assertion. Only assert what is clearly evidenced in the text — do not guess or hallucinate. +Read the user message and evaluate each candidate. For candidates that ARE real domain concepts (project names, system names, proper nouns, technical identifiers), generate a relationship triple ONLY IF the message provides clear evidence. For typos, noise, or common words written oddly, return nothing. -CONVERSATION EXCERPT: +USER MESSAGE: {context} -Return a JSON array of triples. Each triple must have: - - "concept": one of the domain words listed above (exactly as spelled) - - "parent": the broader category or container (e.g. "software-repository", "glitch-university") +Return a JSON array. Each item must have: + - "concept": one of the candidate words above (use the exact spelling, lowercase) + - "parent": what it is or belongs to (e.g. "software-repository", "glitch-university") - "dimension": one of [{dimensions}] or a new specific one if none fit - "is_isa": true if concept IS A type of parent (classification), false if concept BELONGS TO parent (containment) - - "confidence": 0.0–1.0 reflecting how clearly the excerpt supports this assertion + - "confidence": 0.0–1.0 reflecting how clearly the message supports this -Return ONLY the JSON array. No explanation. If no confident assertions can be made, return []. +Only include candidates you are confident are real domain concepts. Return [] if none qualify. +Return ONLY the JSON array. No explanation. Example: [ {{"concept": "gnommoweb", "parent": "software-repository", "dimension": "type", "is_isa": true, "confidence": 0.9}}, @@ -120,7 +121,7 @@ class ContextTriple: def parse_context_triples(response: str) -> list[ContextTriple]: - """Parse JSON array of context-aware triples from LLM response.""" + """Parse JSON array of discovered triples from LLM response.""" try: start = response.find("[") end = response.rfind("]") + 1 diff --git a/plugins/festinger/festinger/main.py b/plugins/festinger/festinger/main.py index 4698773..b68968c 100644 --- a/plugins/festinger/festinger/main.py +++ b/plugins/festinger/festinger/main.py @@ -44,7 +44,7 @@ from .cue_scanner import scan_cues from .recollection import build_recollection_block, inject_recollection from .resolution_job import run_resolution_job, last_run_timestamp from .tokenizer import tokenize -from .write_queue import enqueue_context_extract, enqueue_cue, start_worker, stop_worker +from .write_queue import enqueue_context_discover, enqueue_cue, start_worker, stop_worker from .urd_writer import InsertRequest, insert_urd_edge from .wordnet import import_wordnet, CITATION as WORDNET_CITATION from .test_scenarios import SCENARIOS, seed_scenario, reset_scenario @@ -484,6 +484,21 @@ def extract_prompt_text(body: dict, path: str) -> str: return body.get("prompt", "") +def _last_user_message_text(body: dict, path: str) -> str: + """ + Extract only the last user message for the write path. + Agent responses, thinking traces, and system prompts are excluded — + they are noise for concept discovery. + """ + if path in ("/api/chat", "/v1/chat/completions", "/v1/messages"): + messages = body.get("messages", []) + last_user = next((m for m in reversed(messages) if m.get("role") == "user"), None) + if last_user: + return " ".join(_extract_text_strings(last_user.get("content", ""))) + return "" + return body.get("prompt", "") + + def inject_recollection_anthropic(body: dict, block: str) -> dict: """ Inject a recollection block into an Anthropic Messages API request. @@ -512,69 +527,80 @@ async def process_prompt(body: dict, path: str, pool, cfg: dict) -> dict: Returns a (possibly modified) body dict with the recollection block injected. """ read_threshold = float(await get_config(pool, "saliency_read_threshold", "0.5")) - write_threshold = float(await get_config(pool, "saliency_write_threshold", "1.2")) conf_floor = float(await get_config(pool, "recollection_confidence_floor", "0.6")) recency_days = int(await get_config(pool, "recollection_recency_days", "90")) - prompt_text = extract_prompt_text(body, path) - if not prompt_text.strip(): + # Extract only the last user message — agent responses and reasoning traces + # are noise for both cue scanning and concept discovery. + user_text = _last_user_message_text(body, path) + if not user_text.strip(): return body - # 1. Scan for explicit relationship cues (bypass threshold) - cues = scan_cues(prompt_text) - for cue in cues: + # 1. Scan user message for explicit relationship cues (fast, inline, bypasses LLM). + for cue in scan_cues(user_text): await enqueue_cue(cue) - # 2. Tokenise + update saliency + # 2. Tokenise the recent context (last user + last assistant) for the read path. + # Novel words from the user turn are also collected as LLM candidates. + prompt_text = extract_prompt_text(body, path) tokens = tokenize(prompt_text) salient_for_read: list[int] = [] - # Novel domain words found in this turn — not in the standard dictionary. - # Capped to avoid flooding on unexpectedly large turns. - MAX_NOVEL_PER_PROMPT = 8 - novel_this_prompt: list[str] = [] + # Candidate novel tokens from the USER message only — structural tokens + # (paths, versions, numbers) are filtered out. Capped to avoid flooding + # on very long messages. + MAX_NOVEL_PER_TURN = 8 + novel_candidates: list[str] = [] + + # Only collect candidates from user-side tokens + user_tokens = set(tokenize(user_text)) for token in tokens: soas_row = cache.soas_by_token.get(token) if soas_row is None: - # Token absent from dictionary → candidate novel domain word. - # Skip structural tokens (paths, versions, numbers). - if not _is_structural_token(token) and len(novel_this_prompt) < MAX_NOVEL_PER_PROMPT: - novel_this_prompt.append(token) + # Token absent from cache entirely — candidate domain word. + # Restrict to user-side tokens so we don't mine agent responses. + if ( + token in user_tokens + and not _is_structural_token(token) + and len(novel_candidates) < MAX_NOVEL_PER_TURN + ): + novel_candidates.append(token) continue if soas_row.saliency == 0.0 and soas_row.novelty == 0.0: - # Common English word pre-seeded from the dictionary — not interesting. + # Common English word — skip. continue cache.record_encounter(soas_row.id) + # Only surface in recollection if saliency is above threshold. + # Unconfirmed novel words (saliency=NOVEL_INITIAL_SALIENCY=0.1) are + # deliberately kept below the threshold until the LLM confirms them. if soas_row.saliency >= read_threshold: salient_for_read.append(soas_row.id) - # Create SOAS entries for novel words and add them to the read list. - # Capture first-seen context so zero-hit recollection can include a hint. - for token in novel_this_prompt: - ctx = _sentence_containing(prompt_text, token) - soas_row = await create_novel_soas(pool, token, context=ctx) - salient_for_read.append(soas_row.id) + # Register novel candidates in SOAS at low saliency (below read threshold). + # They become recollection attractors only after the LLM confirms them. + for token in novel_candidates: + ctx = _sentence_containing(user_text, token) + await create_novel_soas(pool, token, context=ctx) + # Do NOT add to salient_for_read — no zero-hit recollection until confirmed. - # Enqueue context-aware LLM extraction for all novel words found this turn. - # The LLM reads the actual conversation text and extracts relationships from - # evidence — one call per turn, not one per concept. - if novel_this_prompt: - await enqueue_context_extract(novel_this_prompt, prompt_text) + # 3. Enqueue for LLM-driven discovery if there are candidates to evaluate. + if novel_candidates and len(user_text) >= 20: + await enqueue_context_discover(user_text, novel_candidates) if not salient_for_read: return body - # 3. Build recollection block + # 5. Build recollection block block = build_recollection_block(salient_for_read, conf_floor, recency_days) if not block: return body - # 4. Inject into messages + # 6. Inject into messages if path == "/api/chat" or path == "/v1/chat/completions": body = dict(body) body["messages"] = inject_recollection(body.get("messages", []), block) diff --git a/plugins/festinger/festinger/write_queue.py b/plugins/festinger/festinger/write_queue.py index d064b0b..76aba60 100644 --- a/plugins/festinger/festinger/write_queue.py +++ b/plugins/festinger/festinger/write_queue.py @@ -15,10 +15,10 @@ import asyncpg from .cache import SoasRow from .cue_scanner import CueTriple -from .db import get_or_create_soas, get_config +from .db import get_or_create_soas, get_config, create_novel_soas, NOVEL_CONFIRMED_SALIENCY from .llm_client import ( ModelConfig, get_model_config, call_llm, - CONTEXT_EXTRACT_PROMPT_TEMPLATE, parse_context_triples, + CONTEXT_DISCOVER_PROMPT_TEMPLATE, parse_context_triples, ) from .urd_writer import InsertRequest, insert_urd_edge from . import cache @@ -27,14 +27,16 @@ log = logging.getLogger("festinger.write_queue") @dataclass -class ContextExtractRequest: +class ContextDiscoverRequest: """ - Novel concepts seen in a single conversation turn, with the turn text as context. - The LLM reads the context and extracts relationship triples from evidence in the text — - much more accurate than asking about concepts in isolation. + User message text plus candidate tokens (words absent from the English + dictionary) submitted for LLM-driven concept discovery. + + candidate_tokens are hints for the LLM — it decides which are real domain + concepts vs typos/noise, and extracts relationship triples from the text. """ - concept_tokens: list[str] - context_text: str + user_text: str + candidate_tokens: list[str] @dataclass @@ -52,12 +54,15 @@ _LLM_CONCURRENCY = 2 _llm_semaphore: asyncio.Semaphore | None = None -async def enqueue_context_extract(tokens: list[str], context: str) -> None: - """Enqueue a context-aware extraction request for a batch of novel concepts.""" +async def enqueue_context_discover(user_text: str, candidate_tokens: list[str]) -> None: + """Enqueue a user message for LLM-driven concept discovery and relation extraction.""" try: - _queue.put_nowait(ContextExtractRequest(concept_tokens=tokens, context_text=context)) + _queue.put_nowait(ContextDiscoverRequest( + user_text=user_text, + candidate_tokens=candidate_tokens, + )) except asyncio.QueueFull: - log.warning("write queue full — dropping context extract for: %s", tokens) + log.warning("write queue full — dropping context discover") async def enqueue_cue(triple: CueTriple) -> None: @@ -94,11 +99,11 @@ async def _worker(pool: asyncpg.Pool) -> None: if isinstance(item, CueWriteRequest): # Fast path: no LLM involved, process inline. await _process_cue(pool, item.triple) - elif isinstance(item, ContextExtractRequest): + elif isinstance(item, ContextDiscoverRequest): # Slow path: fire off without awaiting so the worker stays free. asyncio.create_task( - _process_context_extract_guarded( - pool, item.concept_tokens, item.context_text + _process_context_discover_guarded( + pool, item.user_text, item.candidate_tokens ) ) except Exception as e: @@ -107,18 +112,18 @@ async def _worker(pool: asyncpg.Pool) -> None: _queue.task_done() -async def _process_context_extract_guarded( +async def _process_context_discover_guarded( pool: asyncpg.Pool, - concept_tokens: list[str], - context_text: str, + user_text: str, + candidate_tokens: list[str], ) -> None: - """Wrapper that acquires the LLM semaphore before context extraction.""" + """Wrapper that acquires the LLM semaphore before concept discovery.""" assert _llm_semaphore is not None async with _llm_semaphore: try: - await _process_context_extract(pool, concept_tokens, context_text) + await _process_context_discover(pool, user_text, candidate_tokens) except Exception as e: - log.exception("context extract task error for %s: %s", concept_tokens, e) + log.exception("context discover task error: %s", e) async def stop_worker() -> None: @@ -149,22 +154,24 @@ async def _process_cue(pool: asyncpg.Pool, triple: CueTriple) -> None: log.info("cue triple collision: %s", collision) -async def _process_context_extract( +async def _process_context_discover( pool: asyncpg.Pool, - concept_tokens: list[str], - context_text: str, + user_text: str, + candidate_tokens: list[str], ) -> None: """ - Ask the local LLM to extract relationships for novel concepts from the - actual conversation context. One LLM call per prompt turn, not per concept. + Ask the local LLM to evaluate candidate tokens (dictionary misses) and extract + relationship triples for those it judges to be real domain concepts. - This is fundamentally better than asking about concepts in isolation: - the model reads what was actually said about e.g. 'gnommoweb' and asserts - only what the text supports — no hallucination, evidence-grounded confidence. + Saliency feedback loop: + - Confirmed concepts (triples inserted) → saliency raised to NOVEL_CONFIRMED_SALIENCY + so they surface as recollection hits on subsequent turns. + - Rejected concepts (no triples) → saliency stays low (NOVEL_INITIAL_SALIENCY), + effectively hiding typos and noise from the recollection engine. """ write_model_id = await get_config(pool, "write_model_id") if not write_model_id: - log.debug("no write_model_id configured — skipping context extract") + log.debug("no write_model_id configured — skipping context discover") return model = await get_model_config(pool, write_model_id) @@ -173,31 +180,35 @@ async def _process_context_extract( return seed_dims = ["type", "membership", "runs-on", "tech", "owned-by", "geography"] - prompt = CONTEXT_EXTRACT_PROMPT_TEMPLATE.format( - concepts=", ".join(concept_tokens), - context=context_text[:3000], # cap to avoid exceeding model context + prompt = CONTEXT_DISCOVER_PROMPT_TEMPLATE.format( + candidates=", ".join(candidate_tokens), + context=user_text[:2000], dimensions=", ".join(seed_dims), ) try: response = await call_llm(model, prompt) except Exception as e: - log.warning("LLM call failed for context extract %s: %s", concept_tokens, e) + log.warning("LLM call failed for context discover: %s", e) return triples = parse_context_triples(response) if not triples: - log.info("LLM returned no context triples for: %s", concept_tokens) + log.debug("LLM found no domain concepts among candidates: %s", candidate_tokens) return - concept_set = set(concept_tokens) - inserted = 0 + MIN_CONFIDENCE = 0.6 + candidate_set = set(candidate_tokens) + confirmed: set[str] = set() # candidates confirmed as real concepts + for t in triples: - # Reject any concept the LLM invented that wasn't in our list - if not t.concept or t.concept not in concept_set: - log.debug("context extract: ignoring hallucinated concept %r", t.concept) + if not t.concept or not t.parent or not t.dimension: continue - if not t.parent or not t.dimension: + if t.concept not in candidate_set: + log.debug("context discover: ignoring concept not in candidate list: %r", t.concept) + continue + if t.confidence < MIN_CONFIDENCE: + log.debug("context discover: low confidence triple skipped: %s", t) continue subj_row = await get_or_create_soas(pool, t.concept) @@ -214,9 +225,21 @@ async def _process_context_extract( ) collision = await insert_urd_edge(pool, req) if not collision: - inserted += 1 + confirmed.add(t.concept) + # Raise saliency for confirmed concepts so recollection fires on next encounter. + for concept in confirmed: + async with pool.acquire() as conn: + await conn.execute( + "UPDATE soas SET saliency = $1 WHERE token = $2 AND saliency < $1", + NOVEL_CONFIRMED_SALIENCY, concept, + ) + row = cache.soas_by_token.get(concept) + if row and row.saliency < NOVEL_CONFIRMED_SALIENCY: + row.saliency = NOVEL_CONFIRMED_SALIENCY + + rejected = candidate_set - confirmed log.info( - "context extract complete: concepts=%s → %d triples → %d inserted", - concept_tokens, len(triples), inserted, + "context discover: candidates=%s confirmed=%s rejected=%s", + candidate_tokens, sorted(confirmed), sorted(rejected), )