Simplifying festering

plugins/festinger/festinger/main.py

@@ -38,7 +38,7 @@ from .db import (
     close_pool, get_config, get_or_create_soas,
     get_pool, init_schema, bootstrap_dimensions,
     bootstrap_english_dictionary, warm_cache, reload_urd_cache,
-    flush_encounter_deltas, create_novel_soas, reset_graph,
+    flush_encounter_deltas, reset_graph,
 )
 from .loop_detector import apply_mitigations, record_and_check, session_key
 from .cue_scanner import scan_cues
@@ -46,7 +46,7 @@ from .recollection import build_recollection_block, inject_recollection
 from .resolution_job import run_resolution_job, last_run_timestamp
 from .tokenizer import tokenize
 from .llm_client import ModelConfig
-from .write_queue import enqueue_context_discover, enqueue_cue, start_worker, stop_worker
+from .write_queue import enqueue_cue, start_worker, stop_worker
 from .urd_writer import InsertRequest, insert_urd_edge
 from .wordnet import import_wordnet, CITATION as WORDNET_CITATION
 from .test_scenarios import SCENARIOS, seed_scenario, reset_scenario
@@ -438,32 +438,18 @@ def _extract_text_strings(content) -> list[str]:
     return []
 
 
-# Tokens that look structural/technical — skip novel-word detection for these.
+# Tokens that look structural/technical — kept for potential future use.
-# Matches: paths (foo/bar), emails (a@b), file extensions (foo.py), dotted names (1.2.3),
-# pure numbers, hex literals/colours.
 _STRUCTURAL_RE = re.compile(
-    r"[/@]"                  # URL-like separator or email @
+    r"[/@]"
-    r"|\.\w"                 # dotted extension or namespace (e.g. foo.py, omega.13)
+    r"|\.\w"
-    r"|^\d+$"                # pure digits
+    r"|^\d+$"
-    r"|^\d[\d.]+\d$"         # version string like 1.2 or 3.4.5
+    r"|^\d[\d.]+\d$"
     r"|^#[0-9a-f]{3,6}$"    # hex colour
     r"|^0x[0-9a-f]+$",       # hex literal
     re.IGNORECASE,
 )
 
 
-def _is_structural_token(token: str) -> bool:
-    """Return True if token looks like a path, version, number, or URL fragment."""
-    return bool(_STRUCTURAL_RE.search(token))
-
-
-def _sentence_containing(text: str, token: str, max_chars: int = 80) -> str:
-    """Return a short excerpt of the first sentence containing token (case-insensitive)."""
-    for sentence in re.split(r"(?<=[.!?])\s+|\n+", text):
-        if token.lower() in sentence.lower():
-            return sentence.strip()[:max_chars]
-    return ""
-
 
 def extract_prompt_text(body: dict, path: str) -> str:
     """
@@ -501,6 +487,18 @@ def _last_user_message_text(body: dict, path: str) -> str:
     return body.get("prompt", "")
 
 
+def _last_assistant_message_text(body: dict, path: str) -> str:
+    """Extract the last assistant message for cue scanning."""
+    if path in ("/api/chat", "/v1/chat/completions", "/v1/messages"):
+        messages = body.get("messages", [])
+        last_assistant = next(
+            (m for m in reversed(messages) if m.get("role") == "assistant"), None
+        )
+        if last_assistant:
+            return " ".join(_extract_text_strings(last_assistant.get("content", "")))
+    return ""
+
+
 def inject_recollection_anthropic(body: dict, block: str) -> dict:
     """
     Inject a recollection block into an Anthropic Messages API request.
@@ -689,20 +687,6 @@ def _get_upstream_semaphore(base_url: str) -> asyncio.Semaphore:
     return _local_upstream_semaphores[key]
 
 
-_last_discovery: dict[str, float] = {}
-DISCOVERY_COOLDOWN_SECONDS = 60.0
-
-
-def _discovery_allowed(agent_name: str) -> bool:
-    """Rate-limit context discovery to at most once per cooldown period per agent."""
-    key = agent_name or "_default"
-    now = time.time()
-    if now - _last_discovery.get(key, 0.0) < DISCOVERY_COOLDOWN_SECONDS:
-        log.debug("context discover throttled for agent=%s", key)
-        return False
-    _last_discovery[key] = now
-    return True
-
 
 async def _get_agent_routing_model(pool, agent_name: str) -> ModelConfig | None:
     """
@@ -757,7 +741,6 @@ async def _route_agent_chat(
     # model if you want memory building alongside local inference.)
     body = await process_prompt(
         body, "/v1/chat/completions", pool, cfg, request_headers,
-        skip_discovery=is_local,
     )
 
     sess = session_key(agent_model.model_name, body.get("messages", []))
@@ -862,84 +845,49 @@ async def process_prompt(
     pool,
     cfg: dict,
     request_headers: dict | None = None,
-    skip_discovery: bool = False,
 ) -> dict:
     """
     Run the saliency + recollection pipeline over the prompt.
     Returns a (possibly modified) body dict with the recollection block injected.
-
-    skip_discovery: when True, novel-word candidates are still registered in
-    SOAS (low saliency) but the LLM context-discover call is suppressed.
-    Used when the agent is routed to a local single-GPU model that cannot
-    handle simultaneous requests.
     """
     read_threshold = float(await get_config(pool, "saliency_read_threshold", "0.5"))
     conf_floor = float(await get_config(pool, "recollection_confidence_floor", "0.6"))
     recency_days = int(await get_config(pool, "recollection_recency_days", "90"))
 
     hdrs = request_headers or {}
-    request_model = _extract_request_model_config(path, body, hdrs, cfg)
     agent_name, _ = _extract_agent_name(body, hdrs)  # body already cleaned by route handler
 
-    # Only the last user message — assistant responses and tool outputs are noise.
+    # Last user message — primary source for recollection reads.
     user_text = _last_user_message_text(body, path)
     if not user_text.strip():
         return body
 
-    # 1. Scan user message for explicit relationship cues (fast, no LLM).
+    # 1. Scan user message for explicit relationship cues (fast, regex-only).
     for cue in scan_cues(user_text):
         await enqueue_cue(cue)
 
-    # 2. Single token loop over the user message only.
+    # 2. Also scan the last assistant message for is-a / is-part-of assertions.
-    #    The previous code also tokenised the last assistant message for the read
+    #    Agents frequently state facts in their responses ("X is a Y", "X runs on Z").
-    #    path, but assistant output is large and full of technical terms that look
+    #    No token-loop needed here — just cue extraction.
-    #    novel — it was the main source of spurious discovery calls.
+    assistant_text = _last_assistant_message_text(body, path)
-    #    Concepts the assistant mentioned were already processed when they first
+    if assistant_text:
-    #    appeared in a user turn, so re-scanning is redundant.
+        for cue in scan_cues(assistant_text):
+            await enqueue_cue(cue)
+
+    # 3. Token loop over user message for saliency-triggered recollection.
     tokens = tokenize(user_text)
     salient_for_read: list[int] = []
 
-    # Novel candidates: unknown words that look domain-specific.
-    # Cap at 3 — we want a targeted LLM call, not a flood.
-    MAX_NOVEL_PER_TURN = 3
-    novel_candidates: list[str] = []
-
     for token in tokens:
         soas_row = cache.soas_by_token.get(token)
-
         if soas_row is None:
-            if (
-                not _is_structural_token(token)
-                and len(novel_candidates) < MAX_NOVEL_PER_TURN
-            ):
-                novel_candidates.append(token)
             continue
-
         if soas_row.saliency == 0.0 and soas_row.novelty == 0.0:
             continue  # common English — skip
-
         cache.record_encounter(soas_row.id)
-
         if soas_row.saliency >= read_threshold:
             salient_for_read.append(soas_row.id)
 
-    # Register novel candidates at low saliency (below read threshold).
-    for token in novel_candidates:
-        ctx = _sentence_containing(user_text, token)
-        await create_novel_soas(pool, token, context=ctx)
-
-    # 3. Enqueue LLM-driven discovery — rate-limited per agent.
-    #    At most one discovery call per DISCOVERY_COOLDOWN_SECONDS per agent,
-    #    so a long conversation with many novel words doesn't queue a storm of
-    #    concurrent LM Studio calls.
-    if novel_candidates and len(user_text) >= 20 and not skip_discovery:
-        if _discovery_allowed(agent_name):
-            await enqueue_context_discover(
-                user_text, novel_candidates,
-                agent_name=agent_name,
-                fallback_model=request_model,
-            )
-
     if not salient_for_read:
         return body
 
@@ -1229,6 +1177,107 @@ async def iknowthat(request: Request) -> dict:
     }
 
 
+# ---------------------------------------------------------------------------
+# /concepts — browse and search domain concepts
+# ---------------------------------------------------------------------------
+
+@app.get("/concepts")
+async def list_concepts(
+    request: Request,
+    q: str = "",
+    limit: int = 50,
+    offset: int = 0,
+) -> dict:
+    """
+    Return domain concepts (saliency > 0 or novelty > 0) with their URD edges.
+    Optionally filter by token prefix/substring via ?q=.
+    """
+    pool = request.app.state.pool
+    async with pool.acquire() as conn:
+        if q:
+            rows = await conn.fetch(
+                """
+                SELECT id, token, saliency, novelty, encounter_count, first_seen_context
+                FROM soas
+                WHERE (saliency > 0 OR novelty > 0) AND token ILIKE $1
+                ORDER BY saliency DESC, encounter_count DESC
+                LIMIT $2 OFFSET $3
+                """,
+                f"%{q}%", limit, offset,
+            )
+            total = await conn.fetchval(
+                "SELECT COUNT(*) FROM soas WHERE (saliency > 0 OR novelty > 0) AND token ILIKE $1",
+                f"%{q}%",
+            )
+        else:
+            rows = await conn.fetch(
+                """
+                SELECT id, token, saliency, novelty, encounter_count, first_seen_context
+                FROM soas
+                WHERE saliency > 0 OR novelty > 0
+                ORDER BY saliency DESC, encounter_count DESC
+                LIMIT $1 OFFSET $2
+                """,
+                limit, offset,
+            )
+            total = await conn.fetchval(
+                "SELECT COUNT(*) FROM soas WHERE saliency > 0 OR novelty > 0"
+            )
+
+    result = []
+    for r in rows:
+        cid = r["id"]
+        edges = cache.urd_by_concept.get(cid, [])
+        result.append({
+            "id": cid,
+            "token": r["token"],
+            "saliency": round(r["saliency"], 3),
+            "novelty": round(r["novelty"], 3),
+            "encounter_count": r["encounter_count"],
+            "first_seen_context": r["first_seen_context"] or "",
+            "edges": [
+                {
+                    "dim": e.dim_token,
+                    "parent": e.parent_token,
+                    "is_isa": e.is_isa,
+                    "confidence": round(e.confidence, 3),
+                    "source": e.source,
+                }
+                for e in edges
+            ],
+        })
+
+    return {"total": total, "concepts": result}
+
+
+@app.delete("/concepts/{token:path}")
+async def delete_concept(token: str, request: Request) -> dict:
+    """Remove a concept and all its URD edges from SOAS."""
+    pool = request.app.state.pool
+    token = token.lower().strip()
+    async with pool.acquire() as conn:
+        async with conn.transaction():
+            await conn.execute("DELETE FROM kg_write_log WHERE concept_token = $1", token)
+            soas_id = await conn.fetchval("SELECT id FROM soas WHERE token = $1", token)
+            if soas_id is None:
+                return {"error": f"concept {token!r} not found"}
+            await conn.execute("DELETE FROM resolution_queue WHERE concept_id = $1", soas_id)
+            await conn.execute("DELETE FROM urd WHERE id = $1", soas_id)
+            await conn.execute("DELETE FROM soas WHERE id = $1", soas_id)
+
+    # Evict from cache
+    row = cache.soas_by_token.pop(token, None)
+    if row:
+        cache.soas_by_id.pop(row.id, None)
+        edges = cache.urd_by_concept.pop(row.id, [])
+        for e in edges:
+            cache.urd_by_concept_dim.pop((row.id, e.dim_id), None)
+        cache.urd_by_parent.pop(row.id, None)
+        cache.pending_conflicts.discard(row.id)
+
+    return {"status": "deleted", "token": token}
+
+
 # ---------------------------------------------------------------------------
 # /models — LLM model management
 # ---------------------------------------------------------------------------
@@ -2283,6 +2332,11 @@ ADMIN_HTML = """<!DOCTYPE html>
     .log-nav {{ margin-top: 0.5em; display: flex; gap: 1em; align-items: center; font-size: 0.85em; }}
     footer {{ margin-top: 3em; padding-top: 1em; border-top: 1px solid #ddd; font-size: 0.78em; color: #888; }}
     footer a {{ color: #888; }}
+    .fact-form {{ display: flex; gap: 0.6em; flex-wrap: wrap; align-items: flex-end; margin-top: 0.6em; }}
+    .fact-form input, .fact-form select {{ font-family: monospace; padding: 5px 8px; border: 1px solid #ccc; border-radius: 3px; }}
+    .concept-edges {{ font-size: 0.78em; color: #555; }}
+    .search-row {{ display: flex; gap: 0.6em; align-items: center; margin-bottom: 0.8em; }}
+    .search-row input {{ font-family: monospace; padding: 5px 10px; border: 1px solid #ccc; border-radius: 3px; width: 220px; }}
   </style>
 </head>
 <body>
@@ -2355,6 +2409,51 @@ ADMIN_HTML = """<!DOCTYPE html>
     </div>
   </details>
 
+  <h2>Concepts</h2>
+  <p style="font-size:0.83em;color:#666;margin-bottom:0.6em">
+    Domain concepts with saliency above zero. Add facts manually; the agent's
+    is-a / is-part-of statements are also captured automatically via cue scanning.
+  </p>
+
+  <details open style="margin-bottom:1em">
+    <summary style="cursor:pointer;font-size:0.9em;color:#555;font-weight:bold">Add a fact…</summary>
+    <div class="fact-form" style="margin-top:0.7em">
+      <label style="font-size:0.85em">Concept
+        <input id="f-concept" type="text" placeholder="gnommoweb"
+               style="display:block;margin-top:2px;width:150px">
+      </label>
+      <label style="font-size:0.85em">Relation
+        <select id="f-rel" style="display:block;margin-top:2px;padding:6px 8px">
+          <option value="-isa">is-a (type)</option>
+          <option value="-ispart">is-part-of (membership)</option>
+        </select>
+      </label>
+      <label style="font-size:0.85em">Parent
+        <input id="f-parent" type="text" placeholder="repository"
+               style="display:block;margin-top:2px;width:150px">
+      </label>
+      <label style="font-size:0.85em">Dimension <span style="color:#aaa">(optional)</span>
+        <input id="f-dim" type="text" placeholder="type"
+               style="display:block;margin-top:2px;width:120px">
+      </label>
+      <button onclick="addFact(this)" style="height:32px;margin-top:0">Add fact</button>
+    </div>
+    <div id="fact-result" style="margin-top:0.5em;font-size:0.85em"></div>
+  </details>
+
+  <div class="search-row">
+    <input id="c-search" type="text" placeholder="Search concepts…"
+           oninput="debounceSearch()" onkeydown="if(event.key==='Enter')loadConcepts(0)">
+    <button onclick="loadConcepts(0)">Search</button>
+    <button onclick="document.getElementById('c-search').value='';loadConcepts(0)">Clear</button>
+  </div>
+  <div id="concepts-table-wrap"><em style="color:#aaa">Enter a search term or click Search to browse.</em></div>
+  <div class="log-nav" id="concepts-nav" style="display:none">
+    <button id="c-prev" onclick="conceptsPage(-1)" disabled>← prev</button>
+    <span id="c-page-info"></span>
+    <button id="c-next" onclick="conceptsPage(1)">next →</button>
+  </div>
+
   <h2>Actions</h2>
   <div class="actions">
     <button class="primary" onclick="runResolution(this)">Run conflict resolution now</button>
@@ -2528,6 +2627,125 @@ ADMIN_HTML = """<!DOCTYPE html>
       finally {{ btn.disabled = false; }}
     }}
 
+    // --------------- Concepts ---------------
+
+    const CONCEPT_PAGE_SIZE = 30;
+    let conceptOffset = 0;
+    let conceptTotal = 0;
+    let _searchTimer = null;
+
+    function debounceSearch() {{
+      clearTimeout(_searchTimer);
+      _searchTimer = setTimeout(() => loadConcepts(0), 350);
+    }}
+
+    async function loadConcepts(offset) {{
+      conceptOffset = offset;
+      const q = document.getElementById('c-search').value.trim();
+      const params = new URLSearchParams({{limit: CONCEPT_PAGE_SIZE, offset: conceptOffset}});
+      if (q) params.set('q', q);
+      const r = await fetch('/concepts?' + params);
+      const d = await r.json();
+      conceptTotal = d.total;
+
+      const wrap = document.getElementById('concepts-table-wrap');
+      if (!d.concepts.length) {{
+        wrap.innerHTML = '<em style="color:#aaa">No concepts found.</em>';
+        document.getElementById('concepts-nav').style.display = 'none';
+        return;
+      }}
+
+      let html = '<table><thead><tr>'
+        + '<th>Concept</th><th>Saliency</th><th>Encounters</th>'
+        + '<th>Edges (dim → parent)</th><th>First seen</th><th></th>'
+        + '</tr></thead><tbody>';
+      for (const c of d.concepts) {{
+        const edgeStr = c.edges.map(e =>
+          `<span style="color:#555">[${{e.dim}}]</span> ${{e.parent}}`
+        ).join('&ensp;');
+        const ctx = c.first_seen_context
+          ? `<span title="${{c.first_seen_context.replace(/"/g,'&quot;')}}" style="color:#999;font-size:0.8em">…</span>`
+          : '';
+        html += `<tr>
+          <td><strong>${{c.token}}</strong></td>
+          <td>${{c.saliency}}</td>
+          <td>${{c.encounter_count}}</td>
+          <td class="concept-edges">${{edgeStr || '<span style="color:#bbb">none</span>'}}</td>
+          <td>${{ctx}}</td>
+          <td><button onclick="deleteConcept('${{c.token}}',this)"
+              style="padding:2px 8px;font-size:0.8em;color:#b00;border-color:#b00">✕</button></td>
+        </tr>`;
+      }}
+      html += '</tbody></table>';
+      wrap.innerHTML = html;
+
+      const nav = document.getElementById('concepts-nav');
+      if (conceptTotal > CONCEPT_PAGE_SIZE) {{
+        nav.style.display = 'flex';
+        document.getElementById('c-prev').disabled = conceptOffset === 0;
+        document.getElementById('c-next').disabled = conceptOffset + CONCEPT_PAGE_SIZE >= conceptTotal;
+        document.getElementById('c-page-info').textContent =
+          `${{conceptOffset + 1}}–${{Math.min(conceptOffset + CONCEPT_PAGE_SIZE, conceptTotal)}} of ${{conceptTotal}}`;
+      }} else {{
+        nav.style.display = 'none';
+      }}
+    }}
+
+    function conceptsPage(dir) {{
+      const next = conceptOffset + dir * CONCEPT_PAGE_SIZE;
+      if (next < 0 || next >= conceptTotal) return;
+      loadConcepts(next);
+    }}
+
+    async function addFact(btn) {{
+      const concept = document.getElementById('f-concept').value.trim().toLowerCase();
+      const rel     = document.getElementById('f-rel').value;
+      const parent  = document.getElementById('f-parent').value.trim().toLowerCase();
+      const dim     = document.getElementById('f-dim').value.trim().toLowerCase();
+      if (!concept || !parent) {{ alert('Concept and parent are required.'); return; }}
+      let fact = `${{concept}} ${{rel}} ${{parent}}`;
+      if (dim) fact += ` in context of ${{dim}}`;
+      btn.disabled = true;
+      const el = document.getElementById('fact-result');
+      try {{
+        const r = await fetch('/iknowthat', {{
+          method: 'POST',
+          headers: {{'Content-Type': 'application/json'}},
+          body: JSON.stringify({{fact}})
+        }});
+        const d = await r.json();
+        if (d.error) {{
+          el.style.color = '#b00'; el.textContent = 'Error: ' + d.error;
+        }} else if (d.status === 'collision') {{
+          el.style.color = '#c07000';
+          el.textContent = `Collision (${{d.collision_type}}) — queued for resolution. Current graph unchanged.`;
+        }} else {{
+          el.style.color = '#2a7a2a';
+          el.textContent = `Inserted: ${{d.subject}} [${{d.is_isa ? 'is-a' : 'is-part-of'}}] ${{d.parent}} (dim: ${{d.dimension}})`;
+          document.getElementById('f-concept').value = '';
+          document.getElementById('f-parent').value = '';
+          document.getElementById('f-dim').value = '';
+          await loadStats();
+          loadConcepts(conceptOffset);
+        }}
+      }} catch(e) {{ el.style.color = '#b00'; el.textContent = 'Error: ' + e.message; }}
+      finally {{ btn.disabled = false; }}
+    }}
+
+    async function deleteConcept(token, btn) {{
+      if (!confirm('Delete concept "' + token + '" and all its edges?')) return;
+      btn.disabled = true;
+      try {{
+        const r = await fetch('/concepts/' + encodeURIComponent(token), {{method: 'DELETE'}});
+        const d = await r.json();
+        if (d.error) {{ showResult('Error: ' + d.error, false); return; }}
+        showResult('Deleted: ' + token, true);
+        await loadStats();
+        loadConcepts(conceptOffset);
+      }} catch(e) {{ showResult('Error: ' + e.message, false); }}
+      finally {{ btn.disabled = false; }}
+    }}
+
     async function loadConflicts() {{
       const r = await fetch('/conflicts');
       const d = await r.json();