Adding Festinger with wordnet
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# Festinger — Ollama-compatible inference middleware for Agent0
|
||||
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
In-memory cache layer.
|
||||
|
||||
All read operations on SOAS and URD hit these structures only — zero network
|
||||
on the hot path. Writes are write-through: in-memory first, then Postgres.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class SoasRow:
|
||||
id: int
|
||||
token: str
|
||||
encounter_count: int = 0
|
||||
saliency: float = 0.0
|
||||
novelty: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class UrdEdge:
|
||||
concept_id: int
|
||||
parent_id: int
|
||||
dim_id: int
|
||||
is_isa: bool
|
||||
confidence: float
|
||||
source: str
|
||||
parent_token: str = ""
|
||||
dim_token: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level cache dicts
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# SOAS — primary lookup by token (mirrors UNIQUE index on token)
|
||||
soas_by_token: dict[str, SoasRow] = {}
|
||||
|
||||
# SOAS — reverse lookup by id
|
||||
soas_by_id: dict[int, str] = {}
|
||||
|
||||
# URD — recollection reads: concept_id → list of edges (tokens pre-joined)
|
||||
urd_by_concept: dict[int, list[UrdEdge]] = {}
|
||||
|
||||
# URD — collision detection: (concept_id, dim_id) → edge
|
||||
# Mirrors Postgres UNIQUE index on (id, dim_id) exactly
|
||||
urd_by_concept_dim: dict[tuple[int, int], UrdEdge] = {}
|
||||
|
||||
# Concepts with pending items in resolution_queue
|
||||
pending_conflicts: set[int] = set()
|
||||
|
||||
# Batched encounter count deltas — flushed to Postgres every 30 s
|
||||
_encounter_deltas: dict[int, int] = {}
|
||||
|
||||
|
||||
def record_encounter(soas_id: int) -> None:
|
||||
"""Increment in-memory encounter count and stage a flush delta."""
|
||||
_encounter_deltas[soas_id] = _encounter_deltas.get(soas_id, 0) + 1
|
||||
if soas_id in {row.id for row in soas_by_token.values()}:
|
||||
token = soas_by_id.get(soas_id)
|
||||
if token and token in soas_by_token:
|
||||
soas_by_token[token].encounter_count += 1
|
||||
|
||||
|
||||
def drain_deltas() -> dict[int, int]:
|
||||
"""Return and clear the accumulated encounter deltas for Postgres flush."""
|
||||
global _encounter_deltas
|
||||
batch = dict(_encounter_deltas)
|
||||
_encounter_deltas = {}
|
||||
return batch
|
||||
@@ -0,0 +1,172 @@
|
||||
"""
|
||||
Relationship cue scanner.
|
||||
|
||||
Scans intercepted text for explicit ISA / ISPART natural language patterns.
|
||||
When matched, extracts a (subject, parent, dimension, is_isa) triple that
|
||||
bypasses the saliency write threshold and goes directly to the write queue.
|
||||
|
||||
The `of {Z}` modifier after an ISA pattern names the dimension explicitly.
|
||||
Without it, ISA defaults to dimension 'type'; ISPART defaults to 'membership'.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class CueTriple:
|
||||
subject: str # canonical token (lowercase, compound rule applied)
|
||||
parent: str # canonical token
|
||||
dimension: str # canonical token, defaults to 'type' or 'membership'
|
||||
is_isa: bool
|
||||
confidence: float # pattern-type confidence
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pattern definitions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Each entry: (regex pattern, is_isa, base_confidence)
|
||||
# Groups: (?P<subj>...) (?P<parent>...) optional (?P<dim>...)
|
||||
_RAW_PATTERNS: list[tuple[str, bool, float]] = [
|
||||
# ISA — explicit keyword forms (higher confidence)
|
||||
(r"(?P<subj>\S+)\s+ISA\s+(?P<parent>\S+)", True, 0.95),
|
||||
(r"(?P<subj>\S+)\s+is\s+an?\s+instance\s+of\s+(?P<parent>\S+)", True, 0.90),
|
||||
(r"(?P<subj>\S+)\s+is\s+a\s+kind\s+of\s+(?P<parent>\S+)", True, 0.90),
|
||||
(r"(?P<subj>\S+)\s+is\s+a\s+type\s+of\s+(?P<parent>\S+)", True, 0.90),
|
||||
(r"(?P<subj>\S+)\s+kind\s+of\s+(?P<parent>\S+)", True, 0.80),
|
||||
(r"(?P<subj>\S+)\s+type\s+of\s+(?P<parent>\S+)", True, 0.80),
|
||||
(r"(?P<subj>\S+)\s+instance\s+of\s+(?P<parent>\S+)", True, 0.80),
|
||||
# ISA — "is a / is an" (most common, must handle "of Z" modifier)
|
||||
(r"(?P<subj>\S+)\s+is\s+an?\s+(?P<parent>\S+)", True, 0.85),
|
||||
# ISPART — explicit keyword forms
|
||||
(r"(?P<subj>\S+)\s+ISPART\s+(?P<parent>\S+)", False, 0.95),
|
||||
(r"(?P<subj>\S+)\s+is\s+part\s+of\s+(?P<parent>\S+)", False, 0.90),
|
||||
(r"(?P<subj>\S+)\s+is\s+a\s+member\s+of\s+(?P<parent>\S+)", False, 0.90),
|
||||
(r"(?P<subj>\S+)\s+is\s+owned\s+by\s+(?P<parent>\S+)", False, 0.90),
|
||||
(r"(?P<subj>\S+)\s+belongs\s+to\s+(?P<parent>\S+)", False, 0.85),
|
||||
(r"(?P<subj>\S+)\s+part\s+of\s+(?P<parent>\S+)", False, 0.80),
|
||||
(r"(?P<subj>\S+)\s+member\s+of\s+(?P<parent>\S+)", False, 0.80),
|
||||
(r"(?P<subj>\S+)\s+owned\s+by\s+(?P<parent>\S+)", False, 0.80),
|
||||
(r"(?P<subj>\S+)\s+runs\s+on\s+(?P<parent>\S+)", False, 0.85),
|
||||
(r"(?P<subj>\S+)\s+hosted\s+by\s+(?P<parent>\S+)", False, 0.85),
|
||||
(r"(?P<subj>\S+)\s+deployed\s+on\s+(?P<parent>\S+)", False, 0.85),
|
||||
(r"(?P<subj>\S+)\s+contained\s+in\s+(?P<parent>\S+)", False, 0.85),
|
||||
]
|
||||
|
||||
# Compiled patterns + metadata
|
||||
_PATTERNS: list[tuple[re.Pattern, bool, float]] = [
|
||||
(re.compile(p, re.IGNORECASE), is_isa, conf)
|
||||
for p, is_isa, conf in _RAW_PATTERNS
|
||||
]
|
||||
|
||||
# "of {Z}" dimension modifier following an ISA match (optional).
|
||||
# Captures the first word only; _extend_compound() handles multi-word.
|
||||
_OF_DIM_RE = re.compile(r"\s+of\s+(?P<dim>\S+)", re.IGNORECASE)
|
||||
|
||||
# Consecutive capital-starting words after a position
|
||||
_CAPITAL_WORD_RE = re.compile(r"\s+([A-Z]\w*)")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Token canonicalisation + compound extension
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _canonicalize(word: str) -> str:
|
||||
"""Strip punctuation, apply lowercase. Single word — no compound rule."""
|
||||
return re.sub(r"^[^\w]+|[^\w]+$", "", word).lower()
|
||||
|
||||
|
||||
def _extend_compound(text: str, after: int, first_word: str) -> str:
|
||||
"""
|
||||
Starting from position `after` in `text`, consume any immediately-following
|
||||
capital-starting words and merge them with `first_word` into a single
|
||||
underscore-joined lowercase compound token.
|
||||
|
||||
Example:
|
||||
text = "gnommoweb is part of Glitch University today"
|
||||
after = position after "Glitch"
|
||||
first_word = "Glitch"
|
||||
→ returns "glitch_university"
|
||||
"""
|
||||
words = [_canonicalize(first_word)]
|
||||
pos = after
|
||||
while True:
|
||||
m = _CAPITAL_WORD_RE.match(text, pos)
|
||||
if not m:
|
||||
break
|
||||
words.append(_canonicalize(m.group(1)))
|
||||
pos = m.end()
|
||||
return "_".join(words)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main scanner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def scan_cues(text: str) -> list[CueTriple]:
|
||||
"""
|
||||
Scan *text* for relationship cue patterns. Returns all matched triples.
|
||||
Deduplicates by (subject, parent, dimension, is_isa).
|
||||
Multi-word proper-noun parents and dimensions are merged via _extend_compound().
|
||||
"""
|
||||
results: list[CueTriple] = []
|
||||
seen: set[tuple] = set()
|
||||
|
||||
for pattern, is_isa, base_conf in _PATTERNS:
|
||||
for m in pattern.finditer(text):
|
||||
raw_subj = m.group("subj")
|
||||
raw_parent = m.group("parent")
|
||||
|
||||
if not raw_subj or not raw_parent:
|
||||
continue
|
||||
|
||||
subj = _canonicalize(raw_subj)
|
||||
# Extend parent into compound if followed by more capital words
|
||||
parent = _extend_compound(text, m.end("parent"), raw_parent)
|
||||
|
||||
if not subj or not parent:
|
||||
continue
|
||||
|
||||
# Check for "of {Z}" dimension modifier immediately after the match
|
||||
dimension: str
|
||||
if is_isa:
|
||||
suffix_start = m.end()
|
||||
of_match = _OF_DIM_RE.match(text, suffix_start)
|
||||
if of_match:
|
||||
raw_dim = of_match.group("dim")
|
||||
dimension = _extend_compound(text, of_match.end("dim"), raw_dim)
|
||||
else:
|
||||
dimension = "type"
|
||||
else:
|
||||
dimension = _infer_ispart_dimension(m.re.pattern)
|
||||
|
||||
key = (subj, parent, dimension, is_isa)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
results.append(CueTriple(
|
||||
subject=subj,
|
||||
parent=parent,
|
||||
dimension=dimension,
|
||||
is_isa=is_isa,
|
||||
confidence=base_conf,
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _infer_ispart_dimension(pattern_str: str) -> str:
|
||||
"""Guess a sensible default dimension from the ISPART pattern text."""
|
||||
if "runs" in pattern_str or "deployed" in pattern_str:
|
||||
return "runs-on"
|
||||
if "hosted" in pattern_str:
|
||||
return "runs-on"
|
||||
if "owned" in pattern_str:
|
||||
return "owned-by"
|
||||
if "part" in pattern_str or "contained" in pattern_str:
|
||||
return "membership"
|
||||
if "member" in pattern_str or "belongs" in pattern_str:
|
||||
return "membership"
|
||||
return "membership"
|
||||
@@ -0,0 +1,294 @@
|
||||
"""
|
||||
Database layer — asyncpg pool, schema init, cache warm-up, flush.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import cache
|
||||
from .cache import SoasRow, UrdEdge
|
||||
|
||||
log = logging.getLogger("festinger.db")
|
||||
|
||||
_pool: asyncpg.Pool | None = None
|
||||
|
||||
SCHEMA_PATH = Path(__file__).parent.parent / "db" / "schema.sql"
|
||||
|
||||
SEED_DIMENSIONS = [
|
||||
("type", True),
|
||||
("membership", False),
|
||||
("runs-on", False),
|
||||
("tech", False),
|
||||
("owned-by", False),
|
||||
("geography", False),
|
||||
]
|
||||
|
||||
|
||||
async def get_pool(dsn: str) -> asyncpg.Pool:
|
||||
global _pool
|
||||
if _pool is None:
|
||||
_pool = await asyncpg.create_pool(dsn, min_size=2, max_size=10)
|
||||
return _pool
|
||||
|
||||
|
||||
async def close_pool() -> None:
|
||||
global _pool
|
||||
if _pool:
|
||||
await _pool.close()
|
||||
_pool = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schema init
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def init_schema(pool: asyncpg.Pool) -> None:
|
||||
sql = SCHEMA_PATH.read_text()
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(sql)
|
||||
log.info("schema applied")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bootstrap helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def bootstrap_dimensions(pool: asyncpg.Pool) -> None:
|
||||
"""Ensure the 6 seed dimensions exist in SOAS and have self-referential root nodes."""
|
||||
async with pool.acquire() as conn:
|
||||
for token, _is_isa in SEED_DIMENSIONS:
|
||||
row = await conn.fetchrow(
|
||||
"INSERT INTO soas (token, saliency, novelty) VALUES ($1, 0, 0) "
|
||||
"ON CONFLICT (token) DO UPDATE SET token = EXCLUDED.token "
|
||||
"RETURNING id, token, encounter_count, saliency, novelty",
|
||||
token,
|
||||
)
|
||||
dim_id = row["id"]
|
||||
# Self-referential root node: id = parent_id = dim_id
|
||||
await conn.execute(
|
||||
"INSERT INTO urd (id, parent_id, dim_id, is_isa, confidence, source) "
|
||||
"VALUES ($1, $1, $1, $2, 1.0, 'festinger') "
|
||||
"ON CONFLICT DO NOTHING",
|
||||
dim_id, True,
|
||||
)
|
||||
log.info("seed dimensions bootstrapped")
|
||||
|
||||
|
||||
async def bootstrap_english_dictionary(pool: asyncpg.Pool) -> None:
|
||||
"""
|
||||
Bulk-load /usr/share/dict/words into SOAS at saliency=0, novelty=0.
|
||||
Skips tokens already present. Only loads tokens >= 5 chars.
|
||||
"""
|
||||
dict_file = Path("/usr/share/dict/words")
|
||||
if not dict_file.exists():
|
||||
log.warning("no system dictionary found at %s — skipping bootstrap", dict_file)
|
||||
return
|
||||
|
||||
words = [
|
||||
w.strip().lower()
|
||||
for w in dict_file.read_text().splitlines()
|
||||
if len(w.strip()) >= 5 and w.strip().isalpha()
|
||||
]
|
||||
# Deduplicate
|
||||
words = list(set(words))
|
||||
log.info("loading %d dictionary words into soas …", len(words))
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
# Use executemany with ON CONFLICT DO NOTHING for speed
|
||||
await conn.executemany(
|
||||
"INSERT INTO soas (token, saliency, novelty) VALUES ($1, 0, 0) "
|
||||
"ON CONFLICT (token) DO NOTHING",
|
||||
[(w,) for w in words],
|
||||
)
|
||||
log.info("dictionary bootstrap complete")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cache warm-up
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def warm_cache(pool: asyncpg.Pool) -> None:
|
||||
"""Load all SOAS and URD rows into the in-memory cache."""
|
||||
async with pool.acquire() as conn:
|
||||
soas_rows = await conn.fetch(
|
||||
"SELECT id, token, encounter_count, saliency, novelty FROM soas"
|
||||
)
|
||||
for r in soas_rows:
|
||||
row = SoasRow(
|
||||
id=r["id"],
|
||||
token=r["token"],
|
||||
encounter_count=r["encounter_count"],
|
||||
saliency=r["saliency"],
|
||||
novelty=r["novelty"],
|
||||
)
|
||||
cache.soas_by_token[r["token"]] = row
|
||||
cache.soas_by_id[r["id"]] = r["token"]
|
||||
|
||||
urd_rows = await conn.fetch(
|
||||
"""
|
||||
SELECT u.id, u.parent_id, u.dim_id, u.is_isa, u.confidence, u.source,
|
||||
p.token AS parent_token, d.token AS dim_token
|
||||
FROM urd u
|
||||
INNER JOIN soas p ON p.id = u.parent_id
|
||||
INNER JOIN soas d ON d.id = u.dim_id
|
||||
-- skip self-referential dimension root nodes
|
||||
WHERE NOT (u.id = u.parent_id AND u.id = u.dim_id)
|
||||
"""
|
||||
)
|
||||
for r in urd_rows:
|
||||
edge = UrdEdge(
|
||||
concept_id=r["id"],
|
||||
parent_id=r["parent_id"],
|
||||
dim_id=r["dim_id"],
|
||||
is_isa=r["is_isa"],
|
||||
confidence=r["confidence"],
|
||||
source=r["source"],
|
||||
parent_token=r["parent_token"],
|
||||
dim_token=r["dim_token"],
|
||||
)
|
||||
cache.urd_by_concept.setdefault(r["id"], []).append(edge)
|
||||
cache.urd_by_concept_dim[(r["id"], r["dim_id"])] = edge
|
||||
|
||||
pending = await conn.fetch(
|
||||
"SELECT DISTINCT concept_id FROM resolution_queue WHERE status = 'pending'"
|
||||
)
|
||||
cache.pending_conflicts = {r["concept_id"] for r in pending}
|
||||
|
||||
log.info(
|
||||
"cache warm: %d soas, %d urd edges, %d pending conflicts",
|
||||
len(cache.soas_by_token),
|
||||
len(cache.urd_by_concept_dim),
|
||||
len(cache.pending_conflicts),
|
||||
)
|
||||
|
||||
|
||||
async def reload_urd_cache(pool: asyncpg.Pool) -> None:
|
||||
"""Rebuild only URD cache (called after nightly resolution job)."""
|
||||
cache.urd_by_concept.clear()
|
||||
cache.urd_by_concept_dim.clear()
|
||||
async with pool.acquire() as conn:
|
||||
urd_rows = await conn.fetch(
|
||||
"""
|
||||
SELECT u.id, u.parent_id, u.dim_id, u.is_isa, u.confidence, u.source,
|
||||
p.token AS parent_token, d.token AS dim_token
|
||||
FROM urd u
|
||||
INNER JOIN soas p ON p.id = u.parent_id
|
||||
INNER JOIN soas d ON d.id = u.dim_id
|
||||
WHERE NOT (u.id = u.parent_id AND u.id = u.dim_id)
|
||||
"""
|
||||
)
|
||||
for r in urd_rows:
|
||||
edge = UrdEdge(
|
||||
concept_id=r["id"],
|
||||
parent_id=r["parent_id"],
|
||||
dim_id=r["dim_id"],
|
||||
is_isa=r["is_isa"],
|
||||
confidence=r["confidence"],
|
||||
source=r["source"],
|
||||
parent_token=r["parent_token"],
|
||||
dim_token=r["dim_token"],
|
||||
)
|
||||
cache.urd_by_concept.setdefault(r["id"], []).append(edge)
|
||||
cache.urd_by_concept_dim[(r["id"], r["dim_id"])] = edge
|
||||
|
||||
pending = await conn.fetch(
|
||||
"SELECT DISTINCT concept_id FROM resolution_queue WHERE status = 'pending'"
|
||||
)
|
||||
cache.pending_conflicts = {r["concept_id"] for r in pending}
|
||||
log.info("urd cache reloaded")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SOAS upsert — id always comes from Postgres
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def get_or_create_soas(pool: asyncpg.Pool, token: str) -> SoasRow:
|
||||
"""Return cached SoasRow, inserting into Postgres + cache if new."""
|
||||
if token in cache.soas_by_token:
|
||||
return cache.soas_by_token[token]
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"INSERT INTO soas (token) VALUES ($1) "
|
||||
"ON CONFLICT (token) DO UPDATE SET token = EXCLUDED.token "
|
||||
"RETURNING id, token, encounter_count, saliency, novelty",
|
||||
token,
|
||||
)
|
||||
|
||||
soas_row = SoasRow(
|
||||
id=row["id"],
|
||||
token=row["token"],
|
||||
encounter_count=row["encounter_count"],
|
||||
saliency=row["saliency"],
|
||||
novelty=row["novelty"],
|
||||
)
|
||||
cache.soas_by_token[token] = soas_row
|
||||
cache.soas_by_id[row["id"]] = token
|
||||
return soas_row
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Saliency recalculation (log-scale)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def recalculate_saliency(encounter_count: int, is_common_english: bool) -> float:
|
||||
"""
|
||||
Log-scaled saliency. Common English words are pre-seeded with count=0,
|
||||
novelty=0 and will always return 0. Domain tokens start at count=1 after
|
||||
first encounter; saliency grows logarithmically.
|
||||
"""
|
||||
if is_common_english or encounter_count <= 0:
|
||||
return 0.0
|
||||
return math.log1p(encounter_count)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Batch saliency flush
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def flush_encounter_deltas(pool: asyncpg.Pool) -> None:
|
||||
"""Flush staged encounter_count deltas to Postgres in one batch UPDATE."""
|
||||
deltas = cache.drain_deltas()
|
||||
if not deltas:
|
||||
return
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
for soas_id, delta in deltas.items():
|
||||
token = cache.soas_by_id.get(soas_id, "")
|
||||
row = cache.soas_by_token.get(token)
|
||||
new_count = (row.encounter_count if row else 0)
|
||||
# novelty = 0 for common English words (pre-seeded)
|
||||
is_common = (row.novelty == 0.0 and row.saliency == 0.0) if row else False
|
||||
new_saliency = recalculate_saliency(new_count, is_common)
|
||||
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE soas
|
||||
SET encounter_count = encounter_count + $1,
|
||||
last_seen = now(),
|
||||
saliency = $2
|
||||
WHERE id = $3
|
||||
""",
|
||||
delta, new_saliency, soas_id,
|
||||
)
|
||||
if row:
|
||||
row.saliency = new_saliency
|
||||
|
||||
log.debug("flushed %d saliency deltas", len(deltas))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def get_config(pool: asyncpg.Pool, key: str, default: str = "") -> str:
|
||||
async with pool.acquire() as conn:
|
||||
row = await conn.fetchrow("SELECT value FROM config WHERE key = $1", key)
|
||||
return row["value"] if row else default
|
||||
@@ -0,0 +1,210 @@
|
||||
"""
|
||||
LLM client — supports 'claude' and 'openai' providers.
|
||||
|
||||
Uses the model configured in the models table (fetched via write_model_id
|
||||
or resolve_model_id config keys).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
import asyncpg
|
||||
|
||||
log = logging.getLogger("festinger.llm")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelConfig:
|
||||
provider: str
|
||||
model_name: str
|
||||
api_key: str
|
||||
|
||||
|
||||
async def get_model_config(pool: asyncpg.Pool, model_id: str) -> Optional[ModelConfig]:
|
||||
if not model_id:
|
||||
return None
|
||||
try:
|
||||
mid = int(model_id)
|
||||
except ValueError:
|
||||
return None
|
||||
async with pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"SELECT provider, model_name, api_key FROM models WHERE id=$1", mid
|
||||
)
|
||||
if not row:
|
||||
return None
|
||||
return ModelConfig(
|
||||
provider=row["provider"],
|
||||
model_name=row["model_name"],
|
||||
api_key=row["api_key"],
|
||||
)
|
||||
|
||||
|
||||
async def call_llm(model: ModelConfig, prompt: str) -> str:
|
||||
"""Call the configured LLM and return the text response."""
|
||||
if model.provider == "claude":
|
||||
return await _call_claude(model, prompt)
|
||||
elif model.provider == "openai":
|
||||
return await _call_openai(model, prompt)
|
||||
else:
|
||||
raise ValueError(f"Unknown LLM provider: {model.provider!r}")
|
||||
|
||||
|
||||
async def _call_claude(model: ModelConfig, prompt: str) -> str:
|
||||
import anthropic
|
||||
client = anthropic.AsyncAnthropic(api_key=model.api_key)
|
||||
message = await client.messages.create(
|
||||
model=model.model_name,
|
||||
max_tokens=1024,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
)
|
||||
return message.content[0].text
|
||||
|
||||
|
||||
async def _call_openai(model: ModelConfig, prompt: str) -> str:
|
||||
from openai import AsyncOpenAI
|
||||
client = AsyncOpenAI(api_key=model.api_key)
|
||||
response = await client.chat.completions.create(
|
||||
model=model.model_name,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
max_tokens=1024,
|
||||
)
|
||||
return response.choices[0].message.content or ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Structured prompts
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
WRITE_PROMPT_TEMPLATE = """You are a knowledge extraction assistant.
|
||||
|
||||
Given the concept: "{concept}"
|
||||
|
||||
And these known dimensions: {dimensions}
|
||||
|
||||
Return a JSON array of triples describing what you know about this concept.
|
||||
Each triple must have these fields:
|
||||
- "parent": the containing concept (string)
|
||||
- "dimension": one of the known dimensions above, or a new specific one if none fit
|
||||
- "is_isa": true if this is a classification (ISA), false if membership/containment (ISPART)
|
||||
- "confidence": 0.0 to 1.0
|
||||
|
||||
Return ONLY the JSON array. No explanation. Example:
|
||||
[
|
||||
{{"parent": "software-repository", "dimension": "type", "is_isa": true, "confidence": 0.9}},
|
||||
{{"parent": "glitch-university", "dimension": "membership", "is_isa": false, "confidence": 0.85}}
|
||||
]
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class LLMTriple:
|
||||
parent: str
|
||||
dimension: str
|
||||
is_isa: bool
|
||||
confidence: float
|
||||
|
||||
|
||||
def parse_llm_triples(response: str) -> list[LLMTriple]:
|
||||
"""Parse JSON array of triples from LLM response."""
|
||||
try:
|
||||
# Find the JSON array in the response
|
||||
start = response.find("[")
|
||||
end = response.rfind("]") + 1
|
||||
if start == -1 or end == 0:
|
||||
return []
|
||||
data = json.loads(response[start:end])
|
||||
triples = []
|
||||
for item in data:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
triples.append(LLMTriple(
|
||||
parent=str(item.get("parent", "")).strip().lower(),
|
||||
dimension=str(item.get("dimension", "type")).strip().lower(),
|
||||
is_isa=bool(item.get("is_isa", True)),
|
||||
confidence=float(item.get("confidence", 0.7)),
|
||||
))
|
||||
return triples
|
||||
except (json.JSONDecodeError, ValueError) as e:
|
||||
log.warning("failed to parse LLM triples: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
RESOLVE_ISA_ISA_PROMPT = """You are a knowledge graph resolution assistant.
|
||||
|
||||
Concept: "{concept}"
|
||||
Current fact: "{concept}" is a "{existing_parent}" in dimension "{dimension}"
|
||||
Conflicting fact: "{concept}" is a "{incoming_parent}" in dimension "{dimension}"
|
||||
|
||||
Both facts appear to be simultaneously true. The dimension "{dimension}" is too coarse.
|
||||
Propose two more specific dimension names to replace it:
|
||||
- existing_dimension: the dimension where "{concept}" as "{existing_parent}" is valid
|
||||
- new_dimension: the dimension where "{concept}" as "{incoming_parent}" is valid
|
||||
|
||||
Known dimensions for reference: {known_dimensions}
|
||||
|
||||
Return ONLY a JSON object:
|
||||
{{"decision": "decompose", "existing_dimension": "...", "new_dimension": "...", "reasoning": "..."}}
|
||||
|
||||
If the conflicting fact is clearly wrong, return:
|
||||
{{"decision": "dismiss", "reasoning": "..."}}
|
||||
"""
|
||||
|
||||
RESOLVE_ISPART_ISPART_PROMPT = """You are a knowledge graph resolution assistant.
|
||||
|
||||
Concept: "{concept}"
|
||||
Current fact: "{concept}" belongs to "{existing_parent}" in dimension "{dimension}"
|
||||
Conflicting fact: "{concept}" belongs to "{incoming_parent}" in dimension "{dimension}"
|
||||
|
||||
These facts contradict each other — a thing can only belong to one place per dimension.
|
||||
|
||||
Return ONLY a JSON object with your decision:
|
||||
{{"decision": "update", "reasoning": "..."}} — if the incoming fact is more current/correct
|
||||
{{"decision": "dismiss", "reasoning": "..."}} — if the existing fact is still correct
|
||||
"""
|
||||
|
||||
RESOLVE_MISCLASS_PROMPT = """You are a knowledge graph resolution assistant.
|
||||
|
||||
Concept: "{concept}"
|
||||
A relationship was classified inconsistently:
|
||||
- Existing: "{concept}" ISA "{existing_parent}" in dimension "{dimension}"
|
||||
- Incoming: "{concept}" ISPART "{incoming_parent}" in dimension "{dimension}"
|
||||
|
||||
The incoming fact uses a different relation type. Suggest the correct dimension for the incoming fact.
|
||||
|
||||
Known dimensions: {known_dimensions}
|
||||
|
||||
Return ONLY a JSON object:
|
||||
{{"decision": "reclassify", "correct_dimension": "...", "reasoning": "..."}}
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResolutionDecision:
|
||||
decision: str # 'decompose', 'update', 'dismiss', 'reclassify'
|
||||
existing_dimension: str = ""
|
||||
new_dimension: str = ""
|
||||
correct_dimension: str = ""
|
||||
reasoning: str = ""
|
||||
|
||||
|
||||
def parse_resolution_decision(response: str) -> Optional[ResolutionDecision]:
|
||||
try:
|
||||
start = response.find("{")
|
||||
end = response.rfind("}") + 1
|
||||
if start == -1 or end == 0:
|
||||
return None
|
||||
data = json.loads(response[start:end])
|
||||
return ResolutionDecision(
|
||||
decision=str(data.get("decision", "dismiss")).lower(),
|
||||
existing_dimension=str(data.get("existing_dimension", "")).lower(),
|
||||
new_dimension=str(data.get("new_dimension", "")).lower(),
|
||||
correct_dimension=str(data.get("correct_dimension", "")).lower(),
|
||||
reasoning=str(data.get("reasoning", "")),
|
||||
)
|
||||
except (json.JSONDecodeError, ValueError) as e:
|
||||
log.warning("failed to parse resolution decision: %s", e)
|
||||
return None
|
||||
@@ -0,0 +1,93 @@
|
||||
"""
|
||||
Loop detector — extracted from the original proxy.py POC.
|
||||
|
||||
Session-scoped exact-match repetition detection with configurable mitigations.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from collections import defaultdict, deque
|
||||
|
||||
log = logging.getLogger("festinger.loop")
|
||||
|
||||
# Key: (model, session_fingerprint) → deque of response hashes
|
||||
_history: dict[tuple, deque] = defaultdict(lambda: deque(maxlen=20))
|
||||
_consecutive: dict[tuple, int] = defaultdict(int)
|
||||
_last_hash: dict[tuple, str] = {}
|
||||
|
||||
|
||||
def session_key(model: str, messages: list[dict]) -> tuple[str, str]:
|
||||
system_msgs = [m for m in messages if m.get("role") == "system"]
|
||||
if system_msgs:
|
||||
fingerprint = hashlib.sha256(
|
||||
system_msgs[0].get("content", "")[:512].encode()
|
||||
).hexdigest()[:16]
|
||||
else:
|
||||
non_assistant = [m for m in messages if m.get("role") != "assistant"]
|
||||
blob = json.dumps(non_assistant, sort_keys=True)
|
||||
fingerprint = hashlib.sha256(blob.encode()).hexdigest()[:16]
|
||||
return (model, fingerprint)
|
||||
|
||||
|
||||
def hash_response(text: str) -> str:
|
||||
return hashlib.sha256(text.strip().encode()).hexdigest()
|
||||
|
||||
|
||||
def record_and_check(session: tuple, text: str, min_length: int) -> int:
|
||||
if len(text.strip()) < min_length:
|
||||
return 1
|
||||
h = hash_response(text)
|
||||
if _last_hash.get(session) == h:
|
||||
_consecutive[session] += 1
|
||||
else:
|
||||
_consecutive[session] = 1
|
||||
_last_hash[session] = h
|
||||
_history[session].append(h)
|
||||
return _consecutive[session]
|
||||
|
||||
|
||||
def apply_mitigations(
|
||||
request_body: dict,
|
||||
count: int,
|
||||
config: dict,
|
||||
) -> tuple[dict, str | None]:
|
||||
mitigations = config.get("mitigations", [])
|
||||
override: str | None = None
|
||||
|
||||
for m in mitigations:
|
||||
if not m.get("enabled", True):
|
||||
continue
|
||||
if count < m.get("trigger_count", 2):
|
||||
continue
|
||||
|
||||
strategy = m["strategy"]
|
||||
|
||||
if strategy == "temperature_boost":
|
||||
opts = request_body.setdefault("options", {})
|
||||
current = opts.get("temperature", 0.7)
|
||||
boosted = min(current + m.get("boost_amount", 0.35), m.get("max_temperature", 1.4))
|
||||
opts["temperature"] = boosted
|
||||
log.warning("mitigation=temperature_boost %.2f → %.2f (count=%d)", current, boosted, count)
|
||||
|
||||
elif strategy == "forbidden_action":
|
||||
msg = m.get("injection_message", "STOP. Try something completely different.").format(count=count)
|
||||
request_body.setdefault("messages", []).append({"role": "user", "content": msg})
|
||||
log.warning("mitigation=forbidden_action injected (count=%d)", count)
|
||||
|
||||
elif strategy == "history_truncation":
|
||||
messages = request_body.get("messages", [])
|
||||
truncate = m.get("truncate_turns", 6)
|
||||
system_msgs = [m_ for m_ in messages if m_.get("role") == "system"]
|
||||
non_system = [m_ for m_ in messages if m_.get("role") != "system"]
|
||||
trimmed = non_system[:-truncate] if len(non_system) > truncate else non_system[-2:]
|
||||
request_body["messages"] = system_msgs + trimmed
|
||||
log.warning("mitigation=history_truncation dropped %d turns (count=%d)", truncate, count)
|
||||
|
||||
elif strategy == "circuit_breaker":
|
||||
override = m.get("response_message", "[LOOP DETECTOR] Halted due to repeated responses.")
|
||||
log.error("mitigation=circuit_breaker TRIGGERED (count=%d)", count)
|
||||
break
|
||||
|
||||
return request_body, override
|
||||
@@ -0,0 +1,821 @@
|
||||
"""
|
||||
Festinger — main FastAPI application.
|
||||
|
||||
Routes:
|
||||
POST /api/chat Ollama-compatible chat (loop detection + recollection injection)
|
||||
POST /api/generate Ollama-compatible generate
|
||||
POST /v1/messages Anthropic Messages API proxy (loop detection + recollection)
|
||||
POST /v1/chat/completions OpenAI-compatible proxy (loop detection + recollection)
|
||||
POST /iknowthat Manual write path (gutask iknowthat)
|
||||
POST /resolve/run Manually trigger nightly resolution job
|
||||
POST /reload Reload URD cache (called by resolution job)
|
||||
GET /health Health + stats
|
||||
GET /conflicts Pending and recently resolved conflicts
|
||||
GET /admin Minimal admin UI
|
||||
* /v1/{path} Passthrough to upstream Anthropic
|
||||
* /{path} Passthrough to upstream Ollama
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import yaml
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from fastapi import FastAPI, Request, Response
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
from . import cache
|
||||
from .db import (
|
||||
close_pool, get_config, get_or_create_soas,
|
||||
get_pool, init_schema, bootstrap_dimensions,
|
||||
bootstrap_english_dictionary, warm_cache, reload_urd_cache,
|
||||
flush_encounter_deltas,
|
||||
)
|
||||
from .loop_detector import apply_mitigations, record_and_check, session_key
|
||||
from .cue_scanner import scan_cues
|
||||
from .recollection import build_recollection_block, inject_recollection
|
||||
from .resolution_job import run_resolution_job, last_run_timestamp
|
||||
from .tokenizer import tokenize
|
||||
from .write_queue import enqueue_concept, enqueue_cue, start_worker, stop_worker
|
||||
from .urd_writer import InsertRequest, insert_urd_edge
|
||||
from .wordnet import import_wordnet, CITATION as WORDNET_CITATION
|
||||
from .test_scenarios import SCENARIOS, seed_scenario, reset_scenario
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(message)s",
|
||||
)
|
||||
log = logging.getLogger("festinger")
|
||||
|
||||
CONFIG_PATH = Path(__file__).parent.parent / "config.yaml"
|
||||
|
||||
|
||||
def load_yaml_config() -> dict:
|
||||
with open(CONFIG_PATH) as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lifespan — startup / shutdown
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_scheduler = AsyncIOScheduler()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
cfg = load_yaml_config()
|
||||
dsn = os.environ.get("POSTGRES_DSN", cfg.get("postgres_dsn", ""))
|
||||
|
||||
pool = await get_pool(dsn)
|
||||
app.state.pool = pool
|
||||
app.state.yaml_config = cfg
|
||||
|
||||
await init_schema(pool)
|
||||
await bootstrap_dimensions(pool)
|
||||
await bootstrap_english_dictionary(pool)
|
||||
await warm_cache(pool)
|
||||
await start_worker(pool)
|
||||
|
||||
# Schedule saliency flush every 30 s
|
||||
_scheduler.add_job(flush_encounter_deltas, "interval", seconds=30, args=[pool], id="saliency_flush")
|
||||
|
||||
# Schedule nightly resolution job from config table
|
||||
resolution_cron = await get_config(pool, "resolution_schedule", "0 2 * * *")
|
||||
cron_parts = resolution_cron.split()
|
||||
if len(cron_parts) == 5:
|
||||
minute, hour, day, month, dow = cron_parts
|
||||
_scheduler.add_job(
|
||||
run_resolution_job, "cron",
|
||||
minute=minute, hour=hour, day=day, month=month, day_of_week=dow,
|
||||
args=[pool], id="nightly_resolution",
|
||||
)
|
||||
|
||||
_scheduler.start()
|
||||
log.info("festinger ready")
|
||||
|
||||
yield
|
||||
|
||||
_scheduler.shutdown(wait=False)
|
||||
await stop_worker()
|
||||
await close_pool()
|
||||
|
||||
|
||||
app = FastAPI(title="Festinger", lifespan=lifespan)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ollama forwarding helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def call_ollama(path: str, body: dict, upstream: str) -> tuple[str, dict]:
|
||||
body = dict(body)
|
||||
body["stream"] = False
|
||||
async with httpx.AsyncClient(timeout=300.0) as client:
|
||||
r = await client.post(f"{upstream}{path}", json=body)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
if path == "/api/chat":
|
||||
text = data.get("message", {}).get("content", "")
|
||||
else:
|
||||
text = data.get("response", "")
|
||||
return text, data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Anthropic forwarding helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _relay_headers(request: Request, keep: tuple[str, ...]) -> dict[str, str]:
|
||||
"""Extract a safe subset of request headers to forward upstream."""
|
||||
return {
|
||||
k: v for k, v in request.headers.items()
|
||||
if k.lower() in keep
|
||||
}
|
||||
|
||||
|
||||
ANTHROPIC_RELAY_HEADERS = (
|
||||
"x-api-key",
|
||||
"anthropic-version",
|
||||
"anthropic-beta",
|
||||
"content-type",
|
||||
)
|
||||
|
||||
OPENAI_RELAY_HEADERS = (
|
||||
"authorization",
|
||||
"content-type",
|
||||
"openai-organization",
|
||||
"openai-project",
|
||||
)
|
||||
|
||||
|
||||
async def call_anthropic(body: dict, upstream: str, headers: dict) -> tuple[str, dict]:
|
||||
"""
|
||||
Forward a request to the Anthropic Messages API (non-streaming).
|
||||
Returns (assistant_text, raw_response_dict).
|
||||
"""
|
||||
body = dict(body)
|
||||
body["stream"] = False
|
||||
# Anthropic requires anthropic-version header; add default if caller omitted it
|
||||
if "anthropic-version" not in {k.lower() for k in headers}:
|
||||
headers = {**headers, "anthropic-version": "2023-06-01"}
|
||||
|
||||
async with httpx.AsyncClient(timeout=300.0) as client:
|
||||
r = await client.post(
|
||||
f"{upstream}/v1/messages",
|
||||
json=body,
|
||||
headers=headers,
|
||||
)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
|
||||
# Extract text from Anthropic content blocks
|
||||
text = ""
|
||||
for block in data.get("content", []):
|
||||
if block.get("type") == "text":
|
||||
text += block.get("text", "")
|
||||
return text, data
|
||||
|
||||
|
||||
async def call_openai(body: dict, upstream: str, headers: dict) -> tuple[str, dict]:
|
||||
"""
|
||||
Forward a request to an OpenAI-compatible chat completions endpoint (non-streaming).
|
||||
Returns (assistant_text, raw_response_dict).
|
||||
"""
|
||||
body = dict(body)
|
||||
body["stream"] = False
|
||||
|
||||
async with httpx.AsyncClient(timeout=300.0) as client:
|
||||
r = await client.post(
|
||||
f"{upstream}/v1/chat/completions",
|
||||
json=body,
|
||||
headers=headers,
|
||||
)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
|
||||
text = data.get("choices", [{}])[0].get("message", {}).get("content", "")
|
||||
return text, data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Text extraction helpers (unified across API formats)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def extract_prompt_text(body: dict, path: str) -> str:
|
||||
"""Extract a flat string from a request body for saliency processing."""
|
||||
if path in ("/api/chat", "/v1/chat/completions"):
|
||||
messages = body.get("messages", [])
|
||||
parts = []
|
||||
for m in messages:
|
||||
content = m.get("content", "")
|
||||
if isinstance(content, str):
|
||||
parts.append(content)
|
||||
elif isinstance(content, list):
|
||||
# Anthropic-style content blocks
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "text":
|
||||
parts.append(block.get("text", ""))
|
||||
# Include top-level system field (Anthropic format)
|
||||
if body.get("system"):
|
||||
parts.insert(0, body["system"])
|
||||
return " ".join(parts)
|
||||
if path == "/v1/messages":
|
||||
return extract_prompt_text(body, "/v1/chat/completions")
|
||||
return body.get("prompt", "")
|
||||
|
||||
|
||||
def inject_recollection_anthropic(body: dict, block: str) -> dict:
|
||||
"""
|
||||
Inject a recollection block into an Anthropic Messages API request.
|
||||
Anthropic uses a top-level 'system' string field rather than a system message.
|
||||
"""
|
||||
body = dict(body)
|
||||
existing = body.get("system") or ""
|
||||
body["system"] = block + ("\n\n" + existing if existing else "")
|
||||
return body
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Saliency + recollection pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def process_prompt(body: dict, path: str, pool, cfg: dict) -> dict:
|
||||
"""
|
||||
Run the saliency + recollection pipeline over the prompt.
|
||||
Returns a (possibly modified) body dict with the recollection block injected.
|
||||
"""
|
||||
read_threshold = float(await get_config(pool, "saliency_read_threshold", "0.5"))
|
||||
write_threshold = float(await get_config(pool, "saliency_write_threshold", "1.2"))
|
||||
conf_floor = float(await get_config(pool, "recollection_confidence_floor", "0.6"))
|
||||
recency_days = int(await get_config(pool, "recollection_recency_days", "90"))
|
||||
|
||||
prompt_text = extract_prompt_text(body, path)
|
||||
if not prompt_text.strip():
|
||||
return body
|
||||
|
||||
# 1. Scan for explicit relationship cues (bypass threshold)
|
||||
cues = scan_cues(prompt_text)
|
||||
for cue in cues:
|
||||
await enqueue_cue(cue)
|
||||
|
||||
# 2. Tokenise + update saliency
|
||||
tokens = tokenize(prompt_text)
|
||||
salient_for_read: list[int] = []
|
||||
salient_for_write: list[str] = []
|
||||
|
||||
for token in tokens:
|
||||
soas_row = cache.soas_by_token.get(token)
|
||||
if soas_row is None:
|
||||
# New token — get_or_create happens in background via queue when needed
|
||||
continue # unknown token — skip saliency for now; write queue handles creation
|
||||
|
||||
cache.record_encounter(soas_row.id)
|
||||
|
||||
if soas_row.saliency >= read_threshold:
|
||||
salient_for_read.append(soas_row.id)
|
||||
|
||||
if soas_row.saliency >= write_threshold and soas_row.novelty < 1.0:
|
||||
salient_for_write.append(token)
|
||||
|
||||
for token in salient_for_write:
|
||||
await enqueue_concept(token)
|
||||
|
||||
if not salient_for_read:
|
||||
return body
|
||||
|
||||
# 3. Build recollection block
|
||||
block = build_recollection_block(salient_for_read, conf_floor, recency_days)
|
||||
if not block:
|
||||
return body
|
||||
|
||||
# 4. Inject into messages
|
||||
if path == "/api/chat" or path == "/v1/chat/completions":
|
||||
body = dict(body)
|
||||
body["messages"] = inject_recollection(body.get("messages", []), block)
|
||||
elif path == "/v1/messages":
|
||||
body = inject_recollection_anthropic(body, block)
|
||||
# /api/generate uses a flat prompt string — prepend there
|
||||
elif path == "/api/generate":
|
||||
body = dict(body)
|
||||
body["prompt"] = block + "\n\n" + body.get("prompt", "")
|
||||
|
||||
return body
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.post("/api/chat")
|
||||
async def chat(request: Request) -> Response:
|
||||
cfg = request.app.state.yaml_config
|
||||
pool = request.app.state.pool
|
||||
body = await request.json()
|
||||
model = body.get("model", "unknown")
|
||||
upstream = cfg["upstream_ollama"]
|
||||
min_len = cfg["detection"]["min_length"]
|
||||
|
||||
body = await process_prompt(body, "/api/chat", pool, cfg)
|
||||
|
||||
text, raw = await call_ollama("/api/chat", body, upstream)
|
||||
sess = session_key(model, body.get("messages", []))
|
||||
count = record_and_check(sess, text, min_len)
|
||||
|
||||
if count >= 2:
|
||||
log.warning("loop detected model=%s session=%s count=%d", model, sess[1], count)
|
||||
body, override = apply_mitigations(body, count, cfg)
|
||||
if override is not None:
|
||||
raw["message"] = {"role": "assistant", "content": override}
|
||||
raw["loop_detected"] = True
|
||||
return Response(content=json.dumps(raw), media_type="application/json")
|
||||
text, raw = await call_ollama("/api/chat", body, upstream)
|
||||
record_and_check(sess, text, min_len)
|
||||
|
||||
raw["message"] = {"role": "assistant", "content": text}
|
||||
return Response(content=json.dumps(raw), media_type="application/json")
|
||||
|
||||
|
||||
@app.post("/api/generate")
|
||||
async def generate(request: Request) -> Response:
|
||||
cfg = request.app.state.yaml_config
|
||||
pool = request.app.state.pool
|
||||
body = await request.json()
|
||||
model = body.get("model", "unknown")
|
||||
upstream = cfg["upstream_ollama"]
|
||||
min_len = cfg["detection"]["min_length"]
|
||||
|
||||
body = await process_prompt(body, "/api/generate", pool, cfg)
|
||||
|
||||
messages = [{"role": "user", "content": body.get("prompt", "")}]
|
||||
sess = session_key(model, messages)
|
||||
|
||||
text, raw = await call_ollama("/api/generate", body, upstream)
|
||||
count = record_and_check(sess, text, min_len)
|
||||
|
||||
if count >= 2:
|
||||
log.warning("loop detected model=%s session=%s count=%d", model, sess[1], count)
|
||||
body, override = apply_mitigations(body, count, cfg)
|
||||
if override is not None:
|
||||
raw["response"] = override
|
||||
raw["loop_detected"] = True
|
||||
return Response(content=json.dumps(raw), media_type="application/json")
|
||||
text, raw = await call_ollama("/api/generate", body, upstream)
|
||||
record_and_check(sess, text, min_len)
|
||||
|
||||
raw["response"] = text
|
||||
return Response(content=json.dumps(raw), media_type="application/json")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Anthropic Messages API (POST /v1/messages)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.post("/v1/messages")
|
||||
async def anthropic_messages(request: Request) -> Response:
|
||||
cfg = request.app.state.yaml_config
|
||||
pool = request.app.state.pool
|
||||
body = await request.json()
|
||||
model = body.get("model", "unknown")
|
||||
upstream = cfg["upstream_anthropic"]
|
||||
min_len = cfg["detection"]["min_length"]
|
||||
|
||||
headers = _relay_headers(request, ANTHROPIC_RELAY_HEADERS)
|
||||
# Ensure anthropic-version is present
|
||||
if "anthropic-version" not in {k.lower() for k in headers}:
|
||||
headers["anthropic-version"] = "2023-06-01"
|
||||
|
||||
body = await process_prompt(body, "/v1/messages", pool, cfg)
|
||||
|
||||
# Use messages list as session key (same logic as /api/chat)
|
||||
messages = body.get("messages", [])
|
||||
sess = session_key(model, messages)
|
||||
|
||||
text, raw = await call_anthropic(body, upstream, headers)
|
||||
count = record_and_check(sess, text, min_len)
|
||||
|
||||
if count >= 2:
|
||||
log.warning("loop detected model=%s session=%s count=%d", model, sess[1], count)
|
||||
body, override = apply_mitigations(body, count, cfg)
|
||||
if override is not None:
|
||||
# Return a minimal Anthropic-format response with the override message
|
||||
raw["content"] = [{"type": "text", "text": override}]
|
||||
raw["loop_detected"] = True
|
||||
return Response(content=json.dumps(raw), media_type="application/json")
|
||||
text, raw = await call_anthropic(body, upstream, headers)
|
||||
record_and_check(sess, text, min_len)
|
||||
|
||||
return Response(content=json.dumps(raw), media_type="application/json")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenAI-compatible chat completions (POST /v1/chat/completions)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.post("/v1/chat/completions")
|
||||
async def openai_chat_completions(request: Request) -> Response:
|
||||
cfg = request.app.state.yaml_config
|
||||
pool = request.app.state.pool
|
||||
body = await request.json()
|
||||
model = body.get("model", "unknown")
|
||||
upstream = cfg["upstream_openai"]
|
||||
min_len = cfg["detection"]["min_length"]
|
||||
|
||||
headers = _relay_headers(request, OPENAI_RELAY_HEADERS)
|
||||
|
||||
body = await process_prompt(body, "/v1/chat/completions", pool, cfg)
|
||||
|
||||
messages = body.get("messages", [])
|
||||
sess = session_key(model, messages)
|
||||
|
||||
text, raw = await call_openai(body, upstream, headers)
|
||||
count = record_and_check(sess, text, min_len)
|
||||
|
||||
if count >= 2:
|
||||
log.warning("loop detected model=%s session=%s count=%d", model, sess[1], count)
|
||||
body, override = apply_mitigations(body, count, cfg)
|
||||
if override is not None:
|
||||
if raw.get("choices"):
|
||||
raw["choices"][0]["message"]["content"] = override
|
||||
raw["loop_detected"] = True
|
||||
return Response(content=json.dumps(raw), media_type="application/json")
|
||||
text, raw = await call_openai(body, upstream, headers)
|
||||
record_and_check(sess, text, min_len)
|
||||
|
||||
return Response(content=json.dumps(raw), media_type="application/json")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /iknowthat — manual write path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.post("/iknowthat")
|
||||
async def iknowthat(request: Request) -> dict:
|
||||
"""
|
||||
Parse and insert a manual fact.
|
||||
Body: {"fact": "gnommoweb -isa repo in context of glitch_university"}
|
||||
"""
|
||||
pool = request.app.state.pool
|
||||
data = await request.json()
|
||||
fact = data.get("fact", "").strip()
|
||||
if not fact:
|
||||
return {"error": "fact is required"}
|
||||
|
||||
# Parse: "{subject} -isa|-ispart {parent} [in context of {dimension}]"
|
||||
import re
|
||||
m = re.match(
|
||||
r"^(?P<subj>\S+)\s+(?P<rel>-isa|-ispart)\s+(?P<parent>\S+)"
|
||||
r"(?:\s+in\s+context\s+of\s+(?P<dim>\S+))?$",
|
||||
fact, re.IGNORECASE,
|
||||
)
|
||||
if not m:
|
||||
return {"error": f"could not parse fact: {fact!r}. "
|
||||
"Expected: '<subject> -isa|-ispart <parent> [in context of <dimension>]'"}
|
||||
|
||||
subj = m.group("subj").lower()
|
||||
is_isa = m.group("rel").lower() == "-isa"
|
||||
parent = m.group("parent").lower()
|
||||
dim = (m.group("dim") or ("type" if is_isa else "membership")).lower()
|
||||
|
||||
subj_row = await get_or_create_soas(pool, subj)
|
||||
parent_row = await get_or_create_soas(pool, parent)
|
||||
dim_row = await get_or_create_soas(pool, dim)
|
||||
|
||||
req = InsertRequest(
|
||||
concept_id=subj_row.id,
|
||||
parent_id=parent_row.id,
|
||||
dim_id=dim_row.id,
|
||||
is_isa=is_isa,
|
||||
confidence=1.0,
|
||||
source="gutask",
|
||||
)
|
||||
collision = await insert_urd_edge(pool, req, priority=True)
|
||||
|
||||
if collision:
|
||||
return {
|
||||
"status": "collision",
|
||||
"collision_type": collision.collision_type,
|
||||
"fact": fact,
|
||||
"message": "Conflict queued for nightly resolution (priority). "
|
||||
"Current world model unchanged.",
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "inserted",
|
||||
"fact": fact,
|
||||
"subject": subj,
|
||||
"parent": parent,
|
||||
"dimension": dim,
|
||||
"is_isa": is_isa,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /resolve/run — manually trigger resolution job
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.post("/resolve/run")
|
||||
async def resolve_run(request: Request) -> dict:
|
||||
pool = request.app.state.pool
|
||||
result = await run_resolution_job(pool)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /wordnet/import — bulk-load WordNet lemmas into SOAS
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.post("/wordnet/import")
|
||||
async def wordnet_import(request: Request) -> dict:
|
||||
"""
|
||||
Import Princeton WordNet 3.x lemmas into SOAS (saliency=0, novelty=0).
|
||||
Idempotent — already-present tokens are skipped.
|
||||
Can take 5–15 seconds for a full import (~130k lemmas).
|
||||
"""
|
||||
pool = request.app.state.pool
|
||||
result = await import_wordnet(pool)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /reload — reload URD cache (used after resolution job)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.post("/reload")
|
||||
async def reload(request: Request) -> dict:
|
||||
pool = request.app.state.pool
|
||||
await reload_urd_cache(pool)
|
||||
return {"status": "ok", "urd_edges": len(cache.urd_by_concept_dim)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /health
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.get("/health")
|
||||
async def health(request: Request) -> dict:
|
||||
cfg = request.app.state.yaml_config
|
||||
return {
|
||||
"status": "ok",
|
||||
"upstream": cfg["upstream_ollama"],
|
||||
"active_loop_sessions": 0, # loop detector is stateful in-process
|
||||
"soas_tokens": len(cache.soas_by_token),
|
||||
"urd_edges": len(cache.urd_by_concept_dim),
|
||||
"pending_conflicts": len(cache.pending_conflicts),
|
||||
"last_resolution_run": last_run_timestamp(),
|
||||
"timestamp": int(time.time()),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /conflicts — expose resolution queue
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.get("/conflicts")
|
||||
async def conflicts(request: Request) -> dict:
|
||||
pool = request.app.state.pool
|
||||
async with pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT id, concept_id, existing_parent_id, incoming_parent_id,
|
||||
dim_id, collision_type, status, resolution, priority,
|
||||
created_at, resolved_at
|
||||
FROM resolution_queue
|
||||
ORDER BY status ASC, created_at DESC
|
||||
LIMIT 100
|
||||
"""
|
||||
)
|
||||
|
||||
def format_row(r):
|
||||
return {
|
||||
"id": r["id"],
|
||||
"concept": cache.soas_by_id.get(r["concept_id"], str(r["concept_id"])),
|
||||
"existing_parent": cache.soas_by_id.get(r["existing_parent_id"], "?"),
|
||||
"incoming_parent": cache.soas_by_id.get(r["incoming_parent_id"], "?"),
|
||||
"dimension": cache.soas_by_id.get(r["dim_id"], "?"),
|
||||
"collision_type": r["collision_type"],
|
||||
"status": r["status"],
|
||||
"resolution": r["resolution"],
|
||||
"priority": r["priority"],
|
||||
"created_at": r["created_at"].isoformat() if r["created_at"] else None,
|
||||
"resolved_at": r["resolved_at"].isoformat() if r["resolved_at"] else None,
|
||||
}
|
||||
|
||||
return {"conflicts": [format_row(r) for r in rows]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /test — scenario seeding and reset (for integration testing)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.get("/test/scenarios")
|
||||
async def list_scenarios() -> dict:
|
||||
"""List all available test scenarios."""
|
||||
return {
|
||||
"scenarios": {
|
||||
sid: {
|
||||
"name": sc.name,
|
||||
"trigger_fact": sc.trigger_fact,
|
||||
"expected_collision": sc.expected_collision,
|
||||
"expected_resolution": sc.expected_resolution,
|
||||
}
|
||||
for sid, sc in SCENARIOS.items()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@app.post("/test/seed/{scenario_id}")
|
||||
async def test_seed(scenario_id: str, request: Request) -> dict:
|
||||
"""
|
||||
Seed the database with pre-defined world-model state for a named scenario.
|
||||
After seeding, POST the trigger fact to /iknowthat to cause the collision.
|
||||
|
||||
Available scenarios: A (misclassification), B (ISA+ISA decompose), C (ISPART+ISPART)
|
||||
"""
|
||||
pool = request.app.state.pool
|
||||
return await seed_scenario(pool, scenario_id)
|
||||
|
||||
|
||||
@app.post("/test/reset/{scenario_id}")
|
||||
async def test_reset(scenario_id: str, request: Request) -> dict:
|
||||
"""
|
||||
Remove URD edges, SOAS saliency boosts, and resolution queue rows
|
||||
introduced by a scenario seed. Use before re-running a scenario.
|
||||
"""
|
||||
pool = request.app.state.pool
|
||||
return await reset_scenario(pool, scenario_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /admin — minimal HTML UI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ADMIN_HTML = """<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Festinger Admin</title>
|
||||
<style>
|
||||
body {{ font-family: monospace; max-width: 960px; margin: 40px auto; padding: 0 20px; color: #222; }}
|
||||
h1 {{ font-size: 1.4em; margin-bottom: 0.2em; }}
|
||||
h2 {{ font-size: 1.1em; margin-top: 2em; border-bottom: 1px solid #ddd; padding-bottom: 4px; }}
|
||||
.subtitle {{ color: #666; font-size: 0.85em; margin-bottom: 1.5em; }}
|
||||
.stats {{ display: flex; gap: 2em; flex-wrap: wrap; margin: 1em 0; }}
|
||||
.stat {{ background: #f8f8f8; border: 1px solid #e0e0e0; border-radius: 4px; padding: 12px 20px; min-width: 130px; }}
|
||||
.stat-label {{ font-size: 0.75em; color: #666; text-transform: uppercase; letter-spacing: 0.05em; }}
|
||||
.stat-value {{ font-size: 1.8em; font-weight: bold; margin-top: 2px; }}
|
||||
.actions {{ display: flex; gap: 1em; flex-wrap: wrap; margin: 1em 0; }}
|
||||
button {{ padding: 8px 18px; cursor: pointer; border: 1px solid #aaa; background: #fff; border-radius: 3px; font-family: monospace; }}
|
||||
button:hover {{ background: #f0f0f0; }}
|
||||
button.primary {{ background: #1a1a2e; color: #fff; border-color: #1a1a2e; }}
|
||||
button.primary:hover {{ background: #2a2a4e; }}
|
||||
button:disabled {{ opacity: 0.5; cursor: not-allowed; }}
|
||||
pre {{ background: #f4f4f4; border: 1px solid #e0e0e0; border-radius: 3px; padding: 1em; overflow: auto; font-size: 0.85em; max-height: 400px; }}
|
||||
.status-ok {{ color: #2a7a2a; }}
|
||||
.status-err {{ color: #b00; }}
|
||||
footer {{ margin-top: 3em; padding-top: 1em; border-top: 1px solid #ddd; font-size: 0.78em; color: #888; }}
|
||||
footer a {{ color: #888; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Festinger</h1>
|
||||
<p class="subtitle">Ollama-compatible inference middleware — loop detection & Recollections world model</p>
|
||||
|
||||
<h2>World model stats</h2>
|
||||
<div class="stats" id="stats">
|
||||
<div class="stat"><div class="stat-label">SOAS tokens</div><div class="stat-value" id="s-soas">…</div></div>
|
||||
<div class="stat"><div class="stat-label">URD edges</div><div class="stat-value" id="s-urd">…</div></div>
|
||||
<div class="stat"><div class="stat-label">Pending conflicts</div><div class="stat-value" id="s-conflicts">…</div></div>
|
||||
<div class="stat"><div class="stat-label">Last resolution</div><div class="stat-value" style="font-size:0.85em" id="s-lastrun">…</div></div>
|
||||
</div>
|
||||
|
||||
<h2>Actions</h2>
|
||||
<div class="actions">
|
||||
<button class="primary" onclick="runResolution(this)">Run conflict resolution now</button>
|
||||
<button onclick="runWordnetImport(this)">Import WordNet lemmas</button>
|
||||
</div>
|
||||
<pre id="result" style="display:none"></pre>
|
||||
|
||||
<h2>Pending conflicts</h2>
|
||||
<pre id="conflicts-pre">Loading…</pre>
|
||||
|
||||
<footer>
|
||||
<strong>Vocabulary source:</strong>
|
||||
Princeton University “About WordNet.” <em>WordNet.</em> Princeton University. 2010.
|
||||
<a href="https://wordnet.princeton.edu/" target="_blank">https://wordnet.princeton.edu/</a>
|
||||
— used to pre-seed the SOAS concept vocabulary at saliency 0 (common English baseline).
|
||||
</footer>
|
||||
|
||||
<script>
|
||||
async function loadStats() {{
|
||||
const r = await fetch('/health');
|
||||
const d = await r.json();
|
||||
document.getElementById('s-soas').textContent = d.soas_tokens.toLocaleString();
|
||||
document.getElementById('s-urd').textContent = d.urd_edges.toLocaleString();
|
||||
document.getElementById('s-conflicts').textContent = d.pending_conflicts;
|
||||
document.getElementById('s-lastrun').textContent = d.last_resolution_run
|
||||
? d.last_resolution_run.replace('T', ' ').slice(0, 19)
|
||||
: 'never';
|
||||
}}
|
||||
|
||||
async function loadConflicts() {{
|
||||
const r = await fetch('/conflicts');
|
||||
const d = await r.json();
|
||||
const pending = d.conflicts.filter(c => c.status === 'pending');
|
||||
const el = document.getElementById('conflicts-pre');
|
||||
el.textContent = pending.length
|
||||
? JSON.stringify(pending, null, 2)
|
||||
: '(none)';
|
||||
}}
|
||||
|
||||
function showResult(text, ok) {{
|
||||
const el = document.getElementById('result');
|
||||
el.style.display = 'block';
|
||||
el.className = ok ? 'status-ok' : 'status-err';
|
||||
el.textContent = text;
|
||||
}}
|
||||
|
||||
async function runResolution(btn) {{
|
||||
btn.disabled = true;
|
||||
showResult('Running resolution job…', true);
|
||||
try {{
|
||||
const r = await fetch('/resolve/run', {{method: 'POST'}});
|
||||
const d = await r.json();
|
||||
showResult(JSON.stringify(d, null, 2), r.ok);
|
||||
await loadStats();
|
||||
await loadConflicts();
|
||||
}} catch(e) {{
|
||||
showResult('Error: ' + e.message, false);
|
||||
}} finally {{
|
||||
btn.disabled = false;
|
||||
}}
|
||||
}}
|
||||
|
||||
async function runWordnetImport(btn) {{
|
||||
btn.disabled = true;
|
||||
showResult('Importing WordNet lemmas — this may take 10–20 seconds…', true);
|
||||
try {{
|
||||
const r = await fetch('/wordnet/import', {{method: 'POST'}});
|
||||
const d = await r.json();
|
||||
showResult(JSON.stringify(d, null, 2), r.ok && !d.error);
|
||||
await loadStats();
|
||||
}} catch(e) {{
|
||||
showResult('Error: ' + e.message, false);
|
||||
}} finally {{
|
||||
btn.disabled = false;
|
||||
}}
|
||||
}}
|
||||
|
||||
loadStats();
|
||||
loadConflicts();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
@app.get("/admin", response_class=HTMLResponse)
|
||||
async def admin() -> str:
|
||||
return ADMIN_HTML.format()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Passthrough — everything else forwarded to upstream Ollama
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "HEAD"])
|
||||
async def passthrough(path: str, request: Request) -> Response:
|
||||
cfg = request.app.state.yaml_config
|
||||
# Route /v1/* to Anthropic; everything else (including /api/*) to Ollama
|
||||
if path.startswith("v1/"):
|
||||
upstream = cfg["upstream_anthropic"]
|
||||
relay_headers = ANTHROPIC_RELAY_HEADERS
|
||||
else:
|
||||
upstream = cfg["upstream_ollama"]
|
||||
relay_headers = None
|
||||
|
||||
body = await request.body()
|
||||
if relay_headers:
|
||||
headers = _relay_headers(request, relay_headers)
|
||||
else:
|
||||
headers = {k: v for k, v in request.headers.items() if k.lower() != "host"}
|
||||
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
r = await client.request(
|
||||
request.method,
|
||||
f"{upstream}/{path}",
|
||||
content=body,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
return Response(
|
||||
content=r.content,
|
||||
status_code=r.status_code,
|
||||
media_type=r.headers.get("content-type"),
|
||||
)
|
||||
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
Recollection engine — read path.
|
||||
|
||||
For each salient concept found in an intercepted prompt:
|
||||
- Query URD edges (from in-memory cache, filtered by confidence + recency)
|
||||
- Render hit or zero-hit block
|
||||
- Inject the <recollection> block into the prompt before forwarding to Ollama
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from . import cache
|
||||
from .cache import SoasRow, UrdEdge
|
||||
|
||||
log = logging.getLogger("festinger.recollection")
|
||||
|
||||
ZERO_HIT_TEMPLATE = (
|
||||
"? {concept}: no recollection. If not a typo, store it before proceeding:\n"
|
||||
" gutask iknowthat '{concept} -isa <parent> in context of <dimension>'\n"
|
||||
" gutask iknowthat '{concept} -ispart <system> in context of <dimension>'"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Query (reads from in-memory cache)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def query_edges(
|
||||
concept_id: int,
|
||||
confidence_floor: float,
|
||||
recency_days: int,
|
||||
) -> list[UrdEdge]:
|
||||
"""
|
||||
Return URD edges for *concept_id* that pass confidence + recency filters.
|
||||
All reads are pure in-memory — zero network.
|
||||
Note: we store last_confirmed as a datetime in the edge when warming the
|
||||
cache. For simplicity the cache stores it as a string from Postgres;
|
||||
the recency filter is intentionally lenient when last_confirmed is absent.
|
||||
"""
|
||||
edges = cache.urd_by_concept.get(concept_id, [])
|
||||
if not edges:
|
||||
return []
|
||||
|
||||
cutoff = datetime.now(tz=timezone.utc) - timedelta(days=recency_days)
|
||||
result = []
|
||||
for e in edges:
|
||||
if e.confidence < confidence_floor:
|
||||
continue
|
||||
result.append(e)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Renderers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def render_hit(concept_token: str, edges: list[UrdEdge], concept_id: int) -> str:
|
||||
"""
|
||||
Render one concept's recollection line:
|
||||
gnommoweb: [type] repo [membership] glitch_university
|
||||
Pending-conflict dimensions get a '?' suffix on the dim token.
|
||||
"""
|
||||
has_pending = concept_id in cache.pending_conflicts
|
||||
parts = []
|
||||
for e in edges:
|
||||
dim_label = e.dim_token
|
||||
if has_pending:
|
||||
dim_label += "?"
|
||||
parts.append(f"[{dim_label}] {e.parent_token}")
|
||||
return f"{concept_token}: {' '.join(parts)}"
|
||||
|
||||
|
||||
def render_zero_hit(concept_token: str) -> str:
|
||||
return ZERO_HIT_TEMPLATE.format(concept=concept_token)
|
||||
|
||||
|
||||
def build_recollection_block(
|
||||
salient_concept_ids: list[int],
|
||||
confidence_floor: float,
|
||||
recency_days: int,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Build the full <recollection> block for a list of above-threshold concept IDs.
|
||||
Returns None if there is nothing to say.
|
||||
"""
|
||||
lines: list[str] = []
|
||||
|
||||
for cid in salient_concept_ids:
|
||||
token = cache.soas_by_id.get(cid, str(cid))
|
||||
edges = query_edges(cid, confidence_floor, recency_days)
|
||||
|
||||
if edges:
|
||||
lines.append(render_hit(token, edges, cid))
|
||||
else:
|
||||
lines.append(render_zero_hit(token))
|
||||
|
||||
if not lines:
|
||||
return None
|
||||
|
||||
body = "\n".join(lines)
|
||||
return f"<recollection>\n{body}\n</recollection>"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prompt injection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def inject_recollection(messages: list[dict], block: str) -> list[dict]:
|
||||
"""
|
||||
Prepend the recollection block to the first system message.
|
||||
If no system message exists, insert one at position 0.
|
||||
Returns a new messages list (does not mutate the input).
|
||||
"""
|
||||
messages = list(messages)
|
||||
for i, msg in enumerate(messages):
|
||||
if msg.get("role") == "system":
|
||||
messages[i] = dict(msg)
|
||||
messages[i]["content"] = block + "\n\n" + (msg.get("content") or "")
|
||||
return messages
|
||||
|
||||
# No system message — insert one
|
||||
messages.insert(0, {"role": "system", "content": block})
|
||||
return messages
|
||||
@@ -0,0 +1,230 @@
|
||||
"""
|
||||
Nightly resolution job — drain the resolution queue via cloud LLM.
|
||||
|
||||
For each pending conflict:
|
||||
- ISA+ISA → decompose (two new dimensions) or dismiss
|
||||
- ISPART+ISPART → update (swap to incoming) or dismiss
|
||||
- misclassification → reclassify (insert in correct dimension)
|
||||
|
||||
Triggered by APScheduler on the cron schedule in config table,
|
||||
or manually via POST /resolve/run.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import cache
|
||||
from .db import get_or_create_soas, get_config, reload_urd_cache
|
||||
from .llm_client import (
|
||||
ModelConfig, get_model_config, call_llm,
|
||||
RESOLVE_ISA_ISA_PROMPT, RESOLVE_ISPART_ISPART_PROMPT,
|
||||
RESOLVE_MISCLASS_PROMPT, parse_resolution_decision,
|
||||
)
|
||||
from .urd_writer import InsertRequest, insert_urd_edge
|
||||
|
||||
log = logging.getLogger("festinger.resolution")
|
||||
|
||||
_last_run: datetime | None = None
|
||||
|
||||
|
||||
async def run_resolution_job(pool: asyncpg.Pool) -> dict:
|
||||
"""
|
||||
Process all pending resolution queue items.
|
||||
Returns a summary dict.
|
||||
"""
|
||||
global _last_run
|
||||
log.info("resolution job starting")
|
||||
|
||||
resolve_model_id = await get_config(pool, "resolve_model_id")
|
||||
if not resolve_model_id:
|
||||
log.warning("no resolve_model_id configured — resolution job aborted")
|
||||
return {"status": "aborted", "reason": "no resolve_model_id"}
|
||||
|
||||
model = await get_model_config(pool, resolve_model_id)
|
||||
if not model:
|
||||
log.warning("resolve_model_id=%s not found in models table", resolve_model_id)
|
||||
return {"status": "aborted", "reason": "model not found"}
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
items = await conn.fetch(
|
||||
"""
|
||||
SELECT id, concept_id, existing_parent_id, incoming_parent_id,
|
||||
dim_id, collision_type
|
||||
FROM resolution_queue
|
||||
WHERE status = 'pending'
|
||||
ORDER BY priority DESC, created_at ASC
|
||||
"""
|
||||
)
|
||||
|
||||
counts = {"decompose": 0, "update": 0, "dismiss": 0, "reclassify": 0, "error": 0}
|
||||
|
||||
for item in items:
|
||||
try:
|
||||
outcome = await _resolve_item(pool, model, item)
|
||||
counts[outcome] = counts.get(outcome, 0) + 1
|
||||
except Exception as e:
|
||||
log.exception("resolution error for queue item %d: %s", item["id"], e)
|
||||
counts["error"] += 1
|
||||
|
||||
# Reload URD cache after all resolutions
|
||||
await reload_urd_cache(pool)
|
||||
_last_run = datetime.now(tz=timezone.utc)
|
||||
log.info("resolution job complete: %s", counts)
|
||||
return {"status": "ok", "counts": counts, "processed": len(items)}
|
||||
|
||||
|
||||
async def _resolve_item(pool: asyncpg.Pool, model: ModelConfig, item) -> str:
|
||||
concept_token = cache.soas_by_id.get(item["concept_id"], str(item["concept_id"]))
|
||||
existing_parent = cache.soas_by_id.get(item["existing_parent_id"], "?")
|
||||
incoming_parent = cache.soas_by_id.get(item["incoming_parent_id"], "?")
|
||||
dim_token = cache.soas_by_id.get(item["dim_id"], "?")
|
||||
seed_dims = ["type", "membership", "runs-on", "tech", "owned-by", "geography"]
|
||||
known_dims_str = ", ".join(seed_dims)
|
||||
|
||||
collision_type = item["collision_type"]
|
||||
|
||||
if collision_type == "isa_isa":
|
||||
prompt = RESOLVE_ISA_ISA_PROMPT.format(
|
||||
concept=concept_token,
|
||||
existing_parent=existing_parent,
|
||||
incoming_parent=incoming_parent,
|
||||
dimension=dim_token,
|
||||
known_dimensions=known_dims_str,
|
||||
)
|
||||
elif collision_type == "ispart_ispart":
|
||||
prompt = RESOLVE_ISPART_ISPART_PROMPT.format(
|
||||
concept=concept_token,
|
||||
existing_parent=existing_parent,
|
||||
incoming_parent=incoming_parent,
|
||||
dimension=dim_token,
|
||||
)
|
||||
else: # misclassification
|
||||
prompt = RESOLVE_MISCLASS_PROMPT.format(
|
||||
concept=concept_token,
|
||||
existing_parent=existing_parent,
|
||||
incoming_parent=incoming_parent,
|
||||
dimension=dim_token,
|
||||
known_dimensions=known_dims_str,
|
||||
)
|
||||
|
||||
response = await call_llm(model, prompt)
|
||||
decision = parse_resolution_decision(response)
|
||||
|
||||
if not decision:
|
||||
await _mark_resolved(pool, item["id"], "error", {"raw_response": response[:500]})
|
||||
return "error"
|
||||
|
||||
outcome = decision.decision
|
||||
|
||||
if outcome == "decompose" and collision_type == "isa_isa":
|
||||
await _apply_decompose(pool, item, decision, concept_token, existing_parent, incoming_parent)
|
||||
elif outcome == "update" and collision_type == "ispart_ispart":
|
||||
await _apply_update(pool, item)
|
||||
elif outcome == "reclassify":
|
||||
await _apply_reclassify(pool, item, decision)
|
||||
else:
|
||||
outcome = "dismiss"
|
||||
|
||||
await _mark_resolved(
|
||||
pool, item["id"], "resolved",
|
||||
{"decision": outcome, "reasoning": decision.reasoning},
|
||||
)
|
||||
return outcome
|
||||
|
||||
|
||||
async def _apply_decompose(pool, item, decision, concept_token, existing_parent, incoming_parent):
|
||||
"""Create two new dimensions, migrate existing fact, insert incoming fact."""
|
||||
existing_dim_token = decision.existing_dimension or "type-a"
|
||||
new_dim_token = decision.new_dimension or "type-b"
|
||||
|
||||
existing_dim_row = await get_or_create_soas(pool, existing_dim_token)
|
||||
new_dim_row = await get_or_create_soas(pool, new_dim_token)
|
||||
|
||||
# Create root nodes for new dimensions
|
||||
async with pool.acquire() as conn:
|
||||
for dim_row in [existing_dim_row, new_dim_row]:
|
||||
await conn.execute(
|
||||
"INSERT INTO urd (id, parent_id, dim_id, is_isa, confidence, source) "
|
||||
"VALUES ($1, $1, $1, true, 1.0, 'festinger') ON CONFLICT DO NOTHING",
|
||||
dim_row.id,
|
||||
)
|
||||
|
||||
# Migrate existing edge to new existing_dimension
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"DELETE FROM urd WHERE id=$1 AND dim_id=$2",
|
||||
item["concept_id"], item["dim_id"],
|
||||
)
|
||||
|
||||
existing_parent_row = await get_or_create_soas(pool, cache.soas_by_id.get(item["existing_parent_id"], existing_parent))
|
||||
incoming_parent_row = await get_or_create_soas(pool, cache.soas_by_id.get(item["incoming_parent_id"], incoming_parent))
|
||||
|
||||
for parent_row, dim_row in [
|
||||
(existing_parent_row, existing_dim_row),
|
||||
(incoming_parent_row, new_dim_row),
|
||||
]:
|
||||
req = InsertRequest(
|
||||
concept_id=item["concept_id"],
|
||||
parent_id=parent_row.id,
|
||||
dim_id=dim_row.id,
|
||||
is_isa=True,
|
||||
confidence=0.9,
|
||||
source="festinger",
|
||||
)
|
||||
await insert_urd_edge(pool, req)
|
||||
|
||||
log.info("decompose applied: %s → [%s, %s]", item["concept_id"], existing_dim_token, new_dim_token)
|
||||
|
||||
|
||||
async def _apply_update(pool, item):
|
||||
"""Replace old ISPART edge with incoming fact."""
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"DELETE FROM urd WHERE id=$1 AND dim_id=$2",
|
||||
item["concept_id"], item["dim_id"],
|
||||
)
|
||||
req = InsertRequest(
|
||||
concept_id=item["concept_id"],
|
||||
parent_id=item["incoming_parent_id"],
|
||||
dim_id=item["dim_id"],
|
||||
is_isa=False,
|
||||
confidence=0.85,
|
||||
source="festinger",
|
||||
)
|
||||
await insert_urd_edge(pool, req)
|
||||
log.info("update applied: concept=%d dim=%d → parent=%d",
|
||||
item["concept_id"], item["dim_id"], item["incoming_parent_id"])
|
||||
|
||||
|
||||
async def _apply_reclassify(pool, item, decision):
|
||||
"""Insert incoming fact in the corrected dimension."""
|
||||
correct_dim_token = decision.correct_dimension or "membership"
|
||||
dim_row = await get_or_create_soas(pool, correct_dim_token)
|
||||
req = InsertRequest(
|
||||
concept_id=item["concept_id"],
|
||||
parent_id=item["incoming_parent_id"],
|
||||
dim_id=dim_row.id,
|
||||
is_isa=False,
|
||||
confidence=0.8,
|
||||
source="festinger",
|
||||
)
|
||||
await insert_urd_edge(pool, req)
|
||||
log.info("reclassify applied: concept=%d → dim=%s", item["concept_id"], correct_dim_token)
|
||||
|
||||
|
||||
async def _mark_resolved(pool, queue_id: int, status: str, resolution: dict) -> None:
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"UPDATE resolution_queue SET status=$1, resolution=$2, resolved_at=now() WHERE id=$3",
|
||||
status, json.dumps(resolution), queue_id,
|
||||
)
|
||||
|
||||
|
||||
def last_run_timestamp() -> str | None:
|
||||
if _last_run is None:
|
||||
return None
|
||||
return _last_run.isoformat()
|
||||
@@ -0,0 +1,304 @@
|
||||
"""
|
||||
Test scenario seeder for Festinger.
|
||||
|
||||
Seeds the database and in-memory cache with pre-defined world-model states,
|
||||
then exposes a trigger endpoint so gutasktool (or a test script) can fire the
|
||||
conflicting fact and observe the collision + resolution pipeline.
|
||||
|
||||
Scenarios
|
||||
---------
|
||||
|
||||
A misclassification
|
||||
Seed: michigan ISPART usa in dim:usa (parent_id = dim_id — coarse/degenerate edge)
|
||||
Trigger: gutask iknowthat 'michigan -isa state in context of usa'
|
||||
Expect: misclassification collision → resolution moves ISA fact to dim:type
|
||||
|
||||
B ISA + ISA decomposition
|
||||
Seed: gnommoweb ISA container in dim:type
|
||||
Trigger: gutask iknowthat 'gnommoweb -isa repo in context of type'
|
||||
Expect: isa_isa collision → nightly job decomposes into artifact-type + deployment-type
|
||||
|
||||
C ISPART + ISPART contradiction
|
||||
Seed: dobby ISPART docker_host_1 in dim:runs-on
|
||||
Trigger: gutask iknowthat 'dobby -ispart docker_host_2 in context of runs-on'
|
||||
Expect: ispart_ispart collision → nightly job arbitrates (update or dismiss)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import cache
|
||||
from .cache import SoasRow, UrdEdge
|
||||
from .db import get_or_create_soas
|
||||
|
||||
log = logging.getLogger("festinger.test")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario definitions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class SeedEdge:
|
||||
concept: str
|
||||
parent: str
|
||||
dimension: str
|
||||
is_isa: bool
|
||||
confidence: float = 0.9
|
||||
source: str = "test"
|
||||
note: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Scenario:
|
||||
id: str
|
||||
name: str
|
||||
description: str
|
||||
seed_edges: list[SeedEdge]
|
||||
trigger_fact: str # the fact to POST to /iknowthat to cause the collision
|
||||
expected_collision: str # 'isa_isa' | 'ispart_ispart' | 'misclassification'
|
||||
expected_resolution: str # 'decompose' | 'update' | 'dismiss' | 'reclassify'
|
||||
notes: str = ""
|
||||
|
||||
|
||||
SCENARIOS: dict[str, Scenario] = {
|
||||
|
||||
"A": Scenario(
|
||||
id="A",
|
||||
name="misclassification — michigan/state/usa",
|
||||
description=(
|
||||
"The world model holds a coarse, degenerate fact: michigan ISPART usa "
|
||||
"in the 'usa' dimension (parent_id = dim_id). This is an early, "
|
||||
"undifferentiated encoding — the location context IS the dimension. "
|
||||
"A new fact 'michigan is a state in context of usa' is ISA in the same "
|
||||
"dimension, triggering a misclassification collision. Resolution should "
|
||||
"suggest moving the ISA fact to the 'type' dimension."
|
||||
),
|
||||
seed_edges=[
|
||||
SeedEdge(
|
||||
concept="michigan",
|
||||
parent="usa",
|
||||
dimension="usa", # parent_id = dim_id — the degenerate pattern
|
||||
is_isa=False, # ISPART
|
||||
confidence=0.85,
|
||||
note="coarse pre-knowledge: michigan is part of usa, dim=usa (undifferentiated)",
|
||||
),
|
||||
],
|
||||
trigger_fact="michigan -isa state in context of usa",
|
||||
expected_collision="misclassification",
|
||||
expected_resolution="reclassify",
|
||||
notes=(
|
||||
"The degenerate parent_id=dim_id pattern arises naturally when the world model "
|
||||
"first hears 'michigan is in usa' before any geography dimension exists. "
|
||||
"Resolution should split: ISPART goes to 'geography', ISA goes to 'type'."
|
||||
),
|
||||
),
|
||||
|
||||
"B": Scenario(
|
||||
id="B",
|
||||
name="ISA+ISA decomposition — gnommoweb/container/repo",
|
||||
description=(
|
||||
"The world model holds gnommoweb ISA container in dim:type. "
|
||||
"Both 'container' (deployment artifact) and 'repo' (software artifact) are "
|
||||
"simultaneously true, but the 'type' dimension is too coarse to hold both. "
|
||||
"Resolution should decompose into 'deployment-type' and 'artifact-type'."
|
||||
),
|
||||
seed_edges=[
|
||||
SeedEdge(
|
||||
concept="gnommoweb",
|
||||
parent="container",
|
||||
dimension="type",
|
||||
is_isa=True,
|
||||
confidence=0.9,
|
||||
note="gnommoweb runs as a Docker container",
|
||||
),
|
||||
],
|
||||
trigger_fact="gnommoweb -isa repo in context of type",
|
||||
expected_collision="isa_isa",
|
||||
expected_resolution="decompose",
|
||||
notes="Classic dimension-too-coarse case from PROJECT.md test case B.",
|
||||
),
|
||||
|
||||
"C": Scenario(
|
||||
id="C",
|
||||
name="ISPART+ISPART contradiction — dobby/host migration",
|
||||
description=(
|
||||
"The world model holds dobby ISPART docker_host_1 in dim:runs-on. "
|
||||
"A new fact places dobby on docker_host_2, contradicting the existing "
|
||||
"runs-on edge. One host is correct; the nightly job must arbitrate."
|
||||
),
|
||||
seed_edges=[
|
||||
SeedEdge(
|
||||
concept="dobby",
|
||||
parent="docker_host_1",
|
||||
dimension="runs-on",
|
||||
is_isa=False,
|
||||
confidence=0.9,
|
||||
note="dobby agent originally deployed on docker_host_1",
|
||||
),
|
||||
],
|
||||
trigger_fact="dobby -ispart docker_host_2 in context of runs-on",
|
||||
expected_collision="ispart_ispart",
|
||||
expected_resolution="update",
|
||||
notes="Simulates a container migration where the new host fact should win.",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Seeder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def seed_scenario(pool: asyncpg.Pool, scenario_id: str) -> dict[str, Any]:
|
||||
"""
|
||||
Insert SOAS + URD entries for a named scenario.
|
||||
Returns a summary of what was created, including the token→id mapping
|
||||
and the trigger fact to POST to /iknowthat.
|
||||
"""
|
||||
sc = SCENARIOS.get(scenario_id.upper())
|
||||
if sc is None:
|
||||
return {"error": f"unknown scenario: {scenario_id!r}. Valid: {list(SCENARIOS)}"}
|
||||
|
||||
created_soas: list[dict] = []
|
||||
created_edges: list[dict] = []
|
||||
collisions: list[dict] = []
|
||||
|
||||
for edge in sc.seed_edges:
|
||||
concept_row = await get_or_create_soas(pool, edge.concept)
|
||||
parent_row = await get_or_create_soas(pool, edge.parent)
|
||||
dim_row = await get_or_create_soas(pool, edge.dimension)
|
||||
|
||||
for row in [concept_row, parent_row, dim_row]:
|
||||
created_soas.append({"token": row.token, "id": row.id})
|
||||
|
||||
# Manually set encounter_count + saliency above read threshold so
|
||||
# the concept appears in recollections during the test prompt.
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"UPDATE soas SET encounter_count=50, saliency=1.5 WHERE id=$1",
|
||||
concept_row.id,
|
||||
)
|
||||
if concept_row.token in cache.soas_by_token:
|
||||
cache.soas_by_token[concept_row.token].encounter_count = 50
|
||||
cache.soas_by_token[concept_row.token].saliency = 1.5
|
||||
|
||||
# Insert URD edge — direct Postgres write, bypassing collision detection,
|
||||
# because this is deliberate seed state (not a user-facing write path).
|
||||
key = (concept_row.id, dim_row.id)
|
||||
existing = cache.urd_by_concept_dim.get(key)
|
||||
|
||||
if existing is not None:
|
||||
collisions.append({
|
||||
"concept": edge.concept,
|
||||
"dimension": edge.dimension,
|
||||
"message": "edge already present in cache — skipped",
|
||||
})
|
||||
continue
|
||||
|
||||
try:
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO urd (id, parent_id, dim_id, is_isa, confidence, source)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT DO NOTHING
|
||||
""",
|
||||
concept_row.id, parent_row.id, dim_row.id,
|
||||
edge.is_isa, edge.confidence, edge.source,
|
||||
)
|
||||
except Exception as e:
|
||||
collisions.append({"concept": edge.concept, "error": str(e)})
|
||||
continue
|
||||
|
||||
# Update in-memory cache
|
||||
urd_edge = UrdEdge(
|
||||
concept_id=concept_row.id,
|
||||
parent_id=parent_row.id,
|
||||
dim_id=dim_row.id,
|
||||
is_isa=edge.is_isa,
|
||||
confidence=edge.confidence,
|
||||
source=edge.source,
|
||||
parent_token=parent_row.token,
|
||||
dim_token=dim_row.token,
|
||||
)
|
||||
cache.urd_by_concept.setdefault(concept_row.id, []).append(urd_edge)
|
||||
cache.urd_by_concept_dim[key] = urd_edge
|
||||
|
||||
created_edges.append({
|
||||
"concept": edge.concept,
|
||||
"concept_id": concept_row.id,
|
||||
"parent": edge.parent,
|
||||
"parent_id": parent_row.id,
|
||||
"dimension": edge.dimension,
|
||||
"dim_id": dim_row.id,
|
||||
"is_isa": edge.is_isa,
|
||||
"parent_equals_dim": parent_row.id == dim_row.id,
|
||||
"note": edge.note,
|
||||
})
|
||||
|
||||
return {
|
||||
"scenario": sc.id,
|
||||
"name": sc.name,
|
||||
"description": sc.description,
|
||||
"soas_entries": created_soas,
|
||||
"urd_edges": created_edges,
|
||||
"skipped_collisions": collisions,
|
||||
"next_step": {
|
||||
"action": "POST /iknowthat",
|
||||
"body": {"fact": sc.trigger_fact},
|
||||
"gutask_command": f"gutask iknowthat '{sc.trigger_fact}'",
|
||||
"expected_collision": sc.expected_collision,
|
||||
"expected_resolution": sc.expected_resolution,
|
||||
},
|
||||
"notes": sc.notes,
|
||||
}
|
||||
|
||||
|
||||
async def reset_scenario(pool: asyncpg.Pool, scenario_id: str) -> dict[str, Any]:
|
||||
"""
|
||||
Remove the URD edges and SOAS tokens introduced by a scenario seed.
|
||||
Clears the corresponding cache entries and resolution queue rows.
|
||||
Use before re-running a scenario to get a clean slate.
|
||||
"""
|
||||
sc = SCENARIOS.get(scenario_id.upper())
|
||||
if sc is None:
|
||||
return {"error": f"unknown scenario: {scenario_id!r}"}
|
||||
|
||||
removed_urd = 0
|
||||
removed_rq = 0
|
||||
|
||||
concept_tokens = {e.concept for e in sc.seed_edges}
|
||||
concept_ids = [
|
||||
cache.soas_by_token[t].id
|
||||
for t in concept_tokens
|
||||
if t in cache.soas_by_token
|
||||
]
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
for cid in concept_ids:
|
||||
# Remove resolution queue entries
|
||||
r = await conn.execute(
|
||||
"DELETE FROM resolution_queue WHERE concept_id=$1", cid
|
||||
)
|
||||
removed_rq += int(r.split()[-1])
|
||||
|
||||
# Remove URD edges where this concept is the subject
|
||||
r = await conn.execute("DELETE FROM urd WHERE id=$1", cid)
|
||||
removed_urd += int(r.split()[-1])
|
||||
|
||||
# Clear from in-memory cache
|
||||
cache.urd_by_concept.pop(cid, None)
|
||||
for key in [k for k in cache.urd_by_concept_dim if k[0] == cid]:
|
||||
del cache.urd_by_concept_dim[key]
|
||||
cache.pending_conflicts.discard(cid)
|
||||
|
||||
return {
|
||||
"scenario": scenario_id.upper(),
|
||||
"removed_urd_edges": removed_urd,
|
||||
"removed_resolution_queue_rows": removed_rq,
|
||||
"status": "reset complete",
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
"""
|
||||
Tokeniser — compound token rule + punctuation stripping.
|
||||
|
||||
Rules:
|
||||
1. Split on whitespace.
|
||||
2. Consecutive tokens each starting with a capital letter are merged into a
|
||||
single underscore-joined lowercase token (compound token rule).
|
||||
A lowercase or non-alpha token breaks the run.
|
||||
3. Strip leading/trailing punctuation from each token.
|
||||
4. Lowercase all tokens.
|
||||
5. Discard tokens shorter than 5 characters (unless they surfaced as part of
|
||||
a matched relationship cue — cue scanner handles that separately).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
# Characters that are stripped from token edges
|
||||
_PUNCT_RE = re.compile(r"^[^\w]+|[^\w]+$")
|
||||
|
||||
|
||||
def _strip_punct(s: str) -> str:
|
||||
return _PUNCT_RE.sub("", s)
|
||||
|
||||
|
||||
def tokenize(text: str) -> list[str]:
|
||||
"""
|
||||
Return a deduplicated, ordered list of canonical tokens extracted from
|
||||
*text*, applying the compound token rule.
|
||||
"""
|
||||
raw_words = text.split()
|
||||
tokens: list[str] = []
|
||||
run: list[str] = []
|
||||
|
||||
def flush_run() -> None:
|
||||
if not run:
|
||||
return
|
||||
if len(run) > 1:
|
||||
tokens.append("_".join(w.lower() for w in run))
|
||||
else:
|
||||
tokens.append(run[0].lower())
|
||||
run.clear()
|
||||
|
||||
for word in raw_words:
|
||||
stripped = _strip_punct(word)
|
||||
# Trailing punctuation (sentence boundary) ends a compound run.
|
||||
# Commas/colons/periods after a word mean the next word starts fresh.
|
||||
has_trailing_punct = bool(word) and not word[-1].isalnum() and word[-1] != "_"
|
||||
|
||||
if not stripped:
|
||||
flush_run()
|
||||
continue
|
||||
|
||||
# A word starts a capital run if its first alphabetic character is uppercase
|
||||
first_alpha = next((c for c in stripped if c.isalpha()), None)
|
||||
if first_alpha and first_alpha.isupper():
|
||||
run.append(stripped)
|
||||
if has_trailing_punct:
|
||||
flush_run() # e.g. "FastAPI." ends the run immediately
|
||||
else:
|
||||
flush_run()
|
||||
t = stripped.lower()
|
||||
if t:
|
||||
tokens.append(t)
|
||||
|
||||
flush_run()
|
||||
|
||||
# Filter: keep tokens >= 5 chars, deduplicate preserving order
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for t in tokens:
|
||||
if len(t) >= 5 and t not in seen:
|
||||
seen.add(t)
|
||||
result.append(t)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def tokenize_all(text: str) -> list[str]:
|
||||
"""
|
||||
Like tokenize() but returns ALL tokens (including short ones).
|
||||
Used by the cue scanner which needs short words for pattern matching.
|
||||
"""
|
||||
raw_words = text.split()
|
||||
tokens: list[str] = []
|
||||
run: list[str] = []
|
||||
|
||||
def flush_run() -> None:
|
||||
if not run:
|
||||
return
|
||||
if len(run) > 1:
|
||||
tokens.append("_".join(w.lower() for w in run))
|
||||
else:
|
||||
tokens.append(run[0].lower())
|
||||
run.clear()
|
||||
|
||||
for word in raw_words:
|
||||
stripped = _strip_punct(word)
|
||||
has_trailing_punct = bool(word) and not word[-1].isalnum() and word[-1] != "_"
|
||||
|
||||
if not stripped:
|
||||
flush_run()
|
||||
continue
|
||||
first_alpha = next((c for c in stripped if c.isalpha()), None)
|
||||
if first_alpha and first_alpha.isupper():
|
||||
run.append(stripped)
|
||||
if has_trailing_punct:
|
||||
flush_run()
|
||||
else:
|
||||
flush_run()
|
||||
t = stripped.lower()
|
||||
if t:
|
||||
tokens.append(t)
|
||||
|
||||
flush_run()
|
||||
return tokens
|
||||
@@ -0,0 +1,157 @@
|
||||
"""
|
||||
URD insert pipeline — collision detection and write-through.
|
||||
|
||||
Insert flow:
|
||||
key = (concept_id, dim_id)
|
||||
if key in urd_by_concept_dim → collision detected in-memory
|
||||
→ classify (is_isa flags), route to resolution queue
|
||||
else
|
||||
→ INSERT into Postgres URD
|
||||
→ on success: update urd_by_concept + urd_by_concept_dim
|
||||
→ on UniqueViolation (race): reload row, route to resolution queue
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import cache
|
||||
from .cache import UrdEdge
|
||||
|
||||
log = logging.getLogger("festinger.urd_writer")
|
||||
|
||||
|
||||
@dataclass
|
||||
class InsertRequest:
|
||||
concept_id: int
|
||||
parent_id: int
|
||||
dim_id: int
|
||||
is_isa: bool
|
||||
confidence: float
|
||||
source: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class CollisionInfo:
|
||||
concept_id: int
|
||||
existing_parent_id: int
|
||||
incoming_parent_id: int
|
||||
dim_id: int
|
||||
existing_is_isa: bool
|
||||
incoming_is_isa: bool
|
||||
|
||||
@property
|
||||
def collision_type(self) -> str:
|
||||
if self.existing_is_isa and self.incoming_is_isa:
|
||||
return "isa_isa"
|
||||
if not self.existing_is_isa and not self.incoming_is_isa:
|
||||
return "ispart_ispart"
|
||||
return "misclassification"
|
||||
|
||||
|
||||
async def insert_urd_edge(
|
||||
pool: asyncpg.Pool,
|
||||
req: InsertRequest,
|
||||
priority: bool = False,
|
||||
) -> Optional[CollisionInfo]:
|
||||
"""
|
||||
Attempt to insert a URD edge.
|
||||
Returns CollisionInfo if there was a collision, None on success.
|
||||
"""
|
||||
key = (req.concept_id, req.dim_id)
|
||||
|
||||
# Fast-path collision detection — in-memory
|
||||
existing = cache.urd_by_concept_dim.get(key)
|
||||
if existing is not None:
|
||||
collision = CollisionInfo(
|
||||
concept_id=req.concept_id,
|
||||
existing_parent_id=existing.parent_id,
|
||||
incoming_parent_id=req.parent_id,
|
||||
dim_id=req.dim_id,
|
||||
existing_is_isa=existing.is_isa,
|
||||
incoming_is_isa=req.is_isa,
|
||||
)
|
||||
await _queue_collision(pool, collision, priority)
|
||||
return collision
|
||||
|
||||
# Attempt Postgres insert
|
||||
try:
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO urd (id, parent_id, dim_id, is_isa, confidence, source)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
""",
|
||||
req.concept_id, req.parent_id, req.dim_id,
|
||||
req.is_isa, req.confidence, req.source,
|
||||
)
|
||||
except asyncpg.UniqueViolationError:
|
||||
# Race condition — another process inserted concurrently
|
||||
async with pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"SELECT parent_id, is_isa FROM urd WHERE id=$1 AND dim_id=$2",
|
||||
req.concept_id, req.dim_id,
|
||||
)
|
||||
if row:
|
||||
collision = CollisionInfo(
|
||||
concept_id=req.concept_id,
|
||||
existing_parent_id=row["parent_id"],
|
||||
incoming_parent_id=req.parent_id,
|
||||
dim_id=req.dim_id,
|
||||
existing_is_isa=row["is_isa"],
|
||||
incoming_is_isa=req.is_isa,
|
||||
)
|
||||
await _queue_collision(pool, collision, priority)
|
||||
return collision
|
||||
return None
|
||||
|
||||
# Success — update in-memory cache
|
||||
parent_token = cache.soas_by_id.get(req.parent_id, str(req.parent_id))
|
||||
dim_token = cache.soas_by_id.get(req.dim_id, str(req.dim_id))
|
||||
edge = UrdEdge(
|
||||
concept_id=req.concept_id,
|
||||
parent_id=req.parent_id,
|
||||
dim_id=req.dim_id,
|
||||
is_isa=req.is_isa,
|
||||
confidence=req.confidence,
|
||||
source=req.source,
|
||||
parent_token=parent_token,
|
||||
dim_token=dim_token,
|
||||
)
|
||||
cache.urd_by_concept.setdefault(req.concept_id, []).append(edge)
|
||||
cache.urd_by_concept_dim[key] = edge
|
||||
log.info(
|
||||
"urd insert concept=%d parent=%d dim=%d is_isa=%s source=%s",
|
||||
req.concept_id, req.parent_id, req.dim_id, req.is_isa, req.source,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def _queue_collision(
|
||||
pool: asyncpg.Pool,
|
||||
col: CollisionInfo,
|
||||
priority: bool,
|
||||
) -> None:
|
||||
cache.pending_conflicts.add(col.concept_id)
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO resolution_queue
|
||||
(concept_id, existing_parent_id, incoming_parent_id, dim_id,
|
||||
collision_type, priority)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
""",
|
||||
col.concept_id,
|
||||
col.existing_parent_id,
|
||||
col.incoming_parent_id,
|
||||
col.dim_id,
|
||||
col.collision_type,
|
||||
priority,
|
||||
)
|
||||
log.warning(
|
||||
"collision queued concept=%d type=%s priority=%s",
|
||||
col.concept_id, col.collision_type, priority,
|
||||
)
|
||||
@@ -0,0 +1,125 @@
|
||||
"""
|
||||
WordNet importer — loads Princeton WordNet 3.x index files into SOAS.
|
||||
|
||||
Reads index.noun, index.verb, index.adj, index.adv from the wordnet/ directory.
|
||||
Each non-header line's first field is the lemma (already lowercase, underscores
|
||||
for compound words — matches our compound token convention exactly).
|
||||
|
||||
All tokens are inserted with saliency=0, novelty=0 (common English baseline).
|
||||
Insert is idempotent: ON CONFLICT DO NOTHING.
|
||||
|
||||
Citation:
|
||||
Princeton University "About WordNet." WordNet. Princeton University. 2010.
|
||||
https://wordnet.princeton.edu/
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import cache
|
||||
from .cache import SoasRow
|
||||
|
||||
log = logging.getLogger("festinger.wordnet")
|
||||
|
||||
WORDNET_DIR = Path(__file__).parent.parent / "wordnet"
|
||||
INDEX_FILES = ["index.noun", "index.verb", "index.adj", "index.adv"]
|
||||
BATCH_SIZE = 2000
|
||||
|
||||
CITATION = (
|
||||
'Princeton University "About WordNet." WordNet. '
|
||||
"Princeton University. 2010. https://wordnet.princeton.edu/"
|
||||
)
|
||||
|
||||
|
||||
def _parse_index_file(path: Path) -> list[str]:
|
||||
"""
|
||||
Extract lemma tokens from a WordNet index file.
|
||||
Header lines start with a space or are blank — skip them.
|
||||
Data line format: lemma pos synset_cnt p_cnt ...
|
||||
Lemmas are already lowercase; underscores join compound words.
|
||||
"""
|
||||
tokens: list[str] = []
|
||||
try:
|
||||
with open(path, encoding="utf-8", errors="replace") as f:
|
||||
for line in f:
|
||||
if not line or line[0] in (" ", "\t", "\n"):
|
||||
continue
|
||||
lemma = line.split()[0]
|
||||
# Skip purely numeric tokens and single chars
|
||||
if lemma and not lemma.isdigit() and len(lemma) > 1:
|
||||
tokens.append(lemma)
|
||||
except FileNotFoundError:
|
||||
log.warning("wordnet file not found: %s", path)
|
||||
return tokens
|
||||
|
||||
|
||||
def collect_all_lemmas() -> list[str]:
|
||||
"""Parse all four index files and return a deduplicated list of lemmas."""
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for fname in INDEX_FILES:
|
||||
for token in _parse_index_file(WORDNET_DIR / fname):
|
||||
if token not in seen:
|
||||
seen.add(token)
|
||||
result.append(token)
|
||||
return result
|
||||
|
||||
|
||||
async def import_wordnet(pool: asyncpg.Pool) -> dict:
|
||||
"""
|
||||
Bulk-load all WordNet lemmas into SOAS (saliency=0, novelty=0).
|
||||
Updates the in-memory cache with any newly inserted tokens.
|
||||
Returns a summary dict.
|
||||
"""
|
||||
if not WORDNET_DIR.exists():
|
||||
return {"error": f"wordnet directory not found at {WORDNET_DIR}"}
|
||||
|
||||
lemmas = collect_all_lemmas()
|
||||
total = len(lemmas)
|
||||
log.info("wordnet: %d lemmas collected, beginning import …", total)
|
||||
|
||||
inserted = 0
|
||||
skipped = 0
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
# Process in batches to avoid huge transactions
|
||||
for i in range(0, total, BATCH_SIZE):
|
||||
batch = lemmas[i : i + BATCH_SIZE]
|
||||
|
||||
# INSERT … ON CONFLICT DO NOTHING, then RETURNING to know what was new
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
INSERT INTO soas (token, saliency, novelty)
|
||||
SELECT unnest($1::text[]), 0.0, 0.0
|
||||
ON CONFLICT (token) DO NOTHING
|
||||
RETURNING id, token
|
||||
""",
|
||||
batch,
|
||||
)
|
||||
|
||||
for r in rows:
|
||||
soas_row = SoasRow(id=r["id"], token=r["token"])
|
||||
cache.soas_by_token[r["token"]] = soas_row
|
||||
cache.soas_by_id[r["id"]] = r["token"]
|
||||
inserted += 1
|
||||
|
||||
skipped += len(batch) - len(rows)
|
||||
|
||||
if (i // BATCH_SIZE) % 10 == 0:
|
||||
log.info("wordnet import: %d / %d …", i + len(batch), total)
|
||||
|
||||
log.info(
|
||||
"wordnet import complete: %d inserted, %d already present, %d total",
|
||||
inserted, skipped, total,
|
||||
)
|
||||
return {
|
||||
"status": "ok",
|
||||
"total_lemmas": total,
|
||||
"inserted": inserted,
|
||||
"already_present": skipped,
|
||||
"citation": CITATION,
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
"""
|
||||
Async write queue — background processing of saliency-triggered and
|
||||
cue-triggered write requests.
|
||||
|
||||
Cloud LLM calls and URD inserts never block the proxy response path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
import asyncpg
|
||||
|
||||
from .cache import SoasRow
|
||||
from .cue_scanner import CueTriple
|
||||
from .db import get_or_create_soas, get_config
|
||||
from .llm_client import (
|
||||
ModelConfig, get_model_config, call_llm,
|
||||
WRITE_PROMPT_TEMPLATE, parse_llm_triples,
|
||||
)
|
||||
from .urd_writer import InsertRequest, insert_urd_edge
|
||||
from . import cache
|
||||
|
||||
log = logging.getLogger("festinger.write_queue")
|
||||
|
||||
|
||||
@dataclass
|
||||
class WriteRequest:
|
||||
"""A concept that crossed the write threshold — needs LLM-assisted classification."""
|
||||
concept_token: str
|
||||
trigger: str = "saliency" # 'saliency' | 'cue'
|
||||
|
||||
|
||||
@dataclass
|
||||
class CueWriteRequest:
|
||||
"""A directly extracted CueTriple — bypasses LLM, goes straight to URD insert."""
|
||||
triple: CueTriple
|
||||
|
||||
|
||||
_queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
|
||||
_running: bool = False
|
||||
|
||||
|
||||
async def enqueue_concept(token: str) -> None:
|
||||
try:
|
||||
_queue.put_nowait(WriteRequest(concept_token=token))
|
||||
except asyncio.QueueFull:
|
||||
log.warning("write queue full — dropping concept: %s", token)
|
||||
|
||||
|
||||
async def enqueue_cue(triple: CueTriple) -> None:
|
||||
try:
|
||||
_queue.put_nowait(CueWriteRequest(triple=triple))
|
||||
except asyncio.QueueFull:
|
||||
log.warning("write queue full — dropping cue: %s", triple)
|
||||
|
||||
|
||||
async def start_worker(pool: asyncpg.Pool) -> None:
|
||||
"""Launch background worker. Call once at startup."""
|
||||
global _running
|
||||
_running = True
|
||||
asyncio.create_task(_worker(pool))
|
||||
log.info("write queue worker started")
|
||||
|
||||
|
||||
async def _worker(pool: asyncpg.Pool) -> None:
|
||||
while _running:
|
||||
try:
|
||||
item = await asyncio.wait_for(_queue.get(), timeout=5.0)
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
|
||||
try:
|
||||
if isinstance(item, CueWriteRequest):
|
||||
await _process_cue(pool, item.triple)
|
||||
elif isinstance(item, WriteRequest):
|
||||
await _process_concept(pool, item.concept_token)
|
||||
except Exception as e:
|
||||
log.exception("write queue worker error: %s", e)
|
||||
finally:
|
||||
_queue.task_done()
|
||||
|
||||
|
||||
async def stop_worker() -> None:
|
||||
global _running
|
||||
_running = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Processing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _process_cue(pool: asyncpg.Pool, triple: CueTriple) -> None:
|
||||
"""Insert a cue-extracted triple directly into URD."""
|
||||
subj_row = await get_or_create_soas(pool, triple.subject)
|
||||
parent_row = await get_or_create_soas(pool, triple.parent)
|
||||
dim_row = await get_or_create_soas(pool, triple.dimension)
|
||||
|
||||
req = InsertRequest(
|
||||
concept_id=subj_row.id,
|
||||
parent_id=parent_row.id,
|
||||
dim_id=dim_row.id,
|
||||
is_isa=triple.is_isa,
|
||||
confidence=triple.confidence,
|
||||
source="inferred",
|
||||
)
|
||||
collision = await insert_urd_edge(pool, req)
|
||||
if collision:
|
||||
log.info("cue triple collision: %s", collision)
|
||||
|
||||
|
||||
async def _process_concept(pool: asyncpg.Pool, concept_token: str) -> None:
|
||||
"""Call cloud LLM to classify the concept, then insert all returned triples."""
|
||||
write_model_id = await get_config(pool, "write_model_id")
|
||||
if not write_model_id:
|
||||
log.debug("no write_model_id configured — skipping LLM write for %s", concept_token)
|
||||
return
|
||||
|
||||
model = await get_model_config(pool, write_model_id)
|
||||
if not model:
|
||||
log.warning("write_model_id=%s not found in models table", write_model_id)
|
||||
return
|
||||
|
||||
known_dims = list(cache.soas_by_token.keys())
|
||||
# Keep only seed dimensions + short list for prompt brevity
|
||||
seed_dims = ["type", "membership", "runs-on", "tech", "owned-by", "geography"]
|
||||
dimensions_str = ", ".join(seed_dims)
|
||||
|
||||
prompt = WRITE_PROMPT_TEMPLATE.format(
|
||||
concept=concept_token,
|
||||
dimensions=dimensions_str,
|
||||
)
|
||||
|
||||
try:
|
||||
response = await call_llm(model, prompt)
|
||||
except Exception as e:
|
||||
log.warning("LLM call failed for concept %s: %s", concept_token, e)
|
||||
return
|
||||
|
||||
triples = parse_llm_triples(response)
|
||||
if not triples:
|
||||
log.info("LLM returned no triples for concept: %s", concept_token)
|
||||
return
|
||||
|
||||
subj_row = await get_or_create_soas(pool, concept_token)
|
||||
|
||||
for t in triples:
|
||||
if not t.parent or not t.dimension:
|
||||
continue
|
||||
parent_row = await get_or_create_soas(pool, t.parent)
|
||||
dim_row = await get_or_create_soas(pool, t.dimension)
|
||||
|
||||
req = InsertRequest(
|
||||
concept_id=subj_row.id,
|
||||
parent_id=parent_row.id,
|
||||
dim_id=dim_row.id,
|
||||
is_isa=t.is_isa,
|
||||
confidence=t.confidence,
|
||||
source="cloud_llm",
|
||||
)
|
||||
await insert_urd_edge(pool, req)
|
||||
|
||||
# Mark concept as confirmed — set novelty=1.0
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"UPDATE soas SET novelty = 1.0 WHERE id = $1", subj_row.id
|
||||
)
|
||||
if concept_token in cache.soas_by_token:
|
||||
cache.soas_by_token[concept_token].novelty = 1.0
|
||||
Reference in New Issue
Block a user