Adding migrations and lot of work. Starting work on the demo scope

2026-08-22 14:53:23 +02:00
parent 1893bf23af
commit 94ccbfd1b9
29 changed files with 1280 additions and 122 deletions
@@ -84,11 +84,12 @@ suite('normalized level persistence API', () => {
    state.viewport = { x: 91, y: -42, zoom: 0.85 }

    const document: DocumentExhibit = { id: randomUUID(), type: 'document', title: 'Evidence', publishedAt: '2021-04-17T12:00:00Z', body: ['Extracted body'], regions: [{ id: 'stamp', label: 'Date stamp', excerpt: '17 April 2021', date: '2021-04-17T09:00:00Z' }], fileType: 'image', metadata: {}, ...placed(1051, 417, 174, 145, 2) }
+    const gatedDocument: DocumentExhibit = { id: randomUUID(), type: 'document', title: 'Later tip', body: [], regions: [], fileType: 'image', metadata: {}, requiredFlags: ['tip.received'], ...placed(1260, 417, 174, 145, 3) }
    const folder: FolderExhibit = { id: randomUUID(), type: 'folder', title: 'Folder', content: 'Evidence folder', isOpen: true, ...placed(685, 417, 260, 166) }
    const note: NoteExhibit = { id: randomUUID(), type: 'note', title: 'Extract', content: 'Date matters', ...placed(420, 300, 108, 154) }
    const event: EventExhibit = { id: randomUUID(), type: 'event', title: 'The meeting occurred', content: 'The evidence places the meeting on 18 April.', eventDate: '2021-04-18T14:30:00Z', ...placed(520, 610, 270, 174) }
    const party: PartyExhibit = { id: randomUUID(), type: 'party', partyKind: 'person', title: 'Ada Lovelace', content: 'Named as correspondent.', aliases: ['A. A. L.'], ...placed(720, 250, 280, 190) }
-    state.exhibits = [document, folder, note, event, party]
+    state.exhibits = [document, gatedDocument, folder, note, event, party]
    state.relations = [
      { id: randomUUID(), fromExhibitId: folder.id, toExhibitId: document.id, type: 'contains', sortOrder: 0 },
      { id: randomUUID(), fromExhibitId: note.id, toExhibitId: document.id, type: 'source', sourceRegionId: 'stamp', sortOrder: 0 },
@@ -106,16 +107,55 @@ suite('normalized level persistence API', () => {
    expect(loaded.exhibits.find(item => item.id === folder.id)).toMatchObject({ x: 685, y: 417, isOpen: true })
    expect(loaded.relations).toEqual(expect.arrayContaining([expect.objectContaining({ type: 'supports', fromExhibitId: event.id, toExhibitId: note.id })]))
    expect(loaded.connections[0]).toMatchObject({ fromExhibitId: folder.id, toExhibitId: document.id, label: 'Primary source' })
+    expect(loaded.exhibits.find(item => item.id === gatedDocument.id)).toMatchObject({ requiredFlags: ['tip.received'] })
+
+    const beforeFlag = await (await fetch(`${baseUrl}/api/levels/${state.id}`)).json() as CaseState
+    expect(beforeFlag.exhibits.map(item => item.id)).not.toContain(gatedDocument.id)
+    expect(beforeFlag.newlyVisibleDocumentIds).toContain(document.id)
+    expect(await (await adminFetch(`${baseUrl}/api/levels/${state.id}/flags`)).json()).toEqual([
+      { key: 'tip.received', gatedDocumentCount: 1 },
+    ])
+    expect((await adminFetch(`${baseUrl}/api/levels/${state.id}/flags/tip.received`, { method: 'PUT' })).status).toBe(200)
+    const afterFlag = await (await fetch(`${baseUrl}/api/levels/${state.id}`)).json() as CaseState
+    expect(afterFlag.exhibits.map(item => item.id)).toContain(gatedDocument.id)
+    expect(afterFlag.newlyVisibleDocumentIds).toContain(gatedDocument.id)
+    expect((await fetch(`${baseUrl}/api/levels/${state.id}/reveals/seen`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ documentIds: afterFlag.newlyVisibleDocumentIds }) })).status).toBe(200)
+    expect((await (await fetch(`${baseUrl}/api/levels/${state.id}`)).json() as CaseState).newlyVisibleDocumentIds).toEqual([])
+
+    expect((await adminFetch(`${baseUrl}/api/levels/${state.id}/flags/tip.received`, { method: 'DELETE' })).status).toBe(200)
+    const matchRuleResponse = await adminFetch(`${baseUrl}/api/levels/${state.id}/evidence-match-rules`, {
+      method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({
+        name: 'Smoke source passage', flagKey: 'tip.received', minimumAnchorMatches: 1,
+        anchors: [{ phrase: 'OSINT smoke evidence from the archive', minimumSimilarity: 0.72 }],
+      }),
+    })
+    expect(matchRuleResponse.status).toBe(201)
+    expect(await matchRuleResponse.json()).toMatchObject({ name: 'Smoke source passage', flagKey: 'tip.received', anchors: [{ phrase: 'OSINT smoke evidence from the archive' }] })
+    expect(await (await adminFetch(`${baseUrl}/api/levels/${state.id}/evidence-match-rules`)).json()).toHaveLength(1)

    const upload = new FormData()
-    upload.append('file', new Blob(['OSINT smoke evidence'], { type: 'text/plain' }), 'smoke-evidence.txt')
-    const uploadResponse = await adminFetch(`${baseUrl}/api/levels/${state.id}/documents?edit=1`, { method: 'POST', body: upload })
+    upload.append('file', new Blob(['OSINT smoke evidence from the archlve'], { type: 'text/plain' }), 'smoke-evidence.txt')
+    upload.append('x', '812')
+    upload.append('y', '438')
+    const uploadResponse = await fetch(`${baseUrl}/api/levels/${state.id}/documents`, { method: 'POST', body: upload })
    expect(uploadResponse.status).toBe(201)
-    const uploaded = await uploadResponse.json() as DocumentExhibit
-    expect(uploaded).toMatchObject({ type: 'document', fileName: 'smoke-evidence.txt', fileType: 'text' })
-    expect(await (await fetch(`${baseUrl}/api/assets/${uploaded.assetId}`)).text()).toBe('OSINT smoke evidence')
+    const uploaded = await uploadResponse.json() as DocumentExhibit & { analysis: { extractionStatus: string; matchedFlags: string[]; awardedFlags: string[] } }
+    expect(uploaded).toMatchObject({ type: 'document', fileName: 'smoke-evidence.txt', fileType: 'text', x: 812, y: 438,
+      body: ['OSINT smoke evidence from the archlve'], analysis: { extractionStatus: 'succeeded', matchedFlags: ['tip.received'], awardedFlags: ['tip.received'] } })
+    expect(await (await fetch(`${baseUrl}/api/assets/${uploaded.assetId}`)).text()).toBe('OSINT smoke evidence from the archlve')
    const assetRow = await appPool.query<{ storage_provider: string; content: Buffer | null; object_key: string | null }>('SELECT storage_provider,content,object_key FROM osint.assets WHERE id=$1', [uploaded.assetId])
    expect(assetRow.rows[0]).toMatchObject({ storage_provider: 's3', content: null, object_key: expect.stringMatching(/^assets\//) })
+    const automaticallyRevealed = await (await fetch(`${baseUrl}/api/levels/${state.id}`)).json() as CaseState
+    expect(automaticallyRevealed.exhibits.map(item => item.id)).toContain(gatedDocument.id)
+    const evaluationRows = await appPool.query<{ matched: boolean; matched_anchor_count: number }>(
+      'SELECT matched,matched_anchor_count FROM osint.evidence_match_evaluations WHERE document_exhibit_id=$1', [uploaded.id])
+    expect(evaluationRows.rows).toEqual([{ matched: true, matched_anchor_count: 1 }])
+
+    const screenshot = new FormData()
+    screenshot.append('file', new Blob([Buffer.from('89504e470d0a1a0a', 'hex')], { type: 'image/png' }), 'Screenshot 2026-08-22.png')
+    const screenshotResponse = await fetch(`${baseUrl}/api/levels/${state.id}/documents`, { method: 'POST', body: screenshot })
+    expect(screenshotResponse.status).toBe(201)
+    expect(await screenshotResponse.json()).toMatchObject({ type: 'document', fileType: 'image', fileName: 'Screenshot 2026-08-22.png' })

    const templateResponse = await adminFetch(`${baseUrl}/api/levels/${state.id}/templates?edit=1`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: 'Smoke Template' }) })
    expect(templateResponse.status).toBe(201)
@@ -128,5 +168,10 @@ suite('normalized level persistence API', () => {
    expect(clone.relations.filter(relation => relation.type === 'supports')).toHaveLength(2)
    expect(clone.connections[0]).toMatchObject({ label: 'Primary source', tightness: 85 })
    expect(clone.brief.concepts[0].resolvedPartyExhibitId).not.toBe(party.id)
+    const authoredClone = await (await adminFetch(`${baseUrl}/api/levels/${clone.id}?edit=1`)).json() as CaseState
+    expect(authoredClone.exhibits.find(item => item.type === 'document' && item.title === 'Later tip')).toMatchObject({ requiredFlags: ['tip.received'] })
+    expect(await (await adminFetch(`${baseUrl}/api/levels/${clone.id}/evidence-match-rules`)).json()).toEqual([
+      expect.objectContaining({ name: 'Smoke source passage', flagKey: 'tip.received', anchors: [expect.objectContaining({ phrase: 'OSINT smoke evidence from the archive' })] }),
+    ])
  })
 })
@@ -13,6 +13,7 @@ export async function clearBoard(client: PoolClient, boardId: string) {
  await client.query('DELETE FROM osint.board_views WHERE board_id=$1', [boardId])
  await client.query('DELETE FROM osint.brief_concepts WHERE board_id=$1', [boardId])
  await client.query('DELETE FROM osint.level_briefs WHERE board_id=$1', [boardId])
+  await client.query('DELETE FROM osint.evidence_match_rules WHERE board_id=$1', [boardId])
  await client.query('DELETE FROM osint.metadata_fields WHERE board_id=$1', [boardId])
  await client.query('DELETE FROM osint.exhibits WHERE board_id=$1', [boardId])
  await client.query('UPDATE osint.boards SET revision=0,updated_at=NOW() WHERE id=$1', [boardId])
@@ -68,6 +69,29 @@ export async function cloneBoard(client: PoolClient, sourceBoardId: string, targ
    (exhibit_id,document_type_id,asset_id,title,published_at,captured_at,source_uri) VALUES ($1,$2,$3,$4,$5,$6,$7)`,
  [mapped(exhibitIds, row.exhibit_id, 'document'), row.document_type_id, row.asset_id, row.title, row.published_at, row.captured_at, row.source_uri])

+  const documentRequirements = await client.query<{ document_exhibit_id: string; flag_key: string }>(
+    'SELECT document_exhibit_id,flag_key FROM osint.document_flag_requirements WHERE board_id=$1 ORDER BY document_exhibit_id,flag_key', [sourceBoardId])
+  for (const row of documentRequirements.rows) await client.query(
+    'INSERT INTO osint.document_flag_requirements (board_id,document_exhibit_id,flag_key) VALUES ($1,$2,$3)',
+    [targetBoardId, mapped(exhibitIds, row.document_exhibit_id, 'document reveal requirement'), row.flag_key])
+
+  const ruleIds: IdMap = new Map()
+  const matchRules = await client.query<{
+    id: string; name: string; flag_key: string; matcher_version: string; minimum_anchor_matches: number; enabled: boolean
+  }>('SELECT id,name,flag_key,matcher_version,minimum_anchor_matches,enabled FROM osint.evidence_match_rules WHERE board_id=$1 ORDER BY created_at,id', [sourceBoardId])
+  for (const row of matchRules.rows) {
+    const id = randomUUID(); ruleIds.set(row.id, id)
+    await client.query(`INSERT INTO osint.evidence_match_rules
+      (id,board_id,origin_rule_id,name,flag_key,matcher_version,minimum_anchor_matches,enabled)
+      VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`, [id,targetBoardId,row.id,row.name,row.flag_key,row.matcher_version,row.minimum_anchor_matches,row.enabled])
+  }
+  const matchAnchors = await client.query<{ rule_id: string; phrase_text: string; minimum_similarity: string; sort_order: number }>(
+    `SELECT a.rule_id,a.phrase_text,a.minimum_similarity::text,a.sort_order FROM osint.evidence_match_anchors a
+      JOIN osint.evidence_match_rules r ON r.id=a.rule_id WHERE r.board_id=$1 ORDER BY a.rule_id,a.sort_order,a.id`, [sourceBoardId])
+  for (const row of matchAnchors.rows) await client.query(`INSERT INTO osint.evidence_match_anchors
+    (id,rule_id,phrase_text,minimum_similarity,sort_order) VALUES ($1,$2,$3,$4,$5)`,
+  [randomUUID(), mapped(ruleIds, row.rule_id, 'evidence match rule'), row.phrase_text, row.minimum_similarity, row.sort_order])
+
  const images = await client.query<{ exhibit_id: string; pixel_width: number | null; pixel_height: number | null; alt_text: string }>(
    `SELECT i.* FROM osint.image_documents i JOIN osint.exhibits e ON e.id=i.exhibit_id WHERE e.board_id=$1`, [sourceBoardId])
  for (const row of images.rows) await client.query(
@@ -0,0 +1,48 @@
+import { describe, expect, it } from 'vitest'
+import { evaluateEvidenceRules, normalizeEvidenceText, scoreEvidenceAnchor } from './evidenceMatching.js'
+
+const barricelliRule = {
+  id: 'rule-barricelli',
+  name: 'Contemporary Barricelli fire report',
+  flagKey: 'barricelli.child-rescue-source',
+  minimumAnchorMatches: 1,
+  anchors: [{
+    id: 'parents',
+    phrase: 'den italienske maler og opfinder Barricelli og frue, født Aall',
+    minimumSimilarity: 0.72,
+  }, {
+    id: 'drink',
+    phrase: 'Han vækkede nemlig sin mor for at faa noget at drikke',
+    minimumSimilarity: 0.72,
+  }],
+}
+
+describe('evidence text matching', () => {
+  it('normalizes historical Norwegian characters and page layout noise', () => {
+    expect(normalizeEvidenceText('Født Aall — 2½ aar\n gammel')).toBe('fodt aall 2 1 2 aar gammel')
+  })
+
+  it('tolerates plausible OCR substitutions in a distinctive passage', () => {
+    const result = scoreEvidenceAnchor(
+      normalizeEvidenceText('I kvistleiligheden boede den italienske maler og opfinder Barrioelli og frue, født Aall, med sin lille søn.'),
+      barricelliRule.anchors[0].phrase,
+    )
+    expect(result.similarity).toBeGreaterThan(0.9)
+  })
+
+  it('awards the data-defined flag when one configured anchor is present', () => {
+    const evaluations = evaluateEvidenceRules('Han vækkede nemlig sin mor for at faa noget at drikke, og da ser hun huset brænder.', [barricelliRule])
+    expect(evaluations[0]).toMatchObject({ matched: true, matchedAnchorCount: 1, flagKey: 'barricelli.child-rescue-source' })
+  })
+
+  it('does not match generic words from an unrelated fire report', () => {
+    const evaluations = evaluateEvidenceRules('A family escaped from a boarding-house fire during the night.', [barricelliRule])
+    expect(evaluations[0]).toMatchObject({ matched: false, matchedAnchorCount: 0 })
+  })
+
+  it('can require multiple anchors for a stricter level rule', () => {
+    const rule = { ...barricelliRule, minimumAnchorMatches: 2 }
+    expect(evaluateEvidenceRules(barricelliRule.anchors[0].phrase, [rule])[0].matched).toBe(false)
+    expect(evaluateEvidenceRules(barricelliRule.anchors.map(anchor => anchor.phrase).join(' '), [rule])[0].matched).toBe(true)
+  })
+})
@@ -0,0 +1,115 @@
+export type EvidenceMatchAnchor = {
+  id: string
+  phrase: string
+  minimumSimilarity: number
+}
+
+export type EvidenceMatchRule = {
+  id: string
+  name: string
+  flagKey: string
+  minimumAnchorMatches: number
+  anchors: EvidenceMatchAnchor[]
+}
+
+export type EvidenceAnchorEvaluation = {
+  anchorId: string
+  similarity: number
+  matched: boolean
+  matchedText: string
+}
+
+export type EvidenceRuleEvaluation = {
+  ruleId: string
+  flagKey: string
+  matched: boolean
+  matchedAnchorCount: number
+  score: number
+  anchors: EvidenceAnchorEvaluation[]
+}
+
+const MAX_MATCH_TEXT_CHARACTERS = 200_000
+
+/** Normalize historical spelling characters, punctuation, line breaks, and accents without changing word order. */
+export function normalizeEvidenceText(value: string) {
+  return value.slice(0, MAX_MATCH_TEXT_CHARACTERS)
+    .toLocaleLowerCase('en')
+    .replace(/æ/g, 'ae')
+    .replace(/ø/g, 'o')
+    .replace(/å/g, 'aa')
+    .replace(/½/g, ' 1 2 ')
+    .normalize('NFKD')
+    .replace(/\p{Mark}/gu, '')
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()
+    .replace(/\s+/g, ' ')
+}
+
+function grams(value: string, size = 3) {
+  const compact = value.replace(/\s+/g, ' ')
+  if (compact.length <= size) return [compact]
+  const result: string[] = []
+  for (let index = 0; index <= compact.length - size; index += 1) result.push(compact.slice(index, index + size))
+  return result
+}
+
+function diceSimilarity(left: string, right: string) {
+  if (left === right) return 1
+  if (!left || !right) return 0
+  const leftGrams = grams(left)
+  const rightGrams = grams(right)
+  const rightCounts = new Map<string, number>()
+  for (const gram of rightGrams) rightCounts.set(gram, (rightCounts.get(gram) || 0) + 1)
+  let overlap = 0
+  for (const gram of leftGrams) {
+    const count = rightCounts.get(gram) || 0
+    if (!count) continue
+    overlap += 1
+    rightCounts.set(gram, count - 1)
+  }
+  return (2 * overlap) / (leftGrams.length + rightGrams.length)
+}
+
+export function scoreEvidenceAnchor(normalizedDocument: string, phrase: string) {
+  const normalizedPhrase = normalizeEvidenceText(phrase)
+  if (!normalizedDocument || !normalizedPhrase) return { similarity: 0, matchedText: '' }
+  if (normalizedDocument.includes(normalizedPhrase)) return { similarity: 1, matchedText: normalizedPhrase }
+
+  const documentTokens = normalizedDocument.split(' ')
+  const phraseTokens = normalizedPhrase.split(' ')
+  const spread = Math.max(2, Math.min(8, Math.ceil(phraseTokens.length * 0.2)))
+  const minimumWindow = Math.max(1, phraseTokens.length - spread)
+  const maximumWindow = Math.min(documentTokens.length, phraseTokens.length + spread)
+  let best = { similarity: 0, matchedText: '' }
+
+  for (let windowSize = minimumWindow; windowSize <= maximumWindow; windowSize += 1) {
+    for (let start = 0; start + windowSize <= documentTokens.length; start += 1) {
+      const candidate = documentTokens.slice(start, start + windowSize).join(' ')
+      const similarity = diceSimilarity(normalizedPhrase, candidate)
+      if (similarity > best.similarity) best = { similarity, matchedText: candidate }
+    }
+  }
+  return best
+}
+
+export function evaluateEvidenceRules(text: string, rules: EvidenceMatchRule[]): EvidenceRuleEvaluation[] {
+  const normalizedDocument = normalizeEvidenceText(text)
+  return rules.map(rule => {
+    const anchors = rule.anchors.map(anchor => {
+      const result = scoreEvidenceAnchor(normalizedDocument, anchor.phrase)
+      const similarity = Math.max(0, Math.min(1, result.similarity))
+      return { anchorId: anchor.id, similarity, matched: similarity >= anchor.minimumSimilarity, matchedText: result.matchedText }
+    })
+    const matchedAnchors = anchors.filter(anchor => anchor.matched)
+    const requiredScores = [...anchors].sort((left, right) => right.similarity - left.similarity).slice(0, rule.minimumAnchorMatches)
+    const score = requiredScores.length ? requiredScores.reduce((sum, anchor) => sum + anchor.similarity, 0) / requiredScores.length : 0
+    return {
+      ruleId: rule.id,
+      flagKey: rule.flagKey,
+      matched: matchedAnchors.length >= rule.minimumAnchorMatches,
+      matchedAnchorCount: matchedAnchors.length,
+      score,
+      anchors,
+    }
+  })
+}
@@ -11,6 +11,7 @@ import type { CaseState } from '../src/types.js'
 import { authenticateJwt, createDevelopmentAdminToken, hasAdminClaim, requireAdmin, resolveUserId } from './auth.js'
 import { createLevelRepository } from './levelRepository.js'
 import { createNarrativeRepository } from './narrativeRepository.js'
+import { createTextExtractorFromEnv } from './ocr.js'
 import { createStoryGraphRepository, type StoryNodeType } from './storyGraphRepository.js'
 import { createObjectStorageFromEnv } from './objectStorage.js'

@@ -25,6 +26,7 @@ export const pool = new Pool({ connectionString: databaseUrl })
 const editingEnabled = process.env.LEVEL_EDITING_ENABLED === 'true'
 const objectStorage = createObjectStorageFromEnv()
 await objectStorage.initialize()
+const textExtractor = createTextExtractorFromEnv()
 const levels = createLevelRepository(pool, editingEnabled, objectStorage)
 const narrative = createNarrativeRepository(pool, objectStorage)
 const storyGraph = createStoryGraphRepository(pool)
@@ -49,7 +51,7 @@ const upload = multer({
 })

 app.get('/api/health', async (_req, res) => {
-  try { await pool.query('SELECT 1'); res.json({ ok: true, database: 'connected', objectStorage: objectStorage.provider, schema: 'osint', editingEnabled }) }
+  try { await pool.query('SELECT 1'); res.json({ ok: true, database: 'connected', objectStorage: objectStorage.provider, textExtraction: textExtractor.provider, schema: 'osint', editingEnabled }) }
  catch { res.status(503).json({ ok: false, database: 'unavailable' }) }
 })
 app.get('/api/session', (req, res) => res.json({ authenticated: Boolean(req.authClaims), isAdmin: hasAdminClaim(req) }))
@@ -106,14 +108,41 @@ app.get('/api/assets/:id', async (req, res, next) => {
    asset.stream.pipe(res)
  } catch (error) { next(error) }
 })
-app.post('/api/levels/:id/documents', requireAdmin, upload.single('file'), async (req, res, next) => {
+app.post('/api/levels/:id/documents', upload.single('file'), async (req, res, next) => {
  try {
-    if (!wantsEdit(req)) return res.status(403).json({ error: 'Level editing is disabled' })
    if (!req.file) return res.status(400).json({ error: 'A file is required' })
-    const document = await levels.uploadDocument(String(req.params.id), req.file)
+    const extraction = await textExtractor.extract(req.file)
+    const x = Number(req.body?.x); const y = Number(req.body?.y)
+    const placement = Number.isFinite(x) && Number.isFinite(y) ? { x, y } : undefined
+    const document = await levels.uploadDocument(String(req.params.id), req.file, extraction, placement)
    document ? res.status(201).json(document) : res.status(404).json({ error: 'Level not found' })
  } catch (error) { next(error) }
 })
+app.post('/api/levels/:id/reveals/seen', async (req, res, next) => {
+  try {
+    const ids = Array.isArray(req.body?.documentIds) ? req.body.documentIds.map(String) : []
+    const acknowledged = await levels.acknowledgeRevealedDocuments(String(req.params.id), ids)
+    acknowledged === null ? res.status(404).json({ error: 'Level not found' }) : res.json({ acknowledged })
+  } catch (error) { next(error) }
+})
+app.get('/api/levels/:id/flags', requireAdmin, async (req, res, next) => {
+  try {
+    const flags = await levels.listFlags(String(req.params.id))
+    flags ? res.json(flags) : res.status(404).json({ error: 'Level not found' })
+  } catch (error) { next(error) }
+})
+app.put('/api/levels/:id/flags/:key', requireAdmin, async (req, res, next) => {
+  try {
+    const updated = await levels.setFlag(String(req.params.id), String(req.params.key), true)
+    updated ? res.json({ ok: true }) : res.status(404).json({ error: 'Level not found' })
+  } catch (error) { next(error) }
+})
+app.delete('/api/levels/:id/flags/:key', requireAdmin, async (req, res, next) => {
+  try {
+    const updated = await levels.setFlag(String(req.params.id), String(req.params.key), false)
+    updated ? res.json({ ok: true }) : res.status(404).json({ error: 'Level not found' })
+  } catch (error) { next(error) }
+})
 app.get('/api/levels/:id', async (req, res, next) => {
  try {
    const level = await levels.getLevel(req.params.id, wantsEdit(req))
@@ -142,6 +171,34 @@ function requireEditing(res: express.Response) {
  if (!editingEnabled) { res.status(403).json({ error: 'Level editing is disabled' }); return false }
  return true
 }
+app.get('/api/levels/:id/evidence-match-rules', requireAdmin, async (req, res, next) => {
+  try {
+    const rules = await levels.listEvidenceMatchRules(String(req.params.id))
+    rules ? res.json(rules) : res.status(404).json({ error: 'Level not found' })
+  } catch (error) { next(error) }
+})
+app.post('/api/levels/:id/evidence-match-rules', requireAdmin, async (req, res, next) => {
+  try {
+    if (!requireEditing(res)) return
+    const rule = await levels.createEvidenceMatchRule(String(req.params.id), req.body)
+    rule ? res.status(201).json(rule) : res.status(404).json({ error: 'Level not found' })
+  } catch (error) { next(error) }
+})
+app.put('/api/levels/:id/evidence-match-rules/:ruleId', requireAdmin, async (req, res, next) => {
+  try {
+    if (!requireEditing(res)) return
+    const rule = await levels.updateEvidenceMatchRule(String(req.params.id), String(req.params.ruleId), req.body)
+    rule ? res.json(rule) : res.status(404).json({ error: 'Level or evidence match rule not found' })
+  } catch (error) { next(error) }
+})
+app.delete('/api/levels/:id/evidence-match-rules/:ruleId', requireAdmin, async (req, res, next) => {
+  try {
+    if (!requireEditing(res)) return
+    const removed = await levels.deleteEvidenceMatchRule(String(req.params.id), String(req.params.ruleId))
+    if (removed === null) return res.status(404).json({ error: 'Level not found' })
+    removed ? res.json({ ok: true }) : res.status(404).json({ error: 'Evidence match rule not found' })
+  } catch (error) { next(error) }
+})
 app.get('/api/admin/mysteries', requireAdmin, async (_req, res, next) => {
  try { res.json(await narrative.listMysteries()) } catch (error) { next(error) }
 })
@@ -350,6 +407,23 @@ app.post('/api/playthroughs/:id/advance', async (req, res, next) => {
  } catch (error) { next(error) }
 })

+// The playthrough case-state (achievements). Read is open; granting is a dev-only
+// stand-in until the server-side achievement rule engine drives awards from play.
+app.get('/api/playthroughs/:id/achievements', async (req, res, next) => {
+  try {
+    const flags = await narrative.listAchievements(String(req.params.id))
+    flags ? res.json(flags) : res.status(404).json({ error: 'Playthrough not found' })
+  } catch (error) { next(error) }
+})
+app.post('/api/playthroughs/:id/achievements', async (req, res, next) => {
+  try {
+    if (process.env.NODE_ENV === 'production') return res.status(403).json({ error: 'Manual grants are disabled' })
+    if (!req.body?.flagKey) return res.status(400).json({ error: 'A flagKey is required' })
+    const result = await narrative.awardAchievement(String(req.params.id), String(req.body.flagKey), req.body.nodeId ? String(req.body.nodeId) : null)
+    result.ok ? res.json({ earned: result.earned }) : res.status(result.error === 'Playthrough not found' ? 404 : 400).json({ error: result.error })
+  } catch (error) { next(error) }
+})
+
 app.use((error: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
  if (error instanceof multer.MulterError) {
    return res.status(error.code === 'LIMIT_FILE_SIZE' ? 413 : 400).json({ error: error.code === 'LIMIT_FILE_SIZE' ? 'Document exceeds the upload limit' : error.message })
@@ -1,15 +1,25 @@
 import { createHash, randomUUID } from 'node:crypto'
 import { Readable } from 'node:stream'
 import type { Pool, PoolClient } from 'pg'
-import type { BoardView, BriefConcept, CaseDocument, CaseState, Evidence, Exhibit, ExhibitRelation, OrganizationKind, PartyKind, SourceFileType } from '../src/types.js'
+import type { BoardView, BriefConcept, CaseDocument, CaseState, Evidence, EvidenceMatchRuleDefinition, Exhibit, ExhibitRelation, LevelFlag, OrganizationKind, PartyKind, SourceFileType, UploadedCaseDocument } from '../src/types.js'
 import { isDocumentExhibit, isEventExhibit, isFolderExhibit, isPartyExhibit } from '../src/types.js'
 import { clearBoard, cloneBoard } from './boardClone.js'
+import { evaluateEvidenceRules, type EvidenceMatchRule } from './evidenceMatching.js'
+import { filterLevelVisibility, mergePlayerStateForPersistence } from './levelVisibility.js'
 import type { ObjectStorage } from './objectStorage.js'
+import type { TextExtractionResult } from './ocr.js'

 export type UploadedDocument = { buffer: Buffer; originalname: string; mimetype: string; size: number }
 export type AssetRecord = { original_name: string; mime_type: string; byte_size: string; content: Buffer | null; storage_provider: 'postgres' | 's3'; object_key: string | null }
 export type AssetResponse = { originalName: string; mimeType: string; byteSize: number; stream: NodeJS.ReadableStream }
 export type TemplateSummary = { id: string; slug: string; name: string; currentVersion: number; versionCount: number; updatedAt: string }
+export type EvidenceMatchRuleInput = {
+  name: string
+  flagKey: string
+  minimumAnchorMatches?: number
+  enabled?: boolean
+  anchors: { phrase: string; minimumSimilarity?: number }[]
+}

 export interface LevelRepository {
  listLevels(): Promise<unknown[]>
@@ -21,7 +31,14 @@ export interface LevelRepository {
  saveLevel(state: CaseState, authorMode: boolean): Promise<void>
  resetLevel(levelId: string): Promise<CaseState | null>
  getAsset(assetId: string): Promise<AssetResponse | null>
-  uploadDocument(levelId: string, file: UploadedDocument): Promise<CaseDocument | null>
+  uploadDocument(levelId: string, file: UploadedDocument, extraction: TextExtractionResult, placement?: { x: number; y: number }): Promise<UploadedCaseDocument | null>
+  listFlags(levelId: string): Promise<LevelFlag[] | null>
+  setFlag(levelId: string, key: string, earned: boolean): Promise<boolean>
+  acknowledgeRevealedDocuments(levelId: string, documentIds: string[]): Promise<number | null>
+  listEvidenceMatchRules(levelId: string): Promise<EvidenceMatchRuleDefinition[] | null>
+  createEvidenceMatchRule(levelId: string, input: EvidenceMatchRuleInput): Promise<EvidenceMatchRuleDefinition | null>
+  updateEvidenceMatchRule(levelId: string, ruleId: string, input: EvidenceMatchRuleInput): Promise<EvidenceMatchRuleDefinition | null>
+  deleteEvidenceMatchRule(levelId: string, ruleId: string): Promise<boolean | null>
 }

 type LevelRow = {
@@ -39,6 +56,7 @@ type ExhibitRow = {
 }

 const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
+const flagPattern = /^[a-z][a-z0-9_.-]{0,63}$/
 function requireUuid(value: string, label: string) {
  if (!uuidPattern.test(value)) throw new Error(`${label} must be a UUID`)
  return value
@@ -48,6 +66,26 @@ function timestamp(value: string | undefined) {
  const date = new Date(value)
  return Number.isFinite(date.getTime()) ? date.toISOString() : null
 }
+function requireFlagKey(value: string) {
+  if (!flagPattern.test(value)) throw new Error('Flag keys must start with a lowercase letter and contain only lowercase letters, numbers, dots, dashes, or underscores')
+  return value
+}
+function requireRuleInput(input: EvidenceMatchRuleInput) {
+  const name = String(input.name || '').trim()
+  if (!name || name.length > 160) throw new Error('Evidence match rule names must be between 1 and 160 characters')
+  const flagKey = requireFlagKey(String(input.flagKey || '').trim())
+  if (!Array.isArray(input.anchors) || !input.anchors.length || input.anchors.length > 20) throw new Error('Evidence match rules require between 1 and 20 anchors')
+  const anchors = input.anchors.map(anchor => {
+    const phrase = String(anchor.phrase || '').trim()
+    if (phrase.length < 12 || phrase.length > 1_000) throw new Error('Evidence match anchors must be between 12 and 1000 characters')
+    const minimumSimilarity = anchor.minimumSimilarity === undefined ? 0.72 : Number(anchor.minimumSimilarity)
+    if (!Number.isFinite(minimumSimilarity) || minimumSimilarity < 0.5 || minimumSimilarity > 1) throw new Error('Anchor similarity must be between 0.5 and 1')
+    return { phrase, minimumSimilarity }
+  })
+  const minimumAnchorMatches = input.minimumAnchorMatches === undefined ? 1 : Number(input.minimumAnchorMatches)
+  if (!Number.isInteger(minimumAnchorMatches) || minimumAnchorMatches < 1 || minimumAnchorMatches > anchors.length) throw new Error('Required anchor matches must be between 1 and the number of anchors')
+  return { name, flagKey, minimumAnchorMatches, enabled: input.enabled !== false, anchors }
+}
 function documentType(document: CaseDocument): SourceFileType {
  const allowed: SourceFileType[] = ['image', 'pdf', 'web_capture', 'email', 'article', 'filing', 'price_list', 'text', 'file']
  return allowed.includes(document.fileType) ? document.fileType : 'file'
@@ -67,11 +105,49 @@ export function createLevelRepository(pool: Pool, editingEnabled: boolean, objec
    await client.query("INSERT INTO osint.timeline_views (view_id,range_mode) VALUES ($1,'auto')", [viewId])
  }

+  async function evidenceMatchRules(client: Pool | PoolClient, boardId: string, includeDisabled = false): Promise<EvidenceMatchRuleDefinition[]> {
+    const result = await client.query<{
+      rule_id: string; name: string; flag_key: string; matcher_version: 'char_trigram_v1'; minimum_anchor_matches: number; enabled: boolean
+      anchor_id: string; phrase_text: string; minimum_similarity: string; sort_order: number
+    }>(`SELECT r.id AS rule_id,r.name,r.flag_key,r.matcher_version,r.minimum_anchor_matches,r.enabled,
+        a.id AS anchor_id,a.phrase_text,a.minimum_similarity::text,a.sort_order
+      FROM osint.evidence_match_rules r
+      JOIN osint.evidence_match_anchors a ON a.rule_id=r.id
+      WHERE r.board_id=$1 ${includeDisabled ? '' : 'AND r.enabled'}
+      ORDER BY r.created_at,r.id,a.sort_order,a.id`, [boardId])
+    const rules = new Map<string, EvidenceMatchRuleDefinition>()
+    for (const row of result.rows) {
+      const rule = rules.get(row.rule_id) || { id: row.rule_id, name: row.name, flagKey: row.flag_key,
+        matcherVersion: row.matcher_version, minimumAnchorMatches: row.minimum_anchor_matches, enabled: row.enabled, anchors: [] }
+      rule.anchors.push({ id: row.anchor_id, phrase: row.phrase_text, minimumSimilarity: Number(row.minimum_similarity), sortOrder: row.sort_order })
+      rules.set(row.rule_id, rule)
+    }
+    return [...rules.values()]
+  }
+
+  async function writeEvidenceMatchRule(client: PoolClient, level: LevelRow, ruleId: string, rawInput: EvidenceMatchRuleInput, update: boolean) {
+    const input = requireRuleInput(rawInput)
+    if (update) {
+      const changed = await client.query(`UPDATE osint.evidence_match_rules SET name=$3,flag_key=$4,minimum_anchor_matches=$5,enabled=$6,updated_at=NOW()
+        WHERE id=$1 AND board_id=$2`, [ruleId, level.board_id, input.name, input.flagKey, input.minimumAnchorMatches, input.enabled])
+      if (!changed.rowCount) return null
+      await client.query('DELETE FROM osint.evidence_match_anchors WHERE rule_id=$1', [ruleId])
+    } else {
+      await client.query(`INSERT INTO osint.evidence_match_rules (id,board_id,name,flag_key,minimum_anchor_matches,enabled)
+        VALUES ($1,$2,$3,$4,$5,$6)`, [ruleId, level.board_id, input.name, input.flagKey, input.minimumAnchorMatches, input.enabled])
+    }
+    for (const [sortOrder, anchor] of input.anchors.entries()) await client.query(`INSERT INTO osint.evidence_match_anchors
+      (id,rule_id,phrase_text,minimum_similarity,sort_order) VALUES ($1,$2,$3,$4,$5)`,
+    [randomUUID(), ruleId, anchor.phrase, anchor.minimumSimilarity, sortOrder])
+    await client.query('UPDATE osint.levels SET updated_at=NOW() WHERE id=$1', [level.id])
+    return (await evidenceMatchRules(client, level.board_id, true)).find(rule => rule.id === ruleId) || null
+  }
+
  async function assembleLevel(slug: string, authorMode = false): Promise<CaseState | null> {
    const level = await findLevel(pool, slug)
    if (!level) return null
    const [exhibitsResult, blocksResult, regionsResult, membershipsResult, connectionsResult, metadataResult, eventEvidenceResult,
-      aliasesResult, partyEvidenceResult, briefResult, conceptsResult, viewsResult] = await Promise.all([
+      aliasesResult, partyEvidenceResult, briefResult, conceptsResult, viewsResult, requirementsResult, flagsResult, seenResult] = await Promise.all([
      pool.query<ExhibitRow>(`SELECT e.id,e.exhibit_type_id,e.xpos,e.ypos,e.width,e.height,e.rotation,e.z_index,e.hidden,
        COALESCE(f.title, d.title, n.title, ev.title, p.display_name, '') AS title,
        COALESCE(f.label_text, n.note_text, ev.narrative_text, p.summary, '') AS content,
@@ -122,6 +198,10 @@ export function createLevelRepository(pool: Pool, editingEnabled: boolean, objec
        `SELECT v.id,v.view_type_id,v.placement_mode,v.dock_edge,v.xpos,v.ypos,v.width,v.height,v.z_index,v.visible,
          t.range_mode,t.range_start::text,t.range_end::text FROM osint.board_views v
          JOIN osint.timeline_views t ON t.view_id=v.id WHERE v.board_id=$1 ORDER BY v.z_index,v.created_at`, [level.board_id]),
+      pool.query<{ document_exhibit_id: string; flag_key: string }>(
+        'SELECT document_exhibit_id,flag_key FROM osint.document_flag_requirements WHERE board_id=$1 ORDER BY document_exhibit_id,flag_key', [level.board_id]),
+      pool.query<{ flag_key: string }>('SELECT flag_key FROM osint.level_flags WHERE level_id=$1 ORDER BY flag_key', [level.id]),
+      pool.query<{ document_exhibit_id: string }>('SELECT document_exhibit_id FROM osint.level_seen_documents WHERE level_id=$1', [level.id]),
    ])

    const blocks = new Map<string, string[]>()
@@ -134,6 +214,8 @@ export function createLevelRepository(pool: Pool, editingEnabled: boolean, objec
    for (const row of metadataResult.rows) metadata.set(row.exhibit_id, { ...(metadata.get(row.exhibit_id) || {}), [row.field_key]: row.value })
    const aliases = new Map<string, string[]>()
    for (const row of aliasesResult.rows) aliases.set(row.party_exhibit_id, [...(aliases.get(row.party_exhibit_id) || []), row.alias])
+    const requirements = new Map<string, string[]>()
+    for (const row of requirementsResult.rows) requirements.set(row.document_exhibit_id, [...(requirements.get(row.document_exhibit_id) || []), row.flag_key])
    const relations: ExhibitRelation[] = [
      ...membershipsResult.rows.map(row => ({ id: `contains:${row.folder_exhibit_id}:${row.child_exhibit_id}`, fromExhibitId: row.folder_exhibit_id,
        toExhibitId: row.child_exhibit_id, type: 'contains' as const, sortOrder: row.sort_order })),
@@ -149,13 +231,13 @@ export function createLevelRepository(pool: Pool, editingEnabled: boolean, objec
    const documents: CaseDocument[] = exhibitsResult.rows.filter(row => row.exhibit_type_id === 'document').map(row => {
      const type = row.document_type_id || 'file'
      const publishedAt = row.published_at?.toISOString()
-      return { ...base(row), type: 'document', publishedAt,
+      return { ...base(row), type: 'document', publishedAt, requiredFlags: requirements.get(row.id) || [],
        body: blocks.get(row.id) || [], regions: regions.get(row.id) || [], assetId: row.asset_id || undefined,
        fileName: row.original_name || undefined, mimeType: row.mime_type || undefined,
        fileSize: row.byte_size === null ? undefined : Number(row.byte_size), fileType: type, metadata: metadata.get(row.id) || {} }
    })
    const evidence: Evidence[] = []
-    for (const row of exhibitsResult.rows.filter(row => row.exhibit_type_id !== 'document' && !row.hidden)) {
+    for (const row of exhibitsResult.rows.filter(row => row.exhibit_type_id !== 'document')) {
      const common = { ...base(row), title: row.title, content: row.content }
      if (row.exhibit_type_id === 'folder') evidence.push({ ...common, type:'folder', isOpen:Boolean(row.is_open) })
      else if (row.exhibit_type_id === 'event') evidence.push({ ...common, type:'event', eventDate:row.occurred_at?.toISOString() })
@@ -170,7 +252,7 @@ export function createLevelRepository(pool: Pool, editingEnabled: boolean, objec
      rangeMode: row.range_mode, range: row.range_start && row.range_end ? { start: row.range_start, end: row.range_end } : undefined }))
    const concepts: BriefConcept[] = conceptsResult.rows.map(row => ({ id: row.id, label: row.label, context: row.context_text,
      ...(authorMode && row.expected_party_kind ? { expectedPartyKind: row.expected_party_kind } : {}), resolvedPartyExhibitId: row.resolved_party_exhibit_id || undefined }))
-    return { id: level.slug, title: level.title, subtitle: level.subtitle, exhibits: [...documents, ...evidence], relations,
+    const fullState: CaseState = { id: level.slug, title: level.title, subtitle: level.subtitle, exhibits: [...documents, ...evidence], relations,
      connections: connectionsResult.rows.map(row => ({ id: row.id, fromExhibitId: row.from_exhibit_id, toExhibitId: row.to_exhibit_id,
        label: row.label || undefined, tightness: row.tightness, tagStyle: row.tag_style,
        tagPosition: row.tag_position_percent, tagOffset: row.tag_lateral_offset })),
@@ -178,6 +260,7 @@ export function createLevelRepository(pool: Pool, editingEnabled: boolean, objec
      views, revision: Number(level.revision),
      brief: { body: briefResult.rows[0]?.body || '', concepts }, levelStatus: level.status, editingAllowed: editingEnabled && authorMode,
      sourceTemplateVersionId: level.source_template_version_id || undefined }
+    return filterLevelVisibility(fullState, flagsResult.rows.map(row => row.flag_key), seenResult.rows.map(row => row.document_exhibit_id), authorMode)
  }

  async function templateSummary(slug: string): Promise<TemplateSummary | null> {
@@ -234,6 +317,8 @@ export function createLevelRepository(pool: Pool, editingEnabled: boolean, objec
        VALUES ($1,$2,$3,$4,$5,$6,$7)`, [document.id, documentType(document), document.assetId || null, document.title,
        timestamp(document.publishedAt), timestamp(document.capturedAt), document.sourceUri || null])
      if (document.fileType === 'image') await client.query('INSERT INTO osint.image_documents (exhibit_id) VALUES ($1)', [document.id])
+      for (const flag of new Set((document.requiredFlags || []).map(value => value.trim()).filter(Boolean).map(requireFlagKey))) await client.query(
+        'INSERT INTO osint.document_flag_requirements (board_id,document_exhibit_id,flag_key) VALUES ($1,$2,$3)', [level.board_id, document.id, flag])
      for (const [sortOrder, content] of document.body.entries()) await client.query(
        'INSERT INTO osint.document_content_blocks (id,document_exhibit_id,sort_order,content) VALUES ($1,$2,$3,$4)', [randomUUID(), document.id, sortOrder, content])
      for (const [sortOrder, region] of document.regions.entries()) await client.query(
@@ -411,13 +496,19 @@ export function createLevelRepository(pool: Pool, editingEnabled: boolean, objec
      return templateSummary(input.slug)
    },
    getLevel(levelId, authorMode = false) { return assembleLevel(levelId, authorMode) },
-    async saveLevel(state) {
+    async saveLevel(state, authorMode) {
+      let persistedState = state
+      if (!authorMode) {
+        const [full, visible] = await Promise.all([assembleLevel(state.id, true), assembleLevel(state.id, false)])
+        if (!full || !visible) throw new Error('Level not found')
+        persistedState = mergePlayerStateForPersistence(full, visible, state)
+      }
      const client = await pool.connect()
      try {
        await client.query('BEGIN')
        const level = await findLevel(client, state.id, true)
        if (!level) throw new Error('Level not found')
-        await replaceBoard(client, level, state)
+        await replaceBoard(client, level, persistedState)
        await client.query('COMMIT')
      } catch (error) { await client.query('ROLLBACK'); throw error } finally { client.release() }
    },
@@ -454,7 +545,7 @@ export function createLevelRepository(pool: Pool, editingEnabled: boolean, objec
      const object = await objectStorage.getObject(asset.object_key)
      return object ? { originalName: asset.original_name, mimeType: asset.mime_type, byteSize: Number(asset.byte_size), stream: object.stream } : null
    },
-    async uploadDocument(levelId, file) {
+    async uploadDocument(levelId, file, extraction, placement) {
      const client = await pool.connect()
      try {
        await client.query('BEGIN')
@@ -474,17 +565,149 @@ export function createLevelRepository(pool: Pool, editingEnabled: boolean, objec
          assetId = asset.rows[0].id
        }
        const fileType: SourceFileType = file.mimetype.startsWith('image/') ? 'image' : file.mimetype === 'application/pdf' ? 'pdf' : file.mimetype.startsWith('text/') ? 'text' : 'file'
+        const xpos = Number.isFinite(placement?.x) ? Math.max(0, Math.min(10_000, Number(placement?.x))) : 100
+        const ypos = Number.isFinite(placement?.y) ? Math.max(0, Math.min(10_000, Number(placement?.y))) : 100
        await client.query(`INSERT INTO osint.exhibits (id,board_id,exhibit_type_id,xpos,ypos,width,height,z_index,hidden)
-          VALUES ($1,$2,'document',100,100,174,145,(SELECT COUNT(*) FROM osint.exhibits WHERE board_id=$2),FALSE)`, [exhibitId, level.board_id])
+          VALUES ($1,$2,'document',$3,$4,174,145,(SELECT COUNT(*) FROM osint.exhibits WHERE board_id=$2),FALSE)`, [exhibitId, level.board_id, xpos, ypos])
        await client.query(`INSERT INTO osint.document_exhibits (exhibit_id,document_type_id,asset_id,title) VALUES ($1,$2,$3,$4)`,
          [exhibitId, fileType, assetId, file.originalname])
        if (fileType === 'image') await client.query('INSERT INTO osint.image_documents (exhibit_id) VALUES ($1)', [exhibitId])
+        if (extraction.status === 'succeeded' && extraction.text.trim()) await client.query(
+          'INSERT INTO osint.document_content_blocks (id,document_exhibit_id,sort_order,content) VALUES ($1,$2,0,$3)', [randomUUID(), exhibitId, extraction.text.trim()])
+
+        const extractionResult = await client.query<{ id: string }>(`INSERT INTO osint.asset_text_extractions
+          (id,asset_id,extractor,extractor_version,language,status,extracted_text,error_message)
+          VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
+          ON CONFLICT (asset_id,extractor,extractor_version,language) DO UPDATE SET
+            status=EXCLUDED.status,extracted_text=EXCLUDED.extracted_text,error_message=EXCLUDED.error_message,updated_at=NOW()
+          RETURNING id`, [randomUUID(), assetId, extraction.extractor, extraction.extractorVersion, extraction.language,
+          extraction.status, extraction.text, extraction.error?.slice(0, 2_000) || null])
+        const extractionId = extractionResult.rows[0].id
+        const ruleDefinitions = extraction.status === 'succeeded' && extraction.text.trim()
+          ? await evidenceMatchRules(client, level.board_id)
+          : []
+        const evaluations = evaluateEvidenceRules(extraction.text, ruleDefinitions as EvidenceMatchRule[])
+        const matchedFlags: string[] = []
+        const awardedFlags: string[] = []
+        for (const evaluation of evaluations) {
+          const evaluationId = randomUUID()
+          await client.query(`INSERT INTO osint.evidence_match_evaluations
+            (id,level_id,board_id,document_exhibit_id,extraction_id,rule_id,matched,matched_anchor_count,score)
+            VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`, [evaluationId, level.id, level.board_id, exhibitId, extractionId,
+            evaluation.ruleId, evaluation.matched, evaluation.matchedAnchorCount, evaluation.score])
+          for (const anchor of evaluation.anchors) await client.query(`INSERT INTO osint.evidence_match_anchor_evaluations
+            (evaluation_id,anchor_id,similarity,matched,matched_text) VALUES ($1,$2,$3,$4,$5)`,
+          [evaluationId, anchor.anchorId, anchor.similarity, anchor.matched, anchor.matchedText])
+          if (!evaluation.matched) continue
+          matchedFlags.push(evaluation.flagKey)
+          const awarded = await client.query(`INSERT INTO osint.level_flags (level_id,board_id,flag_key,awarded_by_evidence_match_id)
+            VALUES ($1,$2,$3,$4) ON CONFLICT (level_id,flag_key) DO NOTHING RETURNING flag_key`,
+          [level.id, level.board_id, evaluation.flagKey, evaluationId])
+          if (awarded.rowCount) awardedFlags.push(evaluation.flagKey)
+        }
        await client.query('UPDATE osint.levels SET updated_at=NOW() WHERE id=$1', [level.id])
        await client.query('UPDATE osint.boards SET revision=revision+1,updated_at=NOW() WHERE id=$1', [level.board_id])
        await client.query('COMMIT')
-        return { id: exhibitId,type:'document',title:file.originalname,x:100,y:100,width:174,height:145,rotation:0,zIndex:0,hidden:false,
-          fileType,metadata:{},body:[],regions:[],assetId,fileName:file.originalname,mimeType:file.mimetype,fileSize:file.size }
+        return { id: exhibitId,type:'document',title:file.originalname,x:xpos,y:ypos,width:174,height:145,rotation:0,zIndex:0,hidden:false,
+          fileType,metadata:{},body:extraction.status === 'succeeded' && extraction.text.trim() ? [extraction.text.trim()] : [],regions:[],assetId,
+          fileName:file.originalname,mimeType:file.mimetype,fileSize:file.size,
+          analysis:{ extractionStatus:extraction.status, matchedFlags:[...new Set(matchedFlags)], awardedFlags:[...new Set(awardedFlags)] } }
      } catch (error) { await client.query('ROLLBACK'); throw error } finally { client.release() }
    },
+    async listFlags(levelId) {
+      const level = await findLevel(pool, levelId)
+      if (!level) return null
+      const result = await pool.query<{ flag_key: string; earned_at: Date | null; gated_document_count: number }>(`
+        WITH keys AS (
+          SELECT flag_key FROM osint.level_flags WHERE level_id=$1
+          UNION
+          SELECT flag_key FROM osint.document_flag_requirements WHERE board_id=$2
+          UNION
+          SELECT flag_key FROM osint.evidence_match_rules WHERE board_id=$2
+        )
+        SELECT keys.flag_key,flags.earned_at,COUNT(requirements.document_exhibit_id)::int AS gated_document_count
+        FROM keys
+        LEFT JOIN osint.level_flags flags ON flags.level_id=$1 AND flags.flag_key=keys.flag_key
+        LEFT JOIN osint.document_flag_requirements requirements ON requirements.board_id=$2 AND requirements.flag_key=keys.flag_key
+        GROUP BY keys.flag_key,flags.earned_at ORDER BY keys.flag_key`, [level.id, level.board_id])
+      return result.rows.map(row => ({ key: row.flag_key, earnedAt: row.earned_at?.toISOString(), gatedDocumentCount: row.gated_document_count }))
+    },
+    async setFlag(levelId, rawKey, earned) {
+      const key = requireFlagKey(rawKey.trim())
+      const client = await pool.connect()
+      try {
+        await client.query('BEGIN')
+        const level = await findLevel(client, levelId, true)
+        if (!level) { await client.query('ROLLBACK'); return false }
+        if (earned) await client.query(`INSERT INTO osint.level_flags (level_id,board_id,flag_key) VALUES ($1,$2,$3)
+          ON CONFLICT (level_id,flag_key) DO NOTHING`, [level.id, level.board_id, key])
+        else {
+          await client.query('DELETE FROM osint.level_flags WHERE level_id=$1 AND flag_key=$2', [level.id, key])
+          await client.query(`DELETE FROM osint.level_seen_documents seen USING osint.document_flag_requirements requirement
+            WHERE seen.level_id=$1 AND seen.document_exhibit_id=requirement.document_exhibit_id AND requirement.board_id=$2 AND requirement.flag_key=$3`,
+          [level.id, level.board_id, key])
+        }
+        await client.query('UPDATE osint.levels SET updated_at=NOW() WHERE id=$1', [level.id])
+        await client.query('COMMIT')
+        return true
+      } catch (error) { await client.query('ROLLBACK'); throw error } finally { client.release() }
+    },
+    async listEvidenceMatchRules(levelId) {
+      const level = await findLevel(pool, levelId)
+      return level ? evidenceMatchRules(pool, level.board_id, true) : null
+    },
+    async createEvidenceMatchRule(levelId, input) {
+      const client = await pool.connect()
+      try {
+        await client.query('BEGIN')
+        const level = await findLevel(client, levelId, true)
+        if (!level) { await client.query('ROLLBACK'); return null }
+        const rule = await writeEvidenceMatchRule(client, level, randomUUID(), input, false)
+        await client.query('COMMIT')
+        return rule
+      } catch (error) { await client.query('ROLLBACK'); throw error } finally { client.release() }
+    },
+    async updateEvidenceMatchRule(levelId, ruleId, input) {
+      if (!uuidPattern.test(ruleId)) return null
+      const client = await pool.connect()
+      try {
+        await client.query('BEGIN')
+        const level = await findLevel(client, levelId, true)
+        if (!level) { await client.query('ROLLBACK'); return null }
+        const rule = await writeEvidenceMatchRule(client, level, ruleId, input, true)
+        await client.query('COMMIT')
+        return rule
+      } catch (error) { await client.query('ROLLBACK'); throw error } finally { client.release() }
+    },
+    async deleteEvidenceMatchRule(levelId, ruleId) {
+      if (!uuidPattern.test(ruleId)) return false
+      const client = await pool.connect()
+      try {
+        await client.query('BEGIN')
+        const level = await findLevel(client, levelId, true)
+        if (!level) { await client.query('ROLLBACK'); return null }
+        const removed = await client.query('DELETE FROM osint.evidence_match_rules WHERE id=$1 AND board_id=$2', [ruleId, level.board_id])
+        if (removed.rowCount) await client.query('UPDATE osint.levels SET updated_at=NOW() WHERE id=$1', [level.id])
+        await client.query('COMMIT')
+        return Boolean(removed.rowCount)
+      } catch (error) { await client.query('ROLLBACK'); throw error } finally { client.release() }
+    },
+    async acknowledgeRevealedDocuments(levelId, rawDocumentIds) {
+      const documentIds = [...new Set(rawDocumentIds.filter(id => uuidPattern.test(id)))]
+      const level = await findLevel(pool, levelId)
+      if (!level) return null
+      if (!documentIds.length) return 0
+      const result = await pool.query(`INSERT INTO osint.level_seen_documents (level_id,board_id,document_exhibit_id)
+        SELECT $1,$2,e.id FROM osint.exhibits e
+        WHERE e.board_id=$2 AND e.id=ANY($3::uuid[]) AND e.exhibit_type_id='document' AND NOT e.hidden
+          AND NOT EXISTS (
+            SELECT 1 FROM osint.document_flag_requirements requirement
+            WHERE requirement.document_exhibit_id=e.id AND NOT EXISTS (
+              SELECT 1 FROM osint.level_flags flag WHERE flag.level_id=$1 AND flag.flag_key=requirement.flag_key
+            )
+          )
+        ON CONFLICT (level_id,document_exhibit_id) DO NOTHING`, [level.id, level.board_id, documentIds])
+      return result.rowCount || 0
+    },
  }
 }
@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest'
+import type { CaseState, DocumentExhibit, NoteExhibit } from '../src/types.js'
+import { filterLevelVisibility, mergePlayerStateForPersistence } from './levelVisibility.js'
+
+const placed = { x: 10, y: 20, width: 174, height: 145, rotation: 0, zIndex: 1, hidden: false }
+const open: DocumentExhibit = { id: 'open', type: 'document', title: 'Open', body: [], regions: [], fileType: 'image', metadata: {}, ...placed }
+const gated: DocumentExhibit = { id: 'gated', type: 'document', title: 'Gated', body: [], regions: [], fileType: 'image', metadata: {}, requiredFlags: ['tip.received'], ...placed }
+const note: NoteExhibit = { id: 'note', type: 'note', title: 'Note', content: '', ...placed }
+const state: CaseState = {
+  id: 'demo', title: 'Demo', subtitle: '', exhibits: [open, gated, note], viewport: { x: 0, y: 0, zoom: 1 },
+  relations: [{ id: 'source', type: 'source', fromExhibitId: note.id, toExhibitId: gated.id, sortOrder: 0 }],
+  connections: [{ id: 'thread', fromExhibitId: open.id, toExhibitId: gated.id }],
+  views: [], brief: { body: '', concepts: [] }, revision: 0,
+}
+
+describe('level flag visibility', () => {
+  it('hides gated documents and every edge touching them', () => {
+    const visible = filterLevelVisibility(state, [], [], false)
+    expect(visible.exhibits.map(item => item.id)).toEqual(['open', 'note'])
+    expect(visible.relations).toEqual([])
+    expect(visible.connections).toEqual([])
+    expect(visible.newlyVisibleDocumentIds).toEqual(['open'])
+  })
+
+  it('reveals earned documents once without leaking their gate definition', () => {
+    const visible = filterLevelVisibility(state, ['tip.received'], ['open'], false)
+    expect(visible.exhibits.map(item => item.id)).toEqual(['open', 'gated', 'note'])
+    expect((visible.exhibits[1] as DocumentExhibit).requiredFlags).toBeUndefined()
+    expect(visible.newlyVisibleDocumentIds).toEqual(['gated'])
+  })
+
+  it('returns all documents and requirements to author mode', () => {
+    const authored = filterLevelVisibility(state, [], [], true)
+    expect((authored.exhibits[1] as DocumentExhibit).requiredFlags).toEqual(['tip.received'])
+    expect(authored.newlyVisibleDocumentIds).toEqual([])
+  })
+
+  it('preserves unrevealed documents and their edges during a play-mode save', () => {
+    const visible = filterLevelVisibility(state, [], ['open'], false)
+    const submitted = { ...visible, exhibits: visible.exhibits.map(item => item.id === 'open' ? { ...item, x: 99 } : item) }
+    const merged = mergePlayerStateForPersistence(state, visible, submitted)
+    expect(merged.exhibits.find(item => item.id === 'open')?.x).toBe(99)
+    expect(merged.exhibits.find(item => item.id === 'gated')).toMatchObject({ requiredFlags: ['tip.received'] })
+    expect(merged.relations).toHaveLength(1)
+    expect(merged.connections).toHaveLength(1)
+  })
+})
@@ -0,0 +1,49 @@
+import type { CaseState, DocumentExhibit, ExhibitRelation, Connection } from '../src/types.js'
+
+function requirementsMet(document: DocumentExhibit, earnedFlags: ReadonlySet<string>) {
+  return (document.requiredFlags || []).every(flag => earnedFlags.has(flag))
+}
+
+export function filterLevelVisibility(full: CaseState, earnedFlags: Iterable<string>, seenDocumentIds: Iterable<string>, authorMode: boolean): CaseState {
+  if (authorMode) return { ...full, newlyVisibleDocumentIds: [] }
+  const earned = new Set(earnedFlags)
+  const seen = new Set(seenDocumentIds)
+  const visibleExhibits = full.exhibits.filter(exhibit => !exhibit.hidden && (exhibit.type !== 'document' || requirementsMet(exhibit, earned)))
+  const visibleIds = new Set(visibleExhibits.map(exhibit => exhibit.id))
+  const sanitize = (exhibit: typeof visibleExhibits[number]) => {
+    if (exhibit.type !== 'document') return exhibit
+    const { requiredFlags: _requirements, ...document } = exhibit
+    return document
+  }
+  return {
+    ...full,
+    exhibits: visibleExhibits.map(sanitize),
+    relations: full.relations.filter(relation => visibleIds.has(relation.fromExhibitId) && visibleIds.has(relation.toExhibitId)),
+    connections: full.connections.filter(connection => visibleIds.has(connection.fromExhibitId) && visibleIds.has(connection.toExhibitId)),
+    newlyVisibleDocumentIds: visibleExhibits.flatMap(exhibit => exhibit.type === 'document' && !seen.has(exhibit.id) ? [exhibit.id] : []),
+  }
+}
+
+function appendMissingById<T extends { id: string }>(submitted: T[], preserved: T[]) {
+  const ids = new Set(submitted.map(item => item.id))
+  return [...submitted, ...preserved.filter(item => !ids.has(item.id))]
+}
+
+/** Preserve server-hidden objects during the legacy whole-board PUT used by play mode. */
+export function mergePlayerStateForPersistence(full: CaseState, visible: CaseState, submitted: CaseState): CaseState {
+  const visibleIds = new Set(visible.exhibits.map(exhibit => exhibit.id))
+  const unavailableIds = new Set(full.exhibits.filter(exhibit => !visibleIds.has(exhibit.id)).map(exhibit => exhibit.id))
+  const fullDocuments = new Map(full.exhibits.flatMap(exhibit => exhibit.type === 'document' ? [[exhibit.id, exhibit] as const] : []))
+  const submittedExhibits = submitted.exhibits.map(exhibit => exhibit.type === 'document'
+    ? { ...exhibit, requiredFlags: fullDocuments.get(exhibit.id)?.requiredFlags || exhibit.requiredFlags || [] }
+    : exhibit)
+  const preservedExhibits = full.exhibits.filter(exhibit => unavailableIds.has(exhibit.id))
+  const touchesUnavailable = (item: ExhibitRelation | Connection) => unavailableIds.has(item.fromExhibitId) || unavailableIds.has(item.toExhibitId)
+  return {
+    ...submitted,
+    exhibits: appendMissingById(submittedExhibits, preservedExhibits),
+    relations: appendMissingById(submitted.relations, full.relations.filter(touchesUnavailable)),
+    connections: appendMissingById(submitted.connections, full.connections.filter(touchesUnavailable)),
+    newlyVisibleDocumentIds: [],
+  }
+}
@@ -1,4 +1,5 @@
 import path from 'node:path'
+import fs from 'node:fs/promises'
 import { fileURLToPath } from 'node:url'
 import pg from 'pg'
 import { afterAll, beforeAll, describe, expect, it } from 'vitest'
@@ -31,9 +32,10 @@ suite('PostgreSQL migrations', () => {

  it('applies every migration transactionally and is idempotent', async () => {
    const migrationsDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'migrations')
+    const migrationCount = (await fs.readdir(migrationsDir)).filter(name => /^\d+.*\.sql$/.test(name)).length
    const firstRun: string[] = []
    await runMigrations(testDatabaseUrl, migrationsDir, message => firstRun.push(message))
-    expect(firstRun.filter(message => message.startsWith('apply '))).toHaveLength(24)
+    expect(firstRun.filter(message => message.startsWith('apply '))).toHaveLength(migrationCount)

    const client = new Client({ connectionString: testDatabaseUrl })
    await client.connect()
@@ -46,10 +48,13 @@ suite('PostgreSQL migrations', () => {
      'board_views', 'timeline_views',
      'mysteries', 'npcs', 'npc_poses', 'playthroughs',
      'story_nodes', 'story_node_terminals', 'utterances',
+      'level_flags', 'document_flag_requirements', 'level_seen_documents', 'achievements',
+      'asset_text_extractions', 'evidence_match_rules', 'evidence_match_anchors',
+      'evidence_match_evaluations', 'evidence_match_anchor_evaluations',
    ]))
    expect(tableNames).not.toEqual(expect.arrayContaining(['cases', 'widgets', 'widget_relations', 'cutscenes', 'dialogue_steps', 'mystery_chapters', 'seen_dialogue']))
    const ledger = await client.query<{ count: string }>('SELECT COUNT(*)::text AS count FROM osint.schema_migrations')
-    expect(ledger.rows[0].count).toBe('24')
+    expect(ledger.rows[0].count).toBe(String(migrationCount))
    const connectionColumns = await client.query<{ column_name: string }>(`SELECT column_name FROM information_schema.columns WHERE table_schema='osint' AND table_name='exhibit_connections'`)
    expect(connectionColumns.rows.map(row => row.column_name)).toEqual(expect.arrayContaining(['label', 'tightness', 'tag_style', 'tag_position_percent', 'tag_lateral_offset']))
    const eventOccurrence = await client.query<{ is_nullable: string }>(`SELECT is_nullable FROM information_schema.columns WHERE table_schema='osint' AND table_name='event_exhibits' AND column_name='occurred_at'`)
@@ -58,7 +63,7 @@ suite('PostgreSQL migrations', () => {

    const secondRun: string[] = []
    await runMigrations(testDatabaseUrl, migrationsDir, message => secondRun.push(message))
-    expect(secondRun.filter(message => message.startsWith('skip  '))).toHaveLength(24)
+    expect(secondRun.filter(message => message.startsWith('skip  '))).toHaveLength(migrationCount)
    expect(secondRun.some(message => message.startsWith('apply '))).toBe(false)
  })
 })
@@ -47,6 +47,8 @@ export interface NarrativeRepository {
  createPlaythrough(userId: string, mysterySlug?: string): Promise<PlaythroughState | null>
  getCurrentPlaythrough(userId: string): Promise<PlaythroughState | null>
  advancePlaythrough(userId: string, playthroughId: string, terminalKey?: string): Promise<{ ok: boolean; state?: PlaythroughState; error?: string }>
+  listAchievements(playthroughId: string): Promise<string[] | null>
+  awardAchievement(playthroughId: string, flagKey: string, nodeId?: string | null): Promise<{ ok: boolean; earned: boolean; error?: string }>
  listMysteries(): Promise<MysterySummary[]>
  deleteMystery(id: string): Promise<boolean>
  uploadAsset(file: UploadedFile): Promise<AssetDto>
@@ -231,6 +233,24 @@ export function createNarrativeRepository(pool: Pool, objectStorage: ObjectStora
      return row ? stateForPlaythrough(row.id) : null
    },

+    async listAchievements(playthroughId) {
+      if (!(await pool.query('SELECT 1 FROM osint.playthroughs WHERE id=$1', [playthroughId])).rowCount) return null
+      const rows = (await pool.query<{ flag_key: string }>('SELECT flag_key FROM osint.achievements WHERE playthrough_id=$1 ORDER BY flag_key', [playthroughId])).rows
+      return rows.map(row => row.flag_key)
+    },
+
+    // Grant an achievement (idempotent). `earned` is true only on the first grant.
+    // The eventual server-side rule engine calls this same operation.
+    async awardAchievement(playthroughId, rawKey, nodeId) {
+      const key = rawKey.trim()
+      if (!/^[a-z][a-z0-9_.-]{0,63}$/.test(key)) return { ok: false, earned: false, error: 'Invalid achievement key' }
+      if (!(await pool.query('SELECT 1 FROM osint.playthroughs WHERE id=$1', [playthroughId])).rowCount) return { ok: false, earned: false, error: 'Playthrough not found' }
+      const result = await pool.query(
+        'INSERT INTO osint.achievements (playthrough_id,flag_key,awarded_by_node_id) VALUES ($1,$2,$3) ON CONFLICT (playthrough_id,flag_key) DO NOTHING',
+        [playthroughId, key, nodeId || null])
+      return { ok: true, earned: (result.rowCount || 0) > 0 }
+    },
+
    async advancePlaythrough(userId, playthroughId, terminalKey) {
      const client = await pool.connect()
      try {
@@ -0,0 +1,78 @@
+import { spawn } from 'node:child_process'
+
+export type TextExtractionResult = {
+  extractor: string
+  extractorVersion: string
+  language: string
+  status: 'succeeded' | 'unsupported' | 'failed'
+  text: string
+  error?: string
+}
+
+export interface TextExtractor {
+  provider: string
+  extract(file: { buffer: Buffer; mimetype: string }): Promise<TextExtractionResult>
+}
+
+function plainText(file: { buffer: Buffer; mimetype: string }, maximumCharacters: number): TextExtractionResult | null {
+  if (!file.mimetype.startsWith('text/')) return null
+  return { extractor: 'plain-text', extractorVersion: '1', language: 'und', status: 'succeeded', text: file.buffer.toString('utf8').slice(0, maximumCharacters) }
+}
+
+function runTesseract(command: string, buffer: Buffer, languages: string, pageSegmentationMode: string, timeoutMs: number) {
+  return new Promise<string>((resolve, reject) => {
+    const child = spawn(command, ['stdin', 'stdout', '-l', languages, '--psm', pageSegmentationMode], { stdio: ['pipe', 'pipe', 'pipe'] })
+    const stdout: Buffer[] = []
+    const stderr: Buffer[] = []
+    let settled = false
+    const timer = setTimeout(() => {
+      if (settled) return
+      settled = true
+      child.kill('SIGKILL')
+      reject(new Error(`OCR timed out after ${timeoutMs}ms`))
+    }, timeoutMs)
+    child.stdout.on('data', chunk => stdout.push(Buffer.from(chunk)))
+    child.stderr.on('data', chunk => stderr.push(Buffer.from(chunk)))
+    child.once('error', error => {
+      if (settled) return
+      settled = true
+      clearTimeout(timer)
+      reject(error)
+    })
+    child.once('close', code => {
+      if (settled) return
+      settled = true
+      clearTimeout(timer)
+      if (code === 0) resolve(Buffer.concat(stdout).toString('utf8').trim())
+      else reject(new Error(Buffer.concat(stderr).toString('utf8').trim() || `OCR exited with status ${code}`))
+    })
+    child.stdin.end(buffer)
+  })
+}
+
+export function createTextExtractorFromEnv(): TextExtractor {
+  const enabled = process.env.OCR_ENABLED !== 'false'
+  const command = process.env.OCR_COMMAND || 'tesseract'
+  const languages = process.env.OCR_LANGUAGES || 'nor+eng'
+  const extractorVersion = process.env.OCR_ENGINE_VERSION || 'tesseract-cli-5'
+  const pageSegmentationMode = process.env.OCR_PAGE_SEGMENTATION_MODE || '3'
+  const timeoutMs = Math.max(1_000, Number(process.env.OCR_TIMEOUT_MS || 20_000))
+  const maximumBytes = Math.max(1, Number(process.env.MAX_OCR_BYTES || 15 * 1024 * 1024))
+  const maximumCharacters = Math.max(1_000, Number(process.env.MAX_EXTRACTED_TEXT_CHARACTERS || 200_000))
+  return {
+    provider: enabled ? 'tesseract' : 'disabled',
+    async extract(file) {
+      const direct = plainText(file, maximumCharacters)
+      if (direct) return direct
+      if (!file.mimetype.startsWith('image/')) return { extractor: 'none', extractorVersion: '1', language: 'und', status: 'unsupported', text: '' }
+      if (!enabled) return { extractor: 'tesseract', extractorVersion, language: languages, status: 'unsupported', text: '' }
+      if (file.buffer.byteLength > maximumBytes) return { extractor: 'tesseract', extractorVersion, language: languages, status: 'failed', text: '', error: `Image exceeds the ${maximumBytes}-byte OCR limit` }
+      try {
+        const text = (await runTesseract(command, file.buffer, languages, pageSegmentationMode, timeoutMs)).slice(0, maximumCharacters)
+        return { extractor: 'tesseract', extractorVersion, language: languages, status: 'succeeded', text }
+      } catch (error) {
+        return { extractor: 'tesseract', extractorVersion, language: languages, status: 'failed', text: '', error: error instanceof Error ? error.message : 'OCR failed' }
+      }
+    },
+  }
+}