Implement Scene 7 evidence goal flow
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
import { createServer } from 'node:net'
|
||||
import { createServer as createHttpServer, type Server as HttpServer } from 'node:http'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import pg from 'pg'
|
||||
@@ -17,6 +19,9 @@ let appServer: Awaited<typeof import('./index.js')>['server']
|
||||
let appPool: Awaited<typeof import('./index.js')>['pool']
|
||||
let baseUrl = ''
|
||||
let adminAuthorization = ''
|
||||
let judgeServer: HttpServer
|
||||
let judgeVerdict = { subject:'target',supports_claim:true,evidence_excerpt:'Ada Example patented a pocket telescope',confidence:.96 }
|
||||
let judgeHttpStatus = 200
|
||||
|
||||
function adminFetch(url: string, init: RequestInit = {}) {
|
||||
const headers = new Headers(init.headers)
|
||||
@@ -56,6 +61,17 @@ suite('normalized level persistence API', () => {
|
||||
process.env.JWT_SECRET = 'osint-integration-jwt-secret'
|
||||
process.env.ASSET_STORAGE_DRIVER = 'memory'
|
||||
process.env.PORT = String(port)
|
||||
judgeServer = createHttpServer((_req, res) => {
|
||||
res.statusCode = judgeHttpStatus
|
||||
res.setHeader('content-type', 'application/json')
|
||||
res.end(JSON.stringify({ content: [{ type:'tool_use',name:'record_evidence_verdict',input:judgeVerdict }] }))
|
||||
})
|
||||
await new Promise<void>((resolve, reject) => judgeServer.listen(0, '127.0.0.1', resolve).once('error', reject))
|
||||
const judgeAddress = judgeServer.address()
|
||||
process.env.EVIDENCE_JUDGE_PROVIDER = 'anthropic'
|
||||
process.env.EVIDENCE_JUDGE_MODEL = 'integration-haiku'
|
||||
process.env.ANTHROPIC_API_KEY = 'integration-key'
|
||||
process.env.ANTHROPIC_API_URL = `http://127.0.0.1:${typeof judgeAddress === 'object' && judgeAddress ? judgeAddress.port : 0}`
|
||||
const serverModule = await import('./index.js')
|
||||
appServer = serverModule.server
|
||||
appPool = serverModule.pool
|
||||
@@ -65,6 +81,7 @@ suite('normalized level persistence API', () => {
|
||||
|
||||
afterAll(async () => {
|
||||
if (appServer) await new Promise<void>((resolve, reject) => appServer.close(error => error ? reject(error) : resolve()))
|
||||
if (judgeServer) await new Promise<void>((resolve, reject) => judgeServer.close(error => error ? reject(error) : resolve()))
|
||||
if (appPool) await appPool.end()
|
||||
if (!adminClient) return
|
||||
await adminClient.query(`DROP DATABASE IF EXISTS "${databaseName}"`)
|
||||
@@ -125,13 +142,40 @@ suite('normalized level persistence API', () => {
|
||||
expect((await adminFetch(`${baseUrl}/api/levels/${state.id}/flags/tip.received`, { method: 'DELETE' })).status).toBe(200)
|
||||
const matchRuleResponse = await adminFetch(`${baseUrl}/api/levels/${state.id}/evidence-match-rules`, {
|
||||
method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({
|
||||
name: 'Smoke source passage', flagKey: 'tip.received', minimumAnchorMatches: 1,
|
||||
name: 'Smoke source passage', sourceLabel: 'Archive smoke test', sourceUri: 'https://example.test/archive/smoke',
|
||||
flagKey: 'tip.received', minimumAnchorMatches: 1,
|
||||
anchors: [{ phrase: 'OSINT smoke evidence from the archive', minimumSimilarity: 0.72 }],
|
||||
}),
|
||||
})
|
||||
expect(matchRuleResponse.status).toBe(201)
|
||||
expect(await matchRuleResponse.json()).toMatchObject({ name: 'Smoke source passage', flagKey: 'tip.received', anchors: [{ phrase: 'OSINT smoke evidence from the archive' }] })
|
||||
expect(await matchRuleResponse.json()).toMatchObject({ name: 'Smoke source passage', sourceLabel: 'Archive smoke test',
|
||||
sourceUri: 'https://example.test/archive/smoke', flagKey: 'tip.received', anchors: [{ phrase: 'OSINT smoke evidence from the archive' }] })
|
||||
expect(await (await adminFetch(`${baseUrl}/api/levels/${state.id}/evidence-match-rules`)).json()).toHaveLength(1)
|
||||
const goalResponse = await adminFetch(`${baseUrl}/api/levels/${state.id}/goals`, {
|
||||
method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({
|
||||
key: 'smoke.prove-source', title: 'Prove the source', instructions: 'Paste a matching archival source.',
|
||||
completionMessage: 'Source verified.', requiredFlags: ['tip.received'],
|
||||
}),
|
||||
})
|
||||
expect(goalResponse.status).toBe(201)
|
||||
expect(await goalResponse.json()).toMatchObject({ key: 'smoke.prove-source', status: 'pending', requiredFlags: ['tip.received'] })
|
||||
const playGoalBefore = (await (await fetch(`${baseUrl}/api/levels/${state.id}`)).json() as CaseState).goals[0]
|
||||
expect(playGoalBefore).toMatchObject({ key: 'smoke.prove-source', status: 'pending', newlyCompleted: false })
|
||||
expect(playGoalBefore).not.toHaveProperty('id')
|
||||
expect(playGoalBefore).not.toHaveProperty('requiredFlags')
|
||||
|
||||
const semanticGoalResponse = await adminFetch(`${baseUrl}/api/levels/${state.id}/goals`, {
|
||||
method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({ key:'smoke.semantic-proof',title:'Prove the semantic claim',
|
||||
instructions:'Upload another credible source.',completionMessage:'Claim verified.',requiredFlags:['semantic.proved'] }),
|
||||
})
|
||||
expect(semanticGoalResponse.status).toBe(201)
|
||||
const semanticGoal = await semanticGoalResponse.json() as { id:string }
|
||||
const semanticRuleResponse = await adminFetch(`${baseUrl}/api/levels/${state.id}/evidence-semantic-rules`, {
|
||||
method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({ goalId:semanticGoal.id,name:'Ada inventor claim',
|
||||
targetSubject:'Ada Example',relatedSubject:'Ada Example Senior',assertion:'Ada Example was an inventor',successFlagKey:'semantic.proved',
|
||||
relatedFlagKey:'semantic.father',minimumConfidence:.85 }),
|
||||
})
|
||||
expect(semanticRuleResponse.status).toBe(201)
|
||||
|
||||
const upload = new FormData()
|
||||
upload.append('file', new Blob(['OSINT smoke evidence from the archlve'], { type: 'text/plain' }), 'smoke-evidence.txt')
|
||||
@@ -139,18 +183,39 @@ suite('normalized level persistence API', () => {
|
||||
upload.append('y', '438')
|
||||
const uploadResponse = await fetch(`${baseUrl}/api/levels/${state.id}/documents`, { method: 'POST', body: upload })
|
||||
expect(uploadResponse.status).toBe(201)
|
||||
const uploaded = await uploadResponse.json() as DocumentExhibit & { analysis: { extractionStatus: string; matchedFlags: string[]; awardedFlags: string[] } }
|
||||
const uploaded = await uploadResponse.json() as DocumentExhibit & { analysis: { extractionStatus: string; matchedFlags: string[]; awardedFlags: string[]; goals: CaseState['goals'] } }
|
||||
expect(uploaded).toMatchObject({ type: 'document', fileName: 'smoke-evidence.txt', fileType: 'text', x: 812, y: 438,
|
||||
body: ['OSINT smoke evidence from the archlve'], analysis: { extractionStatus: 'succeeded', matchedFlags: ['tip.received'], awardedFlags: ['tip.received'] } })
|
||||
body: ['OSINT smoke evidence from the archlve'], analysis: { extractionStatus: 'succeeded', matchedFlags: ['tip.received'], awardedFlags: ['tip.received'],
|
||||
goals: expect.arrayContaining([expect.objectContaining({ key: 'smoke.prove-source', status: 'complete', newlyCompleted: true })]) } })
|
||||
expect(await (await fetch(`${baseUrl}/api/assets/${uploaded.assetId}`)).text()).toBe('OSINT smoke evidence from the archlve')
|
||||
const assetRow = await appPool.query<{ storage_provider: string; content: Buffer | null; object_key: string | null }>('SELECT storage_provider,content,object_key FROM osint.assets WHERE id=$1', [uploaded.assetId])
|
||||
expect(assetRow.rows[0]).toMatchObject({ storage_provider: 's3', content: null, object_key: expect.stringMatching(/^assets\//) })
|
||||
const automaticallyRevealed = await (await fetch(`${baseUrl}/api/levels/${state.id}`)).json() as CaseState
|
||||
expect(automaticallyRevealed.exhibits.map(item => item.id)).toContain(gatedDocument.id)
|
||||
expect(automaticallyRevealed.goals).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ key: 'smoke.prove-source', status: 'complete', newlyCompleted: false }),
|
||||
]))
|
||||
const evaluationRows = await appPool.query<{ matched: boolean; matched_anchor_count: number }>(
|
||||
'SELECT matched,matched_anchor_count FROM osint.evidence_match_evaluations WHERE document_exhibit_id=$1', [uploaded.id])
|
||||
expect(evaluationRows.rows).toEqual([{ matched: true, matched_anchor_count: 1 }])
|
||||
|
||||
const semanticUpload = new FormData()
|
||||
semanticUpload.append('file', new Blob(['Archive entry: Ada Example patented a pocket telescope in 1948.'], { type:'text/plain' }), 'semantic-evidence.txt')
|
||||
const semanticDocumentResponse = await fetch(`${baseUrl}/api/levels/${state.id}/documents`, { method:'POST',body:semanticUpload })
|
||||
expect(semanticDocumentResponse.status).toBe(201)
|
||||
const semanticDocument = await semanticDocumentResponse.json() as DocumentExhibit & { analysis:{ goals:CaseState['goals'] } }
|
||||
expect(semanticDocument.analysis.goals.find(goal => goal.key === 'smoke.semantic-proof')?.status).toBe('pending')
|
||||
const judgedResponse = await adminFetch(`${baseUrl}/api/levels/${state.id}/documents/${semanticDocument.id}/judge`, { method:'POST' })
|
||||
expect(judgedResponse.status).toBe(200)
|
||||
expect(await judgedResponse.json()).toMatchObject({ status:'succeeded',subject:'target',supportsClaim:true,confidence:.96,
|
||||
awardedFlags:['semantic.proved'],goals:expect.arrayContaining([expect.objectContaining({ key:'smoke.semantic-proof',status:'complete',newlyCompleted:true })]) })
|
||||
const semanticProvenance = await appPool.query<{ flag_key:string; awarded_by_semantic_evaluation_id:string | null }>(
|
||||
'SELECT flag_key,awarded_by_semantic_evaluation_id FROM osint.level_flags WHERE level_id=(SELECT id FROM osint.levels WHERE slug=$1) AND flag_key=$2',
|
||||
[state.id,'semantic.proved'])
|
||||
expect(semanticProvenance.rows[0]).toMatchObject({ flag_key:'semantic.proved',awarded_by_semantic_evaluation_id:expect.any(String) })
|
||||
expect(await (await adminFetch(`${baseUrl}/api/levels/${state.id}/documents/${semanticDocument.id}/judge`, { method:'POST' })).json())
|
||||
.toMatchObject({ status:'not_needed',awardedFlags:[] })
|
||||
|
||||
const screenshot = new FormData()
|
||||
screenshot.append('file', new Blob([Buffer.from('89504e470d0a1a0a', 'hex')], { type: 'image/png' }), 'Screenshot 2026-08-22.png')
|
||||
const screenshotResponse = await fetch(`${baseUrl}/api/levels/${state.id}/documents`, { method: 'POST', body: screenshot })
|
||||
@@ -171,7 +236,55 @@ suite('normalized level persistence API', () => {
|
||||
const authoredClone = await (await adminFetch(`${baseUrl}/api/levels/${clone.id}?edit=1`)).json() as CaseState
|
||||
expect(authoredClone.exhibits.find(item => item.type === 'document' && item.title === 'Later tip')).toMatchObject({ requiredFlags: ['tip.received'] })
|
||||
expect(await (await adminFetch(`${baseUrl}/api/levels/${clone.id}/evidence-match-rules`)).json()).toEqual([
|
||||
expect.objectContaining({ name: 'Smoke source passage', flagKey: 'tip.received', anchors: [expect.objectContaining({ phrase: 'OSINT smoke evidence from the archive' })] }),
|
||||
expect.objectContaining({ name: 'Smoke source passage', sourceLabel: 'Archive smoke test', sourceUri: 'https://example.test/archive/smoke',
|
||||
flagKey: 'tip.received', anchors: [expect.objectContaining({ phrase: 'OSINT smoke evidence from the archive' })] }),
|
||||
])
|
||||
expect(await (await adminFetch(`${baseUrl}/api/levels/${clone.id}/goals`)).json()).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ key: 'smoke.prove-source', status: 'pending', requiredFlags: ['tip.received'] }),
|
||||
expect.objectContaining({ key: 'smoke.semantic-proof', status: 'pending', requiredFlags: ['semantic.proved'] }),
|
||||
]))
|
||||
expect(await (await adminFetch(`${baseUrl}/api/levels/${clone.id}/evidence-semantic-rules`)).json()).toEqual([
|
||||
expect.objectContaining({ name:'Ada inventor claim',goalKey:'smoke.semantic-proof',successFlagKey:'semantic.proved' }),
|
||||
])
|
||||
})
|
||||
|
||||
it('imports the data-defined Scene 7 template with its private recognition rules', async () => {
|
||||
const { importMysteryTemplate } = await import('../scripts/importMysteryTemplate.js')
|
||||
const manifestPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'mysteries', 'barricelli-scene-7', 'mystery.json')
|
||||
const imported = await importMysteryTemplate(manifestPath, baseUrl, adminAuthorization.replace(/^Bearer /, ''))
|
||||
expect(imported.playableLevel).toMatchObject({ title:'The Barricelli Files',exhibits:[],goals:[expect.objectContaining({
|
||||
key:'barricelli.inventor-proof',status:'pending',newlyCompleted:false,
|
||||
})] })
|
||||
expect(await (await adminFetch(`${baseUrl}/api/levels/${imported.playableLevel.id}/evidence-match-rules`)).json()).toEqual([
|
||||
expect.objectContaining({ name:'Google Patents · GB695913A',sourceUri:'https://patents.google.com/patent/GB695913A/en',
|
||||
flagKey:'scene7.nils_inventor_proved',minimumAnchorMatches:2 }),
|
||||
])
|
||||
expect(await (await adminFetch(`${baseUrl}/api/levels/${imported.playableLevel.id}/evidence-semantic-rules`)).json()).toEqual([
|
||||
expect.objectContaining({ goalKey:'barricelli.inventor-proof',targetSubject:'Nils Aall Barricelli',
|
||||
relatedFlagKey:'scene7.father_inventor_discovered' }),
|
||||
])
|
||||
|
||||
const fixtureDir = path.join(path.dirname(manifestPath), 'fixtures')
|
||||
const fatherUpload = new FormData()
|
||||
fatherUpload.append('file', new Blob([readFileSync(path.join(fixtureDir, 'father-only-negative-ocr.txt'))], { type:'text/plain' }), 'father-source.txt')
|
||||
const fatherDocument = await (await fetch(`${baseUrl}/api/levels/${imported.playableLevel.id}/documents`, { method:'POST',body:fatherUpload })).json() as DocumentExhibit & { analysis:{ goals:CaseState['goals'] } }
|
||||
expect(fatherDocument.analysis.goals[0].status).toBe('pending')
|
||||
judgeVerdict = { subject:'related',supports_claim:true,evidence_excerpt:'den italienske maler og opfinder Barricelli',confidence:.96 }
|
||||
judgeHttpStatus = 429
|
||||
expect(await (await adminFetch(`${baseUrl}/api/levels/${imported.playableLevel.id}/documents/${fatherDocument.id}/judge`, { method:'POST' })).json())
|
||||
.toMatchObject({ status:'failed',retryable:true,awardedFlags:[],goals:[expect.objectContaining({ status:'pending' })] })
|
||||
judgeHttpStatus = 200
|
||||
const fatherJudgment = await (await adminFetch(`${baseUrl}/api/levels/${imported.playableLevel.id}/documents/${fatherDocument.id}/judge`, { method:'POST' })).json()
|
||||
expect(fatherJudgment).toMatchObject({ status:'succeeded',subject:'related',awardedFlags:['scene7.father_inventor_discovered'],
|
||||
goals:[expect.objectContaining({ key:'barricelli.inventor-proof',status:'pending' })] })
|
||||
|
||||
const targetUpload = new FormData()
|
||||
targetUpload.append('file', new Blob([readFileSync(path.join(fixtureDir, 'google-patents-target-ocr.txt'))], { type:'text/plain' }), 'google-patents-source.txt')
|
||||
const targetResponse = await fetch(`${baseUrl}/api/levels/${imported.playableLevel.id}/documents`, { method:'POST',body:targetUpload })
|
||||
expect(targetResponse.status).toBe(201)
|
||||
expect(await targetResponse.json()).toMatchObject({ analysis:{ awardedFlags:['scene7.nils_inventor_proved'],
|
||||
goals:[expect.objectContaining({ key:'barricelli.inventor-proof',status:'complete',newlyCompleted:true })] } })
|
||||
judgeVerdict = { subject:'target',supports_claim:true,evidence_excerpt:'Ada Example patented a pocket telescope',confidence:.96 }
|
||||
judgeHttpStatus = 200
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user