From f455c433b763eee13c30ced5b8bdcf956ce09a6d Mon Sep 17 00:00:00 2001 From: jenstandstad Date: Sat, 22 Aug 2026 20:07:16 +0200 Subject: [PATCH] Fix case report evidence verification --- server/api.integration.test.ts | 37 +++++++++++++- server/caseReports.ts | 89 +++++++++++++++++++++++++++++----- src/App.tsx | 5 +- src/styles.css | 3 +- src/types.ts | 10 ++++ 5 files changed, 127 insertions(+), 17 deletions(-) diff --git a/server/api.integration.test.ts b/server/api.integration.test.ts index 7985271..1978d11 100644 --- a/server/api.integration.test.ts +++ b/server/api.integration.test.ts @@ -293,6 +293,18 @@ suite('normalized level persistence API', () => { const fatherJudgment = await (await adminFetch(`${baseUrl}/api/levels/${imported.playableLevel.id}/documents/${fatherDocument.id}/judge`, { method:'POST' })).json() expect(fatherJudgment).toMatchObject({ status:'succeeded',subject:'related',awardedFlags:['scene7.father_inventor_discovered'], goals:[expect.objectContaining({ key:'barricelli.inventor-proof',status:'pending' })] }) + const relatedState=await (await fetch(`${baseUrl}/api/levels/${imported.playableLevel.id}`)).json() as CaseState + const relatedClaim=relatedState.exhibits.find(exhibit => exhibit.type === 'claim')! + const relatedConnectionId=randomUUID() + relatedState.connections.push({ id:relatedConnectionId,fromExhibitId:relatedClaim.id,toExhibitId:fatherDocument.id,label:'Proof that the Barricelli family included an inventor.',tightness:65,tagStyle:'luggage',tagPosition:50,tagOffset:0 }) + expect((await fetch(`${baseUrl}/api/levels/${relatedState.id}`,{ method:'PUT',headers:{'content-type':'application/json'},body:JSON.stringify(relatedState) })).status).toBe(200) + const relatedSubmission=await (await fetch(`${baseUrl}/api/levels/${relatedState.id}/report/submissions`,{ method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({ + investigatorName:'Test Player',evidence:[{ connectionId:relatedConnectionId,documentExhibitId:fatherDocument.id,relationText:'Proof that the Barricelli family included an inventor.' }], + }) })).json() + expect(relatedSubmission).toMatchObject({ status:'evidence_insufficient',issues:expect.arrayContaining(['missing_accepted_evidence','connected_evidence_unverified']), + feedback:expect.stringContaining('related person rather than the claim subject'),claims:[expect.objectContaining({ evidence:[expect.objectContaining({ + documentExhibitId:fatherDocument.id,evidenceAccepted:false,verification:expect.objectContaining({ status:'semantic_rejected' }), + })] })] }) const targetUpload = new FormData() targetUpload.append('file', new Blob([readFileSync(path.join(fixtureDir, 'google-patents-target-ocr.txt'))], { type:'text/plain' }), 'google-patents-source.txt') @@ -317,9 +329,30 @@ suite('normalized level persistence API', () => { publishedAt:'1953-08-19',sourceCitation:'Google Patents · GB695913A',sourceUri:'https://patents.google.com/patent/GB695913A/en' }], }) }) expect(accepted.status).toBe(201) - expect(await accepted.json()).toMatchObject({ status:'accepted',investigatorName:'Test Player',issues:[],claims:[expect.objectContaining({ evidence:[expect.objectContaining({ + const acceptedBody=await accepted.json() + expect(acceptedBody).toMatchObject({ status:'accepted',investigatorName:'Test Player',issues:[] }) + expect(acceptedBody.claims.flatMap((item:{ evidence:unknown[] }) => item.evidence)).toEqual(expect.arrayContaining([expect.objectContaining({ displayNumber:targetDocument.displayNumber,evidenceAccepted:true,sourceCitation:'Google Patents · GB695913A',publishedAt:'1953-08-19T00:00:00.000Z', - })] })] }) + verification:expect.objectContaining({ status:'accepted' }), + })])) + + // A later copy of the same correct source must be accepted on its own evaluation, + // even though the first copy already owns the one-time level-flag provenance. + const repeatedUpload=new FormData() + repeatedUpload.append('file',new Blob([readFileSync(path.join(fixtureDir,'google-patents-target-ocr.txt'))],{ type:'text/plain' }),'google-patents-second-copy.txt') + const repeatedDocument=await (await fetch(`${baseUrl}/api/levels/${reportState.id}/documents`,{ method:'POST',body:repeatedUpload })).json() as DocumentExhibit & { analysis:{ matchedFlags:string[];awardedFlags:string[] } } + expect(repeatedDocument.analysis).toMatchObject({ matchedFlags:['scene7.nils_inventor_proved'],awardedFlags:[] }) + const repeatedState=await (await fetch(`${baseUrl}/api/levels/${reportState.id}`)).json() as CaseState + const repeatedConnectionId=randomUUID() + repeatedState.connections.push({ id:repeatedConnectionId,fromExhibitId:claim.id,toExhibitId:repeatedDocument.id,label:'Proof that Barricelli is named as the inventor on patent GB695913A.',tightness:65,tagStyle:'luggage',tagPosition:50,tagOffset:0 }) + expect((await fetch(`${baseUrl}/api/levels/${repeatedState.id}`,{ method:'PUT',headers:{'content-type':'application/json'},body:JSON.stringify(repeatedState) })).status).toBe(200) + const repeatedReport=await (await fetch(`${baseUrl}/api/levels/${repeatedState.id}/report/submissions`,{ method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({ + investigatorName:'Test Player',evidence:[{ connectionId:repeatedConnectionId,documentExhibitId:repeatedDocument.id,relationText:'Proof that Barricelli is named as the inventor on patent GB695913A.', + publishedAt:'1953-08-19',sourceCitation:'Google Patents · GB695913A',sourceUri:'https://patents.google.com/patent/GB695913A/en' }], + }) })).json() + expect(repeatedReport).toMatchObject({ status:'accepted',claims:[expect.objectContaining({ evidence:expect.arrayContaining([expect.objectContaining({ + documentExhibitId:repeatedDocument.id,evidenceAccepted:true,verification:expect.objectContaining({ status:'accepted' }), + })]) })] }) judgeVerdict = { subject:'target',supports_claim:true,evidence_excerpt:'Ada Example patented a pocket telescope',confidence:.96 } judgeHttpStatus = 200 }) diff --git a/server/caseReports.ts b/server/caseReports.ts index 95ee18b..cac9bc2 100644 --- a/server/caseReports.ts +++ b/server/caseReports.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto' import type { Pool, PoolClient } from 'pg' -import type { CaseReport, CaseReportEvidence, CaseReportSubmissionInput, CaseReportSubmissionStatus, SourceFileType } from '../src/types.js' +import type { CaseReport, CaseReportEvidence, CaseReportSubmissionInput, CaseReportSubmissionStatus, EvidenceVerification, SourceFileType } from '../src/types.js' type LevelRef = { id:string; board_id:string } @@ -21,6 +21,43 @@ function unfinishedRelation(value: string) { return !value.trim() || /^proof\s+that(?:\s*(?:…|\.{3}))?\s*$/iu.test(value.trim()) } +type RecognitionRow = { + evidence_accepted:boolean;extraction_status:'succeeded'|'unsupported'|'failed'|null + deterministic_evaluated:boolean;deterministic_matched:boolean|null;deterministic_score:string|null + matched_anchor_count:number|null;minimum_anchor_matches:number|null + semantic_status:'pending'|'succeeded'|'failed'|null;semantic_subject:'target'|'related'|'ambiguous'|'neither'|null + semantic_supports_claim:boolean|null;semantic_confidence:string|null;semantic_minimum_confidence:string|null +} + +function verification(row:RecognitionRow):EvidenceVerification { + const score=row.deterministic_score === null ? undefined : Number(row.deterministic_score) + const metrics={ score,matchedMarkers:row.matched_anchor_count ?? undefined,requiredMarkers:row.minimum_anchor_matches ?? undefined } + if (row.evidence_accepted) return { status:'accepted',detail:row.deterministic_matched + ? 'The extracted text matched the source fingerprint for this objective.' + : 'Semantic review found that this exhibit directly supports the target claim.',...metrics } + if (row.semantic_status === 'pending') return { status:'semantic_pending',detail:'Text was extracted, but semantic review is still pending.',...metrics } + if (row.semantic_status === 'failed') return { status:'semantic_failed',detail:'Text was extracted, but semantic review could not be completed. Retry document analysis.',...metrics } + if (row.semantic_status === 'succeeded') { + if (row.semantic_subject === 'target' && row.semantic_supports_claim) { + const confidence=Math.round(Number(row.semantic_confidence || 0) * 100) + const required=Math.round(Number(row.semantic_minimum_confidence || 0) * 100) + return { status:'semantic_rejected',detail:`Semantic review supported the claim, but confidence was ${confidence}% and this objective requires ${required}%.`,...metrics } + } + const subject = row.semantic_subject === 'related' ? 'a related person rather than the claim subject' + : row.semantic_subject === 'ambiguous' ? 'an ambiguous subject' : 'content that does not establish the target claim' + return { status:'semantic_rejected',detail:`Semantic review found ${subject}.`,...metrics } + } + if (row.deterministic_evaluated) { + const matched=row.matched_anchor_count || 0,required=row.minimum_anchor_matches || 0 + const percentage=score === undefined ? null : Math.round(score * 100) + return { status:'text_not_matched',detail:`OCR succeeded, but this exhibit matched ${matched} of ${required} required source markers${percentage === null ? '' : ` (best similarity ${percentage}%)`}.`,...metrics } + } + if (row.extraction_status === 'failed') return { status:'ocr_unavailable',detail:'The image was saved, but OCR could not read usable text from it.' } + if (row.extraction_status === 'unsupported') return { status:'ocr_unavailable',detail:'This file format could not be checked automatically.' } + if (row.extraction_status === 'succeeded') return { status:'not_evaluated',detail:'Text was extracted, but no evidence-recognition rule evaluated this exhibit.' } + return { status:'not_evaluated',detail:'This exhibit has not been analyzed for the objective.' } +} + export async function loadCaseReport(client: Pool | PoolClient, level: LevelRef): Promise { const config = (await client.query<{ title:string; investigator_name:string; required_for_completion:boolean }>( 'SELECT title,investigator_name,required_for_completion FROM osint.case_reports WHERE board_id=$1', [level.board_id])).rows[0] @@ -30,17 +67,22 @@ export async function loadCaseReport(client: Pool | PoolClient, level: LevelRef) WHERE exhibit.board_id=$1 ORDER BY exhibit.created_at,exhibit.id`, [level.board_id]) const evidence = await client.query<{ claim_exhibit_id:string; connection_id:string; document_exhibit_id:string; display_number:number; document_title:string - document_type_id:SourceFileType; relation_text:string; published_at:Date|null; citation_text:string; source_uri:string|null; evidence_accepted:boolean + document_type_id:SourceFileType; relation_text:string; published_at:Date|null; citation_text:string; source_uri:string|null + evidence_accepted:boolean;extraction_status:'succeeded'|'unsupported'|'failed'|null + deterministic_evaluated:boolean;deterministic_matched:boolean|null;deterministic_score:string|null + matched_anchor_count:number|null;minimum_anchor_matches:number|null + semantic_status:'pending'|'succeeded'|'failed'|null;semantic_subject:'target'|'related'|'ambiguous'|'neither'|null + semantic_supports_claim:boolean|null;semantic_confidence:string|null;semantic_minimum_confidence:string|null }>(`SELECT claim.exhibit_id AS claim_exhibit_id,connection.id AS connection_id,document.exhibit_id AS document_exhibit_id, citation.display_number,document.title AS document_title,document.document_type_id,COALESCE(connection.label,'') AS relation_text, document.published_at,document.citation_text,document.source_uri, - (EXISTS (SELECT 1 FROM osint.level_flags flag JOIN osint.evidence_match_evaluations evaluation - ON evaluation.id=flag.awarded_by_evidence_match_id - WHERE flag.level_id=$2 AND evaluation.document_exhibit_id=document.exhibit_id) - OR EXISTS (SELECT 1 FROM osint.level_flags flag JOIN osint.evidence_semantic_evaluations evaluation - ON evaluation.id=flag.awarded_by_semantic_evaluation_id - WHERE flag.level_id=$2 AND evaluation.document_exhibit_id=document.exhibit_id - AND evaluation.status='succeeded' AND evaluation.subject='target' AND evaluation.supports_claim)) AS evidence_accepted + (COALESCE(deterministic.matched,FALSE) OR COALESCE(semantic.status='succeeded' AND semantic.subject='target' + AND semantic.supports_claim AND semantic.confidence >= semantic.minimum_confidence,FALSE)) AS evidence_accepted, + extraction.status AS extraction_status,(deterministic.rule_id IS NOT NULL) AS deterministic_evaluated, + deterministic.matched AS deterministic_matched,deterministic.score::text AS deterministic_score, + deterministic.matched_anchor_count,deterministic.minimum_anchor_matches, + semantic.status AS semantic_status,semantic.subject AS semantic_subject,semantic.supports_claim AS semantic_supports_claim, + semantic.confidence::text AS semantic_confidence,semantic.minimum_confidence::text AS semantic_minimum_confidence FROM osint.claim_exhibits claim JOIN osint.exhibits claim_exhibit ON claim_exhibit.id=claim.exhibit_id AND claim_exhibit.board_id=$1 JOIN osint.exhibit_connections connection ON connection.board_id=$1 @@ -48,6 +90,19 @@ export async function loadCaseReport(client: Pool | PoolClient, level: LevelRef) JOIN osint.document_exhibits document ON document.exhibit_id=CASE WHEN connection.from_exhibit_id=claim.exhibit_id THEN connection.to_exhibit_id ELSE connection.from_exhibit_id END JOIN osint.exhibit_citations citation ON citation.board_id=$1 AND citation.exhibit_id=document.exhibit_id + LEFT JOIN LATERAL (SELECT evaluation.rule_id,evaluation.matched,evaluation.score,evaluation.matched_anchor_count,rule.minimum_anchor_matches + FROM osint.evidence_match_evaluations evaluation + JOIN osint.evidence_match_rules rule ON rule.id=evaluation.rule_id AND rule.board_id=$1 AND rule.enabled + WHERE evaluation.level_id=$2 AND evaluation.document_exhibit_id=document.exhibit_id + ORDER BY evaluation.matched DESC,evaluation.score DESC,evaluation.evaluated_at DESC LIMIT 1) deterministic ON TRUE + LEFT JOIN LATERAL (SELECT evaluation.status,evaluation.subject,evaluation.supports_claim,evaluation.confidence,rule.minimum_confidence + FROM osint.evidence_semantic_evaluations evaluation + JOIN osint.evidence_semantic_rules rule ON rule.id=evaluation.rule_id AND rule.board_id=$1 AND rule.enabled + WHERE evaluation.level_id=$2 AND evaluation.document_exhibit_id=document.exhibit_id + ORDER BY (evaluation.status='succeeded' AND evaluation.subject='target' AND evaluation.supports_claim + AND evaluation.confidence >= rule.minimum_confidence) DESC,evaluation.updated_at DESC LIMIT 1) semantic ON TRUE + LEFT JOIN LATERAL (SELECT candidate.status FROM osint.asset_text_extractions candidate + WHERE candidate.asset_id=document.asset_id ORDER BY candidate.updated_at DESC LIMIT 1) extraction ON TRUE ORDER BY claim_exhibit.created_at,claim.exhibit_id,citation.display_number,connection.created_at`, [level.board_id,level.id]) const latest = (await client.query<{ id:string;status:CaseReportSubmissionStatus; feedback:string }>( 'SELECT id,status,feedback FROM osint.case_report_submissions WHERE level_id=$1 ORDER BY submitted_at DESC,id DESC LIMIT 1', [level.id])).rows[0] @@ -58,7 +113,7 @@ export async function loadCaseReport(client: Pool | PoolClient, level: LevelRef) connectionId:row.connection_id,documentExhibitId:row.document_exhibit_id,displayNumber:row.display_number, documentTitle:row.document_title,fileType:row.document_type_id,relationText:row.relation_text, publishedAt:row.published_at?.toISOString(),sourceCitation:row.citation_text || undefined,sourceUri:row.source_uri || undefined, - evidenceAccepted:row.evidence_accepted, + evidenceAccepted:row.evidence_accepted,verification:verification(row), }]) return { title:config.title,investigatorName:config.investigator_name,requiredForCompletion:config.required_for_completion, status:latest?.status || 'draft',feedback:latest?.feedback,issues, @@ -111,7 +166,11 @@ export async function submitCaseReport(pool: Pool, levelSlug: string, rawInput: if (!assembled.claims.length) blockingIssues.add('missing_claim') for (const claim of assembled.claims) { const accepted = claim.evidence.filter(item => item.evidenceAccepted) - if (!accepted.length) { blockingIssues.add('missing_accepted_evidence'); continue } + if (!accepted.length) { + blockingIssues.add('missing_accepted_evidence') + blockingIssues.add(claim.evidence.length ? 'connected_evidence_unverified' : 'missing_connected_evidence') + continue + } for (const item of accepted) { if (unfinishedRelation(item.relationText)) blockingIssues.add('unfinished_relation') if (!item.publishedAt) blockingIssues.add('missing_date') @@ -122,7 +181,13 @@ export async function submitCaseReport(pool: Pool, levelSlug: string, rawInput: let feedback:string if (pendingGoals || !connectedAccepted) { status='evidence_insufficient' - feedback='The report does not yet connect the claim to evidence that proves it. Find the source, add it to the board, and connect it with red thread.' + const emptyClaim=assembled.claims.find(claim => !claim.evidence.length) + const rejected=assembled.claims.flatMap(claim => claim.evidence).find(item => !item.evidenceAccepted) + feedback=emptyClaim + ? 'No source document is connected to the claim. Return to the board and attach one with red thread.' + : rejected + ? `Exhibit ${rejected.displayNumber} is connected to the claim, but it was not accepted: ${rejected.verification.detail}` + : 'The connected evidence was recognized, but another required level objective is still incomplete.' } else if (blockingIssues.size) { status='evidence_accepted_report_incomplete' feedback=blockingIssues.has('missing_date') || blockingIssues.has('missing_source') diff --git a/src/App.tsx b/src/App.tsx index 0c76060..25a1dfc 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1119,8 +1119,9 @@ function CaseReportPanel({ report, defaultInvestigator, hasNext, onClose, onSubm

CLAIM {claimIndex + 1}{claim.statement}

EVIDENCE

{claim.evidence.length === 0 ?
No source evidence is connected to this claim. Return to the board and use red thread to attach a document.
- : claim.evidence.map(item => { const draft=draftByConnection.get(item.connectionId)!; return
-
Exhibit {item.displayNumber}{item.fileType.replaceAll('_',' ')} · {item.documentTitle}{item.evidenceAccepted && CONTENT VERIFIED}
+ : claim.evidence.map(item => { const draft=draftByConnection.get(item.connectionId)!; const rejected=report.status !== 'draft' && !item.evidenceAccepted; return
+
Exhibit {item.displayNumber}{item.fileType.replaceAll('_',' ')} · {item.documentTitle}{item.evidenceAccepted ? CONTENT VERIFIED : rejected ? NOT VERIFIED : null}
+ {rejected &&

{item.verification.detail}

}