50 lines
2.9 KiB
TypeScript
50 lines
2.9 KiB
TypeScript
|
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||
|
|
import { createEvidenceJudgeFromEnv, EvidenceJudgeError, validateEvidenceVerdict } from './evidenceJudge.js'
|
||
|
|
|
||
|
|
const evidence = 'Patent applicant Nils Aall Barricelli describes an improved chest of drawers with rotating compartments.'
|
||
|
|
|
||
|
|
describe('semantic evidence judge', () => {
|
||
|
|
afterEach(() => {
|
||
|
|
delete process.env.EVIDENCE_JUDGE_PROVIDER
|
||
|
|
delete process.env.EVIDENCE_JUDGE_MODEL
|
||
|
|
delete process.env.ANTHROPIC_API_KEY
|
||
|
|
delete process.env.EVIDENCE_JUDGE_MAX_CHARACTERS
|
||
|
|
vi.restoreAllMocks()
|
||
|
|
})
|
||
|
|
|
||
|
|
it('validates a supported verdict only when its quotation exists in the OCR', () => {
|
||
|
|
expect(validateEvidenceVerdict({ subject:'target',supports_claim:true,evidence_excerpt:'Nils Aall Barricelli describes an improved chest of drawers',confidence:.94 }, evidence)).toEqual({
|
||
|
|
subject:'target', supportsClaim:true, evidenceExcerpt:'Nils Aall Barricelli describes an improved chest of drawers', confidence:.94,
|
||
|
|
})
|
||
|
|
expect(() => validateEvidenceVerdict({ subject:'target',supports_claim:true,evidence_excerpt:'invented quotation',confidence:.99 }, evidence))
|
||
|
|
.toThrow(EvidenceJudgeError)
|
||
|
|
})
|
||
|
|
|
||
|
|
it('is disabled safely without explicit provider configuration', async () => {
|
||
|
|
const judge = createEvidenceJudgeFromEnv()
|
||
|
|
expect(judge.enabled).toBe(false)
|
||
|
|
await expect(judge.judge({ targetSubject:'Nils', assertion:'was an inventor', evidenceText:evidence })).rejects.toMatchObject({ code:'provider_unavailable' })
|
||
|
|
})
|
||
|
|
|
||
|
|
it('uses a constrained Anthropic tool response and truncates untrusted OCR', async () => {
|
||
|
|
process.env.EVIDENCE_JUDGE_PROVIDER = 'anthropic'
|
||
|
|
process.env.EVIDENCE_JUDGE_MODEL = 'configured-cheap-model'
|
||
|
|
process.env.ANTHROPIC_API_KEY = 'test-secret'
|
||
|
|
process.env.EVIDENCE_JUDGE_MAX_CHARACTERS = '1000'
|
||
|
|
const fetcher = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => {
|
||
|
|
const request = JSON.parse(String(init?.body))
|
||
|
|
expect(request.model).toBe('configured-cheap-model')
|
||
|
|
expect(request.tool_choice).toEqual({ type:'tool',name:'record_evidence_verdict' })
|
||
|
|
expect(String(request.messages[0].content)).not.toContain('x'.repeat(1001))
|
||
|
|
expect(new Headers(init?.headers).get('x-api-key')).toBe('test-secret')
|
||
|
|
return new Response(JSON.stringify({ content: [{ type:'tool_use',name:'record_evidence_verdict',input:{
|
||
|
|
subject:'target',supports_claim:true,evidence_excerpt:'Nils Aall Barricelli describes an improved chest of drawers',confidence:.93,
|
||
|
|
} }] }), { status:200,headers:{'content-type':'application/json'} })
|
||
|
|
})
|
||
|
|
const judge = createEvidenceJudgeFromEnv(fetcher)
|
||
|
|
const verdict = await judge.judge({ targetSubject:'Nils Aall Barricelli', relatedSubject:'his father', assertion:'was an inventor', evidenceText:`${evidence}${'x'.repeat(5000)}` })
|
||
|
|
expect(verdict).toMatchObject({ subject:'target',supportsClaim:true,confidence:.93 })
|
||
|
|
expect(fetcher).toHaveBeenCalledOnce()
|
||
|
|
})
|
||
|
|
})
|