export type EvidenceMatchAnchor = { id: string phrase: string minimumSimilarity: number } export type EvidenceMatchRule = { id: string name: string flagKey: string minimumAnchorMatches: number anchors: EvidenceMatchAnchor[] } export type EvidenceAnchorEvaluation = { anchorId: string similarity: number matched: boolean matchedText: string } export type EvidenceRuleEvaluation = { ruleId: string flagKey: string matched: boolean matchedAnchorCount: number score: number anchors: EvidenceAnchorEvaluation[] } const MAX_MATCH_TEXT_CHARACTERS = 200_000 /** Normalize historical spelling characters, punctuation, line breaks, and accents without changing word order. */ export function normalizeEvidenceText(value: string) { return value.slice(0, MAX_MATCH_TEXT_CHARACTERS) .toLocaleLowerCase('en') .replace(/æ/g, 'ae') .replace(/ø/g, 'o') .replace(/å/g, 'aa') .replace(/½/g, ' 1 2 ') .normalize('NFKD') .replace(/\p{Mark}/gu, '') .replace(/[^a-z0-9]+/g, ' ') .trim() .replace(/\s+/g, ' ') } function grams(value: string, size = 3) { const compact = value.replace(/\s+/g, ' ') if (compact.length <= size) return [compact] const result: string[] = [] for (let index = 0; index <= compact.length - size; index += 1) result.push(compact.slice(index, index + size)) return result } function diceSimilarity(left: string, right: string) { if (left === right) return 1 if (!left || !right) return 0 const leftGrams = grams(left) const rightGrams = grams(right) const rightCounts = new Map() for (const gram of rightGrams) rightCounts.set(gram, (rightCounts.get(gram) || 0) + 1) let overlap = 0 for (const gram of leftGrams) { const count = rightCounts.get(gram) || 0 if (!count) continue overlap += 1 rightCounts.set(gram, count - 1) } return (2 * overlap) / (leftGrams.length + rightGrams.length) } export function scoreEvidenceAnchor(normalizedDocument: string, phrase: string) { const normalizedPhrase = normalizeEvidenceText(phrase) if (!normalizedDocument || !normalizedPhrase) return { similarity: 0, matchedText: '' } if (normalizedDocument.includes(normalizedPhrase)) return { similarity: 1, matchedText: normalizedPhrase } const documentTokens = normalizedDocument.split(' ') const phraseTokens = normalizedPhrase.split(' ') const spread = Math.max(2, Math.min(8, Math.ceil(phraseTokens.length * 0.2))) const minimumWindow = Math.max(1, phraseTokens.length - spread) const maximumWindow = Math.min(documentTokens.length, phraseTokens.length + spread) let best = { similarity: 0, matchedText: '' } for (let windowSize = minimumWindow; windowSize <= maximumWindow; windowSize += 1) { for (let start = 0; start + windowSize <= documentTokens.length; start += 1) { const candidate = documentTokens.slice(start, start + windowSize).join(' ') const similarity = diceSimilarity(normalizedPhrase, candidate) if (similarity > best.similarity) best = { similarity, matchedText: candidate } } } return best } export function evaluateEvidenceRules(text: string, rules: EvidenceMatchRule[]): EvidenceRuleEvaluation[] { const normalizedDocument = normalizeEvidenceText(text) return rules.map(rule => { const anchors = rule.anchors.map(anchor => { const result = scoreEvidenceAnchor(normalizedDocument, anchor.phrase) const similarity = Math.max(0, Math.min(1, result.similarity)) return { anchorId: anchor.id, similarity, matched: similarity >= anchor.minimumSimilarity, matchedText: result.matchedText } }) const matchedAnchors = anchors.filter(anchor => anchor.matched) const requiredScores = [...anchors].sort((left, right) => right.similarity - left.similarity).slice(0, rule.minimumAnchorMatches) const score = requiredScores.length ? requiredScores.reduce((sum, anchor) => sum + anchor.similarity, 0) / requiredScores.length : 0 return { ruleId: rule.id, flagKey: rule.flagKey, matched: matchedAnchors.length >= rule.minimumAnchorMatches, matchedAnchorCount: matchedAnchors.length, score, anchors, } }) }