116 lines
4.1 KiB
TypeScript
116 lines
4.1 KiB
TypeScript
export type EvidenceMatchAnchor = {
|
|
id: string
|
|
phrase: string
|
|
minimumSimilarity: number
|
|
}
|
|
|
|
export type EvidenceMatchRule = {
|
|
id: string
|
|
name: string
|
|
flagKey: string
|
|
minimumAnchorMatches: number
|
|
anchors: EvidenceMatchAnchor[]
|
|
}
|
|
|
|
export type EvidenceAnchorEvaluation = {
|
|
anchorId: string
|
|
similarity: number
|
|
matched: boolean
|
|
matchedText: string
|
|
}
|
|
|
|
export type EvidenceRuleEvaluation = {
|
|
ruleId: string
|
|
flagKey: string
|
|
matched: boolean
|
|
matchedAnchorCount: number
|
|
score: number
|
|
anchors: EvidenceAnchorEvaluation[]
|
|
}
|
|
|
|
const MAX_MATCH_TEXT_CHARACTERS = 200_000
|
|
|
|
/** Normalize historical spelling characters, punctuation, line breaks, and accents without changing word order. */
|
|
export function normalizeEvidenceText(value: string) {
|
|
return value.slice(0, MAX_MATCH_TEXT_CHARACTERS)
|
|
.toLocaleLowerCase('en')
|
|
.replace(/æ/g, 'ae')
|
|
.replace(/ø/g, 'o')
|
|
.replace(/å/g, 'aa')
|
|
.replace(/½/g, ' 1 2 ')
|
|
.normalize('NFKD')
|
|
.replace(/\p{Mark}/gu, '')
|
|
.replace(/[^a-z0-9]+/g, ' ')
|
|
.trim()
|
|
.replace(/\s+/g, ' ')
|
|
}
|
|
|
|
function grams(value: string, size = 3) {
|
|
const compact = value.replace(/\s+/g, ' ')
|
|
if (compact.length <= size) return [compact]
|
|
const result: string[] = []
|
|
for (let index = 0; index <= compact.length - size; index += 1) result.push(compact.slice(index, index + size))
|
|
return result
|
|
}
|
|
|
|
function diceSimilarity(left: string, right: string) {
|
|
if (left === right) return 1
|
|
if (!left || !right) return 0
|
|
const leftGrams = grams(left)
|
|
const rightGrams = grams(right)
|
|
const rightCounts = new Map<string, number>()
|
|
for (const gram of rightGrams) rightCounts.set(gram, (rightCounts.get(gram) || 0) + 1)
|
|
let overlap = 0
|
|
for (const gram of leftGrams) {
|
|
const count = rightCounts.get(gram) || 0
|
|
if (!count) continue
|
|
overlap += 1
|
|
rightCounts.set(gram, count - 1)
|
|
}
|
|
return (2 * overlap) / (leftGrams.length + rightGrams.length)
|
|
}
|
|
|
|
export function scoreEvidenceAnchor(normalizedDocument: string, phrase: string) {
|
|
const normalizedPhrase = normalizeEvidenceText(phrase)
|
|
if (!normalizedDocument || !normalizedPhrase) return { similarity: 0, matchedText: '' }
|
|
if (normalizedDocument.includes(normalizedPhrase)) return { similarity: 1, matchedText: normalizedPhrase }
|
|
|
|
const documentTokens = normalizedDocument.split(' ')
|
|
const phraseTokens = normalizedPhrase.split(' ')
|
|
const spread = Math.max(2, Math.min(8, Math.ceil(phraseTokens.length * 0.2)))
|
|
const minimumWindow = Math.max(1, phraseTokens.length - spread)
|
|
const maximumWindow = Math.min(documentTokens.length, phraseTokens.length + spread)
|
|
let best = { similarity: 0, matchedText: '' }
|
|
|
|
for (let windowSize = minimumWindow; windowSize <= maximumWindow; windowSize += 1) {
|
|
for (let start = 0; start + windowSize <= documentTokens.length; start += 1) {
|
|
const candidate = documentTokens.slice(start, start + windowSize).join(' ')
|
|
const similarity = diceSimilarity(normalizedPhrase, candidate)
|
|
if (similarity > best.similarity) best = { similarity, matchedText: candidate }
|
|
}
|
|
}
|
|
return best
|
|
}
|
|
|
|
export function evaluateEvidenceRules(text: string, rules: EvidenceMatchRule[]): EvidenceRuleEvaluation[] {
|
|
const normalizedDocument = normalizeEvidenceText(text)
|
|
return rules.map(rule => {
|
|
const anchors = rule.anchors.map(anchor => {
|
|
const result = scoreEvidenceAnchor(normalizedDocument, anchor.phrase)
|
|
const similarity = Math.max(0, Math.min(1, result.similarity))
|
|
return { anchorId: anchor.id, similarity, matched: similarity >= anchor.minimumSimilarity, matchedText: result.matchedText }
|
|
})
|
|
const matchedAnchors = anchors.filter(anchor => anchor.matched)
|
|
const requiredScores = [...anchors].sort((left, right) => right.similarity - left.similarity).slice(0, rule.minimumAnchorMatches)
|
|
const score = requiredScores.length ? requiredScores.reduce((sum, anchor) => sum + anchor.similarity, 0) / requiredScores.length : 0
|
|
return {
|
|
ruleId: rule.id,
|
|
flagKey: rule.flagKey,
|
|
matched: matchedAnchors.length >= rule.minimumAnchorMatches,
|
|
matchedAnchorCount: matchedAnchors.length,
|
|
score,
|
|
anchors,
|
|
}
|
|
})
|
|
}
|