Adding migrations and lot of work. Starting work on the demo scope

server/evidenceMatching.ts

@@ -0,0 +1,115 @@
+export type EvidenceMatchAnchor = {
+  id: string
+  phrase: string
+  minimumSimilarity: number
+}
+
+export type EvidenceMatchRule = {
+  id: string
+  name: string
+  flagKey: string
+  minimumAnchorMatches: number
+  anchors: EvidenceMatchAnchor[]
+}
+
+export type EvidenceAnchorEvaluation = {
+  anchorId: string
+  similarity: number
+  matched: boolean
+  matchedText: string
+}
+
+export type EvidenceRuleEvaluation = {
+  ruleId: string
+  flagKey: string
+  matched: boolean
+  matchedAnchorCount: number
+  score: number
+  anchors: EvidenceAnchorEvaluation[]
+}
+
+const MAX_MATCH_TEXT_CHARACTERS = 200_000
+
+/** Normalize historical spelling characters, punctuation, line breaks, and accents without changing word order. */
+export function normalizeEvidenceText(value: string) {
+  return value.slice(0, MAX_MATCH_TEXT_CHARACTERS)
+    .toLocaleLowerCase('en')
+    .replace(/æ/g, 'ae')
+    .replace(/ø/g, 'o')
+    .replace(/å/g, 'aa')
+    .replace(/½/g, ' 1 2 ')
+    .normalize('NFKD')
+    .replace(/\p{Mark}/gu, '')
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()
+    .replace(/\s+/g, ' ')
+}
+
+function grams(value: string, size = 3) {
+  const compact = value.replace(/\s+/g, ' ')
+  if (compact.length <= size) return [compact]
+  const result: string[] = []
+  for (let index = 0; index <= compact.length - size; index += 1) result.push(compact.slice(index, index + size))
+  return result
+}
+
+function diceSimilarity(left: string, right: string) {
+  if (left === right) return 1
+  if (!left || !right) return 0
+  const leftGrams = grams(left)
+  const rightGrams = grams(right)
+  const rightCounts = new Map<string, number>()
+  for (const gram of rightGrams) rightCounts.set(gram, (rightCounts.get(gram) || 0) + 1)
+  let overlap = 0
+  for (const gram of leftGrams) {
+    const count = rightCounts.get(gram) || 0
+    if (!count) continue
+    overlap += 1
+    rightCounts.set(gram, count - 1)
+  }
+  return (2 * overlap) / (leftGrams.length + rightGrams.length)
+}
+
+export function scoreEvidenceAnchor(normalizedDocument: string, phrase: string) {
+  const normalizedPhrase = normalizeEvidenceText(phrase)
+  if (!normalizedDocument || !normalizedPhrase) return { similarity: 0, matchedText: '' }
+  if (normalizedDocument.includes(normalizedPhrase)) return { similarity: 1, matchedText: normalizedPhrase }
+
+  const documentTokens = normalizedDocument.split(' ')
+  const phraseTokens = normalizedPhrase.split(' ')
+  const spread = Math.max(2, Math.min(8, Math.ceil(phraseTokens.length * 0.2)))
+  const minimumWindow = Math.max(1, phraseTokens.length - spread)
+  const maximumWindow = Math.min(documentTokens.length, phraseTokens.length + spread)
+  let best = { similarity: 0, matchedText: '' }
+
+  for (let windowSize = minimumWindow; windowSize <= maximumWindow; windowSize += 1) {
+    for (let start = 0; start + windowSize <= documentTokens.length; start += 1) {
+      const candidate = documentTokens.slice(start, start + windowSize).join(' ')
+      const similarity = diceSimilarity(normalizedPhrase, candidate)
+      if (similarity > best.similarity) best = { similarity, matchedText: candidate }
+    }
+  }
+  return best
+}
+
+export function evaluateEvidenceRules(text: string, rules: EvidenceMatchRule[]): EvidenceRuleEvaluation[] {
+  const normalizedDocument = normalizeEvidenceText(text)
+  return rules.map(rule => {
+    const anchors = rule.anchors.map(anchor => {
+      const result = scoreEvidenceAnchor(normalizedDocument, anchor.phrase)
+      const similarity = Math.max(0, Math.min(1, result.similarity))
+      return { anchorId: anchor.id, similarity, matched: similarity >= anchor.minimumSimilarity, matchedText: result.matchedText }
+    })
+    const matchedAnchors = anchors.filter(anchor => anchor.matched)
+    const requiredScores = [...anchors].sort((left, right) => right.similarity - left.similarity).slice(0, rule.minimumAnchorMatches)
+    const score = requiredScores.length ? requiredScores.reduce((sum, anchor) => sum + anchor.similarity, 0) / requiredScores.length : 0
+    return {
+      ruleId: rule.id,
+      flagKey: rule.flagKey,
+      matched: matchedAnchors.length >= rule.minimumAnchorMatches,
+      matchedAnchorCount: matchedAnchors.length,
+      score,
+      anchors,
+    }
+  })
+}