"""
Tokeniser — compound token rule + punctuation stripping.

Rules:
1. Split on whitespace.
2. Consecutive tokens each starting with a capital letter are merged into a
   single underscore-joined lowercase token (compound token rule).
   A lowercase or non-alpha token breaks the run.
3. Strip leading/trailing punctuation from each token.
4. Lowercase all tokens.
5. Discard tokens shorter than 5 characters (unless they surfaced as part of
   a matched relationship cue — cue scanner handles that separately).
"""
from __future__ import annotations

import re
import unicodedata

# Characters that are stripped from token edges
_PUNCT_RE = re.compile(r"^[^\w]+|[^\w]+$")


def _strip_punct(s: str) -> str:
    return _PUNCT_RE.sub("", s)


def tokenize(text: str) -> list[str]:
    """
    Return a deduplicated, ordered list of canonical tokens extracted from
    *text*, applying the compound token rule.
    """
    raw_words = text.split()
    tokens: list[str] = []
    run: list[str] = []

    def flush_run() -> None:
        if not run:
            return
        if len(run) > 1:
            tokens.append("_".join(w.lower() for w in run))
        else:
            tokens.append(run[0].lower())
        run.clear()

    for word in raw_words:
        stripped = _strip_punct(word)
        # Trailing punctuation (sentence boundary) ends a compound run.
        # Commas/colons/periods after a word mean the next word starts fresh.
        has_trailing_punct = bool(word) and not word[-1].isalnum() and word[-1] != "_"

        if not stripped:
            flush_run()
            continue

        # A word starts a capital run if its first alphabetic character is uppercase
        first_alpha = next((c for c in stripped if c.isalpha()), None)
        if first_alpha and first_alpha.isupper():
            run.append(stripped)
            if has_trailing_punct:
                flush_run()  # e.g. "FastAPI." ends the run immediately
        else:
            flush_run()
            t = stripped.lower()
            if t:
                tokens.append(t)

    flush_run()

    # Filter: keep tokens >= 5 chars, deduplicate preserving order
    seen: set[str] = set()
    result: list[str] = []
    for t in tokens:
        if len(t) >= 5 and t not in seen:
            seen.add(t)
            result.append(t)

    return result


def tokenize_all(text: str) -> list[str]:
    """
    Like tokenize() but returns ALL tokens (including short ones).
    Used by the cue scanner which needs short words for pattern matching.
    """
    raw_words = text.split()
    tokens: list[str] = []
    run: list[str] = []

    def flush_run() -> None:
        if not run:
            return
        if len(run) > 1:
            tokens.append("_".join(w.lower() for w in run))
        else:
            tokens.append(run[0].lower())
        run.clear()

    for word in raw_words:
        stripped = _strip_punct(word)
        has_trailing_punct = bool(word) and not word[-1].isalnum() and word[-1] != "_"

        if not stripped:
            flush_run()
            continue
        first_alpha = next((c for c in stripped if c.isalpha()), None)
        if first_alpha and first_alpha.isupper():
            run.append(stripped)
            if has_trailing_punct:
                flush_run()
        else:
            flush_run()
            t = stripped.lower()
            if t:
                tokens.append(t)

    flush_run()
    return tokens
