Intelligent eval primitives

from multivon_eval.auto import (
    auto_evaluators,
    generate_adversarial_cases,
    generate_unicode_obfuscation_cases,
    validate_adversarial_cases,
)

from multivon_eval import EvalCase
from multivon_eval.auto import auto_evaluators

# RAG case → Faithfulness + Hallucination primary, NotEmpty guardrail
rag = EvalCase(input="What's the refund window?", context="30 days from purchase.")
for rec in auto_evaluators(rag):
    print(f"[{rec.tier}] {rec.evaluator.__name__}  ({rec.confidence})")
    print(f"    {rec.rationale}")
# [primary] Faithfulness  (high)
#     input+context → RAG shape, primary faithfulness gate
# [primary] Hallucination  (high)
#     RAG → flag claims not in retrieved context
# [secondary] ContextPrecision  (high)
#     RAG → check retrieved context isn't padded with noise
# [secondary] Relevance  (high)
#     input present → check output addresses the question
# [guardrail] NotEmpty  (high)
#     trivial sanity check — catches empty/whitespace outputs

# Agent case → ToolCallAccuracy primary
agent = EvalCase(input="book the meeting", expected_tool_calls=["create_event"])
auto_evaluators(agent)
# → [ToolCallAccuracy primary] + [ToolCallNecessity secondary] + [NotEmpty guardrail]

# Ambiguous case (multiple shape signals) → confidence drops
mixed = EvalCase(input="?", context="...", expected_tool_calls=["t"], conversation=[...])
auto_evaluators(mixed, strict_mode=True)
# raises AmbiguousCaseShape — multiple primary signals; pin task_type=... explicitly

def auto_evaluators(
    case: EvalCase,
    *,
    task_type: Literal["rag","qa","agent","trajectory","conversation",
                       "multimodal","structured_output","auto"] = "auto",
    strict_mode: bool = False,
    include_pii: bool = False,
    pii_jurisdiction: str = "all",
    include_safety: bool = False,
) -> list[EvaluatorRecommendation]

@dataclass(slots=True)
class EvaluatorRecommendation:
    evaluator: type[Evaluator]
    rationale: str
    tier: Literal["primary", "secondary", "guardrail"] = "primary"
    confidence: Literal["high", "medium", "low"] = "high"

from multivon_eval.auto import generate_adversarial_cases

kb = """
AcmeCorp ships products in 3-5 business days. Refunds are available
within 30 days of purchase. Customer support: Mon-Fri 9-5 PT.
"""

cases = generate_adversarial_cases(
    seed_text=kb,
    target_failure_mode="ungrounded_claim",
    n=10,
)

for c in cases:
    print(c.input)
    print("  expected:", c.expected_output[:60])
    print("  stress_tests:", c.metadata["stress_tests"])

def generate_adversarial_cases(
    seed_text: str,
    target_failure_mode: str,
    n: int = 5,
    *,
    judge: JudgeConfig | None = None,
) -> list[EvalCase]

from multivon_eval.auto import generate_unicode_obfuscation_cases

cases = generate_unicode_obfuscation_cases(
    base_strings=["Aadhaar 1234 5678 9012", "PAN ABCDE1234F"],
    obfuscation_kinds=("homoglyph", "zero_width", "rtlo"),
)
# 6 cases — 2 base strings × 3 obfuscation kinds.
# Each tagged adversarial:unicode_obfuscation:<kind>
# A PIIEvaluator that catches the original SHOULD also catch the obfuscated form.
# Most don't.

from multivon_eval.auto import generate_adversarial_cases, validate_adversarial_cases

def weak_baseline(prompt: str) -> str:
    # Always confabulates — for use in stress-test validation
    return "Based on what I know, the answer is something."

cases = generate_adversarial_cases(kb, "ungrounded_claim", n=20)
kept, reports = validate_adversarial_cases(
    cases,
    weak_baseline,
    n_shots=3,                  # run each case 3× to dampen judge noise
    hardness_band=(0.5, 1.0),   # keep cases the baseline fails ≥ half the shots
)

print(f"{len(kept)}/{len(cases)} cases passed validation")
for r in reports:
    print(f"  case failure_rate={r.failure_rate:.2f}  in_band={r.in_hardness_band}")

def validate_adversarial_cases(
    cases: list[EvalCase],
    baseline_model: callable,
    *,
    n_shots: int = 3,
    hardness_band: tuple[float, float] = (0.5, 1.0),
    judge: JudgeConfig | None = None,
) -> tuple[list[EvalCase], list[HardnessReport]]

@dataclass(slots=True)
class HardnessReport:
    case: EvalCase
    evaluator_name: str
    n_shots: int
    scores: list[float]           # one evaluator score per shot
    baseline_outputs: list[str]   # what the baseline said each shot
    failure_rate: float           # fraction of shots where evaluator.passed was False
    in_hardness_band: bool        # whether this case was kept

    @property
    def baseline_failed(self) -> bool:        # majority-fail heuristic
        return self.failure_rate >= 0.5

    @property
    def baseline_score(self) -> float:        # mean across shots
        ...

shape = infer_product_shape(traces)                # → "rag" | "qa" | ...
heuristic_recs = auto_evaluators(case_from_shape(shape))
llm_recs       = propose_evaluators_via_llm(description, shape, traces)
final_recs     = merge_recommendations(heuristic_recs, llm_recs)

seed_cases     = generate_adversarial_cases(description, mode_for(shape), n=30)
# Optionally:
kept, _        = validate_adversarial_cases(seed_cases, baseline, n_shots=3)