Custom Evaluators

from multivon_eval.evaluators.base import Evaluator
from multivon_eval.case import EvalCase
from multivon_eval.result import EvalResult

class MyEvaluator(Evaluator):
    name = "my_evaluator"   # shown in reports

    def evaluate(self, case: EvalCase, output: str) -> EvalResult:
        # your logic here
        score = 1.0  # between 0.0 and 1.0
        reason = "Passed because..."
        return self._result(score, reason)

class SentenceCount(Evaluator):
    name = "sentence_count"

    def __init__(self, min_sentences: int = 1, max_sentences: int = 10, threshold: float = 1.0):
        super().__init__(threshold)
        self.min_sentences = min_sentences
        self.max_sentences = max_sentences

    def evaluate(self, case: EvalCase, output: str) -> EvalResult:
        count = output.count(".") + output.count("!") + output.count("?")
        in_range = self.min_sentences <= count <= self.max_sentences
        return self._result(
            1.0 if in_range else 0.0,
            f"{count} sentences (expected {self.min_sentences}-{self.max_sentences})",
        )

from multivon_eval import declare_dependencies

graded_sentences = declare_dependencies(
    SentenceCount(), version="sentence-count/v1", dependencies={},
)

from multivon_eval import CheckEvaluator

professional_tone = CheckEvaluator(
    "The response should be professional and respectful",
    questions=["Is the response professional and respectful?"],
    name="professional_tone",
    threshold=1.0,
)
suite.add_evaluators(professional_tone)

@dataclass
class EvalResult:
    evaluator: str      # evaluator name
    score: float        # 0.0 to 1.0
    passed: bool        # score >= threshold
    reason: str         # human-readable explanation
    metadata: dict      # includes skipped=True for unmeasured results

def evaluate(self, case: EvalCase, output: str) -> EvalResult:
    case.input           # the original prompt
    case.expected_output # ground truth (may be None)
    case.context         # retrieved context (may be None)
    case.agent_trace     # list[AgentStep] (may be None)
    case.conversation    # list[dict] (may be None)
    case.tags            # list[str]
    case.metadata        # dict