Versioned cases and trial evidence

from multivon_eval import CaseManifest, EvalCase, EvalSuite, ExactMatch

dataset = CaseManifest("invoice decisions", [
    EvalCase("May this invoice be posted?", "post", context="Approved invoice",
             case_id="invoice-a", revision="1", source_id="document-a"),
    EvalCase("May this invoice be posted?", "hold", context="Missing approval",
             case_id="invoice-b", revision="1", source_id="document-b"),
], splits={"development": ["invoice-a"], "test": ["invoice-b"]})
dataset.save("invoices.manifest.json")
assert CaseManifest.load("invoices.manifest.json").digest == dataset.digest

suite = EvalSuite("invoice decision strings")
suite.add_cases(dataset.split("test")).add_evaluator(ExactMatch())
report = suite.run(lambda prompt: "hold", runs=3, verbose=False)
report.save_json("invoice-results.json")
assert len(report.case_results[0].trials) == 3

from datasets import Dataset, DatasetDict
from multivon_eval import EvalCase
from multivon_eval.integrations.huggingface import from_huggingface

upstream = DatasetDict({"test": Dataset.from_list([
    {"id": "doc-1", "question": "Amount?", "answer": "42.00", "source": "invoice-1"},
])})
manifest = from_huggingface(
    upstream, name="invoice amount fixture",
    record_to_case=lambda row: EvalCase(
        row["question"], row["answer"], case_id=row["id"], source_id=row["source"]),
    provenance={"kind": "local synthetic fixture"},
)
assert manifest.split("test")[0].expected_output == "42.00"

from multivon_eval import declare_dependencies
from multivon_eval.evaluators.base import Evaluator

class HoldDecision(Evaluator):
    name = "hold_decision"
    def evaluate(self, case, output):
        return self._result(float(output.strip() == "hold"), "Requires a hold decision")

# This self-contained grader has no external dependencies.
# The version identifies its complete implementation.
grader = declare_dependencies(HoldDecision(), version="hold-decision/v1", dependencies={})
contract_suite = EvalSuite("declared grader").add_evaluator(grader)
contract_suite.verify_lock(contract_suite.lock())

from pathlib import Path
from multivon_eval import declare_target

policy_file = Path("target-policy.json")
policy_file.write_text('{"review_above": 1000}')

class InvoicePolicyTarget:
    def __call__(self, prompt):
        raise ValueError("This target requires the complete EvalCase")
    def _call_with_case(self, case):
        import json
        policy = json.loads(policy_file.read_text())
        return "review" if case.metadata["amount"] > policy["review_above"] else "post"

invoice_target = declare_target(
    InvoicePolicyTarget(), version="policy-loader/v1", dependencies={},
    configuration={"currency": "USD"}, files={"policy": policy_file},
)
target_suite = EvalSuite("fixed invoice decisions").add_cases([
    EvalCase("invoice A", "post", metadata={"amount": 200}, case_id="invoice-a"),
    EvalCase("invoice B", "review", metadata={"amount": 2000}, case_id="invoice-b"),
]).add_evaluator(ExactMatch())
target_baseline = target_suite.run(invoice_target, verbose=False)

# An implementation candidate changes behavior; the task references stay fixed.
policy_file.write_text('{"review_above": 100}')
target_candidate = target_suite.run(invoice_target, verbose=False)
target_diff = target_baseline.compare(target_candidate)
assert not target_diff.identity_issues
assert "declaration.files.policy" in target_diff.execution_changes["target_paths"]
assert target_baseline.passed == 2 and target_candidate.passed == 1

import json
from multivon_eval import EvalReport, EvalSuite, NotEmpty, regrade

saved = EvalReport.from_dict(json.loads(open("invoice-results.json").read()))
trial = saved.case_results[0].trials[0]
print(trial.data["output"], trial.data["evaluators"])
new_graders = EvalSuite("nonempty diagnostic").add_evaluator(NotEmpty())
reviewed = regrade(saved, new_graders)
assert reviewed.total == 3

from multivon_eval import (
    EvalCase, EvalSuite, ExactMatch, ProviderJournal, capture_provider_events,
)

capture_suite = EvalSuite("capture example").add_case(EvalCase("status", "ready"))
capture_suite.add_evaluator(ExactMatch())

# Replace the local target with your SDK adapter. This example makes no API calls.
# The journal records observed attempts before transport dispatch.
with ProviderJournal("private-provider-events.sqlite") as journal:
    with capture_provider_events(journal=journal):
        captured_report = capture_suite.run(lambda prompt: "ready", verbose=False)
    events = journal.events()

captured_report.save_json("private-results.json")