Experiment Tracking

from multivon_eval import EvalSuite, Experiment, Faithfulness, Relevance

suite = EvalSuite("rag-pipeline")
suite.add_cases(cases)
suite.add_evaluators(Faithfulness(), Relevance())

exp = Experiment("rag-pipeline")

# Baseline run
report_v1 = suite.run(model_v1)
run_v1 = exp.record(report_v1, tags={"model": "gpt-4o", "prompt_v": "2"})

# After your prompt or model change
report_v2 = suite.run(model_v2)
run_v2 = exp.record(report_v2, tags={"model": "gpt-4o", "prompt_v": "3"})

# Compare
exp.compare(run_v1, run_v2)

diff = report_v1.compare(report_v2)  # baseline, then proposal
print(diff.pass_rate_delta)

exp.print_history(n=10)

Experiment: rag-pipeline
Run ID       Timestamp              Model                 Pass rate  Avg score Tags
------------------------------------------------------------------------------------------
e5f6g7h8     2026-04-25 15:30:12    gpt-4o                   92.0%     0.8890 prompt_v=3
a1b2c3d4     2026-04-24 10:15:44    gpt-4o                   84.0%     0.8210 prompt_v=2

exp.record(report, tags={
    "model": "claude-sonnet-4-6",
    "prompt_v": "5",
    "dataset": "v2",
    "deployed": "false",
})

# List all experiments
multivon-eval experiments list

# Show run history
multivon-eval experiments history rag-pipeline

# Compare two runs
multivon-eval experiments compare rag-pipeline a1b2c3d4 e5f6g7h8

# Override storage location
export MULTIVON_HOME=/your/custom/path

import json
import os
from pathlib import Path
from multivon_eval import EvalReport, Experiment

exp = Experiment("prod-eval")
artifact = Path("eval_results.json")
artifact.unlink(missing_ok=True)  # prevent recording an artifact from an older run
try:
    suite.run(model_fn, fail_threshold=0.85, save_json=str(artifact))
finally:
    if artifact.exists():
        exp.record(EvalReport.from_dict(json.loads(artifact.read_text())), tags={
            "git_sha": os.getenv("GITHUB_SHA", "local"),
            "branch": os.getenv("GITHUB_REF_NAME", "local"),
        })