CI/CD Integration

# eval.py
from multivon_eval import EvalSuite, EvalCase, load, Faithfulness, NotEmpty

suite = EvalSuite("Prod Eval")
suite.add_cases(load("tests/cases.jsonl"))
suite.add_evaluators(NotEmpty(), Faithfulness())

report = suite.run(
    my_model,
    fail_threshold=0.85,             # block deployment if < 85% pass
    save_json="eval_results.json",   # written BEFORE the gate fires
    save_html="eval_results.html",
)

# .github/workflows/eval.yml
name: LLM Eval

on:
  push:
    branches: [main]
  pull_request:

jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: pip install multivon-eval

      - name: Run evals
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: multivon-eval run eval.py --html eval_results.html --json eval_results.json

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: eval-results
          path: |
            eval_results.json
            eval_results.html

# Run 8 cases in parallel via threads
report = suite.run(model_fn, workers=8, fail_threshold=0.85)

import asyncio
from multivon_eval import EvalSuite

async def my_async_model(input: str) -> str:
    ...

report = asyncio.run(
    suite.run_async(my_async_model, concurrency=10, fail_threshold=0.85)
)

report.save_json("results.json")
report.save_csv("results.csv")
report.save_html("results.html")   # self-contained HTML report

html = report.to_html()

multivon-eval report results.json

multivon-eval report results.json --html results.html

multivon-eval run eval.py

# Save reports automatically
multivon-eval run eval.py --html results.html
multivon-eval run eval.py --html results.html --json results.json