Fit on development reviews, evaluate held-out sources

from sklearn.model_selection import GroupShuffleSplit
from multivon_eval import CaseManifest

# cases: your EvalCase objects, with explicit case_id and source_id.
# Define them before fitting or inspecting held-out results.
groups = [case.source_id for case in cases]
splitter = GroupShuffleSplit(n_splits=1, test_size=0.3, random_state=42)
development, held_out = next(splitter.split(cases, groups=groups))
manifest = CaseManifest("frozen review protocol", cases, splits={
    "development": [cases[i].case_id for i in development],
    "held_out": [cases[i].case_id for i in held_out],
})
manifest.save("review-manifest.json")

import json
from pathlib import Path
from multivon_eval import EvalReport
from multivon_eval.integrations.label_studio import import_review_annotations
from multivon_eval.review_calibration import collect_reviewed_scores

def read(path):
    return json.loads(Path(path).read_text())

def reviewed_split(split):
    report = EvalReport.from_dict(read(f"{split}-report.json"))
    tasks = read(f"{split}-tasks.json")
    reviews = import_review_annotations(read(f"{split}-export.json"), tasks,
                                        reviewer_kind="human")
    return collect_reviewed_scores(report, tasks, reviews, manifest,
        split=split, reviewer_kind="human", min_reviewers=2)

development_scores = reviewed_split("development")
held_out_scores = reviewed_split("held_out")

from multivon_eval.review_calibration import fit_review_threshold

fit = fit_review_threshold(development_scores,
    false_accept_cost=5, false_reject_cost=1)
Path("frozen-fit.json").write_text(json.dumps(fit.to_dict(), indent=2))

from multivon_eval.review_calibration import evaluate_review_threshold

analysis = evaluate_review_threshold(fit, held_out_scores, confidence=0.95)
Path("held-out-analysis.json").write_text(json.dumps(analysis, indent=2))

python examples/calibrate_reviewed_scores.py --output-dir review-calibration-demo