kulono / multi-model-verifier
blob · tests/test_audit.py · python
tests/test_audit.pypython
1from multi_model_verifier import ModelAnswer, compare_answers, broadcast_and_verify, execute_verify_multi234def _ans(mid, content, err=None):5 return ModelAnswer(model_id=mid, provider_name=mid, content=content, error=err, latency_seconds=0.1)678def test_unanimous():9 a = compare_answers([_ans("a", "42"), _ans("b", "42")])10 assert a.unanimous() and a.divergence_score == 0.0111213def test_divergent():14 a = compare_answers([_ans("a", "42"), _ans("b", "43"), _ans("c", "42")])15 assert not a.unanimous()16 assert a.divergence_score > 017 assert a.consensus == "42" # majority181920def test_all_failed():21 a = compare_answers([_ans("a", "", err="x"), _ans("b", "", err="y")])22 assert a.divergence_score == 1.0232425def test_broadcast_sync():26 def q(model, prompt):27 return ("paris", 0.05)28 audit = broadcast_and_verify(["m1", "m2", "m3"], "capital of France?", q, parallel=False)29 assert audit.unanimous()303132def test_gate_blocks_on_disagreement():33 n = [0]34 def q(model, prompt):35 n[0] += 136 return ("yes" if n[0] < 3 else "no", 0.05)37 res = execute_verify_multi(["m1", "m2", "m3"], "approve?", q, threshold=0.2)38 assert res["blocked"] is True39
100%