kulono / multi-model-verifier
blob · tests/test_audit.py · python
← filesrepo
tests/test_audit.pypython
1from multi_model_verifier import ModelAnswer, compare_answers, broadcast_and_verify, execute_verify_multi
2
3
4def _ans(mid, content, err=None):
5 return ModelAnswer(model_id=mid, provider_name=mid, content=content, error=err, latency_seconds=0.1)
6
7
8def test_unanimous():
9 a = compare_answers([_ans("a", "42"), _ans("b", "42")])
10 assert a.unanimous() and a.divergence_score == 0.0
11
12
13def test_divergent():
14 a = compare_answers([_ans("a", "42"), _ans("b", "43"), _ans("c", "42")])
15 assert not a.unanimous()
16 assert a.divergence_score > 0
17 assert a.consensus == "42" # majority
18
19
20def test_all_failed():
21 a = compare_answers([_ans("a", "", err="x"), _ans("b", "", err="y")])
22 assert a.divergence_score == 1.0
23
24
25def test_broadcast_sync():
26 def q(model, prompt):
27 return ("paris", 0.05)
28 audit = broadcast_and_verify(["m1", "m2", "m3"], "capital of France?", q, parallel=False)
29 assert audit.unanimous()
30
31
32def test_gate_blocks_on_disagreement():
33 n = [0]
34 def q(model, prompt):
35 n[0] += 1
36 return ("yes" if n[0] < 3 else "no", 0.05)
37 res = execute_verify_multi(["m1", "m2", "m3"], "approve?", q, threshold=0.2)
38 assert res["blocked"] is True
39
100%