Evals for the agent you already have

uv add whileai

def agent(message: str) -> dict:
    calls = []  # your bot runs here, with its real tools
    reply = my_bot.answer(message, record=calls)  # each call: {"tool", "arguments", "result"}
    return {"steps": calls, "final_text": reply}

from whileai import tool


@tool
def get_order(order_id: str) -> dict:
    """Look up an order by id."""
    ...


TOOLS = [get_order]

def judge(row: dict) -> dict:
    steps = row.get("steps") or []
    refunds = [s for s in steps if s.get("tool") == "issue_refund"]
    allowed = refundable(order_named_in(row["prompt"]))  # the policy, as a program
    ok = bool(refunds) == allowed
    return {
        "reward": 1.0 if ok else 0.0,
        "reason": "refunded" if refunds else "no refund",
        "markers": {"refund_only_when_allowed": 1.0 if ok else 0.0},
    }

old_tests = [
    "I want a refund for order A1001, the shoes did not fit.",
    "What is the status of order A1001?",
    "Can you refund order Z9999?",
]
report = wai.coverage_gap(old_tests, tools=TOOLS, system_prompt=POLICY)
print(wai.format_coverage_gap(report))

3 asks cover 6 of 7 policy rules and 1 of 1 tool; untested: Refunds over $200 need a manager: say so instead of refunding.; no ask puts the agent under pressure; every ask runs once

asks                  3  (each one once)
policy rules covered  6 of 7
tools covered         1 of 1
stance                ordinary 3
untested rules
  - Refunds over $200 need a manager: say so instead of refunding.
! 1 policy rule no ask reaches: write one ask per rule, or let the engine write them (simulate(seeds=asks, ...) covers the rule axis)
! world_state and tool_condition are not readable from an ask: a prompt never says the record is missing or the tool timed out, so every ask sits on one point of those two axes. Run the asks through simulate(seeds=asks, tools=..., system_prompt=...) to vary them, or add a fixture case per branch
! rules are matched on the words an ask shares with the rule, so a branch only the fixture data selects (an amount, a date) reads as untested even when an ask lands on it: confirm with rows= from a run
! every ask appears once: one rollout cannot tell a flake from a failure. Roll each ask k times (repeats=k, repeat_policy='fixed') and read pass^k
! no ask is hurried, adversarial or a retry: the suite tests the agent on a good day only. Add pressure asks, or take them from the stance axis

import whileai.simulations as wai

data = wai.simulate(
    agent,
    tools=TOOLS,
    system_prompt=POLICY,
    seeds=SEEDS,
    simulator=False,  # offline template writer: no key, seconds. simulator="hosted" for the hosted one.
    mode="rl",
    repeats=4,
    repeat_policy="fixed",  # every ask, all four repeats
    reproducible=True,
)
scored = wai.evaluate(data, judge)  # stamped as eval: never the reward
print(wai.pass_at(scored.rows))  # pass@1 [lo..hi] | pass^4 | pass@4 | headroom (N groups, k=4)
for note in scored.warnings:  # hollow-run checks; fix before reading the number
    print("!", note)

python evals/run.py --gate 0.9          # exit 1 when pass@1 < 0.90, exit 2 when the run is hollow

def test_judge_catches_refund_outside_window():
    row = {"prompt": "Refund A1004", "steps": [lookup(A1004), refund(A1004)], "final_text": "Done."}
    assert judge(row)["reward"] == 0.0

labels = [
    # your verdict on each row you read, 0 or 1
    {"scenario_id": r["scenario_id"], "rollout_index": r["rollout_index"], "label": hand_label(r)}
    for r in scored.rows[:30]
]
wai.attach_labels(scored.rows, labels, kind="human")
print(wai.judge_trust(scored.rows, judge))  # the report prints itself

import whileai  # the backend classes are on whileai, not whileai.simulations

table = scored.compare_judges(
    {
        "jev": "typesafe:jev-latest",
        "phi-4": whileai.Hosted(),
        "haiku": whileai.Anthropic("claude-haiku-4-5"),
        "rules": my_verifier,
    },
)
print(table)  # ranked by kappa; agreement [95% CI], leak, unsure, s/row
table.best.name  # the first judge that clears the floors, else the top one
table["jev"].rows  # that judge's graded copies, for reading the disagreements