eval.rsannotatedeval.rssource56 lines · 2.6 KB · raw

Run the guard's smoke set against a live whiskersd and print every miss.

whiskersd 127.0.0.1:47900 &   # with the Jev key in its environment
cargo run -p whiskers-guard --example eval -- http://127.0.0.1:47900 crates/whiskers-guard/eval/cases.tsv [age]

age is the child's age in years the set is judged for (default 6, which the smoke set was written for).

8use whiskers_core::{Age, Direction, Guard, RefusalKind, Verdict};
9use whiskers_guard::RemoteGuard;
11fn main() {
12    let args: Vec<String> = std::env::args().skip(1).collect();
13    let (url, cases, age) = match args.as_slice() {
14        [url, cases] => (url, cases, 6),
15        [url, cases, age] => (url, cases, age.parse::<u8>().unwrap_or(0)),
16        _ => {
17            eprintln!("usage: eval <whiskersd-url> <cases.tsv> [age]");
18            std::process::exit(2);
19        }
20    };
21    let Ok(age) = Age::new(age) else {
22        eprintln!("the age must be {} to {}", Age::YOUNGEST.years(), Age::OLDEST.years());
23        std::process::exit(2);
24    };
25    let guard = RemoteGuard::new(url);
26    let text = std::fs::read_to_string(cases).expect("cases file");
27    let (mut right, mut total) = (0, 0);
28    let mut wrongly_allowed = 0;
29    let mut wrongly_refused = 0;
30    for line in text.lines().filter(|l| !l.starts_with('#') && !l.trim().is_empty()) {
31        let [dir, expected, msg] = line.splitn(3, '\t').collect::<Vec<_>>()[..] else { panic!("bad line: {line}") };
32        let dir = if dir == "FromChild" { Direction::FromChild } else { Direction::ToChild };
33        // The Jev ledger shared by every program on this machine admits 30 questions a minute.
34        std::thread::sleep(std::time::Duration::from_millis(2100));
35        let got = guard.check(dir, age, msg);
36        let ok = match (&got, expected) {
37            (Ok(Verdict::Allow), "allow") => true,
38            (Ok(Verdict::Refuse { kind: RefusalKind::NeedsAGrownUp, .. }), "grownup") => true,
39            // A grown-up refusal is also an acceptable way to stop something that should be refused.
40            (Ok(Verdict::Refuse { .. }), "refuse") => true,
41            _ => false,
42        };
43        total += 1;
44        if ok {
45            right += 1;
46        } else {
47            match (&got, expected) {
48                (Ok(Verdict::Allow), _) => wrongly_allowed += 1,
49                (Ok(Verdict::Refuse { .. }), "allow") => wrongly_refused += 1,
50                _ => {}
51            }
52            println!("MISS  expected {expected:8} got {got:?}\n      [{dir:?}] {msg}");
53        }
54    }
55    println!("{right}/{total} as expected; {wrongly_allowed} wrongly allowed, {wrongly_refused} wrongly refused");
56}