jev.rsWhat the page says about where Whiskers uses Jev, built from the code that does the using, so the page cannot say what the code does not do:
- the diagram of a turn is
whiskers_core::journey, whose endings a core test runs the real pipeline into; - the topics, the threshold, the two questions and the decision are
whiskers_judge's; - the example requests are the bytes
jev_protocol::request_byteswrites for the real questions and the real state, not text written for the page.
The example responses are not recorded: the repository holds no recorded Jev bodies, and the
page never calls Jev. They are written by hand in Jev's documented shape, with figures chosen
for the example, checked by the real verifier (Response::parse, which refuses a body that does
not answer exactly the questions asked) and decided by the real decide. The page says so.
The age the examples are asked for: the age the guard's smoke set was written for.
22pub const EXAMPLE_AGE: u8 = 6;
The diagram of one turn, one row per step.
25pub fn rows() -> Vec<Row> { 26 TURN.iter() 27 .map(|step| Row { 28 id: step.id.to_owned(), 29 title: match step.jev { 30 Some(check) if check.direction.is_some() => format!("Jev: {}", check.asks), 31 Some(check) => format!("Jev: {}", check.asks), 32 None => match step.place { 33 Place::Model => "The model".to_owned(), 34 _ => "The tablet".to_owned(), 35 }, 36 }, 37 detail: step.what.to_owned() 38 + if step.only_with_pictures { " (Only when she showed a picture.)" } else { "" }, 39 tone: if step.jev.is_some() { 40 Tone::Jev 41 } else if step.id == "say" { 42 Tone::End 43 } else { 44 Tone::Plain 45 }, 46 exits: step 47 .exits 48 .iter() 49 .map(|exit| Exit { cause: exit.cause.to_owned(), then: format!("she hears \u{201c}{}\u{201d}", exit.then.line()), note: exit.parents_see.to_owned() }) 50 .collect(), 51 }) 52 .collect() 53}
A sentence put to the guard and what came of it.
56pub struct Example {
Why this one is shown.
61 pub why: &'static str,
The exact bytes of the request the guard builds for it.
63 pub request: String,
What Jev said the message was about, and how sure it was that it is suitable.
72struct Script { 73 direction: Direction, 74 text: &'static str, 75 why: &'static str, 76 suitable: f64, 77 topic: Topic, 78 confidence: f64, 79} 80 81const SCRIPTS: [Script; 3] = [ 82 Script { 83 direction: Direction::FromChild, 84 text: "Hi Whiskers! My bunny is named Biscuit.", 85 why: "An ordinary sentence. Allowed: the topic is ordinary and Jev is more than 85% sure it is suitable.", 86 suitable: 0.97, 87 topic: Topic::Ordinary, 88 confidence: 0.94, 89 }, 90 Script { 91 direction: Direction::FromChild, 92 text: "Can you keep a secret from my mom and dad?", 93 why: "Asking the cat to keep a secret from her parents. Refused, and as a kind that sends her to a grown-up.", 94 suitable: 0.31, 95 topic: Topic::SecretsFromParents, 96 confidence: 0.88, 97 }, 98 Script { 99 direction: Direction::ToChild, 100 text: "Some grown-ups like to bet money on games. That is called gambling.", 101 why: "Something the model wrote, not something she said. Refused: she never hears it, and the parents' log keeps it.", 102 suitable: 0.42, 103 topic: Topic::AdultTopics, 104 confidence: 0.83, 105 }, 106];
A response body in the shape Jev answers in, written for an example: suitable for the
yes-or-no, and the probabilities of the nine topics with topic taking confidence and the
rest sharing what is left in equal parts.
111fn fixture(suitable: f64, topic: Topic, confidence: f64) -> String { 112 let rest = (1.0 - confidence) / (Topic::ALL.len() - 1) as f64; 113 let probabilities: Vec<String> = Topic::ALL 114 .iter() 115 .map(|(t, label, _)| format!("\"{label}\":{}", if *t == topic { confidence } else { (rest * 10_000.0).round() / 10_000.0 })) 116 .collect(); 117 format!( 118 "{{\"model\":\"{MODEL}\",\"answers\":{{\"suitable\":{{\"type\":\"noul\",\"noul\":{suitable}}},\"topic\":{{\"type\":\"choice\",\"choice\":\"{}\",\"confidence\":{confidence},\"probabilities\":{{{}}}}}}},\"usage\":{{\"input_tokens\":410,\"output_tokens\":2}}}}", 119 topic.label(), 120 probabilities.join(",") 121 ) 122}
The examples, built by running the guard's own code: the request is what the guard sends, the response is parsed by the real verifier, and the verdict is the real decision.
126pub fn examples() -> Vec<Example> { 127 let age = Age::new(EXAMPLE_AGE).expect("a supported age"); 128 let judge = Judge::new(age).expect("the guard's questions are valid"); 129 let model = ModelId::pinned(MODEL).expect("the pinned model is versioned"); 130 SCRIPTS 131 .iter() 132 .map(|s| { 133 let state = Judge::state(s.direction, age, s.text); 134 let request = String::from_utf8(request_bytes(&model, &state, &judge.questions).expect("a valid request")).expect("a request is UTF-8"); 135 let response = fixture(s.suitable, s.topic, s.confidence); 136 let parsed = Response::parse(&model, &judge.questions, response.as_bytes()).expect("the hand-written response answers exactly the questions asked"); 137 let verdict = judge.verdict(s.direction, &parsed); 138 Example { direction: s.direction, text: s.text, why: s.why, request, response, verdict, topic: s.topic, suitable: s.suitable } 139 }) 140 .collect() 141}
143#[cfg(test)] 144mod tests { 145 use super::*; 146 use whiskers_core::RefusalKind; 147 use whiskers_core::journey::{JEV_USES, NOT_JEV}; 148 use whiskers_judge::decide; 149 150 #[test] 151 fn the_diagram_has_the_journeys_steps_in_the_journeys_order() { 152 let ids: Vec<_> = rows().into_iter().map(|r| r.id).collect(); 153 let want: Vec<_> = TURN.iter().map(|s| s.id.to_owned()).collect(); 154 assert_eq!(ids, want); 155 } 156 157 #[test] 158 fn every_exit_says_a_line_the_cat_really_speaks() { 159 for (row, step) in rows().iter().zip(TURN) { 160 assert_eq!(row.exits.len(), step.exits.len()); 161 for (shown, exit) in row.exits.iter().zip(step.exits) { 162 assert!(shown.then.contains(exit.then.line()), "{}", row.id); 163 } 164 } 165 } 166 167 #[test] 168 fn only_the_steps_that_ask_jev_are_drawn_as_jev() { 169 for (row, step) in rows().iter().zip(TURN) { 170 assert_eq!(row.tone == Tone::Jev, step.jev.is_some(), "{}", row.id); 171 } 172 } 173 174 #[test] 175 fn the_three_examples_end_as_their_captions_say() { 176 let all = examples(); 177 assert_eq!(all.len(), 3); 178 assert_eq!(all[0].verdict, Verdict::Allow); 179 assert!(matches!(all[1].verdict, Verdict::Refuse { kind: RefusalKind::NeedsAGrownUp, .. })); 180 // What the model wrote is withheld, never escalated to her. 181 assert!(matches!(all[2].verdict, Verdict::Refuse { kind: RefusalKind::OffLimits, .. })); 182 } 183 184 #[test] 185 fn an_example_verdict_is_what_decide_says_of_what_jev_answered() { 186 for e in examples() { 187 assert_eq!(e.verdict_via_decide(), e.verdict); 188 } 189 } 190 191 #[test] 192 fn the_request_is_the_two_real_questions_and_the_real_state() { 193 let all = examples(); 194 let age = Age::new(EXAMPLE_AGE).unwrap(); 195 let judge = Judge::new(age).unwrap(); 196 let model = ModelId::pinned(MODEL).unwrap(); 197 for e in &all { 198 // Byte for byte what the guard sends for this sentence. 199 let state = Judge::state(e.direction, age, e.text); 200 assert_eq!(e.request.as_bytes(), request_bytes(&model, &state, &judge.questions).unwrap().as_slice()); 201 let body: serde_json::Value = serde_json::from_str(&e.request).unwrap(); 202 assert_eq!(body["model"], MODEL); 203 assert_eq!(body["questions"].as_object().unwrap().len(), 2); 204 assert_eq!(body["questions"]["suitable"]["type"], "noul"); 205 assert_eq!(body["questions"]["topic"]["type"], "choice"); 206 } 207 // The child's words and her age are in the state, and a name never is. 208 assert!(all[0].request.contains("Hi Whiskers! My bunny is named Biscuit.")); 209 assert!(all[0].request.contains("6-year-old child")); 210 } 211 212 #[test] 213 fn every_use_of_jev_has_a_failure_that_is_not_an_allow() { 214 for u in JEV_USES { 215 assert!(!u.on_failure.contains("allowed"), "{}", u.id); 216 } 217 assert!(!NOT_JEV.is_empty()); 218 } 219 220 #[test] 221 fn decide_is_the_policy_the_page_describes() { 222 use whiskers_judge::MIN_SUITABLE; 223 assert_eq!(decide(Direction::FromChild, MIN_SUITABLE, Topic::Ordinary), Verdict::Allow); 224 assert!(matches!(decide(Direction::FromChild, MIN_SUITABLE - 0.001, Topic::Ordinary), Verdict::Refuse { .. })); 225 } 226 227 impl Example { 228 fn verdict_via_decide(&self) -> Verdict { 229 decide(self.direction, self.suitable, self.topic) 230 } 231 } 232}