1//! What the page says about where Whiskers uses Jev, built from the code that does the using, so
2//! the page cannot say what the code does not do:
3//!
4//! - the diagram of a turn is `whiskers_core::journey`, whose endings a core test runs the real
5//!   pipeline into;
6//! - the topics, the threshold, the two questions and the decision are `whiskers_judge`'s;
7//! - the example requests are the bytes `jev_protocol::request_bytes` writes for the real
8//!   questions and the real state, not text written for the page.
9//!
10//! The example **responses** are not recorded: the repository holds no recorded Jev bodies, and the
11//! page never calls Jev. They are written by hand in Jev's documented shape, with figures chosen
12//! for the example, checked by the real verifier (`Response::parse`, which refuses a body that does
13//! not answer exactly the questions asked) and decided by the real `decide`. The page says so.
14
15use jev_protocol::{ModelId, Response, request_bytes};
16use jev_ui::flow::{Exit, Row, Tone};
17use whiskers_core::journey::{Place, TURN};
18use whiskers_core::{Age, Direction, Verdict};
19use whiskers_judge::{Judge, MODEL, Topic};
20
21/// The age the examples are asked for: the age the guard's smoke set was written for.
22pub const EXAMPLE_AGE: u8 = 6;
23
24/// The diagram of one turn, one row per step.
25pub fn rows() -> Vec<Row> {
26    TURN.iter()
27        .map(|step| Row {
28            id: step.id.to_owned(),
29            title: match step.jev {
30                Some(check) if check.direction.is_some() => format!("Jev: {}", check.asks),
31                Some(check) => format!("Jev: {}", check.asks),
32                None => match step.place {
33                    Place::Model => "The model".to_owned(),
34                    _ => "The tablet".to_owned(),
35                },
36            },
37            detail: step.what.to_owned()
38                + if step.only_with_pictures { " (Only when she showed a picture.)" } else { "" },
39            tone: if step.jev.is_some() {
40                Tone::Jev
41            } else if step.id == "say" {
42                Tone::End
43            } else {
44                Tone::Plain
45            },
46            exits: step
47                .exits
48                .iter()
49                .map(|exit| Exit { cause: exit.cause.to_owned(), then: format!("she hears \u{201c}{}\u{201d}", exit.then.line()), note: exit.parents_see.to_owned() })
50                .collect(),
51        })
52        .collect()
53}
54
55/// A sentence put to the guard and what came of it.
56pub struct Example {
57    /// Which way the message travels.
58    pub direction: Direction,
59    pub text: &'static str,
60    /// Why this one is shown.
61    pub why: &'static str,
62    /// The exact bytes of the request the guard builds for it.
63    pub request: String,
64    /// The hand-written response.
65    pub response: String,
66    pub verdict: Verdict,
67    /// What Jev said the message was about, and how sure it was that it is suitable.
68    pub topic: Topic,
69    pub suitable: f64,
70}
71
72struct Script {
73    direction: Direction,
74    text: &'static str,
75    why: &'static str,
76    suitable: f64,
77    topic: Topic,
78    confidence: f64,
79}
80
81const SCRIPTS: [Script; 3] = [
82    Script {
83        direction: Direction::FromChild,
84        text: "Hi Whiskers! My bunny is named Biscuit.",
85        why: "An ordinary sentence. Allowed: the topic is ordinary and Jev is more than 85% sure it is suitable.",
86        suitable: 0.97,
87        topic: Topic::Ordinary,
88        confidence: 0.94,
89    },
90    Script {
91        direction: Direction::FromChild,
92        text: "Can you keep a secret from my mom and dad?",
93        why: "Asking the cat to keep a secret from her parents. Refused, and as a kind that sends her to a grown-up.",
94        suitable: 0.31,
95        topic: Topic::SecretsFromParents,
96        confidence: 0.88,
97    },
98    Script {
99        direction: Direction::ToChild,
100        text: "Some grown-ups like to bet money on games. That is called gambling.",
101        why: "Something the model wrote, not something she said. Refused: she never hears it, and the parents' log keeps it.",
102        suitable: 0.42,
103        topic: Topic::AdultTopics,
104        confidence: 0.83,
105    },
106];
107
108/// A response body in the shape Jev answers in, written for an example: `suitable` for the
109/// yes-or-no, and the probabilities of the nine topics with `topic` taking `confidence` and the
110/// rest sharing what is left in equal parts.
111fn fixture(suitable: f64, topic: Topic, confidence: f64) -> String {
112    let rest = (1.0 - confidence) / (Topic::ALL.len() - 1) as f64;
113    let probabilities: Vec<String> = Topic::ALL
114        .iter()
115        .map(|(t, label, _)| format!("\"{label}\":{}", if *t == topic { confidence } else { (rest * 10_000.0).round() / 10_000.0 }))
116        .collect();
117    format!(
118        "{{\"model\":\"{MODEL}\",\"answers\":{{\"suitable\":{{\"type\":\"noul\",\"noul\":{suitable}}},\"topic\":{{\"type\":\"choice\",\"choice\":\"{}\",\"confidence\":{confidence},\"probabilities\":{{{}}}}}}},\"usage\":{{\"input_tokens\":410,\"output_tokens\":2}}}}",
119        topic.label(),
120        probabilities.join(",")
121    )
122}
123
124/// The examples, built by running the guard's own code: the request is what the guard sends, the
125/// response is parsed by the real verifier, and the verdict is the real decision.
126pub fn examples() -> Vec<Example> {
127    let age = Age::new(EXAMPLE_AGE).expect("a supported age");
128    let judge = Judge::new(age).expect("the guard's questions are valid");
129    let model = ModelId::pinned(MODEL).expect("the pinned model is versioned");
130    SCRIPTS
131        .iter()
132        .map(|s| {
133            let state = Judge::state(s.direction, age, s.text);
134            let request = String::from_utf8(request_bytes(&model, &state, &judge.questions).expect("a valid request")).expect("a request is UTF-8");
135            let response = fixture(s.suitable, s.topic, s.confidence);
136            let parsed = Response::parse(&model, &judge.questions, response.as_bytes()).expect("the hand-written response answers exactly the questions asked");
137            let verdict = judge.verdict(s.direction, &parsed);
138            Example { direction: s.direction, text: s.text, why: s.why, request, response, verdict, topic: s.topic, suitable: s.suitable }
139        })
140        .collect()
141}
142
143#[cfg(test)]
144mod tests {
145    use super::*;
146    use whiskers_core::RefusalKind;
147    use whiskers_core::journey::{JEV_USES, NOT_JEV};
148    use whiskers_judge::decide;
149
150    #[test]
151    fn the_diagram_has_the_journeys_steps_in_the_journeys_order() {
152        let ids: Vec<_> = rows().into_iter().map(|r| r.id).collect();
153        let want: Vec<_> = TURN.iter().map(|s| s.id.to_owned()).collect();
154        assert_eq!(ids, want);
155    }
156
157    #[test]
158    fn every_exit_says_a_line_the_cat_really_speaks() {
159        for (row, step) in rows().iter().zip(TURN) {
160            assert_eq!(row.exits.len(), step.exits.len());
161            for (shown, exit) in row.exits.iter().zip(step.exits) {
162                assert!(shown.then.contains(exit.then.line()), "{}", row.id);
163            }
164        }
165    }
166
167    #[test]
168    fn only_the_steps_that_ask_jev_are_drawn_as_jev() {
169        for (row, step) in rows().iter().zip(TURN) {
170            assert_eq!(row.tone == Tone::Jev, step.jev.is_some(), "{}", row.id);
171        }
172    }
173
174    #[test]
175    fn the_three_examples_end_as_their_captions_say() {
176        let all = examples();
177        assert_eq!(all.len(), 3);
178        assert_eq!(all[0].verdict, Verdict::Allow);
179        assert!(matches!(all[1].verdict, Verdict::Refuse { kind: RefusalKind::NeedsAGrownUp, .. }));
180        // What the model wrote is withheld, never escalated to her.
181        assert!(matches!(all[2].verdict, Verdict::Refuse { kind: RefusalKind::OffLimits, .. }));
182    }
183
184    #[test]
185    fn an_example_verdict_is_what_decide_says_of_what_jev_answered() {
186        for e in examples() {
187            assert_eq!(e.verdict_via_decide(), e.verdict);
188        }
189    }
190
191    #[test]
192    fn the_request_is_the_two_real_questions_and_the_real_state() {
193        let all = examples();
194        let age = Age::new(EXAMPLE_AGE).unwrap();
195        let judge = Judge::new(age).unwrap();
196        let model = ModelId::pinned(MODEL).unwrap();
197        for e in &all {
198            // Byte for byte what the guard sends for this sentence.
199            let state = Judge::state(e.direction, age, e.text);
200            assert_eq!(e.request.as_bytes(), request_bytes(&model, &state, &judge.questions).unwrap().as_slice());
201            let body: serde_json::Value = serde_json::from_str(&e.request).unwrap();
202            assert_eq!(body["model"], MODEL);
203            assert_eq!(body["questions"].as_object().unwrap().len(), 2);
204            assert_eq!(body["questions"]["suitable"]["type"], "noul");
205            assert_eq!(body["questions"]["topic"]["type"], "choice");
206        }
207        // The child's words and her age are in the state, and a name never is.
208        assert!(all[0].request.contains("Hi Whiskers! My bunny is named Biscuit."));
209        assert!(all[0].request.contains("6-year-old child"));
210    }
211
212    #[test]
213    fn every_use_of_jev_has_a_failure_that_is_not_an_allow() {
214        for u in JEV_USES {
215            assert!(!u.on_failure.contains("allowed"), "{}", u.id);
216        }
217        assert!(!NOT_JEV.is_empty());
218    }
219
220    #[test]
221    fn decide_is_the_policy_the_page_describes() {
222        use whiskers_judge::MIN_SUITABLE;
223        assert_eq!(decide(Direction::FromChild, MIN_SUITABLE, Topic::Ordinary), Verdict::Allow);
224        assert!(matches!(decide(Direction::FromChild, MIN_SUITABLE - 0.001, Topic::Ordinary), Verdict::Refuse { .. }));
225    }
226
227    impl Example {
228        fn verdict_via_decide(&self) -> Verdict {
229            decide(self.direction, self.suitable, self.topic)
230        }
231    }
232}