1//! `hostcheck`: asks a running Whiskers backend host the same questions as every other host.
2//!
3//!     hostcheck --base http://127.0.0.1:8801 --label celld --out DIR [--restart-cmd CMD] [--single-household]
4//!
5//! - **laws**: the real conformance laws (`whiskers-conformance`), one by one, through the host's test-mode
6//!   window onto its household objects (`/_hub/..`; a Worker only).
7//! - **script**: the recorded request script (`script.rs`), its answers written to `DIR/<label>.transcript`,
8//!   so two hosts compare with `cmp`.
9//! - **concurrency**: overlapping syncs to one household and the atomicity invariants (`concurrency.rs`).
10//! - **restart**: documents written, the host restarted by `--restart-cmd`, the documents read back.
11//! - **timing**: a cold first request and the median and 95th percentile of warm ones.
12//!
13//! `--single-household` is for `whiskersd`, which has exactly one household and no test mode: only the
14//! script, restart and timing run.
15
16mod concurrency;
17mod host;
18mod hub;
19mod ids;
20mod laws;
21mod script;
22mod transcript;
23
24use std::path::PathBuf;
25use std::process::Command;
26use std::time::Duration;
27
28use host::Host;
29use ids::Households;
30
31struct Args {
32    base: String,
33    label: String,
34    out: PathBuf,
35    restart_cmd: Option<String>,
36    single_household: bool,
37    /// Only these phases (laws, script, concurrency, restart, timing); empty is all.
38    only: Vec<String>,
39}
40
41fn args() -> Result<Args, String> {
42    let mut a = Args { base: String::new(), label: String::new(), out: PathBuf::from("."), restart_cmd: None, single_household: false, only: vec![] };
43    let mut it = std::env::args().skip(1);
44    while let Some(flag) = it.next() {
45        let mut value = |name: &str| it.next().ok_or_else(|| format!("{name} needs a value"));
46        match flag.as_str() {
47            "--base" => a.base = value("--base")?,
48            "--label" => a.label = value("--label")?,
49            "--out" => a.out = PathBuf::from(value("--out")?),
50            "--restart-cmd" => a.restart_cmd = Some(value("--restart-cmd")?),
51            "--single-household" => a.single_household = true,
52            "--only" => a.only = value("--only")?.split(',').map(str::to_owned).collect(),
53            other => return Err(format!("unknown argument {other}")),
54        }
55    }
56    if a.base.is_empty() || a.label.is_empty() {
57        return Err("usage: hostcheck --base URL --label NAME [--out DIR] [--restart-cmd CMD] [--single-household]".into());
58    }
59    Ok(a)
60}
61
62impl Args {
63    fn runs(&self, phase: &str) -> bool {
64        self.only.is_empty() || self.only.iter().any(|p| p == phase)
65    }
66}
67
68fn percentile(sorted: &[Duration], p: f64) -> Duration {
69    let i = ((sorted.len() as f64 - 1.0) * p).round() as usize;
70    sorted[i.min(sorted.len() - 1)]
71}
72
73fn restarter(host: &Host, cmd: Option<String>) -> impl Fn() + '_ {
74    move || {
75        let Some(cmd) = &cmd else {
76            panic!("this host has no --restart-cmd, so a restart cannot be asked of it");
77        };
78        let status = Command::new("sh").arg("-c").arg(cmd).status().expect("run the restart command");
79        assert!(status.success(), "the restart command failed: {status}");
80        let took = host.wait_ready(Duration::from_secs(180)).expect("the host comes back");
81        println!("    (restarted; ready again after {took:?})");
82    }
83}
84
85fn main() {
86    let a = match args() {
87        Ok(a) => a,
88        Err(e) => {
89            eprintln!("hostcheck: {e}");
90            std::process::exit(2);
91        }
92    };
93    std::fs::create_dir_all(&a.out).expect("the output directory");
94    let host = Host::new(&a.base);
95    let ids = Households::new(&a.label);
96    let restart = restarter(&host, a.restart_cmd.clone());
97    let mut failed = 0;
98    println!("== {} at {}", a.label, a.base);
99
100    if let Err(e) = host.wait_ready(Duration::from_secs(120)) {
101        eprintln!("hostcheck: {e}");
102        std::process::exit(2);
103    }
104
105    if !a.single_household && a.runs("laws") {
106        println!("-- laws (whiskers-conformance, through /_hub)");
107        for o in laws::run_all(&host, &ids, &restart) {
108            match &o.failure {
109                None => println!("PASS {}", o.law),
110                Some(why) => {
111                    failed += 1;
112                    println!("FAIL {}: {why}", o.law);
113                }
114            }
115        }
116        for (law, why) in laws::NOT_RUNNABLE_OVER_HTTP {
117            println!("N/A  {law}: {why}");
118        }
119    }
120
121    if a.runs("script") {
122    println!("-- script");
123    let household = if a.single_household { "script".to_owned() } else { ids.next() };
124    let (text, times) = transcript::run(&host, &household);
125    let path = a.out.join(format!("{}.transcript", a.label));
126    std::fs::write(&path, &text).expect("write the transcript");
127    println!("transcript: {} ({} steps, {} bytes)", path.display(), times.len(), text.len());
128    for (step, (name, _)) in script::script().iter().zip(&times) {
129        let line = text.split("### ").find(|b| b.starts_with(name)).and_then(|b| b.lines().nth(1)).unwrap_or("");
130        if !line.starts_with(&step.expect.to_string()) {
131            failed += 1;
132            println!("FAIL script step {name:?}: wanted status {}, got {line:?}", step.expect);
133        }
134    }
135
136    }
137
138    if !a.single_household && a.runs("concurrency") {
139        println!("-- concurrency");
140        for (name, via) in [("through the object's own window (counts changes)", concurrency::Via::Hub), ("through POST /memory/sync", concurrency::Via::Public)] {
141            let before = concurrency::RETRIES.load(std::sync::atomic::Ordering::Relaxed);
142            let t = concurrency::distinct_facts(&host, &ids.next(), via);
143            let retried = concurrency::RETRIES.load(std::sync::atomic::Ordering::Relaxed) - before;
144            let want = concurrency::THREADS * concurrency::EACH;
145            let mut ok = t.failures.is_empty() && t.facts == want && t.distinct_ids == want;
146            if via == concurrency::Via::Hub {
147                ok &= t.claimed_changes == want;
148            }
149            println!(
150                "{} {name}: {} requests from {} threads; {} facts, {} distinct numbers, {} changes claimed, {} retried, {} failed requests {:?}",
151                if ok { "PASS" } else { "FAIL" },
152                want,
153                concurrency::THREADS,
154                t.facts,
155                t.distinct_ids,
156                t.claimed_changes,
157                retried,
158                t.failures.len(),
159                t.failures.first()
160            );
161            failed += usize::from(!ok);
162        }
163        match concurrency::the_same_fact(&host, &ids.next()) {
164            Ok(1) => println!("PASS the same fact from {} devices at once is new to exactly one", concurrency::THREADS),
165            other => {
166                failed += 1;
167                println!("FAIL the same fact from {} devices at once: replies claiming it new: {other:?}", concurrency::THREADS);
168            }
169        }
170    }
171
172    if a.restart_cmd.is_some() && a.runs("restart") {
173        println!("-- restart");
174        let h = if a.single_household { "restart".to_owned() } else { ids.next() };
175        let ok = restart_check(&host, &h, &restart);
176        println!("{} documents survive a restart of the host, and numbering carries on", if ok { "PASS" } else { "FAIL" });
177        failed += usize::from(!ok);
178    }
179
180    if !a.runs("timing") {
181        println!("== {}: {} failure(s)", a.label, failed);
182        std::process::exit(i32::from(failed > 0));
183    }
184    println!("-- timing (sequential POST /memory/sync, empty document, one household)");
185    let h = if a.single_household { "timing".to_owned() } else { ids.next() };
186    let body = r#"{"facts":[],"forgotten":[]}"#;
187    let t0 = std::time::Instant::now();
188    let first = host.post("/memory/sync", &h, body);
189    let cold = t0.elapsed();
190    assert!(matches!(first, Ok(ref r) if r.status == 200), "{first:?}");
191    let mut warm: Vec<Duration> = (0..60)
192        .map(|_| {
193            let t = std::time::Instant::now();
194            let _ = host.post("/memory/sync", &h, body);
195            t.elapsed()
196        })
197        .collect();
198    warm.sort();
199    println!("first request to a new household: {cold:?}; warm n=60: median {:?}, p95 {:?}, max {:?}", percentile(&warm, 0.5), percentile(&warm, 0.95), warm[warm.len() - 1]);
200    let writes: Vec<Duration> = {
201        let mut v: Vec<Duration> = (0..30)
202            .map(|i| {
203                let body = format!(r#"{{"facts":[{{"id":0,"text":"timing fact {i}","learned_at_ms":5,"kind":"Other","who":[],"place":null,"when":null,"pictures":[],"embedding":[],"gid":"tm{i:03}"}}],"forgotten":[]}}"#);
204                let t = std::time::Instant::now();
205                let _ = host.post("/memory/sync", &h, &body);
206                t.elapsed()
207            })
208            .collect();
209        v.sort();
210        v
211    };
212    println!("sync that changes the document (writes), n=30: median {:?}, p95 {:?}", percentile(&writes, 0.5), percentile(&writes, 0.95));
213
214    println!("== {}: {} failure(s)", a.label, failed);
215    std::process::exit(i32::from(failed > 0));
216}
217
218/// Writes all three documents, restarts the host, and reads them back.
219fn restart_check(host: &Host, household: &str, restart: &dyn Fn()) -> bool {
220    let fact = |gid: &str| format!(r#"{{"id":0,"text":"remember {gid}","learned_at_ms":5,"kind":"Other","who":[],"place":null,"when":null,"pictures":[],"embedding":[],"gid":"{gid}"}}"#);
221    let empty_memory = r#"{"facts":[],"forgotten":[]}"#;
222    let chat = r#"{"summary":"kept","turns":[],"last_active_ms":9,"version":7}"#;
223    let read = |path: &str, body: &str| host.post(path, household, body).expect("a reply").body;
224    host.post("/memory/sync", household, &format!(r#"{{"facts":[{},{}],"forgotten":["gone"]}}"#, fact("a1"), fact("a2"))).expect("write memory");
225    host.post("/chat/sync", household, chat).expect("write chat");
226    let before = (read("/memory/sync", empty_memory), read("/chat/sync", r#"{"summary":"","turns":[],"last_active_ms":0,"version":0}"#));
227    restart();
228    let after = (read("/memory/sync", empty_memory), read("/chat/sync", r#"{"summary":"","turns":[],"last_active_ms":0,"version":0}"#));
229    let mut ok = before == after;
230    if !ok {
231        println!("    documents differ across the restart:\n    before {before:?}\n    after  {after:?}");
232    }
233    let ids_of = |text: &str| -> Vec<u64> {
234        let doc: serde_json::Value = serde_json::from_str(text).unwrap_or_default();
235        doc["facts"].as_array().map(|f| f.iter().filter_map(|x| x["id"].as_u64()).collect()).unwrap_or_default()
236    };
237    let old = ids_of(&after.0);
238    let next = ids_of(&read("/memory/sync", &format!(r#"{{"facts":[{}],"forgotten":[]}}"#, fact("a3"))));
239    let distinct: std::collections::BTreeSet<u64> = next.iter().copied().collect();
240    let newest = next.last().copied().unwrap_or(0);
241    let numbered_on = distinct.len() == next.len() && next.len() == old.len() + 1 && old.iter().all(|i| *i < newest);
242    if !numbered_on {
243        println!("    numbering after the restart: before {old:?}, after adding one {next:?}");
244    }
245    ok &= numbered_on;
246    ok
247}