1//! Which Workers AI model is the LLM: the cheapest one that writes a correct
2//! Jev tool call for every case.
3//!
4//! Sends the Worker's own request (`llm::request`) to each candidate over
5//! Cloudflare's REST API and reads the reply with the Worker's own parser
6//! (`llm::parse`), so a pass here is a pass there. Candidates are tried
7//! cheapest first and the run stops at the first that passes every case.
8//!
9//!     lmjtfy-eval              # stop at the first model that passes
10//!     lmjtfy-eval --all        # run every candidate
11//!     lmjtfy-eval <model>      # run one model, verbosely
12//!     op-env-run -- lmjtfy-eval facts   # the questions Jev is asked first (facts.rs)
13//!
14//! `lmjtfy-eval` is the devshell's wrapper: `cargo run -p eval` with the
15//! account and the token's 1Password reference in its environment.
16//!
17//! It spends real neurons from the account's daily allocation: about 15
18//! requests a model. The token comes from 1Password per run and is never
19//! printed.
20
21use std::process::{Command, ExitCode};
22use std::time::{Duration, Instant};
23
24use llm::{CANDIDATES, Draft, Model};
25
26mod facts;
27
28#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
29enum Kind {
30    Noul,
31    Choice,
32    Score,
33}
34
35use Kind::{Choice, Noul, Score};
36
37/// An input, and the sets of tool calls that would answer it. Most inputs
38/// have one right shape; a few are fairly read two ways.
39struct Case {
40    input: &'static str,
41    accept: &'static [&'static [Kind]],
42    /// A count: a Score's lowest level must be none or zero, for a count can be none.
43    none_first: bool,
44}
45
46const CASES: &[Case] = &[
47    Case { input: "is water wet?", accept: &[&[Noul]], none_first: false },
48    Case { input: "should I rewrite it in rust", accept: &[&[Noul]], none_first: false },
49    Case { input: "can penguins fly", accept: &[&[Noul]], none_first: false },
50    Case { input: "is a hot dog a sandwich?", accept: &[&[Noul]], none_first: false },
51    Case { input: "what is the best text editor", accept: &[&[Choice]], none_first: false },
52    Case { input: "which programming language should I learn first?", accept: &[&[Choice]], none_first: false },
53    Case { input: "who would win in a fight, a bear or a shark", accept: &[&[Choice]], none_first: false },
54    Case { input: "best pizza topping", accept: &[&[Choice]], none_first: false },
55    Case { input: "how good is the movie The Matrix?", accept: &[&[Score]], none_first: false },
56    // A Noul's probability is itself "how likely".
57    Case { input: "how likely is it to rain in Seattle in November", accept: &[&[Noul]], none_first: false },
58    Case { input: "rate my startup idea: uber for dogs", accept: &[&[Score]], none_first: false },
59    Case { input: "how spicy is a jalapeño", accept: &[&[Score]], none_first: false },
60    // The levels must reach zero: a model once wrote 1 to 5 for a count.
61    Case { input: "how many sheep do androids dream of?", accept: &[&[Score]], none_first: true },
62    Case { input: "how many moons does Mars have", accept: &[&[Score]], none_first: true },
63    // Quotes and symbols have to survive the model's JSON.
64    Case { input: "is \"C++\" better than C#?", accept: &[&[Noul], &[Choice]], none_first: false },
65    // Two separate questions in one input.
66    Case {
67        input: "is python slower than rust, and which of the two is better for a beginner?",
68        accept: &[&[Noul, Choice], &[Noul, Noul]],
69        none_first: false,
70    },
71];
72
73fn kind(draft: &Draft) -> Kind {
74    match draft {
75        Draft::Noul { .. } => Noul,
76        Draft::Choice { .. } => Choice,
77        Draft::Score { .. } => Score,
78    }
79}
80
81/// Why a reply does not answer its case, or the neurons it cost.
82fn judge(case: &Case, body: &str) -> (Result<(), String>, llm::Usage) {
83    let reply = match llm::parse(body) {
84        Ok(reply) => reply,
85        Err(error) => return (Err(error), llm::Usage::default()),
86    };
87    let verdict = (|| {
88        if reply.calls.is_empty() {
89            return Err("made no tool call".to_owned());
90        }
91        if reply.dropped > 0 {
92            return Err(format!("made {} calls, over the cap", reply.calls.len() + reply.dropped));
93        }
94        let mut kinds = Vec::new();
95        for call in &reply.calls {
96            let draft = call.draft.as_ref().map_err(|e| format!("{}: {e}", call.name))?;
97            // The last word is Jev's own protocol.
98            ask::check(draft).map_err(|e| format!("{}: {e}", call.name))?;
99            if case.none_first {
100                if let Draft::Score { levels, .. } = draft {
101                    let first = levels.first().map(|level| level.to_lowercase()).unwrap_or_default();
102                    let low = ["none", "no ", "zero", "nothing", "0", "not any", "never"];
103                    if !low.iter().any(|word| first == word.trim() || first.starts_with(word)) {
104                        return Err(format!("a count's lowest level is {first:?}, not none or zero"));
105                    }
106                }
107            }
108            kinds.push(kind(draft));
109        }
110        kinds.sort();
111        let accepted = case.accept.iter().any(|want| {
112            let mut want = want.to_vec();
113            want.sort();
114            want == kinds
115        });
116        if accepted { Ok(()) } else { Err(format!("called {kinds:?}, wanted one of {:?}", case.accept)) }
117    })();
118    (verdict, reply.usage)
119}
120
121struct Cloudflare {
122    agent: ureq::Agent,
123    account: String,
124    token: String,
125}
126
127impl Cloudflare {
128    fn from_env() -> Result<Self, String> {
129        let var = |name: &str| std::env::var(name).map_err(|_| format!("{name} is not set; run `lmjtfy-eval` from the devshell"));
130        let account = var("CLOUDFLARE_ACCOUNT_ID")?;
131        let reference = format!("{}/Token", var("LMJTFY_OP_ITEM_REF")?);
132        // op.exe under WSL (it has the desktop app's session), op elsewhere.
133        let op = if Command::new("op.exe").arg("--version").output().is_ok() { "op.exe" } else { "op" };
134        let read = Command::new(op).args(["read", &reference]).output().map_err(|e| format!("{op}: {e}"))?;
135        let token = String::from_utf8_lossy(&read.stdout).trim().to_owned();
136        if !read.status.success() || token.is_empty() {
137            return Err(format!("{op} read gave no token"));
138        }
139        let agent = ureq::Agent::config_builder()
140            .http_status_as_error(false)
141            .timeout_global(Some(Duration::from_secs(120)))
142            .build()
143            .into();
144        Ok(Cloudflare { agent, account, token })
145    }
146
147    fn run(&self, model: &str, request: &str) -> Result<String, String> {
148        let url = format!("https://api.cloudflare.com/client/v4/accounts/{}/ai/run/{model}", self.account);
149        let mut response = self
150            .agent
151            .post(&url)
152            .header("authorization", &format!("Bearer {}", self.token))
153            .header("content-type", "application/json")
154            .send(request)
155            .map_err(|e| e.to_string())?;
156        response.body_mut().read_to_string().map_err(|e| e.to_string())
157    }
158}
159
160struct Scored {
161    passed: usize,
162    neurons: f64,
163    took: Duration,
164}
165
166fn score(cloudflare: &Cloudflare, model: &Model, verbose: bool) -> Scored {
167    let mut scored = Scored { passed: 0, neurons: 0.0, took: Duration::ZERO };
168    for case in CASES {
169        let started = Instant::now();
170        let (verdict, usage) = match cloudflare.run(model.id, &llm::request(case.input, &[])) {
171            Ok(body) => {
172                if verbose {
173                    println!("    {body}");
174                }
175                judge(case, &body)
176            }
177            Err(error) => (Err(error), llm::Usage::default()),
178        };
179        scored.took += started.elapsed();
180        scored.neurons += model.neurons(usage);
181        match verdict {
182            Ok(()) => {
183                scored.passed += 1;
184                println!("  pass  {}", case.input);
185            }
186            Err(why) => println!("  FAIL  {}\n        {why}", case.input),
187        }
188    }
189    scored
190}
191
192fn main() -> ExitCode {
193    let args: Vec<String> = std::env::args().skip(1).collect();
194    // Every run spends Workers AI neurons from the same free daily allowance the live site runs on (about 20 a
195    // call, 20 calls a model per run); ten runs in an afternoon (2026-10-05) used it all and the site could not draft
196    // a question until the allowance reset at 00:00 UTC. So nothing runs unless the caller says so.
197    if !args.iter().any(|arg| arg == "--spend") {
198        eprintln!(
199            "eval: this spends Workers AI neurons from the SAME daily allowance as the live site (a run is roughly 350 neurons \
200             per model of the 10,000 a day). Run it only when asked to, once, and add --spend to confirm."
201        );
202        return ExitCode::FAILURE;
203    }
204    let args: Vec<String> = args.into_iter().filter(|arg| arg != "--spend").collect();
205    match args.first().map(String::as_str) {
206        Some("facts") => return facts::run(false),
207        Some("halves") => return facts::run(true),
208        _ => {}
209    }
210    let all = args.iter().any(|arg| arg == "--all");
211    let named: Vec<&Model> = args.iter().filter_map(|arg| Model::find(arg)).collect();
212    if let Some(unknown) = args.iter().find(|arg| *arg != "--all" && Model::find(arg).is_none()) {
213        eprintln!("eval: {unknown} is not a candidate (see packages/llm/src/models.rs)");
214        return ExitCode::FAILURE;
215    }
216    let cloudflare = match Cloudflare::from_env() {
217        Ok(cloudflare) => cloudflare,
218        Err(error) => {
219            eprintln!("eval: {error}");
220            return ExitCode::FAILURE;
221        }
222    };
223    let models: Vec<&Model> = if named.is_empty() { CANDIDATES.iter().collect() } else { named.clone() };
224
225    let mut table = Vec::new();
226    let mut winner = None;
227    for model in models {
228        println!("\n{}", model.id);
229        let scored = score(&cloudflare, model, !named.is_empty());
230        let perfect = scored.passed == CASES.len();
231        table.push(format!(
232            "{:>2}/{}  {:>7.1} neurons/call  {:>5} ms/call  {}",
233            scored.passed,
234            CASES.len(),
235            scored.neurons / CASES.len() as f64,
236            scored.took.as_millis() / CASES.len() as u128,
237            model.id
238        ));
239        if perfect && winner.is_none() {
240            winner = Some(model.id);
241            if !all && named.is_empty() {
242                break;
243            }
244        }
245    }
246    println!("\n{}", table.join("\n"));
247    match winner {
248        Some(id) => println!("\ncheapest model that passed every case: {id}"),
249        None => println!("\nno model passed every case"),
250    }
251    ExitCode::SUCCESS
252}