21use std::process::{Command, ExitCode};
22use std::time::{Duration, Instant};

24use llm::{CANDIDATES, Draft, Model};
25
26mod facts;
27
28#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
29enum Kind {
30    Noul,
31    Choice,
32    Score,
33}
34
35use Kind::{Choice, Noul, Score};

39struct Case {
40    input: &'static str,
41    accept: &'static [&'static [Kind]],

43    none_first: bool,
44}

46const CASES: &[Case] = &[
47    Case { input: "is water wet?", accept: &[&[Noul]], none_first: false },
48    Case { input: "should I rewrite it in rust", accept: &[&[Noul]], none_first: false },
49    Case { input: "can penguins fly", accept: &[&[Noul]], none_first: false },
50    Case { input: "is a hot dog a sandwich?", accept: &[&[Noul]], none_first: false },
51    Case { input: "what is the best text editor", accept: &[&[Choice]], none_first: false },
52    Case { input: "which programming language should I learn first?", accept: &[&[Choice]], none_first: false },
53    Case { input: "who would win in a fight, a bear or a shark", accept: &[&[Choice]], none_first: false },
54    Case { input: "best pizza topping", accept: &[&[Choice]], none_first: false },
55    Case { input: "how good is the movie The Matrix?", accept: &[&[Score]], none_first: false },
56    // A Noul's probability is itself "how likely".
57    Case { input: "how likely is it to rain in Seattle in November", accept: &[&[Noul]], none_first: false },
58    Case { input: "rate my startup idea: uber for dogs", accept: &[&[Score]], none_first: false },
59    Case { input: "how spicy is a jalapeño", accept: &[&[Score]], none_first: false },
60    // The levels must reach zero: a model once wrote 1 to 5 for a count.
61    Case { input: "how many sheep do androids dream of?", accept: &[&[Score]], none_first: true },
62    Case { input: "how many moons does Mars have", accept: &[&[Score]], none_first: true },
63    // Quotes and symbols have to survive the model's JSON.
64    Case { input: "is \"C++\" better than C#?", accept: &[&[Noul], &[Choice]], none_first: false },
65    // Two separate questions in one input.
66    Case {
67        input: "is python slower than rust, and which of the two is better for a beginner?",
68        accept: &[&[Noul, Choice], &[Noul, Noul]],
69        none_first: false,
70    },
71];
72
73fn kind(draft: &Draft) -> Kind {
74    match draft {
75        Draft::Noul { .. } => Noul,
76        Draft::Choice { .. } => Choice,
77        Draft::Score { .. } => Score,
78    }
79}

82fn judge(case: &Case, body: &str) -> (Result<(), String>, llm::Usage) {
83    let reply = match llm::parse(body) {
84        Ok(reply) => reply,
85        Err(error) => return (Err(error), llm::Usage::default()),
86    };
87    let verdict = (|| {
88        if reply.calls.is_empty() {
89            return Err("made no tool call".to_owned());
90        }
91        if reply.dropped > 0 {
92            return Err(format!("made {} calls, over the cap", reply.calls.len() + reply.dropped));
93        }
94        let mut kinds = Vec::new();
95        for call in &reply.calls {
96            let draft = call.draft.as_ref().map_err(|e| format!("{}: {e}", call.name))?;
97            // The last word is Jev's own protocol.
98            ask::check(draft).map_err(|e| format!("{}: {e}", call.name))?;
99            if case.none_first {
100                if let Draft::Score { levels, .. } = draft {
101                    let first = levels.first().map(|level| level.to_lowercase()).unwrap_or_default();
102                    let low = ["none", "no ", "zero", "nothing", "0", "not any", "never"];
103                    if !low.iter().any(|word| first == word.trim() || first.starts_with(word)) {
104                        return Err(format!("a count's lowest level is {first:?}, not none or zero"));
105                    }
106                }
107            }
108            kinds.push(kind(draft));
109        }
110        kinds.sort();
111        let accepted = case.accept.iter().any(|want| {
112            let mut want = want.to_vec();
113            want.sort();
114            want == kinds
115        });
116        if accepted { Ok(()) } else { Err(format!("called {kinds:?}, wanted one of {:?}", case.accept)) }
117    })();
118    (verdict, reply.usage)
119}

121struct Cloudflare {
122    agent: ureq::Agent,
123    account: String,
124    token: String,
125}
126
127impl Cloudflare {
128    fn from_env() -> Result<Self, String> {
129        let var = |name: &str| std::env::var(name).map_err(|_| format!("{name} is not set; run `lmjtfy-eval` from the devshell"));
130        let account = var("CLOUDFLARE_ACCOUNT_ID")?;
131        let reference = format!("{}/Token", var("LMJTFY_OP_ITEM_REF")?);
132        // op.exe under WSL (it has the desktop app's session), op elsewhere.
133        let op = if Command::new("op.exe").arg("--version").output().is_ok() { "op.exe" } else { "op" };
134        let read = Command::new(op).args(["read", &reference]).output().map_err(|e| format!("{op}: {e}"))?;
135        let token = String::from_utf8_lossy(&read.stdout).trim().to_owned();
136        if !read.status.success() || token.is_empty() {
137            return Err(format!("{op} read gave no token"));
138        }
139        let agent = ureq::Agent::config_builder()
140            .http_status_as_error(false)
141            .timeout_global(Some(Duration::from_secs(120)))
142            .build()
143            .into();
144        Ok(Cloudflare { agent, account, token })
145    }
146
147    fn run(&self, model: &str, request: &str) -> Result<String, String> {
148        let url = format!("https://api.cloudflare.com/client/v4/accounts/{}/ai/run/{model}", self.account);
149        let mut response = self
150            .agent
151            .post(&url)
152            .header("authorization", &format!("Bearer {}", self.token))
153            .header("content-type", "application/json")
154            .send(request)
155            .map_err(|e| e.to_string())?;
156        response.body_mut().read_to_string().map_err(|e| e.to_string())
157    }
158}
159
160struct Scored {
161    passed: usize,
162    neurons: f64,
163    took: Duration,
164}
165
166fn score(cloudflare: &Cloudflare, model: &Model, verbose: bool) -> Scored {
167    let mut scored = Scored { passed: 0, neurons: 0.0, took: Duration::ZERO };
168    for case in CASES {
169        let started = Instant::now();
170        let (verdict, usage) = match cloudflare.run(model.id, &llm::request(case.input, &[])) {
171            Ok(body) => {
172                if verbose {
173                    println!("    {body}");
174                }
175                judge(case, &body)
176            }
177            Err(error) => (Err(error), llm::Usage::default()),
178        };
179        scored.took += started.elapsed();
180        scored.neurons += model.neurons(usage);
181        match verdict {
182            Ok(()) => {
183                scored.passed += 1;
184                println!("  pass  {}", case.input);
185            }
186            Err(why) => println!("  FAIL  {}\n        {why}", case.input),
187        }
188    }
189    scored
190}
191
192fn main() -> ExitCode {
193    let args: Vec<String> = std::env::args().skip(1).collect();
194    match args.first().map(String::as_str) {
195        Some("facts") => return facts::run(false),
196        Some("halves") => return facts::run(true),
197        _ => {}
198    }
199    let all = args.iter().any(|arg| arg == "--all");
200    let named: Vec<&Model> = args.iter().filter_map(|arg| Model::find(arg)).collect();
201    if let Some(unknown) = args.iter().find(|arg| *arg != "--all" && Model::find(arg).is_none()) {
202        eprintln!("eval: {unknown} is not a candidate (see packages/llm/src/models.rs)");
203        return ExitCode::FAILURE;
204    }
205    let cloudflare = match Cloudflare::from_env() {
206        Ok(cloudflare) => cloudflare,
207        Err(error) => {
208            eprintln!("eval: {error}");
209            return ExitCode::FAILURE;
210        }
211    };
212    let models: Vec<&Model> = if named.is_empty() { CANDIDATES.iter().collect() } else { named.clone() };
213
214    let mut table = Vec::new();
215    let mut winner = None;
216    for model in models {
217        println!("\n{}", model.id);
218        let scored = score(&cloudflare, model, !named.is_empty());
219        let perfect = scored.passed == CASES.len();
220        table.push(format!(
221            "{:>2}/{}  {:>7.1} neurons/call  {:>5} ms/call  {}",
222            scored.passed,
223            CASES.len(),
224            scored.neurons / CASES.len() as f64,
225            scored.took.as_millis() / CASES.len() as u128,
226            model.id
227        ));
228        if perfect && winner.is_none() {
229            winner = Some(model.id);
230            if !all && named.is_empty() {
231                break;
232            }
233        }
234    }
235    println!("\n{}", table.join("\n"));
236    match winner {
237        Some(id) => println!("\ncheapest model that passed every case: {id}"),
238        None => println!("\nno model passed every case"),
239    }
240    ExitCode::SUCCESS
241}