1//! Text to speech through ElevenLabs. The key and the voice stay on this machine. The daily cap is not
2//! here: it is the service's (reserved before a line reaches this, settled after), so it cannot be
3//! forgotten by a second caller.
4
5use std::time::{Duration, Instant};
6
7use log::{debug, error, info, warn};
8use whiskers_ports::{
9    Audio, AudioMs, CreditsError, Diagnostic, LineTiming, SecretName, Secrets, SpeakError, SpeechLine, TimedAudio, Voice, VoiceCredits,
10};
11
12const ENDPOINT: &str = "https://api.elevenlabs.io/v1/text-to-speech";
13/// Used when the configured voice is refused (a 402 or 404): Jessica ("Playful, Bright, Warm"), one of ElevenLabs' public
14/// premade voices, which every plan including the free one may use (checked 2026-10-04). It is a public
15/// catalogue id, not an account secret.
16const FALLBACK_VOICE: &str = "cgSgspJ2msm6clMCkdW9";
17/// Flash v2.5 is the low-latency model.
18const MODEL: &str = "eleven_flash_v2_5";
19
20pub struct ElevenLabsVoice<S> {
21    secrets: S,
22    agent: ureq::Agent,
23}
24
25impl<S: Secrets> ElevenLabsVoice<S> {
26    /// Both `ELEVENLABS_API_KEY` and `ELEVENLABS_VOICE_ID` are required; there is no default voice,
27    /// because a voice id is not something to recall from memory.
28    pub fn new(secrets: S) -> Self {
29        let agent = ureq::Agent::config_builder().timeout_global(Some(Duration::from_secs(20))).build().into();
30        Self { secrets, agent }
31    }
32
33    /// The key and the voice, if both are set. A value that is set but empty is as good as absent, but
34    /// is said so, since that is a mistake worth seeing.
35    async fn credentials(&self) -> Option<(String, String)> {
36        let mut found = Vec::new();
37        for name in [SecretName::VoiceKey, SecretName::VoiceId] {
38            match self.secrets.get(name).await {
39                Ok(whiskers_ports::Lookup::Present(v)) => found.push(Some(v.expose().to_owned())),
40                Ok(whiskers_ports::Lookup::Empty) => {
41                    warn!("{name} is set but empty; the natural voice is not configured");
42                    found.push(None);
43                }
44                Ok(whiskers_ports::Lookup::Absent) => found.push(None),
45                Err(e) => {
46                    error!("the credential {name} could not be looked up: {e}");
47                    found.push(None);
48                }
49            }
50        }
51        let voice = found.pop().flatten();
52        let key = found.pop().flatten();
53        key.zip(voice)
54    }
55
56    /// Logs once, at start-up, whether the natural voice can speak and what is missing if not.
57    pub async fn report_configuration(&self) {
58        let mut set = Vec::new();
59        for name in [SecretName::VoiceKey, SecretName::VoiceId] {
60            set.push(matches!(self.secrets.get(name).await, Ok(whiskers_ports::Lookup::Present(_))));
61        }
62        if set.iter().all(|s| *s) {
63            info!("the natural voice is configured");
64        } else {
65            warn!("ElevenLabs is not configured (key set = {}, voice set = {}); /speak will be refused", set[0], set[1]);
66        }
67    }
68
69    async fn synthesise(&self, line: &SpeechLine, timed: bool) -> Result<Vec<u8>, SpeakError> {
70        let Some((key, voice)) = self.credentials().await else {
71            warn!("speech refused: ElevenLabs is not configured");
72            return Err(SpeakError::NotConfigured);
73        };
74        // Speaks with the configured voice; if the account refuses it (a Library voice on the free plan is
75        // a 402, a removed voice a 404) speaks with FALLBACK_VOICE instead, so a line is never silent
76        // because of a voice choice.
77        match self.call_voice(&key, &voice, line, timed) {
78            Err(SpeakError::Refused { status, .. }) if voice != FALLBACK_VOICE && (status == 402 || status == 404) => {
79                warn!("the configured voice was refused ({status}); speaking with the fallback voice");
80                self.call_voice(&key, FALLBACK_VOICE, line, timed)
81            }
82            other => other,
83        }
84    }
85
86    fn call_voice(&self, key: &str, voice: &str, line: &SpeechLine, timed: bool) -> Result<Vec<u8>, SpeakError> {
87        let payload = serde_json::json!({ "text": line.as_str(), "model_id": MODEL }).to_string();
88        let started = Instant::now();
89        let mut resp = self
90            .agent
91            .post(format!("{ENDPOINT}/{voice}{}?output_format=mp3_44100_64", if timed { "/with-timestamps" } else { "" }))
92            .header("xi-api-key", key)
93            .header("content-type", "application/json")
94            .header("accept", if timed { "application/json" } else { "audio/mpeg" })
95            .config()
96            .http_status_as_error(false)
97            .build()
98            .send(payload)
99            .map_err(|e| {
100                error!("ElevenLabs unreachable after {} ms: {e}", started.elapsed().as_millis());
101                SpeakError::Unreachable(Diagnostic::new(format!("ElevenLabs: {e}")))
102            })?;
103        if !resp.status().is_success() {
104            let why = resp.body_mut().read_to_string().unwrap_or_default();
105            error!("ElevenLabs answered {} in {} ms ({} byte body)", resp.status(), started.elapsed().as_millis(), why.len());
106            return Err(SpeakError::Refused {
107                status: resp.status().as_u16(),
108                said: Diagnostic::new(format!("ElevenLabs: {}: {}", resp.status(), why.chars().take(200).collect::<String>())),
109            });
110        }
111        let audio = resp.body_mut().read_to_vec().map_err(|e| {
112            error!("ElevenLabs audio unreadable after {} ms: {e}", started.elapsed().as_millis());
113            SpeakError::Unreadable(Diagnostic::new(format!("ElevenLabs: {e}")))
114        })?;
115        info!("ElevenLabs answered {} in {} ms: {} chars spoken, {} bytes of audio", resp.status(), started.elapsed().as_millis(), line.chars(), audio.len());
116        Ok(audio)
117    }
118}
119
120impl<S: Secrets> Voice for ElevenLabsVoice<S> {
121    async fn configured(&self) -> bool {
122        self.credentials().await.is_some()
123    }
124
125    async fn speak(&self, line: &SpeechLine) -> Result<Audio, SpeakError> {
126        debug!("speak: {} chars", line.chars());
127        self.synthesise(line, false).await.map(Audio)
128    }
129
130    async fn speak_timed(&self, line: &SpeechLine) -> Result<TimedAudio, SpeakError> {
131        debug!("speak (timed): {} chars", line.chars());
132        let reply = self.synthesise(line, true).await?;
133        timed_from_vendor(&reply, line)
134    }
135
136    /// The voice service's monthly credits (a call out, so only asked for when it is configured).
137    async fn credits(&self) -> Result<VoiceCredits, CreditsError> {
138        let Some((key, _)) = self.credentials().await else {
139            return Err(CreditsError::Unreachable(Diagnostic::new("the voice is not configured")));
140        };
141        subscription(&key)
142    }
143}
144
145/// ElevenLabs' `with-timestamps` reply: the audio, and the time each character of the text is spoken.
146/// (The reply also carries a `normalized_alignment` of the text as it was read out; it is not used.)
147#[derive(serde::Deserialize)]
148struct Timestamped {
149    audio_base64: String,
150    alignment: Alignment,
151}
152
153#[derive(serde::Deserialize)]
154struct Alignment {
155    characters: Vec<String>,
156    character_start_times_seconds: Vec<f64>,
157    character_end_times_seconds: Vec<f64>,
158}
159
160/// ElevenLabs' seconds as whole milliseconds. Its times are whole milliseconds already (three decimals), so
161/// this only undoes the float; anything not a time at all is refused.
162fn milliseconds(seconds: f64) -> Option<AudioMs> {
163    let ms = (seconds * 1000.0).round();
164    (ms.is_finite() && (0.0..=f64::from(u32::MAX)).contains(&ms)).then(|| AudioMs::new(ms as u32))
165}
166
167/// The whole of the vendor-shaped part of the timed voice: ElevenLabs' reply to Whiskers' [`TimedAudio`].
168/// Nothing of ElevenLabs' shape leaves this function. A reply that is not that shape, or whose timing is not
169/// of the line that was spoken, is `Unreadable`: the audio is not used with timing that is wrong.
170pub fn timed_from_vendor(reply: &[u8], line: &SpeechLine) -> Result<TimedAudio, SpeakError> {
171    use base64::Engine as _;
172    let unreadable = |why: String| {
173        warn!("ElevenLabs timed reply is not usable: {why}");
174        SpeakError::Unreadable(Diagnostic::new(format!("ElevenLabs: {why}")))
175    };
176    let Timestamped { audio_base64, alignment } = serde_json::from_slice(reply).map_err(|e| unreadable(format!("the reply does not parse: {e}")))?;
177    let audio = base64::engine::general_purpose::STANDARD.decode(audio_base64).map_err(|e| unreadable(format!("the audio is not base64: {e}")))?;
178    let (starts, ends) = (&alignment.character_start_times_seconds, &alignment.character_end_times_seconds);
179    if alignment.characters.len() != starts.len() || starts.len() != ends.len() {
180        return Err(unreadable(format!(
181            "{} characters with {} starts and {} ends",
182            alignment.characters.len(),
183            starts.len(),
184            ends.len()
185        )));
186    }
187    if let Some(at) = alignment.characters.iter().position(|c| c.chars().count() != 1) {
188        return Err(unreadable(format!("alignment entry {at} is not one character")));
189    }
190    let spans = starts
191        .iter()
192        .zip(ends)
193        .enumerate()
194        .map(|(at, (&s, &e))| milliseconds(s).zip(milliseconds(e)).ok_or_else(|| unreadable(format!("the time of character {at} is not a time"))))
195        .collect::<Result<Vec<_>, _>>()?;
196    let timing = LineTiming::new(alignment.characters.concat(), spans).map_err(|e| unreadable(e.to_string()))?;
197    TimedAudio::new(Audio(audio), timing, line).map_err(|e| unreadable(e.to_string()))
198}
199
200#[derive(serde::Deserialize)]
201struct Subscription {
202    tier: Option<String>,
203    character_count: Option<u64>,
204    character_limit: Option<u64>,
205    next_character_count_reset_unix: Option<u64>,
206}
207
208fn subscription(key: &str) -> Result<VoiceCredits, CreditsError> {
209    let agent: ureq::Agent = ureq::Agent::config_builder().timeout_global(Some(Duration::from_secs(8))).http_status_as_error(false).build().into();
210    let started = Instant::now();
211    let mut resp = agent
212        .get("https://api.elevenlabs.io/v1/user/subscription")
213        .header("xi-api-key", key)
214        .call()
215        .map_err(|e| CreditsError::Unreachable(Diagnostic::new(e.to_string())))?;
216    let status = resp.status();
217    let body = resp.body_mut().read_to_string().map_err(|e| CreditsError::Unreadable(Diagnostic::new(e.to_string())))?;
218    debug!("ElevenLabs subscription answered {status} in {} ms, {} bytes", started.elapsed().as_millis(), body.len());
219    if !status.is_success() {
220        return Err(CreditsError::Refused { status: status.as_u16(), said: Diagnostic::new(format!("ElevenLabs answered {status}")) });
221    }
222    let sub: Subscription = serde_json::from_str(&body).map_err(|e| {
223        warn!("ElevenLabs subscription reply does not parse: {e}");
224        CreditsError::Unreadable(Diagnostic::new(e.to_string()))
225    })?;
226    Ok(VoiceCredits { tier: sub.tier, used: sub.character_count, limit: sub.character_limit, resets_at_unix: sub.next_character_count_reset_unix })
227}
228
229#[cfg(test)]
230mod tests {
231    use super::*;
232    use crate::secrets::EnvSecrets;
233    use whiskers_ports::run_ready;
234
235    fn voice(key: Option<&'static str>, id: Option<&'static str>) -> ElevenLabsVoice<EnvSecrets> {
236        ElevenLabsVoice::new(EnvSecrets::from_fn(move |name| match name {
237            "ELEVENLABS_API_KEY" => key.map(str::to_owned),
238            "ELEVENLABS_VOICE_ID" => id.map(str::to_owned),
239            _ => None,
240        }))
241    }
242
243    fn line() -> SpeechLine {
244        SpeechLine::new("hi".into()).unwrap()
245    }
246
247    #[test]
248    fn without_a_key_or_voice_it_says_so_and_never_calls_out() {
249        for (k, v) in [(None, None), (Some("k"), None), (None, Some("v")), (Some("k"), Some("  "))] {
250            let voice = voice(k, v);
251            assert!(!run_ready(voice.configured()));
252            assert_eq!(run_ready(voice.speak(&line())), Err(SpeakError::NotConfigured));
253            assert_eq!(run_ready(voice.speak_timed(&line())), Err(SpeakError::NotConfigured));
254        }
255    }
256
257    /// ElevenLabs' `with-timestamps` reply for "Hi Paula", in the documented shape: the audio, the
258    /// characters and the second each starts and ends, three decimals (whole milliseconds), then the same
259    /// again as it was read out. Written to the documented schema, not captured live.
260    pub(crate) const REPLY: &str = r#"{"audio_base64":"AAEC","alignment":{"characters":["H","i"," ","P","a","u","l","a"],"character_start_times_seconds":[0.0,0.116,0.186,0.267,0.349,0.43,0.511,0.592],"character_end_times_seconds":[0.116,0.186,0.267,0.349,0.43,0.511,0.592,0.801]},"normalized_alignment":{"characters":["H","i"," ","P","a","u","l","a"],"character_start_times_seconds":[0.0,0.116,0.186,0.267,0.349,0.43,0.511,0.592],"character_end_times_seconds":[0.116,0.186,0.267,0.349,0.43,0.511,0.592,0.801]}}"#;
261
262    fn hi_paula() -> SpeechLine {
263        SpeechLine::new("Hi Paula".into()).unwrap()
264    }
265
266    #[test]
267    fn the_vendors_reply_becomes_whiskers_own_timing_and_nothing_else_of_it_survives() {
268        let t = timed_from_vendor(REPLY.as_bytes(), &hi_paula()).unwrap();
269        assert_eq!(t.audio().0, [0, 1, 2]);
270        assert_eq!(t.timing().text(), "Hi Paula");
271        let said: Vec<(char, u32, u32)> = t.timing().characters().map(|(c, s)| (c, s.start().get(), s.end().get())).collect();
272        assert_eq!(said[0], ('H', 0, 116));
273        assert_eq!(said[1], ('i', 116, 186));
274        assert_eq!(said[7], ('a', 592, 801));
275    }
276
277    #[test]
278    fn a_reply_that_is_not_usable_is_unreadable_and_says_why_without_quoting_the_line() {
279        let line = hi_paula();
280        let cases = [
281            ("not json".to_owned(), "does not parse"),
282            (REPLY.replace("AAEC", "!!"), "not base64"),
283            (REPLY.replace(r#""i"," ""#, r#""i","  ""#), "not one character"),
284            (REPLY.replacen("0.116,0.186", "0.116", 1), "starts"),
285            (REPLY.replacen("0.186,0.267,0.349,0.43,0.511,0.592,0.801", "-1.0,0.267,0.349,0.43,0.511,0.592,0.801", 1), "not a time"),
286            (REPLY.replacen("0.116,0.186,0.267", "0.116,0.1,0.267", 1), "is timed before the one ahead"),
287            (REPLY.replace(r#""characters":["H","i"," ","P","a","u","l","a"]"#, r#""characters":["H","o"," ","P","a","u","l","a"]"#), "other text"),
288        ];
289        for (reply, why) in cases {
290            let Err(SpeakError::Unreadable(d)) = timed_from_vendor(reply.as_bytes(), &line) else { panic!("{why}: {reply}") };
291            assert!(d.to_string().contains(why), "{why}: {d}");
292            assert!(!d.to_string().contains("Paula"), "the line is not repeated in an error: {d}");
293        }
294    }
295
296    #[test]
297    fn with_both_it_is_configured() {
298        assert!(run_ready(voice(Some("k"), Some("v")).configured()));
299    }
300}