speak.rsText to speech through ElevenLabs. The key and the voice stay on this machine. The daily cap is not here: it is the service's (reserved before a line reaches this, settled after), so it cannot be forgotten by a second caller.
5use std::time::{Duration, Instant};
Used when the configured voice is refused (a 402 or 404): Jessica ("Playful, Bright, Warm"), one of ElevenLabs' public premade voices, which every plan including the free one may use (checked 2026-10-04). It is a public catalogue id, not an account secret.
16const FALLBACK_VOICE: &str = "cgSgspJ2msm6clMCkdW9";
Flash v2.5 is the low-latency model.
18const MODEL: &str = "eleven_flash_v2_5";
Both ELEVENLABS_API_KEY and ELEVENLABS_VOICE_ID are required; there is no default voice,
because a voice id is not something to recall from memory.
The key and the voice, if both are set. A value that is set but empty is as good as absent, but is said so, since that is a mistake worth seeing.
35 async fn credentials(&self) -> Option<(String, String)> { 36 let mut found = Vec::new(); 37 for name in [SecretName::VoiceKey, SecretName::VoiceId] { 38 match self.secrets.get(name).await { 39 Ok(whiskers_ports::Lookup::Present(v)) => found.push(Some(v.expose().to_owned())), 40 Ok(whiskers_ports::Lookup::Empty) => { 41 warn!("{name} is set but empty; the natural voice is not configured"); 42 found.push(None); 43 } 44 Ok(whiskers_ports::Lookup::Absent) => found.push(None), 45 Err(e) => { 46 error!("the credential {name} could not be looked up: {e}"); 47 found.push(None); 48 } 49 } 50 } 51 let voice = found.pop().flatten(); 52 let key = found.pop().flatten(); 53 key.zip(voice) 54 }
Logs once, at start-up, whether the natural voice can speak and what is missing if not.
57 pub async fn report_configuration(&self) { 58 let mut set = Vec::new(); 59 for name in [SecretName::VoiceKey, SecretName::VoiceId] { 60 set.push(matches!(self.secrets.get(name).await, Ok(whiskers_ports::Lookup::Present(_)))); 61 } 62 if set.iter().all(|s| *s) { 63 info!("the natural voice is configured"); 64 } else { 65 warn!("ElevenLabs is not configured (key set = {}, voice set = {}); /speak will be refused", set[0], set[1]); 66 } 67 }
69 async fn synthesise(&self, line: &SpeechLine, timed: bool) -> Result<Vec<u8>, SpeakError> { 70 let Some((key, voice)) = self.credentials().await else { 71 warn!("speech refused: ElevenLabs is not configured"); 72 return Err(SpeakError::NotConfigured); 73 }; 74 // Speaks with the configured voice; if the account refuses it (a Library voice on the free plan is 75 // a 402, a removed voice a 404) speaks with FALLBACK_VOICE instead, so a line is never silent 76 // because of a voice choice. 77 match self.call_voice(&key, &voice, line, timed) { 78 Err(SpeakError::Refused { status, .. }) if voice != FALLBACK_VOICE && (status == 402 || status == 404) => { 79 warn!("the configured voice was refused ({status}); speaking with the fallback voice"); 80 self.call_voice(&key, FALLBACK_VOICE, line, timed) 81 } 82 other => other, 83 } 84 } 85 86 fn call_voice(&self, key: &str, voice: &str, line: &SpeechLine, timed: bool) -> Result<Vec<u8>, SpeakError> { 87 let payload = serde_json::json!({ "text": line.as_str(), "model_id": MODEL }).to_string(); 88 let started = Instant::now(); 89 let mut resp = self 90 .agent 91 .post(format!("{ENDPOINT}/{voice}{}?output_format=mp3_44100_64", if timed { "/with-timestamps" } else { "" })) 92 .header("xi-api-key", key) 93 .header("content-type", "application/json") 94 .header("accept", if timed { "application/json" } else { "audio/mpeg" }) 95 .config() 96 .http_status_as_error(false) 97 .build() 98 .send(payload) 99 .map_err(|e| { 100 error!("ElevenLabs unreachable after {} ms: {e}", started.elapsed().as_millis()); 101 SpeakError::Unreachable(Diagnostic::new(format!("ElevenLabs: {e}"))) 102 })?; 103 if !resp.status().is_success() { 104 let why = resp.body_mut().read_to_string().unwrap_or_default(); 105 error!("ElevenLabs answered {} in {} ms ({} byte body)", resp.status(), started.elapsed().as_millis(), why.len()); 106 return Err(SpeakError::Refused { 107 status: resp.status().as_u16(), 108 said: Diagnostic::new(format!("ElevenLabs: {}: {}", resp.status(), why.chars().take(200).collect::<String>())), 109 }); 110 } 111 let audio = resp.body_mut().read_to_vec().map_err(|e| { 112 error!("ElevenLabs audio unreadable after {} ms: {e}", started.elapsed().as_millis()); 113 SpeakError::Unreadable(Diagnostic::new(format!("ElevenLabs: {e}"))) 114 })?; 115 info!("ElevenLabs answered {} in {} ms: {} chars spoken, {} bytes of audio", resp.status(), started.elapsed().as_millis(), line.chars(), audio.len()); 116 Ok(audio) 117 } 118} 119 120impl<S: Secrets> Voice for ElevenLabsVoice<S> { 121 async fn configured(&self) -> bool { 122 self.credentials().await.is_some() 123 } 124 125 async fn speak(&self, line: &SpeechLine) -> Result<Audio, SpeakError> { 126 debug!("speak: {} chars", line.chars()); 127 self.synthesise(line, false).await.map(Audio) 128 } 129 130 async fn speak_timed(&self, line: &SpeechLine) -> Result<TimedAudio, SpeakError> { 131 debug!("speak (timed): {} chars", line.chars()); 132 let reply = self.synthesise(line, true).await?; 133 timed_from_vendor(&reply, line) 134 }
The voice service's monthly credits (a call out, so only asked for when it is configured).
ElevenLabs' with-timestamps reply: the audio, and the time each character of the text is spoken.
(The reply also carries a normalized_alignment of the text as it was read out; it is not used.)
ElevenLabs' seconds as whole milliseconds. Its times are whole milliseconds already (three decimals), so this only undoes the float; anything not a time at all is refused.
The whole of the vendor-shaped part of the timed voice: ElevenLabs' reply to Whiskers' [TimedAudio].
Nothing of ElevenLabs' shape leaves this function. A reply that is not that shape, or whose timing is not
of the line that was spoken, is Unreadable: the audio is not used with timing that is wrong.
170pub fn timed_from_vendor(reply: &[u8], line: &SpeechLine) -> Result<TimedAudio, SpeakError> { 171 use base64::Engine as _; 172 let unreadable = |why: String| { 173 warn!("ElevenLabs timed reply is not usable: {why}"); 174 SpeakError::Unreadable(Diagnostic::new(format!("ElevenLabs: {why}"))) 175 }; 176 let Timestamped { audio_base64, alignment } = serde_json::from_slice(reply).map_err(|e| unreadable(format!("the reply does not parse: {e}")))?; 177 let audio = base64::engine::general_purpose::STANDARD.decode(audio_base64).map_err(|e| unreadable(format!("the audio is not base64: {e}")))?; 178 let (starts, ends) = (&alignment.character_start_times_seconds, &alignment.character_end_times_seconds); 179 if alignment.characters.len() != starts.len() || starts.len() != ends.len() { 180 return Err(unreadable(format!( 181 "{} characters with {} starts and {} ends", 182 alignment.characters.len(), 183 starts.len(), 184 ends.len() 185 ))); 186 } 187 if let Some(at) = alignment.characters.iter().position(|c| c.chars().count() != 1) { 188 return Err(unreadable(format!("alignment entry {at} is not one character"))); 189 } 190 let spans = starts 191 .iter() 192 .zip(ends) 193 .enumerate() 194 .map(|(at, (&s, &e))| milliseconds(s).zip(milliseconds(e)).ok_or_else(|| unreadable(format!("the time of character {at} is not a time")))) 195 .collect::<Result<Vec<_>, _>>()?; 196 let timing = LineTiming::new(alignment.characters.concat(), spans).map_err(|e| unreadable(e.to_string()))?; 197 TimedAudio::new(Audio(audio), timing, line).map_err(|e| unreadable(e.to_string())) 198}
200#[derive(serde::Deserialize)] 201struct Subscription { 202 tier: Option<String>, 203 character_count: Option<u64>, 204 character_limit: Option<u64>, 205 next_character_count_reset_unix: Option<u64>, 206} 207 208fn subscription(key: &str) -> Result<VoiceCredits, CreditsError> { 209 let agent: ureq::Agent = ureq::Agent::config_builder().timeout_global(Some(Duration::from_secs(8))).http_status_as_error(false).build().into(); 210 let started = Instant::now(); 211 let mut resp = agent 212 .get("https://api.elevenlabs.io/v1/user/subscription") 213 .header("xi-api-key", key) 214 .call() 215 .map_err(|e| CreditsError::Unreachable(Diagnostic::new(e.to_string())))?; 216 let status = resp.status(); 217 let body = resp.body_mut().read_to_string().map_err(|e| CreditsError::Unreadable(Diagnostic::new(e.to_string())))?; 218 debug!("ElevenLabs subscription answered {status} in {} ms, {} bytes", started.elapsed().as_millis(), body.len()); 219 if !status.is_success() { 220 return Err(CreditsError::Refused { status: status.as_u16(), said: Diagnostic::new(format!("ElevenLabs answered {status}")) }); 221 } 222 let sub: Subscription = serde_json::from_str(&body).map_err(|e| { 223 warn!("ElevenLabs subscription reply does not parse: {e}"); 224 CreditsError::Unreadable(Diagnostic::new(e.to_string())) 225 })?; 226 Ok(VoiceCredits { tier: sub.tier, used: sub.character_count, limit: sub.character_limit, resets_at_unix: sub.next_character_count_reset_unix }) 227} 228 229#[cfg(test)] 230mod tests { 231 use super::*; 232 use crate::secrets::EnvSecrets; 233 use whiskers_ports::run_ready; 234 235 fn voice(key: Option<&'static str>, id: Option<&'static str>) -> ElevenLabsVoice<EnvSecrets> { 236 ElevenLabsVoice::new(EnvSecrets::from_fn(move |name| match name { 237 "ELEVENLABS_API_KEY" => key.map(str::to_owned), 238 "ELEVENLABS_VOICE_ID" => id.map(str::to_owned), 239 _ => None, 240 })) 241 } 242 243 fn line() -> SpeechLine { 244 SpeechLine::new("hi".into()).unwrap() 245 } 246 247 #[test] 248 fn without_a_key_or_voice_it_says_so_and_never_calls_out() { 249 for (k, v) in [(None, None), (Some("k"), None), (None, Some("v")), (Some("k"), Some(" "))] { 250 let voice = voice(k, v); 251 assert!(!run_ready(voice.configured())); 252 assert_eq!(run_ready(voice.speak(&line())), Err(SpeakError::NotConfigured)); 253 assert_eq!(run_ready(voice.speak_timed(&line())), Err(SpeakError::NotConfigured)); 254 } 255 }
ElevenLabs' with-timestamps reply for "Hi Paula", in the documented shape: the audio, the
characters and the second each starts and ends, three decimals (whole milliseconds), then the same
again as it was read out. Written to the documented schema, not captured live.
260 pub(crate) const REPLY: &str = r#"{"audio_base64":"AAEC","alignment":{"characters":["H","i"," ","P","a","u","l","a"],"character_start_times_seconds":[0.0,0.116,0.186,0.267,0.349,0.43,0.511,0.592],"character_end_times_seconds":[0.116,0.186,0.267,0.349,0.43,0.511,0.592,0.801]},"normalized_alignment":{"characters":["H","i"," ","P","a","u","l","a"],"character_start_times_seconds":[0.0,0.116,0.186,0.267,0.349,0.43,0.511,0.592],"character_end_times_seconds":[0.116,0.186,0.267,0.349,0.43,0.511,0.592,0.801]}}"#;
262 fn hi_paula() -> SpeechLine { 263 SpeechLine::new("Hi Paula".into()).unwrap() 264 } 265 266 #[test] 267 fn the_vendors_reply_becomes_whiskers_own_timing_and_nothing_else_of_it_survives() { 268 let t = timed_from_vendor(REPLY.as_bytes(), &hi_paula()).unwrap(); 269 assert_eq!(t.audio().0, [0, 1, 2]); 270 assert_eq!(t.timing().text(), "Hi Paula"); 271 let said: Vec<(char, u32, u32)> = t.timing().characters().map(|(c, s)| (c, s.start().get(), s.end().get())).collect(); 272 assert_eq!(said[0], ('H', 0, 116)); 273 assert_eq!(said[1], ('i', 116, 186)); 274 assert_eq!(said[7], ('a', 592, 801)); 275 } 276 277 #[test] 278 fn a_reply_that_is_not_usable_is_unreadable_and_says_why_without_quoting_the_line() { 279 let line = hi_paula(); 280 let cases = [ 281 ("not json".to_owned(), "does not parse"), 282 (REPLY.replace("AAEC", "!!"), "not base64"), 283 (REPLY.replace(r#""i"," ""#, r#""i"," ""#), "not one character"), 284 (REPLY.replacen("0.116,0.186", "0.116", 1), "starts"), 285 (REPLY.replacen("0.186,0.267,0.349,0.43,0.511,0.592,0.801", "-1.0,0.267,0.349,0.43,0.511,0.592,0.801", 1), "not a time"), 286 (REPLY.replacen("0.116,0.186,0.267", "0.116,0.1,0.267", 1), "is timed before the one ahead"), 287 (REPLY.replace(r#""characters":["H","i"," ","P","a","u","l","a"]"#, r#""characters":["H","o"," ","P","a","u","l","a"]"#), "other text"), 288 ]; 289 for (reply, why) in cases { 290 let Err(SpeakError::Unreadable(d)) = timed_from_vendor(reply.as_bytes(), &line) else { panic!("{why}: {reply}") }; 291 assert!(d.to_string().contains(why), "{why}: {d}"); 292 assert!(!d.to_string().contains("Paula"), "the line is not repeated in an error: {d}"); 293 } 294 } 295 296 #[test] 297 fn with_both_it_is_configured() { 298 assert!(run_ready(voice(Some("k"), Some("v")).configured())); 299 } 300}