1package app.whiskers
2
3import android.content.Context
4import android.media.AudioAttributes
5import android.media.MediaPlayer
6import android.os.Handler
7import android.os.Looper
8import android.os.SystemClock
9import android.speech.tts.TextToSpeech
10import android.speech.tts.UtteranceProgressListener
11import kotlinx.coroutines.Dispatchers
12import kotlinx.coroutines.suspendCancellableCoroutine
13import kotlinx.coroutines.withContext
14import org.json.JSONObject
15import java.io.File
16import java.net.HttpURLConnection
17import java.net.URL
18import java.util.Locale
19import kotlin.coroutines.resume
20
21/** The name of a TextToSpeech error code, for the logs. */
22internal fun ttsErrorName(code: Int): String = when (code) {
23 TextToSpeech.ERROR_SYNTHESIS -> "SYNTHESIS"
24 TextToSpeech.ERROR_SERVICE -> "SERVICE"
25 TextToSpeech.ERROR_OUTPUT -> "OUTPUT"
26 TextToSpeech.ERROR_NETWORK -> "NETWORK"
27 TextToSpeech.ERROR_NETWORK_TIMEOUT -> "NETWORK_TIMEOUT"
28 TextToSpeech.ERROR_INVALID_REQUEST -> "INVALID_REQUEST"
29 TextToSpeech.ERROR_NOT_INSTALLED_YET -> "NOT_INSTALLED_YET"
30 else -> "UNKNOWN"
31} + "($code)"
32
33/** Speaks what the core says. Only text that came back from the core ever reaches this. */
34interface Mouth {
35 /**
36 * Returns when the line has been spoken (or could not be). [onStart] is called at the moment
37 * the sound begins, with a clock for the lyric caption, so the words appear as they are said
38 * and not before.
39 */
40 suspend fun say(text: String, voice: Voice, onStart: (CaptionClock) -> Unit)
41
42 /** Stop speaking at once. */
43 fun stop()
44}
45
46/**
47 * The natural voice (ElevenLabs, fetched through whiskersd so the key stays off the tablet, with
48 * the time of every character), the offline neural voice that lives in the app, and the tablet's
49 * own voice. A line is tried in that order, each only if the one before gave no sound, and a
50 * tablet voice that makes none ends in paced captions, so she always hears or reads something.
51 */
52class Voices(private val context: Context) : Mouth {
53 /** Where the natural voice is fetched from. While [ServiceConfig.Unset] there is none, and a line goes straight to the voices on the device. */
54 @Volatile private var service: ServiceConfig = ServiceConfig.Unset
55
56 fun useService(config: ServiceConfig) {
57 service = config
58 Wlog.i(Area.Mouth, "natural voice source -> ${if (config is ServiceConfig.Set) "service" else "none"}")
59 }
60
61 private var tts: TextToSpeech? = null
62 @Volatile private var ttsReady = false
63 private var player: MediaPlayer? = null
64 private val offline = OfflineVoice.create(context)
65
66 /** Ends the line being spoken, as "done": set while a line is playing, run by [stop]. */
67 @Volatile private var interrupt: (() -> Unit)? = null
68
69 /** Bumped by [stop], so a line still being fetched when she was silenced is not played. */
70 @Volatile private var epoch = 0
71
72 private fun engine(): TextToSpeech {
73 tts?.let { return it }
74 Wlog.i(Area.Mouth, "tts: creating engine")
75 val began = SystemClock.uptimeMillis()
76 val t = TextToSpeech(context) { status ->
77 ttsReady = status == TextToSpeech.SUCCESS
78 if (ttsReady) {
79 val set = tts?.setLanguage(Locale.US)
80 Wlog.i(Area.Mouth, "tts: init SUCCESS after ${SystemClock.uptimeMillis() - began} ms, setLanguage(US)=$set")
81 } else {
82 Wlog.e(Area.Mouth, "tts: init FAILED status=$status (ERROR=${TextToSpeech.ERROR}) after ${SystemClock.uptimeMillis() - began} ms")
83 }
84 }
85 tts = t
86 return t
87 }
88
89 override suspend fun say(text: String, voice: Voice, onStart: (CaptionClock) -> Unit) {
90 // Everything below gives up if she silenced Whiskers meanwhile, including the fallbacks:
91 // a natural line that failed after being stopped must not be spoken again.
92 val asked = epoch
93 Wlog.i(Area.Mouth, "say: voice=$voice, ${text.length} chars")
94 when (voice) {
95 Voice.Natural -> {
96 // Loads while the natural line is fetched, so a failure falls to a voice that is already ready.
97 offline.warm()
98 if (!natural(text, onStart) && asked == epoch) offlineThenTablet(text, onStart, asked)
99 }
100 Voice.OnDevice -> offlineThenTablet(text, onStart, asked)
101 }
102 }
103 private suspend fun offlineThenTablet(text: String, onStart: (CaptionClock) -> Unit, asked: Int) {
104 if (offline.available()) {
105 val spoke = try {
106 offline.say(text, onStart)
107 } catch (e: Exception) {
108 if (e is kotlinx.coroutines.CancellationException) throw e
109 Wlog.w(Area.Mouth, "offline voice threw (${e.javaClass.simpleName})", e)
110 false
111 }
112 if (spoke || asked != epoch) return
113 Wlog.w(Area.Mouth, "offline voice gave no sound: falling back to the tablet voice")
114 } else {
115 Wlog.d(Area.Mouth, "offline voice not available: using the tablet voice")
116 }
117 onDevice(text, onStart, asked)
118 }
119
120 override fun stop() {
121 epoch++
122 Wlog.d(Area.Mouth, "stop: epoch=$epoch, line playing=${interrupt != null}")
123 // Resume whoever is waiting for the line to end; a stopped line is a finished one,
124 // not a failed one (a failure would be spoken again in the tablet's voice).
125 interrupt?.invoke()
126 interrupt = null
127 offline.stop()
128 tts?.stop()
129 }
130
131 // ---- the natural voice -------------------------------------------------
132
133 private suspend fun natural(text: String, onStart: (CaptionClock) -> Unit): Boolean = if (DebugBridge.naturalOff) {
134 Wlog.w(Area.Mouth, "natural voice skipped (debug switch)")
135 false
136 } else try {
137 val asked = epoch
138 val timed = withContext(Dispatchers.IO) { NetActivity.around { fetch(text) } }
139 if (timed == null) {
140 Wlog.w(Area.Mouth, "natural voice unavailable: falling back to the tablet voice")
141 false
142 } else if (asked != epoch) {
143 Wlog.d(Area.Mouth, "natural line fetched but she silenced Whiskers meanwhile; not playing")
144 true
145 } else {
146 play(text, timed, onStart)
147 }
148 } catch (e: Exception) {
149 if (e is kotlinx.coroutines.CancellationException) Wlog.d(Area.Mouth, "natural voice cancelled")
150 else Wlog.w(Area.Mouth, "natural voice failed (${e.javaClass.simpleName}): falling back to the tablet voice", e)
151 false
152 }
153
154 private fun fetch(text: String): TimedSpeech? {
155 val address = (service as? ServiceConfig.Set)?.address ?: run {
156 Wlog.d(Area.Mouth, "natural: no service address, skipping")
157 return null
158 }
159 val began = SystemClock.uptimeMillis()
160 Wlog.d(Area.Mouth, "natural: fetching ${text.length} chars")
161 val conn = URL(address.endpoint(SPEAK_TIMED_PATH)).openConnection() as HttpURLConnection
162 return try {
163 conn.requestMethod = "POST"
164 conn.connectTimeout = 5_000
165 conn.readTimeout = 20_000
166 conn.doOutput = true
167 conn.setRequestProperty("content-type", "application/json")
168 conn.outputStream.use { it.write(JSONObject().put("text", text).toString().toByteArray()) }
169 val code = conn.responseCode
170 if (code != 200) {
171 Wlog.w(Area.Mouth, "natural: service answered HTTP $code after ${SystemClock.uptimeMillis() - began} ms")
172 return null
173 }
174 val body = conn.inputStream.use { it.readBytes() }
175 val parsed = parseTimedSpeech(body.toString(Charsets.UTF_8))
176 Wlog.i(Area.Mouth, "natural: HTTP 200, ${body.size} bytes in ${SystemClock.uptimeMillis() - began} ms, parsed=${parsed != null}, audio=${parsed?.audio?.size ?: 0} bytes")
177 parsed
178 } finally {
179 conn.disconnect()
180 }
181 }
182
183 private suspend fun play(text: String, timed: TimedSpeech, onStart: (CaptionClock) -> Unit): Boolean = withContext(Dispatchers.Main.immediate) {
184 val file = File.createTempFile("voice", ".mp3", context.cacheDir)
185 file.writeBytes(timed.audio)
186 val exact = timed.timelineFor(text)
187 Wlog.d(Area.Mouth, "player: ${timed.audio.size} bytes written, exact timeline=${exact != null}")
188 suspendCancellableCoroutine { cont ->
189 val p = MediaPlayer()
190 player = p
191 var done = false
192 fun finish(ok: Boolean) {
193 if (done) return
194 done = true
195 Wlog.d(Area.Mouth, "player: finish ok=$ok")
196 if (player === p) player = null
197 interrupt = null
198 runCatching { p.release() }.onFailure { Wlog.w(Area.Mouth, "player: release failed", it) }
199 file.delete()
200 if (cont.isActive) cont.resume(ok)
201 }
202 interrupt = {
203 runCatching { p.stop() }.onFailure { Wlog.w(Area.Mouth, "player: stop failed", it) }
204 finish(true)
205 }
206 try {
207 p.setAudioAttributes(AudioAttributes.Builder().setContentType(AudioAttributes.CONTENT_TYPE_SPEECH).build())
208 p.setDataSource(file.path)
209 p.setOnCompletionListener { Wlog.d(Area.Mouth, "player: completed"); finish(true) }
210 p.setOnErrorListener { _, what, extra ->
211 Wlog.e(Area.Mouth, "player error what=$what extra=$extra")
212 finish(false)
213 true
214 }
215 p.setOnPreparedListener {
216 Wlog.d(Area.Mouth, "player: prepared, duration ${it.duration} ms; starting")
217 val syllables = exact ?: estimatedTimeline(text, it.duration.toLong())
218 it.start()
219 // The words appear now, as the sound does.
220 onStart(
221 CaptionClock {
222 if (done) progressAt(syllables, 0, finished = true)
223 else progressAt(syllables, runCatching { p.currentPosition.toLong() }.onFailure { Wlog.v(Area.Mouth, "player: position unreadable (released?)") }.getOrDefault(0L))
224 },
225 )
226 }
227 p.prepareAsync()
228 } catch (e: Exception) {
229 Wlog.e(Area.Mouth, "player: setup failed", e)
230 finish(false)
231 }
232 cont.invokeOnCancellation { Wlog.d(Area.Mouth, "player: cancelled"); finish(false) }
233 }
234 }
235
236 // ---- the tablet's own voice ---------------------------------------------
237
238 private suspend fun onDevice(text: String, onStart: (CaptionClock) -> Unit, asked: Int) = withContext(Dispatchers.Main.immediate) {
239 val t = engine()
240 // The engine starts asynchronously; wait briefly for it on first use.
241 var waited = 0
242 while (!ttsReady && waited < 50 && asked == epoch) {
243 kotlinx.coroutines.delay(100)
244 waited++
245 }
246 if (waited > 0) Wlog.d(Area.Mouth, "tts: waited ${waited * 100} ms for the engine, ready=$ttsReady")
247 if (asked != epoch) {
248 Wlog.d(Area.Mouth, "on-device: silenced before speaking")
249 return@withContext
250 }
251 if (ttsReady && speakOnDevice(t, text, onStart, asked)) return@withContext
252 if (asked != epoch) return@withContext
253 Wlog.w(Area.Mouth, "on-device voice gave no sound (ready=$ttsReady): shutting the engine down and showing paced captions instead")
254 // No working voice on this tablet (it failed to start, or started and never made a sound): throw
255 // the engine away so the next line tries again, and let her read the words, paced as if spoken.
256 runCatching { t.shutdown() }.onFailure { Wlog.w(Area.Mouth, "tts: shutdown failed", it) }
257 tts = null
258 val clock = WordClock(text, SystemClock::uptimeMillis).also { it.begin() }
259 onStart(clock)
260 kotlinx.coroutines.delay(text.length * 75L)
261 clock.finished()
262 }
263
264 /** True if the engine really spoke the line (or was silenced meanwhile); false if it errored or never started. */
265 private suspend fun speakOnDevice(t: TextToSpeech, text: String, onStart: (CaptionClock) -> Unit, asked: Int): Boolean =
266 suspendCancellableCoroutine { cont ->
267 val id = "whiskers-${System.nanoTime()}"
268 Wlog.d(Area.Mouth, "tts: speaking ${text.length} chars as $id")
269 val clock = WordClock(text, SystemClock::uptimeMillis)
270 val handler = Handler(Looper.getMainLooper())
271 var started = false
272 fun end(spoke: Boolean) {
273 handler.removeCallbacksAndMessages(null)
274 clock.finished(); interrupt = null
275 if (cont.isActive) cont.resume(spoke)
276 }
277 interrupt = { t.stop(); end(true) }
278 t.setOnUtteranceProgressListener(object : UtteranceProgressListener() {
279 // Callbacks of a line that was stopped must not end the next one.
280 override fun onStart(utteranceId: String?) {
281 if (utteranceId == id) {
282 Wlog.d(Area.Mouth, "tts: onStart")
283 started = true; handler.removeCallbacksAndMessages(null); clock.begin(); onStart(clock)
284 }
285 }
286 override fun onRangeStart(utteranceId: String?, start: Int, end: Int, frame: Int) { if (utteranceId == id) clock.rangeStarted(start, end) }
287 override fun onDone(utteranceId: String?) {
288 if (utteranceId == id) { Wlog.d(Area.Mouth, "tts: onDone (started=$started)"); end(started) }
289 }
290 override fun onStop(utteranceId: String?, interrupted: Boolean) {
291 if (utteranceId == id) { Wlog.d(Area.Mouth, "tts: onStop interrupted=$interrupted"); end(started || asked != epoch) }
292 }
293 @Deprecated("Deprecated in Java")
294 override fun onError(utteranceId: String?) {
295 if (utteranceId == id) { Wlog.e(Area.Mouth, "tts: onError (no code)"); end(started) }
296 }
297 override fun onError(utteranceId: String?, errorCode: Int) {
298 if (utteranceId == id) { Wlog.e(Area.Mouth, "tts: onError code=${ttsErrorName(errorCode)}"); end(started) }
299 }
300 })
301 if (t.speak(text, TextToSpeech.QUEUE_FLUSH, null, id) == TextToSpeech.ERROR) {
302 Wlog.e(Area.Mouth, "tts: speak() refused the line (returned ERROR)")
303 end(false)
304 } else {
305 // An engine without its voice data can accept a line and never start it.
306 handler.postDelayed({ if (!started) { Wlog.e(Area.Mouth, "tts: never started within 5 s"); t.stop(); end(false) } }, 5_000)
307 }
308 cont.invokeOnCancellation { Wlog.d(Area.Mouth, "tts: cancelled"); handler.removeCallbacksAndMessages(null); t.stop() }
309 }
310
311 fun shutdown() {
312 Wlog.i(Area.Mouth, "shutdown")
313 stop()
314 offline.shutdown()
315 tts?.shutdown()
316 tts = null
317 }
318}
Mouth.kt