whiskers.git/crates/whiskers-ports/src/timing.rs

whiskers.git / crates / whiskers-ports / src / timing.rs

When each character of a spoken line is said: what the app needs to show the words as they are spoken (the bouncing ball over the lyrics, the mouth). This is Whiskers' own vocabulary. It is deliberately not the shape any voice vendor happens to send; an adapter converts into it and nothing vendor-shaped crosses the port.

6use std::fmt;

8use serde::Serialize;
9
10use crate::voice::{Audio, SpeechLine};

A moment inside one spoken line's audio, in milliseconds from its start. Not a Millis, which is an instant on the clock: the two cannot be confused or compared.

14#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
15#[serde(transparent)]
16pub struct AudioMs(u32);

18impl AudioMs {
19    pub const fn new(ms: u32) -> Self {
20        Self(ms)
21    }
22
23    pub const fn get(self) -> u32 {
24        self.0
25    }
26}

The time one character is spoken in: it starts at start and is done at end, never before.

29#[derive(Clone, Copy, Debug, PartialEq, Eq)]
30pub struct Span {
31    start: AudioMs,
32    end: AudioMs,
33}

35impl Span {
36    pub const fn start(self) -> AudioMs {
37        self.start
38    }
39
40    pub const fn end(self) -> AudioMs {
41        self.end
42    }
43}

Why a timing, or a timed line, was refused. index counts characters (Unicode scalar values, the unit [SpeechLine::chars] counts in) from zero.

47#[derive(Clone, Copy, Debug, PartialEq, Eq)]
48pub enum TimingError {

There is no text to time.

50    Empty,

The text has characters characters and spans spans were given: one each, or it is not a timing.

52    Count { characters: usize, spans: usize },

Character index is done before it starts.

54    Reversed { index: usize },

Character index starts, or is done, before the one ahead of it does. Speech does not run backwards.

56    OutOfOrder { index: usize },

The timing is of some other text than the line that was spoken.

58    NotThisLine,
59}

61impl fmt::Display for TimingError {
62    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
63        match self {
64            TimingError::Empty => write!(f, "the timing has no text"),
65            TimingError::Count { characters, spans } => write!(f, "the timing has {spans} spans for {characters} characters"),
66            TimingError::Reversed { index } => write!(f, "character {index} is done before it starts"),
67            TimingError::OutOfOrder { index } => write!(f, "character {index} is timed before the one ahead of it"),
68            TimingError::NotThisLine => write!(f, "the timing is of other text than the line that was spoken"),
69        }
70    }
71}
72
73impl std::error::Error for TimingError {}

A line's text with the time each of its characters is said.

One [Span] per character, by construction (they are pairs of the text's characters and their spans, not two lists that must agree). Every span ends no earlier than it starts, and starts and ends never run backwards from one character to the next. Spans may touch or overlap a little (a space can be said under the end of the word before it); they may not go back in time.

81#[derive(Clone, Debug, PartialEq, Eq)]
82pub struct LineTiming {
83    text: String,
84    spans: Vec<Span>,
85}

87impl LineTiming {

spans are (start, end) per character of text, in order.

89    pub fn new(text: String, spans: Vec<(AudioMs, AudioMs)>) -> Result<Self, TimingError> {
90        let characters = text.chars().count();
91        if characters == 0 {
92            return Err(TimingError::Empty);
93        }
94        if spans.len() != characters {
95            return Err(TimingError::Count { characters, spans: spans.len() });
96        }
97        let mut out: Vec<Span> = Vec::with_capacity(spans.len());
98        for (index, (start, end)) in spans.into_iter().enumerate() {
99            if end < start {
100                return Err(TimingError::Reversed { index });
101            }
102            if let Some(before) = out.last() {
103                if start < before.start || end < before.end {
104                    return Err(TimingError::OutOfOrder { index });
105                }
106            }
107            out.push(Span { start, end });
108        }
109        Ok(Self { text, spans: out })
110    }

112    pub fn text(&self) -> &str {
113        &self.text
114    }

One per character of text.

117    pub fn spans(&self) -> &[Span] {
118        &self.spans
119    }

Each character with the time it is said.

122    pub fn characters(&self) -> impl Iterator<Item = (char, Span)> + '_ {
123        self.text.chars().zip(self.spans.iter().copied())
124    }
125}

Speech (MP3) with the time each character of the line is said.

128#[derive(Clone, Debug, PartialEq, Eq)]
129pub struct TimedAudio {
130    audio: Audio,
131    timing: LineTiming,
132}

134impl TimedAudio {

audio is line spoken, and timing is of exactly that line's text.

136    pub fn new(audio: Audio, timing: LineTiming, line: &SpeechLine) -> Result<Self, TimingError> {
137        if timing.text() != line.as_str() {
138            return Err(TimingError::NotThisLine);
139        }
140        Ok(Self { audio, timing })
141    }

143    pub fn audio(&self) -> &Audio {
144        &self.audio
145    }
146
147    pub fn timing(&self) -> &LineTiming {
148        &self.timing
149    }
150}
151
152#[cfg(test)]
153mod tests {
154    use super::*;
155
156    fn ms(a: &[(u32, u32)]) -> Vec<(AudioMs, AudioMs)> {
157        a.iter().map(|&(s, e)| (AudioMs::new(s), AudioMs::new(e))).collect()
158    }
159
160    #[test]
161    fn a_timing_is_one_span_per_character_in_order() {
162        let t = LineTiming::new("Hi!".into(), ms(&[(0, 100), (100, 180), (180, 400)])).unwrap();
163        let said: Vec<(char, u32, u32)> = t.characters().map(|(c, s)| (c, s.start().get(), s.end().get())).collect();
164        assert_eq!(said, [('H', 0, 100), ('i', 100, 180), ('!', 180, 400)]);
165    }
166
167    #[test]
168    fn characters_are_scalar_values_not_bytes() {
169        assert!(LineTiming::new("h\u{e9}\u{1F408}".into(), ms(&[(0, 1), (1, 2), (2, 3)])).is_ok());
170        assert_eq!(LineTiming::new("h\u{e9}".into(), ms(&[(0, 1), (1, 2), (2, 3)])), Err(TimingError::Count { characters: 2, spans: 3 }));
171    }
172
173    #[test]
174    fn what_is_not_a_timing_is_refused_and_says_where() {
175        assert_eq!(LineTiming::new(String::new(), vec![]), Err(TimingError::Empty));
176        assert_eq!(LineTiming::new("ab".into(), ms(&[(0, 1)])), Err(TimingError::Count { characters: 2, spans: 1 }));
177        assert_eq!(LineTiming::new("ab".into(), ms(&[(0, 1), (5, 4)])), Err(TimingError::Reversed { index: 1 }));
178        assert_eq!(LineTiming::new("ab".into(), ms(&[(10, 20), (5, 30)])), Err(TimingError::OutOfOrder { index: 1 }), "starts early");
179        assert_eq!(LineTiming::new("ab".into(), ms(&[(10, 20), (12, 15)])), Err(TimingError::OutOfOrder { index: 1 }), "ends early");
180    }
181
182    #[test]
183    fn spans_may_touch_overlap_and_be_instants() {
184        assert!(LineTiming::new("abc".into(), ms(&[(0, 10), (10, 10), (10, 30)])).is_ok());
185        assert!(LineTiming::new("ab".into(), ms(&[(0, 10), (5, 20)])).is_ok());
186    }
187
188    #[test]
189    fn a_timed_line_is_the_line_that_was_spoken() {
190        let line = SpeechLine::new("Hi".into()).unwrap();
191        let t = LineTiming::new("Hi".into(), ms(&[(0, 1), (1, 2)])).unwrap();
192        assert!(TimedAudio::new(Audio(vec![1]), t.clone(), &line).is_ok());
193        let other = SpeechLine::new("Ho".into()).unwrap();
194        assert_eq!(TimedAudio::new(Audio(vec![1]), t, &other), Err(TimingError::NotThisLine));
195    }
196}