1//! Forgetting reaches the conversation a memory was told in. A turn has an identity, on every line of it and on the
2//! memory's mentions; forgetting writes the identities (never words) and the schema empties those turns, here and on
3//! every device the forgetting reaches, and in lines of them that arrive later. What waits to be thought about is
4//! dropped, and the bytes of the database and of its write-ahead log are searched for every word.
5
6use std::path::{Path, PathBuf};
7
8use whiskers_core::{Entry, Event, Log, Memory, MemoryDoc, NewFact, Outcome, PictureId};
9use whiskers_core::{ReflectInput, Verdict};
10use whiskers_store::{SqliteLog, SqliteMemory, SqlitePictures, SqliteReflections, Store};
11
12const HEARD: &str = "HEARD-WORDS-3391-QX";
13const MODEL: &str = "MODEL-WORDS-4402-QX";
14const SAID: &str = "SAID-WORDS-5513-QX";
15const LOOK: &str = "LOOKED-LIKE-6624-QX";
16const PROPOSED: &str = "PROPOSED-FACT-7735-QX";
17const AGAIN: &str = "SAID-AGAIN-8846-QX";
18const UNRELATED: &str = "UNRELATED-WORDS-9957-QX";
19const WAITING: &str = "WAITING-WORDS-1068-QX";
20const SHOWN: &[u8] = b"SHOWN-PICTURE-BYTES-2179-QX-0123456789-0123456789";
21const TURN: &str = "0000000000001aaaaaaaaaaaaaaaa1";
22const AGAIN_TURN: &str = "0000000000002bbbbbbbbbbbbbbbb2";
23const OTHER_TURN: &str = "0000000000003cccccccccccccccc3";
24
25fn dir(name: &str) -> PathBuf {
26    use std::sync::atomic::{AtomicUsize, Ordering};
27    static N: AtomicUsize = AtomicUsize::new(0);
28    let d = std::env::temp_dir().join(format!("whiskers-store-turns-{name}-{}-{}", std::process::id(), N.fetch_add(1, Ordering::SeqCst)));
29    let _ = std::fs::remove_dir_all(&d);
30    std::fs::create_dir_all(&d).unwrap();
31    d
32}
33
34fn on_disk(db: &Path) -> (Vec<u8>, Vec<u8>) {
35    let wal = db.with_file_name(format!("{}-wal", db.file_name().unwrap().to_string_lossy()));
36    (std::fs::read(db).unwrap(), std::fs::read(wal).unwrap_or_default())
37}
38
39fn contains(haystack: &[u8], needle: &[u8]) -> bool {
40    haystack.windows(needle.len()).any(|w| w == needle)
41}
42
43fn gone_from_disk(db: &Path, what: &[(&str, &[u8])]) {
44    let (file, wal) = on_disk(db);
45    for (name, n) in what {
46        assert!(!contains(&file, n), "{name} is still in the database file");
47        assert!(!contains(&wal, n), "{name} is still in the write-ahead log");
48    }
49}
50
51fn on_disk_somewhere(db: &Path, what: &[(&str, &[u8])]) {
52    let (file, wal) = on_disk(db);
53    for (name, n) in what {
54        assert!(contains(&file, n) || contains(&wal, n), "{name} should be on disk before it is forgotten");
55    }
56}
57
58fn line(turn: &str, at: u64, event: Event) -> Entry {
59    Entry::in_turn(turn, at, event)
60}
61
62/// The lines of one turn, as the conversation writes them: what she said and showed, what a look found, what the
63/// model wrote, what was said, and a memory proposed from it that was kept out.
64fn the_turn(picture: &PictureId) -> Vec<Entry> {
65    vec![
66        line(TURN, 100, Event::Heard { text: HEARD.into(), pictures: vec![picture.clone()] }),
67        line(TURN, 101, Event::PictureSeen { description: LOOK.into(), pictures: vec![picture.clone()] }),
68        line(TURN, 102, Event::Guarded { direction: whiskers_core::Direction::FromChild, verdict: Verdict::Allow }),
69        line(TURN, 103, Event::ModelWrote { text: MODEL.into() }),
70        line(TURN, 104, Event::Said { text: SAID.into(), outcome: Outcome::Answered }),
71        line(TURN, 105, Event::NotRemembered { fact: PROPOSED.into(), why: "too long".into() }),
72    ]
73}
74
75fn texts_of(entries: &[Entry]) -> Vec<String> {
76    entries
77        .iter()
78        .map(|e| match &e.event {
79            Event::Heard { text, .. } | Event::ModelWrote { text } | Event::Said { text, .. } => text.clone(),
80            Event::PictureSeen { description, .. } => description.clone(),
81            Event::NotRemembered { fact, .. } => fact.clone(),
82            _ => String::new(),
83        })
84        .collect()
85}
86
87fn waiting(turn: &str, words: &str) -> ReflectInput {
88    ReflectInput { turn: turn.into(), heard: words.into(), said: "fine".into(), description: None, pictures: vec![] }
89}
90
91#[test]
92fn forgetting_a_memory_empties_the_turns_it_was_told_in_and_nothing_else() {
93    let d = dir("device");
94    let db = d.join("device.db");
95    let store = Store::open(&db).unwrap();
96    let pictures = SqlitePictures::new(store.clone());
97    let picture = PictureId("0000000000000009-0000.jpg".into());
98    pictures.store(&picture, SHOWN).unwrap();
99    let mut log = SqliteLog::new(store.clone(), "phone");
100    for e in the_turn(&picture) {
101        log.append(&e).unwrap();
102    }
103    // The memory was told again in a second turn, and an unrelated turn and the hello are in the log too.
104    log.append(&line(AGAIN_TURN, 200, Event::Heard { text: AGAIN.into(), pictures: vec![] })).unwrap();
105    log.append(&line(AGAIN_TURN, 201, Event::Said { text: format!("{AGAIN} again"), outcome: Outcome::Answered })).unwrap();
106    log.append(&line(OTHER_TURN, 300, Event::Heard { text: UNRELATED.into(), pictures: vec![] })).unwrap();
107    log.append(&Entry::new(1, Event::Greeted { text: "Hi!".into() })).unwrap();
108
109    let mut memory = SqliteMemory::open(store.clone(), "phone").unwrap();
110    let f = memory.add(NewFact { text: "Biscuit is her bunny".into(), turn: TURN.into(), ..NewFact::default() }, 100).unwrap();
111    memory.mention(f.id, 200, AGAIN_TURN).unwrap();
112    // Exchanges wait: one of the turn the memory came from, one of an unrelated turn.
113    let queue = SqliteReflections::new(store.clone());
114    queue.push(&waiting(TURN, WAITING), 106, 8).unwrap();
115    queue.push(&waiting(OTHER_TURN, UNRELATED), 301, 8).unwrap();
116
117    // The turns are on the memory, by identity, and survive a restart.
118    drop(memory);
119    let mut memory = SqliteMemory::open(store.clone(), "phone").unwrap();
120    let mentions: Vec<String> = memory.facts()[0].mentions.iter().map(|m| m.turn.clone()).collect();
121    assert_eq!(mentions, [TURN, AGAIN_TURN]);
122
123    let words: Vec<(&str, &[u8])> = vec![
124        ("what she said", HEARD.as_bytes()),
125        ("what a look at her picture found", LOOK.as_bytes()),
126        ("what the model wrote", MODEL.as_bytes()),
127        ("what was said back", SAID.as_bytes()),
128        ("a fact proposed from it", PROPOSED.as_bytes()),
129        ("what she said again", AGAIN.as_bytes()),
130        ("the picture she showed", SHOWN),
131        ("the exchange waiting to be thought about", WAITING.as_bytes()),
132    ];
133    on_disk_somewhere(&db, &words);
134    assert_eq!(queue.len().unwrap(), 2);
135
136    memory.forget(f.id, 999).unwrap();
137
138    gone_from_disk(&db, &words);
139    let (_, wal) = on_disk(&db);
140    assert!(wal.is_empty(), "the write-ahead log was truncated: {} bytes", wal.len());
141    // The lines stay, with their times and how the turn ended; what was said in them is empty.
142    let kept = log.own_entries().unwrap();
143    assert_eq!(kept.len(), 6 + 2 + 1 + 1, "no line was removed");
144    let turn: Vec<&Entry> = kept.iter().filter(|e| e.turn == TURN).collect();
145    assert_eq!(turn.iter().map(|e| e.at_ms).collect::<Vec<_>>(), [100, 101, 102, 103, 104, 105]);
146    assert!(texts_of(&turn.iter().map(|e| (*e).clone()).collect::<Vec<_>>()).iter().all(String::is_empty));
147    assert!(matches!(&turn[0].event, Event::Heard { pictures, .. } if pictures.is_empty()), "the picture is let go");
148    assert!(matches!(&turn[4].event, Event::Said { outcome: Outcome::Answered, .. }), "and how it ended stays");
149    assert!(matches!(&turn[5].event, Event::NotRemembered { why, .. } if why == "too long"));
150    assert!(kept.iter().filter(|e| e.turn == AGAIN_TURN).all(|e| texts_of(&[(*e).clone()])[0].is_empty()), "the turn she said it again in is emptied too");
151    let unrelated = kept.iter().find(|e| e.turn == OTHER_TURN).unwrap();
152    assert!(matches!(&unrelated.event, Event::Heard { text, .. } if text == UNRELATED), "an unrelated turn is untouched");
153    assert!(kept.iter().any(|e| matches!(&e.event, Event::Greeted { text } if text == "Hi!")));
154    // Every exchange that waited was dropped: any of them might say the memory again and learn it back.
155    assert!(queue.is_empty().unwrap(), "waiting exchanges are dropped, the unrelated one too");
156    assert!(!queue.has_turn(OTHER_TURN).unwrap());
157    // The picture only that turn showed went with it.
158    assert!(!pictures.has(&picture));
159    // What stays of the memory is its identity, when and by whom, and the identities of its turns.
160    let conn = rusqlite::Connection::open(&db).unwrap();
161    let count = |sql: &str| conn.query_row(sql, [], |r| r.get::<_, i64>(0)).unwrap();
162    assert_eq!(count(&format!("SELECT COUNT(*) FROM tombstone_turn WHERE gid = '{}'", f.gid)), 2);
163    assert_eq!(count("SELECT COUNT(*) FROM journal_picture"), 0);
164    drop((memory, store));
165    let again = SqliteMemory::open(Store::open(&db).unwrap(), "phone").unwrap();
166    assert_eq!(again.snapshot().forgotten_when[0].turns, [TURN, AGAIN_TURN], "and they survive a restart, to be sent on");
167}
168
169#[test]
170fn a_line_of_a_forgotten_turn_that_arrives_late_is_emptied_as_it_is_written() {
171    let d = dir("late");
172    let db = d.join("device.db");
173    let store = Store::open(&db).unwrap();
174    let mut memory = SqliteMemory::open(store.clone(), "phone").unwrap();
175    let f = memory.add(NewFact { text: "Biscuit is her bunny".into(), turn: TURN.into(), ..NewFact::default() }, 100).unwrap();
176    memory.forget(f.id, 999).unwrap();
177    // The tablet had not heard of it, and its lines of that turn reach this device afterwards.
178    let picture = PictureId("0000000000000009-0000.jpg".into());
179    let lines: Vec<String> = the_turn(&picture).iter().map(|e| format!(r#"{{"device":"tablet","entry":{}}}"#, serde_json::to_string(e).unwrap())).collect();
180    let log = SqliteLog::new(store.clone(), "phone");
181    assert_eq!(log.take(0, &lines).unwrap(), 6);
182    store.scrub().unwrap();
183    gone_from_disk(&db, &[("what she said", HEARD.as_bytes()), ("what the model wrote", MODEL.as_bytes()), ("what was said back", SAID.as_bytes()), ("a fact proposed", PROPOSED.as_bytes())]);
184    let (groups, _) = log.read_groups().unwrap();
185    assert!(groups.iter().flatten().all(|e| texts_of(std::slice::from_ref(e))[0].is_empty()));
186    let conn = rusqlite::Connection::open(&db).unwrap();
187    assert_eq!(conn.query_row("SELECT COUNT(*) FROM journal_picture", [], |r| r.get::<_, i64>(0)).unwrap(), 0, "no picture is kept for a line that no longer names one");
188}
189
190#[test]
191fn a_device_that_never_saw_the_memory_empties_the_turns_when_the_forgetting_arrives() {
192    // Phone learned and forgot it; the tablet only has the phone's lines of that turn, taken from the service.
193    let phone_dir = dir("phone");
194    let mut phone = SqliteMemory::open(Store::open(&phone_dir.join("p.db")).unwrap(), "phone").unwrap();
195    let f = phone.add(NewFact { text: "Biscuit is her bunny".into(), turn: TURN.into(), ..NewFact::default() }, 100).unwrap();
196    let before_forgetting: MemoryDoc = MemoryDoc::default();
197    let _ = before_forgetting;
198    phone.forget(f.id, 999).unwrap();
199
200    let tablet_dir = dir("tablet");
201    let db = tablet_dir.join("t.db");
202    let store = Store::open(&db).unwrap();
203    let picture = PictureId("0000000000000009-0000.jpg".into());
204    let lines: Vec<String> = the_turn(&picture).iter().map(|e| format!(r#"{{"device":"phone","entry":{}}}"#, serde_json::to_string(e).unwrap())).collect();
205    let tablet_log = SqliteLog::new(store.clone(), "tablet");
206    tablet_log.take(0, &lines).unwrap();
207    let queue = SqliteReflections::new(store.clone());
208    queue.push(&waiting(OTHER_TURN, WAITING), 1, 8).unwrap();
209    on_disk_somewhere(&db, &[("what she said", HEARD.as_bytes()), ("what the model wrote", MODEL.as_bytes())]);
210
211    let mut tablet = SqliteMemory::open(store.clone(), "tablet").unwrap();
212    tablet.merge(phone.snapshot()).unwrap();
213
214    gone_from_disk(&db, &[("what she said", HEARD.as_bytes()), ("what the model wrote", MODEL.as_bytes()), ("what was said back", SAID.as_bytes()), ("the waiting exchange", WAITING.as_bytes())]);
215    assert!(queue.is_empty().unwrap());
216    // Merging again changes nothing, and is not a second forgetting: nothing waiting is dropped twice.
217    queue.push(&waiting(OTHER_TURN, "later"), 2, 8).unwrap();
218    tablet.merge(phone.snapshot()).unwrap();
219    assert_eq!(queue.len().unwrap(), 1, "the same forgetting arriving again drops nothing");
220}
221
222#[test]
223fn a_copy_dropped_as_a_duplicate_does_not_drop_what_waits_or_empty_its_turns() {
224    let d = dir("duplicate");
225    let store = Store::open(&d.join("d.db")).unwrap();
226    let mut a = SqliteMemory::open(store.clone(), "phone").unwrap();
227    let kept = a.add(NewFact { text: "Biscuit is her bunny".into(), turn: TURN.into(), ..NewFact::default() }, 10).unwrap();
228    let log = SqliteLog::new(store.clone(), "phone");
229    log.clone().append(&line(TURN, 10, Event::Heard { text: HEARD.into(), pictures: vec![] })).unwrap();
230    let queue = SqliteReflections::new(store.clone());
231    queue.push(&waiting(OTHER_TURN, WAITING), 11, 8).unwrap();
232    // The same thing learned on another device arrives as a second fact and is dropped, which writes a tombstone.
233    let mut other = MemoryDoc::default();
234    other.add(NewFact { text: "biscuit is her  bunny".into(), turn: OTHER_TURN.into(), ..NewFact::default() }, 90, "tablet", "zz-other".into());
235    a.merge(other.snapshot()).unwrap();
236    assert_eq!(a.facts().len(), 1);
237    assert_eq!(a.snapshot().forgotten.len(), 1, "a tombstone was written for the copy");
238    assert!(a.snapshot().forgotten_when.is_empty(), "but it is not a forgetting: it has no time");
239    assert_eq!(queue.len().unwrap(), 1, "nothing waiting was dropped");
240    let e = log.own_entries().unwrap();
241    assert!(matches!(&e[0].event, Event::Heard { text, .. } if text == HEARD), "and nothing was emptied");
242    assert_eq!(a.facts()[0].id, kept.id);
243}