1//! Forgetting is a real delete. A memory with a recognisable string in its words and its older words, a
2//! mention, an embedding and a picture is forgotten, and then the bytes of the database file and of its
3//! write-ahead log are searched for every one of them. Done for a device's store and for the service's.
4//!
5//! Journal mode: the database is in write-ahead mode, so a deleted row's old bytes can sit in the `-wal` file
6//! until it is checkpointed. Every forgetting is followed by `wal_checkpoint(TRUNCATE)` (`Store::scrub`), which
7//! copies the log into the file, where `secure_delete` has already zeroed what was deleted, and truncates the log
8//! to nothing. The test checks both files, and that the log really is empty.
9
10use std::path::{Path, PathBuf};
11
12use whiskers_conformance::suite::build;
13use whiskers_core::{HouseholdStore, Log, Memory, NewFact, PictureId, Pictures};
14use whiskers_ports::{Appended, DeviceName, EntryLine, Hub, Journal, PictureShelf, Put, SafePictureId, run_ready};
15use whiskers_store::{SqliteChat, SqliteHousehold, SqliteJournal, SqliteLog, SqliteMemory, SqliteMemoryHub, SqlitePictureShelf, SqlitePictures, Store};
16
17const WORDS: &str = "ZEBRAS-4471-QX";
18const OLD_WORDS: &str = "OLDWORDS-5521-QX";
19/// What a model said a picture showed, which is where a memory made from it first got its words.
20const DESCRIPTION: &str = "DESCRIBED-7731-QX";
21const PICTURE: &[u8] = b"JPEGBYTES-9931-QX-0123456789-0123456789-0123456789-0123456789";
22/// An embedding that cannot be mistaken for anything else in a file.
23const VECTOR: [f32; 4] = [0.123_456_79, -7.654_321, 3.1e7, 9.87e-5];
24
25fn dir(name: &str) -> PathBuf {
26    use std::sync::atomic::{AtomicUsize, Ordering};
27    static N: AtomicUsize = AtomicUsize::new(0);
28    let d = std::env::temp_dir().join(format!("whiskers-store-forget-{name}-{}-{}", std::process::id(), N.fetch_add(1, Ordering::SeqCst)));
29    let _ = std::fs::remove_dir_all(&d);
30    std::fs::create_dir_all(&d).unwrap();
31    d
32}
33
34fn vector_bytes() -> Vec<u8> {
35    VECTOR.iter().flat_map(|x| x.to_le_bytes()).collect()
36}
37
38/// What is on disk for the database at `db`: its file and its write-ahead log, if there is one.
39fn on_disk(db: &Path) -> (Vec<u8>, Vec<u8>) {
40    let wal = db.with_file_name(format!("{}-wal", db.file_name().unwrap().to_string_lossy()));
41    (std::fs::read(db).unwrap(), std::fs::read(wal).unwrap_or_default())
42}
43
44fn contains(haystack: &[u8], needle: &[u8]) -> bool {
45    haystack.windows(needle.len()).any(|w| w == needle)
46}
47
48use whiskers_core::{Entry, Event};
49
50/// The parents' log lines that carry a memory's words, as the engine writes them as it goes: it was
51/// remembered (in its first words), she put it away and a parent restored it, it was called to mind, and
52/// the picture it was made from was described. (A `Heard` line that showed the picture would keep the picture;
53/// that is the next test's.)
54fn lines_with_the_words(gid: &str, picture: &PictureId) -> Vec<Entry> {
55    vec![
56        Entry::new(11, Event::PictureSeen { description: DESCRIPTION.into(), pictures: vec![picture.clone()] }),
57        Entry::new(12, Event::Remembered { fact: format!("{OLD_WORDS} is her toy"), gid: gid.into() }),
58        Entry::new(13, Event::Recalled { facts: vec!["she likes tea".into(), format!("{WORDS} is her big toy")], gids: vec!["other".into(), gid.into()] }),
59        Entry::new(14, Event::PutAway { fact: format!("{WORDS} is her big toy"), gid: gid.into() }),
60        Entry::new(15, Event::Restored { fact: format!("{WORDS} is her big toy"), gid: gid.into() }),
61        Entry::new(16, Event::Remembered { fact: "she likes tea".into(), gid: "other".into() }),
62    ]
63}
64
65/// What the log of `device` holds, as the entries it was written as.
66fn entries(store: &Store, device: &str) -> Vec<Entry> {
67    SqliteLog::new(store.clone(), device).own_entries().unwrap()
68}
69
70/// Every needle that a forgotten memory must take with it.
71fn needles() -> Vec<(&'static str, Vec<u8>)> {
72    vec![
73        ("its words", WORDS.as_bytes().to_vec()),
74        ("its older words", OLD_WORDS.as_bytes().to_vec()),
75        ("its picture", PICTURE.to_vec()),
76        ("its embedding", vector_bytes()),
77        ("the description of its picture", DESCRIPTION.as_bytes().to_vec()),
78    ]
79}
80
81#[test]
82fn a_forgotten_memory_is_in_neither_the_device_database_nor_its_log() {
83    let d = dir("device");
84    let db = d.join("device.db");
85    let store = Store::open(&db).unwrap();
86    let mut memory = SqliteMemory::open(store.clone(), "phone").unwrap();
87    let mut pictures = SqlitePictures::new(store.clone());
88    let kept_picture = pictures.save(&whiskers_core::Image { media_type: "image/jpeg".into(), bytes: b"ANOTHER-PICTURE-1234".to_vec() }).unwrap();
89    let id = PictureId("0000000000000007-0000.jpg".into());
90    pictures.store(&id, PICTURE).unwrap();
91
92    memory.add(NewFact { text: "she likes tea".into(), pictures: vec![kept_picture.clone()], ..NewFact::default() }, 1).unwrap();
93    let f = memory.add(NewFact { text: format!("{OLD_WORDS} is her toy"), pictures: vec![id.clone()], ..NewFact::default() }, 2).unwrap();
94    memory.mention(f.id, 50, "").unwrap();
95    memory.reword(f.id, format!("{WORDS} is her big toy"), 100).unwrap();
96    memory.set_embedding(f.id, VECTOR.to_vec()).unwrap();
97    let before = memory.facts().into_iter().find(|x| x.id == f.id).unwrap();
98    assert_eq!((before.mentions.len(), before.revisions.len(), before.embedding.len(), before.pictures.len()), (2, 1, 4, 1));
99
100    // The log holds the words in its own lines, on this device's own and in the copy it keeps of the tablet's.
101    let mut log = SqliteLog::new(store.clone(), "phone");
102    let lines = lines_with_the_words(&before.gid, &id);
103    for l in &lines {
104        log.append(l).unwrap();
105    }
106    let from_tablet = lines_with_the_words(&before.gid, &id);
107    let mut taken: Vec<String> = from_tablet.iter().map(|e| format!(r#"{{"device":"tablet","entry":{}}}"#, serde_json::to_string(e).unwrap())).collect();
108    // A line from before the lines carried an identity: found by its exact words. And a `Forgot` as it used to be written.
109    taken.push(format!(r#"{{"device":"tablet","entry":{{"at_ms":20,"event":{{"Remembered":{{"fact":"{WORDS} is her big toy"}}}}}}}}"#));
110    taken.push(format!(r#"{{"device":"tablet","entry":{{"at_ms":21,"event":{{"Forgot":{{"fact":"{OLD_WORDS} is her toy"}}}}}}}}"#));
111    assert_eq!(log.take(0, &taken).unwrap(), taken.len());
112
113    // The instrument can see them: before the forgetting every one is in the file or its log.
114    let (file, wal) = on_disk(&db);
115    for (what, n) in needles() {
116        assert!(contains(&file, &n) || contains(&wal, &n), "{what} should be on disk before it is forgotten");
117    }
118
119    memory.forget(f.id, 999).unwrap();
120
121    let (file, wal) = on_disk(&db);
122    for (what, n) in needles() {
123        assert!(!contains(&file, &n), "{what} is still in the database file");
124        assert!(!contains(&wal, &n), "{what} is still in the write-ahead log");
125    }
126    assert!(wal.is_empty(), "the write-ahead log was truncated, not merely scanned: {} bytes", wal.len());
127    // A line that arrives later, from a device that had not heard of the forgetting, is emptied as it is written
128    // (so the words were never in a committed page, which is what the search below checks).
129    for l in lines_with_the_words(&before.gid, &id) {
130        SqliteLog::new(store.clone(), "laptop").append(&l).unwrap();
131    }
132    let (file, wal) = on_disk(&db);
133    for (what, n) in needles() {
134        assert!(!contains(&file, &n) && !contains(&wal, &n), "{what} came back with a late line");
135    }
136    // The log lines are still there, with their times, and say nothing of it: the words are empty. A line about
137    // another memory is untouched.
138    for device in ["phone", "tablet", "laptop"] {
139        let kept = entries(&store, device);
140        assert!(kept.len() >= 6, "{device}: no line was removed");
141        for e in &kept {
142            match &e.event {
143                Event::Remembered { fact, gid } if gid != "other" => assert_eq!(fact, ""),
144                Event::PutAway { fact, .. } | Event::Restored { fact, .. } => assert_eq!(fact, ""),
145                Event::Recalled { facts, .. } => assert_eq!(facts, &vec!["she likes tea".to_string(), String::new()]),
146                Event::PictureSeen { description, .. } => assert_eq!(description, ""),
147                Event::Remembered { fact, .. } => assert_eq!(fact, "she likes tea"),
148                _ => {}
149            }
150        }
151        let times: Vec<u64> = kept.iter().map(|e| e.at_ms).collect();
152        assert!(times.contains(&12) && times.contains(&16), "{device}: a line kept its time ({times:?})");
153    }
154    assert!(entries(&store, "tablet").iter().any(|e| matches!(&e.event, Event::Forgot { gid } if gid.is_empty())), "a Forgot of the old kind has no words left");
155    // What remains of it is its identity, when and by whom.
156    let conn = rusqlite::Connection::open(&db).unwrap();
157    let count = |sql: &str| conn.query_row(sql, [], |r| r.get::<_, i64>(0)).unwrap();
158    assert_eq!(count("SELECT COUNT(*) FROM tombstone"), 1);
159    assert_eq!(count(&format!("SELECT COUNT(*) FROM tombstone WHERE gid = '{}' AND at_ms = 999 AND device = 'phone'", before.gid)), 1);
160    for table in ["fact_mention", "fact_revision", "fact_embedding", "fact_picture"] {
161        assert_eq!(count(&format!("SELECT COUNT(*) FROM {table} WHERE gid = '{}'", before.gid)), 0, "{table} still has rows of the forgotten memory");
162    }
163    assert_eq!(count("SELECT COUNT(*) FROM fact"), 1, "the other memory is untouched");
164    assert_eq!(count("SELECT COUNT(*) FROM picture"), 2 - 1, "its picture is gone and the other memory's stays");
165    assert!(memory.facts()[0].pictures == vec![kept_picture]);
166    // And it is gone after a restart too.
167    drop((memory, pictures, store));
168    let again = SqliteMemory::open(Store::open(&db).unwrap(), "phone").unwrap();
169    assert_eq!(again.facts().len(), 1);
170    assert_eq!(again.snapshot().forgotten, vec![before.gid]);
171}
172
173#[test]
174fn a_forgotten_memory_is_in_neither_the_service_database_nor_its_log_and_stays_forgotten() {
175    let d = dir("service");
176    let db = d.join("service.db");
177    let store = Store::open(&db).unwrap();
178    let hub = SqliteMemoryHub::open(store.clone()).unwrap();
179    let shelf = SqlitePictureShelf::new(store.clone());
180    let picture = SafePictureId::parse("0000000000000007-0000.jpg").unwrap();
181    assert_eq!(run_ready(shelf.put(&picture, PICTURE)).unwrap(), Put::Stored);
182
183    // A device learns it and tells the service.
184    let phone_dir = dir("service-phone");
185    let mut phone = SqliteMemory::open(Store::open(&phone_dir.join("phone.db")).unwrap(), "phone").unwrap();
186    let f = phone.add(NewFact { text: format!("{OLD_WORDS} is her toy"), pictures: vec![PictureId(picture.as_str().into())], ..NewFact::default() }, 2).unwrap();
187    phone.mention(f.id, 50, "").unwrap();
188    phone.reword(f.id, format!("{WORDS} is her big toy"), 100).unwrap();
189    phone.set_embedding(f.id, VECTOR.to_vec()).unwrap();
190    let stale = phone.snapshot();
191    run_ready(hub.merge(stale.clone())).unwrap();
192    // The phone's log lines are on the service too, as every device's are.
193    let journal = SqliteJournal::new(store.clone());
194    let device = DeviceName::new("phone").unwrap();
195    let lines: Vec<EntryLine> = lines_with_the_words(&f.gid, &PictureId(picture.as_str().into())).iter().map(|e| EntryLine::parse(&serde_json::to_string(e).unwrap()).unwrap()).collect();
196    let held = run_ready(journal.held(&device)).unwrap();
197    assert!(matches!(run_ready(journal.append(&device, held, &lines)).unwrap(), Appended::Accepted { .. }));
198    let (file, wal) = on_disk(&db);
199    for (what, n) in needles() {
200        assert!(contains(&file, &n) || contains(&wal, &n), "{what} should be on the service before it is forgotten");
201    }
202
203    // The parents forget it on the phone; the next sync carries the tombstone.
204    phone.forget(f.id, 999).unwrap();
205    let merged = run_ready(hub.merge(phone.snapshot())).unwrap();
206    assert!(merged.document.facts.is_empty());
207
208    let (file, wal) = on_disk(&db);
209    for (what, n) in needles() {
210        assert!(!contains(&file, &n), "{what} is still in the service's database file");
211        assert!(!contains(&wal, &n), "{what} is still in the service's write-ahead log");
212    }
213    assert!(wal.is_empty(), "the service's log was truncated: {} bytes", wal.len());
214    assert_eq!(run_ready(shelf.get(&picture)).unwrap(), None, "the picture nothing else uses went with it");
215    let page = run_ready(journal.pull(whiskers_ports::LogCursor::new(0))).unwrap();
216    assert_eq!(page.lines.len(), 6, "every line is still there");
217    // A device that had not heard yet pushes its lines afterwards: emptied as they arrive.
218    let late = DeviceName::new("laptop").unwrap();
219    let held = run_ready(journal.held(&late)).unwrap();
220    run_ready(journal.append(&late, held, &lines)).unwrap();
221    let (file, wal) = on_disk(&db);
222    for (what, n) in needles() {
223        assert!(!contains(&file, &n) && !contains(&wal, &n), "{what} came back with a late line on the service");
224    }
225
226    // The tombstone wins over any later state from another device: a stale copy that still has it brings nothing back.
227    let again = run_ready(hub.merge(stale)).unwrap();
228    assert!(again.document.facts.is_empty() && again.changed == 0);
229    assert_eq!(again.document.forgotten, vec![f.gid.clone()]);
230    assert_eq!(again.document.forgotten_when.len(), 1, "who and when travel with the identity");
231    // The schema itself refuses to hold it again, whatever code asks.
232    let conn = rusqlite::Connection::open(&db).unwrap();
233    let err = conn.execute("INSERT INTO fact (gid, text, learned_at_ms, kind) VALUES (?1, 'x', 1, 'other')", [&f.gid]).unwrap_err();
234    assert!(err.to_string().contains("forgotten"), "{err}");
235}
236
237#[test]
238fn a_picture_that_another_memory_or_a_logged_conversation_still_shows_is_kept() {
239    let d = dir("shared-picture");
240    let db = d.join("d.db");
241    let store = Store::open(&db).unwrap();
242    let mut memory = SqliteMemory::open(store.clone(), "phone").unwrap();
243    let pictures = SqlitePictures::new(store.clone());
244    let (shared, shown) = (PictureId("0000000000000001-0000.jpg".into()), PictureId("0000000000000002-0000.jpg".into()));
245    pictures.store(&shared, b"SHARED-PICTURE").unwrap();
246    pictures.store(&shown, b"SHOWN-IN-CONVERSATION").unwrap();
247    let a = memory.add(NewFact { text: "one".into(), pictures: vec![shared.clone(), shown.clone()], ..NewFact::default() }, 1).unwrap();
248    memory.add(NewFact { text: "two".into(), pictures: vec![shared.clone()], ..NewFact::default() }, 2).unwrap();
249    let mut log = SqliteLog::new(store.clone(), "phone");
250    log.append(&whiskers_core::Entry::new(3, whiskers_core::Event::Heard { text: "look".into(), pictures: vec![shown.clone()] })).unwrap();
251    memory.forget(a.id, 10).unwrap();
252    assert!(pictures.has(&shared), "another memory still has it");
253    assert!(pictures.has(&shown), "a logged conversation still shows it");
254}
255
256#[test]
257fn hiding_by_the_child_deletes_nothing() {
258    let d = dir("hide");
259    let db = d.join("d.db");
260    let store = Store::open(&db).unwrap();
261    let mut memory = SqliteMemory::open(store.clone(), "phone").unwrap();
262    let f = memory.add(NewFact { text: format!("{WORDS} is her toy"), ..NewFact::default() }, 1).unwrap();
263    memory.set_embedding(f.id, VECTOR.to_vec()).unwrap();
264    memory.hide(f.id, 5).unwrap();
265    store.scrub().unwrap();
266    let (file, _) = on_disk(&db);
267    assert!(contains(&file, WORDS.as_bytes()) && contains(&file, &vector_bytes()), "hiding is a soft delete: everything is kept");
268    assert!(memory.facts()[0].visibility.is_hidden());
269}
270
271#[test]
272fn the_store_leaves_emptying_the_chat_to_the_documents_rule() {
273    // The chat is a document that holders keep in memory and merge (`ChatState::scrub` and `merge`), so a trigger that
274    // emptied its rows would be overwritten by the copy in memory. Forgetting a memory at the store does not touch it;
275    // the engine empties the chat as part of forgetting (see the engine's tests).
276    let d = dir("others");
277    let store = Store::open(&d.join("d.db")).unwrap();
278    let mut chat = SqliteChat::new(store.clone());
279    let mut h = SqliteHousehold::new(store.clone());
280    whiskers_core::ChatStore::save(&mut chat, &build::chat(3, 7, "a summary")).unwrap();
281    h.save(&whiskers_core::Household::default()).unwrap();
282    let mut memory = SqliteMemory::open(store.clone(), "phone").unwrap();
283    let f = memory.add(NewFact { text: "x".into(), ..NewFact::default() }, 1).unwrap();
284    memory.forget(f.id, 2).unwrap();
285    assert_eq!(whiskers_core::ChatStore::load(&chat).unwrap().summary, "a summary");
286}