From 33dbb0c1ef62c39b2471a501dde8b6e2edd5bd95 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Thu, 16 Jul 2026 11:49:46 +1000 Subject: [PATCH 1/3] fix(storage): make snapshot COW and reopen GC durable Signed-off-by: wchwawa --- src/api/db.rs | 966 +++++++-- src/api/errors.rs | 10 + src/api/snapshot.rs | 47 +- src/api/stats.rs | 32 + src/api/tree.rs | 1308 ++++++++++-- src/api/view.rs | 23 +- src/checkpoint/io.rs | 532 ++++- src/checkpoint/mod.rs | 223 ++- src/checkpoint/round.rs | 130 +- src/concurrency/commit_gate.rs | 5 + src/concurrency/gate.rs | 2 +- src/engine/mod.rs | 2 +- src/engine/walker/cow.rs | 44 +- src/engine/walker/erase.rs | 5 +- src/engine/walker/insert.rs | 64 +- src/engine/walker/lookup.rs | 225 ++- src/engine/walker/merge.rs | 146 +- src/engine/walker/migrate.rs | 105 +- src/engine/walker/mod.rs | 1 + src/engine/walker/range.rs | 186 +- src/engine/walker/spillover.rs | 154 +- src/engine/walker/tests.rs | 227 ++- src/metrics.rs | 46 + src/store/blob_frame.rs | 16 + src/store/buffer_manager/mod.rs | 2771 ++++++++++++++++++++++---- src/store/buffer_manager/mutation.rs | 8 + src/store/mod.rs | 4 +- 27 files changed, 6335 insertions(+), 947 deletions(-) diff --git a/src/api/db.rs b/src/api/db.rs index 88888c6..1638b37 100644 --- a/src/api/db.rs +++ b/src/api/db.rs @@ -17,7 +17,7 @@ use super::config::TreeConfig; use super::errors::{Error, Result}; use super::snapshot::Snapshot; use super::stats::{CheckpointerStats, DBStats, JournalStats, OpenStats, VacuumStats}; -use super::tree::{ensure_root_blob, replay_wal, Tree, TreeRuntime}; +use super::tree::{ensure_durable_root_blob, replay_wal, Tree, TreeRuntime}; use super::view::View; use crate::concurrency::{CommitGate, Gate}; use crate::engine::RangeEntry; @@ -37,6 +37,79 @@ const CATALOG_STATE_LIVE: u8 = 1; const CATALOG_STATE_DROPPING: u8 = 2; const CATALOG_VALUE_LEN: usize = 17; const CATALOG_NEXT_ID_LEN: usize = 16; +const AUTO_GC_BATCH_SIZE: usize = 256; + +#[cfg(test)] +struct OpenTreeCatalogBarrier { + entered: std::sync::Barrier, + release: std::sync::Barrier, +} + +#[cfg(test)] +impl OpenTreeCatalogBarrier { + fn new() -> Self { + Self { + entered: std::sync::Barrier::new(2), + release: std::sync::Barrier::new(2), + } + } +} + +#[cfg(test)] +std::thread_local! { + static OPEN_TREE_CATALOG_BARRIER: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(test)] +fn set_open_tree_catalog_barrier_for_current_thread(barrier: Arc) { + OPEN_TREE_CATALOG_BARRIER.with(|slot| *slot.borrow_mut() = Some(barrier)); +} + +#[cfg(test)] +fn pause_open_tree_after_catalog_lookup() { + let barrier = OPEN_TREE_CATALOG_BARRIER.with(|slot| slot.borrow_mut().take()); + if let Some(barrier) = barrier { + barrier.entered.wait(); + barrier.release.wait(); + } +} + +#[cfg(test)] +struct ExportFirstEntryBarrier { + entered: std::sync::Barrier, + release: std::sync::Barrier, +} + +#[cfg(test)] +impl ExportFirstEntryBarrier { + fn new() -> Self { + Self { + entered: std::sync::Barrier::new(2), + release: std::sync::Barrier::new(2), + } + } +} + +#[cfg(test)] +std::thread_local! { + static EXPORT_FIRST_ENTRY_BARRIER: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(test)] +fn set_export_first_entry_barrier_for_current_thread(barrier: Arc) { + EXPORT_FIRST_ENTRY_BARRIER.with(|slot| *slot.borrow_mut() = Some(barrier)); +} + +#[cfg(test)] +fn pause_export_after_first_entry() { + let barrier = EXPORT_FIRST_ENTRY_BARRIER.with(|slot| slot.borrow_mut().take()); + if let Some(barrier) = barrier { + barrier.entered.wait(); + barrier.release.wait(); + } +} #[derive(Clone)] struct OpenTree { @@ -96,6 +169,17 @@ impl DB { cfg.checkpoint.auto_merge = false; let bm = Tree::open_buffer_manager(&cfg)?; + Self::open_with_buffer_manager(cfg, bm) + } + + #[cfg(test)] + fn open_with_blob_store(mut cfg: TreeConfig, store: Arc) -> Result { + cfg.checkpoint.auto_merge = false; + let bm = Arc::new(BufferManager::new(store, cfg.buffer_pool_size)); + Self::open_with_buffer_manager(cfg, bm) + } + + fn open_with_buffer_manager(cfg: TreeConfig, bm: Arc) -> Result { let mut open_stats = OpenStats::default(); let (journal, next_seq) = match cfg.wal_path() { @@ -122,28 +206,37 @@ impl DB { let maintenance_gate = Arc::new(Gate::new()); let commit_gate = Arc::new(CommitGate::new()); - let checkpointer = crate::checkpoint::Checkpointer::spawn( - Arc::clone(&bm), - journal.clone(), - Arc::clone(&maintenance_gate), - Arc::clone(&commit_gate), - cfg.checkpoint.clone(), - ) - .map(Arc::new); - - let db = Self { + let mut db = Self { cfg, store: bm, maintenance_gate, next_seq: Arc::new(AtomicU64::new(next_seq)), commit_gate, journal, - checkpointer, + checkpointer: None, open_stats, trees: Arc::new(Mutex::new(HashMap::new())), catalog_cache: Arc::new(Mutex::new(HashMap::new())), }; - db.stage_dropped_trees()?; + // Replay restores logical parents but the BufferManager epoch starts + // at one in every process. Recover the maximum epoch across every + // frame reachable from the catalog before any snapshot or background + // delta flush can serve. + let epoch_recovery_start = std::time::Instant::now(); + db.restore_epoch_high_water()?; + db.open_stats.epoch_recovery_micros = epoch_recovery_start + .elapsed() + .as_micros() + .min(u128::from(u64::MAX)) as u64; + db.restore_dropping_runtime_fences()?; + db.checkpointer = crate::checkpoint::Checkpointer::spawn( + Arc::clone(&db.store), + db.journal.clone(), + Arc::clone(&db.maintenance_gate), + Arc::clone(&db.commit_gate), + db.cfg.checkpoint.clone(), + ) + .map(Arc::new); Ok(db) } @@ -161,6 +254,13 @@ impl DB { }); } let tree_id = self.allocate_tree_id()?; + let root_guid = root_guid_for_tree_id(tree_id); + + // Publish the deterministic empty root durably before making the + // catalog entry Live. If catalog publication later fails, the root is + // merely unreachable GC debt; the inverse ordering can leave a Live + // catalog entry pointing at a missing root and poison DB reopen. + ensure_durable_root_blob(&self.store, root_guid)?; self.apply_system_batch_unlocked( DB_CATALOG_TREE_ID, @@ -209,11 +309,18 @@ impl DB { /// desired behavior. pub fn open_tree(&self, name: &str) -> Result { let name_bytes = validate_tree_name(name)?; + // Serialize the catalog Live decision with drop_tree's exclusive + // maintenance transition through runtime lookup and Tree construction. + // Once this shared guard releases, a queued drop can mark the exact + // runtime returned below as dropped before it returns to its caller. + let _maintenance = self.maintenance_gate.enter_shared(); let tree_id = self .catalog_lookup_live(name_bytes)? .ok_or_else(|| Error::TreeNotFound { name: name.to_owned(), })?; + #[cfg(test)] + pause_open_tree_after_catalog_lookup(); let open = self.open_tree_state(tree_id)?; self.tree_from_state(tree_id, open) } @@ -244,14 +351,15 @@ impl DB { Ok(names) } - /// Drop a named tree from the catalog and stage its blobs for - /// checkpoint-time deletion. + /// Mark a named tree `Dropping` in the durable catalog. /// /// The catalog tombstone is hidden from [`Self::list_trees`] and /// from [`Self::open_tree`]. Existing handles are fenced before - /// this call returns. Physical cleanup completes in a later - /// [`Self::checkpoint`] after old handles/iterators have dropped - /// their cached root pins. + /// this call returns. A later [`Self::checkpoint`] or [`Self::gc`] + /// reclaims the unreachable closure and removes the catalog tombstone + /// after old handles and iterators release the exact live-root pin. + /// Snapshot roots are protected independently and do not by themselves + /// retain a `Dropping` family's live root. pub fn drop_tree(&self, name: &str) -> Result<()> { let name_bytes = validate_tree_name(name)?; let _maintenance = self.maintenance_gate.enter_exclusive(); @@ -263,8 +371,14 @@ impl DB { }); } }; - let guids = self.collect_tree_guids(entry.tree_id)?; - let seq = self.apply_system_batch_unlocked( + // Publish every acknowledged deferred write before the tree becomes + // Dropping. Later cleanup is reachability-based and must not depend + // on a writer path that `mark_runtime_dropped` has fenced. + { + let _commit = self.commit_gate.enter_writer(); + self.store.flush_write_deltas_for_tree(entry.tree_id)?; + } + self.apply_system_batch_unlocked( DB_CATALOG_TREE_ID, vec![BatchOp::Put { key: name_bytes.to_vec(), @@ -279,7 +393,6 @@ impl DB { }, ); self.mark_runtime_dropped(entry.tree_id); - self.stage_tree_delete_guids(&guids, seq); Ok(()) } @@ -338,70 +451,41 @@ impl DB { .iter() .map(|(_, gate)| gate.enter_exclusive()) .collect::>(); - { - let _commit = self.commit_gate.enter_writer(); - for (tree_id, _, _, _) in &scoped { - self.store.flush_write_deltas_for_tree(*tree_id)?; - } + let _commit = self.commit_gate.enter_writer(); + for (tree_id, _, _, _) in &scoped { + self.store.flush_write_deltas_for_tree(*tree_id)?; } let mut trees = HashMap::with_capacity(scoped.len()); for (_, name, prefix, tree) in scoped { - trees.insert(name, tree.snapshot_unlocked(prefix)); + trees.insert(name, tree.snapshot_unlocked(prefix)?); } DBView { trees } }; read(&view) } - /// Reclaim copy-on-write frames left unreachable by a crash that - /// happened while a snapshot was live — the DB-wide analog of - /// [`crate::Tree::gc`]. Marks every frame reachable from the catalog, - /// each live tree's root, and each live snapshot root, then frees the - /// rest from the shared buffer manager. Returns the count reclaimed. - /// Idempotent. Callers must not create or drop trees concurrently. + /// Reclaim persisted frames not reachable from the catalog, each live + /// tree's root, or a live snapshot root — the DB-wide analog of + /// [`crate::Tree::gc`]. + /// + /// GC freezes every tree, checkpoints the exact parent images it will + /// walk, and only then deletes unreachable blobs. This prevents a + /// durable old parent from losing a child that only a newer in-memory + /// parent stopped referencing. Returns the count reclaimed and is + /// idempotent. Concurrent create/drop operations serialize behind the + /// same DB maintenance fence. pub fn gc(&self) -> Result { - // Read the live tree set gate-free (`catalog_entries` runs a range - // scan that manages its own shared maintenance gate — holding the - // gate here would deadlock against it). Then freeze every tree's - // writers via their mutation gates, taken in tree-id order to - // match `DB::view`/`apply_atomic` and avoid deadlock. Callers must - // not create or drop trees concurrently with gc. - let mut scoped: Vec<(u64, Tree)> = Vec::new(); - for (_, entry) in self.catalog_entries()? { - if entry.state == CatalogState::Live { - let open = self.open_tree_state(entry.tree_id)?; - scoped.push((entry.tree_id, self.tree_from_state(entry.tree_id, open)?)); - } - } - let mut gates: Vec<(u64, Arc)> = scoped - .iter() - .map(|(id, t)| (*id, t.mutation_gate())) - .collect(); - gates.sort_by_key(|(id, _)| *id); - gates.dedup_by_key(|(id, _)| *id); - let _guards: Vec<_> = gates - .iter() - .map(|(_, gate)| gate.enter_exclusive()) - .collect(); - { - let _commit = self.commit_gate.enter_writer(); - for (tree_id, _) in &scoped { - self.store.flush_write_deltas_for_tree(*tree_id)?; - } - } - - let mut reachable: HashSet = HashSet::new(); - reachable.insert(root_guid_for_tree_id(DB_CATALOG_TREE_ID)); - reachable.extend(self.collect_tree_guids(DB_CATALOG_TREE_ID)?); - for (tree_id, _) in &scoped { - reachable.insert(root_guid_for_tree_id(*tree_id)); - reachable.extend(self.collect_tree_guids(*tree_id)?); - } - for snap_root in self.store.snapshot_roots() { - reachable.insert(snap_root); - reachable.extend(crate::engine::collect_blob_guids(&self.store, snap_root)?); + self.restore_dropping_runtime_fences()?; + let (freed, cleanup_complete) = self.gc_reachability_pass(usize::MAX, true)?; + if cleanup_complete && self.finalize_dropped_trees()? { + Tree::checkpoint_shared_store( + &self.store, + self.journal.as_ref(), + &self.maintenance_gate, + &self.commit_gate, + )?; } - self.store.gc_sweep_unreachable(&reachable) + Ok(freed) } /// Reclaim logical garbage and physical space from free store slots. @@ -415,7 +499,6 @@ impl DB { /// unchanged. pub fn vacuum(&self) -> Result { let unreachable = self.gc()?; - self.checkpoint()?; let mut stats = self.store.vacuum_storage()?; stats.unreachable_blobs = unreachable; Ok(stats) @@ -428,33 +511,27 @@ impl DB { /// instant; serialization then runs *outside* the freeze while live /// applies continue (forking the frames the snapshots reference). pub fn export_checkpoint(&self) -> Result { - // Enumerate live families gate-free (the catalog range scan manages - // its own maintenance gate; holding one here would deadlock it). - let mut families: Vec<(Vec, u64, Tree)> = Vec::new(); - for (name, entry) in self.catalog_entries()? { - if entry.state == CatalogState::Live { - let open = self.open_tree_state(entry.tree_id)?; - families.push(( - name, - entry.tree_id, - self.tree_from_state(entry.tree_id, open)?, - )); - } - } - - // Freeze every family's writers (tree-id order, matching - // DB::view/apply_atomic) and snapshot each — O(1) per snapshot. + // The DB maintenance fence covers catalog enumeration, deferred-write + // publication, and every O(1) snapshot capture. A concurrent create + // therefore lands wholly before or after this exported generation; + // it cannot be omitted after its data became visible. let snaps: Vec<(Vec, Snapshot)> = { - let mut gates: Vec<(u64, Arc)> = families - .iter() - .map(|(_, id, t)| (*id, t.mutation_gate())) - .collect(); - gates.sort_by_key(|(id, _)| *id); - gates.dedup_by_key(|(id, _)| *id); - let _guards: Vec<_> = gates - .iter() - .map(|(_, gate)| gate.enter_exclusive()) - .collect(); + let _maintenance = self.maintenance_gate.enter_exclusive(); + { + let _commit = self.commit_gate.enter_writer(); + self.store.flush_write_deltas_for_tree(DB_CATALOG_TREE_ID)?; + } + let mut families: Vec<(Vec, u64, Tree)> = Vec::new(); + for (name, entry) in self.catalog_entries_unlocked()? { + if entry.state == CatalogState::Live { + let open = self.open_tree_state(entry.tree_id)?; + families.push(( + name, + entry.tree_id, + self.tree_from_state(entry.tree_id, open)?, + )); + } + } { let _commit = self.commit_gate.enter_writer(); for (_, tree_id, _) in &families { @@ -464,7 +541,7 @@ impl DB { let mut snaps = Vec::with_capacity(families.len()); for (name, _, tree) in &families { - snaps.push((name.clone(), tree.snapshot_unlocked_unfenced(b""))); + snaps.push((name.clone(), tree.snapshot_unlocked_unfenced(b"")?)); } snaps }; @@ -476,6 +553,8 @@ impl DB { for entry in snap.range() { if let RangeEntry::Key { key, value, .. } = entry? { checkpoint::put_kv(&mut block, &key, &value); + #[cfg(test)] + pause_export_after_first_entry(); } } checkpoint::put_family(&mut buf, name, &block); @@ -505,19 +584,25 @@ impl DB { /// Force one DB-wide checkpoint round. /// - /// This flushes the shared BufferManager, applies pending - /// deletes, and truncates the shared WAL when it is safe. It is - /// not tied to any one named tree. + /// This flushes the shared BufferManager, applies pending deletes, and + /// truncates the shared WAL when it is safe. It then performs one bounded + /// exact-orphan / Dropping-tree cleanup pass and durably finalizes any + /// catalog removals completed by that pass. It is not tied to any one + /// named tree. pub fn checkpoint(&self) -> Result<()> { - self.stage_dropped_trees()?; - Tree::checkpoint_shared_parts( + self.restore_dropping_runtime_fences()?; + Tree::checkpoint_shared_store( &self.store, self.journal.as_ref(), &self.maintenance_gate, &self.commit_gate, )?; - if self.store.pending_delete_count() == 0 && self.finalize_dropped_trees()? { - Tree::checkpoint_shared_parts( + let (_, cleanup_complete) = self.gc_reachability_pass(AUTO_GC_BATCH_SIZE, false)?; + if cleanup_complete + && self.store.pending_delete_count() == 0 + && self.finalize_dropped_trees()? + { + Tree::checkpoint_shared_store( &self.store, self.journal.as_ref(), &self.maintenance_gate, @@ -582,6 +667,9 @@ impl DB { .count(), bm_dirty_count: bm.dirty_count, bm_pending_delete_count: bm.pending_delete_count, + bm_gc_orphan_backlog_count: bm.gc_orphan_backlog_count, + bm_gc_reclaimed_count: bm.gc_reclaimed_count, + bm_gc_last_full_sweep_deferred_count: bm.gc_last_full_sweep_deferred_count, bm_write_delta_count: bm.write_delta_count, bm_read_index_token_count: bm.read_index_token_count, bm_read_index_cache_entries: bm.read_index_cache_entries, @@ -639,6 +727,66 @@ impl DB { self.tree_from_state(DB_CATALOG_TREE_ID, open) } + fn restore_epoch_high_water(&self) -> Result<()> { + let catalog_root = root_guid_for_tree_id(DB_CATALOG_TREE_ID); + if !self.store.has_blob(catalog_root)? { + let durable = self.store.store_blob_guids()?; + let truly_fresh = durable.is_empty() + && self.store.cached_count() == 0 + && self.store.dirty_count() == 0; + if !truly_fresh { + return Err(Error::node_corrupt( + "db catalog root missing from an existing store", + )); + } + ensure_durable_root_blob(&self.store, catalog_root)?; + } + let mut roots = vec![catalog_root]; + for (_, entry) in self.catalog_entries()? { + if entry.state == CatalogState::Dropping { + // Bounded drop GC may durably remove descendants before the + // root itself. A crash in that intermediate state leaves an + // intentionally incomplete Dropping closure, which must not + // make DB open fail before cleanup can resume. No process- + // local handle or snapshot lease survives reopen, and this + // family can never become live again, so its epoch cannot + // constrain future COW decisions for surviving families. + continue; + } + let root_guid = root_guid_for_tree_id(entry.tree_id); + if !self.store.has_blob(root_guid)? { + return Err(Error::node_corrupt( + "db catalog references a missing live tree root", + )); + } + roots.push(root_guid); + } + + // Root high-water is the fast durable summary, but a DB epoch is + // shared across families: a high-epoch frame can be reachable from a + // different root than the snapshot that advanced the epoch. Walk the + // complete live closure so reopen is correct even after the family + // that originally advanced the epoch was dropped in an older session. + let mut frames = HashSet::new(); + for root in roots { + frames.insert(root); + frames.extend(crate::engine::collect_blob_guids(&self.store, root)?); + } + let mut high_water = 1u64; + for guid in frames { + let pin = self.store.pin(guid)?; + let frame = pin.read(); + let root_high_water = crate::layout::frame_epoch_high_water(frame.as_slice()); + let created_epoch = crate::layout::frame_created_epoch(frame.as_slice()); + if root_high_water == u64::MAX || created_epoch == u64::MAX { + return Err(Error::node_corrupt("snapshot epoch exhausted")); + } + high_water = high_water.max(root_high_water).max(created_epoch); + } + self.store.set_current_epoch(high_water); + Ok(()) + } + fn catalog_lookup_live(&self, name: &[u8]) -> Result> { Ok(self .catalog_entry(name)? @@ -683,23 +831,44 @@ impl DB { Ok(entries) } - fn stage_dropped_trees(&self) -> Result<()> { + fn catalog_entries_unlocked(&self) -> Result, CatalogEntry)>> { + let catalog = self.catalog_tree()?; + let mut cursor = catalog.range_unlocked(); + let mut entries = Vec::new(); + while let Some(item) = cursor.next_unlocked() { + if let RangeEntry::Key { key, value, .. } = item? { + if key == CATALOG_NEXT_TREE_ID_KEY { + continue; + } + let entry = decode_catalog_value(&key, &value)?; + let name = String::from_utf8(key.clone()) + .map_err(|_| Error::node_corrupt("db catalog key"))?; + self.catalog_cache.lock().unwrap().insert(name, entry); + entries.push((key, entry)); + } + } + Ok(entries) + } + + fn restore_dropping_runtime_fences(&self) -> Result<()> { for (_, entry) in self.catalog_entries()? { if entry.state == CatalogState::Dropping { - let _maintenance = self.maintenance_gate.enter_exclusive(); self.mark_runtime_dropped(entry.tree_id); - let guids = self.collect_tree_guids(entry.tree_id)?; - self.stage_tree_delete_guids(&guids, self.next_seq.load(Ordering::Acquire)); } } Ok(()) } fn finalize_dropped_trees(&self) -> Result { + let _maintenance = self.maintenance_gate.enter_exclusive(); + { + let _commit = self.commit_gate.enter_writer(); + self.store.flush_write_deltas_for_tree(DB_CATALOG_TREE_ID)?; + } let mut ops = Vec::new(); let mut finalized_tree_ids = Vec::new(); let mut finalized_names = Vec::new(); - for (name, entry) in self.catalog_entries()? { + for (name, entry) in self.catalog_entries_unlocked()? { if entry.state == CatalogState::Dropping && !self .store @@ -715,7 +884,6 @@ impl DB { if ops.is_empty() { return Ok(false); } - let _maintenance = self.maintenance_gate.enter_exclusive(); self.apply_system_batch_unlocked(DB_CATALOG_TREE_ID, ops)?; let mut cache = self.catalog_cache.lock().unwrap(); for name in finalized_names { @@ -729,6 +897,73 @@ impl DB { Ok(true) } + /// Freeze the catalog/tree set, make that exact topology durable, and + /// reclaim at most `limit` blobs outside its pinned-root closure. + /// `Dropping` roots with an exact live pin are treated as canonical for + /// this pass. Snapshot roots are walked independently, so a snapshot in + /// one family never forces traversal of an unrelated Dropping closure. + fn gc_reachability_pass(&self, limit: usize, force_full_scan: bool) -> Result<(usize, bool)> { + let _maintenance = self.maintenance_gate.enter_exclusive(); + { + let _commit = self.commit_gate.enter_writer(); + self.store.flush_write_deltas_for_tree(DB_CATALOG_TREE_ID)?; + } + let entries = self.catalog_entries_unlocked()?; + for (_, entry) in &entries { + if entry.state == CatalogState::Dropping { + self.mark_runtime_dropped(entry.tree_id); + } + } + { + let _commit = self.commit_gate.enter_writer(); + for (_, entry) in &entries { + self.store.flush_write_deltas_for_tree(entry.tree_id)?; + } + } + Tree::checkpoint_shared_store_with_maintenance_held( + &self.store, + self.journal.as_ref(), + &self.commit_gate, + )?; + + let mut reachable = HashSet::new(); + let catalog_root = root_guid_for_tree_id(DB_CATALOG_TREE_ID); + reachable.insert(catalog_root); + reachable.extend(self.collect_tree_guids(DB_CATALOG_TREE_ID)?); + let snapshot_roots = self.store.snapshot_roots_pinned()?; + for (_, entry) in &entries { + let root = root_guid_for_tree_id(entry.tree_id); + let retain = entry.state == CatalogState::Live + || (entry.state == CatalogState::Dropping && self.store.blob_is_pinned(root)); + if retain { + reachable.insert(root); + reachable.extend(self.collect_tree_guids(entry.tree_id)?); + } + } + let canonical_reachable = reachable.clone(); + for snapshot_root in snapshot_roots { + let root = snapshot_root.guid(); + reachable.insert(root); + reachable.extend(crate::engine::collect_blob_guids(&self.store, root)?); + } + if force_full_scan + || entries + .iter() + .any(|(_, entry)| entry.state == CatalogState::Dropping) + { + let outcome = self.store.gc_sweep_unreachable_with_canonical_bounded( + &reachable, + &canonical_reachable, + limit, + )?; + Ok((outcome.freed, outcome.complete)) + } else { + self.store + .reclaim_retired_orphans_bounded(limit) + .map(|freed| (freed, true)) + } + } + fn collect_tree_guids(&self, tree_id: u64) -> Result> { let root_guid = root_guid_for_tree_id(tree_id); if !self.store.has_blob(root_guid)? { @@ -737,12 +972,6 @@ impl DB { crate::engine::collect_blob_guids(&self.store, root_guid) } - fn stage_tree_delete_guids(&self, guids: &[BlobGuid], seq: u64) { - for guid in guids { - self.store.mark_for_delete(*guid, seq); - } - } - fn mark_runtime_dropped(&self, tree_id: u64) { if let Some(open) = self.trees.lock().unwrap().get(&tree_id) { open.runtime.mark_dropped(); @@ -758,7 +987,7 @@ impl DB { return Err(Error::TreeDropped); } let root_guid = root_guid_for_tree_id(tree_id); - ensure_root_blob(&self.store, root_guid)?; + ensure_durable_root_blob(&self.store, root_guid)?; let open = OpenTree { root_guid, runtime: TreeRuntime::new(), @@ -886,6 +1115,7 @@ impl DB { } fn apply_batch_groups_in_memory(&self, groups: &[DBBatchGroup], base_seq: u64) -> Result<()> { + let commit = (self.store.fork_barrier() != 0).then(|| self.commit_gate.enter_writer()); let mut group_base = base_seq; for group in groups { group @@ -893,10 +1123,10 @@ impl DB { .apply_batch_walker_inline(&group.ops, group_base, None)?; group_base += count_group_wal_ops(group); } + drop(commit); if self.cfg.memory_flush_on_write { if let Some(group) = groups.first() { - group.tree.flush_dirty_inline()?; - group.tree.flush_pending_deletes_inline()?; + group.tree.flush_inline()?; } } Ok(()) @@ -909,7 +1139,7 @@ impl DB { open.clone() } else { let root_guid = root_guid_for_tree_id(tree_id); - ensure_root_blob(&self.store, root_guid)?; + ensure_durable_root_blob(&self.store, root_guid)?; let open = OpenTree { root_guid, runtime: TreeRuntime::new(), @@ -1206,3 +1436,489 @@ fn next_allocated_tree_id(tree_id: u64) -> Result { } Ok(next) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::blob_store::{AlignedBlobBuf, MemoryBlobStore}; + use std::collections::BTreeMap; + use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering}; + use std::sync::mpsc; + use std::thread; + use std::time::{Duration, Instant}; + use tempfile::tempdir; + + struct FailFlushOnceStore { + inner: MemoryBlobStore, + fail_next_flush: AtomicBool, + } + + impl FailFlushOnceStore { + fn new() -> Self { + Self { + inner: MemoryBlobStore::new(), + fail_next_flush: AtomicBool::new(false), + } + } + } + + impl BlobStore for FailFlushOnceStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> Result<()> { + self.inner.read_blob(guid, dst) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { + self.inner.write_blob(guid, src) + } + + fn delete_blob(&self, guid: BlobGuid) -> Result<()> { + self.inner.delete_blob(guid) + } + + fn list_blobs(&self) -> Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> Result<()> { + if self.fail_next_flush.swap(false, AtomicOrdering::AcqRel) { + return Err(Error::BlobStoreIo(std::io::Error::other( + "injected root flush failure", + ))); + } + self.inner.flush() + } + + fn needs_flush(&self) -> bool { + self.inner.needs_flush() + } + + fn has_blob(&self, guid: BlobGuid) -> Result { + self.inner.has_blob(guid) + } + } + + #[test] + fn create_tree_durably_publishes_root_before_live_catalog() { + let store = Arc::new(FailFlushOnceStore::new()); + let mut cfg = TreeConfig::memory(); + cfg.checkpoint.enabled = false; + cfg.memory_flush_on_write = true; + let store_dyn: Arc = store.clone(); + let db = DB::open_with_blob_store(cfg.clone(), store_dyn).unwrap(); + + store.fail_next_flush.store(true, AtomicOrdering::Release); + assert!(db.create_tree("empty").is_err()); + assert!(db.catalog_lookup_live(b"empty").unwrap().is_none()); + assert!(matches!( + db.open_tree("empty"), + Err(Error::TreeNotFound { .. }) + )); + assert!(store + .inner + .has_blob(root_guid_for_tree_id(FIRST_USER_TREE_ID)) + .unwrap()); + drop(db); + + // Reopen while the failed attempt's deterministic root is orphaned. + // The catalog stays healthy and retry flushes that root before Live. + let store_dyn: Arc = store.clone(); + let reopened = DB::open_with_blob_store(cfg.clone(), store_dyn).unwrap(); + assert!(matches!( + reopened.open_tree("empty"), + Err(Error::TreeNotFound { .. }) + )); + drop(reopened.create_tree("empty").unwrap()); + drop(reopened); + + // An empty tree has no user WAL record; its root+catalog ordering + // alone must be sufficient for a clean subsequent reopen. + let store_dyn: Arc = store; + let final_db = DB::open_with_blob_store(cfg, store_dyn).unwrap(); + let empty = final_db.open_tree("empty").unwrap(); + assert!(empty.get(b"missing").unwrap().is_none()); + } + + #[test] + fn db_reopens_max_minus_one_as_exhausted_and_rejects_max() { + let dir = tempdir().unwrap(); + let cfg = || { + let mut cfg = TreeConfig::new(dir.path()); + cfg.checkpoint.enabled = false; + cfg.buffer_pool_size = 16; + cfg + }; + + let tree_id; + { + let db = DB::open(cfg()).unwrap(); + let tree = db.create_tree("objects").unwrap(); + tree.put(b"key", b"before").unwrap(); + tree_id = db.catalog_lookup_live(b"objects").unwrap().unwrap(); + db.store.set_current_epoch(u64::MAX - 2); + + let snapshot = tree.snapshot(b"").unwrap(); + assert_eq!(snapshot.epoch(), u64::MAX - 2); + assert_eq!(db.store.current_epoch(), u64::MAX - 1); + drop(snapshot); + db.checkpoint().unwrap(); + } + + { + let db = DB::open(cfg()).unwrap(); + assert_eq!(db.store.current_epoch(), u64::MAX - 1); + let tree = db.open_tree("objects").unwrap(); + let error = tree.snapshot(b"").unwrap_err(); + assert!(matches!(error, Error::SnapshotEpochExhausted)); + assert_eq!(tree.get(b"key").unwrap().as_deref(), Some(&b"before"[..])); + tree.put(b"key", b"after").unwrap(); + + let root_guid = root_guid_for_tree_id(tree_id); + let root = db.store.pin(root_guid).unwrap(); + { + let mut frame = root.write(); + crate::layout::set_frame_epoch_high_water(frame.as_mut_slice(), u64::MAX); + } + db.store + .mark_dirty_cached(root_guid, crate::store::STRUCTURAL_SEQ, root.as_ref()); + drop(root); + db.checkpoint().unwrap(); + } + + let error = DB::open(cfg()).unwrap_err(); + assert!(matches!( + error, + Error::NodeCorrupt { + context: "snapshot epoch exhausted", + .. + } + )); + } + + #[test] + fn open_tree_cannot_resurrect_runtime_after_concurrent_drop() { + let db = DB::open(TreeConfig::memory()).unwrap(); + let existing = db.create_tree("objects").unwrap(); + let tree_id = db.catalog_lookup_live(b"objects").unwrap().unwrap(); + drop(existing); + db.trees.lock().unwrap().remove(&tree_id); + + let barrier = Arc::new(OpenTreeCatalogBarrier::new()); + let opener_db = db.clone(); + let opener_barrier = Arc::clone(&barrier); + let opener = thread::spawn(move || { + set_open_tree_catalog_barrier_for_current_thread(opener_barrier); + opener_db.open_tree("objects") + }); + barrier.entered.wait(); + + let drop_db = db.clone(); + let (drop_done_tx, drop_done_rx) = mpsc::sync_channel(1); + let dropper = thread::spawn(move || { + let result = drop_db.drop_tree("objects"); + drop_done_tx.send(()).unwrap(); + result + }); + let deadline = Instant::now() + Duration::from_secs(2); + while !db.maintenance_gate.writer_pending_for_test() { + assert!( + Instant::now() < deadline, + "drop_tree never queued behind open_tree's shared fence", + ); + thread::yield_now(); + } + assert!( + drop_done_rx + .recv_timeout(Duration::from_millis(30)) + .is_err(), + "drop_tree bypassed the in-flight open_tree construction", + ); + + barrier.release.wait(); + let opened = opener.join().unwrap().unwrap(); + dropper.join().unwrap().unwrap(); + assert!(matches!( + opened.put(b"hidden", b"write"), + Err(Error::TreeDropped) + )); + assert!(matches!( + db.open_tree("objects"), + Err(Error::TreeNotFound { .. }) + )); + assert!(db + .trees + .lock() + .unwrap() + .get(&tree_id) + .unwrap() + .runtime + .is_dropped()); + } + + #[test] + fn export_checkpoint_restarts_after_concurrent_compaction() { + let db = DB::open(TreeConfig::memory()).unwrap(); + let tree = db.create_tree("objects").unwrap(); + let mut expected = BTreeMap::new(); + for index in 0..600u32 { + let key = format!("key/{index:06}").into_bytes(); + let value = vec![(index % 251) as u8; 512]; + tree.put(&key, &value).unwrap(); + expected.insert(key, value); + } + for index in (0..600u32).step_by(4) { + let key = format!("key/{index:06}").into_bytes(); + assert!(tree.delete(&key).unwrap()); + expected.remove(&key); + } + + let barrier = Arc::new(ExportFirstEntryBarrier::new()); + let export_db = db.clone(); + let export_barrier = Arc::clone(&barrier); + let exporter = thread::spawn(move || { + set_export_first_entry_barrier_for_current_thread(export_barrier); + export_db.export_checkpoint() + }); + barrier.entered.wait(); + + let compactions_before = tree.stats().unwrap().total_compactions; + for _ in 0..4 { + tree.compact().unwrap(); + } + let compactions_after = tree.stats().unwrap().total_compactions; + assert!( + compactions_after > compactions_before, + "test must rewrite at least one shared snapshot frame", + ); + barrier.release.wait(); + + let image = exporter.join().unwrap().unwrap(); + let decoded = checkpoint::decode(image.as_bytes()).unwrap(); + let (_, records) = decoded + .families + .iter() + .find(|(name, _)| *name == b"objects") + .unwrap(); + let exported = records + .iter() + .map(|(key, value)| (key.to_vec(), value.to_vec())) + .collect::>(); + assert_eq!(exported, expected); + } + + #[test] + fn export_checkpoint_captures_family_created_before_its_fence() { + let db = DB::open(TreeConfig::memory()).unwrap(); + let blocker = db.maintenance_gate.enter_shared(); + + let create_db = db.clone(); + let (create_done_tx, create_done_rx) = mpsc::channel(); + let creator = thread::spawn(move || { + let result = create_db.create_tree("created-before-export"); + create_done_tx.send(()).unwrap(); + result + }); + + let deadline = Instant::now() + Duration::from_secs(2); + while !db.maintenance_gate.writer_pending_for_test() { + assert!( + Instant::now() < deadline, + "create_tree never queued on the maintenance fence" + ); + thread::yield_now(); + } + + let export_db = db.clone(); + let (export_started_tx, export_started_rx) = mpsc::channel(); + let exporter = thread::spawn(move || { + export_started_tx.send(()).unwrap(); + export_db.export_checkpoint() + }); + export_started_rx.recv().unwrap(); + assert!( + create_done_rx + .recv_timeout(Duration::from_millis(30)) + .is_err(), + "create_tree bypassed an existing maintenance reader" + ); + + // The queued creator owns WRITE_BIT first. Export cannot enumerate + // until that catalog mutation is complete, so the family must be in + // the exported generation (its later user writes may validly fall on + // either side of the boundary). + drop(blocker); + drop(creator.join().unwrap().unwrap()); + let image = exporter.join().unwrap().unwrap(); + let decoded = checkpoint::decode(image.as_bytes()).unwrap(); + assert!( + decoded + .families + .iter() + .any(|(name, _)| *name == b"created-before-export"), + "export omitted a family whose catalog commit preceded its fence" + ); + } + + #[test] + fn db_view_waits_for_checkpoint_delta_flush_before_registering_barrier() { + let db = DB::open(TreeConfig::memory()).unwrap(); + let tree = db.create_tree("objects").unwrap(); + tree.put(b"key", b"old").unwrap(); + let tree_id = db + .catalog_lookup_live(b"objects") + .unwrap() + .expect("created tree id"); + let root_guid = root_guid_for_tree_id(tree_id); + let seq = db.next_seq.fetch_add(1, Ordering::Relaxed); + db.store + .stage_write_delta_put(tree_id, root_guid, b"key", b"checkpoint-value", seq, false); + + let root_pin = db.store.pin(root_guid).unwrap(); + let root_guard = root_pin.write(); + let checkpoint_db = db.clone(); + let (checkpoint_tx, checkpoint_rx) = mpsc::channel(); + let checkpoint = thread::spawn(move || { + checkpoint_tx.send(checkpoint_db.checkpoint()).unwrap(); + }); + let deadline = Instant::now() + Duration::from_secs(2); + while !db.commit_gate.checkpoint_pending_for_test() { + assert!( + Instant::now() < deadline, + "checkpoint never acquired commit-exclusive" + ); + thread::yield_now(); + } + + let view_db = db.clone(); + let (view_tx, view_rx) = mpsc::channel(); + let view_worker = thread::spawn(move || { + let result = view_db.view(&[("objects", b"".as_slice())], |view| { + view.tree("objects") + .expect("captured objects tree") + .get(b"key") + }); + view_tx.send(result).unwrap(); + }); + assert!( + view_rx.recv_timeout(Duration::from_millis(50)).is_err(), + "DB view must not register between delta shared-check and write", + ); + + drop(root_guard); + drop(root_pin); + checkpoint_rx + .recv_timeout(Duration::from_secs(2)) + .unwrap() + .unwrap(); + checkpoint.join().unwrap(); + let value = view_rx + .recv_timeout(Duration::from_secs(2)) + .unwrap() + .unwrap(); + view_worker.join().unwrap(); + assert_eq!(value.as_deref(), Some(&b"checkpoint-value"[..])); + } + + #[test] + fn reopen_skips_partially_reclaimed_dropping_closure() { + let dir = tempdir().unwrap(); + let cfg = || { + let mut cfg = TreeConfig::new(dir.path()); + cfg.checkpoint.enabled = false; + cfg.buffer_pool_size = 32; + cfg + }; + + let tree_id; + { + let db = DB::open(cfg()).unwrap(); + let doomed = db.create_tree("doomed").unwrap(); + tree_id = db + .catalog_lookup_live(b"doomed") + .unwrap() + .expect("created doomed tree id"); + let value = vec![0x5A; 1024]; + for i in 0..1200u32 { + doomed + .put(format!("drop/{i:08}").as_bytes(), &value) + .unwrap(); + } + db.checkpoint().unwrap(); + + let root = root_guid_for_tree_id(tree_id); + let closure = crate::engine::collect_blob_guids(&db.store, root).unwrap(); + assert!( + closure.len() > 1, + "partial-drop recovery test requires a non-root child", + ); + let child = closure[1]; + + db.drop_tree("doomed").unwrap(); + drop(doomed); + // Persist the Dropping catalog state without invoking DB's drop + // sweep, then deterministically reclaim exactly one child while + // retaining the still-referencing root. This is the durable + // intermediate state a bounded sweep can leave at crash time. + Tree::checkpoint_shared_store( + &db.store, + db.journal.as_ref(), + &db.maintenance_gate, + &db.commit_gate, + ) + .unwrap(); + let mut retain: HashSet<_> = db.store.list_blobs().unwrap().into_iter().collect(); + assert!(retain.remove(&child)); + let outcome = db.store.gc_sweep_unreachable_bounded(&retain, 1).unwrap(); + assert_eq!(outcome.freed, 1); + assert!(db.store.store_has_blob(root).unwrap()); + assert!(!db.store.store_has_blob(child).unwrap()); + assert!( + crate::engine::collect_blob_guids(&db.store, root).is_err(), + "test did not create a partially reclaimed Dropping closure", + ); + } + + // Open must ignore the intentionally incomplete Dropping family's + // epoch metadata so the normal recovery sweep can finish it. + { + let db = DB::open(cfg()).unwrap(); + assert!(db.list_trees().unwrap().is_empty()); + + // An unrelated live snapshot must not make the incomplete + // Dropping closure canonical again. Snapshot roots are protected + // by their own pinned closure. + let unrelated = db.create_tree("unrelated").unwrap(); + unrelated.put(b"live", b"value").unwrap(); + let unrelated_snapshot = unrelated.snapshot(b"").unwrap(); + db.gc().unwrap(); + assert!( + !db.store + .store_has_blob(root_guid_for_tree_id(tree_id)) + .unwrap(), + "an unrelated family snapshot must not retain the Dropping root", + ); + assert!( + db.catalog_entry(b"doomed").unwrap().is_none(), + "the completed Dropping family must leave the catalog while the unrelated snapshot is live", + ); + assert!(matches!( + db.open_tree("doomed"), + Err(Error::TreeNotFound { .. }) + )); + assert_eq!( + unrelated_snapshot.get(b"live").unwrap().as_deref(), + Some(&b"value"[..]), + ); + drop(unrelated_snapshot); + db.drop_tree("unrelated").unwrap(); + drop(unrelated); + db.gc().unwrap(); + } + + let db = DB::open(cfg()).unwrap(); + assert!(db.list_trees().unwrap().is_empty()); + assert!(!db + .store + .store_has_blob(root_guid_for_tree_id(tree_id)) + .unwrap()); + } +} diff --git a/src/api/errors.rs b/src/api/errors.rs index 7c08501..2b175b2 100644 --- a/src/api/errors.rs +++ b/src/api/errors.rs @@ -120,6 +120,10 @@ pub enum Error { /// DB trees share one buffer manager, so reclaiming unreachable /// frames safely needs a DB-wide pass rather than a single tree's. GcRequiresStandaloneTree, + /// The monotonic copy-on-write snapshot epoch reached its largest + /// durable value. Existing data remains valid, but no new snapshot can + /// be registered without wrapping the counter. + SnapshotEpochExhausted, } impl Error { @@ -221,6 +225,7 @@ impl std::fmt::Display for Error { f, "gc is only supported on standalone trees; trees opened through a DB share a buffer manager" ), + Self::SnapshotEpochExhausted => write!(f, "snapshot epoch exhausted"), } } } @@ -326,6 +331,10 @@ mod tests { "invalid DB tree name: empty", ), (Error::TreeDropped.to_string(), "DB tree has been dropped"), + ( + Error::SnapshotEpochExhausted.to_string(), + "snapshot epoch exhausted", + ), ]; for (actual, expected) in cases { @@ -355,5 +364,6 @@ mod tests { .source() .is_none()); assert!(Error::TreeDropped.source().is_none()); + assert!(Error::SnapshotEpochExhausted.source().is_none()); } } diff --git a/src/api/snapshot.rs b/src/api/snapshot.rs index 61b4e66..8077118 100644 --- a/src/api/snapshot.rs +++ b/src/api/snapshot.rs @@ -10,15 +10,12 @@ //! Creation is O(one frame copy); the per-write cost is zero while no //! snapshot is live, and bounded by the root→leaf frame path length on //! the first write to each region while one is. Dropping the handle (or -//! calling [`Snapshot::retire`]) retires the snapshot and lowers the -//! global fork barrier. - -use std::ops::Deref; -use std::sync::Arc; - -use crate::store::BufferManager; +//! calling [`Snapshot::retire`]) releases its lease; the global fork +//! barrier lowers after every cloned view and owned cursor derived from +//! that snapshot has also been dropped. use super::view::View; +use std::ops::Deref; /// A stable copy-on-write snapshot of a tree or prefix subtree. /// @@ -27,22 +24,23 @@ use super::view::View; /// writes. All [`View`] read operations are available through `Deref` /// (`snapshot.get(..)`, `snapshot.range()`, `snapshot.scan(..)`, …). /// -/// The snapshot is retired when the handle is dropped; hold it for as -/// long as the stable view is needed. +/// The snapshot epoch is retired after this handle and all cloned views or +/// owned cursors derived from it are dropped. +/// Persisted copy-on-write frames are not reclaimed inline on handle drop. +/// A later standalone or DB checkpoint reclaims a bounded exact batch, while +/// [`crate::Tree::gc`] and [`crate::DB::gc`] provide complete reachability +/// sweeps; [`crate::StoreStats::live_blobs`] may therefore remain elevated +/// until one of those maintenance frontiers completes. pub struct Snapshot { view: Option, - store: Arc, epoch: u64, - retired: bool, } impl Snapshot { - pub(crate) fn new(view: View, store: Arc, epoch: u64) -> Self { + pub(crate) fn new(view: View, epoch: u64) -> Self { Self { view: Some(view), - store, epoch, - retired: false, } } @@ -60,18 +58,11 @@ impl Snapshot { self.view.as_ref().expect("snapshot retired") } - /// Retire the snapshot now, releasing its hold on the fork barrier. - /// Equivalent to dropping the handle, but explicit. Idempotent. + /// Release this snapshot handle now. The fork-barrier epoch retires + /// after the last cloned [`View`] or owned range cursor derived from + /// it is also dropped. pub fn retire(mut self) { - self.retire_inner(); - } - - fn retire_inner(&mut self) { - if !self.retired { - self.retired = true; - drop(self.view.take()); - self.store.retire_snapshot(self.epoch); - } + drop(self.view.take()); } } @@ -83,12 +74,6 @@ impl Deref for Snapshot { } } -impl Drop for Snapshot { - fn drop(&mut self) { - self.retire_inner(); - } -} - impl std::fmt::Debug for Snapshot { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Snapshot") diff --git a/src/api/stats.rs b/src/api/stats.rs index 6a25f73..e49237f 100644 --- a/src/api/stats.rs +++ b/src/api/stats.rs @@ -158,6 +158,7 @@ pub struct VacuumStats { /// root; the totals are pre-summed for the common "how big is the /// tree?" question. All bytes / counts are sums over `blobs`. #[derive(Debug, Clone)] +#[non_exhaustive] pub struct TreeStats { /// Number of distinct blobs reachable from the tree root. pub blob_count: u32, @@ -203,6 +204,19 @@ pub struct TreeStats { /// queued deletes and deletes already claimed by a checkpoint /// epoch but not yet completed. pub bm_pending_delete_count: usize, + /// COW/structural orphan debt: parent-publication staging, + /// snapshot-held candidates, and clean-frontier reclaim backlog. + pub bm_gc_orphan_backlog_count: usize, + /// Cumulative blobs physically reclaimed by full reachability sweeps or + /// clean-frontier exact reclaim. + pub bm_gc_reclaimed_count: u64, + /// Unreachable candidates deferred by the most recent successful full + /// reachability sweep because of its batch limit or a pin. + /// + /// Bounded exact-reclaim passes do not change this value; use + /// [`Self::bm_gc_orphan_backlog_count`] for current COW/structural FIFO + /// debt. + pub bm_gc_last_full_sweep_deferred_count: usize, /// Deferred WAL-backed writes waiting to be merged into ART blob /// frames. WAL truncation waits for this to reach zero. pub bm_write_delta_count: usize, @@ -403,6 +417,7 @@ impl TreeStats { /// `DB`, not any one ART root. Per-tree shape counters still live in /// [`TreeStats`] because shape is root-specific. #[derive(Debug, Clone)] +#[non_exhaustive] pub struct DBStats { /// Number of named trees opened by this process. pub open_tree_count: usize, @@ -411,6 +426,18 @@ pub struct DBStats { pub bm_dirty_count: usize, /// Number of deferred deletes across every tree. pub bm_pending_delete_count: usize, + /// COW/structural orphan debt, including staged and snapshot-held blobs. + pub bm_gc_orphan_backlog_count: usize, + /// Cumulative blobs physically reclaimed by full reachability sweeps or + /// clean-frontier exact reclaim. + pub bm_gc_reclaimed_count: u64, + /// Unreachable candidates deferred by the most recent successful full + /// reachability sweep because of its batch limit or a pin. + /// + /// Bounded exact-reclaim passes do not change this value; use + /// [`Self::bm_gc_orphan_backlog_count`] for current COW/structural FIFO + /// debt. + pub bm_gc_last_full_sweep_deferred_count: usize, /// Deferred WAL-backed writes waiting to be merged into ART blob /// frames. WAL truncation waits for this to reach zero. pub bm_write_delta_count: usize, @@ -556,6 +583,7 @@ pub struct JournalStats { /// Reopen-time recovery telemetry captured while opening a tree. #[derive(Debug, Clone, Copy, Default)] +#[non_exhaustive] pub struct OpenStats { /// WAL records scanned during reopen. pub wal_replay_records: u64, @@ -563,6 +591,10 @@ pub struct OpenStats { pub wal_replay_bytes: u64, /// Wall-clock time spent replaying the WAL, in microseconds. pub wal_replay_micros: u64, + /// Wall-clock time spent restoring the DB-wide snapshot epoch from every + /// frame reachable through the live catalog closure. Standalone + /// trees report zero because their root carries a direct high-water. + pub epoch_recovery_micros: u64, /// Non-zero iff reopen stopped at a torn WAL tail. pub wal_torn_tail: bool, } diff --git a/src/api/tree.rs b/src/api/tree.rs index 12fa9d7..432c51d 100644 --- a/src/api/tree.rs +++ b/src/api/tree.rs @@ -27,6 +27,7 @@ use crate::concurrency::{CommitGate, EndpointLocks, Gate}; use crate::engine; use crate::engine::{ KeyRangeBuilder, KeyRangeEntry, KeyRangeEntryRef, KeyScanOutcome, PrefixCount, RangeBuilder, + RangeIter, }; use crate::journal::codec::{ encode_erase_record, encode_insert_record, encode_rename_object_record, @@ -40,9 +41,11 @@ use crate::layout::{BlobGuid, DATA_AREA_START, PAGE_SIZE, ROOT_BLOB_GUID}; use crate::store::blob_store::{AlignedBlobBuf, BlobStore, FileBlobStore, MemoryBlobStore}; use crate::store::{ BlobFrame, BlobFrameRef, BufferManager, BufferStats, CachedBlob, DirtySnapshotEntry, - WriteDeltaEntry, WriteThroughEntry, WriteThroughStatus, + WriteDeltaEntry, WriteThroughEntry, }; +const AUTO_GC_BATCH_SIZE: usize = 256; + use super::atomic::{AtomicBatch, BatchOp, Record, RecordVersion}; const ONLINE_COMPACT_BLOB_BUDGET: usize = 256; @@ -51,6 +54,44 @@ const SHAPE_UNDERFILLED_CHILD_FILL_PER_MILLE: u32 = 350; const SHAPE_OVERFULL_CHILD_FILL_PER_MILLE: u32 = 850; const WRITE_DELTA_FLUSH_THRESHOLD: usize = 262_144; +#[cfg(test)] +struct FullGcSnapshotCaptureBarrier { + entered: std::sync::Barrier, + release: std::sync::Barrier, +} + +#[cfg(test)] +impl FullGcSnapshotCaptureBarrier { + fn new() -> Self { + Self { + entered: std::sync::Barrier::new(2), + release: std::sync::Barrier::new(2), + } + } +} + +#[cfg(test)] +thread_local! { + static FULL_GC_SNAPSHOT_CAPTURE_BARRIER: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(test)] +fn set_full_gc_snapshot_capture_barrier_for_current_thread( + barrier: Arc, +) { + FULL_GC_SNAPSHOT_CAPTURE_BARRIER.with(|slot| *slot.borrow_mut() = Some(barrier)); +} + +#[cfg(test)] +fn pause_full_gc_after_snapshot_capture() { + let barrier = FULL_GC_SNAPSHOT_CAPTURE_BARRIER.with(|slot| slot.borrow_mut().take()); + if let Some(barrier) = barrier { + barrier.entered.wait(); + barrier.release.wait(); + } +} + type BatchOverlay = HashMap, Option>; type CheckpointMap = HashMap; type CheckpointBytes = Vec<(BlobGuid, u64, u64, AlignedBlobBuf)>; @@ -179,7 +220,9 @@ pub(crate) fn prefix_count_from_seen( /// records the blob content version it was built from; if an /// interleaved writer changes a frame, the iterator discards its /// stack and performs a marker-aware seek from the last emitted -/// key / delimiter lower bound. +/// key / delimiter lower bound. Each `next()` holds the shared +/// maintenance and mutation gates for that cursor step only; a +/// caller-paused iterator does not block structural maintenance. /// - **Writes** (`put`, `delete`) enter the shared side of /// `maintenance_gate`, lock the key's endpoint shard, then hold the /// per-blob `HybridLatch` exclusively for the blobs they touch. @@ -191,9 +234,9 @@ pub(crate) fn prefix_count_from_seen( /// exclusive windows on `maintenance_gate` while folding/deleting /// cross-blob edges. Blob-local compaction runs on the shared /// side under per-blob latches. Point reads rely on parent/child -/// blob latches instead of this tree-wide gate; range scans and -/// foreground writers enter the shared side, while `atomic` and -/// scoped `view` capture enter the exclusive side. +/// blob latches instead of this tree-wide gate; each range-cursor +/// advance and foreground write enters the shared side, while +/// snapshot/view capture enters the exclusive mutation side. /// - **`rename`** locks the two endpoint shards for `src` and /// `dst` in canonical order after entering the shared maintenance /// side. `put` / `delete` lock their single endpoint shard, so a @@ -525,7 +568,7 @@ impl Tree { fn open_inner(cfg: TreeConfig, bm: Arc, attach_journal: bool) -> Result { let root_guid = ROOT_BLOB_GUID; - ensure_root_blob(&bm, root_guid)?; + ensure_durable_root_blob(&bm, root_guid)?; let mut open_stats = OpenStats::default(); // Restore the CoW epoch above every persisted frame's @@ -534,6 +577,9 @@ impl Tree { { let root = bm.pin(root_guid)?; let high_water = crate::layout::frame_epoch_high_water(root.read().as_slice()); + if high_water == u64::MAX { + return Err(Error::node_corrupt("snapshot epoch exhausted")); + } bm.set_current_epoch(high_water); } // File-backed WAL trees replay every durable record onto the @@ -908,10 +954,12 @@ impl Tree { (outcome, None) } } else { - // No WAL — no journal/checkpoint publish boundary to - // race with. `memory_flush_on_write` (if set) - // flushes dirty + pending-delete sets inline before - // returning. + // A no-WAL mutation only needs the publish gate while a + // snapshot can force COW. Keep the guard around edge + // repoint + parent dirty/orphan promotion, then release it + // before any inline flush (CommitGate is non-reentrant). + let commit = + (self.store.fork_barrier() != 0).then(|| self.commit_gate.enter_writer()); let outcome = engine::insert_multi_conditional( &self.store, &self.root_pin, @@ -925,9 +973,9 @@ impl Tree { self.store .mark_dirty_cached(self.root_guid, seq, self.root_pin.as_ref()); } + drop(commit); if outcome.mutated && self.cfg.memory_flush_on_write { - self.flush_dirty_inline()?; - self.flush_pending_deletes_inline()?; + self.flush_inline()?; } (outcome, None) } @@ -1057,6 +1105,8 @@ impl Tree { } // No-op delete (key wasn't there) is not logged. } else { + let commit = + (self.store.fork_barrier() != 0).then(|| self.commit_gate.enter_writer()); let outcome = engine::erase_multi_conditional( &self.store, &self.root_pin, @@ -1069,15 +1119,13 @@ impl Tree { self.store .mark_dirty_cached(self.root_guid, seq, self.root_pin.as_ref()); } + drop(commit); if outcome.mutated && self.cfg.memory_flush_on_write { // Flush every blob the walker touched (root + any // children) — no WAL means this is the sole // durability path. snapshot_dirty drains all // entries; we commit each through the store. - self.flush_dirty_inline()?; - // Plus drain any deferred deletes queued by a - // conservative parent-unlink path. - self.flush_pending_deletes_inline()?; + self.flush_inline()?; } (outcome, None) } @@ -1173,6 +1221,8 @@ impl Tree { encode_rename_object_record(&mut record, seq, self.tree_id, src, dst, force); journal.submit(record, self.cfg.durability.wal_sync())? } else { + let commit = + (self.store.fork_barrier() != 0).then(|| self.commit_gate.enter_writer()); let erase_out = engine::erase_multi( &self.store, &self.root_pin, @@ -1192,12 +1242,12 @@ impl Tree { self.store .mark_dirty_cached(self.root_guid, seq, self.root_pin.as_ref()); } + drop(commit); if self.cfg.memory_flush_on_write { // Walker may have dirtied child blobs across the // erase + insert sequence — drain the full set. // The erase half can also queue SubtreeGone deletes. - self.flush_dirty_inline()?; - self.flush_pending_deletes_inline()?; + self.flush_inline()?; } None } @@ -1302,10 +1352,11 @@ impl Tree { ack.wait()?; } } else { + let commit = (self.store.fork_barrier() != 0).then(|| self.commit_gate.enter_writer()); self.apply_batch_walker_inline(pending, base_seq, None)?; + drop(commit); if self.cfg.memory_flush_on_write { - self.flush_dirty_inline()?; - self.flush_pending_deletes_inline()?; + self.flush_inline()?; } } Ok(()) @@ -1759,7 +1810,9 @@ impl Tree { /// This is not an MVCC snapshot: a long scan can observe keys /// committed after iterator creation if they sort after the /// current cursor. It is, however, monotonic with respect to - /// already-emitted keys and rollups. + /// already-emitted keys and rollups. Each `next()` holds the shared + /// maintenance/mutation gates only for one cursor step; the iterator does + /// not retain those gates while user code is between calls. pub fn range(&self) -> RangeBuilder { let builder = self.range_builder_base(); match self.flush_write_delta_for_tree() { @@ -1780,6 +1833,13 @@ impl Tree { .with_liveness(Arc::clone(&self.dropped)) } + /// Build a record cursor for a caller that already owns the DB maintenance + /// fence on either its shared or exclusive side. The cursor advances with + /// [`RangeIter::next_unlocked`]. + pub(crate) fn range_unlocked(&self) -> RangeIter { + self.range_builder_base().into_iter() + } + /// Shorthand for `tree.range().prefix(p)` — the /// common-90%-of-queries case. /// @@ -1834,11 +1894,13 @@ impl Tree { /// Run a read-only transaction over a prefix snapshot. /// - /// Internally a [`Self::snapshot`] held for the duration of `read`: a - /// copy-on-write capture (O(1) — only the root frame is copied; later - /// live writes fork the frames this view references). Writes committed - /// after the capture are invisible to all reads made through the view, - /// and point lookup / range / list keep using the ART walker. + /// Internally this captures a [`Self::snapshot`] and passes its view to + /// `read`: a copy-on-write capture (O(1) — only the root frame is copied; + /// later live writes fork the frames this view references). Writes + /// committed after the capture are invisible to all reads made through + /// the view, and point lookup / range / list keep using the ART walker. + /// A cloned [`View`] or owned cursor returned through `R` may outlive the + /// callback and keeps the snapshot epoch leased until it is dropped. /// /// A view is scoped: reads outside `prefix` return /// [`Error::OutsideViewScope`]. Use `prefix = b""` only when a @@ -1862,25 +1924,32 @@ impl Tree { /// mechanism exposed as a scoped closure; this returns an owned handle. /// /// Reads outside `prefix` return [`Error::OutsideViewScope`]; use - /// `prefix = b""` for a whole-tree snapshot. The snapshot stays valid - /// until its handle is dropped (or [`Snapshot::retire`] is called). + /// `prefix = b""` for a whole-tree snapshot. Dropping the main handle (or + /// calling [`Snapshot::retire`]) releases that handle; cloned views and + /// owned cursors remain valid and keep the epoch leased until they too are + /// dropped. pub fn snapshot(&self, prefix: &[u8]) -> Result { - self.flush_write_delta_for_tree()?; let _maintenance = self.maintenance_gate.enter_shared(); self.ensure_live()?; - // Freeze live writers so the root frame is byte-stable for the - // copy, and so no writer is mid-overwriting a soon-to-be-shared - // frame at the instant the fork barrier rises. + // Freeze this tree before publishing deferred writes. Holding the + // same commit-writer admission from delta flush through root copy and + // barrier registration linearizes snapshot creation against both + // acknowledged puts and the all-tree background delta flush. let _freeze = self.mutation_gate.enter_exclusive(); - Ok(self.snapshot_unlocked(prefix)) + let _commit = self.commit_gate.enter_writer(); + self.store.flush_write_deltas_for_tree(self.tree_id)?; + self.snapshot_unlocked(prefix) } - /// Reclaim copy-on-write frames left unreachable by a crash that - /// happened while a snapshot was live: the in-memory orphan list is - /// lost on restart, so those forked-away frames would otherwise leak - /// in the store forever. Sweeps every persisted frame not reachable - /// from the live root or a live snapshot root, returning the count - /// freed. Idempotent. + /// Reclaim persisted frames not reachable from the live root or a live + /// snapshot root, returning the count freed. This includes copy-on-write + /// frames left behind when a process crashed with a snapshot live. + /// + /// GC first checkpoints the writer-frozen live root, then computes + /// reachability from that same image, and only then deletes unreachable + /// blobs. This ordering prevents a durable old parent from losing a + /// child that only the newer in-memory parent stopped referencing. + /// Idempotent. /// /// Only supported on standalone trees — trees opened through a `DB` /// share one buffer manager, so a safe sweep needs a DB-wide pass and @@ -1890,25 +1959,40 @@ impl Tree { return Err(Error::GcRequiresStandaloneTree); } self.flush_write_delta_for_tree()?; - let _maintenance = self.maintenance_gate.enter_shared(); + let _maintenance = self.maintenance_gate.enter_exclusive(); self.ensure_live()?; // Freeze writers so the reachable set is stable across the walk. let _freeze = self.mutation_gate.enter_exclusive(); - let mut reachable: HashSet = HashSet::new(); - reachable.insert(self.root_guid); - reachable.extend(engine::collect_blob_guids(&self.store, self.root_guid)?); - for snap_root in self.store.snapshot_roots() { + // The reachable set below is based on the current in-memory parent + // images. Make those exact images durable before deleting anything + // they no longer reference. + Self::checkpoint_shared_store_with_maintenance_held( + &self.store, + self.journal.as_ref(), + &self.commit_gate, + )?; + + let mut canonical_reachable: HashSet = HashSet::new(); + canonical_reachable.insert(self.root_guid); + canonical_reachable.extend(engine::collect_blob_guids(&self.store, self.root_guid)?); + let snapshot_roots = self.store.snapshot_roots_pinned()?; + #[cfg(test)] + pause_full_gc_after_snapshot_capture(); + let mut reachable = canonical_reachable.clone(); + for snapshot_root in snapshot_roots { + let snap_root = snapshot_root.guid(); reachable.insert(snap_root); reachable.extend(engine::collect_blob_guids(&self.store, snap_root)?); } - self.store.gc_sweep_unreachable(&reachable) + self.store + .gc_sweep_unreachable_with_canonical(&reachable, &canonical_reachable) } /// Reclaim logical garbage and physical space from free store slots. /// - /// [`Self::gc`] only makes unreachable blob frames reusable. `vacuum` - /// follows that with a checkpoint and store-level physical cleanup: + /// [`Self::gc`] checkpoints first and then makes unreachable blob frames + /// reusable. `vacuum` follows with store-level physical cleanup: /// live high-water slots may be relocated into lower reusable holes, /// packed-file tails that are durably free are truncated, and on /// Linux any remaining reusable middle slots are hole-punched. This @@ -1920,7 +2004,6 @@ impl Tree { /// there so every live tree participates in the reachable set. pub fn vacuum(&self) -> Result { let unreachable = self.gc()?; - self.checkpoint()?; let mut stats = self.store.vacuum_storage()?; stats.unreachable_blobs = unreachable; Ok(stats) @@ -1930,11 +2013,11 @@ impl Tree { /// gates — the caller must already hold the mutation gate exclusively. /// Used by [`crate::DB::view`] to capture several trees atomically /// under a single coordinated freeze. - pub(crate) fn snapshot_unlocked(&self, prefix: &[u8]) -> Snapshot { + pub(crate) fn snapshot_unlocked(&self, prefix: &[u8]) -> Result { self.snapshot_unlocked_with_scan_fence(prefix, true) } - pub(crate) fn snapshot_unlocked_unfenced(&self, prefix: &[u8]) -> Snapshot { + pub(crate) fn snapshot_unlocked_unfenced(&self, prefix: &[u8]) -> Result { self.snapshot_unlocked_with_scan_fence(prefix, false) } @@ -1942,12 +2025,19 @@ impl Tree { &self, prefix: &[u8], fence_live_writers: bool, - ) -> Snapshot { + ) -> Result { use crate::store::STRUCTURAL_SEQ; let snap_root = engine::fresh_blob_guid(); let root_pin = self.store.install_snapshot_root(snap_root, &self.root_pin); - let epoch = self.store.register_snapshot(snap_root); + let epoch = match self.store.register_snapshot(snap_root, &root_pin) { + Ok(epoch) => epoch, + Err(error) => { + drop(root_pin); + self.store.discard_snapshot_root(snap_root); + return Err(error); + } + }; // Persist the bumped epoch on the live root so a reopened tree // restores `current_epoch` above every frame's `created_epoch`. @@ -1961,11 +2051,18 @@ impl Tree { self.store .mark_dirty_cached(self.root_guid, STRUCTURAL_SEQ, self.root_pin.as_ref()); + let snapshot_lease = crate::store::SnapshotLease::new( + Arc::clone(&self.store), + epoch, + snap_root, + Arc::clone(&root_pin), + ); let view = View::new( prefix.to_vec(), Arc::clone(&self.store), snap_root, root_pin, + snapshot_lease, fence_live_writers.then(|| { ( Arc::clone(&self.maintenance_gate), @@ -1973,7 +2070,7 @@ impl Tree { ) }), ); - Snapshot::new(view, Arc::clone(&self.store), epoch) + Ok(Snapshot::new(view, epoch)) } /// Return `true` if no live key starts with `prefix`. @@ -2002,75 +2099,111 @@ impl Tree { /// tracked for the next round. Write-through with `expected_seq` /// matches checkpoint: retire only the entry captured by this /// snapshot, leaving any racing newer seq for a later flush. - pub(crate) fn flush_dirty_inline(&self) -> Result<()> { - let snap = self.store.snapshot_dirty(); - let mut failed: std::collections::HashMap = std::collections::HashMap::new(); - let mut first_err: Option = None; - let mut entries = Vec::with_capacity(snap.len()); - for (guid, expected_seq) in snap { - // `snapshot_bytes` clones the cached image under a - // brief shared read guard so we hand owned bytes to - // write-through. `None` means the blob was evicted - // between snapshot_dirty and snapshot_bytes — invariant - // I1 regressed, so restore the entry and fail loudly. - if let Some(bytes) = self.store.snapshot_bytes(guid) { - entries.push(WriteThroughEntry { - guid, - bytes, - expected_seq, - content_version: None, - }); - } else { - failed.insert(guid, expected_seq); - first_err.get_or_insert(Error::Internal( - "flush_dirty_inline: dirty entry lost cache image — invariant I1 violated", - )); + /// Commit the no-WAL dirty and pending-delete phases under one global + /// checkpoint-I/O guard. Keeping this guard across both phases prevents a + /// stale concurrent epoch from republishing an old parent between the + /// current parent sync and its child delete. + pub(crate) fn flush_inline(&self) -> Result<()> { + let _checkpoint_io = self.store.enter_checkpoint_io(); + self.flush_dirty_inline_locked()?; + self.flush_pending_deletes_inline_locked() + } + + fn flush_dirty_inline_locked(&self) -> Result<()> { + loop { + let snap = self.store.snapshot_dirty(); + if snap.is_empty() { + return Ok(()); } - } - if !entries.is_empty() { - let expected: Vec<_> = entries - .iter() - .map(|entry| (entry.guid, entry.expected_seq)) - .collect(); - if let Err(e) = self.store.write_through_batch(&entries) { - for (guid, expected_seq) in expected { - failed.insert(guid, expected_seq); + let versioned_snap = match self.store.snapshot_dirty_versions(&snap) { + Ok(versioned) => versioned, + Err(e) => { + self.store.restore_dirty(snap); + return Err(e); } - if first_err.is_none() { - first_err = Some(e); + }; + let mut failed = CheckpointMap::new(); + let mut first_err = None; + let mut entries = Vec::with_capacity(versioned_snap.len()); + for snapshot in versioned_snap { + // Clone only the exact cache image captured for this round. + // A writer may change the frame after the dirty snapshot; in + // that case restore the claimed seq and retry instead of + // retiring the newer bytes against an older store image. + match self + .store + .snapshot_bytes_if_version(snapshot.guid, snapshot.content_version) + { + Ok(Some(bytes)) => entries.push(WriteThroughEntry { + guid: snapshot.guid, + bytes, + expected_seq: snapshot.expected_seq, + content_version: Some(snapshot.content_version), + }), + Ok(None) => { + failed.insert(snapshot.guid, snapshot.expected_seq); + } + Err(e) => { + failed.insert(snapshot.guid, snapshot.expected_seq); + first_err.get_or_insert(e); + } } } + if !entries.is_empty() { + let expected: Vec<_> = entries + .iter() + .map(|entry| (entry.guid, entry.expected_seq)) + .collect(); + match crate::checkpoint::write_entries_child_first(&self.store, entries) { + Ok(deferred) => failed.extend(deferred), + Err(e) => { + failed.extend(expected); + first_err.get_or_insert(e); + } + } + } + let should_retry = !failed.is_empty() && first_err.is_none(); + if !failed.is_empty() { + self.store.restore_dirty(failed); + } + if let Some(e) = first_err { + return Err(e); + } + if !should_retry { + return Ok(()); + } + std::thread::yield_now(); } - if !failed.is_empty() { - self.store.restore_dirty(failed); - } - if let Some(e) = first_err { - return Err(e); - } - Ok(()) } /// Drain the BM pending-delete queue and apply each /// `store.delete_blob` synchronously. /// - /// Companion to [`Self::flush_dirty_inline`] for the deferred + /// Companion to [`Self::flush_dirty_inline_locked`] for the deferred /// delete protocol — `erase` ops that emptied a child blob /// stage the delete here so the manifest mutation can't reach /// disk before the WAL record covering the erase is durable /// (invariant W2D). /// - /// Must run **after** `flush_dirty_inline` (any new bytes in + /// Must run **after** `flush_dirty_inline_locked` (any new bytes in /// dirty land first) and **before** the trailing /// `store.flush` (which persists the manifest deletion). /// Restoration is automatic on individual failures — the /// remaining entries stay queued for the next attempt. - pub(crate) fn flush_pending_deletes_inline(&self) -> Result<()> { + fn flush_pending_deletes_inline_locked(&self) -> Result<()> { + // Publish every rewritten parent before applying a logical user + // delete's manifest mutation. + self.store.flush_inner()?; let pending = self.store.snapshot_pending_deletes(); let mut failed: std::collections::HashMap = std::collections::HashMap::new(); + let mut applied: std::collections::HashMap = + std::collections::HashMap::new(); let mut first_err: Option = None; for (guid, seq) in pending { match self.store.execute_pending_delete(guid, seq) { - Ok(true) => {} + Ok(true) => { + applied.insert(guid, seq); + } Ok(false) => { failed.insert(guid, seq); } @@ -2085,6 +2218,16 @@ impl Tree { if !failed.is_empty() { self.store.restore_pending_deletes(failed); } + // A successful delete only changed the store's in-memory manifest. + // Cross its durability frontier before the no-WAL mutation returns; + // on failure, restore the exact applied set so a later inline flush + // retries the sync/delete idempotently. + if !applied.is_empty() { + if let Err(e) = self.store.flush_inner() { + self.store.restore_pending_deletes(applied); + return Err(e); + } + } if let Some(e) = first_err { return Err(e); } @@ -2108,9 +2251,11 @@ impl Tree { /// 2. **Clone bytes outside `commit_gate`**: if any blob changed /// after intent capture, restore the whole snapshot and retry /// so manual checkpoint remains a durability barrier. - /// 3. **Per-blob write-through** with CAS-on-seq. The CAS - /// retires the dirty entry only if no racing writer bumped - /// it; failures stay in `dirty` for the next round. + /// 3. **Dependency-ordered write-through**. Dirty frames are arranged in + /// child-before-parent waves; each write uses CAS-on-seq/content + /// version and retires the dirty entry only if no racing writer bumped + /// it. A missing/external dirty child defers its parent, and failures + /// stay in `dirty` for the next round. /// If the snapshot had neither dirty blobs nor pending /// deletes and the store reports no outstanding flush /// work, skip the store Sync path entirely. @@ -2126,34 +2271,51 @@ impl Tree { /// deleted slot. Restore pending and return the dirty /// error. The next round will retry the parent write and /// only then process its child's deletion. - /// 6. **Apply pending deletes**. User WAL deletes mutate the - /// manifest in memory; structural deletes only retire their - /// fence and leave orphan cleanup to a future reachability - /// GC. Each `execute_pending_delete` is idempotent against a - /// missing entry; failures are restored. + /// 6. **Apply pending deletes**. These are logical user WAL deletes; + /// structural children never enter this queue and instead use + /// parent-scoped orphan staging. Each `execute_pending_delete` is + /// idempotent against a missing entry; failures are restored. /// 7. **Post-delete sync** — re-`store.flush` iff any manifest /// delete actually applied. Failure → restore the /// already-applied entries so the truncate gate stays /// closed and the next round retries the sync. - /// 8. **Conditional WAL truncate** — only if - /// `dirty_count == 0`, `flushing_count == 0`, - /// `pending_delete_count == 0`, and the store reports no - /// deferred flush work *now*. A racing writer, restored - /// failure, or deferred data/manifest sync must keep the WAL - /// alive until a future flush. + /// 8. **Conditional WAL truncate** — only if dirty, flushing, + /// pending-delete, orphan-staging, and write-delta debt are all zero + /// and the store reports no deferred flush work *now*. A racing writer, + /// restored failure, or deferred data/manifest sync keeps the WAL alive. + /// 9. **Standalone exact reclaim** — with the maintenance fence still + /// exclusive, drain one bounded retired COW/structural FIFO batch. DB + /// tree handles leave shared-store cleanup to [`crate::DB::checkpoint`]. /// /// `memory_flush_on_write = false` callers rely on this to make /// batched writes survive a crash. pub fn checkpoint(&self) -> Result<()> { - Self::checkpoint_shared_parts( + if self.tree_id != 0 { + return Self::checkpoint_shared_store( + &self.store, + self.journal.as_ref(), + &self.maintenance_gate, + &self.commit_gate, + ); + } + + // Standalone checkpoints also make bounded progress on retired COW + // and structural storage. The same exclusive maintenance fence covers + // the durable parent frontier, exact FIFO capture, and physical + // deletion; this fast path does not run a reachability walk. + let _maintenance = self.maintenance_gate.enter_exclusive(); + let _freeze = self.mutation_gate.enter_exclusive(); + Self::checkpoint_shared_store_with_maintenance_held( &self.store, self.journal.as_ref(), - &self.maintenance_gate, &self.commit_gate, - ) + )?; + self.store + .reclaim_retired_orphans_bounded(AUTO_GC_BATCH_SIZE)?; + Ok(()) } - pub(crate) fn checkpoint_shared_parts( + pub(crate) fn checkpoint_shared_store( store: &Arc, journal: Option<&Arc>, maintenance_gate: &Arc, @@ -2161,6 +2323,18 @@ impl Tree { ) -> Result<()> { let _maintenance = maintenance_gate.enter_shared(); + Self::checkpoint_shared_store_with_maintenance_held(store, journal, commit_gate) + } + + /// Complete a checkpoint while the caller already holds the maintenance + /// gate, on either its shared or exclusive side. This lets GC keep writers + /// frozen across both the durability barrier and the reachability walk + /// without re-entering the non-reentrant gate. + pub(crate) fn checkpoint_shared_store_with_maintenance_held( + store: &Arc, + journal: Option<&Arc>, + commit_gate: &Arc, + ) -> Result<()> { loop { let (snap_dirty, snap_pending, versioned_snap, wal_up_to) = Self::capture_checkpoint_intent_shared(store, journal, commit_gate)?; @@ -2228,8 +2402,8 @@ impl Tree { Option, )> { if let Some(journal) = journal { - store.flush_write_deltas()?; let _commit = commit_gate.enter_checkpoint(); + store.flush_write_deltas()?; let wal_up_to = journal.queued_work(); let snap_dirty = store.snapshot_dirty(); let snap_pending = store.snapshot_pending_deletes(); @@ -2244,6 +2418,7 @@ impl Tree { } } } else { + let _commit = commit_gate.enter_checkpoint(); store.flush_write_deltas()?; let snap_dirty = store.snapshot_dirty(); let snap_pending = store.snapshot_pending_deletes(); @@ -2265,6 +2440,10 @@ impl Tree { snap_pending: CheckpointMap, snap_bytes: CheckpointBytes, ) -> Result<()> { + // Capture/CommitGate has already been released. Serialize the complete + // manual write/sync/delete/sync transaction with the background I/O + // worker so an older stale epoch cannot land after a newer delete. + let _checkpoint_io = store.enter_checkpoint_io(); let DirtyWriteOutcome { wrote_any, retry: dirty_retry, @@ -2282,7 +2461,7 @@ impl Tree { // Successful write-throughs already retired their dirty // entries; sync them before any manifest delete can land. - if let Err(e) = store.flush() { + if let Err(e) = store.flush_inner() { store.restore_pending_deletes(snap_pending); return Err(e); } @@ -2327,6 +2506,7 @@ impl Tree { }, ) .collect(); + let wrote_any = !entries.is_empty(); let mut retry = CheckpointMap::new(); let mut first_err = None; if !entries.is_empty() { @@ -2334,13 +2514,9 @@ impl Tree { .iter() .map(|entry| (entry.guid, entry.expected_seq)) .collect(); - match store.write_through_batch(&entries) { - Ok(report) => { - for (entry, status) in entries.iter().zip(report.statuses) { - if matches!(status, WriteThroughStatus::Stale) { - retry.insert(entry.guid, entry.expected_seq); - } - } + match crate::checkpoint::write_entries_child_first(store, entries) { + Ok(deferred) => { + retry.extend(deferred); } Err(e) => { // BlobStore batch failure may have landed any prefix; @@ -2353,7 +2529,7 @@ impl Tree { } } DirtyWriteOutcome { - wrote_any: !entries.is_empty(), + wrote_any, retry, first_err, } @@ -2392,7 +2568,7 @@ impl Tree { .filter(|(guid, seq)| **seq != STRUCTURAL_SEQ && !pending_failed.contains_key(*guid)) .count(); if applied_manifest_deletes > 0 { - if let Err(e) = store.flush() { + if let Err(e) = store.flush_inner() { let restore_applied: CheckpointMap = snap_pending .iter() .filter(|(g, seq)| **seq != STRUCTURAL_SEQ && !pending_failed.contains_key(*g)) @@ -2416,6 +2592,7 @@ impl Tree { if store.dirty_count() == 0 && store.flushing_count() == 0 && store.pending_delete_count() == 0 + && store.orphan_staging_count() == 0 && store.write_delta_count() == 0 && !store.needs_flush() { @@ -2435,6 +2612,7 @@ impl Tree { store.dirty_count() == 0 && store.flushing_count() == 0 && store.pending_delete_count() == 0 + && store.orphan_staging_count() == 0 && store.write_delta_count() == 0 && !store.needs_flush() && journal.is_none_or(|journal| !journal.needs_checkpoint()) @@ -2484,6 +2662,9 @@ impl Tree { blobs: aggregate.blobs, bm_dirty_count: bm.dirty_count, bm_pending_delete_count: bm.pending_delete_count, + bm_gc_orphan_backlog_count: bm.gc_orphan_backlog_count, + bm_gc_reclaimed_count: bm.gc_reclaimed_count, + bm_gc_last_full_sweep_deferred_count: bm.gc_last_full_sweep_deferred_count, bm_write_delta_count: bm.write_delta_count, bm_read_index_token_count: bm.read_index_token_count, bm_read_index_cache_entries: bm.read_index_cache_entries, @@ -2666,8 +2847,9 @@ impl Tree { /// through their versioned cursor frames and restart from the /// last emitted lower bound. /// - /// Both phases stage their changes via `mark_dirty` / - /// `mark_for_delete` on the internal `BufferManager` + /// Both phases stage rewritten parents via `mark_dirty`; structural + /// children use parent-scoped orphan staging, while logical user deletes + /// continue through `mark_for_delete` on the internal `BufferManager` /// rather than writing through to store inline. This keeps /// compact compatible with invariant **W2D**: a naive /// `bm.commit(*guid)` per touched blob would push the cache @@ -2805,16 +2987,28 @@ impl Tree { if !self.store.has_blob(guid)? { return Ok(()); } - let _commit = self - .journal - .as_ref() - .map(|_| self.commit_gate.enter_writer()); + // Parent edge mutation, orphan staging, dirty publication, and + // staging promotion form one clean-frontier transaction even for a + // custom/no-WAL store with background checkpointing enabled. + let _commit = self.commit_gate.enter_writer(); let pin = self.store.pin(guid)?; - let (merged, has_children) = { + let merge_result = { let mut guard = pin.write(); let mut frame = guard.frame(); - let merged = engine::try_merge_children(&self.store, &mut frame, STRUCTURAL_SEQ)?; - (merged, frame.header().num_ext_blobs != 0) + engine::try_merge_children(&self.store, &mut frame, STRUCTURAL_SEQ) + .map(|merged| (merged, frame.header().num_ext_blobs != 0)) + }; + let (merged, has_children) = match merge_result { + Ok(result) => result, + Err(error) => { + // The pass may already have folded earlier children before a + // later child failed. Those rewrites are logically neutral; + // publish the exact partial parent so staged child GUIDs can + // leave parent-scoped limbo and checkpoint can make progress. + self.store.mark_dirty(guid, STRUCTURAL_SEQ); + drop(pin); + return Err(error); + } }; if merged.merged > 0 { self.store.mark_dirty(guid, STRUCTURAL_SEQ); @@ -2834,6 +3028,25 @@ impl Tree { } } +/// Ensure a deterministic empty root exists beyond the store's durability +/// frontier before any catalog or replay path publishes a reference to it. +pub(crate) fn ensure_durable_root_blob(bm: &Arc, root_guid: BlobGuid) -> Result<()> { + if !bm.has_blob(root_guid)? { + let mut scratch = bm.alloc_blob_buf_zeroed(); + BlobFrame::init(scratch.as_mut_slice(), root_guid)?; + bm.write_blob(root_guid, &scratch)?; + } + // A prior create attempt may have submitted the root write and then + // failed its flush. In that case `has_blob` is already true, but callers + // still must cross the root durability frontier before publishing any + // catalog/reference that makes it reachable. + // Always retry the durability barrier. Custom stores may have submitted + // a prior root write before returning a flush error, and `has_blob` alone + // does not imply that mapping survived a crash. + bm.flush()?; + Ok(()) +} + /// Replay `path` onto the BM-cached blobs and return the /// `next_seq` the tree should resume from. /// @@ -2859,16 +3072,6 @@ impl Tree { /// a `Tree::open` → `Tree::checkpoint` immediately after replay /// could find an empty dirty set, write nothing to store, then /// truncate the WAL — silently losing every replayed record. -pub(crate) fn ensure_root_blob(bm: &Arc, root_guid: BlobGuid) -> Result<()> { - if !bm.has_blob(root_guid)? { - let mut scratch = bm.alloc_blob_buf_zeroed(); - BlobFrame::init(scratch.as_mut_slice(), root_guid)?; - bm.write_blob(root_guid, &scratch)?; - bm.flush()?; - } - Ok(()) -} - pub(crate) fn replay_wal( path: &std::path::Path, bm: &Arc, @@ -2887,7 +3090,7 @@ where } std::collections::hash_map::Entry::Vacant(entry) => { let guid = root_for_tree_id(tree_id)?; - ensure_root_blob(bm, guid)?; + ensure_durable_root_blob(bm, guid)?; let pin = bm.pin(guid)?; let (guid, pin) = entry.insert((guid, pin)); (*guid, Arc::clone(pin)) @@ -2981,22 +3184,308 @@ where #[cfg(test)] mod tests { use crate::concurrency::CommitGate; + use crate::engine::{RouteHit, SearchKey}; use crate::journal::Journal; use crate::layout::BlobGuid; use crate::store::blob_store::{AlignedBlobBuf, BlobStore, MemoryBlobStore}; - use crate::store::BufferManager; - use crate::TreeBuilder; - use std::sync::atomic::{AtomicBool, Ordering}; + use crate::store::{BlobFrame, BufferManager}; + use crate::{Durability, Error, Tree, TreeBuilder, TreeConfig}; + use std::collections::{HashMap, HashSet}; + use std::io; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::mpsc::sync_channel; - use std::sync::Arc; + use std::sync::{Arc, Barrier, Mutex}; use std::thread; use std::time::Duration; + #[test] + fn standalone_reopens_exhausted_epoch_without_leaking_failed_snapshot_root() { + let store = Arc::new(MemoryBlobStore::new()); + let mut cfg = TreeConfig::memory(); + cfg.checkpoint.enabled = false; + cfg.memory_flush_on_write = false; + + { + let store_dyn: Arc = store.clone(); + let tree = Tree::open_with_blob_store(cfg.clone(), store_dyn).unwrap(); + tree.put(b"key", b"before").unwrap(); + tree.store.set_current_epoch(u64::MAX - 2); + let snapshot = tree.snapshot(b"").unwrap(); + assert_eq!(snapshot.epoch(), u64::MAX - 2); + assert_eq!(tree.store.current_epoch(), u64::MAX - 1); + drop(snapshot); + tree.checkpoint().unwrap(); + } + + let store_dyn: Arc = store.clone(); + let reopened = Tree::open_with_blob_store(cfg.clone(), store_dyn).unwrap(); + assert_eq!(reopened.store.current_epoch(), u64::MAX - 1); + assert_eq!( + reopened.get(b"key").unwrap().as_deref(), + Some(&b"before"[..]) + ); + let cached_before = reopened.store.cached_count(); + let dirty_before = reopened.store.dirty_count(); + let barrier_before = reopened.store.fork_barrier(); + let live_before = reopened.store.snapshot_roots_pinned().unwrap().len(); + + let error = reopened.snapshot(b"").unwrap_err(); + assert!(matches!(error, Error::SnapshotEpochExhausted)); + assert_eq!(reopened.store.cached_count(), cached_before); + assert_eq!(reopened.store.dirty_count(), dirty_before); + assert_eq!(reopened.store.fork_barrier(), barrier_before); + assert_eq!( + reopened.store.snapshot_roots_pinned().unwrap().len(), + live_before, + ); + assert_eq!(reopened.store.current_epoch(), u64::MAX - 1); + + reopened.put(b"key", b"after").unwrap(); + assert_eq!( + reopened.get(b"key").unwrap().as_deref(), + Some(&b"after"[..]) + ); + + let corrupt_store = Arc::new(MemoryBlobStore::new()); + let mut root = AlignedBlobBuf::zeroed(); + BlobFrame::init(root.as_mut_slice(), [0; 16]).unwrap(); + crate::layout::set_frame_epoch_high_water(root.as_mut_slice(), u64::MAX); + corrupt_store.write_blob([0; 16], &root).unwrap(); + let corrupt_dyn: Arc = corrupt_store; + let error = Tree::open_with_blob_store(cfg, corrupt_dyn).unwrap_err(); + assert!(matches!( + error, + Error::NodeCorrupt { + context: "snapshot epoch exhausted", + .. + } + )); + } + + fn spilled_tree_with_same_child_keys() -> (Tree, Vec, Vec, RouteHit) { + let tree = TreeBuilder::new("ignored") + .memory() + .buffer_pool_size(32) + .open() + .unwrap(); + let old = vec![0x6A; 1024]; + for i in 0..1200u32 { + tree.put(format!("route/{i:08}").as_bytes(), &old).unwrap(); + } + assert!(tree.stats().unwrap().blob_count > 1); + + let mut first_by_child = HashMap::>::new(); + for i in 0..1200u32 { + let key = format!("route/{i:08}").into_bytes(); + assert_eq!(tree.get(&key).unwrap().as_deref(), Some(&old[..])); + let Some(route) = tree.route_cache.lookup(SearchKey::user(&key)) else { + continue; + }; + if let Some(first) = first_by_child.get(&route.child_guid) { + if first != &key { + return (tree, first.clone(), key, route); + } + } else { + first_by_child.insert(route.child_guid, key); + } + } + panic!("test precondition: no two keys routed to the same child"); + } + struct FlushPendingStore { inner: MemoryBlobStore, pending: AtomicBool, } + struct FailReadStore { + inner: Arc, + fail_guids: Mutex>, + last_failed: Mutex>, + } + + struct InlineBlockingStore { + inner: MemoryBlobStore, + block_next_batch: AtomicBool, + entered: Barrier, + release: Barrier, + } + + struct FailTrailingDeleteFlushStore { + inner: MemoryBlobStore, + flush_calls: AtomicUsize, + fail_flush_at: AtomicUsize, + } + + impl FailTrailingDeleteFlushStore { + fn new() -> Self { + Self { + inner: MemoryBlobStore::new(), + flush_calls: AtomicUsize::new(0), + fail_flush_at: AtomicUsize::new(usize::MAX), + } + } + + fn fail_after_one_successful_flush(&self) { + self.fail_flush_at.store( + self.flush_calls.load(Ordering::Acquire) + 2, + Ordering::Release, + ); + } + } + + impl BlobStore for FailTrailingDeleteFlushStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> crate::Result<()> { + self.inner.read_blob(guid, dst) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> crate::Result<()> { + self.inner.write_blob(guid, src) + } + + fn delete_blob(&self, guid: BlobGuid) -> crate::Result<()> { + self.inner.delete_blob(guid) + } + + fn list_blobs(&self) -> crate::Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> crate::Result<()> { + let ordinal = self.flush_calls.fetch_add(1, Ordering::AcqRel) + 1; + if ordinal == self.fail_flush_at.load(Ordering::Acquire) { + self.fail_flush_at.store(usize::MAX, Ordering::Release); + return Err(crate::Error::BlobStoreIo(io::Error::other( + "failpoint: trailing delete flush", + ))); + } + self.inner.flush() + } + + fn needs_flush(&self) -> bool { + self.inner.needs_flush() + } + + fn has_blob(&self, guid: BlobGuid) -> crate::Result { + self.inner.has_blob(guid) + } + } + + impl InlineBlockingStore { + fn new() -> Self { + Self { + inner: MemoryBlobStore::new(), + block_next_batch: AtomicBool::new(false), + entered: Barrier::new(2), + release: Barrier::new(2), + } + } + + fn arm(&self) { + self.block_next_batch.store(true, Ordering::Release); + } + } + + impl BlobStore for InlineBlockingStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> crate::Result<()> { + self.inner.read_blob(guid, dst) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> crate::Result<()> { + self.inner.write_blob(guid, src) + } + + fn write_blobs_with_data_sync( + &self, + writes: &[(BlobGuid, &AlignedBlobBuf)], + ) -> crate::Result<()> { + self.inner.write_blobs_with_data_sync(writes)?; + if self.block_next_batch.swap(false, Ordering::AcqRel) { + self.entered.wait(); + self.release.wait(); + } + Ok(()) + } + + fn delete_blob(&self, guid: BlobGuid) -> crate::Result<()> { + self.inner.delete_blob(guid) + } + + fn list_blobs(&self) -> crate::Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> crate::Result<()> { + self.inner.flush() + } + + fn needs_flush(&self) -> bool { + self.inner.needs_flush() + } + + fn has_blob(&self, guid: BlobGuid) -> crate::Result { + self.inner.has_blob(guid) + } + } + + impl FailReadStore { + fn new(inner: Arc) -> Self { + Self { + inner, + fail_guids: Mutex::new(HashSet::new()), + last_failed: Mutex::new(None), + } + } + + fn arm(&self, guids: impl IntoIterator) { + self.fail_guids.lock().unwrap().extend(guids); + *self.last_failed.lock().unwrap() = None; + } + + fn clear(&self) { + self.fail_guids.lock().unwrap().clear(); + } + + fn take_last_failed(&self) -> Option { + self.last_failed.lock().unwrap().take() + } + } + + impl BlobStore for FailReadStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> crate::Result<()> { + if self.fail_guids.lock().unwrap().contains(&guid) { + *self.last_failed.lock().unwrap() = Some(guid); + return Err(crate::Error::BlobStoreIo(io::Error::other( + "failpoint: deep child read", + ))); + } + self.inner.read_blob(guid, dst) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> crate::Result<()> { + self.inner.write_blob(guid, src) + } + + fn delete_blob(&self, guid: BlobGuid) -> crate::Result<()> { + self.inner.delete_blob(guid) + } + + fn list_blobs(&self) -> crate::Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> crate::Result<()> { + self.inner.flush() + } + + fn needs_flush(&self) -> bool { + self.inner.needs_flush() + } + + fn has_blob(&self, guid: BlobGuid) -> crate::Result { + self.inner.has_blob(guid) + } + } + impl FlushPendingStore { fn new() -> Self { Self { @@ -3041,7 +3530,9 @@ mod tests { let inner: Arc = Arc::new(MemoryBlobStore::new()); let guid = [0x71; 16]; let mut initial = AlignedBlobBuf::zeroed(); - initial.as_mut_slice()[123] = 0x01; + BlobFrame::init(initial.as_mut_slice(), guid).unwrap(); + let value_offset = crate::layout::DATA_AREA_START as usize + 123; + initial.as_mut_slice()[value_offset] = 0x01; inner.write_blob(guid, &initial).unwrap(); let bm = Arc::new(BufferManager::new(Arc::clone(&inner), 4)); @@ -3049,7 +3540,7 @@ mod tests { { let mut guard = pin.write(); - guard.as_mut_slice()[123] = 0x02; + guard.as_mut_slice()[value_offset] = 0x02; } bm.mark_dirty_cached(guid, 10, pin.as_ref()); let snap = bm.snapshot_dirty(); @@ -3062,7 +3553,7 @@ mod tests { { let mut guard = pin.write(); - guard.as_mut_slice()[123] = 0x03; + guard.as_mut_slice()[value_offset] = 0x03; } bm.mark_dirty_cached(guid, 20, pin.as_ref()); @@ -3077,7 +3568,7 @@ mod tests { let mut stored = AlignedBlobBuf::zeroed(); inner.read_blob(guid, &mut stored).unwrap(); assert_eq!( - stored.as_slice()[123], + stored.as_slice()[value_offset], 0x01, "stale checkpoint bytes must not be treated as durable" ); @@ -3088,6 +3579,93 @@ mod tests { ); } + #[test] + fn inline_flush_retries_racing_content_version_and_reopens_latest_value() { + let store = Arc::new(InlineBlockingStore::new()); + let store_dyn: Arc = store.clone(); + let mut cfg = TreeConfig::memory(); + cfg.memory_flush_on_write = false; + let tree = Tree::open_with_blob_store(cfg.clone(), store_dyn).unwrap(); + tree.put(b"inline-race", b"old").unwrap(); + + store.arm(); + let flushing = tree.clone(); + let flush_thread = thread::spawn(move || flushing.flush_inline()); + store.entered.wait(); + tree.put(b"inline-race", b"new").unwrap(); + store.release.wait(); + flush_thread.join().unwrap().unwrap(); + + assert_eq!(tree.stats().unwrap().bm_dirty_count, 0); + drop(tree); + let store_dyn: Arc = store; + let reopened = Tree::open_with_blob_store(cfg, store_dyn).unwrap(); + assert_eq!( + reopened.get(b"inline-race").unwrap().as_deref(), + Some(&b"new"[..]), + "inline flush must retry the raced frame and persist its latest cache image", + ); + } + + #[test] + fn manual_checkpoint_waits_for_global_io_epoch() { + let mut cfg = TreeConfig::memory(); + cfg.memory_flush_on_write = false; + cfg.checkpoint.enabled = false; + let tree = Tree::open(cfg).unwrap(); + tree.put(b"manual-io-lock", b"value").unwrap(); + + let checkpoint_io = tree.store.enter_checkpoint_io(); + let worker_tree = tree.clone(); + let (done_tx, done_rx) = sync_channel(1); + let worker = thread::spawn(move || { + let result = worker_tree.checkpoint(); + done_tx.send(result).unwrap(); + }); + assert!( + done_rx.recv_timeout(Duration::from_millis(100)).is_err(), + "manual checkpoint must not bypass an active checkpoint I/O epoch", + ); + drop(checkpoint_io); + done_rx + .recv_timeout(Duration::from_secs(2)) + .unwrap() + .unwrap(); + worker.join().unwrap(); + assert_eq!( + tree.get(b"manual-io-lock").unwrap().as_deref(), + Some(&b"value"[..]) + ); + } + + #[test] + fn inline_delete_sync_failure_restores_pending_fence_for_retry() { + let store = Arc::new(FailTrailingDeleteFlushStore::new()); + let store_dyn: Arc = store.clone(); + let mut cfg = TreeConfig::memory(); + cfg.memory_flush_on_write = false; + cfg.checkpoint.enabled = false; + let tree = Tree::open_with_blob_store(cfg, store_dyn).unwrap(); + let child = [0x74; 16]; + store + .inner + .write_blob(child, &AlignedBlobBuf::zeroed()) + .unwrap(); + tree.store.mark_for_delete(child, 7); + + store.fail_after_one_successful_flush(); + assert!(tree.flush_inline().is_err()); + assert_eq!( + tree.store.pending_delete_count(), + 1, + "an unsynced applied manifest delete must remain retryable", + ); + + tree.flush_inline().unwrap(); + assert_eq!(tree.store.pending_delete_count(), 0); + assert!(!store.inner.has_blob(child).unwrap()); + } + #[test] fn strict_prefix_point_keys_round_trip_through_public_api() { let tree = TreeBuilder::new("ignored").memory().open().unwrap(); @@ -3217,6 +3795,130 @@ mod tests { handle.join().unwrap(); } + #[test] + fn no_wal_snapshot_mutation_waits_for_clean_frontier_capture() { + let tree = TreeBuilder::new("ignored").memory().open().unwrap(); + tree.put(b"key", b"old").unwrap(); + let snapshot = tree.snapshot(b"").unwrap(); + + let clean_capture = tree.commit_gate.enter_checkpoint(); + let worker_tree = tree.clone(); + let (done_tx, done_rx) = sync_channel(0); + let worker = thread::spawn(move || { + worker_tree.put(b"key", b"new").unwrap(); + done_tx.send(()).unwrap(); + }); + assert!( + done_rx.recv_timeout(Duration::from_millis(50)).is_err(), + "no-WAL mutation under a live snapshot bypassed clean-frontier capture" + ); + drop(clean_capture); + done_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + worker.join().unwrap(); + + assert_eq!(snapshot.get(b"key").unwrap().as_deref(), Some(&b"old"[..])); + assert_eq!(tree.get(b"key").unwrap().as_deref(), Some(&b"new"[..])); + } + + #[test] + fn compact_keeps_snapshot_child_visible_until_first_lazy_read() { + let tree = TreeBuilder::new("ignored") + .memory() + .buffer_pool_size(16) + .open() + .unwrap(); + let big = vec![0xCDu8; 4 * 1024]; + for i in 0..256u32 { + tree.put(format!("k{i:08}").as_bytes(), &big).unwrap(); + } + for i in 0..248u32 { + tree.delete(format!("k{i:08}").as_bytes()).unwrap(); + } + assert!(tree.stats().unwrap().blob_count > 1); + + let snapshot = tree.snapshot(b"").unwrap(); + tree.compact().unwrap(); + assert!( + tree.stats().unwrap().blob_count > 1, + "merge must retain a child that a live snapshot can lazily reach" + ); + assert_eq!( + snapshot.get(b"k00000248").unwrap().as_deref(), + Some(&big[..]), + "snapshot's first post-compact lazy child pin failed" + ); + tree.checkpoint().unwrap(); + assert_eq!( + snapshot.get(b"k00000255").unwrap().as_deref(), + Some(&big[..]), + "checkpoint reclaimed a child held by the snapshot lease" + ); + + drop(snapshot); + tree.compact().unwrap(); + tree.checkpoint().unwrap(); + assert_eq!(tree.stats().unwrap().blob_count, 1); + } + + #[test] + fn full_gc_retains_snapshot_only_orphans_for_post_capture_exact_reclaim() { + let tree = Tree::open(TreeConfig::memory()).unwrap(); + let old = vec![0x41; 1024]; + let new = vec![0x42; 1024]; + for i in 0..1200u32 { + tree.put(format!("gc/{i:08}").as_bytes(), &old).unwrap(); + } + assert!( + tree.stats().unwrap().blob_count > 1, + "test requires snapshot-shared child blobs", + ); + tree.checkpoint().unwrap(); + + let snapshot = tree.snapshot(b"").unwrap(); + for i in 0..1200u32 { + tree.put(format!("gc/{i:08}").as_bytes(), &new).unwrap(); + } + assert!( + tree.stats().unwrap().bm_gc_orphan_backlog_count > 0, + "COW mutations must create snapshot-protected orphan debt", + ); + + let barrier = Arc::new(super::FullGcSnapshotCaptureBarrier::new()); + let worker_barrier = Arc::clone(&barrier); + let worker_tree = tree.clone(); + let worker = thread::spawn(move || { + super::set_full_gc_snapshot_capture_barrier_for_current_thread(worker_barrier); + worker_tree.gc().unwrap() + }); + + // GC now owns strong pins for the snapshot closure but has not yet + // built/swept the union. Retiring the last user lease moves the old + // COW frames into the exact-reclaim FIFO during that window. + barrier.entered.wait(); + drop(snapshot); + barrier.release.wait(); + let _ = worker.join().unwrap(); + + assert!( + tree.stats().unwrap().bm_gc_orphan_backlog_count > 0, + "snapshot-only reachability must not erase retired FIFO entries", + ); + tree.checkpoint().unwrap(); + assert_eq!( + tree.stats().unwrap().bm_gc_orphan_backlog_count, + 0, + "ordinary checkpoint must reclaim the FIFO once GC pins retire", + ); + for i in [0, 599, 1199] { + assert_eq!( + tree.get(format!("gc/{i:08}").as_bytes()) + .unwrap() + .as_deref(), + Some(&new[..]), + ); + } + } + #[test] fn synchronous_checkpoint_truncate_waits_for_store_flush() { let store = Arc::new(FlushPendingStore::new()); @@ -3241,4 +3943,300 @@ mod tests { "WAL should truncate once BM state is clean and store is durable" ); } + + #[test] + fn repeated_snapshot_and_escaped_reader_drops_release_root_cache_entries() { + let tree = TreeBuilder::new("ignored").memory().open().unwrap(); + tree.put(b"key", b"value").unwrap(); + let baseline = tree.store.cached_count(); + + for _ in 0..64 { + drop(tree.snapshot(b"").unwrap()); + + let snapshot = tree.snapshot(b"").unwrap(); + let escaped_view = snapshot.view().clone(); + drop(snapshot); + drop(escaped_view); + + let snapshot = tree.snapshot(b"").unwrap(); + let record_builder = snapshot.range(); + drop(snapshot); + drop(record_builder); + + let snapshot = tree.snapshot(b"").unwrap(); + let record_cursor = snapshot.range().into_iter(); + drop(snapshot); + drop(record_cursor); + + let snapshot = tree.snapshot(b"").unwrap(); + let key_cursor = snapshot.range_keys().into_iter(); + drop(snapshot); + drop(key_cursor); + + let snapshot = tree.snapshot(b"").unwrap(); + let mut partial_cursor = snapshot.range().into_iter(); + drop(snapshot); + assert!(partial_cursor.next().is_some()); + drop(partial_cursor); + } + + assert_eq!( + tree.store.cached_count(), + baseline, + "ephemeral snapshot roots must leave the cache after the final derived reader drops", + ); + } + + #[test] + fn route_cached_atomic_batch_cows_snapshot_shared_child() { + let (tree, first, second, _) = spilled_tree_with_same_child_keys(); + let old = vec![0x6A; 1024]; + let snapshot = tree.snapshot(b"").unwrap(); + let hits_before = tree.route_cache.stats().hits; + + assert!(tree + .atomic(|batch| { + batch.put(&first, b"new-first"); + batch.put(&second, b"new-second"); + }) + .unwrap()); + assert!( + tree.route_cache.stats().hits > hits_before, + "atomic insert run must probe the pre-warmed route fast path", + ); + assert_eq!(snapshot.get(&first).unwrap().as_deref(), Some(&old[..])); + assert_eq!(snapshot.get(&second).unwrap().as_deref(), Some(&old[..])); + assert_eq!( + tree.get(&first).unwrap().as_deref(), + Some(&b"new-first"[..]) + ); + assert_eq!( + tree.get(&second).unwrap().as_deref(), + Some(&b"new-second"[..]) + ); + assert!(tree.store.gc_orphan_backlog_count() > 0); + } + + #[test] + fn cold_deferred_batch_cows_snapshot_shared_child() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg = TreeConfig::new(dir.path()); + cfg.checkpoint.enabled = false; + cfg.durability = Durability::Wal { sync: true }; + cfg.buffer_pool_size = 32; + let tree = Tree::open(cfg).unwrap(); + let old = vec![0x4B; 1024]; + for i in 0..1200u32 { + tree.put(format!("cold/{i:08}").as_bytes(), &old).unwrap(); + } + tree.checkpoint().unwrap(); + assert!(tree.stats().unwrap().blob_count > 1); + + let mut first_by_child = HashMap::>::new(); + let (first, second) = (0..1200u32) + .find_map(|i| { + let key = format!("cold/{i:08}").into_bytes(); + // The baseline was materialized by the deferred-write + // batch, which deliberately has no Tree-owned route cache + // to populate. An existing-key conditional insert is a + // side-effect-free way to make the normal writer learn the + // root crossing before we select two keys from one child. + assert!(!tree.put_if_absent(&key, &old).unwrap()); + let route = tree.route_cache.lookup(SearchKey::user(&key))?; + first_by_child + .insert(route.child_guid, key.clone()) + .map(|first| (first, key)) + }) + .expect("two keys routed through one child"); + tree.route_cache.clear(); + assert_eq!(tree.route_cache.stats().entries, 0); + + let snapshot = tree.snapshot(b"").unwrap(); + tree.put(&first, b"new-first").unwrap(); + tree.put(&second, b"new-second").unwrap(); + assert_eq!(tree.store.write_delta_count_for_tree(tree.tree_id), 2); + assert_eq!(tree.route_cache.stats().entries, 0); + tree.checkpoint().unwrap(); + + assert_eq!(snapshot.get(&first).unwrap().as_deref(), Some(&old[..])); + assert_eq!(snapshot.get(&second).unwrap().as_deref(), Some(&old[..])); + assert_eq!( + tree.get(&first).unwrap().as_deref(), + Some(&b"new-first"[..]) + ); + assert_eq!( + tree.get(&second).unwrap().as_deref(), + Some(&b"new-second"[..]) + ); + assert!(tree.store.gc_orphan_backlog_count() > 0); + } + + #[test] + fn deep_read_failure_after_cow_publishes_structural_parent() { + let inner = Arc::new(MemoryBlobStore::new()); + let failing = Arc::new(FailReadStore::new(Arc::clone(&inner))); + let store: Arc = failing.clone(); + let mut cfg = TreeConfig::memory(); + cfg.buffer_pool_size = 96; + cfg.memory_flush_on_write = false; + let tree = Tree::open_with_blob_store(cfg.clone(), store).unwrap(); + + // Force a two-crossing shape. Values are large enough to make this + // bounded setup converge quickly while still exercising ordinary + // BlobNode spillover rather than the large-value rejection path. + let old = vec![0x35; 8 * 1024]; + let mut keys = Vec::new(); + for i in 0..4096u32 { + let key = format!("deep/read/fail/{i:08}").into_bytes(); + tree.put(&key, &old).unwrap(); + keys.push(key); + if i >= 511 && i % 256 == 255 && tree.stats().unwrap().max_blob_depth >= 2 { + break; + } + } + tree.checkpoint().unwrap(); + assert!( + tree.stats().unwrap().max_blob_depth >= 2, + "test precondition: setup did not produce a grandchild blob", + ); + + // Find a key that descends through a grandchild, then leave that + // exact grandchild cold and armed. The failpoint stays armed until + // explicitly cleared so best-effort read-ahead cannot consume it. + let deep_guids = crate::engine::collect_blob_topology_silent(&tree.store, tree.root_guid) + .unwrap() + .into_iter() + .filter_map(|entry| (entry.depth >= 2).then_some(entry.guid)) + .collect::>(); + assert!(!deep_guids.is_empty()); + for guid in &deep_guids { + assert!( + tree.store.evict_from_cache_for_test(*guid), + "test precondition: deep child remained pinned", + ); + } + failing.arm(deep_guids); + let mut target = None; + for key in &keys { + let read = tree.get(key); + if matches!(read, Err(crate::Error::BlobStoreIo(_))) { + target = failing.take_last_failed().map(|guid| (key.clone(), guid)); + break; + } + } + failing.clear(); + let (target, failed_guid) = target.expect("no key descended through a cold grandchild"); + + // Learn the root crossing while the target is readable, then make + // only its grandchild cold. The failed put must therefore get past + // the root-child fork before the injected read error fires. + assert!(!tree.put_if_absent(&target, &old).unwrap()); + assert!(tree.route_cache.lookup(SearchKey::user(&target)).is_some()); + assert!(tree.store.evict_from_cache_for_test(failed_guid)); + let snapshot = tree.snapshot(b"").unwrap(); + failing.arm([failed_guid]); + let error = tree.put(&target, b"new-after-retry").unwrap_err(); + assert!(matches!(error, crate::Error::BlobStoreIo(_))); + assert_eq!(failing.take_last_failed(), Some(failed_guid)); + failing.clear(); + + assert_eq!(snapshot.get(&target).unwrap().as_deref(), Some(&old[..])); + assert_eq!(tree.get(&target).unwrap().as_deref(), Some(&old[..])); + assert_eq!( + tree.store.orphan_staging_count(), + 0, + "the root parent must publish the successful shape-only fork even when descent fails", + ); + assert!( + tree.store.gc_orphan_backlog_count() > 0, + "the snapshot-retained pre-fork child must be tracked as an orphan", + ); + tree.checkpoint().unwrap(); + + tree.put(&target, b"new-after-retry").unwrap(); + assert_eq!(snapshot.get(&target).unwrap().as_deref(), Some(&old[..])); + assert_eq!( + tree.get(&target).unwrap().as_deref(), + Some(&b"new-after-retry"[..]), + ); + tree.checkpoint().unwrap(); + drop(snapshot); + tree.checkpoint().unwrap(); + drop(tree); + + let store: Arc = failing; + let reopened = Tree::open_with_blob_store(cfg, store).unwrap(); + assert_eq!( + reopened.get(&target).unwrap().as_deref(), + Some(&b"new-after-retry"[..]), + "retry must survive checkpoint and reopen after structural-error recovery", + ); + } + + #[test] + fn checkpoint_delta_flush_linearizes_before_snapshot_barrier() { + let (tree, target, _, route) = spilled_tree_with_same_child_keys(); + let seq = tree.next_seq.fetch_add(1, Ordering::Relaxed); + tree.store.stage_write_delta_put( + tree.tree_id, + tree.root_guid, + &target, + b"checkpoint-value", + seq, + false, + ); + + // Block the checkpoint delta flush exactly at its target child. The + // checkpoint must already own commit-exclusive before it can reach + // this latch; snapshot may own tree-mutation-exclusive meanwhile but + // must not copy/register its root until the flush releases commit. + let child_pin = tree.store.pin(route.child_guid).unwrap(); + let child_guard = child_pin.write(); + let checkpoint_store = Arc::clone(&tree.store); + let checkpoint_commit = Arc::clone(&tree.commit_gate); + let (checkpoint_tx, checkpoint_rx) = sync_channel(0); + let checkpoint = thread::spawn(move || { + let result = + Tree::capture_checkpoint_intent_shared(&checkpoint_store, None, &checkpoint_commit) + .map(|_| ()); + checkpoint_tx.send(result).unwrap(); + }); + + let deadline = std::time::Instant::now() + Duration::from_secs(1); + while !tree.commit_gate.checkpoint_pending_for_test() { + assert!( + std::time::Instant::now() < deadline, + "checkpoint never acquired commit-exclusive" + ); + thread::yield_now(); + } + + let snapshot_tree = tree.clone(); + let (snapshot_tx, snapshot_rx) = sync_channel(0); + let snapshot_worker = thread::spawn(move || { + snapshot_tx.send(snapshot_tree.snapshot(b"")).unwrap(); + }); + assert!( + snapshot_rx.recv_timeout(Duration::from_millis(50)).is_err(), + "snapshot barrier must wait behind the active delta flush", + ); + + drop(child_guard); + drop(child_pin); + checkpoint_rx + .recv_timeout(Duration::from_secs(2)) + .unwrap() + .unwrap(); + checkpoint.join().unwrap(); + let snapshot = snapshot_rx + .recv_timeout(Duration::from_secs(2)) + .unwrap() + .unwrap(); + snapshot_worker.join().unwrap(); + assert_eq!( + snapshot.get(&target).unwrap().as_deref(), + Some(&b"checkpoint-value"[..]), + "snapshot must linearize after the acknowledged delta it waited behind", + ); + } } diff --git a/src/api/view.rs b/src/api/view.rs index 3439a12..0c09e6d 100644 --- a/src/api/view.rs +++ b/src/api/view.rs @@ -12,18 +12,20 @@ use super::tree::{count_scan_limit, prefix_count_from_seen}; use crate::concurrency::Gate; use crate::engine::{self, KeyRangeBuilder, KeyRangeEntryRef, RangeBuilder}; use crate::layout::BlobGuid; -use crate::store::{BufferManager, CachedBlob}; +use crate::store::{BufferManager, CachedBlob, SnapshotLease}; /// Immutable read transaction over one captured prefix. /// /// Created by [`crate::Tree::view`]. Subsequent live-tree writes do -/// not affect it. +/// not affect it. Clones and owned range cursors retain the underlying +/// snapshot epoch until the final derived handle is dropped. #[derive(Clone)] pub struct View { scope: Vec, store: Arc, root_guid: BlobGuid, root_pin: Arc, + snapshot_lease: Arc, range_gate: Arc, scan_fence: Option<(Arc, Arc)>, } @@ -34,6 +36,7 @@ impl View { store: Arc, root_guid: BlobGuid, root_pin: Arc, + snapshot_lease: Arc, scan_fence: Option<(Arc, Arc)>, ) -> Self { Self { @@ -41,6 +44,7 @@ impl View { store, root_guid, root_pin, + snapshot_lease, range_gate: Arc::new(Gate::new()), scan_fence, } @@ -70,7 +74,7 @@ impl View { pub fn get_version(&self, key: &[u8]) -> Result> { self.ensure_in_scope(key)?; let search = engine::SearchKey::user(key); - engine::lookup_multi_with(&self.store, &self.root_pin, None, search, |hit| { + engine::lookup_multi_with_snapshot(&self.store, &self.root_pin, None, search, |hit| { RecordVersion::new(hit.seq) }) } @@ -131,9 +135,11 @@ impl View { fn lookup_record(&self, key: &[u8]) -> Result> { let search = engine::SearchKey::user(key); - engine::lookup_multi_with(&self.store, &self.root_pin, None, search, |hit| Record { - value: hit.value.to_vec(), - version: RecordVersion::new(hit.seq), + engine::lookup_multi_with_snapshot(&self.store, &self.root_pin, None, search, |hit| { + Record { + value: hit.value.to_vec(), + version: RecordVersion::new(hit.seq), + } }) } @@ -146,11 +152,12 @@ impl View { || Arc::clone(&self.range_gate), |(gate, _)| Arc::clone(gate), ), - ); + ) + .with_snapshot_lease(Arc::clone(&self.snapshot_lease)); let builder = if let Some((_, mutation_gate)) = &self.scan_fence { builder.with_mutation_gate(Arc::clone(mutation_gate)) } else { - builder.snapshot_cursor() + builder }; builder.prefix(prefix) } diff --git a/src/checkpoint/io.rs b/src/checkpoint/io.rs index ecc6f02..da440eb 100644 --- a/src/checkpoint/io.rs +++ b/src/checkpoint/io.rs @@ -23,13 +23,16 @@ //! everything through this same queue. use crossbeam_channel::{Receiver, Sender}; -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::Arc; use crate::api::errors::{Error, Result}; use crate::engine; use crate::layout::BlobGuid; -use crate::store::{BlobFrameRef, WriteThroughEntry, WriteThroughStatus, STRUCTURAL_SEQ}; +use crate::store::blob_store::BlobStore; +use crate::store::{ + BlobFrameRef, BufferManager, WriteThroughEntry, WriteThroughStatus, STRUCTURAL_SEQ, +}; use super::Shared; @@ -108,6 +111,10 @@ fn commit_epoch_batch( shared: &Arc, epochs: &mut [CheckpointEpoch], ) -> Vec { + // Serialize the complete write/sync/delete/sync transaction with manual + // checkpoints. A stale older epoch must never overwrite a newer durable + // parent after that newer epoch has already deleted one of its children. + let _checkpoint_io = shared.bm.enter_checkpoint_io(); let mut progresses = Vec::with_capacity(epochs.len()); let mut entries = Vec::new(); let mut collect_error = None; @@ -147,7 +154,7 @@ fn commit_epoch_batch( } vec![0; epochs.len()] } else { - match write_entries_in_dependency_order(shared, &mut entries, epochs.len()) { + match write_entries_in_dependency_order(&shared.bm, &mut entries, epochs.len()) { Ok(report) => { if report.deferred { restore_unflushed_batch_entries(shared, &entries); @@ -235,10 +242,20 @@ fn collect_entry_children(entry: &WriteThroughEntry) -> Result> { } fn write_entries_in_dependency_order( - shared: &Arc, + bm: &Arc, entries: &mut [BatchEntry], epoch_count: usize, ) -> Result { + // A previous data/manifest operation may have left store-local flush + // debt without a corresponding dirty BM entry. Child readiness treats + // any such debt as non-durable, so merely deferring the parent would + // reproduce the same empty dependency wave forever. Cross that frontier + // explicitly before evaluating dependencies; failure is reported to the + // planner, which restores every entry and pending delete for retry. + if bm.needs_flush() { + bm.flush_inner()?; + } + let mut remaining_by_guid = HashMap::::new(); for entry in entries.iter() { *remaining_by_guid.entry(entry.guid).or_insert(0) += 1; @@ -250,12 +267,7 @@ fn write_entries_in_dependency_order( let mut wave = Vec::new(); for (idx, entry) in entries.iter().enumerate() { if !entry.flushed - && children_ready( - shared, - &entry.children, - &remaining_by_guid, - &durable_this_batch, - )? + && children_ready(bm, &entry.children, &remaining_by_guid, &durable_this_batch)? { wave.push(idx); } @@ -263,6 +275,14 @@ fn write_entries_in_dependency_order( if wave.is_empty() { let deferred = entries.iter().any(|entry| !entry.flushed); + if deferred { + validate_no_progress_dependencies( + bm, + entries, + &remaining_by_guid, + &durable_this_batch, + )?; + } return Ok(BatchWriteReport { dirty_flushed_by_epoch, deferred, @@ -278,8 +298,8 @@ fn write_entries_in_dependency_order( .expect("unflushed batch entry owns its write") }) .collect(); - let report = shared.bm.write_through_batch(&wave_entries)?; - shared.bm.flush_inner()?; + let report = bm.write_through_batch(&wave_entries)?; + bm.flush_inner()?; let mut saw_stale = false; for (idx, status) in wave.into_iter().zip(report.statuses) { @@ -310,8 +330,107 @@ fn write_entries_in_dependency_order( } } +/// Classify a dependency planner stall. Only dependencies owned by dirty or +/// flushing work outside this batch may be retried: a missing durable child +/// and any cycle wholly inside the remaining batch are persistent corruption, +/// not scheduling backpressure. +fn validate_no_progress_dependencies( + bm: &Arc, + entries: &[BatchEntry], + remaining_by_guid: &HashMap, + durable_this_batch: &HashSet, +) -> Result<()> { + let mut graph = HashMap::>::new(); + for guid in remaining_by_guid.keys().copied() { + graph.entry(guid).or_default(); + } + let mut external_transient = false; + + for entry in entries.iter().filter(|entry| !entry.flushed) { + for child in &entry.children { + if durable_this_batch.contains(child) { + continue; + } + if remaining_by_guid.contains_key(child) { + graph.entry(entry.guid).or_default().insert(*child); + continue; + } + if bm.has_unflushed_blob(*child) { + external_transient = true; + continue; + } + if !bm.store_has_blob(*child)? { + return Err( + Error::node_corrupt("checkpoint dependency references missing child") + .with_blob_guid(entry.guid), + ); + } + if !bm.store_has_durable_blob(*child)? { + external_transient = true; + } + } + } + + if let Some(guid) = dependency_cycle(&graph) { + return Err(Error::node_corrupt("checkpoint dependency cycle").with_blob_guid(guid)); + } + if external_transient { + return Ok(()); + } + Err(Error::Internal( + "checkpoint dependency planner made no progress", + )) +} + +fn dependency_cycle(graph: &HashMap>) -> Option { + // The graph is derived from potentially corrupt persisted frames, so its + // depth is attacker-/corruption-controlled. Kahn's algorithm keeps the + // diagnostic path iterative and bounded to O(V + E) heap memory instead + // of risking a process stack overflow on a long acyclic chain. + let mut indegree = graph + .keys() + .copied() + .map(|guid| (guid, 0usize)) + .collect::>(); + for children in graph.values() { + for child in children { + if let Some(degree) = indegree.get_mut(child) { + *degree = degree.saturating_add(1); + } + } + } + + let mut ready = indegree + .iter() + .filter_map(|(guid, degree)| (*degree == 0).then_some(*guid)) + .collect::>(); + let mut visited = 0usize; + while let Some(guid) = ready.pop_front() { + visited += 1; + if let Some(children) = graph.get(&guid) { + for child in children { + let Some(degree) = indegree.get_mut(child) else { + continue; + }; + *degree -= 1; + if *degree == 0 { + ready.push_back(*child); + } + } + } + } + + (visited != indegree.len()) + .then(|| { + indegree + .into_iter() + .find_map(|(guid, degree)| (degree != 0).then_some(guid)) + }) + .flatten() +} + fn children_ready( - shared: &Arc, + bm: &Arc, children: &[BlobGuid], remaining_by_guid: &HashMap, durable_this_batch: &HashSet, @@ -320,16 +439,49 @@ fn children_ready( if durable_this_batch.contains(child) { continue; } - if remaining_by_guid.contains_key(child) || shared.bm.has_unflushed_blob(*child) { + if remaining_by_guid.contains_key(child) || bm.has_unflushed_blob(*child) { return Ok(false); } - if !shared.bm.store_has_durable_blob(*child)? { + if !bm.store_has_durable_blob(*child)? { return Ok(false); } } Ok(true) } +/// Write one synchronous checkpoint snapshot in the same child-before-parent +/// durability waves used by the background I/O worker. +/// +/// Every returned entry was either stale or deferred because one of its child +/// references was not yet durable. The caller must restore those entries to +/// the dirty map and retry them in a later checkpoint round. The caller must +/// hold [`BufferManager::enter_checkpoint_io`] across this call and its +/// pending-delete phase. +pub(crate) fn write_entries_child_first( + bm: &Arc, + entries: Vec, +) -> Result> { + let mut batch_entries = Vec::with_capacity(entries.len()); + for entry in entries { + let children = collect_entry_children(&entry)?; + batch_entries.push(BatchEntry { + epoch_idx: 0, + guid: entry.guid, + expected_seq: entry.expected_seq, + entry: Some(entry), + children, + flushed: false, + }); + } + + let _ = write_entries_in_dependency_order(bm, &mut batch_entries, 1)?; + Ok(batch_entries + .iter() + .filter(|entry| !entry.flushed) + .map(|entry| (entry.guid, entry.expected_seq)) + .collect()) +} + fn restore_batch_entries(shared: &Arc, entries: &[BatchEntry]) { if entries.is_empty() { return; @@ -419,13 +571,13 @@ mod tests { use crate::checkpoint::CheckpointConfig; use crate::concurrency::{CommitGate, Gate}; use crate::layout::{BlobNode, NodeType}; - use crate::store::blob_store::{AlignedBlobBuf, BlobStore, MemoryBlobStore}; + use crate::store::blob_store::{AlignedBlobBuf, BlobStore, FileBlobStore, MemoryBlobStore}; use crate::store::{BlobFrame, BufferManager}; use crossbeam_channel::bounded; use std::io; use std::mem::size_of; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; - use std::sync::{Arc, Mutex}; + use std::sync::{Arc, Barrier, Mutex}; #[derive(Debug, PartialEq, Eq)] enum StoreEvent { @@ -443,6 +595,65 @@ mod tests { fail_flush: bool, } + struct BlockingFirstBatchStore { + inner: MemoryBlobStore, + write_batches: AtomicUsize, + first_entered: Barrier, + release_first: Barrier, + } + + impl BlockingFirstBatchStore { + fn new() -> Self { + Self { + inner: MemoryBlobStore::new(), + write_batches: AtomicUsize::new(0), + first_entered: Barrier::new(2), + release_first: Barrier::new(2), + } + } + } + + impl BlobStore for BlockingFirstBatchStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> Result<()> { + self.inner.read_blob(guid, dst) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { + self.inner.write_blob(guid, src) + } + + fn write_blobs_with_data_sync(&self, writes: &[(BlobGuid, &AlignedBlobBuf)]) -> Result<()> { + let ordinal = self.write_batches.fetch_add(1, Ordering::AcqRel) + 1; + if ordinal == 1 { + // Pause the older epoch after its BM content validation but + // before its store write becomes visible. + self.first_entered.wait(); + self.release_first.wait(); + } + self.inner.write_blobs(writes) + } + + fn delete_blob(&self, guid: BlobGuid) -> Result<()> { + self.inner.delete_blob(guid) + } + + fn list_blobs(&self) -> Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> Result<()> { + self.inner.flush() + } + + fn needs_flush(&self) -> bool { + self.inner.needs_flush() + } + + fn has_blob(&self, guid: BlobGuid) -> Result { + self.inner.has_blob(guid) + } + } + impl CountingBatchStore { fn new() -> Self { Self { @@ -766,7 +977,107 @@ mod tests { } #[test] - fn checkpoint_defers_parent_when_existing_child_manifest_is_not_durable() { + fn child_first_checkpoint_rejects_self_dependency_cycle() { + let bm = Arc::new(BufferManager::new(Arc::new(MemoryBlobStore::new()), 8)); + let guid = [0xE7; 16]; + + let error = write_entries_child_first(&bm, vec![entry(guid, 1, parent_blob(guid, guid))]) + .unwrap_err(); + + assert!(matches!( + error, + Error::NodeCorrupt { + context: "checkpoint dependency cycle", + blob_guid: Some(_), + .. + } + )); + } + + #[test] + fn child_first_checkpoint_rejects_two_blob_dependency_cycle() { + let bm = Arc::new(BufferManager::new(Arc::new(MemoryBlobStore::new()), 8)); + let first = [0xE8; 16]; + let second = [0xE9; 16]; + + let error = write_entries_child_first( + &bm, + vec![ + entry(first, 1, parent_blob(first, second)), + entry(second, 1, parent_blob(second, first)), + ], + ) + .unwrap_err(); + + assert!(matches!( + error, + Error::NodeCorrupt { + context: "checkpoint dependency cycle", + blob_guid: Some(_), + .. + } + )); + } + + #[test] + fn dependency_cycle_handles_long_corrupt_graph_without_recursion() { + const NODES: u32 = 50_000; + let guid = |index: u32| { + let mut guid = [0xA7; 16]; + guid[..4].copy_from_slice(&index.to_le_bytes()); + guid + }; + let mut graph = HashMap::>::with_capacity(NODES as usize); + for index in 0..NODES { + let mut children = HashSet::new(); + if index + 1 < NODES { + children.insert(guid(index + 1)); + } + graph.insert(guid(index), children); + } + + assert_eq!(dependency_cycle(&graph), None); + graph.get_mut(&guid(NODES - 1)).unwrap().insert(guid(1)); + assert!(dependency_cycle(&graph).is_some()); + } + + #[test] + fn child_first_checkpoint_rejects_permanently_missing_child() { + let bm = Arc::new(BufferManager::new(Arc::new(MemoryBlobStore::new()), 8)); + let parent = [0xEA; 16]; + let missing = [0xEB; 16]; + + let error = + write_entries_child_first(&bm, vec![entry(parent, 1, parent_blob(parent, missing))]) + .unwrap_err(); + + assert!(matches!( + error, + Error::NodeCorrupt { + context: "checkpoint dependency references missing child", + blob_guid: Some(guid), + .. + } if guid == parent + )); + } + + #[test] + fn child_first_checkpoint_defers_external_dirty_child() { + let bm = Arc::new(BufferManager::new(Arc::new(MemoryBlobStore::new()), 8)); + let parent = [0xEC; 16]; + let child = [0xED; 16]; + bm.install_new_blob(child, child_blob(child, 9), 1); + + let deferred = + write_entries_child_first(&bm, vec![entry(parent, 2, parent_blob(parent, child))]) + .unwrap(); + + assert_eq!(deferred, vec![(parent, 2)]); + assert!(bm.has_unflushed_blob(child)); + } + + #[test] + fn checkpoint_flushes_existing_child_manifest_before_parent_reference() { let store = Arc::new(CountingBatchStore::new()); let shared = test_shared(Arc::clone(&store)); let parent = [0xE3; 16]; @@ -785,18 +1096,191 @@ mod tests { assert_eq!(reports.len(), 1); assert!(reports[0].result.is_ok()); + assert_eq!(reports[0].dirty_flushed, 1); assert_eq!( - reports[0].dirty_flushed, 0, - "parent write must wait until the child's manifest entry is durable" + *store.events.lock().unwrap(), + vec![ + StoreEvent::Flush, + StoreEvent::Write(vec![parent]), + StoreEvent::Flush, + ], + "outstanding child manifest debt must become durable before parent write", ); + assert_eq!(shared.bm.dirty_count(), 0); + } + + #[test] + fn checkpoint_preflush_failure_restores_parent_for_retry() { + let store = Arc::new(CountingBatchStore::failing_flush()); + let shared = test_shared(Arc::clone(&store)); + let parent = [0xE5; 16]; + let child = [0xE6; 16]; + store + .inner + .write_blob(child, &child_blob(child, 9)) + .unwrap(); + store.mark_pending_flush(); + let mut epochs = vec![CheckpointEpoch { + entries: vec![entry(parent, 1, parent_blob(parent, child))], + pending: HashMap::new(), + }]; + + let reports = commit_epoch_batch(&shared, &mut epochs); + + assert_eq!(reports.len(), 1); + assert!(reports[0].result.is_err()); + assert_eq!(reports[0].dirty_flushed, 0); + assert_eq!(*store.events.lock().unwrap(), vec![StoreEvent::Flush]); + assert_eq!(store.write_batches.load(Ordering::Acquire), 0); + assert_eq!(shared.bm.dirty_count(), 1); + } + + #[test] + fn synchronous_child_first_manifest_survives_torn_parent_tail() { + let dir = tempfile::tempdir().unwrap(); + let parent: BlobGuid = [ + 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, + 0x9F, 0xA0, + ]; + let child: BlobGuid = [ + 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, + 0xAF, 0xB0, + ]; + + { + let file = match FileBlobStore::open(dir.path()) { + Ok(store) => Arc::new(store), + Err(Error::BlobStoreIo(e)) if e.raw_os_error() == Some(libc::EINVAL) => { + eprintln!("skipping: O_DIRECT not supported on this fs"); + return; + } + Err(e) => panic!("unexpected open error: {e}"), + }; + // Keep an old durable parent mapping in the manifest. The later + // checkpoint overwrites that same slot with a parent that points + // at a newly-created child. + file.write_blob(parent, &child_blob(parent, 1)).unwrap(); + file.flush().unwrap(); + + let file_dyn: Arc = file; + let bm = Arc::new(BufferManager::new(file_dyn, 8)); + write_entries_child_first( + &bm, + vec![ + entry(parent, 2, parent_blob(parent, child)), + entry(child, 2, child_blob(child, 2)), + ], + ) + .unwrap(); + } + + let log_path = dir.path().join("manifest.log"); + let log = std::fs::read(&log_path).unwrap(); + let positions = |needle: &BlobGuid| { + log.windows(needle.len()) + .enumerate() + .filter_map(|(idx, bytes)| (bytes == needle).then_some(idx)) + .collect::>() + }; + let parent_positions = positions(&parent); + let child_positions = positions(&child); + assert_eq!(parent_positions.len(), 2, "initial + rewritten parent Set"); + assert_eq!(child_positions.len(), 1, "one newly-created child Set"); assert!( - store.events.lock().unwrap().is_empty(), - "no parent write may be issued while the store owes a manifest flush" + child_positions[0] < parent_positions[1], + "the child manifest record must be durable before the rewritten parent record", + ); + + // Keep the child Set and only a torn prefix of the later parent Set. + // Reopen truncates that tail to the last complete record. The parent + // data slot may already contain its new child edge, so the child must + // still be present in the recovered manifest. + const MANIFEST_RECORD_HEADER_SIZE: u64 = 9; + let rewritten_parent_record = + u64::try_from(parent_positions[1]).unwrap() - MANIFEST_RECORD_HEADER_SIZE; + let f = std::fs::OpenOptions::new() + .write(true) + .open(&log_path) + .unwrap(); + f.set_len(rewritten_parent_record + 3).unwrap(); + f.sync_all().unwrap(); + drop(f); + + let reopened = FileBlobStore::open(dir.path()).unwrap(); + let mut parent_bytes = AlignedBlobBuf::zeroed(); + reopened.read_blob(parent, &mut parent_bytes).unwrap(); + let children = + engine::collect_blob_children_from_frame(BlobFrameRef::wrap(parent_bytes.as_slice())) + .unwrap(); + assert_eq!(children, vec![child]); + let mut child_bytes = AlignedBlobBuf::zeroed(); + reopened.read_blob(child, &mut child_bytes).unwrap(); + } + + #[test] + fn checkpoint_io_serializes_old_parent_before_new_parent_delete_epoch() { + let store = Arc::new(BlockingFirstBatchStore::new()); + let parent = [0xB1; 16]; + let child = [0xB2; 16]; + store + .inner + .write_blob(child, &child_blob(child, 1)) + .unwrap(); + let shared = test_shared(Arc::clone(&store)); + shared.bm.mark_for_delete(child, 2); + + let old_shared = Arc::clone(&shared); + let old_epoch = std::thread::spawn(move || { + let mut epochs = vec![CheckpointEpoch { + entries: vec![entry(parent, 1, parent_blob(parent, child))], + pending: HashMap::new(), + }]; + commit_epoch_batch(&old_shared, &mut epochs) + }); + store.first_entered.wait(); + + let new_shared = Arc::clone(&shared); + let (new_done_tx, new_done_rx) = std::sync::mpsc::sync_channel(1); + let new_epoch = std::thread::spawn(move || { + let mut epochs = vec![CheckpointEpoch { + entries: vec![entry(parent, 2, child_blob(parent, 2))], + pending: HashMap::from([(child, 2)]), + }]; + let reports = commit_epoch_batch(&new_shared, &mut epochs); + new_done_tx.send(()).unwrap(); + reports + }); + + assert!( + new_done_rx + .recv_timeout(std::time::Duration::from_millis(100)) + .is_err(), + "the newer parent+delete epoch must wait for the older epoch's complete I/O phase", ); assert_eq!( - shared.bm.dirty_count(), + store.write_batches.load(Ordering::Acquire), 1, - "deferred parent must be restored to dirty state for the next round" + "no second epoch may enter store I/O while the first is paused", + ); + store.release_first.wait(); + + let old_reports = old_epoch.join().unwrap(); + new_done_rx + .recv_timeout(std::time::Duration::from_secs(2)) + .unwrap(); + let new_reports = new_epoch.join().unwrap(); + assert!(old_reports.iter().all(|report| report.result.is_ok())); + assert!(new_reports.iter().all(|report| report.result.is_ok())); + assert!(!store.inner.has_blob(child).unwrap()); + let mut durable_parent = AlignedBlobBuf::zeroed(); + store.inner.read_blob(parent, &mut durable_parent).unwrap(); + assert!( + engine::collect_blob_children_from_frame( + BlobFrameRef::wrap(durable_parent.as_slice(),) + ) + .unwrap() + .is_empty(), + "the final durable parent must be the newer child-free epoch", ); } } diff --git a/src/checkpoint/mod.rs b/src/checkpoint/mod.rs index 919368a..cbe78af 100644 --- a/src/checkpoint/mod.rs +++ b/src/checkpoint/mod.rs @@ -11,7 +11,8 @@ //! │ ├─ snapshot_dirty + journal.flush │ //! │ ├─ submit one CheckpointEpoch │ //! │ ├─ reap completed epochs in FIFO order │ -//! │ └─ journal.truncate iff pipeline empty + clean BM │ +//! │ ├─ journal.truncate iff pipeline empty + clean BM │ +//! │ └─ bounded clean-frontier exact orphan reclaim │ //! └────────┬─────────────────────────────────────────────────┘ //! │ IoTask (bounded crossbeam channel) //! ▼ @@ -86,6 +87,7 @@ use crate::concurrency::{CommitGate, Gate}; use crate::journal::Journal; use crate::store::BufferManager; +pub(crate) use self::io::write_entries_child_first; use self::io::IoTask; // ---------- public config ---------- @@ -116,11 +118,12 @@ pub struct CheckpointConfig { /// Default 512. This bounds dirty growth without forcing an /// early checkpoint for a handful of hot blobs. pub dirty_blob_threshold: usize, - /// Drain queued parent-merge candidates at the start of each - /// round. Parent merge rewrites structural BlobNode edges and - /// queues old child blobs for deferred manifest deletion, so it - /// remains opt-in until reachability-safe delete GC is available - /// for fully asynchronous checkpoint pipelines. + /// Drain queued parent-merge candidates at the start of each round. + /// Parent merge rewrites structural BlobNode edges and stages detached + /// children under their exact parent publication; a clean durable + /// checkpoint frontier later hands them to bounded physical reclamation. + /// The work remains opt-in because it trades foreground tree shape for + /// additional background rewrite and checkpoint I/O. /// /// Default `false`; foreground split/shape control remains /// enabled, and callers can still run explicit maintenance. @@ -414,6 +417,7 @@ impl Drop for Checkpointer { // ---------- checkpoint_thread main loop ---------- fn checkpoint_main(shared: &Arc) { + shared.bm.register_checkpoint_waker(thread::current()); let mut pipeline = round::Pipeline::new(shared.cfg.io_queue_capacity); loop { if shared.checkpoint_stop.load(Ordering::Acquire) { @@ -422,13 +426,17 @@ fn checkpoint_main(shared: &Arc) { if let Err(e) = pipeline.reap_ready(shared) { eprintln!("holt: checkpoint epoch failed: {e}"); } - let has_pressure = shared.bm.dirty_count() >= shared.cfg.dirty_blob_threshold.max(1) + let write_pressure = shared.bm.dirty_count() >= shared.cfg.dirty_blob_threshold.max(1) || shared.bm.pending_delete_count() != 0; - if !has_pressure { + if !write_pressure { if !pipeline.is_empty() { thread::park_timeout(shared.cfg.idle_interval.min(Duration::from_millis(1))); continue; } + // A newly retired orphan unparks this thread once. If every + // candidate is pinned and the round makes no progress, the next + // outer iteration waits a full idle interval instead of spinning + // solely because the backlog gauge remains non-zero. thread::park_timeout(shared.cfg.idle_interval); } if shared.checkpoint_stop.load(Ordering::Acquire) { @@ -452,6 +460,7 @@ fn checkpoint_main(shared: &Arc) { if let Err(e) = pipeline.drain(shared) { eprintln!("holt: checkpoint pipeline drain during shutdown failed: {e}"); } + shared.bm.clear_checkpoint_waker(); } #[cfg(test)] @@ -552,6 +561,58 @@ mod tests { flush_pending: AtomicBool, } + struct BlockingExactDeleteStore { + inner: MemoryBlobStore, + target: BlobGuid, + delete_entered: std::sync::Barrier, + delete_release: std::sync::Barrier, + } + + impl BlockingExactDeleteStore { + fn new(target: BlobGuid) -> Self { + Self { + inner: MemoryBlobStore::new(), + target, + delete_entered: std::sync::Barrier::new(2), + delete_release: std::sync::Barrier::new(2), + } + } + } + + impl BlobStore for BlockingExactDeleteStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> Result<()> { + self.inner.read_blob(guid, dst) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { + self.inner.write_blob(guid, src) + } + + fn write_blobs_with_data_sync(&self, writes: &[(BlobGuid, &AlignedBlobBuf)]) -> Result<()> { + self.inner.write_blobs_with_data_sync(writes) + } + + fn delete_blob(&self, guid: BlobGuid) -> Result<()> { + if guid == self.target { + self.delete_entered.wait(); + self.delete_release.wait(); + } + self.inner.delete_blob(guid) + } + + fn list_blobs(&self) -> Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> Result<()> { + self.inner.flush() + } + + fn needs_flush(&self) -> bool { + self.inner.needs_flush() + } + } + impl FlushGateStore { fn new() -> Self { Self { @@ -759,6 +820,152 @@ mod tests { } } + #[test] + fn pinned_orphan_backlog_parks_and_does_not_starve_later_work() { + let inner = Arc::new(MemoryBlobStore::new()); + let pinned_guid = [0x31; 16]; + let free_guid = [0x32; 16]; + let snapshot_root = [0x33; 16]; + inner + .write_blob(pinned_guid, &test_blob(pinned_guid)) + .unwrap(); + inner.write_blob(free_guid, &test_blob(free_guid)).unwrap(); + inner + .write_blob(snapshot_root, &test_blob(snapshot_root)) + .unwrap(); + let bm = Arc::new(BufferManager::new(inner.clone(), 8)); + let pinned = bm.pin(pinned_guid).unwrap(); + let root_pin = bm.pin(snapshot_root).unwrap(); + + let mut cfg = no_merge_cfg(); + cfg.idle_interval = Duration::from_millis(20); + let ck = Checkpointer::spawn( + Arc::clone(&bm), + None, + maintenance_gate(), + commit_gate(), + cfg, + ) + .expect("spawn"); + + let retire = |guid| { + let epoch = bm.register_snapshot(snapshot_root, &root_pin).unwrap(); + bm.stage_cow_reclaim(snapshot_root, guid, epoch); + bm.mark_dirty_cached(snapshot_root, epoch, root_pin.as_ref()); + bm.retire_snapshot(epoch); + }; + retire(pinned_guid); + + let deadline = Instant::now() + Duration::from_secs(2); + while ck.rounds_attempted() < 2 { + assert!( + Instant::now() < deadline, + "checkpointer never attempted pinned orphan cleanup" + ); + thread::sleep(Duration::from_millis(2)); + } + let rounds_before_wait = ck.rounds_attempted(); + thread::sleep(Duration::from_millis(120)); + let rounds_after_wait = ck.rounds_attempted(); + assert!( + rounds_after_wait - rounds_before_wait <= 10, + "all-pinned orphan backlog busy-spun: {rounds_before_wait} -> {rounds_after_wait}" + ); + assert_eq!(bm.gc_orphan_backlog_count(), 1); + + retire(free_guid); + let deadline = Instant::now() + Duration::from_secs(2); + while inner.has_blob(free_guid).unwrap() { + assert!( + Instant::now() < deadline, + "later reclaimable orphan was starved by the pinned FIFO head" + ); + thread::sleep(Duration::from_millis(2)); + } + assert!(inner.has_blob(pinned_guid).unwrap()); + assert_eq!(bm.gc_orphan_backlog_count(), 1); + + drop(pinned); + ck.wake(); + let deadline = Instant::now() + Duration::from_secs(2); + while inner.has_blob(pinned_guid).unwrap() { + assert!( + Instant::now() < deadline, + "released pinned orphan was not eventually reclaimed" + ); + thread::sleep(Duration::from_millis(2)); + } + assert_eq!(bm.gc_orphan_backlog_count(), 0); + drop(ck); + } + + #[test] + fn background_exact_reclaim_fences_writer_merge_and_drop_admission() { + let orphan = [0x34; 16]; + let snapshot_root = [0x35; 16]; + let store = Arc::new(BlockingExactDeleteStore::new(orphan)); + store.inner.write_blob(orphan, &test_blob(orphan)).unwrap(); + store + .inner + .write_blob(snapshot_root, &test_blob(snapshot_root)) + .unwrap(); + let bm = Arc::new(BufferManager::new(store.clone(), 8)); + let root_pin = bm.pin(snapshot_root).unwrap(); + let epoch = bm.register_snapshot(snapshot_root, &root_pin).unwrap(); + bm.stage_cow_reclaim(snapshot_root, orphan, epoch); + bm.mark_dirty_cached(snapshot_root, epoch, root_pin.as_ref()); + bm.retire_snapshot(epoch); + assert_eq!(bm.gc_orphan_backlog_count(), 1); + + let maintenance = maintenance_gate(); + let commit = commit_gate(); + let mut cfg = no_merge_cfg(); + cfg.idle_interval = Duration::from_secs(10); + let ck = Checkpointer::spawn(Arc::clone(&bm), None, Arc::clone(&maintenance), commit, cfg) + .expect("spawn"); + ck.wake(); + store.delete_entered.wait(); + + // Foreground writers enter maintenance shared; merge and DB drop + // enter maintenance exclusive. Neither admission class may overlap + // the physical exact delete selected from a clean frontier. + let (writer_tx, writer_rx) = std::sync::mpsc::sync_channel(1); + let writer_gate = Arc::clone(&maintenance); + let writer = thread::spawn(move || { + let _guard = writer_gate.enter_shared(); + writer_tx.send(()).unwrap(); + }); + let (topology_tx, topology_rx) = std::sync::mpsc::sync_channel(1); + let topology_gate = Arc::clone(&maintenance); + let topology = thread::spawn(move || { + let _guard = topology_gate.enter_exclusive(); + topology_tx.send(()).unwrap(); + }); + assert!(writer_rx.recv_timeout(Duration::from_millis(50)).is_err()); + assert!( + topology_rx.recv_timeout(Duration::from_millis(50)).is_err(), + "merge/drop admission bypassed exact-reclaim maintenance fence", + ); + + store.delete_release.wait(); + writer_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + topology_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + writer.join().unwrap(); + topology.join().unwrap(); + + let deadline = Instant::now() + Duration::from_secs(2); + while store.inner.has_blob(orphan).unwrap() { + assert!( + Instant::now() < deadline, + "exact reclaim did not finish after releasing the store", + ); + thread::sleep(Duration::from_millis(2)); + } + assert_eq!(bm.gc_orphan_backlog_count(), 0); + drop(root_pin); + drop(ck); + } + #[test] fn planner_queues_second_epoch_while_first_io_is_blocked() { let store = Arc::new(BlockingBatchStore::new()); diff --git a/src/checkpoint/round.rs b/src/checkpoint/round.rs index c9e2939..ad281d0 100644 --- a/src/checkpoint/round.rs +++ b/src/checkpoint/round.rs @@ -7,9 +7,9 @@ //! 0. **Merge pass** (optional, controlled by //! `CheckpointConfig::auto_merge`) — drains queued parent-merge //! candidates and folds mergeable children back into parents. -//! Merge mutations are staged through the same dirty / -//! pending-delete sets as foreground writes, then flushed by -//! this round after the WAL sync. +//! Rewritten parents enter the dirty set; their exact detached children +//! enter parent-scoped orphan staging and become reclaimable only after +//! parent dirty publication and a clean durable frontier. //! 1. **Snapshot dirty + pending deletes + content versions** under //! the exclusive side of the tree's commit-publish gate. //! 2. **Flush WAL** through the journal worker so every record that @@ -22,9 +22,12 @@ //! order. This is the truncate watermark: a later epoch may not //! advance WAL trimming before every older epoch is known to //! have landed or restored. -//! 6. **Truncate WAL** only when the pipeline is empty and -//! `bm.dirty_count() == 0 && bm.pending_delete_count() == 0` -//! under the commit-publish gate. +//! 6. **Truncate WAL** only when the pipeline is empty and dirty, flushing, +//! pending-delete, orphan-staging, and write-delta debt are all zero and +//! the store has no deferred flush work, under the commit-publish gate. +//! 7. **Bounded exact reclaim** — while the maintenance fence still excludes +//! topology changes, drain one retired COW/structural FIFO batch. Pinned +//! candidates return to the queue; no all-store reachability walk occurs. //! //! This function is called from two places: //! @@ -146,36 +149,49 @@ impl Pipeline { if !self.in_flight.is_empty() { return Ok(()); } - let Some(journal) = &shared.journal else { - return Ok(()); + // Lock order is maintenance -> commit everywhere topology may + // change. Keep the exclusive maintenance fence through exact + // reclaim: no writer, merge, compact, or DB drop can publish a new + // edge or topology while a clean frontier's FIFO batch is deleted. + let _maintenance = shared.maintenance_gate.enter_exclusive(); + let candidates = { + let _commit = shared.commit_gate.enter_checkpoint(); + // The dirty/flushing/pending counters are BufferManager state and do + // NOT capture the store's own deferred durability: the I/O worker + // retires a dirty entry right after the data `pwrite` but BEFORE the + // data fsync + manifest-delta persist (`flush_inner`). So all three + // counters can read 0 while `store.needs_flush()` is still true — + // i.e. a just-written blob's new slot mapping is only in the in-memory + // manifest, not yet in `manifest.log`. Truncating the WAL there leaves + // a crashed reopen with the acked write in NEITHER the (truncated) WAL + // nor the (un-persisted) manifest — a lost acknowledged write. The + // SIGKILL crash soak hits exactly this (lazy routing keeps the root + // blob a perpetual compaction target, so it is re-written every round). + // `run_round`'s early-skip gates on the same `needs_flush()` for this + // recovery edge; mirror it here so truncate waits for store durability. + let clean = shared.bm.dirty_count() == 0 + && shared.bm.flushing_count() == 0 + && shared.bm.pending_delete_count() == 0 + && shared.bm.orphan_staging_count() == 0 + && shared.bm.write_delta_count() == 0 + && !shared.bm.needs_flush(); + if !clean { + return Ok(()); + } + if let Some(journal) = &shared.journal { + if journal.needs_checkpoint() { + journal.truncate()?; + use std::sync::atomic::Ordering; + shared.truncates.fetch_add(1, Ordering::Relaxed); + } + } + shared.bm.take_retired_orphans_bounded(256) }; - if !journal.needs_checkpoint() { - return Ok(()); - } - let _commit = shared.commit_gate.enter_checkpoint(); - // The dirty/flushing/pending counters are BufferManager state and do - // NOT capture the store's own deferred durability: the I/O worker - // retires a dirty entry right after the data `pwrite` but BEFORE the - // data fsync + manifest-delta persist (`flush_inner`). So all three - // counters can read 0 while `store.needs_flush()` is still true — - // i.e. a just-written blob's new slot mapping is only in the in-memory - // manifest, not yet in `manifest.log`. Truncating the WAL there leaves - // a crashed reopen with the acked write in NEITHER the (truncated) WAL - // nor the (un-persisted) manifest — a lost acknowledged write. The - // SIGKILL crash soak hits exactly this (lazy routing keeps the root - // blob a perpetual compaction target, so it is re-written every round). - // `run_round`'s early-skip gates on the same `needs_flush()` for this - // recovery edge; mirror it here so truncate waits for store durability. - if shared.bm.dirty_count() == 0 - && shared.bm.flushing_count() == 0 - && shared.bm.pending_delete_count() == 0 - && shared.bm.write_delta_count() == 0 - && !shared.bm.needs_flush() - { - journal.truncate()?; - use std::sync::atomic::Ordering; - shared.truncates.fetch_add(1, Ordering::Relaxed); - } + // A clean frontier durably publishes every current parent image. + // Retired COW GUIDs can now be reclaimed directly from their FIFO; + // this keeps the default background path bounded without rescanning + // the complete blob manifest on every idle checkpoint. + shared.bm.reclaim_retired_orphan_batch(candidates)?; Ok(()) } } @@ -248,6 +264,7 @@ pub(super) fn run_round(shared: &Arc, pipeline: &mut Pipeline) -> Result let (mut snap, mut pending, versioned_snap, wal_up_to) = if let Some(journal) = &shared.journal { let _maintenance = shared.maintenance_gate.enter_shared(); + let _commit = shared.commit_gate.enter_checkpoint(); if let Err(e) = shared.bm.flush_write_deltas() { shared.rounds_failed.fetch_add(1, Ordering::Relaxed); shared @@ -255,7 +272,6 @@ pub(super) fn run_round(shared: &Arc, pipeline: &mut Pipeline) -> Result .store(round_start.elapsed().as_micros() as u64, Ordering::Relaxed); return Err(e); } - let _commit = shared.commit_gate.enter_checkpoint(); let wal_up_to = journal.queued_work(); let snap = shared.bm.snapshot_dirty(); let pending = shared.bm.snapshot_pending_deletes(); @@ -274,6 +290,7 @@ pub(super) fn run_round(shared: &Arc, pipeline: &mut Pipeline) -> Result (snap, pending, versioned_snap, Some(wal_up_to)) } else { let _maintenance = shared.maintenance_gate.enter_shared(); + let _commit = shared.commit_gate.enter_checkpoint(); if let Err(e) = shared.bm.flush_write_deltas() { shared.rounds_failed.fetch_add(1, Ordering::Relaxed); shared @@ -449,9 +466,9 @@ fn clone_versioned_dirty( /// Candidate-driven merge pass — fold mergeable `BlobNode` /// children back into their parents. Stages the mutations via the -/// unified `mark_dirty` + `mark_for_delete` protocol so the round's -/// later checkpoint epoch (WAL flush → data writes → store sync → -/// pending deletes → re-Sync → truncate) handles persistence under W2D. +/// unified parent-scoped staging + `mark_dirty` protocol so the round's +/// later checkpoint epoch durably publishes the parent before the exact child +/// GUID can enter bounded orphan reclamation. /// Takes the exclusive maintenance gate around one parent at a /// time so no foreground writer is lock-coupling through the child /// edge being folded and queued for delete. Foreground spillovers @@ -465,12 +482,11 @@ fn clone_versioned_dirty( /// be wrong here — both happen pre-Sync, pre-WAL. `bm.commit` /// would push cache bytes (potentially including user mutations /// whose WAL records aren't yet durable) directly to store, and -/// `bm.delete_blob` would mutate the manifest in-memory which a -/// later `store.flush` could persist while the corresponding -/// user WAL records still hadn't reached disk. Staging through -/// dirty / pending-delete avoids both: the only flush path is the -/// round's checkpoint epoch, which runs strictly after step 2's -/// WAL flush. +/// `bm.delete_blob` would mutate the manifest in-memory which a later +/// `store.flush` could persist while the corresponding user WAL records still +/// hadn't reached disk. Parent-scoped staging avoids both: dirty publication +/// promotes the child into snapshot-protected or clean-frontier state, and the +/// checkpoint epoch runs strictly after step 2's WAL flush. fn run_merge_pass(shared: &Arc) -> Result { use crate::store::STRUCTURAL_SEQ; @@ -481,20 +497,30 @@ fn run_merge_pass(shared: &Arc) -> Result { if !shared.bm.has_blob(guid)? { continue; } - let _commit = shared - .journal - .as_ref() - .map(|_| shared.commit_gate.enter_writer()); + // Also required for custom/no-WAL stores: clean-frontier capture + // must not interleave parent edge mutation with its dirty publish. + let _commit = shared.commit_gate.enter_writer(); let pin = match shared.bm.pin(guid) { Ok(pin) => pin, Err(e) if is_blob_store_not_found(&e) => continue, Err(e) => return Err(e), }; - let (stats, has_children) = { + let merge_result = { let mut guard = pin.write(); let mut frame = guard.frame(); - let stats = engine::try_merge_children(shared.bm.as_ref(), &mut frame, STRUCTURAL_SEQ)?; - (stats, frame.header().num_ext_blobs != 0) + engine::try_merge_children(shared.bm.as_ref(), &mut frame, STRUCTURAL_SEQ) + .map(|stats| (stats, frame.header().num_ext_blobs != 0)) + }; + let (stats, has_children) = match merge_result { + Ok(result) => result, + Err(error) => { + // Earlier children in this same parent may already have been + // folded. Publishing the semantically-equivalent partial + // parent drains their parent-scoped staging before retry. + shared.bm.mark_dirty(guid, STRUCTURAL_SEQ); + drop(pin); + return Err(error); + } }; if stats.merged > 0 { // Keep the parent pin alive until after dirty diff --git a/src/concurrency/commit_gate.rs b/src/concurrency/commit_gate.rs index 38c3f72..46323b4 100644 --- a/src/concurrency/commit_gate.rs +++ b/src/concurrency/commit_gate.rs @@ -46,6 +46,11 @@ impl CommitGate { _inner: self.gate.enter_exclusive(), } } + + #[cfg(test)] + pub(crate) fn checkpoint_pending_for_test(&self) -> bool { + self.gate.writer_pending_for_test() + } } #[derive(Debug)] diff --git a/src/concurrency/gate.rs b/src/concurrency/gate.rs index 7b01b0f..223d68f 100644 --- a/src/concurrency/gate.rs +++ b/src/concurrency/gate.rs @@ -96,7 +96,7 @@ impl Gate { } #[cfg(test)] - fn writer_pending_for_test(&self) -> bool { + pub(crate) fn writer_pending_for_test(&self) -> bool { self.state.load(Ordering::Acquire) & WRITE_BIT != 0 } } diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 40d4545..d9df19e 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -36,5 +36,5 @@ pub use walker::{ }; pub(crate) use walker::{ collect_blob_children_from_frame, fresh_blob_guid, insert_multi_batch_conditional, - InsertBatchItem, PrefixListCache, SearchKey, + lookup_multi_with_snapshot, InsertBatchItem, PrefixListCache, SearchKey, }; diff --git a/src/engine/walker/cow.rs b/src/engine/walker/cow.rs index 7862455..8caabd1 100644 --- a/src/engine/walker/cow.rs +++ b/src/engine/walker/cow.rs @@ -13,10 +13,11 @@ use std::sync::Arc; use crate::api::errors::Result; -use crate::layout::{frame_created_epoch, BlobGuid}; +use crate::layout::{frame_created_epoch, BlobGuid, BlobNode, NodeType}; use crate::store::{BlobWriteGuard, BufferManager, CachedBlob}; use super::fresh_blob_guid; +use super::readers::ntype_of; use super::writers::repoint_blob_node; /// Whether `child` may be visible to a live snapshot and so must be @@ -49,21 +50,52 @@ pub(super) fn fork_child_if_shared( child_guid: BlobGuid, child_bytes: &[u8], parent_off: u32, - seq: u64, ) -> Result)>> { let barrier = bm.fork_barrier(); let child_epoch = frame_created_epoch(child_bytes); if barrier == 0 || child_epoch > barrier { return Ok(None); } + // Validate the exact edge before allocating a dirty fork. The parent is + // write-latched, so this proof remains stable through repoint and makes + // the later write infallible for every non-corrupt frame. Without this + // ordering a corrupt/stale offset could leak one unattached dirty fork on + // every retry. + { + let frame = parent.frame(); + if ntype_of(frame.as_ref(), parent_off)? != NodeType::Blob { + return Err(crate::api::errors::Error::node_corrupt( + "COW parent edge is not a BlobNode", + )); + } + let body = frame.body_at_offset(parent_off).ok_or_else(|| { + crate::api::errors::Error::node_corrupt("COW parent edge body missing") + })?; + let edge = *super::cast::(body); + if edge.child_blob_guid != child_guid { + return Err(crate::api::errors::Error::node_corrupt( + "COW parent edge child identity changed", + )); + } + } + let parent_guid = { + let frame = parent.frame(); + frame.header().blob_guid + }; let fork_guid = fresh_blob_guid(); - let fork_pin = bm.fork_frame(child_bytes, fork_guid, seq)?; + // The initial fork and parent repoint are logically neutral shape + // changes. Publish them as structural debt before recursion; a successful + // leaf mutation later lowers the fork's dirty seq to the user WAL seq. + let fork_pin = bm.fork_frame(child_bytes, fork_guid, crate::store::STRUCTURAL_SEQ)?; { let mut frame = parent.frame(); + // The pre-allocation validation above ran under this same exclusive + // parent latch, so the body range and node identity cannot change. repoint_blob_node(&mut frame, parent_off, fork_guid)?; } - // The old child is now forked away from the live tree; record it so - // retire reclaims it once no live snapshot can reference it. - bm.record_orphan(child_guid, child_epoch); + // The old child is now forked away from the live tree. Stage it so lease + // retirement can make it eligible for clean-frontier exact reclaim once + // no live snapshot can reference it. + bm.stage_cow_reclaim(parent_guid, child_guid, child_epoch); Ok(Some((fork_guid, fork_pin))) } diff --git a/src/engine/walker/erase.rs b/src/engine/walker/erase.rs index 8c17183..962840d 100644 --- a/src/engine/walker/erase.rs +++ b/src/engine/walker/erase.rs @@ -392,8 +392,11 @@ fn lock_coupled_erase_in_blob( crossing.child_guid, child_guard.as_slice(), crossing.parent_off, - seq, )? { + // Fork + edge repoint is a logically neutral structural + // transaction. Publish the parent before releasing its latch + // so a later deep read/pin error cannot strand cow_pending. + bm.mark_dirty_cached(current_guid, crate::store::STRUCTURAL_SEQ, current_entry); drop(child_guard); drop(child_pin); let fork_guard = fork_pin.write(); diff --git a/src/engine/walker/insert.rs b/src/engine/walker/insert.rs index 3d7b4a5..85af065 100644 --- a/src/engine/walker/insert.rs +++ b/src/engine/walker/insert.rs @@ -553,6 +553,18 @@ fn try_insert_batch_from_first_blob( Err(e) => return Err(e), }; child_pin.prefetch_header(); + if child_is_snapshot_shared(bm, child_pin.as_ref()) { + // The batch fast path only owns a shared parent latch and + // cannot repoint its BlobNode. Fall back to the root-led + // single-item walker, which performs parent-scoped COW before + // acquiring the child's write latch. + drop(child_pin); + drop(root_read); + return Ok(InsertBatchOutcome { + root_dirty: false, + applied: 0, + }); + } let child_guard = child_pin.write(); drop(root_read); @@ -635,6 +647,14 @@ fn try_insert_batch_from_route( Err(e) => return Err(e), }; child_pin.prefetch_header(); + if child_is_snapshot_shared(bm, child_pin.as_ref()) { + // Route fast paths cannot repoint the parent edge under their shared + // latch. Returning a miss routes the operation through the root-led + // COW walker before any child bytes are mutated. + drop(child_pin); + drop(parent_guard); + return Ok(None); + } let child_guard = child_pin.write(); drop(parent_guard); let outcome = insert_batch_in_pinned_blob( @@ -801,8 +821,8 @@ fn should_compact_before_spillover(miss: InsertAllocMiss, before: ReclaimSnapsho fn reclaim_or_spillover( bm: &BufferManager, guard: &mut BlobWriteGuard<'_>, + current_entry: &CachedBlob, current_guid: crate::layout::BlobGuid, - seq: u64, miss: InsertAllocMiss, ) -> Result<()> { let before = reclaim_snapshot(guard); @@ -816,11 +836,15 @@ fn reclaim_or_spillover( { let mut frame = guard.frame(); - spillover_blob(bm, &mut frame, seq).map_err(|e| e.with_blob_guid(current_guid))?; + spillover_blob(bm, &mut frame, crate::store::STRUCTURAL_SEQ) + .map_err(|e| e.with_blob_guid(current_guid))?; } + // `spillover_blob` prepared and compacted the complete parent before it + // installed the fresh child, so publication here cannot be bypassed by + // a later fallible structural step. + bm.mark_dirty_cached(current_guid, crate::store::STRUCTURAL_SEQ, current_entry); bm.note_merge_candidate(current_guid); bm.note_spillover(); - compact_blob(guard).map_err(|e| e.with_blob_guid(current_guid))?; Ok(()) } @@ -893,12 +917,29 @@ fn lock_coupled_insert_in_blob( ); } Err(Error::Alloc(AllocError::OutOfSpace { .. })) => { - reclaim_or_spillover(bm, &mut guard, current_guid, seq, InsertAllocMiss::Space)?; - current_dirty = true; + reclaim_or_spillover( + bm, + &mut guard, + current_entry, + current_guid, + InsertAllocMiss::Space, + )?; + // Compaction/spillover is logically neutral. Publish it now so + // a later deep pin/read error cannot strand parent-scoped COW + // staging or leave an untracked rewritten edge. + bm.mark_dirty_cached(current_guid, crate::store::STRUCTURAL_SEQ, current_entry); + current_dirty = false; } Err(Error::Alloc(AllocError::OutOfSlots)) => { - reclaim_or_spillover(bm, &mut guard, current_guid, seq, InsertAllocMiss::Slots)?; - current_dirty = true; + reclaim_or_spillover( + bm, + &mut guard, + current_entry, + current_guid, + InsertAllocMiss::Slots, + )?; + bm.mark_dirty_cached(current_guid, crate::store::STRUCTURAL_SEQ, current_entry); + current_dirty = false; } Err(e) => return Err(e.with_blob_guid(current_guid)), } @@ -955,9 +996,14 @@ fn cross_and_insert( crossing.child_guid, child_guard.as_slice(), crossing.parent_off, - seq, )? { - parent_dirty = true; + // The fork is byte-identical and the edge repoint is logically + // neutral, so it is safe to publish immediately as STRUCTURAL_SEQ. + // Any recursive error can then return without leaking cow_pending; + // successful user mutation lowers the fork's dirty seq inside the + // recursive walker. + bm.mark_dirty_cached(current_guid, crate::store::STRUCTURAL_SEQ, current_entry); + parent_dirty = false; drop(child_guard); drop(child_pin); let fork_guard = fork_pin.write(); diff --git a/src/engine/walker/lookup.rs b/src/engine/walker/lookup.rs index f37d373..bafdfa2 100644 --- a/src/engine/walker/lookup.rs +++ b/src/engine/walker/lookup.rs @@ -103,15 +103,47 @@ pub fn lookup_multi_with( key: SearchKey<'_>, mut consume: F, ) -> Result> +where + F: FnMut(LookupHit<'_>) -> R, +{ + lookup_multi_with_gc_policy(bm, root_pin, route_cache, key, &mut consume, true) +} + +/// Snapshot/view lookup variant. Its root closure is pinned for the full +/// lease, so a missing descendant is stable corruption rather than a GC +/// race and must not be hidden behind an optimistic restart. +pub(crate) fn lookup_multi_with_snapshot( + bm: &BufferManager, + root_pin: &Arc, + route_cache: Option<&RouteCache>, + key: SearchKey<'_>, + mut consume: F, +) -> Result> +where + F: FnMut(LookupHit<'_>) -> R, +{ + lookup_multi_with_gc_policy(bm, root_pin, route_cache, key, &mut consume, false) +} + +fn lookup_multi_with_gc_policy( + bm: &BufferManager, + root_pin: &Arc, + route_cache: Option<&RouteCache>, + key: SearchKey<'_>, + consume: &mut F, + restart_on_gc: bool, +) -> Result> where F: FnMut(LookupHit<'_>) -> R, { // Outer loop: each iteration is one full attempt; we restart // here when an optimistic snapshot is invalidated. 'restart: loop { + let gc_epoch = restart_on_gc.then(|| bm.gc_read_epoch()); if let Some(cache) = route_cache { if let Some(route) = cache.lookup(key) { - match lookup_from_cached_route(bm, root_pin, cache, key, route, &mut consume)? { + match lookup_from_cached_route(bm, root_pin, cache, key, route, consume, gc_epoch)? + { RouteLookup::Done(result) => return Ok(result), RouteLookup::Stale => {} RouteLookup::Restart => { @@ -153,23 +185,31 @@ where bm.note_optimistic_restart(); continue 'restart; }; - let (child_pin, child_depth) = match indexed_lookup_or_pin(bm, key, crossing, &mut consume)? - { - IndexedLookupOrPin::Done(result) => return Ok(result), - IndexedLookupOrPin::Pin { pin, depth } => (pin, depth), - IndexedLookupOrPin::Restart => { - if let Some(cache) = route_cache { - cache.clear(); + let (child_pin, child_depth) = + match indexed_lookup_or_pin(bm, key, crossing, consume, gc_epoch)? { + IndexedLookupOrPin::Done(result) => return Ok(result), + IndexedLookupOrPin::Pin { pin, depth } => (pin, depth), + IndexedLookupOrPin::Restart => { + if let Some(cache) = route_cache { + cache.clear(); + } + bm.note_optimistic_restart(); + continue 'restart; } - bm.note_optimistic_restart(); - continue 'restart; - } - }; + }; // Cross-blob hops. Same pattern; on a torn read we restart // the whole walk from the root (the parent BlobNode that // pointed us here may also have moved). - match lookup_from_pinned_blob(bm, route_cache, key, child_pin, child_depth, &mut consume)? { + match lookup_from_pinned_blob( + bm, + route_cache, + key, + child_pin, + child_depth, + consume, + gc_epoch, + )? { CrossBlobLookup::Done(result) => return Ok(result), CrossBlobLookup::Restart => { bm.note_optimistic_restart(); @@ -196,6 +236,7 @@ fn lookup_from_cached_route( key: SearchKey<'_>, route: RouteHit, consume: &mut F, + gc_epoch: Option, ) -> Result> where F: FnMut(LookupHit<'_>) -> R, @@ -227,6 +268,7 @@ where child_depth: route.child_depth, }, consume, + gc_epoch, ) { Ok(IndexedLookupOrPin::Done(result)) => return Ok(RouteLookup::Done(result)), Ok(IndexedLookupOrPin::Pin { pin, .. }) => pin, @@ -272,15 +314,24 @@ where else { return Ok(RouteLookup::Restart); }; - let (next_pin, next_depth) = match indexed_lookup_or_pin(bm, key, crossing, consume)? { - IndexedLookupOrPin::Done(result) => return Ok(RouteLookup::Done(result)), - IndexedLookupOrPin::Pin { pin, depth } => (pin, depth), - IndexedLookupOrPin::Restart => { - cache.clear(); - return Ok(RouteLookup::Restart); - } - }; - match lookup_from_pinned_blob(bm, Some(cache), key, next_pin, next_depth, consume)? { + let (next_pin, next_depth) = + match indexed_lookup_or_pin(bm, key, crossing, consume, gc_epoch)? { + IndexedLookupOrPin::Done(result) => return Ok(RouteLookup::Done(result)), + IndexedLookupOrPin::Pin { pin, depth } => (pin, depth), + IndexedLookupOrPin::Restart => { + cache.clear(); + return Ok(RouteLookup::Restart); + } + }; + match lookup_from_pinned_blob( + bm, + Some(cache), + key, + next_pin, + next_depth, + consume, + gc_epoch, + )? { CrossBlobLookup::Done(result) => Ok(RouteLookup::Done(result)), CrossBlobLookup::Restart => Ok(RouteLookup::Restart), } @@ -294,6 +345,7 @@ fn lookup_from_pinned_blob( mut pin: Arc, mut depth: usize, consume: &mut F, + gc_epoch: Option, ) -> Result> where F: FnMut(LookupHit<'_>) -> R, @@ -323,7 +375,7 @@ where else { return Ok(CrossBlobLookup::Restart); }; - match indexed_lookup_or_pin(bm, key, crossing, consume)? { + match indexed_lookup_or_pin(bm, key, crossing, consume, gc_epoch)? { IndexedLookupOrPin::Done(result) => return Ok(CrossBlobLookup::Done(result)), IndexedLookupOrPin::Restart => { if let Some(cache) = route_cache { @@ -392,6 +444,7 @@ fn indexed_lookup_or_pin( key: SearchKey<'_>, crossing: BlobNodeCrossing, consume: &mut F, + gc_epoch: Option, ) -> Result> where F: FnMut(LookupHit<'_>) -> R, @@ -405,7 +458,10 @@ where }); } Ok(None) => {} - Err(e) if is_blob_store_not_found(&e) && bm.has_delete_fence(crossing.child_guid) => { + Err(e) + if is_blob_store_not_found(&e) + && missing_child_is_retryable(bm, crossing.child_guid, gc_epoch) => + { return Ok(IndexedLookupOrPin::Restart); } Err(e) => return Err(e), @@ -416,7 +472,10 @@ where if key.user_bytes().is_none() { let pin = match bm.pin(crossing.child_guid) { Ok(pin) => pin, - Err(e) if is_blob_store_not_found(&e) && bm.has_delete_fence(crossing.child_guid) => { + Err(e) + if is_blob_store_not_found(&e) + && missing_child_is_retryable(bm, crossing.child_guid, gc_epoch) => + { return Ok(IndexedLookupOrPin::Restart); } Err(e) => return Err(e), @@ -437,7 +496,8 @@ where let pin = match bm.pin(crossing.child_guid) { Ok(pin) => pin, Err(e) - if is_blob_store_not_found(&e) && bm.has_delete_fence(crossing.child_guid) => + if is_blob_store_not_found(&e) + && missing_child_is_retryable(bm, crossing.child_guid, gc_epoch) => { return Ok(IndexedLookupOrPin::Restart); } @@ -449,15 +509,36 @@ where depth: crossing.child_depth, }) } - IndexedBlobLookup::NotFound => Ok(IndexedLookupOrPin::Done(None)), + IndexedBlobLookup::NotFound => { + if gc_epoch.is_some_and(|captured| !bm.gc_epoch_still_stable(captured)) { + Ok(IndexedLookupOrPin::Restart) + } else { + Ok(IndexedLookupOrPin::Done(None)) + } + } IndexedBlobLookup::Found { value, seq } => { - let out = consume(LookupHit { value: &value, seq }); - Ok(IndexedLookupOrPin::Done(Some(out))) + let mut hit = || consume(LookupHit { value: &value, seq }); + if let Some(captured) = gc_epoch { + match bm.with_stable_gc_epoch(captured, hit) { + Some(out) => Ok(IndexedLookupOrPin::Done(Some(out))), + None => Ok(IndexedLookupOrPin::Restart), + } + } else { + Ok(IndexedLookupOrPin::Done(Some(hit()))) + } } IndexedBlobLookup::Crossing { .. } => Ok(IndexedLookupOrPin::Restart), } } +fn missing_child_is_retryable( + bm: &BufferManager, + child_guid: BlobGuid, + gc_epoch: Option, +) -> bool { + gc_epoch.is_some_and(|captured| bm.has_delete_fence(child_guid) || bm.gc_raced_since(captured)) +} + // ---------- indexed routed read ---------- const MAX_INDEXED_CHAIN_HOPS: usize = 8; @@ -1694,6 +1775,92 @@ mod tests { assert_eq!(stats.read_index_offset_hits, 0); } + #[test] + fn stale_gc_epoch_does_not_consume_indexed_hit() { + let dir = tempdir().unwrap(); + let guid = [0x49; 16]; + let value = vec![0xC4; 128]; + let mut bytes = AlignedBlobBuf::zeroed(); + { + BlobFrame::init(bytes.as_mut_slice(), guid).unwrap(); + let mut frame = BlobFrame::wrap(bytes.as_mut_slice()); + install_exact_leaf(&mut frame, b"consumer-key\0", &value, 21); + } + + let store = Arc::new(FileBlobStore::open(dir.path()).unwrap()); + let store_dyn: Arc = store.clone(); + { + let bm = BufferManager::new_file(store_dyn.clone(), 128, AlignedBlobBuf::zeroed); + bm.write_through_batch(&[WriteThroughEntry { + guid, + bytes, + expected_seq: 21, + content_version: None, + }]) + .unwrap(); + bm.flush_inner().unwrap(); + } + + let bm = BufferManager::new_file(store_dyn, 128, AlignedBlobBuf::zeroed); + let captured = bm.gc_read_epoch(); + bm.gc_sweep_unreachable(&std::collections::HashSet::from([guid])) + .unwrap(); + + let calls = AtomicUsize::new(0); + let mut consume = |hit: LookupHit<'_>| { + calls.fetch_add(1, Ordering::Relaxed); + hit.value.to_vec() + }; + let crossing = BlobNodeCrossing { + child_guid: guid, + child_depth: 0, + }; + assert!(matches!( + indexed_lookup_or_pin( + &bm, + SearchKey::user(b"consumer-key"), + crossing, + &mut consume, + Some(captured), + ) + .unwrap(), + IndexedLookupOrPin::Restart + )); + assert_eq!(calls.load(Ordering::Relaxed), 0); + + let stable = bm.gc_read_epoch(); + let result = indexed_lookup_or_pin( + &bm, + SearchKey::user(b"consumer-key"), + crossing, + &mut consume, + Some(stable), + ) + .unwrap(); + match result { + IndexedLookupOrPin::Done(Some(got)) => assert_eq!(got, value), + _ => panic!("stable indexed hit was not published"), + } + assert_eq!(calls.load(Ordering::Relaxed), 1); + } + + #[test] + fn stable_point_lookup_never_retries_missing_child_from_delete_fence() { + let child = [0x47; 16]; + let inner: Arc = Arc::new(MemoryBlobStore::new()); + let bm = BufferManager::new(inner, 4); + bm.mark_for_delete(child, 1); + + assert!( + !missing_child_is_retryable(&bm, child, None), + "stable point lookup must surface a missing referenced child", + ); + assert!( + missing_child_is_retryable(&bm, child, Some(bm.gc_read_epoch())), + "GC-fenced readers may restart while the delete fence is live", + ); + } + #[test] fn corrupt_value_segment_is_not_returned() { use std::fs::OpenOptions; diff --git a/src/engine/walker/merge.rs b/src/engine/walker/merge.rs index f9566d6..5386afe 100644 --- a/src/engine/walker/merge.rs +++ b/src/engine/walker/merge.rs @@ -14,7 +14,7 @@ use crate::layout::{BlobNode, NodeType, BLOB_MAX_INLINE}; use crate::store::{decode_child_off, encode_child_off, BlobFrame, BufferManager}; use super::cast; -use super::migrate::{is_mergeable, merge_blob}; +use super::migrate::{is_mergeable, merge_blob, PreparedBlobMerge}; use super::readers::{ child_offset, ntype_of, read_node16, read_node256, read_node4, read_node48, read_prefix, }; @@ -45,10 +45,10 @@ pub struct MergeStats { /// re-checked for their own mergeable descendants. (`is_mergeable` /// rejects any child with its own crossings via `num_ext_blobs`, /// so nested merges are deferred to a future pass.) -/// `seq` is forwarded to [`merge_blob`] so the deferred-delete -/// entry it generates carries the correct WAL stamp. Internal -/// callers (compact / checkpoint round) pass -/// [`crate::store::STRUCTURAL_SEQ`]. +/// `seq` selects the post-rewire reclamation protocol. User-WAL detachments +/// use deferred logical deletion; internal compact/checkpoint callers pass +/// [`crate::store::STRUCTURAL_SEQ`] and stage the exact child against its +/// rewritten parent until a clean durable frontier. pub fn try_merge_children( bm: &BufferManager, parent_frame: &mut BlobFrame<'_>, @@ -56,31 +56,53 @@ pub fn try_merge_children( ) -> Result { let mut stats = MergeStats::default(); let root_off = decode_child_off(parent_frame.header().root_slot); - let new_root = try_merge_subtree(bm, parent_frame, root_off, &mut stats, seq)?; - if new_root != root_off { - parent_frame.header_mut().root_slot = encode_child_off(new_root); + let merged_root = try_merge_subtree(bm, parent_frame, root_off, &mut stats, seq)?; + if merged_root.root_off == root_off { + debug_assert!(merged_root.prepared.is_none()); + } else { + #[cfg(test)] + fail_merge_rewire_if_armed()?; + parent_frame.header_mut().root_slot = encode_child_off(merged_root.root_off); + if let Some(prepared) = merged_root.prepared { + finalize_blob_merge(bm, parent_frame, prepared, seq); + } } Ok(stats) } +#[derive(Debug, Clone, Copy)] +struct SubtreeMerge { + root_off: u32, + prepared: Option, +} + +impl SubtreeMerge { + const fn unchanged(root_off: u32) -> Self { + Self { + root_off, + prepared: None, + } + } +} + fn try_merge_subtree( bm: &BufferManager, frame: &mut BlobFrame<'_>, off: u32, stats: &mut MergeStats, seq: u64, -) -> Result { +) -> Result { let ntype = ntype_of(frame.as_ref(), off)?; match ntype { NodeType::Invalid => Err(Error::node_corrupt( "try_merge_subtree: hit NodeType::Invalid", )), - NodeType::EmptyRoot | NodeType::Leaf => Ok(off), + NodeType::EmptyRoot | NodeType::Leaf => Ok(SubtreeMerge::unchanged(off)), NodeType::Prefix => merge_under_prefix(bm, frame, off, stats, seq), NodeType::Node4 | NodeType::Node16 | NodeType::Node48 | NodeType::Node256 => { merge_under_inner(bm, frame, off, ntype, stats, seq) } - NodeType::Blob => merge_at_blob_node(bm, frame, off, stats, seq), + NodeType::Blob => merge_at_blob_node(bm, frame, off, stats), } } @@ -90,14 +112,21 @@ fn merge_under_prefix( pfx_off: u32, stats: &mut MergeStats, seq: u64, -) -> Result { +) -> Result { let p = read_prefix(frame.as_ref(), pfx_off)?; let child_off = child_offset(p.child as u16); - let new_child = try_merge_subtree(bm, frame, child_off, stats, seq)?; - if new_child != child_off { - set_prefix_child(frame, pfx_off, new_child)?; + let merged_child = try_merge_subtree(bm, frame, child_off, stats, seq)?; + if merged_child.root_off == child_off { + debug_assert!(merged_child.prepared.is_none()); + } else { + #[cfg(test)] + fail_merge_rewire_if_armed()?; + set_prefix_child(frame, pfx_off, merged_child.root_off)?; + if let Some(prepared) = merged_child.prepared { + finalize_blob_merge(bm, frame, prepared, seq); + } } - Ok(pfx_off) + Ok(SubtreeMerge::unchanged(pfx_off)) } #[allow(clippy::too_many_lines)] // one match over 4 inner-node types @@ -108,7 +137,7 @@ fn merge_under_inner( ntype: NodeType, stats: &mut MergeStats, seq: u64, -) -> Result { +) -> Result { // Snapshot child (byte, child_off) pairs once — the inner-node body // stays at a fixed offset through the walk, so reading once + then // mutating via `inner_update_child` is safe. @@ -163,12 +192,19 @@ fn merge_under_inner( }; for (byte, child_off) in pairs { - let new_child = try_merge_subtree(bm, frame, child_off, stats, seq)?; - if new_child != child_off { - inner_update_child(frame, inner_off, ntype, byte, new_child)?; + let merged_child = try_merge_subtree(bm, frame, child_off, stats, seq)?; + if merged_child.root_off == child_off { + debug_assert!(merged_child.prepared.is_none()); + } else { + #[cfg(test)] + fail_merge_rewire_if_armed()?; + inner_update_child(frame, inner_off, ntype, byte, merged_child.root_off)?; + if let Some(prepared) = merged_child.prepared { + finalize_blob_merge(bm, frame, prepared, seq); + } } } - Ok(inner_off) + Ok(SubtreeMerge::unchanged(inner_off)) } fn merge_at_blob_node( @@ -176,8 +212,7 @@ fn merge_at_blob_node( frame: &mut BlobFrame<'_>, bn_off: u32, stats: &mut MergeStats, - seq: u64, -) -> Result { +) -> Result { // Defensive: confirm the body really is a BlobNode + check its // prefix_len fits. If `is_mergeable` returns false, the BlobNode // stays put. @@ -195,17 +230,70 @@ fn merge_at_blob_node( stats.inspected += 1; let mergeable = match is_mergeable(bm, frame, bn_off) { Ok(mergeable) => mergeable, - Err(e) if is_blob_store_not_found(&e) => return Ok(bn_off), + Err(e) if is_blob_store_not_found(&e) => return Ok(SubtreeMerge::unchanged(bn_off)), Err(e) => return Err(e), }; if !mergeable { - return Ok(bn_off); + return Ok(SubtreeMerge::unchanged(bn_off)); } - let new_off = match merge_blob(bm, frame, bn_off, seq) { - Ok(new_off) => new_off, - Err(e) if is_blob_store_not_found(&e) => return Ok(bn_off), + let prepared = match merge_blob(bm, frame, bn_off) { + Ok(prepared) => prepared, + Err(e) if is_blob_store_not_found(&e) => return Ok(SubtreeMerge::unchanged(bn_off)), Err(e) => return Err(e), }; stats.merged += 1; - Ok(new_off) + Ok(SubtreeMerge { + root_off: prepared.inlined_root, + prepared: Some(prepared), + }) +} + +/// Finalize a prepared merge only after the direct owning edge points at the +/// cloned subtree. From this point the old BlobNode is unreachable, so its +/// child can safely enter the deferred reclaim protocol. +fn finalize_blob_merge( + bm: &BufferManager, + parent_frame: &mut BlobFrame<'_>, + prepared: PreparedBlobMerge, + seq: u64, +) { + parent_frame.note_abandoned(prepared.parent_bn_off); + let parent_guid = { + let header = parent_frame.header_mut(); + header.num_ext_blobs = header.num_ext_blobs.saturating_sub(1); + header.blob_guid + }; + if seq == crate::store::STRUCTURAL_SEQ { + bm.stage_structural_reclaim(parent_guid, prepared.child_guid); + } else { + bm.mark_for_delete(prepared.child_guid, seq); + } + + #[cfg(feature = "tracing")] + tracing::debug!( + target: "holt::engine::merge", + child_guid = ?&prepared.child_guid[..4], + parent_bn_off = prepared.parent_bn_off, + inlined_root = prepared.inlined_root, + "merge_blob: rewired child into parent + staged reclaim", + ); +} + +#[cfg(test)] +std::thread_local! { + static FAIL_MERGE_REWIRE: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +#[cfg(test)] +pub(super) fn fail_next_merge_rewire() { + FAIL_MERGE_REWIRE.with(|armed| armed.set(true)); +} + +#[cfg(test)] +fn fail_merge_rewire_if_armed() -> Result<()> { + if FAIL_MERGE_REWIRE.with(|armed| armed.replace(false)) { + Err(Error::node_corrupt("failpoint: merge ancestor rewire")) + } else { + Ok(()) + } } diff --git a/src/engine/walker/migrate.rs b/src/engine/walker/migrate.rs index 41e1a81..6d22548 100644 --- a/src/engine/walker/migrate.rs +++ b/src/engine/walker/migrate.rs @@ -203,7 +203,9 @@ pub fn blob_would_route(frame: BlobFrameRef<'_>) -> bool { /// - `tombstone_leaf_cnt = 0` (every survivor is by definition live). /// - `compact_times` bumped by one. /// - `gap_space` reset to whatever fresh allocations report. -/// - Original `blob_guid` preserved. +/// - Original `blob_guid`, `created_epoch`, and `epoch_high_water` +/// preserved. Compaction changes the frame layout, not its snapshot +/// generation. /// - If every leaf in the source was tombstoned, the root becomes /// the freshly-allocated `EmptyRoot` sentinel. /// @@ -217,10 +219,16 @@ pub fn blob_would_route(frame: BlobFrameRef<'_>) -> bool { /// the heap, lives for the duration of the call) plus one full /// blob memcpy at the end. Roughly tens of µs on a modern machine. pub fn compact_blob(buf: &mut AlignedBlobBuf) -> Result<()> { - let (blob_guid, old_root_off, old_compact_times) = { + let (blob_guid, old_root_off, old_compact_times, old_created_epoch, old_epoch_high_water) = { let old_frame = BlobFrame::wrap(buf.as_mut_slice()); let h = old_frame.header(); - (h.blob_guid, decode_child_off(h.root_slot), h.compact_times) + ( + h.blob_guid, + decode_child_off(h.root_slot), + h.compact_times, + h.created_epoch, + h.epoch_high_water, + ) }; // Pass 0: measure the live subtree so the page-aligned leaf region @@ -240,6 +248,15 @@ pub fn compact_blob(buf: &mut AlignedBlobBuf) -> Result<()> { loop { let overran = { let mut new_frame = BlobFrame::init(new_buf.as_mut_slice(), blob_guid)?; + // `init` deliberately starts a brand-new frame at epoch zero. + // This scratch image is instead a physical rewrite of the same + // logical version, so retain both generation fields before either + // the routed build or its legacy fallback becomes visible. + { + let h = new_frame.header_mut(); + h.created_epoch = old_created_epoch; + h.epoch_high_water = old_epoch_high_water; + } let old_frame = BlobFrame::wrap(buf.as_mut_slice()); let mut leaf_cursor = lrs_opt.unwrap_or(0); let cloned = clone_subtree( @@ -379,16 +396,29 @@ pub fn is_mergeable( Ok(space_ok && slots_ok && no_grandchild && no_tombstones) } -/// Inline a child blob's subtree back into its parent, replacing -/// the cross-blob `BlobNode` crossing with the child's contents. +/// Prepared result of cloning a mergeable child into its parent frame. +/// +/// The old `BlobNode` and child remain live until the direct owner of that +/// edge has successfully rewired its pointer to `inlined_root`. This keeps a +/// failed ancestor rewrite from handing a still-reachable child to GC. +#[derive(Debug, Clone, Copy)] +pub(super) struct PreparedBlobMerge { + pub(super) inlined_root: u32, + pub(super) parent_bn_off: u32, + pub(super) child_guid: BlobGuid, +} + +/// Clone a child blob's subtree into its parent in preparation for replacing +/// the cross-blob `BlobNode` crossing with the cloned contents. /// /// Reads the child via an exclusive guard, deep-clones the child's /// entry-point subtree into `parent_frame` (preserve-mode — caller /// should compact the child first if dropping tombstones matters), -/// optionally wraps the cloned root in the `BlobNode`'s inline -/// prefix, frees the parent's `BlobNode` slot, and drops the child -/// blob from the BM. Returns the slot in `parent_frame` where the -/// inlined subtree's root now lives. +/// optionally wraps the cloned root in the `BlobNode`'s inline prefix, and +/// returns the prepared replacement. It deliberately does not abandon the +/// old `BlobNode`, update external-blob accounting, or queue the child for +/// reclamation. The direct edge owner performs those actions only after its +/// pointer rewrite succeeds. /// /// **The caller's responsibility**: rewire the grandparent's /// pointer to the returned slot. Typical pattern: a recursive @@ -401,24 +431,22 @@ pub fn is_mergeable( /// `true` before this is called. Calling without that check risks /// `OutOfSpace` mid-clone on a too-big merge or wasted work on a /// merge that violates the no-nested-crossings precondition. -/// `seq` is stamped on the deferred-delete entry the merged -/// child generates. Callers from a user op path should pass the -/// op's WAL seq so the W2D protocol can pair the manifest delete -/// with a real WAL record. Internal callers (compact, the -/// checkpoint round's merge pass) pass -/// [`crate::store::STRUCTURAL_SEQ`] — the merge -/// has no WAL record and shouldn't pin the trim watermark. -pub fn merge_blob( +pub(super) fn merge_blob( bm: &BufferManager, parent_frame: &mut BlobFrame<'_>, parent_bn_off: u32, - seq: u64, -) -> Result { +) -> Result { let bn = read_blob_node(parent_frame, parent_bn_off)?; let child_guid = bn.child_blob_guid; let plen = (bn.prefix_len as usize).min(BLOB_MAX_INLINE); let prefix_bytes: Vec = bn.bytes[..plen].to_vec(); + // Any appended clone uses the legacy allocation cursor. De-route before + // the first fallible child read/allocation so an error path that publishes + // the partial frame can never persist a routed header over appended legacy + // nodes. Failure leaves only a safe full-pin layout plus space debt. + parent_frame.header_mut().routing_len = 0; + let new_subtree_root = { let child_pin = bm.pin(child_guid)?; let mut child_guard = child_pin.write(); @@ -444,40 +472,11 @@ pub fn merge_blob( write_prefix_chain(parent_frame, &prefix_bytes, new_subtree_root)? }; - // Abandon-on-free: the parent's BlobNode is unreachable now that - // the grandparent will be repointed at `inlined_root`. - parent_frame.note_abandoned(parent_bn_off); - // Keep external-blob accounting correct — the BlobNode is gone. - { - let h = parent_frame.header_mut(); - h.num_ext_blobs = h.num_ext_blobs.saturating_sub(1); - // A merge appends the cloned subtree's internals + leaves via the - // single `space_used` cursor, so any prior routing layout on the - // parent is now stale (internals would sit above a stale - // `leaf_region_start`). Demote to legacy/full-pin; the next - // compaction re-routes the parent. - h.routing_len = 0; - } - // Hand the now-orphaned child blob to the deferred-delete - // protocol. An inline `bm.delete_blob` here would push the - // manifest mutation to in-memory before the caller's WAL - // flush (or, for internal callers like compact / merge_pass, - // before the next checkpoint round's Sync); on crash that - // would leave the parent in cache pointing at no child - // while manifest persistence raced ahead through any - // unrelated `store.flush`. - bm.mark_for_delete(child_guid, seq); - - #[cfg(feature = "tracing")] - tracing::debug!( - target: "holt::engine::merge", - child_guid = ?&child_guid[..4], - parent_bn_off = parent_bn_off, - inlined_root = inlined_root, - "merge_blob: folded child into parent + queued delete", - ); - - Ok(inlined_root) + Ok(PreparedBlobMerge { + inlined_root, + parent_bn_off, + child_guid, + }) } fn read_blob_node(frame: &BlobFrame<'_>, off: u32) -> Result { diff --git a/src/engine/walker/mod.rs b/src/engine/walker/mod.rs index 312832b..dbbcbbf 100644 --- a/src/engine/walker/mod.rs +++ b/src/engine/walker/mod.rs @@ -66,6 +66,7 @@ pub use insert::{insert_multi, insert_multi_conditional}; pub(crate) use insert::{insert_multi_batch_conditional, InsertBatchItem}; pub(crate) use key::SearchKey; pub use lookup::lookup_multi_with; +pub(crate) use lookup::lookup_multi_with_snapshot; pub use merge::try_merge_children; pub use migrate::{blob_needs_compaction, blob_would_route, compact_blob}; pub(crate) use range::PrefixListCache; diff --git a/src/engine/walker/range.rs b/src/engine/walker/range.rs index e7981ad..8c6105b 100644 --- a/src/engine/walker/range.rs +++ b/src/engine/walker/range.rs @@ -30,7 +30,8 @@ use crate::api::errors::{is_blob_store_not_found, Error, Result}; use crate::concurrency::{CommitGate, Gate}; use crate::layout::{BlobGuid, BlobNode, Leaf, NodeType, BLOB_MAX_INLINE, PREFIX_MAX_INLINE}; use crate::store::{ - decode_child_off, BlobFrameRef, BufferManager, CachedBlob, PrefixLiveness, WriteDeltaKeyState, + decode_child_off, BlobFrameRef, BufferManager, CachedBlob, PrefixLiveness, SnapshotLease, + WriteDeltaKeyState, }; use smallvec::SmallVec; @@ -358,6 +359,7 @@ impl ProjectedRangeEntry { pub struct RangeBuilder { bm: Arc, root_pin: Arc, + snapshot_lease: Option>, root_guid: BlobGuid, tree_id: Option, maintenance_gate: Arc, @@ -365,7 +367,6 @@ pub struct RangeBuilder { commit_gate: Option>, dropped: Option>, preflight_error: Option, - snapshot_cursor: bool, prefix: KeyBuf, start_after: Option, delimiter: Option, @@ -385,6 +386,7 @@ impl RangeBuilder { Self { bm, root_pin, + snapshot_lease: None, root_guid, tree_id: None, maintenance_gate, @@ -392,7 +394,6 @@ impl RangeBuilder { commit_gate: None, dropped: None, preflight_error: None, - snapshot_cursor: false, prefix: KeyBuf::new(), start_after: None, delimiter: None, @@ -404,6 +405,11 @@ impl RangeBuilder { self } + pub(crate) fn with_snapshot_lease(mut self, lease: Arc) -> Self { + self.snapshot_lease = Some(lease); + self + } + pub(crate) fn with_mutation_gate(mut self, mutation_gate: Arc) -> Self { self.mutation_gate = Some(mutation_gate); self @@ -424,11 +430,6 @@ impl RangeBuilder { self } - pub(crate) fn snapshot_cursor(mut self) -> Self { - self.snapshot_cursor = true; - self - } - fn ensure_live(&self) -> Result<()> { if self .dropped @@ -500,15 +501,19 @@ impl IntoIterator for RangeBuilder { impl RangeBuilder { fn into_iter_with_projection(self, projection: RangeProjection) -> RangeIter { + let gc_epoch = self.bm.gc_stable_read_epoch(); + let missing_child_is_corruption = self.snapshot_lease.is_some(); RangeIter { bm: self.bm, root_pin: self.root_pin, + _snapshot_lease: self.snapshot_lease, root_guid: self.root_guid, maintenance_gate: self.maintenance_gate, mutation_gate: self.mutation_gate, dropped: self.dropped, preflight_error: self.preflight_error, - snapshot_cursor: self.snapshot_cursor, + missing_child_is_corruption, + gc_epoch, stack: Vec::with_capacity(8), curr_key: Vec::with_capacity(self.prefix.len().saturating_add(64)), emit_buf: Vec::with_capacity(self.prefix.len().saturating_add(64)), @@ -840,7 +845,12 @@ pub struct RangeIter { mutation_gate: Option>, dropped: Option>, preflight_error: Option, - snapshot_cursor: bool, + /// Snapshot leases make a missing descendant stable corruption. Live + /// cursors may instead restart across a concurrent physical-GC epoch. + missing_child_is_corruption: bool, + /// Physical-GC sequence captured for this live cursor attempt. + /// Snapshot cursors keep strict missing-child semantics instead. + gc_epoch: u64, /// Descent stack. Empty = no init done (if `!initialized`) or /// exhausted (if `terminated`). stack: Vec, @@ -874,6 +884,9 @@ pub struct RangeIter { /// until consumed. Correctness-neutral — a pre-pinned blob is the /// same blob, re-validated by the per-frame `version` check on use. prefetch: HashMap>, + // Declared after every pin-bearing field so stack/prefetch/root drop + // first. The final lease drop can then evict the ephemeral snapshot root. + _snapshot_lease: Option>, } struct Frame { @@ -1158,6 +1171,16 @@ enum InitResult { Restart, } +enum ChildPin { + Pinned(Arc), + Restart, +} + +enum ChildPush { + Pushed, + Restart, +} + enum RangeAdvance { Entry(ProjectedRangeEntry), KeyRef(KeyRefKind), @@ -1229,6 +1252,13 @@ impl RangeIter { self.stats } + /// Advance without entering `maintenance_gate`. + /// Caller must already hold the tree's maintenance guard. + pub(crate) fn next_unlocked(&mut self) -> Option> { + self.next_projected_maybe_guarded(false) + .map(|entry| entry.map(ProjectedRangeEntry::into_record)) + } + fn next_projected_maybe_guarded( &mut self, enter_gate: bool, @@ -1270,16 +1300,39 @@ impl RangeIter { if let Some(error) = self.preflight_error.take() { return Err(error); } + if !self.missing_child_is_corruption { + let current = self.bm.gc_stable_read_epoch(); + if current != self.gc_epoch { + self.gc_epoch = current; + return Ok(RangeAdvance::Restart); + } + } if !self.initialized { match self.init_descent()? { InitResult::Ready => { self.initialized = true; } - InitResult::Empty => return Ok(RangeAdvance::Done), + InitResult::Empty => { + return if !self.missing_child_is_corruption + && self.bm.gc_raced_since(self.gc_epoch) + { + Ok(RangeAdvance::Restart) + } else { + Ok(RangeAdvance::Done) + }; + } InitResult::Restart => return Ok(RangeAdvance::Restart), } } - self.advance_to_next_entry() + let advance = self.advance_to_next_entry()?; + if matches!(advance, RangeAdvance::Done) + && !self.missing_child_is_corruption + && self.bm.gc_raced_since(self.gc_epoch) + { + Ok(RangeAdvance::Restart) + } else { + Ok(advance) + } } fn ensure_live(&self) -> Result<()> { @@ -1664,7 +1717,7 @@ impl RangeIter { let is_candidate = { let top = &self.stack[idx]; let guard = top.pin.read(); - if !self.frame_content_still_valid(top) { + if !Self::frame_content_still_valid(top) { return Ok(InitResult::Restart); } let frame = BlobFrameRef::wrap(guard.as_slice()); @@ -1697,7 +1750,7 @@ impl RangeIter { let top = &self.stack[idx]; let guard = top_pin.read(); let version = top_pin.content_version(); - if !self.frame_version_still_valid(top, version) { + if !Self::frame_version_still_valid(top, version) { return Ok(InitResult::Restart); } let frame = BlobFrameRef::wrap(guard.as_slice()); @@ -1746,7 +1799,7 @@ impl RangeIter { let top = &self.stack[idx]; let guard = top.pin.read(); let version = top.pin.content_version(); - if !self.frame_version_still_valid(top, version) { + if !Self::frame_version_still_valid(top, version) { return Ok(InitResult::Restart); } let frame = BlobFrameRef::wrap(guard.as_slice()); @@ -1803,8 +1856,11 @@ impl RangeIter { continue; } self.stack[idx].next = 1; - if !self.push_in_other_blob(child_guid, p_bytes.as_slice())? { - self.pop_frame(); + if matches!( + self.push_in_other_blob(child_guid, p_bytes.as_slice())?, + ChildPush::Restart + ) { + return Ok(InitResult::Restart); } } } @@ -1830,7 +1886,7 @@ impl RangeIter { let top = &self.stack[idx]; let guard = top_pin.read(); let version = top_pin.content_version(); - if !self.frame_version_still_valid(top, version) { + if !Self::frame_version_still_valid(top, version) { return Ok(InitResult::Restart); } let frame = BlobFrameRef::wrap(guard.as_slice()); @@ -1893,7 +1949,7 @@ impl RangeIter { let top = &self.stack[idx]; let guard = top.pin.read(); let version = top.pin.content_version(); - if !self.frame_version_still_valid(top, version) { + if !Self::frame_version_still_valid(top, version) { return Ok(RangeAdvance::Restart); } subtree_has_live_leaf(BlobFrameRef::wrap(guard.as_slice()), top.off)? @@ -1925,7 +1981,7 @@ impl RangeIter { let kv = { let top = &self.stack[idx]; let guard = top.pin.read(); - if !self.frame_content_still_valid(top) { + if !Self::frame_content_still_valid(top) { return Ok(RangeAdvance::Restart); } let frame = BlobFrameRef::wrap(guard.as_slice()); @@ -2070,7 +2126,7 @@ impl RangeIter { let top = &self.stack[idx]; let guard = top_pin.read(); let version = top_pin.content_version(); - if !self.frame_version_still_valid(top, version) { + if !Self::frame_version_still_valid(top, version) { return Ok(RangeAdvance::Restart); } let frame = BlobFrameRef::wrap(guard.as_slice()); @@ -2130,7 +2186,7 @@ impl RangeIter { let top = &self.stack[idx]; let guard = top.pin.read(); let version = top.pin.content_version(); - if !self.frame_version_still_valid(top, version) { + if !Self::frame_version_still_valid(top, version) { return Ok(RangeAdvance::Restart); } let frame = BlobFrameRef::wrap(guard.as_slice()); @@ -2196,9 +2252,9 @@ impl RangeIter { } } } - let Some(child_pin) = self.pin_scan_child(child_guid)? else { - self.pop_frame(); - continue; + let child_pin = match self.pin_scan_child(child_guid)? { + ChildPin::Pinned(pin) => pin, + ChildPin::Restart => return Ok(RangeAdvance::Restart), }; child_pin.prefetch_header(); if can_rollup { @@ -2237,7 +2293,7 @@ impl RangeIter { let top = &self.stack[idx]; let guard = top_pin.read(); let version = top_pin.content_version(); - if !self.frame_version_still_valid(top, version) { + if !Self::frame_version_still_valid(top, version) { return Ok(RangeAdvance::Restart); } let frame = BlobFrameRef::wrap(guard.as_slice()); @@ -2282,7 +2338,7 @@ impl RangeIter { // delimiter rollup; the iterator can emit the // CommonPrefix from the parent plus read-index // liveness and avoid the child blob read entirely. - let prefetch_guids = if !self.snapshot_cursor + let prefetch_guids = if !self.missing_child_is_corruption && child_ntype == NodeType::Blob && !child_blob_rollup && self.prefetch.is_empty() @@ -2387,23 +2443,38 @@ impl RangeIter { }); } - fn push_in_other_blob(&mut self, child_guid: BlobGuid, prefix_bytes: &[u8]) -> Result { - let Some(child_pin) = self.pin_scan_child(child_guid)? else { - return Ok(false); + fn push_in_other_blob( + &mut self, + child_guid: BlobGuid, + prefix_bytes: &[u8], + ) -> Result { + let child_pin = match self.pin_scan_child(child_guid)? { + ChildPin::Pinned(pin) => pin, + ChildPin::Restart => return Ok(ChildPush::Restart), }; child_pin.prefetch_header(); self.push_pinned_other_blob(child_pin, child_guid, prefix_bytes)?; - Ok(true) + Ok(ChildPush::Pushed) } - fn pin_scan_child(&mut self, child_guid: BlobGuid) -> Result>> { + fn pin_scan_child(&mut self, child_guid: BlobGuid) -> Result { // Consume a read-ahead pin if one was warmed for this child. if let Some(pin) = self.prefetch.remove(&child_guid) { - return Ok(Some(pin)); + return Ok(ChildPin::Pinned(pin)); } match self.bm.pin_scan(child_guid) { - Ok(pin) => Ok(Some(pin)), - Err(e) if is_blob_store_not_found(&e) => Ok(None), + Ok(pin) => Ok(ChildPin::Pinned(pin)), + Err(e) + if is_blob_store_not_found(&e) + && missing_scan_child_is_retryable( + &self.bm, + self.missing_child_is_corruption, + self.gc_epoch, + child_guid, + ) => + { + Ok(ChildPin::Restart) + } Err(e) => Err(e), } } @@ -2435,24 +2506,22 @@ impl RangeIter { } fn path_is_still_valid(&self) -> bool { - if self.snapshot_cursor { - return true; - } self.stack .iter() .all(|frame| frame.pin.validate_content_version(frame.version)) } - fn frame_content_still_valid(&self, frame: &Frame) -> bool { - self.snapshot_cursor || frame.pin.content_version() == frame.version + fn frame_content_still_valid(frame: &Frame) -> bool { + frame.pin.content_version() == frame.version } - fn frame_version_still_valid(&self, frame: &Frame, current_version: u64) -> bool { - self.snapshot_cursor || current_version == frame.version + fn frame_version_still_valid(frame: &Frame, current_version: u64) -> bool { + current_version == frame.version } fn restart_cursor(&mut self) { self.bm.note_range_restart(); + self.gc_epoch = self.bm.gc_stable_read_epoch(); self.stats.restarts += 1; self.stack.clear(); self.curr_key.clear(); @@ -2478,6 +2547,15 @@ impl RangeIter { } } +fn missing_scan_child_is_retryable( + bm: &BufferManager, + missing_child_is_corruption: bool, + gc_epoch: u64, + child_guid: BlobGuid, +) -> bool { + !missing_child_is_corruption && (bm.has_delete_fence(child_guid) || bm.gc_raced_since(gc_epoch)) +} + fn path_seek_relation(path: &[u8], target: &[u8]) -> SeekRelation { let limit = path.len().min(target.len()); let common = simd::longest_common_prefix(path, target); @@ -2733,3 +2811,27 @@ fn next_inner_child_from( )), } } + +#[cfg(test)] +mod gc_tests { + use super::*; + use crate::store::blob_store::{BlobStore, MemoryBlobStore}; + + #[test] + fn live_cursor_restarts_on_delete_fence_without_gc_epoch_change() { + let store: Arc = Arc::new(MemoryBlobStore::new()); + let bm = BufferManager::new(store, 4); + let child_guid = [0xD1; 16]; + let gc_epoch = bm.gc_read_epoch(); + bm.mark_for_delete(child_guid, 7); + + assert_eq!(bm.gc_read_epoch(), gc_epoch); + assert!(missing_scan_child_is_retryable( + &bm, false, gc_epoch, child_guid + )); + assert!( + !missing_scan_child_is_retryable(&bm, true, gc_epoch, child_guid), + "stable snapshot cursors must not hide a genuinely missing descendant" + ); + } +} diff --git a/src/engine/walker/spillover.rs b/src/engine/walker/spillover.rs index 69c91b3..e2d4e39 100644 --- a/src/engine/walker/spillover.rs +++ b/src/engine/walker/spillover.rs @@ -14,7 +14,7 @@ use crate::layout::{ leaf_body_size, size_of_node, BlobGuid, BlobNode, Node16, Node256, Node4, Node48, NodeType, Prefix, DATA_AREA_START, PAGE_SIZE, }; -use crate::store::{decode_child_off, encode_child_off, BlobFrame, BufferManager}; +use crate::store::{decode_child_off, encode_child_off, BlobFrame, BlobFrameRef, BufferManager}; use super::super::simd; use super::cast; @@ -33,6 +33,11 @@ pub(super) use super::migrate::compact_blob; const SPILLOVER_TARGET_CHILD_FILL_PCT: u32 = 70; const SPILLOVER_MIN_CHILD_FILL_PCT: u32 = 35; +#[cfg(test)] +std::thread_local! { + static FAIL_PREPARED_PARENT_COMPACT: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + #[derive(Debug, Clone, Copy)] struct SubtreeFootprint { nodes: u32, @@ -78,11 +83,9 @@ fn spillover_min_child_bytes() -> u32 { /// key scale instead of repeatedly peeling off the largest branch /// into near-full child blobs. /// -/// Returns the BlobNode slot installed in `frame` so callers / -/// tests can verify. The new blob lives in the BM cache + dirty -/// map; its store write happens during the next checkpoint round -/// (after the WAL record for the spillover-triggering op is -/// durable — invariant W2D). +/// The new blob lives in the BM cache + dirty map; its store write happens +/// during the next checkpoint round (after the WAL record for the +/// spillover-triggering op is durable — invariant W2D). /// /// `seq` is the WAL seq the caller pre-allocated for the op that /// triggered spillover (insert / rename / batched put). The new @@ -93,55 +96,83 @@ pub(super) fn spillover_blob( bm: &BufferManager, frame: &mut BlobFrame<'_>, seq: u64, -) -> Result { - let root_off = decode_child_off(frame.header().root_slot); - let victim = pick_victim_subtree(frame, root_off)?; +) -> Result<()> { + // Prepare the complete parent rewrite in scratch. Once the child enters + // the BM dirty set there must be no fallible operation left that could + // strand it unreachable or leave a half-abandoned parent in cache. + let mut prepared_buf = bm.alloc_blob_buf_zeroed(); + frame.copy_bytes_to(prepared_buf.as_mut_slice()); + let mut prepared = BlobFrame::wrap(prepared_buf.as_mut_slice()); + let root_off = decode_child_off(prepared.header().root_slot); + let victim = pick_victim_subtree(&prepared, root_off)?; let new_guid = fresh_blob_guid(); - let outcome = make_blob_from_node_in(bm, frame, victim.victim_off, new_guid)?; - - // Stage the new blob via the unified `mark_dirty → checkpoint - // round` protocol — the bytes stay in cache until the round - // flushes WAL **first** and then writes them through. An - // inline `bm.write_blob(new_guid, ...) + bm.flush()` here - // would violate invariant W2D: a crash between the inline - // write and the user's WAL flush would leave an orphan in - // store AND the parent's BlobNode staged only in cache — - // and a racing checkpointer could flush the parent's - // BlobNode before the user's WAL record was durable, - // leaving the on-disk parent pointing at the pre-spillover - // orphan position. - bm.install_new_blob(new_guid, outcome.buf, seq); + let outcome = make_blob_from_node_in(bm, &prepared, victim.victim_off, new_guid)?; + if BlobFrameRef::wrap(outcome.buf.as_slice()) + .header() + .blob_guid + != new_guid + { + return Err(Error::node_corrupt( + "spillover: prepared child self-GUID mismatch", + )); + } // Abandon the migrated subtree's nodes in the source blob (they've // been copied to the child); reclaimed at next compaction. Bump // dead-bytes so the source still triggers compaction. - abandon_subtree(frame, victim.victim_off)?; + abandon_subtree(&mut prepared, victim.victim_off)?; // Allocate a BlobNode pointing at the child blob. The child // blob's own header.root_slot is the only entry slot. - let bn_alloc = frame.alloc_node(NodeType::Blob)?; - let bn_off = frame + let bn_alloc = prepared.alloc_node(NodeType::Blob)?; + let bn_off = prepared .offset_of_slot(bn_alloc.slot) .ok_or(Error::node_corrupt("spillover: BlobNode offset"))?; let bn = BlobNode::new(&[], new_guid); - write_struct_at(frame, bn_off, &bn)?; + write_struct_at(&mut prepared, bn_off, &bn)?; // Wire the parent of the migrated subtree to point at the new // BlobNode instead of the now-abandoned victim subtree. if victim.parent_off == root_off && victim.via_header_root { - frame.header_mut().root_slot = encode_child_off(bn_off); + prepared.header_mut().root_slot = encode_child_off(bn_off); } else { match victim.kind { VictimEdgeKind::Prefix => { - set_prefix_child(frame, victim.parent_off, bn_off)?; + set_prefix_child(&mut prepared, victim.parent_off, bn_off)?; } VictimEdgeKind::Inner(parent_ntype) => { - inner_update_child(frame, victim.parent_off, parent_ntype, victim.byte, bn_off)?; + inner_update_child( + &mut prepared, + victim.parent_off, + parent_ntype, + victim.byte, + bn_off, + )?; } } } + // Repack the prepared parent before either image becomes visible. This + // is the last fallible structural step and reclaims the abandoned bytes + // needed by the insert retry. A compaction error therefore leaves both + // the live parent and the BM dirty set unchanged. + #[cfg(test)] + if FAIL_PREPARED_PARENT_COMPACT.with(|armed| armed.replace(false)) { + return Err(Error::Internal( + "failpoint: prepared spillover parent compaction", + )); + } + compact_blob(&mut prepared_buf)?; + + // Stage the new blob via the unified `mark_dirty → checkpoint + // round` protocol — the bytes stay in cache until the round + // flushes WAL **first** and then writes them through. Every fallible + // parent transformation above has succeeded, so the only remaining + // parent publication is a fixed-size memcpy. + bm.install_new_blob(new_guid, outcome.buf, seq); + frame.replace_bytes_from(prepared_buf.as_slice()); + #[cfg(feature = "tracing")] tracing::debug!( target: "holt::engine::spillover", @@ -151,7 +182,7 @@ pub(super) fn spillover_blob( "spillover: migrated subtree to fresh child blob", ); - Ok(bn_off) + Ok(()) } /// Memo for per-`pick_victim` subtree footprints, keyed by a node's @@ -708,7 +739,9 @@ mod tests { use super::super::insert::insert; use super::*; use crate::layout::PAGE_SIZE; - use crate::store::BlobFrame; + use crate::store::blob_store::{AlignedBlobBuf, BlobStore, MemoryBlobStore}; + use crate::store::{BlobFrame, BufferManager}; + use std::sync::Arc; fn candidate(bytes: u32, nodes: u32) -> VictimCandidate { candidate_with_boundary(bytes, nodes, BoundaryQuality::Arbitrary, 16) @@ -833,4 +866,61 @@ mod tests { ); assert_eq!(victim.byte, b'x'); } + + fn spillable_frame() -> AlignedBlobBuf { + let mut buf = AlignedBlobBuf::zeroed(); + BlobFrame::init(buf.as_mut_slice(), [0x41; 16]).unwrap(); + let mut frame = BlobFrame::wrap(buf.as_mut_slice()); + let value = vec![0x6B; 1024]; + for i in 0..360u32 { + put( + &mut frame, + format!("spill/two-phase/{i:06}").as_bytes(), + &value, + u64::from(i) + 1, + ); + } + buf + } + + #[test] + fn prepared_spillover_failure_does_not_install_or_rewrite() { + let inner: Arc = Arc::new(MemoryBlobStore::new()); + let bm = BufferManager::new(inner.clone(), 8); + let mut buf = spillable_frame(); + let before = buf.as_slice().to_vec(); + FAIL_PREPARED_PARENT_COMPACT.with(|armed| armed.set(true)); + + let error = { + let mut frame = BlobFrame::wrap(buf.as_mut_slice()); + spillover_blob(&bm, &mut frame, crate::store::STRUCTURAL_SEQ).unwrap_err() + }; + assert!(matches!(error, Error::Internal(_))); + assert_eq!(buf.as_slice(), before); + assert_eq!(bm.dirty_count(), 0); + assert!(inner.list_blobs().unwrap().is_empty()); + } + + #[test] + fn prepared_spillover_installs_self_identifying_reachable_child() { + let inner: Arc = Arc::new(MemoryBlobStore::new()); + let bm = BufferManager::new(inner, 8); + let mut buf = spillable_frame(); + { + let mut frame = BlobFrame::wrap(buf.as_mut_slice()); + spillover_blob(&bm, &mut frame, crate::store::STRUCTURAL_SEQ).unwrap(); + } + let children = super::super::scan::collect_blob_children_from_frame(BlobFrameRef::wrap( + buf.as_slice(), + )) + .unwrap(); + assert_eq!(children.len(), 1); + let child = bm.pin(children[0]).unwrap(); + assert_eq!( + BlobFrameRef::wrap(child.read().as_slice()) + .header() + .blob_guid, + children[0], + ); + } } diff --git a/src/engine/walker/tests.rs b/src/engine/walker/tests.rs index f326e8b..096cc4b 100644 --- a/src/engine/walker/tests.rs +++ b/src/engine/walker/tests.rs @@ -3,21 +3,25 @@ //! BlobNode descent) + `compact_blob`. use super::cast; +use super::cow::fork_child_if_shared; use super::erase::erase; use super::insert::insert; -use super::lookup::{indexed_read_into, lookup, lookup_at}; +use super::lookup::{indexed_read_into, lookup, lookup_at, lookup_multi_with}; +use super::merge::{fail_next_merge_rewire, try_merge_children}; use super::migrate::{blob_needs_compaction, blob_would_route, compact_blob, make_blob_from_node}; use super::readers::{ child_offset, read_node16, read_node256, read_node4, read_node48, read_prefix, }; use super::types::LookupResult; +use super::writers::write_struct_at; use super::SearchKey; use crate::api::errors::Error; use crate::layout::{BlobGuid, BlobNode, NodeType, DATA_AREA_START, PAGE_SIZE}; -use crate::store::blob_store::AlignedBlobBuf; -use crate::store::{decode_child_off, page_align_up, BlobFrame}; +use crate::store::blob_store::{AlignedBlobBuf, BlobStore, MemoryBlobStore}; +use crate::store::{decode_child_off, page_align_up, BlobFrame, BlobFrameRef, BufferManager}; use proptest::prelude::*; use std::collections::BTreeMap; +use std::sync::Arc; /// Decode `header.root_slot` (the encoded root offset) into the root /// node body's absolute byte offset. @@ -82,6 +86,197 @@ fn replace_root_with_blob_node(buf: &mut AlignedBlobBuf, child_guid: BlobGuid) { frame.header_mut().root_slot = crate::store::encode_child_off(off); } +#[test] +fn cow_validates_parent_edge_before_allocating_fork() { + let parent_guid = [0xC1; 16]; + let child_guid = [0xC2; 16]; + let mut parent_buf = AlignedBlobBuf::zeroed(); + BlobFrame::init(parent_buf.as_mut_slice(), parent_guid).unwrap(); + replace_root_with_blob_node(&mut parent_buf, child_guid); + let mut child_buf = AlignedBlobBuf::zeroed(); + BlobFrame::init(child_buf.as_mut_slice(), child_guid).unwrap(); + + let inner: Arc = Arc::new(MemoryBlobStore::new()); + inner.write_blob(parent_guid, &parent_buf).unwrap(); + inner.write_blob(child_guid, &child_buf).unwrap(); + let bm = Arc::new(BufferManager::new(Arc::clone(&inner), 8)); + let parent = bm.pin(parent_guid).unwrap(); + let child = bm.pin(child_guid).unwrap(); + let epoch = bm.register_snapshot(parent_guid, &parent).unwrap(); + let dirty_before = bm.dirty_count(); + + let mut parent_guard = parent.write(); + let child_guard = child.read(); + let Err(error) = fork_child_if_shared( + &bm, + &mut parent_guard, + child_guid, + child_guard.as_slice(), + crate::layout::PAGE_SIZE - 1, + ) else { + panic!("invalid parent edge allocated a COW fork"); + }; + assert!(matches!(error, Error::NodeCorrupt { .. })); + assert_eq!(bm.dirty_count(), dirty_before); + assert_eq!(bm.orphan_staging_count(), 0); + assert_eq!(inner.list_blobs().unwrap().len(), 2); + drop(child_guard); + drop(parent_guard); + bm.retire_snapshot(epoch); +} + +#[test] +#[allow(clippy::too_many_lines)] +fn failed_merge_ancestor_rewire_never_hands_child_to_reclaim() { + let parent_guid = [0xD1; 16]; + let child_guid = [0xD2; 16]; + + let mut child_buf = AlignedBlobBuf::zeroed(); + { + let mut child = BlobFrame::init(child_buf.as_mut_slice(), child_guid).unwrap(); + let root = child.header().root_slot; + insert(&mut child, root, b"pkey", b"value", 1).unwrap(); + } + + let mut parent_buf = AlignedBlobBuf::zeroed(); + { + let mut parent = BlobFrame::init(parent_buf.as_mut_slice(), parent_guid).unwrap(); + let root = parent.header().root_slot; + insert(&mut parent, root, b"pkey", &[0xA5; 112], 1).unwrap(); + for i in 0..60u8 { + let root = parent.header().root_slot; + insert( + &mut parent, + root, + &[b'q', i], + &vec![i; 5000], + u64::from(i) + 2, + ) + .unwrap(); + } + + // Replace the sole `p` leaf with a same-address BlobNode. Its leaf + // allocation is larger than BlobNode, so this is a valid synthetic + // crossing and compaction will rebuild the bookkeeping canonically. + let root_off = decode_child_off(parent.header().root_slot); + let root_node = read_node4(parent.as_ref(), root_off).unwrap(); + let p_index = root_node.keys[..root_node.count as usize] + .iter() + .position(|byte| *byte == b'p') + .unwrap(); + let p_leaf_off = child_offset(root_node.children[p_index]); + write_struct_at(&mut parent, p_leaf_off, &BlobNode::new(b"", child_guid)).unwrap(); + parent.header_mut().num_ext_blobs = 1; + } + assert!( + blob_would_route(BlobFrameRef::wrap(parent_buf.as_slice())), + "test parent must be eligible for a real routed compaction", + ); + compact_blob(&mut parent_buf).unwrap(); + let (ancestor_off, blob_node_off) = { + let parent = BlobFrameRef::wrap(parent_buf.as_slice()); + assert!(parent.header().routing_region().is_some()); + let ancestor_off = decode_child_off(parent.header().root_slot); + let root_node = read_node4(parent, ancestor_off).unwrap(); + let p_index = root_node.keys[..root_node.count as usize] + .iter() + .position(|byte| *byte == b'p') + .unwrap(); + (ancestor_off, child_offset(root_node.children[p_index])) + }; + + let inner: Arc = Arc::new(MemoryBlobStore::new()); + inner.write_blob(parent_guid, &parent_buf).unwrap(); + inner.write_blob(child_guid, &child_buf).unwrap(); + let bm = BufferManager::new(Arc::clone(&inner), 8); + let parent_pin = bm.pin(parent_guid).unwrap(); + + let before = lookup_multi_with(&bm, &parent_pin, None, SearchKey::exact(b"pkey"), |hit| { + hit.value.to_vec() + }) + .unwrap(); + assert_eq!(before.as_deref(), Some(&b"value"[..])); + + fail_next_merge_rewire(); + let error = { + let mut guard = parent_pin.write(); + let mut parent = guard.frame(); + try_merge_children(&bm, &mut parent, crate::store::STRUCTURAL_SEQ).unwrap_err() + }; + assert!(matches!(error, Error::NodeCorrupt { .. })); + + // The prepared clone may consume unreachable scratch space, but the old + // crossing and its accounting stay authoritative until an edge commit. + { + let guard = parent_pin.read(); + let parent = BlobFrameRef::wrap(guard.as_slice()); + assert_eq!(decode_child_off(parent.header().root_slot), ancestor_off); + let root_node = read_node4(parent, ancestor_off).unwrap(); + let p_index = root_node.keys[..root_node.count as usize] + .iter() + .position(|byte| *byte == b'p') + .unwrap(); + assert_eq!(child_offset(root_node.children[p_index]), blob_node_off); + assert_eq!(parent.header().num_ext_blobs, 1); + assert_eq!(parent.header().routing_len, 0); + } + + // Match the production error path, which publishes a partially-mutated + // parent. A failed edge must still never promote the referenced child. + bm.mark_dirty(parent_guid, crate::store::STRUCTURAL_SEQ); + assert_eq!(bm.orphan_staging_count(), 0); + assert_eq!(bm.gc_orphan_backlog_count(), 0); + assert_eq!(bm.pending_delete_count(), 0); + assert!(bm.has_blob(child_guid).unwrap()); + + let value = lookup_multi_with(&bm, &parent_pin, None, SearchKey::exact(b"pkey"), |hit| { + hit.value.to_vec() + }) + .unwrap(); + assert_eq!(value.as_deref(), Some(&b"value"[..])); + + // Persist through the same dirty snapshot/write-through protocol used by + // checkpoint, then prove the authoritative legacy path survives reopen. + let dirty = bm.snapshot_dirty(); + let versioned = bm.snapshot_dirty_versions(&dirty).unwrap(); + let entries: Vec<_> = versioned + .into_iter() + .map(|snapshot| crate::store::WriteThroughEntry { + guid: snapshot.guid, + bytes: bm + .snapshot_bytes_if_version(snapshot.guid, snapshot.content_version) + .unwrap() + .unwrap(), + expected_seq: snapshot.expected_seq, + content_version: Some(snapshot.content_version), + }) + .collect(); + { + let _checkpoint_io = bm.enter_checkpoint_io(); + let report = bm.write_through_batch(&entries).unwrap(); + assert!(report + .statuses + .iter() + .all(|status| { *status == crate::store::WriteThroughStatus::Written })); + bm.flush_inner().unwrap(); + } + drop(parent_pin); + drop(bm); + + let reopened = BufferManager::new(Arc::clone(&inner), 8); + let reopened_parent = reopened.pin(parent_guid).unwrap(); + let reopened_value = lookup_multi_with( + &reopened, + &reopened_parent, + None, + SearchKey::exact(b"pkey"), + |hit| hit.value.to_vec(), + ) + .unwrap(); + assert_eq!(reopened_value.as_deref(), Some(&b"value"[..])); + assert!(reopened.has_blob(child_guid).unwrap()); +} + #[test] fn single_insert_then_lookup() { let (mut buf, _) = fresh_blob(); @@ -994,6 +1189,32 @@ fn compact_blob_is_noop_on_empty_tree() { assert_eq!(frame.header().blob_guid, guid); } +#[test] +fn compact_blob_preserves_snapshot_epoch_fields_exactly() { + let (buf_vec, _) = fresh_blob(); + let mut buf = aligned_from_vec(&buf_vec); + let expected_created = 4_294_967_301_u64; + let expected_high_water = expected_created + 97; + { + let mut frame = BlobFrame::wrap(buf.as_mut_slice()); + let h = frame.header_mut(); + h.created_epoch = expected_created; + h.epoch_high_water = expected_high_water; + let root = h.root_slot; + insert(&mut frame, root, b"epoch-key", b"epoch-value", 1).unwrap(); + } + + compact_blob(&mut buf).unwrap(); + + let frame = BlobFrame::wrap(buf.as_mut_slice()); + assert_eq!(frame.header().created_epoch, expected_created); + assert_eq!(frame.header().epoch_high_water, expected_high_water); + assert_eq!( + get(&frame, b"epoch-key").as_deref(), + Some(&b"epoch-value"[..]) + ); +} + #[test] fn blob_needs_compaction_tracks_tombstones() { let (buf_vec, _) = fresh_blob(); diff --git a/src/metrics.rs b/src/metrics.rs index ec70af6..e6d7155 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -47,6 +47,9 @@ //! | `holt_blob_overfull_children` | gauge | `TreeStats::overfull_child_blobs` | //! | `holt_bm_dirty_count` | gauge | `TreeStats::bm_dirty_count` | //! | `holt_bm_pending_delete_count` | gauge | `TreeStats::bm_pending_delete_count` | +//! | `holt_bm_gc_orphan_backlog_count` | gauge | `TreeStats::bm_gc_orphan_backlog_count` | +//! | `holt_bm_gc_reclaimed_total` | counter | `TreeStats::bm_gc_reclaimed_count` | +//! | `holt_bm_gc_last_full_sweep_deferred_count` | gauge | `TreeStats::bm_gc_last_full_sweep_deferred_count` | //! | `holt_bm_write_delta_count` | gauge | `TreeStats::bm_write_delta_count` | //! | `holt_bm_read_index_token_count` | gauge | `TreeStats::bm_read_index_token_count` | //! | `holt_bm_read_index_cache_entries` | gauge | `TreeStats::bm_read_index_cache_entries` | @@ -120,6 +123,7 @@ //! | `holt_open_wal_replay_records_total` | counter | `OpenStats::wal_replay_records` | //! | `holt_open_wal_replay_bytes` | gauge | `OpenStats::wal_replay_bytes` | //! | `holt_open_wal_replay_duration_seconds` | gauge | `OpenStats::wal_replay_micros` | +//! | `holt_open_epoch_recovery_duration_seconds` | gauge | `OpenStats::epoch_recovery_micros` | //! | `holt_open_wal_torn_tail` | gauge | `OpenStats::wal_torn_tail` | //! | `holt_journal_appends_total` | counter | `JournalStats::appends` | //! | `holt_journal_batches_total` | counter | `JournalStats::batches` | @@ -284,6 +288,27 @@ pub fn render_prometheus(stats: &TreeStats) -> String { "gauge", stats.bm_pending_delete_count as u64, ); + metric( + &mut out, + "holt_bm_gc_orphan_backlog_count", + "COW/structural orphan debt, including staged and snapshot-held blobs.", + "gauge", + stats.bm_gc_orphan_backlog_count as u64, + ); + metric( + &mut out, + "holt_bm_gc_reclaimed_total", + "Cumulative blobs physically reclaimed by full sweeps or clean-frontier exact reclaim.", + "counter", + stats.bm_gc_reclaimed_count, + ); + metric( + &mut out, + "holt_bm_gc_last_full_sweep_deferred_count", + "Unreachable candidates deferred by pins or the bounded limit in the latest successful full reachability sweep.", + "gauge", + stats.bm_gc_last_full_sweep_deferred_count as u64, + ); metric( &mut out, "holt_bm_write_delta_count", @@ -796,6 +821,13 @@ pub fn render_prometheus(stats: &TreeStats) -> String { "gauge", micros_to_seconds(stats.open.wal_replay_micros), ); + metric_f64( + &mut out, + "holt_open_epoch_recovery_duration_seconds", + "Wall-clock time spent restoring DB-wide snapshot epoch state.", + "gauge", + micros_to_seconds(stats.open.epoch_recovery_micros), + ); metric( &mut out, "holt_open_wal_torn_tail", @@ -1024,6 +1056,9 @@ mod tests { blobs: Vec::new(), bm_dirty_count: 2, bm_pending_delete_count: 1, + bm_gc_orphan_backlog_count: 4, + bm_gc_reclaimed_count: 5, + bm_gc_last_full_sweep_deferred_count: 6, bm_write_delta_count: 3, bm_read_index_token_count: 46, bm_read_index_cache_entries: 47, @@ -1075,6 +1110,7 @@ mod tests { wal_replay_records: 21, wal_replay_bytes: 4096, wal_replay_micros: 12_500, + epoch_recovery_micros: 2_500, wal_torn_tail: true, }, journal: with_journal.then_some(JournalStats { @@ -1110,6 +1146,15 @@ mod tests { assert!(out.contains("# TYPE holt_blob_count gauge\n")); assert!(out.contains("holt_blob_count 3\n")); // Monotonic counters keep the `_total` suffix... + assert!(out.contains("holt_bm_gc_orphan_backlog_count 4\n")); + assert!(out.contains( + "# HELP holt_bm_gc_reclaimed_total Cumulative blobs physically reclaimed by full sweeps or clean-frontier exact reclaim.\n" + )); + assert!(out.contains("# TYPE holt_bm_gc_reclaimed_total counter\n")); + assert!(out.contains("holt_bm_gc_reclaimed_total 5\n")); + assert!(!out.contains("holt_bm_gc_reclaimed_count")); + assert!(out.contains("# TYPE holt_bm_gc_last_full_sweep_deferred_count gauge\n")); + assert!(out.contains("holt_bm_gc_last_full_sweep_deferred_count 6\n")); assert!(out.contains("holt_bm_write_delta_count 3\n")); assert!(out.contains("holt_bm_read_index_token_count 46\n")); assert!(out.contains("holt_bm_read_index_cache_entries 47\n")); @@ -1176,6 +1221,7 @@ mod tests { assert!(out.contains("holt_open_wal_replay_records_total 21\n")); assert!(out.contains("holt_open_wal_replay_bytes 4096\n")); assert!(out.contains("holt_open_wal_replay_duration_seconds 0.012500\n")); + assert!(out.contains("holt_open_epoch_recovery_duration_seconds 0.002500\n")); assert!(out.contains("holt_open_wal_torn_tail 1\n")); // ...non-monotonic gauges (sum-over-reachable-blobs) drop it. assert!(out.contains("# TYPE holt_slots gauge\n")); diff --git a/src/store/blob_frame.rs b/src/store/blob_frame.rs index ec15304..ccd9187 100644 --- a/src/store/blob_frame.rs +++ b/src/store/blob_frame.rs @@ -347,6 +347,22 @@ impl<'a> BlobFrame<'a> { BlobFrameRef { buf: self.buf } } + /// Copy the complete frame image into another frame-sized buffer. + /// + /// Structural walkers use this to prepare a fallible rewrite off to + /// the side, then publish it with one infallible replacement only after + /// every allocation and edge update has succeeded. + pub(crate) fn copy_bytes_to(&self, dst: &mut [u8]) { + assert_eq!(dst.len(), PAGE_SIZE as usize); + dst.copy_from_slice(self.buf); + } + + /// Replace this complete frame with a previously prepared frame image. + pub(crate) fn replace_bytes_from(&mut self, src: &[u8]) { + assert_eq!(src.len(), PAGE_SIZE as usize); + self.buf.copy_from_slice(src); + } + /// Initialize a fresh blob from a zeroed buffer. /// /// Writes the header, sets `space_used` to `DATA_AREA_START`, diff --git a/src/store/buffer_manager/mod.rs b/src/store/buffer_manager/mod.rs index 8b96ac8..ff69eab 100644 --- a/src/store/buffer_manager/mod.rs +++ b/src/store/buffer_manager/mod.rs @@ -21,11 +21,11 @@ //! //! The `write_blob` trait method is still write-through (cache + //! store in one call). Internal call sites that produce a new -//! blob (spillover) or unlink one (erase's `SubtreeGone` / -//! merge) go through [`BufferManager::install_new_blob`] / -//! [`BufferManager::mark_for_delete`] instead, so the store -//! write or manifest mutation is deferred until the next flush — -//! invariant **W2D** below. +//! blob (spillover) or unlink one (erase's `SubtreeGone`) goes through +//! [`BufferManager::install_new_blob`] / [`BufferManager::mark_for_delete`]. +//! Structural merge instead stages the detached child against its exact +//! rewritten parent until a clean durable frontier can reclaim it. Store +//! writes and manifest mutations therefore stay behind invariant **W2D**. //! //! ## Dirty tracking + deferred deletes //! @@ -132,9 +132,9 @@ mod residency; mod telemetry; mod write_delta; -use std::collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}; +use std::collections::{hash_map::Entry, BTreeMap, HashMap, HashSet, VecDeque}; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, MutexGuard, Weak}; use dashmap::DashMap; @@ -180,13 +180,103 @@ pub const STRUCTURAL_SEQ: u64 = u64::MAX; /// registration, retirement, and orphan recording stay consistent. #[derive(Default)] struct SnapshotState { - /// Epoch → snapshot root GUID for every live snapshot. - live: BTreeMap, + /// Epoch → snapshot root for every live snapshot. The weak pin can + /// be upgraded under the registry lock so GC holds a stable root while + /// a last derived view/cursor concurrently releases its epoch lease. + live: BTreeMap, + /// COW detachments grouped by the parent whose edge was repointed. + /// A child stays non-reclaimable here until that exact parent publishes + /// dirty/flushing debt; this closes the last-lease-drop window between + /// edge mutation and `mark_dirty(parent)`. + cow_pending: HashMap>, /// Frames forked away from the live tree, tagged with the - /// `created_epoch` of the forked-away version. Safe to free once the - /// fork barrier drops below the tag (no live snapshot can reference - /// it). One entry per fork. + /// `created_epoch` of the forked-away version. Once the fork barrier + /// drops below the tag no live snapshot can reference it. Retirement only + /// moves the exact GUID into the reclaim FIFO; physical deletion waits for + /// a clean durable checkpoint frontier or a full reachability sweep. orphans: Vec<(BlobGuid, u64)>, + /// Structural detachments grouped by their rewritten parent. These do + /// not install a visibility fence because a stable snapshot may still + /// lazily pin the old child. Parent dirty publication promotes them to + /// `structural_orphans` or the reclaim FIFO. + structural_pending: HashMap>, + /// Children detached by structural merge while at least one snapshot + /// lease is live. Unlike a COW fork, a merge has no per-snapshot epoch + /// tag: any copied root may still point at the detached child. Hold the + /// exact GUID until the last lease retires, then move it to the same + /// post-checkpoint FIFO as ordinary retired COW frames. + structural_orphans: HashSet, + /// Copy-on-write and structural frames eligible for exact reclaim after + /// their final snapshot epoch has retired. A clean checkpoint drains this + /// FIFO in bounded batches without an all-store reachability walk; pinned + /// candidates are restored for a later pass. + retired_orphans: VecDeque, + retired_orphan_set: HashSet, +} + +struct SnapshotRoot { + guid: BlobGuid, + pin: Weak, +} + +/// Shared lifetime token for one copy-on-write snapshot epoch. +/// +/// Every `View`, range builder, and owned cursor derived from a snapshot +/// carries this token. The epoch retires only after the final derived read +/// handle drops, preventing GC from collecting descendants that an escaped +/// handle can still reach. +pub(crate) struct SnapshotLease { + store: Arc, + epoch: u64, + root_guid: BlobGuid, + // Keeps the ephemeral root resident for the full lease lifetime. GC + // upgrades the registry's weak pin while holding the registry lock. + root_pin: Option>, +} + +impl SnapshotLease { + pub(crate) fn new( + store: Arc, + epoch: u64, + root_guid: BlobGuid, + root_pin: Arc, + ) -> Arc { + Arc::new(Self { + store, + epoch, + root_guid, + root_pin: Some(root_pin), + }) + } +} + +impl Drop for SnapshotLease { + fn drop(&mut self) { + self.store.retire_snapshot(self.epoch); + drop(self.root_pin.take()); + self.store.discard_snapshot_root(self.root_guid); + } +} + +/// Strong snapshot-root pin owned by one GC reachability pass. +pub(crate) struct PinnedSnapshotRoot { + store: Arc, + guid: BlobGuid, + pin: Option>, +} + +impl PinnedSnapshotRoot { + #[must_use] + pub(crate) fn guid(&self) -> BlobGuid { + self.guid + } +} + +impl Drop for PinnedSnapshotRoot { + fn drop(&mut self) { + drop(self.pin.take()); + self.store.discard_snapshot_root(self.guid); + } } /// One pre-snapshotted blob image ready for checkpoint write-through. @@ -285,6 +375,9 @@ impl CacheBudget { pub(crate) struct BufferStats { pub(crate) dirty_count: usize, pub(crate) pending_delete_count: usize, + pub(crate) gc_orphan_backlog_count: usize, + pub(crate) gc_reclaimed_count: u64, + pub(crate) gc_last_full_sweep_deferred_count: usize, pub(crate) read_index_token_count: usize, pub(crate) read_index_cache_entries: usize, pub(crate) read_index_cache_bytes: usize, @@ -333,6 +426,11 @@ pub(crate) struct BufferStats { pub(crate) store: StoreStats, } +pub(crate) struct GcSweepOutcome { + pub(crate) freed: usize, + pub(crate) complete: bool, +} + /// Frequency-aware blob cache; see the module docs. pub struct BufferManager { store: Arc, @@ -386,6 +484,11 @@ pub struct BufferManager { /// the foreground writer gate. write_delta: WriteDelta, write_delta_flush: Mutex<()>, + /// Serializes one complete checkpoint I/O phase: dependency-ordered + /// dirty waves, their durability syncs, and the following pending-delete + /// phase. Callers acquire this only after releasing CommitGate/capture + /// locks; low-level write/flush helpers never acquire it recursively. + checkpoint_io: Mutex<()>, delete_fence_total: AtomicUsize, /// Rotating shard cursors for advisory maintenance queues. /// Without this, a fixed shard-0-first drain can starve later @@ -419,9 +522,72 @@ pub struct BufferManager { /// may be visible to a snapshot and so must be forked before an /// in-place overwrite; `0` (no live snapshot) disables forking. fork_barrier: AtomicU64, - /// Live CoW snapshot registry + forked-away orphan list. Drives - /// fork-barrier recomputation and orphan reclaim on retire. + /// Live CoW snapshot registry plus detached-frame staging/FIFO state. + /// Retirement lowers the fork barrier and moves eligible COW/structural + /// GUIDs into the exact-reclaim FIFO; it never deletes persisted data + /// inline. Physical deletion requires a clean durable checkpoint frontier + /// or a full reachability sweep. snapshots: Mutex, + /// Even while idle and odd while physical GC deletion is active. + /// Optimistic readers use this sequence to distinguish a concurrent + /// reachability sweep from a stable missing-child corruption. + gc_epoch: AtomicU64, + physical_gc: Mutex<()>, + gc_reclaimed_count: AtomicU64, + /// Unreachable candidates deferred by the most recently completed full + /// reachability sweep. Exact FIFO reclaim deliberately does not overwrite + /// this value: an empty exact batch cannot prove that full-sweep debt is + /// gone. + gc_last_full_sweep_deferred_count: AtomicUsize, + checkpoint_waker: Mutex>, +} + +struct GcEpochGuard<'a> { + _serial: MutexGuard<'a, ()>, + epoch: &'a AtomicU64, +} + +#[cfg(test)] +struct ResidentPinBarrier { + entered: std::sync::Barrier, + release: std::sync::Barrier, +} + +#[cfg(test)] +impl ResidentPinBarrier { + fn new() -> Self { + Self { + entered: std::sync::Barrier::new(2), + release: std::sync::Barrier::new(2), + } + } +} + +#[cfg(test)] +thread_local! { + static RESIDENT_PIN_BARRIER: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(test)] +fn set_resident_pin_barrier_for_current_thread(barrier: Arc) { + RESIDENT_PIN_BARRIER.with(|slot| *slot.borrow_mut() = Some(barrier)); +} + +#[cfg(test)] +fn pause_resident_pin_after_fence_check() { + let barrier = RESIDENT_PIN_BARRIER.with(|slot| slot.borrow_mut().take()); + if let Some(barrier) = barrier { + barrier.entered.wait(); + barrier.release.wait(); + } +} + +impl Drop for GcEpochGuard<'_> { + fn drop(&mut self) { + let previous = self.epoch.fetch_add(1, Ordering::AcqRel); + debug_assert_eq!(previous & 1, 1, "GC epoch must be active on guard drop"); + } } impl BufferManager { @@ -451,6 +617,60 @@ impl BufferManager { self.fork_barrier.load(Ordering::Acquire) } + /// Capture the physical-GC sequence for one optimistic reader attempt. + #[must_use] + pub(crate) fn gc_read_epoch(&self) -> u64 { + self.gc_epoch.load(Ordering::Acquire) + } + + /// Wait for a physical sweep to leave its odd epoch and return the next + /// stable even sequence. Used only when a range cursor crosses a GC + /// barrier; the uncontended path is one atomic load. + #[must_use] + pub(crate) fn gc_stable_read_epoch(&self) -> u64 { + loop { + let epoch = self.gc_read_epoch(); + if epoch & 1 == 0 { + return epoch; + } + std::thread::yield_now(); + } + } + + /// Return `true` when physical deletion was active at capture time or + /// crossed the reader attempt. A stable even value means a missing blob + /// is not explained by GC and must remain a hard error. + #[must_use] + pub(crate) fn gc_raced_since(&self, captured: u64) -> bool { + captured & 1 != 0 || self.gc_epoch.load(Ordering::Acquire) != captured + } + + /// Run a short publication closure while physical GC is excluded, but + /// only if the caller's optimistic generation is still current. + pub(crate) fn with_stable_gc_epoch( + &self, + captured: u64, + publish: impl FnOnce() -> R, + ) -> Option { + let _gc = self.physical_gc.lock().expect("physical GC lock poisoned"); + (!self.gc_raced_since(captured)).then(publish) + } + + #[must_use] + pub(crate) fn gc_epoch_still_stable(&self, captured: u64) -> bool { + self.with_stable_gc_epoch(captured, || ()).is_some() + } + + fn begin_gc_epoch(&self) -> GcEpochGuard<'_> { + let serial = self.physical_gc.lock().expect("physical GC lock poisoned"); + let previous = self.gc_epoch.fetch_add(1, Ordering::AcqRel); + debug_assert_eq!(previous & 1, 0, "physical GC passes must be serialized"); + GcEpochGuard { + _serial: serial, + epoch: &self.gc_epoch, + } + } + /// Fork a frame for copy-on-write: copy `src_bytes` to a fresh GUID /// `new_guid`, repatch the self-GUID, install it, and pin it. The /// install stamps the current epoch — strictly greater than any @@ -498,67 +718,173 @@ impl BufferManager { /// epoch (so frames created afterwards are private to the live /// tree), raises the fork barrier to the snapshot's epoch, and /// returns that epoch. - pub(crate) fn register_snapshot(&self, root_guid: BlobGuid) -> u64 { + pub(crate) fn register_snapshot( + &self, + root_guid: BlobGuid, + root_pin: &Arc, + ) -> Result { let mut snaps = self.snapshots.lock().expect("snapshot registry poisoned"); - // `fetch_add` returns the pre-bump value: that is this snapshot's - // epoch and, because `current_epoch` only ever increases, the - // largest key in the registry — hence the new barrier. - let epoch = self.current_epoch.fetch_add(1, Ordering::AcqRel); - snaps.live.insert(epoch, root_guid); + // The returned pre-bump value is this snapshot's barrier; the + // post-bump value is persisted as the root high-water and stamps new + // frames. MAX-1 is a valid but exhausted durable high-water, while + // MAX can never be emitted by a successful registration. A checked + // atomic update prevents MAX from wrapping to zero and clearing the + // fork barrier while an exhausted snapshot remains live. + let epoch = self + .current_epoch + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < u64::MAX - 1).then_some(current + 1) + }) + .map_err(|_| Error::SnapshotEpochExhausted)?; + snaps.live.insert( + epoch, + SnapshotRoot { + guid: root_guid, + pin: Arc::downgrade(root_pin), + }, + ); self.fork_barrier.store(epoch, Ordering::Release); - epoch + Ok(epoch) } - /// Record a frame forked away from the live tree so it can be freed - /// once no live snapshot can reference it. `created_epoch` is the - /// epoch of the forked-away version (≤ the barrier at fork time). - pub(crate) fn record_orphan(&self, guid: BlobGuid, created_epoch: u64) { - self.snapshots - .lock() - .expect("snapshot registry poisoned") - .orphans - .push((guid, created_epoch)); + /// Stage a frame forked away from the live tree until its rewritten parent + /// publishes dirty/flushing debt. `created_epoch` is the epoch of the + /// forked-away version (≤ the barrier at fork time). Lease retirement only + /// makes it eligible; a clean exact frontier or full GC deletes it. + pub(crate) fn stage_cow_reclaim( + &self, + parent_guid: BlobGuid, + guid: BlobGuid, + created_epoch: u64, + ) { + let mut snapshots = self.snapshots.lock().expect("snapshot registry poisoned"); + let pending = snapshots.cow_pending.entry(parent_guid).or_default(); + if !pending.iter().any(|(candidate, _)| *candidate == guid) { + pending.push((guid, created_epoch)); + } + } + + /// Stage a child detached by structural merge under the parent whose + /// cached edge was rewritten. Staging is deliberately not reclaimable; + /// `mark_dirty(parent)` promotes it only after the new parent image is + /// protected by dirty/flushing checkpoint debt. + pub(crate) fn stage_structural_reclaim(&self, parent_guid: BlobGuid, child_guid: BlobGuid) { + let mut snapshots = self.snapshots.lock().expect("snapshot registry poisoned"); + snapshots + .structural_pending + .entry(parent_guid) + .or_default() + .insert(child_guid); + } + + /// Promote every COW/structural detachment owned by `parent_guid` after + /// that parent's dirty image has been published. A concurrent final + /// snapshot retirement can only make a staged child eligible here, + /// after dirty/flushing state already closes the clean frontier. + fn publish_parent_orphans(&self, parent_guid: BlobGuid) { + let wake = { + let mut snapshots = self.snapshots.lock().expect("snapshot registry poisoned"); + let cow = snapshots + .cow_pending + .remove(&parent_guid) + .unwrap_or_default(); + let structural = snapshots + .structural_pending + .remove(&parent_guid) + .unwrap_or_default(); + let barrier = snapshots.live.keys().next_back().copied().unwrap_or(0); + let mut wake = false; + for (guid, created_epoch) in cow { + if barrier != 0 && created_epoch <= barrier { + snapshots.orphans.push((guid, created_epoch)); + } else if snapshots.retired_orphan_set.insert(guid) { + snapshots.retired_orphans.push_back(guid); + wake = true; + } + } + for guid in structural { + if snapshots.live.is_empty() { + if snapshots.retired_orphan_set.insert(guid) { + snapshots.retired_orphans.push_back(guid); + wake = true; + } + } else { + snapshots.structural_orphans.insert(guid); + } + } + wake + }; + if wake { + self.wake_checkpointer(); + } } /// Retire the snapshot at `epoch`: lower the fork barrier to the - /// highest remaining live snapshot epoch (or `0`), free the - /// snapshot's root frame, and reclaim every orphan whose forked-away - /// version is now newer than the barrier — no live snapshot can - /// reference it. Idempotent for an unknown epoch. + /// highest remaining live snapshot epoch (or `0`), evict the + /// snapshot's root frame, and stop tracking every orphan whose + /// forked-away version is now newer than the barrier. + /// + /// Persisted blobs are deliberately retained. The live parent that + /// stopped referencing a copy-on-write child may still have its older + /// image on stable storage. Retirement only moves eligible COW and + /// structural GUIDs into the exact-reclaim FIFO; physical deletion + /// requires a clean durable checkpoint frontier or a full reachability + /// sweep. Idempotent for an unknown epoch. pub(crate) fn retire_snapshot(&self, epoch: u64) { - let (root, to_free) = { + let (root, wake) = { let mut snaps = self.snapshots.lock().expect("snapshot registry poisoned"); let root = snaps.live.remove(&epoch); let barrier = snaps.live.keys().next_back().copied().unwrap_or(0); self.fork_barrier.store(barrier, Ordering::Release); - let mut to_free = Vec::new(); - snaps.orphans.retain(|&(guid, created_epoch)| { - let still_referenced = created_epoch <= barrier; - if !still_referenced { - to_free.push(guid); + let mut active = Vec::with_capacity(snaps.orphans.len()); + for (guid, created_epoch) in std::mem::take(&mut snaps.orphans) { + if barrier != 0 && created_epoch <= barrier { + active.push((guid, created_epoch)); + } else if snaps.retired_orphan_set.insert(guid) { + snaps.retired_orphans.push_back(guid); } - still_referenced - }); - (root, to_free) + } + snaps.orphans = active; + if barrier == 0 { + let structural = std::mem::take(&mut snaps.structural_orphans); + for guid in structural { + if snaps.retired_orphan_set.insert(guid) { + snaps.retired_orphans.push_back(guid); + } + } + } + (root, !snaps.retired_orphans.is_empty()) }; if let Some(root) = root { - self.reclaim_blob(root); + self.discard_snapshot_root(root.guid); } - for guid in to_free { - self.reclaim_blob(guid); + if wake { + self.wake_checkpointer(); } + // Persistent detachments deliberately keep their cache and dirty + // bookkeeping. A clean checkpoint may reclaim their exact FIFO GUIDs; + // a full GC additionally handles crash leftovers by reachability. } - /// Free a copy-on-write frame no longer referenced by the live tree - /// or any live snapshot. Checkpoint-owned or pending-delete frames - /// are still load-bearing for checkpoint correctness, so reclamation - /// is best-effort: those frames are left for a later DB-wide GC after - /// their checkpoint owner has retired. - fn reclaim_blob(&self, guid: BlobGuid) { + /// Discard an ephemeral snapshot root from process-local state. + /// + /// This must never call `store.delete_blob`: snapshot reachability is + /// based on the current in-memory parent, which can be newer than the last + /// durable parent. Persisted space is reclaimed only after a clean exact + /// frontier or by [`Self::gc_sweep_unreachable_with_canonical`] under a + /// durable reachability barrier. + pub(crate) fn discard_snapshot_root(&self, guid: BlobGuid) { + let _ = self.evict_reclaimable_blob(guid); + } + + /// Evict all process-local state for `guid` when no pin, checkpoint, + /// or pending delete still owns it. Returns `true` when a subsequent + /// reachability-proven physical delete may proceed. + fn evict_reclaimable_blob(&self, guid: BlobGuid) -> bool { { let state = self.mutation_shard(guid).lock().unwrap(); if state.is_protected_or_pending(&guid) { - return; + return false; } } if let Some((_, entry)) = self.cache.remove_if(&guid, |_, entry| { @@ -570,7 +896,7 @@ impl BufferManager { }) { entry.clear_dirty_hint(); } else if self.cache.contains_key(&guid) { - return; + return false; } self.route_resident.remove(guid); let mut state = self.mutation_shard(guid).lock().unwrap(); @@ -578,37 +904,318 @@ impl BufferManager { let removed = state.remove_maintenance_candidates(&guid); drop(state); self.decrement_candidate_totals(removed); - let _ = self.store.delete_blob(guid); + true + } + + /// Whether an API handle or in-flight reader still owns `guid` beyond + /// the cache's own reference. Reachability GC uses this on a Dropping + /// tree root to retain the whole closure until the old handle releases. + pub(crate) fn blob_is_pinned(&self, guid: BlobGuid) -> bool { + self.cache + .get(&guid) + .is_some_and(|entry| Arc::strong_count(entry.value()) > 1) + } + + /// Physically delete one blob that a durability-barrier-protected + /// reachability walk proved unreachable. + fn reclaim_unreachable_blob(&self, guid: BlobGuid) -> Result { + { + let mut state = self.mutation_shard(guid).lock().unwrap(); + if state.is_protected_or_pending(&guid) { + return Ok(false); + } + state.gc_deleting.insert(guid); + self.delete_fence_total.fetch_add(1, Ordering::AcqRel); + } + + let result = (|| { + // Publish invalidation before eviction/delete. A cold indexed + // reader either observes this token change or the delete fence; + // a delayed reader that outlives both is covered by `gc_epoch`. + self.invalidate_indexed_reads(guid); + if let Some((_, entry)) = self.cache.remove_if(&guid, |_, entry| { + if Arc::strong_count(entry) > 1 { + return false; + } + let state = self.mutation_shard(guid).lock().unwrap(); + !state.is_protected(&guid) + && !state.pending_deletes.contains_key(&guid) + && !state.deleting.contains_key(&guid) + }) { + entry.clear_dirty_hint(); + } else if self.cache.contains_key(&guid) { + return Ok(false); + } + self.route_resident.remove(guid); + let mut state = self.mutation_shard(guid).lock().unwrap(); + state.remove_unclaimed_dirty(&guid); + let removed = state.remove_maintenance_candidates(&guid); + drop(state); + self.decrement_candidate_totals(removed); + self.store.delete_blob(guid)?; + Ok(true) + })(); + + let mut state = self.mutation_shard(guid).lock().unwrap(); + if state.gc_deleting.remove(&guid) { + self.delete_fence_total.fetch_sub(1, Ordering::AcqRel); + } + drop(state); + // The helper treats every delete fence as reachability protection. + // Run token cleanup after retiring our GC-owned fence for every + // result. A custom store may physically remove the GUID and then + // report an I/O error; the helper rechecks cache/protection/store + // reachability fail-closed, so it removes only truly dead tokens. self.remove_read_index_token_if_unreachable(guid); + result } - /// GUIDs of every live snapshot's frozen root frame. - pub(crate) fn snapshot_roots(&self) -> Vec { + /// Pinned roots of every live snapshot epoch. + /// + /// Upgrading each weak pin while holding the registry lock closes the + /// race with the last derived `View`/cursor dropping its epoch lease: + /// GC either omits an already-retired root or owns a strong pin for the + /// full reachability walk. + pub(crate) fn snapshot_roots_pinned(self: &Arc) -> Result> { self.snapshots .lock() .expect("snapshot registry poisoned") .live .values() - .copied() + .map(|root| { + root.pin + .upgrade() + .map(|pin| PinnedSnapshotRoot { + store: Arc::clone(self), + guid: root.guid, + pin: Some(pin), + }) + .ok_or(Error::Internal("live snapshot registry lost its root pin")) + }) .collect() } /// Free every persisted frame not in `reachable`, returning the count /// reclaimed. The recovery-time sweep for copy-on-write frames /// orphaned by a crash that lost the in-memory orphan list. The - /// caller must hold the tree quiescent and pass the full reachable - /// set (live root ∪ every live snapshot root). + /// caller must hold the tree quiescent, establish a durable checkpoint + /// of the same root image, and pass the full reachable set (live root + /// ∪ every live snapshot root). + #[cfg(test)] pub(crate) fn gc_sweep_unreachable(&self, reachable: &HashSet) -> Result { - let mut freed = 0; - for guid in self.list_blobs()? { - if !reachable.contains(&guid) { - self.reclaim_blob(guid); + self.gc_sweep_unreachable_bounded(reachable, usize::MAX) + .map(|outcome| outcome.freed) + } + + /// Sweep unreachable blobs while distinguishing the durable canonical + /// topology from frames kept alive only by captured snapshot roots. Both + /// closures protect physical deletion during this pass, but only + /// canonical-live GUIDs invalidate retired-orphan FIFO entries. + pub(crate) fn gc_sweep_unreachable_with_canonical( + &self, + reachable: &HashSet, + canonical_reachable: &HashSet, + ) -> Result { + self.gc_sweep_unreachable_with_canonical_bounded(reachable, canonical_reachable, usize::MAX) + .map(|outcome| outcome.freed) + } + + /// Delete at most `limit` reachability-proven blobs after the caller has + /// frozen and durably checkpointed the topology used for `reachable`. + /// Pins are fail-closed: skipped blobs remain available to the next pass. + #[cfg(test)] + pub(crate) fn gc_sweep_unreachable_bounded( + &self, + reachable: &HashSet, + limit: usize, + ) -> Result { + self.gc_sweep_unreachable_with_canonical_bounded(reachable, reachable, limit) + } + + /// Bounded sweep variant with a separate canonical-live closure. + /// `reachable` is the union used to protect deletion; snapshot-only + /// members remain in the retired FIFO so the ordinary exact-reclaim path + /// can free them immediately after the GC-owned snapshot pins retire. + pub(crate) fn gc_sweep_unreachable_with_canonical_bounded( + &self, + reachable: &HashSet, + canonical_reachable: &HashSet, + limit: usize, + ) -> Result { + debug_assert!(canonical_reachable.is_subset(reachable)); + let _epoch = self.begin_gc_epoch(); + let mut freed = 0usize; + let mut deferred = 0usize; + let mut reclaimed = HashSet::new(); + // The GC epoch already owns `physical_gc`; bypass this type's public + // BlobStore wrapper here so that wrapper can serialize external + // enumerations without recursively taking the same mutex. + let present = self.store.list_blobs()?; + for guid in present.iter().copied() { + if reachable.contains(&guid) { + continue; + } + if freed >= limit { + deferred = deferred.saturating_add(1); + continue; + } + if self.reclaim_unreachable_blob(guid)? { freed += 1; + reclaimed.insert(guid); + } else { + deferred = deferred.saturating_add(1); + } + } + if freed != 0 { + self.store.flush()?; + self.gc_reclaimed_count + .fetch_add(freed as u64, Ordering::Relaxed); + } + self.gc_last_full_sweep_deferred_count + .store(deferred, Ordering::Relaxed); + + // A missing candidate was already reclaimed by an earlier pass or + // recovery. Remove it without requiring another full sweep. + let present: HashSet<_> = present.into_iter().collect(); + let mut snapshots = self.snapshots.lock().expect("snapshot registry poisoned"); + snapshots.retired_orphans.retain(|guid| { + present.contains(guid) + && !reclaimed.contains(guid) + && !canonical_reachable.contains(guid) + }); + snapshots.retired_orphan_set = snapshots.retired_orphans.iter().copied().collect(); + Ok(GcSweepOutcome { + freed, + complete: deferred == 0, + }) + } + + /// Reclaim a FIFO batch of COW or structural frames that are no longer + /// protected by a snapshot lease. The caller must have completed a clean + /// durable checkpoint; that frontier proves current parents no longer + /// reference these exact GUIDs, so the normal path needs no all-store scan. + pub(crate) fn reclaim_retired_orphans_bounded(&self, limit: usize) -> Result { + let candidates = self.take_retired_orphans_bounded(limit); + self.reclaim_retired_orphan_batch(candidates) + } + + /// Move one FIFO batch out of the backlog while the caller owns the + /// clean-frontier commit/maintenance barrier. Separating capture from + /// I/O closes the race where a post-checkpoint writer could enqueue a + /// not-yet-durable orphan between the clean test and queue drain. + pub(crate) fn take_retired_orphans_bounded(&self, limit: usize) -> Vec { + let mut snapshots = self.snapshots.lock().expect("snapshot registry poisoned"); + let take = limit.min(snapshots.retired_orphans.len()); + let mut candidates = Vec::with_capacity(take); + for _ in 0..take { + let guid = snapshots + .retired_orphans + .pop_front() + .expect("bounded retired orphan count"); + snapshots.retired_orphan_set.remove(&guid); + candidates.push(guid); + } + candidates + } + + pub(crate) fn reclaim_retired_orphan_batch(&self, candidates: Vec) -> Result { + if candidates.is_empty() { + return Ok(0); + } + + let _epoch = self.begin_gc_epoch(); + let mut freed = 0usize; + let mut retry = Vec::new(); + for (index, guid) in candidates.iter().copied().enumerate() { + match self.store.has_blob(guid) { + Ok(true) => {} + Ok(false) => continue, + Err(error) => { + retry.extend(candidates[index..].iter().copied()); + self.restore_retired_orphans(retry); + return Err(error); + } + } + match self.reclaim_unreachable_blob(guid) { + Ok(true) => freed += 1, + Ok(false) => retry.push(guid), + Err(error) => { + retry.extend(candidates[index..].iter().copied()); + self.restore_retired_orphans(retry); + return Err(error); + } + } + } + if freed != 0 { + if let Err(error) = self.store.flush() { + self.restore_retired_orphans(candidates); + return Err(error); } + self.gc_reclaimed_count + .fetch_add(freed as u64, Ordering::Relaxed); } + self.restore_retired_orphans(retry); Ok(freed) } + fn restore_retired_orphans(&self, guids: impl IntoIterator) { + let mut snapshots = self.snapshots.lock().expect("snapshot registry poisoned"); + for guid in guids { + if snapshots.retired_orphan_set.insert(guid) { + snapshots.retired_orphans.push_back(guid); + } + } + } + + #[must_use] + pub(crate) fn gc_orphan_backlog_count(&self) -> usize { + let snapshots = self.snapshots.lock().expect("snapshot registry poisoned"); + snapshots.cow_pending.values().map(Vec::len).sum::() + + snapshots.orphans.len() + + snapshots + .structural_pending + .values() + .map(HashSet::len) + .sum::() + + snapshots.structural_orphans.len() + + snapshots.retired_orphans.len() + } + + pub(crate) fn orphan_staging_count(&self) -> usize { + let snapshots = self.snapshots.lock().expect("snapshot registry poisoned"); + snapshots.cow_pending.values().map(Vec::len).sum::() + + snapshots + .structural_pending + .values() + .map(HashSet::len) + .sum::() + } + + pub(crate) fn register_checkpoint_waker(&self, thread: std::thread::Thread) { + *self + .checkpoint_waker + .lock() + .expect("checkpoint waker lock poisoned") = Some(thread); + } + + pub(crate) fn clear_checkpoint_waker(&self) { + self.checkpoint_waker + .lock() + .expect("checkpoint waker lock poisoned") + .take(); + } + + fn wake_checkpointer(&self) { + if let Some(thread) = self + .checkpoint_waker + .lock() + .expect("checkpoint waker lock poisoned") + .as_ref() + { + thread.unpark(); + } + } + pub(crate) fn vacuum_storage(&self) -> Result { self.store.vacuum() } @@ -667,6 +1274,7 @@ impl BufferManager { mutation: std::array::from_fn(|_| Mutex::new(MutationState::default())), write_delta: WriteDelta::default(), write_delta_flush: Mutex::new(()), + checkpoint_io: Mutex::new(()), delete_fence_total: AtomicUsize::new(0), compact_candidate_cursor: AtomicUsize::new(0), merge_candidate_cursor: AtomicUsize::new(0), @@ -677,6 +1285,11 @@ impl BufferManager { current_epoch: AtomicU64::new(1), fork_barrier: AtomicU64::new(0), snapshots: Mutex::new(SnapshotState::default()), + gc_epoch: AtomicU64::new(0), + physical_gc: Mutex::new(()), + gc_reclaimed_count: AtomicU64::new(0), + gc_last_full_sweep_deferred_count: AtomicUsize::new(0), + checkpoint_waker: Mutex::new(None), } } @@ -684,6 +1297,17 @@ impl BufferManager { (self.alloc_uninit)() } + /// Enter the global checkpoint I/O phase. + /// + /// Lock order: capture/CommitGate first, release it, then acquire this + /// guard before any child-first writes or pending deletes. The guard must + /// cover the complete data-write → sync → delete → sync phase. + pub(crate) fn enter_checkpoint_io(&self) -> MutexGuard<'_, ()> { + self.checkpoint_io + .lock() + .expect("checkpoint I/O lock poisoned") + } + /// Current logical clock value. Read by the eviction /// thread to compare against each entry's `last_touched`. The /// returned tick is `Relaxed` — fine for "how cold is this @@ -782,9 +1406,8 @@ impl BufferManager { } /// Current number of cached blobs. - #[cfg(test)] #[must_use] - pub fn cached_count(&self) -> usize { + pub(crate) fn cached_count(&self) -> usize { self.cache.len() } @@ -794,6 +1417,11 @@ impl BufferManager { BufferStats { dirty_count: self.dirty_count(), pending_delete_count: self.pending_delete_count(), + gc_orphan_backlog_count: self.gc_orphan_backlog_count(), + gc_reclaimed_count: self.gc_reclaimed_count.load(Ordering::Relaxed), + gc_last_full_sweep_deferred_count: self + .gc_last_full_sweep_deferred_count + .load(Ordering::Relaxed), read_index_token_count: self.read_index_tokens.len(), read_index_cache_entries: read_index_cache.entries, read_index_cache_bytes: read_index_cache.bytes, @@ -931,14 +1559,6 @@ impl BufferManager { )) } - /// Internal: insert a freshly-loaded blob into the cache. - /// Idempotent under concurrent inserts. Stamps the new entry's - /// `last_touched` so it doesn't look cold to the eviction - /// thread on its very next sweep. - fn insert_into_cache(&self, guid: BlobGuid, contents: &AlignedBlobBuf) { - self.insert_owned_into_cache(guid, contents.clone(), PinAccess::Point); - } - /// Internal: insert a freshly-loaded owned blob into the cache /// without cloning its 512 KB payload. Used on store read /// misses so an allocator-provided registered buffer can become @@ -1131,6 +1751,7 @@ impl BufferManager { /// `delete_blob`, where the blob is going away entirely and /// any pending dirty write would race with the delete in the /// store. + #[cfg(test)] fn evict_from_cache(&self, guid: BlobGuid) -> bool { { let state = self.mutation_shard(guid).lock().unwrap(); @@ -1158,6 +1779,11 @@ impl BufferManager { true } + #[cfg(test)] + pub(crate) fn evict_from_cache_for_test(&self, guid: BlobGuid) -> bool { + self.evict_from_cache(guid) + } + /// Pin a blob in cache and return an `Arc` over it. /// /// On a cache miss, the blob is loaded from the inner store @@ -1183,10 +1809,7 @@ impl BufferManager { /// reading the backing store, so callers can decide whether to use /// read-index/page reads or fall back to a full [`Self::pin`]. pub(crate) fn pin_cached(&self, guid: BlobGuid) -> Result>> { - if self.is_pending_delete(guid) { - return Err(Self::pending_delete_not_found(guid)); - } - Ok(self.get_cached_with_access(guid, PinAccess::Point)) + self.pin_resident_stable(guid, PinAccess::Point) } /// Pin for range/list scans. Hits and misses remain visible in @@ -1223,13 +1846,16 @@ impl BufferManager { let mut miss_guids: Vec = Vec::new(); let mut miss_slots: Vec = Vec::new(); for &guid in guids { - if self.is_pending_delete(guid) { - out.push(None); - continue; - } - if let Some(entry) = self.get_cached_with_access(guid, PinAccess::Scan) { - out.push(Some(entry)); - continue; + match self.pin_resident_stable(guid, PinAccess::Scan) { + Ok(Some(entry)) => { + out.push(Some(entry)); + continue; + } + Err(_) => { + out.push(None); + continue; + } + Ok(None) => {} } out.push(None); miss_slots.push(out.len() - 1); @@ -1238,6 +1864,7 @@ impl BufferManager { if miss_guids.is_empty() { return out; } + let gc_epoch = self.gc_stable_read_epoch(); // Phase 2 — read the cold frames in one batched store call. // SAFETY: `read_blobs` fills every PAGE_SIZE frame whose slot @@ -1257,10 +1884,10 @@ impl BufferManager { } self.note_full_blob_read(PinAccess::Scan); let guid = miss_guids[i]; - if self.is_pending_delete(guid) { - continue; - } - out[miss_slots[i]] = Some(self.insert_owned_into_cache(guid, buf, PinAccess::Scan)); + out[miss_slots[i]] = self + .insert_loaded_after_gc(guid, buf, PinAccess::Scan, gc_epoch) + .ok() + .flatten(); } out } @@ -1282,21 +1909,73 @@ impl BufferManager { } fn pin_with_access(&self, guid: BlobGuid, access: PinAccess) -> Result> { - if self.is_pending_delete(guid) { - return Err(Self::pending_delete_not_found(guid)); + loop { + if let Some(entry) = self.pin_resident_stable(guid, access)? { + return Ok(entry); + } + let gc_epoch = self.gc_stable_read_epoch(); + if self.is_pending_delete(guid) { + return Err(Self::pending_delete_not_found(guid)); + } + // SAFETY: a successful read_blob fills the full PAGE_SIZE frame + // before `scratch` is inserted into the cache or read. + let mut scratch = self.alloc_blob_buf_uninit(); + let read = self.store.read_blob(guid, &mut scratch); + let _gc = self.physical_gc.lock().expect("physical GC lock poisoned"); + if self.gc_raced_since(gc_epoch) { + continue; + } + if self.is_pending_delete(guid) { + return Err(Self::pending_delete_not_found(guid)); + } + read?; + self.note_full_blob_read(access); + return Ok(self.insert_owned_into_cache(guid, scratch, access)); } - if let Some(entry) = self.get_cached_with_access(guid, access) { + } + + /// Return a resident Arc only after revalidating the physical-GC epoch + /// captured before the delete-fence check. Once cloned, the Arc itself + /// makes a later delete fail closed; the terminal epoch check covers a + /// delete that detached the map entry in the check-to-clone window. + fn pin_resident_stable( + &self, + guid: BlobGuid, + access: PinAccess, + ) -> Result>> { + loop { + let gc_epoch = self.gc_stable_read_epoch(); + if self.is_pending_delete(guid) { + return Err(Self::pending_delete_not_found(guid)); + } + #[cfg(test)] + pause_resident_pin_after_fence_check(); + let entry = self.get_cached_with_access(guid, access); + if self.gc_raced_since(gc_epoch) { + continue; + } + if self.is_pending_delete(guid) { + return Err(Self::pending_delete_not_found(guid)); + } return Ok(entry); } - // SAFETY: read_blob fills the full PAGE_SIZE frame before - // `scratch` is inserted into the cache or read. - let mut scratch = self.alloc_blob_buf_uninit(); - self.store.read_blob(guid, &mut scratch)?; - self.note_full_blob_read(access); + } + + fn insert_loaded_after_gc( + &self, + guid: BlobGuid, + contents: AlignedBlobBuf, + access: PinAccess, + captured_gc_epoch: u64, + ) -> Result>> { + let _gc = self.physical_gc.lock().expect("physical GC lock poisoned"); + if self.gc_raced_since(captured_gc_epoch) { + return Ok(None); + } if self.is_pending_delete(guid) { return Err(Self::pending_delete_not_found(guid)); } - Ok(self.insert_owned_into_cache(guid, scratch, access)) + Ok(Some(self.insert_owned_into_cache(guid, contents, access))) } /// Whether `guid` may be served by a cold, page-granular read @@ -1327,7 +2006,21 @@ impl BufferManager { byte_offset: u64, dst: &mut [u8], ) -> Result<()> { - self.store.read_blob_range(guid, byte_offset, dst) + loop { + let gc_epoch = self.gc_stable_read_epoch(); + if self.is_pending_delete(guid) { + return Err(Self::pending_delete_not_found(guid)); + } + let read = self.store.read_blob_range(guid, byte_offset, dst); + let _gc = self.physical_gc.lock().expect("physical GC lock poisoned"); + if self.gc_raced_since(gc_epoch) { + continue; + } + if self.is_pending_delete(guid) { + return Err(Self::pending_delete_not_found(guid)); + } + return read; + } } /// Fill `dst` with a cached 4 KiB indexed-read page. The caller must @@ -1740,32 +2433,40 @@ impl BufferManager { /// [`Self::snapshot_dirty`]. pub fn mark_dirty(&self, guid: BlobGuid, seq: u64) { let cached = self.get_cached_with_access(guid, PinAccess::Silent); - self.mark_dirty_with_hint(guid, seq, cached.as_deref()); + if self.mark_dirty_with_hint(guid, seq, cached.as_deref()) { + self.publish_parent_orphans(guid); + } } /// Same contract as [`Self::mark_dirty`], but the caller /// already holds the cached blob pin from the walker descent. /// This avoids a second DashMap lookup on the mutation hot path. pub(crate) fn mark_dirty_cached(&self, guid: BlobGuid, seq: u64, entry: &CachedBlob) { - self.mark_dirty_with_hint(guid, seq, Some(entry)); + if self.mark_dirty_with_hint(guid, seq, Some(entry)) { + self.publish_parent_orphans(guid); + } } - fn mark_dirty_with_hint(&self, guid: BlobGuid, seq: u64, cached: Option<&CachedBlob>) { + fn mark_dirty_with_hint(&self, guid: BlobGuid, seq: u64, cached: Option<&CachedBlob>) -> bool { let Some(cached) = cached else { // No dirty entry without the newer cache image: that // would violate I1 and make checkpoint unable to // snapshot the bytes it is asked to flush. - return; + return false; }; self.invalidate_indexed_reads(guid); let hint_covers_seq = !cached.dirty_hint_needs_map_publish(seq); let mut state = self.mutation_shard(guid).lock().unwrap(); - if state.has_delete_fence(&guid) { + // Logical unlink owns visibility and discards late writes. A + // transient physical-GC fence is different: GC may defer because + // this writer still pins the frame, so its acknowledged bytes must + // remain dirty for the next checkpoint. + if state.has_logical_delete_fence(&guid) { cached.clear_dirty_hint(); - return; + return false; } if hint_covers_seq && matches!(state.dirty.get(&guid), Some(cur) if *cur <= seq) { - return; + return true; } if hint_covers_seq { cached.clear_dirty_hint(); @@ -1776,6 +2477,7 @@ impl BufferManager { .entry(guid) .and_modify(|cur| *cur = (*cur).min(seq)) .or_insert(seq); + true } /// Drain the current dirty entries from every bookkeeping shard, @@ -1793,25 +2495,41 @@ impl BufferManager { pub fn snapshot_dirty(&self) -> HashMap { let mut out = HashMap::new(); for shard in &self.mutation { - let mut state = shard.lock().unwrap(); - for (guid, seq) in &mut state.dirty { + let (claimed, logically_deleted) = { + let mut state = shard.lock().unwrap(); + let snap = std::mem::take(&mut state.dirty); + let mut claimed = Vec::new(); + let mut logically_deleted = Vec::new(); + for (guid, seq) in snap { + if state.has_logical_delete_fence(&guid) { + logically_deleted.push(guid); + } else if state.gc_deleting.contains(&guid) { + // Transient physical GC may still defer on a pin. + // Keep the row in `dirty` and out of flushing. + state.dirty.insert(guid, seq); + } else { + state.add_flushing(guid); + claimed.push((guid, seq)); + } + } + (claimed, logically_deleted) + }; + + // Never hold a mutation shard while touching DashMap: eviction + // and GC remove cache→mutation, so the inverse order deadlocks. + for guid in logically_deleted { + if let Some(entry) = self.get_cached_with_access(guid, PinAccess::Silent) { + entry.clear_dirty_hint(); + } + } + for (guid, mut seq) in claimed { if let Some(hinted_seq) = self - .get_cached_with_access(*guid, PinAccess::Silent) + .get_cached_with_access(guid, PinAccess::Silent) .and_then(|entry| entry.take_dirty_hint()) { - *seq = (*seq).min(hinted_seq); + seq = seq.min(hinted_seq); } - } - let snap = std::mem::take(&mut state.dirty); - for (guid, seq) in snap { - if state.has_delete_fence(&guid) { - if let Some(entry) = self.get_cached_with_access(guid, PinAccess::Silent) { - entry.clear_dirty_hint(); - } - continue; - } - state.add_flushing(guid); - out.insert(guid, seq); + out.insert(guid, seq); } } out @@ -1858,7 +2576,7 @@ impl BufferManager { let _ = entry.dirty_hint_needs_map_publish(t); } let mut state = self.mutation_shard(guid).lock().unwrap(); - if state.has_delete_fence(&guid) { + if state.has_logical_delete_fence(&guid) { if let Some(entry) = cached { entry.clear_dirty_hint(); } @@ -1917,9 +2635,15 @@ impl BufferManager { /// /// The checkpoint round drains this set after Sync. User WAL /// deletes remove the blob from the store manifest and then - /// re-Sync; `STRUCTURAL_SEQ` deletes only retire the fence and - /// leave the old blob as an orphan for a future reachability GC. + /// re-Sync. `STRUCTURAL_SEQ` never enters this visibility fence: + /// a copied snapshot root may still lazily load the detached child. + /// Structural children instead wait in snapshot-protected orphan + /// state and become exact reclaim candidates at a clean frontier. pub fn mark_for_delete(&self, guid: BlobGuid, seq: u64) { + assert_ne!( + seq, STRUCTURAL_SEQ, + "STRUCTURAL_SEQ requires parent-scoped stage_structural_reclaim" + ); let mut state = self.mutation_shard(guid).lock().unwrap(); if let Some(seq_ref) = state.deleting.get_mut(&guid) { *seq_ref = (*seq_ref).min(seq); @@ -1992,7 +2716,11 @@ impl BufferManager { for (g, t) in entries { let mut state = self.mutation_shard(g).lock().unwrap(); let mut seq = t; - let had_fence = state.has_delete_fence(&g); + // `delete_fence_total` counts the transient GC fence and the + // logical pending/deleting fence independently. Restoring a + // logical row while GC owns the same GUID must therefore add one; + // only an existing logical row suppresses the increment. + let had_logical_fence = state.has_logical_delete_fence(&g); if let Some(claimed) = state.deleting.remove(&g) { seq = seq.min(claimed); } @@ -2003,7 +2731,7 @@ impl BufferManager { } Entry::Vacant(entry) => { entry.insert(seq); - if !had_fence { + if !had_logical_fence { self.delete_fence_total.fetch_add(1, Ordering::AcqRel); } } @@ -2085,13 +2813,17 @@ impl BufferManager { self.merge_candidate_total.load(Ordering::Relaxed) } - /// Execute a queued delete fence. User WAL deletes mutate the - /// inner store manifest; `STRUCTURAL_SEQ` deletes only retire - /// the fence because structural merge/compact has no logical - /// WAL record proving that no durable parent still references - /// the child. Returns `Ok(false)` when the blob still has - /// dirty/flushing state and the caller should requeue it. + /// Execute a queued user-delete fence. Structural orphans must never enter + /// this queue: they use parent-scoped staging so a child cannot become + /// reclaimable before its rewritten parent is published dirty. Returns + /// `Ok(false)` when the blob still has dirty/flushing state and the caller + /// should requeue it. pub(crate) fn execute_pending_delete(&self, guid: BlobGuid, seq: u64) -> Result { + if seq == STRUCTURAL_SEQ { + return Err(Error::Internal( + "STRUCTURAL_SEQ bypassed parent-scoped staging", + )); + } { let state = self.mutation_shard(guid).lock().unwrap(); if state.is_protected(&guid) { @@ -2109,9 +2841,7 @@ impl BufferManager { } else if self.cache.contains_key(&guid) { return Ok(false); } - if seq != STRUCTURAL_SEQ { - self.store.delete_blob(guid)?; - } + self.store.delete_blob(guid)?; self.route_resident.remove(guid); self.finish_pending_delete(guid); self.remove_read_index_token_if_unreachable(guid); @@ -2136,6 +2866,13 @@ impl BufferManager { self.store.has_blob(guid) } + /// Snapshot the GUIDs known by the inner store, bypassing cache-only WAL + /// replay state. DB open uses this only to distinguish a truly fresh + /// store from a corrupted existing store whose catalog root is missing. + pub(crate) fn store_blob_guids(&self) -> Result> { + self.store.list_blobs() + } + /// `true` iff `guid` is visible in the store and the store does /// not owe a data/manifest flush. /// @@ -2163,25 +2900,6 @@ impl BufferManager { state.dirty.contains_key(&guid) || state.flushing.contains_key(&guid) } - /// Snapshot the cached bytes for `guid` into a freshly allocated - /// `AlignedBlobBuf`. Returns `None` if the blob isn't cached. - /// - /// Used by the background checkpointer to hand off bytes to - /// the I/O worker thread without keeping the shared read guard - /// open across the actual `store.write_blob` call. The read - /// guard is held only for the duration of the 512 KB memcpy, so - /// writers don't block on long-running (especially io_uring) - /// I/O. - pub(crate) fn snapshot_bytes(&self, guid: BlobGuid) -> Option { - let entry = self.get_cached_with_access(guid, PinAccess::Silent)?; - let buf = entry.read(); - // SAFETY: copy_from_slice below writes the full PAGE_SIZE - // frame before `out` is returned. - let mut out = self.alloc_blob_buf_uninit(); - out.as_mut_slice().copy_from_slice(buf.as_slice()); - Some(out) - } - /// Clone cached bytes only when the blob still has the /// checkpoint-captured content version. /// @@ -2256,8 +2974,9 @@ impl BufferManager { } for idx in write_indices { let entry = &entries[idx]; - self.retire_write_through(entry.guid, entry.expected_seq); - statuses[idx] = WriteThroughStatus::Written; + if self.retire_write_through(entry.guid, entry.expected_seq, entry.content_version)? { + statuses[idx] = WriteThroughStatus::Written; + } } Ok(WriteThroughBatchReport { statuses }) } @@ -2274,8 +2993,30 @@ impl BufferManager { Ok(cached.validate_content_version(version)) } - fn retire_write_through(&self, guid: BlobGuid, expected_seq: u64) { + fn retire_write_through( + &self, + guid: BlobGuid, + expected_seq: u64, + content_version: Option, + ) -> Result { + let cached = + if content_version.is_some() { + Some(self.get_cached_with_access(guid, PinAccess::Silent).ok_or( + Error::Internal("retire_write_through: flushing entry lost cache image"), + )?) + } else { + None + }; let mut state = self.mutation_shard(guid).lock().unwrap(); + // Revalidate while serializing with dirty publication. The first + // check happens before store I/O; a writer can still update the + // cached frame and publish a lower WAL seq in that I/O window. Such + // bytes were not written by this batch and must remain dirty. + if let (Some(expected), Some(cached)) = (content_version, cached.as_ref()) { + if !cached.validate_content_version(expected) { + return Ok(false); + } + } if expected_seq != STRUCTURAL_SEQ { if let std::collections::hash_map::Entry::Occupied(e) = state.dirty.entry(guid) { // Only retire the entry when no racing writer has @@ -2296,6 +3037,7 @@ impl BufferManager { entry.clear_dirty_hint(); } } + Ok(true) } fn try_publish_read_index(&self, guid: BlobGuid, bytes: &AlignedBlobBuf) { @@ -2330,6 +3072,169 @@ impl BufferManager { self.store.flush() } + /// Write one public `BlobStore` image through to the inner store while + /// preserving the cache's authoritative image on failure. + /// + /// The caller owns `checkpoint_io`. A resident frame is pinned and write + /// latched before entering the physical-GC epoch: a walker may hold that + /// same frame latch while pinning a child, so taking the locks in the + /// opposite order would deadlock it behind the odd GC generation. + fn write_blob_through_locked(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { + loop { + let _stable = self.gc_stable_read_epoch(); + if self.is_pending_delete(guid) { + return Err(Self::pending_delete_not_found(guid)); + } + + if let Some(entry) = self.get_cached_with_access(guid, PinAccess::Silent) { + // The Arc prevents reachability GC from reclaiming this frame + // while we wait for its writer latch. + let mut cached = entry.write(); + let captured_version = entry.content_version(); + let _epoch = self.begin_gc_epoch(); + if self.is_pending_delete(guid) { + return Err(Self::pending_delete_not_found(guid)); + } + self.invalidate_indexed_reads(guid); + self.store.write_blob(guid, src)?; + + // No other cache writer can change the frame while the latch + // is held. Update it only after store success so a failed + // write never exposes bytes that did not reach the store. + debug_assert_eq!(entry.content_version(), captured_version); + cached.as_mut_slice().copy_from_slice(src.as_slice()); + entry.clear_dirty_hint(); + let mut state = self.mutation_shard(guid).lock().unwrap(); + state.remove_unclaimed_dirty(&guid); + let removed = state.remove_maintenance_candidates(&guid); + drop(state); + self.decrement_candidate_totals(removed); + return Ok(()); + } + + // A cold reader can install the blob between the miss above and + // this epoch. Recheck under `physical_gc`; if it won that race, + // release the epoch and take its frame latch in the next attempt. + let _epoch = self.begin_gc_epoch(); + if self.cache.contains_key(&guid) { + continue; + } + if self.is_pending_delete(guid) { + return Err(Self::pending_delete_not_found(guid)); + } + self.invalidate_indexed_reads(guid); + self.store.write_blob(guid, src)?; + let mut state = self.mutation_shard(guid).lock().unwrap(); + state.remove_unclaimed_dirty(&guid); + let removed = state.remove_maintenance_candidates(&guid); + drop(state); + self.decrement_candidate_totals(removed); + return Ok(()); + } + } + + /// Delete one public `BlobStore` GUID through an odd physical-GC epoch. + /// The cached Arc is detached before inner I/O, closing the window where + /// a reader that passed its first fence check could clone a soon-deleted + /// resident entry. Failure reinstalls the same Arc before retiring the + /// epoch and restores every dirty seq published behind the transient + /// fence. + fn delete_blob_through_locked(&self, guid: BlobGuid) -> Result<()> { + let _epoch = self.begin_gc_epoch(); + let claimed_dirty = { + let mut state = self.mutation_shard(guid).lock().unwrap(); + if state.checkpoint_owned_or_pending(&guid) { + return Err(Error::Internal( + "delete_blob: checkpoint or delete fence owns blob", + )); + } + // Claim unflushed debt atomically with the delete fence. A + // concurrent checkpoint planner now sees neither a dirty row it + // could discard because of the fence nor an unfenced row it could + // write after this delete. Failure restores this exact seq. + let claimed_dirty = state.dirty.remove(&guid); + state.gc_deleting.insert(guid); + self.delete_fence_total.fetch_add(1, Ordering::AcqRel); + claimed_dirty + }; + + let detached_entry = self + .cache + .remove_if(&guid, |_, entry| Arc::strong_count(entry) == 1); + let cache_claim_failed = detached_entry.is_none() && self.cache.contains_key(&guid); + let result = (|| { + if cache_claim_failed { + return Err(Error::Internal( + "delete_blob: protected cache image cannot be evicted", + )); + } + self.invalidate_indexed_reads(guid); + self.store.delete_blob(guid)?; + + if let Some((_, entry)) = &detached_entry { + entry.clear_dirty_hint(); + } + self.route_resident.remove(guid); + let mut state = self.mutation_shard(guid).lock().unwrap(); + state.remove_unclaimed_dirty(&guid); + let removed = state.remove_maintenance_candidates(&guid); + drop(state); + self.decrement_candidate_totals(removed); + Ok(()) + })(); + + // Do not take the DashMap shard while holding the mutation shard. + // A logical delete may win this gap; the final mutation-shard check + // below detects that handoff and suppresses dirty resurrection. + let logical_delete_before_reinsert = if result.is_err() { + self.mutation_shard(guid) + .lock() + .unwrap() + .has_logical_delete_fence(&guid) + } else { + false + }; + if result.is_err() && !logical_delete_before_reinsert { + if let Some((_, entry)) = &detached_entry { + self.cache.entry(guid).or_insert_with(|| Arc::clone(entry)); + } + } + let claimed_entry = if result.is_err() { + detached_entry + .as_ref() + .map(|(_, entry)| Arc::clone(entry)) + .or_else(|| self.get_cached_with_access(guid, PinAccess::Silent)) + } else { + None + }; + let mut state = self.mutation_shard(guid).lock().unwrap(); + if state.gc_deleting.remove(&guid) { + self.delete_fence_total.fetch_sub(1, Ordering::AcqRel); + } + let logical_delete_owned = state.has_logical_delete_fence(&guid); + if result.is_err() { + if let (Some(seq), false) = (claimed_dirty, logical_delete_owned) { + if let Some(entry) = &claimed_entry { + let _ = entry.dirty_hint_needs_map_publish(seq); + } + state + .dirty + .entry(guid) + .and_modify(|current| *current = (*current).min(seq)) + .or_insert(seq); + } else if logical_delete_owned { + if let Some(entry) = &claimed_entry { + entry.clear_dirty_hint(); + } + } + } + drop(state); + if result.is_ok() { + self.remove_read_index_token_if_unreachable(guid); + } + result + } + /// Stage a freshly-created blob in cache and tag it dirty at /// `seq` — the unified `mark_dirty → checkpoint round → store /// write` protocol takes ownership from there. @@ -2386,213 +3291,1223 @@ impl BufferManager { impl BlobStore for BufferManager { fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> Result<()> { - if self.is_pending_delete(guid) { - return Err(Self::pending_delete_not_found(guid)); - } - // Cache hit? - if let Some(entry) = self.get_cached_with_access(guid, PinAccess::Point) { - let buf = entry.read(); - dst.as_mut_slice().copy_from_slice(buf.as_slice()); - return Ok(()); - } - // Cache miss — load from inner store and cache. - self.store.read_blob(guid, dst)?; - if self.is_pending_delete(guid) { - return Err(Self::pending_delete_not_found(guid)); - } - self.insert_into_cache(guid, dst); + // `pin` is the authoritative full-frame read path: a cold load is + // published only under a stable physical-GC generation, and the + // returned strong reference keeps GC from reclaiming the frame while + // its bytes are copied to the caller. + let pin = self.pin(guid)?; + let bytes = pin.read(); + dst.as_mut_slice().copy_from_slice(bytes.as_slice()); Ok(()) } fn read_blob_range(&self, guid: BlobGuid, byte_offset: u64, dst: &mut [u8]) -> Result<()> { - if self.is_pending_delete(guid) { - return Err(Self::pending_delete_not_found(guid)); - } - self.store.read_blob_range(guid, byte_offset, dst) + // Public BlobStore reads must observe a newer dirty cache image. The + // inherent range helper is intentionally a cold-store fast path used + // only after indexed-read eligibility has ruled that case out. + let pin = self.pin(guid)?; + let bytes = pin.read(); + let start = usize::try_from(byte_offset).map_err(|_| { + Error::BlobStoreIo(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "blob range offset exceeds addressable memory", + )) + })?; + let end = start.checked_add(dst.len()).ok_or_else(|| { + Error::BlobStoreIo(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "blob range end overflow", + )) + })?; + let src = bytes.as_slice().get(start..end).ok_or_else(|| { + Error::BlobStoreIo(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "blob range exceeds frame", + )) + })?; + dst.copy_from_slice(src); + Ok(()) } fn read_index_range(&self, guid: BlobGuid, byte_offset: u64, dst: &mut [u8]) -> Result { - self.store.read_index_range(guid, byte_offset, dst) + loop { + let gc_epoch = self.gc_stable_read_epoch(); + if !self.read_index_eligible(guid, ReadIndexPolicy::PointRead) { + return Ok(false); + } + let read = self.store.read_index_range(guid, byte_offset, dst); + let _gc = self.physical_gc.lock().expect("physical GC lock poisoned"); + if self.gc_raced_since(gc_epoch) { + continue; + } + if !self.read_index_eligible(guid, ReadIndexPolicy::PointRead) { + return Ok(false); + } + return read; + } + } + + fn read_value_segment_range( + &self, + guid: BlobGuid, + byte_offset: u64, + dst: &mut [u8], + ) -> Result { + loop { + let gc_epoch = self.gc_stable_read_epoch(); + if !self.read_index_eligible(guid, ReadIndexPolicy::PointRead) { + return Ok(false); + } + let read = self.store.read_value_segment_range(guid, byte_offset, dst); + let _gc = self.physical_gc.lock().expect("physical GC lock poisoned"); + if self.gc_raced_since(gc_epoch) { + continue; + } + if !self.read_index_eligible(guid, ReadIndexPolicy::PointRead) { + return Ok(false); + } + return read; + } } fn publish_read_index(&self, guid: BlobGuid, bytes: &[u8], values: &[u8]) -> Result<()> { + let _checkpoint_io = self.enter_checkpoint_io(); + let _epoch = self.begin_gc_epoch(); + if self.is_pending_delete(guid) { + return Err(Self::pending_delete_not_found(guid)); + } + self.invalidate_indexed_reads(guid); self.store.publish_read_index(guid, bytes, values) } fn delete_read_index(&self, guid: BlobGuid) -> Result<()> { + let _checkpoint_io = self.enter_checkpoint_io(); + let _epoch = self.begin_gc_epoch(); self.invalidate_indexed_reads(guid); self.store.delete_read_index(guid) } fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { + let _checkpoint_io = self.enter_checkpoint_io(); + self.write_blob_through_locked(guid, src) + } + + fn write_blobs(&self, writes: &[(BlobGuid, &AlignedBlobBuf)]) -> Result<()> { + let _checkpoint_io = self.enter_checkpoint_io(); + // Preserve the trait's arbitrary-prefix failure contract while + // keeping each successful prefix entry's cache identical to store. + for (guid, src) in writes { + self.write_blob_through_locked(*guid, src)?; + } + Ok(()) + } + + fn delete_blob(&self, guid: BlobGuid) -> Result<()> { + let _checkpoint_io = self.enter_checkpoint_io(); + self.delete_blob_through_locked(guid) + } + + fn list_blobs(&self) -> Result> { + let _gc = self.physical_gc.lock().expect("physical GC lock poisoned"); + self.store.list_blobs() + } + + fn flush(&self) -> Result<()> { + let _checkpoint_io = self.enter_checkpoint_io(); + let _gc = self.physical_gc.lock().expect("physical GC lock poisoned"); + self.store.flush() + } + + fn needs_flush(&self) -> bool { + let _gc = self.physical_gc.lock().expect("physical GC lock poisoned"); + self.store.needs_flush() + } + + fn has_blob(&self, guid: BlobGuid) -> Result { + let _gc = self.physical_gc.lock().expect("physical GC lock poisoned"); if self.is_pending_delete(guid) { - return Err(Self::pending_delete_not_found(guid)); + return Ok(false); } - self.invalidate_indexed_reads(guid); - // Transparent write-through: if cached, refresh the - // cached image; either way, always write to the inner - // store in the same call so durability is unchanged. - if let Some(entry) = self.get_cached_with_access(guid, PinAccess::Silent) { - let mut buf = entry.write(); - buf.as_mut_slice().copy_from_slice(src.as_slice()); - entry.clear_dirty_hint(); + // Fast path: shard-local check without consulting the + // inner store. + if self.cache.contains_key(&guid) { + return Ok(true); } - self.store.write_blob(guid, src)?; - // BlobStore now holds these exact bytes; any pending dirty - // entry for this blob is satisfied. Subsequent writes via - // the pin/write-guard path will re-mark it. - let mut state = self.mutation_shard(guid).lock().unwrap(); - state.remove_unclaimed_dirty(&guid); - let removed = state.remove_maintenance_candidates(&guid); - drop(state); - self.decrement_candidate_totals(removed); - Ok(()) + self.store.has_blob(guid) + } + + fn store_stats(&self) -> StoreStats { + let _gc = self.physical_gc.lock().expect("physical GC lock poisoned"); + self.store.store_stats() + } + + fn vacuum(&self) -> Result { + let _checkpoint_io = self.enter_checkpoint_io(); + let _epoch = self.begin_gc_epoch(); + self.store.vacuum() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::blob_store::{FileBlobStore, MemoryBlobStore}; + use crate::store::{BlobFrame, PAGE_4K}; + use std::sync::atomic::{AtomicBool, AtomicUsize}; + use std::sync::Barrier; + + fn make_buf(byte_at_100: u8) -> AlignedBlobBuf { + let mut b = AlignedBlobBuf::zeroed(); + b.as_mut_slice()[100] = byte_at_100; + b + } + + fn snapshot_current_bytes(bm: &BufferManager, guid: BlobGuid) -> AlignedBlobBuf { + let pin = bm.pin(guid).expect("test blob must be pinnable"); + bm.snapshot_bytes_if_version(guid, pin.content_version()) + .expect("test snapshot must keep its cache image") + .expect("test snapshot version must stay current") + } + + fn persist_all_dirty_for_test(bm: &BufferManager) { + let dirty = bm.snapshot_dirty(); + let versioned = bm.snapshot_dirty_versions(&dirty).unwrap(); + let entries: Vec<_> = versioned + .into_iter() + .map(|snapshot| WriteThroughEntry { + guid: snapshot.guid, + bytes: bm + .snapshot_bytes_if_version(snapshot.guid, snapshot.content_version) + .unwrap() + .unwrap(), + expected_seq: snapshot.expected_seq, + content_version: Some(snapshot.content_version), + }) + .collect(); + let _checkpoint_io = bm.enter_checkpoint_io(); + let report = bm.write_through_batch(&entries).unwrap(); + assert!(report + .statuses + .iter() + .all(|status| *status == WriteThroughStatus::Written),); + bm.flush_inner().unwrap(); + } + + struct FlushPendingStore { + inner: MemoryBlobStore, + pending: AtomicBool, + } + + impl FlushPendingStore { + fn new() -> Self { + Self { + inner: MemoryBlobStore::new(), + pending: AtomicBool::new(false), + } + } + } + + impl BlobStore for FlushPendingStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> Result<()> { + self.inner.read_blob(guid, dst) + } + + fn read_blobs(&self, guids: &[BlobGuid], dsts: &mut [AlignedBlobBuf]) -> Vec> { + self.inner.read_blobs(guids, dsts) + } + + fn read_blob_range(&self, guid: BlobGuid, byte_offset: u64, dst: &mut [u8]) -> Result<()> { + self.inner.read_blob_range(guid, byte_offset, dst) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { + self.pending.store(true, Ordering::Release); + self.inner.write_blob(guid, src) + } + + fn write_blobs(&self, writes: &[(BlobGuid, &AlignedBlobBuf)]) -> Result<()> { + self.pending.store(true, Ordering::Release); + self.inner.write_blobs(writes) + } + + fn delete_blob(&self, guid: BlobGuid) -> Result<()> { + self.pending.store(true, Ordering::Release); + self.inner.delete_blob(guid) + } + + fn list_blobs(&self) -> Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> Result<()> { + self.inner.flush()?; + self.pending.store(false, Ordering::Release); + Ok(()) + } + + fn needs_flush(&self) -> bool { + self.pending.load(Ordering::Acquire) || self.inner.needs_flush() + } + } + + struct BlockingReadStore { + inner: MemoryBlobStore, + block_once: AtomicBool, + entered: Barrier, + release: Barrier, + } + + impl BlockingReadStore { + fn new(inner: MemoryBlobStore) -> Self { + Self { + inner, + block_once: AtomicBool::new(true), + entered: Barrier::new(2), + release: Barrier::new(2), + } + } + } + + struct BlockingWriteStore { + inner: MemoryBlobStore, + block_once: AtomicBool, + entered: Barrier, + release: Barrier, + } + + impl BlockingWriteStore { + fn new(inner: MemoryBlobStore) -> Self { + Self { + inner, + block_once: AtomicBool::new(true), + entered: Barrier::new(2), + release: Barrier::new(2), + } + } + } + + impl BlobStore for BlockingWriteStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> Result<()> { + self.inner.read_blob(guid, dst) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { + self.inner.write_blob(guid, src) + } + + fn write_blobs_with_data_sync(&self, writes: &[(BlobGuid, &AlignedBlobBuf)]) -> Result<()> { + if self.block_once.swap(false, Ordering::AcqRel) { + self.entered.wait(); + self.release.wait(); + } + self.inner.write_blobs(writes)?; + self.inner.flush() + } + + fn delete_blob(&self, guid: BlobGuid) -> Result<()> { + self.inner.delete_blob(guid) + } + + fn list_blobs(&self) -> Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> Result<()> { + self.inner.flush() + } + + fn needs_flush(&self) -> bool { + self.inner.needs_flush() + } + + fn has_blob(&self, guid: BlobGuid) -> Result { + self.inner.has_blob(guid) + } + } + + impl BlobStore for BlockingReadStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> Result<()> { + self.inner.read_blob(guid, dst)?; + if self.block_once.swap(false, Ordering::AcqRel) { + self.entered.wait(); + self.release.wait(); + } + Ok(()) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { + self.inner.write_blob(guid, src) + } + + fn delete_blob(&self, guid: BlobGuid) -> Result<()> { + self.inner.delete_blob(guid) + } + + fn list_blobs(&self) -> Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> Result<()> { + self.inner.flush() + } + + fn needs_flush(&self) -> bool { + self.inner.needs_flush() + } + } + + struct BlockingTraitWriteStore { + inner: MemoryBlobStore, + block_once: AtomicBool, + entered: Barrier, + release: Barrier, + } + + impl BlockingTraitWriteStore { + fn new(inner: MemoryBlobStore) -> Self { + Self { + inner, + block_once: AtomicBool::new(true), + entered: Barrier::new(2), + release: Barrier::new(2), + } + } + } + + impl BlobStore for BlockingTraitWriteStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> Result<()> { + self.inner.read_blob(guid, dst) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { + if self.block_once.swap(false, Ordering::AcqRel) { + self.entered.wait(); + self.release.wait(); + } + self.inner.write_blob(guid, src) + } + + fn delete_blob(&self, guid: BlobGuid) -> Result<()> { + self.inner.delete_blob(guid) + } + + fn list_blobs(&self) -> Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> Result<()> { + self.inner.flush() + } + + fn needs_flush(&self) -> bool { + self.inner.needs_flush() + } + + fn has_blob(&self, guid: BlobGuid) -> Result { + self.inner.has_blob(guid) + } + } + + struct FailingMutationStore { + inner: MemoryBlobStore, + fail_writes: AtomicBool, + fail_deletes: AtomicBool, + block_delete_once: AtomicBool, + delete_entered: Barrier, + delete_release: Barrier, + } + + impl FailingMutationStore { + fn new(inner: MemoryBlobStore) -> Self { + Self { + inner, + fail_writes: AtomicBool::new(false), + fail_deletes: AtomicBool::new(false), + block_delete_once: AtomicBool::new(false), + delete_entered: Barrier::new(2), + delete_release: Barrier::new(2), + } + } + } + + impl BlobStore for FailingMutationStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> Result<()> { + self.inner.read_blob(guid, dst) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { + if self.fail_writes.load(Ordering::Acquire) { + return Err(Error::BlobStoreIo(std::io::Error::other( + "injected write failure", + ))); + } + self.inner.write_blob(guid, src) + } + + fn delete_blob(&self, guid: BlobGuid) -> Result<()> { + if self.block_delete_once.swap(false, Ordering::AcqRel) { + self.delete_entered.wait(); + self.delete_release.wait(); + } + if self.fail_deletes.load(Ordering::Acquire) { + return Err(Error::BlobStoreIo(std::io::Error::other( + "injected delete failure", + ))); + } + self.inner.delete_blob(guid) + } + + fn list_blobs(&self) -> Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> Result<()> { + self.inner.flush() + } + + fn needs_flush(&self) -> bool { + self.inner.needs_flush() + } + + fn has_blob(&self, guid: BlobGuid) -> Result { + self.inner.has_blob(guid) + } + } + + struct FailNthWriteStore { + inner: MemoryBlobStore, + fail_on: usize, + writes: AtomicUsize, + } + + struct FailAfterDeleteStore { + inner: MemoryBlobStore, + fail_once: AtomicBool, + } + + impl FailAfterDeleteStore { + fn new(inner: MemoryBlobStore) -> Self { + Self { + inner, + fail_once: AtomicBool::new(true), + } + } + } + + impl BlobStore for FailAfterDeleteStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> Result<()> { + self.inner.read_blob(guid, dst) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { + self.inner.write_blob(guid, src) + } + + fn delete_blob(&self, guid: BlobGuid) -> Result<()> { + self.inner.delete_blob(guid)?; + if self.fail_once.swap(false, Ordering::AcqRel) { + return Err(Error::BlobStoreIo(std::io::Error::other( + "injected post-delete failure", + ))); + } + Ok(()) + } + + fn list_blobs(&self) -> Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> Result<()> { + self.inner.flush() + } + + fn needs_flush(&self) -> bool { + self.inner.needs_flush() + } + + fn has_blob(&self, guid: BlobGuid) -> Result { + self.inner.has_blob(guid) + } + } + + impl FailNthWriteStore { + fn new(inner: MemoryBlobStore, fail_on: usize) -> Self { + Self { + inner, + fail_on, + writes: AtomicUsize::new(0), + } + } + } + + impl BlobStore for FailNthWriteStore { + fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> Result<()> { + self.inner.read_blob(guid, dst) + } + + fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { + let ordinal = self.writes.fetch_add(1, Ordering::AcqRel) + 1; + if ordinal == self.fail_on { + return Err(Error::BlobStoreIo(std::io::Error::other( + "injected batch-prefix failure", + ))); + } + self.inner.write_blob(guid, src) + } + + fn delete_blob(&self, guid: BlobGuid) -> Result<()> { + self.inner.delete_blob(guid) + } + + fn list_blobs(&self) -> Result> { + self.inner.list_blobs() + } + + fn flush(&self) -> Result<()> { + self.inner.flush() + } + + fn needs_flush(&self) -> bool { + self.inner.needs_flush() + } + + fn has_blob(&self, guid: BlobGuid) -> Result { + self.inner.has_blob(guid) + } + } + + #[test] + fn cold_pin_does_not_reinsert_blob_after_gc_fence_retires() { + let guid = [0xA1; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(guid, &make_buf(7)).unwrap(); + let store = Arc::new(BlockingReadStore::new(inner)); + let bm = Arc::new(BufferManager::new(store.clone(), 4)); + + let reader_bm = Arc::clone(&bm); + let reader = std::thread::spawn(move || reader_bm.pin(guid)); + store.entered.wait(); + + let outcome = bm + .gc_sweep_unreachable_bounded(&HashSet::new(), usize::MAX) + .unwrap(); + assert_eq!(outcome.freed, 1); + assert!(outcome.complete); + store.release.wait(); + + assert!(reader.join().unwrap().is_err()); + assert_eq!(bm.cached_count(), 0, "stale read must not create a zombie"); + assert!(!store.has_blob(guid).unwrap()); + } + + #[test] + fn cold_pin_retries_after_unrelated_gc_epoch_change() { + let target = [0xA6; 16]; + let unreachable = [0xA7; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(target, &make_buf(7)).unwrap(); + inner.write_blob(unreachable, &make_buf(8)).unwrap(); + let store = Arc::new(BlockingReadStore::new(inner)); + let bm = Arc::new(BufferManager::new(store.clone(), 4)); + + let reader_bm = Arc::clone(&bm); + let reader = std::thread::spawn(move || reader_bm.pin(target)); + store.entered.wait(); + + let reachable = HashSet::from([target]); + let outcome = bm + .gc_sweep_unreachable_bounded(&reachable, usize::MAX) + .unwrap(); + assert_eq!(outcome.freed, 1); + assert!(outcome.complete); + store.release.wait(); + + let pin = reader + .join() + .unwrap() + .expect("unrelated GC must be retried inside the cold pin"); + assert_eq!(pin.read().as_slice()[100], 7); + assert!(store.has_blob(target).unwrap()); + assert!(!store.has_blob(unreachable).unwrap()); + } + + #[test] + fn trait_full_read_does_not_publish_gc_zombie() { + let guid = [0xB1; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(guid, &make_buf(7)).unwrap(); + let store = Arc::new(BlockingReadStore::new(inner)); + let bm = Arc::new(BufferManager::new(store.clone(), 4)); + + let reader_bm = Arc::clone(&bm); + let reader = std::thread::spawn(move || { + let mut dst = AlignedBlobBuf::zeroed(); + BlobStore::read_blob(reader_bm.as_ref(), guid, &mut dst) + }); + store.entered.wait(); + + let outcome = bm + .gc_sweep_unreachable_bounded(&HashSet::new(), usize::MAX) + .unwrap(); + assert_eq!(outcome.freed, 1); + store.release.wait(); + + assert!(reader.join().unwrap().is_err()); + assert_eq!(bm.cached_count(), 0); + assert!(!store.has_blob(guid).unwrap()); + } + + #[test] + fn trait_range_read_does_not_publish_gc_zombie() { + let guid = [0xB2; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(guid, &make_buf(7)).unwrap(); + let store = Arc::new(BlockingReadStore::new(inner)); + let bm = Arc::new(BufferManager::new(store.clone(), 4)); + + let reader_bm = Arc::clone(&bm); + let reader = std::thread::spawn(move || { + let mut dst = [0u8; 8]; + BlobStore::read_blob_range(reader_bm.as_ref(), guid, 96, &mut dst) + }); + store.entered.wait(); + + let outcome = bm + .gc_sweep_unreachable_bounded(&HashSet::new(), usize::MAX) + .unwrap(); + assert_eq!(outcome.freed, 1); + store.release.wait(); + + assert!(reader.join().unwrap().is_err()); + assert_eq!(bm.cached_count(), 0); + assert!(!store.has_blob(guid).unwrap()); + } + + #[test] + fn trait_range_read_observes_dirty_cache_image() { + let guid = [0xB3; 16]; + let inner: Arc = Arc::new(MemoryBlobStore::new()); + inner.write_blob(guid, &make_buf(1)).unwrap(); + let bm = BufferManager::new(inner, 4); + let pin = bm.pin(guid).unwrap(); + { + let mut bytes = pin.write(); + bytes.as_mut_slice()[100] = 9; + } + bm.mark_dirty_cached(guid, 10, pin.as_ref()); + + let mut dst = [0u8; 1]; + BlobStore::read_blob_range(&bm, guid, 100, &mut dst).unwrap(); + assert_eq!(dst, [9]); + } + + #[test] + fn trait_write_serializes_with_gc_epoch() { + let guid = [0xB4; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(guid, &make_buf(1)).unwrap(); + let store = Arc::new(BlockingTraitWriteStore::new(inner)); + let bm = Arc::new(BufferManager::new(store.clone(), 4)); + drop(bm.pin(guid).unwrap()); + + let writer_bm = Arc::clone(&bm); + let writer = std::thread::spawn(move || { + BlobStore::write_blob(writer_bm.as_ref(), guid, &make_buf(9)) + }); + store.entered.wait(); + + let gc_bm = Arc::clone(&bm); + let (done_tx, done_rx) = std::sync::mpsc::sync_channel(1); + let gc = std::thread::spawn(move || { + let result = gc_bm.gc_sweep_unreachable_bounded(&HashSet::from([guid]), usize::MAX); + done_tx.send(result).unwrap(); + }); + assert!( + done_rx + .recv_timeout(std::time::Duration::from_millis(100)) + .is_err(), + "GC must wait while the public write owns its odd physical epoch", + ); + + store.release.wait(); + writer.join().unwrap().unwrap(); + let outcome = done_rx + .recv_timeout(std::time::Duration::from_secs(2)) + .unwrap() + .unwrap(); + gc.join().unwrap(); + assert_eq!(outcome.freed, 0); + + let mut stored = AlignedBlobBuf::zeroed(); + store.read_blob(guid, &mut stored).unwrap(); + assert_eq!(stored.as_slice()[100], 9); + assert_eq!(bm.pin(guid).unwrap().read().as_slice()[100], 9); + } + + #[test] + fn trait_write_failure_preserves_cached_bytes_and_dirty_debt() { + let guid = [0xB5; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(guid, &make_buf(1)).unwrap(); + let store = Arc::new(FailingMutationStore::new(inner)); + let bm = BufferManager::new(store.clone(), 4); + let pin = bm.pin(guid).unwrap(); + { + let mut bytes = pin.write(); + bytes.as_mut_slice()[100] = 7; + } + bm.mark_dirty_cached(guid, 10, pin.as_ref()); + drop(pin); + store.fail_writes.store(true, Ordering::Release); + + assert!(BlobStore::write_blob(&bm, guid, &make_buf(9)).is_err()); + assert_eq!(bm.pin(guid).unwrap().read().as_slice()[100], 7); + assert_eq!(bm.dirty_count(), 1); + let mut stored = AlignedBlobBuf::zeroed(); + store.inner.read_blob(guid, &mut stored).unwrap(); + assert_eq!(stored.as_slice()[100], 1); + } + + #[test] + fn trait_delete_failure_restores_planner_hidden_dirty_and_reopens_latest_bytes() { + let guid = [0xB6; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(guid, &make_buf(1)).unwrap(); + let store = Arc::new(FailingMutationStore::new(inner)); + let bm = Arc::new(BufferManager::new(store.clone(), 4)); + let pin = bm.pin(guid).unwrap(); + { + let mut bytes = pin.write(); + bytes.as_mut_slice()[100] = 7; + } + bm.mark_dirty_cached(guid, 10, pin.as_ref()); + drop(pin); + store.fail_deletes.store(true, Ordering::Release); + store.block_delete_once.store(true, Ordering::Release); + + let delete_bm = Arc::clone(&bm); + let delete = std::thread::spawn(move || BlobStore::delete_blob(delete_bm.as_ref(), guid)); + store.delete_entered.wait(); + + assert!( + bm.snapshot_dirty().is_empty(), + "the public delete must hide its claimed dirty row from a planner", + ); + store.delete_release.wait(); + assert!(delete.join().unwrap().is_err()); + + assert_eq!(bm.pin(guid).unwrap().read().as_slice()[100], 7); + assert_eq!(bm.dirty_count(), 1); + assert_eq!(bm.pending_delete_count(), 0); + assert!(store.inner.has_blob(guid).unwrap()); + + persist_all_dirty_for_test(&bm); + drop(bm); + + let store_dyn: Arc = store; + let reopened = BufferManager::new(store_dyn, 4); + assert_eq!(reopened.pin(guid).unwrap().read().as_slice()[100], 7); + } + + #[test] + fn trait_delete_failure_defers_to_concurrent_logical_delete() { + let guid = [0xB9; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(guid, &make_buf(1)).unwrap(); + let store = Arc::new(FailingMutationStore::new(inner)); + let bm = Arc::new(BufferManager::new(store.clone(), 4)); + let pin = bm.pin(guid).unwrap(); + { + let mut bytes = pin.write(); + bytes.as_mut_slice()[100] = 7; + } + bm.mark_dirty_cached(guid, 10, pin.as_ref()); + drop(pin); + store.fail_deletes.store(true, Ordering::Release); + store.block_delete_once.store(true, Ordering::Release); + + let delete_bm = Arc::clone(&bm); + let delete = std::thread::spawn(move || BlobStore::delete_blob(delete_bm.as_ref(), guid)); + store.delete_entered.wait(); + bm.mark_for_delete(guid, 20); + assert_eq!( + bm.pending_delete_count(), + 2, + "generic and logical delete fences must be counted independently", + ); + store.delete_release.wait(); + assert!(delete.join().unwrap().is_err()); + + assert_eq!(bm.dirty_count(), 0, "logical deletion owns the old bytes"); + assert_eq!(bm.pending_delete_count(), 1); + let pending = bm.snapshot_pending_deletes(); + assert_eq!(pending.get(&guid), Some(&20)); + bm.restore_pending_deletes(pending); + assert_eq!(bm.pending_delete_count(), 1); + assert!(store.inner.has_blob(guid).unwrap()); + } + + #[test] + fn restored_logical_delete_counts_independently_from_transient_gc() { + let guid = [0xBE; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(guid, &make_buf(1)).unwrap(); + let store = Arc::new(FailingMutationStore::new(inner)); + let bm = Arc::new(BufferManager::new(store.clone(), 4)); + drop(bm.pin(guid).unwrap()); + store.fail_deletes.store(true, Ordering::Release); + store.block_delete_once.store(true, Ordering::Release); + + let delete_bm = Arc::clone(&bm); + let transient_delete = + std::thread::spawn(move || BlobStore::delete_blob(delete_bm.as_ref(), guid)); + store.delete_entered.wait(); + assert_eq!(bm.pending_delete_count(), 1); + + // Model a failed trailing checkpoint sync restoring a logical row + // while an unrelated transient delete fence still owns this GUID. + bm.restore_pending_deletes(HashMap::from([(guid, 17)])); + assert_eq!( + bm.pending_delete_count(), + 2, + "logical and transient fences require separate references", + ); + + store.delete_release.wait(); + assert!(transient_delete.join().unwrap().is_err()); + assert_eq!(bm.pending_delete_count(), 1); + + let claimed = bm.snapshot_pending_deletes(); + assert_eq!(claimed.get(&guid), Some(&17)); + store.fail_deletes.store(false, Ordering::Release); + assert!(bm.execute_pending_delete(guid, 17).unwrap()); + assert_eq!(bm.pending_delete_count(), 0); } - fn write_blobs(&self, writes: &[(BlobGuid, &AlignedBlobBuf)]) -> Result<()> { - for (guid, _) in writes { - if self.is_pending_delete(*guid) { - return Err(Self::pending_delete_not_found(*guid)); - } - } - for (guid, _) in writes { - self.invalidate_indexed_reads(*guid); - } - for (guid, src) in writes { - if let Some(entry) = self.get_cached_with_access(*guid, PinAccess::Silent) { - let mut buf = entry.write(); - buf.as_mut_slice().copy_from_slice(src.as_slice()); - entry.clear_dirty_hint(); - } - } - self.store.write_blobs(writes)?; - for (guid, _) in writes { - let mut state = self.mutation_shard(*guid).lock().unwrap(); - state.remove_unclaimed_dirty(guid); - let removed = state.remove_maintenance_candidates(guid); - drop(state); - self.decrement_candidate_totals(removed); - } - Ok(()) - } + #[test] + fn trait_delete_success_never_returns_detached_resident_pin() { + let guid = [0xBA; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(guid, &make_buf(1)).unwrap(); + let store = Arc::new(FailingMutationStore::new(inner)); + let bm = Arc::new(BufferManager::new(store.clone(), 4)); + drop(bm.pin(guid).unwrap()); + + let pin_barrier = Arc::new(ResidentPinBarrier::new()); + let reader_bm = Arc::clone(&bm); + let reader_barrier = Arc::clone(&pin_barrier); + let reader = std::thread::spawn(move || { + set_resident_pin_barrier_for_current_thread(reader_barrier); + reader_bm.pin(guid) + }); + pin_barrier.entered.wait(); - fn delete_blob(&self, guid: BlobGuid) -> Result<()> { - if self.is_pending_delete(guid) { - return Err(Self::pending_delete_not_found(guid)); - } - self.invalidate_indexed_reads(guid); - if !self.evict_from_cache(guid) { - return Err(Error::Internal( - "delete_blob: protected cache image cannot be evicted", - )); - } - self.store.delete_blob(guid)?; - self.remove_read_index_token_if_unreachable(guid); - Ok(()) - } + store.block_delete_once.store(true, Ordering::Release); + let delete_bm = Arc::clone(&bm); + let delete = std::thread::spawn(move || BlobStore::delete_blob(delete_bm.as_ref(), guid)); + store.delete_entered.wait(); + pin_barrier.release.wait(); + store.delete_release.wait(); - fn list_blobs(&self) -> Result> { - self.store.list_blobs() + delete.join().unwrap().unwrap(); + assert!( + reader.join().unwrap().is_err(), + "a reader that passed the first fence check must not return the detached Arc", + ); + assert!(!store.inner.has_blob(guid).unwrap()); + assert_eq!(bm.cached_count(), 0); } - fn flush(&self) -> Result<()> { - // Write-through mode: nothing pending in cache. - self.store.flush() + #[test] + fn trait_delete_failure_reinstalls_resident_before_reader_retry() { + let guid = [0xBB; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(guid, &make_buf(3)).unwrap(); + let store = Arc::new(FailingMutationStore::new(inner)); + let bm = Arc::new(BufferManager::new(store.clone(), 4)); + drop(bm.pin(guid).unwrap()); + + let pin_barrier = Arc::new(ResidentPinBarrier::new()); + let reader_bm = Arc::clone(&bm); + let reader_barrier = Arc::clone(&pin_barrier); + let reader = std::thread::spawn(move || { + set_resident_pin_barrier_for_current_thread(reader_barrier); + reader_bm.pin(guid) + }); + pin_barrier.entered.wait(); + + store.fail_deletes.store(true, Ordering::Release); + store.block_delete_once.store(true, Ordering::Release); + let delete_bm = Arc::clone(&bm); + let delete = std::thread::spawn(move || BlobStore::delete_blob(delete_bm.as_ref(), guid)); + store.delete_entered.wait(); + pin_barrier.release.wait(); + store.delete_release.wait(); + + assert!(delete.join().unwrap().is_err()); + let pin = reader + .join() + .unwrap() + .expect("failed delete must make the reinstated resident visible on retry"); + assert_eq!(pin.read().as_slice()[100], 3); + assert!(store.inner.has_blob(guid).unwrap()); } - fn needs_flush(&self) -> bool { - self.store.needs_flush() + #[test] + fn trait_delete_protected_writer_keeps_gc_fence_dirty_debt_through_reopen() { + let guid = [0xBC; 16]; + let inner: Arc = Arc::new(MemoryBlobStore::new()); + inner.write_blob(guid, &make_buf(1)).unwrap(); + let bm = Arc::new(BufferManager::new(inner.clone(), 4)); + let pin = bm.pin(guid).unwrap(); + { + let mut bytes = pin.write(); + bytes.as_mut_slice()[100] = 7; + } + + // Hold the cache shard so generic delete installs gc_deleting but + // cannot complete its atomic remove_if before this writer publishes. + let cache_ref = bm.cache.get(&guid).unwrap(); + let delete_bm = Arc::clone(&bm); + let delete = std::thread::spawn(move || BlobStore::delete_blob(delete_bm.as_ref(), guid)); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + while bm.pending_delete_count() == 0 { + assert!(std::time::Instant::now() < deadline, "delete fence timeout"); + std::thread::yield_now(); + } + + bm.mark_dirty_cached(guid, 30, pin.as_ref()); + assert!( + bm.snapshot_dirty().is_empty(), + "planner must leave transient-GC dirty rows in place", + ); + assert_eq!(bm.dirty_count(), 1); + drop(cache_ref); + assert!(delete.join().unwrap().is_err()); + drop(pin); + + assert_eq!(bm.pending_delete_count(), 0); + assert_eq!(bm.dirty_count(), 1); + persist_all_dirty_for_test(&bm); + drop(bm); + + let reopened = BufferManager::new(inner, 4); + assert_eq!(reopened.pin(guid).unwrap().read().as_slice()[100], 7); } - fn has_blob(&self, guid: BlobGuid) -> Result { - if self.is_pending_delete(guid) { - return Ok(false); + #[test] + fn reachability_gc_protected_writer_keeps_dirty_debt_through_reopen() { + let guid = [0xBD; 16]; + let inner: Arc = Arc::new(MemoryBlobStore::new()); + inner.write_blob(guid, &make_buf(1)).unwrap(); + let bm = Arc::new(BufferManager::new(inner.clone(), 4)); + let pin = bm.pin(guid).unwrap(); + + // Hold the cache shard while reachability GC publishes its transient + // fence. This gives the protected writer a deterministic window to + // update and publish dirty debt before remove_if observes its pin. + let cache_ref = bm.cache.get(&guid).unwrap(); + let gc_bm = Arc::clone(&bm); + let gc = std::thread::spawn(move || { + gc_bm.gc_sweep_unreachable_bounded(&HashSet::new(), usize::MAX) + }); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + while bm.pending_delete_count() == 0 { + assert!(std::time::Instant::now() < deadline, "GC fence timeout"); + std::thread::yield_now(); } - // Fast path: shard-local check without consulting the - // inner store. - if self.cache.contains_key(&guid) { - return Ok(true); + + { + let mut bytes = pin.write(); + bytes.as_mut_slice()[100] = 9; } - self.store.has_blob(guid) - } + bm.mark_dirty_cached(guid, 31, pin.as_ref()); + assert!( + bm.snapshot_dirty().is_empty(), + "planner must not claim a writer protected by transient GC", + ); + assert_eq!(bm.dirty_count(), 1); - fn store_stats(&self) -> StoreStats { - self.store.store_stats() - } + drop(cache_ref); + let outcome = gc.join().unwrap().unwrap(); + assert_eq!(outcome.freed, 0); + assert!(!outcome.complete); + assert_eq!(bm.pending_delete_count(), 0); + assert_eq!(bm.cached_count(), 1); + assert!(inner.has_blob(guid).unwrap()); + assert_eq!(bm.dirty_count(), 1); - fn vacuum(&self) -> Result { - self.store.vacuum() + drop(pin); + persist_all_dirty_for_test(&bm); + drop(bm); + + let reopened = BufferManager::new(inner, 4); + assert_eq!(reopened.pin(guid).unwrap().read().as_slice()[100], 9); } -} -#[cfg(test)] -mod tests { - use super::*; - use crate::store::blob_store::{FileBlobStore, MemoryBlobStore}; - use crate::store::{BlobFrame, PAGE_4K}; - use std::sync::atomic::{AtomicBool, AtomicUsize}; + #[test] + fn trait_batch_write_keeps_successful_prefix_cache_store_consistent() { + let first = [0xB7; 16]; + let second = [0xB8; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(first, &make_buf(1)).unwrap(); + inner.write_blob(second, &make_buf(2)).unwrap(); + let store = Arc::new(FailNthWriteStore::new(inner, 2)); + let bm = BufferManager::new(store.clone(), 4); + drop(bm.pin(first).unwrap()); + drop(bm.pin(second).unwrap()); + let first_new = make_buf(7); + let second_new = make_buf(8); - fn make_buf(byte_at_100: u8) -> AlignedBlobBuf { - let mut b = AlignedBlobBuf::zeroed(); - b.as_mut_slice()[100] = byte_at_100; - b + assert!( + BlobStore::write_blobs(&bm, &[(first, &first_new), (second, &second_new)]).is_err() + ); + assert_eq!(bm.pin(first).unwrap().read().as_slice()[100], 7); + assert_eq!(bm.pin(second).unwrap().read().as_slice()[100], 2); + let mut stored = AlignedBlobBuf::zeroed(); + store.inner.read_blob(first, &mut stored).unwrap(); + assert_eq!(stored.as_slice()[100], 7); + store.inner.read_blob(second, &mut stored).unwrap(); + assert_eq!(stored.as_slice()[100], 2); } - struct FlushPendingStore { - inner: MemoryBlobStore, - pending: AtomicBool, + #[test] + fn concurrent_gc_passes_keep_generation_serial_and_even() { + let inner: Arc = Arc::new(MemoryBlobStore::new()); + inner.write_blob([0xA2; 16], &make_buf(1)).unwrap(); + inner.write_blob([0xA3; 16], &make_buf(2)).unwrap(); + let bm = Arc::new(BufferManager::new(inner.clone(), 4)); + let start = Arc::new(Barrier::new(3)); + + let mut workers = Vec::new(); + for _ in 0..2 { + let bm = Arc::clone(&bm); + let start = Arc::clone(&start); + workers.push(std::thread::spawn(move || { + start.wait(); + bm.gc_sweep_unreachable_bounded(&HashSet::new(), usize::MAX) + .unwrap() + .freed + })); + } + start.wait(); + let total: usize = workers + .into_iter() + .map(|worker| worker.join().unwrap()) + .sum(); + + assert_eq!(total, 2); + assert_eq!(bm.gc_read_epoch(), 4); + assert_eq!(bm.gc_read_epoch() & 1, 0); + assert!(inner.list_blobs().unwrap().is_empty()); } - impl FlushPendingStore { - fn new() -> Self { - Self { - inner: MemoryBlobStore::new(), - pending: AtomicBool::new(false), - } - } - } + #[test] + fn bounded_gc_reports_incomplete_until_all_candidates_are_visited() { + let inner: Arc = Arc::new(MemoryBlobStore::new()); + inner.write_blob([0xA8; 16], &make_buf(1)).unwrap(); + inner.write_blob([0xA9; 16], &make_buf(2)).unwrap(); + let bm = BufferManager::new(inner.clone(), 4); - impl BlobStore for FlushPendingStore { - fn read_blob(&self, guid: BlobGuid, dst: &mut AlignedBlobBuf) -> Result<()> { - self.inner.read_blob(guid, dst) - } + let first = bm.gc_sweep_unreachable_bounded(&HashSet::new(), 1).unwrap(); + assert_eq!(first.freed, 1); + assert!(!first.complete); + assert_eq!(inner.list_blobs().unwrap().len(), 1); + assert_eq!(bm.stats().gc_last_full_sweep_deferred_count, 1); - fn read_blobs(&self, guids: &[BlobGuid], dsts: &mut [AlignedBlobBuf]) -> Vec> { - self.inner.read_blobs(guids, dsts) - } + assert_eq!(bm.reclaim_retired_orphans_bounded(1).unwrap(), 0); + assert_eq!( + bm.stats().gc_last_full_sweep_deferred_count, + 1, + "an empty exact-reclaim batch must not clear full-sweep debt", + ); - fn read_blob_range(&self, guid: BlobGuid, byte_offset: u64, dst: &mut [u8]) -> Result<()> { - self.inner.read_blob_range(guid, byte_offset, dst) - } + let second = bm.gc_sweep_unreachable_bounded(&HashSet::new(), 1).unwrap(); + assert_eq!(second.freed, 1); + assert!(second.complete); + assert!(inner.list_blobs().unwrap().is_empty()); + assert_eq!(bm.stats().gc_last_full_sweep_deferred_count, 0); + } - fn write_blob(&self, guid: BlobGuid, src: &AlignedBlobBuf) -> Result<()> { - self.pending.store(true, Ordering::Release); - self.inner.write_blob(guid, src) + #[test] + fn gc_releases_read_index_tokens_after_delete_fence_retires() { + let inner: Arc = Arc::new(MemoryBlobStore::new()); + for suffix in 0..64u8 { + let mut guid = [0xAB; 16]; + guid[15] = suffix; + inner.write_blob(guid, &make_buf(suffix)).unwrap(); } + let bm = BufferManager::new(inner.clone(), 8); - fn write_blobs(&self, writes: &[(BlobGuid, &AlignedBlobBuf)]) -> Result<()> { - self.pending.store(true, Ordering::Release); - self.inner.write_blobs(writes) - } + let outcome = bm + .gc_sweep_unreachable_bounded(&HashSet::new(), usize::MAX) + .unwrap(); + assert_eq!(outcome.freed, 64); + assert!(outcome.complete); + assert!(inner.list_blobs().unwrap().is_empty()); + assert_eq!( + bm.read_index_tokens.len(), + 0, + "successful GC must not retain per-GUID indexed-read tokens", + ); + } - fn delete_blob(&self, guid: BlobGuid) -> Result<()> { - self.pending.store(true, Ordering::Release); - self.inner.delete_blob(guid) - } + #[test] + fn gc_post_delete_error_still_releases_unreachable_index_token() { + let guid = [0xAC; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(guid, &make_buf(7)).unwrap(); + let store = Arc::new(FailAfterDeleteStore::new(inner)); + let bm = BufferManager::new(store.clone(), 4); + drop(bm.pin(guid).unwrap()); + let token = bm.ensure_read_index_token(guid); + assert_eq!(bm.read_index_tokens.len(), 1); - fn list_blobs(&self) -> Result> { - self.inner.list_blobs() - } + assert!(bm + .gc_sweep_unreachable_bounded(&HashSet::new(), usize::MAX) + .is_err()); + assert!(!store.inner.has_blob(guid).unwrap()); + assert_eq!(bm.cached_count(), 0); + assert_eq!(bm.pending_delete_count(), 0); + assert_eq!(bm.read_index_tokens.len(), 0); + assert!(!bm.read_index_token_valid(guid, token)); - fn flush(&self) -> Result<()> { - self.inner.flush()?; - self.pending.store(false, Ordering::Release); - Ok(()) - } + let retry = bm + .gc_sweep_unreachable_bounded(&HashSet::new(), usize::MAX) + .unwrap(); + assert_eq!(retry.freed, 0); + assert!(retry.complete); + } - fn needs_flush(&self) -> bool { - self.pending.load(Ordering::Acquire) || self.inner.needs_flush() - } + #[test] + fn deferred_fifo_skips_pinned_head_then_reclaims_later_candidates() { + let inner: Arc = Arc::new(MemoryBlobStore::new()); + let pinned_guid = [0xA4; 16]; + let free_guid = [0xA5; 16]; + inner.write_blob(pinned_guid, &make_buf(1)).unwrap(); + inner.write_blob(free_guid, &make_buf(2)).unwrap(); + let bm = BufferManager::new(inner.clone(), 4); + let pin = bm.pin(pinned_guid).unwrap(); + bm.restore_retired_orphans([pinned_guid, free_guid]); + + assert_eq!(bm.reclaim_retired_orphans_bounded(1).unwrap(), 0); + assert_eq!(bm.gc_orphan_backlog_count(), 2); + assert_eq!(bm.stats().gc_last_full_sweep_deferred_count, 0); + assert_eq!(bm.reclaim_retired_orphans_bounded(1).unwrap(), 1); + assert!(!inner.has_blob(free_guid).unwrap()); + assert!(inner.has_blob(pinned_guid).unwrap()); + assert_eq!(bm.gc_orphan_backlog_count(), 1); + assert_eq!( + bm.stats().gc_last_full_sweep_deferred_count, + 0, + "exact reclaim must not overwrite the full-sweep gauge", + ); + + drop(pin); + assert_eq!(bm.reclaim_retired_orphans_bounded(1).unwrap(), 1); + assert_eq!(bm.gc_orphan_backlog_count(), 0); + assert_eq!(bm.stats().gc_last_full_sweep_deferred_count, 0); + assert!(!inner.has_blob(pinned_guid).unwrap()); } struct CountingReadIndexStore { @@ -3160,12 +5075,9 @@ mod tests { "dirty bookkeeping must not be touched by eviction", ); - // And snapshot_bytes(A) must still return Some — the - // invariant downstream checkpoint code relies on. - assert!( - bm.snapshot_bytes(g_a).is_some(), - "dirty entry's cache image must be snapshottable", - ); + // And a versioned snapshot of A must still succeed — the invariant + // downstream checkpoint code relies on. + let _ = snapshot_current_bytes(&bm, g_a); } #[test] @@ -3368,22 +5280,137 @@ mod tests { } #[test] - fn structural_pending_delete_retires_fence_without_manifest_delete() { + fn structural_detach_waits_for_parent_dirty_before_fifo() { let inner: Arc = Arc::new(MemoryBlobStore::new()); let guid = [0x5B; 16]; + let parent_guid = [0x5A; 16]; inner.write_blob(guid, &make_buf(7)).unwrap(); + inner.write_blob(parent_guid, &make_buf(8)).unwrap(); let bm = BufferManager::new(inner.clone(), 4); + let parent = bm.pin(parent_guid).unwrap(); - bm.mark_for_delete(guid, STRUCTURAL_SEQ); - let pending = bm.snapshot_pending_deletes(); - assert_eq!(pending.get(&guid), Some(&STRUCTURAL_SEQ)); - assert!(bm.execute_pending_delete(guid, STRUCTURAL_SEQ).unwrap()); - + bm.stage_structural_reclaim(parent_guid, guid); assert_eq!(bm.pending_delete_count(), 0); + assert_eq!(bm.gc_orphan_backlog_count(), 1); + assert_eq!(bm.orphan_staging_count(), 1); + assert_eq!(bm.reclaim_retired_orphans_bounded(1).unwrap(), 0); + bm.mark_dirty_cached(parent_guid, STRUCTURAL_SEQ, parent.as_ref()); + assert_eq!(bm.orphan_staging_count(), 0); assert!( inner.has_blob(guid).unwrap(), - "structural orphan cleanup must not mutate the store manifest" + "structural detach must not unlink before parent dirty publication" + ); + assert_eq!(bm.reclaim_retired_orphans_bounded(1).unwrap(), 1); + assert_eq!(bm.gc_orphan_backlog_count(), 0); + assert!(!inner.has_blob(guid).unwrap()); + } + + #[test] + fn structural_orphan_stays_visible_until_last_snapshot_lease_retires() { + let inner: Arc = Arc::new(MemoryBlobStore::new()); + let guid = [0x5D; 16]; + let root_guid = [0x5E; 16]; + inner.write_blob(guid, &make_buf(7)).unwrap(); + inner.write_blob(root_guid, &make_buf(8)).unwrap(); + let bm = Arc::new(BufferManager::new(inner.clone(), 4)); + let root_pin = bm.pin(root_guid).unwrap(); + let epoch = bm.register_snapshot(root_guid, &root_pin).unwrap(); + + bm.stage_structural_reclaim(root_guid, guid); + bm.mark_dirty_cached(root_guid, STRUCTURAL_SEQ, root_pin.as_ref()); + assert_eq!(bm.pending_delete_count(), 0); + assert_eq!(bm.gc_orphan_backlog_count(), 1); + assert_eq!(bm.orphan_staging_count(), 0); + assert_eq!( + bm.pin(guid).unwrap().read().as_slice()[100], + 7, + "structural detach must not fence a snapshot's first lazy child pin" ); + assert_eq!(bm.reclaim_retired_orphans_bounded(1).unwrap(), 0); + assert!(inner.has_blob(guid).unwrap()); + + bm.retire_snapshot(epoch); + assert_eq!(bm.gc_orphan_backlog_count(), 1); + assert_eq!(bm.reclaim_retired_orphans_bounded(1).unwrap(), 1); + assert!(!inner.has_blob(guid).unwrap()); + } + + #[test] + fn snapshot_epoch_exhaustion_never_wraps_or_mutates_live_registry() { + let root_guid = [0x63; 16]; + let inner: Arc = Arc::new(MemoryBlobStore::new()); + inner.write_blob(root_guid, &make_buf(1)).unwrap(); + let bm = Arc::new(BufferManager::new(inner, 4)); + let root = bm.pin(root_guid).unwrap(); + bm.set_current_epoch(u64::MAX - 2); + + let epoch = bm.register_snapshot(root_guid, &root).unwrap(); + assert_eq!(epoch, u64::MAX - 2); + assert_eq!(bm.current_epoch(), u64::MAX - 1); + assert_eq!(bm.fork_barrier(), u64::MAX - 2); + assert_eq!(bm.snapshots.lock().unwrap().live.len(), 1); + let cached_before = bm.cached_count(); + let dirty_before = bm.dirty_count(); + + let error = bm.register_snapshot(root_guid, &root).unwrap_err(); + assert!(matches!(error, Error::SnapshotEpochExhausted)); + assert_eq!(bm.current_epoch(), u64::MAX - 1); + assert_eq!(bm.fork_barrier(), u64::MAX - 2); + assert_eq!(bm.snapshots.lock().unwrap().live.len(), 1); + assert_eq!(bm.cached_count(), cached_before); + assert_eq!(bm.dirty_count(), dirty_before); + + bm.retire_snapshot(epoch); + assert_eq!(bm.fork_barrier(), 0); + assert_eq!(bm.current_epoch(), u64::MAX - 1); + } + + #[test] + fn structural_last_lease_drop_before_parent_dirty_stays_staged() { + let inner: Arc = Arc::new(MemoryBlobStore::new()); + let child_guid = [0x5F; 16]; + let parent_guid = [0x60; 16]; + inner.write_blob(child_guid, &make_buf(7)).unwrap(); + inner.write_blob(parent_guid, &make_buf(8)).unwrap(); + let bm = Arc::new(BufferManager::new(inner.clone(), 4)); + let parent = bm.pin(parent_guid).unwrap(); + let epoch = bm.register_snapshot(parent_guid, &parent).unwrap(); + + bm.stage_structural_reclaim(parent_guid, child_guid); + bm.retire_snapshot(epoch); + assert_eq!(bm.orphan_staging_count(), 1); + assert_eq!(bm.reclaim_retired_orphans_bounded(1).unwrap(), 0); + assert!(inner.has_blob(child_guid).unwrap()); + + bm.mark_dirty_cached(parent_guid, STRUCTURAL_SEQ, parent.as_ref()); + assert_eq!(bm.orphan_staging_count(), 0); + assert_eq!(bm.reclaim_retired_orphans_bounded(1).unwrap(), 1); + assert!(!inner.has_blob(child_guid).unwrap()); + } + + #[test] + fn cow_epoch_zero_last_lease_drop_before_parent_dirty_stays_staged() { + let inner: Arc = Arc::new(MemoryBlobStore::new()); + let child_guid = [0x61; 16]; + let parent_guid = [0x62; 16]; + inner.write_blob(child_guid, &make_buf(7)).unwrap(); + inner.write_blob(parent_guid, &make_buf(8)).unwrap(); + let bm = Arc::new(BufferManager::new(inner.clone(), 4)); + let parent = bm.pin(parent_guid).unwrap(); + let epoch = bm.register_snapshot(parent_guid, &parent).unwrap(); + + // Epoch zero is valid for frames written by older Holt versions. + bm.stage_cow_reclaim(parent_guid, child_guid, 0); + bm.retire_snapshot(epoch); + assert_eq!(bm.orphan_staging_count(), 1); + assert_eq!(bm.gc_orphan_backlog_count(), 1); + assert_eq!(bm.reclaim_retired_orphans_bounded(1).unwrap(), 0); + assert!(inner.has_blob(child_guid).unwrap()); + + bm.mark_dirty_cached(parent_guid, 9, parent.as_ref()); + assert_eq!(bm.orphan_staging_count(), 0); + assert_eq!(bm.reclaim_retired_orphans_bounded(1).unwrap(), 1); + assert!(!inner.has_blob(child_guid).unwrap()); } #[test] @@ -3626,9 +5653,7 @@ mod tests { !bm.try_evict_cold(guid), "checkpoint-owned flushing entries must stay cached until write-through", ); - let bytes = bm - .snapshot_bytes(guid) - .expect("flushing protection must preserve cached bytes"); + let bytes = snapshot_current_bytes(&bm, guid); assert_eq!(bytes.as_slice()[123], 0xAB); bm.write_through_batch(&[WriteThroughEntry { @@ -3661,7 +5686,7 @@ mod tests { assert_eq!(snap[&guid], 10); drop(pin); - bm.reclaim_blob(guid); + bm.discard_snapshot_root(guid); let version = bm.snapshot_dirty_versions(&snap).unwrap()[0].content_version; let bytes = bm @@ -3680,7 +5705,7 @@ mod tests { let bm = BufferManager::new(inner.clone(), 1); let pin = bm.pin(guid).unwrap(); - bm.reclaim_blob(guid); + bm.discard_snapshot_root(guid); { let mut guard = pin.write(); @@ -4078,7 +6103,7 @@ mod tests { // Simulate the planner's drain by manually setting up the // "post-drain" state: dirty contains a NEW writer's entry. bm.mark_dirty([0xAA; 16], 200); - let snap_bytes = bm.snapshot_bytes([0xAA; 16]).unwrap(); + let snap_bytes = snapshot_current_bytes(&bm, [0xAA; 16]); // The planner's snap had captured snap_seq=50 (a stale // pre-drain value). Pass that through. @@ -4110,7 +6135,7 @@ mod tests { let _pin = bm.pin([0xA5; 16]).unwrap(); bm.mark_dirty([0xA5; 16], STRUCTURAL_SEQ); - let snap_bytes = bm.snapshot_bytes([0xA5; 16]).unwrap(); + let snap_bytes = snapshot_current_bytes(&bm, [0xA5; 16]); bm.write_through_batch(&[WriteThroughEntry { guid: [0xA5; 16], @@ -4139,7 +6164,7 @@ mod tests { let _pin = bm.pin([0xBB; 16]).unwrap(); bm.mark_dirty([0xBB; 16], 42); - let snap_bytes = bm.snapshot_bytes([0xBB; 16]).unwrap(); + let snap_bytes = snapshot_current_bytes(&bm, [0xBB; 16]); // expected_seq matches the current entry → safe to retire. bm.write_through_batch(&[WriteThroughEntry { @@ -4173,7 +6198,7 @@ mod tests { .iter() .map(|(guid, expected_seq)| WriteThroughEntry { guid: *guid, - bytes: bm.snapshot_bytes(*guid).unwrap(), + bytes: snapshot_current_bytes(&bm, *guid), expected_seq: *expected_seq, content_version: None, }) @@ -4208,7 +6233,7 @@ mod tests { } bm.mark_dirty_cached(guid, 10, pin.as_ref()); let snap = bm.snapshot_dirty(); - let bytes = bm.snapshot_bytes(guid).unwrap(); + let bytes = snapshot_current_bytes(&bm, guid); bm.write_through_batch(&[WriteThroughEntry { guid, @@ -4330,7 +6355,7 @@ mod tests { .iter() .map(|(guid, expected_seq)| WriteThroughEntry { guid: *guid, - bytes: bm.snapshot_bytes(*guid).unwrap(), + bytes: snapshot_current_bytes(&bm, *guid), expected_seq: *expected_seq, content_version: None, }) @@ -4342,6 +6367,80 @@ mod tests { assert_eq!(live[&g1], 200); } + #[test] + fn write_through_revalidates_version_after_store_io() { + let guid = [0xC3; 16]; + let inner = MemoryBlobStore::new(); + inner.write_blob(guid, &make_buf(1)).unwrap(); + let store = Arc::new(BlockingWriteStore::new(inner)); + let store_dyn: Arc = store.clone(); + let bm = Arc::new(BufferManager::new(store_dyn, 4)); + let pin = bm.pin(guid).unwrap(); + + bm.mark_dirty(guid, 100); + let first_dirty = bm.snapshot_dirty(); + let first_version = bm.snapshot_dirty_versions(&first_dirty).unwrap()[0].content_version; + let first_bytes = bm + .snapshot_bytes_if_version(guid, first_version) + .unwrap() + .unwrap(); + let write_bm = Arc::clone(&bm); + let writer = std::thread::spawn(move || { + write_bm.write_through_batch(&[WriteThroughEntry { + guid, + bytes: first_bytes, + expected_seq: 100, + content_version: Some(first_version), + }]) + }); + + // The first content-version check has passed and the stale bytes are + // waiting at store I/O. Publish a lower-seq writer in that exact + // window; seq comparison alone would incorrectly retire it. + store.entered.wait(); + { + let mut frame = pin.write(); + frame.as_mut_slice()[100] = 2; + } + bm.mark_dirty_cached(guid, 50, pin.as_ref()); + store.release.wait(); + + let report = writer.join().unwrap().unwrap(); + assert_eq!(report.statuses, vec![WriteThroughStatus::Stale]); + bm.restore_dirty(first_dirty); + let retained = bm.snapshot_dirty(); + assert_eq!(retained.get(&guid), Some(&50)); + bm.restore_dirty(retained); + + let second_dirty = bm.snapshot_dirty(); + let second_version = bm.snapshot_dirty_versions(&second_dirty).unwrap()[0].content_version; + let second_bytes = bm + .snapshot_bytes_if_version(guid, second_version) + .unwrap() + .unwrap(); + let report = bm + .write_through_batch(&[WriteThroughEntry { + guid, + bytes: second_bytes, + expected_seq: second_dirty[&guid], + content_version: Some(second_version), + }]) + .unwrap(); + assert_eq!(report.statuses, vec![WriteThroughStatus::Written]); + assert_eq!(bm.dirty_count(), 0); + assert_eq!(bm.flushing_count(), 0); + + drop(pin); + drop(bm); + let store_dyn: Arc = store; + let reopened = BufferManager::new(store_dyn, 4); + assert_eq!( + reopened.pin(guid).unwrap().read().as_slice()[100], + 2, + "the racing writer must survive the retry and reopen", + ); + } + #[test] fn write_blob_through_trait_clears_dirty() { let inner: Arc = Arc::new(MemoryBlobStore::new()); @@ -4414,7 +6513,7 @@ mod tests { // After the production checkpoint primitive runs, the inner // store has the bytes and the dirty entry is cleared. let snap = bm.snapshot_dirty(); - let bytes = bm.snapshot_bytes(new_guid).unwrap(); + let bytes = snapshot_current_bytes(&bm, new_guid); bm.write_through_batch(&[WriteThroughEntry { guid: new_guid, bytes, diff --git a/src/store/buffer_manager/mutation.rs b/src/store/buffer_manager/mutation.rs index 10032bc..a2c856e 100644 --- a/src/store/buffer_manager/mutation.rs +++ b/src/store/buffer_manager/mutation.rs @@ -24,6 +24,10 @@ pub(super) struct MutationState { /// in flight, so stale walkers cannot reload or dirty a blob /// that has already been logically unlinked. pub(super) deleting: HashMap, + /// Transient fence owned by reachability GC between cache eviction and + /// durable manifest deletion. Readers that raced the fence restart via + /// the GC generation instead of reinserting a zombie cache entry. + pub(super) gc_deleting: HashSet, /// In-memory maintenance hints for blobs whose local garbage /// is worth checking before the next online compact pass. /// @@ -67,6 +71,10 @@ impl MutationState { } pub(super) fn has_delete_fence(&self, guid: &BlobGuid) -> bool { + self.has_logical_delete_fence(guid) || self.gc_deleting.contains(guid) + } + + pub(super) fn has_logical_delete_fence(&self, guid: &BlobGuid) -> bool { self.pending_deletes.contains_key(guid) || self.deleting.contains_key(guid) } diff --git a/src/store/mod.rs b/src/store/mod.rs index ae288b8..34f4d45 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -21,8 +21,8 @@ pub(crate) use blob_frame::{ pub use blob_frame::{AllocError, BlobFrame, BlobFrameRef, FreeError}; pub(crate) use blob_store::IndexedBlobLookup; pub(crate) use buffer_manager::{ - BlobWriteGuard, BufferStats, DirtySnapshotEntry, WriteDeltaEntry, WriteDeltaKeyState, - WriteThroughEntry, WriteThroughStatus, STRUCTURAL_SEQ, + BlobWriteGuard, BufferStats, DirtySnapshotEntry, SnapshotLease, WriteDeltaEntry, + WriteDeltaKeyState, WriteThroughEntry, WriteThroughStatus, STRUCTURAL_SEQ, }; pub use buffer_manager::{BufferManager, CachedBlob}; pub(crate) use read_index::{ From 4dda614187b9eb0622df6debdf951625650e94ea Mon Sep 17 00:00:00 2001 From: wchwawa Date: Thu, 16 Jul 2026 11:49:53 +1000 Subject: [PATCH 2/3] test(storage): cover snapshot crash and GC recovery Signed-off-by: wchwawa --- tests/checkpoint_failpoint.rs | 144 ++++---- tests/snapshot.rs | 637 ++++++++++++++++++++++++++++++++-- 2 files changed, 662 insertions(+), 119 deletions(-) diff --git a/tests/checkpoint_failpoint.rs b/tests/checkpoint_failpoint.rs index 679c448..6f18c50 100644 --- a/tests/checkpoint_failpoint.rs +++ b/tests/checkpoint_failpoint.rs @@ -1,16 +1,14 @@ //! Fault-injection tests for the checkpoint round's -//! deferred-delete + Sync paths. +//! deferred-delete, structural-orphan, and Sync paths. //! //! Wraps a real store (`MemoryBlobStore` or `FileBlobStore`) //! in a [`FailpointBlobStore`] that can be told to fail the N-th //! `delete_blob` / `flush` / `write_blob` call. The tests verify -//! that structural pending deletes: +//! that structural orphans: //! -//! - retire their in-memory fence without calling -//! `store.delete_blob`; -//! - do not issue a post-delete Sync when no manifest delete -//! happened; -//! - stay queued when dirty write or pre-delete Sync fails. +//! - never enter the user-delete visibility-fence queue; +//! - remain queued when the rewritten parent cannot be made durable; +//! - are deleted only after a clean parent-publication frontier. //! //! The write tests also verify that a failed `write_blob` keeps //! the entry in `dirty` so a subsequent round retries the byte @@ -164,13 +162,7 @@ fn clean_checkpoint_skips_flush_inner() { ); } -/// Build a tree on a failpoint-wrapped memory store and stage at -/// least one structural deferred delete in the BM's `pending_deletes` -/// set. This models the background merge path that caused the -/// nightly crash-soak manifest corruption: the old child blob must -/// remain in the store manifest because no logical WAL record proves -/// that every durable parent has stopped referencing it. -fn setup_with_pending_delete() -> (Arc, Arc, Tree) { +fn setup_after_mass_delete() -> (Arc, Arc, Tree) { let inner: Arc = Arc::new(MemoryBlobStore::new()); let fp = Arc::new(FailpointBlobStore::new(Arc::clone(&inner))); let fp_dyn: Arc = fp.clone(); @@ -185,53 +177,62 @@ fn setup_with_pending_delete() -> (Arc, Arc, tree.put(k.as_bytes(), &payload).unwrap(); } - // Make enough children empty, then compact so the maintenance - // merge path stages STRUCTURAL_SEQ pending deletes. + // Make enough children empty to produce logical user-delete fences and + // leave underfilled children for structural compaction tests. for i in 0..950u32 { let k = format!("k{i:05}"); let _ = tree.delete(k.as_bytes()).unwrap(); } + (inner, fp, tree) +} + +/// Build a tree with at least one parent-scoped structural orphan. The +/// rewritten parent must become durable before the exact old child GUID may +/// leave the store manifest. +fn setup_with_structural_orphan() -> (Arc, Arc, Tree) { + let (inner, fp, tree) = setup_after_mass_delete(); + tree.checkpoint().unwrap(); tree.compact().unwrap(); (inner, fp, tree) } #[test] -fn structural_pending_delete_skips_manifest_delete_and_post_sync() { - let (inner, fp, tree) = setup_with_pending_delete(); +fn structural_orphan_waits_for_parent_durability_before_delete() { + let (inner, fp, tree) = setup_with_structural_orphan(); let stats_before = tree.stats().unwrap(); - assert!( - stats_before.bm_pending_delete_count > 0, - "setup must produce at least one pending delete (got {})", - stats_before.bm_pending_delete_count, - ); + assert_eq!(stats_before.bm_pending_delete_count, 0); + assert!(stats_before.bm_gc_orphan_backlog_count > 0); + assert_eq!(fp.delete_count(), 0); let flushes_before = fp.flush_count(); - // If structural cleanup incorrectly runs a post-delete Sync, - // this second flush will trip. - fp.arm_flush(flushes_before + 2); - // If structural cleanup incorrectly mutates the manifest, - // this delete failpoint will trip. + fp.arm_flush(flushes_before + 1); + assert!(tree.checkpoint().is_err()); + assert_eq!( + fp.delete_count(), + 0, + "a failed parent checkpoint must not delete structural children", + ); + assert!(tree.stats().unwrap().bm_gc_orphan_backlog_count > 0); + fp.arm_delete(fp.delete_count() + 1); + assert!(tree.checkpoint().is_err()); + assert!( + tree.stats().unwrap().bm_gc_orphan_backlog_count > 0, + "a failed exact-child delete must restore the orphan FIFO", + ); tree.checkpoint().unwrap(); - assert_eq!(tree.stats().unwrap().bm_pending_delete_count, 0); + let stats = tree.stats().unwrap(); + assert_eq!(stats.bm_pending_delete_count, 0); + assert_eq!(stats.bm_gc_orphan_backlog_count, 0); + assert!(fp.delete_count() > 0); let store_blobs = inner.list_blobs().unwrap(); - let stats = tree.stats().unwrap(); - assert!( - store_blobs.len() as u32 >= stats.blob_count, - "structural cleanup may leave orphan blobs, but must not \ - delete a durable blob that a parent can still reference", - ); assert_eq!( - fp.delete_count(), - 0, - "structural pending delete must not call store.delete_blob", - ); - assert!( - fp.flush_count() < flushes_before + 2, - "structural pending delete must not trigger a post-delete Sync", + store_blobs.len() as u32, + stats.blob_count, + "clean-frontier reclaim must leave exactly the reachable tree", ); } @@ -306,28 +307,18 @@ fn storage_full_write_failure_is_retried_next_round() { } #[test] -fn dirty_write_failure_does_not_propagate_to_pending_delete() { - // Regression for the bug where a failed parent write_through - // didn't stop the round from applying a dependent child's - // manifest delete — leaving "old parent on-disk still - // references a now-deleted child" on a crash before the next - // round caught up. The fix: on any write failure, the - // pre-delete sync still runs (to fsync the writes that DID - // succeed), but no manifest delete fires; the whole pending - // snapshot is restored for the next round. - let (_inner, fp, tree) = setup_with_pending_delete(); +fn dirty_write_failure_does_not_release_structural_orphan() { + // A failed parent write-through must not release its exact detached child: + // the durable old parent may still reference that GUID. + let (_inner, fp, tree) = setup_with_structural_orphan(); let stats_before = tree.stats().unwrap(); assert!( stats_before.bm_dirty_count > 0, "setup precondition: dirty entry queued (got {})", stats_before.bm_dirty_count, ); - assert!( - stats_before.bm_pending_delete_count > 0, - "setup precondition: pending delete queued (got {})", - stats_before.bm_pending_delete_count, - ); - let pending_before = stats_before.bm_pending_delete_count; + assert!(stats_before.bm_gc_orphan_backlog_count > 0); + let orphan_before = stats_before.bm_gc_orphan_backlog_count; let deletes_before = fp.delete_count(); // Arm the NEXT write_blob to fail — that's the first @@ -348,10 +339,8 @@ fn dirty_write_failure_does_not_propagate_to_pending_delete() { stats_after.bm_dirty_count, ); assert_eq!( - stats_after.bm_pending_delete_count, pending_before, - "dirty failure must NOT apply any pending delete — whole \ - snapshot restored (got {} vs {})", - stats_after.bm_pending_delete_count, pending_before, + stats_after.bm_gc_orphan_backlog_count, orphan_before, + "dirty failure must preserve the exact structural-orphan backlog", ); assert_eq!( @@ -364,19 +353,15 @@ fn dirty_write_failure_does_not_propagate_to_pending_delete() { tree.checkpoint().unwrap(); let stats_done = tree.stats().unwrap(); assert_eq!(stats_done.bm_dirty_count, 0); - assert_eq!(stats_done.bm_pending_delete_count, 0); + assert_eq!(stats_done.bm_gc_orphan_backlog_count, 0); } #[test] -fn pre_delete_sync_failure_restores_pending() { - // Regression for the bug where the pre-delete `store.flush` - // failure path drained `pending` (the checkpoint snapshot) but - // never restored it — losing every queued unlink intent. The - // fix restores `pending` on every Sync-failure return path - // before phase 6. - let (inner, fp, tree) = setup_with_pending_delete(); - let pending_before = tree.stats().unwrap().bm_pending_delete_count; - assert!(pending_before > 0, "setup precondition"); +fn parent_sync_failure_preserves_structural_orphan() { + let (_inner, fp, tree) = setup_with_structural_orphan(); + let orphan_before = tree.stats().unwrap().bm_gc_orphan_backlog_count; + let deletes_before = fp.delete_count(); + assert!(orphan_before > 0, "setup precondition"); // First `store.flush` call inside `tree.checkpoint` is the // pre-delete data Sync at phase 3 — arm to fail it. @@ -391,24 +376,19 @@ fn pre_delete_sync_failure_restores_pending() { let stats_after = tree.stats().unwrap(); assert_eq!( - stats_after.bm_pending_delete_count, pending_before, - "pre-delete Sync failure must restore the entire pending \ - snapshot (got {} vs {})", - stats_after.bm_pending_delete_count, pending_before, + stats_after.bm_gc_orphan_backlog_count, orphan_before, + "parent Sync failure must preserve the exact orphan backlog", ); - // Phase 6 didn't run, so no manifest delete applied. - let store_blobs = inner.list_blobs().unwrap(); - let stats = tree.stats().unwrap(); assert_eq!( - store_blobs.len() as u32, - stats.blob_count, + fp.delete_count(), + deletes_before, "no manifest delete must have applied while pre-delete Sync failed", ); // Recovery: next checkpoint drains the restored snapshot. tree.checkpoint().unwrap(); - assert_eq!(tree.stats().unwrap().bm_pending_delete_count, 0); + assert_eq!(tree.stats().unwrap().bm_gc_orphan_backlog_count, 0); } #[test] diff --git a/tests/snapshot.rs b/tests/snapshot.rs index a29661f..cf0a792 100644 --- a/tests/snapshot.rs +++ b/tests/snapshot.rs @@ -10,7 +10,9 @@ use std::sync::Arc; -use holt::{BlobStore, Durability, Error, MemoryBlobStore, Tree, TreeBuilder, TreeConfig, DB}; +use holt::{ + BlobStore, Durability, Error, FileBlobStore, MemoryBlobStore, Tree, TreeBuilder, TreeConfig, DB, +}; use tempfile::tempdir; #[test] @@ -52,9 +54,7 @@ fn snapshot_reads_across_blob_boundaries() { // the snapshot's copied root crosses `BlobNode`s into shared child // frames on read. let store: Arc = Arc::new(MemoryBlobStore::new()); - let tree = TreeBuilder::new("ignored") - .open_with_blob_store(store.clone()) - .unwrap(); + let tree = Tree::open_with_blob_store(TreeConfig::memory(), store.clone()).unwrap(); const N: u32 = 5000; let value = vec![0xAB_u8; 200]; @@ -124,9 +124,7 @@ fn snapshot_isolates_cross_blob_writes() { // overwrite them. Uses a multi-blob tree so the writes cross into // shared child frames. let store: Arc = Arc::new(MemoryBlobStore::new()); - let tree = TreeBuilder::new("ignored") - .open_with_blob_store(store.clone()) - .unwrap(); + let tree = Tree::open_with_blob_store(TreeConfig::memory(), store.clone()).unwrap(); const N: u32 = 5000; let orig = vec![0xAB_u8; 200]; @@ -315,14 +313,13 @@ fn snapshot_stable_under_randomized_churn() { } #[test] -fn retire_reclaims_forked_frames() { - // Retiring a snapshot must free the frames it kept alive (the - // forked-away originals + the snapshot root), returning the blob - // count to the live working set — no leak. +fn retire_defers_forked_frame_reclamation_to_gc() { + // Retiring a snapshot releases process-local ownership but must keep + // persisted frames until a durability-barrier-protected GC proves them + // unreachable. That fail-closed interval prevents a durable old parent + // from losing a child. let store: Arc = Arc::new(MemoryBlobStore::new()); - let tree = TreeBuilder::new("ignored") - .open_with_blob_store(store.clone()) - .unwrap(); + let tree = Tree::open_with_blob_store(TreeConfig::memory(), store.clone()).unwrap(); const N: u32 = 5000; let orig = vec![0xAB_u8; 200]; @@ -333,6 +330,7 @@ fn retire_reclaims_forked_frames() { let baseline = store.list_blobs().unwrap().len(); assert!(baseline >= 2, "need a multi-blob tree"); + let during; { let snap = tree.snapshot(b"").unwrap(); // Overwrite a spread of keys → forks the shared child frames @@ -342,19 +340,26 @@ fn retire_reclaims_forked_frames() { tree.put(format!("k{i:08}").as_bytes(), b"x").unwrap(); } tree.checkpoint().unwrap(); - let during = store.list_blobs().unwrap().len(); + during = store.list_blobs().unwrap().len(); assert!( during > baseline, "snapshot + forks should add blobs: {during} vs {baseline}", ); assert_eq!(snap.get(b"k00000000").unwrap().as_deref(), Some(&orig[..])); - } // snapshot dropped → retire → reclaim + } // snapshot dropped → process-local retire only - tree.checkpoint().unwrap(); - let after = store.list_blobs().unwrap().len(); + let after_retire = store.list_blobs().unwrap().len(); assert_eq!( - after, baseline, - "retire must reclaim every snapshot frame: {after} vs {baseline}", + after_retire, during, + "retire must preserve persisted COW frames until GC", + ); + + let freed = tree.gc().unwrap(); + assert_eq!(freed, during - baseline); + let after_gc = store.list_blobs().unwrap().len(); + assert_eq!( + after_gc, baseline, + "GC must reclaim every retired snapshot frame: {after_gc} vs {baseline}", ); // Live tree intact. @@ -369,13 +374,11 @@ fn retire_reclaims_forked_frames() { } #[test] -fn overlapping_snapshots_reclaim_after_last_retires() { - // Two overlapping snapshots accumulate forked-away frames; the full - // working set is reclaimed once the last one retires. +fn overlapping_snapshots_reclaim_on_gc_after_last_retires() { + // Two overlapping snapshots accumulate forked-away frames. Retirement + // releases their process-local holds; durable space is reclaimed by GC. let store: Arc = Arc::new(MemoryBlobStore::new()); - let tree = TreeBuilder::new("ignored") - .open_with_blob_store(store.clone()) - .unwrap(); + let tree = Tree::open_with_blob_store(TreeConfig::memory(), store.clone()).unwrap(); const N: u32 = 5000; let v = vec![0xAB_u8; 200]; @@ -403,6 +406,7 @@ fn overlapping_snapshots_reclaim_after_last_retires() { drop(s2); tree.checkpoint().unwrap(); + tree.gc().unwrap(); let after = store.list_blobs().unwrap().len(); assert_eq!( after, baseline, @@ -418,7 +422,7 @@ fn overlapping_snapshots_reclaim_after_last_retires() { } #[test] -fn snapshot_correct_after_reopen() { +fn snapshot_correct_after_multi_blob_compact_checkpoint_reopen() { let dir = tempdir().unwrap(); let cfg = || { let mut c = TreeConfig::new(dir.path()); @@ -441,6 +445,10 @@ fn snapshot_correct_after_reopen() { for i in 0..N { tree.put(format!("k{i:06}").as_bytes(), &v1).unwrap(); } + assert!( + tree.stats().unwrap().blob_count > 1, + "test must exercise compacted child frames", + ); { let snap = tree.snapshot(b"").unwrap(); for i in 0..N { @@ -449,6 +457,21 @@ fn snapshot_correct_after_reopen() { assert_eq!(snap.get(b"k000000").unwrap().as_deref(), Some(&v1[..])); } // retire tree.checkpoint().unwrap(); + + // Rebuild the high-epoch COW frames before persisting them. If + // compact_blob resets either generation field, reopen can allocate a + // snapshot epoch older than a surviving child and a later live write + // will mutate that child underneath the snapshot. + let compactions_before = tree.stats().unwrap().total_compactions; + for _ in 0..4 { + tree.compact().unwrap(); + } + let compactions_after = tree.stats().unwrap().total_compactions; + assert!( + compactions_after > compactions_before, + "test must compact at least one high-epoch frame", + ); + tree.checkpoint().unwrap(); } // Reopen. @@ -558,7 +581,7 @@ fn run_crash_session(child_test: &str, dir: &std::path::Path) { /// Child body for [`crash_leaked_tree_snapshot_does_not_leave_persistent_garbage`]: /// snapshot + writes + checkpoint, then "crash" — forget the snapshot /// so it never retires. Snapshot roots are in-memory only, so reopen -/// should see live data without requiring GC cleanup. +/// should see live data; the storage owner explicitly runs recovery GC. #[test] #[ignore = "child-process body for crash_leaked_tree_snapshot_does_not_leave_persistent_garbage"] fn crash_leak_tree_session() { @@ -589,8 +612,9 @@ fn crash_leaked_tree_snapshot_does_not_leave_persistent_garbage() { run_crash_session("crash_leak_tree_session", dir.path()); - // Reopen: the forgotten snapshot was not a durable root. Only the - // live tree should remain reachable. + // Reopen itself preserves generic Holt startup semantics and does not run + // a full store sweep. The forgotten snapshot was not a durable root, so + // the storage owner's explicit recovery GC can reclaim its old frames. let tree = Tree::open(cfg()).unwrap(); for i in 0..N { assert_eq!( @@ -601,10 +625,7 @@ fn crash_leaked_tree_snapshot_does_not_leave_persistent_garbage() { } let freed = tree.gc().unwrap(); - assert_eq!( - freed, 0, - "ephemeral read snapshots must not leave persistent garbage", - ); + assert!(freed > 0, "explicit recovery gc must reclaim crash orphans"); // Idempotent: GC remains a no-op. assert_eq!(tree.gc().unwrap(), 0, "second gc must be a no-op"); // gc must not have touched live data. @@ -640,6 +661,17 @@ fn crash_leak_db_session() { t1.put(format!("k{i:06}").as_bytes(), b"new").unwrap(); } db.checkpoint().unwrap(); + for i in [0, CRASH_LEAK_N / 2, CRASH_LEAK_N - 1] { + assert_eq!( + snap.get(format!("k{i:06}").as_bytes()).unwrap().as_deref(), + Some(&v[..]), + "deferred batch flush overwrote snapshot child {i}", + ); + } + assert!( + db.stats().bm_gc_orphan_backlog_count > 0, + "snapshot-shared batch updates must create COW orphan debt", + ); std::mem::forget(snap); // crash: read snapshot state is process-local } @@ -655,9 +687,9 @@ fn db_crash_leaked_snapshot_does_not_leave_persistent_garbage() { let db = DB::open(cfg()).unwrap(); let freed = db.gc().unwrap(); - assert_eq!( - freed, 0, - "ephemeral read snapshots must not leave DB-wide persistent garbage", + assert!( + freed > 0, + "explicit DB recovery gc must reclaim crash orphans", ); assert_eq!(db.gc().unwrap(), 0, "second db gc must be a no-op"); @@ -678,6 +710,91 @@ fn db_crash_leaked_snapshot_does_not_leave_persistent_garbage() { } } +#[test] +fn db_reopen_recovers_epoch_from_surviving_other_family_frames() { + let dir = tempdir().unwrap(); + let cfg = || crash_leak_cfg(dir.path()); + let keys = (400..528u32) + .map(|i| format!("epoch/{i:08}").into_bytes()) + .collect::>(); + + { + let db = DB::open(cfg()).unwrap(); + let data = db.create_tree("data").unwrap(); + let epoch_owner = db.create_tree("epoch-owner").unwrap(); + epoch_owner.put(b"anchor", b"owner").unwrap(); + let old = vec![0x31; 1024]; + for i in 0..1200u32 { + data.put(format!("epoch/{i:08}").as_bytes(), &old).unwrap(); + } + db.checkpoint().unwrap(); + assert!(data.stats().unwrap().blob_count > 1); + + // This snapshot advances the DB-global epoch on another family. Data + // then forks high-epoch children while its own root high-water can + // remain older. + let owner_snapshot = epoch_owner.snapshot(b"").unwrap(); + for key in &keys { + data.put(key, b"session-one").unwrap(); + } + db.checkpoint().unwrap(); + assert!(db.stats().bm_gc_orphan_backlog_count > 0); + drop(owner_snapshot); + db.checkpoint().unwrap(); + + // Remove the root that originally carried the maximum high-water. + // Reopen must recover from the created_epoch on data's surviving + // reachable frames rather than assuming that root still exists. + db.drop_tree("epoch-owner").unwrap(); + drop(epoch_owner); + db.gc().unwrap(); + db.checkpoint().unwrap(); + assert_eq!(db.list_trees().unwrap(), vec!["data"]); + } + + let db = DB::open(cfg()).unwrap(); + let data = db.open_tree("data").unwrap(); + db.view(&[("data", b"")], |view| { + for key in &keys { + data.put(key, b"session-two")?; + } + db.checkpoint()?; + + let stable = view.tree("data").expect("captured data family"); + for key in &keys { + assert_eq!(stable.get(key)?.as_deref(), Some(&b"session-one"[..])); + assert_eq!(data.get(key)?.as_deref(), Some(&b"session-two"[..])); + } + assert!( + db.stats().bm_gc_orphan_backlog_count > 0, + "session-two writes must COW the reopened snapshot-shared child", + ); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn db_epoch_recovery_accepts_wal_replayed_cache_only_roots() { + let dir = tempdir().unwrap(); + let cfg = || crash_leak_cfg(dir.path()); + + { + let db = DB::open(cfg()).unwrap(); + let tree = db.create_tree("wal-only").unwrap(); + tree.put(b"key", b"acked-before-checkpoint").unwrap(); + // Deliberately no checkpoint: catalog/root/data exist only through + // the durable WAL plus replayed BM cache in the next session. + } + + let db = DB::open(cfg()).unwrap(); + let tree = db.open_tree("wal-only").unwrap(); + assert_eq!( + tree.get(b"key").unwrap().as_deref(), + Some(&b"acked-before-checkpoint"[..]) + ); +} + #[test] fn gc_rejects_db_trees() { let dir = tempdir().unwrap(); @@ -688,3 +805,449 @@ fn gc_rejects_db_trees() { "gc on a DB tree must be rejected", ); } + +#[test] +fn db_gc_finishes_dropping_tree_protocol_before_global_sweep() { + let dir = tempdir().unwrap(); + let cfg = || crash_leak_cfg(dir.path()); + + { + let db = DB::open(cfg()).unwrap(); + let live = db.create_tree("live").unwrap(); + let doomed = db.create_tree("doomed").unwrap(); + for i in 0..512u32 { + live.put(format!("live/{i:04}").as_bytes(), b"kept") + .unwrap(); + doomed + .put(format!("doomed/{i:04}").as_bytes(), &[0xDD; 256]) + .unwrap(); + } + db.checkpoint().unwrap(); + db.drop_tree("doomed").unwrap(); + drop(doomed); + drop(live); + + db.gc().unwrap(); + assert_eq!(db.list_trees().unwrap(), vec!["live"]); + assert!(matches!( + db.open_tree("doomed"), + Err(Error::TreeNotFound { .. }) + )); + } + + let db = DB::open(cfg()).unwrap(); + assert_eq!(db.list_trees().unwrap(), vec!["live"]); + assert!(matches!( + db.open_tree("doomed"), + Err(Error::TreeNotFound { .. }) + )); + let live = db.open_tree("live").unwrap(); + for i in 0..512u32 { + assert_eq!( + live.get(format!("live/{i:04}").as_bytes()) + .unwrap() + .as_deref(), + Some(&b"kept"[..]), + ); + } +} + +#[test] +fn retained_dropped_tree_handle_does_not_block_checkpoint_or_gc() { + use std::sync::mpsc; + use std::time::Duration; + + let dir = tempdir().unwrap(); + let db = DB::open(crash_leak_cfg(dir.path())).unwrap(); + let doomed = db.create_tree("doomed").unwrap(); + for i in 0..2600u32 { + doomed + .put(format!("doomed/{i:08}").as_bytes(), &[0xD1; 240]) + .unwrap(); + } + db.checkpoint().unwrap(); + db.drop_tree("doomed").unwrap(); + assert!(matches!( + db.create_tree("doomed"), + Err(Error::TreeExists { .. }) + )); + assert!(matches!( + doomed.get(b"doomed/00000000"), + Err(Error::TreeDropped) + )); + + let run_with_timeout = |label: &'static str, op: fn(&DB) -> holt::Result| { + let worker_db = db.clone(); + let (tx, rx) = mpsc::channel(); + let worker = std::thread::spawn(move || tx.send(op(&worker_db)).unwrap()); + let result = rx + .recv_timeout(Duration::from_secs(2)) + .unwrap_or_else(|_| panic!("{label} did not return with a retained dropped handle")) + .unwrap(); + worker.join().unwrap(); + result + }; + + let checkpoint = |db: &DB| db.checkpoint().map(|()| 0); + assert_eq!(run_with_timeout("checkpoint", checkpoint), 0); + assert_eq!(run_with_timeout("gc", DB::gc), 0); + assert!(matches!( + db.open_tree("doomed"), + Err(Error::TreeNotFound { .. }) + )); + + drop(doomed); + assert!(db.gc().unwrap() > 0); + let replacement = db.create_tree("doomed").unwrap(); + assert_eq!(replacement.get(b"doomed/00000000").unwrap(), None); + assert!(db.stats().bm_pending_delete_count == 0); +} + +#[test] +fn dropped_tree_escaped_view_and_cursor_keep_descendants_live() { + let dir = tempdir().unwrap(); + let db = DB::open(crash_leak_cfg(dir.path())).unwrap(); + let doomed = db.create_tree("doomed").unwrap(); + let original = vec![0x6D; 240]; + for i in 0..ESCAPED_READER_N { + doomed + .put(format!("k{i:08}").as_bytes(), &original) + .unwrap(); + } + db.checkpoint().unwrap(); + + let snapshot = doomed.snapshot(b"").unwrap(); + let escaped_view = snapshot.view().clone(); + let mut escaped_cursor = snapshot.range().into_iter(); + drop(snapshot); + db.drop_tree("doomed").unwrap(); + drop(doomed); + + db.checkpoint().unwrap(); + assert_eq!(db.gc().unwrap(), 0); + assert_eq!( + escaped_view.get(b"k00000000").unwrap().as_deref(), + Some(&original[..]) + ); + let mut seen = 0u32; + for entry in &mut escaped_cursor { + match entry.unwrap() { + holt::RangeEntry::Key { key, value, .. } => { + assert_eq!(key, format!("k{seen:08}").into_bytes()); + assert_eq!(value, original); + seen += 1; + } + other => panic!("unexpected escaped range entry: {other:?}"), + } + } + assert_eq!(seen, ESCAPED_READER_N); + + drop(escaped_cursor); + drop(escaped_view); + assert!(db.gc().unwrap() > 0); + assert!(matches!( + db.open_tree("doomed"), + Err(Error::TreeNotFound { .. }) + )); +} + +#[test] +fn live_range_has_no_gap_or_duplicate_across_gc_epochs() { + use std::sync::mpsc; + + let tree = Tree::open(TreeConfig::memory()).unwrap(); + for i in 0..ESCAPED_READER_N { + tree.put(format!("k{i:08}").as_bytes(), &[0x7A; 240]) + .unwrap(); + } + tree.checkpoint().unwrap(); + + let scan_tree = tree.clone(); + let (first_tx, first_rx) = mpsc::sync_channel(0); + let (resume_tx, resume_rx) = mpsc::sync_channel(0); + let scanner = std::thread::spawn(move || { + let mut cursor = scan_tree.range().into_iter(); + let mut keys = Vec::new(); + match cursor.next().expect("first range entry").unwrap() { + holt::RangeEntry::Key { key, .. } => keys.push(key), + other => panic!("unexpected first live range entry: {other:?}"), + } + first_tx.send(()).unwrap(); + resume_rx.recv().unwrap(); + for entry in &mut cursor { + match entry.unwrap() { + holt::RangeEntry::Key { key, .. } => keys.push(key), + other => panic!("unexpected live range entry: {other:?}"), + } + } + (keys, cursor.stats()) + }); + + first_rx.recv().unwrap(); + tree.gc().unwrap(); + resume_tx.send(()).unwrap(); + let (keys, stats) = scanner.join().unwrap(); + let expected = (0..ESCAPED_READER_N) + .map(|i| format!("k{i:08}").into_bytes()) + .collect::>(); + assert_eq!( + keys, expected, + "GC restart skipped or duplicated a range key" + ); + assert!(stats.restarts > 0, "test did not cross a physical GC epoch"); +} + +const ESCAPED_READER_N: u32 = 2600; + +fn escaped_reader_tree() -> (Arc, Tree, Vec) { + let store: Arc = Arc::new(MemoryBlobStore::new()); + let tree = Tree::open_with_blob_store(TreeConfig::memory(), store.clone()).unwrap(); + let original = vec![0x5A_u8; 240]; + for i in 0..ESCAPED_READER_N { + tree.put(format!("k{i:08}").as_bytes(), &original).unwrap(); + } + tree.checkpoint().unwrap(); + assert!( + store.list_blobs().unwrap().len() >= 2, + "escaped-reader tests require child blobs", + ); + (store, tree, original) +} + +fn mutate_escaped_reader_tree(tree: &Tree) { + for i in (0..ESCAPED_READER_N).step_by(3) { + tree.put(format!("k{i:08}").as_bytes(), b"live-generation") + .unwrap(); + } + tree.checkpoint().unwrap(); +} + +#[test] +fn escaped_view_keeps_epoch_and_descendants_live_through_gc() { + let (_store, tree, original) = escaped_reader_tree(); + let snapshot = tree.snapshot(b"").unwrap(); + let escaped = snapshot.view().clone(); + drop(snapshot); + + mutate_escaped_reader_tree(&tree); + tree.gc().unwrap(); + + for i in 0..ESCAPED_READER_N { + assert_eq!( + escaped + .get(format!("k{i:08}").as_bytes()) + .unwrap() + .as_deref(), + Some(&original[..]), + "escaped view lost snapshot key {i}", + ); + } + + drop(escaped); + assert!( + tree.gc().unwrap() > 0, + "last escaped view drop must make retired COW frames reclaimable", + ); +} + +#[test] +fn escaped_record_cursor_keeps_epoch_and_descendants_live_through_gc() { + let (_store, tree, original) = escaped_reader_tree(); + let snapshot = tree.snapshot(b"").unwrap(); + let mut cursor = snapshot.range().into_iter(); + drop(snapshot); + + mutate_escaped_reader_tree(&tree); + tree.gc().unwrap(); + + let mut seen = 0u32; + for entry in &mut cursor { + match entry.unwrap() { + holt::RangeEntry::Key { value, .. } => { + assert_eq!(value, original); + seen += 1; + } + holt::RangeEntry::CommonPrefix(_) => panic!("unexpected delimiter rollup"), + _ => panic!("unexpected range entry variant"), + } + } + assert_eq!(seen, ESCAPED_READER_N); + + drop(cursor); + assert!(tree.gc().unwrap() > 0); +} + +#[test] +fn escaped_key_cursor_keeps_epoch_and_descendants_live_through_gc() { + let (_store, tree, _original) = escaped_reader_tree(); + let snapshot = tree.snapshot(b"").unwrap(); + let mut cursor = snapshot.range_keys().into_iter(); + drop(snapshot); + + mutate_escaped_reader_tree(&tree); + tree.gc().unwrap(); + + let mut seen = 0u32; + for entry in &mut cursor { + match entry.unwrap() { + holt::KeyRangeEntry::Key { key, .. } => { + assert_eq!(key, format!("k{seen:08}").into_bytes()); + seen += 1; + } + holt::KeyRangeEntry::CommonPrefix(_) => panic!("unexpected delimiter rollup"), + _ => panic!("unexpected key-range entry variant"), + } + } + assert_eq!(seen, ESCAPED_READER_N); + + drop(cursor); + assert!(tree.gc().unwrap() > 0); +} + +/// Store directory used by the crash-session child below. +const COW_RETIRE_CRASH_DIR_ENV: &str = "HOLT_COW_RETIRE_CRASH_DIR"; + +fn cow_retire_crash_cfg(dir: &std::path::Path) -> TreeConfig { + let mut cfg = TreeConfig::new(dir); + cfg.checkpoint.enabled = false; + cfg.memory_flush_on_write = false; + cfg +} + +/// Create a durable multi-blob tree, fork its child frames under a snapshot, +/// retire that snapshot, then durably flush only the underlying store +/// manifest. The live parent and its replacement children deliberately stay +/// in the BufferManager's dirty cache. `process::exit` models a crash by +/// skipping every Rust destructor and therefore every orderly Tree shutdown. +#[test] +#[ignore = "child-process body for cow_retire_keeps_durable_parent_children_reopenable"] +fn cow_retire_before_parent_checkpoint_crash_session() { + let Some(dir) = std::env::var_os(COW_RETIRE_CRASH_DIR_ENV) else { + return; + }; + let dir = std::path::PathBuf::from(dir); + let store = Arc::new(FileBlobStore::open(&dir).unwrap()); + let tree = Tree::open_with_blob_store(cow_retire_crash_cfg(&dir), store.clone()).unwrap(); + + const N: u32 = 5000; + let original = [0xAB_u8; 200]; + for i in 0..N { + tree.put(format!("k{i:08}").as_bytes(), &original).unwrap(); + } + tree.checkpoint().unwrap(); + assert!( + store.list_blobs().unwrap().len() >= 2, + "test requires a durable parent with child blobs", + ); + + let snapshot = tree.snapshot(b"").unwrap(); + for i in (0..N).step_by(3) { + tree.put(format!("k{i:08}").as_bytes(), b"new").unwrap(); + } + drop(snapshot); + + // Persist any manifest mutation issued by snapshot retirement without + // checkpointing the dirty live parent or its replacement children. + store.flush().unwrap(); + std::process::exit(0); +} + +#[test] +fn cow_retire_keeps_durable_parent_children_reopenable() { + let dir = tempdir().unwrap(); + let exe = std::env::current_exe().unwrap(); + let status = std::process::Command::new(exe) + .args([ + "cow_retire_before_parent_checkpoint_crash_session", + "--exact", + "--ignored", + "--nocapture", + ]) + .env(COW_RETIRE_CRASH_DIR_ENV, dir.path()) + .status() + .unwrap(); + assert!(status.success(), "crash-session child failed: {status}"); + + // The uncheckpointed live mutation is allowed to disappear. The last + // durable parent must still be able to load every original child. + let store = Arc::new(FileBlobStore::open(dir.path()).unwrap()); + let tree = Tree::open_with_blob_store(cow_retire_crash_cfg(dir.path()), store).unwrap(); + let original = [0xAB_u8; 200]; + for i in 0..5000u32 { + assert_eq!( + tree.get(format!("k{i:08}").as_bytes()).unwrap().as_deref(), + Some(&original[..]), + "durable key {i} was lost after snapshot retirement crash", + ); + } +} + +/// Store directory used by the GC crash-session child below. +const COW_GC_CRASH_DIR_ENV: &str = "HOLT_COW_GC_CRASH_DIR"; + +/// Leave the stable store at generation 1, advance the in-memory parent to +/// generation 2 through copy-on-write, and invoke public GC. GC must first +/// make generation 2 durable before it physically deletes generation 1's +/// children. +#[test] +#[ignore = "child-process body for gc_checkpoints_parent_before_physical_sweep"] +fn gc_before_parent_checkpoint_crash_session() { + let Some(dir) = std::env::var_os(COW_GC_CRASH_DIR_ENV) else { + return; + }; + let dir = std::path::PathBuf::from(dir); + let store = Arc::new(FileBlobStore::open(&dir).unwrap()); + let tree = Tree::open_with_blob_store(cow_retire_crash_cfg(&dir), store.clone()).unwrap(); + + const N: u32 = 5000; + let original = [0x11_u8; 200]; + for i in 0..N { + tree.put(format!("k{i:08}").as_bytes(), &original).unwrap(); + } + tree.checkpoint().unwrap(); + assert!(store.list_blobs().unwrap().len() >= 2); + + let snapshot = tree.snapshot(b"").unwrap(); + for i in (0..N).step_by(3) { + tree.put(format!("k{i:08}").as_bytes(), b"generation-2") + .unwrap(); + } + drop(snapshot); + + tree.gc().unwrap(); + store.flush().unwrap(); + std::process::exit(0); +} + +#[test] +fn gc_checkpoints_parent_before_physical_sweep() { + let dir = tempdir().unwrap(); + let exe = std::env::current_exe().unwrap(); + let status = std::process::Command::new(exe) + .args([ + "gc_before_parent_checkpoint_crash_session", + "--exact", + "--ignored", + "--nocapture", + ]) + .env(COW_GC_CRASH_DIR_ENV, dir.path()) + .status() + .unwrap(); + assert!(status.success(), "GC crash-session child failed: {status}"); + + let store = Arc::new(FileBlobStore::open(dir.path()).unwrap()); + let tree = Tree::open_with_blob_store(cow_retire_crash_cfg(dir.path()), store).unwrap(); + let original = [0x11_u8; 200]; + for i in 0..5000u32 { + let want: &[u8] = if i % 3 == 0 { + b"generation-2" + } else { + &original + }; + assert_eq!( + tree.get(format!("k{i:08}").as_bytes()).unwrap().as_deref(), + Some(want), + "durable key {i} did not match the GC checkpoint generation", + ); + } +} From 6f80b53ed66dccfa1778a0dcc15a7e1b656c2d7e Mon Sep 17 00:00:00 2001 From: wchwawa Date: Thu, 16 Jul 2026 11:50:06 +1000 Subject: [PATCH 3/3] docs(storage): describe durable snapshot reclamation Signed-off-by: wchwawa --- ARCHITECTURE.md | 91 +++++++++++++++++++++++++------------------------ CHANGELOG.md | 16 +++++++++ ROADMAP.md | 7 ++-- 3 files changed, 66 insertions(+), 48 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2532fb3..22307a3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -343,47 +343,45 @@ parent-validated `route_cache` — which collapses a deep walk to a single prefix-anchor edge for hot prefixes without caching raw pointers. -### Checkpoint — 7-phase protocol - -Both `Tree::checkpoint` (synchronous, caller-driven) and the -background `Checkpointer` round follow the same seven-phase -protocol, strictly ordered around the W2D invariant: - -1. **Snapshot + journal flush** under `CommitGate` checkpoint - mode. Drain live dirty entries into the checkpoint snapshot, - move them into the in-flight `flushing` protection set, drain - pending deletes, force the journal durable, and clone the - snapshotted blob bytes before releasing the gate. Flush failure - restores both snapshots. -2. **Batched per-blob write-through** with CAS-on-seq. - Successful writes release the matching `flushing` protection; - racing writers' fresh dirty entries survive for the next round. - Failures are restored to live dirty. -3. **Pre-delete sync** — `store.flush` (data file fdatasync + - manifest persist) so the writes from phase 2 are stable. - Failure restores `pending`. -4. **Abort-on-dirty-failure gate.** Any phase-2 failure restores - `pending` and returns without touching the manifest. A failed - parent write must not propagate to its child's manifest - delete — that would leave the on-disk parent referencing a - slot the manifest no longer has, and WAL replay's walker - descent through the BlobNode crossing would fail. -5. **Apply pending deletes** — manifest mutation in-memory. -6. **Post-delete sync** if any delete actually applied. Failure - restores the already-applied entries (`execute_pending_delete` - is idempotent on retry). -7. **Conditional WAL truncate** — only if `dirty_count == 0` - AND `pending_delete_count == 0` *now*. A racing writer or a - restored failure keeps the WAL alive until a future round. - -The background checkpointer is 3 threads: a planner running the -seven phases, a dedicated I/O worker processing -`IoTask::FlushBatchAndSync` + `IoTask::Sync` from a bounded -`crossbeam-channel` queue, and a cold-blob eviction sweep on the -same `clock_tick` mechanism. -`Drop` joins the planner and runs one final synchronous round on -the calling thread so dirty state between the last bg round and -shutdown doesn't get lost. +### Checkpoint — durable dependency protocol + +`Tree::checkpoint` and the background `Checkpointer` share the same W2D +ordering, while the background path pipelines complete `CheckpointEpoch` +objects through a bounded planner → I/O queue: + +1. **Capture intent** under `CommitGate` checkpoint mode: drain dirty and + pending logical-delete state, install in-flight protection, record content + versions, and force the matching WAL watermark durable. A failure restores + the complete capture. +2. **Clone version-matched bytes** outside the commit gate. A racing mutation + restores the stale item instead of letting old bytes retire newer debt. +3. **Build dependency waves** from physical BlobNode edges. Missing children, + self-cycles, and longer cycles fail closed; children are always submitted + before parents, including no-WAL checkpoints. +4. **Write each wave with CAS-on-version.** Successful writes retire only their + matching dirty/flushing ownership. External dirty children defer their + parents to a later epoch rather than publishing a dangling durable edge. +5. **Pre-delete sync** persists data and manifest additions before any logical + delete can remove an old child mapping. +6. **Apply pending logical deletes**, then run a second sync if any manifest + delete applied. Structural merge children never enter this queue: they stay + in parent-scoped orphan staging until the rewritten parent is dirty and a + clean durable frontier is proven. +7. **Retire epochs in FIFO order.** An epoch error restores all unreported + dirty/delete ownership, and a later epoch cannot advance the WAL watermark + past it. +8. **Conditional WAL truncate** requires an empty pipeline plus zero dirty, + flushing, pending-delete, orphan-staging, and write-delta debt and no store + flush debt. +9. **Bounded exact reclaim** drains retired COW/structural GUIDs while the + maintenance fence still excludes topology changes. Pinned GUIDs return to + the FIFO; crash leftovers are handled by an explicit full reachability GC. + +The background checkpointer has three threads: a planner/orchestrator, an I/O +worker that executes complete dependency-ordered epochs from a bounded +`crossbeam-channel`, and an independent cold-blob eviction sweep. `Drop` joins +the planner and runs a final synchronous round so state admitted after the last +idle round is not lost. `Tree::compact` is online with respect to point reads and foreground writers through `maintenance_gate`. Range iterators @@ -424,12 +422,15 @@ cursor path forces a rebuild from the monotonic lower bound. This is stronger than the upstream-style "invalid iterator" surface because stale paths are handled internally. It is still not MVCC: a long scan can observe keys committed after iterator creation if -they sort after the current cursor. +they sort after the current cursor. Each cursor step holds the shared +maintenance/mutation gates only for that step; an iterator paused in +caller code does not block structural maintenance. For stable read transactions, `Tree::view(prefix, |view| ...)` -captures the prefix's reachable blob frames into a private in-memory -store and then scans that copy. This pays copy cost proportional to -the observed prefix instead of adding per-write MVCC chains. +copies one root frame and shares descendants with the live tree. +Subsequent live writes fork only snapshot-visible frames. Cloned views +and owned cursors keep the snapshot epoch lease alive after the callback +or main snapshot handle returns, until the final derived handle drops. ## 8. BlobStore abstraction diff --git a/CHANGELOG.md b/CHANGELOG.md index 26c046d..d749e0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,22 @@ fine-grained per-commit history is in `git log`. - Added store-space observability for physical allocated bytes, tail reclaimable slots/bytes, reusable middle slots, and vacuum relocation counters. +- Added GC observability for current orphan backlog, cumulative physical + reclamation, and the deferred count from the most recent full reachability + sweep. The latter is intentionally not reset by exact FIFO reclaim. + +### Changed + +- **Breaking stats API hardening.** `TreeStats`, `DBStats`, and `OpenStats` are + now `#[non_exhaustive]`; `TreeStats`/`DBStats` add GC lifecycle fields and + `OpenStats` adds DB epoch-recovery duration. External code that constructed + these returned telemetry structs or destructured them without `..` must use + field access/non-exhaustive matching instead. This one-time break makes + future telemetry additions semver-compatible without compatibility shims. +- GC Prometheus names follow their actual contracts: + `holt_bm_gc_reclaimed_total` is the cumulative full-sweep plus exact-reclaim + counter, while `holt_bm_gc_last_full_sweep_deferred_count` is a gauge for + the most recently completed full reachability sweep only. ### Fixed diff --git a/ROADMAP.md b/ROADMAP.md index 3659c76..dbfab4a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -380,9 +380,10 @@ single-node embedded library, no always-on MVCC version chains. ### Completed — Scoped snapshot view -- `Tree::view(prefix, |view| ...)` captures the blob frames reachable - for a prefix, releases the live tree, and serves point reads and - scans from the private frame set. +- `Tree::view(prefix, |view| ...)` performs one root-frame copy, shares + descendants, and serves stable point reads/scans while first writes fork + snapshot-visible frames. Cloned views and owned cursors extend the epoch + lease beyond the callback until their final handle drops. - Ordinary `range()` / `range_keys()` remain the hot restart-on-conflict iterators; `View` is the explicit stable-read path for list/readdir.