diff --git a/Cargo.lock b/Cargo.lock index 598d584fb..d831f3e8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -811,9 +811,8 @@ dependencies = [ [[package]] name = "holt" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f7180148389fd08f76cdd05ad47640acd1b72b6a5cd2379d1496de96af32e01" +version = "0.8.1" +source = "git+https://github.com/NoKV-Lab/holt.git?rev=6f80b53ed66dccfa1778a0dcc15a7e1b656c2d7e#6f80b53ed66dccfa1778a0dcc15a7e1b656c2d7e" dependencies = [ "crc32fast", "crossbeam-channel", @@ -1383,6 +1382,7 @@ dependencies = [ "nokv-fuse", "nokv-meta", "nokv-object", + "nokv-protocol", "nokv-server", "nokv-types", "serde", diff --git a/Cargo.toml b/Cargo.toml index 67f325b3b..afaa0dbc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ nokv = { path = "crates/nokv", version = "0.1.0" } nokv-python = { path = "crates/nokv-python", version = "0.1.0" } nokv-bench = { path = "bench", version = "0.1.0" } fuser = { version = "0.17.0", default-features = false } -holt = { version = "0.7.3", default-features = false } +holt = { git = "https://github.com/NoKV-Lab/holt.git", rev = "6f80b53ed66dccfa1778a0dcc15a7e1b656c2d7e", default-features = false } etcd-client = { version = "0.19.0", default-features = false } libc = "0.2" opendal = { version = "0.57.0", default-features = false, features = ["blocking", "executors-tokio", "reqwest-rustls-tls", "services-s3"] } diff --git a/bench/src/bin/nokv-bench.rs b/bench/src/bin/nokv-bench.rs index 869db6942..072c1cc52 100644 --- a/bench/src/bin/nokv-bench.rs +++ b/bench/src/bin/nokv-bench.rs @@ -3157,6 +3157,29 @@ fn stats_delta(before: BenchStats, after: BenchStats) -> BenchStats { .metadata_service .metadata_log_archive_bytes_total .saturating_sub(before.metadata_service.metadata_log_archive_bytes_total), + restore_to_fork_requests_total: after + .metadata_service + .restore_to_fork_requests_total + .saturating_sub(before.metadata_service.restore_to_fork_requests_total), + restore_to_fork_success_total: after + .metadata_service + .restore_to_fork_success_total + .saturating_sub(before.metadata_service.restore_to_fork_success_total), + restore_to_fork_failure_total: after + .metadata_service + .restore_to_fork_failure_total + .saturating_sub(before.metadata_service.restore_to_fork_failure_total), + restore_to_fork_elapsed_ns_total: after + .metadata_service + .restore_to_fork_elapsed_ns_total + .saturating_sub(before.metadata_service.restore_to_fork_elapsed_ns_total), + restore_to_fork_elapsed_ns_max: if after.metadata_service.restore_to_fork_requests_total + == before.metadata_service.restore_to_fork_requests_total + { + 0 + } else { + after.metadata_service.restore_to_fork_elapsed_ns_max + }, }, } } @@ -3404,6 +3427,11 @@ fn fetch_server_stats(address: SocketAddr) -> Result { "metadata_log_entries_archived_total", )?, metadata_log_archive_bytes_total: json_u64(body, "metadata_log_archive_bytes_total")?, + restore_to_fork_requests_total: json_u64(body, "restore_to_fork_requests_total")?, + restore_to_fork_success_total: json_u64(body, "restore_to_fork_success_total")?, + restore_to_fork_failure_total: json_u64(body, "restore_to_fork_failure_total")?, + restore_to_fork_elapsed_ns_total: json_u64(body, "restore_to_fork_elapsed_ns_total")?, + restore_to_fork_elapsed_ns_max: json_u64(body, "restore_to_fork_elapsed_ns_max")?, }, }) } @@ -4147,7 +4175,7 @@ mod tests { #[test] fn stats_json_parser_reads_metadata_fields() { - let body = r#"{"object_puts":41,"object_put_bytes":42,"object_gets":43,"object_get_bytes":44,"coalesced_gets":45,"coalesced_get_bytes":46,"cache_hits":47,"cache_hit_bytes":48,"prefetch_enqueued":49,"prefetch_dropped":50,"prefetch_completed":51,"prefetch_failed":52,"prefetch_object_gets":53,"prefetch_object_get_bytes":54,"prefetch_cache_hits":55,"prefetch_cache_hit_bytes":56,"read_plan_cache_hits":57,"read_plan_cache_misses":58,"object_writeback_enqueued":59,"object_writeback_inline":60,"object_writeback_completed":62,"object_writeback_failed":63,"object_writeback_staged_bytes":64,"object_writeback_uploaded_bytes":65,"object_writeback_queue_wait_ns":66,"object_writeback_queue_max_wait_ns":67,"object_writeback_upload_ns":68,"object_writeback_upload_max_ns":69,"object_writeback_collect_ns":70,"object_writeback_digest_ns":71,"object_writeback_store_put_ns":72,"object_writeback_cache_put_ns":73,"manifest_chunks":74,"manifest_blocks":75,"tiered_hot_put_ns":76,"tiered_pending_cold_put_ns":77,"tiered_cold_put_enqueue_ns":78,"local_hot_puts":79,"local_hot_put_bytes":80,"local_hot_put_total_ns":81,"local_hot_put_prepare_ns":82,"local_hot_put_write_ns":83,"local_hot_put_sync_ns":84,"local_hot_put_rename_ns":85,"local_hot_put_record_ns":86,"metadata_store":{"get_total":2,"get_user_strong_total":32,"get_write_plan_local_total":33,"get_snapshot_total":34,"scan_total":3,"scan_user_strong_total":35,"scan_write_plan_local_total":36,"scan_snapshot_total":37,"scan_key_visited_total":4,"scan_key_returned_total":5,"history_lookup_total":40,"active_snapshot_pin_total":0,"commit_total":6,"dedupe_hit_total":7,"predicate_total":8,"prefix_empty_predicate_total":9,"current_put_total":10,"current_delete_total":11,"history_write_total":12,"watch_write_total":13,"dedupe_write_total":14,"commit_prepare_ns_total":15,"atomic_apply_total":16,"atomic_apply_command_total":17,"atomic_apply_max_batch":18,"atomic_apply_ns_total":19},"metadata_service":{"path_index_lookup_total":30,"path_index_hit_total":31,"path_index_miss_total":32,"path_index_stale_total":33,"path_index_scan_stale_total":34,"path_index_fallback_total":35,"create_files_batch_total":36,"create_files_entry_total":37,"create_dirs_batch_total":38,"create_dirs_entry_total":39,"read_dir_plus_total":40,"read_dir_plus_entry_total":41,"read_dir_plus_projection_hit_total":42,"metadata_log_segments_archived_total":43,"metadata_log_entries_archived_total":44,"metadata_log_archive_bytes_total":45}}"#; + let body = r#"{"object_puts":41,"object_put_bytes":42,"object_gets":43,"object_get_bytes":44,"coalesced_gets":45,"coalesced_get_bytes":46,"cache_hits":47,"cache_hit_bytes":48,"prefetch_enqueued":49,"prefetch_dropped":50,"prefetch_completed":51,"prefetch_failed":52,"prefetch_object_gets":53,"prefetch_object_get_bytes":54,"prefetch_cache_hits":55,"prefetch_cache_hit_bytes":56,"read_plan_cache_hits":57,"read_plan_cache_misses":58,"object_writeback_enqueued":59,"object_writeback_inline":60,"object_writeback_completed":62,"object_writeback_failed":63,"object_writeback_staged_bytes":64,"object_writeback_uploaded_bytes":65,"object_writeback_queue_wait_ns":66,"object_writeback_queue_max_wait_ns":67,"object_writeback_upload_ns":68,"object_writeback_upload_max_ns":69,"object_writeback_collect_ns":70,"object_writeback_digest_ns":71,"object_writeback_store_put_ns":72,"object_writeback_cache_put_ns":73,"manifest_chunks":74,"manifest_blocks":75,"tiered_hot_put_ns":76,"tiered_pending_cold_put_ns":77,"tiered_cold_put_enqueue_ns":78,"local_hot_puts":79,"local_hot_put_bytes":80,"local_hot_put_total_ns":81,"local_hot_put_prepare_ns":82,"local_hot_put_write_ns":83,"local_hot_put_sync_ns":84,"local_hot_put_rename_ns":85,"local_hot_put_record_ns":86,"metadata_store":{"get_total":2,"get_user_strong_total":32,"get_write_plan_local_total":33,"get_snapshot_total":34,"scan_total":3,"scan_user_strong_total":35,"scan_write_plan_local_total":36,"scan_snapshot_total":37,"scan_key_visited_total":4,"scan_key_returned_total":5,"history_lookup_total":40,"active_snapshot_pin_total":0,"commit_total":6,"dedupe_hit_total":7,"predicate_total":8,"prefix_empty_predicate_total":9,"current_put_total":10,"current_delete_total":11,"history_write_total":12,"watch_write_total":13,"dedupe_write_total":14,"commit_prepare_ns_total":15,"atomic_apply_total":16,"atomic_apply_command_total":17,"atomic_apply_max_batch":18,"atomic_apply_ns_total":19},"metadata_service":{"path_index_lookup_total":30,"path_index_hit_total":31,"path_index_miss_total":32,"path_index_stale_total":33,"path_index_scan_stale_total":34,"path_index_fallback_total":35,"create_files_batch_total":36,"create_files_entry_total":37,"create_dirs_batch_total":38,"create_dirs_entry_total":39,"read_dir_plus_total":40,"read_dir_plus_entry_total":41,"read_dir_plus_projection_hit_total":42,"metadata_log_segments_archived_total":43,"metadata_log_entries_archived_total":44,"metadata_log_archive_bytes_total":45,"restore_to_fork_requests_total":46,"restore_to_fork_success_total":47,"restore_to_fork_failure_total":48,"restore_to_fork_elapsed_ns_total":49,"restore_to_fork_elapsed_ns_max":50}}"#; assert_eq!(json_u64(body, "object_put_bytes").unwrap(), 42); assert_eq!(json_u64(body, "object_get_bytes").unwrap(), 44); @@ -4220,6 +4248,20 @@ mod tests { json_u64(body, "metadata_log_archive_bytes_total").unwrap(), 45 ); + assert_eq!( + json_u64(body, "restore_to_fork_requests_total").unwrap(), + 46 + ); + assert_eq!(json_u64(body, "restore_to_fork_success_total").unwrap(), 47); + assert_eq!(json_u64(body, "restore_to_fork_failure_total").unwrap(), 48); + assert_eq!( + json_u64(body, "restore_to_fork_elapsed_ns_total").unwrap(), + 49 + ); + assert_eq!( + json_u64(body, "restore_to_fork_elapsed_ns_max").unwrap(), + 50 + ); assert!(json_u64(body, "missing_total").is_err()); } diff --git a/crates/nokv-client/src/lib.rs b/crates/nokv-client/src/lib.rs index 99c3b3268..c657b9a7e 100644 --- a/crates/nokv-client/src/lib.rs +++ b/crates/nokv-client/src/lib.rs @@ -29,11 +29,12 @@ pub use file_client::{ is_artifact_write_conflict, is_artifact_write_retryable, AppendOutcome, NoKvFsClient, PathRangeReadRequest, PathReadRange, PreparedPathRangeBatch, }; +pub use nokv_meta::{RestoreOutcome, RestoreState}; pub use nokv_object::{DataFabricReadStats, ObjectReadPlan}; pub use nokv_protocol::{decode_name_cursor, encode_name_cursor}; pub use service::{ - ClientPreparedArtifact, ClientPreparedArtifactRecovery, CloneOutcome, MetadataClient, - MetadataClientOptions, PathLayoutOpen, PathLayoutOpenRequest, SnapshotOutcome, + ClientPreparedArtifact, ClientPreparedArtifactRecovery, CloneOutcome, MetadataCapabilities, + MetadataClient, MetadataClientOptions, PathLayoutOpen, PathLayoutOpenRequest, SnapshotOutcome, SnapshotPinStatus, }; diff --git a/crates/nokv-client/src/service.rs b/crates/nokv-client/src/service.rs index 6c04fa234..24742abc8 100644 --- a/crates/nokv-client/src/service.rs +++ b/crates/nokv-client/src/service.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::net::SocketAddr; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, RwLock}; @@ -6,39 +6,42 @@ use std::time::Duration; use nokv_control::{ControlStore, ShardId, ShardRecord, ShardState}; use nokv_meta::{ - DentryWithAttr, NamespaceAggregateGroup, NamespaceAggregateMeasure, NamespaceAggregateOp, - NamespaceAggregateOutputMeasure, NamespaceAggregateRequest, NamespaceAggregateResult, - NamespaceAggregateSample, NamespaceAggregateSort, NamespaceAggregateValue, - NamespaceBodyDescriptor, NamespaceCard, NamespaceCardKind, NamespaceFacetSummary, - NamespaceFacetValue, NamespaceFieldSource, NamespaceFieldSourceKind, NamespaceFieldValue, - NamespaceFilterCapability, NamespaceFindField, NamespaceFindRequest, NamespaceFindResult, - NamespaceGrepMatch, NamespaceGrepRequest, NamespaceGrepResult, NamespaceInclude, - NamespaceIndexValue, NamespaceListOptions, NamespaceListPage, NamespacePredicate, - NamespacePredicateOp, NamespacePredicateValue, NamespaceQueryCatalog, NamespaceReadFormat, - NamespaceReadItem, NamespaceReadOptions, NamespaceReadPage, NamespaceRecordCount, - NamespaceRecordType, NamespaceSchema, NamespaceSort, NamespaceSortDirection, - NamespaceSortField, PublishArtifactStagedSession, RecordCountProvenance, RenameReplaceResult, - SnapshotRenewOutcome, SubtreeDelta, UpdateAttr, XattrSetMode, + restore_operation_id, DentryWithAttr, NamespaceAggregateGroup, NamespaceAggregateMeasure, + NamespaceAggregateOp, NamespaceAggregateOutputMeasure, NamespaceAggregateRequest, + NamespaceAggregateResult, NamespaceAggregateSample, NamespaceAggregateSort, + NamespaceAggregateValue, NamespaceBodyDescriptor, NamespaceCard, NamespaceCardKind, + NamespaceFacetSummary, NamespaceFacetValue, NamespaceFieldSource, NamespaceFieldSourceKind, + NamespaceFieldValue, NamespaceFilterCapability, NamespaceFindField, NamespaceFindRequest, + NamespaceFindResult, NamespaceGrepMatch, NamespaceGrepRequest, NamespaceGrepResult, + NamespaceInclude, NamespaceIndexValue, NamespaceListOptions, NamespaceListPage, + NamespacePredicate, NamespacePredicateOp, NamespacePredicateValue, NamespaceQueryCatalog, + NamespaceReadFormat, NamespaceReadItem, NamespaceReadOptions, NamespaceReadPage, + NamespaceRecordCount, NamespaceRecordType, NamespaceSchema, NamespaceSort, + NamespaceSortDirection, NamespaceSortField, PublishArtifactStagedSession, + RecordCountProvenance, RenameReplaceResult, RestoreInitialization, RestoreOutcome, + RestoreState, SnapshotRenewOutcome, SubtreeDelta, UpdateAttr, XattrSetMode, }; use nokv_object::ObjectReadPlan; use nokv_protocol::{ decode_envelope, decode_name_cursor, decode_xattr_name, encode_advisory_lock_kind, encode_file_type, encode_name_cursor, encode_request, encode_xattr_name, request_routing_key, - MetadataRpcRequest, MetadataRpcResult, RoutingKey, WireNamespaceAggregateGroup, - WireNamespaceAggregateMeasure, WireNamespaceAggregateOp, WireNamespaceAggregateOutputMeasure, - WireNamespaceAggregateRequest, WireNamespaceAggregateResult, WireNamespaceAggregateSample, - WireNamespaceAggregateSort, WireNamespaceAggregateValue, WireNamespaceCard, - WireNamespaceCardKind, WireNamespaceFacetSummary, WireNamespaceFacetValue, - WireNamespaceFieldSource, WireNamespaceFieldSourceKind, WireNamespaceFieldValue, - WireNamespaceFilterCapability, WireNamespaceFindField, WireNamespaceFindRequest, - WireNamespaceFindResult, WireNamespaceGrepMatch, WireNamespaceGrepRequest, - WireNamespaceGrepResult, WireNamespaceInclude, WireNamespaceIndexValue, WireNamespaceListPage, - WireNamespacePredicate, WireNamespacePredicateOp, WireNamespacePredicateValue, - WireNamespaceQueryCatalog, WireNamespaceReadFormat, WireNamespaceReadItem, - WireNamespaceReadOptions, WireNamespaceReadPage, WireNamespaceRecordCount, - WireNamespaceRecordType, WireNamespaceSchema, WireNamespaceSort, WireNamespaceSortDirection, - WireNamespaceSortField, WireOpenPathReadPlanRequest, WireRecordCountProvenance, - WireSnapshotRenewOutcome, + MetadataRpcRequest, MetadataRpcResult, RoutingKey, WireMetadataCapabilities, + WireNamespaceAggregateGroup, WireNamespaceAggregateMeasure, WireNamespaceAggregateOp, + WireNamespaceAggregateOutputMeasure, WireNamespaceAggregateRequest, + WireNamespaceAggregateResult, WireNamespaceAggregateSample, WireNamespaceAggregateSort, + WireNamespaceAggregateValue, WireNamespaceCard, WireNamespaceCardKind, + WireNamespaceFacetSummary, WireNamespaceFacetValue, WireNamespaceFieldSource, + WireNamespaceFieldSourceKind, WireNamespaceFieldValue, WireNamespaceFilterCapability, + WireNamespaceFindField, WireNamespaceFindRequest, WireNamespaceFindResult, + WireNamespaceGrepMatch, WireNamespaceGrepRequest, WireNamespaceGrepResult, + WireNamespaceInclude, WireNamespaceIndexValue, WireNamespaceListPage, WireNamespacePredicate, + WireNamespacePredicateOp, WireNamespacePredicateValue, WireNamespaceQueryCatalog, + WireNamespaceReadFormat, WireNamespaceReadItem, WireNamespaceReadOptions, + WireNamespaceReadPage, WireNamespaceRecordCount, WireNamespaceRecordType, WireNamespaceSchema, + WireNamespaceSort, WireNamespaceSortDirection, WireNamespaceSortField, + WireOpenPathReadPlanRequest, WireRecordCountProvenance, WireRestoreInitialization, + WireRestoreInitializationFile, WireRestoreState, WireSnapshotRenewOutcome, + RESTORE_TO_FORK_V1_CAPABILITY, }; use nokv_types::{ AdvisoryLock, AdvisoryLockRequest, BodyDescriptor, ChunkManifest, DentryName, FileType, @@ -54,6 +57,15 @@ use crate::framed::{read_frame, write_frame, FRAMED_RPC_MAGIC}; use crate::wire::*; const DEFAULT_RPC_TIMEOUT: Duration = Duration::from_secs(10); +// A durable restore may legitimately outlive an ordinary metadata RPC while it +// materializes bounded pages or waits for a release/GC safety fence. The public +// operation remains idempotent, but a healthy default client must not abandon +// it at the ordinary ten-second request deadline. +const DEFAULT_RESTORE_RPC_TIMEOUT: Duration = Duration::from_secs(300); + +fn restore_rpc_timeout(configured: Duration) -> Duration { + configured.max(DEFAULT_RESTORE_RPC_TIMEOUT) +} const MAX_BATCH_RPC_REQUESTS: usize = 128; fn snapshot_inode_api_disabled() -> ClientError { @@ -199,6 +211,14 @@ pub struct CloneOutcome { pub snapshot_id: u64, } +/// Feature bits reported by the metadata owner selected for a path. A path is +/// only a routing key for this probe and does not need to exist. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MetadataCapabilities { + pub mount_id: MountId, + pub restore_to_fork_v1: bool, +} + /// Result of pinning a subtree snapshot: the durable `snapshot_id` to pass to a /// later rollback, the read version the snapshot captured, and when the pin's /// lease expires. After `lease_expires_unix_ms`, GC may reap the pin and the @@ -1575,6 +1595,131 @@ impl MetadataClient { } } + pub fn metadata_capabilities(&self, path: &str) -> Result { + match self.call(MetadataRpcRequest::MetadataCapabilities { + path: path.to_owned(), + })? { + MetadataRpcResult::MetadataCapabilities { capabilities } => { + metadata_capabilities(capabilities) + } + other => Err(unexpected_result(other)), + } + } + + /// Query every metadata owner that can win longest-prefix routing at or + /// below `root`. The root probe covers the fallback owner (including an + /// ancestor route); each registered descendant prefix is itself a routing + /// key for the owner that can override that fallback. + /// + /// Fleet routes are refreshed from the control plane before enumeration so + /// capability-gated surfaces fail closed during rolling deployment. In + /// single-shard mode the one root probe covers the only possible owner. + pub fn metadata_capabilities_for_subtree_owners( + &self, + root: &str, + ) -> Result, ClientError> { + let probe_paths = match &self.mode { + RoutingMode::SingleShard { .. } => vec![root.to_owned()], + RoutingMode::Fleet(router) => { + router.refresh(self)?; + router.subtree_owner_probe_paths(root) + } + }; + probe_paths + .into_iter() + .map(|path| self.metadata_capabilities(&path)) + .collect() + } + + pub fn restore_subtree_path_to_fork( + &self, + source: &str, + snapshot_id: u64, + destination: &str, + ) -> Result { + self.restore_subtree_path_to_fork_initialized( + source, + snapshot_id, + destination, + RestoreInitialization::default(), + ) + } + + pub fn restore_subtree_path_to_fork_initialized( + &self, + source: &str, + snapshot_id: u64, + destination: &str, + initialization: RestoreInitialization, + ) -> Result { + self.ensure_same_shard_paths(source, destination)?; + let capabilities = match self.metadata_capabilities(destination) { + Ok(capabilities) => capabilities, + // A pre-v1 owner cannot decode the capability request. Treat that + // as an unavailable capability instead of leaking a generic wire + // error to callers during a rolling deployment. + Err(ClientError::Protocol(_)) => { + return Err(restore_capability_unavailable(destination)); + } + Err(err) => return Err(err), + }; + if !capabilities.restore_to_fork_v1 { + return Err(restore_capability_unavailable(destination)); + } + let expected_operation_id = + restore_operation_id(capabilities.mount_id, source, snapshot_id, destination)?; + match self.call_with_timeout( + MetadataRpcRequest::RestoreSubtreePathToFork { + source_path: source.to_owned(), + snapshot_id, + destination_path: destination.to_owned(), + initialization: WireRestoreInitialization { + remove_relative_paths: initialization.remove_relative_paths, + files: initialization + .files + .into_iter() + .map(|file| WireRestoreInitializationFile { + relative_path: file.relative_path, + bytes: file.bytes, + content_type: file.content_type, + mode: file.mode, + uid: file.uid, + gid: file.gid, + }) + .collect(), + }, + }, + restore_rpc_timeout(self.timeout), + )? { + MetadataRpcResult::Restore { outcome } + if outcome.operation_id == expected_operation_id + && outcome.snapshot_id == snapshot_id + && !outcome.cleanup_pending => + { + Ok(RestoreOutcome { + operation_id: outcome.operation_id, + state: match outcome.state { + WireRestoreState::Complete => RestoreState::Complete, + }, + source_root: inode_id(outcome.source_root)?, + destination_root: inode_id(outcome.destination_root)?, + snapshot_id: outcome.snapshot_id, + read_version: outcome.read_version, + cleanup_pending: false, + }) + } + MetadataRpcResult::Restore { outcome } => Err(ClientError::Protocol(format!( + "restore outcome mismatch: expected operation {} snapshot {} with no pending cleanup, got operation {} snapshot {} cleanup_pending={}", + expected_operation_id, + snapshot_id, + outcome.operation_id, + outcome.snapshot_id, + outcome.cleanup_pending + ))), + other => Err(unexpected_result(other)), + } + } + pub fn diff_subtrees(&self, a: &str, b: &str) -> Result, ClientError> { self.ensure_same_shard_paths(a, b)?; match self.call(MetadataRpcRequest::DiffSubtrees { @@ -2078,11 +2223,19 @@ impl MetadataClient { /// shard endpoint, sends the request, and — in fleet mode — re-resolves the /// routing map and retries on an owner handoff (`NotOwner`/stale owner). fn call(&self, request: MetadataRpcRequest) -> Result { + self.call_with_timeout(request, self.timeout) + } + + fn call_with_timeout( + &self, + request: MetadataRpcRequest, + timeout: Duration, + ) -> Result { let body = encode_request(&request).map_err(|err| ClientError::Protocol(err.to_string()))?; match &self.mode { - RoutingMode::SingleShard { address } => self.call_endpoint(*address, &body), - RoutingMode::Fleet(router) => self.call_fleet(router, &request, &body), + RoutingMode::SingleShard { address } => self.call_endpoint(*address, &body, timeout), + RoutingMode::Fleet(router) => self.call_fleet(router, &request, &body, timeout), } } @@ -2117,6 +2270,7 @@ impl MetadataClient { router: &FleetRouter, request: &MetadataRpcRequest, body: &[u8], + timeout: Duration, ) -> Result { let mut last_resolve_err: Option = None; for attempt in 0..FLEET_MAX_ATTEMPTS { @@ -2138,7 +2292,7 @@ impl MetadataClient { continue; } }; - match self.call_endpoint(address, body) { + match self.call_endpoint(address, body, timeout) { Ok(result) => return Ok(result), Err(err) if attempt + 1 < FLEET_MAX_ATTEMPTS && is_owner_handoff(&err) => { // Owner moved: loop to refresh + re-resolve against the new owner. @@ -2161,10 +2315,11 @@ impl MetadataClient { &self, address: SocketAddr, body: &[u8], + timeout: Duration, ) -> Result { let request_id = self.next_request_id.fetch_add(1, Ordering::Relaxed); let connection = self.connection(address)?; - let body = match connection.call(request_id, body, self.timeout) { + let body = match connection.call(request_id, body, timeout) { Ok(body) => body, Err(err @ ClientError::Io(_)) => { self.drop_connection(address); @@ -2222,6 +2377,24 @@ impl FleetRouter { .route(self.mount, path) } + /// Return one routing key for the subtree fallback owner plus every + /// registered descendant route. Do not hold the route-map lock while the + /// probes run: a probe may observe an owner handoff and refresh the map. + fn subtree_owner_probe_paths(&self, root: &str) -> Vec { + let map = self.shard_map.read().expect("fleet shard map"); + let mut seen = HashSet::new(); + let mut paths = Vec::new(); + seen.insert(root.to_owned()); + paths.push(root.to_owned()); + for route in map.routes() { + let path = &route.prefix.path; + if path_is_strict_descendant(path, root) && seen.insert(path.clone()) { + paths.push(path.clone()); + } + } + paths + } + /// The endpoint currently mapped to `shard_index`, if the cache knows it. fn endpoint(&self, shard_index: u16) -> Option { self.endpoints @@ -2238,7 +2411,7 @@ impl FleetRouter { let (shard_map, endpoints) = resolve_fleet_routes(self.control.as_ref(), self.mount)?; // Drop pooled connections to endpoints that are no longer current so a // handed-off owner's stale socket is not reused. - let live: std::collections::HashSet = endpoints.values().copied().collect(); + let live: HashSet = endpoints.values().copied().collect(); { let mut pool = client.connections.lock().expect("metadata rpc connections"); pool.retain(|address, _| live.contains(address)); @@ -2296,6 +2469,14 @@ fn resolve_fleet_routes( Ok((ShardMap::from_routes(routes), endpoints)) } +fn path_is_strict_descendant(path: &str, root: &str) -> bool { + if root == "/" { + return path != "/" && path.starts_with('/'); + } + path.strip_prefix(root) + .is_some_and(|suffix| suffix.starts_with('/')) +} + /// Whether an error means "the owner moved" — re-resolve the shard map and retry /// against the new owner. Covers an explicit `NotOwner`, a stale owner epoch, and /// a self-fenced expired lease. @@ -2422,6 +2603,22 @@ fn namespace_record_count( }) } +fn metadata_capabilities( + capabilities: WireMetadataCapabilities, +) -> Result { + Ok(MetadataCapabilities { + mount_id: MountId::new(capabilities.mount_id) + .map_err(|err| ClientError::Protocol(err.to_string()))?, + restore_to_fork_v1: capabilities.restore_to_fork_v1, + }) +} + +fn restore_capability_unavailable(path: &str) -> ClientError { + ClientError::Protocol(format!( + "required metadata capability {RESTORE_TO_FORK_V1_CAPABILITY} is unavailable for owner of {path}" + )) +} + fn namespace_schema(schema: WireNamespaceSchema) -> NamespaceSchema { NamespaceSchema { record_type: namespace_record_type(schema.record_type), diff --git a/crates/nokv-client/src/service_tests.rs b/crates/nokv-client/src/service_tests.rs index 6014e43ad..77eed188a 100644 --- a/crates/nokv-client/src/service_tests.rs +++ b/crates/nokv-client/src/service_tests.rs @@ -16,6 +16,18 @@ use std::sync::{Arc, Condvar, Mutex}; use std::thread; use std::time::{Duration, Instant}; +#[test] +fn durable_restore_uses_a_long_rpc_deadline_without_shortening_custom_timeouts() { + assert_eq!( + restore_rpc_timeout(Duration::from_secs(10)), + Duration::from_secs(300) + ); + assert_eq!( + restore_rpc_timeout(Duration::from_secs(600)), + Duration::from_secs(600) + ); +} + fn serve_one(body: &'static str) -> SocketAddr { serve_many(vec![response_body(body)]) } @@ -3024,6 +3036,111 @@ fn service_clone_subtree_path_sends_typed_rpc_and_maps_outcome() { assert_eq!(outcome.snapshot_id, 7); } +#[test] +fn service_restore_probes_owner_capability_and_maps_typed_outcome() { + let addr = serve_request_sequence(vec![ + Box::new(|request| { + assert_eq!( + request, + MetadataRpcRequest::MetadataCapabilities { + path: "/workbenches/restored".to_owned(), + } + ); + response_body( + r#"{"ok":true,"result":{"type":"metadata_capabilities","capabilities":{"mount_id":1,"restore_to_fork_v1":true}}}"#, + ) + }), + Box::new(|request| { + assert_eq!( + request, + MetadataRpcRequest::RestoreSubtreePathToFork { + source_path: "/workbenches/source".to_owned(), + snapshot_id: 17, + destination_path: "/workbenches/restored".to_owned(), + initialization: nokv_protocol::WireRestoreInitialization { + remove_relative_paths: vec!["metadata/checkpoints.jsonl".to_owned()], + files: vec![nokv_protocol::WireRestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: b"manifest".to_vec(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, + } + ); + let operation_id = nokv_meta::restore_operation_id( + nokv_types::MountId::new(1).unwrap(), + "/workbenches/source", + 17, + "/workbenches/restored", + ) + .unwrap(); + response_body(&format!( + r#"{{"ok":true,"result":{{"type":"restore","outcome":{{"operation_id":"{operation_id}","state":"complete","source_root":7,"destination_root":8,"snapshot_id":17,"read_version":42,"cleanup_pending":false}}}}}}"#, + )) + }), + ]); + let client = MetadataClient::connect(addr); + let outcome = client + .restore_subtree_path_to_fork_initialized( + "/workbenches/source", + 17, + "/workbenches/restored", + nokv_meta::RestoreInitialization { + remove_relative_paths: vec!["metadata/checkpoints.jsonl".to_owned()], + files: vec![nokv_meta::RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: b"manifest".to_vec(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, + ) + .unwrap(); + assert_eq!( + outcome.operation_id, + nokv_meta::restore_operation_id( + nokv_types::MountId::new(1).unwrap(), + "/workbenches/source", + 17, + "/workbenches/restored", + ) + .unwrap() + ); + assert_eq!(outcome.state, nokv_meta::RestoreState::Complete); + assert_eq!(outcome.source_root.get(), 7); + assert_eq!(outcome.destination_root.get(), 8); + assert_eq!(outcome.snapshot_id, 17); + assert_eq!(outcome.read_version, 42); + assert!(!outcome.cleanup_pending); +} + +#[test] +fn service_restore_fails_closed_when_owner_lacks_capability() { + let addr = serve_one_request(|request| { + assert_eq!( + request, + MetadataRpcRequest::MetadataCapabilities { + path: "/workbenches/restored".to_owned(), + } + ); + response_body( + r#"{"ok":true,"result":{"type":"metadata_capabilities","capabilities":{"mount_id":1,"restore_to_fork_v1":false}}}"#, + ) + }); + let client = MetadataClient::connect(addr); + let error = client + .restore_subtree_path_to_fork("/workbenches/source", 17, "/workbenches/restored") + .unwrap_err(); + assert!(matches!(error, ClientError::Protocol(ref message) + if message.contains(nokv_protocol::RESTORE_TO_FORK_V1_CAPABILITY) + && message.contains("/workbenches/restored"))); +} + #[test] fn service_diff_subtrees_sends_typed_rpc_and_maps_deltas() { let addr = serve_one_request(|request| { @@ -3193,6 +3310,94 @@ fn fleet_resolves_each_request_to_its_shard_owner_endpoint() { ); } +#[test] +fn fleet_subtree_capabilities_probe_fallback_and_every_descendant_owner() { + let fallback = serve_one_request(|request| { + assert_eq!( + request, + MetadataRpcRequest::MetadataCapabilities { + path: "/workbenches".to_owned(), + } + ); + response_body( + r#"{"ok":true,"result":{"type":"metadata_capabilities","capabilities":{"mount_id":1,"restore_to_fork_v1":true}}}"#, + ) + }); + let nested = serve_one_request(|request| { + assert_eq!( + request, + MetadataRpcRequest::MetadataCapabilities { + path: "/workbenches/special".to_owned(), + } + ); + response_body( + r#"{"ok":true,"result":{"type":"metadata_capabilities","capabilities":{"mount_id":1,"restore_to_fork_v1":false}}}"#, + ) + }); + let deep = serve_one_request(|request| { + assert_eq!( + request, + MetadataRpcRequest::MetadataCapabilities { + path: "/workbenches/special/deep".to_owned(), + } + ); + response_body( + r#"{"ok":true,"result":{"type":"metadata_capabilities","capabilities":{"mount_id":1,"restore_to_fork_v1":true}}}"#, + ) + }); + let outside = serve_one_request(|request| { + assert_eq!( + request, + MetadataRpcRequest::MetadataCapabilities { + path: "/workbenches-other".to_owned(), + } + ); + response_body( + r#"{"ok":true,"result":{"type":"metadata_capabilities","capabilities":{"mount_id":1,"restore_to_fork_v1":true}}}"#, + ) + }); + + let store: Arc = Arc::new(InMemoryControlStore::new()); + register_owned_shard(store.as_ref(), "/", 0, &fallback.to_string()); + register_owned_shard( + store.as_ref(), + "/workbenches/special", + 1, + &nested.to_string(), + ); + register_owned_shard( + store.as_ref(), + "/workbenches/special/deep", + 2, + &deep.to_string(), + ); + // These component-boundary neighbours are outside the requested subtree. + // An accidental broad starts_with match would add a fourth capability. + register_owned_shard( + store.as_ref(), + "/workbenches-other", + 3, + &outside.to_string(), + ); + + let client = MetadataClient::fleet(store, mount_one()).unwrap(); + let capabilities = client + .metadata_capabilities_for_subtree_owners("/workbenches") + .unwrap(); + + assert_eq!(capabilities.len(), 3); + assert_eq!( + capabilities + .iter() + .filter(|capabilities| capabilities.restore_to_fork_v1) + .count(), + 2 + ); + assert!(capabilities + .iter() + .any(|capabilities| !capabilities.restore_to_fork_v1)); +} + #[test] fn fleet_keeps_recovering_owner_unroutable_until_mark_serving() { let store = Arc::new(InMemoryControlStore::new()); diff --git a/crates/nokv-client/src/wire.rs b/crates/nokv-client/src/wire.rs index 722daf2dd..2f8d6e95f 100644 --- a/crates/nokv-client/src/wire.rs +++ b/crates/nokv-client/src/wire.rs @@ -246,6 +246,9 @@ pub(crate) fn client_error_from_wire_error(error: WireMetadataError) -> ClientEr owner_epoch, required_epoch, }), + WireMetadataError::InvalidOwnerEpoch => { + ClientError::Metadata(nokv_meta::MetadError::InvalidOwnerEpoch) + } WireMetadataError::LeaseExpired { now_ms, deadline_ms, @@ -264,6 +267,28 @@ pub(crate) fn client_error_from_wire_error(error: WireMetadataError) -> ClientEr dest_shard, }), WireMetadataError::GraftPoint => ClientError::Metadata(nokv_meta::MetadError::GraftPoint), + WireMetadataError::RestoreInProgress => { + ClientError::Metadata(nokv_meta::MetadError::RestoreInProgress) + } + WireMetadataError::RestoreRootChanged { root } => match inode_id(root) { + Ok(root) => ClientError::Metadata(nokv_meta::MetadError::RestoreRootChanged { root }), + Err(err) => err, + }, + WireMetadataError::RestoreBindingChanged { root } => match inode_id(root) { + Ok(root) => { + ClientError::Metadata(nokv_meta::MetadError::RestoreBindingChanged { root }) + } + Err(err) => err, + }, + WireMetadataError::RestoreResourceLimit { + resource, + limit, + actual, + } => ClientError::Metadata(nokv_meta::MetadError::RestoreResourceLimit { + resource, + limit, + actual, + }), WireMetadataError::StalePreparedArtifactObjectGcEpoch { expected, current } => { ClientError::Metadata(nokv_meta::MetadError::StalePreparedArtifactObjectGcEpoch { expected, @@ -322,6 +347,21 @@ pub(crate) fn client_error_from_wire_error(error: WireMetadataError) -> ClientEr snapshot_id, attempts, }), + WireMetadataError::RestoreHardlinkUnsupported { inode } => match inode_id(inode) { + Ok(inode) => { + ClientError::Metadata(nokv_meta::MetadError::RestoreHardlinkUnsupported { inode }) + } + Err(err) => err, + }, + WireMetadataError::RestoreCrossShardUnsupported { inode } => match inode_id(inode) { + Ok(inode) => { + ClientError::Metadata(nokv_meta::MetadError::RestoreCrossShardUnsupported { inode }) + } + Err(err) => err, + }, + WireMetadataError::RestoreDestinationConflict { destination } => { + ClientError::Metadata(nokv_meta::MetadError::RestoreDestinationConflict { destination }) + } WireMetadataError::SyncLogArchiveFailed { committed, message } => { ClientError::Metadata(nokv_meta::MetadError::SyncLogArchiveFailed { committed, @@ -352,3 +392,16 @@ pub(crate) fn client_error_from_wire_error(error: WireMetadataError) -> ClientEr pub(crate) fn unexpected_result(result: nokv_protocol::MetadataRpcResult) -> ClientError { ClientError::Protocol(format!("unexpected metadata rpc result {result:?}")) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn invalid_owner_epoch_stays_typed_after_wire_decode() { + assert!(matches!( + client_error_from_wire_error(WireMetadataError::InvalidOwnerEpoch), + ClientError::Metadata(nokv_meta::MetadError::InvalidOwnerEpoch) + )); + } +} diff --git a/crates/nokv-meta/src/command.rs b/crates/nokv-meta/src/command.rs index f5a7d8949..50c3cfe48 100644 --- a/crates/nokv-meta/src/command.rs +++ b/crates/nokv-meta/src/command.rs @@ -27,6 +27,9 @@ pub enum ReadPurpose { UserStrong, WritePlanLocal, Snapshot, + /// Internal access to detached restore staging. No ordinary read or write + /// planning path may use this purpose. + RestoreStaging, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/nokv-meta/src/holtstore.rs b/crates/nokv-meta/src/holtstore.rs index 99fb6b1fc..062d5cab1 100644 --- a/crates/nokv-meta/src/holtstore.rs +++ b/crates/nokv-meta/src/holtstore.rs @@ -5,9 +5,10 @@ //! Raft replication, FUSE, or protobuf types. use std::collections::{BTreeMap, HashSet}; +use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; use std::time::Instant; use crate::command::{ @@ -34,12 +35,22 @@ const HISTORY_INDEX_PROGRESS_KEY: &[u8] = b"\0history-key-index-v1-progress"; const HISTORY_INDEX_COMPLETE_VALUE: &[u8] = b"1"; const HISTORY_INDEX_BACKFILL_BATCH_SIZE: usize = 1_024; const HISTORY_PRUNE_CONFLICT_RETRIES: usize = 4; +const KEY_VISIBILITY_STRIPE_COUNT: usize = 256; #[derive(Clone)] pub struct HoltMetadataStore { db: DB, stats: Arc, history_retention: Arc, + /// Holt's exact-key absence assertion is represented as a create/delete + /// sentinel inside one atomic batch. Point gets are intentionally + /// optimistic and can otherwise observe the live sentinel between those + /// two internal operations. Commands lock their exact-key access set from + /// planning through apply; read-only absence guards take the exclusive side + /// only for the affected family/key stripes. + key_visibility_stripes: Arc<[RwLock<()>; KEY_VISIBILITY_STRIPE_COUNT]>, + #[cfg(test)] + before_atomic_apply: Arc>>, } #[derive(Default)] @@ -144,11 +155,38 @@ struct PrefixEmptyGuard { prefix: Vec, } +#[derive(Clone, Debug)] +enum AbsentKeyState { + Missing, + Tombstone(RecordVersion), +} + +#[derive(Clone, Debug)] +struct AbsentKeyGuard { + family: RecordFamily, + key: Vec, + state: AbsentKeyState, +} + +enum KeyVisibilityGuard<'a> { + Read { _guard: RwLockReadGuard<'a, ()> }, + Write { _guard: RwLockWriteGuard<'a, ()> }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum KeyVisibilityLockMode { + Read, + Write, +} + +type KeyVisibilityLockPlan = [Option; KEY_VISIBILITY_STRIPE_COUNT]; + struct CommandPlan { mutations: Vec, history_records: Vec<(RecordFamily, Vec, Vec)>, version_guards: Vec, prefix_empty_guards: Vec, + absent_key_guards: Vec, retain_history: bool, history_retention_delta: HistoryRetentionDelta, adds_history_retention: bool, @@ -296,12 +334,26 @@ impl HoltMetadataStore { } pub fn open_file(path: impl AsRef) -> Result { - let mut config = TreeConfig::new(path.as_ref()); + let path = path.as_ref(); + let recover_existing_store = file_store_has_prior_state(path); + let mut config = TreeConfig::new(path); // A successful metadata RPC is a durability promise. Holt's default // async WAL mode leaves a small ACK-to-flusher window in which SIGKILL // can discard a committed namespace change or restore journal state. config.durability = Durability::Wal { sync: true }; - Self::open(config) + // NoKV runs logical reachability GC on reopen and checkpoint. Keep + // Holt's physical vacuum disabled: reclaimed slots remain reusable, + // while the packed files may retain their high-water allocation. + config.checkpoint.auto_vacuum = false; + let store = Self::open(config)?; + if recover_existing_store { + // WAL replay and NoKV's family migrations above establish the + // authoritative roots. A process crash can lose Holt's in-memory + // COW orphan queue, so sweep that durable root closure before the + // reopened metadata store becomes externally visible. + store.reclaim_unreachable_storage()?; + } + Ok(store) } pub fn open(config: TreeConfig) -> Result { @@ -311,6 +363,9 @@ impl HoltMetadataStore { db, stats: Arc::new(HoltMetadataStoreCounters::default()), history_retention: Arc::new(HistoryRetentionState::new(recovered)), + key_visibility_stripes: Arc::new(std::array::from_fn(|_| RwLock::new(()))), + #[cfg(test)] + before_atomic_apply: Arc::new(std::sync::Mutex::new(None)), }; // Do not create trees in a brand-new database: checkpoint installation // requires a pristine DB. Existing stores, however, are migrated before @@ -323,6 +378,66 @@ impl HoltMetadataStore { Ok(store) } + fn acquire_key_visibility_guards( + &self, + plan: &KeyVisibilityLockPlan, + ) -> Vec> { + plan.iter() + .enumerate() + .filter_map(|(stripe, mode)| match mode { + Some(KeyVisibilityLockMode::Read) => Some(KeyVisibilityGuard::Read { + _guard: self.key_visibility_stripes[stripe] + .read() + .unwrap_or_else(|error| error.into_inner()), + }), + Some(KeyVisibilityLockMode::Write) => Some(KeyVisibilityGuard::Write { + _guard: self.key_visibility_stripes[stripe] + .write() + .unwrap_or_else(|error| error.into_inner()), + }), + None => None, + }) + .collect() + } + + fn acquire_key_visibility_read( + &self, + family: RecordFamily, + key: &[u8], + ) -> RwLockReadGuard<'_, ()> { + self.key_visibility_stripes[key_visibility_stripe(family, key)] + .read() + .unwrap_or_else(|error| error.into_inner()) + } + + #[cfg(test)] + fn set_before_atomic_apply_hook(&self, hook: TestHook) { + *self + .before_atomic_apply + .lock() + .unwrap_or_else(|error| error.into_inner()) = Some(hook); + } + + #[cfg(test)] + fn clear_before_atomic_apply_hook(&self) { + self.before_atomic_apply + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); + } + + #[cfg(test)] + fn run_before_atomic_apply_hook(&self) { + let hook = self + .before_atomic_apply + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + if let Some(hook) = hook { + hook(); + } + } + pub fn checkpoint(&self) -> Result<(), MetadataError> { self.db.checkpoint().map_err(to_backend_error)?; self.reclaim_unreachable_storage()?; @@ -648,6 +763,7 @@ impl MetadataStore for HoltMetadataStore { version: Version, purpose: ReadPurpose, ) -> Result, MetadataError> { + let _key_visibility = self.acquire_key_visibility_read(family, key); self.stats.get_total.fetch_add(1, Ordering::Relaxed); self.stats.record_get_purpose(purpose); read_visible( @@ -851,6 +967,8 @@ impl MetadataStore for HoltMetadataStore { &self, commands: &[MetadataCommand], ) -> Vec> { + let visibility_plan = key_visibility_lock_plan(commands); + let _key_visibility = self.acquire_key_visibility_guards(&visibility_plan); if commands.iter().any(command_mutates_history_retention) { let _fence = self .history_retention @@ -868,6 +986,8 @@ impl MetadataStore for HoltMetadataStore { } fn commit_metadata(&self, command: MetadataCommand) -> Result { + let visibility_plan = key_visibility_lock_plan(std::slice::from_ref(&command)); + let _key_visibility = self.acquire_key_visibility_guards(&visibility_plan); if command_mutates_history_retention(&command) { let _fence = self .history_retention @@ -891,6 +1011,8 @@ impl MetadataStore for HoltMetadataStore { &self, request_id: &[u8], ) -> Result, MetadataError> { + let _key_visibility = + self.acquire_key_visibility_read(RecordFamily::CommandDedupe, request_id); self.current_tree(RecordFamily::CommandDedupe)? .get(request_id) .map_err(to_backend_error)? @@ -938,7 +1060,7 @@ impl HoltMetadataStore { let mut pending = Vec::new(); for (index, command) in commands.iter().cloned().enumerate() { if pending.iter().any(|pending: &PendingPlannedCommand| { - metadata_commands_conflict(&pending.command, &command) + holt_commands_conflict(&pending.command, &command) }) { self.commit_pending_batch(&mut pending, &mut results); } @@ -1106,6 +1228,8 @@ impl HoltMetadataStore { .map(|item| planned_command_stats(&item.command, &item.plan)) .collect::>(); let atomic_start = Instant::now(); + #[cfg(test)] + self.run_before_atomic_apply_hook(); let committed = self .db .atomic(|batch| { @@ -1138,6 +1262,7 @@ impl HoltMetadataStore { .collect::>(); let mut version_guards = Vec::new(); let mut prefix_empty_guards = Vec::new(); + let mut absent_key_guards = Vec::new(); for predicate in &command.predicates { match predicate.predicate { @@ -1154,22 +1279,36 @@ impl HoltMetadataStore { )?; } Predicate::NotExists => { - let index = mutation_index(&mutations, predicate.family, &predicate.key) - .ok_or(MetadataError::PredicateFailed)?; - if mutations[index].mutation.op != MutationOp::Put { - return Err(MetadataError::PredicateFailed); - } - match self.current_record(predicate.family, &predicate.key)? { - None => { - set_mutation_guard(&mut mutations[index], MutationGuard::PutIfAbsent)?; + if let Some(index) = + mutation_index(&mutations, predicate.family, &predicate.key) + { + if mutations[index].mutation.op != MutationOp::Put { + return Err(MetadataError::PredicateFailed); } - Some(record) if record.value.is_none() => { - set_mutation_guard( + match self.current_record(predicate.family, &predicate.key)? { + None => set_mutation_guard( + &mut mutations[index], + MutationGuard::PutIfAbsent, + )?, + Some(record) if record.value.is_none() => set_mutation_guard( &mut mutations[index], MutationGuard::CompareAndPut(record.record_version), - )?; + )?, + Some(_) => return Err(MetadataError::PredicateFailed), } - Some(_) => return Err(MetadataError::PredicateFailed), + } else { + let state = match self.current_record(predicate.family, &predicate.key)? { + None => AbsentKeyState::Missing, + Some(record) if record.value.is_none() => { + AbsentKeyState::Tombstone(record.record_version) + } + Some(_) => return Err(MetadataError::PredicateFailed), + }; + absent_key_guards.push(AbsentKeyGuard { + family: predicate.family, + key: predicate.key.clone(), + state, + }); } } Predicate::PrefixEmpty => { @@ -1235,6 +1374,7 @@ impl HoltMetadataStore { history_records, version_guards, prefix_empty_guards, + absent_key_guards, retain_history, history_retention_delta, adds_history_retention, @@ -1259,6 +1399,8 @@ impl HoltMetadataStore { self.ensure_history_key_index()?; let stats = planned_command_stats(&command, &plan); let atomic_start = Instant::now(); + #[cfg(test)] + self.run_before_atomic_apply_hook(); let committed = self .db .atomic(|batch| { @@ -1464,7 +1606,9 @@ impl HoltMetadataStoreCounters { fn record_get_purpose(&self, purpose: ReadPurpose) { match purpose { ReadPurpose::UserStrong => &self.get_user_strong_total, - ReadPurpose::WritePlanLocal => &self.get_write_plan_local_total, + ReadPurpose::WritePlanLocal | ReadPurpose::RestoreStaging => { + &self.get_write_plan_local_total + } ReadPurpose::Snapshot => &self.get_snapshot_total, } .fetch_add(1, Ordering::Relaxed); @@ -1473,7 +1617,9 @@ impl HoltMetadataStoreCounters { fn record_scan_purpose(&self, purpose: ReadPurpose) { match purpose { ReadPurpose::UserStrong => &self.scan_user_strong_total, - ReadPurpose::WritePlanLocal => &self.scan_write_plan_local_total, + ReadPurpose::WritePlanLocal | ReadPurpose::RestoreStaging => { + &self.scan_write_plan_local_total + } ReadPurpose::Snapshot => &self.scan_snapshot_total, } .fetch_add(1, Ordering::Relaxed); @@ -1574,6 +1720,80 @@ fn command_mutates_history_retention(command: &MetadataCommand) -> bool { .any(|mutation| is_history_retention_family(mutation.family)) } +fn holt_commands_conflict(left: &MetadataCommand, right: &MetadataCommand) -> bool { + metadata_commands_conflict(left, right) + || left.predicates.iter().any(|left_predicate| { + is_read_only_absence_guard(left, left_predicate) + && right.predicates.iter().any(|right_predicate| { + is_read_only_absence_guard(right, right_predicate) + && left_predicate.family == right_predicate.family + && left_predicate.key == right_predicate.key + }) + }) +} + +fn is_read_only_absence_guard( + command: &MetadataCommand, + predicate: &crate::command::PredicateRef, +) -> bool { + matches!(predicate.predicate, Predicate::NotExists) + && !command + .mutations + .iter() + .any(|mutation| mutation.family == predicate.family && mutation.key == predicate.key) +} + +fn key_visibility_lock_plan(commands: &[MetadataCommand]) -> KeyVisibilityLockPlan { + let mut plan = [None; KEY_VISIBILITY_STRIPE_COUNT]; + for command in commands { + add_key_visibility_lock( + &mut plan, + RecordFamily::CommandDedupe, + &command.request_id, + KeyVisibilityLockMode::Read, + ); + for mutation in &command.mutations { + add_key_visibility_lock( + &mut plan, + mutation.family, + &mutation.key, + KeyVisibilityLockMode::Read, + ); + } + for predicate in &command.predicates { + if matches!(predicate.predicate, Predicate::PrefixEmpty) { + continue; + } + let mode = if is_read_only_absence_guard(command, predicate) { + KeyVisibilityLockMode::Write + } else { + KeyVisibilityLockMode::Read + }; + add_key_visibility_lock(&mut plan, predicate.family, &predicate.key, mode); + } + } + plan +} + +fn add_key_visibility_lock( + plan: &mut KeyVisibilityLockPlan, + family: RecordFamily, + key: &[u8], + mode: KeyVisibilityLockMode, +) { + let slot = &mut plan[key_visibility_stripe(family, key)]; + if *slot != Some(KeyVisibilityLockMode::Write) { + *slot = Some(mode); + } +} + +fn key_visibility_stripe(family: RecordFamily, key: &[u8]) -> usize { + let mut hasher = DefaultHasher::new(); + (family as u8).hash(&mut hasher); + key.hash(&mut hasher); + (hasher.finish() as usize) & (KEY_VISIBILITY_STRIPE_COUNT - 1) +} + fn command_advances_history_version(command: &MetadataCommand) -> bool { command.mutations.iter().any(|mutation| { family_requires_history(mutation.family) && !is_history_retention_family(mutation.family) @@ -1694,6 +1914,26 @@ fn enqueue_planned_command( for guard in &plan.prefix_empty_guards { batch.assert_prefix_empty(current_tree_name(guard.family), &guard.prefix); } + // Holt does not expose a read-only exact-key absence assertion. Project a + // sentinel create/delete pair inside the same atomic batch instead. It is + // never externally visible, while a concurrent create makes the guard fail + // and aborts the whole command. Tombstones use their physical record + // version as the compare token. + for guard in &plan.absent_key_guards { + let sentinel = encode_current_value(command.commit_version, &[]); + match guard.state { + AbsentKeyState::Missing => { + batch.put_if_absent(current_tree_name(guard.family), &guard.key, &sentinel) + } + AbsentKeyState::Tombstone(version) => batch.compare_and_put( + current_tree_name(guard.family), + &guard.key, + version, + &sentinel, + ), + } + batch.delete(current_tree_name(guard.family), &guard.key); + } for planned in &plan.mutations { match (planned.mutation.op, planned.guard) { (MutationOp::Put, MutationGuard::Always) => { @@ -1919,6 +2159,12 @@ fn next_current_candidate( Ok(None) } +fn file_store_has_prior_state(path: &Path) -> bool { + std::fs::read_dir(path) + .map(|mut entries| entries.next().is_some()) + .unwrap_or(false) +} + fn next_history_index_candidate( family: RecordFamily, iterator: &mut impl Iterator>, diff --git a/crates/nokv-meta/src/holtstore/tests.rs b/crates/nokv-meta/src/holtstore/tests.rs index 0f4e0f820..a3c751e17 100644 --- a/crates/nokv-meta/src/holtstore/tests.rs +++ b/crates/nokv-meta/src/holtstore/tests.rs @@ -3,10 +3,32 @@ use crate::command::{ CommandKind, DelimitedScanItem, DelimitedScanRequest, HistoryPruneRequest, MetadataCommand, Mutation, PredicateRef, ScanRequest, Value, }; +use std::sync::atomic::{AtomicBool, AtomicUsize}; use std::sync::{Arc, Barrier}; use std::thread; use tempfile::tempdir; +const FILE_REOPEN_GC_DIR_ENV: &str = "NOKV_HOLT_FILE_REOPEN_GC_DIR"; +const FILE_REOPEN_GC_ENTRY_COUNT: u32 = 1_200; + +fn file_reopen_gc_config(path: &std::path::Path) -> TreeConfig { + let mut config = TreeConfig::new(path); + config.checkpoint.enabled = false; + config.durability = Durability::Wal { sync: false }; + config +} + +fn run_file_reopen_gc_crash_session(path: &std::path::Path) { + let executable = std::env::current_exe().unwrap(); + let child_test = "holtstore::tests::file_reopen_gc_crash_session"; + let status = std::process::Command::new(executable) + .args([child_test, "--exact", "--ignored", "--nocapture"]) + .env(FILE_REOPEN_GC_DIR_ENV, path) + .status() + .unwrap(); + assert!(status.success(), "crash-session child test failed"); +} + fn version(raw: u64) -> Version { Version::new(raw).unwrap() } @@ -86,6 +108,35 @@ fn delete_command(key: &[u8], request_id: &[u8], read: u64, commit: u64) -> Meta } } +fn put_with_read_only_absence_guard( + guard_key: &[u8], + output_key: &[u8], + request_id: &[u8], + value: &[u8], + commit: u64, +) -> MetadataCommand { + MetadataCommand { + request_id: request_id.to_vec(), + kind: CommandKind::CreateFile, + read_version: version(commit - 1), + commit_version: version(commit), + primary_family: RecordFamily::Dentry, + primary_key: output_key.to_vec(), + predicates: vec![PredicateRef { + family: RecordFamily::Dentry, + key: guard_key.to_vec(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::Dentry, + key: output_key.to_vec(), + op: MutationOp::Put, + value: Some(Value(value.to_vec())), + }], + watch: Vec::new(), + } +} + fn snapshot_pin_command(request_id: &[u8], commit: u64) -> MetadataCommand { retention_put_command(RecordFamily::Snapshot, b"snapshot/1", request_id, commit) } @@ -398,6 +449,106 @@ fn independent_batch_commits_disjoint_commands() { assert_eq!(stats.atomic_apply_max_batch, 2); } +#[test] +fn independent_batch_splits_commands_with_the_same_read_only_absence_guard() { + let store = HoltMetadataStore::open_memory().unwrap(); + let results = store.commit_independent_batch(&[ + put_with_read_only_absence_guard(b"guard", b"dir/a", b"req-1", b"value-a", 2), + put_with_read_only_absence_guard(b"guard", b"dir/b", b"req-2", b"value-b", 3), + ]); + + assert!(results.iter().all(Result::is_ok)); + assert_eq!( + store + .get( + RecordFamily::Dentry, + b"guard", + version(3), + ReadPurpose::UserStrong, + ) + .unwrap(), + None + ); + assert_eq!( + store + .get( + RecordFamily::Dentry, + b"dir/a", + version(3), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"value-a".to_vec())) + ); + assert_eq!( + store + .get( + RecordFamily::Dentry, + b"dir/b", + version(3), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"value-b".to_vec())) + ); + let stats = store.metadata_store_stats(); + assert_eq!(stats.commit_total, 2); + assert_eq!(stats.atomic_apply_total, 2); + assert_eq!(stats.atomic_apply_command_total, 2); + assert_eq!(stats.atomic_apply_max_batch, 1); +} + +#[test] +fn independent_batch_keeps_different_read_only_absence_guards_together() { + let store = HoltMetadataStore::open_memory().unwrap(); + let results = store.commit_independent_batch(&[ + put_with_read_only_absence_guard(b"guard-a", b"dir/a", b"req-1", b"value-a", 2), + put_with_read_only_absence_guard(b"guard-b", b"dir/b", b"req-2", b"value-b", 3), + ]); + + assert!(results.iter().all(Result::is_ok)); + for guard in [b"guard-a".as_slice(), b"guard-b".as_slice()] { + assert_eq!( + store + .get( + RecordFamily::Dentry, + guard, + version(3), + ReadPurpose::UserStrong, + ) + .unwrap(), + None + ); + } + assert_eq!( + store + .get( + RecordFamily::Dentry, + b"dir/a", + version(3), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"value-a".to_vec())) + ); + assert_eq!( + store + .get( + RecordFamily::Dentry, + b"dir/b", + version(3), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"value-b".to_vec())) + ); + let stats = store.metadata_store_stats(); + assert_eq!(stats.commit_total, 2); + assert_eq!(stats.atomic_apply_total, 1); + assert_eq!(stats.atomic_apply_command_total, 2); + assert_eq!(stats.atomic_apply_max_batch, 2); +} + #[test] fn independent_batch_preserves_conflict_result_boundary() { let store = HoltMetadataStore::open_memory().unwrap(); @@ -1030,6 +1181,163 @@ fn storage_reclaim_is_idempotent_after_checkpoint() { ); } +#[test] +fn file_store_prior_state_detection_ignores_new_empty_directories() { + let directory = tempdir().unwrap(); + let path = directory.path().join("metadata"); + + assert!(!file_store_has_prior_state(&path)); + std::fs::create_dir(&path).unwrap(); + assert!(!file_store_has_prior_state(&path)); + + drop(HoltMetadataStore::open_file(&path).unwrap()); + assert!(file_store_has_prior_state(&path)); +} + +#[test] +fn metadata_apply_preserves_holt_snapshot_children() { + let store = HoltMetadataStore::open_memory().unwrap(); + let tree = store.current_tree(RecordFamily::Dentry).unwrap(); + let original = vec![0xA5; 512]; + let encoded_original = encode_current_value(version(1), &original); + for index in 0..FILE_REOPEN_GC_ENTRY_COUNT { + tree.put(format!("key/{index:08}").as_bytes(), &encoded_original) + .unwrap(); + } + assert!( + tree.stats().unwrap().blob_count > 1, + "snapshot fixture must spill beyond the root frame", + ); + + let snapshot = tree.snapshot(b"").unwrap(); + for index in 0..FILE_REOPEN_GC_ENTRY_COUNT { + let key = format!("key/{index:08}"); + let request_id = format!("snapshot-update/{index:08}"); + store + .commit_metadata(replace_command( + key.as_bytes(), + request_id.as_bytes(), + b"current", + index as u64 + 1, + index as u64 + 2, + )) + .unwrap(); + } + + assert!( + store.db.stats().bm_gc_orphan_backlog_count > 0, + "NoKV metadata apply must fork snapshot-shared Holt children", + ); + for index in 0..FILE_REOPEN_GC_ENTRY_COUNT { + let key = format!("key/{index:08}"); + assert_eq!( + snapshot.view().get(key.as_bytes()).unwrap().as_deref(), + Some(encoded_original.as_slice()), + "snapshot child changed at {key}", + ); + } + let final_key = format!("key/{:08}", FILE_REOPEN_GC_ENTRY_COUNT - 1); + let encoded_live = tree.get(final_key.as_bytes()).unwrap().unwrap(); + assert_eq!( + decode_current_value(&encoded_live).unwrap().1.as_deref(), + Some(&b"current"[..]), + ); +} + +#[test] +#[ignore = "child-process body for file_reopen_reclaims_crash_lost_cow_orphans"] +fn file_reopen_gc_crash_session() { + let Some(path) = std::env::var_os(FILE_REOPEN_GC_DIR_ENV) else { + return; + }; + let store = + HoltMetadataStore::open(file_reopen_gc_config(std::path::Path::new(&path))).unwrap(); + let tree = store.current_tree(RecordFamily::Dentry).unwrap(); + let original = vec![0xAB; 512]; + for index in 0..FILE_REOPEN_GC_ENTRY_COUNT { + tree.put(format!("key/{index:08}").as_bytes(), &original) + .unwrap(); + } + store.db.checkpoint().unwrap(); + assert!( + tree.stats().unwrap().blob_count > 1, + "crash fixture must spill beyond the root frame", + ); + + let snapshot = tree.snapshot(b"").unwrap(); + for index in 0..FILE_REOPEN_GC_ENTRY_COUNT { + tree.put(format!("key/{index:08}").as_bytes(), b"current") + .unwrap(); + } + store.db.checkpoint().unwrap(); + assert!( + store.db.stats().bm_gc_orphan_backlog_count > 0, + "writes under the leaked snapshot must create COW orphan debt", + ); + + // A real crash drops this process-local snapshot lease without retiring + // its old COW frames through Holt's in-memory orphan queue. + std::mem::forget(snapshot); +} + +#[test] +fn file_reopen_reclaims_crash_lost_cow_orphans() { + let directory = tempdir().unwrap(); + let path = directory.path().join("metadata"); + run_file_reopen_gc_crash_session(&path); + assert!(file_store_has_prior_state(&path)); + let data_file = path.join("blobs.dat"); + let data_file_bytes_before = std::fs::metadata(&data_file).unwrap().len(); + + let store = HoltMetadataStore::open_file(&path).unwrap(); + let stats = store.db.stats(); + let followup_reclaimed = store.reclaim_unreachable_storage().unwrap(); + assert!( + stats.bm_gc_reclaimed_count > 0, + "NoKV file reopen must reclaim crash-lost COW frames (follow-up reclaimed {followup_reclaimed})", + ); + assert!( + stats.store.reusable_slots > 0, + "logical GC must publish reclaimed slots for reuse", + ); + assert_eq!( + stats.store.data_file_bytes, data_file_bytes_before, + "auto-vacuum is disabled, so reopen GC must not shrink packed files", + ); + assert_eq!(followup_reclaimed, 0); + assert_eq!( + store + .current_tree(RecordFamily::Dentry) + .unwrap() + .get(b"key/00000000") + .unwrap() + .as_deref(), + Some(&b"current"[..]), + ); + drop(store); + + let reopened = HoltMetadataStore::open_file(&path).unwrap(); + assert_eq!( + reopened.db.stats().bm_gc_reclaimed_count, + 0, + "repeated reopen must find no residual unreachable frames", + ); + let final_key = format!("key/{:08}", FILE_REOPEN_GC_ENTRY_COUNT - 1); + assert_eq!( + reopened + .current_tree(RecordFamily::Dentry) + .unwrap() + .get(final_key.as_bytes()) + .unwrap() + .as_deref(), + Some(&b"current"[..]), + ); + assert_eq!( + std::fs::metadata(data_file).unwrap().len(), + data_file_bytes_before + ); +} + #[test] fn checkpoint_image_rejects_malformed_bytes() { let store = HoltMetadataStore::open_memory().unwrap(); @@ -1345,6 +1653,343 @@ fn not_exists_allows_recreate_after_tombstone() { ); } +#[test] +fn read_only_not_exists_guard_is_atomic_and_leaves_no_sentinel() { + let store = HoltMetadataStore::open_memory().unwrap(); + store + .commit_metadata(put_with_read_only_absence_guard( + b"guard/missing", + b"result/first", + b"guard-missing", + b"committed", + 2, + )) + .unwrap(); + + assert_eq!( + store + .get( + RecordFamily::Dentry, + b"guard/missing", + version(2), + ReadPurpose::UserStrong, + ) + .unwrap(), + None, + "the internal create/delete sentinel must never become a live record", + ); + assert_eq!( + store + .get( + RecordFamily::Dentry, + b"result/first", + version(2), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"committed".to_vec())), + ); + + store + .commit_metadata(put_command(b"guard/existing", b"create-guard", b"value", 3)) + .unwrap(); + assert_eq!( + store.commit_metadata(put_with_read_only_absence_guard( + b"guard/existing", + b"result/must-not-exist", + b"guard-existing", + b"wrong", + 4, + )), + Err(MetadataError::PredicateFailed), + ); + assert!(store + .get( + RecordFamily::Dentry, + b"result/must-not-exist", + version(4), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); +} + +#[test] +fn point_reads_never_observe_read_only_absence_guard_sentinels() { + const GUARD_KEYS: usize = 256; + const COMMITS: u64 = 64; + + let store = HoltMetadataStore::open_memory().unwrap(); + let guard_keys = Arc::new( + (0..GUARD_KEYS) + .map(|index| format!("guard/transient/{index:04}").into_bytes()) + .collect::>(), + ); + let start = Arc::new(Barrier::new(2)); + let done = Arc::new(AtomicBool::new(false)); + + let reader_store = store.clone(); + let reader_keys = Arc::clone(&guard_keys); + let reader_start = Arc::clone(&start); + let reader_done = Arc::clone(&done); + let reader = thread::spawn(move || { + reader_start.wait(); + while !reader_done.load(Ordering::Acquire) { + for key in reader_keys.iter() { + assert_eq!( + reader_store + .get( + RecordFamily::Dentry, + key, + version(u64::MAX), + ReadPurpose::UserStrong, + ) + .unwrap(), + None, + "a point read observed an atomic absence-guard sentinel", + ); + } + } + }); + + start.wait(); + for offset in 0..COMMITS { + let commit = offset + 2; + store + .commit_metadata(MetadataCommand { + request_id: format!("absence-visibility-{offset}").into_bytes(), + kind: CommandKind::SetXattr, + read_version: version(commit - 1), + commit_version: version(commit), + primary_family: RecordFamily::Dentry, + primary_key: b"result/absence-visibility".to_vec(), + predicates: guard_keys + .iter() + .map(|key| PredicateRef { + family: RecordFamily::Dentry, + key: key.clone(), + predicate: Predicate::NotExists, + }) + .collect(), + mutations: vec![Mutation { + family: RecordFamily::Dentry, + key: b"result/absence-visibility".to_vec(), + op: MutationOp::Put, + value: Some(Value(offset.to_be_bytes().to_vec())), + }], + watch: Vec::new(), + }) + .unwrap(); + } + done.store(true, Ordering::Release); + reader.join().unwrap(); + + for key in guard_keys.iter() { + assert!(store + .get( + RecordFamily::Dentry, + key, + version(COMMITS + 1), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + } +} + +#[test] +fn different_absence_guard_stripes_can_enter_atomic_apply_concurrently() { + let left = put_with_read_only_absence_guard( + b"guard/concurrent/left", + b"result/concurrent/left", + b"concurrent-left", + b"left", + 2, + ); + let left_plan = key_visibility_lock_plan(std::slice::from_ref(&left)); + let right = (0..10_000) + .map(|index| { + put_with_read_only_absence_guard( + format!("guard/concurrent/right/{index}").as_bytes(), + format!("result/concurrent/right/{index}").as_bytes(), + format!("concurrent-right-{index}").as_bytes(), + b"right", + 3, + ) + }) + .find(|candidate| { + let right_plan = key_visibility_lock_plan(std::slice::from_ref(candidate)); + left_plan + .iter() + .zip(right_plan.iter()) + .all(|(left_mode, right_mode)| { + !matches!( + (left_mode, right_mode), + (Some(KeyVisibilityLockMode::Write), Some(_)) + | (Some(_), Some(KeyVisibilityLockMode::Write)) + ) + }) + }) + .expect("the 256 stripes must provide a disjoint command access set"); + assert_ne!(left.predicates[0].key, right.predicates[0].key); + assert_ne!( + key_visibility_stripe(left.predicates[0].family, &left.predicates[0].key), + key_visibility_stripe(right.predicates[0].family, &right.predicates[0].key), + ); + + let store = HoltMetadataStore::open_memory().unwrap(); + let apply_barrier = Arc::new(Barrier::new(2)); + let active = Arc::new(AtomicUsize::new(0)); + let max_active = Arc::new(AtomicUsize::new(0)); + store.set_before_atomic_apply_hook(Arc::new({ + let apply_barrier = Arc::clone(&apply_barrier); + let active = Arc::clone(&active); + let max_active = Arc::clone(&max_active); + move || { + let now_active = active.fetch_add(1, Ordering::SeqCst) + 1; + max_active.fetch_max(now_active, Ordering::SeqCst); + apply_barrier.wait(); + active.fetch_sub(1, Ordering::SeqCst); + } + })); + + let left_store = store.clone(); + let left_commit = thread::spawn(move || left_store.commit_metadata(left)); + let right_store = store.clone(); + let right_commit = thread::spawn(move || right_store.commit_metadata(right)); + + assert!(left_commit.join().unwrap().is_ok()); + assert!(right_commit.join().unwrap().is_ok()); + store.clear_before_atomic_apply_hook(); + assert_eq!( + max_active.load(Ordering::SeqCst), + 2, + "different absence-guard stripes must not serialize at the visibility fence", + ); +} + +#[test] +fn read_only_not_exists_guard_accepts_tombstone_and_survives_checkpoint_reopen() { + let directory = tempdir().unwrap(); + let path = directory.path().join("metadata.holt"); + { + let store = HoltMetadataStore::open_file(&path).unwrap(); + store + .commit_metadata(put_command(b"guard/reused", b"create", b"value", 2)) + .unwrap(); + store + .commit_metadata(delete_command(b"guard/reused", b"delete", 2, 3)) + .unwrap(); + let first = store + .commit_metadata(put_with_read_only_absence_guard( + b"guard/reused", + b"result/tombstone", + b"guard-tombstone", + b"safe", + 4, + )) + .unwrap(); + let duplicate = store + .commit_metadata(put_with_read_only_absence_guard( + b"guard/reused", + b"result/ignored-duplicate", + b"guard-tombstone", + b"different", + 5, + )) + .unwrap(); + assert_eq!(duplicate, first, "lost-ACK replay must deduplicate"); + assert!(store + .get( + RecordFamily::Dentry, + b"result/ignored-duplicate", + version(5), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + store.checkpoint().unwrap(); + } + + let reopened = HoltMetadataStore::open_file(&path).unwrap(); + assert!(reopened + .get( + RecordFamily::Dentry, + b"guard/reused", + version(5), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + assert_eq!( + reopened + .get( + RecordFamily::Dentry, + b"result/tombstone", + version(5), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"safe".to_vec())), + ); +} + +#[test] +fn concurrent_cross_absence_guards_commit_exactly_one_command() { + let store = HoltMetadataStore::open_memory().unwrap(); + let barrier = Arc::new(Barrier::new(2)); + let left_store = store.clone(); + let left_barrier = Arc::clone(&barrier); + let left = thread::spawn(move || { + left_barrier.wait(); + left_store.commit_metadata(put_with_read_only_absence_guard( + b"winner/right", + b"winner/left", + b"cross-left", + b"left", + 2, + )) + }); + let right_store = store.clone(); + let right = thread::spawn(move || { + barrier.wait(); + right_store.commit_metadata(put_with_read_only_absence_guard( + b"winner/left", + b"winner/right", + b"cross-right", + b"right", + 3, + )) + }); + + let outcomes = [left.join().unwrap(), right.join().unwrap()]; + assert_eq!(outcomes.iter().filter(|outcome| outcome.is_ok()).count(), 1); + assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome, Err(MetadataError::PredicateFailed))) + .count(), + 1, + ); + let left = store + .get( + RecordFamily::Dentry, + b"winner/left", + version(3), + ReadPurpose::UserStrong, + ) + .unwrap(); + let right = store + .get( + RecordFamily::Dentry, + b"winner/right", + version(3), + ReadPurpose::UserStrong, + ) + .unwrap(); + assert_ne!(left.is_some(), right.is_some()); +} + #[test] fn prefix_empty_predicate_uses_family_prefix() { let store = HoltMetadataStore::open_memory().unwrap(); diff --git a/crates/nokv-meta/src/lib.rs b/crates/nokv-meta/src/lib.rs index 5d5cd44e1..1f68b2c10 100644 --- a/crates/nokv-meta/src/lib.rs +++ b/crates/nokv-meta/src/lib.rs @@ -30,27 +30,30 @@ pub use log::{ METADATA_LOG_ZERO_DIGEST, }; pub use service::{ - BodyReadPlan, CheckpointHandle, CheckpointShard, CloneHandle, CreateInDirPathBatch, - CreatedPreparedArtifact, DanglingBlock, DentryWithAttr, FsckReport, MetadError, - MetadataArchiveConfig, MetadataBackupOutcome, MetadataCheckpointIdentity, - MetadataLogArchiveConfig, MetadataLogPruneOutcome, MetadataLogPublicationState, - MetadataLogRestoreOutcome, MetadataLogSegmentArchiveOutcome, MetadataLogSegmentPointer, - MetadataLogSyncConfig, MetadataLogSyncSnapshot, MetadataRestoreOutcome, MetadataServiceStats, - NamespaceAggregateGroup, NamespaceAggregateMeasure, NamespaceAggregateOp, - NamespaceAggregateOutputMeasure, NamespaceAggregateRequest, NamespaceAggregateResult, - NamespaceAggregateSample, NamespaceAggregateSort, NamespaceAggregateValue, - NamespaceBodyDescriptor, NamespaceCard, NamespaceCardKind, NamespaceFacetSummary, - NamespaceFacetValue, NamespaceFieldSource, NamespaceFieldSourceKind, NamespaceFieldValue, - NamespaceFilterCapability, NamespaceFindField, NamespaceFindRequest, NamespaceFindResult, - NamespaceGrepMatch, NamespaceGrepRequest, NamespaceGrepResult, NamespaceInclude, - NamespaceIndexField, NamespaceIndexRegistration, NamespaceIndexRow, NamespaceIndexValue, - NamespaceListOptions, NamespaceListPage, NamespacePredicate, NamespacePredicateOp, - NamespacePredicateValue, NamespaceQueryCatalog, NamespaceReadFormat, NamespaceReadItem, - NamespaceReadOptions, NamespaceReadPage, NamespaceRecordCount, NamespaceRecordType, - NamespaceSchema, NamespaceSort, NamespaceSortDirection, NamespaceSortField, NoKvFs, - ObjectTransferStats, OpenPathReadPlan, OpenPathReadPlanRequest, PendingObjectCleanupOutcome, - PreparedArtifact, PublishArtifact, PublishArtifactRange, PublishArtifactSession, - PublishArtifactStagedSession, ReadDirPlusPage, RecordCountProvenance, RenameReplaceResult, - SnapshotReapOutcome, SnapshotRenewOutcome, SubtreeDelta, SubtreeDeltaKind, UpdateAttr, - XattrSetMode, DEFAULT_SNAPSHOT_LEASE_MS, + restore_operation_id, BodyReadPlan, CheckpointHandle, CheckpointShard, CloneHandle, + CreateInDirPathBatch, CreatedPreparedArtifact, DanglingBlock, DentryWithAttr, FsckMode, + FsckReport, MetadError, MetadataArchiveConfig, MetadataBackupOutcome, + MetadataCheckpointIdentity, MetadataLogArchiveConfig, MetadataLogPruneOutcome, + MetadataLogPublicationState, MetadataLogRestoreOutcome, MetadataLogSegmentArchiveOutcome, + MetadataLogSegmentPointer, MetadataLogSyncConfig, MetadataLogSyncSnapshot, + MetadataRestoreOutcome, MetadataServiceStats, MismatchedBlock, NamespaceAggregateGroup, + NamespaceAggregateMeasure, NamespaceAggregateOp, NamespaceAggregateOutputMeasure, + NamespaceAggregateRequest, NamespaceAggregateResult, NamespaceAggregateSample, + NamespaceAggregateSort, NamespaceAggregateValue, NamespaceBodyDescriptor, NamespaceCard, + NamespaceCardKind, NamespaceFacetSummary, NamespaceFacetValue, NamespaceFieldSource, + NamespaceFieldSourceKind, NamespaceFieldValue, NamespaceFilterCapability, NamespaceFindField, + NamespaceFindRequest, NamespaceFindResult, NamespaceGrepMatch, NamespaceGrepRequest, + NamespaceGrepResult, NamespaceInclude, NamespaceIndexField, NamespaceIndexRegistration, + NamespaceIndexRow, NamespaceIndexValue, NamespaceListOptions, NamespaceListPage, + NamespacePredicate, NamespacePredicateOp, NamespacePredicateValue, NamespaceQueryCatalog, + NamespaceReadFormat, NamespaceReadItem, NamespaceReadOptions, NamespaceReadPage, + NamespaceRecordCount, NamespaceRecordType, NamespaceSchema, NamespaceSort, + NamespaceSortDirection, NamespaceSortField, NoKvFs, ObjectTransferStats, OpenPathReadPlan, + OpenPathReadPlanRequest, PendingObjectCleanupOutcome, PreparedArtifact, PublishArtifact, + PublishArtifactRange, PublishArtifactSession, PublishArtifactStagedSession, ReadDirPlusPage, + RecordCountProvenance, RenameReplaceResult, RestoreCapabilityDisabled, RestoreDowngradeError, + RestoreDowngradeOutcome, RestoreFsckIssue, RestoreFsckReport, RestoreInitialization, + RestoreInitializationFile, RestoreMetrics, RestoreOutcome, RestoreState, + RestoreWritersQuiesced, SnapshotReapOutcome, SnapshotRenewOutcome, SubtreeDelta, + SubtreeDeltaKind, UpdateAttr, XattrSetMode, DEFAULT_SNAPSHOT_LEASE_MS, }; diff --git a/crates/nokv-meta/src/service.rs b/crates/nokv-meta/src/service.rs index 6d2444661..2a6c37dca 100644 --- a/crates/nokv-meta/src/service.rs +++ b/crates/nokv-meta/src/service.rs @@ -21,6 +21,9 @@ mod log_sync; mod namespace; mod publish; mod read; +mod restore; +mod restore_gc; +mod restore_index; mod rollback; mod snapshot; mod watch; @@ -31,7 +34,11 @@ pub use self::backup::{ MetadataRestoreOutcome, }; pub use self::checkpoint::{CheckpointHandle, CheckpointShard}; -pub use self::fsck::{DanglingBlock, FsckReport}; +pub use self::fsck::{ + DanglingBlock, FsckMode, FsckReport, MismatchedBlock, RestoreCapabilityDisabled, + RestoreDowngradeError, RestoreDowngradeOutcome, RestoreFsckIssue, RestoreFsckReport, + RestoreMetrics, RestoreWritersQuiesced, +}; pub use self::log_archive::{ MetadataLogArchiveConfig, MetadataLogRestoreOutcome, MetadataLogSegmentArchiveOutcome, }; @@ -39,15 +46,19 @@ pub use self::log_sync::{ MetadataLogPruneOutcome, MetadataLogPublicationState, MetadataLogSegmentPointer, MetadataLogSyncConfig, MetadataLogSyncSnapshot, }; +pub use self::restore::{ + restore_operation_id, RestoreInitialization, RestoreInitializationFile, RestoreOutcome, + RestoreState, +}; pub use self::snapshot::DEFAULT_SNAPSHOT_LEASE_MS; use std::collections::hash_map::DefaultHasher; use std::collections::{BTreeMap, HashSet}; use std::fmt; use std::hash::{Hash, Hasher}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Mutex, RwLock}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; use self::lock::AdvisoryLockTable; use crate::command::{ @@ -116,6 +127,11 @@ const PATH_INDEX_VALIDATION_CACHE_MAX_ENTRIES_PER_SHARD: usize = PATH_INDEX_VALIDATION_CACHE_MAX_ENTRIES / PATH_CACHE_SHARD_COUNT; pub(crate) const DEFAULT_READ_LEASE_MS: u64 = 3_600_000; const FAILOVER_DURABILITY_REQUIRED_MARKER: &[u8] = &[1]; +// Appended to the otherwise-stable 24-byte allocator record once durable +// restore-to-fork state has ever been installed. Pre-restore binaries reject +// trailing allocator bytes, so a cold downgrade fails closed instead of +// silently ignoring private restore references and visibility overlays. +const RESTORE_ALLOCATOR_FENCE_MAGIC: &[u8; 8] = b"NKRALV2\0"; fn decode_failover_durability_required_marker(bytes: &[u8]) -> Result<(), MetadError> { if bytes == FAILOVER_DURABILITY_REQUIRED_MARKER { @@ -425,6 +441,7 @@ struct StagedArtifactBody { } struct ReplaceProjectionCommit<'a> { + request_id: Option>, kind: CommandKind, projection: &'a DentryProjection, chunks: &'a [ChunkManifest], @@ -610,11 +627,21 @@ pub struct MetadataServiceStats { pub metadata_log_segments_archived_total: u64, pub metadata_log_entries_archived_total: u64, pub metadata_log_archive_bytes_total: u64, + pub restore_to_fork_requests_total: u64, + pub restore_to_fork_success_total: u64, + pub restore_to_fork_failure_total: u64, + pub restore_to_fork_elapsed_ns_total: u64, + pub restore_to_fork_elapsed_ns_max: u64, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct PendingObjectCleanupOutcome { pub snapshot_reap: SnapshotReapOutcome, + pub restore_release_jobs_processed: usize, + pub restore_release_backlog: usize, + pub restore_release_quarantine: usize, + pub restore_release_mount_wide_quarantine: usize, + pub restore_init_tombstones_scanned: usize, pub scanned: usize, pub blocked_by_snapshots: usize, pub blocked_by_read_leases: usize, @@ -751,6 +778,10 @@ pub enum MetadError { owner_epoch: u64, operation_token: u64, }, + /// Holt checkpoint installation may fail after partially replacing the + /// current database. The service instance is permanently poisoned and must + /// be discarded; reopening a clean store is the only safe recovery. + MetadataCheckpointInstallUncertain, /// The selected metadata checkpoint predates the durable object-GC /// failover fence. It may reference objects deleted after that image was /// produced, so installing it as a recovery source is unsafe. @@ -807,6 +838,33 @@ pub enum MetadError { /// point), NOT `EXDEV` — there is no copy+unlink fallback that would be /// correct here. GraftPoint, + /// The destination is occupied or is durably claimed by another restore. + RestoreDestinationConflict { + destination: String, + }, + /// An identical durable operation is preparing, cleaning, or releasing. + RestoreInProgress, + /// A detached restore inode/root no longer has the operation-scoped + /// membership proof installed before materialization. + RestoreRootChanged { + root: InodeId, + }, + /// The temporary history-retention binding is missing or no longer names + /// the durable restore operation being resumed. + RestoreBindingChanged { + root: InodeId, + }, + RestoreResourceLimit { + resource: String, + limit: u64, + actual: u64, + }, + RestoreHardlinkUnsupported { + inode: InodeId, + }, + RestoreCrossShardUnsupported { + inode: InodeId, + }, SnapshotLeaseExpired { snapshot_id: u64, lease_expires_unix_ms: u64, @@ -844,6 +902,30 @@ pub enum MetadError { }, } +/// Poison a service if checkpoint installation or its post-install recovery +/// exits early. Holt explicitly does not promise atomic installation on error, +/// so a failed instance must never resume serving or accept another repair +/// attempt in place. +struct MetadataCheckpointInstallGuard<'a> { + uncertain: &'a AtomicBool, + complete: bool, +} + +impl MetadataCheckpointInstallGuard<'_> { + fn complete(mut self) { + self.complete = true; + self.uncertain.store(false, Ordering::Release); + } +} + +impl Drop for MetadataCheckpointInstallGuard<'_> { + fn drop(&mut self) { + if !self.complete { + self.uncertain.store(true, Ordering::Release); + } + } +} + pub struct NoKvFs { mount: MountId, /// Stable index of the shard this service owns. Encoded into the high bits @@ -854,6 +936,43 @@ pub struct NoKvFs { objects: O, allocator_gate: Mutex<()>, backup_gate: Mutex<()>, + metadata_checkpoint_install_uncertain: AtomicBool, + /// Process-local serialization only. Durable operation and destination + /// claims remain the authority across owner failover and process restart. + restore_gate: Mutex<()>, + /// Linearizes the no-staging fast path with the first durable restore hold. + /// Readers hold a shared guard through their inode/dentry metadata read; + /// restore/failover recovery takes the exclusive guard before changing the + /// process-local hint below. Durable operation rows remain authoritative. + restore_visibility_fence: RwLock<()>, + /// Fail-closed process-local hint. False is installed only after a durable + /// scan proves that no Preparing/ReadyToAttach/Cleaning/Discarding restore + /// exists. It avoids a System lookup on every ordinary namespace read. + restore_staging_possible: AtomicBool, + /// False only while `new` may still wrap a pristine Holt store awaiting a + /// checkpoint image. Once bootstrap/open/image recovery has explicitly + /// entered restore visibility recovery, owner-epoch installs may safely + /// rebuild the fast-path hint instead of leaving it fail closed forever. + restore_visibility_recovery_ready: AtomicBool, + /// Process-local request counters. Durable restore state remains the source + /// of truth; these metrics expose latency and terminal retry behavior. + restore_to_fork_requests_total: AtomicU64, + restore_to_fork_success_total: AtomicU64, + restore_to_fork_failure_total: AtomicU64, + restore_to_fork_elapsed_ns_total: AtomicU64, + restore_to_fork_elapsed_ns_max: AtomicU64, + /// Process-local seqlock for the current-only restore graph in the System + /// family. Restore fsck/metrics sample this sequence so unrelated namespace + /// writes and scheduler-only cursor movement cannot starve a long scan, + /// while every authoritative restore mutation still invalidates a mixed + /// view. Writers register before metadata apply and leave only after the + /// apply result is known, including lost-ack outcomes. + restore_graph_sequence: AtomicU64, + restore_graph_writers: AtomicUsize, + /// Serializes snapshot publication against restore exact-reference + /// release. Ordinary object GC is fenced by the durable claim-version + /// predicate instead and must remain able to race a paused snapshot commit. + restore_snapshot_gate: RwLock<()>, /// Prevents a live worker/manual GC call from mistaking another in-process /// worker's durable Deleting claim for crash recovery. object_gc_gate: Mutex<()>, @@ -915,6 +1034,12 @@ pub struct NoKvFs { /// clock. Lets lease-deadline fencing be exercised deterministically. clock_override_ms: AtomicU64, metadata_log_sync: Mutex>, + /// Optional control-plane publication callback installed by a controlled + /// server. Restore RPCs temporarily require this callback after each + /// archived command so a crash at an applied-phase barrier cannot strand + /// an unreferenced shared-log tail. + metadata_log_publication_hook: RwLock>, + metadata_log_immediate_publication_depth: AtomicUsize, metadata_log_segments_archived_total: AtomicU64, metadata_log_entries_archived_total: AtomicU64, metadata_log_archive_bytes_total: AtomicU64, @@ -946,6 +1071,9 @@ pub struct NoKvFs { read_dir_plus_projection_hit_total: AtomicU64, } +type MetadataLogPublicationHook = + Arc Result<(), String> + Send + Sync + 'static>; + fn new_path_resolution_cache_shards() -> Vec>> { (0..PATH_CACHE_SHARD_COUNT) .map(|_| Mutex::new(BTreeMap::new())) @@ -977,6 +1105,26 @@ where M: MetadataStore, O: ObjectStore, { + fn ensure_metadata_checkpoint_install_stable(&self) -> Result<(), MetadError> { + if self + .metadata_checkpoint_install_uncertain + .load(Ordering::Acquire) + { + return Err(MetadError::MetadataCheckpointInstallUncertain); + } + Ok(()) + } + + fn begin_metadata_checkpoint_install( + &self, + ) -> Result, MetadError> { + self.ensure_metadata_checkpoint_install_stable()?; + Ok(MetadataCheckpointInstallGuard { + uncertain: &self.metadata_checkpoint_install_uncertain, + complete: false, + }) + } + fn resized_body_digest_uri( &self, inode: InodeId, @@ -1052,14 +1200,32 @@ fn recover_allocator_state( shard_index: u16, ) -> Result { let max_read = Version::new(u64::MAX)?; + let restore_active = metadata.get( + RecordFamily::System, + &restore::restore_active_key(mount), + max_read, + ReadPurpose::UserStrong, + )?; + if let Some(marker) = &restore_active { + if marker.0 != [restore::RESTORE_FORMAT_VERSION] { + return Err(MetadError::Codec( + "invalid restore-to-fork active marker".to_owned(), + )); + } + } if let Some(value) = metadata.get( RecordFamily::System, &allocator_key(mount), max_read, ReadPurpose::UserStrong, )? { - let (last_commit_version, next_inode, epoch) = - decode_allocator_state(&value.0).map_err(|err| MetadError::Codec(err.to_string()))?; + let (last_commit_version, next_inode, epoch, restore_fenced) = + decode_allocator_state_with_restore_fence(&value.0)?; + if restore_active.is_some() != restore_fenced { + return Err(MetadError::Codec( + "restore active marker and allocator downgrade fence disagree".to_owned(), + )); + } Version::new(last_commit_version)?; InodeId::new(next_inode)?; return Ok(AllocatorState { @@ -1069,6 +1235,12 @@ fn recover_allocator_state( }); } + if restore_active.is_some() { + return Err(MetadError::Codec( + "restore active marker exists without an allocator downgrade fence".to_owned(), + )); + } + let mut last_commit_version = 1_u64; let mut max_inode = InodeId::ROOT_RAW; // Only inodes minted by THIS shard may raise the local allocator floor. @@ -1130,6 +1302,40 @@ fn recover_allocator_state( }) } +fn encode_allocator_state_with_restore_fence( + last_commit_version: u64, + next_inode: u64, + epoch: u64, +) -> Vec { + let mut encoded = encode_allocator_state(last_commit_version, next_inode, epoch); + encoded.extend_from_slice(RESTORE_ALLOCATOR_FENCE_MAGIC); + encoded +} + +fn decode_allocator_state_with_restore_fence( + bytes: &[u8], +) -> Result<(u64, u64, u64, bool), MetadError> { + let allocator_bytes = if bytes.len() == 24 { + bytes + } else if bytes.len() == 24 + RESTORE_ALLOCATOR_FENCE_MAGIC.len() + && &bytes[24..] == RESTORE_ALLOCATOR_FENCE_MAGIC + { + &bytes[..24] + } else { + return Err(MetadError::Codec( + "invalid allocator state or restore downgrade fence".to_owned(), + )); + }; + let (last_commit_version, next_inode, epoch) = decode_allocator_state(allocator_bytes) + .map_err(|err| MetadError::Codec(err.to_string()))?; + Ok(( + last_commit_version, + next_inode, + epoch, + bytes.len() != allocator_bytes.len(), + )) +} + fn reservation_upper_bound(required: u64, reservation: u64) -> u64 { required.saturating_add(reservation) } @@ -1750,6 +1956,21 @@ fn request_id(prefix: &[u8], mount: MountId, inode: InodeId, version: Version) - out } +const PREPARED_ARTIFACT_REQUEST_ID_DOMAIN: &[u8] = b":prepared:v1:"; + +fn is_prepared_artifact_request_id(kind: CommandKind, request_id: &[u8]) -> bool { + if !matches!( + kind, + CommandKind::PublishArtifact | CommandKind::ReplaceArtifact + ) { + return false; + } + let prefix = kind_name(kind); + request_id.len() == prefix.len() + PREPARED_ARTIFACT_REQUEST_ID_DOMAIN.len() + 32 + && request_id.starts_with(prefix) + && request_id[prefix.len()..].starts_with(PREPARED_ARTIFACT_REQUEST_ID_DOMAIN) +} + fn allocator_reservation_request_id( mount: MountId, commit_version: Version, @@ -1867,6 +2088,10 @@ impl fmt::Display for MetadError { f, "object GC deletion from owner epoch {owner_epoch} with operation token {operation_token} has an uncertain outcome and requires controlled recovery" ), + Self::MetadataCheckpointInstallUncertain => write!( + f, + "metadata checkpoint installation had an uncertain partial outcome; discard this service instance and reopen a clean store" + ), Self::MetadataArchiveMissingObjectGcFence { checkpoint_key } => write!( f, "metadata checkpoint {checkpoint_key} predates the durable object-GC failover fence and cannot be restored safely" @@ -1926,6 +2151,39 @@ impl fmt::Display for MetadError { f, "path is a cross-shard graft point; use unregister-graft" ), + Self::RestoreDestinationConflict { destination } => write!( + f, + "restore destination is occupied or claimed by another operation: {destination}" + ), + Self::RestoreInProgress => write!(f, "restore operation is still in progress"), + Self::RestoreRootChanged { root } => write!( + f, + "restore staging root or member inode {} changed identity", + root.get() + ), + Self::RestoreBindingChanged { root } => write!( + f, + "restore temporary binding for root inode {} changed identity", + root.get() + ), + Self::RestoreResourceLimit { + resource, + limit, + actual, + } => write!( + f, + "restore resource {resource} exceeds limit {limit} (actual {actual})" + ), + Self::RestoreHardlinkUnsupported { inode } => write!( + f, + "restore does not support hard-linked inode {}", + inode.get() + ), + Self::RestoreCrossShardUnsupported { inode } => write!( + f, + "restore does not support cross-shard inode {}", + inode.get() + ), Self::SnapshotLeaseExpired { snapshot_id, lease_expires_unix_ms, diff --git a/crates/nokv-meta/src/service/agent.rs b/crates/nokv-meta/src/service/agent.rs index d0cd567e5..1e878a4e4 100644 --- a/crates/nokv-meta/src/service/agent.rs +++ b/crates/nokv-meta/src/service/agent.rs @@ -653,7 +653,8 @@ where let metadata = self.stat_path_from_at_version(InodeId::root(), &path, version)?; metadata .map(|metadata| { - let indexes = self.load_namespace_indexes_for_path(&path, version)?; + let indexes = + self.load_namespace_indexes_for_path(&path, metadata.attr.inode, version)?; self.card_for_metadata( path_name_for_card(&path), &path, @@ -730,7 +731,7 @@ where let offset = parse_cursor(request.cursor.as_deref())?; let version = self.read_version()?; let root_inode = self.resolve_directory_path_at_version(&root, version)?; - let indexes = self.load_namespace_indexes_for_path(&root, version)?; + let indexes = self.load_namespace_indexes_for_path(&root, root_inode, version)?; let mut entries = Vec::new(); self.collect_entries(&root, root_inode, version, &mut entries)?; let scanned_entries = entries.len(); @@ -798,7 +799,7 @@ where let limit = bounded_limit(request.limit)?; let version = self.read_version()?; let root_inode = self.resolve_directory_path_at_version(&root, version)?; - let indexes = self.load_namespace_indexes_for_path(&root, version)?; + let indexes = self.load_namespace_indexes_for_path(&root, root_inode, version)?; let mut entries = Vec::new(); self.collect_entries(&root, root_inode, version, &mut entries)?; let scanned_entries = entries.len(); @@ -1057,6 +1058,7 @@ where let path = normalize_card_path(®istration.path)?; let version = self.next_version()?; let read_version = predecessor(version)?; + let catalog_root = self.resolve_directory_path_at_version(&path, read_version)?; let mut mutations = Vec::new(); let catalog_key = path_index_catalog_key(self.mount, &path); let catalog_item = self.metadata.get_versioned( @@ -1065,7 +1067,7 @@ where read_version, ReadPurpose::UserStrong, )?; - let predicates = vec![PredicateRef { + let mut predicates = vec![PredicateRef { family: RecordFamily::PathIndex, key: catalog_key.clone(), predicate: catalog_item @@ -1110,7 +1112,16 @@ where }); } - self.commit_metadata(MetadataCommand { + let restore_write_predicates = + self.restore_namespace_write_predicates(&[catalog_root], read_version)?; + let restore_guarded = + super::restore::restore_write_predicates_include_owner(&restore_write_predicates); + predicates.extend(restore_write_predicates); + let inherited = self.restore_index_replace_catalog_plan(catalog_root, version)?; + predicates.extend(inherited.predicates); + mutations.extend(inherited.mutations); + + let command = MetadataCommand { request_id: request_id( b"register-namespace-index", self.mount, @@ -1125,7 +1136,14 @@ where predicates, mutations, watch: Vec::new(), - })?; + }; + if restore_guarded { + super::restore::validate_restore_command_bounds( + &command, + "restore namespace index registration", + )?; + } + self.commit_metadata(command)?; Ok(()) } @@ -1357,31 +1375,16 @@ where fn load_namespace_indexes_for_path( &self, path: &str, + root: InodeId, version: Version, ) -> Result { - let Some(catalog) = self.metadata.get( - RecordFamily::PathIndex, - &path_index_catalog_key(self.mount, path), - version, - ReadPurpose::UserStrong, - )? + let Some(index) = + self.restore_custom_index_at_path(path, root, version, ReadPurpose::UserStrong)? else { return Ok(LoadedNamespaceIndex::default()); }; - let catalog = decode_path_index_catalog(&catalog.0) - .map_err(|err| MetadError::Codec(err.to_string()))?; - let rows = self.metadata.scan(ScanRequest { - family: RecordFamily::PathIndex, - prefix: path_index_row_prefix(self.mount, path), - start_after: None, - version, - limit: 0, - purpose: ReadPurpose::UserStrong, - })?; let mut indexed_rows = BTreeMap::new(); - for row in rows { - let row = decode_path_index_row(&row.value.0) - .map_err(|err| MetadError::Codec(err.to_string()))?; + for row in index.rows { indexed_rows.insert( row.path, row.values @@ -1394,7 +1397,8 @@ where ); } Ok(LoadedNamespaceIndex { - fields: catalog + fields: index + .catalog .fields .into_iter() .map(namespace_index_field_from_record) diff --git a/crates/nokv-meta/src/service/allocator.rs b/crates/nokv-meta/src/service/allocator.rs index bad477b49..59bb16867 100644 --- a/crates/nokv-meta/src/service/allocator.rs +++ b/crates/nokv-meta/src/service/allocator.rs @@ -67,6 +67,34 @@ where )?; let key = allocator_key(self.mount); let reservation = self.commit_metadata_from_factory(|| { + let read_version = predecessor(commit_version)?; + let active_key = super::restore::restore_active_key(self.mount); + let active = self.metadata.get_versioned( + RecordFamily::System, + &active_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + if let Some(active) = &active { + if active.value.0 != [super::restore::RESTORE_FORMAT_VERSION] { + return Err(MetadError::Codec( + "invalid restore-to-fork active marker".to_owned(), + )); + } + } + let allocator_value = if active.is_some() { + encode_allocator_state_with_restore_fence( + reserved_version, + reserved_next_inode, + self.epoch.load(Ordering::Relaxed), + ) + } else { + encode_allocator_state( + reserved_version, + reserved_next_inode, + self.epoch.load(Ordering::Relaxed), + ) + }; Ok(MetadataCommand { request_id: allocator_reservation_request_id( self.mount, @@ -75,22 +103,22 @@ where reserved_next_inode, ), kind: CommandKind::ReserveAllocator, - read_version: predecessor(commit_version)?, + read_version, commit_version, primary_family: RecordFamily::System, primary_key: key.clone(), - predicates: Vec::new(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: active_key, + predicate: active + .map(|item| Predicate::VersionEquals(item.version)) + .unwrap_or(Predicate::NotExists), + }], mutations: vec![Mutation { family: RecordFamily::System, key, op: MutationOp::Put, - value: Some(Value(encode_allocator_state( - reserved_version, - reserved_next_inode, - // The factory executes under the epoch read fence held - // through metadata apply. - self.epoch.load(Ordering::Relaxed), - ))), + value: Some(Value(allocator_value)), }], watch: Vec::new(), }) @@ -120,6 +148,71 @@ where } } + /// Build the allocator half of the first durable restore hold. The caller + /// holds `allocator_gate` through apply and installs the active marker in + /// the same metadata command, so no reservation can reintroduce a v1 row + /// between the marker CAS and the fenced allocator write. + pub(super) fn restore_allocator_fence_plan( + &self, + read_version: Version, + ) -> Result<(PredicateRef, Mutation), MetadError> { + let key = allocator_key(self.mount); + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "allocator record is missing while installing restore fence".to_owned(), + ) + })?; + let (last_commit_version, next_inode, epoch, restore_fenced) = + decode_allocator_state_with_restore_fence(&item.value.0)?; + let active = self.metadata.get_versioned( + RecordFamily::System, + &super::restore::restore_active_key(self.mount), + read_version, + ReadPurpose::WritePlanLocal, + )?; + if let Some(active) = &active { + if active.value.0 != [super::restore::RESTORE_FORMAT_VERSION] { + return Err(MetadError::Codec( + "invalid restore-to-fork active marker".to_owned(), + )); + } + } + if active.is_some() != restore_fenced { + return Err(MetadError::Codec( + "restore active marker and allocator downgrade fence disagree".to_owned(), + )); + } + let last_commit_version = + last_commit_version.max(self.reserved_version.load(Ordering::Relaxed)); + let next_inode = next_inode.max(self.reserved_next_inode.load(Ordering::Relaxed)); + let epoch = epoch.max(self.epoch.load(Ordering::Relaxed)); + Ok(( + PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(item.version), + }, + Mutation { + family: RecordFamily::System, + key, + op: MutationOp::Put, + value: Some(Value(encode_allocator_state_with_restore_fence( + last_commit_version, + next_inode, + epoch, + ))), + }, + )) + } + pub(super) fn next_version(&self) -> Result { let raw = self.clock.fetch_add(1, Ordering::Relaxed) + 1; self.ensure_allocator_reservation(raw, self.next_inode.load(Ordering::Relaxed))?; @@ -127,6 +220,7 @@ where } pub(super) fn read_version(&self) -> Result { + self.ensure_metadata_checkpoint_install_stable()?; Version::new(self.clock.load(Ordering::Relaxed)).map_err(Into::into) } diff --git a/crates/nokv-meta/src/service/backup.rs b/crates/nokv-meta/src/service/backup.rs index 92d0cafc1..c8e4c4929 100644 --- a/crates/nokv-meta/src/service/backup.rs +++ b/crates/nokv-meta/src/service/backup.rs @@ -187,6 +187,13 @@ where serialize_archive_manifest(&manifest).into_bytes(), )?; + // CURRENT is now the authoritative recovery pointer. Acknowledge its + // exact log boundary before pruning either per-request archive proofs + // or checkpoint objects; a failure here leaves safe retained data and + // never advances local durability coverage ahead of the published + // pointer. + self.acknowledge_published_metadata_checkpoint(exported.log_lsn, exported.log_digest)?; + // Retention deletes happen only after the manifest stops referencing // them; a crash here leaks orphans (reclaimable), never a live pointer. let mut pruned = 0; @@ -336,16 +343,31 @@ where .lock() .unwrap_or_else(|err| err.into_inner()); self.mark_materialization_orphan_possible_under_gc_gate(); + let restore_visibility = self + .restore_visibility_fence + .write() + .unwrap_or_else(|err| err.into_inner()); + let checkpoint_install = self.begin_metadata_checkpoint_install()?; + let _restore_graph_write = self.begin_restore_graph_write(); + self.restore_staging_possible.store(true, Ordering::Release); self.metadata.install_checkpoint_image(&image)?; self.purge_path_caches_after_write(); self.refresh_allocator_state()?; self.verify_failover_durability_required()?; + // Readers that enter after this point still take the restore slow path + // because the hint remains fail closed. Release the write fence before + // either recovery proof: both proofs perform ordinary namespace reads + // that acquire the shared side of this same fence. + drop(restore_visibility); + self.recover_restore_staging_visibility()?; self.reconcile_materialization_orphan_state_under_gc_gate()?; - Ok(MetadataRestoreOutcome { + let outcome = MetadataRestoreOutcome { checkpoint_key: identity.checkpoint_key.clone(), image_bytes: identity.image_bytes, commit_version: self.clock.load(Ordering::SeqCst), - }) + }; + checkpoint_install.complete(); + Ok(outcome) } fn export_metadata_checkpoint(&self) -> Result { @@ -415,6 +437,13 @@ where .lock() .unwrap_or_else(|err| err.into_inner()); self.mark_materialization_orphan_possible_under_gc_gate(); + let restore_visibility = self + .restore_visibility_fence + .write() + .unwrap_or_else(|err| err.into_inner()); + let checkpoint_install = self.begin_metadata_checkpoint_install()?; + let _restore_graph_write = self.begin_restore_graph_write(); + self.restore_staging_possible.store(true, Ordering::Release); self.metadata.install_checkpoint_image(&image)?; // The image replaced the engine state wholesale, bypassing the commit // funnel; entries cached before the install may not exist in it at all. @@ -424,12 +453,16 @@ where // verify the installed state as defense against a mismatched/corrupt // archive assembled outside the supported backup path. self.verify_failover_durability_required()?; + drop(restore_visibility); + self.recover_restore_staging_visibility()?; self.reconcile_materialization_orphan_state_under_gc_gate()?; - Ok(Some(MetadataRestoreOutcome { + let outcome = MetadataRestoreOutcome { checkpoint_key: manifest.current, image_bytes: image.len() as u64, commit_version: manifest.version, - })) + }; + checkpoint_install.complete(); + Ok(Some(outcome)) } fn read_archive_manifest( diff --git a/crates/nokv-meta/src/service/checkpoint.rs b/crates/nokv-meta/src/service/checkpoint.rs index 4e8fa819a..94cfe89c7 100644 --- a/crates/nokv-meta/src/service/checkpoint.rs +++ b/crates/nokv-meta/src/service/checkpoint.rs @@ -165,12 +165,18 @@ where commit_version: Version, object_reference: ObjectReferenceMutation, ) -> Result<(), MetadError> { + let read_version = predecessor(commit_version)?; let mut predicates = vec![PredicateRef { family: RecordFamily::Inode, key: inode_key(self.mount, parent), predicate: Predicate::Exists, }]; predicates.push(object_reference.predicate(self.mount)); + let restore_write_predicates = + self.restore_namespace_write_predicates(&[parent], read_version)?; + let restore_guarded = + super::restore::restore_write_predicates_include_owner(&restore_write_predicates); + predicates.extend(restore_write_predicates); let mut mutations = Vec::new(); for (proj, chunks) in shards { let inode = proj.attr.inode; @@ -214,17 +220,32 @@ where } } } - self.commit_metadata(MetadataCommand { + let projections = shards + .iter() + .map(|(projection, _)| projection.clone()) + .collect::>(); + let enrollment = + self.restore_namespace_enrollment_plan(parent, &projections, read_version)?; + predicates.extend(enrollment.predicates); + mutations.extend(enrollment.mutations); + let command = MetadataCommand { request_id: request_id(b"publish-checkpoint", self.mount, parent, commit_version), kind: CommandKind::PublishArtifact, - read_version: predecessor(commit_version)?, + read_version, commit_version, primary_family: RecordFamily::Dentry, primary_key: dentry_prefix(self.mount, parent), predicates, mutations, watch: Vec::new(), - })?; + }; + if restore_guarded { + super::restore::validate_restore_command_bounds( + &command, + "restore checkpoint publish", + )?; + } + self.commit_metadata(command)?; Ok(()) } } diff --git a/crates/nokv-meta/src/service/command.rs b/crates/nokv-meta/src/service/command.rs index 6101c4fd2..9ad7447f5 100644 --- a/crates/nokv-meta/src/service/command.rs +++ b/crates/nokv-meta/src/service/command.rs @@ -7,6 +7,7 @@ use super::*; /// [`MetadError`] (which is not `Clone`), keeping the check logic single-sourced. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum OwnerLeaseFault { + CheckpointInstallUncertain, StaleEpoch { owner_epoch: u64, required_epoch: u64, @@ -20,6 +21,9 @@ pub(super) enum OwnerLeaseFault { impl OwnerLeaseFault { fn into_error(self) -> MetadError { match self { + OwnerLeaseFault::CheckpointInstallUncertain => { + MetadError::MetadataCheckpointInstallUncertain + } OwnerLeaseFault::StaleEpoch { owner_epoch, required_epoch, @@ -38,27 +42,120 @@ impl OwnerLeaseFault { } } +/// Brackets one authoritative restore-graph metadata apply. The paired +/// sequence increments make an overlapping reader reject its mixed System +/// view; the active count closes the windows on either side of those bumps. +pub(super) struct RestoreGraphWriteGuard<'a> { + sequence: &'a AtomicU64, + writers: &'a AtomicUsize, +} + +impl<'a> RestoreGraphWriteGuard<'a> { + fn new(sequence: &'a AtomicU64, writers: &'a AtomicUsize) -> Self { + writers.fetch_add(1, Ordering::SeqCst); + sequence.fetch_add(1, Ordering::SeqCst); + Self { sequence, writers } + } +} + +impl Drop for RestoreGraphWriteGuard<'_> { + fn drop(&mut self) { + self.sequence.fetch_add(1, Ordering::SeqCst); + self.writers.fetch_sub(1, Ordering::SeqCst); + } +} + impl NoKvFs where M: MetadataStore, O: ObjectStore, { + pub(super) fn begin_restore_graph_write(&self) -> RestoreGraphWriteGuard<'_> { + RestoreGraphWriteGuard::new(&self.restore_graph_sequence, &self.restore_graph_writers) + } + + fn restore_graph_write_for_command( + &self, + command: &MetadataCommand, + ) -> Option> { + super::restore::command_mutates_restore_graph(self.mount, command) + .then(|| self.begin_restore_graph_write()) + } + + fn restore_graph_write_for_batch( + &self, + commands: &[MetadataCommand], + ) -> Option> { + commands + .iter() + .any(|command| super::restore::command_mutates_restore_graph(self.mount, command)) + .then(|| self.begin_restore_graph_write()) + } + + /// Capture a quiescent restore-graph sequence. Writers register before + /// their durable apply, so observing either an active writer or a later + /// sequence change invalidates a current-only System scan. + pub(super) fn restore_graph_read_sequence(&self) -> Option { + if self.restore_graph_writers.load(Ordering::SeqCst) != 0 { + return None; + } + let sequence = self.restore_graph_sequence.load(Ordering::SeqCst); + (self.restore_graph_writers.load(Ordering::SeqCst) == 0).then_some(sequence) + } + + pub(super) fn restore_graph_read_is_stable(&self, sequence: u64) -> bool { + self.restore_graph_writers.load(Ordering::SeqCst) == 0 + && self.restore_graph_sequence.load(Ordering::SeqCst) == sequence + && self.restore_graph_writers.load(Ordering::SeqCst) == 0 + } + pub fn install_owner_epoch(&self, epoch: u64) -> Result<(), MetadError> { validate_owner_epoch(epoch)?; + let _restore_graph_write = self.begin_restore_graph_write(); + // Fail closed while ownership changes. The startup path rebuilds the + // optimization after bootstrap or checkpoint/log recovery. + self.mark_restore_staging_possible(); // Write guard: wait for in-flight commits to finish, then raise both // epochs together so no commit ever observes a torn (epoch, required) // pair or applies under a superseded epoch. - let _fence = self - .epoch_fence - .write() - .unwrap_or_else(|err| err.into_inner()); - self.epoch.fetch_max(epoch, Ordering::Relaxed); - self.required_owner_epoch - .fetch_max(epoch, Ordering::Relaxed); + { + let _fence = self + .epoch_fence + .write() + .unwrap_or_else(|err| err.into_inner()); + self.epoch.fetch_max(epoch, Ordering::Relaxed); + self.required_owner_epoch + .fetch_max(epoch, Ordering::Relaxed); + } + // Controlled failover installs the owner epoch before installing a + // checkpoint into a pristine Holt store. Do not scan in that state: a + // read would create trees and make wholesale image install fail. An + // already-open/bootstrap-complete shard, however, must reconstruct the + // fast-path hint after this fail-closed epoch boundary. + if self + .restore_visibility_recovery_ready + .load(Ordering::Acquire) + { + self.recover_restore_staging_visibility()?; + } Ok(()) } pub fn observe_required_owner_epoch(&self, epoch: u64) -> Result<(), MetadError> { + validate_owner_epoch(epoch)?; + let _restore_graph_write = self.begin_restore_graph_write(); + self.mark_restore_staging_possible(); + self.fence_required_owner_epoch(epoch)?; + self.recover_restore_staging_visibility() + } + + /// Raise only the commit fence after observing a newer control-plane epoch. + /// Publication callbacks use this form because they can run while the + /// current restore command still owns the restore-visibility fence; trying + /// to rebuild that process-local hint on the same thread would deadlock. + /// The stale owner cannot serve through the recovery publication gate, and + /// the successor reconstructs restore visibility from durable state. + pub fn fence_required_owner_epoch(&self, epoch: u64) -> Result<(), MetadError> { validate_owner_epoch(epoch)?; // Write guard: a failover bump waits for in-flight commits, then raises // the required epoch so every subsequent commit is fenced. @@ -120,6 +217,12 @@ where /// Single source of truth for "may this owner commit?": epoch fence first, /// then the wall-clock lease deadline (the partition-safe self-fence). pub(super) fn check_owner_lease(&self) -> Result<(), OwnerLeaseFault> { + if self + .metadata_checkpoint_install_uncertain + .load(Ordering::Acquire) + { + return Err(OwnerLeaseFault::CheckpointInstallUncertain); + } let owner_epoch = self.epoch.load(Ordering::Relaxed); let required_epoch = self.required_owner_epoch.load(Ordering::Relaxed); if owner_epoch < required_epoch { @@ -210,6 +313,7 @@ where message: err.to_string(), })?; } + let _restore_graph_write = self.restore_graph_write_for_command(&command); match self.metadata.commit_metadata(command.clone()) { Ok(result) => { self.purge_path_caches_after_write(); @@ -262,6 +366,7 @@ where .read() .unwrap_or_else(|err| err.into_inner()); self.ensure_owner_epoch_current()?; + let _restore_graph_write = self.restore_graph_write_for_command(&command); let result = self.metadata.commit_metadata(command)?; self.purge_path_caches_after_write(); Ok(result) @@ -315,6 +420,7 @@ where if let Err(fault) = self.check_owner_lease() { return commands.iter().map(|_| Err(fault.into_error())).collect(); } + let _restore_graph_write = self.restore_graph_write_for_batch(commands); self.metadata.commit_independent_batch(commands) }; diff --git a/crates/nokv-meta/src/service/fsck.rs b/crates/nokv-meta/src/service/fsck.rs index 279473964..834dcbe9e 100644 --- a/crates/nokv-meta/src/service/fsck.rs +++ b/crates/nokv-meta/src/service/fsck.rs @@ -1,23 +1,43 @@ -//! Filesystem consistency check (fsck). +//! Filesystem and durable restore consistency checks. //! -//! Verifies the **live** namespace has no dangling block references: every -//! object a live file points at must still exist in the object store. This is -//! the defense-in-depth complement to the object-first write ordering — -//! ordering *prevents* metadata from ever referencing a missing object, and -//! fsck *detects* any drift after the fact (an out-of-band object deletion, an -//! eventual-consistency anomaly in external storage, or a latent bug). -//! -//! The scan walks live inodes at their **current body generation**, so -//! superseded and snapshot-pinned generations are not mistaken for dangling -//! references, and a clone's borrowed (source-owned) block keys resolve against -//! the source objects that still exist. Reclaiming *orphan* objects (written but -//! never referenced) is the opposite direction and requires an object `list`; it -//! is not part of this scan. +//! The object scan verifies every effective manifest reachable from the live +//! namespace. Full mode additionally walks unexpired snapshot pins and +//! `ForkBinding` history roots, follows sparse `base_generation` chains, and +//! includes object-backed symlinks. Restore-private graph validation lives in +//! this module as well so operational diagnostics and the downgrade proof use +//! one definition of consistency. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fmt; use super::*; use crate::layout::inode_prefix; -/// A live file's block reference whose backing object is missing. +const RESTORE_FSCK_PAGE_ROWS: usize = 256; +const MAX_RESTORE_FSCK_OPERATIONS: usize = 65_536; +const OBJECT_FSCK_PAGE_ROWS: usize = 256; +// `read_dir_plus_page_at_version_for_purpose` reads one look-ahead row to +// determine whether another page exists. Keep the physical metadata scan at +// or below `OBJECT_FSCK_PAGE_ROWS`. +const OBJECT_FSCK_DIRECTORY_PAGE_ENTRIES: usize = OBJECT_FSCK_PAGE_ROWS - 1; + +#[derive(Clone, Copy)] +enum RestoreCursorFormat { + RawKey, + ReleaseWorker, +} + +/// Scope of an object-reference fsck. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum FsckMode { + /// Current inode records only. + #[default] + Live, + /// Live records plus every unexpired snapshot and durable fork history root. + Full, +} + +/// A metadata block reference whose backing object is missing. #[derive(Clone, Debug, PartialEq, Eq)] pub struct DanglingBlock { pub inode: u64, @@ -25,18 +45,175 @@ pub struct DanglingBlock { pub object_key: String, } -/// The outcome of an [`NoKvFs::fsck_dangling_blocks`] scan. +/// A metadata block reference whose object has a different byte length. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MismatchedBlock { + pub inode: u64, + pub generation: u64, + pub object_key: String, + pub expected_size: u64, + pub actual_size: u64, +} + +/// The object-reference portion of an fsck scan. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct FsckReport { pub inodes_scanned: usize, pub files_scanned: usize, + pub symlinks_scanned: usize, + pub snapshot_pins_scanned: usize, + pub fork_bindings_scanned: usize, + pub historical_bodies_scanned: usize, pub blocks_checked: usize, pub dangling: Vec, + pub size_mismatches: Vec, +} + +/// Counts and alert inputs for durable restore operations. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RestoreMetrics { + pub read_version: u64, + pub active_marker: bool, + pub allocator_v2_fenced: bool, + pub preparing: usize, + pub ready_to_attach: usize, + pub complete: usize, + pub cleaning: usize, + pub discarding: usize, + pub releasing: usize, + /// Commit-version distance from the oldest non-terminal operation. This is + /// durable across reopen and is the honest long-running alert input because + /// v1 operation records do not contain a wall-clock timestamp. + pub max_preparing_version_age: u64, + pub max_releasing_version_age: u64, + pub control_rows: BTreeMap, + pub staging_rows: usize, + pub exact_reference_rows: usize, + pub index_rows: usize, + pub cleanup_backlog: usize, + pub release_backlog: usize, + pub quarantine_rows: usize, +} + +impl RestoreMetrics { + pub fn operation_count(&self) -> usize { + self.preparing + .saturating_add(self.ready_to_attach) + .saturating_add(self.complete) + .saturating_add(self.cleaning) + .saturating_add(self.discarding) + .saturating_add(self.releasing) + } +} + +/// One fail-closed restore graph violation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RestoreFsckIssue { + pub code: String, + pub message: String, + pub operation_id: Option, + pub ref_set_id: Option, +} + +/// Restore-private consistency report. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RestoreFsckReport { + pub metrics: RestoreMetrics, + pub borrowed_objects_checked: usize, + pub dangling_borrowed_objects: Vec, + pub borrowed_object_size_mismatches: Vec, + pub issues: Vec, +} + +impl RestoreFsckReport { + pub fn is_consistent(&self) -> bool { + self.issues.is_empty() + && self.dangling_borrowed_objects.is_empty() + && self.borrowed_object_size_mismatches.is_empty() + } +} + +/// Explicit acknowledgement required by the destructive downgrade API. +/// +/// Construct this value only after the deployment has stopped advertising and +/// accepting `restore_to_fork_v1`. The metadata layer cannot infer that +/// external routing state, so it requires the operator to make the dependency +/// visible at the call site. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RestoreCapabilityDisabled(()); + +impl RestoreCapabilityDisabled { + pub fn acknowledged() -> Self { + Self(()) + } +} + +/// Explicit proof supplied by deployment orchestration after every metadata +/// owner capable of running restore initialization PUTs has been stopped or +/// fenced. A process-local lock alone cannot establish this global fact. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RestoreWritersQuiesced(()); + +impl RestoreWritersQuiesced { + pub fn acknowledged() -> Self { + Self(()) + } +} + +#[must_use = "a restore downgrade is not safe until full fsck and a fresh metadata checkpoint complete"] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RestoreDowngradeOutcome { + pub already_drained: bool, + pub commit_version: Option, + /// A full restore/object-reference fsck is still required after the fence + /// transaction and before creating the downgrade checkpoint. + pub full_fsck_required: bool, + /// A post-drain metadata checkpoint is still required before an older + /// metadata binary may be started. + pub metadata_checkpoint_required: bool, +} + +#[derive(Debug)] +pub enum RestoreDowngradeError { + Metadata(MetadError), + PrivateStatePresent { + keyspace: String, + /// Lower bound observed by the bounded preflight scan. + observed_rows: usize, + }, + ConcurrentMutation, +} + +impl fmt::Display for RestoreDowngradeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Metadata(error) => write!(formatter, "restore downgrade metadata error: {error}"), + Self::PrivateStatePresent { + keyspace, + observed_rows, + } => write!( + formatter, + "restore downgrade is blocked by at least {observed_rows} row(s) in {keyspace}" + ), + Self::ConcurrentMutation => write!( + formatter, + "restore downgrade raced a metadata mutation; run fsck and retry" + ), + } + } +} + +impl std::error::Error for RestoreDowngradeError {} + +impl From for RestoreDowngradeError { + fn from(error: MetadError) -> Self { + Self::Metadata(error) + } } impl FsckReport { pub fn is_consistent(&self) -> bool { - self.dangling.is_empty() + self.dangling.is_empty() && self.size_mismatches.is_empty() } } @@ -45,62 +222,3677 @@ where M: MetadataStore, O: ObjectStore, { - /// Scan up to `limit` live inodes (`0` = all) and verify that every block a - /// live file references still exists in the object store. + /// Preserve the original live-only entry point. pub fn fsck_dangling_blocks(&self, limit: usize) -> Result { + self.fsck_object_references(FsckMode::Live, limit) + } + + /// Verify object references reachable from metadata. + /// + /// `limit == 0` means unlimited. A non-zero limit bounds the total number + /// of inode bodies inspected across live and historical roots; it is an + /// operational sampling control, not a consistency proof. + pub fn fsck_object_references( + &self, + mode: FsckMode, + limit: usize, + ) -> Result { + let epoch = self.path_cache_epoch.load(Ordering::Acquire); + let report = self.fsck_object_references_at_stable_epoch(mode, limit)?; + if self.path_cache_epoch.load(Ordering::Acquire) != epoch { + return Err(MetadError::Codec( + "object-reference fsck raced a metadata write; retry the scan".to_owned(), + )); + } + Ok(report) + } + + fn fsck_object_references_at_stable_epoch( + &self, + mode: FsckMode, + limit: usize, + ) -> Result { let version = self.read_version()?; - let inode_rows = self.metadata.scan(ScanRequest { - family: RecordFamily::Inode, - prefix: inode_prefix(self.mount), - start_after: None, - version, - limit, - purpose: ReadPurpose::UserStrong, + let mut report = FsckReport::default(); + let mut visited_bodies = HashSet::new(); + + let prefix = inode_prefix(self.mount); + let mut start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::Inode, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: OBJECT_FSCK_PAGE_ROWS, + purpose: ReadPurpose::RestoreStaging, + })?; + if rows.is_empty() { + break; + } + for row in &rows { + if fsck_limit_reached(limit, &report) { + break; + } + let attr = decode_inode_attr(&row.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + report.inodes_scanned = report.inodes_scanned.saturating_add(1); + self.fsck_body_for_attr( + &attr, + version, + ReadPurpose::RestoreStaging, + false, + &mut visited_bodies, + &mut report, + )?; + } + if fsck_limit_reached(limit, &report) { + break; + } + let reached_tail = rows.len() < OBJECT_FSCK_PAGE_ROWS; + start_after = rows.last().map(|row| row.key.clone()); + if reached_tail { + break; + } + } + + if mode == FsckMode::Full && !fsck_limit_reached(limit, &report) { + self.fsck_snapshot_roots(version, limit, &mut visited_bodies, &mut report)?; + self.fsck_fork_binding_roots(version, limit, &mut visited_bodies, &mut report)?; + } + Ok(report) + } + + fn fsck_snapshot_roots( + &self, + version: Version, + limit: usize, + visited_bodies: &mut HashSet<(u64, u64, u64)>, + report: &mut FsckReport, + ) -> Result<(), MetadError> { + let prefix = snapshot_pin_prefix(self.mount); + let mut start_after = None; + let now_ms = self.now_ms(); + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::Snapshot, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: OBJECT_FSCK_PAGE_ROWS, + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + break; + } + for row in &rows { + if fsck_limit_reached(limit, report) { + return Ok(()); + } + let pin = decode_snapshot_pin(&row.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if row.key != snapshot_pin_key(self.mount, pin.snapshot_id) { + return Err(MetadError::Codec(format!( + "snapshot pin key does not match snapshot {}", + pin.snapshot_id + ))); + } + if now_ms >= pin.lease_expires_unix_ms { + continue; + } + report.snapshot_pins_scanned += 1; + self.fsck_namespace_root( + pin.root, + Version::new(pin.read_version)?, + ReadPurpose::Snapshot, + limit, + visited_bodies, + report, + )?; + if fsck_limit_reached(limit, report) { + return Ok(()); + } + } + let reached_tail = rows.len() < OBJECT_FSCK_PAGE_ROWS; + start_after = rows.last().map(|row| row.key.clone()); + if reached_tail { + break; + } + } + Ok(()) + } + + fn fsck_fork_binding_roots( + &self, + version: Version, + limit: usize, + visited_bodies: &mut HashSet<(u64, u64, u64)>, + report: &mut FsckReport, + ) -> Result<(), MetadError> { + self.visit_versioned_fork_bindings_at(version, ReadPurpose::WritePlanLocal, |binding| { + if fsck_limit_reached(limit, report) { + return Ok(true); + } + report.fork_bindings_scanned += 1; + self.fsck_namespace_root( + binding.binding.source_root, + Version::new(binding.binding.pinned_read_version)?, + ReadPurpose::RestoreStaging, + limit, + visited_bodies, + report, + )?; + Ok(fsck_limit_reached(limit, report)) })?; + Ok(()) + } + + fn fsck_namespace_root( + &self, + root: InodeId, + version: Version, + purpose: ReadPurpose, + limit: usize, + visited_bodies: &mut HashSet<(u64, u64, u64)>, + report: &mut FsckReport, + ) -> Result<(), MetadError> { + if fsck_limit_reached(limit, report) { + return Ok(()); + } + let root_attr = self + .get_attr_at_version_for_purpose(root, version, purpose)? + .ok_or_else(|| { + MetadError::Codec(format!( + "retained fsck root {} is absent at version {}", + root.get(), + version.get() + )) + })?; + report.inodes_scanned = report.inodes_scanned.saturating_add(1); + self.fsck_body_for_attr(&root_attr, version, purpose, true, visited_bodies, report)?; + if root_attr.file_type != FileType::Directory || fsck_limit_reached(limit, report) { + return Ok(()); + } + + let mut pending = vec![root]; + let mut visited_directories = HashSet::new(); + while let Some(parent) = pending.pop() { + if !visited_directories.insert(parent) { + continue; + } + let mut after = None; + loop { + let page = self.read_dir_plus_page_at_version_for_purpose( + parent, + after.as_ref(), + OBJECT_FSCK_DIRECTORY_PAGE_ENTRIES, + version, + purpose, + )?; + for child in page.entries { + if fsck_limit_reached(limit, report) { + return Ok(()); + } + report.inodes_scanned = report.inodes_scanned.saturating_add(1); + self.fsck_body_for_attr( + &child.attr, + version, + purpose, + true, + visited_bodies, + report, + )?; + if child.attr.file_type == FileType::Directory + && child.attr.inode.shard_index() == self.shard_index + { + pending.push(child.attr.inode); + } + if fsck_limit_reached(limit, report) { + return Ok(()); + } + } + let Some(next_cursor) = page.next_cursor else { + break; + }; + after = Some(next_cursor); + if fsck_limit_reached(limit, report) { + return Ok(()); + } + } + } + Ok(()) + } + + fn fsck_body_for_attr( + &self, + attr: &InodeAttr, + version: Version, + purpose: ReadPurpose, + historical: bool, + visited_bodies: &mut HashSet<(u64, u64, u64)>, + report: &mut FsckReport, + ) -> Result<(), MetadError> { + if !matches!(attr.file_type, FileType::File | FileType::Symlink) { + return Ok(()); + } + if !visited_bodies.insert((attr.inode.get(), attr.generation, version.get())) { + return Ok(()); + } + let body = self + .body_descriptor_at_version_for_purpose(attr.inode, attr.generation, version, purpose)? + .ok_or(MetadError::MissingBodyDescriptor)?; + if body.generation != attr.generation { + return Err(MetadError::Codec(format!( + "body generation {} does not match inode {} generation {}", + body.generation, + attr.inode.get(), + attr.generation + ))); + } + match attr.file_type { + FileType::File => report.files_scanned += 1, + FileType::Symlink => report.symlinks_scanned += 1, + _ => unreachable!("body-bearing file type checked above"), + } + if historical { + report.historical_bodies_scanned += 1; + } + let manifests = + self.chunk_manifests_for_body_at_version(attr.inode, &body, version, purpose)?; + for block in manifests + .iter() + .flat_map(|manifest| &manifest.slices) + .flat_map(|slice| &slice.blocks) + { + report.blocks_checked += 1; + let key = ObjectKey::new(block.object_key.clone())?; + match self.objects.head(&key)? { + None => report.dangling.push(DanglingBlock { + inode: attr.inode.get(), + generation: body.generation, + object_key: block.object_key.clone(), + }), + Some(info) if info.size != block.len => { + report.size_mismatches.push(MismatchedBlock { + inode: attr.inode.get(), + generation: body.generation, + object_key: block.object_key.clone(), + expected_size: block.len, + actual_size: info.size, + }); + } + Some(_) => {} + } + } + Ok(()) + } +} + +#[derive(Clone)] +struct VersionedRestoreOperation { + operation: super::restore::RestoreOperation, +} + +impl NoKvFs +where + M: MetadataStore, + O: ObjectStore, +{ + /// Durable restore gauges. + /// + /// The authoritative graph is exact across one restore-graph sequence and + /// remains O(private rows). The two worker cursors are scheduler hints and + /// are sampled from their atomic current rows so their round-robin movement + /// cannot starve metrics. Unlike full fsck, this never materializes an + /// entire keyspace: every keyspace is counted in bounded pages and only + /// operation rows are decoded for state/age gauges. + pub fn restore_metrics(&self) -> Result { + self.restore_metrics_with_page_size(4_096) + } - let mut report = FsckReport { - inodes_scanned: inode_rows.len(), - ..FsckReport::default() + fn restore_metrics_with_page_size( + &self, + page_size: usize, + ) -> Result { + if page_size == 0 { + return Err(MetadError::Codec( + "restore metrics page size must be positive".to_owned(), + )); + } + self.ensure_metadata_checkpoint_install_stable()?; + for _ in 0..3 { + let Some(sequence) = self.restore_graph_read_sequence() else { + std::thread::yield_now(); + continue; + }; + let metrics = self.collect_restore_metrics(page_size); + if self.restore_graph_read_is_stable(sequence) { + self.ensure_owner_epoch_current()?; + return metrics; + } + } + Err(MetadError::Codec( + "restore metrics raced a restore graph write; retry the scan".to_owned(), + )) + } + + fn collect_restore_metrics(&self, page_size: usize) -> Result { + let version = self.read_version()?; + self.collect_restore_metrics_at(version, page_size, true) + } + + fn collect_restore_metrics_at( + &self, + version: Version, + page_size: usize, + strict: bool, + ) -> Result { + let mut metrics = RestoreMetrics { + read_version: version.get(), + ..RestoreMetrics::default() }; + if strict { + self.read_restore_fence_metrics(version, &mut metrics)?; + } + + let mut keyspaces = super::restore::restore_control_keyspaces(self.mount); + keyspaces.extend(super::restore_index::restore_index_private_keyspaces( + self.mount, + )); + for (name, prefix) in keyspaces { + let scan_version = if matches!(name, "init_upload_tombstone_cursor" | "release_cursor") + { + // Cursor rows are exact single-key scheduler hints. Read their + // atomic current representation independently from the stable + // ownership graph so a permanent tombstone cannot starve the + // metrics scan by rotating this value forever. + Version::new(u64::MAX)? + } else { + version + }; + let mut count = 0_usize; + let mut start_after = None; + loop { + let page = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: start_after.clone(), + version: scan_version, + limit: page_size, + purpose: ReadPurpose::WritePlanLocal, + })?; + if page.is_empty() { + break; + } + if strict && name == "operation" { + for row in &page { + let operation = super::restore::decode_restore_operation(&row.value.0)?; + if row.key + != super::restore::restore_operation_key( + self.mount, + &operation.operation_digest, + ) + { + return Err(MetadError::Codec( + "restore metrics found an operation key/value mismatch".to_owned(), + )); + } + update_restore_operation_metrics(&mut metrics, &operation); + } + } + count = count.checked_add(page.len()).ok_or_else(|| { + MetadError::Codec("restore metrics row count overflow".to_owned()) + })?; + let reached_tail = page.len() < page_size; + start_after = page.last().map(|row| row.key.clone()); + if reached_tail { + break; + } + } + metrics.control_rows.insert(name.to_owned(), count); + } + metrics.staging_rows = restore_metric_count(&metrics, "staging_member") + .checked_add(restore_metric_count(&metrics, "staging_inode_inverse")) + .and_then(|value| { + value.checked_add(restore_metric_count(&metrics, "staging_inverse_owner")) + }) + .ok_or_else(|| MetadError::Codec("restore staging metric overflow".to_owned()))?; + metrics.exact_reference_rows = restore_metric_count(&metrics, "base_owner") + .checked_add(restore_metric_count(&metrics, "base_inverse")) + .and_then(|value| { + value.checked_add(restore_metric_count(&metrics, "base_inverse_owner")) + }) + .ok_or_else(|| MetadError::Codec("restore reference metric overflow".to_owned()))?; + metrics.cleanup_backlog = restore_metric_count(&metrics, "cleanup_job"); + metrics.release_backlog = restore_metric_count(&metrics, "release_job"); + metrics.quarantine_rows = restore_metric_count(&metrics, "release_quarantine"); + metrics.index_rows = metrics + .control_rows + .iter() + .filter(|(name, _)| name.starts_with("index_")) + .try_fold(0_usize, |total, (_, count)| total.checked_add(*count)) + .ok_or_else(|| MetadError::Codec("restore index metric overflow".to_owned()))?; + Ok(metrics) + } + + fn read_restore_fence_metrics( + &self, + version: Version, + metrics: &mut RestoreMetrics, + ) -> Result<(), MetadError> { + let activation = self + .metadata + .get( + RecordFamily::System, + &super::restore::restore_activation_fence_key(self.mount), + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("restore activation fence is missing".to_owned()))?; + if activation.0 != [super::restore::RESTORE_FORMAT_VERSION] { + return Err(MetadError::Codec( + "restore activation fence has an invalid value".to_owned(), + )); + } + let active = self.metadata.get_versioned( + RecordFamily::System, + &super::restore::restore_active_key(self.mount), + version, + ReadPurpose::WritePlanLocal, + )?; + if active + .as_ref() + .is_some_and(|item| item.value.0 != [super::restore::RESTORE_FORMAT_VERSION]) + { + return Err(MetadError::Codec( + "restore metrics found an invalid active marker".to_owned(), + )); + } + metrics.active_marker = active.is_some(); + let allocator = self + .metadata + .get_versioned( + RecordFamily::System, + &allocator_key(self.mount), + // The allocator is one current-only atomic projection. Its + // ordinary high-water reservations do not change the restore + // fence, and must not invalidate a long graph scan. Restore + // activation/downgrade and owner transitions are separately + // bracketed by the restore graph sequence. + Version::new(u64::MAX)?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore metrics found no allocator record".to_owned()) + })?; + let (_, _, epoch, fenced) = decode_allocator_state_with_restore_fence(&allocator.value.0)?; + if metrics.active_marker != fenced { + return Err(MetadError::Codec( + "restore metrics found an allocator/active-marker mismatch".to_owned(), + )); + } + let current_epoch = self.epoch.load(Ordering::Relaxed); + if epoch != current_epoch { + return Err(MetadError::StaleOwnerEpoch { + owner_epoch: epoch, + required_epoch: current_epoch, + }); + } + metrics.allocator_v2_fenced = fenced; + Ok(()) + } + + /// Validate the complete restore-private control graph. + /// + /// `verify_objects` adds object HEADs and effective-manifest borrower + /// proofs. The structural pass never mutates state and reports independent + /// breaks together instead of stopping at the first malformed row. + pub fn fsck_restore_state( + &self, + verify_objects: bool, + ) -> Result { + self.ensure_metadata_checkpoint_install_stable()?; + // System rows are current-only, so the captured version must be paired + // with the restore-specific writer sequence. Unrelated namespace writes + // and the two scheduler-only cursors do not invalidate this proof. + // Object verification additionally holds the physical-reclaim gate so + // a body cannot disappear between its metadata proof and HEAD. + let _object_gc_guard = if verify_objects { + Some( + self.object_gc_gate + .lock() + .unwrap_or_else(|error| error.into_inner()), + ) + } else { + None + }; + for _ in 0..3 { + let Some(sequence) = self.restore_graph_read_sequence() else { + std::thread::yield_now(); + continue; + }; + let report = self.fsck_restore_state_at_version(verify_objects); + if self.restore_graph_read_is_stable(sequence) { + self.ensure_owner_epoch_current()?; + return report; + } + } + Err(MetadError::Codec( + "restore fsck raced a restore graph write; retry the scan".to_owned(), + )) + } + + fn for_each_restore_system_row( + &self, + prefix: Vec, + version: Version, + mut visit: F, + ) -> Result<(), MetadError> + where + F: FnMut(&crate::command::ScanItem) -> Result<(), MetadError>, + { + let mut start_after = None; + loop { + let page = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: RESTORE_FSCK_PAGE_ROWS, + purpose: ReadPurpose::WritePlanLocal, + })?; + if page.is_empty() { + return Ok(()); + } + for row in &page { + visit(row)?; + } + let reached_tail = page.len() < RESTORE_FSCK_PAGE_ROWS; + start_after = page.last().map(|row| row.key.clone()); + if reached_tail { + return Ok(()); + } + } + } + + fn fsck_restore_state_at_version( + &self, + verify_objects: bool, + ) -> Result { + let version = self.read_version()?; + let mut report = RestoreFsckReport { + metrics: self.collect_restore_metrics_at(version, RESTORE_FSCK_PAGE_ROWS, false)?, + ..RestoreFsckReport::default() + }; + let private_rows = report + .metrics + .control_rows + .values() + .try_fold(0_usize, |total, count| total.checked_add(*count)) + .ok_or_else(|| MetadError::Codec("restore fsck row count overflow".to_owned()))?; + self.fsck_restore_allocator_fence_streaming(version, private_rows, &mut report)?; + + let operation_prefix = super::restore::restore_control_keyspaces(self.mount) + .into_iter() + .find_map(|(name, prefix)| (name == "operation").then_some(prefix)) + .expect("restore operation keyspace is registered"); + let operation_row_prefix = operation_prefix.clone(); + let mut operations = HashMap::<[u8; 32], VersionedRestoreOperation>::new(); + self.for_each_restore_system_row(operation_prefix, version, |row| { + match super::restore::decode_restore_operation(&row.value.0) { + Ok(operation) => { + update_restore_operation_metrics(&mut report.metrics, &operation); + let keyed_digest = row + .key + .strip_prefix(operation_row_prefix.as_slice()) + .filter(|suffix| suffix.len() == 32) + .and_then(|suffix| <[u8; 32]>::try_from(suffix).ok()); + match keyed_digest { + Some(digest) => { + if let Err(error) = + super::restore::validate_restore_operation_identity( + self.mount, + &digest, + &operation, + ) + { + push_restore_issue( + &mut report, + "operation_identity_mismatch", + format!( + "restore operation is not bound to its durable request identity: {error}" + ), + Some(&operation), + ); + } + } + None => push_restore_issue( + &mut report, + "operation_key_mismatch", + "restore operation key has an invalid identity", + Some(&operation), + ), + } + if operations.len() >= MAX_RESTORE_FSCK_OPERATIONS + && !operations.contains_key(&operation.operation_digest) + { + return Err(MetadError::RestoreResourceLimit { + resource: "restore fsck operations".to_owned(), + limit: MAX_RESTORE_FSCK_OPERATIONS as u64, + actual: operations.len().saturating_add(1) as u64, + }); + } + if operations + .insert( + operation.operation_digest, + VersionedRestoreOperation { + operation: operation.clone(), + }, + ) + .is_some() + { + push_restore_issue( + &mut report, + "duplicate_operation", + "duplicate restore operation digest", + Some(&operation), + ); + } + } + Err(error) => push_restore_raw_issue( + &mut report, + "operation_decode_failed", + format!("restore operation row cannot be decoded: {error}"), + ), + } + Ok(()) + })?; + + for versioned in operations.values() { + self.fsck_restore_operation_streaming(versioned, version, verify_objects, &mut report)?; + } + self.fsck_restore_index_state(version, &operations, &mut report)?; + self.fsck_restore_orphan_rows_streaming(version, &operations, verify_objects, &mut report)?; + Ok(report) + } + + fn fsck_restore_index_state( + &self, + version: Version, + operations: &HashMap<[u8; 32], VersionedRestoreOperation>, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + // The index module owns both its logical codecs and physical MVCC + // envelope. Consume its strict inspection result rather than decoding + // either representation in fsck. + let inspection = self.inspect_restore_index_state(version)?; + report.metrics.index_rows = inspection + .counts + .values() + .copied() + .fold(0_usize, usize::saturating_add); + for message in inspection.unowned_issues { + push_restore_raw_issue(report, "index_row_unowned", message); + } - for row in inode_rows { - let attr = decode_inode_attr(&row.value.0) - .map_err(|err| MetadError::Codec(err.to_string()))?; - if attr.file_type != FileType::File { + let mut operation_by_ref_set = + HashMap::>::new(); + for versioned in operations.values() { + operation_by_ref_set + .entry(versioned.operation.ref_set_id) + .or_default() + .push(&versioned.operation); + } + let mut inspected_ref_sets = HashSet::new(); + for (ref_set_id, ref_set) in inspection.ref_sets { + inspected_ref_sets.insert(ref_set_id); + let owners = operation_by_ref_set + .get(&ref_set_id) + .map_or(&[][..], Vec::as_slice); + let operation = match owners { + [operation] => Some(*operation), + [] => { + push_restore_raw_issue( + report, + "orphan_index_ref_set", + format!("restore index ref-set {ref_set_id} has no operation"), + ); + None + } + _ => { + push_restore_raw_issue( + report, + "ambiguous_index_ref_set", + format!( + "restore index ref-set {ref_set_id} is owned by {} operations", + owners.len() + ), + ); + None + } + }; + if let Some(operation) = operation { + if ref_set.operation_digests.len() != 1 + || !ref_set + .operation_digests + .contains(&operation.operation_digest) + { + push_restore_issue( + report, + "index_operation_identity_mismatch", + format!( + "restore index ref-set {ref_set_id} is not owned exclusively by its operation" + ), + Some(operation), + ); + } + for (kind, identity) in [ + ("seal", ref_set.seal_identity.as_ref()), + ("Complete marker", ref_set.complete_identity.as_ref()), + ] { + if identity.is_some_and(|identity| { + identity.operation_digest != operation.operation_digest + || identity.initialization_digest != operation.initialization_digest + || identity.incarnation != operation.created_version + }) { + push_restore_issue( + report, + "index_durable_identity_mismatch", + format!( + "restore index ref-set {ref_set_id} {kind} does not match operation/incarnation" + ), + Some(operation), + ); + } + } + } + for message in ref_set.closure_issues { + push_restore_issue(report, "index_closure_mismatch", message, operation); + } + let seal_count = ref_set.counts.get("index_seal").copied().unwrap_or(0); + let complete_count = ref_set.counts.get("index_complete").copied().unwrap_or(0); + let Some(operation) = operation else { + continue; + }; + let expected = match operation.state { + super::restore::RestoreOperationState::ReadyToAttach => Some((1, 0)), + super::restore::RestoreOperationState::Complete => Some((1, 1)), + super::restore::RestoreOperationState::Preparing => { + if complete_count == 0 && seal_count <= 1 { + None + } else { + Some((seal_count.min(1), 0)) + } + } + super::restore::RestoreOperationState::Cleaning + | super::restore::RestoreOperationState::Discarding => { + if complete_count == 0 && seal_count <= 1 { + None + } else { + Some((seal_count.min(1), 0)) + } + } + // Release is paged and may already have removed either marker. + super::restore::RestoreOperationState::Releasing => { + if seal_count <= 1 && complete_count <= 1 { + None + } else { + Some((seal_count.min(1), complete_count.min(1))) + } + } + }; + if let Some((expected_seal, expected_complete)) = expected { + if seal_count != expected_seal || complete_count != expected_complete { + push_restore_issue( + report, + "index_visibility_state_mismatch", + format!( + "restore index ref-set {ref_set_id} has seal/Complete counts {seal_count}/{complete_count}, expected {expected_seal}/{expected_complete} for {:?}", + operation.state + ), + Some(operation), + ); + } + } + } + for (ref_set_id, owners) in operation_by_ref_set { + if owners.len() != 1 { + if !inspected_ref_sets.contains(&ref_set_id) { + push_restore_raw_issue( + report, + "ambiguous_operation_ref_set", + format!( + "restore ref-set {ref_set_id} is claimed by {} operations", + owners.len() + ), + ); + } continue; } - let Some(body) = self.body_descriptor(attr.inode)? else { - continue; // a file with no body yet has nothing to verify + if inspected_ref_sets.contains(&ref_set_id) { + continue; + } + let operation = owners[0]; + if matches!( + operation.state, + super::restore::RestoreOperationState::ReadyToAttach + | super::restore::RestoreOperationState::Complete + ) { + push_restore_issue( + report, + "index_visibility_state_missing", + format!( + "restore index ref-set {ref_set_id} has no seal/Complete state for {:?}", + operation.state + ), + Some(operation), + ); + } + } + Ok(()) + } + + /// Remove permanent late-PUT tombstones after deployment has globally + /// quiesced restore writers. The caller holds `restore_gate` followed by + /// `object_gc_gate`; `allocator_gate` is deliberately not held because + /// each CAS needs a normally reserved commit version. + fn drain_restore_init_tombstones_quiesced_locked(&self) -> Result<(), RestoreDowngradeError> { + loop { + let read_version = self.read_version()?; + let Some(row) = self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: super::restore::restore_init_upload_tombstone_prefix(self.mount), + start_after: None, + version: read_version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .map_err(MetadError::from)? + .into_iter() + .next() + else { + return Ok(()); + }; + let tombstone = + super::restore::validate_restore_init_upload_tombstone_row(self.mount, &row)?; + self.delete_restore_init_object_range( + tombstone.inode, + tombstone.generation, + tombstone.size, + )?; + if !self.restore_init_object_range_absent( + tombstone.inode, + tombstone.generation, + tombstone.size, + )? { + return Err(RestoreDowngradeError::Metadata(MetadError::Codec( + "restore initialization object reappeared during quiesced drain".to_owned(), + ))); + } + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-drain-init-tombstone", + self.mount, + tombstone.inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).map_err(MetadError::from)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: row.key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }], + mutations: vec![delete_mutation(RecordFamily::System, row.key)], + watch: Vec::new(), }; - report.files_scanned += 1; + match self.commit_metadata(command) { + Ok(_) => {} + Err(MetadError::Metadata(MetadataError::PredicateFailed)) => { + return Err(RestoreDowngradeError::ConcurrentMutation); + } + Err(error) => return Err(error.into()), + } + } + } - let manifest_rows = self.metadata.scan(ScanRequest { - family: RecordFamily::ChunkManifest, - prefix: chunk_manifest_prefix(self.mount, attr.inode, body.generation), + fn drain_restore_cursor_locked( + &self, + cursor_key: Vec, + item_prefix: Vec, + keyspace: &str, + format: RestoreCursorFormat, + ) -> Result<(), RestoreDowngradeError> { + let read_version = self.read_version()?; + let remaining = self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: item_prefix.clone(), start_after: None, + version: read_version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .map_err(MetadError::from)?; + if !remaining.is_empty() { + return Err(RestoreDowngradeError::PrivateStatePresent { + keyspace: keyspace.to_owned(), + observed_rows: remaining.len(), + }); + } + let Some(cursor) = self + .metadata + .get_versioned( + RecordFamily::System, + &cursor_key, + read_version, + ReadPurpose::WritePlanLocal, + ) + .map_err(MetadError::from)? + else { + return Ok(()); + }; + match format { + RestoreCursorFormat::RawKey => { + if cursor.value.0.len() > super::restore::MAX_RESTORE_PATH_BYTES + || (!cursor.value.0.is_empty() && !cursor.value.0.starts_with(&item_prefix)) + { + return Err(RestoreDowngradeError::Metadata(MetadError::Codec(format!( + "restore drain cursor for {keyspace} changed identity" + )))); + } + } + RestoreCursorFormat::ReleaseWorker => { + super::restore::decode_restore_release_worker_cursor_at_version( + self.mount, + &cursor.value.0, + read_version, + )?; + } + } + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-drain-worker-cursor", + self.mount, + InodeId::root(), version, - limit: 0, - purpose: ReadPurpose::UserStrong, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).map_err(MetadError::from)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: cursor_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: item_prefix, + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: cursor_key.clone(), + predicate: Predicate::VersionEquals(cursor.version), + }, + ], + mutations: vec![delete_mutation(RecordFamily::System, cursor_key)], + watch: Vec::new(), + }; + match self.commit_metadata(command) { + Ok(_) => Ok(()), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) => { + Err(RestoreDowngradeError::ConcurrentMutation) + } + Err(error) => Err(error.into()), + } + } + + /// Explicitly drain all restore-to-fork private state and remove the cold + /// downgrade fence. + /// + /// The deployment must first disable the capability and globally quiesce + /// every restore writer. This call does not infer or alter either external + /// condition. On success the caller must run both + /// [`NoKvFs::fsck_restore_state`] and + /// [`NoKvFs::fsck_object_references`] in [`FsckMode::Full`], require clean + /// reports, and create a fresh metadata checkpoint before starting a + /// pre-restore binary. The typed acknowledgements and returned requirement + /// flags make every external safety dependency explicit at the call site. + pub fn drain_restore_to_fork_v1( + &self, + _capability_disabled: RestoreCapabilityDisabled, + _writers_quiesced: RestoreWritersQuiesced, + ) -> Result { + let _restore_guard = self + .restore_gate + .lock() + .unwrap_or_else(|error| error.into_inner()); + let _object_gc_guard = self + .object_gc_gate + .lock() + .unwrap_or_else(|error| error.into_inner()); + self.ensure_owner_epoch_current()?; + + self.drain_restore_init_tombstones_quiesced_locked()?; + self.drain_restore_cursor_locked( + super::restore::restore_init_upload_tombstone_cursor_key(self.mount), + super::restore::restore_init_upload_tombstone_prefix(self.mount), + "init_upload_tombstone", + RestoreCursorFormat::RawKey, + )?; + self.drain_restore_cursor_locked( + super::restore::restore_release_cursor_key(self.mount), + super::restore::restore_release_job_prefix(self.mount), + "release_job", + RestoreCursorFormat::ReleaseWorker, + )?; + + // Allocator reservations may race before this lock. Allocate a version, + // then reject/retry if its predecessor did not include the exact current + // allocator row. Never call next_version while holding allocator_gate. + for _ in 0..8 { + let commit_version = self.next_version()?; + let read_version = predecessor(commit_version).map_err(MetadError::from)?; + let allocator_guard = self + .allocator_gate + .lock() + .unwrap_or_else(|error| error.into_inner()); + self.ensure_owner_epoch_current()?; + let max_version = Version::new(u64::MAX).map_err(MetadError::from)?; + let allocator_key = allocator_key(self.mount); + let current_allocator = self + .metadata + .get_versioned( + RecordFamily::System, + &allocator_key, + max_version, + ReadPurpose::WritePlanLocal, + ) + .map_err(MetadError::from)? + .ok_or_else(|| { + RestoreDowngradeError::Metadata(MetadError::Codec( + "allocator is missing during restore downgrade".to_owned(), + )) + })?; + if current_allocator.version.get() > read_version.get() { + drop(allocator_guard); + continue; + } + return self.finish_restore_downgrade_locked( + read_version, + commit_version, + allocator_key, + current_allocator, + ); + } + Err(RestoreDowngradeError::ConcurrentMutation) + } + + fn finish_restore_downgrade_locked( + &self, + read_version: Version, + commit_version: Version, + allocator_key: Vec, + allocator: crate::command::ReadItem, + ) -> Result { + let active_key = super::restore::restore_active_key(self.mount); + let active = self + .metadata + .get_versioned( + RecordFamily::System, + &active_key, + read_version, + ReadPurpose::WritePlanLocal, + ) + .map_err(MetadError::from)?; + if active + .as_ref() + .is_some_and(|item| item.value.0 != [super::restore::RESTORE_FORMAT_VERSION]) + { + return Err(RestoreDowngradeError::Metadata(MetadError::Codec( + "restore active marker has an invalid value".to_owned(), + ))); + } + let (last_commit_version, next_inode, allocator_epoch, fenced) = + decode_allocator_state_with_restore_fence(&allocator.value.0)?; + if active.is_some() != fenced { + return Err(RestoreDowngradeError::Metadata(MetadError::Codec( + "restore active marker and allocator v2 fence disagree".to_owned(), + ))); + } + if allocator_epoch != self.epoch.load(Ordering::Relaxed) { + return Err(RestoreDowngradeError::Metadata( + MetadError::StaleOwnerEpoch { + owner_epoch: allocator_epoch, + required_epoch: self.epoch.load(Ordering::Relaxed), + }, + )); + } + + let control_keyspaces = super::restore::restore_control_keyspaces(self.mount); + let index_keyspaces = super::restore_index::restore_index_private_keyspaces(self.mount); + for (name, prefix) in control_keyspaces.iter().chain(index_keyspaces.iter()) { + let rows = self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: None, + version: read_version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .map_err(MetadError::from)?; + if !rows.is_empty() { + return Err(RestoreDowngradeError::PrivateStatePresent { + keyspace: (*name).to_owned(), + observed_rows: rows.len(), + }); + } + } + let empty_predicates = restore_downgrade_empty_predicates(self.mount); + + let Some(active) = active else { + return Ok(RestoreDowngradeOutcome { + already_drained: true, + commit_version: None, + full_fsck_required: true, + metadata_checkpoint_required: true, + }); + }; + let claim_key = object_gc_claim_key(self.mount); + let claim = self + .metadata + .get_versioned( + RecordFamily::System, + &claim_key, + read_version, + ReadPurpose::WritePlanLocal, + ) + .map_err(MetadError::from)? + .ok_or_else(|| { + RestoreDowngradeError::Metadata(MetadError::Codec( + "object-GC claim is missing during restore downgrade".to_owned(), + )) })?; - for manifest_row in manifest_rows { - if chunk_index_from_manifest_key(&manifest_row.key)? == BODY_SUMMARY_CHUNK_INDEX { + if decode_object_gc_claim(self.mount, &claim.value.0)? != ObjectGcClaim::Open { + return Err(RestoreDowngradeError::Metadata(MetadError::Codec( + "object-GC claim is not Open during restore downgrade".to_owned(), + ))); + } + let persisted_last_version = last_commit_version + .max(self.reserved_version.load(Ordering::Relaxed)) + .max(commit_version.get()); + let persisted_next_inode = next_inode.max(self.reserved_next_inode.load(Ordering::Relaxed)); + InodeId::new(persisted_next_inode).map_err(MetadError::from)?; + let allocator_v1 = encode_allocator_state( + persisted_last_version, + persisted_next_inode, + allocator_epoch, + ); + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::System, + key: active_key.clone(), + predicate: Predicate::VersionEquals(active.version), + }, + PredicateRef { + family: RecordFamily::System, + key: allocator_key.clone(), + predicate: Predicate::VersionEquals(allocator.version), + }, + PredicateRef { + family: RecordFamily::System, + key: claim_key.clone(), + predicate: Predicate::VersionEquals(claim.version), + }, + ]; + predicates.extend(empty_predicates); + let command = MetadataCommand { + request_id: request_id( + b"restore-downgrade-v1", + self.mount, + InodeId::root(), + commit_version, + ), + kind: CommandKind::ReserveAllocator, + read_version, + commit_version, + primary_family: RecordFamily::System, + primary_key: allocator_key.clone(), + predicates, + mutations: vec![ + Mutation { + family: RecordFamily::System, + key: allocator_key, + op: MutationOp::Put, + value: Some(Value(allocator_v1)), + }, + delete_mutation(RecordFamily::System, active_key), + Mutation { + family: RecordFamily::System, + key: claim_key, + op: MutationOp::Put, + value: Some(Value(encode_object_gc_claim(&ObjectGcClaim::Open)?)), + }, + ], + watch: Vec::new(), + }; + match self.commit_metadata(command) { + Ok(result) => Ok(RestoreDowngradeOutcome { + already_drained: false, + commit_version: Some(result.commit_version.get()), + full_fsck_required: true, + metadata_checkpoint_required: true, + }), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) => { + Err(RestoreDowngradeError::ConcurrentMutation) + } + Err(error) => Err(error.into()), + } + } + + fn fsck_restore_allocator_fence_streaming( + &self, + version: Version, + private_rows: usize, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + match self.metadata.get( + RecordFamily::System, + &super::restore::restore_activation_fence_key(self.mount), + version, + ReadPurpose::WritePlanLocal, + )? { + Some(value) if value.0 == [super::restore::RESTORE_FORMAT_VERSION] => {} + Some(_) => push_restore_raw_issue( + report, + "activation_fence_invalid", + "restore activation fence has an invalid value", + ), + None => push_restore_raw_issue( + report, + "activation_fence_missing", + "restore activation fence is missing", + ), + } + let active = self.metadata.get_versioned( + RecordFamily::System, + &super::restore::restore_active_key(self.mount), + version, + ReadPurpose::WritePlanLocal, + )?; + report.metrics.active_marker = active.is_some(); + if active + .as_ref() + .is_some_and(|item| item.value.0 != [super::restore::RESTORE_FORMAT_VERSION]) + { + push_restore_raw_issue( + report, + "active_marker_invalid", + "restore active marker has an invalid value", + ); + } + let allocator = self.metadata.get_versioned( + RecordFamily::System, + &allocator_key(self.mount), + // Sample the atomic current fence projection independently from + // the captured graph version. System has no history, so an + // unrelated allocator high-water reservation would otherwise make + // the prior row appear missing halfway through fsck. + Version::new(u64::MAX)?, + ReadPurpose::WritePlanLocal, + )?; + match allocator { + Some(item) => match decode_allocator_state_with_restore_fence(&item.value.0) { + Ok((_, _, epoch, fenced)) => { + report.metrics.allocator_v2_fenced = fenced; + if fenced != report.metrics.active_marker { + push_restore_raw_issue( + report, + "allocator_marker_mismatch", + "restore active marker and allocator v2 fence disagree", + ); + } + if fenced && epoch != self.epoch.load(Ordering::Relaxed) { + push_restore_raw_issue( + report, + "allocator_owner_epoch_mismatch", + format!( + "restore-fenced allocator epoch {epoch} differs from owner epoch {}", + self.epoch.load(Ordering::Relaxed) + ), + ); + } + } + Err(error) => push_restore_raw_issue( + report, + "allocator_decode_failed", + format!("allocator cannot be decoded: {error}"), + ), + }, + None => { + push_restore_raw_issue(report, "allocator_missing", "allocator record is missing") + } + } + if private_rows != 0 + && !(report.metrics.active_marker && report.metrics.allocator_v2_fenced) + { + push_restore_raw_issue( + report, + "private_state_without_downgrade_fence", + format!( + "{private_rows} restore-private row(s) exist without both active marker and allocator v2 fence" + ), + ); + } + Ok(()) + } + + fn fsck_restore_operation_streaming( + &self, + versioned: &VersionedRestoreOperation, + read_version: Version, + verify_objects: bool, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + let operation = &versioned.operation; + self.fsck_restore_claim_and_root(operation, read_version, report)?; + self.fsck_restore_temporary_binding(operation, read_version, report)?; + self.fsck_restore_jobs_streaming(operation, read_version, report)?; + self.fsck_restore_staging_members_streaming(operation, read_version, report)?; + self.fsck_restore_base_references(operation, read_version, verify_objects, report)?; + Ok(()) + } + + fn fsck_restore_claim_and_root( + &self, + operation: &super::restore::RestoreOperation, + version: Version, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + let claim_key = + super::restore::restore_destination_claim_key(self.mount, &operation.destination_path); + match self.metadata.get( + RecordFamily::System, + &claim_key, + version, + ReadPurpose::WritePlanLocal, + )? { + Some(value) if value.0 == operation.operation_digest => {} + Some(_) => push_restore_issue( + report, + "destination_claim_mismatch", + "destination claim names another operation", + Some(operation), + ), + None => push_restore_issue( + report, + "destination_claim_missing", + "restore operation has no destination claim", + Some(operation), + ), + } + let root_key = + super::restore::restore_root_index_key(self.mount, operation.destination_root); + match self.metadata.get( + RecordFamily::System, + &root_key, + version, + ReadPurpose::WritePlanLocal, + )? { + Some(value) if value.0 == operation.operation_digest => {} + Some(_) => push_restore_issue( + report, + "root_index_mismatch", + "restore root index names another operation", + Some(operation), + ), + None => push_restore_issue( + report, + "root_index_missing", + "restore operation has no root index", + Some(operation), + ), + } + Ok(()) + } + + fn fsck_restore_temporary_binding( + &self, + operation: &super::restore::RestoreOperation, + version: Version, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + let key = fork_binding_key(self.mount, operation.destination_root); + let binding = self.metadata.get_versioned( + RecordFamily::ForkBinding, + &key, + version, + ReadPurpose::WritePlanLocal, + )?; + let requires_binding = matches!( + operation.state, + super::restore::RestoreOperationState::Preparing + | super::restore::RestoreOperationState::ReadyToAttach + | super::restore::RestoreOperationState::Cleaning + | super::restore::RestoreOperationState::Discarding + ); + match (requires_binding, binding) { + (true, None) => push_restore_issue( + report, + "temporary_binding_missing", + "detached restore operation has no temporary ForkBinding", + Some(operation), + ), + (false, Some(_)) => push_restore_issue( + report, + "temporary_binding_after_attach", + "attached/releasing restore still has a temporary ForkBinding", + Some(operation), + ), + (true, Some(item)) => match crate::layout::decode_fork_binding(&item.value.0) { + Ok(binding) + if binding.fork_root == operation.destination_root + && binding.source_root == operation.source_root + && binding.snapshot_id == operation.snapshot_id + && binding.pinned_read_version == operation.read_version + && binding.created_version == operation.created_version + && item.version.get() == operation.created_version => {} + Ok(_) => push_restore_issue( + report, + "temporary_binding_mismatch", + "temporary ForkBinding identity does not match operation", + Some(operation), + ), + Err(error) => push_restore_issue( + report, + "temporary_binding_decode_failed", + format!("temporary ForkBinding cannot be decoded: {error}"), + Some(operation), + ), + }, + (false, None) => {} + } + Ok(()) + } + + fn fsck_restore_jobs_streaming( + &self, + operation: &super::restore::RestoreOperation, + version: Version, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + let cleanup = self + .metadata + .get( + RecordFamily::System, + &super::restore::restore_cleanup_job_key(self.mount, operation.ref_set_id), + version, + ReadPurpose::WritePlanLocal, + )? + .map(|value| super::restore::decode_restore_cleanup_job(&value.0)) + .transpose()?; + let expects_cleanup = matches!( + operation.state, + super::restore::RestoreOperationState::Cleaning + | super::restore::RestoreOperationState::Discarding + ); + if expects_cleanup != cleanup.is_some() { + push_restore_issue( + report, + if expects_cleanup { + "cleanup_job_missing" + } else { + "unexpected_cleanup_job" + }, + "restore cleanup job does not match operation state", + Some(operation), + ); + } else if cleanup.is_some_and(|job| job.operation_digest != operation.operation_digest) { + push_restore_issue( + report, + "cleanup_job_owner_mismatch", + "restore cleanup job has the wrong operation owner", + Some(operation), + ); + } + + let release = self + .metadata + .get( + RecordFamily::System, + &super::restore::restore_release_job_key(self.mount, operation.ref_set_id), + version, + ReadPurpose::WritePlanLocal, + )? + .map(|value| super::restore::decode_restore_release_job(&value.0)) + .transpose()?; + let expects_release = operation.state == super::restore::RestoreOperationState::Releasing; + if expects_release != release.is_some() { + push_restore_issue( + report, + if expects_release { + "release_job_missing" + } else { + "unexpected_release_job" + }, + "restore release job does not match operation state", + Some(operation), + ); + } else if release.is_some_and(|job| job.operation_digest != operation.operation_digest) { + push_restore_issue( + report, + "release_job_owner_mismatch", + "restore release job has the wrong operation owner", + Some(operation), + ); + } + Ok(()) + } + + fn fsck_restore_staging_members_streaming( + &self, + operation: &super::restore::RestoreOperation, + version: Version, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + let release = self + .metadata + .get( + RecordFamily::System, + &super::restore::restore_release_job_key(self.mount, operation.ref_set_id), + version, + ReadPurpose::WritePlanLocal, + )? + .map(|value| super::restore::decode_restore_release_job(&value.0)) + .transpose()?; + let prefix = + super::restore::restore_staging_member_prefix(self.mount, operation.ref_set_id); + self.for_each_restore_system_row(prefix, version, |row| { + match super::restore::decode_restore_staging_member(&row.value.0) { + Ok(member) => { + let expected = super::restore::restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ); + if row.key != expected || member.operation_digest != operation.operation_digest + { + push_restore_issue( + report, + "staging_member_identity_mismatch", + "restore staging member does not match key/operation", + Some(operation), + ); + return Ok(()); + } + for (code, message, key) in [ + ( + "staging_inverse_missing_or_mismatched", + "restore staging member has no matching inode inverse", + super::restore::restore_staging_inode_key( + self.mount, + member.destination_inode, + ), + ), + ( + "staging_inverse_owner_missing_or_mismatched", + "restore staging member has no matching ref-set-first inverse owner", + super::restore::restore_staging_inverse_owner_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ), + ), + ] { + let matches = self + .metadata + .get( + RecordFamily::System, + &key, + version, + ReadPurpose::WritePlanLocal, + )? + .and_then(|value| { + super::restore::decode_restore_staging_inverse(&value.0).ok() + }) + .is_some_and(|(digest, ref_set)| { + digest == operation.operation_digest + && ref_set == operation.ref_set_id + }); + if !matches { + push_restore_issue(report, code, message, Some(operation)); + } + } + if !member.manifest_cursor.is_empty() { + if operation.state + != super::restore::RestoreOperationState::Releasing + || !matches!( + release.as_ref(), + Some(job) + if job.operation_digest == operation.operation_digest + && job.ref_set_id == operation.ref_set_id + && job.phase + == super::restore::RestoreReleasePhase::Members + ) + { + push_restore_issue( + report, + "staging_manifest_cursor_state_mismatch", + "restore manifest cursor exists outside the member-release phase", + Some(operation), + ); + } + let manifest_prefix = inode_key(self.mount, member.destination_inode); + if member.manifest_cursor.len() != manifest_prefix.len() + 16 + || !member.manifest_cursor.starts_with(&manifest_prefix) + { + push_restore_issue( + report, + "staging_manifest_cursor_shape_invalid", + "restore manifest cursor is not a physical member manifest key", + Some(operation), + ); + return Ok(()); + } + let generation = u64::from_be_bytes( + member.manifest_cursor[manifest_prefix.len() + ..manifest_prefix.len() + 8] + .try_into() + .expect("validated generation width"), + ); + let chunk_index = u64::from_be_bytes( + member.manifest_cursor[manifest_prefix.len() + 8..] + .try_into() + .expect("validated chunk-index width"), + ); + if generation == 0 || chunk_index == BODY_SUMMARY_CHUNK_INDEX { + push_restore_issue( + report, + "staging_manifest_cursor_identity_invalid", + "restore manifest cursor has an invalid generation/chunk identity", + Some(operation), + ); + return Ok(()); + } + let Some(manifest) = self.metadata.get( + RecordFamily::ChunkManifest, + &member.manifest_cursor, + version, + ReadPurpose::RestoreStaging, + )? + else { + push_restore_issue( + report, + "staging_manifest_cursor_missing", + "restore manifest cursor points to a missing manifest", + Some(operation), + ); + return Ok(()); + }; + let manifest = match decode_chunk_manifest(&manifest.0) { + Ok(manifest) => manifest, + Err(error) => { + push_restore_issue( + report, + "staging_manifest_cursor_decode_failed", + format!( + "restore manifest cursor cannot decode its manifest: {error}" + ), + Some(operation), + ); + return Ok(()); + } + }; + match self.restore_owned_manifest_blocks( + member.destination_inode, + chunk_index, + &manifest, + ) { + Ok(blocks) + if usize::try_from(member.manifest_block_cursor) + .is_ok_and(|cursor| cursor > 0 && cursor < blocks.len()) => {} + Ok(_) => push_restore_issue( + report, + "staging_manifest_block_cursor_out_of_range", + "restore manifest block cursor is not a strict partial-page ordinal", + Some(operation), + ), + Err(error) => push_restore_issue( + report, + "staging_manifest_cursor_manifest_invalid", + format!( + "restore manifest cursor has invalid release blocks: {error}" + ), + Some(operation), + ), + } + } + } + Err(error) => push_restore_issue( + report, + "staging_member_decode_failed", + format!("restore staging member cannot be decoded: {error}"), + Some(operation), + ), + } + Ok(()) + }) + } + + fn fsck_restore_base_references( + &self, + operation: &super::restore::RestoreOperation, + version: Version, + verify_objects: bool, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + let prefix = super::restore::restore_base_owner_prefix(self.mount, operation.ref_set_id); + let mut start_after = None; + let mut owner_count = 0_u64; + let mut owner_accumulator = super::restore_gc::initial_restore_base_reference_digest(); + loop { + let page = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: RESTORE_FSCK_PAGE_ROWS, + purpose: ReadPurpose::WritePlanLocal, + })?; + if page.is_empty() { + break; + } + for row in &page { + let reference = match super::restore_gc::decode_restore_base_reference(&row.value.0) + { + Ok(reference) => reference, + Err(error) => { + push_restore_issue( + report, + "base_owner_decode_failed", + format!("restore base owner cannot be decoded: {error}"), + Some(operation), + ); + continue; + } + }; + let object_digest: [u8; 32] = + Sha256::digest(reference.object_key.as_bytes()).into(); + let expected_owner = super::restore::restore_base_owner_key( + self.mount, + operation.ref_set_id, + &object_digest, + reference.borrower_inode, + reference.borrower_generation, + ); + if row.key != expected_owner + || reference.operation_digest != operation.operation_digest + || reference.ref_set_id != operation.ref_set_id + { + push_restore_issue( + report, + "base_owner_identity_mismatch", + "restore base owner does not match key/operation", + Some(operation), + ); continue; } - let manifest = decode_chunk_manifest(&manifest_row.value.0) - .map_err(|err| MetadError::Codec(err.to_string()))?; - for block in manifest.slices.iter().flat_map(|slice| slice.blocks.iter()) { - report.blocks_checked += 1; - let key = ObjectKey::new(block.object_key.clone())?; - if self.objects.head(&key)?.is_none() { - report.dangling.push(DanglingBlock { - inode: attr.inode.get(), - generation: body.generation, - object_key: block.object_key.clone(), - }); + owner_count = owner_count.checked_add(1).ok_or_else(|| { + MetadError::Codec("restore fsck base-owner count overflow".to_owned()) + })?; + owner_accumulator = super::restore_gc::extend_restore_base_reference_digest( + owner_accumulator, + &reference, + )?; + self.fsck_restore_base_reference_inverses( + operation, + version, + &reference, + &object_digest, + report, + )?; + if verify_objects { + report.borrowed_objects_checked = + report.borrowed_objects_checked.saturating_add(1); + let key = ObjectKey::new(reference.object_key.clone())?; + match self.objects.head(&key)? { + None => report.dangling_borrowed_objects.push(DanglingBlock { + inode: reference.borrower_inode.get(), + generation: reference.borrower_generation, + object_key: reference.object_key.clone(), + }), + Some(info) if info.size != reference.size => { + report + .borrowed_object_size_mismatches + .push(MismatchedBlock { + inode: reference.borrower_inode.get(), + generation: reference.borrower_generation, + object_key: reference.object_key.clone(), + expected_size: reference.size, + actual_size: info.size, + }); + } + Some(_) => {} } } } + let reached_tail = page.len() < RESTORE_FSCK_PAGE_ROWS; + start_after = page.last().map(|row| row.key.clone()); + if reached_tail { + break; + } } - Ok(report) + + let seal_key = super::restore::restore_base_seal_key(self.mount, operation.ref_set_id); + let seal = self.metadata.get( + RecordFamily::System, + &seal_key, + version, + ReadPurpose::WritePlanLocal, + )?; + let requires_seal = matches!( + operation.state, + super::restore::RestoreOperationState::ReadyToAttach + | super::restore::RestoreOperationState::Complete + | super::restore::RestoreOperationState::Releasing + ); + match seal { + None if requires_seal => push_restore_issue( + report, + "base_seal_missing", + "sealed/attached restore has no base-reference seal", + Some(operation), + ), + Some(value) => match super::restore_gc::decode_restore_base_seal_record(&value.0) { + Ok(super::restore_gc::RestoreBaseSealRecord::Building(build)) => { + if build.operation_digest != operation.operation_digest + || build.initialization_digest != operation.initialization_digest + || build.ref_set_id != operation.ref_set_id + || build.incarnation != operation.created_version + { + push_restore_issue( + report, + "base_build_identity_mismatch", + "base-reference build identity does not match operation", + Some(operation), + ); + } + if build.reference_count != owner_count + || build.reference_digest != owner_accumulator + { + push_restore_issue( + report, + "base_build_closure_mismatch", + "base-reference build progress does not close over durable owner rows", + Some(operation), + ); + } + if requires_seal { + push_restore_issue( + report, + "base_seal_incomplete", + "attached/releasing restore still has a base-reference build cursor", + Some(operation), + ); + } + } + Ok(super::restore_gc::RestoreBaseSealRecord::Sealed(seal)) => { + if seal.operation_digest != operation.operation_digest + || seal.initialization_digest != operation.initialization_digest + || seal.ref_set_id != operation.ref_set_id + || seal.incarnation != operation.created_version + { + push_restore_issue( + report, + "base_seal_identity_mismatch", + "base-reference seal identity does not match operation", + Some(operation), + ); + } + if matches!( + operation.state, + super::restore::RestoreOperationState::Preparing + | super::restore::RestoreOperationState::ReadyToAttach + | super::restore::RestoreOperationState::Complete + ) { + let owner_digest = + super::restore_gc::finalize_restore_base_reference_digest( + owner_count, + owner_accumulator, + ); + if seal.reference_count != owner_count + || seal.reference_digest != owner_digest + { + push_restore_issue( + report, + "base_seal_closure_mismatch", + "base-reference seal count/digest does not close over owner rows", + Some(operation), + ); + } + } + // Only the detached ReadyToAttach tree is immutable. Once + // Complete, ordinary unlink/publish/link operations may + // legitimately remove or replace a borrower's current + // manifest while its sealed exact-reference rows remain + // as conservative lifetime protection until root release. + // Re-deriving the original seal from the live namespace at + // that point would reject valid post-attach mutations. + if operation.state == super::restore::RestoreOperationState::ReadyToAttach { + let (expected_count, expected_accumulator) = + self.fsck_restore_expected_base_references(operation, version, report)?; + let expected_digest = + super::restore_gc::finalize_restore_base_reference_digest( + expected_count, + expected_accumulator, + ); + if seal.reference_count != expected_count + || seal.reference_digest != expected_digest + { + push_restore_issue( + report, + "base_manifest_closure_mismatch", + "base-reference seal does not close over staging member manifests", + Some(operation), + ); + } + } + } + Err(error) => push_restore_issue( + report, + "base_seal_decode_failed", + format!("base-reference seal/progress cannot be decoded: {error}"), + Some(operation), + ), + }, + None => {} + } + Ok(()) + } + + fn fsck_restore_base_reference_inverses( + &self, + operation: &super::restore::RestoreOperation, + version: Version, + reference: &super::restore_gc::RestoreBaseReference, + object_digest: &[u8; 32], + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + let expected = super::restore_gc::RestoreBaseInverse { + operation_digest: operation.operation_digest, + ref_set_id: operation.ref_set_id, + object_digest: *object_digest, + borrower_inode: reference.borrower_inode, + borrower_generation: reference.borrower_generation, + }; + for (code, message, key) in [ + ( + "base_inverse_missing_or_mismatched", + "restore base owner has no matching inverse", + super::restore::restore_base_inverse_key( + self.mount, + object_digest, + operation.ref_set_id, + reference.borrower_inode, + reference.borrower_generation, + ), + ), + ( + "base_inverse_owner_missing_or_mismatched", + "restore base owner has no matching ref-set-first inverse owner", + super::restore::restore_base_inverse_owner_key( + self.mount, + operation.ref_set_id, + object_digest, + reference.borrower_inode, + reference.borrower_generation, + ), + ), + ] { + let matches = self + .metadata + .get( + RecordFamily::System, + &key, + version, + ReadPurpose::WritePlanLocal, + )? + .and_then(|value| super::restore_gc::decode_restore_base_inverse(&value.0).ok()) + .is_some_and(|inverse| inverse == expected); + if !matches { + push_restore_issue(report, code, message, Some(operation)); + } + } + Ok(()) + } + + fn fsck_restore_expected_base_references( + &self, + operation: &super::restore::RestoreOperation, + version: Version, + report: &mut RestoreFsckReport, + ) -> Result<(u64, [u8; 32]), MetadError> { + let prefix = + super::restore::restore_staging_member_prefix(self.mount, operation.ref_set_id); + let mut start_after = None; + let mut count = 0_u64; + let mut accumulator = super::restore_gc::initial_restore_base_reference_digest(); + loop { + let page = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: RESTORE_FSCK_PAGE_ROWS, + purpose: ReadPurpose::RestoreStaging, + })?; + if page.is_empty() { + break; + } + for row in &page { + let member = match super::restore::decode_restore_staging_member(&row.value.0) { + Ok(member) + if member.operation_digest == operation.operation_digest + && row.key + == super::restore::restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ) => + { + member + } + Ok(_) => { + push_restore_issue( + report, + "staging_member_identity_mismatch", + "restore staging member does not match key/operation", + Some(operation), + ); + continue; + } + Err(error) => { + push_restore_issue( + report, + "staging_member_decode_failed", + format!("restore staging member cannot be decoded: {error}"), + Some(operation), + ); + continue; + } + }; + let Some(layout) = self.restore_member_base_reference_layout(&member, version)? + else { + continue; + }; + let mut chunk_index = 0_u64; + loop { + for (_, reference) in self.restore_member_base_references_for_chunk( + operation, + &member, + &layout, + chunk_index, + version, + )? { + count = count.checked_add(1).ok_or_else(|| { + MetadError::Codec( + "restore fsck expected-reference count overflow".to_owned(), + ) + })?; + accumulator = super::restore_gc::extend_restore_base_reference_digest( + accumulator, + &reference, + )?; + } + if chunk_index == layout.end_chunk { + break; + } + chunk_index = chunk_index.checked_add(1).ok_or_else(|| { + MetadError::Codec( + "restore fsck expected-reference chunk overflow".to_owned(), + ) + })?; + } + } + let reached_tail = page.len() < RESTORE_FSCK_PAGE_ROWS; + start_after = page.last().map(|row| row.key.clone()); + if reached_tail { + break; + } + } + Ok((count, accumulator)) + } + + fn fsck_restore_orphan_rows_streaming( + &self, + version: Version, + operations: &HashMap<[u8; 32], VersionedRestoreOperation>, + verify_objects: bool, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + self.fsck_restore_cursor_row_streaming( + version, + "init_upload_tombstone_cursor", + super::restore::restore_init_upload_tombstone_cursor_key(self.mount), + super::restore::restore_init_upload_tombstone_prefix(self.mount), + RestoreCursorFormat::RawKey, + report, + )?; + self.fsck_restore_cursor_row_streaming( + version, + "release_cursor", + super::restore::restore_release_cursor_key(self.mount), + super::restore::restore_release_job_prefix(self.mount), + RestoreCursorFormat::ReleaseWorker, + report, + )?; + self.fsck_restore_claim_orphans_streaming(version, operations, report)?; + self.fsck_restore_staging_orphans_streaming(version, operations, report)?; + self.fsck_restore_base_orphans_streaming(version, operations, report)?; + self.fsck_restore_auxiliary_orphans_streaming(version, operations, verify_objects, report) + } + + fn fsck_restore_cursor_row_streaming( + &self, + _version: Version, + name: &str, + expected_key: Vec, + target_prefix: Vec, + format: RestoreCursorFormat, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + let current_version = Version::new(u64::MAX)?; + let mut count = 0_usize; + self.for_each_restore_system_row( + restore_control_prefix(self.mount, name), + current_version, + |row| { + count = count.saturating_add(1); + let value_valid = match format { + RestoreCursorFormat::RawKey => { + row.value.0.len() <= super::restore::MAX_RESTORE_PATH_BYTES + && (row.value.0.is_empty() || row.value.0.starts_with(&target_prefix)) + } + RestoreCursorFormat::ReleaseWorker => { + super::restore::decode_restore_release_worker_cursor_at_version( + self.mount, + &row.value.0, + row.version, + ) + .is_ok() + } + }; + if row.key != expected_key || !value_valid { + push_restore_raw_issue( + report, + "restore_cursor_invalid", + format!("restore {name} changed key or target-prefix identity"), + ); + } + Ok(()) + }, + )?; + if count > 1 { + push_restore_raw_issue( + report, + "restore_cursor_duplicate", + format!("restore {name} keyspace contains {count} rows"), + ); + } + Ok(()) + } + + fn fsck_restore_claim_orphans_streaming( + &self, + version: Version, + operations: &HashMap<[u8; 32], VersionedRestoreOperation>, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "destination_claim"), + version, + |row| { + let Ok(digest) = <[u8; 32]>::try_from(row.value.0.as_slice()) else { + push_restore_raw_issue( + report, + "destination_claim_decode_failed", + "destination claim value is not an operation digest", + ); + return Ok(()); + }; + match operations.get(&digest) { + Some(versioned) + if row.key + == super::restore::restore_destination_claim_key( + self.mount, + &versioned.operation.destination_path, + ) => {} + Some(versioned) => push_restore_issue( + report, + "destination_claim_key_mismatch", + "destination claim key does not match operation path", + Some(&versioned.operation), + ), + None => push_restore_raw_issue( + report, + "orphan_destination_claim", + "destination claim has no operation", + ), + } + Ok(()) + }, + )?; + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "root_index"), + version, + |row| { + let Ok(digest) = <[u8; 32]>::try_from(row.value.0.as_slice()) else { + push_restore_raw_issue( + report, + "root_index_decode_failed", + "root index value is not an operation digest", + ); + return Ok(()); + }; + match operations.get(&digest) { + Some(versioned) + if row.key + == super::restore::restore_root_index_key( + self.mount, + versioned.operation.destination_root, + ) => {} + Some(versioned) => push_restore_issue( + report, + "root_index_key_mismatch", + "root index key does not match operation destination root", + Some(&versioned.operation), + ), + None => push_restore_raw_issue( + report, + "orphan_root_index", + "root index has no operation", + ), + } + Ok(()) + }, + ) + } + + fn fsck_restore_staging_orphans_streaming( + &self, + version: Version, + operations: &HashMap<[u8; 32], VersionedRestoreOperation>, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "staging_member"), + version, + |row| { + match super::restore::decode_restore_staging_member(&row.value.0) { + Ok(member) => match operations.get(&member.operation_digest) { + Some(versioned) + if row.key + == super::restore::restore_staging_member_key( + self.mount, + versioned.operation.ref_set_id, + member.destination_inode, + ) => {} + Some(versioned) => push_restore_issue( + report, + "staging_member_key_mismatch", + "staging member key does not match operation ref-set", + Some(&versioned.operation), + ), + None => push_restore_raw_issue( + report, + "orphan_staging_member", + "staging member has no operation", + ), + }, + Err(_) => push_restore_raw_issue( + report, + "staging_member_unowned", + "malformed staging member cannot be attributed to an operation", + ), + } + Ok(()) + }, + )?; + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "staging_inode_inverse"), + version, + |row| { + let (digest, ref_set_id) = + match super::restore::decode_restore_staging_inverse(&row.value.0) { + Ok(decoded) => decoded, + Err(error) => { + push_restore_raw_issue( + report, + "staging_inverse_decode_failed", + format!("staging inode inverse cannot be decoded: {error}"), + ); + return Ok(()); + } + }; + let Some(operation) = operations.get(&digest).map(|item| &item.operation) else { + push_restore_raw_issue( + report, + "orphan_staging_inverse", + "staging inode inverse has no operation", + ); + return Ok(()); + }; + let Some(inode) = restore_key_tail_inode(&row.key) else { + push_restore_issue( + report, + "orphan_staging_inode_inverse", + "staging inode inverse key has no inode identity", + Some(operation), + ); + return Ok(()); + }; + let member_key = + super::restore::restore_staging_member_key(self.mount, ref_set_id, inode); + let member_matches = self + .metadata + .get( + RecordFamily::System, + &member_key, + version, + ReadPurpose::WritePlanLocal, + )? + .and_then(|value| super::restore::decode_restore_staging_member(&value.0).ok()) + .is_some_and(|member| { + member.operation_digest == digest && member.destination_inode == inode + }); + if operation.ref_set_id != ref_set_id + || row.key != super::restore::restore_staging_inode_key(self.mount, inode) + || !member_matches + { + push_restore_issue( + report, + "orphan_staging_inode_inverse", + "staging inode inverse has no matching member owner", + Some(operation), + ); + } + Ok(()) + }, + )?; + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "staging_inverse_owner"), + version, + |row| { + let (digest, ref_set_id) = + match super::restore::decode_restore_staging_inverse(&row.value.0) { + Ok(decoded) => decoded, + Err(error) => { + push_restore_raw_issue( + report, + "staging_inverse_owner_decode_failed", + format!("ref-set-first staging inverse cannot be decoded: {error}"), + ); + return Ok(()); + } + }; + let Some(operation) = operations.get(&digest).map(|item| &item.operation) else { + push_restore_raw_issue( + report, + "unowned_staging_inverse_owner", + "ref-set-first staging inverse has no operation", + ); + return Ok(()); + }; + let Some(inode) = restore_key_tail_inode(&row.key) else { + push_restore_issue( + report, + "orphan_staging_inverse_owner", + "ref-set-first staging inverse key has no inode identity", + Some(operation), + ); + return Ok(()); + }; + let member_matches = self + .metadata + .get( + RecordFamily::System, + &super::restore::restore_staging_member_key(self.mount, ref_set_id, inode), + version, + ReadPurpose::WritePlanLocal, + )? + .and_then(|value| super::restore::decode_restore_staging_member(&value.0).ok()) + .is_some_and(|member| { + member.operation_digest == digest && member.destination_inode == inode + }); + let primary_matches = self + .metadata + .get( + RecordFamily::System, + &super::restore::restore_staging_inode_key(self.mount, inode), + version, + ReadPurpose::WritePlanLocal, + )? + .is_some_and(|value| value.0 == row.value.0); + if operation.ref_set_id != ref_set_id + || row.key + != super::restore::restore_staging_inverse_owner_key( + self.mount, ref_set_id, inode, + ) + || !member_matches + || !primary_matches + { + push_restore_issue( + report, + "orphan_staging_inverse_owner", + "ref-set-first staging inverse has no matching member/primary inverse", + Some(operation), + ); + } + Ok(()) + }, + ) + } + + fn fsck_restore_base_orphans_streaming( + &self, + version: Version, + operations: &HashMap<[u8; 32], VersionedRestoreOperation>, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "base_owner"), + version, + |row| { + match super::restore_gc::decode_restore_base_reference(&row.value.0) { + Ok(reference) => match operations.get(&reference.operation_digest) { + Some(versioned) => { + let object_digest: [u8; 32] = + Sha256::digest(reference.object_key.as_bytes()).into(); + if versioned.operation.ref_set_id != reference.ref_set_id + || row.key + != super::restore::restore_base_owner_key( + self.mount, + reference.ref_set_id, + &object_digest, + reference.borrower_inode, + reference.borrower_generation, + ) + { + push_restore_issue( + report, + "base_owner_ref_set_mismatch", + "base owner key/ref-set does not match operation", + Some(&versioned.operation), + ); + } + } + None => push_restore_raw_issue( + report, + "orphan_base_owner", + "base owner has no operation", + ), + }, + Err(_) => push_restore_raw_issue( + report, + "base_owner_unowned", + "malformed base owner cannot be attributed to an operation", + ), + } + Ok(()) + }, + )?; + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "base_inverse"), + version, + |row| { + let inverse = match super::restore_gc::decode_restore_base_inverse(&row.value.0) { + Ok(inverse) => inverse, + Err(error) => { + push_restore_raw_issue( + report, + "base_inverse_decode_failed", + format!("base inverse cannot be decoded: {error}"), + ); + return Ok(()); + } + }; + let Some(operation) = operations + .get(&inverse.operation_digest) + .map(|item| &item.operation) + else { + push_restore_raw_issue( + report, + "orphan_base_inverse", + "base inverse has no operation", + ); + return Ok(()); + }; + let owner_key = super::restore::restore_base_owner_key( + self.mount, + inverse.ref_set_id, + &inverse.object_digest, + inverse.borrower_inode, + inverse.borrower_generation, + ); + let owner_matches = self + .metadata + .get( + RecordFamily::System, + &owner_key, + version, + ReadPurpose::WritePlanLocal, + )? + .and_then(|value| { + super::restore_gc::decode_restore_base_reference(&value.0).ok() + }) + .is_some_and(|reference| { + reference.operation_digest == inverse.operation_digest + && reference.ref_set_id == inverse.ref_set_id + && reference.borrower_inode == inverse.borrower_inode + && reference.borrower_generation == inverse.borrower_generation + && Sha256::digest(reference.object_key.as_bytes())[..] + == inverse.object_digest + }); + if operation.ref_set_id != inverse.ref_set_id + || row.key + != super::restore::restore_base_inverse_key( + self.mount, + &inverse.object_digest, + inverse.ref_set_id, + inverse.borrower_inode, + inverse.borrower_generation, + ) + || !owner_matches + { + push_restore_issue( + report, + "orphan_or_mismatched_base_inverse", + "base inverse has no matching owner/ref-set", + Some(operation), + ); + } + Ok(()) + }, + )?; + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "base_inverse_owner"), + version, + |row| { + let inverse = match super::restore_gc::decode_restore_base_inverse(&row.value.0) { + Ok(inverse) => inverse, + Err(error) => { + push_restore_raw_issue( + report, + "base_inverse_owner_decode_failed", + format!("ref-set-first base inverse cannot be decoded: {error}"), + ); + return Ok(()); + } + }; + let Some(operation) = operations + .get(&inverse.operation_digest) + .map(|item| &item.operation) + else { + push_restore_raw_issue( + report, + "unowned_base_inverse_owner", + "ref-set-first base inverse has no operation", + ); + return Ok(()); + }; + let expected_key = super::restore::restore_base_inverse_owner_key( + self.mount, + inverse.ref_set_id, + &inverse.object_digest, + inverse.borrower_inode, + inverse.borrower_generation, + ); + let primary_matches = self + .metadata + .get( + RecordFamily::System, + &super::restore::restore_base_inverse_key( + self.mount, + &inverse.object_digest, + inverse.ref_set_id, + inverse.borrower_inode, + inverse.borrower_generation, + ), + version, + ReadPurpose::WritePlanLocal, + )? + .is_some_and(|value| value.0 == row.value.0); + let owner_exists = self + .metadata + .get( + RecordFamily::System, + &super::restore::restore_base_owner_key( + self.mount, + inverse.ref_set_id, + &inverse.object_digest, + inverse.borrower_inode, + inverse.borrower_generation, + ), + version, + ReadPurpose::WritePlanLocal, + )? + .is_some(); + if operation.ref_set_id != inverse.ref_set_id + || row.key != expected_key + || !primary_matches + || !owner_exists + { + push_restore_issue( + report, + "orphan_base_inverse_owner", + "ref-set-first base inverse has no matching owner/primary inverse", + Some(operation), + ); + } + Ok(()) + }, + )?; + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "base_seal"), + version, + |row| { + let (digest, initialization_digest, ref_set_id, incarnation) = + match super::restore_gc::decode_restore_base_seal_record(&row.value.0) { + Ok(super::restore_gc::RestoreBaseSealRecord::Building(build)) => ( + build.operation_digest, + build.initialization_digest, + build.ref_set_id, + build.incarnation, + ), + Ok(super::restore_gc::RestoreBaseSealRecord::Sealed(seal)) => ( + seal.operation_digest, + seal.initialization_digest, + seal.ref_set_id, + seal.incarnation, + ), + Err(error) => { + push_restore_raw_issue( + report, + "base_seal_decode_failed", + format!("base seal/progress cannot be decoded: {error}"), + ); + return Ok(()); + } + }; + match operations.get(&digest) { + Some(versioned) + if versioned.operation.ref_set_id == ref_set_id + && versioned.operation.initialization_digest + == initialization_digest + && versioned.operation.created_version == incarnation + && row.key + == super::restore::restore_base_seal_key( + self.mount, ref_set_id, + ) => {} + Some(versioned) => push_restore_issue( + report, + "base_seal_key_or_owner_mismatch", + "base seal/progress does not match key/operation", + Some(&versioned.operation), + ), + None => push_restore_raw_issue( + report, + "orphan_base_seal", + "base seal/progress has no operation", + ), + } + Ok(()) + }, + ) + } + + fn fsck_restore_auxiliary_orphans_streaming( + &self, + version: Version, + operations: &HashMap<[u8; 32], VersionedRestoreOperation>, + verify_objects: bool, + report: &mut RestoreFsckReport, + ) -> Result<(), MetadError> { + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "init_upload_intent"), + version, + |row| { + match super::restore::decode_restore_init_upload_intent(&row.value.0) { + Ok(intent) => match operations.get(&intent.operation_digest) { + Some(versioned) + if versioned.operation.ref_set_id == intent.ref_set_id + && row.key + == super::restore::restore_init_upload_intent_key( + self.mount, + intent.ref_set_id, + intent.inode, + intent.generation, + ) => {} + Some(versioned) => push_restore_issue( + report, + "init_intent_identity_mismatch", + "initialization upload intent does not match key/operation", + Some(&versioned.operation), + ), + None => push_restore_raw_issue( + report, + "orphan_init_intent", + "initialization upload intent has no operation", + ), + }, + Err(error) => push_restore_raw_issue( + report, + "init_intent_decode_failed", + format!("initialization upload intent cannot be decoded: {error}"), + ), + } + Ok(()) + }, + )?; + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "init_upload_tombstone"), + version, + |row| { + match super::restore::validate_restore_init_upload_tombstone_row(self.mount, row) { + Ok(tombstone) => { + if let Some(versioned) = operations.get(&tombstone.operation_digest) { + if versioned.operation.initialization_digest + != tombstone.initialization_digest + { + push_restore_issue( + report, + "init_tombstone_owner_mismatch", + "initialization tombstone digest does not match live operation", + Some(&versioned.operation), + ); + } + } + if verify_objects + && !self.restore_init_object_range_absent( + tombstone.inode, + tombstone.generation, + tombstone.size, + )? + { + push_restore_raw_issue( + report, + "init_tombstone_object_present", + format!( + "initialization tombstone for inode {} generation {} still has an object", + tombstone.inode.get(), + tombstone.generation, + ), + ); + } + } + Err(error) => push_restore_raw_issue( + report, + "init_tombstone_decode_failed", + format!("initialization tombstone cannot be decoded: {error}"), + ), + } + Ok(()) + }, + )?; + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "cleanup_job"), + version, + |row| { + match super::restore::decode_restore_cleanup_job(&row.value.0) { + Ok(job) => match operations.get(&job.operation_digest) { + Some(versioned) + if versioned.operation.ref_set_id == job.ref_set_id + && row.key + == super::restore::restore_cleanup_job_key( + self.mount, + job.ref_set_id, + ) => {} + Some(versioned) => push_restore_issue( + report, + "cleanup_job_key_or_owner_mismatch", + "cleanup job does not match key/operation", + Some(&versioned.operation), + ), + None => push_restore_raw_issue( + report, + "orphan_cleanup_job", + "cleanup job has no operation", + ), + }, + Err(error) => push_restore_raw_issue( + report, + "cleanup_job_decode_failed", + format!("cleanup job cannot be decoded: {error}"), + ), + } + Ok(()) + }, + )?; + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "release_job"), + version, + |row| { + match super::restore::decode_restore_release_job(&row.value.0) { + Ok(job) => match operations.get(&job.operation_digest) { + Some(versioned) + if versioned.operation.ref_set_id == job.ref_set_id + && row.key + == super::restore::restore_release_job_key( + self.mount, + job.ref_set_id, + ) => {} + Some(versioned) => push_restore_issue( + report, + "release_job_key_or_owner_mismatch", + "release job does not match key/operation", + Some(&versioned.operation), + ), + None => push_restore_raw_issue( + report, + "orphan_release_job", + "release job has no operation", + ), + }, + Err(error) => push_restore_raw_issue( + report, + "release_job_decode_failed", + format!("release job cannot be decoded: {error}"), + ), + } + Ok(()) + }, + )?; + self.for_each_restore_system_row( + restore_control_prefix(self.mount, "release_quarantine"), + version, + |row| { + match super::restore::validate_restore_release_quarantine_row(self.mount, row) { + Ok(quarantine) => { + let scope = match quarantine.scope { + super::restore::RestoreReleaseQuarantineScope::Diagnostic => { + "diagnostic".to_owned() + } + super::restore::RestoreReleaseQuarantineScope::Object(digest) => { + format!("object:{}", restore_digest_hex(&digest)) + } + super::restore::RestoreReleaseQuarantineScope::MountWide => { + "mount-wide".to_owned() + } + }; + push_restore_raw_issue( + report, + "release_quarantine_present", + format!( + "restore release quarantined a {:?} row at version {} ({scope}): {}", + quarantine.family, + quarantine.original_version.get(), + quarantine.reason, + ), + ); + } + Err(error) => push_restore_raw_issue( + report, + "release_quarantine_decode_failed", + format!("restore release quarantine cannot be validated: {error}"), + ), + } + Ok(()) + }, + ) + } +} + +fn restore_control_prefix(mount: MountId, name: &str) -> Vec { + super::restore::restore_control_keyspaces(mount) + .into_iter() + .find_map(|(candidate, prefix)| (candidate == name).then_some(prefix)) + .unwrap_or_else(|| panic!("restore control keyspace {name} is not registered")) +} + +fn restore_key_tail_inode(key: &[u8]) -> Option { + let raw = u64::from_be_bytes(key.get(key.len().checked_sub(8)?..)?.try_into().ok()?); + InodeId::new(raw).ok() +} + +fn restore_metric_count(metrics: &RestoreMetrics, name: &str) -> usize { + metrics.control_rows.get(name).copied().unwrap_or(0) +} + +fn restore_downgrade_empty_predicates(mount: MountId) -> Vec { + let mut predicates = super::restore::restore_control_keyspaces(mount) + .into_iter() + .map(|(_, prefix)| PredicateRef { + family: RecordFamily::System, + key: prefix, + predicate: Predicate::PrefixEmpty, + }) + .collect::>(); + predicates.extend(super::restore_index::restore_index_global_empty_predicates( + mount, + )); + predicates +} + +fn update_restore_operation_metrics( + metrics: &mut RestoreMetrics, + operation: &super::restore::RestoreOperation, +) { + let version_age = metrics + .read_version + .saturating_sub(operation.created_version); + match operation.state { + super::restore::RestoreOperationState::Preparing => { + metrics.preparing += 1; + metrics.max_preparing_version_age = metrics.max_preparing_version_age.max(version_age); + } + super::restore::RestoreOperationState::ReadyToAttach => metrics.ready_to_attach += 1, + super::restore::RestoreOperationState::Complete => metrics.complete += 1, + super::restore::RestoreOperationState::Cleaning => { + metrics.cleaning += 1; + metrics.max_preparing_version_age = metrics.max_preparing_version_age.max(version_age); + } + super::restore::RestoreOperationState::Discarding => { + metrics.discarding += 1; + metrics.max_preparing_version_age = metrics.max_preparing_version_age.max(version_age); + } + super::restore::RestoreOperationState::Releasing => { + metrics.releasing += 1; + metrics.max_releasing_version_age = metrics.max_releasing_version_age.max(version_age); + } + } +} + +fn push_restore_issue( + report: &mut RestoreFsckReport, + code: &str, + message: impl Into, + operation: Option<&super::restore::RestoreOperation>, +) { + report.issues.push(RestoreFsckIssue { + code: code.to_owned(), + message: message.into(), + operation_id: operation.map(|operation| { + format!( + "restore-{}", + restore_digest_hex(&operation.operation_digest) + ) + }), + ref_set_id: operation.map(|operation| operation.ref_set_id), + }); +} + +fn push_restore_raw_issue(report: &mut RestoreFsckReport, code: &str, message: impl Into) { + push_restore_issue(report, code, message, None); +} + +fn restore_digest_hex(digest: &[u8; 32]) -> String { + use std::fmt::Write as _; + let mut encoded = String::with_capacity(64); + for byte in digest { + write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail"); + } + encoded +} + +fn fsck_limit_reached(limit: usize, report: &FsckReport) -> bool { + limit != 0 && report.files_scanned.saturating_add(report.symlinks_scanned) >= limit +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::command::{ReadItem, ScanItem}; + use crate::holtstore::HoltMetadataStore; + use nokv_object::MemoryObjectStore; + use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; + use std::sync::Arc; + + #[derive(Clone)] + struct FsckScanTrackingStore { + inner: HoltMetadataStore, + counts: Arc, + } + + #[derive(Default)] + struct FsckScanCounts { + unbounded_scans: AtomicUsize, + largest_scan: AtomicUsize, + } + + impl FsckScanTrackingStore { + fn new(inner: HoltMetadataStore) -> Self { + Self { + inner, + counts: Arc::new(FsckScanCounts::default()), + } + } + + fn reset(&self) { + self.counts + .unbounded_scans + .store(0, AtomicOrdering::Relaxed); + self.counts.largest_scan.store(0, AtomicOrdering::Relaxed); + } + + fn unbounded_scans(&self) -> usize { + self.counts.unbounded_scans.load(AtomicOrdering::Relaxed) + } + + fn largest_scan(&self) -> usize { + self.counts.largest_scan.load(AtomicOrdering::Relaxed) + } + } + + impl MetadataStore for FsckScanTrackingStore { + fn get_versioned( + &self, + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + self.inner.get_versioned(family, key, version, purpose) + } + + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + if request.limit == 0 { + self.counts + .unbounded_scans + .fetch_add(1, AtomicOrdering::Relaxed); + } else { + self.counts + .largest_scan + .fetch_max(request.limit, AtomicOrdering::Relaxed); + } + self.inner.scan(request) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + self.inner.commit_metadata(command) + } + + fn commit_independent_batch( + &self, + commands: &[MetadataCommand], + ) -> Vec> { + self.inner.commit_independent_batch(commands) + } + + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + self.inner.committed_request_result(request_id) + } + + fn history_retention_epoch(&self) -> Result { + self.inner.history_retention_epoch() + } + + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } + } + + fn service() -> ( + NoKvFs, + HoltMetadataStore, + MemoryObjectStore, + ) { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + (service, metadata, objects) + } + + fn install_restore_downgrade_fence(service: &NoKvFs) { + let object_reference = service.begin_object_reference_mutation().unwrap(); + let version = service.next_version().unwrap(); + let read_version = predecessor(version).unwrap(); + let (allocator_predicate, allocator_mutation) = + service.restore_allocator_fence_plan(read_version).unwrap(); + let active_key = super::super::restore::restore_active_key(service.mount); + service + .commit_metadata(MetadataCommand { + request_id: b"fsck-install-restore-fence".to_vec(), + kind: CommandKind::CleanupObjects, + read_version, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: active_key.clone(), + predicates: vec![ + object_reference.predicate(service.mount), + allocator_predicate, + PredicateRef { + family: RecordFamily::System, + key: active_key.clone(), + predicate: Predicate::NotExists, + }, + ], + mutations: vec![ + allocator_mutation, + Mutation { + family: RecordFamily::System, + key: active_key, + op: MutationOp::Put, + value: Some(Value(vec![super::super::restore::RESTORE_FORMAT_VERSION])), + }, + ], + watch: Vec::new(), + }) + .unwrap(); + } + + #[test] + fn restore_downgrade_cas_covers_every_registered_private_prefix() { + let mount = MountId::new(1).unwrap(); + let expected = super::super::restore::restore_control_keyspaces(mount) + .into_iter() + .chain(super::super::restore_index::restore_index_private_keyspaces(mount)) + .map(|(_, prefix)| prefix) + .collect::>(); + let predicates = restore_downgrade_empty_predicates(mount); + let actual = predicates + .iter() + .map(|predicate| predicate.key.clone()) + .collect::>(); + assert_eq!(actual, expected); + assert_eq!(predicates.len(), actual.len(), "duplicate CAS prefixes"); + assert!(predicates.iter().all(|predicate| { + predicate.family == RecordFamily::System + && predicate.predicate == Predicate::PrefixEmpty + })); + } + + #[test] + fn restore_metrics_strictly_decode_operation_rows_without_running_graph_fsck() { + let (service, _metadata, _objects) = service(); + let mut operation_key = super::super::restore::restore_control_keyspaces(service.mount) + .into_iter() + .find(|(name, _)| *name == "operation") + .unwrap() + .1; + operation_key.extend_from_slice(&[0_u8; 32]); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: b"fsck-malformed-restore-operation".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: operation_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: operation_key, + op: MutationOp::Put, + value: Some(Value(b"malformed-operation".to_vec())), + }], + watch: Vec::new(), + }) + .unwrap(); + + assert!(matches!( + service.restore_metrics_with_page_size(1), + Err(MetadError::Codec(_)) + )); + let report = service.fsck_restore_state(false).unwrap(); + assert!(report + .issues + .iter() + .any(|issue| issue.code == "operation_decode_failed")); + } + + #[test] + fn full_object_fsck_covers_symlinks_and_snapshot_pinned_generations() { + let (service, _metadata, objects) = service(); + let runs = service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let runs_inode = runs.attr.inode; + let file_name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); + let file = service + .publish_artifact(PublishArtifact { + parent: runs_inode, + name: file_name.clone(), + producer: "fsck-test".to_owned(), + digest_uri: "sha256:test".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "fsck/snapshot".to_owned(), + bytes: b"snapshot-body".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + service + .create_symlink( + runs_inode, + DentryName::new(b"latest".to_vec()).unwrap(), + b"artifact.bin".to_vec(), + 0o777, + 1000, + 1000, + ) + .unwrap(); + let _pin = service + .snapshot_subtree_with_lease(runs_inode, DEFAULT_SNAPSHOT_LEASE_MS) + .unwrap(); + service.remove_file(runs_inode, &file_name).unwrap(); + + let lost = ObjectKey::new(format!( + "blocks/{}/{}/{}/0/0", + service.mount.get(), + file.attr.inode.get(), + file.attr.generation, + )) + .unwrap(); + assert!(objects.delete(&lost).unwrap()); + + let live = service.fsck_object_references(FsckMode::Live, 0).unwrap(); + assert!(live.is_consistent(), "live report: {live:?}"); + assert_eq!(live.symlinks_scanned, 1); + + let full = service.fsck_object_references(FsckMode::Full, 0).unwrap(); + assert!(!full.is_consistent()); + assert_eq!(full.snapshot_pins_scanned, 1); + assert!(full.historical_bodies_scanned >= 1); + assert!(full.dangling.iter().any(|entry| { + entry.inode == file.attr.inode.get() && entry.object_key == lost.as_str() + })); + } + + #[test] + fn full_object_fsck_pages_physical_reads_and_limits_bodies_not_inode_rows() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let tracking = FsckScanTrackingStore::new(metadata); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), tracking.clone(), objects); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + + let bulk = service + .create_dir( + InodeId::root(), + DentryName::new(b"bulk".to_vec()).unwrap(), + 0o755, + 1000, + 1000, + ) + .unwrap(); + let pinned = service + .create_dir( + InodeId::root(), + DentryName::new(b"pinned".to_vec()).unwrap(), + 0o755, + 1000, + 1000, + ) + .unwrap(); + for index in 0..270 { + service + .create_dir( + bulk.attr.inode, + DentryName::new(format!("directory-{index:03}").into_bytes()).unwrap(), + 0o755, + 1000, + 1000, + ) + .unwrap(); + } + service + .publish_artifact(PublishArtifact { + parent: bulk.attr.inode, + name: DentryName::new(b"last-body.bin".to_vec()).unwrap(), + producer: "bounded-fsck-test".to_owned(), + digest_uri: "sha256:bounded-fsck".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "fsck/bounded".to_owned(), + bytes: b"bounded-fsck-body".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + for _ in 0..257 { + service + .snapshot_subtree_with_lease(pinned.attr.inode, DEFAULT_SNAPSHOT_LEASE_MS) + .unwrap(); + } + + tracking.reset(); + let limited = service.fsck_object_references(FsckMode::Full, 1).unwrap(); + assert_eq!(limited.files_scanned + limited.symlinks_scanned, 1); + assert!( + limited.inodes_scanned > OBJECT_FSCK_PAGE_ROWS, + "a body limit must not truncate the live inode scan" + ); + assert_eq!(limited.snapshot_pins_scanned, 0); + assert_eq!(tracking.unbounded_scans(), 0); + assert!(tracking.largest_scan() <= OBJECT_FSCK_PAGE_ROWS); + + tracking.reset(); + let full = service.fsck_object_references(FsckMode::Full, 0).unwrap(); + assert!(full.is_consistent(), "full object fsck report: {full:#?}"); + assert_eq!(full.files_scanned, 1); + assert_eq!(full.snapshot_pins_scanned, 257); + assert!(full.inodes_scanned > OBJECT_FSCK_PAGE_ROWS); + assert_eq!(tracking.unbounded_scans(), 0); + assert!(tracking.largest_scan() <= OBJECT_FSCK_PAGE_ROWS); + } + + #[test] + fn restore_fsck_accepts_a_complete_real_restore_graph() { + let (service, _metadata, _objects) = service(); + let source = service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let artifact_name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); + service + .publish_artifact(PublishArtifact { + parent: source.attr.inode, + name: artifact_name.clone(), + producer: "fsck-test".to_owned(), + digest_uri: "sha256:restore-fsck".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "fsck/restore-source".to_owned(), + bytes: b"durable-restore-body".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let pin = service + .snapshot_subtree_with_lease(source.attr.inode, DEFAULT_SNAPSHOT_LEASE_MS) + .unwrap(); + let outcome = service + .restore_subtree_path_to_fork("/source", pin.snapshot_id, "/restored") + .unwrap(); + assert_eq!(outcome.state, RestoreState::Complete); + + service + .remove_file(source.attr.inode, &artifact_name) + .unwrap(); + service.remove_empty_dir_path("/source").unwrap(); + assert!(service.retire_snapshot(pin.snapshot_id).unwrap()); + for _ in 0..4 { + service.cleanup_pending_objects(128).unwrap(); + } + let restored = service + .lookup_path("/restored/artifact.bin") + .unwrap() + .unwrap(); + assert_eq!( + service.read_file(restored.attr.inode, 0, 64).unwrap(), + b"durable-restore-body" + ); + + let report = service.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); + assert_eq!(report.metrics.complete, 1); + assert!(report.metrics.staging_rows > 0); + assert!(report.metrics.index_rows > 0); + let metrics = service.restore_metrics().unwrap(); + assert_eq!(metrics.complete, 1); + assert_eq!(metrics.staging_rows, report.metrics.staging_rows); + assert_eq!( + metrics.exact_reference_rows, + report.metrics.exact_reference_rows + ); + assert_eq!(metrics.index_rows, report.metrics.index_rows); + assert!(service + .fsck_object_references(FsckMode::Full, 0) + .unwrap() + .is_consistent()); + + service + .remove_file(outcome.destination_root, &artifact_name) + .unwrap(); + let mutated = service.fsck_restore_state(true).unwrap(); + assert!( + mutated.is_consistent(), + "post-attach mutation restore fsck report: {mutated:#?}" + ); + assert_eq!(mutated.metrics.complete, 1); + } + + #[test] + fn restore_fsck_and_reopen_fail_closed_on_marker_allocator_mismatch() { + let (service, metadata, objects) = service(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: b"fsck-marker-mismatch".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: super::super::restore::restore_active_key(service.mount), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: super::super::restore::restore_active_key(service.mount), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: super::super::restore::restore_active_key(service.mount), + op: MutationOp::Put, + value: Some(Value(vec![super::super::restore::RESTORE_FORMAT_VERSION])), + }], + watch: Vec::new(), + }) + .unwrap(); + + let report = service.fsck_restore_state(false).unwrap(); + assert!(!report.is_consistent()); + assert!(report + .issues + .iter() + .any(|issue| issue.code == "allocator_marker_mismatch")); + let drain = service.drain_restore_to_fork_v1( + RestoreCapabilityDisabled::acknowledged(), + RestoreWritersQuiesced::acknowledged(), + ); + assert!(matches!( + drain, + Err(RestoreDowngradeError::Metadata(MetadError::Codec(message))) + if message.contains("marker") && message.contains("fence") + )); + assert!(NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0,).is_err()); + } + + #[test] + fn restore_downgrade_drains_fence_and_survives_checkpoint_reopen() { + let (service, metadata, objects) = service(); + install_restore_downgrade_fence(&service); + let claim_key = object_gc_claim_key(service.mount); + let claim_before = metadata + .get_versioned( + RecordFamily::System, + &claim_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + + let outcome = service + .drain_restore_to_fork_v1( + RestoreCapabilityDisabled::acknowledged(), + RestoreWritersQuiesced::acknowledged(), + ) + .unwrap(); + assert!(!outcome.already_drained); + assert!(outcome.commit_version.is_some()); + assert!(outcome.full_fsck_required); + assert!(outcome.metadata_checkpoint_required); + let version = service.read_version().unwrap(); + assert!(metadata + .get( + RecordFamily::System, + &super::super::restore::restore_active_key(service.mount), + version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + let allocator = metadata + .get( + RecordFamily::System, + &allocator_key(service.mount), + version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + assert!( + !decode_allocator_state_with_restore_fence(&allocator.0) + .unwrap() + .3 + ); + let claim_after = metadata + .get_versioned( + RecordFamily::System, + &claim_key, + version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + assert!(claim_after.version.get() > claim_before.version.get()); + assert_eq!( + decode_object_gc_claim(service.mount, &claim_after.value.0).unwrap(), + ObjectGcClaim::Open + ); + assert!(service.fsck_restore_state(false).unwrap().is_consistent()); + + metadata.checkpoint().unwrap(); + let reopened = + NoKvFs::open_existing(MountId::new(1).unwrap(), metadata.clone(), objects, 0).unwrap(); + assert!(reopened.fsck_restore_state(false).unwrap().is_consistent()); + let repeated = reopened + .drain_restore_to_fork_v1( + RestoreCapabilityDisabled::acknowledged(), + RestoreWritersQuiesced::acknowledged(), + ) + .unwrap(); + assert!(repeated.already_drained); + assert!(repeated.commit_version.is_none()); + assert!(repeated.full_fsck_required); + assert!(repeated.metadata_checkpoint_required); + } + + #[test] + fn restore_downgrade_refuses_private_index_state_and_keeps_fence() { + let (service, metadata, _objects) = service(); + install_restore_downgrade_fence(&service); + let (keyspace, mut private_key) = + super::super::restore_index::restore_index_private_keyspaces(service.mount) + .into_iter() + .next() + .unwrap(); + private_key.extend_from_slice(b"fsck-blocker-"); + let private_keys = (0_u8..3) + .map(|suffix| { + let mut key = private_key.clone(); + key.push(suffix); + key + }) + .collect::>(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: b"fsck-private-index-blocker".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: private_keys[0].clone(), + predicates: private_keys + .iter() + .cloned() + .map(|key| PredicateRef { + family: RecordFamily::System, + key, + predicate: Predicate::NotExists, + }) + .collect(), + mutations: private_keys + .into_iter() + .map(|key| Mutation { + family: RecordFamily::System, + key, + op: MutationOp::Put, + value: Some(Value(b"invalid-but-durable".to_vec())), + }) + .collect(), + watch: Vec::new(), + }) + .unwrap(); + + let metrics = service.restore_metrics_with_page_size(2).unwrap(); + assert_eq!(metrics.index_rows, 3); + assert_eq!(metrics.control_rows.get(keyspace), Some(&3)); + let report = service.fsck_restore_state(false).unwrap(); + assert!(!report.is_consistent()); + assert!( + report + .issues + .iter() + .filter(|issue| issue.code == "index_row_unowned") + .count() + >= 3 + ); + + let result = service.drain_restore_to_fork_v1( + RestoreCapabilityDisabled::acknowledged(), + RestoreWritersQuiesced::acknowledged(), + ); + assert!(matches!( + result, + Err(RestoreDowngradeError::PrivateStatePresent { + keyspace: blocked, + observed_rows: 1, + }) if blocked == keyspace + )); + let read_version = service.read_version().unwrap(); + assert!(metadata + .get( + RecordFamily::System, + &super::super::restore::restore_active_key(service.mount), + read_version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_some()); + let allocator = metadata + .get( + RecordFamily::System, + &allocator_key(service.mount), + read_version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + assert!( + decode_allocator_state_with_restore_fence(&allocator.0) + .unwrap() + .3 + ); } } diff --git a/crates/nokv-meta/src/service/gc.rs b/crates/nokv-meta/src/service/gc.rs index c3fadaeda..b6327ac2b 100644 --- a/crates/nokv-meta/src/service/gc.rs +++ b/crates/nokv-meta/src/service/gc.rs @@ -667,6 +667,13 @@ where } fn object_delete_is_protected(&self, record: &ObjectGcRecord) -> Result { + // Completed restore forks release their mount-global history floor. Their + // exact inverse rows are therefore the first durable protection check + // for every candidate object. A malformed row returns an error and the + // caller quarantines the GC candidate instead of issuing DELETE. + if self.restore_object_is_borrowed(&record.object_key)? { + return Ok(true); + } if self .history_retention_floor()? .is_some_and(|floor| floor.get() < record.enqueue_version) @@ -841,6 +848,25 @@ where return Ok(outcome); } let page_size = limit.max(1); + let failover_durability_required = self.failover_durability_required()?; + // Release-to-fork teardown owns exact borrowed-object rows and may + // enqueue ordinary GC candidates. Advance one bounded page while this + // worker already holds object_gc_gate, then let the regular queue consume + // only the resulting durable state. Calling the public wrapper here + // would acquire the same non-reentrant gate twice. + // A failover-durability marker blocks those metadata mutations too, but + // still scan the ordinary GC page below so the pre-existing outcome + // contract reports how much work was held back. + if !failover_durability_required { + outcome.restore_init_tombstones_scanned = + self.cleanup_restore_init_upload_tombstones_locked(page_size)?; + outcome.restore_release_jobs_processed = + self.cleanup_restore_releases_locked(page_size)?; + } + let (backlog, quarantine, mount_wide_quarantine) = self.restore_release_backlog()?; + outcome.restore_release_backlog = backlog; + outcome.restore_release_quarantine = quarantine; + outcome.restore_release_mount_wide_quarantine = mount_wide_quarantine; let cursor_key = object_gc_scan_cursor_key(self.mount); let read_version = self.read_version()?; let cursor = self.metadata.get_versioned( @@ -866,9 +892,11 @@ where purpose: ReadPurpose::UserStrong, })?; outcome.scanned += rows.len(); - if self.failover_durability_required()? { - outcome.blocked_by_failover_durability = - outcome.blocked_by_failover_durability.max(rows.len()); + if failover_durability_required || self.failover_durability_required()? { + outcome.blocked_by_failover_durability = outcome + .blocked_by_failover_durability + .max(rows.len()) + .max(1); return Ok(outcome); } // Reap expired snapshot pins only after the failover marker check, so diff --git a/crates/nokv-meta/src/service/lifecycle.rs b/crates/nokv-meta/src/service/lifecycle.rs index 0d90fb49b..53ec382df 100644 --- a/crates/nokv-meta/src/service/lifecycle.rs +++ b/crates/nokv-meta/src/service/lifecycle.rs @@ -6,6 +6,11 @@ where O: ObjectStore, { pub fn new(mount: MountId, metadata: M, objects: O) -> Self { + // Keep the metadata engine pristine here. Checkpoint restore constructs + // a service around an empty store before installing the image, and even + // a read-only scan would create Holt trees that make wholesale image + // installation fail. Bootstrap/open-existing perform the durable + // restore-visibility proof once creating trees is safe. Self { mount, shard_index: 0, @@ -13,6 +18,19 @@ where objects, allocator_gate: Mutex::new(()), backup_gate: Mutex::new(()), + metadata_checkpoint_install_uncertain: AtomicBool::new(false), + restore_gate: Mutex::new(()), + restore_visibility_fence: RwLock::new(()), + restore_staging_possible: AtomicBool::new(true), + restore_visibility_recovery_ready: AtomicBool::new(false), + restore_to_fork_requests_total: AtomicU64::new(0), + restore_to_fork_success_total: AtomicU64::new(0), + restore_to_fork_failure_total: AtomicU64::new(0), + restore_to_fork_elapsed_ns_total: AtomicU64::new(0), + restore_to_fork_elapsed_ns_max: AtomicU64::new(0), + restore_graph_sequence: AtomicU64::new(0), + restore_graph_writers: AtomicUsize::new(0), + restore_snapshot_gate: RwLock::new(()), object_gc_gate: Mutex::new(()), materialization_orphan_possible: AtomicBool::new(true), #[cfg(test)] @@ -34,6 +52,8 @@ where lease_deadline_ms: AtomicU64::new(0), clock_override_ms: AtomicU64::new(0), metadata_log_sync: Mutex::new(None), + metadata_log_publication_hook: RwLock::new(None), + metadata_log_immediate_publication_depth: AtomicUsize::new(0), metadata_log_segments_archived_total: AtomicU64::new(0), metadata_log_entries_archived_total: AtomicU64::new(0), metadata_log_archive_bytes_total: AtomicU64::new(0), @@ -120,6 +140,19 @@ where objects, allocator_gate: Mutex::new(()), backup_gate: Mutex::new(()), + metadata_checkpoint_install_uncertain: AtomicBool::new(false), + restore_gate: Mutex::new(()), + restore_visibility_fence: RwLock::new(()), + restore_staging_possible: AtomicBool::new(true), + restore_visibility_recovery_ready: AtomicBool::new(false), + restore_to_fork_requests_total: AtomicU64::new(0), + restore_to_fork_success_total: AtomicU64::new(0), + restore_to_fork_failure_total: AtomicU64::new(0), + restore_to_fork_elapsed_ns_total: AtomicU64::new(0), + restore_to_fork_elapsed_ns_max: AtomicU64::new(0), + restore_graph_sequence: AtomicU64::new(0), + restore_graph_writers: AtomicUsize::new(0), + restore_snapshot_gate: RwLock::new(()), object_gc_gate: Mutex::new(()), materialization_orphan_possible: AtomicBool::new(true), #[cfg(test)] @@ -141,6 +174,8 @@ where lease_deadline_ms: AtomicU64::new(0), clock_override_ms: AtomicU64::new(0), metadata_log_sync: Mutex::new(None), + metadata_log_publication_hook: RwLock::new(None), + metadata_log_immediate_publication_depth: AtomicUsize::new(0), metadata_log_segments_archived_total: AtomicU64::new(0), metadata_log_entries_archived_total: AtomicU64::new(0), metadata_log_archive_bytes_total: AtomicU64::new(0), @@ -171,6 +206,10 @@ where read_dir_plus_entry_total: AtomicU64::new(0), read_dir_plus_projection_hit_total: AtomicU64::new(0), }; + // Prove the restore visibility fast path before namespace orphan + // recovery performs ordinary inode reads. Both reconstructions are + // durable-state based and leave their hints fail closed on error. + service.recover_restore_staging_visibility()?; service.recover_materialization_orphan_state()?; Ok(service) } @@ -240,6 +279,21 @@ where metadata_log_archive_bytes_total: self .metadata_log_archive_bytes_total .load(Ordering::Relaxed), + restore_to_fork_requests_total: self + .restore_to_fork_requests_total + .load(Ordering::Relaxed), + restore_to_fork_success_total: self + .restore_to_fork_success_total + .load(Ordering::Relaxed), + restore_to_fork_failure_total: self + .restore_to_fork_failure_total + .load(Ordering::Relaxed), + restore_to_fork_elapsed_ns_total: self + .restore_to_fork_elapsed_ns_total + .load(Ordering::Relaxed), + restore_to_fork_elapsed_ns_max: self + .restore_to_fork_elapsed_ns_max + .load(Ordering::Relaxed), } } @@ -247,6 +301,13 @@ where self.mount } + /// Highest metadata commit version currently installed in this service. + /// Server recovery uses it to bind an authoritative checkpoint/log image + /// to terminal request markers restored from that same image. + pub fn metadata_version(&self) -> Result { + self.read_version().map(Version::get) + } + pub fn metadata_store(&self) -> &M { &self.metadata } diff --git a/crates/nokv-meta/src/service/live_test_barrier.rs b/crates/nokv-meta/src/service/live_test_barrier.rs index e0b9777ff..01bbb18cb 100644 --- a/crates/nokv-meta/src/service/live_test_barrier.rs +++ b/crates/nokv-meta/src/service/live_test_barrier.rs @@ -10,11 +10,44 @@ use std::io::Write as _; use std::path::PathBuf; use std::time::{Duration, Instant}; +#[cfg(test)] +use std::cell::Cell; + use super::MetadError; const SNAPSHOT_BARRIER_DIR_ENV: &str = "NOKV_TEST_SNAPSHOT_BARRIER_DIR"; +const RESTORE_BARRIER_DIR_ENV: &str = "NOKV_TEST_RESTORE_BARRIER_DIR"; const BARRIER_TIMEOUT_MS_ENV: &str = "NOKV_TEST_BARRIER_TIMEOUT_MS"; const DEFAULT_BARRIER_TIMEOUT: Duration = Duration::from_secs(60); +const RESTORE_OPERATION_DIGEST_HEX_LEN: usize = 64; +const MAX_RESTORE_BARRIER_BATCH_INDEX: u64 = 999_999; + +#[cfg(test)] +thread_local! { + static TEST_ATTACH_APPLIED_CALLS: Cell = const { Cell::new(0) }; +} + +/// A restore phase that may only be observed after its metadata command has +/// durably applied. Keeping PUT boundaries out of this enum makes it harder to +/// accidentally move an object-store crash point across its write. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum RestoreAppliedPhase { + Hold, + MaterializeBatch(u64), + ReferenceBatch(u64), + ReferencesSealed, + IndexSealed, + Attach, + CleanupBatch(u64), + ReleaseBatch(u64), +} + +/// The explicit side of an initialization object PUT. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum RestoreInitializationPutBoundary { + Before, + After, +} pub(super) fn snapshot(snapshot_id: u64, phase: &str) -> Result<(), MetadError> { let Some(directory) = barrier_directory(SNAPSHOT_BARRIER_DIR_ENV)? else { @@ -23,6 +56,109 @@ pub(super) fn snapshot(snapshot_id: u64, phase: &str) -> Result<(), MetadError> wait(directory, format!("{snapshot_id}.{phase}")) } +/// Wait at a restore crash point after the corresponding metadata command has +/// durably applied. Callers must place this after `commit_metadata` succeeds; +/// in particular, `Attach` belongs after apply and before the RPC ACK. +pub(super) fn restore_applied( + operation_id: &str, + phase: RestoreAppliedPhase, +) -> Result<(), MetadError> { + #[cfg(test)] + if phase == RestoreAppliedPhase::Attach { + TEST_ATTACH_APPLIED_CALLS.with(|calls| calls.set(calls.get().saturating_add(1))); + } + let Some(directory) = barrier_directory(RESTORE_BARRIER_DIR_ENV)? else { + return Ok(()); + }; + let operation_id = validated_restore_operation_id(operation_id)?; + wait( + directory, + format!("{operation_id}.{}", restore_applied_phase_name(phase)?), + ) +} + +#[cfg(test)] +pub(super) fn reset_test_attach_applied_calls() { + TEST_ATTACH_APPLIED_CALLS.with(|calls| calls.set(0)); +} + +#[cfg(test)] +pub(super) fn test_attach_applied_calls() -> u64 { + TEST_ATTACH_APPLIED_CALLS.with(Cell::get) +} + +/// Wait immediately before or after an initialization object PUT. The same +/// deterministic batch index must be used on both sides of one PUT. +pub(super) fn restore_initialization_put( + operation_id: &str, + batch_index: u64, + boundary: RestoreInitializationPutBoundary, +) -> Result<(), MetadError> { + let Some(directory) = barrier_directory(RESTORE_BARRIER_DIR_ENV)? else { + return Ok(()); + }; + let operation_id = validated_restore_operation_id(operation_id)?; + let batch_index = restore_batch_index(batch_index)?; + let boundary = match boundary { + RestoreInitializationPutBoundary::Before => "before", + RestoreInitializationPutBoundary::After => "after", + }; + wait( + directory, + format!("{operation_id}.initialization-put-{boundary}-{batch_index}"), + ) +} + +fn validated_restore_operation_id(operation_id: &str) -> Result<&str, MetadError> { + let Some(digest) = operation_id.strip_prefix("restore-") else { + return Err(MetadError::Codec( + "live-test restore barrier operation id must start with restore-".to_owned(), + )); + }; + if digest.len() != RESTORE_OPERATION_DIGEST_HEX_LEN + || !digest + .as_bytes() + .iter() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + { + return Err(MetadError::Codec( + "live-test restore barrier operation id must contain 64 lowercase hex digits" + .to_owned(), + )); + } + Ok(operation_id) +} + +fn restore_applied_phase_name(phase: RestoreAppliedPhase) -> Result { + Ok(match phase { + RestoreAppliedPhase::Hold => "hold-applied".to_owned(), + RestoreAppliedPhase::MaterializeBatch(index) => { + format!("materialize-batch-{}", restore_batch_index(index)?) + } + RestoreAppliedPhase::ReferenceBatch(index) => { + format!("reference-batch-{}", restore_batch_index(index)?) + } + RestoreAppliedPhase::ReferencesSealed => "references-sealed".to_owned(), + RestoreAppliedPhase::IndexSealed => "index-sealed".to_owned(), + RestoreAppliedPhase::Attach => "attach-applied".to_owned(), + RestoreAppliedPhase::CleanupBatch(index) => { + format!("cleanup-batch-{}", restore_batch_index(index)?) + } + RestoreAppliedPhase::ReleaseBatch(index) => { + format!("release-batch-{}", restore_batch_index(index)?) + } + }) +} + +fn restore_batch_index(index: u64) -> Result { + if index > MAX_RESTORE_BARRIER_BATCH_INDEX { + return Err(MetadError::Codec(format!( + "live-test restore barrier batch index {index} exceeds {MAX_RESTORE_BARRIER_BATCH_INDEX}" + ))); + } + Ok(format!("{index:06}")) +} + fn barrier_directory(variable: &str) -> Result, MetadError> { let Some(raw) = std::env::var_os(variable) else { return Ok(None); @@ -83,3 +219,154 @@ fn wait(directory: PathBuf, stem: String) -> Result<(), MetadError> { } Ok(()) } + +#[cfg(test)] +mod tests { + use std::sync::{Mutex, MutexGuard}; + use std::thread; + + use tempfile::TempDir; + + use super::*; + + const OPERATION_ID: &str = + "restore-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + static ENV_LOCK: Mutex<()> = Mutex::new(()); + + struct RestoreBarrierEnvironment { + _guard: MutexGuard<'static, ()>, + directory: TempDir, + previous_directory: Option, + previous_timeout: Option, + } + + impl RestoreBarrierEnvironment { + fn enabled() -> Self { + let guard = ENV_LOCK.lock().expect("restore barrier env lock"); + let directory = tempfile::tempdir().expect("restore barrier tempdir"); + let previous_directory = std::env::var_os(RESTORE_BARRIER_DIR_ENV); + let previous_timeout = std::env::var_os(BARRIER_TIMEOUT_MS_ENV); + std::env::set_var(RESTORE_BARRIER_DIR_ENV, directory.path()); + std::env::remove_var(BARRIER_TIMEOUT_MS_ENV); + Self { + _guard: guard, + directory, + previous_directory, + previous_timeout, + } + } + + fn path(&self, phase: &str, suffix: &str) -> PathBuf { + self.directory + .path() + .join(format!("{OPERATION_ID}.{phase}.{suffix}")) + } + } + + impl Drop for RestoreBarrierEnvironment { + fn drop(&mut self) { + restore_environment_variable(RESTORE_BARRIER_DIR_ENV, &self.previous_directory); + restore_environment_variable(BARRIER_TIMEOUT_MS_ENV, &self.previous_timeout); + } + } + + fn restore_environment_variable(variable: &str, previous: &Option) { + match previous { + Some(value) => std::env::set_var(variable, value), + None => std::env::remove_var(variable), + } + } + + #[test] + fn restore_phase_names_match_live_harness_contract() { + let cases = [ + (RestoreAppliedPhase::Hold, "hold-applied"), + ( + RestoreAppliedPhase::MaterializeBatch(0), + "materialize-batch-000000", + ), + ( + RestoreAppliedPhase::MaterializeBatch(12), + "materialize-batch-000012", + ), + ( + RestoreAppliedPhase::ReferenceBatch(1), + "reference-batch-000001", + ), + (RestoreAppliedPhase::ReferencesSealed, "references-sealed"), + (RestoreAppliedPhase::IndexSealed, "index-sealed"), + (RestoreAppliedPhase::Attach, "attach-applied"), + (RestoreAppliedPhase::CleanupBatch(0), "cleanup-batch-000000"), + ( + RestoreAppliedPhase::ReleaseBatch(999_999), + "release-batch-999999", + ), + ]; + for (phase, expected) in cases { + assert_eq!(restore_applied_phase_name(phase).unwrap(), expected); + } + } + + #[test] + fn restore_applied_publishes_ready_and_waits_for_continue() { + let environment = RestoreBarrierEnvironment::enabled(); + let arm = environment.path("materialize-batch-000007", "arm"); + std::fs::write(&arm, b"armed\n").unwrap(); + + let waiter = thread::spawn(|| { + restore_applied(OPERATION_ID, RestoreAppliedPhase::MaterializeBatch(7)) + }); + let ready = environment.path("materialize-batch-000007", "ready"); + let deadline = Instant::now() + Duration::from_secs(5); + while !ready.exists() { + assert!( + Instant::now() < deadline, + "restore barrier did not become ready" + ); + thread::sleep(Duration::from_millis(2)); + } + assert!(!waiter.is_finished()); + std::fs::write( + environment.path("materialize-batch-000007", "continue"), + b"continue\n", + ) + .unwrap(); + waiter.join().unwrap().unwrap(); + } + + #[test] + fn restore_initialization_put_uses_explicit_boundaries() { + let environment = RestoreBarrierEnvironment::enabled(); + for (boundary, phase) in [ + ( + RestoreInitializationPutBoundary::Before, + "initialization-put-before-000003", + ), + ( + RestoreInitializationPutBoundary::After, + "initialization-put-after-000003", + ), + ] { + std::fs::write(environment.path(phase, "arm"), b"armed\n").unwrap(); + std::fs::write(environment.path(phase, "continue"), b"continue\n").unwrap(); + restore_initialization_put(OPERATION_ID, 3, boundary).unwrap(); + assert!(environment.path(phase, "ready").exists()); + } + } + + #[test] + fn restore_barrier_rejects_ambiguous_names_and_indexes() { + assert!(validated_restore_operation_id("restore-ABC").is_err()); + assert!(validated_restore_operation_id("../../restore-deadbeef").is_err()); + assert!(restore_batch_index(1_000_000).is_err()); + } + + #[test] + fn disabled_restore_barrier_has_no_filesystem_side_effect() { + let _guard = ENV_LOCK.lock().expect("restore barrier env lock"); + let previous = std::env::var_os(RESTORE_BARRIER_DIR_ENV); + std::env::remove_var(RESTORE_BARRIER_DIR_ENV); + assert!(restore_applied("not-an-operation-id", RestoreAppliedPhase::Hold).is_ok()); + restore_environment_variable(RESTORE_BARRIER_DIR_ENV, &previous); + } +} diff --git a/crates/nokv-meta/src/service/log_archive.rs b/crates/nokv-meta/src/service/log_archive.rs index 0726c493a..c195d99a8 100644 --- a/crates/nokv-meta/src/service/log_archive.rs +++ b/crates/nokv-meta/src/service/log_archive.rs @@ -200,6 +200,12 @@ where .lock() .unwrap_or_else(|err| err.into_inner()); self.mark_materialization_orphan_possible_under_gc_gate(); + let restore_visibility = self + .restore_visibility_fence + .write() + .unwrap_or_else(|err| err.into_inner()); + let checkpoint_replay = self.begin_metadata_checkpoint_install()?; + self.restore_staging_possible.store(true, Ordering::Release); let mut durable_lsn = checkpoint_lsn; let mut last_digest = checkpoint_digest; let mut replay_next_inode = None; @@ -226,14 +232,21 @@ where .fetch_max(next_inode, Ordering::Relaxed); } self.refresh_allocator_state()?; + // Keep the slow-path hint set while releasing the exclusive fence. + // Both recovery proofs perform namespace reads and would deadlock if + // invoked while this thread still owned the write side. + drop(restore_visibility); + self.recover_restore_staging_visibility()?; self.reconcile_materialization_orphan_state_under_gc_gate()?; - Ok(MetadataLogRestoreOutcome { + let outcome = MetadataLogRestoreOutcome { checkpoint, replayed_entries: entries.len(), durable_lsn, last_digest, - }) + }; + checkpoint_replay.complete(); + Ok(outcome) } } diff --git a/crates/nokv-meta/src/service/log_sync.rs b/crates/nokv-meta/src/service/log_sync.rs index 648c24977..08f479c85 100644 --- a/crates/nokv-meta/src/service/log_sync.rs +++ b/crates/nokv-meta/src/service/log_sync.rs @@ -4,6 +4,42 @@ use super::log_archive::archive_metadata_log_segment_to_store; use super::*; use crate::{MetadataLogEntry, MetadataLogSegment}; +const PREPARED_TERMINAL_PROOF_CACHE_LIMIT: usize = 4_096; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct ArchivedPreparedRequestResult { + result: CommitResult, + lsn: u64, +} + +#[derive(Clone, Debug)] +struct PreparedArchiveProofSnapshot { + archive: MetadataLogArchiveConfig, + shard_id: String, + segments: Vec, + baseline_lsn: u64, + baseline_digest: [u8; 32], + durable_tail_lsn: u64, + durable_tail_digest: [u8; 32], +} + +fn trim_prepared_terminal_proof_cache( + proofs: &mut BTreeMap, ArchivedPreparedRequestResult>, +) { + while proofs.len() > PREPARED_TERMINAL_PROOF_CACHE_LIMIT { + let Some(oldest_request_id) = proofs + .iter() + .min_by(|(left_id, left), (right_id, right)| { + left.lsn.cmp(&right.lsn).then_with(|| left_id.cmp(right_id)) + }) + .map(|(request_id, _)| request_id.clone()) + else { + break; + }; + proofs.remove(&oldest_request_id); + } +} + /// A pointer to one archived logical-log segment in the live chain. /// /// The sync state keeps the ordered chain of every segment archived above the @@ -24,9 +60,18 @@ pub struct MetadataLogSyncConfig { pub epoch: u64, pub durable_lsn: u64, pub last_digest: [u8; 32], + /// Exact checkpoint boundary below the inherited active segment chain. + /// This differs from `durable_lsn` after failover, where `durable_lsn` + /// names the tail and the baseline names the checkpoint before replay. + pub durable_recovery_baseline_lsn: u64, + pub durable_recovery_baseline_digest: [u8; 32], /// Segment chain inherited from the control record (e.g. after failover), /// so the new owner's future `LogRef` publishes keep the full chain. pub segments: Vec, + /// Whether `durable_recovery_baseline_*` names an externally authoritative + /// checkpoint. Prepared terminal durability never relies on metadata + /// versions; it uses this baseline plus the exact active LSN chain. + pub has_authoritative_recovery_baseline: bool, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -37,6 +82,7 @@ pub struct MetadataLogSyncSnapshot { pub last_digest: [u8; 32], /// Ordered (oldest first) segment chain above the latest checkpoint. pub segments: Vec, + pub has_authoritative_recovery_baseline: bool, } /// Atomic local state used to decide whether a control-published recovery tail @@ -72,6 +118,12 @@ pub(super) struct MetadataLogSyncState { next_lsn: u64, prev_digest: [u8; 32], segments: Vec, + has_authoritative_recovery_baseline: bool, + durable_recovery_covered_lsn: u64, + durable_recovery_covered_digest: [u8; 32], + /// Prepared-publish terminal results whose exact commands are present in + /// an archived segment above the latest authoritative checkpoint. + archived_prepared_request_results: BTreeMap, ArchivedPreparedRequestResult>, /// A locally committed segment whose object-store archive has not yet /// completed. No later metadata command may apply until this exact segment /// is durably retried, otherwise its LSN could be reused and recovery would @@ -124,6 +176,7 @@ impl MetadataLogSyncState { durable_lsn: self.next_lsn - 1, last_digest: self.prev_digest, segments: self.segments.clone(), + has_authoritative_recovery_baseline: self.has_authoritative_recovery_baseline, } } } @@ -142,7 +195,10 @@ impl MetadataLogSyncConfig { epoch, durable_lsn, last_digest, + durable_recovery_baseline_lsn: durable_lsn, + durable_recovery_baseline_digest: last_digest, segments: Vec::new(), + has_authoritative_recovery_baseline: false, } } @@ -152,6 +208,31 @@ impl MetadataLogSyncConfig { self.segments = segments; self } + + /// Set the authoritative checkpoint boundary below an inherited log tail. + /// The active segment chain must connect this digest to `durable_lsn`. + pub fn with_durable_recovery_baseline(mut self, lsn: u64, digest: [u8; 32]) -> Self { + self.durable_recovery_baseline_lsn = lsn; + self.durable_recovery_baseline_digest = digest; + self + } + + /// Mark the configured recovery baseline as externally authoritative. + pub fn with_authoritative_recovery_baseline(mut self) -> Self { + self.has_authoritative_recovery_baseline = true; + self + } +} + +struct ImmediateMetadataLogPublication<'a> { + depth: &'a AtomicUsize, +} + +impl Drop for ImmediateMetadataLogPublication<'_> { + fn drop(&mut self) { + let previous = self.depth.fetch_sub(1, Ordering::AcqRel); + debug_assert!(previous > 0, "metadata log publication scope underflow"); + } } impl NoKvFs @@ -159,6 +240,86 @@ where M: MetadataStore, O: ObjectStore, { + /// Install the owner-fenced callback used to publish an exact archived log + /// tail to the external control plane. A controlled server installs this + /// once, after enabling synchronous logging and before accepting requests. + pub fn install_sync_metadata_log_publication_hook(&self, hook: F) -> Result<(), MetadError> + where + F: Fn(&MetadataLogSyncSnapshot) -> Result<(), String> + Send + Sync + 'static, + { + if self + .metadata_log_sync + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_none() + { + return Err(MetadError::Codec( + "cannot install metadata log publication hook before enabling the log".to_owned(), + )); + } + let mut installed = self + .metadata_log_publication_hook + .write() + .unwrap_or_else(|error| error.into_inner()); + if installed.is_some() { + return Err(MetadError::Codec( + "metadata log publication hook is already installed".to_owned(), + )); + } + *installed = Some(Arc::new(hook)); + Ok(()) + } + + /// Run a multi-command operation with a control publication barrier after + /// every command whose shared-log segment archived successfully. The + /// callback is invoked before the metadata API reports that command as + /// applied, so an applied-phase crash point is always reachable from the + /// control record rather than only from an unreferenced object. + pub fn with_immediate_sync_metadata_log_publication( + &self, + execute: impl FnOnce() -> Result, + ) -> Result { + self.metadata_log_immediate_publication_depth + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |depth| { + depth.checked_add(1) + }) + .map_err(|_| { + MetadError::Codec("metadata log publication scope depth is exhausted".to_owned()) + })?; + let _publication = ImmediateMetadataLogPublication { + depth: &self.metadata_log_immediate_publication_depth, + }; + execute() + } + + fn publish_archived_metadata_log_tail_immediately( + &self, + snapshot: &MetadataLogSyncSnapshot, + ) -> Result<(), MetadError> { + if self + .metadata_log_immediate_publication_depth + .load(Ordering::Acquire) + == 0 + { + return Ok(()); + } + let hook = self + .metadata_log_publication_hook + .read() + .unwrap_or_else(|error| error.into_inner()) + .clone() + .ok_or_else(|| { + MetadError::Codec( + "immediate metadata log publication has no control-plane hook".to_owned(), + ) + })?; + hook(snapshot).map_err(|message| { + MetadError::Codec(format!( + "immediate metadata log control publication failed: {message}" + )) + }) + } + /// Verify that an installed/restored metadata image already carries the /// failover fence. This never creates or repairs the marker: callers use it /// to reject checkpoints produced before failover-safe object GC existed. @@ -202,6 +363,53 @@ where if config.epoch == 0 { return Err(MetadError::InvalidOwnerEpoch); } + if config.durable_recovery_baseline_lsn > config.durable_lsn { + return Err(MetadError::Codec( + "metadata recovery baseline is above the durable log tail".to_owned(), + )); + } + if config.durable_recovery_baseline_lsn == config.durable_lsn + && config.durable_recovery_baseline_digest != config.last_digest + { + return Err(MetadError::Codec( + "metadata recovery baseline digest differs from the durable log tail".to_owned(), + )); + } + if config.durable_recovery_baseline_lsn < config.durable_lsn { + let mut expected_lsn = config + .durable_recovery_baseline_lsn + .checked_add(1) + .ok_or_else(|| MetadError::Codec("metadata log LSN is exhausted".to_owned()))?; + let mut observed_tail_digest = None; + for pointer in config + .segments + .iter() + .filter(|pointer| pointer.last_lsn > config.durable_recovery_baseline_lsn) + { + if pointer.first_lsn <= config.durable_recovery_baseline_lsn + || pointer.first_lsn > pointer.last_lsn + || pointer.first_lsn != expected_lsn + { + return Err(MetadError::Codec( + "metadata recovery segment pointers do not form a continuous LSN chain" + .to_owned(), + )); + } + expected_lsn = pointer + .last_lsn + .checked_add(1) + .ok_or_else(|| MetadError::Codec("metadata log LSN is exhausted".to_owned()))?; + observed_tail_digest = Some(pointer.last_digest); + } + if expected_lsn - 1 != config.durable_lsn + || observed_tail_digest != Some(config.last_digest) + { + return Err(MetadError::Codec( + "metadata recovery segment pointers do not reach the durable log tail" + .to_owned(), + )); + } + } let next_lsn = config .durable_lsn .checked_add(1) @@ -227,6 +435,10 @@ where next_lsn, prev_digest: config.last_digest, segments: config.segments, + has_authoritative_recovery_baseline: config.has_authoritative_recovery_baseline, + durable_recovery_covered_lsn: config.durable_recovery_baseline_lsn, + durable_recovery_covered_digest: config.durable_recovery_baseline_digest, + archived_prepared_request_results: BTreeMap::new(), pending_segment: None, unresolved_commit_group: None, }; @@ -413,6 +625,10 @@ where .metadata_log_sync .lock() .unwrap_or_else(|err| err.into_inner()) = None; + *self + .metadata_log_publication_hook + .write() + .unwrap_or_else(|err| err.into_inner()) = None; Ok(()) } @@ -424,6 +640,15 @@ where .map(MetadataLogSyncState::snapshot) } + #[cfg(test)] + pub(super) fn prepared_terminal_proof_cache_len(&self) -> Option { + self.metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()) + .as_ref() + .map(|state| state.archived_prepared_request_results.len()) + } + /// Read the logical tail and its unpublishable local states under one lock. /// A caller may treat a cached control tail as clean only when both flags /// are false and the snapshot identity matches exactly. @@ -439,6 +664,272 @@ where }) } + /// Return an already-committed prepared publish only when its local Holt + /// apply marker also has a cross-machine durability proof. This is kept out + /// of the generic commit funnel: ordinary request ids do not bind the full + /// prepared payload and a raw local marker is not a shared-log proof. + pub(super) fn prepared_terminal_commit_result( + &self, + request_id: &[u8], + expected_commit_version: Version, + ) -> Result, MetadError> { + let _log_enable_fence = self + .metadata_log_enable_fence + .read() + .unwrap_or_else(|err| err.into_inner()); + let log_enabled = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()) + .is_some(); + let _commit_log_guard = log_enabled.then(|| { + self.metadata_commit_log_gate + .lock() + .unwrap_or_else(|err| err.into_inner()) + }); + if log_enabled { + self.resolve_unresolved_metadata_commit_group_locked()?; + self.flush_pending_metadata_log_segment_locked() + .map_err(|err| MetadError::SyncLogArchiveFailed { + committed: true, + message: err.to_string(), + })?; + } + + { + let _epoch_fence = self + .epoch_fence + .read() + .unwrap_or_else(|err| err.into_inner()); + self.ensure_owner_epoch_current()?; + } + let Some(result) = self.metadata.committed_request_result(request_id)? else { + return Ok(None); + }; + if result.commit_version != expected_commit_version { + return Err(MetadError::Codec(format!( + "prepared publish terminal version mismatch: expected {}, got {}", + expected_commit_version.get(), + result.commit_version.get() + ))); + } + if log_enabled { + let guard = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()); + let state = guard.as_ref().ok_or_else(|| { + MetadError::Codec( + "synchronous metadata log was disabled during prepared terminal lookup" + .to_owned(), + ) + })?; + let cached_lsn = state + .archived_prepared_request_results + .get(request_id) + .map(|archived| { + if archived.result != result { + return Err(MetadError::Codec( + "prepared publish cached archive result changed identity".to_owned(), + )); + } + Ok(archived.lsn) + }) + .transpose()?; + let proof = PreparedArchiveProofSnapshot { + archive: state.config.clone(), + shard_id: state.shard_id.clone(), + segments: state.segments.clone(), + baseline_lsn: state.durable_recovery_covered_lsn, + baseline_digest: state.durable_recovery_covered_digest, + durable_tail_lsn: state.next_lsn - 1, + durable_tail_digest: state.prev_digest, + }; + let has_authoritative_baseline = state.has_authoritative_recovery_baseline; + drop(guard); + if !self + .prepared_request_is_in_archived_segments(request_id, &result, cached_lsn, &proof)? + && !has_authoritative_baseline + { + return Ok(None); + } + } + // Object-store proof can be slow, so it deliberately runs without the + // owner-epoch read fence. Reacquire the fence afterwards, re-read the + // immutable local terminal marker, and perform the lease/epoch check + // last so failover or lease expiry during proof cannot return success + // from a stale owner. + let _epoch_fence = self + .epoch_fence + .read() + .unwrap_or_else(|err| err.into_inner()); + let final_result = self.metadata.committed_request_result(request_id)?; + if final_result.as_ref() != Some(&result) { + return Err(MetadError::Codec( + "prepared publish terminal marker changed during archive proof".to_owned(), + )); + } + self.ensure_owner_epoch_current()?; + Ok(Some(result)) + } + + /// Record that a checkpoint became authoritative in the external control + /// plane. The exact logical-log boundary is validated before archived + /// per-request proofs covered by the checkpoint are compacted away. + pub fn acknowledge_published_metadata_checkpoint( + &self, + log_lsn: u64, + log_digest: [u8; 32], + ) -> Result<(), MetadError> { + let _log_enable_fence = self + .metadata_log_enable_fence + .read() + .unwrap_or_else(|err| err.into_inner()); + let log_enabled = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()) + .is_some(); + if !log_enabled { + return Ok(()); + } + let _commit_log_guard = self + .metadata_commit_log_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + self.resolve_unresolved_metadata_commit_group_locked()?; + self.flush_pending_metadata_log_segment_locked() + .map_err(|err| MetadError::SyncLogArchiveFailed { + committed: true, + message: err.to_string(), + })?; + self.ensure_owner_epoch_current()?; + let mut guard = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()); + let state = guard.as_mut().ok_or_else(|| { + MetadError::Codec( + "synchronous metadata log was disabled during checkpoint acknowledgement" + .to_owned(), + ) + })?; + if log_lsn < state.durable_recovery_covered_lsn { + return Err(MetadError::Codec( + "published checkpoint regresses durable recovery coverage".to_owned(), + )); + } + let current_lsn = state.next_lsn - 1; + let boundary_matches = if log_lsn == state.durable_recovery_covered_lsn { + log_digest == state.durable_recovery_covered_digest + } else if log_lsn == current_lsn { + log_digest == state.prev_digest + } else { + state + .segments + .iter() + .any(|segment| segment.last_lsn == log_lsn && segment.last_digest == log_digest) + }; + if !boundary_matches { + return Err(MetadError::Codec(format!( + "published checkpoint log boundary {log_lsn} does not match the durable local chain" + ))); + } + + state.has_authoritative_recovery_baseline = true; + state.durable_recovery_covered_lsn = log_lsn; + state.durable_recovery_covered_digest = log_digest; + state + .archived_prepared_request_results + .retain(|_, proof| proof.lsn > log_lsn); + Ok(()) + } + + fn prepared_request_is_in_archived_segments( + &self, + request_id: &[u8], + expected: &CommitResult, + cached_lsn: Option, + proof: &PreparedArchiveProofSnapshot, + ) -> Result { + if cached_lsn.is_some_and(|lsn| lsn <= proof.baseline_lsn) { + return Err(MetadError::Codec( + "prepared publish cache retained a checkpoint-covered LSN".to_owned(), + )); + } + let mut expected_first_lsn = proof.baseline_lsn.checked_add(1); + let mut expected_prev_digest = proof.baseline_digest; + let mut found = false; + for pointer in &proof.segments { + if pointer.last_lsn <= proof.baseline_lsn { + continue; + } + let Some(first_lsn) = expected_first_lsn else { + return Err(MetadError::Codec( + "prepared publish archive LSN is exhausted".to_owned(), + )); + }; + if pointer.first_lsn <= proof.baseline_lsn + || pointer.first_lsn > pointer.last_lsn + || pointer.first_lsn != first_lsn + { + return Err(MetadError::Codec( + "prepared publish archive chain has an LSN gap".to_owned(), + )); + } + proof.archive.validate_segment_key(&pointer.segment_key)?; + let segment = self.load_metadata_log_segment(&pointer.segment_key)?; + if segment.first_lsn != pointer.first_lsn + || segment.last_lsn != pointer.last_lsn + || segment.last_digest != pointer.last_digest + || segment.prev_digest != expected_prev_digest + || segment.shard_id != proof.shard_id + { + return Err(MetadError::Codec( + "prepared publish archive pointer changed segment identity".to_owned(), + )); + } + for entry in &segment.entries { + if entry.request_id != request_id { + continue; + } + if !is_prepared_artifact_request_id(entry.command.kind, &entry.request_id) + || entry.result != *expected + || cached_lsn.is_some_and(|lsn| entry.lsn != lsn) + || found + { + return Err(MetadError::Codec( + "prepared publish archived result changed identity".to_owned(), + )); + } + found = true; + } + if !found && cached_lsn.is_some_and(|lsn| lsn <= pointer.last_lsn) { + return Err(MetadError::Codec( + "prepared publish cached archive locator is missing".to_owned(), + )); + } + expected_first_lsn = pointer.last_lsn.checked_add(1); + expected_prev_digest = pointer.last_digest; + } + if cached_lsn.is_some() && !found { + return Err(MetadError::Codec( + "prepared publish cached archive segment is missing".to_owned(), + )); + } + let observed_tail_lsn = expected_first_lsn + .map(|next_lsn| next_lsn - 1) + .unwrap_or(u64::MAX); + if observed_tail_lsn != proof.durable_tail_lsn + || expected_prev_digest != proof.durable_tail_digest + { + return Err(MetadError::Codec( + "prepared publish archive chain does not reach the durable log tail".to_owned(), + )); + } + Ok(found) + } + /// Resolve every ambiguous apply and flush the exact committed segment /// before a control-plane recovery reference is selected. This uses the /// same lock order as metadata commit and checkpoint export. @@ -714,6 +1205,32 @@ where .last_lsn .checked_add(1) .ok_or_else(|| MetadError::Codec("metadata log LSN is exhausted".to_owned()))?; + let mut prepared_results: BTreeMap, ArchivedPreparedRequestResult> = + BTreeMap::new(); + for entry in &segment.entries { + if !is_prepared_artifact_request_id(entry.command.kind, &entry.request_id) { + continue; + } + if let Some(existing) = state + .archived_prepared_request_results + .get(&entry.request_id) + .or_else(|| prepared_results.get(&entry.request_id)) + { + if existing.result != entry.result { + return Err(MetadError::Codec( + "prepared publish request id has conflicting archived results".to_owned(), + )); + } + } else { + prepared_results.insert( + entry.request_id.clone(), + ArchivedPreparedRequestResult { + result: entry.result.clone(), + lsn: entry.lsn, + }, + ); + } + } let archived = archive_metadata_log_segment_to_store(&self.objects, &state.config, segment)?; self.metadata_log_segments_archived_total @@ -730,6 +1247,10 @@ where last_lsn: segment.last_lsn, last_digest: segment.last_digest, }); + state + .archived_prepared_request_results + .extend(prepared_results); + trim_prepared_terminal_proof_cache(&mut state.archived_prepared_request_results); Ok(()) } @@ -795,6 +1316,39 @@ where state.pending_segment = Some(segment.clone()); self.archive_and_advance_metadata_log_state(state, &segment)?; state.pending_segment = None; - Ok(Some(state.snapshot())) + let snapshot = state.snapshot(); + drop(guard); + self.publish_archived_metadata_log_tail_immediately(&snapshot)?; + Ok(Some(snapshot)) + } +} + +#[cfg(test)] +mod prepared_terminal_proof_cache_tests { + use super::*; + + #[test] + fn proof_cache_is_bounded_and_evicts_oldest_lsn_first() { + let mut proofs = BTreeMap::new(); + for index in 0..(PREPARED_TERMINAL_PROOF_CACHE_LIMIT + 2) { + proofs.insert( + (index as u64).to_be_bytes().to_vec(), + ArchivedPreparedRequestResult { + result: CommitResult { + commit_version: Version::new(1).unwrap(), + applied_mutations: 1, + watch_events: 0, + }, + lsn: index as u64 + 1, + }, + ); + } + + trim_prepared_terminal_proof_cache(&mut proofs); + + assert_eq!(proofs.len(), PREPARED_TERMINAL_PROOF_CACHE_LIMIT); + assert!(!proofs.contains_key(0_u64.to_be_bytes().as_slice())); + assert!(!proofs.contains_key(1_u64.to_be_bytes().as_slice())); + assert_eq!(proofs.values().map(|proof| proof.lsn).min(), Some(3)); } } diff --git a/crates/nokv-meta/src/service/namespace.rs b/crates/nokv-meta/src/service/namespace.rs index 8843b260c..026ef7dbe 100644 --- a/crates/nokv-meta/src/service/namespace.rs +++ b/crates/nokv-meta/src/service/namespace.rs @@ -1,4 +1,7 @@ use super::*; +use std::collections::HashMap; + +const NAMESPACE_PROOF_PAGE_ROWS: usize = 256; struct PreparedCreateBatch { entries: Vec, @@ -13,13 +16,20 @@ struct PreparedRemoveFile { struct PreparedRemoveEmptyDir { entry: DentryWithAttr, command: MetadataCommand, + starts_restore_release: bool, +} + +#[derive(Default)] +struct FinalBodyCleanupPlan { + predicates: Vec, + mutations: Vec, } #[derive(Clone, Debug)] -struct LinkedDentryProjection { - key: Vec, - projection: DentryProjection, - version: Version, +pub(super) struct LinkedDentryProjection { + pub(super) key: Vec, + pub(super) projection: DentryProjection, + pub(super) version: Version, } #[derive(Default)] @@ -63,6 +73,7 @@ where // `new` starts fail-closed because callers may hand it a // pre-populated store. Once a root exists, one recovery proof // establishes the healthy fast-path or keeps slow checking on. + self.recover_restore_staging_visibility()?; self.recover_materialization_orphan_state()?; Ok(root) } @@ -168,22 +179,25 @@ where )); } let version = self.next_version()?; + let read_version = predecessor(version)?; let key = dentry_key(self.mount, parent, name); + let mut predicates = vec![PredicateRef { + family: RecordFamily::Dentry, + key: key.clone(), + predicate: Predicate::VersionEquals(dentry_version), + }]; + predicates.extend(self.restore_namespace_write_predicates(&[parent], read_version)?); let commit = self.commit_metadata(MetadataCommand { request_id: request_id(b"remove-graft", self.mount, entry.attr.inode, version), kind: CommandKind::RemoveEmptyDir, - read_version: predecessor(version)?, + read_version, commit_version: version, primary_family: RecordFamily::Dentry, primary_key: key.clone(), // Only the dentry version is guarded: there is no Inode record and no // local subtree to assert empty (the child's contents live on the // owning shard). - predicates: vec![PredicateRef { - family: RecordFamily::Dentry, - key: key.clone(), - predicate: Predicate::VersionEquals(dentry_version), - }], + predicates, mutations: vec![delete_mutation(RecordFamily::Dentry, key)], watch: self .watch_projection( @@ -456,12 +470,24 @@ where }, object_reference.predicate(self.mount), ]; + let restore_write_predicates = + self.restore_namespace_write_predicates(&[inode, new_parent], read_version)?; + let restore_guarded = + super::restore::restore_write_predicates_include_owner(&restore_write_predicates); + predicates.extend(restore_write_predicates); + let enrollment = self.restore_namespace_enrollment_plan( + new_parent, + std::slice::from_ref(&new_projection), + read_version, + )?; + predicates.extend(enrollment.predicates); let mut mutations = vec![Mutation { family: RecordFamily::Inode, key: source_inode_key, op: MutationOp::Put, value: Some(Value(encode_inode_attr(&attr))), }]; + let mut updated_existing_links = Vec::with_capacity(linked.len()); for linked in &linked { predicates.push(PredicateRef { family: RecordFamily::Dentry, @@ -471,6 +497,7 @@ where let mut projection = linked.projection.clone(); projection.attr = attr.clone(); projection.dentry.attr_generation = attr.generation; + updated_existing_links.push((linked.projection.clone(), projection.clone())); mutations.push(put_projection_mutation( RecordFamily::Dentry, linked.key.clone(), @@ -482,7 +509,11 @@ where destination_key.clone(), &new_projection, )); - self.commit_metadata(MetadataCommand { + mutations.extend(enrollment.mutations); + let restore_index = self.restore_index_link_plan(&updated_existing_links, version)?; + predicates.extend(restore_index.predicates); + mutations.extend(restore_index.mutations); + let command = MetadataCommand { request_id: request_id(b"link", self.mount, inode, version), kind: CommandKind::Link, read_version, @@ -504,7 +535,11 @@ where ) .into_iter() .collect(), - })?; + }; + if restore_guarded { + super::restore::validate_restore_command_bounds(&command, "restore namespace link")?; + } + self.commit_metadata(command)?; Ok(new_projection.into()) } @@ -604,6 +639,7 @@ where let projection = projection(parent, name.clone(), attr, body); self.commit_replace_projection_with_chunks(ReplaceProjectionCommit { + request_id: None, kind: CommandKind::UpdateAttr, projection: &projection, chunks: &chunks, @@ -1024,6 +1060,7 @@ where let mut canonical_attr = decode_inode_attr(&inode_item.value.0) .map_err(|err| MetadError::Codec(err.to_string()))?; let key = dentry_key(self.mount, parent, name); + let mut predicates = Vec::new(); let mut mutations = vec![delete_mutation(RecordFamily::Dentry, key.clone())]; if let Some(path_index) = self.live_path_index_key_for_entry(path_components, parent, name, &entry, version)? @@ -1036,9 +1073,9 @@ where )); } let final_link = canonical_attr.nlink == 1; - let linked = if final_link { + let (linked, updated_remaining) = if final_link { mutations.push(delete_mutation(RecordFamily::Inode, inode_key.clone())); - Vec::new() + (Vec::new(), Vec::new()) } else { let linked = self.linked_dentry_projections_for_inode(entry.attr.inode, predecessor(version)?)?; @@ -1051,6 +1088,7 @@ where op: MutationOp::Put, value: Some(Value(encode_inode_attr(&canonical_attr))), }); + let mut updated_remaining = Vec::with_capacity(linked.len().saturating_sub(1)); for linked in &linked { if linked.key == key { continue; @@ -1058,25 +1096,28 @@ where let mut projection = linked.projection.clone(); projection.attr = canonical_attr.clone(); projection.dentry.attr_generation = canonical_attr.generation; + updated_remaining.push((linked.projection.clone(), projection.clone())); mutations.push(put_projection_mutation( RecordFamily::Dentry, linked.key.clone(), &projection, )); } - linked + (linked, updated_remaining) }; if final_link { if let Some(body) = &entry.body { - mutations.extend(self.chunk_manifest_delete_and_gc_mutations( + let cleanup = self.final_body_cleanup_plan( entry.attr.inode, - body.generation, + body, + predecessor(version)?, version, - &HashSet::new(), - )?); + )?; + predicates.extend(cleanup.predicates); + mutations.extend(cleanup.mutations); } } - let mut predicates = vec![ + predicates.extend([ PredicateRef { family: RecordFamily::Dentry, key: key.clone(), @@ -1087,7 +1128,14 @@ where key: inode_key, predicate: Predicate::VersionEquals(inode_item.version), }, - ]; + ]); + let restore_write_predicates = self.restore_namespace_write_predicates( + &[parent, entry.attr.inode], + predecessor(version)?, + )?; + let restore_guarded = + super::restore::restore_write_predicates_include_owner(&restore_write_predicates); + predicates.extend(restore_write_predicates); for linked in linked { if linked.key == dentry_key(self.mount, parent, name) { continue; @@ -1098,6 +1146,15 @@ where predicate: Predicate::VersionEquals(linked.version), }); } + let removed_projection = DentryProjection { + dentry: entry.dentry.clone(), + attr: entry.attr.clone(), + body: entry.body.clone(), + }; + let restore_index = + self.restore_index_remove_plan(&removed_projection, &updated_remaining, version)?; + predicates.extend(restore_index.predicates); + mutations.extend(restore_index.mutations); let command = MetadataCommand { request_id: request_id(b"remove-file", self.mount, entry.attr.inode, version), kind: CommandKind::RemoveFile, @@ -1121,9 +1178,151 @@ where .into_iter() .collect(), }; + if restore_guarded { + super::restore::validate_restore_command_bounds( + &command, + "restore namespace remove file", + )?; + } Ok(PreparedRemoveFile { entry, command }) } + /// Build the atomic metadata closure for deleting the last namespace link + /// to a file body. An append or sparse publish stores only its dirty chunks + /// in the top generation and falls through `base_generation`; deleting the + /// top rows alone strands the older summaries/manifests and leaves their + /// objects without a fresh GC candidate. + /// + /// Every physical row in the reachable generation chain is version-fenced + /// and deleted by the caller's namespace transaction. Every canonical block + /// minted by this inode is enqueued using the generation encoded in its + /// object key. That distinction matters after chain compaction: a new + /// self-contained manifest can retain an unchanged block from an older + /// generation whose physical manifest row was already deleted. Cloned + /// manifests contain keys owned by another inode; those borrowed keys stay + /// protected by their source plus snapshot/exact-reference policy. + fn final_body_cleanup_plan( + &self, + inode: InodeId, + body: &BodyDescriptor, + read_version: Version, + enqueue_version: Version, + ) -> Result { + let generations = + self.resolve_generation_chain(inode, body, read_version, ReadPurpose::WritePlanLocal)?; + let enqueue_unix_ms = current_time_ms(); + let mut queued_objects: HashMap = HashMap::new(); + let mut plan = FinalBodyCleanupPlan::default(); + + for generation in generations { + let prefix = chunk_manifest_prefix(self.mount, inode, generation); + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::ChunkManifest, + prefix: prefix.clone(), + start_after: None, + version: read_version, + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + })?; + let mut saw_summary = false; + + for row in rows { + if row.key.len() != prefix.len() + std::mem::size_of::() + || !row.key.starts_with(&prefix) + { + return Err(MetadError::Codec( + "chunk manifest scan returned a key outside its generation prefix" + .to_owned(), + )); + } + let chunk_index = chunk_index_from_manifest_key(&row.key)?; + plan.predicates.push(PredicateRef { + family: RecordFamily::ChunkManifest, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }); + plan.mutations + .push(delete_mutation(RecordFamily::ChunkManifest, row.key)); + + if chunk_index == BODY_SUMMARY_CHUNK_INDEX { + let summary = decode_body_descriptor(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + if summary.generation != generation { + return Err(MetadError::Codec( + "body summary generation does not match its manifest key".to_owned(), + )); + } + saw_summary = true; + continue; + } + + let manifest = decode_chunk_manifest(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + if manifest.chunk_index != chunk_index { + return Err(MetadError::Codec( + "chunk manifest index does not match its manifest key".to_owned(), + )); + } + for block in manifest.slices.iter().flat_map(|slice| &slice.blocks) { + if !self.block_object_is_owned_by_inode(inode, &block.object_key)? { + continue; + } + let (owner, object_generation, object_chunk, block_index) = + self.canonical_block_object_identity(&block.object_key)?; + if owner != inode || object_chunk != chunk_index { + return Err(MetadError::Codec( + "owned block object identity does not match its manifest".to_owned(), + )); + } + + match queued_objects.get(&block.object_key) { + Some((_, digest_uri)) if digest_uri != &block.digest_uri => { + return Err(MetadError::Codec( + "one block object key has inconsistent manifest identity" + .to_owned(), + )); + } + Some(_) => continue, + None => { + queued_objects.insert( + block.object_key.clone(), + (block.len, block.digest_uri.clone()), + ); + } + } + let record = ObjectGcRecord { + inode: owner, + generation: object_generation, + object_key: block.object_key.clone(), + size: block.len, + digest_uri: block.digest_uri.clone(), + enqueue_version: enqueue_version.get(), + enqueue_unix_ms, + }; + plan.mutations.push(Mutation { + family: RecordFamily::Gc, + key: gc_object_key( + self.mount, + enqueue_version.get(), + owner, + object_generation, + object_chunk, + block_index, + ), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_record(&record))), + }); + } + } + + if !saw_summary { + return Err(MetadError::MissingBodyDescriptor); + } + } + + Ok(plan) + } + pub fn remove_file_path(&self, path: &str) -> Result { let components = parse_absolute_path(path)?; let Some((name, parent_components)) = components.split_last() else { @@ -1195,7 +1394,12 @@ where ) -> Result { let version = self.next_version()?; let prepared = self.prepare_remove_empty_dir(parent, name, path_components, version)?; - map_remove_empty_dir_commit(self.commit_metadata(prepared.command))?; + map_remove_empty_dir_commit( + self.commit_restore_release_transition( + prepared.command, + prepared.starts_restore_release, + ), + )?; Ok(prepared.entry) } @@ -1233,6 +1437,40 @@ where { mutations.push(delete_mutation(RecordFamily::PathIndex, path_index)); } + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::Dentry, + key: source_key.clone(), + predicate: Predicate::VersionEquals(dentry_version), + }, + PredicateRef { + family: RecordFamily::Dentry, + key: child_prefix, + predicate: Predicate::PrefixEmpty, + }, + ]; + let restore_write_predicates = self.restore_namespace_write_predicates( + &[parent, entry.attr.inode], + predecessor(version)?, + )?; + let restore_guarded = + super::restore::restore_write_predicates_include_owner(&restore_write_predicates); + predicates.extend(restore_write_predicates); + let restore_release = self.prepare_restore_root_release(entry.attr.inode, version)?; + if let Some(release) = restore_release.as_ref() { + predicates.extend(release.predicates.clone()); + mutations.extend(release.mutations.clone()); + } + let restore_index = self.restore_index_unlink_plan( + &DentryProjection { + dentry: entry.dentry.clone(), + attr: entry.attr.clone(), + body: entry.body.clone(), + }, + version, + )?; + predicates.extend(restore_index.predicates); + mutations.extend(restore_index.mutations); let command = MetadataCommand { request_id: request_id(b"remove-empty-dir", self.mount, entry.attr.inode, version), kind: CommandKind::RemoveEmptyDir, @@ -1240,18 +1478,7 @@ where commit_version: version, primary_family: RecordFamily::Dentry, primary_key: source_key.clone(), - predicates: vec![ - PredicateRef { - family: RecordFamily::Dentry, - key: source_key.clone(), - predicate: Predicate::VersionEquals(dentry_version), - }, - PredicateRef { - family: RecordFamily::Dentry, - key: child_prefix, - predicate: Predicate::PrefixEmpty, - }, - ], + predicates, mutations, watch: self .watch_projection( @@ -1267,7 +1494,17 @@ where .into_iter() .collect(), }; - Ok(PreparedRemoveEmptyDir { entry, command }) + if restore_guarded { + super::restore::validate_restore_command_bounds( + &command, + "restore namespace remove directory", + )?; + } + Ok(PreparedRemoveEmptyDir { + entry, + command, + starts_restore_release: restore_release.is_some(), + }) } pub fn remove_empty_dir_path(&self, path: &str) -> Result { @@ -1303,11 +1540,26 @@ where } } - let commands = prepared + let committed = if prepared .iter() - .map(|(_, remove)| remove.command.clone()) - .collect::>(); - let committed = self.commit_independent_metadata_batch(&commands); + .any(|(_, remove)| remove.starts_restore_release) + { + prepared + .iter() + .map(|(_, remove)| { + self.commit_restore_release_transition( + remove.command.clone(), + remove.starts_restore_release, + ) + }) + .collect() + } else { + let commands = prepared + .iter() + .map(|(_, remove)| remove.command.clone()) + .collect::>(); + self.commit_independent_metadata_batch(&commands) + }; for ((index, remove), result) in prepared.into_iter().zip(committed) { results[index] = Some(map_remove_empty_dir_commit(result).map(|_| remove.entry)); } @@ -1480,20 +1732,61 @@ where return Err(MetadError::GraftPoint); } } - if replace { - if source.attr.file_type == FileType::Directory { - return Err(MetadError::NotFile); - } - if let Some((entry, _)) = &destination { - if entry.attr.file_type == FileType::Directory { - return Err(MetadError::NotFile); - } - } + let replacing_complete_restore_directory = if replace + && source.attr.file_type == FileType::Directory + && destination + .as_ref() + .is_some_and(|(entry, _)| entry.attr.file_type == FileType::Directory) + { + self.is_complete_restore_root( + destination + .as_ref() + .expect("checked destination") + .0 + .attr + .inode, + )? + } else { + false + }; + let moving_complete_restore_directory = if replace + && source.attr.file_type == FileType::Directory + && destination + .as_ref() + .is_some_and(|(entry, _)| entry.attr.file_type == FileType::Directory) + { + self.is_complete_restore_root(source.attr.inode)? + } else { + false + }; + if replace + && (source.attr.file_type == FileType::Directory + || destination + .as_ref() + .is_some_and(|(entry, _)| entry.attr.file_type == FileType::Directory)) + && !replacing_complete_restore_directory + && !moving_complete_restore_directory + { + return Err(MetadError::NotFile); } let object_reference = self.begin_object_reference_mutation()?; let version = self.next_version()?; let read_version = predecessor(version)?; + let restore_release = if replacing_complete_restore_directory { + let victim = destination + .as_ref() + .expect("checked restore victim") + .0 + .attr + .inode; + Some( + self.prepare_restore_root_release(victim, version)? + .ok_or(MetadError::RestoreRootChanged { root: victim })?, + ) + } else { + None + }; let source_key = dentry_key(self.mount, parent, name); let destination_key = dentry_key(self.mount, new_parent, &new_name); let reachable = self.namespace_reachability_for_exposure(read_version)?; @@ -1523,6 +1816,21 @@ where }, object_reference.predicate(self.mount), ]; + let mut guarded_inodes = vec![parent, new_parent, source.attr.inode]; + if let Some((entry, _)) = &destination { + guarded_inodes.push(entry.attr.inode); + } + let restore_write_predicates = + self.restore_namespace_write_predicates(&guarded_inodes, read_version)?; + let restore_guarded = + super::restore::restore_write_predicates_include_owner(&restore_write_predicates); + predicates.extend(restore_write_predicates); + let enrollment = self.restore_namespace_enrollment_plan( + new_parent, + std::slice::from_ref(&projection), + read_version, + )?; + predicates.extend(enrollment.predicates); let replaced = if let Some((entry, destination_version)) = destination { predicates.push(PredicateRef { family: RecordFamily::Dentry, @@ -1548,6 +1856,8 @@ where value: Some(Value(encode_dentry_projection(&projection))), }, ]; + mutations.extend(enrollment.mutations); + let mut replaced_link_updates = Vec::new(); if let Some(source_path) = self.live_path_index_key_for_entry( path_index.map(|(source, _)| source), parent, @@ -1592,15 +1902,35 @@ where key: replaced_inode_key.clone(), predicate: Predicate::VersionEquals(replaced_inode_item.version), }); - if replaced_attr.nlink == 1 { + if moving_complete_restore_directory && !replacing_complete_restore_directory { + // POSIX directory replacement requires an empty ordinary + // victim. The restored source stays Complete and merely moves; + // the victim is removed with explicit Dentry/Xattr closure so + // it cannot leave a hidden namespace tail behind. + predicates.extend([ + PredicateRef { + family: RecordFamily::Dentry, + key: dentry_prefix(self.mount, replaced.attr.inode), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::Xattr, + key: xattr_prefix(self.mount, replaced.attr.inode), + predicate: Predicate::PrefixEmpty, + }, + ]); + mutations.push(delete_mutation(RecordFamily::Inode, replaced_inode_key)); + } else if replaced_attr.nlink == 1 { mutations.push(delete_mutation(RecordFamily::Inode, replaced_inode_key)); if let Some(body) = &replaced.body { - mutations.extend(self.chunk_manifest_delete_and_gc_mutations( + let cleanup = self.final_body_cleanup_plan( replaced.attr.inode, - body.generation, + body, + read_version, version, - &HashSet::new(), - )?); + )?; + predicates.extend(cleanup.predicates); + mutations.extend(cleanup.mutations); } } else { replaced_attr.nlink -= 1; @@ -1623,17 +1953,42 @@ where key: linked.key.clone(), predicate: Predicate::VersionEquals(linked.version), }); - let mut projection = linked.projection; - projection.attr = replaced_attr.clone(); - projection.dentry.attr_generation = replaced_attr.generation; + let old_projection = linked.projection; + let mut updated_projection = old_projection.clone(); + updated_projection.attr = replaced_attr.clone(); + updated_projection.dentry.attr_generation = replaced_attr.generation; + replaced_link_updates.push((old_projection, updated_projection.clone())); mutations.push(put_projection_mutation( RecordFamily::Dentry, linked.key, - &projection, + &updated_projection, )); } } } + if let Some(release) = restore_release { + predicates.extend(release.predicates); + mutations.extend(release.mutations); + } + let source_projection = DentryProjection { + dentry: source.dentry.clone(), + attr: source.attr.clone(), + body: source.body.clone(), + }; + let replaced_projection = replaced.as_ref().map(|entry| DentryProjection { + dentry: entry.dentry.clone(), + attr: entry.attr.clone(), + body: entry.body.clone(), + }); + let mut restore_index = self.restore_index_rename_plan( + &source_projection, + &projection, + replaced_projection.as_ref(), + version, + )?; + restore_index.extend(self.restore_index_link_plan(&replaced_link_updates, version)?)?; + predicates.extend(restore_index.predicates); + mutations.extend(restore_index.mutations); let mut watch = Vec::new(); if let Some(replaced) = &replaced { if let Some(event) = self.watch_projection( @@ -1674,7 +2029,7 @@ where watch.push(event); } - self.commit_metadata(MetadataCommand { + let command = MetadataCommand { request_id: request_id( if replace { b"rename-replace" @@ -1697,7 +2052,11 @@ where predicates, mutations, watch, - })?; + }; + if restore_guarded { + super::restore::validate_restore_command_bounds(&command, "restore namespace rename")?; + } + self.commit_restore_release_transition(command, replacing_complete_restore_directory)?; Ok(RenameReplaceResult { entry: projection.into(), replaced, @@ -1736,34 +2095,135 @@ where Ok(matches_canonical.then_some(key)) } - fn linked_dentry_projections_for_inode( + pub(super) fn linked_dentry_projections_for_inode( &self, inode: InodeId, version: Version, ) -> Result, MetadError> { - let rows = self.metadata.scan(ScanRequest { - family: RecordFamily::Dentry, - prefix: dentry_mount_prefix(self.mount), - start_after: None, - version, - limit: 0, - purpose: ReadPurpose::WritePlanLocal, - })?; let mut linked = Vec::new(); - for row in rows { - let projection = decode_dentry_projection(&row.value.0) - .map_err(|err| MetadError::Codec(err.to_string()))?; - if projection.attr.inode == inode { + self.visit_linked_dentry_projections_for_inode(inode, version, |projection| { + linked.push(projection); + Ok(false) + })?; + Ok(linked) + } + + /// Return at most `limit` current links for restore cleanup. The physical + /// mount scan is always paged and stops as soon as the requested number of + /// matching links is found; callers never materialize an unbounded hardlink + /// fanout merely to delete one bounded release batch. + pub(super) fn restore_linked_dentry_projection_page( + &self, + inode: InodeId, + version: Version, + limit: usize, + ) -> Result, MetadError> { + let limit = limit.clamp(1, NAMESPACE_PROOF_PAGE_ROWS); + let mut linked = Vec::with_capacity(limit); + let prefix = dentry_mount_prefix(self.mount); + let mut start_after = None; + 'pages: loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::Dentry, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: NAMESPACE_PROOF_PAGE_ROWS, + purpose: ReadPurpose::RestoreStaging, + })?; + if rows.is_empty() { + break; + } + for row in &rows { + let projection = decode_dentry_projection(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + if projection.attr.inode != inode { + continue; + } + let expected_key = dentry_key( + self.mount, + projection.dentry.parent, + &projection.dentry.name, + ); + if row.key != expected_key + || projection.dentry.child != projection.attr.inode + || projection.dentry.child_type != projection.attr.file_type + || projection.dentry.attr_generation != projection.attr.generation + || projection.body.as_ref().is_some_and(|body| { + body.size != projection.attr.size + || body.generation == 0 + || body.generation > projection.attr.generation + }) + { + return Err(MetadError::Codec(format!( + "restore release dentry changed identity for inode {}", + inode.get() + ))); + } linked.push(LinkedDentryProjection { - key: row.key, + key: row.key.clone(), projection, version: row.version, }); + if linked.len() >= limit { + break 'pages; + } + } + let reached_tail = rows.len() < NAMESPACE_PROOF_PAGE_ROWS; + start_after = rows.last().map(|row| row.key.clone()); + if reached_tail { + break; } } Ok(linked) } + fn visit_linked_dentry_projections_for_inode( + &self, + inode: InodeId, + version: Version, + mut visit: F, + ) -> Result + where + F: FnMut(LinkedDentryProjection) -> Result, + { + let prefix = dentry_mount_prefix(self.mount); + let mut start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::Dentry, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: NAMESPACE_PROOF_PAGE_ROWS, + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + return Ok(false); + } + for row in &rows { + let projection = decode_dentry_projection(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + if projection.attr.inode != inode { + continue; + } + self.validate_current_dentry_projection(&row.key, &projection, version)?; + if visit(LinkedDentryProjection { + key: row.key.clone(), + projection, + version: row.version, + })? { + return Ok(true); + } + } + let reached_tail = rows.len() < NAMESPACE_PROOF_PAGE_ROWS; + start_after = rows.last().map(|row| row.key.clone()); + if reached_tail { + return Ok(false); + } + } + } + /// Enter fail-closed mode before the first metadata write of a detached /// materialization. Callers hold `object_gc_gate`, so a link/rename cannot /// observe the old healthy state and then race the first orphan-producing @@ -1888,39 +2348,58 @@ where version: Version, ) -> Result { let purpose = ReadPurpose::WritePlanLocal; - let dentry_rows = self.metadata.scan(ScanRequest { - family: RecordFamily::Dentry, - prefix: dentry_mount_prefix(self.mount), - start_after: None, - version, - limit: 0, - purpose, - })?; - if dentry_rows - .iter() - .any(|row| !reachable.dentries.contains(&row.key)) - { - return Ok(false); + let dentry_prefix = dentry_mount_prefix(self.mount); + let mut start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::Dentry, + prefix: dentry_prefix.clone(), + start_after: start_after.clone(), + version, + limit: NAMESPACE_PROOF_PAGE_ROWS, + purpose, + })?; + if rows + .iter() + .any(|row| !reachable.dentries.contains(&row.key)) + { + return Ok(false); + } + let reached_tail = rows.len() < NAMESPACE_PROOF_PAGE_ROWS; + start_after = rows.last().map(|row| row.key.clone()); + if reached_tail { + break; + } } - for row in self.metadata.scan(ScanRequest { - family: RecordFamily::Inode, - prefix: inode_prefix(self.mount), - start_after: None, - version, - limit: 0, - purpose, - })? { - let attr = decode_inode_attr(&row.value.0) - .map_err(|err| MetadError::Codec(err.to_string()))?; - if row.key != inode_key(self.mount, attr.inode) { - return Err(MetadError::Codec(format!( - "inode row key does not match inode {} during materialization recovery", - attr.inode.get() - ))); + let inode_prefix = inode_prefix(self.mount); + let mut start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::Inode, + prefix: inode_prefix.clone(), + start_after: start_after.clone(), + version, + limit: NAMESPACE_PROOF_PAGE_ROWS, + purpose, + })?; + for row in &rows { + let attr = decode_inode_attr(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + if row.key != inode_key(self.mount, attr.inode) { + return Err(MetadError::Codec(format!( + "inode row key does not match inode {} during materialization recovery", + attr.inode.get() + ))); + } + if !reachable.inodes.contains(&attr.inode) { + return Ok(false); + } } - if !reachable.inodes.contains(&attr.inode) { - return Ok(false); + let reached_tail = rows.len() < NAMESPACE_PROOF_PAGE_ROWS; + start_after = rows.last().map(|row| row.key.clone()); + if reached_tail { + break; } } Ok(true) @@ -1950,12 +2429,79 @@ where &self, version: Version, ) -> Result { + let mut reachable = NamespaceReachability::default(); + self.visit_namespace_reachable_entries_at(version, |key, attr, _| { + reachable.inodes.insert(attr.inode); + if attr.inode.shard_index() == self.shard_index() + && attr.file_type == FileType::Directory + { + reachable.directories.insert(attr.inode); + } + if let Some(key) = key { + reachable.dentries.insert(key.to_vec()); + } + Ok(false) + })?; + Ok(reachable) + } + + /// Prove a batch of inode reachability with one namespace walk. The walk + /// stops as soon as every candidate is found; an unreachable candidate + /// requires a complete but physically paged traversal at one stable MVCC + /// version. + pub(super) fn restore_reachable_inodes_at( + &self, + candidates: &HashSet, + version: Version, + ) -> Result, MetadError> { + if candidates.is_empty() { + return Ok(HashSet::new()); + } + let mut found = HashSet::with_capacity(candidates.len()); + self.visit_namespace_reachable_entries_at(version, |_, attr, _| { + if candidates.contains(&attr.inode) { + found.insert(attr.inode); + } + Ok(found.len() == candidates.len()) + })?; + Ok(found) + } + + /// Return one reachable body descriptor per candidate inode in one walk. + /// A reachable body-less inode is represented by `None`; an absent map key + /// means the inode is not reachable from any namespace anchor. + pub(super) fn restore_reachable_inode_bodies_at( + &self, + candidates: &HashSet, + version: Version, + ) -> Result>, MetadError> { + if candidates.is_empty() { + return Ok(HashMap::new()); + } + let mut found = HashMap::with_capacity(candidates.len()); + self.visit_namespace_reachable_entries_at(version, |_, attr, body| { + if candidates.contains(&attr.inode) { + found.entry(attr.inode).or_insert_with(|| body.cloned()); + } + Ok(found.len() == candidates.len()) + })?; + Ok(found) + } + + fn visit_namespace_reachable_entries_at( + &self, + version: Version, + mut visit: F, + ) -> Result + where + F: FnMut(Option<&[u8]>, &InodeAttr, Option<&BodyDescriptor>) -> Result, + { #[cfg(test)] self.namespace_reachability_scans .fetch_add(1, Ordering::Relaxed); let purpose = ReadPurpose::WritePlanLocal; let Some(root_attr) = - self.get_attr_at_version_for_purpose(InodeId::root(), version, purpose)? + self.namespace_reachability_inode_attr_at(InodeId::root(), version, purpose)? else { return Err(MetadError::Codec( "mount root is missing during namespace reachability proof".to_owned(), @@ -1966,12 +2512,19 @@ where "mount root is not a directory during namespace reachability proof".to_owned(), )); } + if visit(None, &root_attr, None)? { + return Ok(true); + } let mut pending = vec![InodeId::root()]; - for versioned in self.versioned_fork_bindings_at(version, purpose)? { + if self.visit_versioned_fork_bindings_at(version, purpose, |versioned| { + if !self.restore_fork_binding_is_namespace_anchor(&versioned.binding, version)? { + return Ok(false); + } let root = versioned.binding.fork_root; - let Some(attr) = self.get_attr_at_version_for_purpose(root, version, purpose)? else { - continue; + let Some(attr) = self.namespace_reachability_inode_attr_at(root, version, purpose)? + else { + return Ok(false); }; if attr.file_type != FileType::Directory { return Err(MetadError::Codec(format!( @@ -1980,35 +2533,88 @@ where ))); } pending.push(root); + visit(None, &attr, None) + })? { + return Ok(true); } - let mut reachable = NamespaceReachability::default(); + let mut visited_directories = HashSet::new(); while let Some(parent) = pending.pop() { - if !reachable.directories.insert(parent) { + if !visited_directories.insert(parent) { continue; } - reachable.inodes.insert(parent); - for row in self.metadata.scan(ScanRequest { - family: RecordFamily::Dentry, - prefix: dentry_prefix(self.mount, parent), - start_after: None, - version, - limit: 0, - purpose, - })? { - let projection = decode_dentry_projection(&row.value.0) - .map_err(|err| MetadError::Codec(err.to_string()))?; - self.validate_current_dentry_projection(&row.key, &projection, version)?; - reachable.dentries.insert(row.key); - reachable.inodes.insert(projection.attr.inode); - if projection.attr.inode.shard_index() == self.shard_index() - && projection.attr.file_type == FileType::Directory - { - pending.push(projection.attr.inode); + let prefix = dentry_prefix(self.mount, parent); + let mut start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::Dentry, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: NAMESPACE_PROOF_PAGE_ROWS, + purpose, + })?; + for row in &rows { + let projection = decode_dentry_projection(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + self.validate_current_dentry_projection(&row.key, &projection, version)?; + if visit(Some(&row.key), &projection.attr, projection.body.as_ref())? { + return Ok(true); + } + if projection.attr.inode.shard_index() == self.shard_index() + && projection.attr.file_type == FileType::Directory + { + pending.push(projection.attr.inode); + } + } + let reached_tail = rows.len() < NAMESPACE_PROOF_PAGE_ROWS; + start_after = rows.last().map(|row| row.key.clone()); + if reached_tail { + break; } } } - Ok(reachable) + Ok(false) + } + + /// Read an inode for the internal reachability graph without re-entering + /// restore visibility. The graph itself is the visibility proof, and it is + /// also called while the restore visibility fence may already be held. + fn namespace_reachability_inode_attr_at( + &self, + inode: InodeId, + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadError> { + let Some(value) = self.metadata.get( + RecordFamily::Inode, + &inode_key(self.mount, inode), + version, + purpose, + )? + else { + return Ok(None); + }; + let attr = + decode_inode_attr(&value.0).map_err(|error| MetadError::Codec(error.to_string()))?; + if attr.inode != inode { + return Err(MetadError::Codec(format!( + "inode row key does not match inode {} during namespace reachability proof", + inode.get() + ))); + } + Ok(Some(attr)) + } + + pub(super) fn restore_inode_reachable_from_mount( + &self, + inode: InodeId, + version: Version, + ) -> Result { + let candidates = HashSet::from([inode]); + Ok(self + .restore_reachable_inodes_at(&candidates, version)? + .contains(&inode)) } pub(super) fn commit_create_projection( @@ -2024,6 +2630,7 @@ where version, None, None, + None, ) } @@ -2064,6 +2671,11 @@ where key: inode_key(self.mount, parent), predicate: Predicate::Exists, }]; + let restore_write_predicates = + self.restore_namespace_write_predicates(&[parent], predecessor(version)?)?; + let restore_guarded = + super::restore::restore_write_predicates_include_owner(&restore_write_predicates); + predicates.extend(restore_write_predicates); let mut mutations = Vec::with_capacity(projections.len() * if path_indexes.is_some() { 3 } else { 2 }); let mut watch = Vec::with_capacity(projections.len()); @@ -2115,7 +2727,11 @@ where watch.push(event); } } - Ok(MetadataCommand { + let enrollment = + self.restore_namespace_enrollment_plan(parent, projections, predecessor(version)?)?; + predicates.extend(enrollment.predicates); + mutations.extend(enrollment.mutations); + let command = MetadataCommand { request_id: request_id(kind_name(kind), self.mount, parent, version), kind, read_version: predecessor(version)?, @@ -2125,7 +2741,14 @@ where predicates, mutations, watch, - }) + }; + if restore_guarded { + super::restore::validate_restore_command_bounds( + &command, + "restore namespace create batch", + )?; + } + Ok(command) } /// Build the cross-shard graft command. Unlike every other create path this @@ -2141,7 +2764,36 @@ where ) -> Result { let parent = projection.dentry.parent; let dentry = dentry_key(self.mount, parent, &projection.dentry.name); - Ok(MetadataCommand { + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, parent), + predicate: Predicate::Exists, + }, + PredicateRef { + family: RecordFamily::Dentry, + key: dentry.clone(), + predicate: Predicate::NotExists, + }, + ]; + let restore_write_predicates = + self.restore_namespace_write_predicates(&[parent], predecessor(version)?)?; + let restore_guarded = + super::restore::restore_write_predicates_include_owner(&restore_write_predicates); + predicates.extend(restore_write_predicates); + let enrollment = self.restore_namespace_enrollment_plan( + parent, + std::slice::from_ref(projection), + predecessor(version)?, + )?; + predicates.extend(enrollment.predicates); + let mut mutations = vec![put_projection_mutation( + RecordFamily::Dentry, + dentry.clone(), + projection, + )]; + mutations.extend(enrollment.mutations); + let command = MetadataCommand { request_id: request_id( kind_name(CommandKind::CreateDir), self.mount, @@ -2153,25 +2805,10 @@ where commit_version: version, primary_family: RecordFamily::Dentry, primary_key: dentry.clone(), - predicates: vec![ - PredicateRef { - family: RecordFamily::Inode, - key: inode_key(self.mount, parent), - predicate: Predicate::Exists, - }, - PredicateRef { - family: RecordFamily::Dentry, - key: dentry.clone(), - predicate: Predicate::NotExists, - }, - ], + predicates, // Exactly one mutation: the dentry projection. No Inode record for the // foreign child — that is the allocator-safety invariant of a graft. - mutations: vec![put_projection_mutation( - RecordFamily::Dentry, - dentry, - projection, - )], + mutations, watch: self .watch_projection( parent, @@ -2185,7 +2822,14 @@ where ) .into_iter() .collect(), - }) + }; + if restore_guarded { + super::restore::validate_restore_command_bounds( + &command, + "restore namespace create graft", + )?; + } + Ok(command) } pub(super) fn commit_create_projection_with_chunks( @@ -2203,6 +2847,7 @@ where version, None, Some(object_reference), + None, ) } @@ -2215,6 +2860,7 @@ where version: Version, path_index: Option>, object_reference: Option, + request_id_override: Option>, ) -> Result<(), MetadError> { let inode = projection.attr.inode; let dentry = dentry_key( @@ -2271,11 +2917,26 @@ where predicate: Predicate::NotExists, }, ]; + let restore_write_predicates = self.restore_namespace_write_predicates( + &[projection.dentry.parent], + predecessor(version)?, + )?; + let restore_guarded = + super::restore::restore_write_predicates_include_owner(&restore_write_predicates); + predicates.extend(restore_write_predicates); + let enrollment = self.restore_namespace_enrollment_plan( + projection.dentry.parent, + std::slice::from_ref(projection), + predecessor(version)?, + )?; + predicates.extend(enrollment.predicates); + mutations.extend(enrollment.mutations); if let Some(object_reference) = object_reference { predicates.push(object_reference.predicate(self.mount)); } - self.commit_metadata(MetadataCommand { - request_id: request_id(kind_name(kind), self.mount, inode, version), + let command = MetadataCommand { + request_id: request_id_override + .unwrap_or_else(|| request_id(kind_name(kind), self.mount, inode, version)), kind, read_version: predecessor(version)?, commit_version: version, @@ -2296,7 +2957,14 @@ where ) .into_iter() .collect(), - })?; + }; + if restore_guarded { + super::restore::validate_restore_command_bounds( + &command, + "restore namespace publish create", + )?; + } + self.commit_metadata(command)?; Ok(()) } @@ -2305,6 +2973,7 @@ where commit: ReplaceProjectionCommit<'_>, ) -> Result<(), MetadError> { let ReplaceProjectionCommit { + request_id: request_id_override, kind, projection, chunks, @@ -2322,10 +2991,37 @@ where &projection.dentry.name, ); let read_version = predecessor(version)?; + let restore_write_predicates = self + .restore_namespace_write_predicates(&[projection.dentry.parent, inode], read_version)?; + let restore_guarded = + super::restore::restore_write_predicates_include_owner(&restore_write_predicates); let linked = if projection.attr.nlink <= 1 { + let old_projection = if restore_guarded { + let old = self + .metadata + .get_versioned( + RecordFamily::Dentry, + &dentry, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + if old.version != dentry_version { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + decode_dentry_projection(&old.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))? + } else { + // Preserve the original prepared-publish contract outside a + // completed restore. In particular, do not require historical + // dentry state which may already have been pruned: the exact + // VersionEquals CAS below reports a stale plan, and Holt's + // request-id dedupe makes an identical retry idempotent. + projection.clone() + }; vec![LinkedDentryProjection { key: dentry.clone(), - projection: projection.clone(), + projection: old_projection, version: dentry_version, }] } else { @@ -2343,6 +3039,20 @@ where predicate: Predicate::Exists, }, ]; + predicates.extend(restore_write_predicates); + let primary_overlay_managed = if restore_guarded && path_index.is_some() { + let primary = linked + .iter() + .find(|linked| linked.key == dentry) + .ok_or_else(|| { + MetadError::Codec( + "restore publish primary dentry is missing from its inode links".to_owned(), + ) + })?; + self.restore_index_manages_projection_location(&primary.projection, read_version)? + } else { + false + }; if let Some(object_reference) = object_reference { predicates.push(object_reference.predicate(self.mount)); } @@ -2353,6 +3063,7 @@ where value: Some(Value(encode_inode_attr(&projection.attr))), }]; let mut primary_projection_updated = false; + let mut updated_link_projections = Vec::with_capacity(linked.len()); for linked in linked { if linked.key != dentry { predicates.push(PredicateRef { @@ -2363,10 +3074,12 @@ where } else { primary_projection_updated = true; } - let mut updated = linked.projection; + let old = linked.projection; + let mut updated = old.clone(); updated.attr = projection.attr.clone(); updated.dentry.attr_generation = projection.attr.generation; updated.body = projection.body.clone(); + updated_link_projections.push((old, updated.clone())); mutations.push(put_projection_mutation( RecordFamily::Dentry, linked.key, @@ -2380,7 +3093,10 @@ where projection, )); } - if let Some(path_index) = path_index { + let restore_index = self.restore_index_link_plan(&updated_link_projections, version)?; + predicates.extend(restore_index.predicates); + mutations.extend(restore_index.mutations); + if let Some(path_index) = path_index.filter(|_| !primary_overlay_managed) { mutations.push(put_projection_mutation( RecordFamily::PathIndex, path_index, @@ -2425,8 +3141,9 @@ where }); } } - self.commit_metadata(MetadataCommand { - request_id: request_id(kind_name(kind), self.mount, inode, version), + let command = MetadataCommand { + request_id: request_id_override + .unwrap_or_else(|| request_id(kind_name(kind), self.mount, inode, version)), kind, read_version, commit_version: version, @@ -2447,7 +3164,14 @@ where ) .into_iter() .collect(), - })?; + }; + if restore_guarded { + super::restore::validate_restore_command_bounds( + &command, + "restore namespace publish replace", + )?; + } + self.commit_metadata(command)?; Ok(()) } } diff --git a/crates/nokv-meta/src/service/publish.rs b/crates/nokv-meta/src/service/publish.rs index ac8822f77..22586d20d 100644 --- a/crates/nokv-meta/src/service/publish.rs +++ b/crates/nokv-meta/src/service/publish.rs @@ -16,6 +16,7 @@ where O: ObjectStore, { pub fn publish_artifact(&self, request: PublishArtifact) -> Result { + self.restore_namespace_write_predicates(&[request.parent], self.read_version()?)?; let object_reference = self.begin_object_reference_mutation()?; let version = self.next_version()?; let inode = self.next_inode()?; @@ -66,6 +67,10 @@ where if existing.attr.file_type != FileType::File { return Err(MetadError::NotFile); } + self.restore_namespace_write_predicates( + &[request.parent, existing.attr.inode], + self.read_version()?, + )?; let version = self.next_version()?; let StagedArtifactBody { body, @@ -90,6 +95,7 @@ where let projection = projection(request.parent, request.name, attr, Some(body)); let old_generation = existing.body.as_ref().map(|body| body.generation); if let Err(err) = self.commit_replace_projection_with_chunks(ReplaceProjectionCommit { + request_id: None, kind: CommandKind::ReplaceArtifact, projection: &projection, chunks: &chunks, @@ -126,9 +132,11 @@ where name: DentryName, object_reference: ObjectReferenceMutation, ) -> Result { + let read_version = self.read_version()?; + self.restore_namespace_write_predicates(&[parent], read_version)?; let Some(parent_attr) = self.get_attr_at_version_for_purpose( parent, - self.read_version()?, + read_version, ReadPurpose::WritePlanLocal, )? else { @@ -189,6 +197,10 @@ where if existing.attr.file_type != FileType::File { return Err(MetadError::NotFile); } + self.restore_namespace_write_predicates( + &[parent, existing.attr.inode], + self.read_version()?, + )?; let generation = self.next_version()?; let now_ms = current_time_ms(); Ok(PreparedArtifact { @@ -228,6 +240,10 @@ where &self, mut prepared: PreparedArtifact, ) -> Result { + self.restore_namespace_write_predicates( + &[prepared.parent, prepared.inode], + self.read_version()?, + )?; if let Some(path) = prepared.path.as_deref() { let components = parse_absolute_path(path)?; let Some((name, parent_components)) = components.split_last() else { @@ -303,21 +319,33 @@ where gid: u32, ) -> Result { validate_new_prepared_block_identities(self.mount, &prepared, &chunks)?; - self.publish_prepared_artifact_impl(PreparedArtifactPublish { - prepared, - body, - chunks, - old_chunks: &[], - mode, - uid, - gid, - }) + self.publish_prepared_artifact_impl( + PreparedArtifactPublish { + prepared, + body, + chunks, + old_chunks: &[], + mode, + uid, + gid, + }, + None, + ) } fn publish_prepared_artifact_impl( &self, request: PreparedArtifactPublish<'_>, + request_id_override: Option>, ) -> Result { + validate_prepared_artifact( + self.mount, + &request.prepared, + &request.body, + &request.chunks, + )?; + let request_id = request_id_override + .unwrap_or_else(|| prepared_artifact_request_id(self.mount, &request)); let PreparedArtifactPublish { prepared, body, @@ -327,9 +355,28 @@ where uid, gid, } = request; - validate_prepared_artifact(self.mount, &prepared, &body, &chunks)?; - self.ensure_prepared_object_gc_epoch(prepared.object_gc_claim_version)?; let version = Version::new(prepared.generation)?; + let path_index = prepared + .path + .as_deref() + .map(|path| { + parse_absolute_path(path).map(|components| path_index_key(self.mount, &components)) + }) + .transpose()?; + let expected_dentry_version = if prepared.replace { + Some(Version::new(prepared.dentry_version.ok_or_else(|| { + MetadError::InvalidPreparedArtifact( + "replace artifact is missing dentry version".to_owned(), + ) + })?)?) + } else { + if prepared.dentry_version.is_some() || prepared.old_generation.is_some() { + return Err(MetadError::InvalidPreparedArtifact( + "create artifact must not carry replace state".to_owned(), + )); + } + None + }; let mut attr = InodeAttr { inode: prepared.inode, file_type: FileType::File, @@ -351,81 +398,145 @@ where } } let projection = projection(prepared.parent, prepared.name.clone(), attr, Some(body)); - if prepared.replace { - let expected_dentry_version = - Version::new(prepared.dentry_version.ok_or_else(|| { - MetadError::InvalidPreparedArtifact( - "replace artifact is missing dentry version".to_owned(), - ) - })?)?; - let replaced = self - .lookup_plus_for_write_plan(prepared.parent, &prepared.name)? - .and_then(|(existing, current_dentry_version)| { - (existing.attr.file_type == FileType::File - && existing.attr.inode == prepared.inode - && current_dentry_version == expected_dentry_version) - .then_some(existing) - }); - self.commit_replace_projection_with_chunks(ReplaceProjectionCommit { - kind: CommandKind::ReplaceArtifact, - projection: &projection, - chunks: &chunks, - old_chunks, - dentry_version: expected_dentry_version, - old_generation: prepared.old_generation, - version, - path_index: prepared - .path - .as_deref() - .map(|path| { - parse_absolute_path(path) - .map(|components| path_index_key(self.mount, &components)) + let terminal_result = || { + self.prepared_terminal_commit_result(&request_id, version) + .map(|terminal| { + terminal.map(|_| RenameReplaceResult { + entry: projection.clone().into(), + replaced: None, }) - .transpose()?, - object_reference: Some(ObjectReferenceMutation::from_version(Version::new( - prepared.object_gc_claim_version, - )?)), - }) - .map_err(|err| { - self.classify_prepared_object_gc_epoch_error(err, prepared.object_gc_claim_version) - })?; - Ok(RenameReplaceResult { - entry: projection.into(), - replaced, - }) - } else { - if prepared.dentry_version.is_some() || prepared.old_generation.is_some() { - return Err(MetadError::InvalidPreparedArtifact( - "create artifact must not carry replace state".to_owned(), - )); + }) + }; + if let Some(result) = terminal_result()? { + return Ok(result); + } + + let publish = (|| { + self.ensure_prepared_object_gc_epoch(prepared.object_gc_claim_version)?; + if let Some(expected_dentry_version) = expected_dentry_version { + let replaced = self + .lookup_plus_for_write_plan(prepared.parent, &prepared.name)? + .and_then(|(existing, current_dentry_version)| { + (existing.attr.file_type == FileType::File + && existing.attr.inode == prepared.inode + && current_dentry_version == expected_dentry_version) + .then_some(existing) + }); + self.commit_replace_projection_with_chunks(ReplaceProjectionCommit { + request_id: Some(request_id.clone()), + kind: CommandKind::ReplaceArtifact, + projection: &projection, + chunks: &chunks, + old_chunks, + dentry_version: expected_dentry_version, + old_generation: prepared.old_generation, + version, + path_index, + object_reference: Some(ObjectReferenceMutation::from_version(Version::new( + prepared.object_gc_claim_version, + )?)), + }) + .map_err(|err| { + self.classify_prepared_object_gc_epoch_error( + err, + prepared.object_gc_claim_version, + ) + })?; + Ok(RenameReplaceResult { + entry: projection.clone().into(), + replaced, + }) + } else { + self.commit_create_projection_with_chunks_and_path_index( + CommandKind::PublishArtifact, + &projection, + &chunks, + version, + path_index, + Some(ObjectReferenceMutation::from_version(Version::new( + prepared.object_gc_claim_version, + )?)), + Some(request_id.clone()), + ) + .map_err(|err| { + self.classify_prepared_object_gc_epoch_error( + err, + prepared.object_gc_claim_version, + ) + })?; + Ok(RenameReplaceResult { + entry: projection.clone().into(), + replaced: None, + }) } - self.commit_create_projection_with_chunks_and_path_index( - CommandKind::PublishArtifact, - &projection, - &chunks, - version, - prepared - .path - .as_deref() - .map(|path| { - parse_absolute_path(path) - .map(|components| path_index_key(self.mount, &components)) - }) - .transpose()?, - Some(ObjectReferenceMutation::from_version(Version::new( - prepared.object_gc_claim_version, - )?)), - ) - .map_err(|err| { - self.classify_prepared_object_gc_epoch_error(err, prepared.object_gc_claim_version) - })?; - Ok(RenameReplaceResult { - entry: projection.into(), - replaced: None, - }) + })(); + match publish { + Ok(result) => Ok(result), + Err(error) => terminal_result()?.ok_or(error), } } + #[allow(clippy::too_many_arguments)] + fn prepared_terminal_rename_result( + &self, + request_id: &[u8], + version: Version, + prepared: &PreparedArtifact, + body: &BodyDescriptor, + mode: u32, + uid: u32, + gid: u32, + ) -> Result, MetadError> { + if self + .prepared_terminal_commit_result(request_id, version)? + .is_none() + { + return Ok(None); + } + if let Some((current, _)) = + self.lookup_plus_for_write_plan(prepared.parent, &prepared.name)? + { + if current.attr.inode == prepared.inode + && current.attr.generation == prepared.generation + { + return Ok(Some(RenameReplaceResult { + entry: current, + replaced: None, + })); + } + } + let nlink = if prepared.replace { + self.lookup_plus_for_write_plan(prepared.parent, &prepared.name)? + .map(|(entry, _)| entry.attr.nlink) + .unwrap_or_else(|| FileType::File.initial_link_count()) + } else { + FileType::File.initial_link_count() + }; + let attr = InodeAttr { + inode: prepared.inode, + file_type: FileType::File, + mode, + uid, + gid, + rdev: 0, + nlink, + size: body.size, + generation: prepared.generation, + mtime_ms: prepared.mtime_ms, + ctime_ms: prepared.ctime_ms, + }; + Ok(Some(RenameReplaceResult { + entry: projection( + prepared.parent, + prepared.name.clone(), + attr, + Some(body.clone()), + ) + .into(), + replaced: None, + })) + } + fn ensure_prepared_object_gc_epoch(&self, expected: u64) -> Result<(), MetadError> { let key = object_gc_claim_key(self.mount); let item = self @@ -482,22 +593,52 @@ where "prepared artifact target does not match publish session".to_owned(), )); } + let request_id = prepared_artifact_session_request_id(self.mount, &prepared, &request); let version = Version::new(prepared.generation)?; + let terminal_body = BodyDescriptor { + producer: request.producer.clone(), + digest_uri: request.digest_uri.clone(), + size: request.size, + content_type: request.content_type.clone(), + manifest_id: request.manifest_id.clone(), + generation: version.get(), + base_generation: 0, + chunk_size: DEFAULT_CHUNK_SIZE, + block_size: DEFAULT_BLOCK_SIZE as u64, + }; + if let Some(result) = self.prepared_terminal_rename_result( + &request_id, + version, + &prepared, + &terminal_body, + request.mode, + request.uid, + request.gid, + )? { + return Ok(result); + } + self.restore_namespace_write_predicates( + &[prepared.parent, prepared.inode], + self.read_version()?, + )?; let StagedArtifactBody { body, chunks, old_chunks, staged, } = self.stage_artifact_session(&request, &prepared, version)?; - self.publish_prepared_artifact_impl(PreparedArtifactPublish { - prepared, - body, - chunks, - old_chunks: &old_chunks, - mode: request.mode, - uid: request.uid, - gid: request.gid, - }) + self.publish_prepared_artifact_impl( + PreparedArtifactPublish { + prepared, + body, + chunks, + old_chunks: &old_chunks, + mode: request.mode, + uid: request.uid, + gid: request.gid, + }, + Some(request_id), + ) .map_err(|err| MetadError::PublishArtifactFailed { source: Box::new(err), staged, @@ -511,6 +652,10 @@ where ranges: &[PublishArtifactRange], block_index_base: u64, ) -> Result { + self.restore_namespace_write_predicates( + &[prepared.parent, prepared.inode], + self.read_version()?, + )?; let dirty_ranges = ranges .iter() .filter(|range| !range.bytes.is_empty()) @@ -557,8 +702,32 @@ where "prepared artifact target does not match staged publish session".to_owned(), )); } - validate_new_prepared_block_identities(self.mount, &prepared, &request.chunks)?; + let request_id = + prepared_artifact_staged_session_request_id(self.mount, &prepared, &request); let version = Version::new(prepared.generation)?; + let terminal_body = BodyDescriptor { + producer: request.producer.clone(), + digest_uri: request.digest_uri.clone(), + size: request.size, + content_type: request.content_type.clone(), + manifest_id: request.manifest_id.clone(), + generation: version.get(), + base_generation: prepared.old_generation.unwrap_or(0), + chunk_size: DEFAULT_CHUNK_SIZE, + block_size: DEFAULT_BLOCK_SIZE as u64, + }; + if let Some(result) = self.prepared_terminal_rename_result( + &request_id, + version, + &prepared, + &terminal_body, + request.mode, + request.uid, + request.gid, + )? { + return Ok(result); + } + validate_new_prepared_block_identities(self.mount, &prepared, &request.chunks)?; let (chunks, base_generation) = self.resolve_session_chunks(&prepared, request.size, request.chunks)?; self.manifest_chunks @@ -579,15 +748,18 @@ where // A delta publish preserves its base generation; the prior generation's // blocks are reclaimed later at chain collapse (compaction), not here. let old_chunks: Vec = Vec::new(); - self.publish_prepared_artifact_impl(PreparedArtifactPublish { - prepared, - body, - chunks, - old_chunks: &old_chunks, - mode: request.mode, - uid: request.uid, - gid: request.gid, - }) + self.publish_prepared_artifact_impl( + PreparedArtifactPublish { + prepared, + body, + chunks, + old_chunks: &old_chunks, + mode: request.mode, + uid: request.uid, + gid: request.gid, + }, + Some(request_id), + ) .map_err(|err| MetadError::PublishArtifactFailed { source: Box::new(err), staged: request.staged, @@ -862,3 +1034,193 @@ where Ok(chunks.into_values().collect()) } } + +fn prepared_artifact_request_id(mount: MountId, request: &PreparedArtifactPublish<'_>) -> Vec { + let PreparedArtifactPublish { + prepared, + body, + chunks, + old_chunks, + mode, + uid, + gid, + } = request; + let (kind, mut hasher) = + prepared_request_hasher(mount, prepared, b"nokv.prepared-artifact.request.v1"); + hash_prepared_field(&mut hasher, b"body", &encode_body_descriptor(body)); + hash_prepared_field( + &mut hasher, + b"chunk_count", + &(chunks.len() as u64).to_be_bytes(), + ); + for chunk in chunks { + hash_prepared_field(&mut hasher, b"chunk", &encode_chunk_manifest(chunk)); + } + hash_prepared_field( + &mut hasher, + b"old_chunk_count", + &(old_chunks.len() as u64).to_be_bytes(), + ); + for chunk in *old_chunks { + hash_prepared_field(&mut hasher, b"old_chunk", &encode_chunk_manifest(chunk)); + } + hash_prepared_field(&mut hasher, b"mode", &mode.to_be_bytes()); + hash_prepared_field(&mut hasher, b"uid", &uid.to_be_bytes()); + hash_prepared_field(&mut hasher, b"gid", &gid.to_be_bytes()); + + finish_prepared_request_id(kind, hasher) +} + +fn prepared_artifact_session_request_id( + mount: MountId, + prepared: &PreparedArtifact, + request: &PublishArtifactSession, +) -> Vec { + let (kind, mut hasher) = prepared_request_hasher( + mount, + prepared, + b"nokv.prepared-artifact.session.request.v1", + ); + hash_prepared_field(&mut hasher, b"producer", request.producer.as_bytes()); + hash_prepared_field(&mut hasher, b"digest_uri", request.digest_uri.as_bytes()); + hash_prepared_field( + &mut hasher, + b"content_type", + request.content_type.as_bytes(), + ); + hash_prepared_field(&mut hasher, b"manifest_id", request.manifest_id.as_bytes()); + hash_prepared_field(&mut hasher, b"size", &request.size.to_be_bytes()); + hash_prepared_field( + &mut hasher, + b"range_count", + &(request.ranges.len() as u64).to_be_bytes(), + ); + for range in &request.ranges { + hash_prepared_field(&mut hasher, b"range_offset", &range.offset.to_be_bytes()); + hash_prepared_field(&mut hasher, b"range_bytes", &range.bytes); + } + hash_prepared_field(&mut hasher, b"mode", &request.mode.to_be_bytes()); + hash_prepared_field(&mut hasher, b"uid", &request.uid.to_be_bytes()); + hash_prepared_field(&mut hasher, b"gid", &request.gid.to_be_bytes()); + finish_prepared_request_id(kind, hasher) +} + +fn prepared_artifact_staged_session_request_id( + mount: MountId, + prepared: &PreparedArtifact, + request: &PublishArtifactStagedSession, +) -> Vec { + let (kind, mut hasher) = prepared_request_hasher( + mount, + prepared, + b"nokv.prepared-artifact.staged-session.request.v1", + ); + hash_prepared_field(&mut hasher, b"producer", request.producer.as_bytes()); + hash_prepared_field(&mut hasher, b"digest_uri", request.digest_uri.as_bytes()); + hash_prepared_field( + &mut hasher, + b"content_type", + request.content_type.as_bytes(), + ); + hash_prepared_field(&mut hasher, b"manifest_id", request.manifest_id.as_bytes()); + hash_prepared_field(&mut hasher, b"size", &request.size.to_be_bytes()); + hash_prepared_field( + &mut hasher, + b"chunk_count", + &(request.chunks.len() as u64).to_be_bytes(), + ); + for chunk in &request.chunks { + hash_prepared_field(&mut hasher, b"chunk", &encode_chunk_manifest(chunk)); + } + hash_prepared_field( + &mut hasher, + b"staged_count", + &(request.staged.len() as u64).to_be_bytes(), + ); + for object in request.staged.objects() { + hash_prepared_field(&mut hasher, b"staged_key", object.key.as_str().as_bytes()); + hash_prepared_field(&mut hasher, b"staged_size", &object.size.to_be_bytes()); + } + hash_prepared_field(&mut hasher, b"mode", &request.mode.to_be_bytes()); + hash_prepared_field(&mut hasher, b"uid", &request.uid.to_be_bytes()); + hash_prepared_field(&mut hasher, b"gid", &request.gid.to_be_bytes()); + finish_prepared_request_id(kind, hasher) +} + +fn prepared_request_hasher( + mount: MountId, + prepared: &PreparedArtifact, + domain: &[u8], +) -> (CommandKind, Sha256) { + let kind = if prepared.replace { + CommandKind::ReplaceArtifact + } else { + CommandKind::PublishArtifact + }; + let mut hasher = Sha256::new(); + hash_prepared_field(&mut hasher, b"domain", domain); + hash_prepared_field(&mut hasher, b"kind", kind_name(kind)); + hash_prepared_field(&mut hasher, b"mount", &mount.get().to_be_bytes()); + hash_prepared_field(&mut hasher, b"parent", &prepared.parent.get().to_be_bytes()); + hash_prepared_field(&mut hasher, b"name", prepared.name.as_bytes()); + hash_prepared_optional_bytes( + &mut hasher, + b"path", + prepared.path.as_deref().map(str::as_bytes), + ); + hash_prepared_field(&mut hasher, b"inode", &prepared.inode.get().to_be_bytes()); + hash_prepared_field( + &mut hasher, + b"generation", + &prepared.generation.to_be_bytes(), + ); + hash_prepared_field(&mut hasher, b"mtime_ms", &prepared.mtime_ms.to_be_bytes()); + hash_prepared_field(&mut hasher, b"ctime_ms", &prepared.ctime_ms.to_be_bytes()); + hash_prepared_field(&mut hasher, b"replace", &[u8::from(prepared.replace)]); + hash_prepared_optional_u64(&mut hasher, b"dentry_version", prepared.dentry_version); + hash_prepared_optional_u64(&mut hasher, b"old_generation", prepared.old_generation); + hash_prepared_field( + &mut hasher, + b"object_gc_claim_version", + &prepared.object_gc_claim_version.to_be_bytes(), + ); + (kind, hasher) +} + +fn finish_prepared_request_id(kind: CommandKind, hasher: Sha256) -> Vec { + let prefix = kind_name(kind); + let digest = hasher.finalize(); + let mut out = + Vec::with_capacity(prefix.len() + PREPARED_ARTIFACT_REQUEST_ID_DOMAIN.len() + digest.len()); + out.extend_from_slice(prefix); + out.extend_from_slice(PREPARED_ARTIFACT_REQUEST_ID_DOMAIN); + out.extend_from_slice(&digest); + out +} + +fn hash_prepared_optional_u64(hasher: &mut Sha256, name: &[u8], value: Option) { + match value { + Some(value) => { + hash_prepared_field(hasher, name, &[1]); + hash_prepared_field(hasher, name, &value.to_be_bytes()); + } + None => hash_prepared_field(hasher, name, &[0]), + } +} + +fn hash_prepared_optional_bytes(hasher: &mut Sha256, name: &[u8], value: Option<&[u8]>) { + match value { + Some(value) => { + hash_prepared_field(hasher, name, &[1]); + hash_prepared_field(hasher, name, value); + } + None => hash_prepared_field(hasher, name, &[0]), + } +} + +fn hash_prepared_field(hasher: &mut Sha256, name: &[u8], value: &[u8]) { + hasher.update((name.len() as u64).to_be_bytes()); + hasher.update(name); + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); +} diff --git a/crates/nokv-meta/src/service/read.rs b/crates/nokv-meta/src/service/read.rs index a1cf75a1d..782d90419 100644 --- a/crates/nokv-meta/src/service/read.rs +++ b/crates/nokv-meta/src/service/read.rs @@ -24,6 +24,13 @@ where version: Version, purpose: ReadPurpose, ) -> Result, MetadError> { + let _visibility = self + .restore_visibility_fence + .read() + .unwrap_or_else(|error| error.into_inner()); + if !self.restore_inode_visible_at(inode, version, purpose)? { + return Ok(None); + } let Some(value) = self.metadata.get( RecordFamily::Inode, &inode_key(self.mount, inode), @@ -106,6 +113,13 @@ where version: Version, purpose: ReadPurpose, ) -> Result, MetadError> { + let _visibility = self + .restore_visibility_fence + .read() + .unwrap_or_else(|error| error.into_inner()); + if !self.restore_inode_visible_at(parent, version, purpose)? { + return Ok(None); + } let key = dentry_key(self.mount, parent, name); let Some(item) = self.metadata @@ -181,6 +195,13 @@ where version: Version, purpose: ReadPurpose, ) -> Result, MetadError> { + let _visibility = self + .restore_visibility_fence + .read() + .unwrap_or_else(|error| error.into_inner()); + if !self.restore_inode_visible_at(parent, version, purpose)? { + return Ok(Vec::new()); + } let rows = self.metadata.scan(ScanRequest { family: RecordFamily::Dentry, prefix: dentry_prefix(self.mount, parent), @@ -229,6 +250,16 @@ where version: Version, purpose: ReadPurpose, ) -> Result { + let _visibility = self + .restore_visibility_fence + .read() + .unwrap_or_else(|error| error.into_inner()); + if !self.restore_inode_visible_at(parent, version, purpose)? { + return Ok(ReadDirPlusPage { + entries: Vec::new(), + next_cursor: None, + }); + } let requested = limit.max(1); let rows = self.metadata.scan(ScanRequest { family: RecordFamily::Dentry, @@ -273,6 +304,75 @@ where version: Version, ) -> Result { let requested = limit.max(1); + let overlay_inverse_prefix = + super::restore_index::restore_index_parent_inverse_prefix_for_read(self.mount, parent); + let has_overlay = !self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: overlay_inverse_prefix, + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::UserStrong, + })? + .is_empty() + || self.restore_index_parent_has_fork_source(parent)?; + if !has_overlay { + return self.list_canonical_indexed_components_page( + parent, components, after, requested, version, + ); + } + let keep = requested.saturating_add(1); + let (canonical, stale_rows) = + self.scan_canonical_indexed_components_page(parent, components, after, keep, version)?; + let mut entries = BTreeMap::, DentryWithAttr>::new(); + for entry in canonical.entries { + entries.insert(entry.dentry.name.as_bytes().to_vec(), entry); + } + let inherited = self.restore_indexed_children_page( + parent, + after, + keep, + version, + ReadPurpose::UserStrong, + )?; + for entry in inherited.entries { + // A canonical PathIndex row is authoritative for the same child; + // inherited overlay only fills names that have not subsequently + // been published into the canonical index. + entries + .entry(entry.dentry.name.as_bytes().to_vec()) + .or_insert(entry); + } + let has_more = entries.len() > requested; + let entries = entries.into_values().take(requested).collect::>(); + let next_cursor = if has_more { + entries.last().map(|entry| entry.dentry.name.clone()) + } else { + None + }; + self.read_dir_plus_total.fetch_add(1, Ordering::Relaxed); + self.read_dir_plus_entry_total + .fetch_add(entries.len() as u64, Ordering::Relaxed); + self.read_dir_plus_projection_hit_total + .fetch_add(entries.len() as u64, Ordering::Relaxed); + self.path_index_scan_stale_total + .fetch_add(stale_rows, Ordering::Relaxed); + Ok(ReadDirPlusPage { + entries, + next_cursor, + }) + } + + fn scan_canonical_indexed_components_page( + &self, + parent: InodeId, + components: &[DentryName], + after: Option<&DentryName>, + requested: usize, + version: Version, + ) -> Result<(ReadDirPlusPage, u64), MetadError> { let prefix = path_index_prefix(self.mount, components); let mut start_after = after.map(|name| { let mut marker = components.to_vec(); @@ -325,17 +425,34 @@ where } else { None }; + Ok(( + ReadDirPlusPage { + entries, + next_cursor, + }, + stale_rows, + )) + } + + fn list_canonical_indexed_components_page( + &self, + parent: InodeId, + components: &[DentryName], + after: Option<&DentryName>, + requested: usize, + version: Version, + ) -> Result { + let (page, stale_rows) = self.scan_canonical_indexed_components_page( + parent, components, after, requested, version, + )?; self.read_dir_plus_total.fetch_add(1, Ordering::Relaxed); self.read_dir_plus_entry_total - .fetch_add(entries.len() as u64, Ordering::Relaxed); + .fetch_add(page.entries.len() as u64, Ordering::Relaxed); self.read_dir_plus_projection_hit_total - .fetch_add(entries.len() as u64, Ordering::Relaxed); + .fetch_add(page.entries.len() as u64, Ordering::Relaxed); self.path_index_scan_stale_total .fetch_add(stale_rows, Ordering::Relaxed); - Ok(ReadDirPlusPage { - entries, - next_cursor, - }) + Ok(page) } fn indexed_path_child( diff --git a/crates/nokv-meta/src/service/restore.rs b/crates/nokv-meta/src/service/restore.rs new file mode 100644 index 000000000..c3dd620bc --- /dev/null +++ b/crates/nokv-meta/src/service/restore.rs @@ -0,0 +1,10745 @@ +//! Durable restore-to-fork metadata state and service entry points. +//! +//! Restore state is private to `nokv-meta` and stored in `RecordFamily::System`. +//! The public API only exposes a completed outcome; no partially materialized +//! inode is a valid user-visible result. + +use super::*; +use crate::command::{ReadItem, ScanItem}; +use std::collections::{BTreeMap, HashMap}; + +pub(super) const RESTORE_FORMAT_VERSION: u8 = 1; +pub(super) const MAX_RESTORE_PATH_BYTES: usize = 4096; +pub(super) const MAX_RESTORE_INITIALIZATION_ENTRIES: usize = 1024; +pub(super) const MAX_RESTORE_INITIALIZATION_BYTES: usize = 8 * 1024 * 1024; +pub(super) const MAX_RESTORE_SUBTREE_ENTRIES: usize = 1_000_000; +pub(super) const RESTORE_BATCH_ENTRIES: usize = 64; +const RESTORE_BACKLOG_PAGE_ROWS: usize = 256; + +const OPERATION_MAGIC: &[u8; 8] = b"NKRSTOP\0"; +const STAGING_MEMBER_MAGIC: &[u8; 8] = b"NKRSTGM\0"; +const OPERATION_KEY_LABEL: &[u8] = b"restore-operation\0"; +const CLAIM_KEY_LABEL: &[u8] = b"restore-destination\0"; +const ACTIVE_KEY_LABEL: &[u8] = b"restore-to-fork-v1-active\0"; +const ACTIVATION_FENCE_KEY_LABEL: &[u8] = b"restore-to-fork-v1-activation-fence\0"; +const STAGING_KEY_LABEL: &[u8] = b"restore-staging-member\0"; +const STAGING_INVERSE_KEY_LABEL: &[u8] = b"restore-staging-inode\0"; +const STAGING_INVERSE_OWNER_KEY_LABEL: &[u8] = b"restore-staging-inverse-owner\0"; +const ROOT_KEY_LABEL: &[u8] = b"restore-root-index\0"; +const BASE_OWNER_KEY_LABEL: &[u8] = b"restore-base-owner\0"; +const BASE_INVERSE_KEY_LABEL: &[u8] = b"restore-base-inverse\0"; +const BASE_INVERSE_OWNER_KEY_LABEL: &[u8] = b"restore-base-inverse-owner\0"; +const BASE_SEAL_KEY_LABEL: &[u8] = b"restore-base-seal\0"; +const CLEANUP_KEY_LABEL: &[u8] = b"restore-cleanup-job\0"; +const CLEANUP_MAGIC: &[u8; 8] = b"NKRCLNJ\0"; +const INIT_UPLOAD_KEY_LABEL: &[u8] = b"restore-init-upload-intent\0"; +const INIT_UPLOAD_MAGIC: &[u8; 8] = b"NKRINIT\0"; +const INIT_UPLOAD_TOMBSTONE_KEY_LABEL: &[u8] = b"restore-init-upload-tombstone\0"; +const INIT_UPLOAD_TOMBSTONE_MAGIC: &[u8; 8] = b"NKRITMB\0"; +const INIT_UPLOAD_TOMBSTONE_CURSOR_KEY_LABEL: &[u8] = b"restore-init-upload-tombstone-cursor\0"; +const RELEASE_KEY_LABEL: &[u8] = b"restore-release-job\0"; +const RELEASE_MAGIC: &[u8; 8] = b"NKRRELJ\0"; +const RELEASE_CURSOR_KEY_LABEL: &[u8] = b"restore-release-cursor\0"; +const RELEASE_CURSOR_MAGIC: &[u8; 8] = b"NKRRCUR\0"; +const RELEASE_QUARANTINE_KEY_LABEL: &[u8] = b"restore-release-quarantine\0"; +const RELEASE_QUARANTINE_MAGIC: &[u8; 8] = b"NKRRELQ\0"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RestoreState { + Complete, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RestoreOutcome { + pub operation_id: String, + pub state: RestoreState, + pub source_root: InodeId, + pub destination_root: InodeId, + pub snapshot_id: u64, + pub read_version: u64, + pub cleanup_pending: bool, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RestoreInitialization { + pub remove_relative_paths: Vec, + pub files: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RestoreInitializationFile { + pub relative_path: String, + pub bytes: Vec, + pub content_type: String, + pub mode: u32, + pub uid: u32, + pub gid: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum RestoreOperationState { + Preparing, + ReadyToAttach, + Complete, + Cleaning, + Discarding, + Releasing, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreOperation { + pub(super) operation_digest: [u8; 32], + pub(super) initialization_digest: [u8; 32], + pub(super) state: RestoreOperationState, + pub(super) source_root: InodeId, + pub(super) destination_root: InodeId, + pub(super) snapshot_id: u64, + pub(super) read_version: u64, + pub(super) created_version: u64, + pub(super) ref_set_id: u64, + pub(super) source_path: String, + pub(super) destination_path: String, +} + +struct RestorePathProof { + inode: InodeId, + predicates: Vec, +} + +struct RestoreHold<'a> { + operation: &'a RestoreOperation, + object_reference: ObjectReferenceMutation, + pin_version: Version, + source_attr: &'a InodeAttr, + source_predicates: &'a [PredicateRef], + destination_parent: InodeId, + destination_name: &'a DentryName, + destination_predicates: &'a [PredicateRef], +} + +struct RestoreInitializationRemoveCommand<'a> { + operation: &'a RestoreOperation, + parent: InodeId, + name: &'a DentryName, + entry: &'a DentryWithAttr, + dentry_version: Version, + staging: &'a RestoreStagingProof, + chunks: &'a [ChunkManifest], + xattr_keys: &'a [Vec], + version: Version, +} + +struct RestoreInitializationPublishCommand<'a> { + operation: &'a RestoreOperation, + parent: InodeId, + name: &'a DentryName, + file: &'a RestoreInitializationFile, + existing: Option<&'a (DentryWithAttr, Version)>, + inode: InodeId, + object_generation: Version, + intent_version: Version, + object_reference: ObjectReferenceMutation, + body: &'a BodyDescriptor, + chunks: &'a [ChunkManifest], + old_chunks: &'a [ChunkManifest], + staging: Option<&'a RestoreStagingProof>, + version: Version, +} + +struct RestoreCloneFrame { + source: InodeId, + destination: InodeId, + relative_components: Vec, + after: Option, +} + +struct RestoreCloneEntry { + source: DentryWithAttr, + destination: InodeId, + relative_path: String, + body: Option, + chunks: Vec, +} + +#[derive(Clone)] +struct RestoreReferenceReleaseEntry { + owner_key: Vec, + owner_version: Version, + inverse_key: Vec, + inverse_version: Version, + inverse_owner_key: Vec, + inverse_owner_version: Version, + reference: super::restore_gc::RestoreBaseReference, + object_digest: [u8; 32], + identity: (InodeId, u64, u64, u64), +} + +#[derive(Clone)] +struct RestoreReferenceReleaseGuard { + owner_key: Vec, + owner_version: Version, + inverse_key: Vec, + inverse_version: Version, + inverse_owner_key: Vec, + inverse_owner_version: Version, + reference: super::restore_gc::RestoreBaseReference, + object_digest: [u8; 32], + identity: (InodeId, u64, u64, u64), +} + +enum RestoreReferenceReleaseGuardValidation { + Valid(Box), + Corrupt { + reason: String, + object_digest: Option<[u8; 32]>, + }, +} + +impl From<&RestoreReferenceReleaseEntry> for RestoreReferenceReleaseGuard { + fn from(entry: &RestoreReferenceReleaseEntry) -> Self { + Self { + owner_key: entry.owner_key.clone(), + owner_version: entry.owner_version, + inverse_key: entry.inverse_key.clone(), + inverse_version: entry.inverse_version, + inverse_owner_key: entry.inverse_owner_key.clone(), + inverse_owner_version: entry.inverse_owner_version, + reference: entry.reference.clone(), + object_digest: entry.object_digest, + identity: entry.identity, + } + } +} + +fn restore_release_error_is_retryable(error: &MetadError) -> bool { + matches!( + error, + MetadError::Metadata(MetadataError::PredicateFailed) + | MetadError::Metadata(MetadataError::Backend(_)) + | MetadError::SyncLogArchiveFailed { .. } + | MetadError::StaleOwnerEpoch { .. } + | MetadError::LeaseExpired { .. } + | MetadError::NotOwner { .. } + | MetadError::MetadataCheckpointInstallUncertain + ) +} + +struct RestoreReferenceReleaseCommand<'a> { + operation: &'a RestoreOperation, + operation_version: Version, + job: &'a RestoreReleaseJob, + job_version: Version, + entries: &'a [RestoreReferenceReleaseEntry], + defer_gc_for: Option<[u8; 32]>, + deferred_guard: Option<&'a RestoreReferenceReleaseGuard>, + object_reference: ObjectReferenceMutation, + version: Version, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreStagingMember { + pub(super) operation_digest: [u8; 32], + pub(super) source_inode: Option, + pub(super) destination_inode: InodeId, + pub(super) destination_parent: Option, + pub(super) name: Option, + pub(super) relative_path: String, + /// Cursor over ordinary (canonical) PathIndex rows created after attach. + /// Snapshot-materialized members use only the private restore overlay and + /// start complete; dynamically enrolled members are drained page by page. + pub(super) canonical_index_cursor: Vec, + pub(super) canonical_index_complete: bool, + /// Physical ChunkManifest row currently being drained during release. + /// The row is retained until every owned block has been durably enqueued. + pub(super) manifest_cursor: Vec, + /// Ordinal of the next canonical owned block in `manifest_cursor`. + pub(super) manifest_block_cursor: u64, +} + +struct RestoreStagingProof { + member: RestoreStagingMember, + member_version: Version, + inverse_version: Version, + inverse_owner_version: Version, +} + +enum RestoreNamespaceActivityFence { + Inactive(PredicateRef), + Active { key: Vec, version: Version }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreInitUploadIntent { + pub(super) operation_digest: [u8; 32], + pub(super) ref_set_id: u64, + pub(super) inode: InodeId, + pub(super) generation: u64, + pub(super) size: u64, + pub(super) relative_path: String, + /// 0 = upload may still become visible; 1 = the permanent global cleanup + /// tombstone is durable, so this operation-local intent may be retired. + pub(super) cleanup_pass: u8, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreInitUploadTombstone { + pub(super) operation_digest: [u8; 32], + pub(super) initialization_digest: [u8; 32], + pub(super) inode: InodeId, + pub(super) generation: u64, + pub(super) size: u64, + pub(super) relative_path: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreCleanupJob { + pub(super) operation_digest: [u8; 32], + pub(super) ref_set_id: u64, + pub(super) index_complete: bool, + pub(super) index_cursor: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum RestoreReleasePhase { + ExactReferences, + Members, + Overlay, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreReleaseJob { + pub(super) operation_digest: [u8; 32], + pub(super) ref_set_id: u64, + pub(super) phase: RestoreReleasePhase, + pub(super) cursor: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreReleaseWorkerCursor { + /// Frozen metadata row-version frontier for one fair scan cycle. Jobs + /// inserted or rewritten after this boundary are deferred until the next + /// cycle, including old restores whose release starts late. + pub(super) cycle_high_water: u64, + /// Last physical release-job key visited in this cycle. The row may have + /// been deleted or rewritten after the cursor was committed. + pub(super) start_after: Vec, +} + +pub(super) struct RestoreReleaseTransition { + pub(super) predicates: Vec, + pub(super) mutations: Vec, +} + +pub(super) struct RestoreNamespaceEnrollmentPlan { + pub(super) predicates: Vec, + pub(super) mutations: Vec, +} + +/// Stable public request identity. Initialization is intentionally excluded: +/// changing it for an existing request is a destination conflict, not a new +/// operation. +pub fn restore_operation_id( + mount: MountId, + source_path: &str, + snapshot_id: u64, + destination_path: &str, +) -> Result { + let source = canonical_path(&parse_absolute_path(source_path)?)?; + let destination = canonical_path(&parse_absolute_path(destination_path)?)?; + Ok(format!( + "restore-{}", + hex_digest(&restore_operation_digest( + mount, + &source, + snapshot_id, + &destination, + )) + )) +} + +pub(super) fn restore_operation_digest( + mount: MountId, + source_path: &str, + snapshot_id: u64, + destination_path: &str, +) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(b"nokv-restore-to-fork-request-v1\0"); + hasher.update(mount.get().to_be_bytes()); + put_hash_field(&mut hasher, source_path.as_bytes()); + hasher.update(snapshot_id.to_be_bytes()); + put_hash_field(&mut hasher, destination_path.as_bytes()); + hasher.finalize().into() +} + +pub(super) fn restore_operation_key(mount: MountId, digest: &[u8; 32]) -> Vec { + system_key_with_digest(mount, OPERATION_KEY_LABEL, digest) +} + +pub(super) fn restore_destination_claim_key(mount: MountId, path: &str) -> Vec { + let digest: [u8; 32] = Sha256::digest(path.as_bytes()).into(); + system_key_with_digest(mount, CLAIM_KEY_LABEL, &digest) +} + +pub(super) fn restore_active_key(mount: MountId) -> Vec { + system_key(mount, ACTIVE_KEY_LABEL) +} + +pub(super) fn restore_activation_fence_key(mount: MountId) -> Vec { + system_key(mount, ACTIVATION_FENCE_KEY_LABEL) +} + +pub(super) fn restore_staging_member_prefix(mount: MountId, ref_set_id: u64) -> Vec { + system_key_with_u64(mount, STAGING_KEY_LABEL, ref_set_id) +} + +pub(super) fn restore_staging_member_key( + mount: MountId, + ref_set_id: u64, + inode: InodeId, +) -> Vec { + let mut key = restore_staging_member_prefix(mount, ref_set_id); + key.extend_from_slice(&inode.get().to_be_bytes()); + key +} + +pub(super) fn restore_staging_inode_key(mount: MountId, inode: InodeId) -> Vec { + system_key_with_u64(mount, STAGING_INVERSE_KEY_LABEL, inode.get()) +} + +pub(super) fn restore_staging_inverse_owner_prefix(mount: MountId, ref_set_id: u64) -> Vec { + system_key_with_u64(mount, STAGING_INVERSE_OWNER_KEY_LABEL, ref_set_id) +} + +pub(super) fn restore_staging_inverse_owner_key( + mount: MountId, + ref_set_id: u64, + inode: InodeId, +) -> Vec { + let mut key = restore_staging_inverse_owner_prefix(mount, ref_set_id); + key.extend_from_slice(&inode.get().to_be_bytes()); + key +} + +pub(super) fn encode_restore_staging_member( + member: &RestoreStagingMember, +) -> Result, MetadError> { + validate_restore_staging_member(member)?; + let relative = member.relative_path.as_bytes(); + let name = member.name.as_ref().map_or(&[][..], DentryName::as_bytes); + let name_len = u32::try_from(name.len()) + .map_err(|_| MetadError::Codec("restore member name is too long".to_owned()))?; + let relative_len = u32::try_from(relative.len()) + .map_err(|_| MetadError::Codec("restore relative path is too long".to_owned()))?; + let cursor_len = u32::try_from(member.canonical_index_cursor.len()) + .map_err(|_| MetadError::Codec("restore member index cursor is too long".to_owned()))?; + let manifest_cursor_len = u32::try_from(member.manifest_cursor.len()) + .map_err(|_| MetadError::Codec("restore member manifest cursor is too long".to_owned()))?; + let mut value = Vec::with_capacity( + 92 + name.len() + + relative.len() + + member.canonical_index_cursor.len() + + member.manifest_cursor.len(), + ); + value.extend_from_slice(STAGING_MEMBER_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.extend_from_slice(&member.operation_digest); + value.push(u8::from(member.source_inode.is_some())); + value.extend_from_slice(&member.source_inode.map_or(0, InodeId::get).to_be_bytes()); + value.extend_from_slice(&member.destination_inode.get().to_be_bytes()); + value.push(u8::from(member.destination_parent.is_some())); + value.extend_from_slice( + &member + .destination_parent + .map_or(0, InodeId::get) + .to_be_bytes(), + ); + value.extend_from_slice(&name_len.to_be_bytes()); + value.extend_from_slice(name); + value.extend_from_slice(&relative_len.to_be_bytes()); + value.extend_from_slice(relative); + value.push(u8::from(member.canonical_index_complete)); + value.extend_from_slice(&cursor_len.to_be_bytes()); + value.extend_from_slice(&member.canonical_index_cursor); + value.extend_from_slice(&manifest_cursor_len.to_be_bytes()); + value.extend_from_slice(&member.manifest_cursor); + value.extend_from_slice(&member.manifest_block_cursor.to_be_bytes()); + Ok(value) +} + +pub(super) fn decode_restore_staging_member( + value: &[u8], +) -> Result { + let mut decoder = RestoreDecoder::new(value); + if decoder.take(8)? != STAGING_MEMBER_MAGIC { + return Err(MetadError::Codec( + "invalid restore staging member magic".to_owned(), + )); + } + if decoder.u8()? != RESTORE_FORMAT_VERSION { + return Err(MetadError::Codec( + "unsupported restore staging member version".to_owned(), + )); + } + let operation_digest = decoder.array_32()?; + let source_tag = decoder.u8()?; + let source_raw = decoder.u64()?; + let source_inode = match (source_tag, source_raw) { + (0, 0) => None, + (1, raw) => Some(InodeId::new(raw)?), + _ => { + return Err(MetadError::Codec( + "restore staging member has an invalid source inode tag".to_owned(), + )) + } + }; + let destination_inode = InodeId::new(decoder.u64()?)?; + let parent_tag = decoder.u8()?; + let parent_raw = decoder.u64()?; + let destination_parent = match (parent_tag, parent_raw) { + (0, 0) => None, + (1, raw) => Some(InodeId::new(raw)?), + _ => { + return Err(MetadError::Codec( + "restore staging member has an invalid parent inode tag".to_owned(), + )) + } + }; + let name_bytes = decoder.bytes()?; + let name = if name_bytes.is_empty() { + None + } else { + Some(DentryName::new(name_bytes).map_err(|error| { + MetadError::Codec(format!("invalid restore staging member name: {error}")) + })?) + }; + let relative_path = decoder.string()?; + let canonical_index_complete = match decoder.u8()? { + 0 => false, + 1 => true, + _ => { + return Err(MetadError::Codec( + "restore staging member has an invalid index-complete tag".to_owned(), + )) + } + }; + let canonical_index_cursor = decoder.bytes()?; + let manifest_cursor = decoder.bytes()?; + let manifest_block_cursor = decoder.u64()?; + let member = RestoreStagingMember { + operation_digest, + source_inode, + destination_inode, + destination_parent, + name, + relative_path, + canonical_index_cursor, + canonical_index_complete, + manifest_cursor, + manifest_block_cursor, + }; + decoder.finish()?; + validate_restore_staging_member(&member)?; + Ok(member) +} + +fn validate_restore_staging_member(member: &RestoreStagingMember) -> Result<(), MetadError> { + if member.relative_path.len() > MAX_RESTORE_PATH_BYTES { + return Err(MetadError::Codec( + "restore staging member relative path is too long".to_owned(), + )); + } + if member.canonical_index_cursor.len() > MAX_RESTORE_PATH_BYTES + 16 { + return Err(MetadError::Codec( + "restore staging member index cursor is too long".to_owned(), + )); + } + if member.canonical_index_complete && !member.canonical_index_cursor.is_empty() { + return Err(MetadError::Codec( + "completed restore member index scan retains a cursor".to_owned(), + )); + } + if member.manifest_cursor.len() > MAX_RESTORE_PATH_BYTES + 16 { + return Err(MetadError::Codec( + "restore staging member manifest cursor is too long".to_owned(), + )); + } + if member.manifest_cursor.is_empty() != (member.manifest_block_cursor == 0) { + return Err(MetadError::Codec( + "restore staging member manifest cursor shape is invalid".to_owned(), + )); + } + if !member.relative_path.is_empty() + && canonical_restore_relative_path(&member.relative_path)? != member.relative_path + { + return Err(MetadError::Codec( + "restore staging member relative path is not canonical".to_owned(), + )); + } + match (&member.destination_parent, &member.name) { + (None, None) if member.relative_path.is_empty() => {} + (Some(_), Some(_)) if member.source_inode.is_none() && member.relative_path.is_empty() => {} + (Some(_), Some(name)) if !member.relative_path.is_empty() => { + let components = restore_relative_components(&member.relative_path)?; + if components.last() != Some(name) { + return Err(MetadError::Codec( + "restore staging member name does not match relative path".to_owned(), + )); + } + } + _ => { + return Err(MetadError::Codec( + "restore staging member parent/name shape is invalid".to_owned(), + )) + } + } + Ok(()) +} + +pub(super) fn encode_restore_staging_inverse(operation: &RestoreOperation) -> Vec { + let mut value = Vec::with_capacity(40); + value.extend_from_slice(&operation.operation_digest); + value.extend_from_slice(&operation.ref_set_id.to_be_bytes()); + value +} + +pub(super) fn decode_restore_staging_inverse(value: &[u8]) -> Result<([u8; 32], u64), MetadError> { + if value.len() != 40 { + return Err(MetadError::Codec( + "restore staging inverse has an invalid length".to_owned(), + )); + } + let digest = value[..32].try_into().expect("digest width"); + let ref_set_id = u64::from_be_bytes(value[32..].try_into().expect("u64 width")); + if ref_set_id == 0 { + return Err(MetadError::Codec( + "restore staging inverse has a zero set id".to_owned(), + )); + } + Ok((digest, ref_set_id)) +} + +pub(super) fn restore_root_index_key(mount: MountId, root: InodeId) -> Vec { + system_key_with_u64(mount, ROOT_KEY_LABEL, root.get()) +} + +pub(super) fn restore_base_owner_prefix(mount: MountId, ref_set_id: u64) -> Vec { + system_key_with_u64(mount, BASE_OWNER_KEY_LABEL, ref_set_id) +} + +pub(super) fn restore_base_owner_key( + mount: MountId, + ref_set_id: u64, + object_digest: &[u8; 32], + borrower_inode: InodeId, + borrower_generation: u64, +) -> Vec { + let mut key = restore_base_owner_prefix(mount, ref_set_id); + key.extend_from_slice(object_digest); + key.extend_from_slice(&borrower_inode.get().to_be_bytes()); + key.extend_from_slice(&borrower_generation.to_be_bytes()); + key +} + +fn restore_base_owner_object_digest_from_key( + mount: MountId, + ref_set_id: u64, + key: &[u8], +) -> Option<[u8; 32]> { + let prefix = restore_base_owner_prefix(mount, ref_set_id); + if !key.starts_with(&prefix) || key.len() != prefix.len() + 32 + 8 + 8 { + return None; + } + key[prefix.len()..prefix.len() + 32].try_into().ok() +} + +pub(super) fn restore_base_inverse_prefix(mount: MountId, object_digest: &[u8; 32]) -> Vec { + let mut key = system_key(mount, BASE_INVERSE_KEY_LABEL); + key.extend_from_slice(object_digest); + key +} + +pub(super) fn restore_base_inverse_key( + mount: MountId, + object_digest: &[u8; 32], + ref_set_id: u64, + borrower_inode: InodeId, + borrower_generation: u64, +) -> Vec { + let mut key = restore_base_inverse_prefix(mount, object_digest); + key.extend_from_slice(&ref_set_id.to_be_bytes()); + key.extend_from_slice(&borrower_inode.get().to_be_bytes()); + key.extend_from_slice(&borrower_generation.to_be_bytes()); + key +} + +pub(super) fn restore_base_inverse_owner_prefix(mount: MountId, ref_set_id: u64) -> Vec { + system_key_with_u64(mount, BASE_INVERSE_OWNER_KEY_LABEL, ref_set_id) +} + +pub(super) fn restore_base_inverse_owner_key( + mount: MountId, + ref_set_id: u64, + object_digest: &[u8; 32], + borrower_inode: InodeId, + borrower_generation: u64, +) -> Vec { + let mut key = restore_base_inverse_owner_prefix(mount, ref_set_id); + key.extend_from_slice(object_digest); + key.extend_from_slice(&borrower_inode.get().to_be_bytes()); + key.extend_from_slice(&borrower_generation.to_be_bytes()); + key +} + +pub(super) fn restore_base_seal_key(mount: MountId, ref_set_id: u64) -> Vec { + system_key_with_u64(mount, BASE_SEAL_KEY_LABEL, ref_set_id) +} + +pub(super) fn restore_cleanup_job_key(mount: MountId, ref_set_id: u64) -> Vec { + system_key_with_u64(mount, CLEANUP_KEY_LABEL, ref_set_id) +} + +pub(super) fn restore_init_upload_intent_prefix(mount: MountId, ref_set_id: u64) -> Vec { + system_key_with_u64(mount, INIT_UPLOAD_KEY_LABEL, ref_set_id) +} + +pub(super) fn restore_init_upload_intent_key( + mount: MountId, + ref_set_id: u64, + inode: InodeId, + generation: u64, +) -> Vec { + let mut key = restore_init_upload_intent_prefix(mount, ref_set_id); + key.extend_from_slice(&inode.get().to_be_bytes()); + key.extend_from_slice(&generation.to_be_bytes()); + key +} + +pub(super) fn restore_init_upload_tombstone_prefix(mount: MountId) -> Vec { + system_key(mount, INIT_UPLOAD_TOMBSTONE_KEY_LABEL) +} + +pub(super) fn restore_init_upload_tombstone_key( + mount: MountId, + operation_digest: &[u8; 32], + inode: InodeId, + generation: u64, +) -> Vec { + let mut key = restore_init_upload_tombstone_prefix(mount); + key.extend_from_slice(operation_digest); + key.extend_from_slice(&inode.get().to_be_bytes()); + key.extend_from_slice(&generation.to_be_bytes()); + key +} + +pub(super) fn restore_init_upload_tombstone_cursor_key(mount: MountId) -> Vec { + system_key(mount, INIT_UPLOAD_TOMBSTONE_CURSOR_KEY_LABEL) +} + +pub(super) fn restore_release_job_prefix(mount: MountId) -> Vec { + system_key(mount, RELEASE_KEY_LABEL) +} + +pub(super) fn restore_release_job_key(mount: MountId, ref_set_id: u64) -> Vec { + system_key_with_u64(mount, RELEASE_KEY_LABEL, ref_set_id) +} + +fn restore_release_job_ref_set_id(mount: MountId, key: &[u8]) -> Option { + let prefix = restore_release_job_prefix(mount); + if !key.starts_with(&prefix) || key.len() != prefix.len() + 8 { + return None; + } + let ref_set_id = u64::from_be_bytes(key[prefix.len()..].try_into().ok()?); + (ref_set_id != 0).then_some(ref_set_id) +} + +pub(super) fn restore_release_cursor_key(mount: MountId) -> Vec { + system_key(mount, RELEASE_CURSOR_KEY_LABEL) +} + +pub(super) fn restore_release_quarantine_prefix(mount: MountId) -> Vec { + system_key(mount, RELEASE_QUARANTINE_KEY_LABEL) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum RestoreReleaseQuarantineScope { + Diagnostic, + Object([u8; 32]), + MountWide, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreReleaseQuarantine { + pub(super) family: RecordFamily, + pub(super) scope: RestoreReleaseQuarantineScope, + pub(super) original_version: Version, + pub(super) original_key: Vec, + pub(super) original_value: Vec, + pub(super) reason: String, +} + +pub(super) fn restore_release_object_quarantine_prefix( + mount: MountId, + object_digest: &[u8; 32], +) -> Vec { + let mut key = restore_release_quarantine_prefix(mount); + key.push(2); + key.extend_from_slice(object_digest); + key +} + +pub(super) fn restore_release_mount_wide_quarantine_prefix(mount: MountId) -> Vec { + let mut key = restore_release_quarantine_prefix(mount); + key.push(3); + key +} + +fn restore_release_quarantine_key( + mount: MountId, + family: RecordFamily, + row: &crate::command::ScanItem, + scope: RestoreReleaseQuarantineScope, +) -> Vec { + let mut hasher = Sha256::new(); + hasher.update(b"nokv-restore-release-quarantine-v1\0"); + hasher.update([restore_record_family_tag(family)]); + hasher.update((row.key.len() as u64).to_be_bytes()); + hasher.update(&row.key); + hasher.update(row.version.get().to_be_bytes()); + hasher.update((row.value.0.len() as u64).to_be_bytes()); + hasher.update(&row.value.0); + let digest: [u8; 32] = hasher.finalize().into(); + let mut key = match scope { + RestoreReleaseQuarantineScope::Diagnostic => { + let mut key = restore_release_quarantine_prefix(mount); + key.push(1); + key + } + RestoreReleaseQuarantineScope::Object(object_digest) => { + restore_release_object_quarantine_prefix(mount, &object_digest) + } + RestoreReleaseQuarantineScope::MountWide => { + restore_release_mount_wide_quarantine_prefix(mount) + } + }; + key.extend_from_slice(&digest); + key +} + +fn encode_restore_release_quarantine( + family: RecordFamily, + row: &crate::command::ScanItem, + reason: &str, + scope: RestoreReleaseQuarantineScope, +) -> Result, MetadError> { + let key_len = u32::try_from(row.key.len()) + .map_err(|_| MetadError::Codec("restore release quarantine key is too long".to_owned()))?; + let value_len = u32::try_from(row.value.0.len()).map_err(|_| { + MetadError::Codec("restore release quarantine value is too long".to_owned()) + })?; + let mut reason_len = reason.len().min(MAX_RESTORE_PATH_BYTES); + while !reason.is_char_boundary(reason_len) { + reason_len -= 1; + } + let reason = reason.as_bytes(); + let reason_len = u32::try_from(reason_len).map_err(|_| { + MetadError::Codec("restore release quarantine reason is too long".to_owned()) + })?; + let mut value = Vec::with_capacity( + 22_usize + .saturating_add(row.key.len()) + .saturating_add(row.value.0.len()) + .saturating_add(reason.len().min(MAX_RESTORE_PATH_BYTES)), + ); + value.extend_from_slice(RELEASE_QUARANTINE_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.push(restore_record_family_tag(family)); + match scope { + RestoreReleaseQuarantineScope::Diagnostic => value.push(1), + RestoreReleaseQuarantineScope::Object(object_digest) => { + value.push(2); + value.extend_from_slice(&object_digest); + } + RestoreReleaseQuarantineScope::MountWide => value.push(3), + } + value.extend_from_slice(&row.version.get().to_be_bytes()); + value.extend_from_slice(&key_len.to_be_bytes()); + value.extend_from_slice(&row.key); + value.extend_from_slice(&value_len.to_be_bytes()); + value.extend_from_slice(&row.value.0); + value.extend_from_slice(&reason_len.to_be_bytes()); + value.extend_from_slice(&reason[..reason_len as usize]); + Ok(value) +} + +fn restore_record_family_tag(family: RecordFamily) -> u8 { + match family { + RecordFamily::System => 0, + RecordFamily::Mount => 1, + RecordFamily::Inode => 2, + RecordFamily::Dentry => 3, + RecordFamily::Parent => 4, + RecordFamily::Xattr => 5, + RecordFamily::ChunkManifest => 6, + RecordFamily::Session => 7, + RecordFamily::PathIndex => 8, + RecordFamily::Watch => 9, + RecordFamily::Snapshot => 10, + RecordFamily::Gc => 11, + RecordFamily::CommandDedupe => 12, + RecordFamily::History => 13, + RecordFamily::ForkBinding => 14, + RecordFamily::ForkShadow => 15, + } +} + +fn restore_record_family_from_tag(tag: u8) -> Result { + match tag { + 0 => Ok(RecordFamily::System), + 1 => Ok(RecordFamily::Mount), + 2 => Ok(RecordFamily::Inode), + 3 => Ok(RecordFamily::Dentry), + 4 => Ok(RecordFamily::Parent), + 5 => Ok(RecordFamily::Xattr), + 6 => Ok(RecordFamily::ChunkManifest), + 7 => Ok(RecordFamily::Session), + 8 => Ok(RecordFamily::PathIndex), + 9 => Ok(RecordFamily::Watch), + 10 => Ok(RecordFamily::Snapshot), + 11 => Ok(RecordFamily::Gc), + 12 => Ok(RecordFamily::CommandDedupe), + 13 => Ok(RecordFamily::History), + 14 => Ok(RecordFamily::ForkBinding), + 15 => Ok(RecordFamily::ForkShadow), + tag => Err(MetadError::Codec(format!( + "invalid restore quarantine record family {tag}" + ))), + } +} + +pub(super) fn validate_restore_release_quarantine_row( + mount: MountId, + row: &crate::command::ScanItem, +) -> Result { + let mut decoder = RestoreDecoder::new(&row.value.0); + if decoder.take(8)? != RELEASE_QUARANTINE_MAGIC || decoder.u8()? != RESTORE_FORMAT_VERSION { + return Err(MetadError::Codec( + "invalid restore release quarantine header".to_owned(), + )); + } + let family = restore_record_family_from_tag(decoder.u8()?)?; + let scope = match decoder.u8()? { + 1 => RestoreReleaseQuarantineScope::Diagnostic, + 2 => RestoreReleaseQuarantineScope::Object(decoder.array_32()?), + 3 => RestoreReleaseQuarantineScope::MountWide, + tag => { + return Err(MetadError::Codec(format!( + "invalid restore release quarantine scope {tag}" + ))) + } + }; + let original_version = Version::new(decoder.u64()?)?; + let original_key = decoder.bytes()?; + let original_value = decoder.bytes()?; + let reason_bytes = decoder.bytes()?; + decoder.finish()?; + if reason_bytes.len() > MAX_RESTORE_PATH_BYTES { + return Err(MetadError::Codec( + "restore release quarantine reason is too long".to_owned(), + )); + } + let reason = String::from_utf8(reason_bytes).map_err(|_| { + MetadError::Codec("restore release quarantine reason is not utf-8".to_owned()) + })?; + let original_row = crate::command::ScanItem { + key: original_key.clone(), + value: Value(original_value.clone()), + version: original_version, + }; + if row.key != restore_release_quarantine_key(mount, family, &original_row, scope) { + return Err(MetadError::Codec( + "restore release quarantine key changed identity".to_owned(), + )); + } + // Re-encoding closes over all length fields and prevents accepting a + // non-canonical envelope that merely hashes to the same physical key. + if row.value.0 != encode_restore_release_quarantine(family, &original_row, &reason, scope)? { + return Err(MetadError::Codec( + "restore release quarantine value is not canonical".to_owned(), + )); + } + Ok(RestoreReleaseQuarantine { + family, + scope, + original_version, + original_key, + original_value, + reason, + }) +} + +fn encode_restore_release_job(job: &RestoreReleaseJob) -> Result, MetadError> { + if job.ref_set_id == 0 || job.cursor.len() > MAX_RESTORE_PATH_BYTES { + return Err(MetadError::Codec( + "restore release job has an invalid identity or cursor".to_owned(), + )); + } + let cursor_len = u32::try_from(job.cursor.len()) + .map_err(|_| MetadError::Codec("restore release cursor is too long".to_owned()))?; + let mut value = Vec::with_capacity(54 + job.cursor.len()); + value.extend_from_slice(RELEASE_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.extend_from_slice(&job.operation_digest); + value.extend_from_slice(&job.ref_set_id.to_be_bytes()); + value.push(match job.phase { + RestoreReleasePhase::ExactReferences => 1, + RestoreReleasePhase::Members => 2, + RestoreReleasePhase::Overlay => 3, + }); + value.extend_from_slice(&cursor_len.to_be_bytes()); + value.extend_from_slice(&job.cursor); + Ok(value) +} + +pub(super) fn decode_restore_release_job(value: &[u8]) -> Result { + let mut decoder = RestoreDecoder::new(value); + if decoder.take(8)? != RELEASE_MAGIC || decoder.u8()? != RESTORE_FORMAT_VERSION { + return Err(MetadError::Codec( + "invalid restore release job header".to_owned(), + )); + } + let operation_digest = decoder.array_32()?; + let ref_set_id = decoder.u64()?; + let phase = match decoder.u8()? { + 1 => RestoreReleasePhase::ExactReferences, + 2 => RestoreReleasePhase::Members, + 3 => RestoreReleasePhase::Overlay, + tag => { + return Err(MetadError::Codec(format!( + "invalid restore release phase {tag}" + ))) + } + }; + let cursor = decoder.bytes()?; + decoder.finish()?; + let job = RestoreReleaseJob { + operation_digest, + ref_set_id, + phase, + cursor, + }; + encode_restore_release_job(&job)?; + Ok(job) +} + +pub(super) fn encode_restore_release_worker_cursor( + mount: MountId, + cursor: &RestoreReleaseWorkerCursor, +) -> Result, MetadError> { + if cursor.cycle_high_water == 0 || cursor.start_after.len() > MAX_RESTORE_PATH_BYTES { + return Err(MetadError::Codec( + "restore release worker cursor has an invalid boundary".to_owned(), + )); + } + if !cursor.start_after.is_empty() + && restore_release_job_ref_set_id(mount, &cursor.start_after).is_none() + { + return Err(MetadError::Codec( + "restore release worker cursor has a non-canonical job key".to_owned(), + )); + } + let start_after_len = u32::try_from(cursor.start_after.len()) + .map_err(|_| MetadError::Codec("restore release worker cursor is too long".to_owned()))?; + let mut value = Vec::with_capacity(21 + cursor.start_after.len()); + value.extend_from_slice(RELEASE_CURSOR_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.extend_from_slice(&cursor.cycle_high_water.to_be_bytes()); + value.extend_from_slice(&start_after_len.to_be_bytes()); + value.extend_from_slice(&cursor.start_after); + Ok(value) +} + +pub(super) fn decode_restore_release_worker_cursor( + mount: MountId, + value: &[u8], +) -> Result { + let mut decoder = RestoreDecoder::new(value); + if decoder.take(8)? != RELEASE_CURSOR_MAGIC || decoder.u8()? != RESTORE_FORMAT_VERSION { + return Err(MetadError::Codec( + "invalid restore release worker cursor header".to_owned(), + )); + } + let cursor = RestoreReleaseWorkerCursor { + cycle_high_water: decoder.u64()?, + start_after: decoder.bytes()?, + }; + decoder.finish()?; + // Reuse the canonical encoder as the single shape and keyspace validator. + encode_restore_release_worker_cursor(mount, &cursor)?; + Ok(cursor) +} + +pub(super) fn decode_restore_release_worker_cursor_at_version( + mount: MountId, + value: &[u8], + read_version: Version, +) -> Result { + let cursor = decode_restore_release_worker_cursor(mount, value)?; + if cursor.cycle_high_water > read_version.get() { + return Err(MetadError::Codec( + "restore release worker cursor is ahead of metadata".to_owned(), + )); + } + Ok(cursor) +} + +fn encode_restore_cleanup_job(job: &RestoreCleanupJob) -> Result, MetadError> { + if job.ref_set_id == 0 || job.index_cursor.len() > MAX_RESTORE_PATH_BYTES { + return Err(MetadError::Codec( + "restore cleanup job has an invalid identity or cursor".to_owned(), + )); + } + if job.index_complete && !job.index_cursor.is_empty() { + return Err(MetadError::Codec( + "completed restore cleanup job has a cursor".to_owned(), + )); + } + let cursor_len = u32::try_from(job.index_cursor.len()) + .map_err(|_| MetadError::Codec("restore cleanup cursor is too long".to_owned()))?; + let mut value = Vec::with_capacity(54 + job.index_cursor.len()); + value.extend_from_slice(CLEANUP_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.extend_from_slice(&job.operation_digest); + value.extend_from_slice(&job.ref_set_id.to_be_bytes()); + value.push(u8::from(job.index_complete)); + value.extend_from_slice(&cursor_len.to_be_bytes()); + value.extend_from_slice(&job.index_cursor); + Ok(value) +} + +pub(super) fn decode_restore_cleanup_job(value: &[u8]) -> Result { + let mut decoder = RestoreDecoder::new(value); + if decoder.take(8)? != CLEANUP_MAGIC || decoder.u8()? != RESTORE_FORMAT_VERSION { + return Err(MetadError::Codec( + "invalid restore cleanup job header".to_owned(), + )); + } + let operation_digest = decoder.array_32()?; + let ref_set_id = decoder.u64()?; + let index_complete = match decoder.u8()? { + 0 => false, + 1 => true, + _ => { + return Err(MetadError::Codec( + "restore cleanup job has an invalid completion flag".to_owned(), + )) + } + }; + let index_cursor = decoder.bytes()?; + decoder.finish()?; + let job = RestoreCleanupJob { + operation_digest, + ref_set_id, + index_complete, + index_cursor, + }; + encode_restore_cleanup_job(&job)?; + Ok(job) +} + +fn encode_restore_init_upload_intent( + intent: &RestoreInitUploadIntent, +) -> Result, MetadError> { + if intent.ref_set_id == 0 || intent.generation == 0 || intent.cleanup_pass > 1 { + return Err(MetadError::Codec( + "restore init upload intent contains a zero identity".to_owned(), + )); + } + let relative_path = canonical_restore_relative_path(&intent.relative_path)?; + if relative_path != intent.relative_path { + return Err(MetadError::Codec( + "restore init upload intent path is not canonical".to_owned(), + )); + } + let path = relative_path.as_bytes(); + let path_len = u32::try_from(path.len()) + .map_err(|_| MetadError::Codec("restore init upload path is too long".to_owned()))?; + let mut value = Vec::with_capacity(77 + path.len()); + value.extend_from_slice(INIT_UPLOAD_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.extend_from_slice(&intent.operation_digest); + value.extend_from_slice(&intent.ref_set_id.to_be_bytes()); + value.extend_from_slice(&intent.inode.get().to_be_bytes()); + value.extend_from_slice(&intent.generation.to_be_bytes()); + value.extend_from_slice(&intent.size.to_be_bytes()); + value.push(intent.cleanup_pass); + value.extend_from_slice(&path_len.to_be_bytes()); + value.extend_from_slice(path); + Ok(value) +} + +pub(super) fn decode_restore_init_upload_intent( + value: &[u8], +) -> Result { + let mut decoder = RestoreDecoder::new(value); + if decoder.take(8)? != INIT_UPLOAD_MAGIC || decoder.u8()? != RESTORE_FORMAT_VERSION { + return Err(MetadError::Codec( + "invalid restore init upload intent header".to_owned(), + )); + } + let intent = RestoreInitUploadIntent { + operation_digest: decoder.array_32()?, + ref_set_id: decoder.u64()?, + inode: InodeId::new(decoder.u64()?)?, + generation: decoder.u64()?, + size: decoder.u64()?, + cleanup_pass: decoder.u8()?, + relative_path: decoder.string()?, + }; + decoder.finish()?; + encode_restore_init_upload_intent(&intent)?; + Ok(intent) +} + +pub(super) fn encode_restore_init_upload_tombstone( + tombstone: &RestoreInitUploadTombstone, +) -> Result, MetadError> { + if tombstone.generation == 0 { + return Err(MetadError::Codec( + "restore init upload tombstone has a zero generation".to_owned(), + )); + } + let relative_path = canonical_restore_relative_path(&tombstone.relative_path)?; + if relative_path != tombstone.relative_path { + return Err(MetadError::Codec( + "restore init upload tombstone path is not canonical".to_owned(), + )); + } + let path = relative_path.as_bytes(); + let path_len = u32::try_from(path.len()) + .map_err(|_| MetadError::Codec("restore tombstone path is too long".to_owned()))?; + let mut value = Vec::with_capacity(101 + path.len()); + value.extend_from_slice(INIT_UPLOAD_TOMBSTONE_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.extend_from_slice(&tombstone.operation_digest); + value.extend_from_slice(&tombstone.initialization_digest); + value.extend_from_slice(&tombstone.inode.get().to_be_bytes()); + value.extend_from_slice(&tombstone.generation.to_be_bytes()); + value.extend_from_slice(&tombstone.size.to_be_bytes()); + value.extend_from_slice(&path_len.to_be_bytes()); + value.extend_from_slice(path); + Ok(value) +} + +pub(super) fn decode_restore_init_upload_tombstone( + value: &[u8], +) -> Result { + let mut decoder = RestoreDecoder::new(value); + if decoder.take(8)? != INIT_UPLOAD_TOMBSTONE_MAGIC || decoder.u8()? != RESTORE_FORMAT_VERSION { + return Err(MetadError::Codec( + "invalid restore init upload tombstone header".to_owned(), + )); + } + let tombstone = RestoreInitUploadTombstone { + operation_digest: decoder.array_32()?, + initialization_digest: decoder.array_32()?, + inode: InodeId::new(decoder.u64()?)?, + generation: decoder.u64()?, + size: decoder.u64()?, + relative_path: decoder.string()?, + }; + decoder.finish()?; + encode_restore_init_upload_tombstone(&tombstone)?; + Ok(tombstone) +} + +pub(super) fn validate_restore_init_upload_tombstone_row( + mount: MountId, + row: &crate::command::ScanItem, +) -> Result { + let tombstone = decode_restore_init_upload_tombstone(&row.value.0)?; + if row.key + != restore_init_upload_tombstone_key( + mount, + &tombstone.operation_digest, + tombstone.inode, + tombstone.generation, + ) + { + return Err(MetadError::Codec( + "restore init upload tombstone key changed identity".to_owned(), + )); + } + Ok(tombstone) +} + +pub(super) fn encode_restore_operation( + operation: &RestoreOperation, +) -> Result, MetadError> { + validate_restore_operation(operation)?; + let source = operation.source_path.as_bytes(); + let destination = operation.destination_path.as_bytes(); + let source_len = u32::try_from(source.len()) + .map_err(|_| MetadError::Codec("restore source path is too long".to_owned()))?; + let destination_len = u32::try_from(destination.len()) + .map_err(|_| MetadError::Codec("restore destination path is too long".to_owned()))?; + let mut out = Vec::with_capacity(166 + source.len() + destination.len()); + out.extend_from_slice(OPERATION_MAGIC); + out.push(RESTORE_FORMAT_VERSION); + out.push(operation_state_tag(operation.state)); + out.extend_from_slice(&operation.operation_digest); + out.extend_from_slice(&operation.initialization_digest); + for value in [ + operation.source_root.get(), + operation.destination_root.get(), + operation.snapshot_id, + operation.read_version, + operation.created_version, + operation.ref_set_id, + ] { + out.extend_from_slice(&value.to_be_bytes()); + } + out.extend_from_slice(&source_len.to_be_bytes()); + out.extend_from_slice(source); + out.extend_from_slice(&destination_len.to_be_bytes()); + out.extend_from_slice(destination); + Ok(out) +} + +pub(super) fn decode_restore_operation(bytes: &[u8]) -> Result { + let mut decoder = RestoreDecoder::new(bytes); + if decoder.take(8)? != OPERATION_MAGIC { + return Err(MetadError::Codec( + "invalid restore operation magic".to_owned(), + )); + } + if decoder.u8()? != RESTORE_FORMAT_VERSION { + return Err(MetadError::Codec( + "unsupported restore operation format version".to_owned(), + )); + } + let state = operation_state(decoder.u8()?)?; + let operation_digest = decoder.array_32()?; + let initialization_digest = decoder.array_32()?; + let source_root = InodeId::new(decoder.u64()?)?; + let destination_root = InodeId::new(decoder.u64()?)?; + let snapshot_id = decoder.u64()?; + let read_version = decoder.u64()?; + let created_version = decoder.u64()?; + let ref_set_id = decoder.u64()?; + let source_path = decoder.string()?; + let destination_path = decoder.string()?; + decoder.finish()?; + let operation = RestoreOperation { + operation_digest, + initialization_digest, + state, + source_root, + destination_root, + snapshot_id, + read_version, + created_version, + ref_set_id, + source_path, + destination_path, + }; + validate_restore_operation(&operation)?; + Ok(operation) +} + +fn validate_restore_operation(operation: &RestoreOperation) -> Result<(), MetadError> { + if operation.snapshot_id == 0 + || operation.read_version == 0 + || operation.created_version == 0 + || operation.ref_set_id == 0 + { + return Err(MetadError::Codec( + "restore operation contains a zero durable identity".to_owned(), + )); + } + let source = canonical_path(&parse_absolute_path(&operation.source_path)?)?; + let destination = canonical_path(&parse_absolute_path(&operation.destination_path)?)?; + if source != operation.source_path || destination != operation.destination_path { + return Err(MetadError::Codec( + "restore operation path is not canonical".to_owned(), + )); + } + if source == destination { + return Err(MetadError::Codec( + "restore source and destination are identical".to_owned(), + )); + } + Ok(()) +} + +/// Bind a decoded operation to both its physical durable key and the canonical +/// public request identity. The codec cannot perform this check because the +/// mount is deliberately not duplicated inside the operation value. +pub(super) fn validate_restore_operation_identity( + mount: MountId, + expected_digest: &[u8; 32], + operation: &RestoreOperation, +) -> Result<(), MetadError> { + if operation.operation_digest != *expected_digest { + return Err(MetadError::Codec( + "restore operation value does not match its durable key".to_owned(), + )); + } + let canonical_digest = restore_operation_digest( + mount, + &operation.source_path, + operation.snapshot_id, + &operation.destination_path, + ); + if canonical_digest != *expected_digest { + return Err(MetadError::Codec( + "restore operation digest does not match its request identity".to_owned(), + )); + } + Ok(()) +} + +fn operation_state_tag(state: RestoreOperationState) -> u8 { + match state { + RestoreOperationState::Preparing => 1, + RestoreOperationState::ReadyToAttach => 2, + RestoreOperationState::Complete => 3, + RestoreOperationState::Cleaning => 4, + RestoreOperationState::Discarding => 5, + RestoreOperationState::Releasing => 6, + } +} + +fn operation_state(tag: u8) -> Result { + match tag { + 1 => Ok(RestoreOperationState::Preparing), + 2 => Ok(RestoreOperationState::ReadyToAttach), + 3 => Ok(RestoreOperationState::Complete), + 4 => Ok(RestoreOperationState::Cleaning), + 5 => Ok(RestoreOperationState::Discarding), + 6 => Ok(RestoreOperationState::Releasing), + _ => Err(MetadError::Codec(format!( + "invalid restore operation state {tag}" + ))), + } +} + +fn system_key(mount: MountId, label: &[u8]) -> Vec { + let mut key = Vec::with_capacity(8 + label.len()); + key.extend_from_slice(&mount.get().to_be_bytes()); + key.extend_from_slice(label); + key +} + +fn system_key_with_digest(mount: MountId, label: &[u8], digest: &[u8; 32]) -> Vec { + let mut key = system_key(mount, label); + key.extend_from_slice(digest); + key +} + +fn system_key_with_u64(mount: MountId, label: &[u8], value: u64) -> Vec { + let mut key = system_key(mount, label); + key.extend_from_slice(&value.to_be_bytes()); + key +} + +/// Authoritative registry of restore-private control keyspaces, excluding the +/// active marker itself. Diagnostics and the explicit downgrade transaction use +/// this list so a newly-added durable row cannot be omitted from the drain +/// proof. Index-overlay keyspaces are registered by `restore_index` because +/// that module owns their physical MVCC layout. +pub(super) fn restore_control_keyspaces(mount: MountId) -> Vec<(&'static str, Vec)> { + [ + ("operation", OPERATION_KEY_LABEL), + ("destination_claim", CLAIM_KEY_LABEL), + ("staging_member", STAGING_KEY_LABEL), + ("staging_inode_inverse", STAGING_INVERSE_KEY_LABEL), + ("staging_inverse_owner", STAGING_INVERSE_OWNER_KEY_LABEL), + ("root_index", ROOT_KEY_LABEL), + ("base_owner", BASE_OWNER_KEY_LABEL), + ("base_inverse", BASE_INVERSE_KEY_LABEL), + ("base_inverse_owner", BASE_INVERSE_OWNER_KEY_LABEL), + ("base_seal", BASE_SEAL_KEY_LABEL), + ("cleanup_job", CLEANUP_KEY_LABEL), + ("init_upload_intent", INIT_UPLOAD_KEY_LABEL), + ("init_upload_tombstone", INIT_UPLOAD_TOMBSTONE_KEY_LABEL), + ( + "init_upload_tombstone_cursor", + INIT_UPLOAD_TOMBSTONE_CURSOR_KEY_LABEL, + ), + ("release_job", RELEASE_KEY_LABEL), + ("release_cursor", RELEASE_CURSOR_KEY_LABEL), + ("release_quarantine", RELEASE_QUARANTINE_KEY_LABEL), + ] + .into_iter() + .map(|(name, label)| (name, system_key(mount, label))) + .collect() +} + +/// Return whether a command can change the authoritative restore graph. +/// +/// The two mount-global worker cursors are scheduling hints rather than graph +/// ownership. Excluding their exact keys lets long fsck/object HEAD scans make +/// progress while permanent upload tombstones rotate round-robin. Corrupt +/// suffixed rows still match their registered keyspace and therefore fence the +/// scan like any other restore mutation. +pub(super) fn command_mutates_restore_graph(mount: MountId, command: &MetadataCommand) -> bool { + if !command + .mutations + .iter() + .any(|mutation| mutation.family == RecordFamily::System) + { + return false; + } + let init_cursor = restore_init_upload_tombstone_cursor_key(mount); + let release_cursor = restore_release_cursor_key(mount); + let active = restore_active_key(mount); + let activation_fence = restore_activation_fence_key(mount); + let control = restore_control_keyspaces(mount); + let index = super::restore_index::restore_index_private_keyspaces(mount); + + command.mutations.iter().any(|mutation| { + if mutation.family != RecordFamily::System { + return false; + } + if mutation.key == init_cursor || mutation.key == release_cursor { + return false; + } + mutation.key == active + || mutation.key == activation_fence + || control + .iter() + .any(|(_, prefix)| mutation.key.starts_with(prefix)) + || index + .iter() + .any(|(_, prefix)| mutation.key.starts_with(prefix)) + }) +} + +fn put_hash_field(hasher: &mut Sha256, bytes: &[u8]) { + hasher.update((bytes.len() as u64).to_be_bytes()); + hasher.update(bytes); +} + +fn preview_restore_initialization_artifact( + mount: MountId, + file: &RestoreInitializationFile, +) -> Result<(InodeId, Version, BodyDescriptor, Vec), MetadError> { + // Use the widest decimal inode/generation in object keys. Metadata ids are + // binary fixed-width, while object keys are embedded strings in chunk + // manifests; using u64::MAX therefore makes this preview conservative for + // every allocator state without writing an object. + let inode = InodeId::new(u64::MAX)?; + let generation = Version::new(u64::MAX)?; + let size = file.bytes.len() as u64; + let mut chunks = Vec::new(); + let mut chunk_offset = 0_u64; + while chunk_offset < size { + let chunk_index = chunk_offset / DEFAULT_CHUNK_SIZE; + let chunk_len = DEFAULT_CHUNK_SIZE.min(size - chunk_offset); + let mut blocks = Vec::new(); + let mut block_offset = 0_u64; + while block_offset < chunk_len { + let block_index = block_offset / DEFAULT_BLOCK_SIZE as u64; + let block_len = (DEFAULT_BLOCK_SIZE as u64).min(chunk_len - block_offset); + blocks.push(BlockDescriptor { + object_key: format!( + "blocks/{}/{}/{}/{}/{}", + mount.get(), + inode.get(), + generation.get(), + chunk_index, + block_index + ), + logical_offset: chunk_offset + block_offset, + object_offset: 0, + len: block_len, + digest_uri: format!("sha256:{}", "0".repeat(64)), + }); + block_offset = block_offset.saturating_add(block_len); + } + chunks.push(ChunkManifest { + chunk_index, + logical_offset: chunk_offset, + len: chunk_len, + slices: vec![SliceManifest { + slice_id: 1, + logical_offset: chunk_offset, + len: chunk_len, + blocks, + }], + }); + chunk_offset = chunk_offset.saturating_add(chunk_len); + } + Ok(( + inode, + generation, + BodyDescriptor { + producer: "nokv-restore-initialization".to_owned(), + digest_uri: body_digest_uri(&file.bytes), + size, + content_type: file.content_type.clone(), + manifest_id: format!("restore-init/{}", file.relative_path), + generation: generation.get(), + base_generation: 0, + chunk_size: DEFAULT_CHUNK_SIZE, + block_size: DEFAULT_BLOCK_SIZE as u64, + }, + chunks, + )) +} + +fn hex_digest(bytes: &[u8; 32]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(64); + for byte in bytes { + write!(&mut out, "{byte:02x}").expect("writing to String cannot fail"); + } + out +} + +pub(super) fn restore_barrier_operation_id(operation: &RestoreOperation) -> String { + format!("restore-{}", hex_digest(&operation.operation_digest)) +} + +struct RestoreDecoder<'a> { + input: &'a [u8], +} + +impl<'a> RestoreDecoder<'a> { + fn new(input: &'a [u8]) -> Self { + Self { input } + } + + fn take(&mut self, len: usize) -> Result<&'a [u8], MetadError> { + let Some((head, tail)) = self.input.split_at_checked(len) else { + return Err(MetadError::Codec( + "restore operation record is truncated".to_owned(), + )); + }; + self.input = tail; + Ok(head) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn u32(&mut self) -> Result { + Ok(u32::from_be_bytes( + self.take(4)?.try_into().expect("u32 width"), + )) + } + + fn u64(&mut self) -> Result { + Ok(u64::from_be_bytes( + self.take(8)?.try_into().expect("u64 width"), + )) + } + + fn array_32(&mut self) -> Result<[u8; 32], MetadError> { + Ok(self.take(32)?.try_into().expect("digest width")) + } + + fn bytes(&mut self) -> Result, MetadError> { + let len = self.u32()? as usize; + Ok(self.take(len)?.to_vec()) + } + + fn string(&mut self) -> Result { + String::from_utf8(self.bytes()?) + .map_err(|_| MetadError::Codec("restore operation path is not utf-8".to_owned())) + } + + fn finish(self) -> Result<(), MetadError> { + if self.input.is_empty() { + Ok(()) + } else { + Err(MetadError::Codec( + "restore operation record has trailing bytes".to_owned(), + )) + } + } +} + +fn canonical_restore_initialization( + mut initialization: RestoreInitialization, +) -> Result<(RestoreInitialization, [u8; 32]), MetadError> { + let count = initialization + .remove_relative_paths + .len() + .saturating_add(initialization.files.len()); + if count > MAX_RESTORE_INITIALIZATION_ENTRIES { + return Err(MetadError::RestoreResourceLimit { + resource: "restore initialization entries".to_owned(), + limit: MAX_RESTORE_INITIALIZATION_ENTRIES as u64, + actual: count as u64, + }); + } + for path in &mut initialization.remove_relative_paths { + *path = canonical_restore_relative_path(path)?; + } + for file in &mut initialization.files { + file.relative_path = canonical_restore_relative_path(&file.relative_path)?; + } + initialization.remove_relative_paths.sort(); + initialization + .files + .sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); + + // Bound the complete canonical initialization payload, not only file + // contents. This mirrors the digest framing below: every path/string/blob + // contributes its length field and fixed mode/uid/gid metadata. + let bytes = initialization + .remove_relative_paths + .iter() + .fold(0_usize, |total, path| { + total + .saturating_add(1) + .saturating_add(std::mem::size_of::()) + .saturating_add(path.len()) + }); + let bytes = initialization.files.iter().fold(bytes, |total, file| { + total + .saturating_add(1) + .saturating_add(std::mem::size_of::()) + .saturating_add(file.relative_path.len()) + .saturating_add(std::mem::size_of::()) + .saturating_add(file.content_type.len()) + .saturating_add(3 * std::mem::size_of::()) + .saturating_add(std::mem::size_of::()) + .saturating_add(file.bytes.len()) + }); + if bytes > MAX_RESTORE_INITIALIZATION_BYTES { + return Err(MetadError::RestoreResourceLimit { + resource: "restore initialization bytes".to_owned(), + limit: MAX_RESTORE_INITIALIZATION_BYTES as u64, + actual: bytes as u64, + }); + } + let mut paths = initialization.remove_relative_paths.clone(); + paths.extend( + initialization + .files + .iter() + .map(|file| file.relative_path.clone()), + ); + paths.sort(); + for pair in paths.windows(2) { + if pair[0] == pair[1] { + return Err(MetadError::InvalidPath( + "restore initialization contains duplicate paths".to_owned(), + )); + } + if pair[1] + .strip_prefix(&pair[0]) + .is_some_and(|suffix| suffix.starts_with('/')) + { + return Err(MetadError::InvalidPath( + "restore initialization paths overlap as ancestor and descendant".to_owned(), + )); + } + } + + let mut hasher = Sha256::new(); + hasher.update(b"nokv-restore-initialization-v1\0"); + for path in &initialization.remove_relative_paths { + hasher.update([1]); + put_hash_field(&mut hasher, path.as_bytes()); + } + for file in &initialization.files { + hasher.update([2]); + put_hash_field(&mut hasher, file.relative_path.as_bytes()); + put_hash_field(&mut hasher, file.content_type.as_bytes()); + hasher.update(file.mode.to_be_bytes()); + hasher.update(file.uid.to_be_bytes()); + hasher.update(file.gid.to_be_bytes()); + put_hash_field(&mut hasher, &file.bytes); + } + Ok((initialization, hasher.finalize().into())) +} + +fn canonical_restore_relative_path(path: &str) -> Result { + if path.is_empty() || path.starts_with('/') { + return Err(MetadError::InvalidPath( + "restore initialization path must be non-empty and relative".to_owned(), + )); + } + let absolute = format!("/{path}"); + let components = parse_absolute_path(&absolute)?; + if components.is_empty() { + return Err(MetadError::InvalidPath( + "restore initialization cannot address its root".to_owned(), + )); + } + let canonical = canonical_path(&components)?; + let relative = canonical + .strip_prefix('/') + .expect("canonical non-root path starts with slash") + .to_owned(); + if relative.len() > MAX_RESTORE_PATH_BYTES { + return Err(MetadError::RestoreResourceLimit { + resource: "restore initialization relative path bytes".to_owned(), + limit: MAX_RESTORE_PATH_BYTES as u64, + actual: relative.len() as u64, + }); + } + Ok(relative) +} + +fn restore_relative_components(path: &str) -> Result, MetadError> { + let canonical = canonical_restore_relative_path(path)?; + Ok(parse_absolute_path(&format!("/{canonical}"))?) +} + +fn canonical_restore_relative_components(components: &[DentryName]) -> Result { + if components.is_empty() { + return Ok(String::new()); + } + let absolute = canonical_path(components)?; + let relative = absolute + .strip_prefix('/') + .expect("canonical non-root path starts with slash") + .to_owned(); + if relative.len() > MAX_RESTORE_PATH_BYTES { + return Err(MetadError::RestoreResourceLimit { + resource: "restore relative path bytes".to_owned(), + limit: MAX_RESTORE_PATH_BYTES as u64, + actual: relative.len() as u64, + }); + } + Ok(relative) +} + +// The state machine implementation is added below this durable contract. Keep +// the entry points present while protocol/server work compiles against it. +impl NoKvFs +where + M: MetadataStore, + O: ObjectStore, +{ + pub(super) fn ensure_restore_snapshot_retirable( + &self, + snapshot_id: u64, + bindings: &[super::snapshot::VersionedForkBinding], + version: Version, + ) -> Result<(), MetadError> { + for binding in bindings { + let root = binding.binding.fork_root; + let Some(root_index) = self.metadata.get( + RecordFamily::System, + &restore_root_index_key(self.mount, root), + version, + ReadPurpose::WritePlanLocal, + )? + else { + continue; + }; + let digest: [u8; 32] = root_index + .0 + .as_slice() + .try_into() + .map_err(|_| MetadError::RestoreRootChanged { root })?; + let operation = self + .metadata + .get( + RecordFamily::System, + &restore_operation_key(self.mount, &digest), + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreRootChanged { root })?; + let operation = decode_restore_operation(&operation.0)?; + if operation.operation_digest != digest + || operation.destination_root != root + || operation.snapshot_id != snapshot_id + || binding.binding.snapshot_id != snapshot_id + || binding.binding.source_root != operation.source_root + || binding.binding.pinned_read_version != operation.read_version + { + return Err(MetadError::RestoreBindingChanged { root }); + } + // A restore's temporary ForkBinding is the only durable history + // floor while staging. It may not be retired through the generic + // clone path, even if no borrowed manifest has been written yet. + return Err(MetadError::RestoreInProgress); + } + Ok(()) + } + + pub(super) fn is_complete_restore_root(&self, root: InodeId) -> Result { + let Some((operation, _, _)) = self.restore_root_operation_at(root, self.read_version()?)? + else { + return Ok(false); + }; + match operation.state { + RestoreOperationState::Complete => Ok(true), + RestoreOperationState::Releasing => Err(MetadError::RestoreInProgress), + _ => Err(MetadError::RestoreRootChanged { root }), + } + } + + /// Install the stable pre-activation fence once metadata is safe to mutate. + /// + /// `NoKvFs::new` may wrap a pristine store which is about to receive a + /// checkpoint image, so construction cannot create this row. Bootstrap, + /// reopen, and recovery call this method before admitting namespace work. + /// The first restore rewrites the row in the same command as the active + /// marker; ordinary writes therefore conflict only with restore activation, + /// never with unrelated allocator reservations. + pub(super) fn ensure_restore_activation_fence(&self) -> Result<(), MetadError> { + const MAX_ATTEMPTS: usize = 8; + + let key = restore_activation_fence_key(self.mount); + if let Some(item) = self.metadata.get( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? { + if item.0 != [RESTORE_FORMAT_VERSION] { + return Err(MetadError::Codec( + "restore activation fence has an invalid value".to_owned(), + )); + } + return Ok(()); + } + for attempt in 0..MAX_ATTEMPTS { + let version = self.next_version()?; + let read_version = predecessor(version)?; + if let Some(item) = self.metadata.get_versioned( + RecordFamily::System, + &key, + read_version, + ReadPurpose::WritePlanLocal, + )? { + if item.value.0 != [RESTORE_FORMAT_VERSION] { + return Err(MetadError::Codec( + "restore activation fence has an invalid value".to_owned(), + )); + } + return Ok(()); + } + let active_key = restore_active_key(self.mount); + let active = self.metadata.get_versioned( + RecordFamily::System, + &active_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + if active + .as_ref() + .is_some_and(|item| item.value.0 != [RESTORE_FORMAT_VERSION]) + { + return Err(MetadError::Codec( + "restore active marker has an invalid value".to_owned(), + )); + } + let command = MetadataCommand { + request_id: request_id( + b"restore-activation-fence", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::ReserveAllocator, + read_version, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: active_key, + predicate: active + .map(|item| Predicate::VersionEquals(item.version)) + .unwrap_or(Predicate::NotExists), + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(vec![RESTORE_FORMAT_VERSION])), + }], + watch: Vec::new(), + }; + match self.commit_metadata(command) { + Ok(_) => return Ok(()), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + if attempt + 1 < MAX_ATTEMPTS => + { + continue; + } + Err( + error @ MetadError::SyncLogArchiveFailed { + committed: true, .. + }, + ) => { + let installed = self.metadata.get( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )?; + if installed + .as_ref() + .is_some_and(|value| value.0 == [RESTORE_FORMAT_VERSION]) + { + return Ok(()); + } + return Err(error); + } + Err(error) => return Err(error), + } + } + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + } + + /// Build the durable fence required by an ordinary inode-addressed write. + /// + /// Restore materialization deliberately creates real inode/dentry records + /// before the destination is attached. An inode id is not an authority to + /// mutate those records: a caller can guess one, and an attach can race a + /// read-only visibility check. Every ordinary namespace mutation therefore + /// joins the owning operation, member, and inverse rows in the same + /// `MetadataCommand` CAS. + /// + /// Complete operations behave like an ordinary namespace. During release, + /// only an inode which is still reachable through an escaped link may be + /// mutated; this lets the last escaped holder be removed without reopening + /// access to the detached tree. All materialization/discard states are + /// fail-closed. + pub(super) fn restore_namespace_write_predicates( + &self, + inodes: &[InodeId], + read_version: Version, + ) -> Result, MetadError> { + self.restore_namespace_write_predicates_with_policy(inodes, read_version, true, true) + } + + /// A restore destination parent is already proven reachable by its full + /// path-dentry CAS. This variant is also used while the first-hold + /// visibility write fence is held, so it must not recursively enter the + /// visibility read path. A Releasing parent is never a valid destination. + fn restore_destination_parent_predicates( + &self, + parent: InodeId, + read_version: Version, + ) -> Result, MetadError> { + self.restore_namespace_write_predicates_with_policy(&[parent], read_version, false, false) + } + + fn restore_namespace_write_predicates_with_policy( + &self, + inodes: &[InodeId], + read_version: Version, + require_complete_reachability: bool, + allow_releasing_reachable: bool, + ) -> Result, MetadError> { + let (active_key, active_version) = match self.restore_namespace_activity_fence()? { + RestoreNamespaceActivityFence::Inactive(fence) => return Ok(vec![fence]), + RestoreNamespaceActivityFence::Active { key, version } => (key, version), + }; + let mut predicates = Vec::new(); + let mut guarded_keys = HashSet::>::new(); + let mut operation_states = HashMap::<[u8; 32], (RestoreOperationState, Version)>::new(); + let unique_inodes = inodes.iter().copied().collect::>(); + let mut reachable_inodes = None::>; + let active_marker = (active_key, active_version); + + for inode in unique_inodes.iter().copied() { + let inverse_key = restore_staging_inode_key(self.mount, inode); + let Some(inverse) = self.metadata.get_versioned( + RecordFamily::System, + &inverse_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + else { + // Absence is also part of the write plan. Without this CAS an + // ordinary inode could be enrolled into a completed restore + // after planning, then become detached by release while the + // stale ordinary command still commits against it. + if guarded_keys.insert(inverse_key.clone()) { + predicates.push(PredicateRef { + family: RecordFamily::System, + key: inverse_key, + predicate: Predicate::NotExists, + }); + } + continue; + }; + let (operation_digest, ref_set_id) = decode_restore_staging_inverse(&inverse.value.0)?; + let operation_key = restore_operation_key(self.mount, &operation_digest); + let (state, operation_version) = match operation_states.get(&operation_digest) { + Some(state) => *state, + None => { + let operation_item = self + .metadata + .get_versioned( + RecordFamily::System, + &operation_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "restore staging inverse has no owning operation".to_owned(), + ) + })?; + let operation = decode_restore_operation(&operation_item.value.0)?; + if operation.operation_digest != operation_digest + || operation.ref_set_id != ref_set_id + { + return Err(MetadError::Codec( + "restore staging inverse changed operation identity".to_owned(), + )); + } + let state = (operation.state, operation_item.version); + operation_states.insert(operation_digest, state); + state + } + }; + match state { + RestoreOperationState::Complete => { + if require_complete_reachability + && self.restore_staging_possible.load(Ordering::Acquire) + { + if reachable_inodes.is_none() { + reachable_inodes = Some( + self.restore_reachable_inodes_at(&unique_inodes, read_version)?, + ); + } + if !reachable_inodes + .as_ref() + .is_some_and(|reachable| reachable.contains(&inode)) + { + return Err(MetadError::RestoreInProgress); + } + } + } + RestoreOperationState::Releasing => { + if !allow_releasing_reachable { + return Err(MetadError::RestoreInProgress); + } + if reachable_inodes.is_none() { + reachable_inodes = + Some(self.restore_reachable_inodes_at(&unique_inodes, read_version)?); + } + if !reachable_inodes + .as_ref() + .is_some_and(|reachable| reachable.contains(&inode)) + { + return Err(MetadError::RestoreInProgress); + } + } + RestoreOperationState::Preparing + | RestoreOperationState::ReadyToAttach + | RestoreOperationState::Cleaning + | RestoreOperationState::Discarding => { + return Err(MetadError::RestoreInProgress); + } + } + + let member_key = restore_staging_member_key(self.mount, ref_set_id, inode); + let member = self + .metadata + .get_versioned( + RecordFamily::System, + &member_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore staging inverse has no owning member".to_owned()) + })?; + let decoded_member = decode_restore_staging_member(&member.value.0)?; + if decoded_member.operation_digest != operation_digest + || decoded_member.destination_inode != inode + { + return Err(MetadError::Codec( + "restore staging member changed operation identity".to_owned(), + )); + } + let inverse_owner_key = + restore_staging_inverse_owner_key(self.mount, ref_set_id, inode); + let inverse_owner = self + .metadata + .get_versioned( + RecordFamily::System, + &inverse_owner_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore staging inverse has no ref-set owner".to_owned()) + })?; + if inverse_owner.value != inverse.value { + return Err(MetadError::Codec( + "restore staging inverse owner changed identity".to_owned(), + )); + } + + let (active_key, active_version) = active_marker.clone(); + + for (key, version) in [ + (active_key, active_version), + (operation_key.clone(), operation_version), + (inverse_key, inverse.version), + (member_key, member.version), + (inverse_owner_key, inverse_owner.version), + ] { + if guarded_keys.insert(key.clone()) { + predicates.push(PredicateRef { + family: RecordFamily::System, + key, + predicate: Predicate::VersionEquals(version), + }); + } + } + } + Ok(predicates) + } + + /// Fence the first restore hold without probing an inode-private key when + /// restore-to-fork has never been activated on this mount. + /// + /// Predicate-only `NotExists` guards are implemented by Holt as an atomic + /// empty sentinel create/delete pair. Making every ordinary write use such + /// a guard on a missing staging-inverse key lets concurrent commands race + /// on the same physical key. A dedicated, stable activation row avoids + /// that sentinel and is rewritten only by restore activation; allocator + /// reservations must not invalidate unrelated ordinary writes. + fn restore_namespace_activity_fence( + &self, + ) -> Result { + // The first restore hold can land while an ordinary write is staged. + // Read the latest control state; the returned exact-version predicate + // then invalidates only a command which crossed that activation. + let control_version = self.read_version()?; + let active_key = restore_active_key(self.mount); + if let Some(active) = self.metadata.get_versioned( + RecordFamily::System, + &active_key, + control_version, + ReadPurpose::WritePlanLocal, + )? { + if active.value.0 != [RESTORE_FORMAT_VERSION] { + return Err(MetadError::Codec( + "restore active marker has an invalid value".to_owned(), + )); + } + return Ok(RestoreNamespaceActivityFence::Active { + key: active_key, + version: active.version, + }); + } + + let key = restore_activation_fence_key(self.mount); + let activation = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + control_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("restore activation fence is missing".to_owned()))?; + if activation.value.0 != [RESTORE_FORMAT_VERSION] { + return Err(MetadError::Codec( + "restore activation fence has an invalid value".to_owned(), + )); + } + Ok(RestoreNamespaceActivityFence::Inactive(PredicateRef { + family: RecordFamily::System, + key, + predicate: Predicate::VersionEquals(activation.version), + })) + } + + /// Enrol newly named local inodes in the ref-set of a completed restore. + /// The rows are written with the namespace mutation which first exposes the + /// inode below that restore, so release never has a visibility/membership + /// gap. Directory renames only enrol their root here; the bounded release + /// walker discovers descendants before it is allowed to remove the parent. + pub(super) fn restore_namespace_enrollment_plan( + &self, + parent: InodeId, + projections: &[DentryProjection], + read_version: Version, + ) -> Result { + if matches!( + self.restore_namespace_activity_fence()?, + RestoreNamespaceActivityFence::Inactive(_) + ) { + // Every enrollment caller first joins + // `restore_namespace_write_predicates`, whose allocator-version + // fence invalidates the command if the first restore hold races + // this fast path. + return Ok(RestoreNamespaceEnrollmentPlan { + predicates: Vec::new(), + mutations: Vec::new(), + }); + } + let inverse_key = restore_staging_inode_key(self.mount, parent); + let Some(inverse) = self.metadata.get_versioned( + RecordFamily::System, + &inverse_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(RestoreNamespaceEnrollmentPlan { + predicates: Vec::new(), + mutations: Vec::new(), + }); + }; + let (operation_digest, ref_set_id) = decode_restore_staging_inverse(&inverse.value.0)?; + let operation_key = restore_operation_key(self.mount, &operation_digest); + let operation_item = self + .metadata + .get_versioned( + RecordFamily::System, + &operation_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore namespace parent has no operation".to_owned()) + })?; + let operation = decode_restore_operation(&operation_item.value.0)?; + if operation.operation_digest != operation_digest || operation.ref_set_id != ref_set_id { + return Err(MetadError::Codec( + "restore namespace parent changed operation identity".to_owned(), + )); + } + if operation.state != RestoreOperationState::Complete { + return Err(MetadError::RestoreInProgress); + } + self.restore_namespace_enrollment_plan_for_operation( + &operation, + operation_item.version, + projections, + read_version, + ) + } + + fn restore_namespace_enrollment_plan_for_operation( + &self, + operation: &RestoreOperation, + operation_version: Version, + projections: &[DentryProjection], + read_version: Version, + ) -> Result { + let mut predicates = vec![PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }]; + let mut mutations = Vec::new(); + let inverse_value = Value(encode_restore_staging_inverse(operation)); + let mut seen = HashSet::new(); + for projection in projections { + let inode = projection.attr.inode; + if !seen.insert(inode) { + continue; + } + if inode.shard_index() != self.shard_index { + return Err(MetadError::RestoreCrossShardUnsupported { inode }); + } + let member_key = restore_staging_member_key(self.mount, operation.ref_set_id, inode); + let inverse_key = restore_staging_inode_key(self.mount, inode); + let inverse_owner_key = + restore_staging_inverse_owner_key(self.mount, operation.ref_set_id, inode); + if let Some(existing) = self.metadata.get_versioned( + RecordFamily::System, + &inverse_key, + read_version, + ReadPurpose::WritePlanLocal, + )? { + let (digest, ref_set_id) = decode_restore_staging_inverse(&existing.value.0)?; + if digest != operation.operation_digest || ref_set_id != operation.ref_set_id { + return Err(MetadError::InvalidPath( + "inode cannot be enrolled by two restore ref-sets".to_owned(), + )); + } + let member = self + .metadata + .get_versioned( + RecordFamily::System, + &member_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore namespace inverse has no member".to_owned()) + })?; + let inverse_owner = self + .metadata + .get_versioned( + RecordFamily::System, + &inverse_owner_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "restore namespace inverse has no ref-set owner".to_owned(), + ) + })?; + let decoded = decode_restore_staging_member(&member.value.0)?; + if decoded.operation_digest != operation.operation_digest + || decoded.destination_inode != inode + || inverse_owner.value != existing.value + { + return Err(MetadError::Codec( + "restore namespace membership changed identity".to_owned(), + )); + } + predicates.extend([ + PredicateRef { + family: RecordFamily::System, + key: inverse_key, + predicate: Predicate::VersionEquals(existing.version), + }, + PredicateRef { + family: RecordFamily::System, + key: member_key, + predicate: Predicate::VersionEquals(member.version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_owner_key, + predicate: Predicate::VersionEquals(inverse_owner.version), + }, + ]); + continue; + } + for key in [&member_key, &inverse_key, &inverse_owner_key] { + if self + .metadata + .get( + RecordFamily::System, + key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .is_some() + { + return Err(MetadError::Codec( + "restore namespace membership is only partially present".to_owned(), + )); + } + predicates.push(PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }); + } + let member = RestoreStagingMember { + operation_digest: operation.operation_digest, + source_inode: None, + destination_inode: inode, + destination_parent: Some(projection.dentry.parent), + name: Some(projection.dentry.name.clone()), + // Dynamic membership is keyed by inode and retains its exact + // raw parent/name. It intentionally has no snapshot-relative + // path, which may not be UTF-8 for ordinary namespace writes. + relative_path: String::new(), + canonical_index_cursor: Vec::new(), + canonical_index_complete: false, + manifest_cursor: Vec::new(), + manifest_block_cursor: 0, + }; + mutations.extend([ + Mutation { + family: RecordFamily::System, + key: member_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_member(&member)?)), + }, + Mutation { + family: RecordFamily::System, + key: inverse_key, + op: MutationOp::Put, + value: Some(inverse_value.clone()), + }, + Mutation { + family: RecordFamily::System, + key: inverse_owner_key, + op: MutationOp::Put, + value: Some(inverse_value.clone()), + }, + ]); + } + Ok(RestoreNamespaceEnrollmentPlan { + predicates, + mutations, + }) + } + + /// Whether a durable `ForkBinding` is allowed to anchor an inode-addressed + /// namespace reachability proof. Generic clone/rollback bindings have no + /// restore root-index row and remain legal anchors. A restore binding is a + /// history-retention hold only: its detached tree must stay unreachable + /// until the single attach transaction removes the binding and publishes + /// the destination dentry. + pub(super) fn restore_fork_binding_is_namespace_anchor( + &self, + binding: &ForkBinding, + version: Version, + ) -> Result { + let root_key = restore_root_index_key(self.mount, binding.fork_root); + let Some(root_index) = self.metadata.get( + RecordFamily::System, + &root_key, + version, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(true); + }; + let operation_digest: [u8; 32] = + root_index + .0 + .as_slice() + .try_into() + .map_err(|_| MetadError::RestoreRootChanged { + root: binding.fork_root, + })?; + let operation = self + .metadata + .get( + RecordFamily::System, + &restore_operation_key(self.mount, &operation_digest), + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreRootChanged { + root: binding.fork_root, + })?; + let operation = decode_restore_operation(&operation.0)?; + if operation.operation_digest != operation_digest + || operation.destination_root != binding.fork_root + || operation.source_root != binding.source_root + || operation.snapshot_id != binding.snapshot_id + || operation.read_version != binding.pinned_read_version + || operation.created_version != binding.created_version + { + return Err(MetadError::RestoreBindingChanged { + root: binding.fork_root, + }); + } + match operation.state { + RestoreOperationState::Preparing + | RestoreOperationState::ReadyToAttach + | RestoreOperationState::Cleaning + | RestoreOperationState::Discarding => Ok(false), + RestoreOperationState::Complete | RestoreOperationState::Releasing => { + Err(MetadError::Codec( + "attached restore operation still has a temporary ForkBinding".to_owned(), + )) + } + } + } + + pub(super) fn prepare_restore_root_release( + &self, + root: InodeId, + version: Version, + ) -> Result, MetadError> { + let read_version = predecessor(version)?; + let Some((mut operation, operation_item, root_item)) = + self.restore_root_operation_at(root, read_version)? + else { + return Ok(None); + }; + if operation.state != RestoreOperationState::Complete { + return Err(MetadError::RestoreInProgress); + } + let active_key = restore_active_key(self.mount); + let active_item = self + .metadata + .get_versioned( + RecordFamily::System, + &active_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("complete restore has no active marker".to_owned()))?; + if active_item.value.0 != [RESTORE_FORMAT_VERSION] { + return Err(MetadError::Codec( + "restore active marker has an invalid value".to_owned(), + )); + } + let operation_key = restore_operation_key(self.mount, &operation.operation_digest); + let root_key = restore_root_index_key(self.mount, root); + let release_key = restore_release_job_key(self.mount, operation.ref_set_id); + operation.state = RestoreOperationState::Releasing; + let job = RestoreReleaseJob { + operation_digest: operation.operation_digest, + ref_set_id: operation.ref_set_id, + phase: RestoreReleasePhase::ExactReferences, + cursor: Vec::new(), + }; + Ok(Some(RestoreReleaseTransition { + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::VersionEquals(operation_item.version), + }, + PredicateRef { + family: RecordFamily::System, + key: root_key, + predicate: Predicate::VersionEquals(root_item.version), + }, + PredicateRef { + family: RecordFamily::System, + key: release_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: active_key.clone(), + predicate: Predicate::VersionEquals(active_item.version), + }, + ], + mutations: vec![ + Mutation { + family: RecordFamily::System, + key: operation_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_operation(&operation)?)), + }, + Mutation { + family: RecordFamily::System, + key: release_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_release_job(&job)?)), + }, + Mutation { + family: RecordFamily::System, + key: active_key, + op: MutationOp::Put, + value: Some(Value(vec![RESTORE_FORMAT_VERSION])), + }, + ], + })) + } + + fn restore_root_operation_at( + &self, + root: InodeId, + version: Version, + ) -> Result, MetadError> { + let root_key = restore_root_index_key(self.mount, root); + let Some(root_item) = self.metadata.get_versioned( + RecordFamily::System, + &root_key, + version, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(None); + }; + let digest: [u8; 32] = root_item + .value + .0 + .as_slice() + .try_into() + .map_err(|_| MetadError::RestoreRootChanged { root })?; + let operation_key = restore_operation_key(self.mount, &digest); + let operation_item = self + .metadata + .get_versioned( + RecordFamily::System, + &operation_key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreRootChanged { root })?; + let operation = decode_restore_operation(&operation_item.value.0)?; + validate_restore_operation_identity(self.mount, &digest, &operation)?; + if operation.destination_root != root { + return Err(MetadError::RestoreRootChanged { root }); + } + Ok(Some((operation, operation_item, root_item))) + } + + pub fn restore_subtree_path_to_fork( + &self, + source_path: &str, + snapshot_id: u64, + destination_path: &str, + ) -> Result { + self.restore_subtree_path_to_fork_initialized( + source_path, + snapshot_id, + destination_path, + RestoreInitialization::default(), + ) + } + + pub fn restore_subtree_path_to_fork_initialized( + &self, + source_path: &str, + snapshot_id: u64, + destination_path: &str, + initialization: RestoreInitialization, + ) -> Result { + let started = Instant::now(); + self.restore_to_fork_requests_total + .fetch_add(1, Ordering::Relaxed); + let _restore = self + .restore_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + let result = self.restore_subtree_path_to_fork_initialized_locked( + source_path, + snapshot_id, + destination_path, + initialization, + ); + match &result { + Ok(_) => { + self.restore_to_fork_success_total + .fetch_add(1, Ordering::Relaxed); + } + Err(_) => { + self.restore_to_fork_failure_total + .fetch_add(1, Ordering::Relaxed); + } + } + let elapsed_ns = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX); + self.restore_to_fork_elapsed_ns_total + .fetch_add(elapsed_ns, Ordering::Relaxed); + self.restore_to_fork_elapsed_ns_max + .fetch_max(elapsed_ns, Ordering::Relaxed); + result + } + + fn restore_subtree_path_to_fork_initialized_locked( + &self, + source_path: &str, + snapshot_id: u64, + destination_path: &str, + initialization: RestoreInitialization, + ) -> Result { + let source_components = parse_absolute_path(source_path)?; + let source_path = canonical_path(&source_components)?; + let destination_components = parse_absolute_path(destination_path)?; + let destination_path = canonical_path(&destination_components)?; + if source_path == destination_path { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path, + }); + } + let path_bytes = source_path.len().saturating_add(destination_path.len()); + if path_bytes > MAX_RESTORE_PATH_BYTES { + return Err(MetadError::RestoreResourceLimit { + resource: "restore source and destination path bytes".to_owned(), + limit: MAX_RESTORE_PATH_BYTES as u64, + actual: path_bytes as u64, + }); + } + let (initialization, initialization_digest) = + canonical_restore_initialization(initialization)?; + let operation_digest = + restore_operation_digest(self.mount, &source_path, snapshot_id, &destination_path); + let operation_id = + restore_operation_id(self.mount, &source_path, snapshot_id, &destination_path)?; + + // Terminal lookup is intentionally first. A completed retry does not + // need the source namespace or caller-owned snapshot pin to survive. + if let Some(operation) = self.restore_operation(operation_digest)? { + if operation.initialization_digest != initialization_digest + || operation.source_path != source_path + || operation.destination_path != destination_path + { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path, + }); + } + return match operation.state { + RestoreOperationState::Complete => { + self.completed_restore_outcome(&operation_id, &operation) + } + RestoreOperationState::Preparing + | RestoreOperationState::Cleaning + | RestoreOperationState::Discarding => { + self.discard_preparing_restore(&operation)?; + self.restore_subtree_path_to_fork_initialized_locked( + &source_path, + snapshot_id, + &destination_path, + initialization, + ) + } + RestoreOperationState::ReadyToAttach => { + self.attach_restore_destination(&operation_id, &operation) + } + RestoreOperationState::Releasing => Err(MetadError::RestoreInProgress), + }; + } + + // The destination claim is a durable cross-owner serialization point. + // Check it before resolving the source or validating its pin so a + // conflicting request never leaks a generic predicate failure after + // expensive preflight work. + if let Some(claim) = self.metadata.get( + RecordFamily::System, + &restore_destination_claim_key(self.mount, &destination_path), + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? { + let claim_digest: [u8; 32] = claim.0.as_slice().try_into().map_err(|_| { + MetadError::Codec("restore destination claim has an invalid length".to_owned()) + })?; + if claim_digest != operation_digest { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path, + }); + } + // Claim and operation are installed atomically. A matching orphan + // claim is therefore a fail-closed recovery condition, never a + // license to start a second materialization. + return Err(MetadError::RestoreInProgress); + } + + let Some((destination_name, destination_parent_components)) = + destination_components.split_last() + else { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path, + }); + }; + let destination_proof = self.restore_directory_path_proof(destination_parent_components)?; + if destination_proof.inode.shard_index() != self.shard_index { + return Err(MetadError::RestoreCrossShardUnsupported { + inode: destination_proof.inode, + }); + } + if self.lookup_path(&destination_path)?.is_some() { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path, + }); + } + let source_proof = self.restore_directory_path_proof(&source_components)?; + if source_proof.inode.shard_index() != self.shard_index { + return Err(MetadError::RestoreCrossShardUnsupported { + inode: source_proof.inode, + }); + } + let pin_item = self + .versioned_snapshot_pin_at( + snapshot_id, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + self.ensure_snapshot_id_shard(snapshot_id, source_proof.inode)?; + if pin_item.pin.root != source_proof.inode { + return Err(MetadError::SnapshotRootMismatch { + snapshot_id, + expected_root: source_proof.inode, + actual_root: Some(pin_item.pin.root), + actual_shard: self.shard_index, + }); + } + self.ensure_snapshot_pin_live(&pin_item.pin)?; + let source_read_version = Version::new(pin_item.pin.read_version)?; + let snapshot_path_root = match self + .resolve_components_as_directory_from_at_version_for_purpose( + InodeId::root(), + &source_components, + source_read_version, + ReadPurpose::Snapshot, + ) { + Ok(root) => Some(root), + Err(MetadError::NotFound | MetadError::NotDirectory) => None, + Err(error) => return Err(error), + }; + if snapshot_path_root != Some(pin_item.pin.root) { + return Err(MetadError::SnapshotRootMismatch { + snapshot_id, + expected_root: pin_item.pin.root, + actual_root: snapshot_path_root, + actual_shard: self.shard_index, + }); + } + let source_attr = self + .get_attr_at_version_for_purpose( + source_proof.inode, + source_read_version, + ReadPurpose::Snapshot, + )? + .ok_or(MetadError::NotFound)?; + if source_attr.file_type != FileType::Directory { + return Err(MetadError::NotDirectory); + } + self.preflight_restore_subtree( + source_proof.inode, + source_read_version, + &source_path, + &destination_path, + destination_proof.inode, + destination_name, + &initialization, + )?; + + // A fresh mount may need to durably initialize the object-GC Open + // claim. Do that before allocating the operation commit version: the + // claim's exact record version must be visible at the hold command's + // read version. + let object_reference = self.begin_object_reference_mutation()?; + let destination_root = self.next_inode()?; + let created_version = self.next_version()?; + let operation = RestoreOperation { + operation_digest, + initialization_digest, + state: RestoreOperationState::Preparing, + source_root: source_proof.inode, + destination_root, + snapshot_id, + read_version: pin_item.pin.read_version, + created_version: created_version.get(), + ref_set_id: created_version.get(), + source_path, + destination_path: destination_path.clone(), + }; + let install_result = { + // Linearize the fast-path hint with the first durable staging + // write. Recovery/failover takes this same exclusive fence, so it + // cannot prove an empty keyspace and clear the hint immediately + // before the hold becomes visible. + let _visibility = self + .restore_visibility_fence + .write() + .unwrap_or_else(|error| error.into_inner()); + self.restore_staging_possible.store(true, Ordering::Release); + self.install_restore_hold(RestoreHold { + operation: &operation, + object_reference, + pin_version: pin_item.version, + source_attr: &source_attr, + source_predicates: &source_proof.predicates, + destination_parent: destination_proof.inode, + destination_name, + destination_predicates: &destination_proof.predicates, + }) + }; + if let Err(error) = install_result { + // Predicate failure may mean no hold was installed, or that a + // concurrent owner installed one. Re-prove rather than guessing; + // failure leaves the hint true. + let _ = self.recover_restore_staging_visibility(); + if matches!(error, MetadError::Metadata(MetadataError::PredicateFailed)) { + if let Some(durable) = self.restore_operation(operation_digest)? { + if durable.initialization_digest != initialization_digest { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path, + }); + } + return match durable.state { + RestoreOperationState::Complete => { + self.completed_restore_outcome(&operation_id, &durable) + } + _ => Err(MetadError::RestoreInProgress), + }; + } + if let Some(claim) = self.metadata.get( + RecordFamily::System, + &restore_destination_claim_key(self.mount, &destination_path), + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? { + let claim_digest: [u8; 32] = claim.0.as_slice().try_into().map_err(|_| { + MetadError::Codec( + "restore destination claim has an invalid length".to_owned(), + ) + })?; + return if claim_digest == operation_digest { + Err(MetadError::RestoreInProgress) + } else { + Err(MetadError::RestoreDestinationConflict { + destination: destination_path, + }) + }; + } + if self + .lookup_plus_for_write_plan(destination_proof.inode, destination_name)? + .is_some() + { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path, + }); + } + } + return Err(error); + } + + self.materialize_restore_tree(&operation)?; + self.apply_restore_initialization(&operation, &initialization)?; + self.materialize_and_seal_restore_indexes( + &operation, + destination_proof.inode, + destination_name, + )?; + live_test_barrier::restore_applied( + &restore_barrier_operation_id(&operation), + live_test_barrier::RestoreAppliedPhase::IndexSealed, + )?; + self.seal_restore_base_references(&operation)?; + self.mark_restore_ready_to_attach(&operation)?; + self.attach_restore_destination(&operation_id, &operation) + } + + fn restore_operation( + &self, + operation_digest: [u8; 32], + ) -> Result, MetadError> { + let Some(value) = self.metadata.get( + RecordFamily::System, + &restore_operation_key(self.mount, &operation_digest), + self.read_version()?, + ReadPurpose::UserStrong, + )? + else { + return Ok(None); + }; + let operation = decode_restore_operation(&value.0)?; + validate_restore_operation_identity(self.mount, &operation_digest, &operation)?; + Ok(Some(operation)) + } + + /// Enter the fail-closed visibility mode while owner/process-local state is + /// being reconstructed. The first restore hold uses the same write fence + /// across both this store and its durable commit, so a concurrent recovery + /// proof cannot clear the hint between those two actions. + pub(super) fn mark_restore_staging_possible(&self) { + let _visibility = self + .restore_visibility_fence + .write() + .unwrap_or_else(|error| error.into_inner()); + self.restore_staging_possible.store(true, Ordering::Release); + } + + /// Commit a Complete -> Releasing namespace transition under the same + /// visibility write fence used by the first restore hold. Setting the hint + /// before the durable command prevents a Complete fast-path reader/writer + /// from crossing the transition and addressing the newly detached tree. + pub(super) fn commit_restore_release_transition( + &self, + command: MetadataCommand, + starts_release: bool, + ) -> Result { + if !starts_release { + return self.commit_metadata(command); + } + let result = { + let _visibility = self + .restore_visibility_fence + .write() + .unwrap_or_else(|error| error.into_inner()); + self.restore_staging_possible.store(true, Ordering::Release); + self.commit_metadata(command) + }; + match result { + // A successful Complete -> Releasing transition deliberately + // leaves the process-local hint in the slow path. The release + // worker clears it only after the final durable command removes + // the last Releasing operation. + Ok(committed) => Ok(committed), + Err(error) => { + // Predicate failure may mean a concurrent transition won. A + // failed reconstruction deliberately leaves the hint true. + let _ = self.recover_restore_staging_visibility(); + Err(error) + } + } + } + + /// Rebuild the process-local visibility hint from durable restore-private + /// rows. The hint is only an optimization: malformed or incomplete durable + /// state returns an error with the hint left fail closed. + pub(super) fn recover_restore_staging_visibility(&self) -> Result<(), MetadError> { + const RECOVERY_PAGE_ROWS: usize = 256; + + self.ensure_restore_activation_fence()?; + let _visibility = self + .restore_visibility_fence + .write() + .unwrap_or_else(|error| error.into_inner()); + // Callers invoke recovery only after bootstrap or after an image/log + // install has made metadata scans safe. Remember that lifecycle fact + // separately from whether this particular proof succeeds. + self.restore_visibility_recovery_ready + .store(true, Ordering::Release); + self.restore_staging_possible.store(true, Ordering::Release); + + let version = self.read_version()?; + let keyspaces = restore_control_keyspaces(self.mount); + let operation_prefix = keyspaces + .iter() + .find_map(|(name, prefix)| (*name == "operation").then_some(prefix.clone())) + .ok_or_else(|| MetadError::Codec("restore operation keyspace is missing".to_owned()))?; + let inverse_prefix = keyspaces + .iter() + .find_map(|(name, prefix)| (*name == "staging_inode_inverse").then_some(prefix.clone())) + .ok_or_else(|| { + MetadError::Codec("restore staging inverse keyspace is missing".to_owned()) + })?; + + let mut staging_possible = false; + let mut start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: operation_prefix.clone(), + start_after: start_after.clone(), + version, + limit: RECOVERY_PAGE_ROWS, + purpose: ReadPurpose::WritePlanLocal, + })?; + for row in &rows { + let digest_bytes = row + .key + .strip_prefix(operation_prefix.as_slice()) + .filter(|bytes| bytes.len() == 32) + .ok_or_else(|| { + MetadError::Codec( + "restore operation row has an invalid durable key".to_owned(), + ) + })?; + let digest: [u8; 32] = digest_bytes.try_into().map_err(|_| { + MetadError::Codec("restore operation digest has an invalid length".to_owned()) + })?; + if restore_operation_key(self.mount, &digest) != row.key { + return Err(MetadError::Codec( + "restore operation row key changed identity".to_owned(), + )); + } + let operation = decode_restore_operation(&row.value.0)?; + validate_restore_operation_identity(self.mount, &digest, &operation)?; + if operation.state == RestoreOperationState::Complete { + self.validate_restore_complete_visibility_marker(&operation, version)?; + } else { + staging_possible = true; + } + } + start_after = rows.last().map(|row| row.key.clone()); + if rows.len() < RECOVERY_PAGE_ROWS { + break; + } + } + + let mut start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: inverse_prefix.clone(), + start_after: start_after.clone(), + version, + limit: RECOVERY_PAGE_ROWS, + purpose: ReadPurpose::WritePlanLocal, + })?; + for row in &rows { + let inode_bytes = row + .key + .strip_prefix(inverse_prefix.as_slice()) + .filter(|bytes| bytes.len() == 8) + .ok_or_else(|| { + MetadError::Codec( + "restore staging inverse has an invalid durable key".to_owned(), + ) + })?; + let inode = InodeId::new(u64::from_be_bytes(inode_bytes.try_into().map_err( + |_| { + MetadError::Codec( + "restore staging inverse inode has an invalid length".to_owned(), + ) + }, + )?))?; + if restore_staging_inode_key(self.mount, inode) != row.key { + return Err(MetadError::Codec( + "restore staging inverse key changed identity".to_owned(), + )); + } + let (operation_digest, ref_set_id) = decode_restore_staging_inverse(&row.value.0)?; + let operation = self + .metadata + .get( + RecordFamily::System, + &restore_operation_key(self.mount, &operation_digest), + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "restore staging inverse has no durable operation".to_owned(), + ) + })?; + let operation = decode_restore_operation(&operation.0)?; + if operation.operation_digest != operation_digest { + return Err(MetadError::Codec( + "restore staging inverse operation changed identity".to_owned(), + )); + } + if operation.ref_set_id != ref_set_id { + return Err(MetadError::Codec( + "restore staging inverse changed operation identity".to_owned(), + )); + } + } + start_after = rows.last().map(|row| row.key.clone()); + if rows.len() < RECOVERY_PAGE_ROWS { + break; + } + } + self.restore_staging_possible + .store(staging_possible, Ordering::Release); + Ok(()) + } + + #[cfg(test)] + pub(super) fn restore_staging_slow_path_enabled(&self) -> bool { + self.restore_staging_possible.load(Ordering::Acquire) + } + + fn completed_restore_outcome( + &self, + operation_id: &str, + operation: &RestoreOperation, + ) -> Result { + // The terminal operation row is authoritative. The restored root may + // have been renamed since completion, and exact retry remains valid + // after source pin retirement/source deletion. + Ok(RestoreOutcome { + operation_id: operation_id.to_owned(), + state: RestoreState::Complete, + source_root: operation.source_root, + destination_root: operation.destination_root, + snapshot_id: operation.snapshot_id, + read_version: operation.read_version, + cleanup_pending: false, + }) + } + + fn restore_directory_path_proof( + &self, + components: &[DentryName], + ) -> Result { + let version = self.read_version()?; + let mut inode = InodeId::root(); + let mut predicates = vec![PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, inode), + predicate: Predicate::Exists, + }]; + for component in components { + let (entry, dentry_version) = self + .lookup_plus_at_version_for_purpose( + inode, + component, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + if entry.attr.file_type != FileType::Directory { + return Err(MetadError::NotDirectory); + } + predicates.push(PredicateRef { + family: RecordFamily::Dentry, + key: dentry_key(self.mount, inode, component), + predicate: Predicate::VersionEquals(dentry_version), + }); + inode = entry.attr.inode; + } + Ok(RestorePathProof { inode, predicates }) + } + + fn install_restore_hold(&self, hold: RestoreHold<'_>) -> Result<(), MetadError> { + let RestoreHold { + operation, + object_reference, + pin_version, + source_attr, + source_predicates, + destination_parent, + destination_name, + destination_predicates, + } = hold; + let version = Version::new(operation.created_version)?; + let read_version = predecessor(version)?; + // All inode/version allocation happened before entering this critical + // section. Hold allocator_gate through the hold apply so a reservation + // cannot rewrite a v1 allocator row between the active-marker CAS and + // the atomic v2 downgrade-fence mutation. + let _allocator_guard = self.allocator_gate.lock().map_err(|error| { + MetadataError::Backend(format!("metadata allocator gate poisoned: {error}")) + })?; + let operation_key = restore_operation_key(self.mount, &operation.operation_digest); + let claim_key = restore_destination_claim_key(self.mount, &operation.destination_path); + let binding_key = fork_binding_key(self.mount, operation.destination_root); + let root_index_key = restore_root_index_key(self.mount, operation.destination_root); + let member_key = restore_staging_member_key( + self.mount, + operation.ref_set_id, + operation.destination_root, + ); + let staging_inode_key = restore_staging_inode_key(self.mount, operation.destination_root); + let staging_inverse_owner_key = restore_staging_inverse_owner_key( + self.mount, + operation.ref_set_id, + operation.destination_root, + ); + let active_key = restore_active_key(self.mount); + let active = self.metadata.get_versioned( + RecordFamily::System, + &active_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + if let Some(active) = &active { + if active.value.0 != [RESTORE_FORMAT_VERSION] { + return Err(MetadError::Codec( + "invalid restore-to-fork active marker".to_owned(), + )); + } + } + let activation_key = restore_activation_fence_key(self.mount); + let activation = self + .metadata + .get_versioned( + RecordFamily::System, + &activation_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("restore activation fence is missing".to_owned()))?; + if activation.value.0 != [RESTORE_FORMAT_VERSION] { + return Err(MetadError::Codec( + "restore activation fence has an invalid value".to_owned(), + )); + } + let mut root_attr = source_attr.clone(); + root_attr.inode = operation.destination_root; + root_attr.nlink = FileType::Directory.initial_link_count(); + root_attr.generation = version.get(); + root_attr.ctime_ms = current_time_ms(); + let binding = ForkBinding { + fork_root: operation.destination_root, + source_root: operation.source_root, + pinned_read_version: operation.read_version, + snapshot_id: operation.snapshot_id, + created_version: version.get(), + }; + let mut predicates = source_predicates.to_vec(); + predicates.extend_from_slice(destination_predicates); + predicates + .extend(self.restore_destination_parent_predicates(destination_parent, read_version)?); + predicates.extend([ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::Snapshot, + key: snapshot_pin_key(self.mount, operation.snapshot_id), + predicate: Predicate::VersionEquals(pin_version), + }, + PredicateRef { + family: RecordFamily::Dentry, + key: dentry_key(self.mount, destination_parent, destination_name), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, operation.destination_root), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: claim_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: root_index_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: member_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: staging_inode_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: staging_inverse_owner_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: active_key.clone(), + predicate: active + .as_ref() + .map(|item| Predicate::VersionEquals(item.version)) + .unwrap_or(Predicate::NotExists), + }, + PredicateRef { + family: RecordFamily::System, + key: activation_key.clone(), + predicate: Predicate::VersionEquals(activation.version), + }, + ]); + let mut command = MetadataCommand { + request_id: request_id( + b"restore-install-hold", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::SnapshotSubtree, + read_version, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: operation_key.clone(), + predicates, + mutations: vec![ + Mutation { + family: RecordFamily::Inode, + key: inode_key(self.mount, operation.destination_root), + op: MutationOp::Put, + value: Some(Value(encode_inode_attr(&root_attr))), + }, + Mutation { + family: RecordFamily::ForkBinding, + key: binding_key, + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(&binding))), + }, + Mutation { + family: RecordFamily::System, + key: operation_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_operation(operation)?)), + }, + Mutation { + family: RecordFamily::System, + key: claim_key, + op: MutationOp::Put, + value: Some(Value(operation.operation_digest.to_vec())), + }, + Mutation { + family: RecordFamily::System, + key: root_index_key, + op: MutationOp::Put, + value: Some(Value(operation.operation_digest.to_vec())), + }, + Mutation { + family: RecordFamily::System, + key: member_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_member( + &RestoreStagingMember { + operation_digest: operation.operation_digest, + source_inode: Some(operation.source_root), + destination_inode: operation.destination_root, + destination_parent: None, + name: None, + relative_path: String::new(), + canonical_index_cursor: Vec::new(), + canonical_index_complete: true, + manifest_cursor: Vec::new(), + manifest_block_cursor: 0, + }, + )?)), + }, + Mutation { + family: RecordFamily::System, + key: staging_inode_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_inverse(operation))), + }, + Mutation { + family: RecordFamily::System, + key: staging_inverse_owner_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_inverse(operation))), + }, + Mutation { + family: RecordFamily::System, + key: active_key, + op: MutationOp::Put, + value: Some(Value(vec![RESTORE_FORMAT_VERSION])), + }, + Mutation { + family: RecordFamily::System, + key: activation_key, + op: MutationOp::Put, + value: Some(Value(vec![RESTORE_FORMAT_VERSION])), + }, + ], + watch: Vec::new(), + }; + self.commit_metadata_from_factory(|| { + // Build the allocator mutation inside the owner epoch fence used by + // apply; its encoded epoch can therefore never lag a concurrent + // failover. The exact allocator version joins the same command as + // the first restore marker and temporary ForkBinding. + let (allocator_predicate, allocator_mutation) = + self.restore_allocator_fence_plan(read_version)?; + command.predicates.push(allocator_predicate); + command.mutations.push(allocator_mutation); + validate_restore_command_bounds(&command, "restore install hold")?; + Ok(command) + })?; + live_test_barrier::restore_applied( + &restore_barrier_operation_id(operation), + live_test_barrier::RestoreAppliedPhase::Hold, + )?; + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn preflight_restore_subtree( + &self, + source_root: InodeId, + read_version: Version, + source_path: &str, + destination_path: &str, + destination_parent: InodeId, + destination_name: &DentryName, + initialization: &RestoreInitialization, + ) -> Result<(), MetadError> { + // Build the same command shape used by materialization before any + // durable restore state or inode allocation exists. All ids are + // fixed-width in metadata keys/values, so source ids are exact size + // stand-ins for the detached destination ids. + let bounds_version = Version::new(2)?; + let bounds_operation = RestoreOperation { + operation_digest: [0; 32], + initialization_digest: [0; 32], + state: RestoreOperationState::Preparing, + source_root, + destination_root: source_root, + snapshot_id: 1, + read_version: read_version.get(), + created_version: bounds_version.get(), + ref_set_id: bounds_version.get(), + source_path: source_path.to_owned(), + destination_path: destination_path.to_owned(), + }; + let mut base_reference_preflight = + self.begin_restore_base_reference_preflight(&bounds_operation); + let bounds_object_reference = ObjectReferenceMutation::from_version(Version::new(1)?); + let initialization_paths = initialization + .remove_relative_paths + .iter() + .cloned() + .chain( + initialization + .files + .iter() + .map(|file| file.relative_path.clone()), + ) + .collect::>(); + let removed_paths = initialization + .remove_relative_paths + .iter() + .cloned() + .collect::>(); + let initialization_files = initialization + .files + .iter() + .map(|file| (file.relative_path.as_str(), file)) + .collect::>(); + let mut matched_initialization_files = HashSet::::new(); + let mut canonical_index_batch = + super::restore_index::RestoreCanonicalIndexPreflightBatch::default(); + self.preflight_restore_xattrs(source_root, read_version)?; + let mut root_attr = self + .get_attr_at_version_for_purpose(source_root, read_version, ReadPurpose::Snapshot)? + .ok_or(MetadError::NotFound)?; + root_attr.inode = source_root; + root_attr.nlink = FileType::Directory.initial_link_count(); + root_attr.generation = bounds_version.get(); + root_attr.ctime_ms = current_time_ms(); + self.push_restore_canonical_index_preflight_member( + &bounds_operation, + &RestoreStagingMember { + operation_digest: bounds_operation.operation_digest, + source_inode: Some(source_root), + destination_inode: source_root, + destination_parent: None, + name: None, + relative_path: String::new(), + canonical_index_cursor: Vec::new(), + canonical_index_complete: true, + manifest_cursor: Vec::new(), + manifest_block_cursor: 0, + }, + &projection( + destination_parent, + destination_name.clone(), + root_attr, + None, + ), + &mut canonical_index_batch, + )?; + let mut queue = vec![RestoreCloneFrame { + source: source_root, + destination: source_root, + relative_components: Vec::new(), + after: None, + }]; + self.preflight_restore_custom_index_catalog( + &bounds_operation, + source_root, + &[], + &initialization_paths, + )?; + let mut entries = 1_usize; + while let Some(frame) = queue.pop() { + let page = self.read_dir_plus_page_at_version_for_purpose( + frame.source, + frame.after.as_ref(), + RESTORE_BATCH_ENTRIES, + read_version, + ReadPurpose::Snapshot, + )?; + let children = page.entries; + entries = entries.saturating_add(children.len()); + if entries > MAX_RESTORE_SUBTREE_ENTRIES { + return Err(MetadError::RestoreResourceLimit { + resource: "restore subtree entries".to_owned(), + limit: MAX_RESTORE_SUBTREE_ENTRIES as u64, + actual: entries as u64, + }); + } + if let Some(after) = page.next_cursor { + queue.push(RestoreCloneFrame { + source: frame.source, + destination: frame.destination, + relative_components: frame.relative_components.clone(), + after: Some(after), + }); + } + if !children.is_empty() { + let mut bounds_batch = Vec::with_capacity(children.len()); + for child in &children { + let mut child_relative_components = frame.relative_components.clone(); + child_relative_components.push(child.dentry.name.clone()); + let relative_path = + canonical_restore_relative_components(&child_relative_components)?; + if child.attr.inode.shard_index() != self.shard_index { + return Err(MetadError::RestoreCrossShardUnsupported { + inode: child.attr.inode, + }); + } + if child.attr.file_type != FileType::Directory && child.attr.nlink != 1 { + return Err(MetadError::RestoreHardlinkUnsupported { + inode: child.attr.inode, + }); + } + let (body, chunks) = match &child.body { + Some(source_body) => { + let mut body = source_body.clone(); + body.base_generation = 0; + let chunks = self.chunk_manifests_for_body_at_version( + child.attr.inode, + source_body, + read_version, + ReadPurpose::Snapshot, + )?; + (Some(body), chunks) + } + None => (None, Vec::new()), + }; + if !removed_paths.contains(&relative_path) + && !initialization_files.contains_key(relative_path.as_str()) + { + if let Some(source_body) = &child.body { + // Source manifests are borrowed in the detached clone even + // when their canonical object key embeds the source inode. + // Initialization replacements are newly owned destination + // objects and are intentionally excluded above. + self.preflight_restore_base_references_for_entry( + &bounds_operation, + child.attr.inode, + source_body.generation, + None, + &chunks, + &mut base_reference_preflight, + )?; + } + } + self.preflight_restore_xattrs(child.attr.inode, read_version)?; + if child.attr.file_type == FileType::Directory { + self.preflight_restore_custom_index_catalog( + &bounds_operation, + child.attr.inode, + &child_relative_components, + &initialization_paths, + )?; + queue.push(RestoreCloneFrame { + source: child.attr.inode, + destination: child.attr.inode, + relative_components: child_relative_components, + after: None, + }); + } + if !removed_paths.contains(&relative_path) { + let (source_inode, destination_body) = + if let Some(file) = initialization_files.get(relative_path.as_str()) { + if child.attr.file_type != FileType::File { + return Err(MetadError::NotFile); + } + matched_initialization_files.insert(relative_path.clone()); + let (_, _, body, _) = + preview_restore_initialization_artifact(self.mount, file)?; + (None, Some(body)) + } else { + (Some(child.attr.inode), body.clone()) + }; + let mut destination_attr = child.attr.clone(); + destination_attr.inode = child.attr.inode; + destination_attr.nlink = destination_attr.file_type.initial_link_count(); + destination_attr.generation = destination_body + .as_ref() + .map_or(bounds_version.get(), |body| body.generation); + destination_attr.ctime_ms = current_time_ms(); + let destination_projection = projection( + frame.destination, + child.dentry.name.clone(), + destination_attr, + destination_body, + ); + self.push_restore_canonical_index_preflight_member( + &bounds_operation, + &RestoreStagingMember { + operation_digest: bounds_operation.operation_digest, + source_inode, + destination_inode: child.attr.inode, + destination_parent: Some(frame.destination), + name: Some(child.dentry.name.clone()), + relative_path: relative_path.clone(), + canonical_index_cursor: Vec::new(), + canonical_index_complete: true, + manifest_cursor: Vec::new(), + manifest_block_cursor: 0, + }, + &destination_projection, + &mut canonical_index_batch, + )?; + } + bounds_batch.push(RestoreCloneEntry { + source: child.clone(), + destination: child.attr.inode, + relative_path, + body, + chunks, + }); + } + let command = self.build_restore_children_command( + &bounds_operation, + frame.source, + &bounds_batch, + bounds_object_reference, + bounds_version, + )?; + validate_restore_command_bounds(&command, "restore materialization batch")?; + } + } + for (index, file) in initialization.files.iter().enumerate() { + if matched_initialization_files.contains(&file.relative_path) { + continue; + } + let components = restore_relative_components(&file.relative_path)?; + let (name, parent_components) = components.split_last().ok_or_else(|| { + MetadError::InvalidPath("restore initialization cannot write root".to_owned()) + })?; + let parent = self.resolve_components_as_directory_from_at_version_for_purpose( + source_root, + parent_components, + read_version, + ReadPurpose::Snapshot, + )?; + let (_, generation, body, _) = + preview_restore_initialization_artifact(self.mount, file)?; + let synthetic_inode = + InodeId::new(u64::MAX.checked_sub(index as u64).ok_or_else(|| { + MetadError::Codec("restore preview inode underflow".to_owned()) + })?)?; + let now_ms = current_time_ms(); + let destination_projection = projection( + parent, + name.clone(), + InodeAttr { + inode: synthetic_inode, + file_type: FileType::File, + mode: file.mode, + uid: file.uid, + gid: file.gid, + rdev: 0, + nlink: FileType::File.initial_link_count(), + size: body.size, + generation: generation.get(), + mtime_ms: now_ms, + ctime_ms: now_ms, + }, + Some(body), + ); + self.push_restore_canonical_index_preflight_member( + &bounds_operation, + &RestoreStagingMember { + operation_digest: bounds_operation.operation_digest, + source_inode: None, + destination_inode: synthetic_inode, + destination_parent: Some(parent), + name: Some(name.clone()), + relative_path: file.relative_path.clone(), + canonical_index_cursor: Vec::new(), + canonical_index_complete: true, + manifest_cursor: Vec::new(), + manifest_block_cursor: 0, + }, + &destination_projection, + &mut canonical_index_batch, + )?; + } + self.preflight_restore_initialization(source_root, read_version, initialization)?; + self.finish_restore_canonical_index_preflight( + &bounds_operation, + &mut canonical_index_batch, + )?; + self.finish_restore_base_reference_preflight( + &bounds_operation, + &mut base_reference_preflight, + ) + } + + fn preflight_restore_xattrs( + &self, + inode: InodeId, + read_version: Version, + ) -> Result<(), MetadError> { + self.restore_xattrs_for_copy(inode, read_version, ReadPurpose::Snapshot)?; + Ok(()) + } + + fn restore_xattrs_for_copy( + &self, + inode: InodeId, + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadError> { + const PAGE_ROWS: usize = 64; + let prefix = xattr_prefix(self.mount, inode); + let mut rows = Vec::new(); + let mut start_after = None; + loop { + let page = self.metadata.scan(ScanRequest { + family: RecordFamily::Xattr, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: PAGE_ROWS, + purpose, + })?; + if page.is_empty() { + break; + } + let page_len = page.len(); + start_after = page.last().map(|row| row.key.clone()); + rows.extend(page); + self.validate_restore_xattr_rows(inode, &rows)?; + if page_len < PAGE_ROWS { + break; + } + } + Ok(rows) + } + + fn validate_restore_xattr_rows( + &self, + inode: InodeId, + rows: &[ScanItem], + ) -> Result<(), MetadError> { + // Runtime copy has four fixed predicates (GC claim, operation, + // temporary binding, destination inode), plus one destination + // NotExists predicate and one Put per xattr. + let items = 4_usize.saturating_add(rows.len().saturating_mul(2)); + if items > 4096 { + return Err(MetadError::RestoreResourceLimit { + resource: "restore inode xattr command items".to_owned(), + limit: 4096, + actual: items as u64, + }); + } + let operation_key_len = restore_operation_key(self.mount, &[0; 32]).len(); + let object_gc_claim_key_len = object_gc_claim_key(self.mount).len(); + let binding_key_len = fork_binding_key(self.mount, inode).len(); + let inode_key_len = inode_key(self.mount, inode).len(); + let primary_key_len = xattr_prefix(self.mount, inode).len(); + let bytes = b"restore-copy-xattrs" + .len() + .saturating_add(24) + .saturating_add(primary_key_len) + .saturating_add(object_gc_claim_key_len) + .saturating_add(32) + .saturating_add(operation_key_len) + .saturating_add(32) + .saturating_add(binding_key_len) + .saturating_add(32) + .saturating_add(inode_key_len) + .saturating_add(32) + .saturating_add(rows.iter().fold(0_usize, |total, row| { + // Source and destination xattr keys have identical widths. + total + .saturating_add(row.key.len()) + .saturating_add(32) + .saturating_add(row.key.len()) + .saturating_add(row.value.0.len()) + .saturating_add(16) + })); + if bytes > MAX_RESTORE_INITIALIZATION_BYTES { + return Err(MetadError::RestoreResourceLimit { + resource: "restore inode xattr command bytes".to_owned(), + limit: MAX_RESTORE_INITIALIZATION_BYTES as u64, + actual: bytes as u64, + }); + } + Ok(()) + } + + fn preflight_restore_initialization( + &self, + source_root: InodeId, + read_version: Version, + initialization: &RestoreInitialization, + ) -> Result<(), MetadError> { + let bounds_version = Version::new(2)?; + let bounds_operation = RestoreOperation { + operation_digest: [0; 32], + initialization_digest: [0; 32], + state: RestoreOperationState::Preparing, + source_root, + destination_root: source_root, + snapshot_id: 1, + read_version: read_version.get(), + created_version: bounds_version.get(), + ref_set_id: bounds_version.get(), + source_path: "/".to_owned(), + destination_path: "/".to_owned(), + }; + let bounds_object_reference = ObjectReferenceMutation::from_version(Version::new(1)?); + for path in &initialization.remove_relative_paths { + let components = restore_relative_components(path)?; + let (name, parents) = components.split_last().ok_or_else(|| { + MetadError::InvalidPath("restore initialization cannot remove root".to_owned()) + })?; + let parent = match self.resolve_components_as_directory_from_at_version_for_purpose( + source_root, + parents, + read_version, + ReadPurpose::Snapshot, + ) { + Ok(parent) => parent, + Err(MetadError::NotFound) => continue, + Err(err) => return Err(err), + }; + if let Some((entry, dentry_version)) = self.lookup_plus_at_version_for_purpose( + parent, + name, + read_version, + ReadPurpose::Snapshot, + )? { + if entry.attr.file_type == FileType::Directory { + return Err(MetadError::InvalidPath(format!( + "restore initialization cannot remove directory {path}" + ))); + } + let chunks = if let Some(body) = &entry.body { + self.chunk_manifests_for_body_at_version( + entry.attr.inode, + body, + read_version, + ReadPurpose::Snapshot, + )? + } else { + Vec::new() + }; + let xattr_keys = self + .restore_xattrs_for_copy(entry.attr.inode, read_version, ReadPurpose::Snapshot)? + .into_iter() + .map(|row| row.key) + .collect::>(); + let staging = RestoreStagingProof { + member: RestoreStagingMember { + operation_digest: bounds_operation.operation_digest, + source_inode: Some(entry.attr.inode), + destination_inode: entry.attr.inode, + destination_parent: Some(parent), + name: Some(name.clone()), + relative_path: path.clone(), + canonical_index_cursor: Vec::new(), + canonical_index_complete: true, + manifest_cursor: Vec::new(), + manifest_block_cursor: 0, + }, + member_version: bounds_version, + inverse_version: bounds_version, + inverse_owner_version: bounds_version, + }; + let command = self.build_restore_initialization_remove_command( + RestoreInitializationRemoveCommand { + operation: &bounds_operation, + parent, + name, + entry: &entry, + dentry_version, + staging: &staging, + chunks: &chunks, + xattr_keys: &xattr_keys, + version: bounds_version, + }, + )?; + validate_restore_command_bounds(&command, "restore initialization remove")?; + } + } + for file in &initialization.files { + let components = restore_relative_components(&file.relative_path)?; + let (name, parents) = components.split_last().ok_or_else(|| { + MetadError::InvalidPath("restore initialization cannot write root".to_owned()) + })?; + let parent = self.resolve_components_as_directory_from_at_version_for_purpose( + source_root, + parents, + read_version, + ReadPurpose::Snapshot, + )?; + let existing = self.lookup_plus_at_version_for_purpose( + parent, + name, + read_version, + ReadPurpose::Snapshot, + )?; + if existing + .as_ref() + .is_some_and(|(entry, _)| entry.attr.file_type != FileType::File) + { + return Err(MetadError::NotFile); + } + let (preview_inode, object_generation, body, chunks) = + preview_restore_initialization_artifact(self.mount, file)?; + let old_chunks = match existing.as_ref().and_then(|(entry, _)| entry.body.as_ref()) { + Some(old_body) => self.chunk_manifests_for_body_at_version( + existing + .as_ref() + .expect("body implies an existing entry") + .0 + .attr + .inode, + old_body, + read_version, + ReadPurpose::Snapshot, + )?, + None => Vec::new(), + }; + let staging = existing.as_ref().map(|(entry, _)| RestoreStagingProof { + member: RestoreStagingMember { + operation_digest: bounds_operation.operation_digest, + source_inode: Some(entry.attr.inode), + destination_inode: preview_inode, + destination_parent: Some(parent), + name: Some(name.clone()), + relative_path: file.relative_path.clone(), + canonical_index_cursor: Vec::new(), + canonical_index_complete: true, + manifest_cursor: Vec::new(), + manifest_block_cursor: 0, + }, + member_version: bounds_version, + inverse_version: bounds_version, + inverse_owner_version: bounds_version, + }); + let command = self.build_restore_initialization_publish_command( + RestoreInitializationPublishCommand { + operation: &bounds_operation, + parent, + name, + file, + existing: existing.as_ref(), + inode: preview_inode, + object_generation, + intent_version: bounds_version, + object_reference: bounds_object_reference, + body: &body, + chunks: &chunks, + old_chunks: &old_chunks, + staging: staging.as_ref(), + version: bounds_version, + }, + )?; + validate_restore_command_bounds(&command, "restore initialization publish")?; + } + Ok(()) + } + + fn materialize_restore_tree(&self, operation: &RestoreOperation) -> Result<(), MetadError> { + let source_version = Version::new(operation.read_version)?; + let mut batch_index = 0_u64; + self.copy_restore_xattrs( + operation, + operation.source_root, + operation.destination_root, + source_version, + )?; + let mut queue = vec![RestoreCloneFrame { + source: operation.source_root, + destination: operation.destination_root, + relative_components: Vec::new(), + after: None, + }]; + while let Some(frame) = queue.pop() { + let page = self.read_dir_plus_page_at_version_for_purpose( + frame.source, + frame.after.as_ref(), + RESTORE_BATCH_ENTRIES, + source_version, + ReadPurpose::Snapshot, + )?; + if let Some(after) = page.next_cursor { + queue.push(RestoreCloneFrame { + source: frame.source, + destination: frame.destination, + relative_components: frame.relative_components.clone(), + after: Some(after), + }); + } + if !page.entries.is_empty() { + let mut batch = Vec::with_capacity(page.entries.len()); + for source in &page.entries { + let mut relative_components = frame.relative_components.clone(); + relative_components.push(source.dentry.name.clone()); + let relative_path = + canonical_restore_relative_components(&relative_components)?; + let destination = self.next_inode()?; + let (body, chunks) = match &source.body { + Some(source_body) => { + // Effective manifests are materialized under the + // destination's top generation. Clearing the base + // pointer is mandatory: copied sparse/append bodies + // do not have the source inode's older summaries. + let mut body = source_body.clone(); + body.base_generation = 0; + let chunks = self.chunk_manifests_for_body_at_version( + source.attr.inode, + source_body, + source_version, + ReadPurpose::Snapshot, + )?; + (Some(body), chunks) + } + None => (None, Vec::new()), + }; + batch.push(RestoreCloneEntry { + source: source.clone(), + destination, + relative_path, + body, + chunks, + }); + } + self.commit_restore_children(operation, frame.destination, &batch, batch_index)?; + batch_index = batch_index.checked_add(1).ok_or_else(|| { + MetadError::Codec("restore materialization batch index overflow".to_owned()) + })?; + for entry in batch { + self.copy_restore_xattrs( + operation, + entry.source.attr.inode, + entry.destination, + source_version, + )?; + if entry.source.attr.file_type == FileType::Directory { + queue.push(RestoreCloneFrame { + source: entry.source.attr.inode, + destination: entry.destination, + relative_components: restore_relative_components(&entry.relative_path)?, + after: None, + }); + } + } + } + } + Ok(()) + } + + fn commit_restore_children( + &self, + operation: &RestoreOperation, + destination_parent: InodeId, + entries: &[RestoreCloneEntry], + batch_index: u64, + ) -> Result<(), MetadError> { + if entries.is_empty() { + return Ok(()); + } + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let command = self.build_restore_children_command( + operation, + destination_parent, + entries, + object_reference, + version, + )?; + validate_restore_command_bounds(&command, "restore materialization batch")?; + self.commit_metadata(command)?; + live_test_barrier::restore_applied( + &restore_barrier_operation_id(operation), + live_test_barrier::RestoreAppliedPhase::MaterializeBatch(batch_index), + )?; + Ok(()) + } + + fn build_restore_children_command( + &self, + operation: &RestoreOperation, + destination_parent: InodeId, + entries: &[RestoreCloneEntry], + object_reference: ObjectReferenceMutation, + version: Version, + ) -> Result { + let read_version = predecessor(version)?; + let operation_key = restore_operation_key(self.mount, &operation.operation_digest); + let binding_key = fork_binding_key(self.mount, operation.destination_root); + let mut predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: operation_key, + predicate: Predicate::VersionEquals(Version::new(operation.created_version)?), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key, + predicate: Predicate::VersionEquals(Version::new(operation.created_version)?), + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, destination_parent), + predicate: Predicate::Exists, + }, + ]; + let mut mutations = Vec::new(); + for entry in entries { + let mut attr = entry.source.attr.clone(); + attr.inode = entry.destination; + attr.nlink = attr.file_type.initial_link_count(); + attr.generation = entry + .body + .as_ref() + .map_or(version.get(), |body| body.generation); + attr.ctime_ms = current_time_ms(); + let projection = projection( + destination_parent, + entry.source.dentry.name.clone(), + attr, + entry.body.clone(), + ); + let dentry = dentry_key(self.mount, destination_parent, &entry.source.dentry.name); + let inode = inode_key(self.mount, entry.destination); + let member = + restore_staging_member_key(self.mount, operation.ref_set_id, entry.destination); + let staging_inode = restore_staging_inode_key(self.mount, entry.destination); + let staging_inverse_owner = restore_staging_inverse_owner_key( + self.mount, + operation.ref_set_id, + entry.destination, + ); + predicates.extend([ + PredicateRef { + family: RecordFamily::Dentry, + key: dentry.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: member.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: staging_inode.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: staging_inverse_owner.clone(), + predicate: Predicate::NotExists, + }, + ]); + mutations.extend([ + Mutation { + family: RecordFamily::Inode, + key: inode, + op: MutationOp::Put, + value: Some(Value(encode_inode_attr(&projection.attr))), + }, + put_projection_mutation(RecordFamily::Dentry, dentry, &projection), + Mutation { + family: RecordFamily::System, + key: member, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_member( + &RestoreStagingMember { + operation_digest: operation.operation_digest, + source_inode: Some(entry.source.attr.inode), + destination_inode: entry.destination, + destination_parent: Some(destination_parent), + name: Some(entry.source.dentry.name.clone()), + relative_path: entry.relative_path.clone(), + canonical_index_cursor: Vec::new(), + canonical_index_complete: true, + manifest_cursor: Vec::new(), + manifest_block_cursor: 0, + }, + )?)), + }, + Mutation { + family: RecordFamily::System, + key: staging_inode, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_inverse(operation))), + }, + Mutation { + family: RecordFamily::System, + key: staging_inverse_owner, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_inverse(operation))), + }, + ]); + if let Some(body) = &projection.body { + mutations.push(Mutation { + family: RecordFamily::ChunkManifest, + key: chunk_manifest_key( + self.mount, + entry.destination, + body.generation, + BODY_SUMMARY_CHUNK_INDEX, + ), + op: MutationOp::Put, + value: Some(Value(encode_body_descriptor(body))), + }); + mutations.extend(entry.chunks.iter().map(|chunk| Mutation { + family: RecordFamily::ChunkManifest, + key: chunk_manifest_key( + self.mount, + entry.destination, + body.generation, + chunk.chunk_index, + ), + op: MutationOp::Put, + value: Some(Value(encode_chunk_manifest(chunk))), + })); + } + } + Ok(MetadataCommand { + request_id: request_id( + b"restore-materialize-batch", + self.mount, + destination_parent, + version, + ), + kind: CommandKind::CreateFiles, + read_version, + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: dentry_prefix(self.mount, destination_parent), + predicates, + mutations, + // Detached materialization never emits user-visible watch events. + watch: Vec::new(), + }) + } + + fn copy_restore_xattrs( + &self, + operation: &RestoreOperation, + source: InodeId, + destination: InodeId, + source_version: Version, + ) -> Result<(), MetadError> { + let source_prefix = xattr_prefix(self.mount, source); + let rows = self.restore_xattrs_for_copy(source, source_version, ReadPurpose::Snapshot)?; + if rows.is_empty() { + return Ok(()); + } + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let operation_key = restore_operation_key(self.mount, &operation.operation_digest); + let mut predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: operation_key, + predicate: Predicate::VersionEquals(Version::new(operation.created_version)?), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: fork_binding_key(self.mount, operation.destination_root), + predicate: Predicate::VersionEquals(Version::new(operation.created_version)?), + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, destination), + predicate: Predicate::Exists, + }, + ]; + let mut mutations = Vec::with_capacity(rows.len()); + for row in rows { + let name = row + .key + .strip_prefix(source_prefix.as_slice()) + .ok_or_else(|| { + MetadError::Codec("restore xattr scan escaped source prefix".to_owned()) + })?; + let key = xattr_key(self.mount, destination, name); + predicates.push(PredicateRef { + family: RecordFamily::Xattr, + key: key.clone(), + predicate: Predicate::NotExists, + }); + mutations.push(Mutation { + family: RecordFamily::Xattr, + key, + op: MutationOp::Put, + value: Some(row.value), + }); + } + let command = MetadataCommand { + request_id: request_id(b"restore-copy-xattrs", self.mount, destination, version), + kind: CommandKind::SetXattr, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::Xattr, + primary_key: xattr_prefix(self.mount, destination), + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore xattr copy")?; + self.commit_metadata(command)?; + Ok(()) + } + + fn apply_restore_initialization( + &self, + operation: &RestoreOperation, + initialization: &RestoreInitialization, + ) -> Result<(), MetadError> { + for relative_path in &initialization.remove_relative_paths { + let components = restore_relative_components(relative_path)?; + let (name, parents) = components.split_last().ok_or_else(|| { + MetadError::InvalidPath("restore initialization cannot remove root".to_owned()) + })?; + let version = self.read_version()?; + let parent = match self.resolve_components_as_directory_from_at_version_for_purpose( + operation.destination_root, + parents, + version, + ReadPurpose::RestoreStaging, + ) { + Ok(parent) => parent, + Err(MetadError::NotFound) => continue, + Err(err) => return Err(err), + }; + match self.lookup_plus_at_version_for_purpose( + parent, + name, + version, + ReadPurpose::RestoreStaging, + )? { + None => continue, + Some((entry, _)) if entry.attr.file_type == FileType::Directory => { + return Err(MetadError::InvalidPath(format!( + "restore initialization cannot remove directory {relative_path}" + ))) + } + Some((entry, dentry_version)) => { + self.remove_restore_initialization_entry( + operation, + parent, + name, + &entry, + dentry_version, + )?; + } + } + } + + for (file_index, file) in initialization.files.iter().enumerate() { + let components = restore_relative_components(&file.relative_path)?; + let (name, parents) = components.split_last().ok_or_else(|| { + MetadError::InvalidPath("restore initialization cannot write root".to_owned()) + })?; + let version = self.read_version()?; + let parent = self.resolve_components_as_directory_from_at_version_for_purpose( + operation.destination_root, + parents, + version, + ReadPurpose::RestoreStaging, + )?; + let existing = self.lookup_plus_at_version_for_purpose( + parent, + name, + version, + ReadPurpose::RestoreStaging, + )?; + if existing + .as_ref() + .is_some_and(|(entry, _)| entry.attr.file_type != FileType::File) + { + return Err(MetadError::NotFile); + } + self.publish_restore_initialization_file( + operation, + parent, + name, + file, + existing, + file_index as u64, + )?; + } + Ok(()) + } + + fn restore_staging_proof( + &self, + operation: &RestoreOperation, + inode: InodeId, + version: Version, + ) -> Result { + let member_key = restore_staging_member_key(self.mount, operation.ref_set_id, inode); + let member = self + .metadata + .get_versioned( + RecordFamily::System, + &member_key, + version, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreRootChanged { root: inode })?; + let decoded_member = decode_restore_staging_member(&member.value.0)?; + if decoded_member.operation_digest != operation.operation_digest + || decoded_member.destination_inode != inode + { + return Err(MetadError::RestoreRootChanged { root: inode }); + } + let inverse_key = restore_staging_inode_key(self.mount, inode); + let inverse = self + .metadata + .get_versioned( + RecordFamily::System, + &inverse_key, + version, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreRootChanged { root: inode })?; + let (digest, ref_set_id) = decode_restore_staging_inverse(&inverse.value.0)?; + if digest != operation.operation_digest || ref_set_id != operation.ref_set_id { + return Err(MetadError::RestoreRootChanged { root: inode }); + } + let inverse_owner_key = + restore_staging_inverse_owner_key(self.mount, operation.ref_set_id, inode); + let inverse_owner = self + .metadata + .get_versioned( + RecordFamily::System, + &inverse_owner_key, + version, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreRootChanged { root: inode })?; + if inverse_owner.value != inverse.value { + return Err(MetadError::RestoreRootChanged { root: inode }); + } + Ok(RestoreStagingProof { + member: decoded_member, + member_version: member.version, + inverse_version: inverse.version, + inverse_owner_version: inverse_owner.version, + }) + } + + fn remove_restore_initialization_entry( + &self, + operation: &RestoreOperation, + parent: InodeId, + name: &DentryName, + entry: &DentryWithAttr, + dentry_version: Version, + ) -> Result<(), MetadError> { + let read_version = self.read_version()?; + let staging = self.restore_staging_proof(operation, entry.attr.inode, read_version)?; + let chunks = if let Some(body) = &entry.body { + self.chunk_manifests_for_body_at_version( + entry.attr.inode, + body, + read_version, + ReadPurpose::RestoreStaging, + )? + } else { + Vec::new() + }; + let xattr_keys = self + .restore_xattrs_for_copy(entry.attr.inode, read_version, ReadPurpose::RestoreStaging)? + .into_iter() + .map(|row| row.key) + .collect::>(); + let version = self.next_version()?; + let command = + self.build_restore_initialization_remove_command(RestoreInitializationRemoveCommand { + operation, + parent, + name, + entry, + dentry_version, + staging: &staging, + chunks: &chunks, + xattr_keys: &xattr_keys, + version, + })?; + validate_restore_command_bounds(&command, "restore initialization remove")?; + self.commit_metadata(command)?; + Ok(()) + } + + fn build_restore_initialization_remove_command( + &self, + plan: RestoreInitializationRemoveCommand<'_>, + ) -> Result { + let RestoreInitializationRemoveCommand { + operation, + parent, + name, + entry, + dentry_version, + staging, + chunks, + xattr_keys, + version, + } = plan; + let mut mutations = vec![ + delete_mutation(RecordFamily::Dentry, dentry_key(self.mount, parent, name)), + delete_mutation(RecordFamily::Inode, inode_key(self.mount, entry.attr.inode)), + delete_mutation( + RecordFamily::System, + restore_staging_member_key(self.mount, operation.ref_set_id, entry.attr.inode), + ), + delete_mutation( + RecordFamily::System, + restore_staging_inode_key(self.mount, entry.attr.inode), + ), + delete_mutation( + RecordFamily::System, + restore_staging_inverse_owner_key( + self.mount, + operation.ref_set_id, + entry.attr.inode, + ), + ), + ]; + if let Some(body) = &entry.body { + mutations.push(delete_mutation( + RecordFamily::ChunkManifest, + chunk_manifest_key( + self.mount, + entry.attr.inode, + body.generation, + BODY_SUMMARY_CHUNK_INDEX, + ), + )); + mutations.extend(chunks.iter().map(|chunk| { + delete_mutation( + RecordFamily::ChunkManifest, + chunk_manifest_key( + self.mount, + entry.attr.inode, + body.generation, + chunk.chunk_index, + ), + ) + })); + } + mutations.extend( + xattr_keys + .iter() + .cloned() + .map(|key| delete_mutation(RecordFamily::Xattr, key)), + ); + Ok(MetadataCommand { + request_id: request_id( + b"restore-init-remove", + self.mount, + entry.attr.inode, + version, + ), + kind: CommandKind::RemoveFile, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: dentry_key(self.mount, parent, name), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(Version::new(operation.created_version)?), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: fork_binding_key(self.mount, operation.destination_root), + predicate: Predicate::VersionEquals(Version::new(operation.created_version)?), + }, + PredicateRef { + family: RecordFamily::Dentry, + key: dentry_key(self.mount, parent, name), + predicate: Predicate::VersionEquals(dentry_version), + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, entry.attr.inode), + predicate: Predicate::Exists, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_member_key( + self.mount, + operation.ref_set_id, + entry.attr.inode, + ), + predicate: Predicate::VersionEquals(staging.member_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_inode_key(self.mount, entry.attr.inode), + predicate: Predicate::VersionEquals(staging.inverse_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_inverse_owner_key( + self.mount, + operation.ref_set_id, + entry.attr.inode, + ), + predicate: Predicate::VersionEquals(staging.inverse_owner_version), + }, + ], + mutations, + watch: Vec::new(), + }) + } + + fn publish_restore_initialization_file( + &self, + operation: &RestoreOperation, + parent: InodeId, + name: &DentryName, + file: &RestoreInitializationFile, + existing: Option<(DentryWithAttr, Version)>, + file_index: u64, + ) -> Result<(), MetadError> { + let object_reference = self.begin_object_reference_mutation()?; + let inode = existing + .as_ref() + .map_or_else(|| self.next_inode(), |(entry, _)| Ok(entry.attr.inode))?; + let object_generation = self.next_version()?; + let (_intent, intent_version) = self.persist_restore_init_upload_intent( + operation, + inode, + object_generation, + file, + object_reference, + )?; + let request = PublishArtifact { + parent, + name: name.clone(), + producer: "nokv-restore-initialization".to_owned(), + digest_uri: body_digest_uri(&file.bytes), + content_type: file.content_type.clone(), + manifest_id: format!("restore-init/{}", file.relative_path), + bytes: file.bytes.clone(), + mode: file.mode, + uid: file.uid, + gid: file.gid, + }; + let operation_id = restore_barrier_operation_id(operation); + live_test_barrier::restore_initialization_put( + &operation_id, + file_index, + live_test_barrier::RestoreInitializationPutBoundary::Before, + )?; + let StagedArtifactBody { + body, + chunks, + old_chunks: _, + staged, + } = self.stage_artifact_body(&request, inode, object_generation)?; + if let Err(error) = live_test_barrier::restore_initialization_put( + &operation_id, + file_index, + live_test_barrier::RestoreInitializationPutBoundary::After, + ) { + return Err(MetadError::PublishArtifactFailed { + source: Box::new(error), + staged, + }); + } + let result = self.commit_restore_initialization_file( + operation, + parent, + name, + file, + existing, + inode, + object_generation, + intent_version, + object_reference, + body, + chunks, + ); + if let Err(error) = result { + // Preserve both bytes and the durable intent for every error. Even + // a proven metadata predicate failure does not prove object-store + // writer quiescence or immediate PUT visibility. Explicit discard + // first publishes a permanent tombstone ledger, then may delete + // these operation-local rows; the GC sweeper keeps re-deleting old + // keys after arbitrarily late PUT completion. + return Err(MetadError::PublishArtifactFailed { + source: Box::new(error), + staged, + }); + } + Ok(()) + } + + fn persist_restore_init_upload_intent( + &self, + operation: &RestoreOperation, + inode: InodeId, + generation: Version, + file: &RestoreInitializationFile, + object_reference: ObjectReferenceMutation, + ) -> Result<(RestoreInitUploadIntent, Version), MetadError> { + let intent = RestoreInitUploadIntent { + operation_digest: operation.operation_digest, + ref_set_id: operation.ref_set_id, + inode, + generation: generation.get(), + size: file.bytes.len() as u64, + relative_path: file.relative_path.clone(), + cleanup_pass: 0, + }; + let key = restore_init_upload_intent_key( + self.mount, + operation.ref_set_id, + inode, + generation.get(), + ); + let version = self.next_version()?; + self.commit_metadata(MetadataCommand { + request_id: request_id(b"restore-init-upload-intent", self.mount, inode, version), + kind: CommandKind::PublishArtifact, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(Version::new(operation.created_version)?), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: fork_binding_key(self.mount, operation.destination_root), + predicate: Predicate::VersionEquals(Version::new(operation.created_version)?), + }, + PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key, + op: MutationOp::Put, + value: Some(Value(encode_restore_init_upload_intent(&intent)?)), + }], + watch: Vec::new(), + })?; + Ok((intent, version)) + } + + #[allow(clippy::too_many_arguments)] + fn commit_restore_initialization_file( + &self, + operation: &RestoreOperation, + parent: InodeId, + name: &DentryName, + file: &RestoreInitializationFile, + existing: Option<(DentryWithAttr, Version)>, + inode: InodeId, + object_generation: Version, + intent_version: Version, + object_reference: ObjectReferenceMutation, + body: BodyDescriptor, + chunks: Vec, + ) -> Result<(), MetadError> { + let version = self.next_version()?; + let (staging, old_chunks) = if let Some((existing, _)) = existing.as_ref() { + let staging = + self.restore_staging_proof(operation, existing.attr.inode, predecessor(version)?)?; + let old_chunks = if let Some(old_body) = &existing.body { + self.chunk_manifests_for_body_at_version( + existing.attr.inode, + old_body, + predecessor(version)?, + ReadPurpose::RestoreStaging, + )? + } else { + Vec::new() + }; + (Some(staging), old_chunks) + } else { + (None, Vec::new()) + }; + let command = self.build_restore_initialization_publish_command( + RestoreInitializationPublishCommand { + operation, + parent, + name, + file, + existing: existing.as_ref(), + inode, + object_generation, + intent_version, + object_reference, + body: &body, + chunks: &chunks, + old_chunks: &old_chunks, + staging: staging.as_ref(), + version, + }, + )?; + validate_restore_command_bounds(&command, "restore initialization publish")?; + self.commit_metadata(command)?; + Ok(()) + } + + fn build_restore_initialization_publish_command( + &self, + plan: RestoreInitializationPublishCommand<'_>, + ) -> Result { + let RestoreInitializationPublishCommand { + operation, + parent, + name, + file, + existing, + inode, + object_generation, + intent_version, + object_reference, + body, + chunks, + old_chunks, + staging, + version, + } = plan; + let now_ms = current_time_ms(); + let attr = InodeAttr { + inode, + file_type: FileType::File, + mode: file.mode, + uid: file.uid, + gid: file.gid, + rdev: 0, + nlink: FileType::File.initial_link_count(), + size: body.size, + generation: object_generation.get(), + mtime_ms: now_ms, + ctime_ms: now_ms, + }; + let projection = projection(parent, name.clone(), attr, Some(body.clone())); + let dentry = dentry_key(self.mount, parent, name); + let mut predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(Version::new(operation.created_version)?), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: fork_binding_key(self.mount, operation.destination_root), + predicate: Predicate::VersionEquals(Version::new(operation.created_version)?), + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, parent), + predicate: Predicate::Exists, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_init_upload_intent_key( + self.mount, + operation.ref_set_id, + inode, + object_generation.get(), + ), + predicate: Predicate::VersionEquals(intent_version), + }, + ]; + let mut mutations = vec![ + Mutation { + family: RecordFamily::Inode, + key: inode_key(self.mount, inode), + op: MutationOp::Put, + value: Some(Value(encode_inode_attr(&projection.attr))), + }, + put_projection_mutation(RecordFamily::Dentry, dentry.clone(), &projection), + Mutation { + family: RecordFamily::ChunkManifest, + key: chunk_manifest_key( + self.mount, + inode, + body.generation, + BODY_SUMMARY_CHUNK_INDEX, + ), + op: MutationOp::Put, + value: Some(Value(encode_body_descriptor(body))), + }, + delete_mutation( + RecordFamily::System, + restore_init_upload_intent_key( + self.mount, + operation.ref_set_id, + inode, + object_generation.get(), + ), + ), + ]; + mutations.extend(chunks.iter().map(|chunk| Mutation { + family: RecordFamily::ChunkManifest, + key: chunk_manifest_key(self.mount, inode, body.generation, chunk.chunk_index), + op: MutationOp::Put, + value: Some(Value(encode_chunk_manifest(chunk))), + })); + let kind = if let Some((existing, dentry_version)) = existing { + let staging = staging.ok_or_else(|| { + MetadError::Codec( + "restore initialization replace command is missing staging proof".to_owned(), + ) + })?; + if staging.member.relative_path != file.relative_path { + return Err(MetadError::RestoreRootChanged { root: inode }); + } + let member_key = restore_staging_member_key(self.mount, operation.ref_set_id, inode); + predicates.extend([ + PredicateRef { + family: RecordFamily::Dentry, + key: dentry.clone(), + predicate: Predicate::VersionEquals(*dentry_version), + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, inode), + predicate: Predicate::Exists, + }, + PredicateRef { + family: RecordFamily::System, + key: member_key.clone(), + predicate: Predicate::VersionEquals(staging.member_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_inode_key(self.mount, inode), + predicate: Predicate::VersionEquals(staging.inverse_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_inverse_owner_key(self.mount, operation.ref_set_id, inode), + predicate: Predicate::VersionEquals(staging.inverse_owner_version), + }, + ]); + mutations.push(Mutation { + family: RecordFamily::System, + key: member_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_member( + &RestoreStagingMember { + operation_digest: operation.operation_digest, + source_inode: None, + destination_inode: inode, + destination_parent: Some(parent), + name: Some(name.clone()), + relative_path: file.relative_path.clone(), + canonical_index_cursor: Vec::new(), + canonical_index_complete: true, + manifest_cursor: Vec::new(), + manifest_block_cursor: 0, + }, + )?)), + }); + if let Some(old_body) = &existing.body { + mutations.push(delete_mutation( + RecordFamily::ChunkManifest, + chunk_manifest_key( + self.mount, + inode, + old_body.generation, + BODY_SUMMARY_CHUNK_INDEX, + ), + )); + mutations.extend(old_chunks.iter().map(|chunk| { + delete_mutation( + RecordFamily::ChunkManifest, + chunk_manifest_key( + self.mount, + inode, + old_body.generation, + chunk.chunk_index, + ), + ) + })); + } + CommandKind::ReplaceArtifact + } else { + let member = restore_staging_member_key(self.mount, operation.ref_set_id, inode); + let inverse = restore_staging_inode_key(self.mount, inode); + let inverse_owner = + restore_staging_inverse_owner_key(self.mount, operation.ref_set_id, inode); + predicates.extend([ + PredicateRef { + family: RecordFamily::Dentry, + key: dentry.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, inode), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: member.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: inverse.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_owner.clone(), + predicate: Predicate::NotExists, + }, + ]); + mutations.extend([ + Mutation { + family: RecordFamily::System, + key: member, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_member( + &RestoreStagingMember { + operation_digest: operation.operation_digest, + source_inode: None, + destination_inode: inode, + destination_parent: Some(parent), + name: Some(name.clone()), + relative_path: file.relative_path.clone(), + canonical_index_cursor: Vec::new(), + canonical_index_complete: true, + manifest_cursor: Vec::new(), + manifest_block_cursor: 0, + }, + )?)), + }, + Mutation { + family: RecordFamily::System, + key: inverse, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_inverse(operation))), + }, + Mutation { + family: RecordFamily::System, + key: inverse_owner, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_inverse(operation))), + }, + ]); + CommandKind::PublishArtifact + }; + Ok(MetadataCommand { + request_id: request_id(b"restore-init-publish", self.mount, inode, version), + kind, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: dentry, + predicates, + mutations, + watch: Vec::new(), + }) + } + + fn discard_preparing_restore(&self, operation: &RestoreOperation) -> Result<(), MetadError> { + let operation_version = self.begin_restore_discard(operation)?; + live_test_barrier::restore_applied( + &restore_barrier_operation_id(operation), + live_test_barrier::RestoreAppliedPhase::CleanupBatch(0), + )?; + let mut discarding = operation.clone(); + discarding.state = RestoreOperationState::Discarding; + if !self.cleanup_restore_init_upload_intents(&discarding, operation_version)? { + return Err(MetadError::RestoreInProgress); + } + if !self.cleanup_restore_staging_members(&discarding, operation_version)? { + return Err(MetadError::RestoreInProgress); + } + if !self.cleanup_restore_base_references(&discarding, operation_version)? { + return Err(MetadError::RestoreInProgress); + } + if !self.cleanup_restore_index_page(&discarding, operation_version)? { + return Err(MetadError::RestoreInProgress); + } + self.finish_restore_discard(&discarding, operation_version)?; + self.recover_restore_staging_visibility()?; + Ok(()) + } + + fn begin_restore_discard(&self, operation: &RestoreOperation) -> Result { + let key = restore_operation_key(self.mount, &operation.operation_digest); + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreRootChanged { + root: operation.destination_root, + })?; + let current = decode_restore_operation(&item.value.0)?; + if current.operation_digest != operation.operation_digest + || current.initialization_digest != operation.initialization_digest + { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + if matches!( + current.state, + RestoreOperationState::Cleaning | RestoreOperationState::Discarding + ) { + let cleanup = self + .metadata + .get( + RecordFamily::System, + &restore_cleanup_job_key(self.mount, operation.ref_set_id), + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("discarding restore has no cleanup job".to_owned()) + })?; + let cleanup = decode_restore_cleanup_job(&cleanup.0)?; + if cleanup.operation_digest != operation.operation_digest + || cleanup.ref_set_id != operation.ref_set_id + { + return Err(MetadError::Codec( + "restore cleanup job changed identity".to_owned(), + )); + } + return Ok(item.version); + } + if current.state != RestoreOperationState::Preparing { + return Err(MetadError::RestoreInProgress); + } + let binding_key = fork_binding_key(self.mount, operation.destination_root); + let binding = self + .metadata + .get_versioned( + RecordFamily::ForkBinding, + &binding_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreBindingChanged { + root: operation.destination_root, + })?; + let decoded = crate::layout::decode_fork_binding(&binding.value.0).map_err(|_| { + MetadError::RestoreBindingChanged { + root: operation.destination_root, + } + })?; + if decoded.fork_root != operation.destination_root + || decoded.source_root != operation.source_root + || decoded.snapshot_id != operation.snapshot_id + || decoded.pinned_read_version != operation.read_version + { + return Err(MetadError::RestoreBindingChanged { + root: operation.destination_root, + }); + } + let mut discarding = current; + discarding.state = RestoreOperationState::Discarding; + let cleanup_key = restore_cleanup_job_key(self.mount, operation.ref_set_id); + let cleanup = RestoreCleanupJob { + operation_digest: operation.operation_digest, + ref_set_id: operation.ref_set_id, + index_complete: false, + index_cursor: Vec::new(), + }; + let version = self.next_version()?; + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"restore-begin-discard", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(item.version), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key, + predicate: Predicate::VersionEquals(binding.version), + }, + PredicateRef { + family: RecordFamily::System, + key: cleanup_key.clone(), + predicate: Predicate::NotExists, + }, + ], + mutations: vec![ + Mutation { + family: RecordFamily::System, + key, + op: MutationOp::Put, + value: Some(Value(encode_restore_operation(&discarding)?)), + }, + Mutation { + family: RecordFamily::System, + key: cleanup_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_cleanup_job(&cleanup)?)), + }, + ], + watch: Vec::new(), + })?; + Ok(version) + } + + fn cleanup_restore_init_upload_intents( + &self, + operation: &RestoreOperation, + operation_version: Version, + ) -> Result { + let prefix = restore_init_upload_intent_prefix(self.mount, operation.ref_set_id); + let read_version = self.read_version()?; + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: None, + version: read_version, + limit: RESTORE_BATCH_ENTRIES, + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + return Ok(true); + } + let mut predicates = vec![PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }]; + let mut mutations = Vec::with_capacity(rows.len() * 2); + for row in rows { + let mut intent = decode_restore_init_upload_intent(&row.value.0)?; + if intent.operation_digest != operation.operation_digest + || intent.ref_set_id != operation.ref_set_id + || restore_init_upload_intent_key( + self.mount, + intent.ref_set_id, + intent.inode, + intent.generation, + ) != row.key + { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + predicates.push(PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }); + let tombstone = RestoreInitUploadTombstone { + operation_digest: intent.operation_digest, + initialization_digest: operation.initialization_digest, + inode: intent.inode, + generation: intent.generation, + size: intent.size, + relative_path: intent.relative_path.clone(), + }; + let tombstone_key = restore_init_upload_tombstone_key( + self.mount, + &tombstone.operation_digest, + tombstone.inode, + tombstone.generation, + ); + let existing_tombstone = self.metadata.get_versioned( + RecordFamily::System, + &tombstone_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + match intent.cleanup_pass { + 0 => { + match existing_tombstone { + Some(existing) => { + if decode_restore_init_upload_tombstone(&existing.value.0)? != tombstone + { + return Err(MetadError::Codec( + "restore init upload tombstone changed identity".to_owned(), + )); + } + predicates.push(PredicateRef { + family: RecordFamily::System, + key: tombstone_key, + predicate: Predicate::VersionEquals(existing.version), + }); + } + None => { + predicates.push(PredicateRef { + family: RecordFamily::System, + key: tombstone_key.clone(), + predicate: Predicate::NotExists, + }); + mutations.push(Mutation { + family: RecordFamily::System, + key: tombstone_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_init_upload_tombstone( + &tombstone, + )?)), + }); + } + } + // The first cleanup pass only publishes the permanent + // tombstone. No object DELETE may happen before that CAS: + // an older owner can still be blocked inside PUT. + intent.cleanup_pass = 1; + mutations.push(Mutation { + family: RecordFamily::System, + key: row.key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_restore_init_upload_intent(&intent)?)), + }); + } + 1 => { + let existing = existing_tombstone.ok_or_else(|| { + MetadError::Codec( + "restore init intent is tombstoned without a global ledger".to_owned(), + ) + })?; + if decode_restore_init_upload_tombstone(&existing.value.0)? != tombstone { + return Err(MetadError::Codec( + "restore init upload tombstone changed identity".to_owned(), + )); + } + predicates.push(PredicateRef { + family: RecordFamily::System, + key: tombstone_key, + predicate: Predicate::VersionEquals(existing.version), + }); + self.delete_restore_init_object_range( + tombstone.inode, + tombstone.generation, + tombstone.size, + )?; + mutations.push(delete_mutation(RecordFamily::System, row.key.clone())); + } + _ => unreachable!("restore init intent codec validates cleanup pass"), + } + } + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-discard-init-intents", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: prefix, + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore discard init intents")?; + self.commit_metadata(command)?; + Ok(false) + } + + pub(super) fn delete_restore_init_object_range( + &self, + inode: InodeId, + generation: u64, + size: u64, + ) -> Result { + let chunk_size = DEFAULT_CHUNK_SIZE; + let block_size = DEFAULT_BLOCK_SIZE as u64; + let mut chunk_offset = 0_u64; + let mut deleted_any = false; + while chunk_offset < size { + let chunk_index = chunk_offset / chunk_size; + let chunk_len = chunk_size.min(size - chunk_offset); + let block_count = chunk_len.div_ceil(block_size); + for block_index in 0..block_count { + let key = ObjectKey::new(format!( + "blocks/{}/{}/{}/{}/{}", + self.mount.get(), + inode.get(), + generation, + chunk_index, + block_index + ))?; + deleted_any |= self.objects.delete(&key)?; + } + chunk_offset = chunk_offset.saturating_add(chunk_len); + } + Ok(deleted_any) + } + + pub(super) fn restore_init_object_range_absent( + &self, + inode: InodeId, + generation: u64, + size: u64, + ) -> Result { + let chunk_size = DEFAULT_CHUNK_SIZE; + let block_size = DEFAULT_BLOCK_SIZE as u64; + let mut chunk_offset = 0_u64; + while chunk_offset < size { + let chunk_index = chunk_offset / chunk_size; + let chunk_len = chunk_size.min(size - chunk_offset); + let block_count = chunk_len.div_ceil(block_size); + for block_index in 0..block_count { + let key = ObjectKey::new(format!( + "blocks/{}/{}/{}/{}/{}", + self.mount.get(), + inode.get(), + generation, + chunk_index, + block_index + ))?; + if self.objects.head(&key)?.is_some() { + return Ok(false); + } + } + chunk_offset = chunk_offset.saturating_add(chunk_len); + } + Ok(true) + } + + /// Re-delete object keys tracked by permanent initialization tombstones. + /// This automatic sweep deliberately never removes ledger rows: only an + /// explicit global-writer drain can prove that no old owner remains blocked + /// inside PUT and safely CAS the tombstone away. + pub(super) fn cleanup_restore_init_upload_tombstones_locked( + &self, + limit: usize, + ) -> Result { + let version = self.read_version()?; + let prefix = restore_init_upload_tombstone_prefix(self.mount); + let cursor_key = restore_init_upload_tombstone_cursor_key(self.mount); + let cursor_item = self.metadata.get_versioned( + RecordFamily::System, + &cursor_key, + version, + ReadPurpose::WritePlanLocal, + )?; + let cursor = cursor_item + .as_ref() + .map_or_else(Vec::new, |item| item.value.0.clone()); + if cursor.len() > MAX_RESTORE_PATH_BYTES + || (!cursor.is_empty() && !cursor.starts_with(&prefix)) + { + return Err(MetadError::Codec( + "restore init tombstone cursor changed identity".to_owned(), + )); + } + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: (!cursor.is_empty()).then_some(cursor.clone()), + version, + limit: limit.max(1), + purpose: ReadPurpose::WritePlanLocal, + })?; + for row in &rows { + let tombstone = match validate_restore_init_upload_tombstone_row(self.mount, row) { + Ok(tombstone) => tombstone, + Err(error) => { + self.quarantine_restore_release_job(row, &error.to_string())?; + continue; + } + }; + if let Err(error) = self.delete_restore_init_object_range( + tombstone.inode, + tombstone.generation, + tombstone.size, + ) { + self.quarantine_restore_release_job(row, &error.to_string())?; + } + } + let next_cursor = rows.last().map_or_else(Vec::new, |row| row.key.clone()); + if !rows.is_empty() || !cursor.is_empty() { + self.update_restore_round_robin_cursor( + cursor_key, + cursor_item, + next_cursor, + &prefix, + b"restore-init-tombstone-cursor", + )?; + } + Ok(rows.len()) + } + + fn update_restore_round_robin_cursor( + &self, + cursor_key: Vec, + cursor_item: Option, + next_cursor: Vec, + item_prefix: &[u8], + request_prefix: &[u8], + ) -> Result<(), MetadError> { + if next_cursor.len() > MAX_RESTORE_PATH_BYTES + || (!next_cursor.is_empty() && !next_cursor.starts_with(item_prefix)) + { + return Err(MetadError::Codec( + "restore round-robin cursor escaped its keyspace".to_owned(), + )); + } + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id(request_prefix, self.mount, InodeId::root(), version), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: cursor_key.clone(), + predicates: vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: cursor_key.clone(), + predicate: cursor_item + .as_ref() + .map(|item| Predicate::VersionEquals(item.version)) + .unwrap_or(Predicate::NotExists), + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key: cursor_key, + op: MutationOp::Put, + value: Some(Value(next_cursor)), + }], + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore round-robin cursor")?; + self.commit_metadata(command)?; + Ok(()) + } + + fn update_restore_release_worker_cursor( + &self, + cursor_key: Vec, + cursor_item: Option, + next_cursor: &RestoreReleaseWorkerCursor, + ) -> Result<(), MetadError> { + let encoded = encode_restore_release_worker_cursor(self.mount, next_cursor)?; + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-release-worker-cursor", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: cursor_key.clone(), + predicates: vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: cursor_key.clone(), + predicate: cursor_item + .as_ref() + .map(|item| Predicate::VersionEquals(item.version)) + .unwrap_or(Predicate::NotExists), + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key: cursor_key, + op: MutationOp::Put, + value: Some(Value(encoded)), + }], + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore release worker cursor")?; + self.commit_metadata(command)?; + Ok(()) + } + + fn cleanup_restore_staging_members( + &self, + operation: &RestoreOperation, + operation_version: Version, + ) -> Result { + let prefix = restore_staging_member_prefix(self.mount, operation.ref_set_id); + let Some(row) = self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix, + start_after: None, + version: self.read_version()?, + limit: 1, + purpose: ReadPurpose::RestoreStaging, + })? + .into_iter() + .next() + else { + return Ok(true); + }; + let member = decode_restore_staging_member(&row.value.0)?; + if member.operation_digest != operation.operation_digest + || restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ) != row.key + { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + self.cleanup_restore_staging_member(operation, operation_version, member, row.version)?; + Ok(false) + } + + fn cleanup_restore_staging_member( + &self, + operation: &RestoreOperation, + operation_version: Version, + member: RestoreStagingMember, + member_version: Version, + ) -> Result { + let read_version = self.read_version()?; + if member.source_inode.is_none() { + if let (Some(parent), Some(name)) = (member.destination_parent, member.name.as_ref()) { + if let Some((entry, _)) = self.lookup_plus_at_version_for_purpose( + parent, + name, + read_version, + ReadPurpose::RestoreStaging, + )? { + if let Some(body) = entry.body { + let chunks = self.chunk_manifests_for_body_at_version( + member.destination_inode, + &body, + read_version, + ReadPurpose::RestoreStaging, + )?; + for block in chunks + .iter() + .flat_map(|chunk| chunk.slices.iter()) + .flat_map(|slice| slice.blocks.iter()) + { + if !self.owns_block_object_key( + member.destination_inode, + body.generation, + &block.object_key, + ) { + return Err(MetadError::RestoreRootChanged { + root: member.destination_inode, + }); + } + self.objects + .delete(&ObjectKey::new(block.object_key.clone())?)?; + } + } + } + } + } + if !self.delete_restore_metadata_prefix( + operation, + operation_version, + RecordFamily::ChunkManifest, + inode_key(self.mount, member.destination_inode), + )? { + return Ok(false); + } + if !self.delete_restore_metadata_prefix( + operation, + operation_version, + RecordFamily::Xattr, + xattr_prefix(self.mount, member.destination_inode), + )? { + return Ok(false); + } + let inverse_key = restore_staging_inode_key(self.mount, member.destination_inode); + let inverse = self + .metadata + .get_versioned( + RecordFamily::System, + &inverse_key, + self.read_version()?, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreRootChanged { + root: member.destination_inode, + })?; + let (digest, ref_set_id) = decode_restore_staging_inverse(&inverse.value.0)?; + if digest != operation.operation_digest || ref_set_id != operation.ref_set_id { + return Err(MetadError::RestoreRootChanged { + root: member.destination_inode, + }); + } + let inverse_owner_key = restore_staging_inverse_owner_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ); + let inverse_owner = self + .metadata + .get_versioned( + RecordFamily::System, + &inverse_owner_key, + self.read_version()?, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreRootChanged { + root: member.destination_inode, + })?; + if inverse_owner.value != inverse.value { + return Err(MetadError::RestoreRootChanged { + root: member.destination_inode, + }); + } + let member_key = + restore_staging_member_key(self.mount, operation.ref_set_id, member.destination_inode); + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: member_key.clone(), + predicate: Predicate::VersionEquals(member_version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.clone(), + predicate: Predicate::VersionEquals(inverse.version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_owner_key.clone(), + predicate: Predicate::VersionEquals(inverse_owner.version), + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, member.destination_inode), + predicate: Predicate::Exists, + }, + ]; + let mut mutations = vec![ + delete_mutation( + RecordFamily::Inode, + inode_key(self.mount, member.destination_inode), + ), + delete_mutation(RecordFamily::System, member_key), + delete_mutation(RecordFamily::System, inverse_key), + delete_mutation(RecordFamily::System, inverse_owner_key), + ]; + if let (Some(parent), Some(name)) = (member.destination_parent, member.name) { + let dentry_key = dentry_key(self.mount, parent, &name); + let dentry = self + .metadata + .get_versioned( + RecordFamily::Dentry, + &dentry_key, + self.read_version()?, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreRootChanged { + root: member.destination_inode, + })?; + let projection = + crate::layout::decode_dentry_projection(&dentry.value.0).map_err(|_| { + MetadError::RestoreRootChanged { + root: member.destination_inode, + } + })?; + if projection.attr.inode != member.destination_inode { + return Err(MetadError::RestoreRootChanged { + root: member.destination_inode, + }); + } + predicates.push(PredicateRef { + family: RecordFamily::Dentry, + key: dentry_key.clone(), + predicate: Predicate::VersionEquals(dentry.version), + }); + mutations.push(delete_mutation(RecordFamily::Dentry, dentry_key)); + } + let version = self.next_version()?; + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"restore-discard-member", + self.mount, + member.destination_inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: restore_staging_member_prefix(self.mount, operation.ref_set_id), + predicates, + mutations, + watch: Vec::new(), + })?; + Ok(true) + } + + fn delete_restore_metadata_prefix( + &self, + operation: &RestoreOperation, + operation_version: Version, + family: RecordFamily, + prefix: Vec, + ) -> Result { + let rows = self.metadata.scan(ScanRequest { + family, + prefix: prefix.clone(), + start_after: None, + version: self.read_version()?, + limit: RESTORE_BATCH_ENTRIES, + purpose: ReadPurpose::RestoreStaging, + })?; + if rows.is_empty() { + return Ok(true); + } + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-discard-metadata", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: family, + primary_key: prefix, + predicates: std::iter::once(PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }) + .chain(rows.iter().map(|row| PredicateRef { + family, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + })) + .collect(), + mutations: rows + .into_iter() + .map(|row| delete_mutation(family, row.key)) + .collect(), + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore discard metadata")?; + self.commit_metadata(command)?; + Ok(false) + } + + fn cleanup_restore_base_references( + &self, + operation: &RestoreOperation, + operation_version: Version, + ) -> Result { + // Reference-page creation validates existing inverse/owner pairs under + // this same gate. Serialize bounded Preparing cleanup with that read so + // a legitimate discard cannot be mistaken for durable corruption. + let _gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|error| error.into_inner()); + let prefix = restore_base_owner_prefix(self.mount, operation.ref_set_id); + let read_version = self.read_version()?; + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: None, + version: read_version, + limit: RESTORE_BATCH_ENTRIES, + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + return Ok(true); + } + let mut predicates = vec![PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }]; + let mut mutations = Vec::with_capacity(rows.len() * 2); + for row in &rows { + let reference = super::restore_gc::decode_restore_base_reference(&row.value.0)?; + if reference.operation_digest != operation.operation_digest + || reference.ref_set_id != operation.ref_set_id + { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + let object_digest: [u8; 32] = Sha256::digest(reference.object_key.as_bytes()).into(); + let expected_owner = restore_base_owner_key( + self.mount, + reference.ref_set_id, + &object_digest, + reference.borrower_inode, + reference.borrower_generation, + ); + if row.key != expected_owner { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + let inverse_key = restore_base_inverse_key( + self.mount, + &object_digest, + reference.ref_set_id, + reference.borrower_inode, + reference.borrower_generation, + ); + let inverse = self + .metadata + .get_versioned( + RecordFamily::System, + &inverse_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreRootChanged { + root: operation.destination_root, + })?; + let decoded_inverse = super::restore_gc::decode_restore_base_inverse(&inverse.value.0)?; + if decoded_inverse.operation_digest != operation.operation_digest + || decoded_inverse.ref_set_id != reference.ref_set_id + || decoded_inverse.object_digest != object_digest + || decoded_inverse.borrower_inode != reference.borrower_inode + || decoded_inverse.borrower_generation != reference.borrower_generation + { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + let inverse_owner_key = restore_base_inverse_owner_key( + self.mount, + reference.ref_set_id, + &object_digest, + reference.borrower_inode, + reference.borrower_generation, + ); + let inverse_owner = self + .metadata + .get_versioned( + RecordFamily::System, + &inverse_owner_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreRootChanged { + root: operation.destination_root, + })?; + if inverse_owner.value != inverse.value { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + predicates.extend([ + PredicateRef { + family: RecordFamily::System, + key: expected_owner.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.clone(), + predicate: Predicate::VersionEquals(inverse.version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_owner_key.clone(), + predicate: Predicate::VersionEquals(inverse_owner.version), + }, + ]); + mutations.extend([ + delete_mutation(RecordFamily::System, expected_owner), + delete_mutation(RecordFamily::System, inverse_key), + delete_mutation(RecordFamily::System, inverse_owner_key), + ]); + } + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-discard-base-refs", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: prefix, + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore discard base refs")?; + self.commit_metadata(command)?; + Ok(false) + } + + /// Advance exactly one durable private-index cleanup page. The index helper + /// may commit its own bounded delete batch first; persisting the opaque + /// cursor in a second CAS is ACK-loss safe because replaying an older cursor + /// only rescans already-absent keys. + fn cleanup_restore_index_page( + &self, + operation: &RestoreOperation, + operation_version: Version, + ) -> Result { + let cleanup_key = restore_cleanup_job_key(self.mount, operation.ref_set_id); + let cleanup_item = self + .metadata + .get_versioned( + RecordFamily::System, + &cleanup_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("restore cleanup job is missing".to_owned()))?; + let mut cleanup = decode_restore_cleanup_job(&cleanup_item.value.0)?; + if cleanup.operation_digest != operation.operation_digest + || cleanup.ref_set_id != operation.ref_set_id + { + return Err(MetadError::Codec( + "restore cleanup job changed identity".to_owned(), + )); + } + if cleanup.index_complete { + return Ok(true); + } + let outcome = self.release_restore_index_page( + operation, + operation_version, + &cleanup.index_cursor, + RESTORE_BATCH_ENTRIES, + )?; + cleanup.index_complete = outcome.complete; + cleanup.index_cursor = if outcome.complete { + Vec::new() + } else { + outcome.cursor + }; + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-discard-index-cursor", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: cleanup_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: cleanup_key.clone(), + predicate: Predicate::VersionEquals(cleanup_item.version), + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key: cleanup_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_cleanup_job(&cleanup)?)), + }], + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore discard index cursor")?; + self.commit_metadata(command)?; + Ok(cleanup.index_complete) + } + + fn finish_restore_discard( + &self, + operation: &RestoreOperation, + operation_version: Version, + ) -> Result<(), MetadError> { + let read_version = self.read_version()?; + let binding_key = fork_binding_key(self.mount, operation.destination_root); + let binding = self + .metadata + .get_versioned( + RecordFamily::ForkBinding, + &binding_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreBindingChanged { + root: operation.destination_root, + })?; + let decoded_binding = + crate::layout::decode_fork_binding(&binding.value.0).map_err(|_| { + MetadError::RestoreBindingChanged { + root: operation.destination_root, + } + })?; + if decoded_binding.fork_root != operation.destination_root + || decoded_binding.source_root != operation.source_root + || decoded_binding.pinned_read_version != operation.read_version + || decoded_binding.snapshot_id != operation.snapshot_id + || decoded_binding.created_version != operation.created_version + { + return Err(MetadError::RestoreBindingChanged { + root: operation.destination_root, + }); + } + let claim_key = restore_destination_claim_key(self.mount, &operation.destination_path); + let claim = self + .metadata + .get_versioned( + RecordFamily::System, + &claim_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreRootChanged { + root: operation.destination_root, + })?; + let root_key = restore_root_index_key(self.mount, operation.destination_root); + let root = self + .metadata + .get_versioned( + RecordFamily::System, + &root_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreRootChanged { + root: operation.destination_root, + })?; + if claim.value.0 != operation.operation_digest || root.value.0 != operation.operation_digest + { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + let operation_key = restore_operation_key(self.mount, &operation.operation_digest); + let seal_key = restore_base_seal_key(self.mount, operation.ref_set_id); + let cleanup_key = restore_cleanup_job_key(self.mount, operation.ref_set_id); + let cleanup = self + .metadata + .get_versioned( + RecordFamily::System, + &cleanup_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("restore cleanup job is missing".to_owned()))?; + let cleanup_job = decode_restore_cleanup_job(&cleanup.value.0)?; + if cleanup_job.operation_digest != operation.operation_digest + || cleanup_job.ref_set_id != operation.ref_set_id + || !cleanup_job.index_complete + { + return Err(MetadError::Codec( + "restore cleanup job is not complete".to_owned(), + )); + } + let seal = self.metadata.get_versioned( + RecordFamily::System, + &seal_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + if let Some(item) = &seal { + let identity_matches = + match super::restore_gc::decode_restore_base_seal_record(&item.value.0)? { + super::restore_gc::RestoreBaseSealRecord::Building(build) => { + build.operation_digest == operation.operation_digest + && build.initialization_digest == operation.initialization_digest + && build.ref_set_id == operation.ref_set_id + && build.incarnation == operation.created_version + } + super::restore_gc::RestoreBaseSealRecord::Sealed(seal) => { + seal.operation_digest == operation.operation_digest + && seal.initialization_digest == operation.initialization_digest + && seal.ref_set_id == operation.ref_set_id + && seal.incarnation == operation.created_version + } + }; + if !identity_matches { + return Err(MetadError::Codec( + "restore discard base seal/progress changed identity".to_owned(), + )); + } + } + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + predicate: Predicate::VersionEquals(binding.version), + }, + PredicateRef { + family: RecordFamily::System, + key: claim_key.clone(), + predicate: Predicate::VersionEquals(claim.version), + }, + PredicateRef { + family: RecordFamily::System, + key: root_key.clone(), + predicate: Predicate::VersionEquals(root.version), + }, + PredicateRef { + family: RecordFamily::System, + key: cleanup_key.clone(), + predicate: Predicate::VersionEquals(cleanup.version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_member_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_inverse_owner_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_base_owner_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_base_inverse_owner_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_init_upload_intent_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: seal_key.clone(), + predicate: seal + .as_ref() + .map(|item| Predicate::VersionEquals(item.version)) + .unwrap_or(Predicate::NotExists), + }, + ]; + predicates.extend( + super::restore_index::restore_index_release_empty_predicates( + self.mount, + operation.ref_set_id, + ), + ); + let mut mutations = vec![ + delete_mutation(RecordFamily::System, operation_key.clone()), + delete_mutation(RecordFamily::ForkBinding, binding_key), + delete_mutation(RecordFamily::System, claim_key), + delete_mutation(RecordFamily::System, root_key), + delete_mutation(RecordFamily::System, cleanup_key), + ]; + if seal.is_some() { + mutations.push(delete_mutation(RecordFamily::System, seal_key)); + } + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-finish-discard", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: operation_key.clone(), + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore finish discard")?; + self.commit_metadata(command)?; + Ok(()) + } + + pub fn cleanup_restore_releases(&self, limit: usize) -> Result { + let _gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|error| error.into_inner()); + self.cleanup_restore_releases_locked(limit) + } + + pub(super) fn cleanup_restore_releases_locked( + &self, + limit: usize, + ) -> Result { + let version = self.read_version()?; + let prefix = restore_release_job_prefix(self.mount); + let cursor_key = restore_release_cursor_key(self.mount); + let cursor_item = self.metadata.get_versioned( + RecordFamily::System, + &cursor_key, + version, + ReadPurpose::WritePlanLocal, + )?; + let persisted_cursor = cursor_item + .as_ref() + .map(|item| { + decode_restore_release_worker_cursor_at_version(self.mount, &item.value.0, version) + }) + .transpose()?; + let mut active_cursor = + persisted_cursor + .clone() + .unwrap_or_else(|| RestoreReleaseWorkerCursor { + cycle_high_water: version.get(), + start_after: Vec::new(), + }); + let effective_limit = limit.max(1); + let scan_cycle = |cursor: &RestoreReleaseWorkerCursor| { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: (!cursor.start_after.is_empty()).then_some(cursor.start_after.clone()), + version, + limit: effective_limit, + purpose: ReadPurpose::WritePlanLocal, + })?; + let reached_physical_tail = rows.len() < effective_limit; + let mut visited = Vec::with_capacity(rows.len()); + let mut reached_cycle_boundary = false; + for row in rows { + if row.version.get() > cursor.cycle_high_water { + // Both newly-created operations and old Complete restores + // released late are outside this frozen cycle. Stop at the + // first newer row version instead of chasing a key tail + // that can grow forever. + reached_cycle_boundary = true; + break; + } + visited.push(row); + } + Ok::<_, MetadError>((visited, reached_physical_tail, reached_cycle_boundary)) + }; + + let (mut visited, mut reached_physical_tail, mut reached_version_boundary) = + scan_cycle(&active_cursor)?; + let rolled_cursor = RestoreReleaseWorkerCursor { + cycle_high_water: version.get(), + start_after: Vec::new(), + }; + // If the prior page ended exactly at the high-water boundary, start + // the next cycle in this same bounded worker call. The first scan did + // not visit a row, so at most `effective_limit` jobs are processed. + if visited.is_empty() + && (reached_physical_tail || reached_version_boundary) + && active_cursor != rolled_cursor + { + active_cursor = rolled_cursor; + (visited, reached_physical_tail, reached_version_boundary) = + scan_cycle(&active_cursor)?; + } + if visited.is_empty() { + // Keep an idle cursor unchanged. Advancing only its high-water on + // every background GC tick would create an unbounded metadata-log + // write loop and continually invalidate stable-epoch fsck scans. + return Ok(0); + } + + let mut processed = 0_usize; + for row in &visited { + let Some(ref_set_id) = restore_release_job_ref_set_id(self.mount, &row.key) else { + self.quarantine_restore_invalid_release_job_key( + row, + "restore release job key has an invalid identity", + )?; + processed = processed.saturating_add(1); + continue; + }; + let job = match (|| { + if ref_set_id > row.version.get() { + return Err(MetadError::Codec( + "restore release job identity is newer than its row".to_owned(), + )); + } + decode_restore_release_job(&row.value.0) + })() + .and_then(|job| { + if restore_release_job_key(self.mount, job.ref_set_id) != row.key { + return Err(MetadError::Codec( + "restore release job key does not match its identity".to_owned(), + )); + } + Ok(job) + }) { + Ok(job) => job, + Err(error) => { + self.quarantine_restore_release_job(row, &error.to_string())?; + processed = processed.saturating_add(1); + continue; + } + }; + if let Err(error) = self.process_restore_release_job(job, row.version) { + if restore_release_error_is_retryable(&error) { + // A concurrent worker, lost acknowledgement, owner fence, + // or recovery-log outage is resolved by a later durable + // scan. Never quarantine a healthy job for an external or + // retryable control-plane fault. + processed = processed.saturating_add(1); + continue; + } + // Keep the active job for fsck and a later retry, but record + // the failure and advance the mount-global cursor. One damaged + // or repeatedly failing ref-set must not starve unrelated + // release jobs behind it. + self.quarantine_restore_release_job(row, &error.to_string())?; + processed = processed.saturating_add(1); + continue; + } + processed = processed.saturating_add(1); + } + + let next_cursor = if reached_version_boundary { + RestoreReleaseWorkerCursor { + // A newer row ended this frozen cycle. Raise the frontier to + // the scan snapshot, even when older blocked jobs preceded the + // boundary, so that mixed queues cannot pin it forever. + cycle_high_water: version.get(), + start_after: Vec::new(), + } + } else if reached_physical_tail { + RestoreReleaseWorkerCursor { + // Reset the key position but keep the frontier frozen. If a + // job actually changed, its newer row version becomes the + // next call's explicit boundary and raises the frontier then. + // A snapshot-blocked job therefore produces no idle writes. + cycle_high_water: active_cursor.cycle_high_water, + start_after: Vec::new(), + } + } else { + RestoreReleaseWorkerCursor { + cycle_high_water: active_cursor.cycle_high_water, + start_after: visited + .iter() + .rev() + .find(|row| restore_release_job_ref_set_id(self.mount, &row.key).is_some()) + .map_or_else(|| active_cursor.start_after.clone(), |row| row.key.clone()), + } + }; + let cursor_changed = persisted_cursor.as_ref() != Some(&next_cursor); + if cursor_changed { + self.update_restore_release_worker_cursor(cursor_key, cursor_item, &next_cursor)?; + } + Ok(processed) + } + + pub(super) fn restore_release_backlog(&self) -> Result<(usize, usize, usize), MetadError> { + let version = self.read_version()?; + Ok(( + self.count_restore_control_rows(restore_release_job_prefix(self.mount), version)?, + self.count_restore_control_rows( + restore_release_quarantine_prefix(self.mount), + version, + )?, + self.count_restore_control_rows( + restore_release_mount_wide_quarantine_prefix(self.mount), + version, + )?, + )) + } + + fn count_restore_control_rows( + &self, + prefix: Vec, + version: Version, + ) -> Result { + let mut count = 0_usize; + let mut start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: RESTORE_BACKLOG_PAGE_ROWS, + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + return Ok(count); + } + count = count.saturating_add(rows.len()); + start_after = rows.last().map(|row| row.key.clone()); + if rows.len() < RESTORE_BACKLOG_PAGE_ROWS { + return Ok(count); + } + } + } + + fn quarantine_restore_release_job( + &self, + row: &crate::command::ScanItem, + reason: &str, + ) -> Result<(), MetadError> { + self.quarantine_restore_release_job_with_disposition(row, reason, false) + } + + fn quarantine_restore_invalid_release_job_key( + &self, + row: &crate::command::ScanItem, + reason: &str, + ) -> Result<(), MetadError> { + self.quarantine_restore_release_job_with_disposition(row, reason, true) + } + + fn quarantine_restore_release_job_with_disposition( + &self, + row: &crate::command::ScanItem, + reason: &str, + remove_original: bool, + ) -> Result<(), MetadError> { + let scope = RestoreReleaseQuarantineScope::Diagnostic; + let quarantine_key = + restore_release_quarantine_key(self.mount, RecordFamily::System, row, scope); + let version = self.next_version()?; + let mut mutations = vec![Mutation { + family: RecordFamily::System, + key: quarantine_key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_restore_release_quarantine( + RecordFamily::System, + row, + reason, + scope, + )?)), + }]; + if remove_original { + // A non-canonical physical key cannot own a release operation. + // Move its exact bytes into quarantine atomically so the damaged + // row remains diagnosable without blocking the bounded scanner. + mutations.push(delete_mutation(RecordFamily::System, row.key.clone())); + } + let command = MetadataCommand { + request_id: request_id( + b"restore-quarantine-release-job", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: row.key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + PredicateRef { + family: RecordFamily::System, + key: quarantine_key.clone(), + predicate: Predicate::NotExists, + }, + ], + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore release job quarantine")?; + match self.commit_metadata(command) { + Ok(_) + | Err(MetadError::Metadata(MetadataError::PredicateFailed)) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + }) => Ok(()), + Err(error) => Err(error), + } + } + + fn process_restore_release_job( + &self, + mut job: RestoreReleaseJob, + mut job_version: Version, + ) -> Result<(), MetadError> { + // A snapshot pin is a new historical borrower. Keep its publication + // mutually exclusive with the reachability proof and exact-reference + // CAS below, without serializing ordinary object GC against snapshot + // minting (the durable object-GC claim version already fences that). + let _restore_snapshot_gate = self + .restore_snapshot_gate + .write() + .unwrap_or_else(|error| error.into_inner()); + let (operation, operation_version) = + self.releasing_restore_operation(&job.operation_digest, job.ref_set_id)?; + // System-family restore rows have no historical MVCC view. If a live + // snapshot can still reach any member in this ref-set, retain the whole + // operation (members, exact refs, overlay, seals, and root index) until + // that pin retires. Snapshot minting and this worker share + // `restore_snapshot_gate`, so the proof cannot race a newly published + // pin. + // Members -> Overlay is a durable, prefix-empty cut made while this + // worker holds both object_gc_gate and restore_snapshot_gate. Before + // that cut, every historical holder is discoverable through a staging + // inverse and must be rechecked on every page. After it, no current + // namespace inode can identify this ref-set and a newly minted + // snapshot/ForkBinding cannot resurrect one, so repeating a full + // historical subtree walk for every one of the index-release stages is + // both redundant and pathologically expensive on a deep Holt history. + if job.phase != RestoreReleasePhase::Overlay + && self.restore_live_snapshot_holds_ref_set(&operation)? + { + return Ok(()); + } + match job.phase { + RestoreReleasePhase::ExactReferences => { + self.release_restore_reference_page( + &operation, + operation_version, + &mut job, + &mut job_version, + )?; + } + RestoreReleasePhase::Members => { + self.release_restore_member_page( + &operation, + operation_version, + &mut job, + &mut job_version, + )?; + } + RestoreReleasePhase::Overlay => { + let outcome = self.release_restore_index_page( + &operation, + operation_version, + &job.cursor, + RESTORE_BATCH_ENTRIES, + )?; + job.cursor = outcome.cursor; + if outcome.complete { + self.finish_restore_release(&operation, operation_version, &job, job_version)?; + } else { + self.update_restore_release_job( + &operation, + operation_version, + &job, + &mut job_version, + )?; + } + } + } + live_test_barrier::restore_applied( + &restore_barrier_operation_id(&operation), + live_test_barrier::RestoreAppliedPhase::ReleaseBatch(0), + )?; + Ok(()) + } + + fn releasing_restore_operation( + &self, + digest: &[u8; 32], + ref_set_id: u64, + ) -> Result<(RestoreOperation, Version), MetadError> { + let key = restore_operation_key(self.mount, digest); + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::Codec( + "restore release job has no operation".to_owned(), + ))?; + let operation = decode_restore_operation(&item.value.0)?; + if operation.operation_digest != *digest + || operation.ref_set_id != ref_set_id + || operation.state != RestoreOperationState::Releasing + { + return Err(MetadError::Codec( + "restore release job operation identity changed".to_owned(), + )); + } + Ok((operation, item.version)) + } + + fn restore_reference_release_command( + &self, + input: RestoreReferenceReleaseCommand<'_>, + ) -> Result { + let RestoreReferenceReleaseCommand { + operation, + operation_version, + job, + job_version, + entries, + defer_gc_for, + deferred_guard, + object_reference, + version, + } = input; + let first = entries.first().ok_or_else(|| { + MetadError::Codec("restore reference release batch is empty".to_owned()) + })?; + let mut predicates = Vec::with_capacity(3 + entries.len().saturating_mul(3)); + predicates.extend([ + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + predicate: Predicate::VersionEquals(job_version), + }, + object_reference.predicate(self.mount), + ]); + match (defer_gc_for, deferred_guard) { + (Some(object_digest), Some(guard)) if guard.object_digest == object_digest => { + predicates.extend([ + PredicateRef { + family: RecordFamily::System, + key: guard.owner_key.clone(), + predicate: Predicate::VersionEquals(guard.owner_version), + }, + PredicateRef { + family: RecordFamily::System, + key: guard.inverse_key.clone(), + predicate: Predicate::VersionEquals(guard.inverse_version), + }, + PredicateRef { + family: RecordFamily::System, + key: guard.inverse_owner_key.clone(), + predicate: Predicate::VersionEquals(guard.inverse_owner_version), + }, + ]); + } + (None, None) => {} + _ => { + return Err(MetadError::Codec( + "restore release deferred GC has no matching continuation proof".to_owned(), + )); + } + } + let mut mutations = Vec::with_capacity(1 + entries.len().saturating_mul(4)); + mutations.push(Mutation { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + op: MutationOp::Put, + value: Some(Value(encode_restore_release_job(job)?)), + }); + let enqueue_unix_ms = current_time_ms(); + let mut gc_records = BTreeMap::new(); + for entry in entries { + predicates.extend([ + PredicateRef { + family: RecordFamily::System, + key: entry.owner_key.clone(), + predicate: Predicate::VersionEquals(entry.owner_version), + }, + PredicateRef { + family: RecordFamily::System, + key: entry.inverse_key.clone(), + predicate: Predicate::VersionEquals(entry.inverse_version), + }, + PredicateRef { + family: RecordFamily::System, + key: entry.inverse_owner_key.clone(), + predicate: Predicate::VersionEquals(entry.inverse_owner_version), + }, + ]); + mutations.extend([ + delete_mutation(RecordFamily::System, entry.owner_key.clone()), + delete_mutation(RecordFamily::System, entry.inverse_key.clone()), + delete_mutation(RecordFamily::System, entry.inverse_owner_key.clone()), + ]); + if defer_gc_for == Some(entry.object_digest) { + // The next owner row belongs to the same canonical object. + // Defer enqueue until its final owner page so a malformed + // continuation cannot be hidden between release pages. + continue; + } + let (owner_inode, owner_generation, chunk_index, block_index) = entry.identity; + let record = ObjectGcRecord { + inode: owner_inode, + generation: owner_generation, + object_key: entry.reference.object_key.clone(), + size: entry.reference.size, + digest_uri: entry.reference.digest_uri.clone(), + enqueue_version: version.get(), + enqueue_unix_ms, + }; + let key = gc_object_key( + self.mount, + version.get(), + owner_inode, + owner_generation, + chunk_index, + block_index, + ); + if let Some(existing) = gc_records.insert(key, record.clone()) { + if existing != record { + return Err(MetadError::Codec( + "restore release batch has inconsistent canonical object identity" + .to_owned(), + )); + } + } + } + mutations.extend(gc_records.into_iter().map(|(key, record)| Mutation { + family: RecordFamily::Gc, + key, + op: MutationOp::Put, + value: Some(Value(encode_object_gc_record(&record))), + })); + Ok(MetadataCommand { + request_id: request_id( + b"restore-release-reference-batch", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: first.owner_key.clone(), + predicates, + mutations, + watch: Vec::new(), + }) + } + + #[allow(clippy::too_many_arguments)] + fn commit_restore_reference_release_batch( + &self, + operation: &RestoreOperation, + operation_version: Version, + job: &RestoreReleaseJob, + job_version: &mut Version, + entries: &mut Vec, + defer_gc_for: Option<[u8; 32]>, + deferred_guard: Option<&RestoreReferenceReleaseGuard>, + ) -> Result<(), MetadError> { + if entries.is_empty() { + return Ok(()); + } + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let command = self.restore_reference_release_command(RestoreReferenceReleaseCommand { + operation, + operation_version, + job, + job_version: *job_version, + entries, + defer_gc_for, + deferred_guard, + object_reference, + version, + })?; + validate_restore_command_bounds(&command, "restore release reference batch")?; + let request_id = command.request_id.clone(); + let expected_mutations = command.mutations.len(); + let commit = self.commit_metadata(command); + match commit { + Ok(_) => {} + Err(MetadError::Metadata(MetadataError::Backend(message))) => { + let applied = self + .metadata + .committed_request_result(&request_id)? + .is_some_and(|result| { + result.commit_version == version + && result.applied_mutations == expected_mutations + && result.watch_events == 0 + }); + if !applied { + return Err(MetadError::Metadata(MetadataError::Backend(message))); + } + } + Err(error) => return Err(error), + } + *job_version = version; + entries.clear(); + Ok(()) + } + + fn quarantine_restore_release_row_and_advance( + &self, + operation: &RestoreOperation, + operation_version: Version, + job: &mut RestoreReleaseJob, + job_version: &mut Version, + row: &crate::command::ScanItem, + reason: &str, + ) -> Result<(), MetadError> { + self.quarantine_restore_release_row_and_advance_scoped( + operation, + operation_version, + job, + job_version, + row, + reason, + RestoreReleaseQuarantineScope::Diagnostic, + ) + } + + #[allow(clippy::too_many_arguments)] + fn quarantine_restore_release_object_row_and_advance( + &self, + operation: &RestoreOperation, + operation_version: Version, + job: &mut RestoreReleaseJob, + job_version: &mut Version, + pending: &mut Vec, + row: &crate::command::ScanItem, + reason: &str, + object_digest: Option<[u8; 32]>, + ) -> Result<(), MetadError> { + self.quarantine_restore_release_row_and_advance_scoped( + operation, + operation_version, + job, + job_version, + row, + reason, + object_digest.map_or( + RestoreReleaseQuarantineScope::MountWide, + RestoreReleaseQuarantineScope::Object, + ), + )?; + // The marker must become durable before a preceding valid row can + // enqueue this object's GC candidate. Otherwise a crash between the + // batch and this quarantine could hide the only malformed borrower. + self.commit_restore_reference_release_batch( + operation, + operation_version, + job, + job_version, + pending, + None, + None, + ) + } + + #[allow(clippy::too_many_arguments)] + fn quarantine_restore_release_row_and_advance_scoped( + &self, + operation: &RestoreOperation, + operation_version: Version, + job: &mut RestoreReleaseJob, + job_version: &mut Version, + row: &crate::command::ScanItem, + reason: &str, + scope: RestoreReleaseQuarantineScope, + ) -> Result<(), MetadError> { + job.cursor = row.key.clone(); + let quarantine_key = + restore_release_quarantine_key(self.mount, RecordFamily::System, row, scope); + let quarantine = self.metadata.get_versioned( + RecordFamily::System, + &quarantine_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )?; + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + predicate: Predicate::VersionEquals(*job_version), + }, + PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + ]; + let mut mutations = vec![ + Mutation { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + op: MutationOp::Put, + value: Some(Value(encode_restore_release_job(job)?)), + }, + // Retain the malformed active row. Removing it could make the + // ref-set owner prefix appear empty while a target-first inverse + // still exists, allowing finalization to erase the operation that + // fsck needs for repair. The per-job cursor advances past it, and + // the mount-global round-robin cursor keeps unrelated jobs moving. + ]; + match quarantine { + Some(item) => predicates.push(PredicateRef { + family: RecordFamily::System, + key: quarantine_key, + predicate: Predicate::VersionEquals(item.version), + }), + None => { + predicates.push(PredicateRef { + family: RecordFamily::System, + key: quarantine_key.clone(), + predicate: Predicate::NotExists, + }); + mutations.push(Mutation { + family: RecordFamily::System, + key: quarantine_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_release_quarantine( + RecordFamily::System, + row, + reason, + scope, + )?)), + }); + } + } + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-quarantine-release-row", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: row.key.clone(), + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore release row quarantine")?; + self.commit_metadata(command)?; + *job_version = version; + Ok(()) + } + + fn restore_reference_release_guard_for_row( + &self, + operation: &RestoreOperation, + row: &crate::command::ScanItem, + read_version: Version, + ) -> Result { + let keyed_object_digest = + restore_base_owner_object_digest_from_key(self.mount, operation.ref_set_id, &row.key); + let Some(keyed_object_digest) = keyed_object_digest else { + return Ok(RestoreReferenceReleaseGuardValidation::Corrupt { + reason: "restore release owner continuation has an invalid key identity".to_owned(), + object_digest: None, + }); + }; + let reference = match super::restore_gc::decode_restore_base_reference(&row.value.0) { + Ok(reference) => reference, + Err(error) => { + return Ok(RestoreReferenceReleaseGuardValidation::Corrupt { + reason: error.to_string(), + object_digest: None, + }); + } + }; + let object_digest: [u8; 32] = Sha256::digest(reference.object_key.as_bytes()).into(); + if reference.operation_digest != operation.operation_digest + || reference.ref_set_id != operation.ref_set_id + || object_digest != keyed_object_digest + || restore_base_owner_key( + self.mount, + reference.ref_set_id, + &object_digest, + reference.borrower_inode, + reference.borrower_generation, + ) != row.key + { + return Ok(RestoreReferenceReleaseGuardValidation::Corrupt { + reason: "restore release owner continuation changed identity".to_owned(), + object_digest: (object_digest == keyed_object_digest).then_some(object_digest), + }); + } + let inverse_key = restore_base_inverse_key( + self.mount, + &object_digest, + reference.ref_set_id, + reference.borrower_inode, + reference.borrower_generation, + ); + let inverse = self.metadata.get_versioned( + RecordFamily::System, + &inverse_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + let Some(inverse) = inverse else { + return Ok(RestoreReferenceReleaseGuardValidation::Corrupt { + reason: "restore release owner continuation has no inverse row".to_owned(), + object_digest: Some(object_digest), + }); + }; + let decoded_inverse = match super::restore_gc::decode_restore_base_inverse(&inverse.value.0) + { + Ok(inverse) => inverse, + Err(error) => { + return Ok(RestoreReferenceReleaseGuardValidation::Corrupt { + reason: error.to_string(), + object_digest: Some(object_digest), + }); + } + }; + if decoded_inverse.operation_digest != operation.operation_digest + || decoded_inverse.ref_set_id != operation.ref_set_id + || decoded_inverse.object_digest != object_digest + || decoded_inverse.borrower_inode != reference.borrower_inode + || decoded_inverse.borrower_generation != reference.borrower_generation + { + return Ok(RestoreReferenceReleaseGuardValidation::Corrupt { + reason: "restore release inverse continuation changed identity".to_owned(), + object_digest: Some(object_digest), + }); + } + let inverse_owner_key = restore_base_inverse_owner_key( + self.mount, + operation.ref_set_id, + &object_digest, + reference.borrower_inode, + reference.borrower_generation, + ); + let inverse_owner = self.metadata.get_versioned( + RecordFamily::System, + &inverse_owner_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + let Some(inverse_owner) = inverse_owner else { + return Ok(RestoreReferenceReleaseGuardValidation::Corrupt { + reason: "restore release inverse continuation has no ref-set owner".to_owned(), + object_digest: Some(object_digest), + }); + }; + if inverse_owner.value != inverse.value { + return Ok(RestoreReferenceReleaseGuardValidation::Corrupt { + reason: "restore release inverse-owner continuation changed identity".to_owned(), + object_digest: Some(object_digest), + }); + } + let identity = match self.canonical_block_object_identity(&reference.object_key) { + Ok(identity) => identity, + Err(error) => { + return Ok(RestoreReferenceReleaseGuardValidation::Corrupt { + reason: error.to_string(), + object_digest: Some(object_digest), + }); + } + }; + Ok(RestoreReferenceReleaseGuardValidation::Valid(Box::new( + RestoreReferenceReleaseGuard { + owner_key: row.key.clone(), + owner_version: row.version, + inverse_key, + inverse_version: inverse.version, + inverse_owner_key, + inverse_owner_version: inverse_owner.version, + reference, + object_digest, + identity, + }, + ))) + } + + fn release_restore_reference_page( + &self, + operation: &RestoreOperation, + operation_version: Version, + job: &mut RestoreReleaseJob, + job_version: &mut Version, + ) -> Result<(), MetadError> { + let prefix = restore_base_owner_prefix(self.mount, operation.ref_set_id); + let read_version = self.read_version()?; + let scanned_rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: (!job.cursor.is_empty()).then_some(job.cursor.clone()), + version: read_version, + limit: RESTORE_BATCH_ENTRIES.saturating_add(1), + purpose: ReadPurpose::WritePlanLocal, + })?; + let rows = &scanned_rows[..scanned_rows.len().min(RESTORE_BATCH_ENTRIES)]; + let borrower_inodes = rows + .iter() + .filter_map(|row| { + super::restore_gc::decode_restore_base_reference(&row.value.0) + .ok() + .map(|reference| reference.borrower_inode) + }) + .collect::>(); + let reachable_bodies = + self.restore_reachable_inode_bodies_at(&borrower_inodes, read_version)?; + let mut cursor_dirty = false; + let mut processed_all_rows = true; + let mut pending = Vec::new(); + for row in rows { + let keyed_object_digest = restore_base_owner_object_digest_from_key( + self.mount, + operation.ref_set_id, + &row.key, + ); + let reference = match super::restore_gc::decode_restore_base_reference(&row.value.0) { + Ok(reference) => reference, + Err(error) => { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + row, + &error.to_string(), + None, + )?; + cursor_dirty = false; + continue; + } + }; + let object_digest: [u8; 32] = Sha256::digest(reference.object_key.as_bytes()).into(); + if reference.operation_digest != operation.operation_digest + || reference.ref_set_id != operation.ref_set_id + || restore_base_owner_key( + self.mount, + reference.ref_set_id, + &object_digest, + reference.borrower_inode, + reference.borrower_generation, + ) != row.key + { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + row, + "restore release owner row changed identity", + (keyed_object_digest == Some(object_digest)).then_some(object_digest), + )?; + cursor_dirty = false; + continue; + } + let inverse_key = restore_base_inverse_key( + self.mount, + &object_digest, + reference.ref_set_id, + reference.borrower_inode, + reference.borrower_generation, + ); + let inverse = self.metadata.get_versioned( + RecordFamily::System, + &inverse_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + let Some(inverse) = inverse else { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + row, + "restore release owner has no inverse row", + keyed_object_digest, + )?; + cursor_dirty = false; + continue; + }; + let decoded_inverse = + match super::restore_gc::decode_restore_base_inverse(&inverse.value.0) { + Ok(inverse) => inverse, + Err(error) => { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + row, + &error.to_string(), + keyed_object_digest, + )?; + cursor_dirty = false; + continue; + } + }; + if decoded_inverse.operation_digest != operation.operation_digest + || decoded_inverse.ref_set_id != operation.ref_set_id + || decoded_inverse.object_digest != object_digest + || decoded_inverse.borrower_inode != reference.borrower_inode + || decoded_inverse.borrower_generation != reference.borrower_generation + { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + row, + "restore release inverse row changed identity", + keyed_object_digest, + )?; + cursor_dirty = false; + continue; + } + let inverse_owner_key = restore_base_inverse_owner_key( + self.mount, + operation.ref_set_id, + &object_digest, + reference.borrower_inode, + reference.borrower_generation, + ); + let inverse_owner = self.metadata.get_versioned( + RecordFamily::System, + &inverse_owner_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + let Some(inverse_owner) = inverse_owner else { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + row, + "restore release base inverse has no ref-set owner", + keyed_object_digest, + )?; + cursor_dirty = false; + continue; + }; + if inverse_owner.value != inverse.value { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + row, + "restore release base inverse owner changed identity", + keyed_object_digest, + )?; + cursor_dirty = false; + continue; + } + let retained = match self.restore_borrower_references_object( + &reference, + read_version, + &reachable_bodies, + ) { + Ok(retained) => retained, + Err(error) if restore_release_error_is_retryable(&error) => return Err(error), + Err(error) => { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + row, + &error.to_string(), + keyed_object_digest, + )?; + cursor_dirty = false; + continue; + } + }; + if retained { + // Retained rows only advance the release frontier. Persist one + // frontier per page instead of issuing one metadata command per + // row. A crash before the page commit causes a bounded rescan; + // object deletion and GC enqueue remain atomic in the batch. + job.cursor = row.key.clone(); + cursor_dirty = true; + continue; + } + let identity = match self.canonical_block_object_identity(&reference.object_key) { + Ok(identity) => identity, + Err(error) => { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + row, + &error.to_string(), + keyed_object_digest, + )?; + cursor_dirty = false; + continue; + } + }; + if pending.iter().any(|entry: &RestoreReferenceReleaseEntry| { + entry.reference.object_key == reference.object_key + && (entry.identity != identity + || entry.reference.size != reference.size + || entry.reference.digest_uri != reference.digest_uri) + }) { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + row, + "restore release references disagree on canonical object identity", + keyed_object_digest, + )?; + cursor_dirty = false; + continue; + } + let previous_cursor = job.cursor.clone(); + job.cursor = row.key.clone(); + cursor_dirty = true; + pending.push(RestoreReferenceReleaseEntry { + owner_key: row.key.clone(), + owner_version: row.version, + inverse_key, + inverse_version: inverse.version, + inverse_owner_key, + inverse_owner_version: inverse_owner.version, + reference, + object_digest, + identity, + }); + let bounds_version = Version::new(2)?; + let reserve_guard = RestoreReferenceReleaseGuard::from( + pending + .last() + .expect("the candidate release entry was just appended"), + ); + let bounds_command = + self.restore_reference_release_command(RestoreReferenceReleaseCommand { + operation, + operation_version, + job, + job_version: *job_version, + entries: &pending, + defer_gc_for: None, + deferred_guard: None, + object_reference: ObjectReferenceMutation::from_version(bounds_version), + version: bounds_version, + })?; + let deferred_bounds_command = + self.restore_reference_release_command(RestoreReferenceReleaseCommand { + operation, + operation_version, + job, + job_version: *job_version, + entries: &pending, + defer_gc_for: Some(reserve_guard.object_digest), + deferred_guard: Some(&reserve_guard), + object_reference: ObjectReferenceMutation::from_version(bounds_version), + version: bounds_version, + })?; + let bounds = + validate_restore_command_bounds(&bounds_command, "restore release reference batch") + .and_then(|()| { + validate_restore_command_bounds( + &deferred_bounds_command, + "restore release reference batch with continuation proof", + ) + }); + match bounds { + Ok(()) => {} + Err(MetadError::RestoreResourceLimit { .. }) => { + let continuation = pending + .pop() + .expect("the candidate release entry was just appended"); + job.cursor = previous_cursor; + if pending.is_empty() { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + row, + "restore release reference exceeds the command byte budget", + keyed_object_digest, + )?; + cursor_dirty = false; + continue; + } + let defer_gc_for = pending + .last() + .is_some_and(|entry| entry.object_digest == object_digest) + .then_some(object_digest); + let deferred_guard = + defer_gc_for.map(|_| RestoreReferenceReleaseGuard::from(&continuation)); + self.commit_restore_reference_release_batch( + operation, + operation_version, + job, + job_version, + &mut pending, + defer_gc_for, + deferred_guard.as_ref(), + )?; + cursor_dirty = false; + processed_all_rows = false; + break; + } + Err(error) => return Err(error), + } + } + let mut deferred_guard = None; + if processed_all_rows && scanned_rows.len() > RESTORE_BATCH_ENTRIES { + let lookahead = &scanned_rows[RESTORE_BATCH_ENTRIES]; + match self.restore_reference_release_guard_for_row( + operation, + lookahead, + read_version, + )? { + RestoreReferenceReleaseGuardValidation::Valid(guard) => { + let continuation_needed = pending + .last() + .is_some_and(|entry| entry.object_digest == guard.object_digest); + if continuation_needed + && pending.iter().any(|entry| { + entry.reference.object_key == guard.reference.object_key + && (entry.identity != guard.identity + || entry.reference.size != guard.reference.size + || entry.reference.digest_uri != guard.reference.digest_uri) + }) + { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + lookahead, + "restore release continuation disagrees on canonical object identity", + Some(guard.object_digest), + )?; + cursor_dirty = false; + processed_all_rows = false; + } else if continuation_needed { + deferred_guard = Some(guard); + } + } + RestoreReferenceReleaseGuardValidation::Corrupt { + reason, + object_digest, + } => { + self.quarantine_restore_release_object_row_and_advance( + operation, + operation_version, + job, + job_version, + &mut pending, + lookahead, + &reason, + object_digest, + )?; + cursor_dirty = false; + processed_all_rows = false; + } + } + } + if !pending.is_empty() { + let defer_gc_for = deferred_guard.as_ref().map(|guard| guard.object_digest); + self.commit_restore_reference_release_batch( + operation, + operation_version, + job, + job_version, + &mut pending, + defer_gc_for, + deferred_guard.as_deref(), + )?; + cursor_dirty = false; + } + let any_owner = self.restore_ref_set_has_base_owners(operation.ref_set_id)?; + if !any_owner || processed_all_rows && rows.len() < RESTORE_BATCH_ENTRIES { + // A reachable escaped borrower may keep a small set of exact + // references indefinitely. Once one full owner pass reaches its + // tail, let the member worker reclaim unrelated detached members + // (including initialization artifacts), then return here for the + // next reachability pass. This prevents one live borrower from + // starving the rest of the release job. + job.phase = RestoreReleasePhase::Members; + job.cursor.clear(); + cursor_dirty = true; + } + if cursor_dirty { + self.update_restore_release_job(operation, operation_version, job, job_version)?; + } + Ok(()) + } + + fn restore_ref_set_has_base_owners(&self, ref_set_id: u64) -> Result { + let version = self.read_version()?; + for prefix in [ + restore_base_owner_prefix(self.mount, ref_set_id), + restore_base_inverse_owner_prefix(self.mount, ref_set_id), + ] { + if !self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix, + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })? + .is_empty() + { + return Ok(true); + } + } + Ok(false) + } + + fn release_restore_member_page( + &self, + operation: &RestoreOperation, + operation_version: Version, + job: &mut RestoreReleaseJob, + job_version: &mut Version, + ) -> Result<(), MetadError> { + let prefix = restore_staging_member_prefix(self.mount, operation.ref_set_id); + let read_version = self.read_version()?; + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: (!job.cursor.is_empty()).then_some(job.cursor.clone()), + version: read_version, + limit: RESTORE_BATCH_ENTRIES, + purpose: ReadPurpose::RestoreStaging, + })?; + let candidate_inodes = rows + .iter() + .filter_map(|row| { + let member = decode_restore_staging_member(&row.value.0).ok()?; + (member.operation_digest == operation.operation_digest + && restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ) == row.key) + .then_some(member.destination_inode) + }) + .collect::>(); + let reachable_inodes = self.restore_reachable_inodes_at(&candidate_inodes, read_version)?; + for row in &rows { + let member = match decode_restore_staging_member(&row.value.0) { + Ok(member) + if member.operation_digest == operation.operation_digest + && restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ) == row.key => + { + member + } + Ok(_) => { + self.quarantine_restore_release_row_and_advance( + operation, + operation_version, + job, + job_version, + row, + "restore release member row changed identity", + )?; + continue; + } + Err(error) => { + self.quarantine_restore_release_row_and_advance( + operation, + operation_version, + job, + job_version, + row, + &error.to_string(), + )?; + continue; + } + }; + let inverse_key = restore_staging_inode_key(self.mount, member.destination_inode); + let inverse = self.metadata.get_versioned( + RecordFamily::System, + &inverse_key, + read_version, + ReadPurpose::RestoreStaging, + )?; + let Some(inverse) = inverse else { + self.quarantine_restore_release_row_and_advance( + operation, + operation_version, + job, + job_version, + row, + "restore release member has no staging inverse", + )?; + continue; + }; + let inverse_identity = decode_restore_staging_inverse(&inverse.value.0); + if !matches!( + inverse_identity, + Ok((digest, ref_set_id)) + if digest == operation.operation_digest + && ref_set_id == operation.ref_set_id + ) { + let reason = inverse_identity.err().map_or_else( + || "restore release staging inverse changed identity".to_owned(), + |error| error.to_string(), + ); + self.quarantine_restore_release_row_and_advance( + operation, + operation_version, + job, + job_version, + row, + &reason, + )?; + continue; + } + let inverse_owner_key = restore_staging_inverse_owner_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ); + let inverse_owner = self.metadata.get_versioned( + RecordFamily::System, + &inverse_owner_key, + read_version, + ReadPurpose::RestoreStaging, + )?; + let Some(inverse_owner) = inverse_owner else { + self.quarantine_restore_release_row_and_advance( + operation, + operation_version, + job, + job_version, + row, + "restore release member has no ref-set inverse owner", + )?; + continue; + }; + if inverse_owner.value != inverse.value { + self.quarantine_restore_release_row_and_advance( + operation, + operation_version, + job, + job_version, + row, + "restore release staging inverse owner changed identity", + )?; + continue; + } + let reachable = reachable_inodes.contains(&member.destination_inode); + if !reachable { + if self.release_restore_directory_children( + operation, + operation_version, + job, + job_version, + &member, + row.version, + &inverse_key, + inverse.version, + &inverse_owner_key, + inverse_owner.version, + )? { + return Ok(()); + } + if self.release_restore_member_canonical_index_page( + operation, + operation_version, + job, + *job_version, + &member, + row.version, + &inverse_key, + inverse.version, + )? { + return Ok(()); + } + if self.release_restore_member_manifest_page( + operation, + operation_version, + job, + *job_version, + &member, + row.version, + &inverse_key, + inverse.version, + )? { + return Ok(()); + } + if self.release_restore_member_metadata_page( + operation, + operation_version, + job, + *job_version, + &member, + row.version, + &inverse_key, + inverse.version, + RecordFamily::Xattr, + xattr_prefix(self.mount, member.destination_inode), + b"restore-release-member-xattrs", + )? { + return Ok(()); + } + if self.release_restore_member_dentry_page( + operation, + operation_version, + job, + *job_version, + &member, + row.version, + &inverse_key, + inverse.version, + )? { + return Ok(()); + } + } + if let Err(error) = self.finish_restore_release_member( + operation, + operation_version, + job, + job_version, + &member, + row.version, + &inverse_key, + inverse.version, + &inverse_owner_key, + inverse_owner.version, + reachable, + ) { + if matches!( + &error, + MetadError::Codec(_) + | MetadError::MissingBodyDescriptor + | MetadError::RestoreResourceLimit { .. } + ) { + self.quarantine_restore_release_row_and_advance( + operation, + operation_version, + job, + job_version, + row, + &error.to_string(), + )?; + continue; + } + return Err(error); + } + } + let any_member = !self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix, + start_after: None, + version: self.read_version()?, + limit: 1, + purpose: ReadPurpose::RestoreStaging, + })? + .is_empty() + || !self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_staging_inverse_owner_prefix(self.mount, operation.ref_set_id), + start_after: None, + version: self.read_version()?, + limit: 1, + purpose: ReadPurpose::RestoreStaging, + })? + .is_empty(); + let any_owner = self.restore_ref_set_has_base_owners(operation.ref_set_id)?; + if !any_member { + if any_owner { + return Err(MetadError::Codec( + "restore release base references have no staging members".to_owned(), + )); + } + job.phase = RestoreReleasePhase::Overlay; + job.cursor.clear(); + self.transition_restore_release_to_overlay( + operation, + operation_version, + job, + job_version, + )?; + } else if rows.len() < RESTORE_BATCH_ENTRIES { + if any_owner { + job.phase = RestoreReleasePhase::ExactReferences; + } + job.cursor.clear(); + self.update_restore_release_job(operation, operation_version, job, job_version)?; + } + Ok(()) + } + + /// Persistently discover one page below a detached directory before its + /// inode can be reclaimed. Each child is enrolled in the same ref-set in + /// the command that advances the release cursor. Advancing past the parent + /// prevents a low-id directory from starving newly discovered children; + /// the ordinary end-of-keyspace wrap revisits it after the children drain. + #[allow(clippy::too_many_arguments)] + fn release_restore_directory_children( + &self, + operation: &RestoreOperation, + operation_version: Version, + job: &mut RestoreReleaseJob, + job_version: &mut Version, + member: &RestoreStagingMember, + member_version: Version, + inverse_key: &[u8], + inverse_version: Version, + inverse_owner_key: &[u8], + inverse_owner_version: Version, + ) -> Result { + let read_version = self.read_version()?; + let inode_key = inode_key(self.mount, member.destination_inode); + let Some(inode) = self.metadata.get_versioned( + RecordFamily::Inode, + &inode_key, + read_version, + ReadPurpose::RestoreStaging, + )? + else { + return Ok(false); + }; + let attr = decode_inode_attr(&inode.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if attr.inode != member.destination_inode || attr.file_type != FileType::Directory { + return Ok(false); + } + let prefix = dentry_prefix(self.mount, member.destination_inode); + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::Dentry, + prefix: prefix.clone(), + start_after: None, + version: read_version, + limit: RESTORE_BATCH_ENTRIES, + purpose: ReadPurpose::RestoreStaging, + })?; + if rows.is_empty() { + return Ok(false); + } + let mut projections = Vec::with_capacity(rows.len()); + for row in &rows { + let projection = decode_dentry_projection(&row.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if projection.dentry.parent != member.destination_inode + || dentry_key( + self.mount, + projection.dentry.parent, + &projection.dentry.name, + ) != row.key + { + return Err(MetadError::Codec( + "restore release child dentry changed identity".to_owned(), + )); + } + let child_inverse_key = restore_staging_inode_key(self.mount, projection.attr.inode); + if let Some(child_inverse) = self.metadata.get_versioned( + RecordFamily::System, + &child_inverse_key, + read_version, + ReadPurpose::RestoreStaging, + )? { + let (child_digest, child_ref_set_id) = + decode_restore_staging_inverse(&child_inverse.value.0)?; + if child_digest != operation.operation_digest + || child_ref_set_id != operation.ref_set_id + { + return self.cascade_nested_restore_release( + operation, + operation_version, + job, + job_version, + member, + member_version, + inverse_key, + inverse_version, + inverse_owner_key, + inverse_owner_version, + &inode_key, + inode.version, + row, + &projection, + &child_inverse_key, + child_inverse, + read_version, + ); + } + } + projections.push(projection); + } + let enrollment = self.restore_namespace_enrollment_plan_for_operation( + operation, + operation_version, + &projections, + read_version, + )?; + job.cursor = + restore_staging_member_key(self.mount, operation.ref_set_id, member.destination_inode); + let version = self.next_version()?; + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + predicate: Predicate::VersionEquals(*job_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ), + predicate: Predicate::VersionEquals(member_version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.to_vec(), + predicate: Predicate::VersionEquals(inverse_version), + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode_key, + predicate: Predicate::VersionEquals(inode.version), + }, + ]; + predicates.extend(enrollment.predicates); + predicates.extend(rows.iter().map(|row| PredicateRef { + family: RecordFamily::Dentry, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + })); + let mut mutations = enrollment.mutations; + mutations.push(Mutation { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + op: MutationOp::Put, + value: Some(Value(encode_restore_release_job(job)?)), + }); + let command = MetadataCommand { + request_id: request_id( + b"restore-release-discover-children", + self.mount, + member.destination_inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: prefix, + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore release child discovery")?; + self.commit_metadata(command)?; + *job_version = version; + Ok(true) + } + + /// Detach one nested restore root from an already-detached outer restore. + /// The child operation owns its inode/ref-set, so the outer worker must not + /// enrol or reclaim it. Instead, one command starts (or joins) the child's + /// release, removes the outer parent dentry, and advances the outer cursor. + #[allow(clippy::too_many_arguments)] + fn cascade_nested_restore_release( + &self, + operation: &RestoreOperation, + operation_version: Version, + job: &mut RestoreReleaseJob, + job_version: &mut Version, + member: &RestoreStagingMember, + member_version: Version, + inverse_key: &[u8], + inverse_version: Version, + inverse_owner_key: &[u8], + inverse_owner_version: Version, + parent_inode_key: &[u8], + parent_inode_version: Version, + dentry_row: &crate::command::ScanItem, + projection: &DentryProjection, + child_inverse_key: &[u8], + child_inverse: ReadItem, + read_version: Version, + ) -> Result { + if projection.attr.file_type != FileType::Directory + || projection.attr.inode.shard_index() != self.shard_index() + { + return Err(MetadError::Codec( + "restore release found a foreign ref-set on a non-local directory".to_owned(), + )); + } + let child_root = projection.attr.inode; + let (child_digest, child_ref_set_id) = + decode_restore_staging_inverse(&child_inverse.value.0)?; + if child_digest == operation.operation_digest + || child_ref_set_id == operation.ref_set_id + || child_ref_set_id == 0 + { + return Err(MetadError::Codec( + "restore release nested inverse has an invalid owner".to_owned(), + )); + } + + let child_operation_key = restore_operation_key(self.mount, &child_digest); + let child_operation_item = self + .metadata + .get_versioned( + RecordFamily::System, + &child_operation_key, + read_version, + ReadPurpose::RestoreStaging, + )? + .ok_or_else(|| MetadError::Codec("nested restore root has no operation".to_owned()))?; + let mut child_operation = decode_restore_operation(&child_operation_item.value.0)?; + if child_operation.operation_digest != child_digest + || child_operation.ref_set_id != child_ref_set_id + || child_operation.destination_root != child_root + { + return Err(MetadError::Codec( + "nested restore operation changed identity".to_owned(), + )); + } + + let child_root_key = restore_root_index_key(self.mount, child_root); + let child_root_item = self + .metadata + .get_versioned( + RecordFamily::System, + &child_root_key, + read_version, + ReadPurpose::RestoreStaging, + )? + .ok_or_else(|| { + MetadError::Codec("nested restore operation has no root index".to_owned()) + })?; + if child_root_item.value.0 != child_digest { + return Err(MetadError::Codec( + "nested restore root index changed identity".to_owned(), + )); + } + + let child_member_key = restore_staging_member_key(self.mount, child_ref_set_id, child_root); + let child_member_item = self + .metadata + .get_versioned( + RecordFamily::System, + &child_member_key, + read_version, + ReadPurpose::RestoreStaging, + )? + .ok_or_else(|| { + MetadError::Codec("nested restore root has no staging member".to_owned()) + })?; + let child_member = decode_restore_staging_member(&child_member_item.value.0)?; + if child_member.operation_digest != child_digest + || child_member.destination_inode != child_root + || child_member.destination_parent.is_some() + || child_member.name.is_some() + { + return Err(MetadError::Codec( + "nested restore root member changed identity".to_owned(), + )); + } + let child_inverse_owner_key = + restore_staging_inverse_owner_key(self.mount, child_ref_set_id, child_root); + let child_inverse_owner = self + .metadata + .get_versioned( + RecordFamily::System, + &child_inverse_owner_key, + read_version, + ReadPurpose::RestoreStaging, + )? + .ok_or_else(|| { + MetadError::Codec("nested restore root has no inverse owner".to_owned()) + })?; + if child_inverse_owner.value != child_inverse.value { + return Err(MetadError::Codec( + "nested restore root inverse owner changed identity".to_owned(), + )); + } + + let child_release_key = restore_release_job_key(self.mount, child_ref_set_id); + let child_release = self.metadata.get_versioned( + RecordFamily::System, + &child_release_key, + read_version, + ReadPurpose::RestoreStaging, + )?; + let mut predicates = vec![ + self.begin_object_reference_mutation()? + .predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + predicate: Predicate::VersionEquals(*job_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ), + predicate: Predicate::VersionEquals(member_version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.to_vec(), + predicate: Predicate::VersionEquals(inverse_version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_owner_key.to_vec(), + predicate: Predicate::VersionEquals(inverse_owner_version), + }, + PredicateRef { + family: RecordFamily::Inode, + key: parent_inode_key.to_vec(), + predicate: Predicate::VersionEquals(parent_inode_version), + }, + PredicateRef { + family: RecordFamily::Dentry, + key: dentry_row.key.clone(), + predicate: Predicate::VersionEquals(dentry_row.version), + }, + PredicateRef { + family: RecordFamily::System, + key: child_operation_key.clone(), + predicate: Predicate::VersionEquals(child_operation_item.version), + }, + PredicateRef { + family: RecordFamily::System, + key: child_root_key, + predicate: Predicate::VersionEquals(child_root_item.version), + }, + PredicateRef { + family: RecordFamily::System, + key: child_member_key, + predicate: Predicate::VersionEquals(child_member_item.version), + }, + PredicateRef { + family: RecordFamily::System, + key: child_inverse_key.to_vec(), + predicate: Predicate::VersionEquals(child_inverse.version), + }, + PredicateRef { + family: RecordFamily::System, + key: child_inverse_owner_key, + predicate: Predicate::VersionEquals(child_inverse_owner.version), + }, + ]; + + job.cursor = + restore_staging_member_key(self.mount, operation.ref_set_id, member.destination_inode); + let mut mutations = vec![ + delete_mutation(RecordFamily::Dentry, dentry_row.key.clone()), + Mutation { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + op: MutationOp::Put, + value: Some(Value(encode_restore_release_job(job)?)), + }, + ]; + match child_operation.state { + RestoreOperationState::Complete => { + if child_release.is_some() { + return Err(MetadError::Codec( + "complete nested restore already has a release job".to_owned(), + )); + } + predicates.push(PredicateRef { + family: RecordFamily::System, + key: child_release_key.clone(), + predicate: Predicate::NotExists, + }); + child_operation.state = RestoreOperationState::Releasing; + let child_job = RestoreReleaseJob { + operation_digest: child_digest, + ref_set_id: child_ref_set_id, + phase: RestoreReleasePhase::ExactReferences, + cursor: Vec::new(), + }; + mutations.extend([ + Mutation { + family: RecordFamily::System, + key: child_operation_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_operation(&child_operation)?)), + }, + Mutation { + family: RecordFamily::System, + key: child_release_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_release_job(&child_job)?)), + }, + ]); + } + RestoreOperationState::Releasing => { + let child_release = child_release.ok_or_else(|| { + MetadError::Codec("releasing nested restore has no release job".to_owned()) + })?; + let child_job = decode_restore_release_job(&child_release.value.0)?; + if child_job.operation_digest != child_digest + || child_job.ref_set_id != child_ref_set_id + { + return Err(MetadError::Codec( + "nested restore release job changed identity".to_owned(), + )); + } + predicates.push(PredicateRef { + family: RecordFamily::System, + key: child_release_key, + predicate: Predicate::VersionEquals(child_release.version), + }); + } + RestoreOperationState::Preparing + | RestoreOperationState::ReadyToAttach + | RestoreOperationState::Cleaning + | RestoreOperationState::Discarding => { + return Err(MetadError::Codec( + "nested restore root is attached in a non-visible state".to_owned(), + )); + } + } + + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-release-cascade-nested", + self.mount, + child_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: dentry_row.key.clone(), + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore nested release cascade")?; + self.commit_metadata(command)?; + *job_version = version; + Ok(true) + } + + /// Drain canonical PathIndex rows for a post-attach member without an + /// unbounded mount scan. The cursor lives in the member row itself, so a + /// crash or owner failover resumes after the last physical key. Ordinary + /// writes cannot create another link to an unreachable Releasing inode. + #[allow(clippy::too_many_arguments)] + fn release_restore_member_canonical_index_page( + &self, + operation: &RestoreOperation, + operation_version: Version, + _job: &RestoreReleaseJob, + job_version: Version, + member: &RestoreStagingMember, + member_version: Version, + inverse_key: &[u8], + inverse_version: Version, + ) -> Result { + if member.canonical_index_complete { + return Ok(false); + } + let read_version = self.read_version()?; + let prefix = path_index_prefix(self.mount, &[]); + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::PathIndex, + prefix: prefix.clone(), + start_after: (!member.canonical_index_cursor.is_empty()) + .then(|| member.canonical_index_cursor.clone()), + version: read_version, + limit: RESTORE_BATCH_ENTRIES, + purpose: ReadPurpose::RestoreStaging, + })?; + let mut next_member = member.clone(); + if rows.len() < RESTORE_BATCH_ENTRIES { + next_member.canonical_index_complete = true; + next_member.canonical_index_cursor.clear(); + } else { + next_member.canonical_index_cursor = rows + .last() + .expect("a full restore index page has a last row") + .key + .clone(); + } + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + predicate: Predicate::VersionEquals(job_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ), + predicate: Predicate::VersionEquals(member_version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.to_vec(), + predicate: Predicate::VersionEquals(inverse_version), + }, + ]; + let mut mutations = vec![Mutation { + family: RecordFamily::System, + key: restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ), + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_member(&next_member)?)), + }]; + for row in rows { + if !row.key.starts_with(&prefix) { + return Err(MetadError::Codec( + "restore release canonical index scan escaped its mount".to_owned(), + )); + } + let projection = decode_dentry_projection(&row.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if projection.attr.inode != member.destination_inode { + continue; + } + predicates.push(PredicateRef { + family: RecordFamily::PathIndex, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }); + mutations.push(delete_mutation(RecordFamily::PathIndex, row.key)); + } + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-release-member-canonical-index", + self.mount, + member.destination_inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::PathIndex, + primary_key: prefix, + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore release canonical path index")?; + self.commit_metadata(command)?; + Ok(true) + } + + #[allow(clippy::too_many_arguments)] + pub(super) fn restore_owned_manifest_blocks( + &self, + destination_inode: InodeId, + chunk_index: u64, + manifest: &ChunkManifest, + ) -> Result, MetadError> { + if manifest.chunk_index != chunk_index { + return Err(MetadError::Codec( + "restore release chunk manifest changed identity".to_owned(), + )); + } + let mut blocks = BTreeMap::<(u64, u64), BlockDescriptor>::new(); + for block in manifest.slices.iter().flat_map(|slice| slice.blocks.iter()) { + if !self.block_object_is_owned_by_inode(destination_inode, &block.object_key)? { + // Inherited manifests retain the source object's canonical key. + // The source owner is responsible for enqueueing that object; + // this restore only releases its exact borrower reference. + continue; + } + let (object_owner, object_generation, object_chunk, block_index) = + self.canonical_block_object_identity(&block.object_key)?; + if object_owner != destination_inode || object_chunk != chunk_index { + return Err(MetadError::Codec( + "restore release owned object changed manifest identity".to_owned(), + )); + } + match blocks.entry((object_generation, block_index)) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(block.clone()); + } + std::collections::btree_map::Entry::Occupied(entry) + if entry.get().object_key == block.object_key + && entry.get().digest_uri == block.digest_uri => {} + std::collections::btree_map::Entry::Occupied(_) => { + return Err(MetadError::Codec( + "restore release manifest has conflicting owned block identities" + .to_owned(), + )); + } + } + } + Ok(blocks.into_values().collect()) + } + + #[allow(clippy::too_many_arguments)] + fn release_restore_member_manifest_page( + &self, + operation: &RestoreOperation, + operation_version: Version, + _job: &RestoreReleaseJob, + job_version: Version, + member: &RestoreStagingMember, + member_version: Version, + inverse_key: &[u8], + inverse_version: Version, + ) -> Result { + let prefix = inode_key(self.mount, member.destination_inode); + let read_version = self.read_version()?; + let row = if member.manifest_cursor.is_empty() { + self.metadata + .scan(ScanRequest { + family: RecordFamily::ChunkManifest, + prefix: prefix.clone(), + start_after: None, + version: read_version, + limit: 1, + purpose: ReadPurpose::RestoreStaging, + })? + .into_iter() + .next() + } else { + if member.manifest_cursor.len() != prefix.len() + 16 + || !member.manifest_cursor.starts_with(&prefix) + { + return Err(MetadError::Codec( + "restore release manifest cursor has an invalid shape".to_owned(), + )); + } + self.metadata + .get_versioned( + RecordFamily::ChunkManifest, + &member.manifest_cursor, + read_version, + ReadPurpose::RestoreStaging, + )? + .map(|item| ScanItem { + key: member.manifest_cursor.clone(), + value: item.value, + version: item.version, + }) + }; + let Some(row) = row else { + if member.manifest_cursor.is_empty() { + return Ok(false); + } + return Err(MetadError::Codec( + "restore release manifest cursor points to a missing row".to_owned(), + )); + }; + if row.key.len() != prefix.len() + 16 || !row.key.starts_with(&prefix) { + return Err(MetadError::Codec( + "restore release chunk-manifest key has an invalid shape".to_owned(), + )); + } + let generation = u64::from_be_bytes( + row.key[prefix.len()..prefix.len() + 8] + .try_into() + .expect("validated generation width"), + ); + let chunk_index = u64::from_be_bytes( + row.key[prefix.len() + 8..] + .try_into() + .expect("validated chunk-index width"), + ); + if generation == 0 { + return Err(MetadError::Codec( + "restore release chunk manifest has a zero generation".to_owned(), + )); + } + + let release_blocks = if chunk_index == BODY_SUMMARY_CHUNK_INDEX { + if !member.manifest_cursor.is_empty() { + return Err(MetadError::Codec( + "restore release body-summary row retains a block cursor".to_owned(), + )); + } + let body = decode_body_descriptor(&row.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if body.generation != generation { + return Err(MetadError::Codec( + "restore release body summary generation changed identity".to_owned(), + )); + } + Vec::new() + } else { + let manifest = decode_chunk_manifest(&row.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + self.restore_owned_manifest_blocks(member.destination_inode, chunk_index, &manifest)? + }; + let block_cursor = usize::try_from(member.manifest_block_cursor).map_err(|_| { + MetadError::Codec("restore release manifest block cursor overflow".to_owned()) + })?; + if block_cursor > release_blocks.len() + || (!member.manifest_cursor.is_empty() && block_cursor == release_blocks.len()) + { + return Err(MetadError::Codec( + "restore release manifest block cursor is out of range".to_owned(), + )); + } + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + predicate: Predicate::VersionEquals(job_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ), + predicate: Predicate::VersionEquals(member_version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.to_vec(), + predicate: Predicate::VersionEquals(inverse_version), + }, + PredicateRef { + family: RecordFamily::ChunkManifest, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + ]; + let mut next_member = member.clone(); + next_member.manifest_cursor = row.key.clone(); + next_member.manifest_block_cursor = member.manifest_block_cursor.saturating_add(1); + let member_key = + restore_staging_member_key(self.mount, operation.ref_set_id, member.destination_inode); + // Reserve both the durable cursor update and the eventual manifest + // deletion while packing GC rows. A partial page removes the delete, + // so every committed command remains within both advertised bounds. + let mutations = vec![ + Mutation { + family: RecordFamily::System, + key: member_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_member(&next_member)?)), + }, + delete_mutation(RecordFamily::ChunkManifest, row.key.clone()), + ]; + let mut command = MetadataCommand { + request_id: request_id( + b"restore-release-member-manifests", + self.mount, + member.destination_inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::ChunkManifest, + primary_key: prefix, + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore release member manifests")?; + + let mut accepted = 0_usize; + for block in release_blocks.iter().skip(block_cursor) { + let (object_owner, object_generation, object_chunk, block_index) = + self.canonical_block_object_identity(&block.object_key)?; + let record = ObjectGcRecord { + inode: object_owner, + generation: object_generation, + object_key: block.object_key.clone(), + size: block.len, + digest_uri: block.digest_uri.clone(), + enqueue_version: version.get(), + enqueue_unix_ms: current_time_ms(), + }; + command.mutations.push(Mutation { + family: RecordFamily::Gc, + key: gc_object_key( + self.mount, + version.get(), + object_owner, + object_generation, + object_chunk, + block_index, + ), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_record(&record))), + }); + match validate_restore_command_bounds(&command, "restore release member manifests") { + Ok(()) => accepted += 1, + Err(error) => { + command.mutations.pop(); + if accepted == 0 { + return Err(error); + } + break; + } + } + } + + let next_block_cursor = block_cursor.checked_add(accepted).ok_or_else(|| { + MetadError::Codec("restore release manifest block cursor overflow".to_owned()) + })?; + if next_block_cursor == release_blocks.len() { + next_member.manifest_cursor.clear(); + next_member.manifest_block_cursor = 0; + } else { + next_member.manifest_cursor = row.key.clone(); + next_member.manifest_block_cursor = u64::try_from(next_block_cursor).map_err(|_| { + MetadError::Codec("restore release manifest block cursor overflow".to_owned()) + })?; + command.mutations.remove(1); + } + let encoded_next_member = encode_restore_staging_member(&next_member)?; + command.mutations[0].value = Some(Value(encoded_next_member.clone())); + validate_restore_command_bounds(&command, "restore release member manifests")?; + match self.commit_metadata(command) { + Ok(_) => {} + Err(error @ MetadError::Metadata(MetadataError::Backend(_))) => { + // A backend error may be a lost acknowledgement after the + // atomic cursor+GC apply. Reconcile from the durable member and + // manifest visibility instead of replaying or quarantining a + // healthy release job. The member CAS is the transaction's + // commit witness, so observing it also proves every GC row in + // this page landed exactly once. + let read_version = self.read_version()?; + let member_matches = self + .metadata + .get( + RecordFamily::System, + &restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ), + read_version, + ReadPurpose::RestoreStaging, + )? + .is_some_and(|value| value.0 == encoded_next_member); + let manifest_exists = self + .metadata + .get( + RecordFamily::ChunkManifest, + &row.key, + read_version, + ReadPurpose::RestoreStaging, + )? + .is_some(); + let manifest_complete = next_block_cursor == release_blocks.len(); + if !member_matches || manifest_exists == manifest_complete { + return Err(error); + } + } + Err(error) => return Err(error), + } + Ok(true) + } + + #[allow(clippy::too_many_arguments)] + fn release_restore_member_metadata_page( + &self, + operation: &RestoreOperation, + operation_version: Version, + _job: &RestoreReleaseJob, + job_version: Version, + member: &RestoreStagingMember, + member_version: Version, + inverse_key: &[u8], + inverse_version: Version, + family: RecordFamily, + prefix: Vec, + request_prefix: &[u8], + ) -> Result { + let rows = self.metadata.scan(ScanRequest { + family, + prefix: prefix.clone(), + start_after: None, + version: self.read_version()?, + limit: RESTORE_BATCH_ENTRIES, + purpose: ReadPurpose::RestoreStaging, + })?; + if rows.is_empty() { + return Ok(false); + } + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + request_prefix, + self.mount, + member.destination_inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: family, + primary_key: prefix, + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + predicate: Predicate::VersionEquals(job_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ), + predicate: Predicate::VersionEquals(member_version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.to_vec(), + predicate: Predicate::VersionEquals(inverse_version), + }, + ] + .into_iter() + .chain(rows.iter().map(|row| PredicateRef { + family, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + })) + .collect(), + mutations: rows + .into_iter() + .map(|row| delete_mutation(family, row.key)) + .collect(), + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore release member metadata")?; + self.commit_metadata(command)?; + Ok(true) + } + + #[allow(clippy::too_many_arguments)] + fn release_restore_member_dentry_page( + &self, + operation: &RestoreOperation, + operation_version: Version, + _job: &RestoreReleaseJob, + job_version: Version, + member: &RestoreStagingMember, + member_version: Version, + inverse_key: &[u8], + inverse_version: Version, + ) -> Result { + let version = self.read_version()?; + let rows = self.restore_linked_dentry_projection_page( + member.destination_inode, + version, + RESTORE_BATCH_ENTRIES, + )?; + if rows.is_empty() { + return Ok(false); + } + let commit_version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-release-member-dentries", + self.mount, + member.destination_inode, + commit_version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(commit_version)?, + commit_version, + primary_family: RecordFamily::Dentry, + primary_key: rows[0].key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + predicate: Predicate::VersionEquals(job_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ), + predicate: Predicate::VersionEquals(member_version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.to_vec(), + predicate: Predicate::VersionEquals(inverse_version), + }, + ] + .into_iter() + .chain(rows.iter().map(|linked| PredicateRef { + family: RecordFamily::Dentry, + key: linked.key.clone(), + predicate: Predicate::VersionEquals(linked.version), + })) + .collect(), + mutations: rows + .into_iter() + .map(|linked| delete_mutation(RecordFamily::Dentry, linked.key)) + .collect(), + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore release member dentries")?; + self.commit_metadata(command)?; + Ok(true) + } + + #[allow(clippy::too_many_arguments)] + fn finish_restore_release_member( + &self, + operation: &RestoreOperation, + operation_version: Version, + job: &mut RestoreReleaseJob, + job_version: &mut Version, + member: &RestoreStagingMember, + member_version: Version, + inverse_key: &[u8], + inverse_version: Version, + inverse_owner_key: &[u8], + inverse_owner_version: Version, + reachable: bool, + ) -> Result<(), MetadError> { + if !member.manifest_cursor.is_empty() || member.manifest_block_cursor != 0 { + return Err(MetadError::Codec( + "restore release cannot finish a member with an active manifest cursor".to_owned(), + )); + } + if self.restore_ref_set_has_base_owners(operation.ref_set_id)? { + // Exact references are released in a separate paged pass. The + // current member may already be detached and fully cleaned, but + // keep its durable identity until every exact row is gone so fsck + // can still validate borrower ownership. Advancing the cursor lets + // other members make progress before the next exact-reference + // sweep. + job.cursor = restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ); + let retained_inode_key = inode_key(self.mount, member.destination_inode); + let read_version = self.read_version()?; + let retained_inode = if reachable { + None + } else { + self.metadata.get_versioned( + RecordFamily::Inode, + &retained_inode_key, + read_version, + ReadPurpose::RestoreStaging, + )? + }; + let version = self.next_version()?; + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + predicate: Predicate::VersionEquals(*job_version), + }, + PredicateRef { + family: RecordFamily::System, + key: job.cursor.clone(), + predicate: Predicate::VersionEquals(member_version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.to_vec(), + predicate: Predicate::VersionEquals(inverse_version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_owner_key.to_vec(), + predicate: Predicate::VersionEquals(inverse_owner_version), + }, + ]; + let mut mutations = vec![Mutation { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + op: MutationOp::Put, + value: Some(Value(encode_restore_release_job(job)?)), + }]; + if !reachable { + predicates.extend([ + PredicateRef { + family: RecordFamily::ChunkManifest, + key: retained_inode_key.clone(), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::Xattr, + key: xattr_prefix(self.mount, member.destination_inode), + predicate: Predicate::PrefixEmpty, + }, + ]); + match retained_inode { + Some(inode) => { + let attr = decode_inode_attr(&inode.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if attr.inode != member.destination_inode { + return Err(MetadError::Codec( + "restore retained member inode changed identity".to_owned(), + )); + } + if attr.file_type == FileType::Directory { + predicates.push(PredicateRef { + family: RecordFamily::Dentry, + key: dentry_prefix(self.mount, member.destination_inode), + predicate: Predicate::PrefixEmpty, + }); + } + predicates.push(PredicateRef { + family: RecordFamily::Inode, + key: retained_inode_key.clone(), + predicate: Predicate::VersionEquals(inode.version), + }); + mutations.push(delete_mutation( + RecordFamily::Inode, + retained_inode_key.clone(), + )); + } + None => predicates.push(PredicateRef { + family: RecordFamily::Inode, + key: retained_inode_key, + predicate: Predicate::NotExists, + }), + } + } + let command = MetadataCommand { + request_id: request_id( + b"restore-retain-release-member", + self.mount, + member.destination_inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: job.cursor.clone(), + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore retained member cursor")?; + self.commit_metadata(command)?; + *job_version = version; + return Ok(()); + } + let read_version = self.read_version()?; + let member_inode_key = inode_key(self.mount, member.destination_inode); + let inode = self.metadata.get_versioned( + RecordFamily::Inode, + &member_inode_key, + read_version, + ReadPurpose::RestoreStaging, + )?; + let inode_attr = inode + .as_ref() + .map(|item| { + decode_inode_attr(&item.value.0) + .map_err(|error| MetadError::Codec(error.to_string())) + }) + .transpose()?; + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + job.cursor = + restore_staging_member_key(self.mount, operation.ref_set_id, member.destination_inode); + let member_key = job.cursor.clone(); + let mut predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + predicate: Predicate::VersionEquals(*job_version), + }, + PredicateRef { + family: RecordFamily::System, + key: member_key.clone(), + predicate: Predicate::VersionEquals(member_version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.to_vec(), + predicate: Predicate::VersionEquals(inverse_version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_owner_key.to_vec(), + predicate: Predicate::VersionEquals(inverse_owner_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_base_owner_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + ]; + let mut mutations = vec![Mutation { + family: RecordFamily::System, + key: restore_release_job_key(self.mount, operation.ref_set_id), + op: MutationOp::Put, + value: Some(Value(encode_restore_release_job(job)?)), + }]; + if !reachable { + mutations.extend([ + delete_mutation(RecordFamily::System, member_key), + delete_mutation(RecordFamily::System, inverse_key.to_vec()), + delete_mutation(RecordFamily::System, inverse_owner_key.to_vec()), + ]); + predicates.extend([ + PredicateRef { + family: RecordFamily::ChunkManifest, + key: inode_key(self.mount, member.destination_inode), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::Xattr, + key: xattr_prefix(self.mount, member.destination_inode), + predicate: Predicate::PrefixEmpty, + }, + ]); + if inode_attr + .as_ref() + .is_some_and(|attr| attr.file_type == FileType::Directory) + { + predicates.push(PredicateRef { + family: RecordFamily::Dentry, + key: dentry_prefix(self.mount, member.destination_inode), + predicate: Predicate::PrefixEmpty, + }); + } + if let Some(inode) = inode { + predicates.push(PredicateRef { + family: RecordFamily::Inode, + key: member_inode_key.clone(), + predicate: Predicate::VersionEquals(inode.version), + }); + mutations.push(delete_mutation(RecordFamily::Inode, member_inode_key)); + } else { + predicates.push(PredicateRef { + family: RecordFamily::Inode, + key: member_inode_key, + predicate: Predicate::NotExists, + }); + } + } + let command = MetadataCommand { + request_id: request_id( + b"restore-release-member", + self.mount, + member.destination_inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: restore_staging_member_prefix(self.mount, operation.ref_set_id), + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore release member")?; + self.commit_metadata(command)?; + *job_version = version; + Ok(()) + } + + fn finish_restore_release( + &self, + operation: &RestoreOperation, + operation_version: Version, + _job: &RestoreReleaseJob, + job_version: Version, + ) -> Result<(), MetadError> { + let read_version = self.read_version()?; + let operation_key = restore_operation_key(self.mount, &operation.operation_digest); + let release_key = restore_release_job_key(self.mount, operation.ref_set_id); + let claim_key = restore_destination_claim_key(self.mount, &operation.destination_path); + let claim = self + .metadata + .get_versioned( + RecordFamily::System, + &claim_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreRootChanged { + root: operation.destination_root, + })?; + let root_key = restore_root_index_key(self.mount, operation.destination_root); + let root = self + .metadata + .get_versioned( + RecordFamily::System, + &root_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreRootChanged { + root: operation.destination_root, + })?; + if claim.value.0 != operation.operation_digest || root.value.0 != operation.operation_digest + { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + let seal_key = restore_base_seal_key(self.mount, operation.ref_set_id); + let seal = self + .metadata + .get_versioned( + RecordFamily::System, + &seal_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore release base-reference seal is missing".to_owned()) + })?; + let decoded_seal = super::restore_gc::decode_restore_base_seal(&seal.value.0)?; + if decoded_seal.operation_digest != operation.operation_digest + || decoded_seal.initialization_digest != operation.initialization_digest + || decoded_seal.ref_set_id != operation.ref_set_id + || decoded_seal.incarnation != operation.created_version + { + return Err(MetadError::Codec( + "restore release base-reference seal changed identity".to_owned(), + )); + } + let cleanup_key = restore_cleanup_job_key(self.mount, operation.ref_set_id); + let cleanup = self.metadata.get_versioned( + RecordFamily::System, + &cleanup_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + if cleanup + .as_ref() + .is_some_and(|item| item.value.0 != operation.operation_digest) + { + return Err(MetadError::Codec( + "restore release cleanup marker changed identity".to_owned(), + )); + } + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: release_key.clone(), + predicate: Predicate::VersionEquals(job_version), + }, + PredicateRef { + family: RecordFamily::System, + key: claim_key.clone(), + predicate: Predicate::VersionEquals(claim.version), + }, + PredicateRef { + family: RecordFamily::System, + key: root_key.clone(), + predicate: Predicate::VersionEquals(root.version), + }, + PredicateRef { + family: RecordFamily::System, + key: seal_key.clone(), + predicate: Predicate::VersionEquals(seal.version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_base_owner_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_base_inverse_owner_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_member_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_inverse_owner_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_init_upload_intent_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + ]; + match &cleanup { + Some(item) => predicates.push(PredicateRef { + family: RecordFamily::System, + key: cleanup_key.clone(), + predicate: Predicate::VersionEquals(item.version), + }), + None => predicates.push(PredicateRef { + family: RecordFamily::System, + key: cleanup_key.clone(), + predicate: Predicate::NotExists, + }), + } + predicates.extend( + super::restore_index::restore_index_release_empty_predicates( + self.mount, + operation.ref_set_id, + ), + ); + let mut mutations = vec![ + delete_mutation(RecordFamily::System, operation_key.clone()), + delete_mutation(RecordFamily::System, release_key), + delete_mutation(RecordFamily::System, claim_key), + delete_mutation(RecordFamily::System, root_key), + delete_mutation(RecordFamily::System, seal_key), + ]; + if cleanup.is_some() { + mutations.push(delete_mutation(RecordFamily::System, cleanup_key)); + } + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-finish-release", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: operation_key, + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore finish release")?; + match self.commit_metadata(command) { + Ok(_) => { + // This is the only successful release boundary that may return + // the process to the fast path. Recovery proves globally that + // no detached restore state remains; malformed state stays + // fail closed. + self.recover_restore_staging_visibility()?; + Ok(()) + } + Err( + error @ MetadError::SyncLogArchiveFailed { + committed: true, .. + }, + ) => { + // Metadata is already durable even though its archive ACK was + // lost. There is no release-job row left to trigger another + // worker pass, so reconcile the hint at this durable boundary + // before preserving the caller-visible acknowledgement error. + let _ = self.recover_restore_staging_visibility(); + Err(error) + } + Err(error) => Err(error), + } + } + + /// Return true when any unexpired snapshot's pinned namespace contains an + /// inode owned by this restore ref-set. Checking staging inverses rather + /// than only `destination_root` also covers an escaped child that was + /// renamed or hard-linked outside the restored root before root deletion. + fn restore_live_snapshot_holds_ref_set( + &self, + operation: &RestoreOperation, + ) -> Result { + const RETENTION_CONTROL_PAGE: usize = 64; + + let current_version = self.read_version()?; + let now_ms = self.now_ms(); + let pin_prefix = snapshot_pin_prefix(self.mount); + let mut pin_start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::Snapshot, + prefix: pin_prefix.clone(), + start_after: pin_start_after.clone(), + version: current_version, + limit: RETENTION_CONTROL_PAGE, + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + break; + } + for row in &rows { + let pin = decode_snapshot_pin(&row.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if row.key != snapshot_pin_key(self.mount, pin.snapshot_id) + || pin.read_version == 0 + || pin.created_version != pin.read_version.saturating_add(1) + { + return Err(MetadError::Codec( + "restore release snapshot pin changed identity".to_owned(), + )); + } + self.ensure_snapshot_id_shard(pin.snapshot_id, pin.root)?; + if now_ms >= pin.lease_expires_unix_ms { + continue; + } + if self.restore_snapshot_subtree_holds_ref_set( + pin.root, + Version::new(pin.read_version)?, + current_version, + operation, + )? { + return Ok(true); + } + } + pin_start_after = rows.last().map(|row| row.key.clone()); + if rows.len() < RETENTION_CONTROL_PAGE { + break; + } + } + + if self.visit_versioned_fork_bindings_at( + current_version, + ReadPurpose::WritePlanLocal, + |versioned| { + // The restore's own temporary binding is removed atomically at + // attach and must never make staging a namespace anchor. A binding + // still carrying that exact fork-root identity here is corruption, + // not a generic holder to silently accept. + if versioned.binding.fork_root == operation.destination_root + && versioned.binding.source_root == operation.source_root + && versioned.binding.snapshot_id == operation.snapshot_id + { + return Err(MetadError::Codec( + "releasing restore still has its temporary ForkBinding".to_owned(), + )); + } + self.restore_snapshot_subtree_holds_ref_set( + versioned.binding.source_root, + Version::new(versioned.binding.pinned_read_version)?, + current_version, + operation, + ) + }, + )? { + return Ok(true); + } + Ok(false) + } + + fn restore_snapshot_subtree_holds_ref_set( + &self, + root: InodeId, + snapshot_version: Version, + current_version: Version, + operation: &RestoreOperation, + ) -> Result { + let mut pending = vec![root]; + let mut visited = HashSet::new(); + let mut entries = 0_usize; + while let Some(inode) = pending.pop() { + if !visited.insert(inode) { + continue; + } + if let Some(inverse) = self.metadata.get( + RecordFamily::System, + &restore_staging_inode_key(self.mount, inode), + current_version, + ReadPurpose::WritePlanLocal, + )? { + let (digest, ref_set_id) = decode_restore_staging_inverse(&inverse.0)?; + if digest == operation.operation_digest && ref_set_id == operation.ref_set_id { + return Ok(true); + } + } + let Some(attr) = self.get_attr_at_version_for_purpose( + inode, + snapshot_version, + ReadPurpose::Snapshot, + )? + else { + return Err(MetadError::Codec(format!( + "live snapshot retention inode {} is missing", + inode.get() + ))); + }; + if attr.file_type != FileType::Directory { + continue; + } + let mut after = None; + loop { + let page = self.read_dir_plus_page_at_version_for_purpose( + inode, + after.as_ref(), + RESTORE_BATCH_ENTRIES, + snapshot_version, + ReadPurpose::Snapshot, + )?; + entries = entries.saturating_add(page.entries.len()); + if entries > MAX_RESTORE_SUBTREE_ENTRIES { + return Err(MetadError::RestoreResourceLimit { + resource: "restore release snapshot retention entries".to_owned(), + limit: MAX_RESTORE_SUBTREE_ENTRIES as u64, + actual: entries as u64, + }); + } + for child in page.entries { + if let Some(inverse) = self.metadata.get( + RecordFamily::System, + &restore_staging_inode_key(self.mount, child.attr.inode), + current_version, + ReadPurpose::WritePlanLocal, + )? { + let (digest, ref_set_id) = decode_restore_staging_inverse(&inverse.0)?; + if digest == operation.operation_digest + && ref_set_id == operation.ref_set_id + { + return Ok(true); + } + } + if child.attr.file_type == FileType::Directory + && child.attr.inode.shard_index() == self.shard_index() + { + pending.push(child.attr.inode); + } + } + let Some(next) = page.next_cursor else { + break; + }; + after = Some(next); + } + } + Ok(false) + } + + fn restore_borrower_references_object( + &self, + reference: &super::restore_gc::RestoreBaseReference, + version: Version, + reachable_bodies: &HashMap>, + ) -> Result { + let Some(body) = reachable_bodies.get(&reference.borrower_inode) else { + return Ok(false); + }; + let Some(body) = body.as_ref() else { + return Ok(false); + }; + if body.chunk_size == 0 || body.block_size == 0 { + return Err(ObjectError::InvalidChunkLayout.into()); + } + if body.size == 0 { + return Ok(false); + } + let (_, _, chunk_index, _) = self.canonical_block_object_identity(&reference.object_key)?; + if chunk_index > (body.size - 1) / body.chunk_size { + return Ok(false); + } + let chain = self.resolve_generation_chain( + reference.borrower_inode, + body, + version, + ReadPurpose::RestoreStaging, + )?; + let Some(manifest) = self.chain_chunk_manifest( + reference.borrower_inode, + &chain, + chunk_index, + version, + ReadPurpose::RestoreStaging, + )? + else { + return Ok(false); + }; + if manifest.chunk_index != chunk_index { + return Err(MetadError::Codec( + "restore borrower manifest changed chunk identity".to_owned(), + )); + } + let manifest_len = usize::try_from(manifest.len).map_err(|_| ObjectError::InvalidRange)?; + let plan = plan_chunk_manifest_reads( + std::slice::from_ref(&manifest), + manifest.logical_offset, + manifest_len, + )?; + if let Some(block) = plan + .blocks + .into_iter() + .find(|block| block.object_key == reference.object_key) + { + if block.digest_uri != reference.digest_uri || block.object_len != reference.size { + return Err(MetadError::Codec( + "restore borrower object identity changed".to_owned(), + )); + } + return Ok(true); + } + // Historical holders are gated once per ref-set page by + // `restore_live_snapshot_holds_ref_set` before this function runs. At + // that point every borrower still has its staging inverse (member + // release is a later phase), and snapshot/ForkBinding publication shares + // `object_gc_gate` with this worker. Rewalking every historical subtree + // here would turn release into O(exact refs × historical namespace) + // without closing any additional race. + Ok(false) + } + + fn update_restore_release_job( + &self, + operation: &RestoreOperation, + operation_version: Version, + job: &RestoreReleaseJob, + job_version: &mut Version, + ) -> Result<(), MetadError> { + let key = restore_release_job_key(self.mount, operation.ref_set_id); + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-update-release-job", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(*job_version), + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key, + op: MutationOp::Put, + value: Some(Value(encode_restore_release_job(job)?)), + }], + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore release cursor")?; + self.commit_metadata(command)?; + *job_version = version; + Ok(()) + } + + /// Publish the one-way cut after which historical retention no longer + /// needs to walk namespace subtrees. The four ref-set-first owner prefixes + /// make the proof durable in the same CAS as the phase transition; global + /// inverse keyspaces remain fail-closed through their paired owner audits. + fn transition_restore_release_to_overlay( + &self, + operation: &RestoreOperation, + operation_version: Version, + job: &RestoreReleaseJob, + job_version: &mut Version, + ) -> Result<(), MetadError> { + if job.phase != RestoreReleasePhase::Overlay || !job.cursor.is_empty() { + return Err(MetadError::Codec( + "restore release overlay transition has an invalid job state".to_owned(), + )); + } + let key = restore_release_job_key(self.mount, operation.ref_set_id); + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-enter-release-overlay", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(*job_version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_member_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_inverse_owner_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_base_owner_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_base_inverse_owner_prefix(self.mount, operation.ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key, + op: MutationOp::Put, + value: Some(Value(encode_restore_release_job(job)?)), + }], + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore release overlay transition")?; + self.commit_metadata(command)?; + *job_version = version; + Ok(()) + } + + fn attach_restore_destination( + &self, + operation_id: &str, + operation: &RestoreOperation, + ) -> Result { + validate_restore_operation_identity(self.mount, &operation.operation_digest, operation)?; + let destination_components = parse_absolute_path(&operation.destination_path)?; + let (destination_name, parent_components) = destination_components + .split_last() + .ok_or_else(|| MetadError::RestoreDestinationConflict { + destination: operation.destination_path.clone(), + })?; + let destination_proof = self.restore_directory_path_proof(parent_components)?; + if destination_proof.inode.shard_index() != self.shard_index { + return Err(MetadError::RestoreCrossShardUnsupported { + inode: destination_proof.inode, + }); + } + let version = self.next_version()?; + let read_version = predecessor(version)?; + let operation_key = restore_operation_key(self.mount, &operation.operation_digest); + let operation_item = self + .metadata + .get_versioned( + RecordFamily::System, + &operation_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + let durable_operation = decode_restore_operation(&operation_item.value.0)?; + validate_restore_operation_identity( + self.mount, + &operation.operation_digest, + &durable_operation, + )?; + if durable_operation.state != RestoreOperationState::ReadyToAttach { + return Err(MetadError::RestoreInProgress); + } + let mut expected_operation = operation.clone(); + expected_operation.state = RestoreOperationState::ReadyToAttach; + if durable_operation != expected_operation { + return Err(MetadError::Codec( + "restore ReadyToAttach operation changed identity".to_owned(), + )); + } + let binding_key = fork_binding_key(self.mount, operation.destination_root); + let binding_item = self + .metadata + .get_versioned( + RecordFamily::ForkBinding, + &binding_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + let binding = crate::layout::decode_fork_binding(&binding_item.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + if binding.fork_root != operation.destination_root + || binding.source_root != operation.source_root + || binding.pinned_read_version != operation.read_version + || binding.snapshot_id != operation.snapshot_id + { + return Err(MetadError::Codec( + "restore temporary ForkBinding changed identity".to_owned(), + )); + } + let seal_key = restore_base_seal_key(self.mount, operation.ref_set_id); + let seal_item = self + .metadata + .get_versioned( + RecordFamily::System, + &seal_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore base-reference seal is missing".to_owned()) + })?; + let seal = super::restore_gc::decode_restore_base_seal(&seal_item.value.0)?; + if seal.operation_digest != operation.operation_digest + || seal.initialization_digest != operation.initialization_digest + || seal.ref_set_id != operation.ref_set_id + || seal.incarnation != operation.created_version + { + return Err(MetadError::Codec( + "restore base-reference seal changed identity".to_owned(), + )); + } + let base_seal_predicate = self.restore_base_seal_predicate(operation, read_version)?; + let index_seal_predicate = self.restore_index_seal_predicate(operation, read_version)?; + let claim_key = restore_destination_claim_key(self.mount, &operation.destination_path); + let claim = self + .metadata + .get_versioned( + RecordFamily::System, + &claim_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("restore destination claim is missing".to_owned()))?; + if claim.value.0 != operation.operation_digest { + return Err(MetadError::RestoreDestinationConflict { + destination: operation.destination_path.clone(), + }); + } + let root_index_key = restore_root_index_key(self.mount, operation.destination_root); + let root_index = self + .metadata + .get_versioned( + RecordFamily::System, + &root_index_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("restore root index is missing".to_owned()))?; + if root_index.value.0 != operation.operation_digest { + return Err(MetadError::Codec( + "restore root index changed identity".to_owned(), + )); + } + let mut root_attr = self + .get_attr_at_version_for_purpose( + operation.destination_root, + read_version, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::NotFound)?; + root_attr.ctime_ms = current_time_ms(); + let projection = projection( + destination_proof.inode, + destination_name.clone(), + root_attr, + None, + ); + let dentry = dentry_key(self.mount, destination_proof.inode, destination_name); + let cleanup_job = restore_cleanup_job_key(self.mount, operation.ref_set_id); + let object_reference = self.begin_object_reference_mutation()?; + let index_visibility = self.restore_index_complete_plan(operation, version)?; + let mut predicates = destination_proof.predicates; + predicates.extend( + self.restore_destination_parent_predicates(destination_proof.inode, read_version)?, + ); + predicates.extend([ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::Dentry, + key: dentry.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::VersionEquals(operation_item.version), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + predicate: Predicate::VersionEquals(binding_item.version), + }, + base_seal_predicate, + index_seal_predicate, + PredicateRef { + family: RecordFamily::System, + key: claim_key, + predicate: Predicate::VersionEquals(claim.version), + }, + PredicateRef { + family: RecordFamily::System, + key: root_index_key, + predicate: Predicate::VersionEquals(root_index.version), + }, + PredicateRef { + family: RecordFamily::System, + key: cleanup_job.clone(), + predicate: Predicate::NotExists, + }, + ]); + predicates.extend(index_visibility.predicates); + let mut complete = durable_operation; + complete.state = RestoreOperationState::Complete; + let mut mutations = vec![ + put_projection_mutation(RecordFamily::Dentry, dentry.clone(), &projection), + Mutation { + family: RecordFamily::System, + key: operation_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_operation(&complete)?)), + }, + delete_mutation(RecordFamily::ForkBinding, binding_key), + ]; + mutations.extend(index_visibility.mutations); + let command = MetadataCommand { + request_id: request_id( + b"restore-attach-destination", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CreateDir, + read_version, + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: dentry.clone(), + predicates, + mutations, + watch: self + .watch_projection( + projection.dentry.parent, + WatchEvent { + kind: WatchEventKind::Create, + parent: Some(projection.dentry.parent), + name: Some(projection.dentry.name.clone()), + inode: operation.destination_root, + version: version.get(), + }, + ) + .into_iter() + .collect(), + }; + validate_restore_command_bounds(&command, "restore final attach")?; + match self.commit_metadata(command) { + Ok(_) => { + // Reconcile once at the durable state transition, before a + // crash barrier can interrupt the acknowledgement path. + // Exact terminal retries stay O(1). + self.recover_restore_staging_visibility()?; + live_test_barrier::restore_applied( + operation_id, + live_test_barrier::RestoreAppliedPhase::Attach, + )?; + self.completed_restore_outcome(operation_id, &complete) + } + Err( + error @ MetadError::SyncLogArchiveFailed { + committed: true, .. + }, + ) => { + let terminal = self.restore_operation(operation.operation_digest)?; + match terminal { + Some(terminal) if terminal.state == RestoreOperationState::Complete => { + self.recover_restore_staging_visibility()?; + // The local attach applied, but its archived log tail was + // not proven in the control plane. Preserve the committed + // publication error and do not expose the post-publication + // crash barrier. An exact retry observes Complete after + // the server repairs the recovery reference. + Err(error) + } + _ => Err(MetadError::RestoreInProgress), + } + } + Err(MetadError::Metadata(MetadataError::PredicateFailed)) => self + .reprove_restore_attach_predicate_failure( + operation_id, + operation, + object_reference, + ), + Err(error) => Err(error), + } + } + + /// Classify a failed final-attach CAS from a bounded set of durable facts. + /// + /// The destination dentry is the only public namespace mutation in the + /// command. Re-reading it after checking the terminal operation turns a + /// concurrent occupation into the public restore conflict instead of + /// leaking the storage engine's generic predicate failure. Other races + /// remain retryable and are resolved by an exact request retry. + fn reprove_restore_attach_predicate_failure( + &self, + operation_id: &str, + operation: &RestoreOperation, + object_reference: ObjectReferenceMutation, + ) -> Result { + let Some(durable_operation) = self.restore_operation(operation.operation_digest)? else { + return Err(MetadError::RestoreInProgress); + }; + match durable_operation.state { + RestoreOperationState::Complete => { + self.recover_restore_staging_visibility()?; + return self.completed_restore_outcome(operation_id, &durable_operation); + } + RestoreOperationState::ReadyToAttach => { + let mut expected_operation = operation.clone(); + expected_operation.state = RestoreOperationState::ReadyToAttach; + if durable_operation != expected_operation { + return Err(MetadError::Codec( + "restore ReadyToAttach operation changed identity".to_owned(), + )); + } + } + RestoreOperationState::Preparing + | RestoreOperationState::Cleaning + | RestoreOperationState::Discarding + | RestoreOperationState::Releasing => { + return Err(MetadError::RestoreInProgress); + } + } + + let claim_key = restore_destination_claim_key(self.mount, &operation.destination_path); + match self.metadata.get( + RecordFamily::System, + &claim_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? { + Some(claim) if claim.0 == operation.operation_digest => {} + Some(_) => { + return Err(MetadError::RestoreDestinationConflict { + destination: operation.destination_path.clone(), + }); + } + None => { + return Err(MetadError::Codec( + "restore destination claim is missing".to_owned(), + )); + } + } + + if self.lookup_path(&operation.destination_path)?.is_some() { + return Err(MetadError::RestoreDestinationConflict { + destination: operation.destination_path.clone(), + }); + } + + let current_object_reference = match self.begin_object_reference_mutation() { + Ok(reference) => reference, + Err(MetadError::Metadata(MetadataError::PredicateFailed)) => { + return Err(MetadError::RestoreInProgress); + } + Err(error) => return Err(error), + }; + if current_object_reference.version() != object_reference.version() { + return Err(MetadError::StalePreparedArtifactObjectGcEpoch { + expected: object_reference.version().get(), + current: current_object_reference.version().get(), + }); + } + + Err(MetadError::RestoreInProgress) + } +} + +pub(super) fn validate_restore_command_bounds( + command: &MetadataCommand, + resource: &str, +) -> Result<(), MetadError> { + const MAX_ITEMS: usize = 4096; + const MAX_BYTES: usize = 8 * 1024 * 1024; + let items = command + .predicates + .len() + .saturating_add(command.mutations.len()) + .saturating_add(command.watch.len()); + if items > MAX_ITEMS { + return Err(MetadError::RestoreResourceLimit { + resource: format!("{resource} items"), + limit: MAX_ITEMS as u64, + actual: items as u64, + }); + } + // Bound the durable command encoding, including its three collection + // counts and watch payloads. Predicate rows use the largest on-wire form + // (`VersionEquals`), while mutation rows include the optional value length + // prefix even for deletes. The estimate is therefore conservative but can + // never admit a log command larger than the advertised limit. + let bytes = 58_usize + .saturating_add(command.request_id.len()) + .saturating_add(command.primary_key.len()) + .saturating_add(command.predicates.iter().fold(0_usize, |total, predicate| { + total.saturating_add(predicate.key.len()).saturating_add(18) + })) + .saturating_add(command.mutations.iter().fold(0_usize, |total, mutation| { + total + .saturating_add(mutation.key.len()) + .saturating_add(mutation.value.as_ref().map_or(0, |value| value.0.len())) + .saturating_add(19) + })) + .saturating_add(command.watch.iter().fold(0_usize, |total, watch| { + total + .saturating_add(watch.key.len()) + .saturating_add(watch.event.len()) + .saturating_add(17) + })); + if bytes > MAX_BYTES { + return Err(MetadError::RestoreResourceLimit { + resource: format!("{resource} bytes"), + limit: MAX_BYTES as u64, + actual: bytes as u64, + }); + } + Ok(()) +} + +/// Only an existing staging-inverse row opts the merged ordinary namespace +/// command into the restore-specific command budget. Before the first restore, +/// ordinary commands carry an allocator `VersionEquals` activation fence; that +/// global predicate is not restore membership. +pub(super) fn restore_write_predicates_include_owner(predicates: &[PredicateRef]) -> bool { + predicates.iter().any(|predicate| { + predicate.family == RecordFamily::System + && matches!(predicate.predicate, Predicate::VersionEquals(_)) + && predicate.key.len() == 8 + STAGING_INVERSE_KEY_LABEL.len() + 8 + && predicate.key[8..].starts_with(STAGING_INVERSE_KEY_LABEL) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::holtstore::HoltMetadataStore; + use nokv_object::MemoryObjectStore; + + fn mount(raw: u64) -> MountId { + MountId::new(raw).unwrap() + } + + fn operation() -> RestoreOperation { + RestoreOperation { + operation_digest: [7; 32], + initialization_digest: [8; 32], + state: RestoreOperationState::Preparing, + source_root: InodeId::new(10).unwrap(), + destination_root: InodeId::new(20).unwrap(), + snapshot_id: 30, + read_version: 40, + created_version: 50, + ref_set_id: 50, + source_path: "/workbenches/source".to_owned(), + destination_path: "/workbenches/destination".to_owned(), + } + } + + #[test] + fn restore_operation_codec_round_trips_and_fails_closed() { + let operation = operation(); + let encoded = encode_restore_operation(&operation).unwrap(); + assert_eq!(decode_restore_operation(&encoded).unwrap(), operation); + assert!(decode_restore_operation(&encoded[..encoded.len() - 1]).is_err()); + let mut trailing = encoded; + trailing.push(0); + assert!(decode_restore_operation(&trailing).is_err()); + } + + #[test] + fn restore_release_worker_cursor_codec_is_strict_and_mount_scoped() { + let mount_id = mount(1); + let cursor = RestoreReleaseWorkerCursor { + cycle_high_water: 50, + start_after: restore_release_job_key(mount_id, 7), + }; + let encoded = encode_restore_release_worker_cursor(mount_id, &cursor).unwrap(); + assert_eq!( + decode_restore_release_worker_cursor(mount_id, &encoded).unwrap(), + cursor + ); + assert!(decode_restore_release_worker_cursor(mount(2), &encoded).is_err()); + assert!( + decode_restore_release_worker_cursor(mount_id, &encoded[..encoded.len() - 1]).is_err() + ); + + let mut invalid_magic = encoded.clone(); + invalid_magic[0] ^= 0xff; + assert!(decode_restore_release_worker_cursor(mount_id, &invalid_magic).is_err()); + let mut invalid_version = encoded.clone(); + invalid_version[8] = RESTORE_FORMAT_VERSION.saturating_add(1); + assert!(decode_restore_release_worker_cursor(mount_id, &invalid_version).is_err()); + let mut trailing = encoded; + trailing.push(0); + assert!(decode_restore_release_worker_cursor(mount_id, &trailing).is_err()); + assert!(encode_restore_release_worker_cursor( + mount_id, + &RestoreReleaseWorkerCursor { + cycle_high_water: 0, + start_after: Vec::new(), + }, + ) + .is_err()); + assert!(encode_restore_release_worker_cursor( + mount_id, + &RestoreReleaseWorkerCursor { + cycle_high_water: 1, + start_after: vec![b'x'; MAX_RESTORE_PATH_BYTES + 1], + }, + ) + .is_err()); + let mut malformed_job_key = restore_release_job_prefix(mount_id); + malformed_job_key.push(1); + assert!(encode_restore_release_worker_cursor( + mount_id, + &RestoreReleaseWorkerCursor { + cycle_high_water: 1, + start_after: malformed_job_key, + }, + ) + .is_err()); + } + + #[test] + fn restore_staging_member_codec_preserves_manifest_release_cursor() { + let operation = operation(); + let member = RestoreStagingMember { + operation_digest: operation.operation_digest, + source_inode: Some(operation.source_root), + destination_inode: operation.destination_root, + destination_parent: None, + name: None, + relative_path: String::new(), + canonical_index_cursor: Vec::new(), + canonical_index_complete: true, + manifest_cursor: vec![9; 32], + manifest_block_cursor: 4_087, + }; + let encoded = encode_restore_staging_member(&member).unwrap(); + assert_eq!(decode_restore_staging_member(&encoded).unwrap(), member); + assert!(decode_restore_staging_member(&encoded[..encoded.len() - 1]).is_err()); + + let mut invalid = member.clone(); + invalid.manifest_cursor.clear(); + assert!(encode_restore_staging_member(&invalid).is_err()); + invalid.manifest_block_cursor = 0; + invalid.manifest_cursor = vec![1]; + assert!(encode_restore_staging_member(&invalid).is_err()); + } + + #[test] + fn operation_id_is_mount_scoped_and_canonical() { + let first = restore_operation_id(mount(1), "/a/", 7, "/b").unwrap(); + let canonical = restore_operation_id(mount(1), "/a", 7, "/b").unwrap(); + let other_mount = restore_operation_id(mount(2), "/a", 7, "/b").unwrap(); + assert_eq!(first, canonical); + assert_ne!(first, other_mount); + } + + #[test] + fn private_restore_keys_are_mount_scoped_and_disjoint() { + let digest = [9; 32]; + let operation = restore_operation_key(mount(1), &digest); + let claim = restore_destination_claim_key(mount(1), "/dst"); + let borrower = InodeId::new(11).unwrap(); + let owner = restore_base_owner_key(mount(1), 7, &digest, borrower, 13); + let inverse = restore_base_inverse_key(mount(1), &digest, 7, borrower, 13); + assert_ne!(operation, claim); + assert_ne!(owner, inverse); + assert_ne!(operation, restore_operation_key(mount(2), &digest)); + } + + #[test] + fn restore_graph_command_classifier_covers_every_private_keyspace() { + let mount_id = mount(1); + let command = |mutations: Vec| MetadataCommand { + request_id: b"restore-graph-classifier".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: Version::new(1).unwrap(), + commit_version: Version::new(2).unwrap(), + primary_family: mutations + .first() + .map_or(RecordFamily::System, |mutation| mutation.family), + primary_key: mutations + .first() + .map_or_else(|| b"none".to_vec(), |mutation| mutation.key.clone()), + predicates: Vec::new(), + mutations, + watch: Vec::new(), + }; + let put = |family, key| Mutation { + family, + key, + op: MutationOp::Put, + value: Some(Value(vec![1])), + }; + + for key in [ + restore_active_key(mount_id), + restore_activation_fence_key(mount_id), + ] { + assert!(command_mutates_restore_graph( + mount_id, + &command(vec![put(RecordFamily::System, key)]) + )); + } + for (name, prefix) in restore_control_keyspaces(mount_id) { + if matches!(name, "init_upload_tombstone_cursor" | "release_cursor") { + assert!(!command_mutates_restore_graph( + mount_id, + &command(vec![put(RecordFamily::System, prefix.clone())]) + )); + } + let mut suffixed = prefix; + suffixed.push(1); + assert!(command_mutates_restore_graph( + mount_id, + &command(vec![put(RecordFamily::System, suffixed)]) + )); + } + for (_, mut prefix) in super::restore_index::restore_index_private_keyspaces(mount_id) { + prefix.push(1); + assert!(command_mutates_restore_graph( + mount_id, + &command(vec![put(RecordFamily::System, prefix)]) + )); + } + + assert!(!command_mutates_restore_graph( + mount_id, + &command(vec![put(RecordFamily::System, allocator_key(mount_id))]) + )); + assert!(!command_mutates_restore_graph( + mount_id, + &command(vec![put( + RecordFamily::System, + restore_operation_key(mount(2), &[4; 32]), + )]) + )); + assert!(!command_mutates_restore_graph( + mount_id, + &command(vec![put( + RecordFamily::Dentry, + restore_active_key(mount_id), + )]) + )); + assert!(command_mutates_restore_graph( + mount_id, + &command(vec![ + put( + RecordFamily::System, + restore_init_upload_tombstone_cursor_key(mount_id), + ), + put(RecordFamily::System, restore_active_key(mount_id)), + ]) + )); + } + + #[test] + fn materialization_preflight_counts_the_complete_sixty_four_entry_command() { + let service = NoKvFs::new( + mount(1), + HoltMetadataStore::open_memory().unwrap(), + MemoryObjectStore::new(), + ); + let operation = operation(); + let entries = (0_u64..RESTORE_BATCH_ENTRIES as u64) + .map(|index| { + let inode = InodeId::new(100 + index).unwrap(); + let generation = 1_000 + index; + let name = DentryName::new(format!("file-{index:02}")).unwrap(); + let body = BodyDescriptor { + producer: "unit-test".to_owned(), + digest_uri: "sha256:test".to_owned(), + size: 60 * DEFAULT_CHUNK_SIZE, + content_type: "application/octet-stream".to_owned(), + manifest_id: format!("manifest-{index}"), + generation, + base_generation: 0, + chunk_size: DEFAULT_CHUNK_SIZE, + block_size: DEFAULT_BLOCK_SIZE as u64, + }; + let chunks = (0_u64..60) + .map(|chunk_index| ChunkManifest { + chunk_index, + logical_offset: chunk_index * DEFAULT_CHUNK_SIZE, + len: DEFAULT_CHUNK_SIZE, + slices: Vec::new(), + }) + .collect(); + RestoreCloneEntry { + source: DentryWithAttr { + dentry: DentryRecord { + parent: InodeId::root(), + name: name.clone(), + child: inode, + child_type: FileType::File, + attr_generation: generation, + }, + attr: InodeAttr { + inode, + file_type: FileType::File, + mode: 0o644, + uid: 1000, + gid: 1000, + rdev: 0, + nlink: 1, + size: body.size, + generation, + mtime_ms: 0, + ctime_ms: 0, + }, + body: Some(body.clone()), + }, + destination: inode, + relative_path: String::from_utf8_lossy(name.as_bytes()).into_owned(), + body: Some(body), + chunks, + } + }) + .collect::>(); + let version = Version::new(60).unwrap(); + let object_reference = ObjectReferenceMutation::from_version(Version::new(1).unwrap()); + + let single = service + .build_restore_children_command( + &operation, + InodeId::root(), + std::slice::from_ref(&entries[0]), + object_reference, + version, + ) + .unwrap(); + validate_restore_command_bounds(&single, "restore materialization batch").unwrap(); + + let aggregate = service + .build_restore_children_command( + &operation, + InodeId::root(), + &entries, + object_reference, + version, + ) + .unwrap(); + assert!(matches!( + validate_restore_command_bounds(&aggregate, "restore materialization batch"), + Err(MetadError::RestoreResourceLimit { resource, .. }) + if resource == "restore materialization batch items" + )); + } + + #[test] + fn restore_command_byte_limit_includes_watch_payloads() { + let version = Version::new(2).unwrap(); + let command = MetadataCommand { + request_id: b"restore-watch-bound".to_vec(), + kind: CommandKind::CreateDir, + read_version: Version::new(1).unwrap(), + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: vec![0; 32], + predicates: Vec::new(), + mutations: Vec::new(), + watch: vec![WatchProjection { + family: RecordFamily::Watch, + key: vec![0; 32], + event: vec![0; 8 * 1024 * 1024], + }], + }; + + assert!(matches!( + validate_restore_command_bounds(&command, "restore watch bound"), + Err(MetadError::RestoreResourceLimit { resource, .. }) + if resource == "restore watch bound bytes" + )); + } +} diff --git a/crates/nokv-meta/src/service/restore_gc.rs b/crates/nokv-meta/src/service/restore_gc.rs new file mode 100644 index 000000000..40e9f19fb --- /dev/null +++ b/crates/nokv-meta/src/service/restore_gc.rs @@ -0,0 +1,1943 @@ +//! Exact borrowed-object references and bounded restore release. + +use super::restore::{ + decode_restore_operation, encode_restore_operation, restore_barrier_operation_id, + restore_base_inverse_key, restore_base_inverse_prefix, restore_base_owner_key, + restore_base_seal_key, restore_operation_key, RestoreOperation, RestoreOperationState, +}; +use super::*; + +const BASE_REFERENCE_FORMAT_VERSION: u8 = 1; +const BASE_REFERENCE_BUILD_FORMAT_VERSION: u8 = 3; +const BASE_REFERENCE_BATCH: usize = 64; +const BASE_MEMBER_SCAN_BATCH: usize = 64; +const BASE_CHUNK_SCAN_BATCH: usize = 64; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreBaseReference { + pub(super) operation_digest: [u8; 32], + pub(super) ref_set_id: u64, + pub(super) borrower_inode: InodeId, + pub(super) borrower_generation: u64, + pub(super) object_key: String, + pub(super) digest_uri: String, + pub(super) size: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreBaseSeal { + pub(super) operation_digest: [u8; 32], + pub(super) initialization_digest: [u8; 32], + pub(super) ref_set_id: u64, + pub(super) incarnation: u64, + pub(super) reference_count: u64, + pub(super) reference_digest: [u8; 32], +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreBaseBuild { + pub(super) operation_digest: [u8; 32], + pub(super) initialization_digest: [u8; 32], + pub(super) ref_set_id: u64, + pub(super) incarnation: u64, + pub(super) completed_member: Option, + pub(super) active_member: Option, + pub(super) chunk_cursor: Option, + pub(super) object_cursor: Option<[u8; 32]>, + pub(super) reference_count: u64, + pub(super) reference_digest: [u8; 32], + pub(super) batch_index: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) enum RestoreBaseSealRecord { + Building(RestoreBaseBuild), + Sealed(RestoreBaseSeal), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreBaseInverse { + pub(super) operation_digest: [u8; 32], + pub(super) ref_set_id: u64, + pub(super) object_digest: [u8; 32], + pub(super) borrower_inode: InodeId, + pub(super) borrower_generation: u64, +} + +#[derive(Debug)] +struct RestoreBaseReferencePage { + next: RestoreBaseBuild, + references: Vec, + member_predicates: Vec, + complete: bool, +} + +#[derive(Debug)] +pub(super) struct RestoreMemberBaseReferenceLayout { + borrower_generation: u64, + generation_chain: Vec, + pub(super) end_chunk: u64, +} + +#[derive(Debug)] +struct RestoreChunkReferenceCache { + inode: InodeId, + chunk_index: u64, + references: Vec<([u8; 32], RestoreBaseReference)>, +} + +pub(super) struct RestoreBaseReferencePreflight { + build: RestoreBaseBuild, + pending: Vec, +} + +impl NoKvFs +where + M: MetadataStore, + O: ObjectStore, +{ + pub(super) fn begin_restore_base_reference_preflight( + &self, + operation: &RestoreOperation, + ) -> RestoreBaseReferencePreflight { + RestoreBaseReferencePreflight { + build: RestoreBaseBuild { + operation_digest: operation.operation_digest, + initialization_digest: operation.initialization_digest, + ref_set_id: operation.ref_set_id, + incarnation: operation.created_version, + completed_member: None, + active_member: None, + chunk_cursor: None, + object_cursor: None, + reference_count: 0, + reference_digest: initial_restore_base_reference_digest(), + batch_index: 0, + }, + pending: Vec::with_capacity(BASE_REFERENCE_BATCH), + } + } + + pub(super) fn preflight_restore_base_references_for_entry( + &self, + operation: &RestoreOperation, + borrower_inode: InodeId, + borrower_generation: u64, + owned_object_inode: Option, + chunks: &[ChunkManifest], + preflight: &mut RestoreBaseReferencePreflight, + ) -> Result<(), MetadError> { + for manifest in chunks { + let manifest_len = + usize::try_from(manifest.len).map_err(|_| ObjectError::InvalidRange)?; + let plan = plan_chunk_manifest_reads( + std::slice::from_ref(manifest), + manifest.logical_offset, + manifest_len, + )?; + let mut selected = BTreeMap::<[u8; 32], RestoreBaseReference>::new(); + for block in plan.blocks { + if owned_object_inode.is_some_and(|inode| { + self.owns_block_object_key(inode, borrower_generation, &block.object_key) + }) { + continue; + } + let (_, _, object_chunk, _) = + self.canonical_block_object_identity(&block.object_key)?; + if object_chunk != manifest.chunk_index { + return Err(MetadError::Codec( + "restore preflight object changed chunk identity".to_owned(), + )); + } + let digest: [u8; 32] = Sha256::digest(block.object_key.as_bytes()).into(); + let reference = RestoreBaseReference { + operation_digest: operation.operation_digest, + ref_set_id: operation.ref_set_id, + borrower_inode, + borrower_generation, + object_key: block.object_key, + digest_uri: block.digest_uri, + size: block.object_len, + }; + if let Some(existing) = selected.get(&digest) { + if existing != &reference { + return Err(MetadError::Codec( + "restore preflight found an inconsistent object identity".to_owned(), + )); + } + continue; + } + selected.insert(digest, reference); + } + for reference in selected.into_values() { + preflight.pending.push(reference); + while preflight.pending.len() >= BASE_REFERENCE_BATCH { + self.flush_restore_base_reference_preflight(operation, preflight)?; + } + } + } + Ok(()) + } + + pub(super) fn finish_restore_base_reference_preflight( + &self, + operation: &RestoreOperation, + preflight: &mut RestoreBaseReferencePreflight, + ) -> Result<(), MetadError> { + while !preflight.pending.is_empty() { + self.flush_restore_base_reference_preflight(operation, preflight)?; + } + Ok(()) + } + + fn flush_restore_base_reference_preflight( + &self, + operation: &RestoreOperation, + preflight: &mut RestoreBaseReferencePreflight, + ) -> Result<(), MetadError> { + if preflight.pending.is_empty() { + return Ok(()); + } + if preflight.build.operation_digest != operation.operation_digest + || preflight.build.initialization_digest != operation.initialization_digest + || preflight.build.ref_set_id != operation.ref_set_id + || preflight.build.incarnation != operation.created_version + { + return Err(MetadError::Codec( + "restore base-reference preflight changed identity".to_owned(), + )); + } + let take = preflight.pending.len().min(BASE_REFERENCE_BATCH); + let references = preflight.pending[..take].to_vec(); + let last = references.last().ok_or_else(|| { + MetadError::Codec("restore base-reference preflight made no progress".to_owned()) + })?; + let (_, _, chunk_index, _) = self.canonical_block_object_identity(&last.object_key)?; + let mut next = preflight.build.clone(); + next.active_member = Some(last.borrower_inode); + next.chunk_cursor = Some(chunk_index); + next.object_cursor = Some(Sha256::digest(last.object_key.as_bytes()).into()); + // Runtime may carry proof for up to 64 scanned staging members, + // including empty members before the first reference. Pad preflight to + // that exact worst-case predicate cardinality so the hold cannot land + // on a source whose first real reference page is a few KiB larger. + let mut member_predicates = Vec::with_capacity(BASE_MEMBER_SCAN_BATCH); + for index in 0..BASE_MEMBER_SCAN_BATCH { + let inode = InodeId::new(u64::MAX - index as u64)?; + let key = + super::restore::restore_staging_member_key(self.mount, operation.ref_set_id, inode); + member_predicates.push(PredicateRef { + family: RecordFamily::System, + key, + predicate: Predicate::VersionEquals(Version::new(2)?), + }); + } + let page = RestoreBaseReferencePage { + next, + references, + member_predicates, + complete: false, + }; + let bounds_version = Version::new(2)?; + let binding_key = fork_binding_key(self.mount, operation.destination_root); + let seal_key = restore_base_seal_key(self.mount, operation.ref_set_id); + let fitted = self.fit_restore_base_reference_page( + operation, + bounds_version, + &binding_key, + &seal_key, + Version::new(1)?, + &preflight.build, + page, + true, + )?; + let accepted = fitted.references.len(); + let (command, next, _) = self.build_restore_base_reference_page_command( + operation, + bounds_version, + &binding_key, + &seal_key, + Version::new(1)?, + ObjectReferenceMutation::from_version(Version::new(1)?), + bounds_version, + &fitted, + )?; + super::restore::validate_restore_command_bounds(&command, "restore base-reference batch")?; + preflight.build = next; + preflight.pending.drain(..accepted); + Ok(()) + } + + pub(super) fn seal_restore_base_references( + &self, + operation: &RestoreOperation, + ) -> Result<(), MetadError> { + let operation_key = restore_operation_key(self.mount, &operation.operation_digest); + let binding_key = fork_binding_key(self.mount, operation.destination_root); + let seal_key = restore_base_seal_key(self.mount, operation.ref_set_id); + let operation_version = Version::new(operation.created_version)?; + let mut chunk_cache = None; + loop { + let read_version = self.read_version()?; + let record = self.metadata.get_versioned( + RecordFamily::System, + &seal_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + let Some(record) = record else { + let build = RestoreBaseBuild { + operation_digest: operation.operation_digest, + initialization_digest: operation.initialization_digest, + ref_set_id: operation.ref_set_id, + incarnation: operation.created_version, + completed_member: None, + active_member: None, + chunk_cursor: None, + object_cursor: None, + reference_count: 0, + reference_digest: initial_restore_base_reference_digest(), + batch_index: 0, + }; + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-start-base-refs", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: seal_key.clone(), + predicates: vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: seal_key.clone(), + predicate: Predicate::NotExists, + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key: seal_key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_restore_base_build(&build)?)), + }], + watch: Vec::new(), + }; + super::restore::validate_restore_command_bounds( + &command, + "restore base-reference build start", + )?; + self.commit_metadata(command)?; + continue; + }; + match decode_restore_base_seal_record(&record.value.0)? { + RestoreBaseSealRecord::Sealed(seal) => { + validate_restore_base_seal(operation, &seal)?; + return Ok(()); + } + RestoreBaseSealRecord::Building(build) => { + validate_restore_base_build(operation, &build)?; + let page = self.plan_restore_base_reference_page( + operation, + &build, + read_version, + &mut chunk_cache, + )?; + if page.complete && page.next == build { + self.finish_restore_base_reference_seal( + operation, + operation_version, + &binding_key, + &seal_key, + record.version, + &build, + )?; + return Ok(()); + } + let page = self.fit_restore_base_reference_page( + operation, + operation_version, + &binding_key, + &seal_key, + record.version, + &build, + page, + false, + )?; + self.persist_restore_base_reference_page( + operation, + operation_version, + &binding_key, + &seal_key, + record.version, + page, + )?; + } + } + } + } + + pub(super) fn restore_base_seal_predicate( + &self, + operation: &RestoreOperation, + version: Version, + ) -> Result { + let seal_key = restore_base_seal_key(self.mount, operation.ref_set_id); + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &seal_key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore base-reference seal is missing".to_owned()) + })?; + let seal = decode_restore_base_seal(&item.value.0)?; + validate_restore_base_seal(operation, &seal)?; + Ok(PredicateRef { + family: RecordFamily::System, + key: seal_key, + predicate: Predicate::VersionEquals(item.version), + }) + } + + #[allow(clippy::too_many_arguments)] + fn finish_restore_base_reference_seal( + &self, + operation: &RestoreOperation, + operation_version: Version, + binding_key: &[u8], + seal_key: &[u8], + build_version: Version, + build: &RestoreBaseBuild, + ) -> Result<(), MetadError> { + let seal = RestoreBaseSeal { + operation_digest: operation.operation_digest, + initialization_digest: operation.initialization_digest, + ref_set_id: operation.ref_set_id, + incarnation: operation.created_version, + reference_count: build.reference_count, + reference_digest: finalize_restore_base_reference_digest( + build.reference_count, + build.reference_digest, + ), + }; + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-seal-base-refs", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: seal_key.to_vec(), + predicates: vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.to_vec(), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: seal_key.to_vec(), + predicate: Predicate::VersionEquals(build_version), + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key: seal_key.to_vec(), + op: MutationOp::Put, + value: Some(Value(encode_restore_base_seal(&seal))), + }], + watch: Vec::new(), + }; + super::restore::validate_restore_command_bounds(&command, "restore base seal")?; + self.commit_metadata(command)?; + live_test_barrier::restore_applied( + &restore_barrier_operation_id(operation), + live_test_barrier::RestoreAppliedPhase::ReferencesSealed, + )?; + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn persist_restore_base_reference_page( + &self, + operation: &RestoreOperation, + operation_version: Version, + binding_key: &[u8], + seal_key: &[u8], + build_version: Version, + page: RestoreBaseReferencePage, + ) -> Result<(), MetadError> { + // Another restore's release removes its owner/inverse projections + // atomically while holding object_gc_gate. Keep the cross-ref-set + // identity read and this bounded page commit under the same gate so a + // prefix scan cannot observe the old inverse immediately before its + // canonical owner disappears. The GC-claim predicate remains the + // durable failover fence; this mutex only supplies local multi-key + // read consistency. + let _gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|error| error.into_inner()); + let read_version = self.read_version()?; + self.validate_restore_base_reference_identities(operation, &page.references, read_version)?; + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let (command, next, batch_index) = self.build_restore_base_reference_page_command( + operation, + operation_version, + binding_key, + seal_key, + build_version, + object_reference, + version, + &page, + )?; + super::restore::validate_restore_command_bounds(&command, "restore base-reference batch")?; + let encoded_next = encode_restore_base_build(&next)?; + match self.commit_metadata(command) { + Ok(_) => {} + Err(error @ MetadError::Metadata(MetadataError::Backend(_))) => { + let applied = self + .metadata + .get( + RecordFamily::System, + seal_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .is_some_and(|value| value.0 == encoded_next); + if !applied { + return Err(error); + } + } + Err(error) => return Err(error), + } + live_test_barrier::restore_applied( + &restore_barrier_operation_id(operation), + live_test_barrier::RestoreAppliedPhase::ReferenceBatch(batch_index), + )?; + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn build_restore_base_reference_page_command( + &self, + operation: &RestoreOperation, + operation_version: Version, + binding_key: &[u8], + seal_key: &[u8], + build_version: Version, + object_reference: ObjectReferenceMutation, + version: Version, + page: &RestoreBaseReferencePage, + ) -> Result<(MetadataCommand, RestoreBaseBuild, u64), MetadError> { + let mut next = page.next.clone(); + for reference in &page.references { + next.reference_digest = + extend_restore_base_reference_digest(next.reference_digest, reference)?; + next.reference_count = next.reference_count.checked_add(1).ok_or_else(|| { + MetadError::Codec("restore base-reference count overflow".to_owned()) + })?; + } + let batch_index = next.batch_index; + next.batch_index = next + .batch_index + .checked_add(1) + .ok_or_else(|| MetadError::Codec("restore base-reference batch overflow".to_owned()))?; + let mut predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.to_vec(), + predicate: Predicate::VersionEquals(operation_version), + }, + PredicateRef { + family: RecordFamily::System, + key: seal_key.to_vec(), + predicate: Predicate::VersionEquals(build_version), + }, + ]; + predicates.extend(page.member_predicates.iter().cloned()); + let mut mutations = Vec::with_capacity(page.references.len() * 3 + 1); + for reference in &page.references { + let object_digest: [u8; 32] = Sha256::digest(reference.object_key.as_bytes()).into(); + let owner_key = restore_base_owner_key( + self.mount, + operation.ref_set_id, + &object_digest, + reference.borrower_inode, + reference.borrower_generation, + ); + let inverse_key = restore_base_inverse_key( + self.mount, + &object_digest, + operation.ref_set_id, + reference.borrower_inode, + reference.borrower_generation, + ); + let inverse_owner_key = super::restore::restore_base_inverse_owner_key( + self.mount, + operation.ref_set_id, + &object_digest, + reference.borrower_inode, + reference.borrower_generation, + ); + predicates.extend([ + PredicateRef { + family: RecordFamily::System, + key: owner_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_owner_key.clone(), + predicate: Predicate::NotExists, + }, + ]); + let inverse_value = encode_restore_base_inverse(&RestoreBaseInverse { + operation_digest: operation.operation_digest, + ref_set_id: operation.ref_set_id, + object_digest, + borrower_inode: reference.borrower_inode, + borrower_generation: reference.borrower_generation, + }); + mutations.extend([ + Mutation { + family: RecordFamily::System, + key: owner_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_base_reference(reference)?)), + }, + Mutation { + family: RecordFamily::System, + key: inverse_key, + op: MutationOp::Put, + value: Some(Value(inverse_value.clone())), + }, + Mutation { + family: RecordFamily::System, + key: inverse_owner_key, + op: MutationOp::Put, + value: Some(Value(inverse_value)), + }, + ]); + } + mutations.push(Mutation { + family: RecordFamily::System, + key: seal_key.to_vec(), + op: MutationOp::Put, + value: Some(Value(encode_restore_base_build(&next)?)), + }); + Ok(( + MetadataCommand { + request_id: request_id( + b"restore-persist-base-refs", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: super::restore::restore_base_owner_prefix( + self.mount, + operation.ref_set_id, + ), + predicates, + mutations, + watch: Vec::new(), + }, + next, + batch_index, + )) + } + + #[allow(clippy::too_many_arguments)] + fn fit_restore_base_reference_page( + &self, + operation: &RestoreOperation, + operation_version: Version, + binding_key: &[u8], + seal_key: &[u8], + build_version: Version, + build: &RestoreBaseBuild, + page: RestoreBaseReferencePage, + retain_all_member_predicates: bool, + ) -> Result { + let bounds_object_reference = ObjectReferenceMutation::from_version(Version::new(1)?); + let bounds_version = Version::new(2)?; + let full = self.build_restore_base_reference_page_command( + operation, + operation_version, + binding_key, + seal_key, + build_version, + bounds_object_reference, + bounds_version, + &page, + )?; + match super::restore::validate_restore_command_bounds( + &full.0, + "restore base-reference batch", + ) { + Ok(()) => return Ok(page), + Err(error) if page.references.len() <= 1 => return Err(error), + Err(_) => {} + } + + for accepted in (1..page.references.len()).rev() { + let last = &page.references[accepted - 1]; + let (_, _, chunk_index, _) = self.canonical_block_object_identity(&last.object_key)?; + let mut next = build.clone(); + next.active_member = Some(last.borrower_inode); + next.chunk_cursor = Some(chunk_index); + next.object_cursor = Some(Sha256::digest(last.object_key.as_bytes()).into()); + let member_predicates = if retain_all_member_predicates { + page.member_predicates.clone() + } else { + let member_cursor = super::restore::restore_staging_member_key( + self.mount, + operation.ref_set_id, + last.borrower_inode, + ); + page.member_predicates + .iter() + .filter(|predicate| predicate.key.as_slice() <= member_cursor.as_slice()) + .cloned() + .collect() + }; + let candidate = RestoreBaseReferencePage { + next, + references: page.references[..accepted].to_vec(), + member_predicates, + complete: false, + }; + let command = self.build_restore_base_reference_page_command( + operation, + operation_version, + binding_key, + seal_key, + build_version, + bounds_object_reference, + bounds_version, + &candidate, + )?; + match super::restore::validate_restore_command_bounds( + &command.0, + "restore base-reference batch", + ) { + Ok(()) => return Ok(candidate), + Err(error) if accepted == 1 => return Err(error), + Err(_) => {} + } + } + Err(MetadError::Codec( + "restore base-reference packer made no progress".to_owned(), + )) + } + + fn plan_restore_base_reference_page( + &self, + operation: &RestoreOperation, + build: &RestoreBaseBuild, + version: Version, + chunk_cache: &mut Option, + ) -> Result { + let mut next = build.clone(); + let mut references = Vec::with_capacity(BASE_REFERENCE_BATCH); + let mut member_predicates = Vec::with_capacity(BASE_MEMBER_SCAN_BATCH); + let mut members_examined = 0_usize; + let mut chunks_examined = 0_usize; + loop { + if references.len() == BASE_REFERENCE_BATCH + || members_examined == BASE_MEMBER_SCAN_BATCH + || chunks_examined == BASE_CHUNK_SCAN_BATCH + { + return Ok(RestoreBaseReferencePage { + next, + references, + member_predicates, + complete: false, + }); + } + let active = next.active_member; + let rows = if let Some(active) = active { + let key = super::restore::restore_staging_member_key( + self.mount, + operation.ref_set_id, + active, + ); + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + version, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreRootChanged { root: active })?; + vec![crate::command::ScanItem { + key, + value: item.value, + version: item.version, + }] + } else { + let prefix = + super::restore::restore_staging_member_prefix(self.mount, operation.ref_set_id); + self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix, + start_after: next.completed_member.map(|inode| { + super::restore::restore_staging_member_key( + self.mount, + operation.ref_set_id, + inode, + ) + }), + version, + limit: BASE_MEMBER_SCAN_BATCH - members_examined, + purpose: ReadPurpose::RestoreStaging, + })? + }; + if rows.is_empty() { + return Ok(RestoreBaseReferencePage { + next, + references, + member_predicates, + complete: true, + }); + } + let requested = BASE_MEMBER_SCAN_BATCH - members_examined; + let reached_tail = active.is_none() && rows.len() < requested; + for row in rows { + let member = super::restore::decode_restore_staging_member(&row.value.0)?; + if member.operation_digest != operation.operation_digest + || row.key + != super::restore::restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ) + || next + .completed_member + .is_some_and(|completed| member.destination_inode <= completed) + || active.is_some_and(|active| active != member.destination_inode) + { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + member_predicates.push(PredicateRef { + family: RecordFamily::System, + key: row.key, + predicate: Predicate::VersionEquals(row.version), + }); + members_examined += 1; + + let Some(layout) = self.restore_member_base_reference_layout(&member, version)? + else { + next.completed_member = Some(member.destination_inode); + next.active_member = None; + next.chunk_cursor = None; + next.object_cursor = None; + *chunk_cache = None; + continue; + }; + let mut chunk_index = if active == Some(member.destination_inode) { + next.chunk_cursor.ok_or_else(|| { + MetadError::Codec( + "restore base-reference build lost its chunk cursor".to_owned(), + ) + })? + } else { + 0 + }; + if chunk_index > layout.end_chunk { + return Err(MetadError::Codec( + "restore base-reference chunk cursor exceeds borrower size".to_owned(), + )); + } + let mut after = if active == Some(member.destination_inode) { + next.object_cursor + } else { + None + }; + loop { + if references.len() == BASE_REFERENCE_BATCH + || chunks_examined == BASE_CHUNK_SCAN_BATCH + { + next.active_member = Some(member.destination_inode); + next.chunk_cursor = Some(chunk_index); + next.object_cursor = after; + return Ok(RestoreBaseReferencePage { + next, + references, + member_predicates, + complete: false, + }); + } + if chunk_cache.as_ref().is_none_or(|cache| { + cache.inode != member.destination_inode || cache.chunk_index != chunk_index + }) { + *chunk_cache = Some(RestoreChunkReferenceCache { + inode: member.destination_inode, + chunk_index, + references: self.restore_member_base_references_for_chunk( + operation, + &member, + &layout, + chunk_index, + version, + )?, + }); + } + let cached = &chunk_cache + .as_ref() + .expect("restore chunk cache installed") + .references; + let start = cached.partition_point(|(digest, _)| { + after.is_some_and(|cursor| *digest <= cursor) + }); + let remaining = BASE_REFERENCE_BATCH - references.len(); + let end = start.saturating_add(remaining).min(cached.len()); + references.extend( + cached[start..end] + .iter() + .map(|(_, reference)| reference.clone()), + ); + if end < cached.len() { + let digest = + cached + .get(end - 1) + .map(|(digest, _)| *digest) + .ok_or_else(|| { + MetadError::Codec( + "restore member reference page made no cursor progress" + .to_owned(), + ) + })?; + next.active_member = Some(member.destination_inode); + next.chunk_cursor = Some(chunk_index); + next.object_cursor = Some(digest); + return Ok(RestoreBaseReferencePage { + next, + references, + member_predicates, + complete: false, + }); + } + + chunks_examined += 1; + after = None; + *chunk_cache = None; + if chunk_index == layout.end_chunk { + next.completed_member = Some(member.destination_inode); + next.active_member = None; + next.chunk_cursor = None; + next.object_cursor = None; + break; + } + chunk_index = chunk_index.checked_add(1).ok_or_else(|| { + MetadError::Codec("restore base-reference chunk cursor overflow".to_owned()) + })?; + next.active_member = Some(member.destination_inode); + next.chunk_cursor = Some(chunk_index); + next.object_cursor = None; + } + if references.len() == BASE_REFERENCE_BATCH + || members_examined == BASE_MEMBER_SCAN_BATCH + || chunks_examined == BASE_CHUNK_SCAN_BATCH + { + return Ok(RestoreBaseReferencePage { + next, + references, + member_predicates, + complete: false, + }); + } + } + if reached_tail { + return Ok(RestoreBaseReferencePage { + next, + references, + member_predicates, + complete: true, + }); + } + } + } + + pub(super) fn restore_member_base_reference_layout( + &self, + member: &super::restore::RestoreStagingMember, + version: Version, + ) -> Result, MetadError> { + let attr = self + .get_attr_at_version_for_purpose( + member.destination_inode, + version, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreRootChanged { + root: member.destination_inode, + })?; + let summary_key = chunk_manifest_key( + self.mount, + member.destination_inode, + attr.generation, + BODY_SUMMARY_CHUNK_INDEX, + ); + let Some(body) = self.metadata.get( + RecordFamily::ChunkManifest, + &summary_key, + version, + ReadPurpose::RestoreStaging, + )? + else { + return Ok(None); + }; + let body = decode_body_descriptor(&body.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if body.generation != attr.generation { + return Err(MetadError::Codec( + "restore borrower body generation changed identity".to_owned(), + )); + } + if body.chunk_size == 0 || body.block_size == 0 { + return Err(ObjectError::InvalidChunkLayout.into()); + } + if body.size == 0 { + return Ok(None); + } + Ok(Some(RestoreMemberBaseReferenceLayout { + borrower_generation: body.generation, + generation_chain: self.resolve_generation_chain( + member.destination_inode, + &body, + version, + ReadPurpose::RestoreStaging, + )?, + end_chunk: (body.size - 1) / body.chunk_size, + })) + } + + pub(super) fn restore_member_base_references_for_chunk( + &self, + operation: &RestoreOperation, + member: &super::restore::RestoreStagingMember, + layout: &RestoreMemberBaseReferenceLayout, + chunk_index: u64, + version: Version, + ) -> Result, MetadError> { + if chunk_index > layout.end_chunk { + return Err(MetadError::Codec( + "restore borrower chunk exceeds effective body".to_owned(), + )); + } + let Some(manifest) = self.chain_chunk_manifest( + member.destination_inode, + &layout.generation_chain, + chunk_index, + version, + ReadPurpose::RestoreStaging, + )? + else { + return Ok(Vec::new()); + }; + if manifest.chunk_index != chunk_index { + return Err(MetadError::Codec( + "restore borrower manifest changed chunk identity".to_owned(), + )); + } + let manifest_len = usize::try_from(manifest.len).map_err(|_| ObjectError::InvalidRange)?; + let plan = plan_chunk_manifest_reads( + std::slice::from_ref(&manifest), + manifest.logical_offset, + manifest_len, + )?; + let mut selected = BTreeMap::<[u8; 32], RestoreBaseReference>::new(); + for block in plan.blocks { + if self.owns_block_object_key( + member.destination_inode, + layout.borrower_generation, + &block.object_key, + ) { + continue; + } + let (_, _, object_chunk, _) = + self.canonical_block_object_identity(&block.object_key)?; + if object_chunk != chunk_index { + return Err(MetadError::Codec( + "restore borrower object changed chunk identity".to_owned(), + )); + } + let digest: [u8; 32] = Sha256::digest(block.object_key.as_bytes()).into(); + let reference = RestoreBaseReference { + operation_digest: operation.operation_digest, + ref_set_id: operation.ref_set_id, + borrower_inode: member.destination_inode, + borrower_generation: layout.borrower_generation, + object_key: block.object_key, + digest_uri: block.digest_uri, + size: block.object_len, + }; + if let Some(existing) = selected.get(&digest) { + if existing != &reference { + return Err(MetadError::Codec( + "restore object-key digest collision or inconsistent borrower".to_owned(), + )); + } + continue; + } + selected.insert(digest, reference); + } + Ok(selected.into_iter().collect()) + } + + fn validate_restore_base_reference_identities( + &self, + operation: &RestoreOperation, + references: &[RestoreBaseReference], + version: Version, + ) -> Result<(), MetadError> { + let mut batch = BTreeMap::new(); + for reference in references { + let object_digest: [u8; 32] = Sha256::digest(reference.object_key.as_bytes()).into(); + let identity = ( + reference.object_key.clone(), + self.canonical_block_object_identity(&reference.object_key)?, + reference.digest_uri.clone(), + reference.size, + ); + if let Some(existing) = batch.insert(object_digest, identity.clone()) { + if existing != identity { + return Err(MetadError::Codec( + "restore object-key digest collision or inconsistent object identity" + .to_owned(), + )); + } + } + let inverse_rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_base_inverse_prefix(self.mount, &object_digest), + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })?; + let Some(inverse_row) = inverse_rows.first() else { + continue; + }; + let inverse = decode_restore_base_inverse(&inverse_row.value.0)?; + if inverse.object_digest != object_digest { + return Err(MetadError::Codec( + "restore base inverse changed object identity".to_owned(), + )); + } + let owner = self + .metadata + .get( + RecordFamily::System, + &restore_base_owner_key( + self.mount, + inverse.ref_set_id, + &object_digest, + inverse.borrower_inode, + inverse.borrower_generation, + ), + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore base inverse has no canonical owner".to_owned()) + })?; + let owner = decode_restore_base_reference(&owner.0)?; + let owner_identity = ( + owner.object_key.clone(), + self.canonical_block_object_identity(&owner.object_key)?, + owner.digest_uri.clone(), + owner.size, + ); + if owner_identity != identity { + return Err(MetadError::Codec( + "restore object-key digest collision or inconsistent object identity" + .to_owned(), + )); + } + if owner.operation_digest == operation.operation_digest + && owner.ref_set_id == operation.ref_set_id + && owner.borrower_inode == reference.borrower_inode + && owner.borrower_generation == reference.borrower_generation + { + return Err(MetadError::Codec( + "restore base-reference builder revisited a durable owner".to_owned(), + )); + } + } + Ok(()) + } + + pub(super) fn mark_restore_ready_to_attach( + &self, + operation: &RestoreOperation, + ) -> Result<(), MetadError> { + let object_reference = self.begin_object_reference_mutation()?; + let read_version = self.read_version()?; + let operation_key = restore_operation_key(self.mount, &operation.operation_digest); + let operation_item = self + .metadata + .get_versioned( + RecordFamily::System, + &operation_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreInProgress)?; + let durable = decode_restore_operation(&operation_item.value.0)?; + if durable != *operation || durable.state != RestoreOperationState::Preparing { + return Err(MetadError::RestoreInProgress); + } + let binding_key = fork_binding_key(self.mount, operation.destination_root); + let binding = self + .metadata + .get_versioned( + RecordFamily::ForkBinding, + &binding_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::RestoreBindingChanged { + root: operation.destination_root, + })?; + let decoded_binding = + crate::layout::decode_fork_binding(&binding.value.0).map_err(|_| { + MetadError::RestoreBindingChanged { + root: operation.destination_root, + } + })?; + if binding.version != Version::new(operation.created_version)? + || decoded_binding.fork_root != operation.destination_root + || decoded_binding.source_root != operation.source_root + || decoded_binding.pinned_read_version != operation.read_version + || decoded_binding.snapshot_id != operation.snapshot_id + || decoded_binding.created_version != operation.created_version + { + return Err(MetadError::RestoreBindingChanged { + root: operation.destination_root, + }); + } + let mut predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::VersionEquals(operation_item.version), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key, + predicate: Predicate::VersionEquals(binding.version), + }, + self.restore_base_seal_predicate(operation, read_version)?, + self.restore_index_seal_predicate(operation, read_version)?, + ]; + let mut ready = durable; + ready.state = RestoreOperationState::ReadyToAttach; + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-ready-to-attach", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: operation_key.clone(), + predicates: std::mem::take(&mut predicates), + mutations: vec![Mutation { + family: RecordFamily::System, + key: operation_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_operation(&ready)?)), + }], + watch: Vec::new(), + }; + super::restore::validate_restore_command_bounds(&command, "restore ready-to-attach")?; + self.commit_metadata(command)?; + Ok(()) + } + + /// Fail closed while any exact inverse row exists and is consistent with its + /// owner and operation records. Corruption is surfaced to the generic GC + /// worker, which quarantines the candidate instead of issuing DELETE. + pub(super) fn restore_object_is_borrowed(&self, object_key: &str) -> Result { + let object_digest: [u8; 32] = Sha256::digest(object_key.as_bytes()).into(); + let version = self.read_version()?; + let object_quarantine = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: super::restore::restore_release_object_quarantine_prefix( + self.mount, + &object_digest, + ), + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })?; + let mount_wide_quarantine = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: super::restore::restore_release_mount_wide_quarantine_prefix(self.mount), + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })?; + if !object_quarantine.is_empty() || !mount_wide_quarantine.is_empty() { + // An object-scoped malformed owner row can hide this candidate's + // only durable borrower identity. Mount-wide quarantine is reserved + // for the rarer case where even the canonical owner-key digest is + // unrecoverable. Diagnostic member/job quarantines do not block + // unrelated object GC. + return Ok(true); + } + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_base_inverse_prefix(self.mount, &object_digest), + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })?; + let Some(row) = rows.first() else { + return Ok(false); + }; + let inverse = decode_restore_base_inverse(&row.value.0)?; + if inverse.object_digest != object_digest { + return Err(MetadError::Codec( + "restore inverse row object digest mismatch".to_owned(), + )); + } + let owner_key = restore_base_owner_key( + self.mount, + inverse.ref_set_id, + &inverse.object_digest, + inverse.borrower_inode, + inverse.borrower_generation, + ); + let owner = self + .metadata + .get( + RecordFamily::System, + &owner_key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("restore inverse row has no owner row".to_owned()))?; + let owner = decode_restore_base_reference(&owner.0)?; + if owner.operation_digest != inverse.operation_digest + || owner.ref_set_id != inverse.ref_set_id + || owner.borrower_inode != inverse.borrower_inode + || owner.borrower_generation != inverse.borrower_generation + || owner.object_key != object_key + { + return Err(MetadError::Codec( + "restore inverse/owner row mismatch".to_owned(), + )); + } + let inverse_owner_key = super::restore::restore_base_inverse_owner_key( + self.mount, + inverse.ref_set_id, + &inverse.object_digest, + inverse.borrower_inode, + inverse.borrower_generation, + ); + let inverse_owner = self + .metadata + .get( + RecordFamily::System, + &inverse_owner_key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore inverse has no ref-set owner row".to_owned()) + })?; + if decode_restore_base_inverse(&inverse_owner.0)? != inverse { + return Err(MetadError::Codec( + "restore inverse/ref-set owner row mismatch".to_owned(), + )); + } + let operation = self + .metadata + .get( + RecordFamily::System, + &restore_operation_key(self.mount, &inverse.operation_digest), + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore base reference has no operation".to_owned()) + })?; + let operation = decode_restore_operation(&operation.0)?; + if operation.ref_set_id != inverse.ref_set_id { + return Err(MetadError::Codec( + "restore base reference set does not match operation".to_owned(), + )); + } + Ok(true) + } +} + +pub(super) fn initial_restore_base_reference_digest() -> [u8; 32] { + [0; 32] +} + +pub(super) fn extend_restore_base_reference_digest( + previous: [u8; 32], + reference: &RestoreBaseReference, +) -> Result<[u8; 32], MetadError> { + let encoded = encode_restore_base_reference(reference)?; + let mut leaf = Sha256::new(); + leaf.update(b"nokv-restore-base-reference-set-leaf-v2\0"); + leaf.update((encoded.len() as u64).to_be_bytes()); + leaf.update(encoded); + let leaf: [u8; 32] = leaf.finalize().into(); + Ok(std::array::from_fn(|index| previous[index] ^ leaf[index])) +} + +pub(super) fn finalize_restore_base_reference_digest( + reference_count: u64, + accumulator: [u8; 32], +) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(b"nokv-restore-base-reference-set-seal-v2\0"); + hasher.update(reference_count.to_be_bytes()); + hasher.update(accumulator); + hasher.finalize().into() +} + +fn validate_restore_base_build( + operation: &RestoreOperation, + build: &RestoreBaseBuild, +) -> Result<(), MetadError> { + if build.operation_digest != operation.operation_digest + || build.initialization_digest != operation.initialization_digest + || build.ref_set_id != operation.ref_set_id + || build.incarnation != operation.created_version + { + return Err(MetadError::Codec( + "restore base-reference build changed identity".to_owned(), + )); + } + if build.active_member.is_some() != build.chunk_cursor.is_some() + || (build.object_cursor.is_some() && build.active_member.is_none()) + || build + .completed_member + .zip(build.active_member) + .is_some_and(|(completed, active)| active <= completed) + || (build.reference_count == 0 + && build.reference_digest != initial_restore_base_reference_digest()) + { + return Err(MetadError::Codec( + "restore base-reference build cursor is invalid".to_owned(), + )); + } + Ok(()) +} + +fn validate_restore_base_seal( + operation: &RestoreOperation, + seal: &RestoreBaseSeal, +) -> Result<(), MetadError> { + if seal.operation_digest != operation.operation_digest + || seal.initialization_digest != operation.initialization_digest + || seal.ref_set_id != operation.ref_set_id + || seal.incarnation != operation.created_version + || (seal.reference_count == 0 + && seal.reference_digest + != finalize_restore_base_reference_digest( + 0, + initial_restore_base_reference_digest(), + )) + { + return Err(MetadError::Codec( + "restore base-reference seal changed identity".to_owned(), + )); + } + Ok(()) +} + +pub(super) fn encode_restore_base_build(build: &RestoreBaseBuild) -> Result, MetadError> { + if build.ref_set_id == 0 || build.incarnation == 0 { + return Err(MetadError::Codec( + "restore base-reference build contains a zero identity".to_owned(), + )); + } + if build.active_member.is_some() != build.chunk_cursor.is_some() + || (build.object_cursor.is_some() && build.active_member.is_none()) + || build + .completed_member + .zip(build.active_member) + .is_some_and(|(completed, active)| active <= completed) + { + return Err(MetadError::Codec( + "restore base-reference build cursor is invalid".to_owned(), + )); + } + let mut out = Vec::with_capacity(189); + out.push(BASE_REFERENCE_BUILD_FORMAT_VERSION); + out.extend_from_slice(&build.operation_digest); + out.extend_from_slice(&build.initialization_digest); + out.extend_from_slice(&build.ref_set_id.to_be_bytes()); + out.extend_from_slice(&build.incarnation.to_be_bytes()); + for inode in [build.completed_member, build.active_member] { + out.push(u8::from(inode.is_some())); + out.extend_from_slice(&inode.map_or(0, InodeId::get).to_be_bytes()); + } + out.push(u8::from(build.chunk_cursor.is_some())); + out.extend_from_slice(&build.chunk_cursor.unwrap_or(0).to_be_bytes()); + out.push(u8::from(build.object_cursor.is_some())); + out.extend_from_slice(&build.object_cursor.unwrap_or([0; 32])); + out.extend_from_slice(&build.reference_count.to_be_bytes()); + out.extend_from_slice(&build.reference_digest); + out.extend_from_slice(&build.batch_index.to_be_bytes()); + Ok(out) +} + +pub(super) fn decode_restore_base_build(bytes: &[u8]) -> Result { + let mut decoder = BaseDecoder::new(bytes); + if decoder.u8()? != BASE_REFERENCE_BUILD_FORMAT_VERSION { + return Err(MetadError::Codec( + "unsupported restore base-reference build version".to_owned(), + )); + } + let operation_digest = decoder.array_32()?; + let initialization_digest = decoder.array_32()?; + let ref_set_id = decoder.u64()?; + let incarnation = decoder.u64()?; + let decode_inode = |decoder: &mut BaseDecoder<'_>| -> Result, MetadError> { + match (decoder.u8()?, decoder.u64()?) { + (0, 0) => Ok(None), + (1, raw) => Ok(Some(InodeId::new(raw)?)), + _ => Err(MetadError::Codec( + "restore base-reference build has an invalid inode cursor".to_owned(), + )), + } + }; + let completed_member = decode_inode(&mut decoder)?; + let active_member = decode_inode(&mut decoder)?; + let chunk_cursor = match (decoder.u8()?, decoder.u64()?) { + (0, 0) => None, + (1, raw) => Some(raw), + _ => { + return Err(MetadError::Codec( + "restore base-reference build has an invalid chunk cursor".to_owned(), + )) + } + }; + let object_tag = decoder.u8()?; + let object_raw = decoder.array_32()?; + let object_cursor = match object_tag { + 0 if object_raw == [0; 32] => None, + 1 => Some(object_raw), + _ => { + return Err(MetadError::Codec( + "restore base-reference build has an invalid object cursor".to_owned(), + )) + } + }; + let build = RestoreBaseBuild { + operation_digest, + initialization_digest, + ref_set_id, + incarnation, + completed_member, + active_member, + chunk_cursor, + object_cursor, + reference_count: decoder.u64()?, + reference_digest: decoder.array_32()?, + batch_index: decoder.u64()?, + }; + decoder.finish()?; + if build.ref_set_id == 0 + || build.incarnation == 0 + || build.active_member.is_some() != build.chunk_cursor.is_some() + || (build.object_cursor.is_some() && build.active_member.is_none()) + || build + .completed_member + .zip(build.active_member) + .is_some_and(|(completed, active)| active <= completed) + { + return Err(MetadError::Codec( + "restore base-reference build contains an invalid identity".to_owned(), + )); + } + Ok(build) +} + +pub(super) fn decode_restore_base_seal_record( + bytes: &[u8], +) -> Result { + match bytes.first().copied() { + Some(BASE_REFERENCE_FORMAT_VERSION) => { + decode_restore_base_seal(bytes).map(RestoreBaseSealRecord::Sealed) + } + Some(BASE_REFERENCE_BUILD_FORMAT_VERSION) => { + decode_restore_base_build(bytes).map(RestoreBaseSealRecord::Building) + } + _ => Err(MetadError::Codec( + "unsupported restore base seal/progress version".to_owned(), + )), + } +} + +fn encode_restore_base_reference(reference: &RestoreBaseReference) -> Result, MetadError> { + if reference.ref_set_id == 0 || reference.borrower_generation == 0 { + return Err(MetadError::Codec( + "restore base reference contains a zero identity".to_owned(), + )); + } + let object_key = reference.object_key.as_bytes(); + let digest_uri = reference.digest_uri.as_bytes(); + let object_len = u32::try_from(object_key.len()) + .map_err(|_| MetadError::Codec("restore object key is too long".to_owned()))?; + let digest_len = u32::try_from(digest_uri.len()) + .map_err(|_| MetadError::Codec("restore digest URI is too long".to_owned()))?; + let mut out = Vec::with_capacity(102 + object_key.len() + digest_uri.len()); + out.push(BASE_REFERENCE_FORMAT_VERSION); + out.extend_from_slice(&reference.operation_digest); + out.extend_from_slice(&reference.ref_set_id.to_be_bytes()); + out.extend_from_slice(&reference.borrower_inode.get().to_be_bytes()); + out.extend_from_slice(&reference.borrower_generation.to_be_bytes()); + out.extend_from_slice(&reference.size.to_be_bytes()); + out.extend_from_slice(&object_len.to_be_bytes()); + out.extend_from_slice(object_key); + out.extend_from_slice(&digest_len.to_be_bytes()); + out.extend_from_slice(digest_uri); + Ok(out) +} + +pub(super) fn decode_restore_base_reference( + bytes: &[u8], +) -> Result { + let mut decoder = BaseDecoder::new(bytes); + if decoder.u8()? != BASE_REFERENCE_FORMAT_VERSION { + return Err(MetadError::Codec( + "unsupported restore base-reference version".to_owned(), + )); + } + let reference = RestoreBaseReference { + operation_digest: decoder.array_32()?, + ref_set_id: decoder.u64()?, + borrower_inode: InodeId::new(decoder.u64()?)?, + borrower_generation: decoder.u64()?, + size: decoder.u64()?, + object_key: decoder.string()?, + digest_uri: decoder.string()?, + }; + decoder.finish()?; + if reference.ref_set_id == 0 || reference.borrower_generation == 0 { + return Err(MetadError::Codec( + "restore base reference contains a zero identity".to_owned(), + )); + } + Ok(reference) +} + +fn encode_restore_base_inverse(inverse: &RestoreBaseInverse) -> Vec { + let mut out = Vec::with_capacity(89); + out.push(BASE_REFERENCE_FORMAT_VERSION); + out.extend_from_slice(&inverse.operation_digest); + out.extend_from_slice(&inverse.ref_set_id.to_be_bytes()); + out.extend_from_slice(&inverse.object_digest); + out.extend_from_slice(&inverse.borrower_inode.get().to_be_bytes()); + out.extend_from_slice(&inverse.borrower_generation.to_be_bytes()); + out +} + +pub(super) fn decode_restore_base_inverse(bytes: &[u8]) -> Result { + let mut decoder = BaseDecoder::new(bytes); + if decoder.u8()? != BASE_REFERENCE_FORMAT_VERSION { + return Err(MetadError::Codec( + "unsupported restore base-inverse version".to_owned(), + )); + } + let inverse = RestoreBaseInverse { + operation_digest: decoder.array_32()?, + ref_set_id: decoder.u64()?, + object_digest: decoder.array_32()?, + borrower_inode: InodeId::new(decoder.u64()?)?, + borrower_generation: decoder.u64()?, + }; + decoder.finish()?; + if inverse.ref_set_id == 0 || inverse.borrower_generation == 0 { + return Err(MetadError::Codec( + "restore base inverse contains a zero identity".to_owned(), + )); + } + Ok(inverse) +} + +pub(super) fn encode_restore_base_seal(seal: &RestoreBaseSeal) -> Vec { + let mut out = Vec::with_capacity(121); + out.push(BASE_REFERENCE_FORMAT_VERSION); + out.extend_from_slice(&seal.operation_digest); + out.extend_from_slice(&seal.initialization_digest); + out.extend_from_slice(&seal.ref_set_id.to_be_bytes()); + out.extend_from_slice(&seal.incarnation.to_be_bytes()); + out.extend_from_slice(&seal.reference_count.to_be_bytes()); + out.extend_from_slice(&seal.reference_digest); + out +} + +pub(super) fn decode_restore_base_seal(bytes: &[u8]) -> Result { + let mut decoder = BaseDecoder::new(bytes); + if decoder.u8()? != BASE_REFERENCE_FORMAT_VERSION { + return Err(MetadError::Codec( + "unsupported restore base-seal version".to_owned(), + )); + } + let seal = RestoreBaseSeal { + operation_digest: decoder.array_32()?, + initialization_digest: decoder.array_32()?, + ref_set_id: decoder.u64()?, + incarnation: decoder.u64()?, + reference_count: decoder.u64()?, + reference_digest: decoder.array_32()?, + }; + decoder.finish()?; + if seal.ref_set_id == 0 || seal.incarnation == 0 { + return Err(MetadError::Codec( + "restore base seal contains a zero identity".to_owned(), + )); + } + Ok(seal) +} + +struct BaseDecoder<'a> { + input: &'a [u8], +} + +impl<'a> BaseDecoder<'a> { + fn new(input: &'a [u8]) -> Self { + Self { input } + } + + fn take(&mut self, len: usize) -> Result<&'a [u8], MetadError> { + let Some((head, tail)) = self.input.split_at_checked(len) else { + return Err(MetadError::Codec( + "restore base-reference record is truncated".to_owned(), + )); + }; + self.input = tail; + Ok(head) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn u32(&mut self) -> Result { + Ok(u32::from_be_bytes( + self.take(4)?.try_into().expect("u32 width"), + )) + } + + fn u64(&mut self) -> Result { + Ok(u64::from_be_bytes( + self.take(8)?.try_into().expect("u64 width"), + )) + } + + fn array_32(&mut self) -> Result<[u8; 32], MetadError> { + Ok(self.take(32)?.try_into().expect("digest width")) + } + + fn string(&mut self) -> Result { + let len = self.u32()? as usize; + String::from_utf8(self.take(len)?.to_vec()) + .map_err(|_| MetadError::Codec("restore base-reference string is not utf-8".to_owned())) + } + + fn finish(self) -> Result<(), MetadError> { + if self.input.is_empty() { + Ok(()) + } else { + Err(MetadError::Codec( + "restore base-reference record has trailing bytes".to_owned(), + )) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::holtstore::HoltMetadataStore; + use nokv_object::MemoryObjectStore; + + #[test] + fn base_reference_codecs_fail_closed() { + let reference = RestoreBaseReference { + operation_digest: [1; 32], + ref_set_id: 9, + borrower_inode: InodeId::new(7).unwrap(), + borrower_generation: 11, + object_key: "blocks/1/2/3/4/5".to_owned(), + digest_uri: "sha256:abc".to_owned(), + size: 12, + }; + let encoded = encode_restore_base_reference(&reference).unwrap(); + assert_eq!(decode_restore_base_reference(&encoded).unwrap(), reference); + assert!(decode_restore_base_reference(&encoded[..encoded.len() - 1]).is_err()); + + let seal = RestoreBaseSeal { + operation_digest: [2; 32], + initialization_digest: [3; 32], + ref_set_id: 9, + incarnation: 10, + reference_count: 4, + reference_digest: [5; 32], + }; + assert_eq!( + decode_restore_base_seal(&encode_restore_base_seal(&seal)).unwrap(), + seal + ); + + let build = RestoreBaseBuild { + operation_digest: [6; 32], + initialization_digest: [7; 32], + ref_set_id: 12, + incarnation: 13, + completed_member: Some(InodeId::new(20).unwrap()), + active_member: Some(InodeId::new(21).unwrap()), + chunk_cursor: Some(4), + object_cursor: Some([8; 32]), + reference_count: 65, + reference_digest: [9; 32], + batch_index: 2, + }; + let encoded = encode_restore_base_build(&build).unwrap(); + assert_eq!(decode_restore_base_build(&encoded).unwrap(), build); + assert!(decode_restore_base_build(&encoded[..encoded.len() - 1]).is_err()); + + let mut invalid = build; + invalid.active_member = None; + assert!(encode_restore_base_build(&invalid).is_err()); + } + + #[test] + fn base_reference_packer_rejects_one_reference_larger_than_the_command_budget() { + let service = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + MemoryObjectStore::new(), + ); + let operation = RestoreOperation { + operation_digest: [7; 32], + initialization_digest: [8; 32], + state: RestoreOperationState::Preparing, + source_root: InodeId::new(2).unwrap(), + destination_root: InodeId::new(3).unwrap(), + snapshot_id: 4, + read_version: 5, + created_version: 6, + ref_set_id: 6, + source_path: "/source".to_owned(), + destination_path: "/destination".to_owned(), + }; + let build = service + .begin_restore_base_reference_preflight(&operation) + .build; + let page = RestoreBaseReferencePage { + next: build.clone(), + references: vec![RestoreBaseReference { + operation_digest: operation.operation_digest, + ref_set_id: operation.ref_set_id, + borrower_inode: InodeId::new(9).unwrap(), + borrower_generation: 10, + object_key: "blocks/1/2/10/0/0".to_owned(), + digest_uri: "d".repeat(8 * 1024 * 1024), + size: 1, + }], + member_predicates: Vec::new(), + complete: false, + }; + let binding_key = fork_binding_key(service.mount, operation.destination_root); + let seal_key = restore_base_seal_key(service.mount, operation.ref_set_id); + + // A public restore carrying this descriptor is normally rejected even + // earlier by the larger materialization-command preflight. Exercise the + // shared exact-reference packer directly so its own one-item no-progress + // boundary remains independently fail-closed. + let error = service + .fit_restore_base_reference_page( + &operation, + Version::new(operation.created_version).unwrap(), + &binding_key, + &seal_key, + Version::new(1).unwrap(), + &build, + page, + false, + ) + .unwrap_err(); + assert!(matches!( + error, + MetadError::RestoreResourceLimit { + resource, + limit, + actual, + } if resource == "restore base-reference batch bytes" + && limit == 8 * 1024 * 1024 + && actual > limit + )); + } +} diff --git a/crates/nokv-meta/src/service/restore_index.rs b/crates/nokv-meta/src/service/restore_index.rs new file mode 100644 index 000000000..4eae7f361 --- /dev/null +++ b/crates/nokv-meta/src/service/restore_index.rs @@ -0,0 +1,7765 @@ +//! Visibility fences and operation-scoped indexes for detached restores. +//! +//! Restore indexes deliberately live in the private `System` family. A row is +//! useful only while its owning operation is durably `Complete`; writing rows +//! during detached materialization therefore cannot make the destination +//! visible early. The owner-side keyspace makes release page-able, while the +//! inverse keyspaces let normal namespace mutations find every nested restore +//! that owns an index row for a dentry. + +use super::restore::{ + decode_restore_operation, decode_restore_staging_inverse, decode_restore_staging_member, + encode_restore_staging_member, restore_operation_key, restore_staging_inode_key, + restore_staging_member_key, restore_staging_member_prefix, RestoreOperation, + RestoreOperationState, RestoreStagingMember, MAX_RESTORE_SUBTREE_ENTRIES, + RESTORE_BATCH_ENTRIES, RESTORE_FORMAT_VERSION, +}; +use super::*; + +const INDEX_ENTRY_MAGIC: &[u8; 8] = b"NKRIDXE\0"; +const INDEX_CATALOG_MAGIC: &[u8; 8] = b"NKRIDXC\0"; +const INDEX_ROW_MAGIC: &[u8; 8] = b"NKRIDXR\0"; +const INDEX_SEAL_MAGIC: &[u8; 8] = b"NKRIDXS\0"; +const INDEX_MVCC_MAGIC: &[u8; 8] = b"NKRIDXM\0"; +const INDEX_COMPLETE_MAGIC: &[u8; 8] = b"NKRIDXP\0"; +const INDEX_SOURCE_MEMBER_MAGIC: &[u8; 8] = b"NKRIDXI\0"; + +const INDEX_ENTRY_LABEL: &[u8] = b"restore-index-entry\0"; +const INDEX_PARENT_OWNER_LABEL: &[u8] = b"restore-index-parent-owner\0"; +const INDEX_PARENT_INVERSE_LABEL: &[u8] = b"restore-index-parent-inverse\0"; +const INDEX_CATALOG_LABEL: &[u8] = b"restore-index-catalog\0"; +const INDEX_CATALOG_INVERSE_LABEL: &[u8] = b"restore-index-catalog-inverse\0"; +const INDEX_ROW_LABEL: &[u8] = b"restore-index-row\0"; +const INDEX_TARGET_INVERSE_LABEL: &[u8] = b"restore-index-target-inverse\0"; +const INDEX_SEAL_LABEL: &[u8] = b"restore-index-seal\0"; +const INDEX_COMPLETE_LABEL: &[u8] = b"restore-index-complete\0"; +const INDEX_SOURCE_MEMBER_LABEL: &[u8] = b"restore-index-source-member\0"; + +// `RecordFamily::System` intentionally has no Holt history. Every mutable +// restore-index head therefore has an append-only physical history keyspace. +// The physical key mirrors the logical key layout and appends the metadata +// command's commit version. This keeps parent/target and ref-set scans +// bounded without enabling history for unrelated System records. +const INDEX_MVCC_ENTRY_LABEL: &[u8] = b"restore-index-mvcc-entry\0"; +const INDEX_MVCC_PARENT_OWNER_LABEL: &[u8] = b"restore-index-mvcc-parent-owner\0"; +const INDEX_MVCC_PARENT_INVERSE_LABEL: &[u8] = b"restore-index-mvcc-parent-inverse\0"; +const INDEX_MVCC_CATALOG_LABEL: &[u8] = b"restore-index-mvcc-catalog\0"; +const INDEX_MVCC_CATALOG_INVERSE_LABEL: &[u8] = b"restore-index-mvcc-catalog-inverse\0"; +const INDEX_MVCC_ROW_LABEL: &[u8] = b"restore-index-mvcc-row\0"; +const INDEX_MVCC_TARGET_INVERSE_LABEL: &[u8] = b"restore-index-mvcc-target-inverse\0"; +const INDEX_MVCC_SOURCE_MEMBER_LABEL: &[u8] = b"restore-index-mvcc-source-member\0"; + +const OWNER_BYTES: usize = 40; +const RESTORE_INDEX_SCAN_PAGE: usize = 256; +// The namespace page helper asks the backing store for `requested + 1` rows to +// determine whether a next cursor exists. Keep that physical scan within the +// same 256-row bound as every private restore-index scan. +const RESTORE_INDEX_DENTRY_PAGE: usize = RESTORE_INDEX_SCAN_PAGE - 1; +// Keep this in lock-step with `restore::validate_restore_command_bounds`. +// Overlay plans are merged into an ordinary namespace command, so they must +// fail before the caller can publish a namespace mutation that cannot carry +// the complete owner/inverse/MVCC update atomically. +const MAX_RESTORE_INDEX_PLAN_ITEMS: usize = 4_096; +const MAX_RESTORE_INDEX_PLAN_BYTES: usize = 8 * 1024 * 1024; +const RESTORE_INDEX_MVCC_VERSION_DELIMITER: u8 = 0; +const MAX_RESTORE_INDEX_INSPECTION_REF_SETS: usize = 65_536; +const MAX_RESTORE_INDEX_INSPECTION_ISSUES: usize = 4_096; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct RestoreIndexOwner { + operation_digest: [u8; 32], + ref_set_id: u64, +} + +impl RestoreIndexOwner { + fn from_operation(operation: &RestoreOperation) -> Self { + Self { + operation_digest: operation.operation_digest, + ref_set_id: operation.ref_set_id, + } + } + + fn validate(self) -> Result { + if self.ref_set_id == 0 { + return Err(MetadError::Codec( + "restore index owner has a zero ref-set id".to_owned(), + )); + } + Ok(self) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RestoreIndexEntry { + owner: RestoreIndexOwner, + projection: DentryProjection, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RestoreIndexCatalog { + owner: RestoreIndexOwner, + catalog_root: InodeId, + record: PathIndexCatalogRecord, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RestoreIndexTarget { + parent: Option, + name: Vec, + inode: InodeId, + file_type: FileType, + attr_generation: u64, + body_digest: Option<[u8; 32]>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RestoreIndexRow { + owner: RestoreIndexOwner, + catalog_root: InodeId, + target: RestoreIndexTarget, + record: PathIndexRowRecord, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RestoreIndexSourceMember { + owner: RestoreIndexOwner, + source_inode: InodeId, + member: RestoreStagingMember, +} + +/// Source-member rows seal the immutable snapshot-to-destination identity. +/// Release may advance only the live staging member's manifest cursor; that +/// progress must not invalidate fsck or prevent retaining a reachable source +/// row. Canonical-index cleanup remains exact because it has different +/// enrollment semantics and is never legal for snapshot source members. +fn restore_index_source_member_matches_staging( + source: &RestoreStagingMember, + staging: &RestoreStagingMember, +) -> bool { + source.manifest_cursor.is_empty() + && source.manifest_block_cursor == 0 + && source.operation_digest == staging.operation_digest + && source.source_inode == staging.source_inode + && source.destination_inode == staging.destination_inode + && source.destination_parent == staging.destination_parent + && source.name == staging.name + && source.relative_path == staging.relative_path + && source.canonical_index_cursor == staging.canonical_index_cursor + && source.canonical_index_complete == staging.canonical_index_complete +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RestoreIndexMvccKind { + Put = 1, + Tombstone = 2, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RestoreIndexMvccRecord { + commit_version: u64, + kind: RestoreIndexMvccKind, + logical_key: Vec, + /// A tombstone retains the last full typed value. Release/fsck can thus + /// still prove owner/inverse identity instead of trusting only a digest. + value: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RestoreIndexCompleteMarker { + operation_digest: [u8; 32], + initialization_digest: [u8; 32], + ref_set_id: u64, + incarnation: u64, + complete_version: u64, +} + +#[derive(Clone, Debug)] +struct SelectedRestoreIndexMvcc { + logical_key: Vec, + record: RestoreIndexMvccRecord, +} + +#[derive(Clone, Debug)] +struct RestoreIndexForkSource { + binding: ForkBinding, + operation: RestoreOperation, +} + +type RestoreIndexCounterpart = (&'static [u8], Vec); + +struct RestoreIndexInspectedRowIdentity { + ref_set_id: u64, + operation_digest: [u8; 32], + is_put: bool, + is_tombstone: bool, + seal_identity: Option, + complete_identity: Option, +} + +#[derive(Default)] +struct RestoreIndexInspectionIssueBudget { + emitted: usize, + truncated: bool, +} + +impl RestoreIndexInspectionIssueBudget { + fn push(&mut self, issues: &mut Vec, issue: String) { + if self.emitted < MAX_RESTORE_INDEX_INSPECTION_ISSUES { + issues.push(issue); + self.emitted += 1; + } else { + self.truncated = true; + } + } +} + +struct RestoreIndexReleasePage<'a> { + operation: &'a RestoreOperation, + operation_version: Version, + stage: RestoreIndexReleaseStage, + logical_prefix: Vec, + start_after: Option>, + limit: usize, + version: Version, +} + +#[derive(Clone, Copy)] +struct RestoreIndexChildrenQuery<'a> { + parent: InodeId, + after: Option<&'a DentryName>, + keep: usize, + version: Version, + purpose: ReadPurpose, +} + +#[derive(Clone, Debug)] +struct VersionedRestoreIndexEntry { + key: Vec, + version: Version, + entry: RestoreIndexEntry, + operation_key: Vec, + operation_version: Version, +} + +#[derive(Clone, Debug)] +struct VersionedRestoreIndexRow { + owner_key: Vec, + owner_version: Version, + inverse_key: Vec, + inverse_version: Version, + row: RestoreIndexRow, + operation_key: Vec, + operation_version: Version, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreIndexSeal { + pub(super) operation_digest: [u8; 32], + pub(super) initialization_digest: [u8; 32], + pub(super) ref_set_id: u64, + pub(super) incarnation: u64, + pub(super) entry_count: u64, + pub(super) catalog_count: u64, + pub(super) row_count: u64, + pub(super) digest: [u8; 32], +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(super) struct RestoreIndexMutationPlan { + pub(super) predicates: Vec, + pub(super) mutations: Vec, +} + +#[derive(Default)] +pub(super) struct RestoreCanonicalIndexPreflightBatch { + members: usize, + desired: Vec, +} + +impl RestoreIndexMutationPlan { + pub(super) fn extend(&mut self, other: Self) -> Result<(), MetadError> { + for predicate in other.predicates { + self.push_predicate(predicate)?; + } + for mutation in other.mutations { + self.set_mutation(mutation); + } + self.validate_budget("restore index mutation plan") + } + + fn push_predicate(&mut self, predicate: PredicateRef) -> Result<(), MetadError> { + if let Some(existing) = self + .predicates + .iter() + .find(|existing| existing.family == predicate.family && existing.key == predicate.key) + { + if existing.predicate != predicate.predicate { + return Err(MetadError::Codec( + "restore index mutation plan has conflicting predicates".to_owned(), + )); + } + return Ok(()); + } + self.predicates.push(predicate); + Ok(()) + } + + fn set_mutation(&mut self, mutation: Mutation) { + if let Some(existing) = self + .mutations + .iter_mut() + .find(|existing| existing.family == mutation.family && existing.key == mutation.key) + { + *existing = mutation; + } else { + self.mutations.push(mutation); + } + } + + fn validate_budget(&self, resource: &str) -> Result<(), MetadError> { + let items = self.predicates.len().saturating_add(self.mutations.len()); + if items > MAX_RESTORE_INDEX_PLAN_ITEMS { + return Err(MetadError::RestoreResourceLimit { + resource: format!("{resource} items"), + limit: MAX_RESTORE_INDEX_PLAN_ITEMS as u64, + actual: items as u64, + }); + } + let bytes = self + .predicates + .iter() + .fold(0_usize, |total, predicate| { + total.saturating_add(predicate.key.len()).saturating_add(32) + }) + .saturating_add(self.mutations.iter().fold(0_usize, |total, mutation| { + total + .saturating_add(mutation.key.len()) + .saturating_add(mutation.value.as_ref().map_or(0, |value| value.0.len())) + .saturating_add(16) + })); + if bytes > MAX_RESTORE_INDEX_PLAN_BYTES { + return Err(MetadError::RestoreResourceLimit { + resource: format!("{resource} bytes"), + limit: MAX_RESTORE_INDEX_PLAN_BYTES as u64, + actual: bytes as u64, + }); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreCustomIndex { + pub(super) catalog: PathIndexCatalogRecord, + pub(super) rows: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(super) struct RestoreIndexReleaseOutcome { + /// Opaque cursor to persist in the restore release job. An empty cursor + /// means a retained escaped row requires a later full pass. + pub(super) cursor: Vec, + /// True only after every owner/inverse row and the index seal are absent. + pub(super) complete: bool, + /// Reachable escaped rows observed in this page/cycle. + pub(super) retained: usize, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(super) struct RestoreIndexInspection { + pub(super) counts: BTreeMap, + pub(super) mvcc_puts: usize, + pub(super) mvcc_tombstones: usize, + pub(super) ref_sets: BTreeMap, + pub(super) unowned_issues: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(super) struct RestoreIndexRefSetInspection { + pub(super) counts: BTreeMap, + pub(super) operation_digests: std::collections::BTreeSet<[u8; 32]>, + pub(super) seal_identity: Option, + pub(super) complete_identity: Option, + pub(super) closure_issues: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct RestoreIndexDurableIdentity { + pub(super) operation_digest: [u8; 32], + pub(super) initialization_digest: [u8; 32], + pub(super) incarnation: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RestoreIndexReleaseStage { + Entries = 1, + Rows = 2, + Catalogs = 3, + Parents = 4, + MvccEntries = 5, + MvccRows = 6, + MvccCatalogs = 7, + MvccParents = 8, + AuditParentInverses = 9, + AuditCatalogInverses = 10, + AuditTargetInverses = 11, + AuditMvccParentInverses = 12, + AuditMvccCatalogInverses = 13, + AuditMvccTargetInverses = 14, + Seal = 15, + SourceMembers = 16, + MvccSourceMembers = 17, +} + +impl RestoreIndexReleaseStage { + fn decode(tag: u8) -> Result { + match tag { + 1 => Ok(Self::Entries), + 2 => Ok(Self::Rows), + 3 => Ok(Self::Catalogs), + 4 => Ok(Self::Parents), + 5 => Ok(Self::MvccEntries), + 6 => Ok(Self::MvccRows), + 7 => Ok(Self::MvccCatalogs), + 8 => Ok(Self::MvccParents), + 9 => Ok(Self::AuditParentInverses), + 10 => Ok(Self::AuditCatalogInverses), + 11 => Ok(Self::AuditTargetInverses), + 12 => Ok(Self::AuditMvccParentInverses), + 13 => Ok(Self::AuditMvccCatalogInverses), + 14 => Ok(Self::AuditMvccTargetInverses), + 15 => Ok(Self::Seal), + 16 => Ok(Self::SourceMembers), + 17 => Ok(Self::MvccSourceMembers), + _ => Err(MetadError::Codec(format!( + "restore index release cursor has invalid stage {tag}" + ))), + } + } + + fn next(self) -> Option { + match self { + Self::Entries => Some(Self::Rows), + Self::Rows => Some(Self::Catalogs), + Self::Catalogs => Some(Self::Parents), + Self::Parents => Some(Self::MvccEntries), + Self::MvccEntries => Some(Self::MvccRows), + Self::MvccRows => Some(Self::MvccCatalogs), + Self::MvccCatalogs => Some(Self::MvccParents), + Self::MvccParents => Some(Self::AuditParentInverses), + Self::AuditParentInverses => Some(Self::AuditCatalogInverses), + Self::AuditCatalogInverses => Some(Self::AuditTargetInverses), + Self::AuditTargetInverses => Some(Self::AuditMvccParentInverses), + Self::AuditMvccParentInverses => Some(Self::AuditMvccCatalogInverses), + Self::AuditMvccCatalogInverses => Some(Self::AuditMvccTargetInverses), + Self::AuditMvccTargetInverses => Some(Self::SourceMembers), + Self::SourceMembers => Some(Self::MvccSourceMembers), + Self::MvccSourceMembers => Some(Self::Seal), + Self::Seal => None, + } + } +} + +fn encode_restore_index_release_cursor(stage: RestoreIndexReleaseStage, key: &[u8]) -> Vec { + let mut cursor = Vec::with_capacity(1 + key.len()); + cursor.push(stage as u8); + cursor.extend_from_slice(key); + cursor +} + +fn decode_restore_index_release_cursor( + cursor: &[u8], +) -> Result<(RestoreIndexReleaseStage, Option>), MetadError> { + if cursor.is_empty() { + return Ok((RestoreIndexReleaseStage::Entries, None)); + } + let stage = RestoreIndexReleaseStage::decode(cursor[0])?; + Ok((stage, (cursor.len() > 1).then(|| cursor[1..].to_vec()))) +} + +fn restore_index_release_page_outcome( + stage: RestoreIndexReleaseStage, + rows: &[crate::command::ScanItem], + limit: usize, + retained: usize, +) -> RestoreIndexReleaseOutcome { + let cursor = if rows.len() >= limit { + encode_restore_index_release_cursor( + stage, + rows.last().map_or(&[][..], |row| row.key.as_slice()), + ) + } else { + encode_restore_index_release_cursor( + stage + .next() + .expect("all paged restore index release stages have a successor"), + &[], + ) + }; + RestoreIndexReleaseOutcome { + cursor, + complete: false, + retained, + } +} + +fn restore_index_system_key(mount: MountId, label: &[u8]) -> Vec { + let mut key = Vec::with_capacity(8 + label.len()); + key.extend_from_slice(&mount.get().to_be_bytes()); + key.extend_from_slice(label); + key +} + +/// Authoritative registry for every private restore-index keyspace. Downgrade, +/// drain, metrics and fsck must consume this list instead of duplicating labels +/// or assuming System's current view represents append-only MVCC state. +pub(super) fn restore_index_private_keyspaces(mount: MountId) -> Vec<(&'static str, Vec)> { + [ + ("index_entry", INDEX_ENTRY_LABEL), + ("index_parent_owner", INDEX_PARENT_OWNER_LABEL), + ("index_parent_inverse", INDEX_PARENT_INVERSE_LABEL), + ("index_catalog", INDEX_CATALOG_LABEL), + ("index_catalog_inverse", INDEX_CATALOG_INVERSE_LABEL), + ("index_row", INDEX_ROW_LABEL), + ("index_target_inverse", INDEX_TARGET_INVERSE_LABEL), + ("index_source_member", INDEX_SOURCE_MEMBER_LABEL), + ("index_seal", INDEX_SEAL_LABEL), + ("index_complete", INDEX_COMPLETE_LABEL), + ("index_mvcc_entry", INDEX_MVCC_ENTRY_LABEL), + ("index_mvcc_parent_owner", INDEX_MVCC_PARENT_OWNER_LABEL), + ("index_mvcc_parent_inverse", INDEX_MVCC_PARENT_INVERSE_LABEL), + ("index_mvcc_catalog", INDEX_MVCC_CATALOG_LABEL), + ( + "index_mvcc_catalog_inverse", + INDEX_MVCC_CATALOG_INVERSE_LABEL, + ), + ("index_mvcc_row", INDEX_MVCC_ROW_LABEL), + ("index_mvcc_target_inverse", INDEX_MVCC_TARGET_INVERSE_LABEL), + ("index_mvcc_source_member", INDEX_MVCC_SOURCE_MEMBER_LABEL), + ] + .into_iter() + .map(|(name, label)| (name, restore_index_system_key(mount, label))) + .collect() +} + +pub(super) fn restore_index_global_empty_predicates(mount: MountId) -> Vec { + restore_index_private_keyspaces(mount) + .into_iter() + .map(|(_, key)| PredicateRef { + family: RecordFamily::System, + key, + predicate: Predicate::PrefixEmpty, + }) + .collect() +} + +fn restore_index_ref_set_prefix(mount: MountId, label: &[u8], ref_set_id: u64) -> Vec { + let mut key = restore_index_system_key(mount, label); + key.extend_from_slice(&ref_set_id.to_be_bytes()); + key +} + +fn restore_index_entry_prefix(mount: MountId, ref_set_id: u64, parent: Option) -> Vec { + let mut key = restore_index_ref_set_prefix(mount, INDEX_ENTRY_LABEL, ref_set_id); + if let Some(parent) = parent { + key.extend_from_slice(&parent.get().to_be_bytes()); + } + key +} + +fn restore_index_entry_key( + mount: MountId, + ref_set_id: u64, + parent: InodeId, + name: &DentryName, +) -> Vec { + let mut key = restore_index_entry_prefix(mount, ref_set_id, Some(parent)); + key.extend_from_slice(name.as_bytes()); + key +} + +fn restore_index_parent_owner_prefix(mount: MountId, ref_set_id: u64) -> Vec { + restore_index_ref_set_prefix(mount, INDEX_PARENT_OWNER_LABEL, ref_set_id) +} + +fn restore_index_parent_owner_key(mount: MountId, ref_set_id: u64, parent: InodeId) -> Vec { + let mut key = restore_index_parent_owner_prefix(mount, ref_set_id); + key.extend_from_slice(&parent.get().to_be_bytes()); + key +} + +pub(super) fn restore_index_parent_inverse_prefix_for_read( + mount: MountId, + parent: InodeId, +) -> Vec { + let mut key = restore_index_system_key(mount, INDEX_PARENT_INVERSE_LABEL); + key.extend_from_slice(&parent.get().to_be_bytes()); + key +} + +fn restore_index_parent_inverse_key(mount: MountId, parent: InodeId, ref_set_id: u64) -> Vec { + let mut key = restore_index_parent_inverse_prefix_for_read(mount, parent); + key.extend_from_slice(&ref_set_id.to_be_bytes()); + key +} + +fn restore_index_catalog_prefix(mount: MountId, ref_set_id: u64) -> Vec { + restore_index_ref_set_prefix(mount, INDEX_CATALOG_LABEL, ref_set_id) +} + +fn restore_index_catalog_key(mount: MountId, ref_set_id: u64, catalog_root: InodeId) -> Vec { + let mut key = restore_index_catalog_prefix(mount, ref_set_id); + key.extend_from_slice(&catalog_root.get().to_be_bytes()); + key +} + +fn restore_index_catalog_inverse_prefix(mount: MountId, catalog_root: InodeId) -> Vec { + let mut key = restore_index_system_key(mount, INDEX_CATALOG_INVERSE_LABEL); + key.extend_from_slice(&catalog_root.get().to_be_bytes()); + key +} + +fn restore_index_catalog_inverse_key( + mount: MountId, + catalog_root: InodeId, + ref_set_id: u64, +) -> Vec { + let mut key = restore_index_catalog_inverse_prefix(mount, catalog_root); + key.extend_from_slice(&ref_set_id.to_be_bytes()); + key +} + +fn restore_index_row_prefix( + mount: MountId, + ref_set_id: u64, + catalog_root: Option, +) -> Vec { + let mut key = restore_index_ref_set_prefix(mount, INDEX_ROW_LABEL, ref_set_id); + if let Some(catalog_root) = catalog_root { + key.extend_from_slice(&catalog_root.get().to_be_bytes()); + } + key +} + +fn restore_index_target_digest(parent: Option, name: &[u8]) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(b"nokv-restore-index-target-v1\0"); + hasher.update(parent.map_or(0, InodeId::get).to_be_bytes()); + hasher.update((name.len() as u64).to_be_bytes()); + hasher.update(name); + hasher.finalize().into() +} + +fn restore_index_row_key( + mount: MountId, + ref_set_id: u64, + catalog_root: InodeId, + target: &RestoreIndexTarget, +) -> Vec { + let mut key = restore_index_row_prefix(mount, ref_set_id, Some(catalog_root)); + key.extend_from_slice(&restore_index_target_digest(target.parent, &target.name)); + key +} + +fn restore_index_target_inverse_prefix( + mount: MountId, + parent: Option, + name: &[u8], +) -> Vec { + let mut key = restore_index_system_key(mount, INDEX_TARGET_INVERSE_LABEL); + key.extend_from_slice(&restore_index_target_digest(parent, name)); + key +} + +fn restore_index_target_inverse_key( + mount: MountId, + target: &RestoreIndexTarget, + ref_set_id: u64, + catalog_root: InodeId, +) -> Vec { + let mut key = restore_index_target_inverse_prefix(mount, target.parent, &target.name); + key.extend_from_slice(&ref_set_id.to_be_bytes()); + key.extend_from_slice(&catalog_root.get().to_be_bytes()); + key +} + +fn restore_index_source_member_prefix(mount: MountId, ref_set_id: u64) -> Vec { + restore_index_ref_set_prefix(mount, INDEX_SOURCE_MEMBER_LABEL, ref_set_id) +} + +fn restore_index_source_member_key( + mount: MountId, + ref_set_id: u64, + source_inode: InodeId, +) -> Vec { + let mut key = restore_index_source_member_prefix(mount, ref_set_id); + key.extend_from_slice(&source_inode.get().to_be_bytes()); + key +} + +pub(super) fn restore_index_seal_key(mount: MountId, ref_set_id: u64) -> Vec { + restore_index_ref_set_prefix(mount, INDEX_SEAL_LABEL, ref_set_id) +} + +pub(super) fn restore_index_complete_key(mount: MountId, ref_set_id: u64) -> Vec { + restore_index_ref_set_prefix(mount, INDEX_COMPLETE_LABEL, ref_set_id) +} + +fn restore_index_mvcc_label(logical_label: &[u8]) -> Option<&'static [u8]> { + match logical_label { + INDEX_ENTRY_LABEL => Some(INDEX_MVCC_ENTRY_LABEL), + INDEX_PARENT_OWNER_LABEL => Some(INDEX_MVCC_PARENT_OWNER_LABEL), + INDEX_PARENT_INVERSE_LABEL => Some(INDEX_MVCC_PARENT_INVERSE_LABEL), + INDEX_CATALOG_LABEL => Some(INDEX_MVCC_CATALOG_LABEL), + INDEX_CATALOG_INVERSE_LABEL => Some(INDEX_MVCC_CATALOG_INVERSE_LABEL), + INDEX_ROW_LABEL => Some(INDEX_MVCC_ROW_LABEL), + INDEX_TARGET_INVERSE_LABEL => Some(INDEX_MVCC_TARGET_INVERSE_LABEL), + INDEX_SOURCE_MEMBER_LABEL => Some(INDEX_MVCC_SOURCE_MEMBER_LABEL), + _ => None, + } +} + +fn restore_index_logical_label_for_key(mount: MountId, key: &[u8]) -> Option<&'static [u8]> { + [ + INDEX_ENTRY_LABEL, + INDEX_PARENT_OWNER_LABEL, + INDEX_PARENT_INVERSE_LABEL, + INDEX_CATALOG_LABEL, + INDEX_CATALOG_INVERSE_LABEL, + INDEX_ROW_LABEL, + INDEX_TARGET_INVERSE_LABEL, + INDEX_SOURCE_MEMBER_LABEL, + ] + .into_iter() + .find(|label| key.starts_with(&restore_index_system_key(mount, label))) +} + +fn restore_index_owner_for_logical_value( + label: &[u8], + value: &[u8], +) -> Result { + if label == INDEX_ENTRY_LABEL { + return Ok(decode_restore_index_entry(value)?.owner); + } + if label == INDEX_PARENT_OWNER_LABEL || label == INDEX_PARENT_INVERSE_LABEL { + return decode_restore_index_owner(value); + } + if label == INDEX_CATALOG_LABEL || label == INDEX_CATALOG_INVERSE_LABEL { + return Ok(decode_restore_index_catalog(value)?.owner); + } + if label == INDEX_ROW_LABEL || label == INDEX_TARGET_INVERSE_LABEL { + return Ok(decode_restore_index_row(value)?.owner); + } + if label == INDEX_SOURCE_MEMBER_LABEL { + return Ok(decode_restore_index_source_member(value)?.owner); + } + Err(MetadError::Codec( + "restore index value has no ref-set owner".to_owned(), + )) +} + +fn restore_index_inspection_keyspace_name( + logical_label: &[u8], + mvcc: bool, +) -> Result<&'static str, MetadError> { + match (logical_label, mvcc) { + (INDEX_ENTRY_LABEL, false) => Ok("index_entry"), + (INDEX_PARENT_OWNER_LABEL, false) => Ok("index_parent_owner"), + (INDEX_PARENT_INVERSE_LABEL, false) => Ok("index_parent_inverse"), + (INDEX_CATALOG_LABEL, false) => Ok("index_catalog"), + (INDEX_CATALOG_INVERSE_LABEL, false) => Ok("index_catalog_inverse"), + (INDEX_ROW_LABEL, false) => Ok("index_row"), + (INDEX_TARGET_INVERSE_LABEL, false) => Ok("index_target_inverse"), + (INDEX_SOURCE_MEMBER_LABEL, false) => Ok("index_source_member"), + (INDEX_ENTRY_LABEL, true) => Ok("index_mvcc_entry"), + (INDEX_PARENT_OWNER_LABEL, true) => Ok("index_mvcc_parent_owner"), + (INDEX_PARENT_INVERSE_LABEL, true) => Ok("index_mvcc_parent_inverse"), + (INDEX_CATALOG_LABEL, true) => Ok("index_mvcc_catalog"), + (INDEX_CATALOG_INVERSE_LABEL, true) => Ok("index_mvcc_catalog_inverse"), + (INDEX_ROW_LABEL, true) => Ok("index_mvcc_row"), + (INDEX_TARGET_INVERSE_LABEL, true) => Ok("index_mvcc_target_inverse"), + (INDEX_SOURCE_MEMBER_LABEL, true) => Ok("index_mvcc_source_member"), + _ => Err(MetadError::Codec( + "restore index inspection found an unknown keyspace label".to_owned(), + )), + } +} + +/// Validate a logical row's own key and derive the exact paired logical row. +/// Entries point at their parent owner in the current view, but historical +/// entry tombstones do not require a same-command parent-owner tombstone. +fn restore_index_inspection_counterpart( + mount: MountId, + logical_label: &[u8], + logical_key: &[u8], + value: &[u8], + include_entry_parent: bool, +) -> Result, MetadError> { + if logical_label == INDEX_ENTRY_LABEL { + let entry = decode_restore_index_entry(value)?; + let expected = restore_index_entry_key( + mount, + entry.owner.ref_set_id, + entry.projection.dentry.parent, + &entry.projection.dentry.name, + ); + if logical_key != expected { + return Err(MetadError::Codec( + "restore index entry key/value identity mismatch".to_owned(), + )); + } + return Ok(include_entry_parent.then(|| { + ( + INDEX_PARENT_OWNER_LABEL, + restore_index_parent_owner_key( + mount, + entry.owner.ref_set_id, + entry.projection.dentry.parent, + ), + ) + })); + } + if logical_label == INDEX_PARENT_OWNER_LABEL { + let owner = decode_restore_index_owner(value)?; + let prefix = restore_index_parent_owner_prefix(mount, owner.ref_set_id); + if logical_key.len() != prefix.len() + 8 || !logical_key.starts_with(&prefix) { + return Err(MetadError::Codec( + "restore index parent owner key/value identity mismatch".to_owned(), + )); + } + let parent = InodeId::new(u64::from_be_bytes( + logical_key[prefix.len()..].try_into().expect("u64 width"), + ))?; + return Ok(Some(( + INDEX_PARENT_INVERSE_LABEL, + restore_index_parent_inverse_key(mount, parent, owner.ref_set_id), + ))); + } + if logical_label == INDEX_PARENT_INVERSE_LABEL { + let owner = decode_restore_index_owner(value)?; + let prefix = restore_index_system_key(mount, INDEX_PARENT_INVERSE_LABEL); + if logical_key.len() != prefix.len() + 16 || !logical_key.starts_with(&prefix) { + return Err(MetadError::Codec( + "restore index parent inverse key/value identity mismatch".to_owned(), + )); + } + let parent = InodeId::new(u64::from_be_bytes( + logical_key[prefix.len()..prefix.len() + 8] + .try_into() + .expect("u64 width"), + ))?; + let encoded_ref_set = u64::from_be_bytes( + logical_key[prefix.len() + 8..] + .try_into() + .expect("u64 width"), + ); + if encoded_ref_set != owner.ref_set_id { + return Err(MetadError::Codec( + "restore index parent inverse ref-set mismatch".to_owned(), + )); + } + return Ok(Some(( + INDEX_PARENT_OWNER_LABEL, + restore_index_parent_owner_key(mount, owner.ref_set_id, parent), + ))); + } + if logical_label == INDEX_CATALOG_LABEL || logical_label == INDEX_CATALOG_INVERSE_LABEL { + let catalog = decode_restore_index_catalog(value)?; + let owner_key = + restore_index_catalog_key(mount, catalog.owner.ref_set_id, catalog.catalog_root); + let inverse_key = restore_index_catalog_inverse_key( + mount, + catalog.catalog_root, + catalog.owner.ref_set_id, + ); + let (expected, counterpart_label, counterpart) = if logical_label == INDEX_CATALOG_LABEL { + (owner_key, INDEX_CATALOG_INVERSE_LABEL, inverse_key) + } else { + (inverse_key, INDEX_CATALOG_LABEL, owner_key) + }; + if logical_key != expected { + return Err(MetadError::Codec( + "restore index catalog key/value identity mismatch".to_owned(), + )); + } + return Ok(Some((counterpart_label, counterpart))); + } + if logical_label == INDEX_ROW_LABEL || logical_label == INDEX_TARGET_INVERSE_LABEL { + let row = decode_restore_index_row(value)?; + let owner_key = + restore_index_row_key(mount, row.owner.ref_set_id, row.catalog_root, &row.target); + let inverse_key = restore_index_target_inverse_key( + mount, + &row.target, + row.owner.ref_set_id, + row.catalog_root, + ); + let (expected, counterpart_label, counterpart) = if logical_label == INDEX_ROW_LABEL { + (owner_key, INDEX_TARGET_INVERSE_LABEL, inverse_key) + } else { + (inverse_key, INDEX_ROW_LABEL, owner_key) + }; + if logical_key != expected { + return Err(MetadError::Codec( + "restore index row key/value identity mismatch".to_owned(), + )); + } + return Ok(Some((counterpart_label, counterpart))); + } + if logical_label == INDEX_SOURCE_MEMBER_LABEL { + let source = decode_restore_index_source_member(value)?; + if logical_key + != restore_index_source_member_key(mount, source.owner.ref_set_id, source.source_inode) + { + return Err(MetadError::Codec( + "restore index source-member key/value identity mismatch".to_owned(), + )); + } + return Ok(None); + } + Err(MetadError::Codec( + "restore index inspection cannot validate an unknown logical row".to_owned(), + )) +} + +fn restore_index_mvcc_key( + mount: MountId, + logical_key: &[u8], + commit_version: Version, +) -> Result, MetadError> { + let logical_label = + restore_index_logical_label_for_key(mount, logical_key).ok_or_else(|| { + MetadError::Codec("restore index MVCC key is outside a tracked keyspace".to_owned()) + })?; + let logical_prefix = restore_index_system_key(mount, logical_label); + let mvcc_label = restore_index_mvcc_label(logical_label).expect("tracked label has MVCC label"); + let mut key = restore_index_system_key(mount, mvcc_label); + key.extend_from_slice(&logical_key[logical_prefix.len()..]); + // Dentry names reject NUL, while every other logical suffix has a fixed + // width. This delimiter keeps all versions of one logical row contiguous + // and lets a caller seek past an exact logical key without skipping names + // that merely share its byte prefix. + key.push(RESTORE_INDEX_MVCC_VERSION_DELIMITER); + key.extend_from_slice(&commit_version.get().to_be_bytes()); + Ok(key) +} + +fn restore_index_mvcc_prefix(mount: MountId, logical_prefix: &[u8]) -> Result, MetadError> { + let logical_label = + restore_index_logical_label_for_key(mount, logical_prefix).ok_or_else(|| { + MetadError::Codec("restore index MVCC prefix is outside a tracked keyspace".to_owned()) + })?; + let base = restore_index_system_key(mount, logical_label); + let mvcc_label = restore_index_mvcc_label(logical_label).expect("tracked label has MVCC label"); + let mut prefix = restore_index_system_key(mount, mvcc_label); + prefix.extend_from_slice(&logical_prefix[base.len()..]); + Ok(prefix) +} + +fn restore_index_mvcc_owner_prefixes(mount: MountId, ref_set_id: u64) -> Vec> { + [ + restore_index_entry_prefix(mount, ref_set_id, None), + restore_index_parent_owner_prefix(mount, ref_set_id), + restore_index_catalog_prefix(mount, ref_set_id), + restore_index_row_prefix(mount, ref_set_id, None), + restore_index_source_member_prefix(mount, ref_set_id), + ] + .into_iter() + .map(|prefix| restore_index_mvcc_prefix(mount, &prefix).expect("owner prefix is MVCC tracked")) + .collect() +} + +fn encode_restore_index_mvcc(record: &RestoreIndexMvccRecord) -> Vec { + let mut value = Vec::with_capacity(26 + record.logical_key.len() + record.value.len()); + value.extend_from_slice(INDEX_MVCC_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.push(record.kind as u8); + value.extend_from_slice(&record.commit_version.to_be_bytes()); + push_restore_index_bytes(&mut value, &record.logical_key); + push_restore_index_bytes(&mut value, &record.value); + value +} + +fn decode_restore_index_mvcc(value: &[u8]) -> Result { + let mut decoder = RestoreIndexDecoder::new(value); + decoder.header(INDEX_MVCC_MAGIC, "MVCC record")?; + let kind = match decoder.u8()? { + 1 => RestoreIndexMvccKind::Put, + 2 => RestoreIndexMvccKind::Tombstone, + tag => { + return Err(MetadError::Codec(format!( + "restore index MVCC record has invalid kind {tag}" + ))) + } + }; + let commit_version = decoder.u64()?; + if commit_version == 0 { + return Err(MetadError::Codec( + "restore index MVCC record has a zero commit version".to_owned(), + )); + } + let logical_key = decoder.bytes()?.to_vec(); + let value = decoder.bytes()?.to_vec(); + decoder.finish("MVCC record")?; + if logical_key.is_empty() || value.is_empty() { + return Err(MetadError::Codec( + "restore index MVCC record has an empty key/value".to_owned(), + )); + } + Ok(RestoreIndexMvccRecord { + commit_version, + kind, + logical_key, + value, + }) +} + +fn encode_restore_index_complete(marker: &RestoreIndexCompleteMarker) -> Vec { + let mut value = Vec::with_capacity(97); + value.extend_from_slice(INDEX_COMPLETE_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.extend_from_slice(&marker.operation_digest); + value.extend_from_slice(&marker.initialization_digest); + value.extend_from_slice(&marker.ref_set_id.to_be_bytes()); + value.extend_from_slice(&marker.incarnation.to_be_bytes()); + value.extend_from_slice(&marker.complete_version.to_be_bytes()); + value +} + +fn decode_restore_index_complete(value: &[u8]) -> Result { + let mut decoder = RestoreIndexDecoder::new(value); + decoder.header(INDEX_COMPLETE_MAGIC, "complete marker")?; + let marker = RestoreIndexCompleteMarker { + operation_digest: decoder.array_32()?, + initialization_digest: decoder.array_32()?, + ref_set_id: decoder.u64()?, + incarnation: decoder.u64()?, + complete_version: decoder.u64()?, + }; + decoder.finish("complete marker")?; + if marker.ref_set_id == 0 || marker.incarnation == 0 || marker.complete_version == 0 { + return Err(MetadError::Codec( + "restore index complete marker has an invalid identity".to_owned(), + )); + } + Ok(marker) +} + +fn restore_index_mvcc_inverse_logical_key( + mount: MountId, + record: &RestoreIndexMvccRecord, +) -> Result>, MetadError> { + let label = + restore_index_logical_label_for_key(mount, &record.logical_key).ok_or_else(|| { + MetadError::Codec("restore index MVCC record has an unknown logical label".to_owned()) + })?; + if label == INDEX_ENTRY_LABEL { + let entry = decode_restore_index_entry(&record.value)?; + if record.logical_key + != restore_index_entry_key( + mount, + entry.owner.ref_set_id, + entry.projection.dentry.parent, + &entry.projection.dentry.name, + ) + { + return Err(MetadError::Codec( + "restore index MVCC entry changed logical identity".to_owned(), + )); + } + return Ok(None); + } + if label == INDEX_PARENT_OWNER_LABEL { + let owner = decode_restore_index_owner(&record.value)?; + let prefix = restore_index_parent_owner_prefix(mount, owner.ref_set_id); + if record.logical_key.len() != prefix.len() + 8 || !record.logical_key.starts_with(&prefix) + { + return Err(MetadError::Codec( + "restore index MVCC parent owner changed logical identity".to_owned(), + )); + } + let parent = InodeId::new(u64::from_be_bytes( + record.logical_key[prefix.len()..] + .try_into() + .expect("u64 width"), + ))?; + return Ok(Some(restore_index_parent_inverse_key( + mount, + parent, + owner.ref_set_id, + ))); + } + if label == INDEX_CATALOG_LABEL { + let catalog = decode_restore_index_catalog(&record.value)?; + if record.logical_key + != restore_index_catalog_key(mount, catalog.owner.ref_set_id, catalog.catalog_root) + { + return Err(MetadError::Codec( + "restore index MVCC catalog changed logical identity".to_owned(), + )); + } + return Ok(Some(restore_index_catalog_inverse_key( + mount, + catalog.catalog_root, + catalog.owner.ref_set_id, + ))); + } + if label == INDEX_ROW_LABEL { + let row = decode_restore_index_row(&record.value)?; + if record.logical_key + != restore_index_row_key(mount, row.owner.ref_set_id, row.catalog_root, &row.target) + { + return Err(MetadError::Codec( + "restore index MVCC row changed logical identity".to_owned(), + )); + } + return Ok(Some(restore_index_target_inverse_key( + mount, + &row.target, + row.owner.ref_set_id, + row.catalog_root, + ))); + } + if label == INDEX_SOURCE_MEMBER_LABEL { + let source = decode_restore_index_source_member(&record.value)?; + if record.logical_key + != restore_index_source_member_key(mount, source.owner.ref_set_id, source.source_inode) + { + return Err(MetadError::Codec( + "restore index MVCC source member changed logical identity".to_owned(), + )); + } + return Ok(None); + } + Err(MetadError::Codec( + "restore index MVCC release encountered an inverse-side owner".to_owned(), + )) +} + +/// Final release fence. Inverse keyspaces are target-first and cannot be +/// expressed as one ref-set prefix; `release_restore_index_page` deletes each +/// inverse under an exact owner-row CAS before these owner-side predicates can +/// become true. +pub(super) fn restore_index_release_empty_predicates( + mount: MountId, + ref_set_id: u64, +) -> Vec { + [ + INDEX_ENTRY_LABEL, + INDEX_PARENT_OWNER_LABEL, + INDEX_CATALOG_LABEL, + INDEX_ROW_LABEL, + INDEX_SOURCE_MEMBER_LABEL, + ] + .into_iter() + .map(|label| PredicateRef { + family: RecordFamily::System, + key: restore_index_ref_set_prefix(mount, label, ref_set_id), + predicate: Predicate::PrefixEmpty, + }) + .chain(std::iter::once(PredicateRef { + family: RecordFamily::System, + key: restore_index_seal_key(mount, ref_set_id), + predicate: Predicate::NotExists, + })) + .chain(std::iter::once(PredicateRef { + family: RecordFamily::System, + key: restore_index_complete_key(mount, ref_set_id), + predicate: Predicate::NotExists, + })) + .chain( + restore_index_mvcc_owner_prefixes(mount, ref_set_id) + .into_iter() + .map(|key| PredicateRef { + family: RecordFamily::System, + key, + predicate: Predicate::PrefixEmpty, + }), + ) + .collect() +} + +fn encode_restore_index_owner(owner: RestoreIndexOwner) -> Vec { + let mut value = Vec::with_capacity(OWNER_BYTES); + value.extend_from_slice(&owner.operation_digest); + value.extend_from_slice(&owner.ref_set_id.to_be_bytes()); + value +} + +fn decode_restore_index_owner(value: &[u8]) -> Result { + if value.len() != OWNER_BYTES { + return Err(MetadError::Codec( + "restore index owner has an invalid length".to_owned(), + )); + } + RestoreIndexOwner { + operation_digest: value[..32].try_into().expect("digest width"), + ref_set_id: u64::from_be_bytes(value[32..].try_into().expect("u64 width")), + } + .validate() +} + +fn encode_restore_index_entry(entry: &RestoreIndexEntry) -> Vec { + let projection = encode_dentry_projection(&entry.projection); + let mut value = Vec::with_capacity(53 + projection.len()); + value.extend_from_slice(INDEX_ENTRY_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.extend_from_slice(&encode_restore_index_owner(entry.owner)); + push_restore_index_bytes(&mut value, &projection); + value +} + +fn decode_restore_index_entry(value: &[u8]) -> Result { + let mut decoder = RestoreIndexDecoder::new(value); + decoder.header(INDEX_ENTRY_MAGIC, "entry")?; + let owner = decoder.owner()?; + let projection = decode_dentry_projection(decoder.bytes()?) + .map_err(|error| MetadError::Codec(error.to_string()))?; + decoder.finish("entry")?; + Ok(RestoreIndexEntry { owner, projection }) +} + +fn encode_restore_index_catalog(catalog: &RestoreIndexCatalog) -> Vec { + let record = encode_path_index_catalog(&catalog.record); + let mut value = Vec::with_capacity(61 + record.len()); + value.extend_from_slice(INDEX_CATALOG_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.extend_from_slice(&encode_restore_index_owner(catalog.owner)); + value.extend_from_slice(&catalog.catalog_root.get().to_be_bytes()); + push_restore_index_bytes(&mut value, &record); + value +} + +fn decode_restore_index_catalog(value: &[u8]) -> Result { + let mut decoder = RestoreIndexDecoder::new(value); + decoder.header(INDEX_CATALOG_MAGIC, "catalog")?; + let owner = decoder.owner()?; + let catalog_root = InodeId::new(decoder.u64()?)?; + let record = decode_path_index_catalog(decoder.bytes()?) + .map_err(|error| MetadError::Codec(error.to_string()))?; + decoder.finish("catalog")?; + Ok(RestoreIndexCatalog { + owner, + catalog_root, + record, + }) +} + +fn encode_restore_index_target(value: &mut Vec, target: &RestoreIndexTarget) { + value.push(u8::from(target.parent.is_some())); + value.extend_from_slice(&target.parent.map_or(0, InodeId::get).to_be_bytes()); + push_restore_index_bytes(value, &target.name); + value.extend_from_slice(&target.inode.get().to_be_bytes()); + value.push(restore_index_file_type_tag(target.file_type)); + value.extend_from_slice(&target.attr_generation.to_be_bytes()); + match target.body_digest { + Some(digest) => { + value.push(1); + value.extend_from_slice(&digest); + } + None => value.push(0), + } +} + +fn decode_restore_index_target( + decoder: &mut RestoreIndexDecoder<'_>, +) -> Result { + let parent_tag = decoder.u8()?; + let parent_raw = decoder.u64()?; + let parent = match (parent_tag, parent_raw) { + (0, 0) => None, + (1, raw) => Some(InodeId::new(raw)?), + _ => { + return Err(MetadError::Codec( + "restore index target has an invalid parent tag".to_owned(), + )) + } + }; + let name = decoder.bytes()?.to_vec(); + if parent.is_some() == name.is_empty() { + return Err(MetadError::Codec( + "restore index target has an invalid parent/name shape".to_owned(), + )); + } + if !name.is_empty() { + DentryName::new(name.clone()).map_err(|error| MetadError::Codec(error.to_string()))?; + } + let inode = InodeId::new(decoder.u64()?)?; + let file_type = restore_index_file_type(decoder.u8()?)?; + let attr_generation = decoder.u64()?; + let body_digest = match decoder.u8()? { + 0 => None, + 1 => Some(decoder.array_32()?), + _ => { + return Err(MetadError::Codec( + "restore index target has an invalid body tag".to_owned(), + )) + } + }; + Ok(RestoreIndexTarget { + parent, + name, + inode, + file_type, + attr_generation, + body_digest, + }) +} + +fn encode_restore_index_row(row: &RestoreIndexRow) -> Vec { + let record = encode_path_index_row(&row.record); + let mut value = Vec::with_capacity(112 + row.target.name.len() + record.len()); + value.extend_from_slice(INDEX_ROW_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.extend_from_slice(&encode_restore_index_owner(row.owner)); + value.extend_from_slice(&row.catalog_root.get().to_be_bytes()); + encode_restore_index_target(&mut value, &row.target); + push_restore_index_bytes(&mut value, &record); + value +} + +fn decode_restore_index_row(value: &[u8]) -> Result { + let mut decoder = RestoreIndexDecoder::new(value); + decoder.header(INDEX_ROW_MAGIC, "row")?; + let owner = decoder.owner()?; + let catalog_root = InodeId::new(decoder.u64()?)?; + let target = decode_restore_index_target(&mut decoder)?; + let record = decode_path_index_row(decoder.bytes()?) + .map_err(|error| MetadError::Codec(error.to_string()))?; + decoder.finish("row")?; + Ok(RestoreIndexRow { + owner, + catalog_root, + target, + record, + }) +} + +fn encode_restore_index_source_member( + source: &RestoreIndexSourceMember, +) -> Result, MetadError> { + let member = encode_restore_staging_member(&source.member)?; + let mut value = Vec::with_capacity(61 + member.len()); + value.extend_from_slice(INDEX_SOURCE_MEMBER_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.extend_from_slice(&encode_restore_index_owner(source.owner)); + value.extend_from_slice(&source.source_inode.get().to_be_bytes()); + push_restore_index_bytes(&mut value, &member); + Ok(value) +} + +fn decode_restore_index_source_member( + value: &[u8], +) -> Result { + let mut decoder = RestoreIndexDecoder::new(value); + decoder.header(INDEX_SOURCE_MEMBER_MAGIC, "source member")?; + let owner = decoder.owner()?; + let source_inode = InodeId::new(decoder.u64()?)?; + let member = decode_restore_staging_member(decoder.bytes()?)?; + decoder.finish("source member")?; + if member.operation_digest != owner.operation_digest + || member.source_inode != Some(source_inode) + { + return Err(MetadError::Codec( + "restore index source member changed identity".to_owned(), + )); + } + Ok(RestoreIndexSourceMember { + owner, + source_inode, + member, + }) +} + +pub(super) fn encode_restore_index_seal(seal: &RestoreIndexSeal) -> Vec { + let mut value = Vec::with_capacity(153); + value.extend_from_slice(INDEX_SEAL_MAGIC); + value.push(RESTORE_FORMAT_VERSION); + value.extend_from_slice(&seal.operation_digest); + value.extend_from_slice(&seal.initialization_digest); + value.extend_from_slice(&seal.ref_set_id.to_be_bytes()); + value.extend_from_slice(&seal.incarnation.to_be_bytes()); + value.extend_from_slice(&seal.entry_count.to_be_bytes()); + value.extend_from_slice(&seal.catalog_count.to_be_bytes()); + value.extend_from_slice(&seal.row_count.to_be_bytes()); + value.extend_from_slice(&seal.digest); + value +} + +pub(super) fn decode_restore_index_seal(value: &[u8]) -> Result { + let mut decoder = RestoreIndexDecoder::new(value); + decoder.header(INDEX_SEAL_MAGIC, "seal")?; + let seal = RestoreIndexSeal { + operation_digest: decoder.array_32()?, + initialization_digest: decoder.array_32()?, + ref_set_id: decoder.u64()?, + incarnation: decoder.u64()?, + entry_count: decoder.u64()?, + catalog_count: decoder.u64()?, + row_count: decoder.u64()?, + digest: decoder.array_32()?, + }; + decoder.finish("seal")?; + if seal.ref_set_id == 0 || seal.incarnation == 0 { + return Err(MetadError::Codec( + "restore index seal has an invalid identity".to_owned(), + )); + } + Ok(seal) +} + +fn restore_index_file_type_tag(file_type: FileType) -> u8 { + match file_type { + FileType::File => 1, + FileType::Directory => 2, + FileType::Symlink => 3, + FileType::NamedPipe => 4, + FileType::CharDevice => 5, + FileType::BlockDevice => 6, + FileType::Socket => 7, + } +} + +fn restore_index_file_type(tag: u8) -> Result { + match tag { + 1 => Ok(FileType::File), + 2 => Ok(FileType::Directory), + 3 => Ok(FileType::Symlink), + 4 => Ok(FileType::NamedPipe), + 5 => Ok(FileType::CharDevice), + 6 => Ok(FileType::BlockDevice), + 7 => Ok(FileType::Socket), + _ => Err(MetadError::Codec(format!( + "restore index target has invalid file type {tag}" + ))), + } +} + +fn restore_index_body_digest(body: Option<&BodyDescriptor>) -> Option<[u8; 32]> { + body.map(|body| { + let mut hasher = Sha256::new(); + hasher.update(b"nokv-restore-index-body-v1\0"); + hasher.update(encode_body_descriptor(body)); + hasher.finalize().into() + }) +} + +fn restore_index_relative_components( + base: &str, + path: &str, +) -> Result>, MetadError> { + let base = parse_absolute_path(base)?; + let path = parse_absolute_path(path)?; + if path.len() < base.len() + || !base + .iter() + .zip(path.iter()) + .all(|(left, right)| left == right) + { + return Ok(None); + } + Ok(Some(path[base.len()..].to_vec())) +} + +fn restore_index_relative_string(components: &[DentryName]) -> Result { + let absolute = canonical_path(components)?; + Ok(absolute.strip_prefix('/').unwrap_or(&absolute).to_owned()) +} + +fn restore_index_join_path(base: &str, relative: &[DentryName]) -> Result { + let mut components = parse_absolute_path(base)?; + components.extend_from_slice(relative); + canonical_path(&components) +} + +fn restore_index_target_from_projection(projection: &DentryProjection) -> RestoreIndexTarget { + RestoreIndexTarget { + parent: Some(projection.dentry.parent), + name: projection.dentry.name.as_bytes().to_vec(), + inode: projection.dentry.child, + file_type: projection.dentry.child_type, + attr_generation: projection.dentry.attr_generation, + body_digest: restore_index_body_digest(projection.body.as_ref()), + } +} + +fn restore_index_root_target(metadata: &PathMetadata) -> RestoreIndexTarget { + RestoreIndexTarget { + parent: None, + name: Vec::new(), + inode: metadata.attr.inode, + file_type: metadata.attr.file_type, + attr_generation: metadata.attr.generation, + body_digest: restore_index_body_digest(metadata.body.as_ref()), + } +} + +/// A generic clone remints inode ids while preserving the source entry's +/// metadata and body descriptor. Compare only fields that survive that +/// reminting so an inherited index row is not applied to a later replacement +/// that merely reused the same path. +fn restore_index_clone_metadata_matches_source(fork: &PathMetadata, source: &PathMetadata) -> bool { + fork.attr.file_type == source.attr.file_type + && fork.attr.mode == source.attr.mode + && fork.attr.uid == source.attr.uid + && fork.attr.gid == source.attr.gid + && fork.attr.rdev == source.attr.rdev + && fork.attr.size == source.attr.size + && fork.attr.mtime_ms == source.attr.mtime_ms + && fork.body == source.body +} + +fn restore_index_clone_entry_matches_source( + fork: &DentryWithAttr, + source: &DentryWithAttr, +) -> bool { + restore_index_clone_metadata_matches_source( + &PathMetadata { + attr: fork.attr.clone(), + body: fork.body.clone(), + }, + &PathMetadata { + attr: source.attr.clone(), + body: source.body.clone(), + }, + ) +} + +fn restore_index_target_matches_projection( + target: &RestoreIndexTarget, + projection: &DentryProjection, +) -> bool { + target.parent == Some(projection.dentry.parent) + && target.name == projection.dentry.name.as_bytes() + && target.inode == projection.dentry.child + && target.file_type == projection.dentry.child_type + && target.attr_generation == projection.dentry.attr_generation + && target.body_digest == restore_index_body_digest(projection.body.as_ref()) +} + +fn push_restore_index_bytes(out: &mut Vec, bytes: &[u8]) { + out.extend_from_slice(&(bytes.len() as u32).to_be_bytes()); + out.extend_from_slice(bytes); +} + +fn fold_restore_index_seal_row(hasher: &mut Sha256, kind: u8, key: &[u8], value: &[u8]) { + hasher.update([kind]); + hasher.update((key.len() as u64).to_be_bytes()); + hasher.update(key); + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); +} + +fn account_restore_index_buffer( + resource: &str, + items: usize, + bytes: &mut usize, + additional_bytes: usize, +) -> Result<(), MetadError> { + if items > MAX_RESTORE_INDEX_PLAN_ITEMS { + return Err(MetadError::RestoreResourceLimit { + resource: format!("{resource} items"), + limit: MAX_RESTORE_INDEX_PLAN_ITEMS as u64, + actual: items as u64, + }); + } + *bytes = bytes.saturating_add(additional_bytes); + if *bytes > MAX_RESTORE_INDEX_PLAN_BYTES { + return Err(MetadError::RestoreResourceLimit { + resource: format!("{resource} bytes"), + limit: MAX_RESTORE_INDEX_PLAN_BYTES as u64, + actual: *bytes as u64, + }); + } + Ok(()) +} + +fn validate_restore_index_collection(resource: &str, actual: usize) -> Result<(), MetadError> { + if actual > MAX_RESTORE_SUBTREE_ENTRIES { + return Err(MetadError::RestoreResourceLimit { + resource: resource.to_owned(), + limit: MAX_RESTORE_SUBTREE_ENTRIES as u64, + actual: actual as u64, + }); + } + Ok(()) +} + +struct RestoreIndexDecoder<'a> { + input: &'a [u8], +} + +impl<'a> RestoreIndexDecoder<'a> { + fn new(input: &'a [u8]) -> Self { + Self { input } + } + + fn take(&mut self, len: usize) -> Result<&'a [u8], MetadError> { + let Some((head, tail)) = self.input.split_at_checked(len) else { + return Err(MetadError::Codec( + "restore index record is truncated".to_owned(), + )); + }; + self.input = tail; + Ok(head) + } + + fn header(&mut self, magic: &[u8; 8], kind: &str) -> Result<(), MetadError> { + if self.take(8)? != magic { + return Err(MetadError::Codec(format!( + "invalid restore index {kind} magic" + ))); + } + if self.u8()? != RESTORE_FORMAT_VERSION { + return Err(MetadError::Codec(format!( + "unsupported restore index {kind} version" + ))); + } + Ok(()) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn u32(&mut self) -> Result { + Ok(u32::from_be_bytes( + self.take(4)?.try_into().expect("u32 width"), + )) + } + + fn u64(&mut self) -> Result { + Ok(u64::from_be_bytes( + self.take(8)?.try_into().expect("u64 width"), + )) + } + + fn array_32(&mut self) -> Result<[u8; 32], MetadError> { + Ok(self.take(32)?.try_into().expect("digest width")) + } + + fn bytes(&mut self) -> Result<&'a [u8], MetadError> { + let len = self.u32()? as usize; + self.take(len) + } + + fn owner(&mut self) -> Result { + decode_restore_index_owner(self.take(OWNER_BYTES)?) + } + + fn finish(self, kind: &str) -> Result<(), MetadError> { + if self.input.is_empty() { + Ok(()) + } else { + Err(MetadError::Codec(format!( + "restore index {kind} has trailing bytes" + ))) + } + } +} + +fn restore_index_owner_mutations( + mount: MountId, + owner: RestoreIndexOwner, + parent: InodeId, +) -> [Mutation; 2] { + let value = Value(encode_restore_index_owner(owner)); + [ + Mutation { + family: RecordFamily::System, + key: restore_index_parent_owner_key(mount, owner.ref_set_id, parent), + op: MutationOp::Put, + value: Some(value.clone()), + }, + Mutation { + family: RecordFamily::System, + key: restore_index_parent_inverse_key(mount, parent, owner.ref_set_id), + op: MutationOp::Put, + value: Some(value), + }, + ] +} + +fn restore_index_catalog_mutations(mount: MountId, catalog: &RestoreIndexCatalog) -> [Mutation; 2] { + let value = Value(encode_restore_index_catalog(catalog)); + [ + Mutation { + family: RecordFamily::System, + key: restore_index_catalog_key(mount, catalog.owner.ref_set_id, catalog.catalog_root), + op: MutationOp::Put, + value: Some(value.clone()), + }, + Mutation { + family: RecordFamily::System, + key: restore_index_catalog_inverse_key( + mount, + catalog.catalog_root, + catalog.owner.ref_set_id, + ), + op: MutationOp::Put, + value: Some(value), + }, + ] +} + +fn restore_index_row_mutations(mount: MountId, row: &RestoreIndexRow) -> [Mutation; 2] { + let value = Value(encode_restore_index_row(row)); + [ + Mutation { + family: RecordFamily::System, + key: restore_index_row_key(mount, row.owner.ref_set_id, row.catalog_root, &row.target), + op: MutationOp::Put, + value: Some(value.clone()), + }, + Mutation { + family: RecordFamily::System, + key: restore_index_target_inverse_key( + mount, + &row.target, + row.owner.ref_set_id, + row.catalog_root, + ), + op: MutationOp::Put, + value: Some(value), + }, + ] +} + +fn restore_index_source_member_mutation( + mount: MountId, + source: &RestoreIndexSourceMember, +) -> Result { + Ok(Mutation { + family: RecordFamily::System, + key: restore_index_source_member_key(mount, source.owner.ref_set_id, source.source_inode), + op: MutationOp::Put, + value: Some(Value(encode_restore_index_source_member(source)?)), + }) +} + +/// Mutations used while building a detached tree. Callers must place them in +/// the same command as the corresponding dentry, or in a later hidden staging +/// command guarded by the operation and temporary binding versions. +pub(super) fn restore_index_materialization_mutations( + mount: MountId, + operation: &RestoreOperation, + projection: &DentryProjection, +) -> RestoreIndexMutationPlan { + let owner = RestoreIndexOwner::from_operation(operation); + let key = restore_index_entry_key( + mount, + operation.ref_set_id, + projection.dentry.parent, + &projection.dentry.name, + ); + let mut mutations = restore_index_owner_mutations(mount, owner, projection.dentry.parent) + .into_iter() + .collect::>(); + mutations.push(Mutation { + family: RecordFamily::System, + key, + op: MutationOp::Put, + value: Some(Value(encode_restore_index_entry(&RestoreIndexEntry { + owner, + projection: projection.clone(), + }))), + }); + RestoreIndexMutationPlan { + predicates: Vec::new(), + mutations, + } +} + +impl NoKvFs +where + M: MetadataStore, + O: ObjectStore, +{ + /// Resolve an exact generic clone root back to a durably Complete restore. + /// The existing 40-byte ForkBinding remains the sole clone contract; the + /// restore staging inverse is only used to prove that its source owns a + /// sealed private index overlay. + fn restore_index_fork_source( + &self, + fork_root: InodeId, + ) -> Result, MetadError> { + let control_version = self.read_version()?; + let binding_key = fork_binding_key(self.mount, fork_root); + let Some(binding_item) = self.metadata.get_versioned( + RecordFamily::ForkBinding, + &binding_key, + control_version, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(None); + }; + let binding = crate::layout::decode_fork_binding(&binding_item.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if binding.fork_root != fork_root + || binding_key != fork_binding_key(self.mount, binding.fork_root) + || binding.created_version != binding_item.version.get() + || binding.fork_root.shard_index() != self.shard_index + || binding.source_root.shard_index() != self.shard_index + { + return Err(MetadError::Codec( + "restore index ForkBinding changed identity".to_owned(), + )); + } + let pinned_version = Version::new(binding.pinned_read_version)?; + if !self.restore_fork_binding_is_namespace_anchor(&binding, control_version)? { + // A detached restore's temporary binding is a retention fence, not + // a generic clone whose reads may fall through to a source index. + return Ok(None); + } + + let Some(inverse) = self.metadata.get( + RecordFamily::System, + &restore_staging_inode_key(self.mount, binding.source_root), + control_version, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(None); + }; + let (operation_digest, ref_set_id) = decode_restore_staging_inverse(&inverse.0)?; + let owner = RestoreIndexOwner { + operation_digest, + ref_set_id, + } + .validate()?; + let Some(operation) = + self.visible_restore_index_operation(owner, pinned_version, ReadPurpose::Snapshot)? + else { + return Ok(None); + }; + if operation.destination_root != binding.source_root + || operation.operation_digest != operation_digest + || operation.ref_set_id != ref_set_id + { + return Err(MetadError::Codec( + "restore index ForkBinding source changed restore identity".to_owned(), + )); + } + Ok(Some(RestoreIndexForkSource { binding, operation })) + } + + pub(super) fn restore_index_parent_has_fork_source( + &self, + parent: InodeId, + ) -> Result { + let control_version = self.read_version()?; + let binding_key = fork_binding_key(self.mount, parent); + if self + .metadata + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: binding_key, + start_after: None, + version: control_version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })? + .is_empty() + { + return Ok(false); + } + Ok(self.restore_index_fork_source(parent)?.is_some()) + } + + pub(super) fn inspect_restore_index_state( + &self, + read_version: Version, + ) -> Result { + let mut inspection = RestoreIndexInspection::default(); + let mut issue_budget = RestoreIndexInspectionIssueBudget::default(); + for (name, prefix) in restore_index_private_keyspaces(self.mount) { + let mut count = 0_usize; + let mut start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: start_after.clone(), + version: read_version, + limit: RESTORE_INDEX_SCAN_PAGE, + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + break; + } + count = count.checked_add(rows.len()).ok_or_else(|| { + MetadError::Codec("restore index inspection row count overflow".to_owned()) + })?; + for row in &rows { + let decoded = self.restore_index_inspected_row_identity(name, row); + let decoded = match decoded { + Ok(decoded) => decoded, + Err(error) => { + issue_budget.push( + &mut inspection.unowned_issues, + format!( + "{name} row (key bytes {}) cannot be attributed: {error}", + row.key.len() + ), + ); + continue; + } + }; + + if !inspection.ref_sets.contains_key(&decoded.ref_set_id) + && inspection.ref_sets.len() >= MAX_RESTORE_INDEX_INSPECTION_REF_SETS + { + return Err(MetadError::RestoreResourceLimit { + resource: "restore index fsck ref sets".to_owned(), + limit: MAX_RESTORE_INDEX_INSPECTION_REF_SETS as u64, + actual: inspection.ref_sets.len().saturating_add(1) as u64, + }); + } + let ref_set = inspection.ref_sets.entry(decoded.ref_set_id).or_default(); + let keyspace_count = ref_set.counts.entry(name.to_owned()).or_default(); + *keyspace_count = keyspace_count.checked_add(1).ok_or_else(|| { + MetadError::Codec( + "restore index inspection ref-set count overflow".to_owned(), + ) + })?; + if !ref_set + .operation_digests + .contains(&decoded.operation_digest) + && ref_set.operation_digests.len() < 2 + { + ref_set.operation_digests.insert(decoded.operation_digest); + } + if decoded.is_put { + inspection.mvcc_puts = + inspection.mvcc_puts.checked_add(1).ok_or_else(|| { + MetadError::Codec( + "restore index inspection MVCC PUT count overflow".to_owned(), + ) + })?; + } + if decoded.is_tombstone { + inspection.mvcc_tombstones = + inspection.mvcc_tombstones.checked_add(1).ok_or_else(|| { + MetadError::Codec( + "restore index inspection MVCC tombstone count overflow" + .to_owned(), + ) + })?; + } + if let Some(identity) = decoded.seal_identity { + match &ref_set.seal_identity { + Some(existing) if existing != &identity => issue_budget.push( + &mut ref_set.closure_issues, + format!( + "ref-set {} has conflicting index seal identities", + decoded.ref_set_id + ), + ), + None => ref_set.seal_identity = Some(identity), + Some(_) => {} + } + } + if let Some(identity) = decoded.complete_identity { + match &ref_set.complete_identity { + Some(existing) if existing != &identity => issue_budget.push( + &mut ref_set.closure_issues, + format!( + "ref-set {} has conflicting Complete marker identities", + decoded.ref_set_id + ), + ), + None => ref_set.complete_identity = Some(identity), + Some(_) => {} + } + } + + if !matches!(name, "index_seal" | "index_complete") { + match self.restore_index_inspected_row_closure(name, row, read_version) { + Ok((owner, Some(issue))) => { + if let Some(ref_set) = + inspection.ref_sets.get_mut(&owner.ref_set_id) + { + issue_budget.push(&mut ref_set.closure_issues, issue); + } else { + issue_budget.push( + &mut inspection.unowned_issues, + format!( + "{name} row closure resolved an unknown ref-set {}", + owner.ref_set_id + ), + ); + } + } + Ok((_, None)) => {} + Err(error) => issue_budget.push( + &mut inspection.unowned_issues, + format!( + "{name} row (key bytes {}) fails exact closure: {error}", + row.key.len() + ), + ), + } + } + } + let reached_tail = rows.len() < RESTORE_INDEX_SCAN_PAGE; + start_after = rows.last().map(|row| row.key.clone()); + if reached_tail { + break; + } + } + inspection.counts.insert(name.to_owned(), count); + } + + self.inspect_complete_restore_index_source_closure( + read_version, + &mut inspection, + &mut issue_budget, + )?; + + for (ref_set_id, ref_set) in &mut inspection.ref_sets { + if ref_set.operation_digests.len() != 1 { + issue_budget.push( + &mut ref_set.closure_issues, + format!( + "ref-set {ref_set_id} has at least {} operation identities", + ref_set.operation_digests.len() + ), + ); + } + if let (Some(seal), Some(complete)) = + (&ref_set.seal_identity, &ref_set.complete_identity) + { + if seal != complete { + issue_budget.push( + &mut ref_set.closure_issues, + format!( + "ref-set {ref_set_id} index seal and Complete marker identities disagree" + ), + ); + } + } + for (owner, inverse) in [ + ("index_parent_owner", "index_parent_inverse"), + ("index_catalog", "index_catalog_inverse"), + ("index_row", "index_target_inverse"), + ("index_mvcc_parent_owner", "index_mvcc_parent_inverse"), + ("index_mvcc_catalog", "index_mvcc_catalog_inverse"), + ("index_mvcc_row", "index_mvcc_target_inverse"), + ] { + let owner_count = ref_set.counts.get(owner).copied().unwrap_or(0); + let inverse_count = ref_set.counts.get(inverse).copied().unwrap_or(0); + if owner_count != inverse_count { + issue_budget.push( + &mut ref_set.closure_issues, + format!( + "ref-set {ref_set_id} {owner}/{inverse} count mismatch: {owner_count}/{inverse_count}" + ), + ); + } + } + } + if issue_budget.truncated { + inspection.unowned_issues.push(format!( + "restore index inspection diagnostics exceeded the {}-issue limit", + MAX_RESTORE_INDEX_INSPECTION_ISSUES + )); + } + Ok(inspection) + } + + fn inspect_complete_restore_index_source_closure( + &self, + read_version: Version, + inspection: &mut RestoreIndexInspection, + issue_budget: &mut RestoreIndexInspectionIssueBudget, + ) -> Result<(), MetadError> { + let complete_ref_sets = inspection + .ref_sets + .iter() + .filter_map(|(ref_set_id, ref_set)| { + ref_set + .complete_identity + .as_ref() + .map(|identity| (*ref_set_id, identity.operation_digest)) + }) + .collect::>(); + for (ref_set_id, operation_digest) in complete_ref_sets { + let prefix = restore_staging_member_prefix(self.mount, ref_set_id); + let mut scanned = 0_usize; + self.restore_index_for_each_raw( + RecordFamily::System, + &prefix, + None, + read_version, + ReadPurpose::WritePlanLocal, + RESTORE_INDEX_SCAN_PAGE, + |row| { + scanned = scanned.saturating_add(1); + validate_restore_index_collection( + "restore index fsck staging members", + scanned, + )?; + let Ok(member) = decode_restore_staging_member(&row.value.0) else { + // The staging-member fsck owns malformed staging rows. + return Ok(true); + }; + let Some(source_inode) = member.source_inode else { + return Ok(true); + }; + let source_key = + restore_index_source_member_key(self.mount, ref_set_id, source_inode); + let matches = self + .metadata + .get( + RecordFamily::System, + &source_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .is_some_and(|value| { + decode_restore_index_source_member(&value.0).is_ok_and(|source| { + source.owner.operation_digest == operation_digest + && source.owner.ref_set_id == ref_set_id + && source.source_inode == source_inode + && restore_index_source_member_matches_staging( + &source.member, + &member, + ) + }) + }); + if !matches { + let Some(ref_set) = inspection.ref_sets.get_mut(&ref_set_id) else { + return Err(MetadError::Codec( + "restore index source closure lost its ref-set".to_owned(), + )); + }; + issue_budget.push( + &mut ref_set.closure_issues, + format!( + "ref-set {ref_set_id} staging member for source inode {} has no exact source member", + source_inode.get() + ), + ); + } + Ok(true) + }, + )?; + } + Ok(()) + } + + fn restore_index_inspected_row_identity( + &self, + name: &str, + row: &crate::command::ScanItem, + ) -> Result { + if name == "index_seal" { + let seal = decode_restore_index_seal(&row.value.0)?; + if row.key != restore_index_seal_key(self.mount, seal.ref_set_id) { + return Err(MetadError::Codec( + "restore index inspection found an invalid seal key".to_owned(), + )); + } + return Ok(RestoreIndexInspectedRowIdentity { + ref_set_id: seal.ref_set_id, + operation_digest: seal.operation_digest, + is_put: false, + is_tombstone: false, + seal_identity: Some(RestoreIndexDurableIdentity { + operation_digest: seal.operation_digest, + initialization_digest: seal.initialization_digest, + incarnation: seal.incarnation, + }), + complete_identity: None, + }); + } + if name == "index_complete" { + let marker = decode_restore_index_complete(&row.value.0)?; + if row.key != restore_index_complete_key(self.mount, marker.ref_set_id) { + return Err(MetadError::Codec( + "restore index inspection found an invalid Complete key".to_owned(), + )); + } + return Ok(RestoreIndexInspectedRowIdentity { + ref_set_id: marker.ref_set_id, + operation_digest: marker.operation_digest, + is_put: false, + is_tombstone: false, + seal_identity: None, + complete_identity: Some(RestoreIndexDurableIdentity { + operation_digest: marker.operation_digest, + initialization_digest: marker.initialization_digest, + incarnation: marker.incarnation, + }), + }); + } + if name.starts_with("index_mvcc_") { + let record = decode_restore_index_mvcc(&row.value.0)?; + let commit_version = Version::new(record.commit_version)?; + if row.version != commit_version + || row.key + != restore_index_mvcc_key(self.mount, &record.logical_key, commit_version)? + { + return Err(MetadError::Codec( + "restore index inspection found an invalid MVCC physical key".to_owned(), + )); + } + let label = restore_index_logical_label_for_key(self.mount, &record.logical_key) + .ok_or_else(|| { + MetadError::Codec( + "restore index inspection found an unknown MVCC logical key".to_owned(), + ) + })?; + let owner = restore_index_owner_for_logical_value(label, &record.value)?; + return Ok(RestoreIndexInspectedRowIdentity { + ref_set_id: owner.ref_set_id, + operation_digest: owner.operation_digest, + is_put: record.kind == RestoreIndexMvccKind::Put, + is_tombstone: record.kind == RestoreIndexMvccKind::Tombstone, + seal_identity: None, + complete_identity: None, + }); + } + let label = restore_index_logical_label_for_key(self.mount, &row.key).ok_or_else(|| { + MetadError::Codec("restore index inspection found an unknown logical key".to_owned()) + })?; + let owner = restore_index_owner_for_logical_value(label, &row.value.0)?; + Ok(RestoreIndexInspectedRowIdentity { + ref_set_id: owner.ref_set_id, + operation_digest: owner.operation_digest, + is_put: false, + is_tombstone: false, + seal_identity: None, + complete_identity: None, + }) + } + + fn restore_index_inspected_row_closure( + &self, + name: &str, + row: &crate::command::ScanItem, + read_version: Version, + ) -> Result<(RestoreIndexOwner, Option), MetadError> { + if name.starts_with("index_mvcc_") { + let record = decode_restore_index_mvcc(&row.value.0)?; + let commit_version = Version::new(record.commit_version)?; + let logical_label = + restore_index_logical_label_for_key(self.mount, &record.logical_key).ok_or_else( + || { + MetadError::Codec( + "restore index MVCC inspection found an unknown logical key".to_owned(), + ) + }, + )?; + let owner = restore_index_owner_for_logical_value(logical_label, &record.value)?; + if row.version != commit_version + || row.key + != restore_index_mvcc_key(self.mount, &record.logical_key, commit_version)? + { + return Err(MetadError::Codec( + "restore index MVCC inspection found a physical identity mismatch".to_owned(), + )); + } + let Some((counterpart_label, counterpart_logical_key)) = + restore_index_inspection_counterpart( + self.mount, + logical_label, + &record.logical_key, + &record.value, + false, + )? + else { + return Ok((owner, None)); + }; + let counterpart_name = restore_index_inspection_keyspace_name(counterpart_label, true)?; + let counterpart_key = + restore_index_mvcc_key(self.mount, &counterpart_logical_key, commit_version)?; + let mut counterpart_record = record; + counterpart_record.logical_key = counterpart_logical_key; + let expected = encode_restore_index_mvcc(&counterpart_record); + let matches = self + .metadata + .get_versioned( + RecordFamily::System, + &counterpart_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .is_some_and(|item| item.value.0 == expected && item.version == commit_version); + let issue = (!matches).then(|| { + format!( + "ref-set {} {name} row has no exact {counterpart_name} at commit {}", + owner.ref_set_id, + commit_version.get() + ) + }); + return Ok((owner, issue)); + } + + let logical_label = + restore_index_logical_label_for_key(self.mount, &row.key).ok_or_else(|| { + MetadError::Codec( + "restore index inspection found an unknown logical key".to_owned(), + ) + })?; + let owner = restore_index_owner_for_logical_value(logical_label, &row.value.0)?; + if logical_label == INDEX_SOURCE_MEMBER_LABEL { + let source = decode_restore_index_source_member(&row.value.0)?; + let staging_key = restore_staging_member_key( + self.mount, + owner.ref_set_id, + source.member.destination_inode, + ); + let matches = self + .metadata + .get( + RecordFamily::System, + &staging_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .is_some_and(|value| { + decode_restore_staging_member(&value.0).is_ok_and(|member| { + restore_index_source_member_matches_staging(&source.member, &member) + }) + }); + return Ok(( + owner, + (!matches).then(|| { + format!( + "ref-set {} source member has no exact staging member", + owner.ref_set_id + ) + }), + )); + } + let Some((counterpart_label, counterpart_key)) = restore_index_inspection_counterpart( + self.mount, + logical_label, + &row.key, + &row.value.0, + true, + )? + else { + return Ok((owner, None)); + }; + let counterpart_name = restore_index_inspection_keyspace_name(counterpart_label, false)?; + let expected = if logical_label == INDEX_ENTRY_LABEL { + encode_restore_index_owner(owner) + } else { + row.value.0.clone() + }; + let matches = self + .metadata + .get_versioned( + RecordFamily::System, + &counterpart_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .is_some_and(|item| { + item.value.0 == expected + && (logical_label == INDEX_ENTRY_LABEL || item.version == row.version) + }); + let issue = (!matches).then(|| { + format!( + "ref-set {} {name} row has no exact {counterpart_name}", + owner.ref_set_id + ) + }); + Ok((owner, issue)) + } + + #[allow(clippy::too_many_arguments)] + fn restore_index_for_each_raw( + &self, + family: RecordFamily, + prefix: &[u8], + initial_start_after: Option<&[u8]>, + version: Version, + purpose: ReadPurpose, + page_limit: usize, + mut visit: F, + ) -> Result<(), MetadError> + where + F: FnMut(crate::command::ScanItem) -> Result, + { + if page_limit == 0 || page_limit > RESTORE_INDEX_SCAN_PAGE { + return Err(MetadError::Codec(format!( + "restore index scan page {page_limit} is outside 1..={RESTORE_INDEX_SCAN_PAGE}" + ))); + } + let mut start_after = initial_start_after.map(ToOwned::to_owned); + loop { + let page = self.metadata.scan(ScanRequest { + family, + prefix: prefix.to_vec(), + start_after: start_after.clone(), + version, + limit: page_limit, + purpose, + })?; + if page.is_empty() { + break; + } + let reached_tail = page.len() < page_limit; + let mut previous = start_after.as_deref(); + for row in &page { + if !row.key.starts_with(prefix) + || previous.is_some_and(|previous| previous >= row.key.as_slice()) + { + return Err(MetadError::Codec( + "restore index paged scan returned an invalid key order".to_owned(), + )); + } + previous = Some(&row.key); + } + start_after = page.last().map(|row| row.key.clone()); + for row in page { + if !visit(row)? { + return Ok(()); + } + } + if reached_tail { + break; + } + } + Ok(()) + } + + fn restore_index_for_each_mvcc_latest( + &self, + logical_prefix: &[u8], + logical_start_after: Option<&[u8]>, + exact: bool, + requested_version: Version, + purpose: ReadPurpose, + mut visit: F, + ) -> Result<(), MetadError> + where + F: FnMut(SelectedRestoreIndexMvcc) -> Result, + { + let mut physical_prefix = restore_index_mvcc_prefix(self.mount, logical_prefix)?; + if exact { + physical_prefix.push(RESTORE_INDEX_MVCC_VERSION_DELIMITER); + } + let mut start_after = logical_start_after + .map(|logical_key| { + restore_index_mvcc_key( + self.mount, + logical_key, + Version::new(u64::MAX).expect("u64::MAX is a valid non-zero version"), + ) + }) + .transpose()?; + // System is current-only, so the physical COW rows are always read at + // the current control version and selected by their embedded commit + // version. Reading them at `requested_version` would silently drop the + // very history this keyspace exists to preserve. + let control_version = self.read_version()?; + let scan_purpose = match purpose { + ReadPurpose::Snapshot => ReadPurpose::WritePlanLocal, + other => other, + }; + let mut current_logical = None::>; + let mut current_candidate = None::; + loop { + let page = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: physical_prefix.clone(), + start_after: start_after.clone(), + version: control_version, + limit: RESTORE_INDEX_SCAN_PAGE, + purpose: scan_purpose, + })?; + if page.is_empty() { + break; + } + let reached_tail = page.len() < RESTORE_INDEX_SCAN_PAGE; + for item in &page { + let record = decode_restore_index_mvcc(&item.value.0)?; + let commit_version = Version::new(record.commit_version)?; + if item.version != commit_version + || item.key + != restore_index_mvcc_key(self.mount, &record.logical_key, commit_version)? + || !record.logical_key.starts_with(logical_prefix) + || (exact && record.logical_key != logical_prefix) + { + return Err(MetadError::Codec( + "restore index MVCC physical/logical identity changed".to_owned(), + )); + } + if current_logical + .as_ref() + .is_some_and(|logical| logical != &record.logical_key) + { + if current_logical + .as_ref() + .is_some_and(|logical| logical >= &record.logical_key) + { + return Err(MetadError::Codec( + "restore index MVCC logical rows are not strictly ordered".to_owned(), + )); + } + if let Some(candidate) = current_candidate.take() { + if !visit(candidate)? { + return Ok(()); + } + } + current_logical = Some(record.logical_key.clone()); + } else if current_logical.is_none() { + current_logical = Some(record.logical_key.clone()); + } + if record.commit_version <= requested_version.get() { + if current_candidate.as_ref().is_some_and(|candidate| { + candidate.record.commit_version >= record.commit_version + }) { + return Err(MetadError::Codec( + "restore index MVCC versions are not strictly ordered".to_owned(), + )); + } + current_candidate = Some(SelectedRestoreIndexMvcc { + logical_key: record.logical_key.clone(), + record, + }); + } + } + start_after = page.last().map(|item| item.key.clone()); + if reached_tail { + break; + } + } + if let Some(candidate) = current_candidate { + let _ = visit(candidate)?; + } + Ok(()) + } + + fn restore_index_mvcc_value( + &self, + logical_key: &[u8], + requested_version: Version, + purpose: ReadPurpose, + ) -> Result, MetadError> { + let mut selected = None; + self.restore_index_for_each_mvcc_latest( + logical_key, + None, + true, + requested_version, + purpose, + |row| { + if selected.replace(row).is_some() { + return Err(MetadError::Codec( + "restore index MVCC exact lookup is ambiguous".to_owned(), + )); + } + Ok(true) + }, + )?; + Ok(selected) + } + + fn restore_index_effective_for_each( + &self, + logical_prefix: &[u8], + logical_start_after: Option<&[u8]>, + requested_version: Version, + purpose: ReadPurpose, + mut visit: F, + ) -> Result<(), MetadError> + where + F: FnMut(crate::command::ScanItem) -> Result, + { + if purpose == ReadPurpose::Snapshot { + return self.restore_index_for_each_mvcc_latest( + logical_prefix, + logical_start_after, + false, + requested_version, + purpose, + |row| { + if row.record.kind == RestoreIndexMvccKind::Tombstone { + return Ok(true); + } + visit(crate::command::ScanItem { + key: row.logical_key, + value: Value(row.record.value), + version: Version::new(row.record.commit_version)?, + }) + }, + ); + } + + let mut start_after = logical_start_after.map(ToOwned::to_owned); + loop { + let page = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: logical_prefix.to_vec(), + start_after: start_after.clone(), + version: requested_version, + limit: RESTORE_INDEX_SCAN_PAGE, + purpose, + })?; + if page.is_empty() { + break; + } + let reached_tail = page.len() < RESTORE_INDEX_SCAN_PAGE; + for row in &page { + if !visit(row.clone())? { + return Ok(()); + } + } + start_after = page.last().map(|row| row.key.clone()); + if reached_tail { + break; + } + } + Ok(()) + } + + fn restore_index_effective_get( + &self, + logical_key: &[u8], + requested_version: Version, + purpose: ReadPurpose, + ) -> Result, MetadError> { + if purpose != ReadPurpose::Snapshot { + return Ok(self.metadata.get_versioned( + RecordFamily::System, + logical_key, + requested_version, + purpose, + )?); + } + let Some(row) = self.restore_index_mvcc_value(logical_key, requested_version, purpose)? + else { + return Ok(None); + }; + if row.record.kind == RestoreIndexMvccKind::Tombstone { + return Ok(None); + } + Ok(Some(crate::command::ReadItem { + value: Value(row.record.value), + version: Version::new(row.record.commit_version)?, + })) + } + + fn finalize_restore_index_mvcc_plan( + &self, + mut plan: RestoreIndexMutationPlan, + read_version: Version, + commit_version: Version, + ) -> Result { + plan.validate_budget("restore index mutation plan")?; + let head_mutations = plan.mutations.clone(); + for mutation in head_mutations { + if mutation.family != RecordFamily::System + || restore_index_logical_label_for_key(self.mount, &mutation.key).is_none() + { + continue; + } + let (kind, value) = match mutation.op { + MutationOp::Put => ( + RestoreIndexMvccKind::Put, + mutation + .value + .as_ref() + .ok_or_else(|| { + MetadError::Codec("restore index put mutation has no value".to_owned()) + })? + .0 + .clone(), + ), + MutationOp::Delete => { + let existing = self + .metadata + .get( + RecordFamily::System, + &mutation.key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "restore index tombstone has no durable head".to_owned(), + ) + })?; + (RestoreIndexMvccKind::Tombstone, existing.0) + } + }; + let physical_key = restore_index_mvcc_key(self.mount, &mutation.key, commit_version)?; + plan.push_predicate(PredicateRef { + family: RecordFamily::System, + key: physical_key.clone(), + predicate: Predicate::NotExists, + })?; + plan.set_mutation(Mutation { + family: RecordFamily::System, + key: physical_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_index_mvcc(&RestoreIndexMvccRecord { + commit_version: commit_version.get(), + kind, + logical_key: mutation.key, + value, + }))), + }); + plan.validate_budget("restore index mutation plan with MVCC")?; + } + Ok(plan) + } + + /// Attach-time visibility pointer. The caller must merge this plan into + /// the single command that publishes the destination dentry and changes + /// the operation to `Complete`. + pub(super) fn restore_index_complete_plan( + &self, + operation: &RestoreOperation, + commit_version: Version, + ) -> Result { + let key = restore_index_complete_key(self.mount, operation.ref_set_id); + let marker = RestoreIndexCompleteMarker { + operation_digest: operation.operation_digest, + initialization_digest: operation.initialization_digest, + ref_set_id: operation.ref_set_id, + incarnation: operation.created_version, + complete_version: commit_version.get(), + }; + Ok(RestoreIndexMutationPlan { + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key, + op: MutationOp::Put, + value: Some(Value(encode_restore_index_complete(&marker))), + }], + }) + } + + /// Ordinary inode-addressed reads must not bypass the detached restore's + /// first-visibility boundary. Only explicit restore staging reads may + /// materialize and seal the tree before attach. Callers hold + /// `restore_visibility_fence` through this check and the namespace read, + /// which linearizes the false fast path with the first durable hold. + pub(super) fn restore_inode_visible_at( + &self, + inode: InodeId, + version: Version, + purpose: ReadPurpose, + ) -> Result { + self.ensure_metadata_checkpoint_install_stable()?; + if purpose == ReadPurpose::RestoreStaging { + return Ok(true); + } + if !self.restore_staging_possible.load(Ordering::Acquire) { + return Ok(true); + } + let control_version = self.read_version()?; + let Some(inverse) = self.metadata.get( + RecordFamily::System, + &restore_staging_inode_key(self.mount, inode), + control_version, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(true); + }; + let (operation_digest, ref_set_id) = decode_restore_staging_inverse(&inverse.0)?; + let operation = self + .metadata + .get( + RecordFamily::System, + &restore_operation_key(self.mount, &operation_digest), + control_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore staging inode has no operation".to_owned()) + })?; + let operation = decode_restore_operation(&operation.0)?; + if operation.operation_digest != operation_digest || operation.ref_set_id != ref_set_id { + return Err(MetadError::Codec( + "restore staging inode does not match its operation".to_owned(), + )); + } + let marker_key = restore_index_complete_key(self.mount, ref_set_id); + let marker = self + .metadata + .get( + RecordFamily::System, + &marker_key, + control_version, + ReadPurpose::WritePlanLocal, + )? + .map(|value| decode_restore_index_complete(&value.0)) + .transpose()?; + let marker_visible = marker.as_ref().is_some_and(|marker| { + marker.operation_digest == operation.operation_digest + && marker.initialization_digest == operation.initialization_digest + && marker.ref_set_id == operation.ref_set_id + && marker.incarnation == operation.created_version + && marker.complete_version <= version.get() + }); + match operation.state { + RestoreOperationState::Complete if purpose == ReadPurpose::Snapshot => { + Ok(marker_visible) + } + RestoreOperationState::Complete => Ok(marker_visible + && self.restore_inode_reachable_from_mount(inode, control_version)?), + RestoreOperationState::Releasing if purpose == ReadPurpose::Snapshot => { + // Release is fenced by live SnapshotPin/ref-set retention. A + // historical snapshot must not be filtered through current + // namespace reachability after the root was removed. + Ok(marker_visible) + } + RestoreOperationState::Releasing => Ok(marker_visible + && self.restore_inode_reachable_from_mount(inode, control_version)?), + RestoreOperationState::Preparing + | RestoreOperationState::ReadyToAttach + | RestoreOperationState::Cleaning + | RestoreOperationState::Discarding => Ok(false), + } + } + + /// A Complete operation may enable the process-local read fast path only + /// when its visibility marker is durably present and matches the exact + /// operation incarnation. This is checked during explicit recovery hooks, + /// never on an ordinary read. + pub(super) fn validate_restore_complete_visibility_marker( + &self, + operation: &RestoreOperation, + version: Version, + ) -> Result<(), MetadError> { + let marker = self + .metadata + .get( + RecordFamily::System, + &restore_index_complete_key(self.mount, operation.ref_set_id), + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("complete restore operation has no visibility marker".to_owned()) + })?; + let marker = decode_restore_index_complete(&marker.0)?; + if marker.operation_digest != operation.operation_digest + || marker.initialization_digest != operation.initialization_digest + || marker.ref_set_id != operation.ref_set_id + || marker.incarnation != operation.created_version + || marker.complete_version > version.get() + { + return Err(MetadError::Codec( + "complete restore visibility marker changed identity".to_owned(), + )); + } + Ok(()) + } + + fn visible_restore_index_operation( + &self, + owner: RestoreIndexOwner, + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadError> { + Ok(self + .visible_restore_index_operation_versioned(owner, version, purpose)? + .map(|(operation, _)| operation)) + } + + fn visible_restore_index_operation_versioned( + &self, + owner: RestoreIndexOwner, + version: Version, + _purpose: ReadPurpose, + ) -> Result, MetadError> { + let key = restore_operation_key(self.mount, &owner.operation_digest); + let control_version = self.read_version()?; + let Some(item) = self.metadata.get_versioned( + RecordFamily::System, + &key, + control_version, + ReadPurpose::WritePlanLocal, + )? + else { + return Err(MetadError::Codec( + "restore index owner has no operation".to_owned(), + )); + }; + let operation = decode_restore_operation(&item.value.0)?; + if operation.operation_digest != owner.operation_digest + || operation.ref_set_id != owner.ref_set_id + { + return Err(MetadError::Codec( + "restore index owner does not match its operation".to_owned(), + )); + } + if !matches!( + operation.state, + RestoreOperationState::Complete | RestoreOperationState::Releasing + ) { + return Ok(None); + } + let marker_key = restore_index_complete_key(self.mount, owner.ref_set_id); + let marker = self + .metadata + .get( + RecordFamily::System, + &marker_key, + control_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("visible restore index has no Complete marker".to_owned()) + })?; + let marker = decode_restore_index_complete(&marker.0)?; + if marker.operation_digest != operation.operation_digest + || marker.initialization_digest != operation.initialization_digest + || marker.ref_set_id != operation.ref_set_id + || marker.incarnation != operation.created_version + { + return Err(MetadError::Codec( + "restore index Complete marker changed identity".to_owned(), + )); + } + if marker.complete_version > version.get() { + return Ok(None); + } + Ok(Some((operation, item.version))) + } + + fn restore_index_staging_guards( + &self, + operation: &RestoreOperation, + version: Version, + ) -> Result, MetadError> { + let operation_key = restore_operation_key(self.mount, &operation.operation_digest); + let operation_item = self + .metadata + .get_versioned( + RecordFamily::System, + &operation_key, + version, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreInProgress)?; + let durable = decode_restore_operation(&operation_item.value.0)?; + if durable != *operation || durable.state != RestoreOperationState::Preparing { + return Err(MetadError::RestoreInProgress); + } + let binding_key = fork_binding_key(self.mount, operation.destination_root); + let binding_item = self + .metadata + .get_versioned( + RecordFamily::ForkBinding, + &binding_key, + version, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreBindingChanged { + root: operation.destination_root, + })?; + let binding = crate::layout::decode_fork_binding(&binding_item.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if binding.fork_root != operation.destination_root + || binding.source_root != operation.source_root + || binding.pinned_read_version != operation.read_version + || binding.snapshot_id != operation.snapshot_id + || binding.created_version != operation.created_version + { + return Err(MetadError::RestoreBindingChanged { + root: operation.destination_root, + }); + } + Ok(vec![ + PredicateRef { + family: RecordFamily::System, + key: operation_key, + predicate: Predicate::VersionEquals(operation_item.version), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key, + predicate: Predicate::VersionEquals(binding_item.version), + }, + ]) + } + + fn restore_index_unique_puts( + desired: Vec, + context: &str, + ) -> Result, Mutation>, MetadError> { + let mut unique = BTreeMap::, Mutation>::new(); + for mutation in desired { + if mutation.family != RecordFamily::System + || mutation.op != MutationOp::Put + || mutation.value.is_none() + { + return Err(MetadError::Codec(format!( + "restore index {context} emitted an invalid mutation" + ))); + } + match unique.entry(mutation.key.clone()) { + std::collections::btree_map::Entry::Vacant(slot) => { + slot.insert(mutation); + } + std::collections::btree_map::Entry::Occupied(slot) + if slot.get().value != mutation.value => + { + return Err(MetadError::Codec(format!( + "restore index {context} disagrees on a durable row" + ))); + } + std::collections::btree_map::Entry::Occupied(_) => {} + } + } + Ok(unique) + } + + fn append_fresh_restore_index_put( + &self, + mutation: Mutation, + version: Version, + predicates: &mut Vec, + mutations: &mut Vec, + ) -> Result<(), MetadError> { + let desired_value = mutation + .value + .as_ref() + .ok_or_else(|| MetadError::Codec("restore index fresh put has no value".to_owned()))?; + let physical_key = restore_index_mvcc_key(self.mount, &mutation.key, version)?; + predicates.extend([ + PredicateRef { + family: RecordFamily::System, + key: mutation.key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: physical_key.clone(), + predicate: Predicate::NotExists, + }, + ]); + mutations.push(Mutation { + family: RecordFamily::System, + key: physical_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_index_mvcc(&RestoreIndexMvccRecord { + commit_version: version.get(), + kind: RestoreIndexMvccKind::Put, + logical_key: mutation.key.clone(), + value: desired_value.0.clone(), + }))), + }); + mutations.push(mutation); + Ok(()) + } + + fn restore_index_materialization_command( + &self, + operation: &RestoreOperation, + primary_key: Vec, + version: Version, + predicates: Vec, + mutations: Vec, + ) -> Result { + Ok(MetadataCommand { + request_id: request_id( + b"restore-materialize-index", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::RegisterNamespaceIndex, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key, + predicates, + mutations, + watch: Vec::new(), + }) + } + + /// Validate the fresh-operation command shape without consulting or + /// mutating durable restore state. A new ref-set has no existing heads, so + /// every desired owner/inverse row carries the same head + physical-MVCC + /// predicates and puts that `commit_restore_index_put_batch` will emit + /// after the hold. + fn preflight_restore_index_put_batch( + &self, + operation: &RestoreOperation, + primary_key: Vec, + desired: Vec, + ) -> Result<(), MetadError> { + if desired.is_empty() { + return Ok(()); + } + let unique = Self::restore_index_unique_puts(desired, "preflight")?; + + let version = Version::new(operation.created_version)?; + let read_version = predecessor(version)?; + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(read_version), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: fork_binding_key(self.mount, operation.destination_root), + predicate: Predicate::VersionEquals(read_version), + }, + ]; + let mut mutations = Vec::with_capacity(unique.len() * 2); + for (_, mutation) in unique { + self.append_fresh_restore_index_put( + mutation, + version, + &mut predicates, + &mut mutations, + )?; + } + let command = self.restore_index_materialization_command( + operation, + primary_key, + version, + predicates, + mutations, + )?; + super::restore::validate_restore_command_bounds( + &command, + "restore index materialization batch", + ) + } + + fn commit_restore_index_put_batch( + &self, + operation: &RestoreOperation, + primary_key: Vec, + desired: Vec, + ) -> Result<(), MetadError> { + if desired.is_empty() { + return Ok(()); + } + let unique = Self::restore_index_unique_puts(desired, "materialization")?; + + let version = self.next_version()?; + let read_version = predecessor(version)?; + let mut predicates = self.restore_index_staging_guards(operation, read_version)?; + let mut mutations = Vec::with_capacity(unique.len() * 2); + for (_, mutation) in unique { + let desired_value = mutation.value.as_ref().expect("put value"); + match self.metadata.get_versioned( + RecordFamily::System, + &mutation.key, + read_version, + ReadPurpose::RestoreStaging, + )? { + Some(existing) if existing.value == *desired_value => { + let history = self + .restore_index_mvcc_value( + &mutation.key, + read_version, + ReadPurpose::RestoreStaging, + )? + .ok_or_else(|| { + MetadError::Codec( + "restore index durable head has no MVCC Put".to_owned(), + ) + })?; + if history.record.kind != RestoreIndexMvccKind::Put + || history.record.commit_version != existing.version.get() + || history.record.value != desired_value.0 + { + return Err(MetadError::Codec( + "restore index durable head/MVCC Put mismatch".to_owned(), + )); + } + predicates.push(PredicateRef { + family: RecordFamily::System, + key: mutation.key, + predicate: Predicate::VersionEquals(existing.version), + }); + } + Some(_) => { + return Err(MetadError::Codec( + "restore index materialization found a conflicting row".to_owned(), + )) + } + None => { + self.append_fresh_restore_index_put( + mutation, + version, + &mut predicates, + &mut mutations, + )?; + } + } + } + if mutations.is_empty() { + return Ok(()); + } + let command = self.restore_index_materialization_command( + operation, + primary_key, + version, + predicates, + mutations, + )?; + super::restore::validate_restore_command_bounds( + &command, + "restore index materialization batch", + )?; + self.commit_metadata(command)?; + Ok(()) + } + + fn restore_index_staging_member_from_row( + &self, + operation: &RestoreOperation, + version: Version, + row: &crate::command::ScanItem, + ) -> Result { + let member = decode_restore_staging_member(&row.value.0)?; + if member.operation_digest != operation.operation_digest + || row.key + != restore_staging_member_key( + self.mount, + operation.ref_set_id, + member.destination_inode, + ) + { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + let inverse_key = restore_staging_inode_key(self.mount, member.destination_inode); + let inverse = self + .metadata + .get( + RecordFamily::System, + &inverse_key, + version, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreRootChanged { + root: member.destination_inode, + })?; + let (digest, ref_set_id) = decode_restore_staging_inverse(&inverse.0)?; + if digest != operation.operation_digest || ref_set_id != operation.ref_set_id { + return Err(MetadError::RestoreRootChanged { + root: member.destination_inode, + }); + } + Ok(member) + } + + fn restore_index_projection_for_member( + &self, + operation: &RestoreOperation, + member: &RestoreStagingMember, + destination_parent: InodeId, + destination_name: &DentryName, + version: Version, + ) -> Result { + if member.destination_inode == operation.destination_root { + if member.destination_parent.is_some() + || member.name.is_some() + || !member.relative_path.is_empty() + { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + let attr = self + .get_attr_at_version_for_purpose( + operation.destination_root, + version, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreRootChanged { + root: operation.destination_root, + })?; + return Ok(projection( + destination_parent, + destination_name.clone(), + attr, + None, + )); + } + let (Some(parent), Some(name)) = (member.destination_parent, member.name.as_ref()) else { + return Err(MetadError::RestoreRootChanged { + root: member.destination_inode, + }); + }; + let (entry, _) = self + .lookup_plus_at_version_for_purpose(parent, name, version, ReadPurpose::RestoreStaging)? + .ok_or(MetadError::RestoreRootChanged { + root: member.destination_inode, + })?; + if entry.attr.inode != member.destination_inode { + return Err(MetadError::RestoreRootChanged { + root: member.destination_inode, + }); + } + Ok(DentryProjection { + dentry: entry.dentry, + attr: entry.attr, + body: entry.body, + }) + } + + fn flush_restore_canonical_index_preflight_batch( + &self, + operation: &RestoreOperation, + batch: &mut RestoreCanonicalIndexPreflightBatch, + ) -> Result<(), MetadError> { + if batch.members == 0 { + return Ok(()); + } + let desired = std::mem::take(&mut batch.desired); + batch.members = 0; + self.preflight_restore_index_put_batch( + operation, + restore_index_entry_prefix(self.mount, operation.ref_set_id, None), + desired, + ) + } + + /// Feed the initialization-adjusted final staging-member stream in + /// destination-inode order. The indexed predicate and mutation builders are + /// shared with runtime materialization; only the durable reads/commit are + /// replaced by the fresh-ref-set command-shape validator. + pub(super) fn push_restore_canonical_index_preflight_member( + &self, + operation: &RestoreOperation, + member: &RestoreStagingMember, + projection: &DentryProjection, + batch: &mut RestoreCanonicalIndexPreflightBatch, + ) -> Result<(), MetadError> { + if let Some(source_inode) = member.source_inode { + batch.desired.push(restore_index_source_member_mutation( + self.mount, + &RestoreIndexSourceMember { + owner: RestoreIndexOwner::from_operation(operation), + source_inode, + member: member.clone(), + }, + )?); + } + if self.restore_source_member_is_indexed(operation, member, projection)? { + batch.desired.extend( + restore_index_materialization_mutations(self.mount, operation, projection) + .mutations, + ); + } + batch.members = batch.members.saturating_add(1); + if batch.members == RESTORE_BATCH_ENTRIES { + self.flush_restore_canonical_index_preflight_batch(operation, batch)?; + } + Ok(()) + } + + pub(super) fn finish_restore_canonical_index_preflight( + &self, + operation: &RestoreOperation, + batch: &mut RestoreCanonicalIndexPreflightBatch, + ) -> Result<(), MetadError> { + self.flush_restore_canonical_index_preflight_batch(operation, batch) + } + + fn materialize_restore_canonical_indexes( + &self, + operation: &RestoreOperation, + destination_parent: InodeId, + destination_name: &DentryName, + ) -> Result<(), MetadError> { + let scan_version = self.read_version()?; + let prefix = restore_staging_member_prefix(self.mount, operation.ref_set_id); + let mut start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after: start_after.clone(), + version: scan_version, + limit: RESTORE_BATCH_ENTRIES, + purpose: ReadPurpose::RestoreStaging, + })?; + if rows.is_empty() { + break; + } + let reached_tail = rows.len() < RESTORE_BATCH_ENTRIES; + let version = self.read_version()?; + let mut desired = Vec::with_capacity(rows.len() * 4); + for row in &rows { + let member = + self.restore_index_staging_member_from_row(operation, scan_version, row)?; + if let Some(source_inode) = member.source_inode { + desired.push(restore_index_source_member_mutation( + self.mount, + &RestoreIndexSourceMember { + owner: RestoreIndexOwner::from_operation(operation), + source_inode, + member: member.clone(), + }, + )?); + } + let projection = self.restore_index_projection_for_member( + operation, + &member, + destination_parent, + destination_name, + version, + )?; + if self.restore_source_member_is_indexed(operation, &member, &projection)? { + desired.extend( + restore_index_materialization_mutations(self.mount, operation, &projection) + .mutations, + ); + } + } + self.commit_restore_index_put_batch( + operation, + restore_index_entry_prefix(self.mount, operation.ref_set_id, None), + desired, + )?; + start_after = rows.last().map(|row| row.key.clone()); + if reached_tail { + break; + } + } + Ok(()) + } + + fn restore_source_member_is_indexed( + &self, + operation: &RestoreOperation, + member: &RestoreStagingMember, + destination: &DentryProjection, + ) -> Result { + // Initialization-created/replaced artifacts follow the normal + // path-aware publish behavior. Directories and bodiless nodes are not + // introduced into an index merely because restore traversed them. + let Some(source_inode) = member.source_inode else { + return Ok(destination.body.is_some()); + }; + let mut components = parse_absolute_path(&operation.source_path)?; + if !member.relative_path.is_empty() { + components.extend(parse_absolute_path(&format!("/{}", member.relative_path))?); + } + let Some((name, parent_components)) = components.split_last() else { + return Ok(false); + }; + let source_version = Version::new(operation.read_version)?; + let parent = match self.resolve_components_as_directory_from_at_version_for_purpose( + InodeId::root(), + parent_components, + source_version, + ReadPurpose::Snapshot, + ) { + Ok(parent) => parent, + Err(MetadError::NotFound | MetadError::NotDirectory) => return Ok(false), + Err(error) => return Err(error), + }; + let Some((source, _)) = self.lookup_plus_at_version_for_purpose( + parent, + name, + source_version, + ReadPurpose::Snapshot, + )? + else { + return Ok(false); + }; + if source.attr.inode != source_inode { + return Err(MetadError::RestoreRootChanged { + root: operation.source_root, + }); + } + let source_projection = DentryProjection { + dentry: source.dentry, + attr: source.attr, + body: source.body, + }; + let path_key = path_index_key(self.mount, &components); + if let Some(value) = self.metadata.get( + RecordFamily::PathIndex, + &path_key, + source_version, + ReadPurpose::Snapshot, + )? { + let indexed = decode_dentry_projection(&value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if indexed == source_projection { + return Ok(true); + } + } + if self + .restore_index_entries_for_location_at( + parent, + name, + source_version, + ReadPurpose::Snapshot, + )? + .into_iter() + .any(|indexed| indexed.entry.projection == source_projection) + { + return Ok(true); + } + + let Some(fork_source) = self.restore_index_fork_source(operation.source_root)? else { + return Ok(false); + }; + if member.relative_path.is_empty() { + return Ok(false); + } + let relative = parse_absolute_path(&format!("/{}", member.relative_path))?; + let Some((source_name, source_parent_components)) = relative.split_last() else { + return Ok(false); + }; + let pinned_version = Version::new(fork_source.binding.pinned_read_version)?; + let source_parent = self.resolve_components_as_directory_from_at_version_for_purpose( + fork_source.binding.source_root, + source_parent_components, + pinned_version, + ReadPurpose::Snapshot, + )?; + let Some((fork_origin, _)) = self.lookup_plus_at_version_for_purpose( + source_parent, + source_name, + pinned_version, + ReadPurpose::Snapshot, + )? + else { + return Ok(false); + }; + let clone_source: DentryWithAttr = source_projection.into(); + if !restore_index_clone_entry_matches_source(&clone_source, &fork_origin) { + return Ok(false); + } + let origin_projection = DentryProjection { + dentry: fork_origin.dentry.clone(), + attr: fork_origin.attr.clone(), + body: fork_origin.body.clone(), + }; + let origin_components = { + let mut components = parse_absolute_path(&fork_source.operation.destination_path)?; + components.extend_from_slice(&relative); + components + }; + if let Some(value) = self.metadata.get( + RecordFamily::PathIndex, + &path_index_key(self.mount, &origin_components), + pinned_version, + ReadPurpose::Snapshot, + )? { + let indexed = decode_dentry_projection(&value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if indexed == origin_projection { + return Ok(true); + } + } + Ok(self + .restore_index_entries_for_location_at( + source_parent, + source_name, + pinned_version, + ReadPurpose::Snapshot, + )? + .into_iter() + .any(|indexed| indexed.entry.projection == origin_projection)) + } + + fn restore_index_metadata_for_inode( + &self, + inode: InodeId, + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadError> { + let Some(attr) = self.get_attr_at_version_for_purpose(inode, version, purpose)? else { + return Ok(None); + }; + let body = if attr.file_type == FileType::File { + self.body_descriptor_at_version_for_purpose(inode, attr.generation, version, purpose)? + } else { + None + }; + Ok(Some(PathMetadata { attr, body })) + } + + fn restore_index_staging_member_for_inode( + &self, + operation: &RestoreOperation, + inode: InodeId, + version: Version, + ) -> Result, MetadError> { + let key = restore_staging_member_key(self.mount, operation.ref_set_id, inode); + let Some(item) = self.metadata.get_versioned( + RecordFamily::System, + &key, + version, + ReadPurpose::RestoreStaging, + )? + else { + return Ok(None); + }; + let row = crate::command::ScanItem { + key, + value: item.value, + version: item.version, + }; + Ok(Some(self.restore_index_staging_member_from_row( + operation, version, &row, + )?)) + } + + fn restore_index_staging_member_for_relative( + &self, + operation: &RestoreOperation, + relative: &[DentryName], + version: Version, + ) -> Result, MetadError> { + let relative_string = restore_index_relative_string(relative)?; + let inode = if relative.is_empty() { + operation.destination_root + } else { + let relative_path = canonical_path(relative)?; + let Some(metadata) = self.stat_path_from_at_version_for_purpose( + operation.destination_root, + &relative_path, + version, + ReadPurpose::RestoreStaging, + )? + else { + return Ok(None); + }; + metadata.attr.inode + }; + let Some(member) = + self.restore_index_staging_member_for_inode(operation, inode, version)? + else { + return Ok(None); + }; + if member.relative_path != relative_string { + return Err(MetadError::RestoreRootChanged { root: inode }); + } + Ok(Some(member)) + } + + fn restore_index_staging_member_for_source_inode( + &self, + operation: &RestoreOperation, + source_inode: InodeId, + version: Version, + ) -> Result, MetadError> { + let key = restore_index_source_member_key(self.mount, operation.ref_set_id, source_inode); + let Some(value) = self.metadata.get( + RecordFamily::System, + &key, + version, + ReadPurpose::RestoreStaging, + )? + else { + return Ok(None); + }; + let source = decode_restore_index_source_member(&value.0)?; + if source.owner != RestoreIndexOwner::from_operation(operation) + || source.source_inode != source_inode + || key + != restore_index_source_member_key( + self.mount, + source.owner.ref_set_id, + source.source_inode, + ) + { + return Err(MetadError::RestoreRootChanged { + root: operation.destination_root, + }); + } + let Some(current) = self.restore_index_staging_member_for_inode( + operation, + source.member.destination_inode, + version, + )? + else { + return Err(MetadError::RestoreRootChanged { + root: source.member.destination_inode, + }); + }; + if current != source.member { + return Err(MetadError::RestoreRootChanged { + root: source.member.destination_inode, + }); + } + Ok(Some(source.member)) + } + + fn preflight_restore_index_target( + &self, + catalog_root: InodeId, + relative: &[DentryName], + version: Version, + ) -> Result { + if relative.is_empty() { + let metadata = self + .restore_index_metadata_for_inode(catalog_root, version, ReadPurpose::Snapshot)? + .ok_or(MetadError::RestoreRootChanged { root: catalog_root })?; + return Ok(restore_index_root_target(&metadata)); + } + let relative_path = canonical_path(relative)?; + let Some((entry, _)) = self.lookup_path_from_at_version_for_purpose( + catalog_root, + &relative_path, + version, + ReadPurpose::Snapshot, + )? + else { + return Err(MetadError::RestoreRootChanged { root: catalog_root }); + }; + Ok(restore_index_target_from_projection(&DentryProjection { + dentry: entry.dentry, + attr: entry.attr, + body: entry.body, + })) + } + + /// Stream one directory discovered by the existing subtree preflight. + /// The cheap exact-root probes avoid an O(subtree) custom-index merge for + /// directories with no catalog, while canonical, completed-overlay, and + /// generic-fork catalogs all flow through the same effective snapshot view + /// used by runtime materialization. + pub(super) fn preflight_restore_custom_index_catalog( + &self, + operation: &RestoreOperation, + source_catalog_root: InodeId, + catalog_relative: &[DentryName], + initialization_paths: &HashSet, + ) -> Result<(), MetadError> { + let source_version = Version::new(operation.read_version)?; + let source_catalog_path = + restore_index_join_path(&operation.source_path, catalog_relative)?; + let canonical_exists = self + .metadata + .get( + RecordFamily::PathIndex, + &path_index_catalog_key(self.mount, &source_catalog_path), + source_version, + ReadPurpose::Snapshot, + )? + .is_some(); + let mut overlay_exists = false; + let inverse_prefix = restore_index_catalog_inverse_prefix(self.mount, source_catalog_root); + self.restore_index_effective_for_each( + &inverse_prefix, + None, + source_version, + ReadPurpose::Snapshot, + |_| { + overlay_exists = true; + Ok(false) + }, + )?; + let generic_fork_exists = self + .restore_index_fork_source(source_catalog_root)? + .is_some(); + if !canonical_exists && !overlay_exists && !generic_fork_exists { + return Ok(()); + } + + let Some(effective) = self.restore_custom_index_at_path( + &source_catalog_path, + source_catalog_root, + source_version, + ReadPurpose::Snapshot, + )? + else { + return Ok(()); + }; + let destination_catalog_path = + restore_index_join_path(&operation.destination_path, catalog_relative)?; + let owner = RestoreIndexOwner::from_operation(operation); + let mut restored_batch = Vec::::with_capacity(RESTORE_BATCH_ENTRIES); + let mut restored_count = 0_u64; + for row in effective.rows { + let Some(row_relative) = + restore_index_relative_components(&source_catalog_path, &row.path)? + else { + return Err(MetadError::Codec( + "source namespace row escaped its effective catalog".to_owned(), + )); + }; + let mut operation_relative = catalog_relative.to_vec(); + operation_relative.extend_from_slice(&row_relative); + if initialization_paths.contains(&restore_index_relative_string(&operation_relative)?) { + continue; + } + let target = self.preflight_restore_index_target( + source_catalog_root, + &row_relative, + source_version, + )?; + restored_batch.push(RestoreIndexRow { + owner, + catalog_root: source_catalog_root, + target, + record: PathIndexRowRecord { + path: restore_index_join_path(&destination_catalog_path, &row_relative)?, + values: row.values, + }, + }); + restored_count = restored_count.checked_add(1).ok_or_else(|| { + MetadError::Codec("restore custom index row count overflow".to_owned()) + })?; + if restored_batch.len() == RESTORE_BATCH_ENTRIES { + let desired = restored_batch + .drain(..) + .flat_map(|row| restore_index_row_mutations(self.mount, &row)) + .collect(); + self.preflight_restore_index_put_batch( + operation, + restore_index_row_prefix( + self.mount, + operation.ref_set_id, + Some(source_catalog_root), + ), + desired, + )?; + } + } + if !restored_batch.is_empty() { + let desired = restored_batch + .drain(..) + .flat_map(|row| restore_index_row_mutations(self.mount, &row)) + .collect(); + self.preflight_restore_index_put_batch( + operation, + restore_index_row_prefix( + self.mount, + operation.ref_set_id, + Some(source_catalog_root), + ), + desired, + )?; + } + let catalog = RestoreIndexCatalog { + owner, + catalog_root: source_catalog_root, + record: PathIndexCatalogRecord { + path: destination_catalog_path, + fields: effective.catalog.fields, + row_count: restored_count, + }, + }; + self.preflight_restore_index_put_batch( + operation, + restore_index_catalog_prefix(self.mount, operation.ref_set_id), + restore_index_catalog_mutations(self.mount, &catalog) + .into_iter() + .collect(), + ) + } + + #[allow(clippy::too_many_arguments)] + fn materialize_restore_custom_index_catalog( + &self, + operation: &RestoreOperation, + destination_parent: InodeId, + destination_name: &DentryName, + staging_version: Version, + source_catalog_root: InodeId, + catalog_version: Version, + source_catalog_path: &str, + catalog_relative: &[DentryName], + generic_fork: bool, + ) -> Result<(), MetadError> { + let Some(destination_catalog_member) = self.restore_index_staging_member_for_relative( + operation, + catalog_relative, + staging_version, + )? + else { + return Ok(()); + }; + let destination_catalog_root = destination_catalog_member.destination_inode; + if generic_fork { + let existing_key = restore_index_catalog_key( + self.mount, + operation.ref_set_id, + destination_catalog_root, + ); + if let Some(existing) = self.metadata.get( + RecordFamily::System, + &existing_key, + self.read_version()?, + ReadPurpose::RestoreStaging, + )? { + let catalog = decode_restore_index_catalog(&existing.0)?; + if catalog.owner != RestoreIndexOwner::from_operation(operation) + || catalog.catalog_root != destination_catalog_root + { + return Err(MetadError::Codec( + "restore custom index catalog conflicts with an earlier source".to_owned(), + )); + } + return Ok(()); + } + } + let Some(effective) = self.restore_custom_index_at_path( + source_catalog_path, + source_catalog_root, + catalog_version, + ReadPurpose::Snapshot, + )? + else { + return Ok(()); + }; + let destination_catalog_path = + restore_index_join_path(&operation.destination_path, catalog_relative)?; + let owner = RestoreIndexOwner::from_operation(operation); + let mut restored_count = 0_u64; + let mut restored_batch = Vec::::with_capacity(RESTORE_BATCH_ENTRIES); + for row in effective.rows { + let Some(row_relative) = + restore_index_relative_components(source_catalog_path, &row.path)? + else { + return Err(MetadError::Codec( + "source namespace row escaped its effective catalog".to_owned(), + )); + }; + let mut operation_relative = catalog_relative.to_vec(); + operation_relative.extend_from_slice(&row_relative); + let Some(member) = self.restore_index_staging_member_for_relative( + operation, + &operation_relative, + staging_version, + )? + else { + // Initialization removed the indexed source entry. + continue; + }; + let source_relative_path = canonical_path(&row_relative)?; + let Some(source_metadata) = self.stat_path_from_at_version_for_purpose( + source_catalog_root, + &source_relative_path, + catalog_version, + ReadPurpose::Snapshot, + )? + else { + continue; + }; + let source_matches = if generic_fork { + match member.source_inode { + Some(member_source_inode) => self + .restore_index_metadata_for_inode( + member_source_inode, + Version::new(operation.read_version)?, + ReadPurpose::Snapshot, + )? + .is_some_and(|clone_metadata| { + restore_index_clone_metadata_matches_source( + &clone_metadata, + &source_metadata, + ) + }), + None => false, + } + } else { + member.source_inode == Some(source_metadata.attr.inode) + }; + if !source_matches { + continue; + } + let target = if row_relative.is_empty() { + let metadata = self + .restore_index_metadata_for_inode( + destination_catalog_root, + staging_version, + ReadPurpose::RestoreStaging, + )? + .ok_or(MetadError::RestoreRootChanged { + root: destination_catalog_root, + })?; + restore_index_root_target(&metadata) + } else { + let projection = self.restore_index_projection_for_member( + operation, + &member, + destination_parent, + destination_name, + staging_version, + )?; + restore_index_target_from_projection(&projection) + }; + restored_batch.push(RestoreIndexRow { + owner, + catalog_root: destination_catalog_root, + target, + record: PathIndexRowRecord { + path: restore_index_join_path(&destination_catalog_path, &row_relative)?, + values: row.values, + }, + }); + restored_count = restored_count.checked_add(1).ok_or_else(|| { + MetadError::Codec("restore custom index row count overflow".to_owned()) + })?; + if restored_count > MAX_RESTORE_SUBTREE_ENTRIES as u64 { + return Err(MetadError::RestoreResourceLimit { + resource: "restore custom index rows".to_owned(), + limit: MAX_RESTORE_SUBTREE_ENTRIES as u64, + actual: restored_count, + }); + } + if restored_batch.len() == RESTORE_BATCH_ENTRIES { + let desired = restored_batch + .drain(..) + .flat_map(|row| restore_index_row_mutations(self.mount, &row)) + .collect(); + self.commit_restore_index_put_batch( + operation, + restore_index_row_prefix( + self.mount, + operation.ref_set_id, + Some(destination_catalog_root), + ), + desired, + )?; + } + } + if !restored_batch.is_empty() { + let desired = restored_batch + .drain(..) + .flat_map(|row| restore_index_row_mutations(self.mount, &row)) + .collect(); + self.commit_restore_index_put_batch( + operation, + restore_index_row_prefix( + self.mount, + operation.ref_set_id, + Some(destination_catalog_root), + ), + desired, + )?; + } + let catalog = RestoreIndexCatalog { + owner, + catalog_root: destination_catalog_root, + record: PathIndexCatalogRecord { + path: destination_catalog_path, + fields: effective.catalog.fields, + row_count: restored_count, + }, + }; + self.commit_restore_index_put_batch( + operation, + restore_index_catalog_prefix(self.mount, operation.ref_set_id), + restore_index_catalog_mutations(self.mount, &catalog) + .into_iter() + .collect(), + ) + } + + fn materialize_restore_custom_indexes( + &self, + operation: &RestoreOperation, + destination_parent: InodeId, + destination_name: &DentryName, + ) -> Result<(), MetadError> { + let staging_version = self.read_version()?; + let source_version = Version::new(operation.read_version)?; + let namespace_catalog_prefix = path_index_catalog_key(self.mount, ""); + self.restore_index_for_each_raw( + RecordFamily::PathIndex, + &namespace_catalog_prefix, + None, + source_version, + ReadPurpose::Snapshot, + RESTORE_INDEX_SCAN_PAGE, + |item| { + let catalog = decode_path_index_catalog(&item.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if item.key != path_index_catalog_key(self.mount, &catalog.path) { + return Err(MetadError::Codec( + "source namespace catalog path does not match its key".to_owned(), + )); + } + let Some(relative) = + restore_index_relative_components(&operation.source_path, &catalog.path)? + else { + return Ok(true); + }; + let Some(member) = self.restore_index_staging_member_for_relative( + operation, + &relative, + staging_version, + )? + else { + return Ok(true); + }; + let Some(source_inode) = member.source_inode else { + return Ok(true); + }; + self.materialize_restore_custom_index_catalog( + operation, + destination_parent, + destination_name, + staging_version, + source_inode, + source_version, + &catalog.path, + &relative, + false, + )?; + Ok(true) + }, + )?; + + // A source subtree may itself be a completed restore whose custom + // catalog path predates a later namespace move. Resolve its current + // relative location through this operation's durable staging members + // instead of retaining a whole-subtree source-inode map. + let catalog_owner_prefix = restore_index_system_key(self.mount, INDEX_CATALOG_LABEL); + self.restore_index_for_each_mvcc_latest( + &catalog_owner_prefix, + None, + false, + source_version, + ReadPurpose::Snapshot, + |item| { + if item.record.kind == RestoreIndexMvccKind::Tombstone { + return Ok(true); + } + let catalog = decode_restore_index_catalog(&item.record.value)?; + if item.logical_key + != restore_index_catalog_key( + self.mount, + catalog.owner.ref_set_id, + catalog.catalog_root, + ) + { + return Err(MetadError::Codec( + "source restore catalog path does not match its key".to_owned(), + )); + } + if self + .visible_restore_index_operation( + catalog.owner, + source_version, + ReadPurpose::Snapshot, + )? + .is_none() + { + return Ok(true); + } + let Some(source_member) = self.restore_index_staging_member_for_source_inode( + operation, + catalog.catalog_root, + staging_version, + )? + else { + return Ok(true); + }; + let relative = if source_member.relative_path.is_empty() { + Vec::new() + } else { + parse_absolute_path(&format!("/{}", source_member.relative_path))? + }; + let source_catalog_path = + restore_index_join_path(&operation.source_path, &relative)?; + self.materialize_restore_custom_index_catalog( + operation, + destination_parent, + destination_name, + staging_version, + catalog.catalog_root, + source_version, + &source_catalog_path, + &relative, + false, + )?; + Ok(true) + }, + )?; + + if let Some(fork_source) = self.restore_index_fork_source(operation.source_root)? { + let pinned_version = Version::new(fork_source.binding.pinned_read_version)?; + self.restore_index_for_each_mvcc_latest( + &catalog_owner_prefix, + None, + false, + pinned_version, + ReadPurpose::Snapshot, + |item| { + if item.record.kind == RestoreIndexMvccKind::Tombstone { + return Ok(true); + } + let catalog = decode_restore_index_catalog(&item.record.value)?; + if item.logical_key + != restore_index_catalog_key( + self.mount, + catalog.owner.ref_set_id, + catalog.catalog_root, + ) + { + return Err(MetadError::Codec( + "generic fork source catalog path does not match its key".to_owned(), + )); + } + if self + .visible_restore_index_operation( + catalog.owner, + pinned_version, + ReadPurpose::Snapshot, + )? + .is_none() + { + return Ok(true); + } + let Some(relative) = restore_index_relative_components( + &fork_source.operation.destination_path, + &catalog.record.path, + )? + else { + return Ok(true); + }; + if self + .restore_index_staging_member_for_relative( + operation, + &relative, + staging_version, + )? + .is_none() + { + return Ok(true); + } + self.materialize_restore_custom_index_catalog( + operation, + destination_parent, + destination_name, + staging_version, + catalog.catalog_root, + pinned_version, + &catalog.record.path, + &relative, + true, + )?; + Ok(true) + }, + )?; + } + Ok(()) + } + + /// Materialize navigation and custom catalog overlays while the tree is + /// detached. The returned seal proof must be included in the later + /// ReadyToAttach transition and final attach CAS. + pub(super) fn materialize_and_seal_restore_indexes( + &self, + operation: &RestoreOperation, + destination_parent: InodeId, + destination_name: &DentryName, + ) -> Result<(RestoreIndexSeal, Version), MetadError> { + self.materialize_restore_canonical_indexes( + operation, + destination_parent, + destination_name, + )?; + self.materialize_restore_custom_indexes(operation, destination_parent, destination_name)?; + self.seal_restore_indexes(operation) + } + + fn restore_index_catalog_row_count( + &self, + owner: RestoreIndexOwner, + catalog_root: InodeId, + version: Version, + purpose: ReadPurpose, + ) -> Result { + let prefix = restore_index_row_prefix(self.mount, owner.ref_set_id, Some(catalog_root)); + let mut count = 0_u64; + self.restore_index_for_each_raw( + RecordFamily::System, + &prefix, + None, + version, + purpose, + RESTORE_INDEX_SCAN_PAGE, + |item| { + let row = decode_restore_index_row(&item.value.0)?; + if row.owner != owner + || row.catalog_root != catalog_root + || item.key + != restore_index_row_key( + self.mount, + owner.ref_set_id, + catalog_root, + &row.target, + ) + { + return Err(MetadError::Codec( + "restore index catalog row changed identity".to_owned(), + )); + } + count = count.checked_add(1).ok_or_else(|| { + MetadError::Codec("restore index catalog row count overflow".to_owned()) + })?; + Ok(true) + }, + )?; + Ok(count) + } + + fn compute_restore_index_seal( + &self, + operation: &RestoreOperation, + version: Version, + purpose: ReadPurpose, + ) -> Result { + let owner = RestoreIndexOwner::from_operation(operation); + let mut hasher = Sha256::new(); + hasher.update(b"nokv-restore-index-seal-v1\0"); + let mut entry_count = 0_u64; + let mut catalog_count = 0_u64; + let mut row_count = 0_u64; + + let entry_prefix = restore_index_entry_prefix(self.mount, owner.ref_set_id, None); + self.restore_index_for_each_raw( + RecordFamily::System, + &entry_prefix, + None, + version, + purpose, + RESTORE_INDEX_SCAN_PAGE, + |item| { + let entry = decode_restore_index_entry(&item.value.0)?; + if entry.owner != owner + || item.key + != restore_index_entry_key( + self.mount, + owner.ref_set_id, + entry.projection.dentry.parent, + &entry.projection.dentry.name, + ) + { + return Err(MetadError::Codec( + "restore index seal found an invalid entry owner/key".to_owned(), + )); + } + for key in [ + restore_index_parent_owner_key( + self.mount, + owner.ref_set_id, + entry.projection.dentry.parent, + ), + restore_index_parent_inverse_key( + self.mount, + entry.projection.dentry.parent, + owner.ref_set_id, + ), + ] { + let value = self + .metadata + .get(RecordFamily::System, &key, version, purpose)? + .ok_or_else(|| { + MetadError::Codec( + "restore index entry has no parent owner/inverse".to_owned(), + ) + })?; + if decode_restore_index_owner(&value.0)? != owner { + return Err(MetadError::Codec( + "restore index entry parent owner/inverse mismatch".to_owned(), + )); + } + } + fold_restore_index_seal_row(&mut hasher, 1, &item.key, &item.value.0); + entry_count = entry_count.checked_add(1).ok_or_else(|| { + MetadError::Codec("restore index entry count overflow".to_owned()) + })?; + Ok(true) + }, + )?; + + let catalog_prefix = restore_index_catalog_prefix(self.mount, owner.ref_set_id); + self.restore_index_for_each_raw( + RecordFamily::System, + &catalog_prefix, + None, + version, + purpose, + RESTORE_INDEX_SCAN_PAGE, + |item| { + let catalog = decode_restore_index_catalog(&item.value.0)?; + if catalog.owner != owner + || item.key + != restore_index_catalog_key( + self.mount, + owner.ref_set_id, + catalog.catalog_root, + ) + { + return Err(MetadError::Codec( + "restore index seal found an invalid catalog owner/key".to_owned(), + )); + } + let inverse_key = restore_index_catalog_inverse_key( + self.mount, + catalog.catalog_root, + owner.ref_set_id, + ); + let inverse = self + .metadata + .get(RecordFamily::System, &inverse_key, version, purpose)? + .ok_or_else(|| { + MetadError::Codec("restore index catalog has no inverse".to_owned()) + })?; + if decode_restore_index_catalog(&inverse.0)? != catalog { + return Err(MetadError::Codec( + "restore index catalog owner/inverse mismatch".to_owned(), + )); + } + let actual = self.restore_index_catalog_row_count( + owner, + catalog.catalog_root, + version, + purpose, + )?; + if catalog.record.row_count != actual { + return Err(MetadError::Codec(format!( + "restore index catalog row count mismatch: expected {}, found {actual}", + catalog.record.row_count + ))); + } + fold_restore_index_seal_row(&mut hasher, 2, &item.key, &item.value.0); + catalog_count = catalog_count.checked_add(1).ok_or_else(|| { + MetadError::Codec("restore index catalog count overflow".to_owned()) + })?; + Ok(true) + }, + )?; + + let row_prefix = restore_index_row_prefix(self.mount, owner.ref_set_id, None); + self.restore_index_for_each_raw( + RecordFamily::System, + &row_prefix, + None, + version, + purpose, + RESTORE_INDEX_SCAN_PAGE, + |item| { + let row = decode_restore_index_row(&item.value.0)?; + if row.owner != owner + || item.key + != restore_index_row_key( + self.mount, + owner.ref_set_id, + row.catalog_root, + &row.target, + ) + { + return Err(MetadError::Codec( + "restore index seal found an invalid custom row owner/key".to_owned(), + )); + } + let catalog_key = + restore_index_catalog_key(self.mount, owner.ref_set_id, row.catalog_root); + let catalog = self + .metadata + .get(RecordFamily::System, &catalog_key, version, purpose)? + .ok_or_else(|| { + MetadError::Codec("restore index row has no catalog owner".to_owned()) + })?; + if decode_restore_index_catalog(&catalog.0)?.owner != owner { + return Err(MetadError::Codec( + "restore index row catalog changed identity".to_owned(), + )); + } + let inverse_key = restore_index_target_inverse_key( + self.mount, + &row.target, + owner.ref_set_id, + row.catalog_root, + ); + let inverse = self + .metadata + .get(RecordFamily::System, &inverse_key, version, purpose)? + .ok_or_else(|| { + MetadError::Codec("restore index row has no target inverse".to_owned()) + })?; + if decode_restore_index_row(&inverse.0)? != row { + return Err(MetadError::Codec( + "restore index row owner/inverse mismatch".to_owned(), + )); + } + fold_restore_index_seal_row(&mut hasher, 3, &item.key, &item.value.0); + row_count = row_count.checked_add(1).ok_or_else(|| { + MetadError::Codec("restore index row count overflow".to_owned()) + })?; + Ok(true) + }, + )?; + + let parent_prefix = restore_index_parent_owner_prefix(self.mount, owner.ref_set_id); + self.restore_index_for_each_raw( + RecordFamily::System, + &parent_prefix, + None, + version, + purpose, + RESTORE_INDEX_SCAN_PAGE, + |item| { + if item.key.len() != parent_prefix.len() + 8 + || decode_restore_index_owner(&item.value.0)? != owner + { + return Err(MetadError::Codec( + "restore index seal found an invalid parent owner".to_owned(), + )); + } + let parent = InodeId::new(u64::from_be_bytes( + item.key[parent_prefix.len()..] + .try_into() + .expect("u64 width"), + ))?; + let inverse_key = + restore_index_parent_inverse_key(self.mount, parent, owner.ref_set_id); + let inverse = self + .metadata + .get(RecordFamily::System, &inverse_key, version, purpose)? + .ok_or_else(|| { + MetadError::Codec("restore index parent owner has no inverse".to_owned()) + })?; + if decode_restore_index_owner(&inverse.0)? != owner + || self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_index_entry_prefix( + self.mount, + owner.ref_set_id, + Some(parent), + ), + start_after: None, + version, + limit: 1, + purpose, + })? + .is_empty() + { + return Err(MetadError::Codec( + "restore index parent owner is orphaned".to_owned(), + )); + } + Ok(true) + }, + )?; + + let source_member_prefix = restore_index_source_member_prefix(self.mount, owner.ref_set_id); + self.restore_index_for_each_raw( + RecordFamily::System, + &source_member_prefix, + None, + version, + purpose, + RESTORE_INDEX_SCAN_PAGE, + |item| { + let source = decode_restore_index_source_member(&item.value.0)?; + if source.owner != owner + || item.key + != restore_index_source_member_key( + self.mount, + owner.ref_set_id, + source.source_inode, + ) + { + return Err(MetadError::Codec( + "restore index seal found an invalid source member".to_owned(), + )); + } + let staging_key = restore_staging_member_key( + self.mount, + owner.ref_set_id, + source.member.destination_inode, + ); + let staging = self + .metadata + .get(RecordFamily::System, &staging_key, version, purpose)? + .ok_or_else(|| { + MetadError::Codec( + "restore index source member has no staging member".to_owned(), + ) + })?; + if decode_restore_staging_member(&staging.0)? != source.member { + return Err(MetadError::Codec( + "restore index source/staging member mismatch".to_owned(), + )); + } + fold_restore_index_seal_row(&mut hasher, 4, &item.key, &item.value.0); + Ok(true) + }, + )?; + + let staging_prefix = restore_staging_member_prefix(self.mount, owner.ref_set_id); + self.restore_index_for_each_raw( + RecordFamily::System, + &staging_prefix, + None, + version, + purpose, + RESTORE_INDEX_SCAN_PAGE, + |item| { + let member = + self.restore_index_staging_member_from_row(operation, version, &item)?; + let Some(source_inode) = member.source_inode else { + return Ok(true); + }; + let source_key = + restore_index_source_member_key(self.mount, owner.ref_set_id, source_inode); + let source = self + .metadata + .get(RecordFamily::System, &source_key, version, purpose)? + .ok_or_else(|| { + MetadError::Codec( + "restore staging member has no source-member index".to_owned(), + ) + })?; + let source = decode_restore_index_source_member(&source.0)?; + if source.owner != owner + || source.source_inode != source_inode + || source.member != member + { + return Err(MetadError::Codec( + "restore staging/source member mismatch".to_owned(), + )); + } + Ok(true) + }, + )?; + + self.validate_restore_index_inverse_closure(operation.ref_set_id, version, purpose)?; + + Ok(RestoreIndexSeal { + operation_digest: operation.operation_digest, + initialization_digest: operation.initialization_digest, + ref_set_id: operation.ref_set_id, + incarnation: operation.created_version, + entry_count, + catalog_count, + row_count, + digest: hasher.finalize().into(), + }) + } + + fn validate_restore_index_inverse_closure( + &self, + ref_set_id: u64, + version: Version, + purpose: ReadPurpose, + ) -> Result<(), MetadError> { + let parent_prefix = restore_index_system_key(self.mount, INDEX_PARENT_INVERSE_LABEL); + self.restore_index_for_each_raw( + RecordFamily::System, + &parent_prefix, + None, + version, + purpose, + RESTORE_INDEX_SCAN_PAGE, + |item| { + if item.key.len() != parent_prefix.len() + 16 { + return Err(MetadError::Codec( + "restore index parent inverse key has an invalid length".to_owned(), + )); + } + let row_ref_set = u64::from_be_bytes( + item.key[item.key.len() - 8..] + .try_into() + .expect("u64 width"), + ); + if row_ref_set != ref_set_id { + return Ok(true); + } + let parent = InodeId::new(u64::from_be_bytes( + item.key[parent_prefix.len()..parent_prefix.len() + 8] + .try_into() + .expect("u64 width"), + ))?; + let owner = decode_restore_index_owner(&item.value.0)?; + if owner.ref_set_id != ref_set_id + || item.key != restore_index_parent_inverse_key(self.mount, parent, ref_set_id) + { + return Err(MetadError::Codec( + "restore index parent inverse changed identity".to_owned(), + )); + } + let owner_value = self + .metadata + .get( + RecordFamily::System, + &restore_index_parent_owner_key(self.mount, ref_set_id, parent), + version, + purpose, + )? + .ok_or_else(|| { + MetadError::Codec("restore index parent inverse is orphaned".to_owned()) + })?; + if decode_restore_index_owner(&owner_value.0)? != owner { + return Err(MetadError::Codec( + "restore index parent inverse/owner mismatch".to_owned(), + )); + } + Ok(true) + }, + )?; + + let catalog_prefix = restore_index_system_key(self.mount, INDEX_CATALOG_INVERSE_LABEL); + self.restore_index_for_each_raw( + RecordFamily::System, + &catalog_prefix, + None, + version, + purpose, + RESTORE_INDEX_SCAN_PAGE, + |item| { + if item.key.len() != catalog_prefix.len() + 16 { + return Err(MetadError::Codec( + "restore index catalog inverse key has an invalid length".to_owned(), + )); + } + let row_ref_set = u64::from_be_bytes( + item.key[item.key.len() - 8..] + .try_into() + .expect("u64 width"), + ); + if row_ref_set != ref_set_id { + return Ok(true); + } + let catalog = decode_restore_index_catalog(&item.value.0)?; + if catalog.owner.ref_set_id != ref_set_id + || item.key + != restore_index_catalog_inverse_key( + self.mount, + catalog.catalog_root, + ref_set_id, + ) + { + return Err(MetadError::Codec( + "restore index catalog inverse changed identity".to_owned(), + )); + } + let owner_value = self + .metadata + .get( + RecordFamily::System, + &restore_index_catalog_key(self.mount, ref_set_id, catalog.catalog_root), + version, + purpose, + )? + .ok_or_else(|| { + MetadError::Codec("restore index catalog inverse is orphaned".to_owned()) + })?; + if decode_restore_index_catalog(&owner_value.0)? != catalog { + return Err(MetadError::Codec( + "restore index catalog inverse/owner mismatch".to_owned(), + )); + } + Ok(true) + }, + )?; + + let target_prefix = restore_index_system_key(self.mount, INDEX_TARGET_INVERSE_LABEL); + self.restore_index_for_each_raw( + RecordFamily::System, + &target_prefix, + None, + version, + purpose, + RESTORE_INDEX_SCAN_PAGE, + |item| { + if item.key.len() != target_prefix.len() + 48 { + return Err(MetadError::Codec( + "restore index target inverse key has an invalid length".to_owned(), + )); + } + let row_ref_set = u64::from_be_bytes( + item.key[item.key.len() - 16..item.key.len() - 8] + .try_into() + .expect("u64 width"), + ); + if row_ref_set != ref_set_id { + return Ok(true); + } + let row = decode_restore_index_row(&item.value.0)?; + if row.owner.ref_set_id != ref_set_id + || item.key + != restore_index_target_inverse_key( + self.mount, + &row.target, + ref_set_id, + row.catalog_root, + ) + { + return Err(MetadError::Codec( + "restore index target inverse changed identity".to_owned(), + )); + } + let owner_value = self + .metadata + .get( + RecordFamily::System, + &restore_index_row_key( + self.mount, + ref_set_id, + row.catalog_root, + &row.target, + ), + version, + purpose, + )? + .ok_or_else(|| { + MetadError::Codec("restore index target inverse is orphaned".to_owned()) + })?; + if decode_restore_index_row(&owner_value.0)? != row { + return Err(MetadError::Codec( + "restore index target inverse/owner mismatch".to_owned(), + )); + } + Ok(true) + }, + )?; + Ok(()) + } + + fn seal_restore_indexes( + &self, + operation: &RestoreOperation, + ) -> Result<(RestoreIndexSeal, Version), MetadError> { + let version = self.next_version()?; + let read_version = predecessor(version)?; + let seal = + self.compute_restore_index_seal(operation, read_version, ReadPurpose::RestoreStaging)?; + let key = restore_index_seal_key(self.mount, operation.ref_set_id); + if let Some(existing) = self.metadata.get_versioned( + RecordFamily::System, + &key, + read_version, + ReadPurpose::RestoreStaging, + )? { + let decoded = decode_restore_index_seal(&existing.value.0)?; + if decoded != seal { + return Err(MetadError::Codec( + "restore index seal conflicts with materialized rows".to_owned(), + )); + } + return Ok((decoded, existing.version)); + } + let mut predicates = self.restore_index_staging_guards(operation, read_version)?; + predicates.push(PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }); + let command = MetadataCommand { + request_id: request_id( + b"restore-seal-index", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::RegisterNamespaceIndex, + read_version, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates, + mutations: vec![Mutation { + family: RecordFamily::System, + key, + op: MutationOp::Put, + value: Some(Value(encode_restore_index_seal(&seal))), + }], + watch: Vec::new(), + }; + super::restore::validate_restore_command_bounds(&command, "restore index seal")?; + self.commit_metadata(command)?; + Ok((seal, version)) + } + + /// Recompute the attach-time seal at `version` and return its exact CAS + /// predicate. This is intentionally stronger than checking only the seal's + /// identity: a missing/tampered owner or inverse row fails before attach. + pub(super) fn restore_index_seal_predicate( + &self, + operation: &RestoreOperation, + version: Version, + ) -> Result { + let key = restore_index_seal_key(self.mount, operation.ref_set_id); + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("restore index seal is missing".to_owned()))?; + let durable = decode_restore_index_seal(&item.value.0)?; + let computed = + self.compute_restore_index_seal(operation, version, ReadPurpose::WritePlanLocal)?; + if durable != computed { + return Err(MetadError::Codec( + "restore index seal no longer matches owner rows".to_owned(), + )); + } + Ok(PredicateRef { + family: RecordFamily::System, + key, + predicate: Predicate::VersionEquals(item.version), + }) + } + + fn restore_index_entries_for_location_at( + &self, + parent: InodeId, + name: &DentryName, + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadError> { + let inverse_prefix = restore_index_parent_inverse_prefix_for_read(self.mount, parent); + let mut entries = Vec::new(); + let mut buffered_bytes = 0_usize; + self.restore_index_effective_for_each( + &inverse_prefix, + None, + version, + purpose, + |inverse| { + if inverse.key.len() != inverse_prefix.len() + 8 { + return Err(MetadError::Codec( + "restore index parent inverse key has an invalid length".to_owned(), + )); + } + let owner = decode_restore_index_owner(&inverse.value.0)?; + let ref_set_id = u64::from_be_bytes( + inverse.key[inverse_prefix.len()..] + .try_into() + .expect("u64 width"), + ); + if owner.ref_set_id != ref_set_id + || inverse.key + != restore_index_parent_inverse_key(self.mount, parent, ref_set_id) + { + return Err(MetadError::Codec( + "restore index parent inverse changed identity".to_owned(), + )); + } + let Some((_, operation_version)) = + self.visible_restore_index_operation_versioned(owner, version, purpose)? + else { + return Ok(true); + }; + let parent_owner_key = + restore_index_parent_owner_key(self.mount, owner.ref_set_id, parent); + let parent_owner = self + .restore_index_effective_get(&parent_owner_key, version, purpose)? + .ok_or_else(|| { + MetadError::Codec("restore index parent inverse has no owner".to_owned()) + })?; + if decode_restore_index_owner(&parent_owner.value.0)? != owner { + return Err(MetadError::Codec( + "restore index parent owner/inverse mismatch".to_owned(), + )); + } + let key = restore_index_entry_key(self.mount, owner.ref_set_id, parent, name); + let Some(item) = self.restore_index_effective_get(&key, version, purpose)? else { + return Ok(true); + }; + let entry = decode_restore_index_entry(&item.value.0)?; + if entry.owner != owner + || entry.projection.dentry.parent != parent + || entry.projection.dentry.name != *name + || key + != restore_index_entry_key( + self.mount, + owner.ref_set_id, + entry.projection.dentry.parent, + &entry.projection.dentry.name, + ) + { + return Err(MetadError::Codec( + "restore index entry changed identity".to_owned(), + )); + } + account_restore_index_buffer( + "restore index location candidates", + entries.len().saturating_add(1), + &mut buffered_bytes, + inverse + .key + .len() + .saturating_add(inverse.value.0.len()) + .saturating_add(key.len()) + .saturating_add(item.value.0.len()), + )?; + entries.push(VersionedRestoreIndexEntry { + key, + version: item.version, + entry, + operation_key: restore_operation_key(self.mount, &owner.operation_digest), + operation_version, + }); + Ok(true) + }, + )?; + Ok(entries) + } + + fn restore_index_entries_for_location( + &self, + parent: InodeId, + name: &DentryName, + version: Version, + ) -> Result, MetadError> { + self.restore_index_entries_for_location_at( + parent, + name, + version, + ReadPurpose::WritePlanLocal, + ) + } + + /// Return whether this exact namespace location is backed by a visible + /// restore overlay entry. An inode may have another inherited hardlink + /// while `parent/name` itself is a later canonical link, so inode ownership + /// or a non-empty aggregate publish plan is not sufficient evidence. + pub(super) fn restore_index_manages_projection_location( + &self, + projection: &DentryProjection, + version: Version, + ) -> Result { + let entries = self.restore_index_entries_for_location( + projection.dentry.parent, + &projection.dentry.name, + version, + )?; + for indexed in &entries { + if indexed.entry.projection != *projection { + return Err(MetadError::Codec( + "restore index location projection changed identity".to_owned(), + )); + } + } + Ok(!entries.is_empty()) + } + + fn restore_index_rows_for_target( + &self, + target: &RestoreIndexTarget, + version: Version, + ) -> Result, MetadError> { + let inverse_prefix = + restore_index_target_inverse_prefix(self.mount, target.parent, &target.name); + let mut rows = Vec::new(); + let mut buffered_bytes = 0_usize; + self.restore_index_for_each_raw( + RecordFamily::System, + &inverse_prefix, + None, + version, + ReadPurpose::WritePlanLocal, + RESTORE_INDEX_SCAN_PAGE, + |inverse| { + if inverse.key.len() != inverse_prefix.len() + 16 { + return Err(MetadError::Codec( + "restore index target inverse key has an invalid length".to_owned(), + )); + } + let row = decode_restore_index_row(&inverse.value.0)?; + let ref_set_id = u64::from_be_bytes( + inverse.key[inverse_prefix.len()..inverse_prefix.len() + 8] + .try_into() + .expect("u64 width"), + ); + let catalog_root = InodeId::new(u64::from_be_bytes( + inverse.key[inverse_prefix.len() + 8..] + .try_into() + .expect("u64 width"), + ))?; + if inverse.key + != restore_index_target_inverse_key( + self.mount, + &row.target, + row.owner.ref_set_id, + row.catalog_root, + ) + || row.owner.ref_set_id != ref_set_id + || row.catalog_root != catalog_root + { + return Err(MetadError::Codec( + "restore index target inverse changed identity".to_owned(), + )); + } + // A SHA-256 target-prefix collision is not corruption; the full + // encoded identity disambiguates it before mutation planning. + if row.target != *target { + return Ok(true); + } + let Some((_, operation_version)) = self.visible_restore_index_operation_versioned( + row.owner, + version, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(true); + }; + let owner_key = restore_index_row_key( + self.mount, + row.owner.ref_set_id, + row.catalog_root, + &row.target, + ); + let owner = self + .metadata + .get_versioned( + RecordFamily::System, + &owner_key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore index target inverse has no owner".to_owned()) + })?; + if decode_restore_index_row(&owner.value.0)? != row { + return Err(MetadError::Codec( + "restore index custom row owner/inverse mismatch".to_owned(), + )); + } + let catalog_key = + restore_index_catalog_key(self.mount, row.owner.ref_set_id, row.catalog_root); + let catalog = self + .metadata + .get( + RecordFamily::System, + &catalog_key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore index custom row has no catalog".to_owned()) + })?; + if decode_restore_index_catalog(&catalog.0)?.owner != row.owner { + return Err(MetadError::Codec( + "restore index custom row catalog changed identity".to_owned(), + )); + } + account_restore_index_buffer( + "restore index target candidates", + rows.len().saturating_add(1), + &mut buffered_bytes, + inverse + .key + .len() + .saturating_add(inverse.value.0.len()) + .saturating_add(owner_key.len()) + .saturating_add(owner.value.0.len()), + )?; + rows.push(VersionedRestoreIndexRow { + owner_key, + owner_version: owner.version, + inverse_key: inverse.key, + inverse_version: inverse.version, + operation_key: restore_operation_key(self.mount, &row.owner.operation_digest), + operation_version, + row, + }); + Ok(true) + }, + )?; + Ok(rows) + } + + fn restore_index_plan_row_cas( + plan: &mut RestoreIndexMutationPlan, + family: RecordFamily, + key: Vec, + version: Version, + ) -> Result<(), MetadError> { + plan.push_predicate(PredicateRef { + family, + key, + predicate: Predicate::VersionEquals(version), + }) + } + + fn restore_index_plan_operation_cas( + plan: &mut RestoreIndexMutationPlan, + key: Vec, + version: Version, + ) -> Result<(), MetadError> { + Self::restore_index_plan_row_cas(plan, RecordFamily::System, key, version) + } + + fn restore_index_plan_put_at( + &self, + plan: &mut RestoreIndexMutationPlan, + key: Vec, + value: Vec, + version: Version, + ) -> Result<(), MetadError> { + match self.metadata.get_versioned( + RecordFamily::System, + &key, + version, + ReadPurpose::WritePlanLocal, + )? { + Some(existing) => Self::restore_index_plan_row_cas( + plan, + RecordFamily::System, + key.clone(), + existing.version, + )?, + None => plan.push_predicate(PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + })?, + } + plan.set_mutation(Mutation { + family: RecordFamily::System, + key, + op: MutationOp::Put, + value: Some(Value(value)), + }); + Ok(()) + } + + fn restore_index_plan_parent_owner( + &self, + plan: &mut RestoreIndexMutationPlan, + owner: RestoreIndexOwner, + parent: InodeId, + version: Version, + ) -> Result<(), MetadError> { + let value = encode_restore_index_owner(owner); + for key in [ + restore_index_parent_owner_key(self.mount, owner.ref_set_id, parent), + restore_index_parent_inverse_key(self.mount, parent, owner.ref_set_id), + ] { + if let Some(existing) = self.metadata.get_versioned( + RecordFamily::System, + &key, + version, + ReadPurpose::WritePlanLocal, + )? { + if existing.value.0 != value { + return Err(MetadError::Codec( + "restore index parent owner/inverse changed identity".to_owned(), + )); + } + Self::restore_index_plan_row_cas( + plan, + RecordFamily::System, + key, + existing.version, + )?; + } else { + plan.push_predicate(PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + })?; + plan.set_mutation(Mutation { + family: RecordFamily::System, + key, + op: MutationOp::Put, + value: Some(Value(value.clone())), + }); + } + } + Ok(()) + } + + fn restore_index_directory_contains( + &self, + root: InodeId, + candidate: InodeId, + version: Version, + ) -> Result { + if root == candidate { + return Ok(true); + } + let mut visited = HashSet::new(); + let mut pending = vec![root]; + let mut traversed = 0_usize; + while let Some(parent) = pending.pop() { + if !visited.insert(parent) { + continue; + } + validate_restore_index_collection( + "restore index containment directories", + visited.len(), + )?; + let mut after = None; + loop { + let page = self.read_dir_plus_page_at_version_for_purpose( + parent, + after.as_ref(), + RESTORE_INDEX_DENTRY_PAGE, + version, + ReadPurpose::WritePlanLocal, + )?; + for entry in page.entries { + traversed = traversed.saturating_add(1); + validate_restore_index_collection( + "restore index containment entries", + traversed, + )?; + if entry.attr.inode == candidate { + return Ok(true); + } + if entry.attr.file_type == FileType::Directory + && entry.attr.inode.shard_index() == self.shard_index + { + pending.push(entry.attr.inode); + validate_restore_index_collection( + "restore index containment pending directories", + pending.len(), + )?; + } + } + let Some(next) = page.next_cursor else { + break; + }; + if after + .as_ref() + .is_some_and(|cursor: &DentryName| cursor.as_bytes() >= next.as_bytes()) + { + return Err(MetadError::Codec( + "restore index containment cursor did not advance".to_owned(), + )); + } + after = Some(next); + } + } + Ok(false) + } + + pub(super) fn restore_index_unlink_plan( + &self, + projection: &DentryProjection, + commit_version: Version, + ) -> Result { + let read_version = predecessor(commit_version)?; + let mut plan = RestoreIndexMutationPlan::default(); + for indexed in self.restore_index_entries_for_location( + projection.dentry.parent, + &projection.dentry.name, + read_version, + )? { + if indexed.entry.projection != *projection { + return Err(MetadError::Codec( + "restore index unlink projection changed identity".to_owned(), + )); + } + Self::restore_index_plan_operation_cas( + &mut plan, + indexed.operation_key, + indexed.operation_version, + )?; + Self::restore_index_plan_row_cas( + &mut plan, + RecordFamily::System, + indexed.key.clone(), + indexed.version, + )?; + plan.set_mutation(delete_mutation(RecordFamily::System, indexed.key)); + } + let target = restore_index_target_from_projection(projection); + for indexed in self.restore_index_rows_for_target(&target, read_version)? { + Self::restore_index_plan_operation_cas( + &mut plan, + indexed.operation_key, + indexed.operation_version, + )?; + for (key, row_version) in [ + (indexed.owner_key, indexed.owner_version), + (indexed.inverse_key, indexed.inverse_version), + ] { + Self::restore_index_plan_row_cas( + &mut plan, + RecordFamily::System, + key.clone(), + row_version, + )?; + plan.set_mutation(delete_mutation(RecordFamily::System, key)); + } + } + self.finalize_restore_index_mvcc_plan(plan, read_version, commit_version) + } + + pub(super) fn restore_index_publish_plan( + &self, + old: &DentryProjection, + new: &DentryProjection, + commit_version: Version, + ) -> Result { + if old.dentry.parent != new.dentry.parent || old.dentry.name != new.dentry.name { + return Err(MetadError::Codec( + "restore index publish plan cannot move a dentry".to_owned(), + )); + } + let read_version = predecessor(commit_version)?; + let mut plan = RestoreIndexMutationPlan::default(); + for indexed in self.restore_index_entries_for_location( + old.dentry.parent, + &old.dentry.name, + read_version, + )? { + if indexed.entry.projection != *old { + return Err(MetadError::Codec( + "restore index publish projection changed identity".to_owned(), + )); + } + Self::restore_index_plan_operation_cas( + &mut plan, + indexed.operation_key, + indexed.operation_version, + )?; + Self::restore_index_plan_row_cas( + &mut plan, + RecordFamily::System, + indexed.key.clone(), + indexed.version, + )?; + plan.set_mutation(Mutation { + family: RecordFamily::System, + key: indexed.key, + op: MutationOp::Put, + value: Some(Value(encode_restore_index_entry(&RestoreIndexEntry { + owner: indexed.entry.owner, + projection: new.clone(), + }))), + }); + } + let old_target = restore_index_target_from_projection(old); + let new_target = restore_index_target_from_projection(new); + for indexed in self.restore_index_rows_for_target(&old_target, read_version)? { + Self::restore_index_plan_operation_cas( + &mut plan, + indexed.operation_key, + indexed.operation_version, + )?; + for (key, row_version) in [ + (indexed.owner_key.clone(), indexed.owner_version), + (indexed.inverse_key.clone(), indexed.inverse_version), + ] { + Self::restore_index_plan_row_cas( + &mut plan, + RecordFamily::System, + key, + row_version, + )?; + } + let mut row = indexed.row; + row.target = new_target.clone(); + let encoded = encode_restore_index_row(&row); + plan.set_mutation(Mutation { + family: RecordFamily::System, + key: indexed.owner_key, + op: MutationOp::Put, + value: Some(Value(encoded.clone())), + }); + plan.set_mutation(Mutation { + family: RecordFamily::System, + key: indexed.inverse_key, + op: MutationOp::Put, + value: Some(Value(encoded)), + }); + } + self.finalize_restore_index_mvcc_plan(plan, read_version, commit_version) + } + + pub(super) fn restore_index_link_plan( + &self, + updated_existing_links: &[(DentryProjection, DentryProjection)], + commit_version: Version, + ) -> Result { + let mut plan = RestoreIndexMutationPlan::default(); + for (old, new) in updated_existing_links { + plan.extend(self.restore_index_publish_plan(old, new, commit_version)?)?; + } + // The newly-created link deliberately has no inherited custom row; its + // normal PathIndex mutation remains owned by the namespace command. + Ok(plan) + } + + pub(super) fn restore_index_remove_plan( + &self, + removed: &DentryProjection, + updated_remaining_links: &[(DentryProjection, DentryProjection)], + commit_version: Version, + ) -> Result { + let mut plan = self.restore_index_unlink_plan(removed, commit_version)?; + plan.extend(self.restore_index_link_plan(updated_remaining_links, commit_version)?)?; + Ok(plan) + } + + /// A normal namespace-index registration is a replacement, not an + /// additive merge. Delete inherited custom catalog/rows for this exact + /// root in the same command that writes the canonical registration so + /// omitted old rows cannot reappear from the overlay. + pub(super) fn restore_index_replace_catalog_plan( + &self, + catalog_root: InodeId, + commit_version: Version, + ) -> Result { + let version = predecessor(commit_version)?; + let mut plan = RestoreIndexMutationPlan::default(); + let inverse_prefix = restore_index_catalog_inverse_prefix(self.mount, catalog_root); + self.restore_index_for_each_raw( + RecordFamily::System, + &inverse_prefix, + None, + version, + ReadPurpose::WritePlanLocal, + RESTORE_INDEX_SCAN_PAGE, + |inverse| { + if inverse.key.len() != inverse_prefix.len() + 8 { + return Err(MetadError::Codec( + "restore index catalog inverse key has an invalid length".to_owned(), + )); + } + let catalog = decode_restore_index_catalog(&inverse.value.0)?; + if catalog.catalog_root != catalog_root + || inverse.key + != restore_index_catalog_inverse_key( + self.mount, + catalog_root, + catalog.owner.ref_set_id, + ) + { + return Err(MetadError::Codec( + "restore index catalog inverse changed identity".to_owned(), + )); + } + let Some((_, operation_version)) = self.visible_restore_index_operation_versioned( + catalog.owner, + version, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(true); + }; + Self::restore_index_plan_operation_cas( + &mut plan, + restore_operation_key(self.mount, &catalog.owner.operation_digest), + operation_version, + )?; + let owner_key = + restore_index_catalog_key(self.mount, catalog.owner.ref_set_id, catalog_root); + let owner = self + .metadata + .get_versioned( + RecordFamily::System, + &owner_key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore index catalog inverse has no owner".to_owned()) + })?; + if decode_restore_index_catalog(&owner.value.0)? != catalog { + return Err(MetadError::Codec( + "restore index catalog owner/inverse mismatch".to_owned(), + )); + } + for (key, row_version) in + [(owner_key, owner.version), (inverse.key, inverse.version)] + { + Self::restore_index_plan_row_cas( + &mut plan, + RecordFamily::System, + key.clone(), + row_version, + )?; + plan.set_mutation(delete_mutation(RecordFamily::System, key)); + } + plan.validate_budget("restore index catalog replacement")?; + let row_prefix = restore_index_row_prefix( + self.mount, + catalog.owner.ref_set_id, + Some(catalog_root), + ); + self.restore_index_for_each_raw( + RecordFamily::System, + &row_prefix, + None, + version, + ReadPurpose::WritePlanLocal, + RESTORE_INDEX_SCAN_PAGE, + |item| { + let row = decode_restore_index_row(&item.value.0)?; + if row.owner != catalog.owner + || row.catalog_root != catalog_root + || item.key + != restore_index_row_key( + self.mount, + row.owner.ref_set_id, + catalog_root, + &row.target, + ) + { + return Err(MetadError::Codec( + "restore index replacement found an invalid row".to_owned(), + )); + } + let target_inverse_key = restore_index_target_inverse_key( + self.mount, + &row.target, + row.owner.ref_set_id, + catalog_root, + ); + let target_inverse = self + .metadata + .get_versioned( + RecordFamily::System, + &target_inverse_key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "restore index replacement row has no inverse".to_owned(), + ) + })?; + if decode_restore_index_row(&target_inverse.value.0)? != row { + return Err(MetadError::Codec( + "restore index replacement row/inverse mismatch".to_owned(), + )); + } + for (key, row_version) in [ + (item.key, item.version), + (target_inverse_key, target_inverse.version), + ] { + Self::restore_index_plan_row_cas( + &mut plan, + RecordFamily::System, + key.clone(), + row_version, + )?; + plan.set_mutation(delete_mutation(RecordFamily::System, key)); + } + plan.validate_budget("restore index catalog replacement")?; + Ok(true) + }, + )?; + Ok(true) + }, + )?; + self.finalize_restore_index_mvcc_plan(plan, version, commit_version) + } + + pub(super) fn restore_index_rename_plan( + &self, + source: &DentryProjection, + destination: &DentryProjection, + replaced: Option<&DentryProjection>, + commit_version: Version, + ) -> Result { + let version = predecessor(commit_version)?; + let mut plan = RestoreIndexMutationPlan::default(); + if let Some(replaced) = replaced { + plan.extend(self.restore_index_unlink_plan(replaced, commit_version)?)?; + } + for indexed in self.restore_index_entries_for_location( + source.dentry.parent, + &source.dentry.name, + version, + )? { + if indexed.entry.projection != *source { + return Err(MetadError::Codec( + "restore index rename projection changed identity".to_owned(), + )); + } + Self::restore_index_plan_operation_cas( + &mut plan, + indexed.operation_key, + indexed.operation_version, + )?; + Self::restore_index_plan_row_cas( + &mut plan, + RecordFamily::System, + indexed.key.clone(), + indexed.version, + )?; + let destination_key = restore_index_entry_key( + self.mount, + indexed.entry.owner.ref_set_id, + destination.dentry.parent, + &destination.dentry.name, + ); + if destination_key != indexed.key { + plan.set_mutation(delete_mutation(RecordFamily::System, indexed.key)); + self.restore_index_plan_put_at( + &mut plan, + destination_key, + encode_restore_index_entry(&RestoreIndexEntry { + owner: indexed.entry.owner, + projection: destination.clone(), + }), + version, + )?; + self.restore_index_plan_parent_owner( + &mut plan, + indexed.entry.owner, + destination.dentry.parent, + version, + )?; + } else { + plan.set_mutation(Mutation { + family: RecordFamily::System, + key: destination_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_index_entry(&RestoreIndexEntry { + owner: indexed.entry.owner, + projection: destination.clone(), + }))), + }); + } + } + + let source_target = restore_index_target_from_projection(source); + let destination_target = restore_index_target_from_projection(destination); + let mut containment = BTreeMap::::new(); + for indexed in self.restore_index_rows_for_target(&source_target, version)? { + Self::restore_index_plan_operation_cas( + &mut plan, + indexed.operation_key, + indexed.operation_version, + )?; + for (key, row_version) in [ + (indexed.owner_key.clone(), indexed.owner_version), + (indexed.inverse_key.clone(), indexed.inverse_version), + ] { + Self::restore_index_plan_row_cas( + &mut plan, + RecordFamily::System, + key.clone(), + row_version, + )?; + plan.set_mutation(delete_mutation(RecordFamily::System, key)); + } + let within_catalog = + if let Some(within) = containment.get(&indexed.row.catalog_root.get()).copied() { + within + } else { + let within = self.restore_index_directory_contains( + indexed.row.catalog_root, + destination.dentry.parent, + version, + )?; + containment.insert(indexed.row.catalog_root.get(), within); + within + }; + if !within_catalog { + continue; + } + let mut row = indexed.row; + row.target = destination_target.clone(); + let owner_key = restore_index_row_key( + self.mount, + row.owner.ref_set_id, + row.catalog_root, + &row.target, + ); + let inverse_key = restore_index_target_inverse_key( + self.mount, + &row.target, + row.owner.ref_set_id, + row.catalog_root, + ); + let encoded = encode_restore_index_row(&row); + self.restore_index_plan_put_at(&mut plan, owner_key, encoded.clone(), version)?; + self.restore_index_plan_put_at(&mut plan, inverse_key, encoded, version)?; + } + self.finalize_restore_index_mvcc_plan(plan, version, commit_version) + } + + fn restore_index_target_is_current( + &self, + target: &RestoreIndexTarget, + version: Version, + ) -> Result { + let Some(parent) = target.parent else { + let Some(attr) = self.get_attr_at_version_for_purpose( + target.inode, + version, + ReadPurpose::RestoreStaging, + )? + else { + return Ok(false); + }; + let body = if attr.file_type == FileType::File { + self.body_descriptor_at_version_for_purpose( + target.inode, + attr.generation, + version, + ReadPurpose::RestoreStaging, + )? + } else { + None + }; + return Ok(target.inode == attr.inode + && target.file_type == attr.file_type + && target.attr_generation == attr.generation + && target.body_digest == restore_index_body_digest(body.as_ref())); + }; + let name = DentryName::new(target.name.clone()) + .map_err(|error| MetadError::Codec(error.to_string()))?; + let Some((entry, _)) = self.lookup_plus_at_version_for_purpose( + parent, + &name, + version, + ReadPurpose::RestoreStaging, + )? + else { + return Ok(false); + }; + Ok(restore_index_target_matches_projection( + target, + &DentryProjection { + dentry: entry.dentry, + attr: entry.attr, + body: entry.body, + }, + )) + } + + fn commit_restore_index_release_deletes( + &self, + operation: &RestoreOperation, + operation_version: Version, + primary_key: Vec, + mut predicates: Vec, + mutations: Vec, + ) -> Result<(), MetadError> { + if mutations.is_empty() { + return Ok(()); + } + predicates.push(PredicateRef { + family: RecordFamily::System, + key: restore_operation_key(self.mount, &operation.operation_digest), + predicate: Predicate::VersionEquals(operation_version), + }); + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"restore-release-index", + self.mount, + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key, + predicates, + mutations, + watch: Vec::new(), + }; + super::restore::validate_restore_command_bounds(&command, "restore index release page")?; + self.commit_metadata(command)?; + Ok(()) + } + + fn release_restore_index_mvcc_page( + &self, + page: RestoreIndexReleasePage<'_>, + ) -> Result { + let RestoreIndexReleasePage { + operation, + operation_version, + stage, + logical_prefix, + start_after, + limit, + version, + } = page; + let physical_prefix = restore_index_mvcc_prefix(self.mount, &logical_prefix)?; + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: physical_prefix.clone(), + start_after, + version, + limit, + purpose: ReadPurpose::WritePlanLocal, + })?; + let mut predicates = Vec::with_capacity(rows.len() * 2); + let mut mutations = Vec::with_capacity(rows.len() * 2); + for row in &rows { + let record = decode_restore_index_mvcc(&row.value.0)?; + let commit_version = Version::new(record.commit_version)?; + if row.version != commit_version + || row.key + != restore_index_mvcc_key(self.mount, &record.logical_key, commit_version)? + || !record.logical_key.starts_with(&logical_prefix) + { + return Err(MetadError::Codec( + "restore index MVCC release row changed identity".to_owned(), + )); + } + predicates.push(PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }); + mutations.push(delete_mutation(RecordFamily::System, row.key.clone())); + if let Some(inverse_logical) = + restore_index_mvcc_inverse_logical_key(self.mount, &record)? + { + let inverse_key = + restore_index_mvcc_key(self.mount, &inverse_logical, commit_version)?; + let inverse = self + .metadata + .get_versioned( + RecordFamily::System, + &inverse_key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "restore index MVCC owner has no inverse version".to_owned(), + ) + })?; + let inverse_record = decode_restore_index_mvcc(&inverse.value.0)?; + if inverse.version != commit_version + || inverse_record.commit_version != record.commit_version + || inverse_record.kind != record.kind + || inverse_record.logical_key != inverse_logical + || inverse_record.value != record.value + { + return Err(MetadError::Codec( + "restore index MVCC owner/inverse version mismatch".to_owned(), + )); + } + predicates.push(PredicateRef { + family: RecordFamily::System, + key: inverse_key.clone(), + predicate: Predicate::VersionEquals(inverse.version), + }); + mutations.push(delete_mutation(RecordFamily::System, inverse_key)); + } + } + self.commit_restore_index_release_deletes( + operation, + operation_version, + physical_prefix, + predicates, + mutations, + )?; + Ok(restore_index_release_page_outcome(stage, &rows, limit, 0)) + } + + fn audit_restore_index_inverse_page( + &self, + operation: &RestoreOperation, + stage: RestoreIndexReleaseStage, + start_after: Option>, + limit: usize, + version: Version, + ) -> Result { + let (label, logical_label, kind, mvcc) = match stage { + RestoreIndexReleaseStage::AuditParentInverses => ( + INDEX_PARENT_INVERSE_LABEL, + INDEX_PARENT_INVERSE_LABEL, + 1_u8, + false, + ), + RestoreIndexReleaseStage::AuditCatalogInverses => ( + INDEX_CATALOG_INVERSE_LABEL, + INDEX_CATALOG_INVERSE_LABEL, + 2_u8, + false, + ), + RestoreIndexReleaseStage::AuditTargetInverses => ( + INDEX_TARGET_INVERSE_LABEL, + INDEX_TARGET_INVERSE_LABEL, + 3_u8, + false, + ), + RestoreIndexReleaseStage::AuditMvccParentInverses => ( + INDEX_MVCC_PARENT_INVERSE_LABEL, + INDEX_PARENT_INVERSE_LABEL, + 1_u8, + true, + ), + RestoreIndexReleaseStage::AuditMvccCatalogInverses => ( + INDEX_MVCC_CATALOG_INVERSE_LABEL, + INDEX_CATALOG_INVERSE_LABEL, + 2_u8, + true, + ), + RestoreIndexReleaseStage::AuditMvccTargetInverses => ( + INDEX_MVCC_TARGET_INVERSE_LABEL, + INDEX_TARGET_INVERSE_LABEL, + 3_u8, + true, + ), + _ => { + return Err(MetadError::Codec( + "restore index inverse audit has an invalid phase".to_owned(), + )) + } + }; + let prefix = restore_index_system_key(self.mount, label); + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after, + version, + limit, + purpose: ReadPurpose::WritePlanLocal, + })?; + for row in &rows { + let suffix_len = match (kind, mvcc) { + (1 | 2, false) => 16, + (3, false) => 48, + (1 | 2, true) => 24, + (3, true) => 56, + _ => unreachable!(), + }; + if row.key.len() != prefix.len() + suffix_len { + // The row cannot be attributed to this ref-set from its key. + // Mount-wide fsck reports it; stranding every unrelated release + // job on one unowned malformed key would violate isolation. + continue; + } + let ref_set_range = match (kind, mvcc) { + (1 | 2, false) => row.key.len() - 8..row.key.len(), + (3, false) => row.key.len() - 16..row.key.len() - 8, + (1 | 2, true) => row.key.len() - 16..row.key.len() - 8, + (3, true) => row.key.len() - 24..row.key.len() - 16, + _ => unreachable!(), + }; + let key_ref_set = + u64::from_be_bytes(row.key[ref_set_range].try_into().expect("u64 width")); + if key_ref_set != operation.ref_set_id { + continue; + } + let (owner, logical_key) = if mvcc { + let record = decode_restore_index_mvcc(&row.value.0)?; + let commit_version = Version::new(record.commit_version)?; + if row.version != commit_version + || row.key + != restore_index_mvcc_key(self.mount, &record.logical_key, commit_version)? + || restore_index_logical_label_for_key(self.mount, &record.logical_key) + != Some(logical_label) + { + return Err(MetadError::Codec( + "restore index MVCC inverse audit changed identity".to_owned(), + )); + } + let owner = match kind { + 1 => decode_restore_index_owner(&record.value)?, + 2 => decode_restore_index_catalog(&record.value)?.owner, + 3 => decode_restore_index_row(&record.value)?.owner, + _ => unreachable!(), + }; + (owner, record.logical_key) + } else { + let owner = match kind { + 1 => decode_restore_index_owner(&row.value.0)?, + 2 => decode_restore_index_catalog(&row.value.0)?.owner, + 3 => decode_restore_index_row(&row.value.0)?.owner, + _ => unreachable!(), + }; + (owner, row.key.clone()) + }; + if owner.ref_set_id != operation.ref_set_id { + return Err(MetadError::Codec( + "restore index inverse audit owner/key ref-set mismatch".to_owned(), + )); + } + return Err(MetadError::Codec(format!( + "restore index inverse remains after owner release for ref-set {} (logical key bytes {})", + operation.ref_set_id, + logical_key.len() + ))); + } + Ok(restore_index_release_page_outcome(stage, &rows, limit, 0)) + } + + /// Delete one bounded page of operation-scoped index state. Releasing + /// operations retain escaped, still-reachable identities; cleanup and + /// discard operations delete hidden rows unconditionally. A retained row + /// advances the cursor exactly like a deleted row so one borrower cannot + /// starve later entries. + pub(super) fn release_restore_index_page( + &self, + operation: &RestoreOperation, + operation_version: Version, + cursor: &[u8], + limit: usize, + ) -> Result { + if !matches!( + operation.state, + RestoreOperationState::Releasing + | RestoreOperationState::Cleaning + | RestoreOperationState::Discarding + ) { + return Err(MetadError::RestoreInProgress); + } + let retain_reachable = operation.state == RestoreOperationState::Releasing; + let limit = limit.clamp(1, super::restore::RESTORE_BATCH_ENTRIES); + let (stage, start_after) = decode_restore_index_release_cursor(cursor)?; + let version = self.read_version()?; + let owner = RestoreIndexOwner::from_operation(operation); + let mut retained = 0_usize; + + match stage { + RestoreIndexReleaseStage::Entries => { + let prefix = restore_index_entry_prefix(self.mount, operation.ref_set_id, None); + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after, + version, + limit, + purpose: ReadPurpose::WritePlanLocal, + })?; + let mut predicates = Vec::with_capacity(rows.len()); + let mut mutations = Vec::with_capacity(rows.len()); + let mut decoded = Vec::with_capacity(rows.len()); + let mut candidates = HashSet::with_capacity(rows.len()); + for row in &rows { + let entry = decode_restore_index_entry(&row.value.0)?; + if entry.owner != owner + || row.key + != restore_index_entry_key( + self.mount, + owner.ref_set_id, + entry.projection.dentry.parent, + &entry.projection.dentry.name, + ) + { + return Err(MetadError::Codec( + "restore index release entry changed identity".to_owned(), + )); + } + let target = restore_index_target_from_projection(&entry.projection); + if retain_reachable { + candidates.insert(target.inode); + } + decoded.push(target); + } + let reachable = self.restore_reachable_inodes_at(&candidates, version)?; + for (row, target) in rows.iter().zip(decoded) { + if retain_reachable + && reachable.contains(&target.inode) + && self.restore_index_target_is_current(&target, version)? + { + retained = retained.saturating_add(1); + continue; + } + predicates.push(PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }); + mutations.push(delete_mutation(RecordFamily::System, row.key.clone())); + } + self.commit_restore_index_release_deletes( + operation, + operation_version, + prefix, + predicates, + mutations, + )?; + return Ok(restore_index_release_page_outcome( + stage, &rows, limit, retained, + )); + } + RestoreIndexReleaseStage::Rows => { + let prefix = restore_index_row_prefix(self.mount, operation.ref_set_id, None); + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after, + version, + limit, + purpose: ReadPurpose::WritePlanLocal, + })?; + let mut predicates = Vec::with_capacity(rows.len() * 2); + let mut mutations = Vec::with_capacity(rows.len() * 2); + let mut decoded_rows = Vec::with_capacity(rows.len()); + let mut candidates = HashSet::with_capacity(rows.len()); + for row in &rows { + let decoded = decode_restore_index_row(&row.value.0)?; + if decoded.owner != owner + || row.key + != restore_index_row_key( + self.mount, + owner.ref_set_id, + decoded.catalog_root, + &decoded.target, + ) + { + return Err(MetadError::Codec( + "restore index release custom row changed identity".to_owned(), + )); + } + let inverse_key = restore_index_target_inverse_key( + self.mount, + &decoded.target, + owner.ref_set_id, + decoded.catalog_root, + ); + let inverse = self + .metadata + .get_versioned( + RecordFamily::System, + &inverse_key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "restore index custom row has no target inverse".to_owned(), + ) + })?; + if decode_restore_index_row(&inverse.value.0)? != decoded { + return Err(MetadError::Codec( + "restore index custom row/inverse mismatch".to_owned(), + )); + } + if retain_reachable { + candidates.insert(decoded.target.inode); + } + decoded_rows.push((decoded, inverse_key, inverse.version)); + } + let reachable = self.restore_reachable_inodes_at(&candidates, version)?; + for (row, (decoded, inverse_key, inverse_version)) in rows.iter().zip(decoded_rows) + { + if retain_reachable + && reachable.contains(&decoded.target.inode) + && self.restore_index_target_is_current(&decoded.target, version)? + { + retained = retained.saturating_add(1); + continue; + } + predicates.extend([ + PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.clone(), + predicate: Predicate::VersionEquals(inverse_version), + }, + ]); + mutations.extend([ + delete_mutation(RecordFamily::System, row.key.clone()), + delete_mutation(RecordFamily::System, inverse_key), + ]); + } + self.commit_restore_index_release_deletes( + operation, + operation_version, + prefix, + predicates, + mutations, + )?; + return Ok(restore_index_release_page_outcome( + stage, &rows, limit, retained, + )); + } + RestoreIndexReleaseStage::Catalogs => { + let prefix = restore_index_catalog_prefix(self.mount, operation.ref_set_id); + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after, + version, + limit, + purpose: ReadPurpose::WritePlanLocal, + })?; + let mut predicates = Vec::with_capacity(rows.len() * 2); + let mut mutations = Vec::with_capacity(rows.len() * 2); + let mut decoded_rows = Vec::with_capacity(rows.len()); + let mut candidates = HashSet::with_capacity(rows.len()); + for row in &rows { + let decoded = decode_restore_index_catalog(&row.value.0)?; + if decoded.owner != owner + || row.key + != restore_index_catalog_key( + self.mount, + owner.ref_set_id, + decoded.catalog_root, + ) + { + return Err(MetadError::Codec( + "restore index release catalog changed identity".to_owned(), + )); + } + let inverse_key = restore_index_catalog_inverse_key( + self.mount, + decoded.catalog_root, + owner.ref_set_id, + ); + let inverse = self + .metadata + .get_versioned( + RecordFamily::System, + &inverse_key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("restore index catalog has no inverse".to_owned()) + })?; + if decode_restore_index_catalog(&inverse.value.0)? != decoded { + return Err(MetadError::Codec( + "restore index catalog/inverse mismatch".to_owned(), + )); + } + let has_rows = !self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_index_row_prefix( + self.mount, + owner.ref_set_id, + Some(decoded.catalog_root), + ), + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })? + .is_empty(); + if retain_reachable && !has_rows { + candidates.insert(decoded.catalog_root); + } + decoded_rows.push((decoded, inverse_key, inverse.version, has_rows)); + } + let reachable = self.restore_reachable_inodes_at(&candidates, version)?; + for (row, (decoded, inverse_key, inverse_version, has_rows)) in + rows.iter().zip(decoded_rows) + { + if retain_reachable && (has_rows || reachable.contains(&decoded.catalog_root)) { + retained = retained.saturating_add(1); + continue; + } + predicates.extend([ + PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.clone(), + predicate: Predicate::VersionEquals(inverse_version), + }, + ]); + mutations.extend([ + delete_mutation(RecordFamily::System, row.key.clone()), + delete_mutation(RecordFamily::System, inverse_key), + ]); + } + self.commit_restore_index_release_deletes( + operation, + operation_version, + prefix, + predicates, + mutations, + )?; + return Ok(restore_index_release_page_outcome( + stage, &rows, limit, retained, + )); + } + RestoreIndexReleaseStage::Parents => { + let prefix = restore_index_parent_owner_prefix(self.mount, operation.ref_set_id); + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after, + version, + limit, + purpose: ReadPurpose::WritePlanLocal, + })?; + let mut predicates = Vec::with_capacity(rows.len() * 2); + let mut mutations = Vec::with_capacity(rows.len() * 2); + for row in &rows { + if row.key.len() != prefix.len() + 8 { + return Err(MetadError::Codec( + "restore index parent owner key has an invalid length".to_owned(), + )); + } + let decoded_owner = decode_restore_index_owner(&row.value.0)?; + if decoded_owner != owner { + return Err(MetadError::Codec( + "restore index parent owner changed identity".to_owned(), + )); + } + let parent = InodeId::new(u64::from_be_bytes( + row.key[prefix.len()..].try_into().expect("u64 width"), + ))?; + if !self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_index_entry_prefix( + self.mount, + owner.ref_set_id, + Some(parent), + ), + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })? + .is_empty() + { + retained = retained.saturating_add(1); + continue; + } + let inverse_key = + restore_index_parent_inverse_key(self.mount, parent, owner.ref_set_id); + let inverse = self + .metadata + .get_versioned( + RecordFamily::System, + &inverse_key, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "restore index parent owner has no inverse".to_owned(), + ) + })?; + if decode_restore_index_owner(&inverse.value.0)? != owner { + return Err(MetadError::Codec( + "restore index parent owner/inverse mismatch".to_owned(), + )); + } + predicates.extend([ + PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.clone(), + predicate: Predicate::VersionEquals(inverse.version), + }, + ]); + mutations.extend([ + delete_mutation(RecordFamily::System, row.key.clone()), + delete_mutation(RecordFamily::System, inverse_key), + ]); + } + self.commit_restore_index_release_deletes( + operation, + operation_version, + prefix, + predicates, + mutations, + )?; + return Ok(restore_index_release_page_outcome( + stage, &rows, limit, retained, + )); + } + RestoreIndexReleaseStage::MvccEntries => { + return self.release_restore_index_mvcc_page(RestoreIndexReleasePage { + operation, + operation_version, + stage, + logical_prefix: restore_index_entry_prefix( + self.mount, + operation.ref_set_id, + None, + ), + start_after, + limit, + version, + }); + } + RestoreIndexReleaseStage::MvccRows => { + return self.release_restore_index_mvcc_page(RestoreIndexReleasePage { + operation, + operation_version, + stage, + logical_prefix: restore_index_row_prefix( + self.mount, + operation.ref_set_id, + None, + ), + start_after, + limit, + version, + }); + } + RestoreIndexReleaseStage::MvccCatalogs => { + return self.release_restore_index_mvcc_page(RestoreIndexReleasePage { + operation, + operation_version, + stage, + logical_prefix: restore_index_catalog_prefix(self.mount, operation.ref_set_id), + start_after, + limit, + version, + }); + } + RestoreIndexReleaseStage::MvccParents => { + return self.release_restore_index_mvcc_page(RestoreIndexReleasePage { + operation, + operation_version, + stage, + logical_prefix: restore_index_parent_owner_prefix( + self.mount, + operation.ref_set_id, + ), + start_after, + limit, + version, + }); + } + RestoreIndexReleaseStage::AuditParentInverses + | RestoreIndexReleaseStage::AuditCatalogInverses + | RestoreIndexReleaseStage::AuditTargetInverses + | RestoreIndexReleaseStage::AuditMvccParentInverses + | RestoreIndexReleaseStage::AuditMvccCatalogInverses + | RestoreIndexReleaseStage::AuditMvccTargetInverses => { + return self.audit_restore_index_inverse_page( + operation, + stage, + start_after, + limit, + version, + ); + } + RestoreIndexReleaseStage::SourceMembers => { + let prefix = restore_index_source_member_prefix(self.mount, operation.ref_set_id); + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: prefix.clone(), + start_after, + version, + limit, + purpose: ReadPurpose::WritePlanLocal, + })?; + let mut predicates = Vec::with_capacity(rows.len()); + let mut mutations = Vec::with_capacity(rows.len()); + for row in &rows { + let source = decode_restore_index_source_member(&row.value.0)?; + if source.owner != owner + || row.key + != restore_index_source_member_key( + self.mount, + owner.ref_set_id, + source.source_inode, + ) + { + return Err(MetadError::Codec( + "restore index release source member changed identity".to_owned(), + )); + } + if retain_reachable { + let staging_key = restore_staging_member_key( + self.mount, + owner.ref_set_id, + source.member.destination_inode, + ); + if let Some(staging) = self.metadata.get( + RecordFamily::System, + &staging_key, + version, + ReadPurpose::WritePlanLocal, + )? { + if !restore_index_source_member_matches_staging( + &source.member, + &decode_restore_staging_member(&staging.0)?, + ) { + return Err(MetadError::Codec( + "restore index release source/staging member mismatch" + .to_owned(), + )); + } + retained = retained.saturating_add(1); + continue; + } + } + predicates.push(PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }); + mutations.push(delete_mutation(RecordFamily::System, row.key.clone())); + } + self.commit_restore_index_release_deletes( + operation, + operation_version, + prefix, + predicates, + mutations, + )?; + return Ok(restore_index_release_page_outcome( + stage, &rows, limit, retained, + )); + } + RestoreIndexReleaseStage::MvccSourceMembers => { + return self.release_restore_index_mvcc_page(RestoreIndexReleasePage { + operation, + operation_version, + stage, + logical_prefix: restore_index_source_member_prefix( + self.mount, + operation.ref_set_id, + ), + start_after, + limit, + version, + }); + } + RestoreIndexReleaseStage::Seal => {} + } + + let owner_prefixes = [ + restore_index_entry_prefix(self.mount, operation.ref_set_id, None), + restore_index_parent_owner_prefix(self.mount, operation.ref_set_id), + restore_index_catalog_prefix(self.mount, operation.ref_set_id), + restore_index_row_prefix(self.mount, operation.ref_set_id, None), + restore_index_source_member_prefix(self.mount, operation.ref_set_id), + ]; + for prefix in owner_prefixes { + if !self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix, + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })? + .is_empty() + { + return Ok(RestoreIndexReleaseOutcome { + cursor: Vec::new(), + complete: false, + retained: 1, + }); + } + } + // Target-first inverse keyspaces cannot participate in the final + // ref-set PrefixEmpty predicates. Audit their full closure here; an + // orphan is corruption and must strand/quarantine the release rather + // than letting operation deletion hide it. + let seal_key = restore_index_seal_key(self.mount, operation.ref_set_id); + let marker_key = restore_index_complete_key(self.mount, operation.ref_set_id); + let marker = self.metadata.get_versioned( + RecordFamily::System, + &marker_key, + version, + ReadPurpose::WritePlanLocal, + )?; + if let Some(marker) = &marker { + let decoded = decode_restore_index_complete(&marker.value.0)?; + if decoded.operation_digest != operation.operation_digest + || decoded.initialization_digest != operation.initialization_digest + || decoded.ref_set_id != operation.ref_set_id + || decoded.incarnation != operation.created_version + { + return Err(MetadError::Codec( + "restore index release Complete marker changed identity".to_owned(), + )); + } + } + if let Some(seal) = self.metadata.get_versioned( + RecordFamily::System, + &seal_key, + version, + ReadPurpose::WritePlanLocal, + )? { + let decoded = decode_restore_index_seal(&seal.value.0)?; + if decoded.operation_digest != operation.operation_digest + || decoded.initialization_digest != operation.initialization_digest + || decoded.ref_set_id != operation.ref_set_id + || decoded.incarnation != operation.created_version + { + return Err(MetadError::Codec( + "restore index release seal changed identity".to_owned(), + )); + } + let mut predicates = vec![PredicateRef { + family: RecordFamily::System, + key: seal_key.clone(), + predicate: Predicate::VersionEquals(seal.version), + }]; + let mut mutations = vec![delete_mutation(RecordFamily::System, seal_key.clone())]; + match marker { + Some(marker) => { + predicates.push(PredicateRef { + family: RecordFamily::System, + key: marker_key.clone(), + predicate: Predicate::VersionEquals(marker.version), + }); + mutations.push(delete_mutation(RecordFamily::System, marker_key)); + } + None => predicates.push(PredicateRef { + family: RecordFamily::System, + key: marker_key, + predicate: Predicate::NotExists, + }), + } + self.commit_restore_index_release_deletes( + operation, + operation_version, + seal_key, + predicates, + mutations, + )?; + } else if let Some(marker) = marker { + self.commit_restore_index_release_deletes( + operation, + operation_version, + marker_key.clone(), + vec![PredicateRef { + family: RecordFamily::System, + key: marker_key.clone(), + predicate: Predicate::VersionEquals(marker.version), + }], + vec![delete_mutation(RecordFamily::System, marker_key)], + )?; + } + Ok(RestoreIndexReleaseOutcome { + cursor: Vec::new(), + complete: true, + retained: 0, + }) + } + + fn canonical_custom_index_at_path( + &self, + path: &str, + catalog_value: &Value, + version: Version, + purpose: ReadPurpose, + ) -> Result { + let catalog = decode_path_index_catalog(&catalog_value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if catalog.path != path { + return Err(MetadError::Codec( + "namespace index catalog path does not match its key".to_owned(), + )); + } + let mut rows = Vec::new(); + let row_prefix = path_index_row_prefix(self.mount, path); + self.restore_index_for_each_raw( + RecordFamily::PathIndex, + &row_prefix, + None, + version, + purpose, + RESTORE_INDEX_SCAN_PAGE, + |item| { + let row = decode_path_index_row(&item.value.0) + .map_err(|error| MetadError::Codec(error.to_string()))?; + if item.key != path_index_row_key(self.mount, path, &row.path) { + return Err(MetadError::Codec( + "namespace index row path does not match its key".to_owned(), + )); + } + if restore_index_relative_components(path, &row.path)?.is_none() { + return Err(MetadError::Codec( + "namespace index row escaped its catalog root".to_owned(), + )); + } + rows.push(row); + Ok(true) + }, + )?; + Ok(RestoreCustomIndex { + catalog: PathIndexCatalogRecord { + path: catalog.path, + fields: catalog.fields, + row_count: rows.len() as u64, + }, + rows, + }) + } + + fn restore_index_path_metadata_if_current( + &self, + root: InodeId, + relative_path: &str, + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadError> { + match self.stat_path_from_at_version_for_purpose(root, relative_path, version, purpose) { + Ok(metadata) => Ok(metadata), + Err(MetadError::NotFound | MetadError::NotDirectory) => Ok(None), + Err(error) => Err(error), + } + } + + /// Rebase one durable row from the catalog path recorded at restore time to + /// the catalog's current query path, then prove that the exact dentry/body + /// identity still occupies that path. Runtime queries are intentionally + /// driven by index rows: they never enumerate unrelated namespace entries + /// and therefore never inherit the restore creation-time one-million-entry + /// resource limit. + fn current_restore_index_row_path( + &self, + query_root: InodeId, + query_path: &str, + catalog_path: &str, + row: &RestoreIndexRow, + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadError> { + let Some(relative) = restore_index_relative_components(catalog_path, &row.record.path)? + else { + return Err(MetadError::Codec( + "restore index row escaped its recorded catalog root".to_owned(), + )); + }; + let current_path = restore_index_join_path(query_path, &relative)?; + if relative.is_empty() { + let Some(metadata) = + self.restore_index_path_metadata_if_current(query_root, "/", version, purpose)? + else { + return Ok(None); + }; + return Ok((restore_index_root_target(&metadata) == row.target).then_some(current_path)); + } + + let relative_path = canonical_path(&relative)?; + let entry = match self.lookup_path_from_at_version_for_purpose( + query_root, + &relative_path, + version, + purpose, + ) { + Ok(entry) => entry, + Err(MetadError::NotFound | MetadError::NotDirectory) => None, + Err(error) => return Err(error), + }; + let Some((entry, _)) = entry else { + return Ok(None); + }; + let projection = DentryProjection { + dentry: entry.dentry, + attr: entry.attr, + body: entry.body, + }; + Ok( + restore_index_target_matches_projection(&row.target, &projection) + .then_some(current_path), + ) + } + + fn restore_index_recorded_path_is_current( + &self, + query_root: InodeId, + query_path: &str, + recorded_path: &str, + version: Version, + purpose: ReadPurpose, + ) -> Result { + let Some(relative) = restore_index_relative_components(query_path, recorded_path)? else { + return Ok(false); + }; + let relative_path = canonical_path(&relative)?; + Ok(self + .restore_index_path_metadata_if_current(query_root, &relative_path, version, purpose)? + .is_some()) + } + + /// Effective custom index at an exact namespace root. Canonical records + /// override inherited overlay rows of the same current path; otherwise + /// fields and rows are merged across complete nested restores. Every + /// overlay row is checked against both its inverse and the current dentry / + /// body identity before it is returned. + pub(super) fn restore_custom_index_at_path( + &self, + path: &str, + root: InodeId, + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadError> { + let canonical_value = self.metadata.get( + RecordFamily::PathIndex, + &path_index_catalog_key(self.mount, path), + version, + purpose, + )?; + let fork_source = self.restore_index_fork_source(root)?; + let inverse_prefix = restore_index_catalog_inverse_prefix(self.mount, root); + let mut has_visible_overlay = false; + self.restore_index_effective_for_each( + &inverse_prefix, + None, + version, + purpose, + |inverse| { + if inverse.key.len() != inverse_prefix.len() + 8 { + return Err(MetadError::Codec( + "restore index catalog inverse key has an invalid length".to_owned(), + )); + } + let catalog = decode_restore_index_catalog(&inverse.value.0)?; + let ref_set_id = u64::from_be_bytes( + inverse.key[inverse_prefix.len()..] + .try_into() + .expect("u64 width"), + ); + if catalog.catalog_root != root + || catalog.owner.ref_set_id != ref_set_id + || inverse.key + != restore_index_catalog_inverse_key(self.mount, root, ref_set_id) + { + return Err(MetadError::Codec( + "restore index catalog inverse changed identity".to_owned(), + )); + } + if self + .visible_restore_index_operation(catalog.owner, version, purpose)? + .is_some() + { + has_visible_overlay = true; + return Ok(false); + } + Ok(true) + }, + )?; + + if canonical_value.is_none() && !has_visible_overlay && fork_source.is_none() { + return Ok(None); + } + if !has_visible_overlay && fork_source.is_none() { + return canonical_value + .as_ref() + .map(|catalog| self.canonical_custom_index_at_path(path, catalog, version, purpose)) + .transpose(); + } + + // Overlay and generic-fork queries validate only index candidates. They + // must not build a map of every dentry under `root`: unindexed growth is + // unrelated to the index result and may continue beyond restore's + // creation-time subtree limit. + let mut fields = BTreeMap::::new(); + let mut rows = BTreeMap::::new(); + let mut found_catalog = false; + + self.restore_index_effective_for_each( + &inverse_prefix, + None, + version, + purpose, + |inverse| { + if inverse.key.len() != inverse_prefix.len() + 8 { + return Err(MetadError::Codec( + "restore index catalog inverse key has an invalid length".to_owned(), + )); + } + let catalog = decode_restore_index_catalog(&inverse.value.0)?; + let ref_set_id = u64::from_be_bytes( + inverse.key[inverse_prefix.len()..] + .try_into() + .expect("u64 width"), + ); + if catalog.catalog_root != root + || catalog.owner.ref_set_id != ref_set_id + || inverse.key + != restore_index_catalog_inverse_key(self.mount, root, ref_set_id) + { + return Err(MetadError::Codec( + "restore index catalog inverse changed identity".to_owned(), + )); + } + if self + .visible_restore_index_operation(catalog.owner, version, purpose)? + .is_none() + { + return Ok(true); + } + let owner_key = restore_index_catalog_key(self.mount, ref_set_id, root); + let owner_item = self + .restore_index_effective_get(&owner_key, version, purpose)? + .ok_or_else(|| { + MetadError::Codec("restore index catalog inverse has no owner".to_owned()) + })?; + if decode_restore_index_catalog(&owner_item.value.0)? != catalog { + return Err(MetadError::Codec( + "restore index catalog owner/inverse mismatch".to_owned(), + )); + } + found_catalog = true; + for field in catalog.record.fields.iter().cloned() { + match fields.entry(field.field.clone()) { + std::collections::btree_map::Entry::Vacant(slot) => { + slot.insert(field); + } + std::collections::btree_map::Entry::Occupied(slot) + if slot.get() != &field => + { + return Err(MetadError::Codec( + "nested restore custom catalogs disagree on a field".to_owned(), + )); + } + std::collections::btree_map::Entry::Occupied(_) => {} + } + } + let row_prefix = restore_index_row_prefix(self.mount, ref_set_id, Some(root)); + self.restore_index_effective_for_each( + &row_prefix, + None, + version, + purpose, + |item| { + let mut row = decode_restore_index_row(&item.value.0)?; + if row.owner != catalog.owner + || row.catalog_root != root + || item.key + != restore_index_row_key(self.mount, ref_set_id, root, &row.target) + { + return Err(MetadError::Codec( + "restore index custom row changed identity".to_owned(), + )); + } + let target_inverse_key = restore_index_target_inverse_key( + self.mount, + &row.target, + ref_set_id, + root, + ); + let target_inverse = self + .restore_index_effective_get(&target_inverse_key, version, purpose)? + .ok_or_else(|| { + MetadError::Codec( + "restore index custom row has no target inverse".to_owned(), + ) + })?; + if decode_restore_index_row(&target_inverse.value.0)? != row { + return Err(MetadError::Codec( + "restore index custom row/inverse mismatch".to_owned(), + )); + } + let Some(current_path) = self.current_restore_index_row_path( + root, + path, + &catalog.record.path, + &row, + version, + purpose, + )? + else { + return Ok(true); + }; + row.record.path = current_path; + match rows.entry(row.record.path.clone()) { + std::collections::btree_map::Entry::Vacant(slot) => { + slot.insert(row.record); + } + std::collections::btree_map::Entry::Occupied(slot) + if slot.get() != &row.record => + { + return Err(MetadError::Codec( + "nested restore custom rows disagree on a current path" + .to_owned(), + )); + } + std::collections::btree_map::Entry::Occupied(_) => {} + } + Ok(true) + }, + )?; + Ok(true) + }, + )?; + + if let Some(fork_source) = fork_source { + let source_version = Version::new(fork_source.binding.pinned_read_version)?; + let source_path = &fork_source.operation.destination_path; + if let Some(inherited) = self.restore_custom_index_at_path( + source_path, + fork_source.binding.source_root, + source_version, + ReadPurpose::Snapshot, + )? { + found_catalog = true; + for field in inherited.catalog.fields { + match fields.entry(field.field.clone()) { + std::collections::btree_map::Entry::Vacant(slot) => { + slot.insert(field); + } + std::collections::btree_map::Entry::Occupied(slot) + if slot.get() != &field => + { + return Err(MetadError::Codec( + "generic fork custom catalogs disagree on a field".to_owned(), + )); + } + std::collections::btree_map::Entry::Occupied(_) => {} + } + } + for mut row in inherited.rows { + let Some(relative) = restore_index_relative_components(source_path, &row.path)? + else { + return Err(MetadError::Codec( + "generic fork custom row escaped its source catalog".to_owned(), + )); + }; + let fork_path = restore_index_join_path(path, &relative)?; + let relative_path = canonical_path(&relative)?; + let Some(source_metadata) = self.restore_index_path_metadata_if_current( + fork_source.binding.source_root, + &relative_path, + source_version, + ReadPurpose::Snapshot, + )? + else { + continue; + }; + let Some(fork_metadata) = self.restore_index_path_metadata_if_current( + root, + &relative_path, + version, + purpose, + )? + else { + continue; + }; + if !restore_index_clone_metadata_matches_source( + &fork_metadata, + &source_metadata, + ) { + continue; + } + row.path = fork_path; + match rows.entry(row.path.clone()) { + std::collections::btree_map::Entry::Vacant(slot) => { + slot.insert(row); + } + std::collections::btree_map::Entry::Occupied(slot) + if slot.get() != &row => + { + return Err(MetadError::Codec( + "generic fork and restore overlay disagree on a custom row" + .to_owned(), + )); + } + std::collections::btree_map::Entry::Occupied(_) => {} + } + } + } + } + + if let Some(catalog_value) = canonical_value { + let canonical = + self.canonical_custom_index_at_path(path, &catalog_value, version, purpose)?; + found_catalog = true; + for field in canonical.catalog.fields { + fields.insert(field.field.clone(), field); + } + for row in canonical.rows { + if self.restore_index_recorded_path_is_current( + root, + &canonical.catalog.path, + &row.path, + version, + purpose, + )? { + rows.insert(row.path.clone(), row); + } + } + } + + if !found_catalog { + return Ok(None); + } + let rows = rows.into_values().collect::>(); + Ok(Some(RestoreCustomIndex { + catalog: PathIndexCatalogRecord { + path: path.to_owned(), + fields: fields.into_values().collect(), + row_count: rows.len() as u64, + }, + rows, + })) + } + + fn collect_restore_index_owner_children( + &self, + query: RestoreIndexChildrenQuery<'_>, + owner: RestoreIndexOwner, + visible_operation: &RestoreOperation, + entries: &mut BTreeMap, DentryWithAttr>, + releasing_parent_reachable: &mut Option, + ) -> Result<(), MetadError> { + if visible_operation.state == RestoreOperationState::Releasing + && query.purpose != ReadPurpose::Snapshot + { + let parent_reachable = if let Some(reachable) = *releasing_parent_reachable { + reachable + } else { + let candidates = HashSet::from([query.parent]); + let reachable = self + .restore_reachable_inodes_at(&candidates, query.version)? + .contains(&query.parent); + *releasing_parent_reachable = Some(reachable); + reachable + }; + if !parent_reachable { + return Ok(()); + } + } + let entry_prefix = + restore_index_entry_prefix(self.mount, owner.ref_set_id, Some(query.parent)); + let start_after = query + .after + .map(|name| restore_index_entry_key(self.mount, owner.ref_set_id, query.parent, name)); + let mut accepted = 0_usize; + self.restore_index_effective_for_each( + &entry_prefix, + start_after.as_deref(), + query.version, + query.purpose, + |item| { + let entry = decode_restore_index_entry(&item.value.0)?; + if entry.owner != owner + || item.key + != restore_index_entry_key( + self.mount, + owner.ref_set_id, + entry.projection.dentry.parent, + &entry.projection.dentry.name, + ) + || entry.projection.dentry.parent != query.parent + { + return Err(MetadError::Codec( + "restore index entry does not match its key/owner".to_owned(), + )); + } + let Some((current, _)) = self.lookup_plus_at_version_for_purpose( + query.parent, + &entry.projection.dentry.name, + query.version, + query.purpose, + )? + else { + return Ok(true); + }; + let indexed: DentryWithAttr = entry.projection.into(); + if current != indexed { + return Ok(true); + } + match entries.entry(current.dentry.name.as_bytes().to_vec()) { + std::collections::btree_map::Entry::Vacant(slot) => { + slot.insert(current); + } + std::collections::btree_map::Entry::Occupied(slot) + if slot.get() != ¤t => + { + return Err(MetadError::Codec( + "nested restore indexes disagree on a current dentry".to_owned(), + )); + } + std::collections::btree_map::Entry::Occupied(_) => {} + } + if entries.len() > query.keep { + entries.pop_last(); + } + accepted = accepted.saturating_add(1); + Ok(accepted < query.keep) + }, + ) + } + + /// Returns only rows owned by a durably complete operation and still equal + /// to the authoritative current dentry. Corrupt owner/inverse pairs fail + /// closed instead of silently falling back to a partial namespace view. + /// Every owner stream contributes at most `limit + 1` candidates and MVCC + /// history is folded in fixed pages, so pagination memory is O(limit), not + /// O(directory entries) or O(history rows). + pub(super) fn restore_indexed_children_page( + &self, + parent: InodeId, + after: Option<&DentryName>, + limit: usize, + version: Version, + purpose: ReadPurpose, + ) -> Result { + let requested = limit.max(1); + let keep = requested.saturating_add(1); + let query = RestoreIndexChildrenQuery { + parent, + after, + keep, + version, + purpose, + }; + let inverse_prefix = restore_index_parent_inverse_prefix_for_read(self.mount, parent); + let mut entries = BTreeMap::, DentryWithAttr>::new(); + let mut releasing_parent_reachable = None; + self.restore_index_effective_for_each( + &inverse_prefix, + None, + version, + purpose, + |inverse| { + if inverse.key.len() != inverse_prefix.len() + 8 { + return Err(MetadError::Codec( + "restore index parent inverse key has an invalid length".to_owned(), + )); + } + let owner = decode_restore_index_owner(&inverse.value.0)?; + let ref_set_id = u64::from_be_bytes( + inverse.key[inverse_prefix.len()..] + .try_into() + .expect("u64 width"), + ); + if owner.ref_set_id != ref_set_id + || inverse.key + != restore_index_parent_inverse_key(self.mount, parent, ref_set_id) + { + return Err(MetadError::Codec( + "restore index parent inverse does not match its owner".to_owned(), + )); + } + let Some(visible_operation) = + self.visible_restore_index_operation(owner, version, purpose)? + else { + return Ok(true); + }; + let owner_key = restore_index_parent_owner_key(self.mount, ref_set_id, parent); + let owner_value = self + .restore_index_effective_get(&owner_key, version, purpose)? + .ok_or_else(|| { + MetadError::Codec("restore index parent inverse has no owner".to_owned()) + })?; + if decode_restore_index_owner(&owner_value.value.0)? != owner { + return Err(MetadError::Codec( + "restore index parent owner/inverse mismatch".to_owned(), + )); + } + self.collect_restore_index_owner_children( + query, + owner, + &visible_operation, + &mut entries, + &mut releasing_parent_reachable, + )?; + Ok(true) + }, + )?; + if let Some(fork_source) = self.restore_index_fork_source(parent)? { + let source_version = Version::new(fork_source.binding.pinned_read_version)?; + let mut source_after = after.cloned(); + let mut accepted = 0_usize; + loop { + let source_page = self.restore_indexed_children_page( + fork_source.binding.source_root, + source_after.as_ref(), + keep, + source_version, + ReadPurpose::Snapshot, + )?; + for source in source_page.entries { + let Some((fork, _)) = self.lookup_plus_at_version_for_purpose( + parent, + &source.dentry.name, + version, + purpose, + )? + else { + continue; + }; + if !restore_index_clone_entry_matches_source(&fork, &source) { + continue; + } + match entries.entry(fork.dentry.name.as_bytes().to_vec()) { + std::collections::btree_map::Entry::Vacant(slot) => { + slot.insert(fork); + } + std::collections::btree_map::Entry::Occupied(slot) + if slot.get() != &fork => + { + return Err(MetadError::Codec( + "generic fork and restore overlay disagree on a current dentry" + .to_owned(), + )); + } + std::collections::btree_map::Entry::Occupied(_) => {} + } + if entries.len() > keep { + entries.pop_last(); + } + accepted = accepted.saturating_add(1); + if accepted >= keep { + break; + } + } + if accepted >= keep { + break; + } + let Some(next) = source_page.next_cursor else { + break; + }; + if source_after + .as_ref() + .is_some_and(|after| after.as_bytes() >= next.as_bytes()) + { + return Err(MetadError::Codec( + "generic fork inherited index cursor did not advance".to_owned(), + )); + } + source_after = Some(next); + } + } + let has_more = entries.len() > requested; + let mut entries = entries.into_values().take(requested).collect::>(); + let next_cursor = has_more + .then(|| entries.last().map(|entry| entry.dentry.name.clone())) + .flatten(); + Ok(ReadDirPlusPage { + entries: std::mem::take(&mut entries), + next_cursor, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn mount() -> MountId { + MountId::new(7).unwrap() + } + + fn operation() -> RestoreOperation { + RestoreOperation { + operation_digest: [3; 32], + initialization_digest: [5; 32], + state: RestoreOperationState::Preparing, + source_root: InodeId::new(10).unwrap(), + destination_root: InodeId::new(20).unwrap(), + snapshot_id: 30, + read_version: 40, + created_version: 50, + ref_set_id: 50, + source_path: "/source".to_owned(), + destination_path: "/destination".to_owned(), + } + } + + fn projection() -> DentryProjection { + let inode = InodeId::new(22).unwrap(); + DentryProjection { + dentry: DentryRecord { + parent: InodeId::new(20).unwrap(), + name: DentryName::new(b"file.txt".to_vec()).unwrap(), + child: inode, + child_type: FileType::File, + attr_generation: 11, + }, + attr: InodeAttr { + inode, + file_type: FileType::File, + mode: 0o644, + uid: 1, + gid: 2, + rdev: 0, + nlink: 1, + size: 3, + generation: 11, + mtime_ms: 12, + ctime_ms: 13, + }, + body: Some(BodyDescriptor { + producer: "test".to_owned(), + digest_uri: "sha256:abc".to_owned(), + size: 3, + content_type: "text/plain".to_owned(), + manifest_id: "manifest".to_owned(), + generation: 11, + base_generation: 0, + chunk_size: 64, + block_size: 16, + }), + } + } + + #[test] + fn restore_index_entry_codec_is_strict() { + let entry = RestoreIndexEntry { + owner: RestoreIndexOwner::from_operation(&operation()), + projection: projection(), + }; + let encoded = encode_restore_index_entry(&entry); + assert_eq!(decode_restore_index_entry(&encoded).unwrap(), entry); + assert!(decode_restore_index_entry(&encoded[..encoded.len() - 1]).is_err()); + let mut trailing = encoded; + trailing.push(0); + assert!(decode_restore_index_entry(&trailing).is_err()); + } + + #[test] + fn restore_index_custom_codecs_preserve_path_specific_identity() { + let projection = projection(); + let owner = RestoreIndexOwner::from_operation(&operation()); + let target = restore_index_target_from_projection(&projection); + let catalog = RestoreIndexCatalog { + owner, + catalog_root: InodeId::new(20).unwrap(), + record: PathIndexCatalogRecord { + path: "/destination".to_owned(), + fields: Vec::new(), + row_count: 1, + }, + }; + let row = RestoreIndexRow { + owner, + catalog_root: catalog.catalog_root, + target, + record: PathIndexRowRecord { + path: "/destination/file.txt".to_owned(), + values: vec![( + "kind".to_owned(), + PathIndexValueRecord::String("report".to_owned()), + )], + }, + }; + assert_eq!( + decode_restore_index_catalog(&encode_restore_index_catalog(&catalog)).unwrap(), + catalog + ); + assert_eq!( + decode_restore_index_row(&encode_restore_index_row(&row)).unwrap(), + row + ); + assert!(restore_index_target_matches_projection( + &row.target, + &projection + )); + let mut changed_generation = projection.clone(); + changed_generation.dentry.attr_generation += 1; + changed_generation.attr.generation += 1; + assert!(!restore_index_target_matches_projection( + &row.target, + &changed_generation + )); + + let mut trailing = encode_restore_index_row(&row); + trailing.push(0); + assert!(decode_restore_index_row(&trailing).is_err()); + } + + #[test] + fn restore_index_source_member_codec_is_strict() { + let operation = operation(); + let source_inode = InodeId::new(11).unwrap(); + let source = RestoreIndexSourceMember { + owner: RestoreIndexOwner::from_operation(&operation), + source_inode, + member: RestoreStagingMember { + operation_digest: operation.operation_digest, + source_inode: Some(source_inode), + destination_inode: InodeId::new(21).unwrap(), + destination_parent: Some(operation.destination_root), + name: Some(DentryName::new(b"child".to_vec()).unwrap()), + relative_path: "child".to_owned(), + canonical_index_cursor: Vec::new(), + canonical_index_complete: true, + manifest_cursor: Vec::new(), + manifest_block_cursor: 0, + }, + }; + let encoded = encode_restore_index_source_member(&source).unwrap(); + assert_eq!( + decode_restore_index_source_member(&encoded).unwrap(), + source + ); + assert!(decode_restore_index_source_member(&encoded[..encoded.len() - 1]).is_err()); + } + + #[test] + fn restore_index_seal_codec_is_strict() { + let seal = RestoreIndexSeal { + operation_digest: [1; 32], + initialization_digest: [2; 32], + ref_set_id: 7, + incarnation: 8, + entry_count: 9, + catalog_count: 10, + row_count: 11, + digest: [12; 32], + }; + let encoded = encode_restore_index_seal(&seal); + assert_eq!(decode_restore_index_seal(&encoded).unwrap(), seal); + assert!(decode_restore_index_seal(&encoded[..encoded.len() - 1]).is_err()); + } + + #[test] + fn restore_index_mvcc_codec_and_physical_key_are_strict() { + let logical_key = restore_index_entry_key( + mount(), + 50, + InodeId::new(20).unwrap(), + &DentryName::new(b"file.txt".to_vec()).unwrap(), + ); + let record = RestoreIndexMvccRecord { + commit_version: 70, + kind: RestoreIndexMvccKind::Tombstone, + logical_key: logical_key.clone(), + value: encode_restore_index_entry(&RestoreIndexEntry { + owner: RestoreIndexOwner::from_operation(&operation()), + projection: projection(), + }), + }; + let encoded = encode_restore_index_mvcc(&record); + assert_eq!(decode_restore_index_mvcc(&encoded).unwrap(), record); + assert!(decode_restore_index_mvcc(&encoded[..encoded.len() - 1]).is_err()); + + let version = Version::new(record.commit_version).unwrap(); + let physical = restore_index_mvcc_key(mount(), &logical_key, version).unwrap(); + assert!(physical.starts_with(&restore_index_mvcc_prefix(mount(), &logical_key).unwrap())); + assert_ne!( + physical, + restore_index_mvcc_key(mount(), &logical_key, Version::new(71).unwrap()).unwrap() + ); + } + + #[test] + fn restore_index_complete_marker_codec_is_strict() { + let marker = RestoreIndexCompleteMarker { + operation_digest: [1; 32], + initialization_digest: [2; 32], + ref_set_id: 3, + incarnation: 4, + complete_version: 5, + }; + let encoded = encode_restore_index_complete(&marker); + assert_eq!(decode_restore_index_complete(&encoded).unwrap(), marker); + assert!(decode_restore_index_complete(&encoded[..encoded.len() - 1]).is_err()); + } + + #[test] + fn restore_index_keys_keep_nested_refsets_distinct() { + let parent = InodeId::new(90).unwrap(); + let name = DentryName::new(b"child".to_vec()).unwrap(); + assert_ne!( + restore_index_entry_key(mount(), 1, parent, &name), + restore_index_entry_key(mount(), 2, parent, &name) + ); + assert_ne!( + restore_index_parent_inverse_key(mount(), parent, 1), + restore_index_parent_inverse_key(mount(), parent, 2) + ); + } + + #[test] + fn restore_index_path_boundaries_are_component_aware() { + assert_eq!( + restore_index_relative_string( + &restore_index_relative_components("/work/a", "/work/a/child") + .unwrap() + .unwrap() + ) + .unwrap(), + "child" + ); + assert!(restore_index_relative_components("/work/a", "/work/ab") + .unwrap() + .is_none()); + } + + #[test] + fn restore_index_release_cursor_rejects_unknown_stage() { + assert!(decode_restore_index_release_cursor(&[0]).is_err()); + assert!(decode_restore_index_release_cursor(&[18]).is_err()); + assert_eq!( + decode_restore_index_release_cursor(&[]).unwrap(), + (RestoreIndexReleaseStage::Entries, None) + ); + } + + #[test] + fn restore_index_mutation_plan_deduplicates_and_rejects_conflicting_cas() { + let key = b"key".to_vec(); + let mut plan = RestoreIndexMutationPlan::default(); + let version = Version::new(7).unwrap(); + plan.push_predicate(PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(version), + }) + .unwrap(); + plan.push_predicate(PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(version), + }) + .unwrap(); + assert_eq!(plan.predicates.len(), 1); + assert!(plan + .push_predicate(PredicateRef { + family: RecordFamily::System, + key, + predicate: Predicate::NotExists, + }) + .is_err()); + } + + #[test] + fn restore_index_mutation_plan_rejects_item_and_byte_overflow() { + let mut too_many = RestoreIndexMutationPlan::default(); + for index in 0..=MAX_RESTORE_INDEX_PLAN_ITEMS { + too_many.predicates.push(PredicateRef { + family: RecordFamily::System, + key: (index as u64).to_be_bytes().to_vec(), + predicate: Predicate::NotExists, + }); + } + assert!(matches!( + too_many.validate_budget("test restore index plan"), + Err(MetadError::RestoreResourceLimit { + resource, + limit, + actual, + }) if resource.ends_with(" items") + && limit == MAX_RESTORE_INDEX_PLAN_ITEMS as u64 + && actual == limit + 1 + )); + + let too_large = RestoreIndexMutationPlan { + predicates: Vec::new(), + mutations: vec![Mutation { + family: RecordFamily::System, + key: b"large-row".to_vec(), + op: MutationOp::Put, + value: Some(Value(vec![0; MAX_RESTORE_INDEX_PLAN_BYTES])), + }], + }; + assert!(matches!( + too_large.validate_budget("test restore index plan"), + Err(MetadError::RestoreResourceLimit { + resource, + limit, + actual, + }) if resource.ends_with(" bytes") + && limit == MAX_RESTORE_INDEX_PLAN_BYTES as u64 + && actual > limit + )); + } +} diff --git a/crates/nokv-meta/src/service/snapshot.rs b/crates/nokv-meta/src/service/snapshot.rs index 5a1be6219..bf5975d8d 100644 --- a/crates/nokv-meta/src/service/snapshot.rs +++ b/crates/nokv-meta/src/service/snapshot.rs @@ -6,6 +6,7 @@ use crate::layout::{decode_fork_binding, fork_binding_prefix}; pub const DEFAULT_SNAPSHOT_LEASE_MS: u64 = 3_600_000; const SNAPSHOT_MINT_MAX_ATTEMPTS: usize = 8; const SNAPSHOT_RENEW_MAX_ATTEMPTS: usize = 8; +const FORK_BINDING_SCAN_PAGE: usize = 64; const SNAPSHOT_ID_SHARD_BITS: u32 = 16; const SNAPSHOT_ID_LOCAL_BITS: u32 = u64::BITS - SNAPSHOT_ID_SHARD_BITS; const SNAPSHOT_ID_LOCAL_MASK: u64 = (1_u64 << SNAPSHOT_ID_LOCAL_BITS) - 1; @@ -37,6 +38,15 @@ where root: InodeId, lease_ms: u64, ) -> Result { + // Release-to-fork proves every current and historical borrower while + // holding this gate before it drops an exact object reference. Keep + // snapshot minting in the same critical section so a new pin cannot + // commit after that proof but before the exact-reference CAS. Ordinary + // object GC remains concurrent and is fenced by its durable claim CAS. + let _restore_snapshot_gate = self + .restore_snapshot_gate + .read() + .unwrap_or_else(|error| error.into_inner()); for attempt in 0..SNAPSHOT_MINT_MAX_ATTEMPTS { let object_reference = self.begin_object_reference_mutation()?; let Some(attr) = self.get_attr_at_version_for_purpose( @@ -109,6 +119,10 @@ where path: &str, lease_ms: u64, ) -> Result { + let _restore_snapshot_gate = self + .restore_snapshot_gate + .read() + .unwrap_or_else(|error| error.into_inner()); for attempt in 0..SNAPSHOT_MINT_MAX_ATTEMPTS { let object_reference = self.begin_object_reference_mutation()?; let binding_version = self.read_version()?; @@ -186,6 +200,7 @@ where return Ok(false); } self.validate_snapshot_retention_roots(snapshot_id, pin.as_ref(), &bindings, None)?; + self.ensure_restore_snapshot_retirable(snapshot_id, &bindings, read_version)?; self.ensure_fork_bindings_releasable(&bindings, read_version)?; let pin_key = snapshot_pin_key(self.mount, snapshot_id); @@ -285,6 +300,7 @@ where &bindings, Some(expected_root), )?; + self.ensure_restore_snapshot_retirable(snapshot_id, &bindings, read_version)?; self.ensure_fork_bindings_releasable(&bindings, read_version)?; let pin_key = snapshot_pin_key(self.mount, snapshot_id); @@ -386,6 +402,16 @@ where snapshot_id: u64, lease_ms: u64, ) -> Result { + // A renewal that read a live pin immediately before its old lease + // deadline may commit after the deadline. Serialize that whole proof + // and CAS with restore release, which treats an expired pin as no + // longer retaining its private ref-set. Otherwise release could prove + // the old lease dead, remove the operation/overlay, and then let this + // call acknowledge an extension of the now-incomplete snapshot. + let _restore_snapshot_gate = self + .restore_snapshot_gate + .read() + .unwrap_or_else(|error| error.into_inner()); let requested_expiry = self.now_ms().saturating_add(lease_ms); let key = snapshot_pin_key(self.mount, snapshot_id); for _attempt in 0..SNAPSHOT_RENEW_MAX_ATTEMPTS { @@ -767,17 +793,41 @@ where version: Version, purpose: ReadPurpose, ) -> Result, MetadError> { - self.metadata - .scan(ScanRequest { + let mut bindings = Vec::new(); + self.visit_versioned_fork_bindings_at(version, purpose, |binding| { + bindings.push(binding); + Ok(false) + })?; + Ok(bindings) + } + + /// Visit fork bindings at one stable version in bounded pages. Returning + /// `true` from the visitor stops the scan, which lets retention proofs keep + /// O(page) memory and avoid reading bindings after the first live holder. + pub(super) fn visit_versioned_fork_bindings_at( + &self, + version: Version, + purpose: ReadPurpose, + mut visit: F, + ) -> Result + where + F: FnMut(VersionedForkBinding) -> Result, + { + let prefix = fork_binding_prefix(self.mount); + let mut start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { family: RecordFamily::ForkBinding, - prefix: fork_binding_prefix(self.mount), - start_after: None, + prefix: prefix.clone(), + start_after: start_after.clone(), version, - limit: 0, + limit: FORK_BINDING_SCAN_PAGE, purpose, - })? - .into_iter() - .map(|row| { + })?; + if rows.is_empty() { + break; + } + for row in &rows { let binding = decode_fork_binding(&row.value.0) .map_err(|err| MetadError::Codec(err.to_string()))?; let expected_key = fork_binding_key(self.mount, binding.fork_root); @@ -807,13 +857,20 @@ where binding.snapshot_id ))); } - Ok(VersionedForkBinding { + if visit(VersionedForkBinding { binding, - key: row.key, + key: row.key.clone(), version: row.version, - }) - }) - .collect() + })? { + return Ok(true); + } + } + start_after = rows.last().map(|row| row.key.clone()); + if rows.len() < FORK_BINDING_SCAN_PAGE { + break; + } + } + Ok(false) } fn fork_bindings_for_snapshot_at( @@ -953,7 +1010,7 @@ where Ok((u64::from(self.shard_index()) << SNAPSHOT_ID_LOCAL_BITS) | local) } - fn ensure_snapshot_id_shard( + pub(super) fn ensure_snapshot_id_shard( &self, snapshot_id: u64, expected_root: InodeId, diff --git a/crates/nokv-meta/src/service/xattr.rs b/crates/nokv-meta/src/service/xattr.rs index ce2d581c4..e30c4c4f2 100644 --- a/crates/nokv-meta/src/service/xattr.rs +++ b/crates/nokv-meta/src/service/xattr.rs @@ -14,6 +14,7 @@ where ) -> Result<(), MetadError> { validate_xattr_name(name)?; let version = self.next_version()?; + let read_version = predecessor(version)?; let key = xattr_key(self.mount, inode, name); let mut predicates = vec![PredicateRef { family: RecordFamily::Inode, @@ -33,10 +34,11 @@ where predicate: Predicate::Exists, }), } + predicates.extend(self.restore_namespace_write_predicates(&[inode], read_version)?); self.commit_metadata(MetadataCommand { request_id: request_id(b"set-xattr", self.mount, inode, version), kind: CommandKind::SetXattr, - read_version: predecessor(version)?, + read_version, commit_version: version, primary_family: RecordFamily::Xattr, primary_key: key.clone(), @@ -90,26 +92,29 @@ where pub fn remove_xattr(&self, inode: InodeId, name: &[u8]) -> Result<(), MetadError> { validate_xattr_name(name)?; let version = self.next_version()?; + let read_version = predecessor(version)?; let key = xattr_key(self.mount, inode, name); + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, inode), + predicate: Predicate::Exists, + }, + PredicateRef { + family: RecordFamily::Xattr, + key: key.clone(), + predicate: Predicate::Exists, + }, + ]; + predicates.extend(self.restore_namespace_write_predicates(&[inode], read_version)?); self.commit_metadata(MetadataCommand { request_id: request_id(b"remove-xattr", self.mount, inode, version), kind: CommandKind::RemoveXattr, - read_version: predecessor(version)?, + read_version, commit_version: version, primary_family: RecordFamily::Xattr, primary_key: key.clone(), - predicates: vec![ - PredicateRef { - family: RecordFamily::Inode, - key: inode_key(self.mount, inode), - predicate: Predicate::Exists, - }, - PredicateRef { - family: RecordFamily::Xattr, - key: key.clone(), - predicate: Predicate::Exists, - }, - ], + predicates, mutations: vec![Mutation { family: RecordFamily::Xattr, key, diff --git a/crates/nokv-meta/src/service_tests.rs b/crates/nokv-meta/src/service_tests.rs index 1c57e9885..60729a067 100644 --- a/crates/nokv-meta/src/service_tests.rs +++ b/crates/nokv-meta/src/service_tests.rs @@ -5,63 +5,36 @@ use crate::layout::object_gc_quarantine_prefix; use crate::{MetadataLogEntry, MetadataLogSegment, METADATA_LOG_ZERO_DIGEST}; use nokv_object::{MemoryObjectStore, ObjectBytes}; use nokv_types::{AdvisoryLockKind, AdvisoryLockRequest}; +use std::collections::BTreeMap; use std::sync::{Arc, Barrier, Condvar}; use std::time::{Duration, Instant}; #[derive(Clone)] -struct SnapshotCommitBarrierStore { +struct RestoreInverseProbeStore { inner: HoltMetadataStore, - kind: CommandKind, - request_prefix: Option>, - rejected_kind: Option, - remaining: Arc, - predicate_failures: Arc, - entered: Arc, - release: Arc, + inverse_prefix: Vec, + inverse_gets: Arc, } -impl SnapshotCommitBarrierStore { - fn new(kind: CommandKind, blocked_commits: u64, parties: usize) -> Self { +impl RestoreInverseProbeStore { + fn new(mount: MountId) -> Self { + let inverse_prefix = restore::restore_control_keyspaces(mount) + .into_iter() + .find_map(|(name, prefix)| (name == "staging_inode_inverse").then_some(prefix)) + .expect("restore staging inverse keyspace is registered"); Self { inner: HoltMetadataStore::open_memory().unwrap(), - kind, - request_prefix: None, - rejected_kind: None, - remaining: Arc::new(AtomicU64::new(blocked_commits)), - predicate_failures: Arc::new(AtomicU64::new(0)), - entered: Arc::new(Barrier::new(parties)), - release: Arc::new(Barrier::new(parties)), + inverse_prefix, + inverse_gets: Arc::new(AtomicU64::new(0)), } } - fn wait_until_blocked(&self) { - self.entered.wait(); - } - - fn arm(&self, blocked_commits: u64) { - self.remaining.store(blocked_commits, Ordering::SeqCst); - } - - fn predicate_failures(&self) -> u64 { - self.predicate_failures.load(Ordering::SeqCst) - } - - fn rejecting(mut self, kind: CommandKind) -> Self { - self.rejected_kind = Some(kind); - self - } - - fn matching_request_prefix(mut self, prefix: &[u8]) -> Self { - self.request_prefix = Some(prefix.to_vec()); - self - } - - fn release_blocked(&self) { - self.release.wait(); + fn inverse_gets(&self) -> u64 { + self.inverse_gets.load(Ordering::SeqCst) } } -impl MetadataStore for SnapshotCommitBarrierStore { +impl MetadataStore for RestoreInverseProbeStore { fn get_versioned( &self, family: RecordFamily, @@ -69,6 +42,9 @@ impl MetadataStore for SnapshotCommitBarrierStore { version: Version, purpose: ReadPurpose, ) -> Result, MetadataError> { + if family == RecordFamily::System && key.starts_with(&self.inverse_prefix) { + self.inverse_gets.fetch_add(1, Ordering::SeqCst); + } self.inner.get_versioned(family, key, version, purpose) } @@ -77,29 +53,7 @@ impl MetadataStore for SnapshotCommitBarrierStore { } fn commit_metadata(&self, command: MetadataCommand) -> Result { - if command.kind == self.kind - && self - .request_prefix - .as_ref() - .is_none_or(|prefix| command.request_id.starts_with(prefix)) - && self - .remaining - .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { - remaining.checked_sub(1) - }) - .is_ok() - { - self.entered.wait(); - self.release.wait(); - } - if self.rejected_kind == Some(command.kind) { - return Err(MetadataError::PredicateFailed); - } - let result = self.inner.commit_metadata(command); - if matches!(&result, Err(MetadataError::PredicateFailed)) { - self.predicate_failures.fetch_add(1, Ordering::SeqCst); - } - result + self.inner.commit_metadata(command) } fn commit_independent_batch( @@ -128,45 +82,36 @@ impl MetadataStore for SnapshotCommitBarrierStore { } } -impl MetadataStoreStatsProvider for SnapshotCommitBarrierStore { - fn metadata_store_stats(&self) -> MetadataStoreStats { - self.inner.metadata_store_stats() - } -} - -/// Metadata store wrapper that pauses one matching command only after Holt has -/// durably applied it. This deterministically exposes apply-vs-log ordering -/// races without relying on scheduler timing. #[derive(Clone)] -struct PostCommitBarrierStore { +struct RestoreBorrowerManifestReadFaultStore { inner: HoltMetadataStore, - request_id: Vec, - remaining: Arc, - applied: Arc, - release: Arc, + fail_key: Arc>>>, + failures: Arc, } -impl PostCommitBarrierStore { - fn new(request_id: &[u8]) -> Self { +impl RestoreBorrowerManifestReadFaultStore { + fn new() -> Self { Self { inner: HoltMetadataStore::open_memory().unwrap(), - request_id: request_id.to_vec(), - remaining: Arc::new(AtomicU64::new(1)), - applied: Arc::new(Barrier::new(2)), - release: Arc::new(Barrier::new(2)), + fail_key: Arc::new(Mutex::new(None)), + failures: Arc::new(AtomicU64::new(0)), } } - fn wait_until_applied(&self) { - self.applied.wait(); + fn fail_next_restore_manifest_read(&self, key: Vec) { + let replaced = self.fail_key.lock().unwrap().replace(key); + assert!( + replaced.is_none(), + "a restore manifest read fault is already armed" + ); } - fn release_after_apply(&self) { - self.release.wait(); + fn failures(&self) -> u64 { + self.failures.load(Ordering::SeqCst) } } -impl MetadataStore for PostCommitBarrierStore { +impl MetadataStore for RestoreBorrowerManifestReadFaultStore { fn get_versioned( &self, family: RecordFamily, @@ -174,6 +119,24 @@ impl MetadataStore for PostCommitBarrierStore { version: Version, purpose: ReadPurpose, ) -> Result, MetadataError> { + let should_fail = + if family == RecordFamily::ChunkManifest && purpose == ReadPurpose::RestoreStaging { + let mut fail_key = self.fail_key.lock().unwrap(); + if fail_key.as_deref() == Some(key) { + fail_key.take(); + true + } else { + false + } + } else { + false + }; + if should_fail { + self.failures.fetch_add(1, Ordering::SeqCst); + return Err(MetadataError::Backend( + "injected restore borrower manifest read failure".to_owned(), + )); + } self.inner.get_versioned(family, key, version, purpose) } @@ -182,19 +145,7 @@ impl MetadataStore for PostCommitBarrierStore { } fn commit_metadata(&self, command: MetadataCommand) -> Result { - let should_block = command.request_id == self.request_id - && self - .remaining - .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { - remaining.checked_sub(1) - }) - .is_ok(); - let result = self.inner.commit_metadata(command); - if should_block && result.is_ok() { - self.applied.wait(); - self.release.wait(); - } - result + self.inner.commit_metadata(command) } fn commit_independent_batch( @@ -223,84 +174,48 @@ impl MetadataStore for PostCommitBarrierStore { } } -impl MetadataCheckpointStore for PostCommitBarrierStore { - fn checkpoint(&self) -> Result<(), MetadataError> { - self.inner.checkpoint() - } - - fn export_checkpoint_image(&self) -> Result, MetadataError> { - self.inner.export_checkpoint_image() - } - - fn install_checkpoint_image(&self, image: &[u8]) -> Result<(), MetadataError> { - self.inner.install_checkpoint_image(image) - } - - fn reclaim_unreachable_storage(&self) -> Result { - self.inner.reclaim_unreachable_storage() - } -} - -impl MetadataStoreStatsProvider for PostCommitBarrierStore { - fn metadata_store_stats(&self) -> MetadataStoreStats { - self.inner.metadata_store_stats() - } -} - +/// Pause one non-empty exact-reference inverse scan after Holt has produced its +/// snapshot. A concurrent release can then attempt to delete that inverse and +/// its canonical owner, reproducing the scan/point-get TOCTOU seen by the live +/// crash matrix. #[derive(Clone)] -struct PostCommitErrorStore { +struct RestoreBaseInverseScanBarrierStore { inner: HoltMetadataStore, - kind: CommandKind, - remaining: Arc, - readback_failures: Arc, - readback_mismatches: Arc, - batch_backend_indices: Arc>>>, + inverse_prefix: Vec, + armed: Arc, + entered: Arc, + release: Arc, } -impl PostCommitErrorStore { - fn new(kind: CommandKind) -> Self { +impl RestoreBaseInverseScanBarrierStore { + fn new(mount: MountId) -> Self { + let inverse_prefix = restore::restore_control_keyspaces(mount) + .into_iter() + .find_map(|(name, prefix)| (name == "base_inverse").then_some(prefix)) + .expect("restore base inverse keyspace is registered"); Self { inner: HoltMetadataStore::open_memory().unwrap(), - kind, - remaining: Arc::new(AtomicU64::new(1)), - readback_failures: Arc::new(AtomicU64::new(0)), - readback_mismatches: Arc::new(AtomicU64::new(0)), - batch_backend_indices: Arc::new(Mutex::new(None)), + inverse_prefix, + armed: Arc::new(AtomicBool::new(false)), + entered: Arc::new(Barrier::new(2)), + release: Arc::new(Barrier::new(2)), } } - fn new_disarmed(kind: CommandKind) -> Self { - let store = Self::new(kind); - store.remaining.store(0, Ordering::SeqCst); - store - } - fn arm(&self) { - self.remaining.store(1, Ordering::SeqCst); - } - - fn fail_next_readbacks(&self, count: u64) { - self.readback_failures.store(count, Ordering::SeqCst); - } - - fn clear_readback_failures(&self) { - self.readback_failures.store(0, Ordering::SeqCst); - } - - fn mismatch_next_readbacks(&self, count: u64) { - self.readback_mismatches.store(count, Ordering::SeqCst); + self.armed.store(true, Ordering::SeqCst); } - fn clear_readback_mismatches(&self) { - self.readback_mismatches.store(0, Ordering::SeqCst); + fn wait_until_blocked(&self) { + self.entered.wait(); } - fn fail_next_batch_results(&self, indices: Vec) { - *self.batch_backend_indices.lock().unwrap() = Some(indices); + fn release_blocked(&self) { + self.release.wait(); } } -impl MetadataStore for PostCommitErrorStore { +impl MetadataStore for RestoreBaseInverseScanBarrierStore { fn get_versioned( &self, family: RecordFamily, @@ -312,74 +227,38 @@ impl MetadataStore for PostCommitErrorStore { } fn scan(&self, request: ScanRequest) -> Result, MetadataError> { - self.inner.scan(request) - } - - fn commit_metadata(&self, command: MetadataCommand) -> Result { - let kind = command.kind; - let result = self.inner.commit_metadata(command); - if kind == self.kind - && result.is_ok() + let matches = request.family == RecordFamily::System + && request.prefix.starts_with(&self.inverse_prefix); + let rows = self.inner.scan(request)?; + if matches + && !rows.is_empty() && self - .remaining - .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { - remaining.checked_sub(1) - }) + .armed + .compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst) .is_ok() { - return Err(MetadataError::Backend( - "injected journal acknowledgement failure".to_owned(), - )); + self.entered.wait(); + self.release.wait(); } - result + Ok(rows) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + self.inner.commit_metadata(command) } fn commit_independent_batch( &self, commands: &[MetadataCommand], ) -> Vec> { - let mut results = self.inner.commit_independent_batch(commands); - if let Some(indices) = self.batch_backend_indices.lock().unwrap().take() { - for index in indices { - if matches!(results.get(index), Some(Ok(_))) { - results[index] = Err(MetadataError::Backend( - "injected batch journal acknowledgement failure".to_owned(), - )); - } - } - } - results + self.inner.commit_independent_batch(commands) } fn committed_request_result( &self, request_id: &[u8], ) -> Result, MetadataError> { - if self - .readback_failures - .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { - remaining.checked_sub(1) - }) - .is_ok() - { - return Err(MetadataError::Backend( - "injected authoritative readback failure".to_owned(), - )); - } - let result = self.inner.committed_request_result(request_id)?; - if self - .readback_mismatches - .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { - remaining.checked_sub(1) - }) - .is_ok() - { - return Ok(result.map(|mut result| { - result.applied_mutations = result.applied_mutations.saturating_add(1); - result - })); - } - Ok(result) + self.inner.committed_request_result(request_id) } fn history_retention_epoch(&self) -> Result { @@ -394,46 +273,101 @@ impl MetadataStore for PostCommitErrorStore { } } -impl MetadataStoreStatsProvider for PostCommitErrorStore { +impl MetadataStoreStatsProvider for RestoreBaseInverseScanBarrierStore { fn metadata_store_stats(&self) -> MetadataStoreStats { self.inner.metadata_store_stats() } } -impl MetadataCheckpointStore for PostCommitErrorStore { - fn checkpoint(&self) -> Result<(), MetadataError> { - self.inner.checkpoint() - } +#[derive(Clone)] +struct RestorePostApplyFaultStore { + inner: M, + state: Arc>, +} - fn export_checkpoint_image(&self) -> Result, MetadataError> { - self.inner.export_checkpoint_image() - } +#[derive(Default)] +struct RestorePostApplyFaultState { + fault: Option, + fail_checkpoint_after_install: bool, + applied_by_request_id: BTreeMap, usize>, + applied_mutation_counts: Vec<(Vec, usize)>, +} - fn install_checkpoint_image(&self, image: &[u8]) -> Result<(), MetadataError> { - self.inner.install_checkpoint_image(image) - } - - fn reclaim_unreachable_storage(&self) -> Result { - self.inner.reclaim_unreachable_storage() - } +struct RestorePostApplyFault { + request_prefix: Vec, + remaining_matches: usize, } -#[derive(Clone)] -struct SnapshotPredicateOnceStore { - inner: HoltMetadataStore, - remaining_failures: Arc, +impl RestorePostApplyFaultStore { + fn new() -> Self { + Self::wrapping(HoltMetadataStore::open_memory().unwrap()) + } } -impl SnapshotPredicateOnceStore { - fn new() -> Self { +impl RestorePostApplyFaultStore { + fn wrapping(inner: M) -> Self { Self { - inner: HoltMetadataStore::open_memory().unwrap(), - remaining_failures: Arc::new(AtomicU64::new(1)), + inner, + state: Arc::new(Mutex::new(RestorePostApplyFaultState::default())), } } + + fn fail_after_apply(&self, request_prefix: &[u8], zero_based_match: usize) { + self.state.lock().unwrap().fault = Some(RestorePostApplyFault { + request_prefix: request_prefix.to_vec(), + remaining_matches: zero_based_match, + }); + } + + fn clear_fault(&self) { + self.state.lock().unwrap().fault = None; + } + + fn fail_checkpoint_after_install(&self) { + self.state.lock().unwrap().fail_checkpoint_after_install = true; + } + + fn applied_with_prefix(&self, request_prefix: &[u8]) -> usize { + self.state + .lock() + .unwrap() + .applied_by_request_id + .iter() + .filter_map(|(request_id, count)| { + request_id.starts_with(request_prefix).then_some(*count) + }) + .sum() + } + + fn applied_mutation_counts_with_prefix(&self, request_prefix: &[u8]) -> Vec { + self.state + .lock() + .unwrap() + .applied_mutation_counts + .iter() + .filter_map(|(request_id, count)| { + request_id.starts_with(request_prefix).then_some(*count) + }) + .collect() + } + + fn applied_request_counts_with_prefix(&self, request_prefix: &[u8]) -> Vec { + self.state + .lock() + .unwrap() + .applied_by_request_id + .iter() + .filter_map(|(request_id, count)| { + request_id.starts_with(request_prefix).then_some(*count) + }) + .collect() + } } -impl MetadataStore for SnapshotPredicateOnceStore { +impl MetadataStore for RestorePostApplyFaultStore +where + M: MetadataStore, +{ fn get_versioned( &self, family: RecordFamily, @@ -448,18 +382,47 @@ impl MetadataStore for SnapshotPredicateOnceStore { self.inner.scan(request) } + fn scan_delimited( + &self, + request: crate::command::DelimitedScanRequest, + ) -> Result, MetadataError> { + self.inner.scan_delimited(request) + } + fn commit_metadata(&self, command: MetadataCommand) -> Result { - if command.kind == CommandKind::SnapshotSubtree - && self - .remaining_failures - .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { - remaining.checked_sub(1) - }) - .is_ok() - { - return Err(MetadataError::PredicateFailed); + let request_id = command.request_id.clone(); + let mutation_count = command.mutations.len(); + let result = self.inner.commit_metadata(command); + if result.is_ok() { + let should_fail = { + let mut state = self.state.lock().unwrap(); + *state + .applied_by_request_id + .entry(request_id.clone()) + .or_default() += 1; + state + .applied_mutation_counts + .push((request_id.clone(), mutation_count)); + match state.fault.as_mut() { + Some(fault) if request_id.starts_with(&fault.request_prefix) => { + if fault.remaining_matches == 0 { + state.fault = None; + true + } else { + fault.remaining_matches -= 1; + false + } + } + _ => false, + } + }; + if should_fail { + return Err(MetadataError::Backend( + "injected restore crash after durable apply".to_owned(), + )); + } } - self.inner.commit_metadata(command) + result } fn commit_independent_batch( @@ -488,77 +451,97 @@ impl MetadataStore for SnapshotPredicateOnceStore { } } -impl MetadataStoreStatsProvider for SnapshotPredicateOnceStore { - fn metadata_store_stats(&self) -> MetadataStoreStats { - self.inner.metadata_store_stats() +impl MetadataCheckpointStore for RestorePostApplyFaultStore +where + M: MetadataCheckpointStore, +{ + fn checkpoint(&self) -> Result<(), MetadataError> { + self.inner.checkpoint() } -} -#[derive(Clone)] -struct PurposeTrackingStore { - inner: HoltMetadataStore, - counts: Arc, + fn export_checkpoint_image(&self) -> Result, MetadataError> { + self.inner.export_checkpoint_image() + } + + fn install_checkpoint_image(&self, image: &[u8]) -> Result<(), MetadataError> { + self.inner.install_checkpoint_image(image)?; + if std::mem::take(&mut self.state.lock().unwrap().fail_checkpoint_after_install) { + return Err(MetadataError::Backend( + "injected checkpoint install error after apply".to_owned(), + )); + } + Ok(()) + } + + fn reclaim_unreachable_storage(&self) -> Result { + self.inner.reclaim_unreachable_storage() + } } -#[derive(Default)] -struct PurposeCounts { - user_strong_gets: AtomicU64, - write_plan_gets: AtomicU64, - snapshot_gets: AtomicU64, - user_strong_scans: AtomicU64, - write_plan_scans: AtomicU64, - snapshot_scans: AtomicU64, +impl MetadataStoreStatsProvider for RestorePostApplyFaultStore +where + M: MetadataStoreStatsProvider, +{ + fn metadata_store_stats(&self) -> MetadataStoreStats { + self.inner.metadata_store_stats() + } } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct PurposeCountSnapshot { - user_strong_gets: u64, - write_plan_gets: u64, - snapshot_gets: u64, - user_strong_scans: u64, - write_plan_scans: u64, - snapshot_scans: u64, +/// Holt intentionally caps one value at 64 KiB, while the MetadataStore +/// contract permits other backends to expose larger manifest, restore-control, +/// and object-GC values. Keep those test-only values out-of-line while +/// delegating physical keys and versions to Holt so release CAS, ACK +/// reconciliation, and reopen exercise the real service state machine. +#[derive(Clone)] +struct OversizedManifestStore { + inner: RestorePostApplyFaultStore, + manifests: Arc, ScanItem>>>, + system_rows: Arc, ScanItem>>>, + gc_rows: Arc, ScanItem>>>, } -impl PurposeTrackingStore { +impl OversizedManifestStore { fn new() -> Self { Self { - inner: HoltMetadataStore::open_memory().unwrap(), - counts: Arc::new(PurposeCounts::default()), + inner: RestorePostApplyFaultStore::new(), + manifests: Arc::new(Mutex::new(BTreeMap::new())), + system_rows: Arc::new(Mutex::new(BTreeMap::new())), + gc_rows: Arc::new(Mutex::new(BTreeMap::new())), } } - fn counts(&self) -> PurposeCountSnapshot { - PurposeCountSnapshot { - user_strong_gets: self.counts.user_strong_gets.load(Ordering::Relaxed), - write_plan_gets: self.counts.write_plan_gets.load(Ordering::Relaxed), - snapshot_gets: self.counts.snapshot_gets.load(Ordering::Relaxed), - user_strong_scans: self.counts.user_strong_scans.load(Ordering::Relaxed), - write_plan_scans: self.counts.write_plan_scans.load(Ordering::Relaxed), - snapshot_scans: self.counts.snapshot_scans.load(Ordering::Relaxed), - } + fn install_manifest(&self, item: ScanItem) { + self.manifests + .lock() + .unwrap() + .insert(item.key.clone(), item); } - fn record_get(&self, purpose: ReadPurpose) { - match purpose { - ReadPurpose::UserStrong => &self.counts.user_strong_gets, - ReadPurpose::WritePlanLocal => &self.counts.write_plan_gets, - ReadPurpose::Snapshot => &self.counts.snapshot_gets, - } - .fetch_add(1, Ordering::Relaxed); + fn fail_after_apply(&self, request_prefix: &[u8], zero_based_match: usize) { + self.inner + .fail_after_apply(request_prefix, zero_based_match); } - fn record_scan(&self, purpose: ReadPurpose) { - match purpose { - ReadPurpose::UserStrong => &self.counts.user_strong_scans, - ReadPurpose::WritePlanLocal => &self.counts.write_plan_scans, - ReadPurpose::Snapshot => &self.counts.snapshot_scans, - } - .fetch_add(1, Ordering::Relaxed); + fn clear_fault(&self) { + self.inner.clear_fault(); + } + + fn applied_with_prefix(&self, request_prefix: &[u8]) -> usize { + self.inner.applied_with_prefix(request_prefix) + } + + fn applied_mutation_counts_with_prefix(&self, request_prefix: &[u8]) -> Vec { + self.inner + .applied_mutation_counts_with_prefix(request_prefix) + } + + fn applied_request_counts_with_prefix(&self, request_prefix: &[u8]) -> Vec { + self.inner + .applied_request_counts_with_prefix(request_prefix) } } -impl MetadataStore for PurposeTrackingStore { +impl MetadataStore for OversizedManifestStore { fn get_versioned( &self, family: RecordFamily, @@ -566,25 +549,140 @@ impl MetadataStore for PurposeTrackingStore { version: Version, purpose: ReadPurpose, ) -> Result, MetadataError> { - self.record_get(purpose); + if family == RecordFamily::ChunkManifest { + if let Some(item) = self.manifests.lock().unwrap().get(key) { + if item.version <= version { + return Ok(Some(ReadItem { + value: item.value.clone(), + version: item.version, + })); + } + } + } else if family == RecordFamily::System { + if let Some(item) = self.system_rows.lock().unwrap().get(key) { + if item.version <= version { + return Ok(Some(ReadItem { + value: item.value.clone(), + version: item.version, + })); + } + } + } else if family == RecordFamily::Gc { + if let Some(item) = self.gc_rows.lock().unwrap().get(key) { + if item.version <= version { + return Ok(Some(ReadItem { + value: item.value.clone(), + version: item.version, + })); + } + } + } self.inner.get_versioned(family, key, version, purpose) } fn scan(&self, request: ScanRequest) -> Result, MetadataError> { - self.record_scan(request.purpose); - self.inner.scan(request) - } - - fn scan_delimited( - &self, - request: crate::command::DelimitedScanRequest, - ) -> Result, MetadataError> { - self.record_scan(request.purpose); - self.inner.scan_delimited(request) + let mut rows = self.inner.scan(request.clone())?; + if request.family == RecordFamily::ChunkManifest { + for manifest in self.manifests.lock().unwrap().values() { + if manifest.version <= request.version { + if let Some(row) = rows.iter_mut().find(|row| row.key == manifest.key) { + *row = manifest.clone(); + } + } + } + } else if request.family == RecordFamily::System { + for item in self.system_rows.lock().unwrap().values() { + if item.version <= request.version { + if let Some(row) = rows.iter_mut().find(|row| row.key == item.key) { + *row = item.clone(); + } + } + } + } else if request.family == RecordFamily::Gc { + for item in self.gc_rows.lock().unwrap().values() { + if item.version <= request.version { + if let Some(row) = rows.iter_mut().find(|row| row.key == item.key) { + *row = item.clone(); + } + } + } + } + Ok(rows) } - fn commit_metadata(&self, command: MetadataCommand) -> Result { - self.inner.commit_metadata(command) + fn commit_metadata(&self, mut command: MetadataCommand) -> Result { + let request_id = command.request_id.clone(); + let mut oversized_manifests = Vec::new(); + let mut oversized_system_rows = Vec::new(); + let mut oversized_gc_rows = Vec::new(); + let mut deleted = Vec::new(); + for mutation in &mut command.mutations { + if !matches!( + mutation.family, + RecordFamily::ChunkManifest | RecordFamily::System | RecordFamily::Gc + ) { + continue; + } + match mutation.op { + MutationOp::Put + if mutation + .value + .as_ref() + .is_some_and(|value| value.0.len() > u16::MAX as usize) => + { + let value = mutation + .value + .replace(Value(Vec::new())) + .expect("oversized Put has a value"); + let item = ScanItem { + key: mutation.key.clone(), + value, + version: command.commit_version, + }; + if mutation.family == RecordFamily::ChunkManifest { + oversized_manifests.push(item); + } else if mutation.family == RecordFamily::System { + oversized_system_rows.push(item); + } else { + oversized_gc_rows.push(item); + } + } + MutationOp::Delete => deleted.push((mutation.family, mutation.key.clone())), + MutationOp::Put => {} + } + } + let result = self.inner.commit_metadata(command); + let applied = result.is_ok() + || self + .inner + .committed_request_result(&request_id) + .ok() + .flatten() + .is_some(); + if applied { + let mut manifests = self.manifests.lock().unwrap(); + let mut system_rows = self.system_rows.lock().unwrap(); + let mut gc_rows = self.gc_rows.lock().unwrap(); + for (family, key) in deleted { + if family == RecordFamily::ChunkManifest { + manifests.remove(&key); + } else if family == RecordFamily::System { + system_rows.remove(&key); + } else { + gc_rows.remove(&key); + } + } + for item in oversized_manifests { + manifests.insert(item.key.clone(), item); + } + for item in oversized_system_rows { + system_rows.insert(item.key.clone(), item); + } + for item in oversized_gc_rows { + gc_rows.insert(item.key.clone(), item); + } + } + result } fn commit_independent_batch( @@ -613,81 +711,67 @@ impl MetadataStore for PurposeTrackingStore { } } -impl MetadataStoreStatsProvider for PurposeTrackingStore { +impl MetadataCheckpointStore for OversizedManifestStore { + fn checkpoint(&self) -> Result<(), MetadataError> { + self.inner.checkpoint() + } + + fn export_checkpoint_image(&self) -> Result, MetadataError> { + self.inner.export_checkpoint_image() + } + + fn install_checkpoint_image(&self, image: &[u8]) -> Result<(), MetadataError> { + self.inner.install_checkpoint_image(image) + } + + fn reclaim_unreachable_storage(&self) -> Result { + self.inner.reclaim_unreachable_storage() + } +} + +impl MetadataStoreStatsProvider for OversizedManifestStore { fn metadata_store_stats(&self) -> MetadataStoreStats { self.inner.metadata_store_stats() } } +/// Keep one PathIndex row out-of-line so preflight can cover metadata backends +/// whose value limit is larger than Holt's 64 KiB test backend. #[derive(Clone)] -struct PausingObjectGcStore { - inner: HoltMetadataStore, - gate: Arc<(Mutex, Condvar)>, -} - -#[derive(Default)] -struct PausingObjectGcState { - armed: bool, - reached: bool, - released: bool, +struct OversizedPathIndexStore { + inner: RestorePostApplyFaultStore, + rows: Arc>>, } -impl PausingObjectGcStore { +impl OversizedPathIndexStore { fn new() -> Self { Self { - inner: HoltMetadataStore::open_memory().unwrap(), - gate: Arc::new((Mutex::new(PausingObjectGcState::default()), Condvar::new())), + inner: RestorePostApplyFaultStore::new(), + rows: Arc::new(Mutex::new(Vec::new())), } } - fn arm(&self) { - let (lock, _) = &*self.gate; - *lock.lock().unwrap() = PausingObjectGcState { - armed: true, - reached: false, - released: false, - }; + fn install_row(&self, row: ScanItem) { + self.install_override(RecordFamily::PathIndex, row); } - fn wait_until_reached(&self) { - let (lock, changed) = &*self.gate; - let mut state = lock.lock().unwrap(); - let deadline = Instant::now() + Duration::from_secs(10); - while !state.reached { - let now = Instant::now(); - assert!(now < deadline, "timed out waiting for durable GC claim"); - let (next, timed_out) = changed.wait_timeout(state, deadline - now).unwrap(); - state = next; - assert!( - !timed_out.timed_out() || state.reached, - "timed out waiting for durable GC claim" - ); + fn install_override(&self, family: RecordFamily, row: ScanItem) { + let mut rows = self.rows.lock().unwrap(); + if let Some((_, existing)) = rows.iter_mut().find(|(candidate_family, candidate)| { + *candidate_family == family && candidate.key == row.key + }) { + *existing = row; + } else { + rows.push((family, row)); } } - fn release(&self) { - let (lock, changed) = &*self.gate; - let mut state = lock.lock().unwrap(); - state.released = true; - changed.notify_all(); - } - - fn pause_after_deleting_claim(&self) { - let (lock, changed) = &*self.gate; - let mut state = lock.lock().unwrap(); - if !state.armed { - return; - } - state.armed = false; - state.reached = true; - changed.notify_all(); - while !state.released { - state = changed.wait(state).unwrap(); - } + fn applied_with_prefix(&self, request_prefix: &[u8]) -> usize { + self.inner.applied_with_prefix(request_prefix) } } -impl MetadataStore for PausingObjectGcStore { +impl MetadataStore for OversizedPathIndexStore { fn get_versioned( &self, family: RecordFamily, @@ -695,29 +779,37 @@ impl MetadataStore for PausingObjectGcStore { version: Version, purpose: ReadPurpose, ) -> Result, MetadataError> { - self.inner.get_versioned(family, key, version, purpose) - } + if let Some((_, row)) = self + .rows + .lock() + .unwrap() + .iter() + .find(|(candidate_family, row)| { + *candidate_family == family && row.key == key && row.version <= version + }) + { + return Ok(Some(ReadItem { + value: row.value.clone(), + version: row.version, + })); + } + self.inner.get_versioned(family, key, version, purpose) + } fn scan(&self, request: ScanRequest) -> Result, MetadataError> { - self.inner.scan(request) + let mut rows = self.inner.scan(request.clone())?; + for (family, oversized) in self.rows.lock().unwrap().iter() { + if *family == request.family && oversized.version <= request.version { + if let Some(row) = rows.iter_mut().find(|row| row.key == oversized.key) { + *row = oversized.clone(); + } + } + } + Ok(rows) } fn commit_metadata(&self, command: MetadataCommand) -> Result { - let deleting_claim = command.primary_family == RecordFamily::System - && command.primary_key == object_gc_claim_key(MountId::new(1).unwrap()) - && command.mutations.iter().any(|mutation| { - mutation.family == RecordFamily::System - && mutation.key == command.primary_key - && mutation - .value - .as_ref() - .is_some_and(|value| value.0.first() == Some(&2)) - }); - let result = self.inner.commit_metadata(command); - if deleting_claim && result.is_ok() { - self.pause_after_deleting_claim(); - } - result + self.inner.commit_metadata(command) } fn commit_independent_batch( @@ -746,11190 +838,22586 @@ impl MetadataStore for PausingObjectGcStore { } } -/// Simulates a remote DELETE that takes effect but loses its acknowledgement. -/// The retry observes the object as missing, matching an idempotent S3 DELETE. +impl MetadataStoreStatsProvider for OversizedPathIndexStore { + fn metadata_store_stats(&self) -> MetadataStoreStats { + self.inner.metadata_store_stats() + } +} + #[derive(Clone)] -struct DeleteAckLostObjectStore { - inner: MemoryObjectStore, - lose_next_delete_ack: Arc, - delete_calls: Arc, +struct SnapshotCommitBarrierStore { + inner: HoltMetadataStore, + kind: CommandKind, + request_prefix: Option>, + rejected_kind: Option, + remaining: Arc, + predicate_failures: Arc, + entered: Arc, + release: Arc, } -impl DeleteAckLostObjectStore { - fn new(inner: MemoryObjectStore) -> Self { +impl SnapshotCommitBarrierStore { + fn new(kind: CommandKind, blocked_commits: u64, parties: usize) -> Self { Self { - inner, - lose_next_delete_ack: Arc::new(AtomicBool::new(true)), - delete_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + inner: HoltMetadataStore::open_memory().unwrap(), + kind, + request_prefix: None, + rejected_kind: None, + remaining: Arc::new(AtomicU64::new(blocked_commits)), + predicate_failures: Arc::new(AtomicU64::new(0)), + entered: Arc::new(Barrier::new(parties)), + release: Arc::new(Barrier::new(parties)), } } - fn delete_calls(&self) -> usize { - self.delete_calls.load(Ordering::SeqCst) + fn wait_until_blocked(&self) { + self.entered.wait(); + } + + fn arm(&self, blocked_commits: u64) { + self.remaining.store(blocked_commits, Ordering::SeqCst); + } + + fn predicate_failures(&self) -> u64 { + self.predicate_failures.load(Ordering::SeqCst) + } + + fn rejecting(mut self, kind: CommandKind) -> Self { + self.rejected_kind = Some(kind); + self + } + + fn matching_request_prefix(mut self, prefix: &[u8]) -> Self { + self.request_prefix = Some(prefix.to_vec()); + self + } + + fn release_blocked(&self) { + self.release.wait(); } } -impl ObjectStore for DeleteAckLostObjectStore { - fn put( +impl MetadataStore for SnapshotCommitBarrierStore { + fn get_versioned( &self, - key: &ObjectKey, - bytes: impl Into, - ) -> Result { - self.inner.put(key, bytes) + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + self.inner.get_versioned(family, key, version, purpose) } - fn get( + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + self.inner.scan(request) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + if command.kind == self.kind + && self + .request_prefix + .as_ref() + .is_none_or(|prefix| command.request_id.starts_with(prefix)) + && self + .remaining + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + self.entered.wait(); + self.release.wait(); + } + if self.rejected_kind == Some(command.kind) + && self + .request_prefix + .as_ref() + .is_none_or(|prefix| command.request_id.starts_with(prefix)) + { + return Err(MetadataError::PredicateFailed); + } + let result = self.inner.commit_metadata(command); + if matches!(&result, Err(MetadataError::PredicateFailed)) { + self.predicate_failures.fetch_add(1, Ordering::SeqCst); + } + result + } + + fn commit_independent_batch( &self, - key: &ObjectKey, - range: Option, - ) -> Result, ObjectError> { - self.inner.get(key, range) + commands: &[MetadataCommand], + ) -> Vec> { + self.inner.commit_independent_batch(commands) } - fn head(&self, key: &ObjectKey) -> Result, ObjectError> { - self.inner.head(key) + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + self.inner.committed_request_result(request_id) } - fn delete(&self, key: &ObjectKey) -> Result { - self.delete_calls.fetch_add(1, Ordering::SeqCst); - let deleted = self.inner.delete(key)?; - if self.lose_next_delete_ack.swap(false, Ordering::SeqCst) { - return Err(ObjectError::Backend( - "injected lost DELETE acknowledgement".to_owned(), - )); - } - Ok(deleted) + fn history_retention_epoch(&self) -> Result { + self.inner.history_retention_epoch() } -} -fn service() -> NoKvFs { - service_with_objects().0 + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } } -fn service_with_objects() -> ( - NoKvFs, - MemoryObjectStore, -) { - let objects = MemoryObjectStore::new(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - (service, objects) +impl MetadataStoreStatsProvider for SnapshotCommitBarrierStore { + fn metadata_store_stats(&self) -> MetadataStoreStats { + self.inner.metadata_store_stats() + } } -fn enqueue_gc_candidate(service: &NoKvFs, mut record: ObjectGcRecord) -> Vec -where - M: MetadataStore, - O: ObjectStore, -{ - let version = service.next_version().unwrap(); - record.enqueue_version = version.get(); - record.enqueue_unix_ms = service.now_ms(); - let key = gc_object_key( - service.mount, - version.get(), - record.inode, - record.generation, - 0, - 0, - ); - service - .commit_metadata(MetadataCommand { - request_id: request_id( - b"test-enqueue-gc-candidate", - service.mount, - record.inode, - version, - ), - kind: CommandKind::CleanupObjects, - read_version: predecessor(version).unwrap(), - commit_version: version, - primary_family: RecordFamily::Gc, - primary_key: key.clone(), - predicates: vec![PredicateRef { - family: RecordFamily::Gc, - key: key.clone(), - predicate: Predicate::NotExists, - }], - mutations: vec![Mutation { - family: RecordFamily::Gc, - key: key.clone(), - op: MutationOp::Put, - value: Some(Value(encode_object_gc_record(&record))), - }], - watch: Vec::new(), - }) - .unwrap(); - key +/// Metadata store wrapper that pauses one matching command only after Holt has +/// durably applied it. This deterministically exposes apply-vs-log ordering +/// races without relying on scheduler timing. +#[derive(Clone)] +struct PostCommitBarrierStore { + inner: HoltMetadataStore, + request_id: Vec, + remaining: Arc, + applied: Arc, + release: Arc, } -fn leave_object_gc_deleting(service: &NoKvFs, gc_row: &ScanItem) -> Vec -where - M: MetadataStore, - O: ObjectStore, -{ - let claim_key = object_gc_claim_key(service.mount); - let claim = service - .metadata - .get_versioned( - RecordFamily::System, - &claim_key, - service.read_version().unwrap(), +impl PostCommitBarrierStore { + fn new(request_id: &[u8]) -> Self { + Self { + inner: HoltMetadataStore::open_memory().unwrap(), + request_id: request_id.to_vec(), + remaining: Arc::new(AtomicU64::new(1)), + applied: Arc::new(Barrier::new(2)), + release: Arc::new(Barrier::new(2)), + } + } + + fn wait_until_applied(&self) { + self.applied.wait(); + } + + fn release_after_apply(&self) { + self.release.wait(); + } +} + +impl MetadataStore for PostCommitBarrierStore { + fn get_versioned( + &self, + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + self.inner.get_versioned(family, key, version, purpose) + } + + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + self.inner.scan(request) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + let should_block = command.request_id == self.request_id + && self + .remaining + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok(); + let result = self.inner.commit_metadata(command); + if should_block && result.is_ok() { + self.applied.wait(); + self.release.wait(); + } + result + } + + fn commit_independent_batch( + &self, + commands: &[MetadataCommand], + ) -> Vec> { + self.inner.commit_independent_batch(commands) + } + + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + self.inner.committed_request_result(request_id) + } + + fn history_retention_epoch(&self) -> Result { + self.inner.history_retention_epoch() + } + + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } +} + +impl MetadataCheckpointStore for PostCommitBarrierStore { + fn checkpoint(&self) -> Result<(), MetadataError> { + self.inner.checkpoint() + } + + fn export_checkpoint_image(&self) -> Result, MetadataError> { + self.inner.export_checkpoint_image() + } + + fn install_checkpoint_image(&self, image: &[u8]) -> Result<(), MetadataError> { + self.inner.install_checkpoint_image(image) + } + + fn reclaim_unreachable_storage(&self) -> Result { + self.inner.reclaim_unreachable_storage() + } +} + +impl MetadataStoreStatsProvider for PostCommitBarrierStore { + fn metadata_store_stats(&self) -> MetadataStoreStats { + self.inner.metadata_store_stats() + } +} + +#[derive(Clone)] +struct PostCommitErrorStore { + inner: HoltMetadataStore, + kind: CommandKind, + remaining: Arc, + readback_failures: Arc, + readback_mismatches: Arc, + batch_backend_indices: Arc>>>, +} + +impl PostCommitErrorStore { + fn new(kind: CommandKind) -> Self { + Self { + inner: HoltMetadataStore::open_memory().unwrap(), + kind, + remaining: Arc::new(AtomicU64::new(1)), + readback_failures: Arc::new(AtomicU64::new(0)), + readback_mismatches: Arc::new(AtomicU64::new(0)), + batch_backend_indices: Arc::new(Mutex::new(None)), + } + } + + fn new_disarmed(kind: CommandKind) -> Self { + let store = Self::new(kind); + store.remaining.store(0, Ordering::SeqCst); + store + } + + fn arm(&self) { + self.remaining.store(1, Ordering::SeqCst); + } + + fn fail_next_readbacks(&self, count: u64) { + self.readback_failures.store(count, Ordering::SeqCst); + } + + fn clear_readback_failures(&self) { + self.readback_failures.store(0, Ordering::SeqCst); + } + + fn mismatch_next_readbacks(&self, count: u64) { + self.readback_mismatches.store(count, Ordering::SeqCst); + } + + fn clear_readback_mismatches(&self) { + self.readback_mismatches.store(0, Ordering::SeqCst); + } + + fn fail_next_batch_results(&self, indices: Vec) { + *self.batch_backend_indices.lock().unwrap() = Some(indices); + } +} + +impl MetadataStore for PostCommitErrorStore { + fn get_versioned( + &self, + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + self.inner.get_versioned(family, key, version, purpose) + } + + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + self.inner.scan(request) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + let kind = command.kind; + let result = self.inner.commit_metadata(command); + if kind == self.kind + && result.is_ok() + && self + .remaining + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return Err(MetadataError::Backend( + "injected journal acknowledgement failure".to_owned(), + )); + } + result + } + + fn commit_independent_batch( + &self, + commands: &[MetadataCommand], + ) -> Vec> { + let mut results = self.inner.commit_independent_batch(commands); + if let Some(indices) = self.batch_backend_indices.lock().unwrap().take() { + for index in indices { + if matches!(results.get(index), Some(Ok(_))) { + results[index] = Err(MetadataError::Backend( + "injected batch journal acknowledgement failure".to_owned(), + )); + } + } + } + results + } + + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + if self + .readback_failures + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return Err(MetadataError::Backend( + "injected authoritative readback failure".to_owned(), + )); + } + let result = self.inner.committed_request_result(request_id)?; + if self + .readback_mismatches + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return Ok(result.map(|mut result| { + result.applied_mutations = result.applied_mutations.saturating_add(1); + result + })); + } + Ok(result) + } + + fn history_retention_epoch(&self) -> Result { + self.inner.history_retention_epoch() + } + + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } +} + +impl MetadataStoreStatsProvider for PostCommitErrorStore { + fn metadata_store_stats(&self) -> MetadataStoreStats { + self.inner.metadata_store_stats() + } +} + +impl MetadataCheckpointStore for PostCommitErrorStore { + fn checkpoint(&self) -> Result<(), MetadataError> { + self.inner.checkpoint() + } + + fn export_checkpoint_image(&self) -> Result, MetadataError> { + self.inner.export_checkpoint_image() + } + + fn install_checkpoint_image(&self, image: &[u8]) -> Result<(), MetadataError> { + self.inner.install_checkpoint_image(image) + } + + fn reclaim_unreachable_storage(&self) -> Result { + self.inner.reclaim_unreachable_storage() + } +} + +#[derive(Clone)] +struct SnapshotPredicateOnceStore { + inner: HoltMetadataStore, + remaining_failures: Arc, +} + +impl SnapshotPredicateOnceStore { + fn new() -> Self { + Self { + inner: HoltMetadataStore::open_memory().unwrap(), + remaining_failures: Arc::new(AtomicU64::new(1)), + } + } +} + +impl MetadataStore for SnapshotPredicateOnceStore { + fn get_versioned( + &self, + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + self.inner.get_versioned(family, key, version, purpose) + } + + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + self.inner.scan(request) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + if command.kind == CommandKind::SnapshotSubtree + && self + .remaining_failures + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return Err(MetadataError::PredicateFailed); + } + self.inner.commit_metadata(command) + } + + fn commit_independent_batch( + &self, + commands: &[MetadataCommand], + ) -> Vec> { + self.inner.commit_independent_batch(commands) + } + + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + self.inner.committed_request_result(request_id) + } + + fn history_retention_epoch(&self) -> Result { + self.inner.history_retention_epoch() + } + + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } +} + +impl MetadataStoreStatsProvider for SnapshotPredicateOnceStore { + fn metadata_store_stats(&self) -> MetadataStoreStats { + self.inner.metadata_store_stats() + } +} + +#[derive(Clone)] +struct PurposeTrackingStore { + inner: HoltMetadataStore, + counts: Arc, +} + +#[derive(Default)] +struct PurposeCounts { + user_strong_gets: AtomicU64, + write_plan_gets: AtomicU64, + snapshot_gets: AtomicU64, + user_strong_scans: AtomicU64, + write_plan_scans: AtomicU64, + snapshot_scans: AtomicU64, + dentry_scans: AtomicU64, + unbounded_scans: AtomicU64, + restore_index_unbounded_scans: AtomicU64, + largest_bounded_scan: AtomicU64, + largest_restore_index_scan: AtomicU64, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct PurposeCountSnapshot { + user_strong_gets: u64, + write_plan_gets: u64, + snapshot_gets: u64, + user_strong_scans: u64, + write_plan_scans: u64, + snapshot_scans: u64, + dentry_scans: u64, + unbounded_scans: u64, + restore_index_unbounded_scans: u64, + largest_bounded_scan: u64, + largest_restore_index_scan: u64, +} + +impl PurposeTrackingStore { + fn new() -> Self { + Self { + inner: HoltMetadataStore::open_memory().unwrap(), + counts: Arc::new(PurposeCounts::default()), + } + } + + fn counts(&self) -> PurposeCountSnapshot { + PurposeCountSnapshot { + user_strong_gets: self.counts.user_strong_gets.load(Ordering::Relaxed), + write_plan_gets: self.counts.write_plan_gets.load(Ordering::Relaxed), + snapshot_gets: self.counts.snapshot_gets.load(Ordering::Relaxed), + user_strong_scans: self.counts.user_strong_scans.load(Ordering::Relaxed), + write_plan_scans: self.counts.write_plan_scans.load(Ordering::Relaxed), + snapshot_scans: self.counts.snapshot_scans.load(Ordering::Relaxed), + dentry_scans: self.counts.dentry_scans.load(Ordering::Relaxed), + unbounded_scans: self.counts.unbounded_scans.load(Ordering::Relaxed), + restore_index_unbounded_scans: self + .counts + .restore_index_unbounded_scans + .load(Ordering::Relaxed), + largest_bounded_scan: self.counts.largest_bounded_scan.load(Ordering::Relaxed), + largest_restore_index_scan: self + .counts + .largest_restore_index_scan + .load(Ordering::Relaxed), + } + } + + fn reset_scan_bounds(&self) { + self.counts.unbounded_scans.store(0, Ordering::Relaxed); + self.counts + .restore_index_unbounded_scans + .store(0, Ordering::Relaxed); + self.counts.largest_bounded_scan.store(0, Ordering::Relaxed); + self.counts + .largest_restore_index_scan + .store(0, Ordering::Relaxed); + } + + fn record_get(&self, purpose: ReadPurpose) { + match purpose { + ReadPurpose::UserStrong => &self.counts.user_strong_gets, + ReadPurpose::WritePlanLocal | ReadPurpose::RestoreStaging => { + &self.counts.write_plan_gets + } + ReadPurpose::Snapshot => &self.counts.snapshot_gets, + } + .fetch_add(1, Ordering::Relaxed); + } + + fn record_scan(&self, family: RecordFamily, purpose: ReadPurpose, limit: usize) { + match purpose { + ReadPurpose::UserStrong => &self.counts.user_strong_scans, + ReadPurpose::WritePlanLocal | ReadPurpose::RestoreStaging => { + &self.counts.write_plan_scans + } + ReadPurpose::Snapshot => &self.counts.snapshot_scans, + } + .fetch_add(1, Ordering::Relaxed); + if family == RecordFamily::Dentry { + self.counts.dentry_scans.fetch_add(1, Ordering::Relaxed); + } + if limit == 0 { + self.counts.unbounded_scans.fetch_add(1, Ordering::Relaxed); + if matches!( + family, + RecordFamily::System | RecordFamily::PathIndex | RecordFamily::Dentry + ) { + self.counts + .restore_index_unbounded_scans + .fetch_add(1, Ordering::Relaxed); + } + } else { + self.counts + .largest_bounded_scan + .fetch_max(limit as u64, Ordering::Relaxed); + if matches!( + family, + RecordFamily::System | RecordFamily::PathIndex | RecordFamily::Dentry + ) { + self.counts + .largest_restore_index_scan + .fetch_max(limit as u64, Ordering::Relaxed); + } + } + } +} + +impl MetadataStore for PurposeTrackingStore { + fn get_versioned( + &self, + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + self.record_get(purpose); + self.inner.get_versioned(family, key, version, purpose) + } + + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + self.record_scan(request.family, request.purpose, request.limit); + self.inner.scan(request) + } + + fn scan_delimited( + &self, + request: crate::command::DelimitedScanRequest, + ) -> Result, MetadataError> { + self.record_scan(request.family, request.purpose, request.limit); + self.inner.scan_delimited(request) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + self.inner.commit_metadata(command) + } + + fn commit_independent_batch( + &self, + commands: &[MetadataCommand], + ) -> Vec> { + self.inner.commit_independent_batch(commands) + } + + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + self.inner.committed_request_result(request_id) + } + + fn history_retention_epoch(&self) -> Result { + self.inner.history_retention_epoch() + } + + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } +} + +impl MetadataStoreStatsProvider for PurposeTrackingStore { + fn metadata_store_stats(&self) -> MetadataStoreStats { + self.inner.metadata_store_stats() + } +} + +#[derive(Clone)] +struct PausingObjectGcStore { + inner: HoltMetadataStore, + gate: Arc<(Mutex, Condvar)>, +} + +#[derive(Default)] +struct PausingObjectGcState { + armed: bool, + reached: bool, + released: bool, +} + +impl PausingObjectGcStore { + fn new() -> Self { + Self { + inner: HoltMetadataStore::open_memory().unwrap(), + gate: Arc::new((Mutex::new(PausingObjectGcState::default()), Condvar::new())), + } + } + + fn arm(&self) { + let (lock, _) = &*self.gate; + *lock.lock().unwrap() = PausingObjectGcState { + armed: true, + reached: false, + released: false, + }; + } + + fn wait_until_reached(&self) { + let (lock, changed) = &*self.gate; + let mut state = lock.lock().unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while !state.reached { + let now = Instant::now(); + assert!(now < deadline, "timed out waiting for durable GC claim"); + let (next, timed_out) = changed.wait_timeout(state, deadline - now).unwrap(); + state = next; + assert!( + !timed_out.timed_out() || state.reached, + "timed out waiting for durable GC claim" + ); + } + } + + fn release(&self) { + let (lock, changed) = &*self.gate; + let mut state = lock.lock().unwrap(); + state.released = true; + changed.notify_all(); + } + + fn pause_after_deleting_claim(&self) { + let (lock, changed) = &*self.gate; + let mut state = lock.lock().unwrap(); + if !state.armed { + return; + } + state.armed = false; + state.reached = true; + changed.notify_all(); + while !state.released { + state = changed.wait(state).unwrap(); + } + } +} + +impl MetadataStore for PausingObjectGcStore { + fn get_versioned( + &self, + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + self.inner.get_versioned(family, key, version, purpose) + } + + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + self.inner.scan(request) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + let deleting_claim = command.primary_family == RecordFamily::System + && command.primary_key == object_gc_claim_key(MountId::new(1).unwrap()) + && command.mutations.iter().any(|mutation| { + mutation.family == RecordFamily::System + && mutation.key == command.primary_key + && mutation + .value + .as_ref() + .is_some_and(|value| value.0.first() == Some(&2)) + }); + let result = self.inner.commit_metadata(command); + if deleting_claim && result.is_ok() { + self.pause_after_deleting_claim(); + } + result + } + + fn commit_independent_batch( + &self, + commands: &[MetadataCommand], + ) -> Vec> { + self.inner.commit_independent_batch(commands) + } + + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + self.inner.committed_request_result(request_id) + } + + fn history_retention_epoch(&self) -> Result { + self.inner.history_retention_epoch() + } + + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } +} + +/// Simulates a remote DELETE that takes effect but loses its acknowledgement. +/// The retry observes the object as missing, matching an idempotent S3 DELETE. +#[derive(Clone)] +struct DeleteAckLostObjectStore { + inner: MemoryObjectStore, + lose_next_delete_ack: Arc, + delete_calls: Arc, +} + +impl DeleteAckLostObjectStore { + fn new(inner: MemoryObjectStore) -> Self { + Self { + inner, + lose_next_delete_ack: Arc::new(AtomicBool::new(true)), + delete_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + } + } + + fn delete_calls(&self) -> usize { + self.delete_calls.load(Ordering::SeqCst) + } +} + +impl ObjectStore for DeleteAckLostObjectStore { + fn put( + &self, + key: &ObjectKey, + bytes: impl Into, + ) -> Result { + self.inner.put(key, bytes) + } + + fn get( + &self, + key: &ObjectKey, + range: Option, + ) -> Result, ObjectError> { + self.inner.get(key, range) + } + + fn head(&self, key: &ObjectKey) -> Result, ObjectError> { + self.inner.head(key) + } + + fn delete(&self, key: &ObjectKey) -> Result { + self.delete_calls.fetch_add(1, Ordering::SeqCst); + let deleted = self.inner.delete(key)?; + if self.lose_next_delete_ack.swap(false, Ordering::SeqCst) { + return Err(ObjectError::Backend( + "injected lost DELETE acknowledgement".to_owned(), + )); + } + Ok(deleted) + } +} + +/// Pause one object HEAD so restore fsck can be exercised across an unrelated +/// metadata commit while its physical-reclaim fence remains held. +#[derive(Clone)] +struct HeadBarrierObjectStore { + inner: MemoryObjectStore, + armed: Arc, + entered: Arc, + release: Arc, +} + +impl HeadBarrierObjectStore { + fn new(inner: MemoryObjectStore) -> Self { + Self { + inner, + armed: Arc::new(AtomicBool::new(false)), + entered: Arc::new(Barrier::new(2)), + release: Arc::new(Barrier::new(2)), + } + } + + fn arm(&self) { + self.armed.store(true, Ordering::SeqCst); + } + + fn wait_until_head(&self) { + self.entered.wait(); + } + + fn release_head(&self) { + self.release.wait(); + } +} + +impl ObjectStore for HeadBarrierObjectStore { + fn put( + &self, + key: &ObjectKey, + bytes: impl Into, + ) -> Result { + self.inner.put(key, bytes) + } + + fn get( + &self, + key: &ObjectKey, + range: Option, + ) -> Result, ObjectError> { + self.inner.get(key, range) + } + + fn head(&self, key: &ObjectKey) -> Result, ObjectError> { + let info = self.inner.head(key)?; + if self + .armed + .compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + self.entered.wait(); + self.release.wait(); + } + Ok(info) + } + + fn delete(&self, key: &ObjectKey) -> Result { + self.inner.delete(key) + } +} + +/// Pause after the current active-marker read so an allocator high-water +/// reservation can replace the current-only System allocator before metrics +/// reads its fence projection. +#[derive(Clone)] +struct AllocatorFenceBarrierStore { + inner: HoltMetadataStore, + active_key: Vec, + armed: Arc, + entered: Arc, + release: Arc, +} + +impl AllocatorFenceBarrierStore { + fn new(mount: MountId) -> Self { + Self { + inner: HoltMetadataStore::open_memory().unwrap(), + active_key: restore::restore_active_key(mount), + armed: Arc::new(AtomicBool::new(false)), + entered: Arc::new(Barrier::new(2)), + release: Arc::new(Barrier::new(2)), + } + } + + fn arm(&self) { + self.armed.store(true, Ordering::SeqCst); + } + + fn wait_until_active_read(&self) { + self.entered.wait(); + } + + fn release_active_read(&self) { + self.release.wait(); + } +} + +impl MetadataStore for AllocatorFenceBarrierStore { + fn get_versioned( + &self, + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + let item = self.inner.get_versioned(family, key, version, purpose)?; + if family == RecordFamily::System + && key == self.active_key + && self + .armed + .compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + self.entered.wait(); + self.release.wait(); + } + Ok(item) + } + + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + self.inner.scan(request) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + self.inner.commit_metadata(command) + } + + fn commit_independent_batch( + &self, + commands: &[MetadataCommand], + ) -> Vec> { + self.inner.commit_independent_batch(commands) + } + + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + self.inner.committed_request_result(request_id) + } + + fn history_retention_epoch(&self) -> Result { + self.inner.history_retention_epoch() + } + + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } +} + +fn service() -> NoKvFs { + service_with_objects().0 +} + +fn service_with_objects() -> ( + NoKvFs, + MemoryObjectStore, +) { + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + (service, objects) +} + +fn enqueue_gc_candidate(service: &NoKvFs, mut record: ObjectGcRecord) -> Vec +where + M: MetadataStore, + O: ObjectStore, +{ + let version = service.next_version().unwrap(); + record.enqueue_version = version.get(); + record.enqueue_unix_ms = service.now_ms(); + let key = gc_object_key( + service.mount, + version.get(), + record.inode, + record.generation, + 0, + 0, + ); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-enqueue-gc-candidate", + service.mount, + record.inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Gc, + primary_key: key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::Gc, + key: key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::Gc, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_record(&record))), + }], + watch: Vec::new(), + }) + .unwrap(); + key +} + +fn leave_object_gc_deleting(service: &NoKvFs, gc_row: &ScanItem) -> Vec +where + M: MetadataStore, + O: ObjectStore, +{ + let claim_key = object_gc_claim_key(service.mount); + let claim = service + .metadata + .get_versioned( + RecordFamily::System, + &claim_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-leave-object-gc-deleting", + service.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: claim_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: claim_key.clone(), + predicate: Predicate::VersionEquals(claim.version), + }, + PredicateRef { + family: RecordFamily::Gc, + key: gc_row.key.clone(), + predicate: Predicate::VersionEquals(gc_row.version), + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key: claim_key.clone(), + op: MutationOp::Put, + value: Some(Value( + encode_object_gc_claim(&ObjectGcClaim::Deleting { + owner_epoch: service.epoch.load(Ordering::Relaxed), + operation_token: claim.version.get(), + gc_record_key: gc_row.key.clone(), + gc_record_version: gc_row.version.get(), + }) + .unwrap(), + )), + }], + watch: Vec::new(), + }) + .unwrap(); + claim_key +} + +fn delete_object_gc_claim(service: &NoKvFs) +where + M: MetadataStore, + O: ObjectStore, +{ + let key = object_gc_claim_key(service.mount); + let claim = service + .metadata + .get_versioned( + RecordFamily::System, + &key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-delete-object-gc-claim", + service.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(claim.version), + }], + mutations: vec![delete_mutation(RecordFamily::System, key)], + watch: Vec::new(), + }) + .unwrap(); +} + +fn artifact_request(name: DentryName, manifest_id: &str, bytes: &[u8]) -> PublishArtifact { + PublishArtifact { + parent: InodeId::root(), + name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:test".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: manifest_id.to_owned(), + bytes: bytes.to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + } +} + +fn publish_path_artifact( + service: &NoKvFs, + path: &str, + manifest_id: &str, + bytes: &[u8], +) -> DentryWithAttr { + let prepared = service.prepare_artifact_create_path(path).unwrap(); + service + .publish_prepared_artifact_session( + prepared.clone(), + PublishArtifactSession { + parent: prepared.parent, + name: prepared.name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:test".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: manifest_id.to_owned(), + size: bytes.len() as u64, + ranges: vec![PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap() + .entry +} + +/// Supersede an existing artifact in `parent` (replace -> a fresh generation). +fn republish_path_artifact( + service: &NoKvFs, + parent: InodeId, + name: &str, + manifest_id: &str, + bytes: &[u8], +) -> DentryWithAttr { + let prepared = service + .prepare_artifact_replace(parent, DentryName::new(name.as_bytes().to_vec()).unwrap()) + .unwrap(); + service + .publish_prepared_artifact_session( + prepared.clone(), + PublishArtifactSession { + parent: prepared.parent, + name: prepared.name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:test".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: manifest_id.to_owned(), + size: bytes.len() as u64, + ranges: vec![PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap() + .entry +} + +#[test] +fn publish_multichunk_artifact_succeeds() { + let service = service(); + // 128 MiB spans two 64 MiB chunks: the multi-chunk publish path the FUSE + // bigfile workload hits (and currently EIOs on via InvalidPreparedArtifact). + let size = 128 * 1024 * 1024_usize; + let bytes = vec![0u8; size]; + let prepared = service.prepare_artifact_create_path("/big.bin").unwrap(); + let result = service.publish_prepared_artifact_session( + prepared.clone(), + PublishArtifactSession { + parent: prepared.parent, + name: prepared.name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:test".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "fuse/big".to_owned(), + size: size as u64, + ranges: vec![PublishArtifactRange { offset: 0, bytes }], + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ); + assert!( + result.is_ok(), + "multi-chunk publish failed: {:?}", + result.err() + ); +} + +fn block_key(inode: InodeId, generation: u64, chunk: u64, block: u64) -> ObjectKey { + ObjectKey::new(format!( + "blocks/1/{}/{}/{}/{}", + inode.get(), + generation, + chunk, + block + )) + .unwrap() +} + +fn body_descriptor(generation: u64, size: u64) -> BodyDescriptor { + BodyDescriptor { + producer: "unit-test".to_owned(), + digest_uri: "sha256:test".to_owned(), + size, + content_type: "application/octet-stream".to_owned(), + manifest_id: format!("manifest-{generation}"), + generation, + base_generation: 0, + chunk_size: DEFAULT_CHUNK_SIZE, + block_size: DEFAULT_BLOCK_SIZE as u64, + } +} + +fn one_chunk_manifest(inode: InodeId, generation: u64, len: u64) -> ChunkManifest { + ChunkManifest { + chunk_index: 0, + logical_offset: 0, + len, + slices: vec![SliceManifest { + slice_id: 1, + logical_offset: 0, + len, + blocks: vec![BlockDescriptor { + object_key: block_key(inode, generation, 0, 0).as_str().to_owned(), + logical_offset: 0, + object_offset: 0, + len, + digest_uri: "sha256:block".to_owned(), + }], + }], + } +} + +#[test] +fn create_dir_then_lookup_and_readdir_use_dentry_projection() { + let service = service(); + let name = DentryName::new(b"runs".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), name.clone(), 0o755, 1000, 1000) + .unwrap(); + + let lookup = service + .lookup_plus(InodeId::root(), &name) + .unwrap() + .unwrap(); + assert_eq!(lookup, created); + + let entries = service.read_dir_plus(InodeId::root()).unwrap(); + assert_eq!(entries, vec![created]); + let stats = service.metadata_service_stats(); + assert_eq!(stats.read_dir_plus_total, 1); + assert_eq!(stats.read_dir_plus_entry_total, 1); + assert_eq!(stats.read_dir_plus_projection_hit_total, 1); +} + +#[test] +fn write_planning_reads_are_marked_local_while_user_reads_stay_strong() { + let metadata = PurposeTrackingStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let file_name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + service + .create_file(InodeId::root(), file_name.clone(), 0o644, 1000, 1000) + .unwrap(); + let dir_name = DentryName::new(b"runs".to_vec()).unwrap(); + let dir = service + .create_dir(InodeId::root(), dir_name, 0o755, 1000, 1000) + .unwrap(); + + let before_lookup = metadata.counts(); + assert!(service + .lookup_plus(InodeId::root(), &file_name) + .unwrap() + .is_some()); + let after_lookup = metadata.counts(); + assert!(after_lookup.user_strong_gets > before_lookup.user_strong_gets); + assert_eq!(after_lookup.write_plan_gets, before_lookup.write_plan_gets); + + service + .remove_file(InodeId::root(), &file_name) + .expect("remove file"); + let after_remove = metadata.counts(); + assert_eq!(after_remove.user_strong_gets, after_lookup.user_strong_gets); + assert!(after_remove.write_plan_gets > after_lookup.write_plan_gets); + + let snapshot = service + .snapshot_subtree(dir.attr.inode) + .expect("snapshot subtree"); + let after_snapshot = metadata.counts(); + assert_eq!( + after_snapshot.user_strong_gets, + after_remove.user_strong_gets + ); + assert!(after_snapshot.write_plan_gets > after_remove.write_plan_gets); + + assert!(service + .get_attr_at_snapshot("/runs", snapshot.snapshot_id, &[]) + .unwrap() + .is_some()); + assert!(service + .read_dir_plus_at_snapshot("/runs", snapshot.snapshot_id, &[]) + .unwrap() + .is_empty()); + let after_snapshot_reads = metadata.counts(); + assert!( + after_snapshot_reads.user_strong_gets > after_snapshot.user_strong_gets, + "root-path binding is resolved with strong reads before snapshot-purpose reads" + ); + assert!(after_snapshot_reads.snapshot_gets > after_snapshot.snapshot_gets); + assert!(after_snapshot_reads.snapshot_scans > after_snapshot.snapshot_scans); +} + +#[test] +fn xattr_round_trips_lists_replaces_and_removes() { + let service = service(); + let entry = service + .create_file( + InodeId::root(), + DentryName::new(b"note.txt".to_vec()).unwrap(), + 0o644, + 1000, + 1000, + ) + .unwrap(); + + service + .set_xattr( + entry.attr.inode, + b"user.comment", + b"first".to_vec(), + XattrSetMode::Create, + ) + .unwrap(); + assert_eq!( + service + .get_xattr(entry.attr.inode, b"user.comment") + .unwrap(), + Some(b"first".to_vec()) + ); + assert_eq!( + service.list_xattr(entry.attr.inode).unwrap(), + vec![b"user.comment".to_vec()] + ); + assert!(matches!( + service.set_xattr( + entry.attr.inode, + b"user.comment", + b"duplicate".to_vec(), + XattrSetMode::Create, + ), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + + service + .set_xattr( + entry.attr.inode, + b"user.comment", + b"second".to_vec(), + XattrSetMode::Replace, + ) + .unwrap(); + assert_eq!( + service + .get_xattr(entry.attr.inode, b"user.comment") + .unwrap(), + Some(b"second".to_vec()) + ); + assert!(matches!( + service.set_xattr( + entry.attr.inode, + b"user.missing", + b"value".to_vec(), + XattrSetMode::Replace, + ), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + + service + .remove_xattr(entry.attr.inode, b"user.comment") + .unwrap(); + assert_eq!( + service + .get_xattr(entry.attr.inode, b"user.comment") + .unwrap(), + None + ); + assert!(service.list_xattr(entry.attr.inode).unwrap().is_empty()); + assert!(matches!( + service.remove_xattr(entry.attr.inode, b"user.comment"), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); +} + +#[test] +fn path_methods_resolve_current_namespace_on_server_side() { + let service = service(); + let runs = service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let artifact = service + .create_file_path("/runs/checkpoint.bin", 0o644, 1000, 1000) + .unwrap(); + + assert_eq!(service.lookup_path("/runs").unwrap(), Some(runs.clone())); + assert_eq!( + service.lookup_path("/runs/checkpoint.bin").unwrap(), + Some(artifact.clone()) + ); + assert_eq!(service.read_dir_plus_path("/runs").unwrap(), vec![artifact]); + assert!(matches!( + service.create_file_path("relative", 0o644, 1000, 1000), + Err(MetadError::InvalidPath(_)) + )); +} + +#[test] +fn plain_path_create_uses_canonical_namespace_without_path_index() { + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let artifact = service + .create_file_path("/runs/checkpoint.bin", 0o644, 1000, 1000) + .unwrap(); + let components = parse_absolute_path("/runs/checkpoint.bin").unwrap(); + let key = path_index_key(MountId::new(1).unwrap(), &components); + assert!(metadata + .get( + RecordFamily::PathIndex, + &key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + + let before = service.metadata_service_stats(); + assert_eq!( + service.lookup_path("/runs/checkpoint.bin").unwrap(), + Some(artifact) + ); + let after = service.metadata_service_stats(); + assert_eq!( + after.path_index_lookup_total - before.path_index_lookup_total, + 0 + ); + assert_eq!( + after.path_index_fallback_total - before.path_index_fallback_total, + 0 + ); +} + +#[test] +fn prepared_artifact_path_publish_writes_and_uses_validated_path_index() { + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let prepared = service + .prepare_artifact_create_path("/runs/checkpoint.bin") + .unwrap(); + let body = body_descriptor(prepared.generation, 6); + let artifact = service + .publish_prepared_artifact( + prepared.clone(), + body, + vec![one_chunk_manifest(prepared.inode, prepared.generation, 6)], + 0o644, + 1000, + 1000, + ) + .unwrap() + .entry; + let components = parse_absolute_path("/runs/checkpoint.bin").unwrap(); + let key = path_index_key(MountId::new(1).unwrap(), &components); + let indexed = metadata + .get( + RecordFamily::PathIndex, + &key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .expect("artifact path index entry"); + let projection = decode_dentry_projection(&indexed.0).unwrap(); + assert_eq!(DentryWithAttr::from(projection), artifact); + + let before = service.metadata_service_stats(); + let metadata = service + .stat_path("/runs/checkpoint.bin") + .unwrap() + .expect("artifact stat"); + assert_eq!(metadata.attr, artifact.attr); + assert_eq!(metadata.body, artifact.body); + let after = service.metadata_service_stats(); + assert_eq!( + after.path_index_lookup_total - before.path_index_lookup_total, + 1 + ); + assert_eq!(after.path_index_hit_total - before.path_index_hit_total, 1); + assert_eq!( + after.path_index_fallback_total - before.path_index_fallback_total, + 0 + ); + + let before = service.metadata_service_stats(); + assert_eq!(service.stat_path("/runs/missing.bin").unwrap(), None); + let after = service.metadata_service_stats(); + assert_eq!( + after.path_index_lookup_total - before.path_index_lookup_total, + 1 + ); + assert_eq!( + after.path_index_miss_total - before.path_index_miss_total, + 1 + ); + assert_eq!( + after.path_index_fallback_total - before.path_index_fallback_total, + 1 + ); +} + +#[test] +fn artifact_path_rename_moves_live_path_index() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let archive = service + .create_dir_path("/archive", 0o755, 1000, 1000) + .unwrap(); + let artifact = publish_path_artifact(&service, "/runs/a.bin", "runs/a.bin", b"a"); + let old_components = parse_absolute_path("/runs/a.bin").unwrap(); + let new_components = parse_absolute_path("/archive/a.bin").unwrap(); + let old_key = path_index_key(MountId::new(1).unwrap(), &old_components); + let new_key = path_index_key(MountId::new(1).unwrap(), &new_components); + assert!(metadata + .get( + RecordFamily::PathIndex, + &old_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_some()); + + let renamed = service + .rename_path("/runs/a.bin", "/archive/a.bin") + .unwrap(); + let old_index = metadata + .get( + RecordFamily::PathIndex, + &old_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(); + let new_index = metadata + .get( + RecordFamily::PathIndex, + &new_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .expect("renamed artifact path index"); + + assert!(old_index.is_none()); + assert_eq!(renamed.attr.inode, artifact.attr.inode); + let indexed = decode_dentry_projection(&new_index.0).unwrap(); + assert_eq!(indexed.dentry.parent, archive.attr.inode); + assert_eq!(indexed.dentry.name.as_bytes(), b"a.bin"); + assert_eq!(indexed.attr.inode, artifact.attr.inode); +} + +#[test] +fn plain_directory_path_rename_does_not_create_path_index() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let runs = service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let source_components = parse_absolute_path("/runs").unwrap(); + let destination_components = parse_absolute_path("/archive").unwrap(); + let source_key = path_index_key(MountId::new(1).unwrap(), &source_components); + let destination_key = path_index_key(MountId::new(1).unwrap(), &destination_components); + let before = metadata.metadata_store_stats(); + + let renamed = service.rename_path("/runs", "/archive").unwrap(); + let after = metadata.metadata_store_stats(); + + assert_eq!(renamed.attr.inode, runs.attr.inode); + assert_eq!(after.current_put_total - before.current_put_total, 1); + assert_eq!(after.current_delete_total - before.current_delete_total, 1); + assert!(metadata + .get( + RecordFamily::PathIndex, + &source_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + assert!(metadata + .get( + RecordFamily::PathIndex, + &destination_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); +} + +#[test] +fn artifact_path_remove_deletes_live_path_index() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/runs/a.bin", "runs/a.bin", b"a"); + let components = parse_absolute_path("/runs/a.bin").unwrap(); + let key = path_index_key(MountId::new(1).unwrap(), &components); + assert!(metadata + .get( + RecordFamily::PathIndex, + &key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_some()); + + service.remove_file_path("/runs/a.bin").unwrap(); + + assert!(metadata + .get( + RecordFamily::PathIndex, + &key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); +} + +#[test] +fn path_resolution_cache_reuses_parent_directory_for_indexed_stats() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/runs/a.bin", "runs/a.bin", b"a"); + publish_path_artifact(&service, "/runs/b.bin", "runs/b.bin", b"b"); + service.clear_read_path_caches_for_test(); + + let before_store = metadata.metadata_store_stats(); + let before_service = service.metadata_service_stats(); + assert!(service.stat_path("/runs/a.bin").unwrap().is_some()); + let after_first_store = metadata.metadata_store_stats(); + assert!(service.stat_path("/runs/b.bin").unwrap().is_some()); + let after_second_store = metadata.metadata_store_stats(); + let after_service = service.metadata_service_stats(); + + let first_gets = after_first_store.get_total - before_store.get_total; + let second_gets = after_second_store.get_total - after_first_store.get_total; + assert_eq!(first_gets, 3); + assert_eq!(second_gets, 2); + assert_eq!( + after_service.path_index_hit_total - before_service.path_index_hit_total, + 2 + ); + assert_eq!( + after_service.path_index_fallback_total - before_service.path_index_fallback_total, + 0 + ); +} + +#[test] +fn validated_path_index_cache_reuses_stat_validation_for_indexed_list() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let first = publish_path_artifact(&service, "/runs/a.bin", "runs/a.bin", b"a"); + let second = publish_path_artifact(&service, "/runs/b.bin", "runs/b.bin", b"b"); + service.clear_read_path_caches_for_test(); + + assert!(service.stat_path("/runs/a.bin").unwrap().is_some()); + assert!(service.stat_path("/runs/b.bin").unwrap().is_some()); + + let before_store = metadata.metadata_store_stats(); + let page = service.list_indexed_path_page("/runs", None, 10).unwrap(); + let after_store = metadata.metadata_store_stats(); + + assert_eq!(page.entries, vec![first, second]); + assert_eq!(page.next_cursor, None); + assert_eq!(after_store.get_total - before_store.get_total, 0); +} + +#[test] +fn validated_path_index_lookup_cache_reuses_repeated_stat_result() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let artifact = publish_path_artifact(&service, "/runs/a.bin", "runs/a.bin", b"a"); + service.clear_read_path_caches_for_test(); + + let first = service + .stat_path("/runs/a.bin") + .unwrap() + .expect("first stat"); + assert_eq!(first.attr, artifact.attr); + + let before_store = metadata.metadata_store_stats(); + let second = service + .stat_path("/runs/a.bin") + .unwrap() + .expect("second stat"); + let after_store = metadata.metadata_store_stats(); + + assert_eq!(second.attr, artifact.attr); + assert_eq!(after_store.get_total - before_store.get_total, 0); +} + +#[test] +fn namespace_find_body_facets_do_not_require_body_projection() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/runs/a.json", "runs/a.json", br#"{"loss":0.4}"#); + publish_path_artifact(&service, "/runs/b.log", "runs/b.log", b"loss=0.3\n"); + + let result = service + .find_paths(NamespaceFindRequest { + path: "/runs".to_owned(), + predicates: Vec::new(), + sort: Vec::new(), + include: Vec::new(), + facets: vec![NamespaceFindField::body_content_type()], + cursor: None, + limit: 10, + }) + .unwrap(); + + assert_eq!(result.match_count, 2); + assert!(result.matches.iter().all(|card| card.body.is_none())); + assert_eq!(result.facets.len(), 1); + assert_eq!( + result.facets[0].field, + NamespaceFindField::body_content_type() + ); + assert_eq!(result.facets[0].values[0].count, 2); +} + +#[test] +fn namespace_find_tolerates_exists_predicate_payloads() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/runs/a.json", "runs/a.json", br#"{"loss":0.4}"#); + + let result = service + .find_paths(NamespaceFindRequest { + path: "/runs".to_owned(), + predicates: vec![NamespacePredicate { + field: NamespaceFindField::body_content_type(), + op: NamespacePredicateOp::Exists, + value: Some(NamespacePredicateValue::String("ignored".to_owned())), + }], + sort: Vec::new(), + include: Vec::new(), + facets: Vec::new(), + cursor: None, + limit: 10, + }) + .unwrap(); + + assert_eq!(result.match_count, 1); + assert_eq!(result.matches[0].path, "/runs/a.json"); +} + +#[test] +fn namespace_grep_cursor_resumes_at_next_unemitted_match() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/runs/train.log", + "runs/train.log", + b"loss=1\nloss=2\n", + ); + + let first = service + .grep_paths(NamespaceGrepRequest { + path: "/runs/train.log".to_owned(), + pattern: "loss".to_owned(), + patterns: Vec::new(), + recursive: false, + name_glob: None, + cursor: None, + limit: 1, + max_files: None, + max_bytes: None, + }) + .unwrap(); + assert_eq!(first.matches.len(), 1); + assert_eq!(first.matches[0].line_number, 1); + assert_eq!(first.pattern, "loss"); + assert!(first.patterns.is_empty()); + + let second = service + .grep_paths(NamespaceGrepRequest { + path: "/runs/train.log".to_owned(), + pattern: "loss".to_owned(), + patterns: Vec::new(), + recursive: false, + name_glob: None, + cursor: first.next_cursor, + limit: 1, + max_files: None, + max_bytes: None, + }) + .unwrap(); + assert_eq!(second.matches.len(), 1); + assert_eq!(second.matches[0].line_number, 2); +} + +#[test] +fn namespace_grep_multiple_patterns_match_any() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/runs/a.log", "runs/a.log", b"alpha metric\n"); + publish_path_artifact( + &service, + "/runs/b.log", + "runs/b.log", + b"nothing here\nbeta metric\n", + ); + + let result = service + .grep_paths(NamespaceGrepRequest { + path: "/runs".to_owned(), + pattern: String::new(), + patterns: vec!["alpha".to_owned(), "beta".to_owned()], + recursive: false, + name_glob: None, + cursor: None, + limit: 10, + max_files: None, + max_bytes: None, + }) + .unwrap(); + + assert_eq!(result.matches.len(), 2); + assert_eq!(result.matches[0].path, "/runs/a.log"); + assert_eq!(result.matches[0].line_number, 1); + assert_eq!(result.matches[1].path, "/runs/b.log"); + assert_eq!(result.matches[1].line_number, 2); + assert_eq!(result.patterns, vec!["alpha", "beta"]); +} + +#[test] +fn namespace_grep_multiple_patterns_match_cjk() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/runs/notes.txt", + "runs/notes.txt", + "今日营养记录\n普通一行\n食谱更新完成\n".as_bytes(), + ); + + let result = service + .grep_paths(NamespaceGrepRequest { + path: "/runs/notes.txt".to_owned(), + pattern: String::new(), + patterns: vec!["营养".to_owned(), "食谱".to_owned()], + recursive: false, + name_glob: None, + cursor: None, + limit: 10, + max_files: None, + max_bytes: None, + }) + .unwrap(); + + let lines = result + .matches + .iter() + .map(|entry| entry.line_number) + .collect::>(); + assert_eq!(lines, vec![1, 3]); +} + +/// `patterns` adds OR alternatives to `pattern` (union semantics); a non-empty +/// `pattern` must keep matching when `patterns` is also provided instead of +/// being silently dropped. +#[test] +fn namespace_grep_unions_pattern_with_patterns() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/runs/notes.txt", + "runs/notes.txt", + "食谱更新完成\n无关的一行\n食材已备齐\n新 recipe 上线\n".as_bytes(), + ); + + let result = service + .grep_paths(NamespaceGrepRequest { + path: "/runs/notes.txt".to_owned(), + pattern: "食谱".to_owned(), + patterns: vec!["食材".to_owned(), "recipe".to_owned()], + recursive: false, + name_glob: None, + cursor: None, + limit: 10, + max_files: None, + max_bytes: None, + }) + .unwrap(); + + let lines = result + .matches + .iter() + .map(|entry| entry.line_number) + .collect::>(); + assert_eq!(lines, vec![1, 3, 4]); + // The echo reports the request fields verbatim. + assert_eq!(result.pattern, "食谱"); + assert_eq!(result.patterns, vec!["食材", "recipe"]); +} + +/// The workbench pipe-split forwards `pattern: "a|b"` together with +/// `patterns: ["a", "b"]`. Any line containing the literal "a|b" also contains +/// each split alternative, so union semantics must return the same match set +/// as the split alternatives alone. +#[test] +fn namespace_grep_piped_pattern_union_matches_split_alternatives() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/runs/mixed.log", + "runs/mixed.log", + b"alpha metric\nliteral alpha|beta row\nbeta metric\nnothing\n", + ); + let request = |pattern: &str, patterns: Vec| NamespaceGrepRequest { + path: "/runs/mixed.log".to_owned(), + pattern: pattern.to_owned(), + patterns, + recursive: false, + name_glob: None, + cursor: None, + limit: 10, + max_files: None, + max_bytes: None, + }; + + let split_only = service + .grep_paths(request("", vec!["alpha".to_owned(), "beta".to_owned()])) + .unwrap(); + let piped_union = service + .grep_paths(request( + "alpha|beta", + vec!["alpha".to_owned(), "beta".to_owned()], + )) + .unwrap(); + + let lines = |result: &NamespaceGrepResult| { + result + .matches + .iter() + .map(|entry| (entry.path.clone(), entry.line_number)) + .collect::>() + }; + assert_eq!(lines(&piped_union), lines(&split_only)); + assert_eq!( + lines(&split_only), + vec![ + ("/runs/mixed.log".to_owned(), 1), + ("/runs/mixed.log".to_owned(), 2), + ("/runs/mixed.log".to_owned(), 3), + ] + ); +} + +#[test] +fn namespace_grep_name_glob_skips_unmatched_files_without_reading() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/runs/a.md", "runs/a.md", b"needle in md\n"); + publish_path_artifact(&service, "/runs/b.log", "runs/b.log", b"needle in log\n"); + + let result = service + .grep_paths(NamespaceGrepRequest { + path: "/runs".to_owned(), + pattern: "needle".to_owned(), + patterns: Vec::new(), + recursive: false, + name_glob: Some("*.md".to_owned()), + cursor: None, + limit: 10, + max_files: None, + max_bytes: None, + }) + .unwrap(); + + assert_eq!(result.matches.len(), 1); + assert_eq!(result.matches[0].path, "/runs/a.md"); + assert_eq!(result.files_scanned, 1); + assert_eq!(result.bytes_read, b"needle in md\n".len()); +} + +#[test] +fn namespace_grep_name_glob_matches_cjk_substring() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/runs/营养日志.txt", + "runs/nutrition.txt", + "记录一\n".as_bytes(), + ); + publish_path_artifact( + &service, + "/runs/训练日志.txt", + "runs/training.txt", + "记录二\n".as_bytes(), + ); + + let result = service + .grep_paths(NamespaceGrepRequest { + path: "/runs".to_owned(), + pattern: "记录".to_owned(), + patterns: Vec::new(), + recursive: false, + name_glob: Some("*营养*".to_owned()), + cursor: None, + limit: 10, + max_files: None, + max_bytes: None, + }) + .unwrap(); + + assert_eq!(result.matches.len(), 1); + assert_eq!(result.matches[0].path, "/runs/营养日志.txt"); + assert_eq!(result.files_scanned, 1); +} + +#[test] +fn namespace_grep_rejects_more_than_sixteen_patterns() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + + let err = service + .grep_paths(NamespaceGrepRequest { + path: "/runs".to_owned(), + pattern: String::new(), + patterns: (0..17).map(|index| format!("p{index}")).collect(), + recursive: false, + name_glob: None, + cursor: None, + limit: 10, + max_files: None, + max_bytes: None, + }) + .unwrap_err(); + + assert!(matches!(err, MetadError::InvalidQuery(message) if message.contains("16"))); +} + +#[test] +fn namespace_grep_rejects_empty_pattern_entry() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + + let err = service + .grep_paths(NamespaceGrepRequest { + path: "/runs".to_owned(), + pattern: String::new(), + patterns: vec!["ok".to_owned(), String::new()], + recursive: false, + name_glob: None, + cursor: None, + limit: 10, + max_files: None, + max_bytes: None, + }) + .unwrap_err(); + + assert!(matches!(err, MetadError::InvalidQuery(message) if message.contains("empty"))); +} + +#[test] +fn namespace_grep_rejects_missing_pattern_and_patterns() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + + let err = service + .grep_paths(NamespaceGrepRequest { + path: "/runs".to_owned(), + pattern: String::new(), + patterns: Vec::new(), + recursive: false, + name_glob: None, + cursor: None, + limit: 10, + max_files: None, + max_bytes: None, + }) + .unwrap_err(); + + assert!(matches!(err, MetadError::InvalidQuery(message) if message.contains("pattern"))); +} + +#[test] +fn namespace_read_structured_offset_without_cursor_skips_items() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/runs/a.json", "runs/a.json", br#"["a","b","c"]"#); + + let page = service + .read_page( + "/runs/a.json", + NamespaceReadOptions { + format: NamespaceReadFormat::Structured, + cursor: None, + offset: 1, + limit: 1, + expected_generation: None, + }, + ) + .unwrap(); + + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].index, 1); + assert_eq!(page.items[0].value_json, r#""b""#); + assert!(page.truncated); +} + +#[test] +fn namespace_read_bytes_honors_returned_cursor() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/runs/a.bin", "runs/a.bin", b"abcdef"); + + let first = service + .read_page( + "/runs/a.bin", + NamespaceReadOptions { + format: NamespaceReadFormat::Bytes, + cursor: None, + offset: 0, + limit: 2, + expected_generation: None, + }, + ) + .unwrap(); + assert_eq!(first.bytes.as_deref(), Some(b"ab".as_slice())); + + let second = service + .read_page( + "/runs/a.bin", + NamespaceReadOptions { + format: NamespaceReadFormat::Bytes, + cursor: first.next_cursor, + offset: 0, + limit: 2, + expected_generation: None, + }, + ) + .unwrap(); + assert_eq!(second.bytes.as_deref(), Some(b"cd".as_slice())); +} + +#[test] +fn register_namespace_index_rejects_rows_outside_registered_path() { + let service = service(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + + let err = service + .register_namespace_index(NamespaceIndexRegistration { + path: "/runs".to_owned(), + fields: vec![NamespaceIndexField { + field: NamespaceFindField::new("run.status"), + operators: vec![NamespacePredicateOp::Eq], + sortable: false, + facetable: true, + }], + rows: vec![NamespaceIndexRow { + path: "/archive/a.json".to_owned(), + values: Vec::new(), + }], + }) + .unwrap_err(); + + assert!( + matches!(err, MetadError::InvalidQuery(message) if message.contains("outside registered namespace")) + ); +} + +#[test] +fn register_namespace_index_uses_metadata_predicate_fence() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + + let before = metadata.metadata_store_stats(); + service + .register_namespace_index(NamespaceIndexRegistration { + path: "/runs".to_owned(), + fields: vec![NamespaceIndexField { + field: NamespaceFindField::new("run.status"), + operators: vec![NamespacePredicateOp::Eq], + sortable: false, + facetable: true, + }], + rows: vec![NamespaceIndexRow { + path: "/runs/a.json".to_owned(), + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("run.status"), + value: NamespacePredicateValue::String("completed".to_owned()), + }], + }], + }) + .unwrap(); + let after = metadata.metadata_store_stats(); + + assert_eq!( + after.predicate_total - before.predicate_total, + 2, + "catalog registration fences both the canonical catalog and absent restore membership" + ); +} + +#[test] +fn stale_path_index_falls_back_to_canonical_namespace() { + let service = service(); + let runs = service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let archive = service + .create_dir_path("/archive", 0o755, 1000, 1000) + .unwrap(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create_path("/runs/checkpoint.bin") + .unwrap(); + let artifact = service + .publish_prepared_artifact( + prepared.clone(), + body_descriptor(prepared.generation, 6), + vec![one_chunk_manifest(prepared.inode, prepared.generation, 6)], + 0o644, + 1000, + 1000, + ) + .unwrap() + .entry; + + assert!(service.stat_path("/runs/checkpoint.bin").unwrap().is_some()); + + service + .rename(runs.attr.inode, &name, archive.attr.inode, name.clone()) + .unwrap(); + + let before = service.metadata_service_stats(); + assert_eq!(service.stat_path("/runs/checkpoint.bin").unwrap(), None); + let after = service.metadata_service_stats(); + assert_eq!( + after.path_index_lookup_total - before.path_index_lookup_total, + 1 + ); + assert_eq!( + after.path_index_stale_total - before.path_index_stale_total, + 1 + ); + assert_eq!( + after.path_index_fallback_total - before.path_index_fallback_total, + 1 + ); + + let mut moved_artifact = artifact; + moved_artifact.dentry.parent = archive.attr.inode; + + let before = service.metadata_service_stats(); + let metadata = service + .stat_path("/archive/checkpoint.bin") + .unwrap() + .expect("moved artifact stat"); + assert_eq!(metadata.attr, moved_artifact.attr); + assert_eq!(metadata.body, moved_artifact.body); + let after = service.metadata_service_stats(); + assert_eq!( + after.path_index_miss_total - before.path_index_miss_total, + 1 + ); + assert_eq!( + after.path_index_fallback_total - before.path_index_fallback_total, + 1 + ); +} + +#[test] +fn path_index_page_lists_immediate_indexed_children_with_holt_delimiter() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let epoch = service + .create_dir_path("/runs/epoch", 0o755, 1000, 1000) + .unwrap(); + service + .create_file_path("/runs/plain.txt", 0o644, 1000, 1000) + .unwrap(); + publish_path_artifact(&service, "/runs/top.bin", "runs/top.bin", b"top"); + publish_path_artifact( + &service, + "/runs/epoch/ckpt.bin", + "runs/epoch/ckpt.bin", + b"ckpt", + ); + + let before = metadata.metadata_store_stats(); + let first = service.list_indexed_path_page("/runs", None, 1).unwrap(); + let after_first = metadata.metadata_store_stats(); + assert_eq!(first.entries, vec![epoch]); + assert_eq!( + first.next_cursor.as_ref().map(DentryName::as_bytes), + Some(b"epoch".as_slice()) + ); + assert_eq!( + after_first.scan_key_returned_total - before.scan_key_returned_total, + 2 + ); + + let second = service + .list_indexed_path_page("/runs", first.next_cursor.as_ref(), 10) + .unwrap(); + assert_eq!(second.entries.len(), 1); + assert_eq!(second.entries[0].dentry.name.as_bytes(), b"top.bin"); + assert_eq!(second.entries[0].attr.file_type, FileType::File); + assert_eq!(second.next_cursor, None); +} + +#[test] +fn path_index_page_skips_stale_rows_without_truncating_visible_children() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/archive", 0o755, 1000, 1000) + .unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/runs/aaa", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/runs/aaa/stale.bin", + "runs/aaa/stale.bin", + b"stale", + ); + service.rename_path("/runs/aaa", "/archive/aaa").unwrap(); + let first_valid = publish_path_artifact(&service, "/runs/bbb.bin", "runs/bbb.bin", b"bbb"); + let second_valid = publish_path_artifact(&service, "/runs/ccc.bin", "runs/ccc.bin", b"ccc"); + + let before_store = metadata.metadata_store_stats(); + let before_service = service.metadata_service_stats(); + let first = service.list_indexed_path_page("/runs", None, 1).unwrap(); + let after_first_store = metadata.metadata_store_stats(); + let after_first_service = service.metadata_service_stats(); + assert_eq!(first.entries, vec![first_valid]); + assert_eq!( + first.next_cursor.as_ref().map(DentryName::as_bytes), + Some(b"bbb.bin".as_slice()) + ); + assert!( + after_first_store.scan_key_returned_total - before_store.scan_key_returned_total > 2, + "stale index row should force an extra delimiter scan page" + ); + assert_eq!( + after_first_service.read_dir_plus_entry_total - before_service.read_dir_plus_entry_total, + 1 + ); + assert_eq!( + after_first_service.read_dir_plus_projection_hit_total + - before_service.read_dir_plus_projection_hit_total, + 1 + ); + assert!( + after_first_service.path_index_scan_stale_total + - before_service.path_index_scan_stale_total + >= 1, + "stale derived path-index rows should be reported separately from live entries" + ); + + let second = service + .list_indexed_path_page("/runs", first.next_cursor.as_ref(), 1) + .unwrap(); + assert_eq!(second.entries, vec![second_valid]); + assert_eq!(second.next_cursor, None); +} + +#[test] +fn directory_rename_leaves_descendant_path_index_as_derived_stale_cache() { + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let prepared = service + .prepare_artifact_create_path("/runs/checkpoint.bin") + .unwrap(); + let artifact = service + .publish_prepared_artifact( + prepared.clone(), + body_descriptor(prepared.generation, 6), + vec![one_chunk_manifest(prepared.inode, prepared.generation, 6)], + 0o644, + 1000, + 1000, + ) + .unwrap() + .entry; + let old_components = parse_absolute_path("/runs/checkpoint.bin").unwrap(); + let old_key = path_index_key(MountId::new(1).unwrap(), &old_components); + assert!(metadata + .get( + RecordFamily::PathIndex, + &old_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_some()); + + service.rename_path("/runs", "/archive").unwrap(); + + let renamed_dir_key = path_index_key( + MountId::new(1).unwrap(), + &parse_absolute_path("/runs").unwrap(), + ); + assert!(metadata + .get( + RecordFamily::PathIndex, + &renamed_dir_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + assert!(metadata + .get( + RecordFamily::PathIndex, + &old_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_some()); + assert!(matches!( + service.lookup_path("/runs/checkpoint.bin"), + Err(MetadError::NotFound) + )); + assert_eq!( + service.lookup_path("/archive/checkpoint.bin").unwrap(), + Some(artifact) + ); +} + +#[test] +fn create_file_publishes_metadata_without_body_descriptor() { + let service = service(); + let name = DentryName::new(b"empty.txt".to_vec()).unwrap(); + let created = service + .create_file(InodeId::root(), name.clone(), 0o644, 1000, 1000) + .unwrap(); + assert_eq!(created.attr.file_type, FileType::File); + assert_eq!(created.attr.size, 0); + assert!(created.body.is_none()); + assert_eq!( + service.lookup_plus(InodeId::root(), &name).unwrap(), + Some(created) + ); +} + +#[test] +fn new_file_attrs_use_wall_clock_timestamps() { + let service = service(); + let before = current_time_ms().saturating_sub(1_000); + + let created = service + .create_file( + InodeId::root(), + DentryName::new(b"empty.txt".to_vec()).unwrap(), + 0o644, + 1000, + 1000, + ) + .unwrap(); + assert!(created.attr.mtime_ms >= before); + assert!(created.attr.ctime_ms >= before); + assert!(created.attr.mtime_ms > created.attr.generation); + + let published = service + .publish_artifact(artifact_request( + DentryName::new(b"artifact.bin".to_vec()).unwrap(), + "artifact", + b"body", + )) + .unwrap(); + assert!(published.attr.mtime_ms >= before); + assert!(published.attr.ctime_ms >= before); + assert!(published.attr.mtime_ms > published.attr.generation); +} + +#[test] +fn create_symlink_round_trips_target_and_unlinks_like_file() { + let service = service(); + let name = DentryName::new(b"latest".to_vec()).unwrap(); + let created = service + .create_symlink( + InodeId::root(), + name.clone(), + b"runs/42/checkpoint.bin".to_vec(), + 0o777, + 1000, + 1000, + ) + .unwrap(); + + assert_eq!(created.attr.file_type, FileType::Symlink); + assert_eq!(created.attr.size, 22); + assert_eq!( + service.read_symlink(created.attr.inode).unwrap(), + b"runs/42/checkpoint.bin" + ); + assert_eq!( + created.body.as_ref().unwrap().digest_uri, + "sha256:15a533489b90109ab69bd64dabcc260602c854b6b4a472b20aefa0eabcee3a24" + ); + assert_eq!( + service.lookup_plus(InodeId::root(), &name).unwrap(), + Some(created.clone()) + ); + + let removed = service.remove_file(InodeId::root(), &name).unwrap(); + assert_eq!(removed.attr.file_type, FileType::Symlink); + assert_eq!(service.lookup_plus(InodeId::root(), &name).unwrap(), None); +} + +#[test] +fn create_special_node_persists_type_and_rdev_without_body() { + let service = service(); + let fifo_name = DentryName::new(b"events.fifo".to_vec()).unwrap(); + let fifo = service + .create_special_node( + InodeId::root(), + fifo_name.clone(), + SpecialNodeSpec { + file_type: FileType::NamedPipe, + mode: 0o644, + rdev: 0, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + assert_eq!(fifo.attr.file_type, FileType::NamedPipe); + assert_eq!(fifo.attr.rdev, 0); + assert_eq!(fifo.attr.size, 0); + assert!(fifo.body.is_none()); + assert_eq!( + service.lookup_plus(InodeId::root(), &fifo_name).unwrap(), + Some(fifo.clone()) + ); + + let char_name = DentryName::new(b"accelerator0".to_vec()).unwrap(); + let char_device = service + .create_special_node( + InodeId::root(), + char_name.clone(), + SpecialNodeSpec { + file_type: FileType::CharDevice, + mode: 0o660, + rdev: 0x1234, + uid: 0, + gid: 44, + }, + ) + .unwrap(); + assert_eq!(char_device.attr.file_type, FileType::CharDevice); + assert_eq!(char_device.attr.rdev, 0x1234); + assert!(char_device.body.is_none()); + assert!(service + .read_dir_plus(InodeId::root()) + .unwrap() + .iter() + .any(|entry| entry.attr == char_device.attr)); + + let removed = service.remove_file(InodeId::root(), &char_name).unwrap(); + assert_eq!(removed.attr.file_type, FileType::CharDevice); + assert_eq!( + service.lookup_plus(InodeId::root(), &char_name).unwrap(), + None + ); +} + +#[test] +fn advisory_locks_detect_conflicts_and_support_partial_unlock() { + let service = service(); + let name = DentryName::new(b"locked.bin".to_vec()).unwrap(); + let file = service + .create_file(InodeId::root(), name, 0o644, 1000, 1000) + .unwrap(); + let inode = file.attr.inode; + let read_owner = 11; + let write_owner = 22; + + service + .set_advisory_lock(AdvisoryLockRequest { + inode, + owner: read_owner, + start: 0, + end: 99, + kind: AdvisoryLockKind::Read, + pid: 1100, + wait: false, + }) + .unwrap(); + service + .set_advisory_lock(AdvisoryLockRequest { + inode, + owner: 33, + start: 20, + end: 30, + kind: AdvisoryLockKind::Read, + pid: 3300, + wait: false, + }) + .unwrap(); + + let conflict = service + .get_advisory_lock(AdvisoryLockRequest { + inode, + owner: write_owner, + start: 50, + end: 60, + kind: AdvisoryLockKind::Write, + pid: 2200, + wait: false, + }) + .unwrap() + .unwrap(); + assert_eq!(conflict.owner, read_owner); + assert_eq!(conflict.kind, AdvisoryLockKind::Read); + assert!(matches!( + service.set_advisory_lock(AdvisoryLockRequest { + inode, + owner: write_owner, + start: 50, + end: 60, + kind: AdvisoryLockKind::Write, + pid: 2200, + wait: false, + }), + Err(MetadError::LockConflict(_)) + )); + + service + .set_advisory_lock(AdvisoryLockRequest { + inode, + owner: read_owner, + start: 40, + end: 70, + kind: AdvisoryLockKind::Unlock, + pid: 1100, + wait: false, + }) + .unwrap(); + assert!(service + .get_advisory_lock(AdvisoryLockRequest { + inode, + owner: write_owner, + start: 50, + end: 60, + kind: AdvisoryLockKind::Write, + pid: 2200, + wait: false, + }) + .unwrap() + .is_none()); + assert!(service + .get_advisory_lock(AdvisoryLockRequest { + inode, + owner: write_owner, + start: 10, + end: 20, + kind: AdvisoryLockKind::Write, + pid: 2200, + wait: false, + }) + .unwrap() + .is_some()); +} + +#[test] +fn snapshot_preserves_symlink_target() { + let service = service(); + let name = DentryName::new(b"latest".to_vec()).unwrap(); + service + .create_symlink( + InodeId::root(), + name.clone(), + b"runs/old".to_vec(), + 0o777, + 1000, + 1000, + ) + .unwrap(); + let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + service.remove_file(InodeId::root(), &name).unwrap(); + service + .create_symlink( + InodeId::root(), + name.clone(), + b"runs/new".to_vec(), + 0o777, + 1000, + 1000, + ) + .unwrap(); + + assert_eq!( + service + .read_symlink_at_snapshot("/", snapshot.snapshot_id, std::slice::from_ref(&name)) + .unwrap(), + b"runs/old" + ); +} + +#[test] +fn update_attrs_truncates_and_extends_sparse_file() { + let service = service(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "checkpoint-v1", b"abcdef")) + .unwrap(); + + let shrunk = service + .update_attrs( + InodeId::root(), + &name, + UpdateAttr { + size: Some(3), + ..UpdateAttr::default() + }, + ) + .unwrap(); + assert_eq!(shrunk.attr.inode, published.attr.inode); + assert_eq!(shrunk.attr.size, 3); + assert_eq!(service.read_file(shrunk.attr.inode, 0, 8).unwrap(), b"abc"); + assert_eq!( + shrunk.body.as_ref().unwrap().digest_uri, + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + + let grown = service + .update_attrs( + InodeId::root(), + &name, + UpdateAttr { + size: Some(6), + ..UpdateAttr::default() + }, + ) + .unwrap(); + assert_eq!(grown.attr.size, 6); + assert_eq!( + service.read_file(grown.attr.inode, 0, 8).unwrap(), + b"abc\0\0\0" + ); + assert_eq!( + grown.body.as_ref().unwrap().digest_uri, + "sha256:dd0b251b2bf91037a1e4fc8416a24ae00bcb9a8c252dc7e2361f2fc015f51c16" + ); +} + +#[test] +fn attr_only_update_preserves_body_generation_and_readability() { + // `cp` preserves metadata, so it chmods a file it just wrote. An attribute- + // only `update_attrs` (no size change) must not advance `attr.generation`: + // the body summary / chunk manifests are keyed by generation and reads + // resolve the body via `attr.generation`, so bumping it would point the + // dentry at a generation that has no body, surfacing as MissingBodyDescriptor + // on the next read (the cp corruption this regression guards). + let service = service(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "checkpoint-v1", b"abcdef")) + .unwrap(); + let body_generation = published.body.as_ref().unwrap().generation; + assert_eq!(published.attr.generation, body_generation); + + let chmodded = service + .update_attrs( + InodeId::root(), + &name, + UpdateAttr { + mode: Some(0o600), + ..UpdateAttr::default() + }, + ) + .unwrap(); + + assert_eq!(chmodded.attr.mode, 0o600); + assert_eq!(chmodded.attr.size, published.attr.size); + // Generation is the content version; an attribute-only change keeps it. + assert_eq!(chmodded.attr.generation, body_generation); + assert_eq!(chmodded.body.as_ref().unwrap().generation, body_generation); + // The body is still resolvable and intact after the metadata-only update. + assert_eq!( + service.read_file(chmodded.attr.inode, 0, 6).unwrap(), + b"abcdef" + ); + + // A size change still advances the generation (new body content). + let resized = service + .update_attrs( + InodeId::root(), + &name, + UpdateAttr { + size: Some(3), + ..UpdateAttr::default() + }, + ) + .unwrap(); + assert!(resized.attr.generation > body_generation); + assert_eq!( + resized.attr.generation, + resized.body.as_ref().unwrap().generation + ); + assert_eq!(service.read_file(resized.attr.inode, 0, 8).unwrap(), b"abc"); +} + +#[test] +fn replace_publish_refreshes_stale_dentry_version_after_attr_update() { + // Reproduces the cp setattr-mid-write -> release publish CAS: a write handle + // prepares an artifact-replace (pinning the dentry version), then a `setattr` + // (here a chmod via update_attrs) advances the dentry version out-of-band. + // Publishing with the stale pinned version must fail the CAS (PredicateFailed + // -> EIO), and re-reading the live version via `current_dentry_version` (what + // publish_handle now does) before publishing must make it succeed without + // losing the body. + let service = service(); + let name = DentryName::new(b"y.bin".to_vec()).unwrap(); + service + .publish_artifact(artifact_request(name.clone(), "y-v1", b"abcdef")) + .unwrap(); + + // The write handle's prepared-replace, capturing the current dentry version. + let mut prepared = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + let pinned_version = prepared.dentry_version.unwrap(); + + // An intervening chmod advances the dentry version, stranding `prepared`. + service + .update_attrs( + InodeId::root(), + &name, + UpdateAttr { + mode: Some(0o600), + ..UpdateAttr::default() + }, + ) + .unwrap(); + let current_version = service + .current_dentry_version(InodeId::root(), &name) + .unwrap() + .unwrap(); + assert_ne!( + current_version, pinned_version, + "chmod must advance the dentry version" + ); + + let new_body = body_descriptor(prepared.generation, 3); + let new_chunks = vec![one_chunk_manifest(prepared.inode, prepared.generation, 3)]; + + // Publishing with the stale pinned version fails the CAS, exactly the cp EIO. + let stale = service.publish_prepared_artifact( + prepared.clone(), + new_body.clone(), + new_chunks.clone(), + 0o600, + 1000, + 1000, + ); + assert!( + matches!( + stale, + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + ), + "stale dentry version must fail the replace CAS, got {stale:?}" + ); + + // Rebinding the guard to the live version (the publish_handle refresh) lets the + // replace CAS pass and commit the new body. + prepared.dentry_version = Some(current_version); + let published = service + .publish_prepared_artifact(prepared, new_body, new_chunks, 0o600, 1000, 1000) + .unwrap() + .entry; + assert_eq!(published.attr.size, 3); + assert_eq!(published.attr.mode, 0o600); + let committed = service + .stat_path("/y.bin") + .unwrap() + .expect("artifact still resolvable after refreshed publish"); + assert_eq!(committed.attr.inode, published.attr.inode); + assert_eq!(committed.attr.size, 3); + assert_eq!(committed.body.as_ref().unwrap().size, 3); +} + +#[test] +fn update_root_attrs_changes_root_inode_without_dentry_projection() { + let service = service(); + let updated = service + .update_root_attrs(UpdateAttr { + mode: Some(0o700), + uid: Some(42), + gid: Some(43), + ..UpdateAttr::default() + }) + .unwrap(); + + assert_eq!(updated.mode, 0o700); + assert_eq!(updated.uid, 42); + assert_eq!(updated.gid, 43); + assert_eq!(service.get_attr(InodeId::root()).unwrap().unwrap(), updated); +} + +#[test] +fn history_writes_are_snapshot_retention_driven() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + + let before_hot = metadata.metadata_store_stats(); + service + .update_root_attrs(UpdateAttr { + mode: Some(0o700), + ..UpdateAttr::default() + }) + .unwrap(); + let after_hot = metadata.metadata_store_stats(); + assert_eq!( + after_hot.history_write_total - before_hot.history_write_total, + 0 + ); + + let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + assert_eq!(metadata.metadata_store_stats().active_snapshot_pin_total, 1); + let snapshot_attr = service + .get_attr_at_snapshot("/", snapshot.snapshot_id, &[]) + .unwrap() + .unwrap(); + let before_retained = metadata.metadata_store_stats(); + service + .update_root_attrs(UpdateAttr { + mode: Some(0o750), + ..UpdateAttr::default() + }) + .unwrap(); + let after_retained = metadata.metadata_store_stats(); + + assert_eq!( + after_retained.history_write_total - before_retained.history_write_total, + 1 + ); + assert_eq!( + service + .get_attr_at_snapshot("/", snapshot.snapshot_id, &[]) + .unwrap() + .unwrap(), + snapshot_attr + ); + assert_eq!( + service.get_attr(InodeId::root()).unwrap().unwrap().mode, + 0o750 + ); + + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + assert_eq!(metadata.metadata_store_stats().active_snapshot_pin_total, 0); + let before_retired_hot = metadata.metadata_store_stats(); + service + .update_root_attrs(UpdateAttr { + mode: Some(0o755), + ..UpdateAttr::default() + }) + .unwrap(); + let after_retired_hot = metadata.metadata_store_stats(); + assert_eq!( + after_retired_hot.history_write_total - before_retired_hot.history_write_total, + 0 + ); +} + +#[test] +fn create_file_hot_path_write_attribution_is_bounded() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let before = metadata.metadata_store_stats(); + + service + .create_file( + InodeId::root(), + DentryName::new(b"empty.txt".to_vec()).unwrap(), + 0o644, + 1000, + 1000, + ) + .unwrap(); + + let after = metadata.metadata_store_stats(); + assert_eq!(after.commit_total - before.commit_total, 1); + assert_eq!(after.current_put_total - before.current_put_total, 2); + assert_eq!(after.current_delete_total - before.current_delete_total, 0); + assert_eq!(after.history_write_total - before.history_write_total, 0); + assert_eq!(after.watch_write_total - before.watch_write_total, 0); + assert_eq!(after.dedupe_write_total - before.dedupe_write_total, 1); +} + +#[test] +fn create_files_in_dir_coalesces_into_one_metadata_command() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let before = metadata.metadata_store_stats(); + let before_service = service.metadata_service_stats(); + + let entries = service + .create_files_in_dir_path( + "/runs", + vec![ + DentryName::new(b"a.bin".to_vec()).unwrap(), + DentryName::new(b"b.bin".to_vec()).unwrap(), + ], + 0o644, + 1000, + 1000, + ) + .unwrap(); + + let after = metadata.metadata_store_stats(); + let after_service = service.metadata_service_stats(); + assert_eq!(entries.len(), 2); + assert_eq!(after.commit_total - before.commit_total, 1); + assert_eq!(after.current_put_total - before.current_put_total, 4); + assert_eq!(after.current_delete_total - before.current_delete_total, 0); + assert_eq!(after.history_write_total - before.history_write_total, 0); + assert_eq!(after.watch_write_total - before.watch_write_total, 0); + assert_eq!(after.dedupe_write_total - before.dedupe_write_total, 1); + assert_eq!( + after_service.create_files_batch_total - before_service.create_files_batch_total, + 1 + ); + assert_eq!( + after_service.create_files_entry_total - before_service.create_files_entry_total, + 2 + ); + let listed = service.read_dir_plus_path("/runs").unwrap(); + assert_eq!(listed.len(), 2); +} + +#[test] +fn create_dirs_in_dir_coalesces_into_one_metadata_command() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let before = metadata.metadata_store_stats(); + let before_service = service.metadata_service_stats(); + + let entries = service + .create_dirs_in_dir_path( + "/runs", + vec![ + DentryName::new(b"a".to_vec()).unwrap(), + DentryName::new(b"b".to_vec()).unwrap(), + ], + 0o755, + 1000, + 1000, + ) + .unwrap(); + + let after = metadata.metadata_store_stats(); + let after_service = service.metadata_service_stats(); + assert_eq!(entries.len(), 2); + assert!(entries + .iter() + .all(|entry| entry.attr.file_type == FileType::Directory)); + assert_eq!(after.commit_total - before.commit_total, 1); + assert_eq!(after.current_put_total - before.current_put_total, 4); + assert_eq!(after.current_delete_total - before.current_delete_total, 0); + assert_eq!(after.history_write_total - before.history_write_total, 0); + assert_eq!(after.watch_write_total - before.watch_write_total, 0); + assert_eq!(after.dedupe_write_total - before.dedupe_write_total, 1); + assert_eq!( + after_service.create_dirs_batch_total - before_service.create_dirs_batch_total, + 1 + ); + assert_eq!( + after_service.create_dirs_entry_total - before_service.create_dirs_entry_total, + 2 + ); + let listed = service.read_dir_plus_path("/runs").unwrap(); + assert_eq!(listed.len(), 2); +} + +#[test] +fn remove_files_in_dir_coalesces_into_one_holt_apply() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + service + .create_files_in_dir_path( + "/runs", + vec![ + DentryName::new(b"a.bin".to_vec()).unwrap(), + DentryName::new(b"b.bin".to_vec()).unwrap(), + DentryName::new(b"keep.bin".to_vec()).unwrap(), + ], + 0o644, + 1000, + 1000, + ) + .unwrap(); + let before = metadata.metadata_store_stats(); + + let removed = service + .remove_files_in_dir_path( + "/runs", + vec![ + DentryName::new(b"a.bin".to_vec()).unwrap(), + DentryName::new(b"b.bin".to_vec()).unwrap(), + ], + ) + .unwrap(); + + let after = metadata.metadata_store_stats(); + assert_eq!(removed.len(), 2); + assert!(removed.iter().all(Result::is_ok)); + assert_eq!(after.commit_total - before.commit_total, 2); + assert_eq!(after.current_delete_total - before.current_delete_total, 4); + assert_eq!(after.history_write_total - before.history_write_total, 0); + assert_eq!(after.watch_write_total - before.watch_write_total, 0); + assert_eq!(after.dedupe_write_total - before.dedupe_write_total, 2); + assert_eq!(after.atomic_apply_total - before.atomic_apply_total, 1); + assert_eq!( + after.atomic_apply_command_total - before.atomic_apply_command_total, + 2 + ); + let listed = service.read_dir_plus_path("/runs").unwrap(); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].dentry.name.as_bytes(), b"keep.bin"); +} + +#[test] +fn remove_empty_dirs_in_dir_coalesces_into_one_holt_apply() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + service + .create_dir_batches_in_dir_path(vec![CreateInDirPathBatch { + parent_path: "/runs".to_owned(), + names: vec![ + DentryName::new(b"a".to_vec()).unwrap(), + DentryName::new(b"b".to_vec()).unwrap(), + DentryName::new(b"keep".to_vec()).unwrap(), + ], + mode: 0o755, + uid: 1000, + gid: 1000, + }]) + .remove(0) + .unwrap(); + let before = metadata.metadata_store_stats(); + + let removed = service + .remove_empty_dirs_in_dir_path( + "/runs", + vec![ + DentryName::new(b"a".to_vec()).unwrap(), + DentryName::new(b"b".to_vec()).unwrap(), + ], + ) + .unwrap(); + + let after = metadata.metadata_store_stats(); + assert_eq!(removed.len(), 2); + assert!(removed[0].is_ok()); + assert!(removed[1].is_ok()); + assert_eq!(after.commit_total - before.commit_total, 2); + assert_eq!(after.current_delete_total - before.current_delete_total, 4); + assert_eq!(after.history_write_total - before.history_write_total, 0); + assert_eq!(after.watch_write_total - before.watch_write_total, 0); + assert_eq!(after.dedupe_write_total - before.dedupe_write_total, 2); + assert_eq!(after.atomic_apply_total - before.atomic_apply_total, 1); + assert_eq!( + after.atomic_apply_command_total - before.atomic_apply_command_total, + 2 + ); + let listed = service.read_dir_plus_path("/runs").unwrap(); + let names = listed + .iter() + .map(|entry| entry.dentry.name.as_bytes()) + .collect::>(); + assert_eq!(names, vec![b"keep".as_slice()]); +} + +#[test] +fn read_dir_plus_page_returns_cursor_without_materializing_full_directory() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + service + .create_files_in_dir_path( + "/runs", + vec![ + DentryName::new(b"a.bin".to_vec()).unwrap(), + DentryName::new(b"b.bin".to_vec()).unwrap(), + DentryName::new(b"c.bin".to_vec()).unwrap(), + ], + 0o644, + 1000, + 1000, + ) + .unwrap(); + let runs = service.lookup_path("/runs").unwrap().unwrap(); + + let before_store = metadata.metadata_store_stats(); + let first = service + .read_dir_plus_page(runs.attr.inode, None, 2) + .unwrap(); + let after_first_store = metadata.metadata_store_stats(); + assert_eq!( + first + .entries + .iter() + .map(|entry| entry.dentry.name.as_bytes()) + .collect::>(), + vec![b"a.bin".as_slice(), b"b.bin".as_slice()] + ); + assert_eq!( + first.next_cursor.as_ref().map(DentryName::as_bytes), + Some(b"b.bin".as_slice()) + ); + assert_eq!( + after_first_store.scan_key_returned_total - before_store.scan_key_returned_total, + 3 + ); + + let before_service = service.metadata_service_stats(); + let second = service + .read_dir_plus_page(runs.attr.inode, first.next_cursor.as_ref(), 2) + .unwrap(); + let after_service = service.metadata_service_stats(); + assert_eq!( + second + .entries + .iter() + .map(|entry| entry.dentry.name.as_bytes()) + .collect::>(), + vec![b"c.bin".as_slice()] + ); + assert_eq!(second.next_cursor, None); + assert_eq!( + after_service.read_dir_plus_entry_total - before_service.read_dir_plus_entry_total, + 1 + ); + assert_eq!( + after_service.read_dir_plus_projection_hit_total + - before_service.read_dir_plus_projection_hit_total, + 1 + ); +} + +#[test] +fn publish_artifact_stores_body_then_publishes_metadata() { + let service = service(); + let name = DentryName::new(b"checkpoint.json".to_vec()).unwrap(); + let before_publish = service.object_stats(); + let published = service + .publish_artifact(PublishArtifact { + content_type: "application/json".to_owned(), + ..artifact_request(name.clone(), "runs/1/checkpoint.json", b"{\"x\":1}") + }) + .unwrap(); + assert_eq!( + service.object_stats().object_put_bytes, + before_publish.object_put_bytes + 7 + ); + + let lookup = service + .lookup_plus(InodeId::root(), &name) + .unwrap() + .unwrap(); + assert_eq!(lookup, published); + assert_eq!(lookup.attr.size, 7); + assert_eq!( + lookup.body.as_ref().unwrap().manifest_id, + "runs/1/checkpoint.json" + ); + + let bytes = service + .read_artifact(InodeId::root(), &name) + .expect("read artifact body"); + assert_eq!(bytes, b"{\"x\":1}"); + + let body = service + .body_descriptor(published.attr.inode) + .expect("read body descriptor") + .expect("body descriptor exists"); + assert_eq!(body.manifest_id, "runs/1/checkpoint.json"); + assert_eq!(body.generation, published.attr.generation); + let range = service + .read_file(published.attr.inode, 2, 3) + .expect("read file range"); + assert_eq!(range, b"x\":"); + let before_cache = service.object_stats(); + let cached = service + .read_file(published.attr.inode, 2, 3) + .expect("read cached file range"); + assert_eq!(cached, b"x\":"); + let after_cache = service.object_stats(); + assert!(after_cache.cache_hits > before_cache.cache_hits); + assert_eq!( + after_cache.cache_hit_bytes, + before_cache.cache_hit_bytes + 3 + ); +} + +#[test] +fn read_file_uses_one_attr_read_for_body_and_manifest_plan() { + let metadata = PurposeTrackingStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let published = service + .publish_artifact(artifact_request( + DentryName::new(b"checkpoint.bin".to_vec()).unwrap(), + "checkpoint/body", + b"abcdef", + )) + .unwrap(); + + let before = metadata.counts(); + assert_eq!( + service.read_file(published.attr.inode, 1, 3).unwrap(), + b"bcd" + ); + let after = metadata.counts(); + assert_eq!( + after.user_strong_gets - before.user_strong_gets, + 3, + "read_file should read inode, body summary, and one chunk manifest" + ); + assert_eq!(after.write_plan_gets, before.write_plan_gets); +} + +#[test] +fn read_artifact_uses_dentry_projection_body_descriptor() { + let metadata = PurposeTrackingStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + service + .publish_artifact(artifact_request(name.clone(), "checkpoint/body", b"abcdef")) + .unwrap(); + + let before = metadata.counts(); + assert_eq!( + service.read_artifact(InodeId::root(), &name).unwrap(), + b"abcdef" + ); + let after = metadata.counts(); + assert_eq!( + after.user_strong_gets - before.user_strong_gets, + 2, + "read_artifact should read dentry projection and one chunk manifest" + ); + assert_eq!(after.write_plan_gets, before.write_plan_gets); +} + +#[test] +fn open_path_read_plan_uses_dentry_projection_body_descriptor() { + let metadata = PurposeTrackingStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let published = service + .publish_artifact(artifact_request( + DentryName::new(b"checkpoint.bin".to_vec()).unwrap(), + "checkpoint/body", + b"abcdef", + )) + .unwrap(); + + let before = metadata.counts(); + let open = service + .open_path_read_plan("/checkpoint.bin", 1, 3, Some(published.attr.generation)) + .unwrap(); + let after = metadata.counts(); + assert_eq!(open.metadata.attr.inode, published.attr.inode); + assert_eq!(open.plan.output_len, 3); + assert_eq!(open.plan.blocks.len(), 1); + assert_eq!( + after.user_strong_gets - before.user_strong_gets, + 2, + "open_path_read_plan should read dentry projection and one chunk manifest" + ); + assert_eq!(after.write_plan_gets, before.write_plan_gets); +} + +#[test] +fn open_path_read_plan_batch_uses_one_read_version() { + let metadata = PurposeTrackingStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let first = service + .publish_artifact(artifact_request( + DentryName::new(b"sample-0.bin".to_vec()).unwrap(), + "dataset/sample-0", + b"abcdefgh", + )) + .unwrap(); + let second = service + .publish_artifact(artifact_request( + DentryName::new(b"sample-1.bin".to_vec()).unwrap(), + "dataset/sample-1", + b"uvwxyz", + )) + .unwrap(); + + let before = metadata.counts(); + let opens = service + .open_path_read_plan_batch(&[ + OpenPathReadPlanRequest { + path: "/sample-0.bin".to_owned(), + offset: 1, + len: 3, + expected_generation: Some(first.attr.generation), + }, + OpenPathReadPlanRequest { + path: "/sample-1.bin".to_owned(), + offset: 2, + len: 2, + expected_generation: Some(second.attr.generation), + }, + ]) + .unwrap(); + let after = metadata.counts(); + + assert_eq!(opens.len(), 2); + assert_eq!(opens[0].metadata.attr.inode, first.attr.inode); + assert_eq!(opens[1].metadata.attr.inode, second.attr.inode); + assert_eq!(opens[0].lease.read_version, opens[1].lease.read_version); + assert_eq!(opens[0].plan.output_len, 3); + assert_eq!(opens[1].plan.output_len, 2); + assert_eq!( + after.user_strong_gets - before.user_strong_gets, + 4, + "batch open should read each dentry projection and chunk manifest once" + ); + assert_eq!(after.write_plan_gets, before.write_plan_gets); +} + +#[test] +fn open_path_read_plan_returns_zero_write_lease_and_projected_plan() { + let metadata = PurposeTrackingStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let published = service + .publish_artifact(artifact_request( + DentryName::new(b"checkpoint.bin".to_vec()).unwrap(), + "checkpoint/body", + b"abcdef", + )) + .unwrap(); + + let before_counts = metadata.counts(); + let before_commits = service.metadata_store_stats().commit_total; + let open = service + .open_path_read_plan("/checkpoint.bin", 1, 3, Some(published.attr.generation)) + .unwrap(); + let after_counts = metadata.counts(); + + assert_eq!(open.metadata.attr.inode, published.attr.inode); + assert_eq!(open.lease.inode, published.attr.inode); + assert_eq!(open.lease.generation, published.attr.generation); + assert!(open.lease.read_version >= published.attr.generation); + assert_eq!(open.plan.output_len, 3); + assert_eq!(open.plan.blocks.len(), 1); + assert_eq!( + service.metadata_store_stats().commit_total, + before_commits, + "layout-open must not persist read state" + ); + assert_eq!( + after_counts.user_strong_gets - before_counts.user_strong_gets, + 2, + "layout-open should read dentry projection and one chunk manifest" + ); + assert_eq!(after_counts.write_plan_gets, before_counts.write_plan_gets); +} + +#[test] +fn read_file_plan_returns_ranges_without_fetching_objects() { + let service = service(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name, "checkpoint/body", b"hello metadata")) + .unwrap(); + let before = service.object_stats(); + let plan = service + .read_file_plan(published.attr.inode, published.attr.generation, 6, 6) + .unwrap(); + assert_eq!(plan.output_len, 6); + assert_eq!(plan.blocks.len(), 1); + assert_eq!(plan.blocks[0].object_offset, 6); + assert_eq!(plan.blocks[0].len, 6); + assert_eq!(plan.blocks[0].output_offset, 0); + assert!(plan.blocks[0].digest_uri.starts_with("xxh3-64:")); + assert_eq!(service.object_stats().object_gets, before.object_gets); + + let stale = service + .read_file_plan(published.attr.inode, published.attr.generation - 1, 0, 1) + .unwrap_err(); + assert!( + matches!(stale, MetadError::StaleBodyGeneration { .. }), + "unexpected error: {stale:?}" + ); +} + +#[test] +fn prepared_artifact_publish_commits_manifest_without_object_fetch() { + let service = service(); + let name = DentryName::new(b"metadata.bin".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + let body = body_descriptor(prepared.generation, 6); + let result = service + .publish_prepared_artifact( + prepared.clone(), + body, + vec![one_chunk_manifest(prepared.inode, prepared.generation, 6)], + 0o644, + 1000, + 1000, + ) + .unwrap(); + assert_eq!(result.replaced, None); + assert_eq!(result.entry.attr.inode, prepared.inode); + let lookup = service + .lookup_plus(InodeId::root(), &name) + .unwrap() + .unwrap(); + assert_eq!(lookup, result.entry); + let plan = service + .read_file_plan(prepared.inode, prepared.generation, 1, 3) + .unwrap(); + assert_eq!(plan.output_len, 3); + assert_eq!(plan.blocks[0].object_offset, 1); +} + +#[test] +fn prepared_artifact_publish_rejects_foreign_block_identity_before_commit() { + let service = service(); + let name = DentryName::new(b"foreign-block.bin".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), name) + .unwrap(); + let body = body_descriptor(prepared.generation, 6); + let foreign_keys = [ + format!( + "blocks/2/{}/{}/0/0", + prepared.inode.get(), + prepared.generation + ), + format!("blocks/1/999/{}/0/0", prepared.generation), + format!( + "blocks/1/{}/{}/0/0", + prepared.inode.get(), + prepared.generation + 1 + ), + format!( + "blocks/1/{}/{}/1/0", + prepared.inode.get(), + prepared.generation + ), + format!( + "blocks/1/{}/{}/0", + prepared.inode.get(), + prepared.generation + ), + ]; + + for object_key in foreign_keys { + let mut chunk = one_chunk_manifest(prepared.inode, prepared.generation, 6); + chunk.slices[0].blocks[0].object_key = object_key; + let err = service + .publish_prepared_artifact( + prepared.clone(), + body.clone(), + vec![chunk], + 0o644, + 1000, + 1000, + ) + .unwrap_err(); + assert!( + matches!(err, MetadError::InvalidPreparedArtifact(_)), + "unexpected error: {err:?}" + ); + assert!(service + .lookup_plus(InodeId::root(), &prepared.name) + .unwrap() + .is_none()); + } + + let mut valid_nonzero_block = one_chunk_manifest(prepared.inode, prepared.generation, 6); + valid_nonzero_block.slices[0].blocks[0].object_key = + block_key(prepared.inode, prepared.generation, 0, 17) + .as_str() + .to_owned(); + service + .publish_prepared_artifact(prepared, body, vec![valid_nonzero_block], 0o644, 1000, 1000) + .unwrap(); +} + +#[test] +fn prepared_artifact_publish_rejects_duplicate_chunk_range() { + let service = service(); + let name = DentryName::new(b"duplicate-chunk.bin".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), name) + .unwrap(); + let body = body_descriptor(prepared.generation, 12); + let chunks = vec![ + one_chunk_manifest(prepared.inode, prepared.generation, 6), + one_chunk_manifest(prepared.inode, prepared.generation, 6), + ]; + + let err = service + .publish_prepared_artifact(prepared, body, chunks, 0o644, 1000, 1000) + .unwrap_err(); + assert!( + matches!(err, MetadError::InvalidPreparedArtifact(_)), + "unexpected error: {err:?}" + ); +} + +#[test] +fn prepared_artifact_publish_rejects_slice_block_gap() { + let service = service(); + let name = DentryName::new(b"slice-gap.bin".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), name) + .unwrap(); + let body = body_descriptor(prepared.generation, 6); + let mut chunk = one_chunk_manifest(prepared.inode, prepared.generation, 6); + chunk.slices[0].blocks[0].len = 3; + + let err = service + .publish_prepared_artifact(prepared, body, vec![chunk], 0o644, 1000, 1000) + .unwrap_err(); + assert!( + matches!(err, MetadError::InvalidPreparedArtifact(_)), + "unexpected error: {err:?}" + ); +} + +#[test] +fn prepared_artifact_publish_rejects_block_larger_than_manifest_block_size() { + let service = service(); + let name = DentryName::new(b"oversized-block.bin".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), name) + .unwrap(); + let mut body = body_descriptor(prepared.generation, 6); + body.block_size = 3; + let chunk = one_chunk_manifest(prepared.inode, prepared.generation, 6); + + let err = service + .publish_prepared_artifact(prepared, body, vec![chunk], 0o644, 1000, 1000) + .unwrap_err(); + assert!( + matches!(err, MetadError::InvalidPreparedArtifact(_)), + "unexpected error: {err:?}" + ); +} + +#[test] +fn prepared_artifact_replace_rejects_stale_dentry_version() { + let service = service(); + let name = DentryName::new(b"replace-metadata.bin".to_vec()).unwrap(); + service + .publish_artifact(artifact_request(name.clone(), "old", b"old")) + .unwrap(); + let prepared = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + service + .replace_artifact(artifact_request(name, "newer", b"newer")) + .unwrap(); + let err = service + .publish_prepared_artifact( + prepared.clone(), + body_descriptor(prepared.generation, 6), + vec![one_chunk_manifest(prepared.inode, prepared.generation, 6)], + 0o644, + 1000, + 1000, + ) + .unwrap_err(); + assert!( + matches!(err, MetadError::Metadata(MetadataError::PredicateFailed)), + "unexpected error: {err:?}" + ); +} + +#[test] +fn prepared_artifact_replace_retry_is_idempotent() { + let service = service(); + let name = DentryName::new(b"retry-metadata.bin".to_vec()).unwrap(); + service + .publish_artifact(artifact_request(name.clone(), "old", b"old")) + .unwrap(); + let prepared = service + .prepare_artifact_replace(InodeId::root(), name) + .unwrap(); + let body = body_descriptor(prepared.generation, 6); + let chunks = vec![one_chunk_manifest(prepared.inode, prepared.generation, 6)]; + let first = service + .publish_prepared_artifact( + prepared.clone(), + body.clone(), + chunks.clone(), + 0o644, + 1000, + 1000, + ) + .unwrap(); + assert!(first.replaced.is_some()); + let second = service + .publish_prepared_artifact(prepared, body, chunks, 0o644, 1000, 1000) + .unwrap(); + assert_eq!(second.entry, first.entry); + assert_eq!(second.replaced, None); +} + +#[test] +fn prepared_artifact_session_uploads_only_dirty_ranges_and_reuses_old_blocks() { + let service = service(); + let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request( + name.clone(), + "artifact.bin", + b"abcdefghij", + )) + .unwrap(); + let before = service.object_stats(); + let prepared = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + let before_scan = service.metadata_store_stats(); + let replaced = service + .publish_prepared_artifact_session( + prepared, + PublishArtifactSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "unknown".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "artifact.bin".to_owned(), + size: 10, + ranges: vec![PublishArtifactRange { + offset: 3, + bytes: b"XYZ".to_vec(), + }], + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + let after = service.object_stats(); + let after_scan = service.metadata_store_stats(); + assert_eq!(after.object_puts, before.object_puts + 1); + assert_eq!(after.object_put_bytes, before.object_put_bytes + 3); + assert_eq!( + after_scan.scan_key_visited_total, + before_scan.scan_key_visited_total + ); + assert_eq!( + after_scan.scan_key_returned_total, + before_scan.scan_key_returned_total + ); + assert_eq!(replaced.entry.attr.inode, published.attr.inode); + assert_eq!( + service.read_file(published.attr.inode, 0, 10).unwrap(), + b"abcXYZghij" + ); + let gc = service.cleanup_pending_objects(10).unwrap(); + assert_eq!(gc.attempted, 0); +} + +#[test] +fn prepared_artifact_session_splits_noncontiguous_dirty_blocks() { + let service = service(); + let name = DentryName::new(b"sparse-dirty.bin".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request( + name.clone(), + "sparse-dirty-v1", + b"abcdefghijklmnop", + )) + .unwrap(); + let prepared = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + + service + .publish_prepared_artifact_session( + prepared, + PublishArtifactSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "unknown".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "sparse-dirty-v2".to_owned(), + size: 16, + ranges: vec![ + PublishArtifactRange { + offset: 2, + bytes: b"XY".to_vec(), + }, + PublishArtifactRange { + offset: 10, + bytes: b"UV".to_vec(), + }, + ], + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + + assert_eq!( + service.read_file(published.attr.inode, 0, 16).unwrap(), + b"abXYefghijUVmnop" + ); +} + +#[test] +fn prepared_artifact_session_replace_exact_retry_does_not_restage_or_reapply() { + let service = service(); + let name = DentryName::new(b"session-retry.bin".to_vec()).unwrap(); + service + .publish_artifact(artifact_request( + name.clone(), + "session-retry-v1", + b"original bytes", + )) + .unwrap(); + let prepared = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + let session = PublishArtifactSession { + parent: prepared.parent, + name: prepared.name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:session-retry".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "session-retry-v2".to_owned(), + size: 12, + ranges: vec![PublishArtifactRange { + offset: 0, + bytes: b"retry bytes!".to_vec(), + }], + mode: 0o640, + uid: 1001, + gid: 1002, + }; + + let first = service + .publish_prepared_artifact_session(prepared.clone(), session.clone()) + .unwrap(); + let object_puts = service.object_puts.load(Ordering::Relaxed); + let stats = service.metadata_store_stats(); + let retry = service + .publish_prepared_artifact_session(prepared, session) + .unwrap(); + + assert_eq!(retry.entry, first.entry); + assert!(retry.replaced.is_none()); + assert_eq!(service.object_puts.load(Ordering::Relaxed), object_puts); + let retry_stats = service.metadata_store_stats(); + assert_eq!(retry_stats.commit_total, stats.commit_total); + assert_eq!(retry_stats.dedupe_write_total, stats.dedupe_write_total); + assert_eq!(retry_stats.current_put_total, stats.current_put_total); + assert_eq!(retry_stats.current_delete_total, stats.current_delete_total); + assert_eq!(retry_stats.history_write_total, stats.history_write_total); + assert_eq!(retry_stats.watch_write_total, stats.watch_write_total); + assert_eq!( + service.read_file(first.entry.attr.inode, 0, 12).unwrap(), + b"retry bytes!" + ); +} + +#[test] +fn prepared_artifact_staged_session_preserves_dirty_slice_overlay() { + let service = service(); + let name = DentryName::new(b"staged-dirty.bin".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request( + name.clone(), + "staged-dirty-v1", + b"abcdefghijklmnop", + )) + .unwrap(); + let prepared = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "staged-dirty-v2", + &[ + PublishArtifactRange { + offset: 2, + bytes: b"XY".to_vec(), + }, + PublishArtifactRange { + offset: 10, + bytes: b"UV".to_vec(), + }, + ], + 0, + ) + .unwrap(); + let staged = written.staged_objects().unwrap(); + let chunks = written.chunk_manifests(); + + service + .publish_prepared_artifact_staged_session( + prepared, + PublishArtifactStagedSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "unknown".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "staged-dirty-v2".to_owned(), + size: 16, + chunks, + staged, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + + assert_eq!( + service.read_file(published.attr.inode, 0, 16).unwrap(), + b"abXYefghijUVmnop" + ); + let metadata = service.lookup_path("/staged-dirty.bin").unwrap().unwrap(); + let body = metadata.body.as_ref().unwrap(); + let manifests = service + .chunk_manifests_for_body_at_version( + published.attr.inode, + body, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(); + assert_eq!(manifests[0].slices.len(), 3); + assert_eq!(manifests[0].slices[1].logical_offset, 2); + assert_eq!(manifests[0].slices[2].logical_offset, 10); +} + +#[test] +fn stat_path_sees_append_after_read_during_prepared_publish_window() { + // Regression for the "concurrent read + append silently drops the last + // append" visibility bug: prepare pre-allocates the commit version (the + // clock bump), so a stat between prepare and publish caches the pre-append + // dentry under the exact version the publish then applies at. The publish + // never advances the clock past that version, so without apply-time cache + // purging the poisoned entry is served for the process lifetime. + let service = service(); + service.create_dir_path("/w", 0o755, 1000, 1000).unwrap(); + + let staged_session = + |prepared: &PreparedArtifact, written: &ChunkedWrite, manifest_id: &str, size: u64| { + PublishArtifactStagedSession { + parent: prepared.parent, + name: prepared.name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "unknown".to_owned(), + content_type: "text/plain".to_owned(), + manifest_id: manifest_id.to_owned(), + size, + chunks: written.chunk_manifests(), + staged: written.staged_objects().unwrap(), + mode: 0o644, + uid: 1000, + gid: 1000, + } + }; + + let prepared = service.prepare_artifact_create_path("/w/log.txt").unwrap(); + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "log-v1", + &[PublishArtifactRange { + offset: 0, + bytes: b"seg0|".to_vec(), + }], + 0, + ) + .unwrap(); + let session = staged_session(&prepared, &written, "log-v1", 5); + service + .publish_prepared_artifact_staged_session(prepared, session) + .unwrap(); + + // Append: the replace prepare allocates the commit version V. + let prepared = service.prepare_artifact_replace_path("/w/log.txt").unwrap(); + // A read inside the staging window resolves at read_version == V and + // caches the pre-append entry under it. + let before = service.stat_path("/w/log.txt").unwrap().unwrap(); + assert_eq!(before.attr.size, 5); + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "log-v2", + &[PublishArtifactRange { + offset: 5, + bytes: b"seg1|".to_vec(), + }], + 0, + ) + .unwrap(); + let session = staged_session(&prepared, &written, "log-v2", 10); + service + .publish_prepared_artifact_staged_session(prepared, session) + .unwrap(); + + // No later write bumps the clock here: the applied publish itself must + // have invalidated the poisoned entry. + let after = service.stat_path("/w/log.txt").unwrap().unwrap(); + assert_eq!(after.attr.size, 10); + assert_eq!( + service + .lookup_path("/w/log.txt") + .unwrap() + .unwrap() + .attr + .size, + 10 + ); +} + +#[test] +fn delta_publish_writes_only_dirty_chunks_and_preserves_base() { + let service = service(); + let name = DentryName::new(b"multi.bin".to_vec()).unwrap(); + + // Generation 1: a two-chunk file (a few bytes in chunk 0 and chunk 1). + let create = service + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + let inode = create.inode; + let g1 = create.generation; + let written = service + .stage_prepared_artifact_ranges( + &create, + "multi-v1", + &[ + PublishArtifactRange { + offset: 0, + bytes: b"aa".to_vec(), + }, + PublishArtifactRange { + offset: DEFAULT_CHUNK_SIZE, + bytes: b"bb".to_vec(), + }, + ], + 0, + ) + .unwrap(); + let staged = written.staged_objects().unwrap(); + let chunks = written.chunk_manifests(); + service + .publish_prepared_artifact_staged_session( + create, + PublishArtifactStagedSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "unknown".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "multi-v1".to_owned(), + size: DEFAULT_CHUNK_SIZE + 2, + chunks, + staged, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + + // Generation 2: overwrite only chunk 0 — a delta over generation 1. + let replace = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + let g2 = replace.generation; + assert_eq!(replace.old_generation, Some(g1)); + let written2 = service + .stage_prepared_artifact_ranges( + &replace, + "multi-v2", + &[PublishArtifactRange { + offset: 0, + bytes: b"XY".to_vec(), + }], + 0, + ) + .unwrap(); + let staged2 = written2.staged_objects().unwrap(); + let chunks2 = written2.chunk_manifests(); + service + .publish_prepared_artifact_staged_session( + replace, + PublishArtifactStagedSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "unknown".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "multi-v2".to_owned(), + size: DEFAULT_CHUNK_SIZE + 2, + chunks: chunks2, + staged: staged2, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + + let version = service.read_version().unwrap(); + let body = service + .lookup_path("/multi.bin") + .unwrap() + .unwrap() + .body + .unwrap(); + assert_eq!(body.generation, g2); + // The delta falls through to generation 1 for untouched chunks. + assert_eq!(body.base_generation, g1); + + // O(write): generation 2 stores ONLY the dirty chunk (chunk 0), not chunk 1. + assert!(service + .chain_chunk_manifest(inode, &[g2], 0, version, ReadPurpose::UserStrong) + .unwrap() + .is_some()); + assert!(service + .chain_chunk_manifest(inode, &[g2], 1, version, ReadPurpose::UserStrong) + .unwrap() + .is_none()); + + // The base generation is preserved intact — not eagerly deleted. + assert!(service + .chain_chunk_manifest(inode, &[g1], 0, version, ReadPurpose::UserStrong) + .unwrap() + .is_some()); + assert!(service + .chain_chunk_manifest(inode, &[g1], 1, version, ReadPurpose::UserStrong) + .unwrap() + .is_some()); + + // Reads resolve across the chain: chunk 0 from the delta, chunk 1 inherited. + assert_eq!(service.read_file(inode, 0, 2).unwrap(), b"XY"); + assert_eq!( + service.read_file(inode, DEFAULT_CHUNK_SIZE, 2).unwrap(), + b"bb" + ); +} + +fn overwrite_staged( + service: &NoKvFs, + prepared: PreparedArtifact, + name: &DentryName, + manifest_id: &str, + offset: u64, + bytes: &[u8], + size: u64, +) { + let parent = prepared.parent; + let written = service + .stage_prepared_artifact_ranges( + &prepared, + manifest_id, + &[PublishArtifactRange { + offset, + bytes: bytes.to_vec(), + }], + 0, + ) + .unwrap(); + let staged = written.staged_objects().unwrap(); + let chunks = written.chunk_manifests(); + service + .publish_prepared_artifact_staged_session( + prepared, + PublishArtifactStagedSession { + parent, + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "unknown".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: manifest_id.to_owned(), + size, + chunks, + staged, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); +} + +#[test] +fn final_unlink_cleans_the_full_owner_generation_chain_and_snapshot_protects_objects() { + let (service, objects) = service_with_objects(); + let workspace = service + .create_dir_path("/unlink-chain", 0o755, 1000, 1000) + .unwrap(); + let name = dname(b"artifact.bin"); + let base = publish_path_artifact( + &service, + "/unlink-chain/artifact.bin", + "unlink-chain/base", + b"base", + ); + let base_body = base.body.as_ref().unwrap(); + let base_generation = base_body.generation; + let base_block = block_key(base.attr.inode, base_generation, 0, 0); + + let append = service + .prepare_artifact_replace(workspace.attr.inode, name.clone()) + .unwrap(); + let append_generation = append.generation; + overwrite_staged( + &service, + append, + &name, + "unlink-chain/append", + base_body.size, + b"+delta", + base_body.size + 6, + ); + let appended = service + .lookup_path("/unlink-chain/artifact.bin") + .unwrap() + .unwrap(); + assert_eq!( + appended.body.as_ref().unwrap().base_generation, + base_generation + ); + let append_block = block_key(appended.attr.inode, append_generation, 0, 0); + assert!(objects.head(&base_block).unwrap().is_some()); + assert!(objects.head(&append_block).unwrap().is_some()); + + let snapshot = service.snapshot_subtree_path("/unlink-chain").unwrap(); + service.remove_file(workspace.attr.inode, &name).unwrap(); + + for generation in [base_generation, append_generation] { + let rows = service + .metadata + .scan(ScanRequest { + family: RecordFamily::ChunkManifest, + prefix: chunk_manifest_prefix(service.mount, appended.attr.inode, generation), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap(); + assert!( + rows.is_empty(), + "final unlink left generation {generation} manifests: {rows:?}" + ); + } + let queued = service + .metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .into_iter() + .map(|row| decode_object_gc_record(&row.value.0).unwrap().object_key) + .collect::>(); + assert!(queued.contains(base_block.as_str())); + assert!(queued.contains(append_block.as_str())); + + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&base_block).unwrap().is_some()); + assert!(objects.head(&append_block).unwrap().is_some()); + + assert!(service + .retire_snapshot_path("/unlink-chain", snapshot.snapshot_id) + .unwrap()); + for _ in 0..8 { + service.cleanup_pending_objects(100).unwrap(); + if objects.head(&base_block).unwrap().is_none() + && objects.head(&append_block).unwrap().is_none() + { + break; + } + } + assert!(objects.head(&base_block).unwrap().is_none()); + assert!(objects.head(&append_block).unwrap().is_none()); +} + +#[test] +fn rename_replace_cleans_the_full_victim_generation_chain() { + let service = service(); + let victim = publish_path_artifact(&service, "/victim.bin", "rename-chain/base", b"victim"); + let victim_body = victim.body.as_ref().unwrap(); + let base_generation = victim_body.generation; + let base_block = block_key(victim.attr.inode, base_generation, 0, 0); + let name = dname(b"victim.bin"); + let append = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + let append_generation = append.generation; + overwrite_staged( + &service, + append, + &name, + "rename-chain/append", + victim_body.size, + b"+delta", + victim_body.size + 6, + ); + let append_block = block_key(victim.attr.inode, append_generation, 0, 0); + let source = publish_path_artifact( + &service, + "/source.bin", + "rename-chain/source", + b"replacement", + ); + + let outcome = service + .rename_replace_path("/source.bin", "/victim.bin") + .unwrap(); + assert_eq!( + outcome.replaced.as_ref().unwrap().attr.inode, + victim.attr.inode + ); + assert_eq!( + service + .lookup_path("/victim.bin") + .unwrap() + .unwrap() + .attr + .inode, + source.attr.inode + ); + for generation in [base_generation, append_generation] { + assert!( + service + .metadata + .scan(ScanRequest { + family: RecordFamily::ChunkManifest, + prefix: chunk_manifest_prefix(service.mount, victim.attr.inode, generation), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty(), + "rename-replace left victim generation {generation} manifests" + ); + } + let queued = service + .metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .into_iter() + .map(|row| decode_object_gc_record(&row.value.0).unwrap().object_key) + .collect::>(); + assert!(queued.contains(base_block.as_str())); + assert!(queued.contains(append_block.as_str())); +} + +#[test] +fn final_unlink_enqueues_a_compaction_retained_old_generation_object() { + let (service, objects) = service_with_objects(); + let name = dname(b"compacted.bin"); + let create = service + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + let inode = create.inode; + let base_generation = create.generation; + let written = service + .stage_prepared_artifact_ranges( + &create, + "final-unlink-compaction/base", + &[ + PublishArtifactRange { + offset: 0, + bytes: b"aa".to_vec(), + }, + PublishArtifactRange { + offset: DEFAULT_CHUNK_SIZE, + bytes: b"bb".to_vec(), + }, + ], + 0, + ) + .unwrap(); + service + .publish_prepared_artifact_staged_session( + create, + PublishArtifactStagedSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:compaction-base".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "final-unlink-compaction/base".to_owned(), + size: DEFAULT_CHUNK_SIZE + 2, + chunks: written.chunk_manifests(), + staged: written.staged_objects().unwrap(), + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + let retained_old_block = block_key(inode, base_generation, 1, 0); + + let mut compacted_generation = None; + for index in 1..=12 { + let replace = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + overwrite_staged( + &service, + replace, + &name, + &format!("final-unlink-compaction/{index}"), + 0, + &[b'a' + index as u8, b'a' + index as u8], + DEFAULT_CHUNK_SIZE + 2, + ); + let body = service + .lookup_path("/compacted.bin") + .unwrap() + .unwrap() + .body + .unwrap(); + if body.generation != base_generation && body.base_generation == 0 { + compacted_generation = Some(body.generation); + break; + } + } + let compacted_generation = compacted_generation.expect("delta chain must compact"); + assert!(objects.head(&retained_old_block).unwrap().is_some()); + assert!(service + .metadata + .scan(ScanRequest { + family: RecordFamily::ChunkManifest, + prefix: chunk_manifest_prefix(service.mount, inode, base_generation), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); + let compacted_manifests = service + .metadata + .scan(ScanRequest { + family: RecordFamily::ChunkManifest, + prefix: chunk_manifest_prefix(service.mount, inode, compacted_generation), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap(); + assert!(compacted_manifests.iter().any(|row| { + chunk_index_from_manifest_key(&row.key) + .ok() + .filter(|chunk| *chunk != BODY_SUMMARY_CHUNK_INDEX) + .and_then(|_| decode_chunk_manifest(&row.value.0).ok()) + .is_some_and(|manifest| { + manifest + .slices + .iter() + .flat_map(|slice| &slice.blocks) + .any(|block| block.object_key == retained_old_block.as_str()) + }) + })); + + service.remove_file(InodeId::root(), &name).unwrap(); + let queued = service + .metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .into_iter() + .map(|row| decode_object_gc_record(&row.value.0).unwrap().object_key) + .collect::>(); + assert!( + queued.contains(retained_old_block.as_str()), + "final unlink must enqueue an old-generation object retained by the compacted manifest" + ); + // Staging the delta chain may install short read leases while it merges + // prior slices. Advance past that grace period, then prove this is not only + // a metadata-queue assertion: the regular durable GC worker deletes the + // retained old-generation object after the final namespace reference dies. + service.set_clock_override_ms( + service + .now_ms() + .saturating_add(DEFAULT_READ_LEASE_MS) + .saturating_add(1), + ); + for _ in 0..64 { + service.cleanup_pending_objects(256).unwrap(); + if objects.head(&retained_old_block).unwrap().is_none() { + break; + } + } + assert!( + objects.head(&retained_old_block).unwrap().is_none(), + "GC must physically delete the compacted manifest's retained old-generation object" + ); +} + +#[test] +fn final_unlink_skips_clone_and_restore_borrowed_objects() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let source = publish_path_artifact( + &service, + "/source/data.bin", + "final-unlink-borrowed/source", + b"borrowed", + ); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + + let clone = service + .clone_subtree_path_into("/source", "/clone") + .unwrap(); + let clone_file = service.lookup_path("/clone/data.bin").unwrap().unwrap(); + let append = service + .prepare_artifact_replace(clone.root, dname(b"data.bin")) + .unwrap(); + let append_generation = append.generation; + overwrite_staged( + &service, + append, + &dname(b"data.bin"), + "final-unlink-borrowed/clone-delta", + source_body.size, + b"+owned", + source_body.size + 6, + ); + let clone_owned_block = block_key(clone_file.attr.inode, append_generation, 0, 0); + service.remove_file_path("/clone/data.bin").unwrap(); + + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + service.remove_file_path("/restored/data.bin").unwrap(); + + let queued = service + .metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .into_iter() + .map(|row| decode_object_gc_record(&row.value.0).unwrap().object_key) + .collect::>(); + assert!(queued.contains(clone_owned_block.as_str())); + assert!( + !queued.contains(source_block.as_str()), + "clone/restore borrowers must not enqueue the source-owned object" + ); +} + +#[test] +fn delta_chain_compacts_to_self_contained_at_depth_threshold() { + let service = service(); + let name = DentryName::new(b"hot.bin".to_vec()).unwrap(); + let create = service + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + let inode = create.inode; + overwrite_staged(&service, create, &name, "hot-0", 0, b"AAAA", 4); + + // Overwrite the same region many times. Each delta extends the fall-through + // chain by one; at the depth threshold the publish must re-materialize a + // self-contained generation (base_generation == 0) instead of growing the + // chain without bound. Every read must stay correct throughout. + let mut saw_compaction = false; + for i in 1..=12u32 { + let replace = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + let byte = b'A' + (i % 16) as u8; + let want = [byte; 4]; + overwrite_staged(&service, replace, &name, &format!("hot-{i}"), 0, &want, 4); + assert_eq!(service.read_file(inode, 0, 4).unwrap(), want.to_vec()); + let body = service + .lookup_path("/hot.bin") + .unwrap() + .unwrap() + .body + .unwrap(); + if body.base_generation == 0 { + saw_compaction = true; + // Compaction must coalesce the hot chunk's accumulated slices, not + // just collapse the chain — otherwise slice count grows unbounded + // across compaction cycles. The fully-overwritten chunk collapses to + // a single newest-wins slice. + let chunk0 = service + .chain_chunk_manifest( + inode, + &[body.generation], + 0, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert_eq!(chunk0.slices.len(), 1); + } + } + assert!( + saw_compaction, + "deep delta chain must compact to a self-contained generation" + ); +} + +#[test] +fn compacting_staged_publish_exact_retry_uses_stable_terminal_identity() { + let service = service(); + let name = DentryName::new(b"compaction-retry.bin".to_vec()).unwrap(); + let create = service + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + let inode = create.inode; + overwrite_staged(&service, create, &name, "compaction-retry-0", 0, b"0000", 4); + for index in 1..8 { + let prepared = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + let bytes = [b'0' + index; 4]; + overwrite_staged( + &service, + prepared, + &name, + &format!("compaction-retry-{index}"), + 0, + &bytes, + 4, + ); + } + + let prepared = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "compaction-retry-8", + &[PublishArtifactRange { + offset: 0, + bytes: b"8888".to_vec(), + }], + 0, + ) + .unwrap(); + let session = PublishArtifactStagedSession { + parent: prepared.parent, + name: prepared.name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:compaction-retry".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "compaction-retry-8".to_owned(), + size: 4, + chunks: written.chunk_manifests(), + staged: written.staged_objects().unwrap(), + mode: 0o644, + uid: 1000, + gid: 1000, + }; + let first = service + .publish_prepared_artifact_staged_session(prepared.clone(), session.clone()) + .unwrap(); + assert_eq!(first.entry.body.as_ref().unwrap().base_generation, 0); + let stats = service.metadata_store_stats(); + + let retry = service + .publish_prepared_artifact_staged_session(prepared, session) + .unwrap(); + + assert_eq!(retry.entry, first.entry); + assert!(retry.replaced.is_none()); + let retry_stats = service.metadata_store_stats(); + assert_eq!(retry_stats.commit_total, stats.commit_total); + assert_eq!(retry_stats.dedupe_write_total, stats.dedupe_write_total); + assert_eq!(retry_stats.current_put_total, stats.current_put_total); + assert_eq!(retry_stats.current_delete_total, stats.current_delete_total); + assert_eq!(retry_stats.history_write_total, stats.history_write_total); + assert_eq!(retry_stats.watch_write_total, stats.watch_write_total); + assert_eq!(service.read_file(inode, 0, 4).unwrap(), b"8888"); +} + +#[test] +fn chain_collapse_gc_is_snapshot_safe() { + let service = service(); + let name = DentryName::new(b"snap.bin".to_vec()).unwrap(); + let create = service + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + let inode = create.inode; + overwrite_staged(&service, create, &name, "snap-0", 0, b"AAAA", 4); + + // Pin generation 1, then overwrite enough to trigger a chain-collapse + // compaction. The compaction enqueues the superseded chain blocks for GC. + let pin = service.snapshot_subtree(InodeId::root()).unwrap(); + for i in 1..=10u32 { + let replace = service + .prepare_artifact_replace(InodeId::root(), name.clone()) + .unwrap(); + let byte = b'A' + (i % 16) as u8; + overwrite_staged( + &service, + replace, + &name, + &format!("snap-{i}"), + 0, + &[byte; 4], + 4, + ); + } + + // The snapshot still resolves generation-1 content, and a GC pass must NOT + // delete any block the snapshot can still reach — the version retention + // floor blocks reclamation of everything enqueued after the snapshot. + assert_eq!( + service + .read_file_at_snapshot("/", pin.snapshot_id, std::slice::from_ref(&name), 0, 4) + .unwrap(), + b"AAAA" + ); + let blocked = service.cleanup_pending_objects(1024).unwrap(); + assert!( + blocked.blocked_by_snapshots > 0, + "snapshot must block reclamation of still-reachable chain blocks" + ); + assert_eq!(blocked.deleted, 0); + // Snapshot read still works after the (blocked) GC pass. + assert_eq!( + service + .read_file_at_snapshot("/", pin.snapshot_id, std::slice::from_ref(&name), 0, 4) + .unwrap(), + b"AAAA" + ); + + // Retiring the snapshot raises the floor; the superseded chain blocks now + // reclaim — proving the whole chain (not just its top) was enqueued. + assert!(service.retire_snapshot(pin.snapshot_id).unwrap()); + let reclaimed = service.cleanup_pending_objects(1024).unwrap(); + assert!( + reclaimed.deleted > 0, + "retiring the snapshot must let superseded chain blocks reclaim" + ); + + // The live file reads correctly throughout (last write was i=10 -> 'K'). + assert_eq!(service.read_file(inode, 0, 4).unwrap(), b"KKKK"); +} + +#[test] +fn replace_artifact_preserves_inode_and_returns_old_body() { + let service = service(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let first = service + .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) + .unwrap(); + let replaced = service + .replace_artifact(artifact_request( + name.clone(), + "checkpoint/new", + b"new-body", + )) + .unwrap(); + + assert_eq!(replaced.entry.attr.inode, first.attr.inode); + assert!(replaced.entry.attr.generation > first.attr.generation); + assert_eq!(replaced.replaced, Some(first.clone())); + assert_eq!( + service.lookup_plus(InodeId::root(), &name).unwrap(), + Some(replaced.entry.clone()) + ); + assert_eq!( + service.read_artifact(InodeId::root(), &name).unwrap(), + b"new-body" + ); + assert_eq!( + replaced.replaced.unwrap().body.unwrap().manifest_id, + "checkpoint/old" + ); +} + +#[test] +fn get_attr_reads_root_inode() { + let service = service(); + let root = service.get_attr(InodeId::root()).unwrap().unwrap(); + assert_eq!(root.inode, InodeId::root()); + assert_eq!(root.file_type, FileType::Directory); +} + +#[test] +fn remove_file_deletes_namespace_and_returns_old_body() { + let service = service(); + let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "artifact.bin", b"old")) + .unwrap(); + + let removed = service.remove_file(InodeId::root(), &name).unwrap(); + assert_eq!(removed, published); + assert_eq!(removed.body.as_ref().unwrap().manifest_id, "artifact.bin"); + assert!(service + .lookup_plus(InodeId::root(), &name) + .unwrap() + .is_none()); + assert!(service.get_attr(removed.attr.inode).unwrap().is_none()); +} + +#[test] +fn hardlink_updates_link_count_and_defers_body_gc_until_last_unlink() { + let (service, objects) = service_with_objects(); + let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); + let link_name = DentryName::new(b"artifact.link".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "artifact.bin", b"old")) + .unwrap(); + let body = published.body.clone().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + + let linked = service + .link(published.attr.inode, InodeId::root(), link_name.clone()) + .unwrap(); + assert_eq!(linked.attr.inode, published.attr.inode); + assert_eq!(linked.attr.nlink, 2); + assert_eq!( + service + .lookup_plus(InodeId::root(), &name) + .unwrap() + .unwrap() + .attr + .nlink, + 2 + ); + assert_eq!( + service + .lookup_plus(InodeId::root(), &link_name) + .unwrap() + .unwrap() + .attr + .nlink, + 2 + ); + + let removed = service.remove_file(InodeId::root(), &name).unwrap(); + assert_eq!(removed.attr.inode, published.attr.inode); + assert!(service + .lookup_plus(InodeId::root(), &name) + .unwrap() + .is_none()); + let remaining = service + .lookup_plus(InodeId::root(), &link_name) + .unwrap() + .unwrap(); + assert_eq!(remaining.attr.nlink, 1); + assert_eq!( + service + .get_attr(published.attr.inode) + .unwrap() + .unwrap() + .nlink, + 1 + ); + assert_eq!( + service.read_artifact(InodeId::root(), &link_name).unwrap(), + b"old" + ); + assert!(objects.head(&object).unwrap().is_some()); + assert_eq!( + service.cleanup_pending_objects(100).unwrap(), + PendingObjectCleanupOutcome::default() + ); + + let removed_last = service.remove_file(InodeId::root(), &link_name).unwrap(); + assert_eq!(removed_last.attr.inode, published.attr.inode); + assert!(service.get_attr(published.attr.inode).unwrap().is_none()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&object).unwrap().is_none()); +} + +#[test] +fn hardlink_rejects_directories() { + let service = service(); + let dir = service + .create_dir( + InodeId::root(), + DentryName::new(b"dir".to_vec()).unwrap(), + 0o755, + 1000, + 1000, + ) + .unwrap(); + let err = service + .link( + dir.attr.inode, + InodeId::root(), + DentryName::new(b"dir-link".to_vec()).unwrap(), + ) + .unwrap_err(); + assert!(matches!(err, MetadError::NotFile)); +} + +#[test] +fn remove_file_queues_old_body_for_object_cleanup() { + let (service, objects) = service_with_objects(); + let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "artifact.bin", b"old")) + .unwrap(); + let body = published.body.clone().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + assert!(objects.head(&object).unwrap().is_some()); + + let removed = service.remove_file(InodeId::root(), &name).unwrap(); + assert_eq!(removed, published); + assert!(objects.head(&object).unwrap().is_some()); + + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.scanned, 1); + assert_eq!(cleanup.attempted, 1); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.missing, 0); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&object).unwrap().is_none()); + assert_eq!( + service.cleanup_pending_objects(100).unwrap(), + PendingObjectCleanupOutcome::default() + ); +} + +#[test] +fn gc_uses_the_canonical_nonzero_object_block_index() { + let (service, objects) = service_with_objects(); + let name = DentryName::new(b"sparse-gc.bin".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + let inode = prepared.inode; + let generation = prepared.generation; + let block_index = 7_u64; + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "sparse-gc", + &[PublishArtifactRange { + offset: 0, + bytes: b"sparse".to_vec(), + }], + block_index, + ) + .unwrap(); + let staged = written.staged_objects().unwrap(); + let chunks = written.chunk_manifests(); + service + .publish_prepared_artifact_staged_session( + prepared, + PublishArtifactStagedSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "unknown".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "sparse-gc".to_owned(), + size: 6, + chunks, + staged, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + + let object = block_key(inode, generation, 0, block_index); + assert!(objects.head(&object).unwrap().is_some()); + service.remove_file(InodeId::root(), &name).unwrap(); + + let rows = service + .metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap(); + assert_eq!(rows.len(), 1); + let key = decode_object_gc_record_key(service.mount, &rows[0].key).unwrap(); + assert_eq!(key.chunk_index, 0); + assert_eq!(key.block_index, block_index); + + let cleanup = service.cleanup_pending_objects(10).unwrap(); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&object).unwrap().is_none()); + assert!(service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: object_gc_quarantine_prefix(service.mount), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); +} + +#[test] +fn durable_object_gc_claim_codec_round_trips_and_rejects_trailing_bytes() { + let mount = MountId::new(1).unwrap(); + let deleting = ObjectGcClaim::Deleting { + owner_epoch: 7, + operation_token: 11, + gc_record_key: gc_object_key(mount, 10, InodeId::new(2).unwrap(), 3, 4, 5), + gc_record_version: 13, + }; + assert_eq!( + decode_object_gc_claim( + mount, + &encode_object_gc_claim(&ObjectGcClaim::Open).unwrap() + ) + .unwrap(), + ObjectGcClaim::Open + ); + assert_eq!( + decode_object_gc_claim(mount, &encode_object_gc_claim(&deleting).unwrap()).unwrap(), + deleting + ); + let mut malformed = encode_object_gc_claim(&deleting).unwrap(); + malformed.push(0); + assert!(matches!( + decode_object_gc_claim(mount, &malformed), + Err(MetadError::Codec(_)) + )); +} + +#[test] +fn durable_object_gc_claim_codec_rejects_zero_identity_fields() { + let mount = MountId::new(1).unwrap(); + let gc_record_key = gc_object_key(mount, 10, InodeId::new(2).unwrap(), 3, 4, 5); + for invalid in [ + ObjectGcClaim::Deleting { + owner_epoch: 0, + operation_token: 11, + gc_record_key: gc_record_key.clone(), + gc_record_version: 13, + }, + ObjectGcClaim::Deleting { + owner_epoch: 7, + operation_token: 0, + gc_record_key: gc_record_key.clone(), + gc_record_version: 13, + }, + ObjectGcClaim::Deleting { + owner_epoch: 7, + operation_token: 11, + gc_record_key: gc_record_key.clone(), + gc_record_version: 0, + }, + ] { + assert!(matches!( + decode_object_gc_claim(mount, &encode_object_gc_claim(&invalid).unwrap()), + Err(MetadError::Codec(_)) + )); + } +} + +#[test] +fn durable_object_gc_claim_codec_rejects_non_local_or_malformed_gc_keys() { + let mount = MountId::new(1).unwrap(); + let valid_key = gc_object_key(mount, 10, InodeId::new(2).unwrap(), 3, 4, 5); + let mut invalid_keys = vec![ + gc_object_key( + MountId::new(2).unwrap(), + 10, + InodeId::new(2).unwrap(), + 3, + 4, + 5, + ), + gc_queue_prefix(mount), + ]; + for field in [1, 2, 3] { + let mut key = valid_key.clone(); + key[field * 8..(field + 1) * 8].fill(0); + invalid_keys.push(key); + } + + for gc_record_key in invalid_keys { + let invalid = ObjectGcClaim::Deleting { + owner_epoch: 7, + operation_token: 11, + gc_record_key, + gc_record_version: 13, + }; + assert!(matches!( + decode_object_gc_claim(mount, &encode_object_gc_claim(&invalid).unwrap()), + Err(MetadError::Codec(_)) + )); + } +} + +#[test] +fn durable_gc_claim_blocks_new_object_reference_planning_while_deleting() { + let metadata = PausingObjectGcStore::new(); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let published = service + .publish_artifact(artifact_request( + DentryName::new(b"old".to_vec()).unwrap(), + "old", + b"old", + )) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + let resize_name = DentryName::new(b"resize".to_vec()).unwrap(); + service + .publish_artifact(artifact_request( + resize_name.clone(), + "resize", + b"resize-body", + )) + .unwrap(); + service.remove_file_path("/old").unwrap(); + + metadata.arm(); + let cleaner = { + let service = Arc::clone(&service); + std::thread::spawn(move || service.cleanup_pending_objects(1)) + }; + metadata.wait_until_reached(); + + assert!(matches!( + service.prepare_artifact_create(InodeId::root(), DentryName::new(b"new".to_vec()).unwrap()), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!(matches!( + service.publish_artifact(artifact_request( + DentryName::new(b"new-publish".to_vec()).unwrap(), + "new-publish", + b"new" + )), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!(matches!( + service.create_symlink( + InodeId::root(), + DentryName::new(b"new-link".to_vec()).unwrap(), + b"target".to_vec(), + 0o777, + 1000, + 1000 + ), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!(matches!( + service.create_symlink( + InodeId::root(), + DentryName::new(b"invalid-link".to_vec()).unwrap(), + Vec::new(), + 0o777, + 1000, + 1000 + ), + Err(MetadError::InvalidPath(_)) + )); + assert!(matches!( + service.update_attrs( + InodeId::root(), + &resize_name, + UpdateAttr { + size: Some(1), + ..UpdateAttr::default() + } + ), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!(matches!( + service.publish_checkpoint( + InodeId::root(), + vec![CheckpointShard { + name: DentryName::new(b"checkpoint-shard".to_vec()).unwrap(), + bytes: b"checkpoint".to_vec(), + }], + 1000, + 1000 + ), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!(matches!( + service.snapshot_subtree(InodeId::root()), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + let attr_only = service + .update_attrs( + InodeId::root(), + &resize_name, + UpdateAttr { + mode: Some(0o600), + ..UpdateAttr::default() + }, + ) + .unwrap(); + assert_eq!(attr_only.attr.mode, 0o600); + metadata.release(); + let cleaned = cleaner.join().unwrap().unwrap(); + assert_eq!(cleaned.deleted, 1); + assert!(objects.head(&object).unwrap().is_none()); +} + +#[test] +fn gc_recheck_preserves_an_object_referenced_by_the_current_manifest() { + let (service, objects) = service_with_objects(); + let name = DentryName::new(b"live".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "live", b"live")) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: published.attr.inode, + generation: body.generation, + object_key: object.as_str().to_owned(), + size: body.size, + digest_uri: body.digest_uri.clone(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + + let claim_key = object_gc_claim_key(service.mount); + let claim_version_before = service + .metadata + .get_versioned( + RecordFamily::System, + &claim_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap() + .version; + + let cleanup = service.cleanup_pending_objects(1).unwrap(); + assert_eq!(cleanup.blocked_by_snapshots, 1); + assert_eq!(cleanup.deleted, 0); + assert_eq!(cleanup.records_removed, 0); + assert!(objects.head(&object).unwrap().is_some()); + assert_eq!( + service.read_artifact(InodeId::root(), &name).unwrap(), + b"live" + ); + + // The first pass advances the durable cursor, the second reaches the tail + // and resets it, and the third sees the protected row again. None of those + // advisory scans may rotate the global object-reference epoch. + assert_eq!(service.cleanup_pending_objects(1).unwrap().scanned, 0); + assert_eq!( + service + .cleanup_pending_objects(1) + .unwrap() + .blocked_by_snapshots, + 1 + ); + let claim_version_after = service + .metadata + .get_versioned( + RecordFamily::System, + &claim_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap() + .version; + assert_eq!(claim_version_after, claim_version_before); +} + +#[test] +fn snapshot_mint_retries_after_an_intervening_object_delete_epoch() { + let store = SnapshotCommitBarrierStore::new(CommandKind::SnapshotSubtree, 1, 2); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"snapshot-race".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "snapshot-race", b"old")) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + + let mint_service = Arc::clone(&service); + let mint = std::thread::spawn(move || mint_service.snapshot_subtree(InodeId::root())); + store.wait_until_blocked(); + service.remove_file(InodeId::root(), &name).unwrap(); + assert_eq!(service.cleanup_pending_objects(1).unwrap().deleted, 1); + assert!(objects.head(&object).unwrap().is_none()); + store.release_blocked(); + + let snapshot = mint.join().unwrap().unwrap(); + assert!(matches!( + service.read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/snapshot-race"), + Err(MetadError::NotFound) + )); +} + +#[test] +fn reopen_resumes_a_durable_object_gc_claim_after_crash() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let published = service + .publish_artifact(artifact_request( + DentryName::new(b"old".to_vec()).unwrap(), + "old", + b"old", + )) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file_path("/old").unwrap(); + let gc_row = metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .pop() + .unwrap(); + let claim_key = leave_object_gc_deleting(&service, &gc_row); + drop(service); + + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + assert!(objects.head(&object).unwrap().is_some()); + let deferred_claim = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert!(matches!( + decode_object_gc_claim(reopened.mount, &deferred_claim.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); + + let cleanup = reopened.cleanup_pending_objects(1).unwrap(); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&object).unwrap().is_none()); + let open_claim = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert_eq!( + decode_object_gc_claim(reopened.mount, &open_claim.0).unwrap(), + ObjectGcClaim::Open + ); +} + +#[test] +fn history_cleanup_does_not_invalidate_a_prepared_object_upload() { + let (service, _) = service_with_objects(); + let name = DentryName::new(b"upload-during-history-prune.bin".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + + service.cleanup_history(100).unwrap(); + assert_eq!( + service + .refresh_prepared_artifact_object_gc_epoch(prepared.clone()) + .unwrap(), + prepared + ); + + let bytes = b"history-prune-independent"; + service + .publish_prepared_artifact_session( + prepared, + PublishArtifactSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:history-prune-independent".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "history-prune-independent".to_owned(), + size: bytes.len() as u64, + ranges: vec![PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + assert_eq!( + service.read_artifact(InodeId::root(), &name).unwrap(), + bytes + ); +} + +#[test] +fn prepared_publish_rejects_an_intervening_object_gc_epoch_and_refreshes_identity() { + let (service, objects) = service_with_objects(); + let victim_name = DentryName::new(b"gc-victim".to_vec()).unwrap(); + service + .publish_artifact(artifact_request( + victim_name.clone(), + "gc-victim", + b"victim", + )) + .unwrap(); + service.remove_file(InodeId::root(), &victim_name).unwrap(); + + let name = DentryName::new(b"late-after-gc".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + let bytes = b"late-after-gc"; + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "late-after-gc", + &[PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + 0, + ) + .unwrap(); + let staged = written.staged_objects().unwrap(); + + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 1); + let error = service + .publish_prepared_artifact_staged_session( + prepared.clone(), + PublishArtifactStagedSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:late-after-gc".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "late-after-gc".to_owned(), + size: bytes.len() as u64, + chunks: written.chunk_manifests(), + staged, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap_err(); + let staged = match error { + MetadError::PublishArtifactFailed { source, staged } => { + assert!(matches!( + *source, + MetadError::StalePreparedArtifactObjectGcEpoch { .. } + )); + staged + } + error => panic!("unexpected prepared publish error: {error:?}"), + }; + assert!(service.lookup_path("/late-after-gc").unwrap().is_none()); + let cleanup = service.cleanup_staged_objects(&staged).unwrap(); + assert_eq!(cleanup.deleted, staged.len()); + for object in staged.objects() { + assert!(objects.head(&object.key).unwrap().is_none()); + } + + let refreshed = service + .refresh_prepared_artifact_object_gc_epoch(prepared.clone()) + .unwrap(); + assert!(refreshed.generation > prepared.generation); + assert_ne!( + refreshed.object_gc_claim_version, + prepared.object_gc_claim_version + ); +} + +#[test] +fn failover_claim_rotation_rejects_old_prepared_upload_and_allows_cleanup() { + let (source, objects) = service_with_objects(); + let name = dname(b"prepared-before-failover.bin"); + let prepared = source + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + let bytes = b"staged by the failed owner"; + let written = source + .stage_prepared_artifact_ranges( + &prepared, + "prepared-before-failover", + &[PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + 0, + ) + .unwrap(); + let staged_before_failover = written.staged_objects().unwrap(); + let staged_key = staged_before_failover.objects()[0].key.clone(); + assert!(objects.head(&staged_key).unwrap().is_some()); + + let checkpoint = MetadataArchiveConfig::new("meta/prepared-failover", 2); + source.backup_metadata(&checkpoint).unwrap(); + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + // Match controlled startup: install the acquired epoch before restoring the + // old owner's checkpoint, recover any crash-left deletion, then rotate Open. + recovered.install_owner_epoch(2).unwrap(); + recovered.restore_metadata(&checkpoint).unwrap().unwrap(); + recovered.recover_object_gc_claim().unwrap(); + recovered.rotate_object_gc_claim_for_failover().unwrap(); + + let error = recovered + .publish_prepared_artifact_staged_session( + prepared, + PublishArtifactStagedSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:prepared-before-failover".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "prepared-before-failover".to_owned(), + size: bytes.len() as u64, + chunks: written.chunk_manifests(), + staged: staged_before_failover, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap_err(); + let staged = match error { + MetadError::PublishArtifactFailed { source, staged } => { + assert!(matches!( + *source, + MetadError::StalePreparedArtifactObjectGcEpoch { .. } + | MetadError::Metadata(MetadataError::PredicateFailed) + )); + staged + } + other => panic!("unexpected old prepared publish error: {other:?}"), + }; + assert!(recovered + .lookup_plus(InodeId::root(), &name) + .unwrap() + .is_none()); + recovered.cleanup_staged_objects(&staged).unwrap(); + assert!(objects.head(&staged_key).unwrap().is_none()); +} + +#[test] +fn failover_claim_rotation_confirms_a_lost_backend_ack_by_exact_readback() { + let metadata = PostCommitErrorStore::new_disarmed(CommandKind::CleanupObjects); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), dname(b"old-token.bin")) + .unwrap(); + let old_claim_version = prepared.object_gc_claim_version; + + metadata.arm(); + service.rotate_object_gc_claim_for_failover().unwrap(); + let refreshed = service + .refresh_prepared_artifact_object_gc_epoch(prepared) + .unwrap(); + assert_ne!(refreshed.object_gc_claim_version, old_claim_version); +} + +#[test] +fn blocked_head_gc_row_does_not_starve_later_reclaimable_candidate() { + let (service, objects) = service_with_objects(); + let live_name = DentryName::new(b"live-head".to_vec()).unwrap(); + let live = service + .publish_artifact(artifact_request(live_name, "live-head", b"live")) + .unwrap(); + let live_body = live.body.as_ref().unwrap(); + let live_object = block_key(live.attr.inode, live_body.generation, 0, 0); + enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: live.attr.inode, + generation: live_body.generation, + object_key: live_object.as_str().to_owned(), + size: live_body.size, + digest_uri: live_body.digest_uri.clone(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + + let reclaimable = ObjectKey::new("blocks/1/900/1/0/0").unwrap(); + objects.put(&reclaimable, b"stale".to_vec()).unwrap(); + enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: InodeId::new(900).unwrap(), + generation: 1, + object_key: reclaimable.as_str().to_owned(), + size: 5, + digest_uri: "sha256:stale".to_owned(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + + let first = service.cleanup_pending_objects(1).unwrap(); + assert_eq!(first.scanned, 1); + assert_eq!(first.blocked_by_snapshots, 1); + assert_eq!(first.deleted, 0); + assert!(objects.head(&live_object).unwrap().is_some()); + assert!(objects.head(&reclaimable).unwrap().is_some()); + + let second = service.cleanup_pending_objects(1).unwrap(); + assert_eq!(second.scanned, 1); + assert_eq!(second.deleted, 1); + assert!(objects.head(&live_object).unwrap().is_some()); + assert!(objects.head(&reclaimable).unwrap().is_none()); +} + +#[test] +fn local_gc_row_key_cannot_delete_an_object_owned_by_another_mount() { + let (service, objects) = service_with_objects(); + let foreign_object = ObjectKey::new("blocks/2/902/1/0/0").unwrap(); + objects.put(&foreign_object, b"foreign".to_vec()).unwrap(); + let row_key = enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: InodeId::new(902).unwrap(), + generation: 1, + object_key: foreign_object.as_str().to_owned(), + size: 7, + digest_uri: "sha256:foreign".to_owned(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + assert_eq!(row_key.len(), 48); + assert!(row_key.starts_with(&gc_queue_prefix(service.mount))); + + let outcome = service.cleanup_pending_objects(1).unwrap(); + assert_eq!(outcome.attempted, 0); + assert_eq!(outcome.deleted, 0); + assert_eq!(outcome.records_removed, 1); + assert!(objects.head(&foreign_object).unwrap().is_some()); + assert!(service + .metadata + .get( + RecordFamily::Gc, + &row_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + assert_eq!( + service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: object_gc_quarantine_prefix(service.mount), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn gc_row_key_cannot_delete_a_different_block_of_the_same_generation() { + let (service, objects) = service_with_objects(); + let inode = InodeId::new(904).unwrap(); + let object = ObjectKey::new("blocks/1/904/1/0/0").unwrap(); + objects.put(&object, b"protected".to_vec()).unwrap(); + + let version = service.next_version().unwrap(); + let row_key = gc_object_key(service.mount, version.get(), inode, 1, 0, 1); + let record = ObjectGcRecord { + inode, + generation: 1, + object_key: object.as_str().to_owned(), + size: 9, + digest_uri: "sha256:protected".to_owned(), + enqueue_version: version.get(), + enqueue_unix_ms: service.now_ms(), + }; + service + .commit_metadata(MetadataCommand { + request_id: b"mismatched-block-gc-row".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Gc, + primary_key: row_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::Gc, + key: row_key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::Gc, + key: row_key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_record(&record))), + }], + watch: Vec::new(), + }) + .unwrap(); + + let outcome = service.cleanup_pending_objects(1).unwrap(); + assert_eq!(outcome.attempted, 0); + assert_eq!(outcome.deleted, 0); + assert_eq!(outcome.records_removed, 1); + assert!(objects.head(&object).unwrap().is_some()); + assert!(service + .metadata + .get( + RecordFamily::Gc, + &row_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + assert_eq!( + service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: object_gc_quarantine_prefix(service.mount), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn crash_left_gc_claim_quarantines_a_foreign_object_key_without_deleting_it() { + let (service, objects) = service_with_objects(); + let metadata = service.metadata.clone(); + let foreign_object = ObjectKey::new("blocks/2/903/1/0/0").unwrap(); + objects.put(&foreign_object, b"foreign".to_vec()).unwrap(); + let row_key = enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: InodeId::new(903).unwrap(), + generation: 1, + object_key: foreign_object.as_str().to_owned(), + size: 7, + digest_uri: "sha256:foreign".to_owned(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + let row = metadata + .get_versioned( + RecordFamily::Gc, + &row_key, + service.read_version().unwrap(), ReadPurpose::WritePlanLocal, ) .unwrap() .unwrap(); - let version = service.next_version().unwrap(); - service - .commit_metadata(MetadataCommand { - request_id: request_id( - b"test-leave-object-gc-deleting", - service.mount, - InodeId::root(), - version, - ), - kind: CommandKind::CleanupObjects, - read_version: predecessor(version).unwrap(), - commit_version: version, - primary_family: RecordFamily::System, - primary_key: claim_key.clone(), - predicates: vec![ - PredicateRef { - family: RecordFamily::System, - key: claim_key.clone(), - predicate: Predicate::VersionEquals(claim.version), - }, - PredicateRef { - family: RecordFamily::Gc, - key: gc_row.key.clone(), - predicate: Predicate::VersionEquals(gc_row.version), - }, - ], - mutations: vec![Mutation { - family: RecordFamily::System, - key: claim_key.clone(), - op: MutationOp::Put, - value: Some(Value( - encode_object_gc_claim(&ObjectGcClaim::Deleting { - owner_epoch: service.epoch.load(Ordering::Relaxed), - operation_token: claim.version.get(), - gc_record_key: gc_row.key.clone(), - gc_record_version: gc_row.version.get(), - }) - .unwrap(), - )), - }], - watch: Vec::new(), + leave_object_gc_deleting( + &service, + &ScanItem { + key: row_key.clone(), + value: row.value, + version: row.version, + }, + ); + drop(service); + + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + let outcome = reopened.recover_object_gc_claim().unwrap(); + assert_eq!(outcome.attempted, 0); + assert_eq!(outcome.deleted, 0); + assert_eq!(outcome.records_removed, 1); + assert!(objects.head(&foreign_object).unwrap().is_some()); + assert!(metadata + .get( + RecordFamily::Gc, + &row_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + assert_eq!( + metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: object_gc_quarantine_prefix(reopened.mount), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); + let claim = metadata + .get( + RecordFamily::System, + &object_gc_claim_key(reopened.mount), + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert_eq!( + decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), + ObjectGcClaim::Open + ); +} + +#[test] +fn delete_error_keeps_the_durable_claim_closed_until_same_claim_recovery() { + let backing = MemoryObjectStore::new(); + let objects = DeleteAckLostObjectStore::new(backing.clone()); + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let published = service + .publish_artifact(artifact_request( + DentryName::new(b"lost-delete-ack.bin".to_vec()).unwrap(), + "lost-delete-ack", + b"payload", + )) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service + .remove_file( + InodeId::root(), + &DentryName::new(b"lost-delete-ack.bin".to_vec()).unwrap(), + ) + .unwrap(); + + assert!(matches!( + service.cleanup_pending_objects(1), + Err(MetadError::Object(ObjectError::Backend(message))) + if message == "injected lost DELETE acknowledgement" + )); + assert_eq!(objects.delete_calls(), 1); + assert!(backing.head(&object).unwrap().is_none()); + let claim_key = object_gc_claim_key(service.mount); + let deleting = metadata + .get( + RecordFamily::System, + &claim_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert!(matches!( + decode_object_gc_claim(service.mount, &deleting.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); + assert!(matches!( + service.begin_object_reference_mutation(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert_eq!( + metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); + + let recovered = service.recover_object_gc_claim().unwrap(); + assert_eq!(objects.delete_calls(), 2); + assert_eq!(recovered.attempted, 1); + assert_eq!(recovered.missing, 1); + assert_eq!(recovered.records_removed, 1); + let open = metadata + .get( + RecordFamily::System, + &claim_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert_eq!( + decode_object_gc_claim(service.mount, &open.0).unwrap(), + ObjectGcClaim::Open + ); + assert!(metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); +} + +#[test] +fn malformed_gc_row_is_quarantined_without_starving_a_valid_candidate() { + let (service, objects) = service_with_objects(); + let mut malformed_key = gc_queue_prefix(service.mount); + malformed_key.extend_from_slice(&0_u64.to_be_bytes()); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: b"malformed-gc-row".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Gc, + primary_key: malformed_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::Gc, + key: malformed_key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::Gc, + key: malformed_key, + op: MutationOp::Put, + value: Some(Value(b"not-an-object-gc-record".to_vec())), + }], + watch: Vec::new(), + }) + .unwrap(); + + let object = ObjectKey::new("blocks/1/901/1/0/0").unwrap(); + objects.put(&object, b"stale".to_vec()).unwrap(); + enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: InodeId::new(901).unwrap(), + generation: 1, + object_key: object.as_str().to_owned(), + size: 5, + digest_uri: "sha256:stale".to_owned(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + + let outcome = service.cleanup_pending_objects(10).unwrap(); + assert_eq!(outcome.scanned, 2); + assert_eq!(outcome.records_removed, 2); + assert_eq!(outcome.deleted, 1); + assert!(objects.head(&object).unwrap().is_none()); + assert_eq!( + service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: object_gc_quarantine_prefix(service.mount), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn failover_durability_marker_survives_disable_and_reopen_and_blocks_object_gc() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"ha-victim".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "ha-victim", b"victim")) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file(InodeId::root(), &name).unwrap(); + + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata-log", + "shard-0", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + service.disable_sync_metadata_log().unwrap(); + let marker_key = failover_durability_required_key(service.mount); + assert_eq!( + metadata + .get( + RecordFamily::System, + &marker_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap() + .0, + FAILOVER_DURABILITY_REQUIRED_MARKER + ); + drop(service); + + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + let outcome = reopened.cleanup_pending_objects(1).unwrap(); + assert_eq!(outcome.scanned, 1); + assert_eq!(outcome.blocked_by_failover_durability, 1); + assert_eq!(outcome.attempted, 0); + assert_eq!(outcome.records_removed, 0); + assert!(objects.head(&object).unwrap().is_some()); + assert_eq!( + metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn failover_marker_committing_first_invalidates_shared_gc_claim_without_delete() { + let metadata = SnapshotCommitBarrierStore::new(CommandKind::CleanupObjects, 0, 2); + let memory = MemoryObjectStore::new(); + let objects = DeleteAckLostObjectStore::new(memory.clone()); + let cleaner = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + )); + cleaner.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"marker-first".to_vec()).unwrap(); + let published = cleaner + .publish_artifact(artifact_request(name.clone(), "marker-first", b"victim")) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + cleaner.remove_file(InodeId::root(), &name).unwrap(); + let marker = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + + metadata.arm(1); + let cleanup = { + let cleaner = Arc::clone(&cleaner); + std::thread::spawn(move || cleaner.cleanup_pending_objects(1)) + }; + metadata.wait_until_blocked(); + marker.require_failover_durability().unwrap(); + metadata.release_blocked(); + + let outcome = cleanup.join().unwrap().unwrap(); + assert_eq!(outcome.attempted, 0); + assert_eq!(objects.delete_calls(), 0); + assert!(memory.head(&object).unwrap().is_some()); + assert_eq!(metadata.predicate_failures(), 1); + cleaner.refresh_allocator_state().unwrap(); + let blocked = cleaner.cleanup_pending_objects(1).unwrap(); + assert_eq!(blocked.blocked_by_failover_durability, 1); + assert_eq!(objects.delete_calls(), 0); + let marker_key = failover_durability_required_key(cleaner.mount); + assert_eq!( + metadata + .get( + RecordFamily::System, + &marker_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap() + .0, + FAILOVER_DURABILITY_REQUIRED_MARKER + ); +} + +#[test] +fn shared_gc_claim_committing_first_invalidates_preplanned_marker_cas() { + let metadata = SnapshotCommitBarrierStore::new(CommandKind::CleanupObjects, 0, 2); + let objects = MemoryObjectStore::new(); + let cleaner = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + cleaner.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"claim-first".to_vec()).unwrap(); + let published = cleaner + .publish_artifact(artifact_request(name.clone(), "claim-first", b"victim")) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + cleaner.remove_file(InodeId::root(), &name).unwrap(); + let marker = Arc::new( + NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(), + ); + + metadata.arm(1); + let marker_commit = { + let marker = Arc::clone(&marker); + std::thread::spawn(move || marker.require_failover_durability()) + }; + metadata.wait_until_blocked(); + let outcome = cleaner.cleanup_pending_objects(1).unwrap(); + assert_eq!(outcome.deleted, 1); + assert!(objects.head(&object).unwrap().is_none()); + metadata.release_blocked(); + marker_commit.join().unwrap().unwrap(); + + assert_eq!( + metadata.predicate_failures(), + 1, + "the marker command planned against the old Open claim must not cross the Deleting transition" + ); + assert_eq!( + metadata + .get( + RecordFamily::System, + &failover_durability_required_key(cleaner.mount), + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap() + .0, + FAILOVER_DURABILITY_REQUIRED_MARKER + ); +} + +#[test] +fn startup_recovery_keeps_interrupted_claim_closed_under_failover_marker() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"interrupted-ha-victim".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request( + name.clone(), + "interrupted-ha-victim", + b"victim", + )) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file(InodeId::root(), &name).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata-log", + "shard-0", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + service.disable_sync_metadata_log().unwrap(); + let gc_row = metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .pop() + .unwrap(); + let claim_key = leave_object_gc_deleting(&service, &gc_row); + drop(service); + + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + assert!(matches!( + reopened.recover_object_gc_claim(), + Err(MetadError::ObjectGcRecoveryRequiresIntervention { .. }) + )); + let claim = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert!(matches!( + decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); + assert!(objects.head(&object).unwrap().is_some()); + assert_eq!( + metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn enabling_failover_durability_fences_a_crash_left_deleting_claim_before_recovery() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"pre-marker-interrupted".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request( + name.clone(), + "pre-marker-interrupted", + b"victim", + )) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file(InodeId::root(), &name).unwrap(); + let gc_row = metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, }) + .unwrap() + .pop() .unwrap(); - claim_key -} + let claim_key = leave_object_gc_deleting(&service, &gc_row); + drop(service); -fn delete_object_gc_claim(service: &NoKvFs) -where - M: MetadataStore, - O: ObjectStore, -{ - let key = object_gc_claim_key(service.mount); - let claim = service - .metadata - .get_versioned( + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + reopened + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata-log", + "shard-0", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + reopened.disable_sync_metadata_log().unwrap(); + let deleting = metadata + .get( RecordFamily::System, - &key, - service.read_version().unwrap(), - ReadPurpose::WritePlanLocal, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, ) .unwrap() .unwrap(); - let version = service.next_version().unwrap(); - service - .commit_metadata(MetadataCommand { - request_id: request_id( - b"test-delete-object-gc-claim", - service.mount, - InodeId::root(), - version, - ), - kind: CommandKind::CleanupObjects, - read_version: predecessor(version).unwrap(), - commit_version: version, - primary_family: RecordFamily::System, - primary_key: key.clone(), - predicates: vec![PredicateRef { - family: RecordFamily::System, - key: key.clone(), - predicate: Predicate::VersionEquals(claim.version), - }], - mutations: vec![delete_mutation(RecordFamily::System, key)], - watch: Vec::new(), - }) - .unwrap(); -} + assert!(matches!( + decode_object_gc_claim(reopened.mount, &deleting.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); -fn artifact_request(name: DentryName, manifest_id: &str, bytes: &[u8]) -> PublishArtifact { - PublishArtifact { - parent: InodeId::root(), - name, - producer: "unit-test".to_owned(), - digest_uri: "sha256:test".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: manifest_id.to_owned(), - bytes: bytes.to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - } + assert!(matches!( + reopened.recover_object_gc_claim(), + Err(MetadError::ObjectGcRecoveryRequiresIntervention { .. }) + )); + let still_deleting = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert!(matches!( + decode_object_gc_claim(reopened.mount, &still_deleting.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); + assert!(objects.head(&object).unwrap().is_some()); + assert_eq!( + metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); } -fn publish_path_artifact( - service: &NoKvFs, - path: &str, - manifest_id: &str, - bytes: &[u8], -) -> DentryWithAttr { - let prepared = service.prepare_artifact_create_path(path).unwrap(); +#[test] +fn failover_recovery_does_not_reopen_after_external_delete_completed() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"deleted-before-crash".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request( + name.clone(), + "deleted-before-crash", + b"victim", + )) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file(InodeId::root(), &name).unwrap(); + let gc_row = metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .pop() + .unwrap(); + let claim_key = leave_object_gc_deleting(&service, &gc_row); + assert!(objects.delete(&object).unwrap()); service - .publish_prepared_artifact_session( - prepared.clone(), - PublishArtifactSession { - parent: prepared.parent, - name: prepared.name, - producer: "unit-test".to_owned(), - digest_uri: "sha256:test".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: manifest_id.to_owned(), - size: bytes.len() as u64, - ranges: vec![PublishArtifactRange { - offset: 0, - bytes: bytes.to_vec(), - }], - mode: 0o644, - uid: 1000, - gid: 1000, - }, + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata-log", + "shard-0", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + service.disable_sync_metadata_log().unwrap(); + drop(service); + + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + assert!(matches!( + reopened.recover_object_gc_claim(), + Err(MetadError::ObjectGcRecoveryRequiresIntervention { .. }) + )); + assert!(objects.head(&object).unwrap().is_none()); + let claim = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, ) .unwrap() - .entry + .unwrap(); + assert!(matches!( + decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); + assert_eq!( + metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); } -/// Supersede an existing artifact in `parent` (replace -> a fresh generation). -fn republish_path_artifact( - service: &NoKvFs, - parent: InodeId, - name: &str, - manifest_id: &str, - bytes: &[u8], -) -> DentryWithAttr { - let prepared = service - .prepare_artifact_replace(parent, DentryName::new(name.as_bytes().to_vec()).unwrap()) +#[test] +fn reopen_missing_claim_is_initialized_before_enabling_failover_durability() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + delete_object_gc_claim(&service); + drop(service); + + let reopened = + NoKvFs::open_existing(MountId::new(1).unwrap(), metadata.clone(), objects, 0).unwrap(); + assert!(metadata + .get( + RecordFamily::System, + &object_gc_claim_key(reopened.mount), + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + reopened + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata-log", + "shard-0", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - service - .publish_prepared_artifact_session( - prepared.clone(), - PublishArtifactSession { - parent: prepared.parent, - name: prepared.name, - producer: "unit-test".to_owned(), - digest_uri: "sha256:test".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: manifest_id.to_owned(), - size: bytes.len() as u64, - ranges: vec![PublishArtifactRange { - offset: 0, - bytes: bytes.to_vec(), - }], - mode: 0o644, - uid: 1000, - gid: 1000, - }, + let claim = metadata + .get( + RecordFamily::System, + &object_gc_claim_key(reopened.mount), + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, ) .unwrap() - .entry + .unwrap(); + assert_eq!( + decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), + ObjectGcClaim::Open + ); + assert!(metadata + .get( + RecordFamily::System, + &failover_durability_required_key(reopened.mount), + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_some()); } #[test] -fn publish_multichunk_artifact_succeeds() { - let service = service(); - // 128 MiB spans two 64 MiB chunks: the multi-chunk publish path the FUSE - // bigfile workload hits (and currently EIOs on via InvalidPreparedArtifact). - let size = 128 * 1024 * 1024_usize; - let bytes = vec![0u8; size]; - let prepared = service.prepare_artifact_create_path("/big.bin").unwrap(); - let result = service.publish_prepared_artifact_session( - prepared.clone(), - PublishArtifactSession { - parent: prepared.parent, - name: prepared.name, - producer: "unit-test".to_owned(), - digest_uri: "sha256:test".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "fuse/big".to_owned(), - size: size as u64, - ranges: vec![PublishArtifactRange { offset: 0, bytes }], - mode: 0o644, - uid: 1000, - gid: 1000, - }, - ); - assert!( - result.is_ok(), - "multi-chunk publish failed: {:?}", - result.err() +fn reopen_missing_claim_is_initialized_by_cleanup_recovery() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"legacy-gc-victim".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request( + name.clone(), + "legacy-gc-victim", + b"victim", + )) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file(InodeId::root(), &name).unwrap(); + delete_object_gc_claim(&service); + drop(service); + + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + let outcome = reopened.cleanup_pending_objects(1).unwrap(); + assert_eq!(outcome.deleted, 1); + assert!(objects.head(&object).unwrap().is_none()); + let claim = metadata + .get( + RecordFamily::System, + &object_gc_claim_key(reopened.mount), + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert_eq!( + decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), + ObjectGcClaim::Open ); } -fn block_key(inode: InodeId, generation: u64, chunk: u64, block: u64) -> ObjectKey { - ObjectKey::new(format!( - "blocks/1/{}/{}/{}/{}", - inode.get(), - generation, - chunk, - block - )) - .unwrap() +#[test] +fn read_lease_grace_blocks_recent_object_gc_records() { + let (service, objects) = service_with_objects(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let first = service + .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) + .unwrap(); + let old_body = first.body.clone().unwrap(); + let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); + let replaced = service + .replace_artifact(artifact_request( + name.clone(), + "checkpoint/new", + b"new-body", + )) + .unwrap(); + let new_body = replaced.entry.body.clone().unwrap(); + let new_object = block_key(replaced.entry.attr.inode, new_body.generation, 0, 0); + + let blocked = service + .cleanup_pending_objects_with_grace(100, std::time::Duration::from_secs(3_600)) + .unwrap(); + assert_eq!(blocked.scanned, 1); + assert_eq!(blocked.blocked_by_snapshots, 0); + assert_eq!(blocked.blocked_by_read_leases, 1); + assert_eq!(blocked.attempted, 0); + assert_eq!(blocked.records_removed, 0); + assert!(objects.head(&old_object).unwrap().is_some()); + assert!(objects.head(&new_object).unwrap().is_some()); + + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&old_object).unwrap().is_none()); + assert!(objects.head(&new_object).unwrap().is_some()); } -fn body_descriptor(generation: u64, size: u64) -> BodyDescriptor { - BodyDescriptor { - producer: "unit-test".to_owned(), - digest_uri: "sha256:test".to_owned(), - size, - content_type: "application/octet-stream".to_owned(), - manifest_id: format!("manifest-{generation}"), - generation, - base_generation: 0, - chunk_size: DEFAULT_CHUNK_SIZE, - block_size: DEFAULT_BLOCK_SIZE as u64, - } +#[test] +fn replace_artifact_cleanup_deletes_only_old_generation() { + let (service, objects) = service_with_objects(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let first = service + .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) + .unwrap(); + let old_body = first.body.clone().unwrap(); + let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); + let replaced = service + .replace_artifact(artifact_request( + name.clone(), + "checkpoint/new", + b"new-body", + )) + .unwrap(); + let new_body = replaced.entry.body.clone().unwrap(); + let new_object = block_key(replaced.entry.attr.inode, new_body.generation, 0, 0); + assert!(objects.head(&old_object).unwrap().is_some()); + assert!(objects.head(&new_object).unwrap().is_some()); + + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&old_object).unwrap().is_none()); + assert!(objects.head(&new_object).unwrap().is_some()); + assert_eq!( + service.read_artifact(InodeId::root(), &name).unwrap(), + b"new-body" + ); } -fn one_chunk_manifest(inode: InodeId, generation: u64, len: u64) -> ChunkManifest { - ChunkManifest { - chunk_index: 0, - logical_offset: 0, - len, - slices: vec![SliceManifest { - slice_id: 1, - logical_offset: 0, - len, - blocks: vec![BlockDescriptor { - object_key: block_key(inode, generation, 0, 0).as_str().to_owned(), - logical_offset: 0, - object_offset: 0, - len, - digest_uri: "sha256:block".to_owned(), - }], - }], - } +#[test] +fn snapshot_after_replace_does_not_block_old_object_cleanup() { + let (service, objects) = service_with_objects(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let first = service + .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) + .unwrap(); + let old_body = first.body.clone().unwrap(); + let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); + let replaced = service + .replace_artifact(artifact_request( + name.clone(), + "checkpoint/new", + b"new-body", + )) + .unwrap(); + let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + + assert_eq!( + service + .read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/checkpoint.bin") + .unwrap(), + b"new-body" + ); + assert!(objects.head(&old_object).unwrap().is_some()); + + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.scanned, 1); + assert_eq!(cleanup.blocked_by_snapshots, 0); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&old_object).unwrap().is_none()); + assert_eq!( + service.read_artifact(InodeId::root(), &name).unwrap(), + b"new-body" + ); + assert_eq!( + replaced.entry.body.unwrap().generation, + snapshot.read_version + ); } #[test] -fn create_dir_then_lookup_and_readdir_use_dentry_projection() { - let service = service(); - let name = DentryName::new(b"runs".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), name.clone(), 0o755, 1000, 1000) +fn snapshot_preserves_old_artifact_and_blocks_object_gc_until_retired() { + let (service, objects) = service_with_objects(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let first = service + .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) .unwrap(); + let old_body = first.body.clone().unwrap(); + let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); + let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); - let lookup = service - .lookup_plus(InodeId::root(), &name) - .unwrap() + let replaced = service + .replace_artifact(artifact_request( + name.clone(), + "checkpoint/new", + b"new-body", + )) .unwrap(); - assert_eq!(lookup, created); + let new_body = replaced.entry.body.clone().unwrap(); + let new_object = block_key(replaced.entry.attr.inode, new_body.generation, 0, 0); - let entries = service.read_dir_plus(InodeId::root()).unwrap(); - assert_eq!(entries, vec![created]); - let stats = service.metadata_service_stats(); - assert_eq!(stats.read_dir_plus_total, 1); - assert_eq!(stats.read_dir_plus_entry_total, 1); - assert_eq!(stats.read_dir_plus_projection_hit_total, 1); + assert_eq!( + service + .read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/checkpoint.bin") + .unwrap(), + b"old" + ); + assert_eq!( + service + .get_attr_at_snapshot("/", snapshot.snapshot_id, std::slice::from_ref(&name)) + .unwrap(), + Some(first.attr.clone()) + ); + assert_eq!( + service + .read_file_at_snapshot("/", snapshot.snapshot_id, std::slice::from_ref(&name), 0, 3,) + .unwrap(), + b"old" + ); + assert_eq!( + service + .read_dir_plus_at_snapshot("/", snapshot.snapshot_id, &[]) + .unwrap(), + vec![first.clone()] + ); + assert_eq!( + service.read_artifact(InodeId::root(), &name).unwrap(), + b"new-body" + ); + let blocked = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(blocked.scanned, 1); + assert_eq!(blocked.blocked_by_snapshots, 1); + assert_eq!(blocked.attempted, 0); + assert!(objects.head(&old_object).unwrap().is_some()); + assert!(objects.head(&new_object).unwrap().is_some()); + + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + assert!(!service.retire_snapshot(snapshot.snapshot_id).unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&old_object).unwrap().is_none()); + assert!(objects.head(&new_object).unwrap().is_some()); } #[test] -fn write_planning_reads_are_marked_local_while_user_reads_stay_strong() { - let metadata = PurposeTrackingStore::new(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let file_name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); +fn snapshot_path_reads_are_rooted_at_snapshot_subtree_and_support_ranges() { + let service = service(); + let scope = service + .create_dir_path("/scope", 0o755, 1000, 1000) + .unwrap(); + let nested = service + .create_dir_path("/scope/nested", 0o755, 1000, 1000) + .unwrap(); + let outside = service + .create_dir_path("/outside", 0o755, 1000, 1000) + .unwrap(); + let name = DentryName::new(b"model.bin".to_vec()).unwrap(); + let inside_old = service + .publish_artifact(PublishArtifact { + parent: nested.attr.inode, + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:inside-old".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "inside-old".to_owned(), + bytes: b"inside-old".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); service - .create_file(InodeId::root(), file_name.clone(), 0o644, 1000, 1000) + .publish_artifact(PublishArtifact { + parent: outside.attr.inode, + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:outside".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "outside".to_owned(), + bytes: b"outside".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) .unwrap(); - let dir_name = DentryName::new(b"runs".to_vec()).unwrap(); - let dir = service - .create_dir(InodeId::root(), dir_name, 0o755, 1000, 1000) + let snapshot = service.snapshot_subtree_path("/scope").unwrap(); + service + .replace_artifact(PublishArtifact { + parent: nested.attr.inode, + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:inside-new".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "inside-new".to_owned(), + bytes: b"inside-new".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) .unwrap(); - let before_lookup = metadata.counts(); - assert!(service - .lookup_plus(InodeId::root(), &file_name) + let root = service + .stat_path_at_snapshot("/scope", snapshot.snapshot_id, "/") .unwrap() - .is_some()); - let after_lookup = metadata.counts(); - assert!(after_lookup.user_strong_gets > before_lookup.user_strong_gets); - assert_eq!(after_lookup.write_plan_gets, before_lookup.write_plan_gets); - - service - .remove_file(InodeId::root(), &file_name) - .expect("remove file"); - let after_remove = metadata.counts(); - assert_eq!(after_remove.user_strong_gets, after_lookup.user_strong_gets); - assert!(after_remove.write_plan_gets > after_lookup.write_plan_gets); - - let snapshot = service - .snapshot_subtree(dir.attr.inode) - .expect("snapshot subtree"); - let after_snapshot = metadata.counts(); + .unwrap(); + assert_eq!(root.attr.inode, scope.attr.inode); assert_eq!( - after_snapshot.user_strong_gets, - after_remove.user_strong_gets + service + .read_dir_plus_path_at_snapshot("/scope", snapshot.snapshot_id, "/") + .unwrap(), + vec![nested.clone()] ); - assert!(after_snapshot.write_plan_gets > after_remove.write_plan_gets); - - assert!(service - .get_attr_at_snapshot("/runs", snapshot.snapshot_id, &[]) - .unwrap() - .is_some()); - assert!(service - .read_dir_plus_at_snapshot("/runs", snapshot.snapshot_id, &[]) + let file = service + .stat_path_at_snapshot("/scope", snapshot.snapshot_id, "/nested/model.bin") .unwrap() - .is_empty()); - let after_snapshot_reads = metadata.counts(); - assert!( - after_snapshot_reads.user_strong_gets > after_snapshot.user_strong_gets, - "root-path binding is resolved with strong reads before snapshot-purpose reads" + .unwrap(); + assert_eq!(file.attr.generation, inside_old.attr.generation); + assert_eq!(file.body, inside_old.body); + assert_eq!( + service + .read_file_path_at_snapshot("/scope", snapshot.snapshot_id, "/nested/model.bin", 7, 3,) + .unwrap(), + b"old" ); - assert!(after_snapshot_reads.snapshot_gets > after_snapshot.snapshot_gets); - assert!(after_snapshot_reads.snapshot_scans > after_snapshot.snapshot_scans); + assert!(matches!( + service.read_file_path_at_snapshot( + "/scope", + snapshot.snapshot_id, + "/outside/model.bin", + 0, + 7, + ), + Err(MetadError::NotFound) + )); } #[test] -fn xattr_round_trips_lists_replaces_and_removes() { +fn snapshot_path_list_pages_include_entries_deleted_after_snapshot() { let service = service(); - let entry = service - .create_file( + for name in [ + b"a.txt".as_slice(), + b"b.txt".as_slice(), + b"c.txt".as_slice(), + ] { + service + .create_file( + InodeId::root(), + DentryName::new(name.to_vec()).unwrap(), + 0o644, + 1000, + 1000, + ) + .unwrap(); + } + let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + service + .rename( InodeId::root(), - DentryName::new(b"note.txt".to_vec()).unwrap(), - 0o644, - 1000, - 1000, + &DentryName::new(b"b.txt".to_vec()).unwrap(), + InodeId::root(), + DentryName::new(b"z.txt".to_vec()).unwrap(), ) .unwrap(); - service - .set_xattr( - entry.attr.inode, - b"user.comment", - b"first".to_vec(), - XattrSetMode::Create, + .remove_file( + InodeId::root(), + &DentryName::new(b"c.txt".to_vec()).unwrap(), ) .unwrap(); + + let first = service + .read_dir_plus_path_at_snapshot_page("/", snapshot.snapshot_id, "/", None, 2) + .unwrap(); assert_eq!( - service - .get_xattr(entry.attr.inode, b"user.comment") - .unwrap(), - Some(b"first".to_vec()) - ); - assert_eq!( - service.list_xattr(entry.attr.inode).unwrap(), - vec![b"user.comment".to_vec()] + first + .entries + .iter() + .map(|entry| entry.dentry.name.as_bytes()) + .collect::>(), + vec![b"a.txt".as_slice(), b"b.txt".as_slice()] ); - assert!(matches!( - service.set_xattr( - entry.attr.inode, - b"user.comment", - b"duplicate".to_vec(), - XattrSetMode::Create, - ), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - )); + let cursor = first.next_cursor.unwrap(); + assert_eq!(cursor.as_bytes(), b"b.txt"); + + let second = service + .read_dir_plus_path_at_snapshot_page("/", snapshot.snapshot_id, "/", Some(&cursor), 2) + .unwrap(); + assert_eq!(second.entries.len(), 1); + assert_eq!(second.entries[0].dentry.name.as_bytes(), b"c.txt"); + assert!(second.next_cursor.is_none()); +} +#[test] +fn history_cleanup_keeps_snapshot_reads_until_snapshot_retired() { + let service = service(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); service - .set_xattr( - entry.attr.inode, - b"user.comment", - b"second".to_vec(), - XattrSetMode::Replace, - ) + .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) + .unwrap(); + let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + service + .replace_artifact(artifact_request( + name.clone(), + "checkpoint/new", + b"new-body", + )) .unwrap(); + + let retained = service.cleanup_history(100).unwrap(); + assert!(retained.retained_by_snapshots > 0); assert_eq!( service - .get_xattr(entry.attr.inode, b"user.comment") + .read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/checkpoint.bin") .unwrap(), - Some(b"second".to_vec()) + b"old" ); - assert!(matches!( - service.set_xattr( - entry.attr.inode, - b"user.missing", - b"value".to_vec(), - XattrSetMode::Replace, - ), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - )); - service - .remove_xattr(entry.attr.inode, b"user.comment") - .unwrap(); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + let pruned = service.cleanup_history(100).unwrap(); + assert!(pruned.removed > 0); assert_eq!( service - .get_xattr(entry.attr.inode, b"user.comment") + .metadata + .get( + RecordFamily::Dentry, + &dentry_key(service.mount, InodeId::root(), &name), + Version::new(snapshot.read_version).unwrap(), + ReadPurpose::Snapshot, + ) .unwrap(), None ); - assert!(service.list_xattr(entry.attr.inode).unwrap().is_empty()); - assert!(matches!( - service.remove_xattr(entry.attr.inode, b"user.comment"), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - )); } #[test] -fn path_methods_resolve_current_namespace_on_server_side() { +fn remove_empty_dir_rejects_non_empty_directory() { let service = service(); - let runs = service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let artifact = service - .create_file_path("/runs/checkpoint.bin", 0o644, 1000, 1000) + let dir = DentryName::new(b"runs".to_vec()).unwrap(); + let child = DentryName::new(b"1".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), dir.clone(), 0o755, 1000, 1000) + .unwrap(); + service + .create_dir(created.attr.inode, child, 0o755, 1000, 1000) .unwrap(); - assert_eq!(service.lookup_path("/runs").unwrap(), Some(runs.clone())); + let err = service.remove_empty_dir(InodeId::root(), &dir).unwrap_err(); + assert!(matches!(err, MetadError::DirectoryNotEmpty)); + assert!(service + .lookup_plus(InodeId::root(), &dir) + .unwrap() + .is_some()); +} + +#[test] +fn remove_empty_dir_deletes_empty_directory() { + let service = service(); + let dir = DentryName::new(b"runs".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), dir.clone(), 0o755, 1000, 1000) + .unwrap(); + + let removed = service.remove_empty_dir(InodeId::root(), &dir).unwrap(); + assert_eq!(removed, created); + assert!(service + .lookup_plus(InodeId::root(), &dir) + .unwrap() + .is_none()); + assert!(service.get_attr(created.attr.inode).unwrap().is_none()); +} + +#[test] +fn remove_empty_dir_allows_directory_after_last_child_removed() { + let service = service(); + let dir = DentryName::new(b"runs".to_vec()).unwrap(); + let child = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), dir.clone(), 0o755, 1000, 1000) + .unwrap(); + service + .publish_artifact(PublishArtifact { + parent: created.attr.inode, + name: child.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:test".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "runs/checkpoint.bin".to_owned(), + bytes: b"payload".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + + service.remove_file(created.attr.inode, &child).unwrap(); + let removed = service.remove_empty_dir(InodeId::root(), &dir).unwrap(); + + assert_eq!(removed, created); + assert!(service + .lookup_plus(InodeId::root(), &dir) + .unwrap() + .is_none()); +} + +#[test] +fn rename_moves_dentry_without_changing_inode() { + let service = service(); + let old_name = DentryName::new(b"old".to_vec()).unwrap(); + let new_name = DentryName::new(b"new".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), old_name.clone(), 0o755, 1000, 1000) + .unwrap(); + + let renamed = service + .rename( + InodeId::root(), + &old_name, + InodeId::root(), + new_name.clone(), + ) + .unwrap(); + assert_eq!(renamed.attr.inode, created.attr.inode); + assert!(service + .lookup_plus(InodeId::root(), &old_name) + .unwrap() + .is_none()); assert_eq!( - service.lookup_path("/runs/checkpoint.bin").unwrap(), - Some(artifact.clone()) + service.lookup_plus(InodeId::root(), &new_name).unwrap(), + Some(renamed) ); - assert_eq!(service.read_dir_plus_path("/runs").unwrap(), vec![artifact]); - assert!(matches!( - service.create_file_path("relative", 0o644, 1000, 1000), - Err(MetadError::InvalidPath(_)) - )); } #[test] -fn plain_path_create_uses_canonical_namespace_without_path_index() { - let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let artifact = service - .create_file_path("/runs/checkpoint.bin", 0o644, 1000, 1000) - .unwrap(); - let components = parse_absolute_path("/runs/checkpoint.bin").unwrap(); - let key = path_index_key(MountId::new(1).unwrap(), &components); - assert!(metadata - .get( - RecordFamily::PathIndex, - &key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, +fn rename_replace_returns_replaced_file_body() { + let service = service(); + let source_name = DentryName::new(b"stage".to_vec()).unwrap(); + let final_name = DentryName::new(b"final".to_vec()).unwrap(); + let source = service + .publish_artifact(artifact_request(source_name.clone(), "stage", b"new")) + .unwrap(); + let old = service + .publish_artifact(artifact_request(final_name.clone(), "final-old", b"old")) + .unwrap(); + + let result = service + .rename_replace( + InodeId::root(), + &source_name, + InodeId::root(), + final_name.clone(), ) + .unwrap(); + assert_eq!(result.entry.attr.inode, source.attr.inode); + assert_eq!(result.replaced, Some(old.clone())); + assert!(service + .lookup_plus(InodeId::root(), &source_name) .unwrap() .is_none()); - - let before = service.metadata_service_stats(); - assert_eq!( - service.lookup_path("/runs/checkpoint.bin").unwrap(), - Some(artifact) - ); - let after = service.metadata_service_stats(); - assert_eq!( - after.path_index_lookup_total - before.path_index_lookup_total, - 0 - ); assert_eq!( - after.path_index_fallback_total - before.path_index_fallback_total, - 0 + service.lookup_plus(InodeId::root(), &final_name).unwrap(), + Some(result.entry) ); + assert!(service.get_attr(old.attr.inode).unwrap().is_none()); } #[test] -fn prepared_artifact_path_publish_writes_and_uses_validated_path_index() { - let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let prepared = service - .prepare_artifact_create_path("/runs/checkpoint.bin") +fn watch_replay_returns_typed_events_after_cursor() { + let service = service(); + let cursor = service.watch_subtree(InodeId::root()).unwrap(); + let name = DentryName::new(b"runs".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), name.clone(), 0o755, 1000, 1000) .unwrap(); - let body = body_descriptor(prepared.generation, 6); - let artifact = service - .publish_prepared_artifact( - prepared.clone(), - body, - vec![one_chunk_manifest(prepared.inode, prepared.generation, 6)], - 0o644, - 1000, - 1000, - ) - .unwrap() - .entry; - let components = parse_absolute_path("/runs/checkpoint.bin").unwrap(); - let key = path_index_key(MountId::new(1).unwrap(), &components); - let indexed = metadata - .get( - RecordFamily::PathIndex, - &key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .expect("artifact path index entry"); - let projection = decode_dentry_projection(&indexed.0).unwrap(); - assert_eq!(DentryWithAttr::from(projection), artifact); - let before = service.metadata_service_stats(); - let metadata = service - .stat_path("/runs/checkpoint.bin") - .unwrap() - .expect("artifact stat"); - assert_eq!(metadata.attr, artifact.attr); - assert_eq!(metadata.body, artifact.body); - let after = service.metadata_service_stats(); + let events = service.replay_watch(InodeId::root(), cursor, 100).unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].event.kind, WatchEventKind::Create); + assert_eq!(events[0].event.parent, Some(InodeId::root())); + assert_eq!(events[0].event.name, Some(name.clone())); + assert_eq!(events[0].event.inode, created.attr.inode); + + let next_name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + service + .publish_artifact(artifact_request( + next_name.clone(), + "checkpoint.bin", + b"body", + )) + .unwrap(); + let resumed = service + .replay_watch(InodeId::root(), events[0].cursor, 100) + .unwrap(); + assert_eq!(resumed.len(), 1); + assert_eq!(resumed[0].event.kind, WatchEventKind::PublishArtifact); + assert_eq!(resumed[0].event.name, Some(next_name)); +} + +#[test] +fn watch_replay_resumes_from_cursor_without_scanning_old_events() { + let service = service(); + let cursor = service.watch_subtree(InodeId::root()).unwrap(); + for name in ["a", "b", "c"] { + service + .create_dir( + InodeId::root(), + DentryName::new(name.as_bytes().to_vec()).unwrap(), + 0o755, + 1000, + 1000, + ) + .unwrap(); + } + + let before_first = service.metadata_store_stats(); + let first = service.replay_watch(InodeId::root(), cursor, 1).unwrap(); + let after_first = service.metadata_store_stats(); + assert_eq!(first.len(), 1); assert_eq!( - after.path_index_lookup_total - before.path_index_lookup_total, + first[0].event.name.as_ref().map(DentryName::as_bytes), + Some(b"a".as_slice()) + ); + assert_eq!( + after_first.scan_key_visited_total - before_first.scan_key_visited_total, 1 ); - assert_eq!(after.path_index_hit_total - before.path_index_hit_total, 1); assert_eq!( - after.path_index_fallback_total - before.path_index_fallback_total, - 0 + after_first.scan_key_returned_total - before_first.scan_key_returned_total, + 1 ); - let before = service.metadata_service_stats(); - assert_eq!(service.stat_path("/runs/missing.bin").unwrap(), None); - let after = service.metadata_service_stats(); + let before_second = service.metadata_store_stats(); + let second = service + .replay_watch(InodeId::root(), first[0].cursor, 1) + .unwrap(); + let after_second = service.metadata_store_stats(); + assert_eq!(second.len(), 1); assert_eq!( - after.path_index_lookup_total - before.path_index_lookup_total, - 1 + second[0].event.name.as_ref().map(DentryName::as_bytes), + Some(b"b".as_slice()) ); assert_eq!( - after.path_index_miss_total - before.path_index_miss_total, + after_second.scan_key_visited_total - before_second.scan_key_visited_total, 1 ); assert_eq!( - after.path_index_fallback_total - before.path_index_fallback_total, + after_second.scan_key_returned_total - before_second.scan_key_returned_total, 1 ); } #[test] -fn artifact_path_rename_moves_live_path_index() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let archive = service - .create_dir_path("/archive", 0o755, 1000, 1000) - .unwrap(); - let artifact = publish_path_artifact(&service, "/runs/a.bin", "runs/a.bin", b"a"); - let old_components = parse_absolute_path("/runs/a.bin").unwrap(); - let new_components = parse_absolute_path("/archive/a.bin").unwrap(); - let old_key = path_index_key(MountId::new(1).unwrap(), &old_components); - let new_key = path_index_key(MountId::new(1).unwrap(), &new_components); - assert!(metadata - .get( - RecordFamily::PathIndex, - &old_key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .is_some()); - - let renamed = service - .rename_path("/runs/a.bin", "/archive/a.bin") - .unwrap(); - let old_index = metadata - .get( - RecordFamily::PathIndex, - &old_key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) +fn rename_replay_notifies_old_and_new_parent_scopes() { + let service = service(); + let old_parent_name = DentryName::new(b"old-parent".to_vec()).unwrap(); + let new_parent_name = DentryName::new(b"new-parent".to_vec()).unwrap(); + let old_parent = service + .create_dir(InodeId::root(), old_parent_name, 0o755, 1000, 1000) .unwrap(); - let new_index = metadata - .get( - RecordFamily::PathIndex, - &new_key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .expect("renamed artifact path index"); - - assert!(old_index.is_none()); - assert_eq!(renamed.attr.inode, artifact.attr.inode); - let indexed = decode_dentry_projection(&new_index.0).unwrap(); - assert_eq!(indexed.dentry.parent, archive.attr.inode); - assert_eq!(indexed.dentry.name.as_bytes(), b"a.bin"); - assert_eq!(indexed.attr.inode, artifact.attr.inode); -} - -#[test] -fn plain_directory_path_rename_does_not_create_path_index() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let runs = service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let source_components = parse_absolute_path("/runs").unwrap(); - let destination_components = parse_absolute_path("/archive").unwrap(); - let source_key = path_index_key(MountId::new(1).unwrap(), &source_components); - let destination_key = path_index_key(MountId::new(1).unwrap(), &destination_components); - let before = metadata.metadata_store_stats(); - - let renamed = service.rename_path("/runs", "/archive").unwrap(); - let after = metadata.metadata_store_stats(); - - assert_eq!(renamed.attr.inode, runs.attr.inode); - assert_eq!(after.current_put_total - before.current_put_total, 1); - assert_eq!(after.current_delete_total - before.current_delete_total, 1); - assert!(metadata - .get( - RecordFamily::PathIndex, - &source_key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .is_none()); - assert!(metadata - .get( - RecordFamily::PathIndex, - &destination_key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .is_none()); -} + let new_parent = service + .create_dir(InodeId::root(), new_parent_name, 0o755, 1000, 1000) + .unwrap(); + let file_name = DentryName::new(b"artifact".to_vec()).unwrap(); + let source = service + .create_file(old_parent.attr.inode, file_name.clone(), 0o644, 1000, 1000) + .unwrap(); + let old_cursor = service.watch_subtree(old_parent.attr.inode).unwrap(); + let new_cursor = service.watch_subtree(new_parent.attr.inode).unwrap(); -#[test] -fn artifact_path_remove_deletes_live_path_index() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/runs/a.bin", "runs/a.bin", b"a"); - let components = parse_absolute_path("/runs/a.bin").unwrap(); - let key = path_index_key(MountId::new(1).unwrap(), &components); - assert!(metadata - .get( - RecordFamily::PathIndex, - &key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, + service + .rename( + old_parent.attr.inode, + &file_name, + new_parent.attr.inode, + file_name.clone(), ) - .unwrap() - .is_some()); + .unwrap(); - service.remove_file_path("/runs/a.bin").unwrap(); + let old_events = service + .replay_watch(old_parent.attr.inode, old_cursor, 100) + .unwrap(); + assert_eq!(old_events.len(), 1); + assert_eq!(old_events[0].event.kind, WatchEventKind::Remove); + assert_eq!(old_events[0].event.inode, source.attr.inode); - assert!(metadata - .get( - RecordFamily::PathIndex, - &key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .is_none()); + let new_events = service + .replay_watch(new_parent.attr.inode, new_cursor, 100) + .unwrap(); + assert_eq!(new_events.len(), 1); + assert_eq!(new_events[0].event.kind, WatchEventKind::Rename); + assert_eq!(new_events[0].event.name, Some(file_name)); + assert_eq!(new_events[0].event.inode, source.attr.inode); } #[test] -fn path_resolution_cache_reuses_parent_directory_for_indexed_stats() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); +fn watch_replay_survives_service_reopen() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/runs/a.bin", "runs/a.bin", b"a"); - publish_path_artifact(&service, "/runs/b.bin", "runs/b.bin", b"b"); - service.clear_read_path_caches_for_test(); - - let before_store = metadata.metadata_store_stats(); - let before_service = service.metadata_service_stats(); - assert!(service.stat_path("/runs/a.bin").unwrap().is_some()); - let after_first_store = metadata.metadata_store_stats(); - assert!(service.stat_path("/runs/b.bin").unwrap().is_some()); - let after_second_store = metadata.metadata_store_stats(); - let after_service = service.metadata_service_stats(); + let cursor = service.watch_subtree(InodeId::root()).unwrap(); + let name = DentryName::new(b"runs".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), name.clone(), 0o755, 1000, 1000) + .unwrap(); + drop(service); - let first_gets = after_first_store.get_total - before_store.get_total; - let second_gets = after_second_store.get_total - after_first_store.get_total; - assert_eq!(first_gets, 3); - assert_eq!(second_gets, 2); - assert_eq!( - after_service.path_index_hit_total - before_service.path_index_hit_total, - 2 - ); - assert_eq!( - after_service.path_index_fallback_total - before_service.path_index_fallback_total, - 0 - ); + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let events = reopened.replay_watch(InodeId::root(), cursor, 100).unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].event.kind, WatchEventKind::Create); + assert_eq!(events[0].event.name, Some(name)); + assert_eq!(events[0].event.inode, created.attr.inode); } #[test] -fn validated_path_index_cache_reuses_stat_validation_for_indexed_list() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); +fn open_existing_recovers_inode_and_version_allocators() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let first = publish_path_artifact(&service, "/runs/a.bin", "runs/a.bin", b"a"); - let second = publish_path_artifact(&service, "/runs/b.bin", "runs/b.bin", b"b"); - service.clear_read_path_caches_for_test(); + let first = service + .create_dir( + InodeId::root(), + DentryName::new(b"first".to_vec()).unwrap(), + 0o755, + 1000, + 1000, + ) + .unwrap(); + drop(service); - assert!(service.stat_path("/runs/a.bin").unwrap().is_some()); - assert!(service.stat_path("/runs/b.bin").unwrap().is_some()); + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let second = reopened + .create_dir( + InodeId::root(), + DentryName::new(b"second".to_vec()).unwrap(), + 0o755, + 1000, + 1000, + ) + .unwrap(); + assert!(second.attr.inode > first.attr.inode); + assert!(second.attr.generation > first.attr.generation); +} - let before_store = metadata.metadata_store_stats(); - let page = service.list_indexed_path_page("/runs", None, 10).unwrap(); - let after_store = metadata.metadata_store_stats(); +#[test] +fn refresh_allocator_state_advances_live_read_version_after_external_commit() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let original = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + original.bootstrap_root(0o755, 1000, 1000).unwrap(); - assert_eq!(page.entries, vec![first, second]); - assert_eq!(page.next_cursor, None); - assert_eq!(after_store.get_total - before_store.get_total, 0); + let external = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let external_file = external + .create_file_path("/external.bin", 0o644, 1000, 1000) + .unwrap(); + assert!(original.stat_path("/external.bin").unwrap().is_none()); + + original.refresh_allocator_state().unwrap(); + let visible = original + .stat_path("/external.bin") + .unwrap() + .expect("external commit should be visible after refresh"); + assert_eq!(visible.attr, external_file.attr); + let local_file = original + .create_file_path("/after-refresh.bin", 0o644, 1000, 1000) + .unwrap(); + assert!(local_file.attr.inode > external_file.attr.inode); + assert!(local_file.attr.generation > external_file.attr.generation); } #[test] -fn validated_path_index_lookup_cache_reuses_repeated_stat_result() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); +fn open_existing_recovers_after_dentry_only_rename() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let artifact = publish_path_artifact(&service, "/runs/a.bin", "runs/a.bin", b"a"); - service.clear_read_path_caches_for_test(); - - let first = service - .stat_path("/runs/a.bin") - .unwrap() - .expect("first stat"); - assert_eq!(first.attr, artifact.attr); + let old_name = DentryName::new(b"old".to_vec()).unwrap(); + let new_name = DentryName::new(b"new".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), old_name.clone(), 0o755, 1000, 1000) + .unwrap(); + let renamed = service + .rename( + InodeId::root(), + &old_name, + InodeId::root(), + new_name.clone(), + ) + .unwrap(); + assert_eq!(renamed.attr.inode, created.attr.inode); + drop(service); - let before_store = metadata.metadata_store_stats(); - let second = service - .stat_path("/runs/a.bin") + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + assert!(reopened + .lookup_plus(InodeId::root(), &old_name) .unwrap() - .expect("second stat"); - let after_store = metadata.metadata_store_stats(); - - assert_eq!(second.attr, artifact.attr); - assert_eq!(after_store.get_total - before_store.get_total, 0); + .is_none()); + assert_eq!( + reopened.lookup_plus(InodeId::root(), &new_name).unwrap(), + Some(renamed) + ); + assert_eq!(reopened.read_dir_plus(InodeId::root()).unwrap().len(), 1); } #[test] -fn namespace_find_body_facets_do_not_require_body_projection() { - let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/runs/a.json", "runs/a.json", br#"{"loss":0.4}"#); - publish_path_artifact(&service, "/runs/b.log", "runs/b.log", b"loss=0.3\n"); - - let result = service - .find_paths(NamespaceFindRequest { - path: "/runs".to_owned(), - predicates: Vec::new(), - sort: Vec::new(), - include: Vec::new(), - facets: vec![NamespaceFindField::body_content_type()], - cursor: None, - limit: 10, - }) +fn open_existing_does_not_reuse_removed_inode() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let first_name = DentryName::new(b"first".to_vec()).unwrap(); + let second_name = DentryName::new(b"second".to_vec()).unwrap(); + let first = service + .create_file(InodeId::root(), first_name.clone(), 0o644, 1000, 1000) .unwrap(); + service.remove_file(InodeId::root(), &first_name).unwrap(); + drop(service); - assert_eq!(result.match_count, 2); - assert!(result.matches.iter().all(|card| card.body.is_none())); - assert_eq!(result.facets.len(), 1); + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let second = reopened + .create_file(InodeId::root(), second_name.clone(), 0o644, 1000, 1000) + .unwrap(); + assert!(second.attr.inode > first.attr.inode); + assert!(second.attr.generation > first.attr.generation); + assert!(reopened + .lookup_plus(InodeId::root(), &first_name) + .unwrap() + .is_none()); assert_eq!( - result.facets[0].field, - NamespaceFindField::body_content_type() + reopened.lookup_plus(InodeId::root(), &second_name).unwrap(), + Some(second) ); - assert_eq!(result.facets[0].values[0].count, 2); } #[test] -fn namespace_find_tolerates_exists_predicate_payloads() { - let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/runs/a.json", "runs/a.json", br#"{"loss":0.4}"#); - - let result = service - .find_paths(NamespaceFindRequest { - path: "/runs".to_owned(), - predicates: vec![NamespacePredicate { - field: NamespaceFindField::body_content_type(), - op: NamespacePredicateOp::Exists, - value: Some(NamespacePredicateValue::String("ignored".to_owned())), - }], - sort: Vec::new(), - include: Vec::new(), - facets: Vec::new(), - cursor: None, - limit: 10, - }) +fn pending_object_gc_survives_service_reopen() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "artifact.bin", b"old")) .unwrap(); + let body = published.body.clone().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file(InodeId::root(), &name).unwrap(); + drop(service); - assert_eq!(result.match_count, 1); - assert_eq!(result.matches[0].path, "/runs/a.json"); + let reopened = + NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects.clone(), 0).unwrap(); + let cleanup = reopened.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&object).unwrap().is_none()); } #[test] -fn namespace_grep_cursor_resumes_at_next_unemitted_match() { - let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact( - &service, - "/runs/train.log", - "runs/train.log", - b"loss=1\nloss=2\n", +fn snapshot_pin_survives_service_reopen() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + drop(service); + + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + assert_eq!( + reopened.snapshot_pin(snapshot.snapshot_id).unwrap(), + Some(snapshot) ); + assert_eq!(reopened.metadata_store_stats().active_snapshot_pin_total, 1); +} +#[test] +fn failed_publish_returns_staged_objects_for_cleanup_and_does_not_reuse_identity() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); let first = service - .grep_paths(NamespaceGrepRequest { - path: "/runs/train.log".to_owned(), - pattern: "loss".to_owned(), - patterns: Vec::new(), - recursive: false, - name_glob: None, - cursor: None, - limit: 1, - max_files: None, - max_bytes: None, - }) + .publish_artifact(artifact_request(name.clone(), "first", b"first")) .unwrap(); - assert_eq!(first.matches.len(), 1); - assert_eq!(first.matches[0].line_number, 1); - assert_eq!(first.pattern, "loss"); - assert!(first.patterns.is_empty()); + let err = service + .publish_artifact(artifact_request(name.clone(), "duplicate", b"duplicate")) + .unwrap_err(); + let staged = match err { + MetadError::PublishArtifactFailed { source, staged } => { + assert!(matches!( + *source, + MetadError::Metadata(MetadataError::PredicateFailed) + )); + staged + } + err => panic!("unexpected publish error: {err:?}"), + }; + assert_eq!(staged.len(), 1); + for object in staged.objects() { + assert!(objects.head(&object.key).unwrap().is_some()); + } + assert_eq!( + service.lookup_plus(InodeId::root(), &name).unwrap(), + Some(first.clone()) + ); - let second = service - .grep_paths(NamespaceGrepRequest { - path: "/runs/train.log".to_owned(), - pattern: "loss".to_owned(), - patterns: Vec::new(), - recursive: false, - name_glob: None, - cursor: first.next_cursor, - limit: 1, - max_files: None, - max_bytes: None, - }) + let cleanup = service.cleanup_staged_objects(&staged).unwrap(); + assert_eq!(cleanup.attempted, staged.len()); + assert_eq!(cleanup.deleted, staged.len()); + assert_eq!(cleanup.missing, 0); + for object in staged.objects() { + assert!(objects.head(&object.key).unwrap().is_none()); + } + drop(service); + + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let next_name = DentryName::new(b"next.bin".to_vec()).unwrap(); + let next = reopened + .publish_artifact(artifact_request(next_name, "next", b"next")) .unwrap(); - assert_eq!(second.matches.len(), 1); - assert_eq!(second.matches[0].line_number, 2); -} -#[test] -fn namespace_grep_multiple_patterns_match_any() { - let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/runs/a.log", "runs/a.log", b"alpha metric\n"); - publish_path_artifact( - &service, - "/runs/b.log", - "runs/b.log", - b"nothing here\nbeta metric\n", - ); + assert!(next.attr.inode.get() > first.attr.inode.get() + 1); + assert!(next.attr.generation > first.attr.generation + 1); +} - let result = service - .grep_paths(NamespaceGrepRequest { - path: "/runs".to_owned(), - pattern: String::new(), - patterns: vec!["alpha".to_owned(), "beta".to_owned()], - recursive: false, - name_glob: None, - cursor: None, - limit: 10, - max_files: None, - max_bytes: None, - }) - .unwrap(); +fn dname(raw: &[u8]) -> DentryName { + DentryName::new(raw.to_vec()).unwrap() +} - assert_eq!(result.matches.len(), 2); - assert_eq!(result.matches[0].path, "/runs/a.log"); - assert_eq!(result.matches[0].line_number, 1); - assert_eq!(result.matches[1].path, "/runs/b.log"); - assert_eq!(result.matches[1].line_number, 2); - assert_eq!(result.patterns, vec!["alpha", "beta"]); +fn block_count_for(objects: &MemoryObjectStore, inode: InodeId, generation: u64) -> usize { + // Count the published blocks the base file owns under its (inode, generation). + let mut count = 0; + let mut block = 0; + while objects + .head(&block_key(inode, generation, 0, block)) + .unwrap() + .is_some() + { + count += 1; + block += 1; + } + count } #[test] -fn namespace_grep_multiple_patterns_match_cjk() { - let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact( - &service, - "/runs/notes.txt", - "runs/notes.txt", - "今日营养记录\n普通一行\n食谱更新完成\n".as_bytes(), +fn clone_subtree_shares_base_blocks_diverges_on_write_and_keeps_gc_safe() { + let (service, objects) = service_with_objects(); + // 1. Base namespace: /base with files a ("AAA..") and b ("BBB.."). + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let a = publish_path_artifact(&service, "/base/a", "base/a", &vec![b'A'; 4096]); + let b = publish_path_artifact(&service, "/base/b", "base/b", &vec![b'B'; 4096]); + let a_gen = a.body.as_ref().unwrap().generation; + let b_gen = b.body.as_ref().unwrap().generation; + let a_block = block_key(a.attr.inode, a_gen, 0, 0); + let b_block = block_key(b.attr.inode, b_gen, 0, 0); + assert!(objects.head(&a_block).unwrap().is_some()); + assert!(objects.head(&b_block).unwrap().is_some()); + let objects_after_base = objects.object_count(); + + // 2. Writable O(1)-ish fork of /base. + let fork = service.clone_subtree_path("/base").unwrap(); + assert_ne!(fork.root, base.attr.inode); + + // 3. Sharing: the fork sees the base content, with NO duplicate blocks. + let fork_a = service + .lookup_plus(fork.root, &dname(b"a")) + .unwrap() + .unwrap(); + let fork_b = service + .lookup_plus(fork.root, &dname(b"b")) + .unwrap() + .unwrap(); + assert_ne!( + fork_a.attr.inode, a.attr.inode, + "fork must use a fresh inode" + ); + // Shared files keep the source's content generation (the CoW sharing signal). + assert_eq!(fork_a.attr.generation, a_gen); + assert_eq!(fork_b.attr.generation, b_gen); + assert_eq!(fork_b.body.as_ref().unwrap().generation, b_gen); + assert_eq!( + service.read_artifact(fork.root, &dname(b"a")).unwrap(), + vec![b'A'; 4096] + ); + assert_eq!( + service.read_artifact(fork.root, &dname(b"b")).unwrap(), + vec![b'B'; 4096] + ); + // Zero-copy: clone added metadata only, not object blocks. + assert_eq!( + objects.object_count(), + objects_after_base, + "clone must share base blocks, not copy them" + ); + // The fork's a/b manifests reference the SAME object keys as the base. + assert_eq!( + service + .read_file_plan(fork_a.attr.inode, fork_a.attr.generation, 0, 4096) + .unwrap() + .blocks[0] + .object_key, + a_block.as_str() ); - let result = service - .grep_paths(NamespaceGrepRequest { - path: "/runs/notes.txt".to_owned(), - pattern: String::new(), - patterns: vec!["营养".to_owned(), "食谱".to_owned()], - recursive: false, - name_glob: None, - cursor: None, - limit: 10, - max_files: None, - max_bytes: None, + // 4. Divergence: rewrite a in the fork and add a new file c. + service + .replace_artifact(PublishArtifact { + parent: fork.root, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:zzz".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "fork/a".to_owned(), + bytes: vec![b'Z'; 4096], + mode: 0o644, + uid: 1000, + gid: 1000, }) .unwrap(); - - let lines = result - .matches - .iter() - .map(|entry| entry.line_number) - .collect::>(); - assert_eq!(lines, vec![1, 3]); -} - -/// `patterns` adds OR alternatives to `pattern` (union semantics); a non-empty -/// `pattern` must keep matching when `patterns` is also provided instead of -/// being silently dropped. -#[test] -fn namespace_grep_unions_pattern_with_patterns() { - let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact( - &service, - "/runs/notes.txt", - "runs/notes.txt", - "食谱更新完成\n无关的一行\n食材已备齐\n新 recipe 上线\n".as_bytes(), - ); - - let result = service - .grep_paths(NamespaceGrepRequest { - path: "/runs/notes.txt".to_owned(), - pattern: "食谱".to_owned(), - patterns: vec!["食材".to_owned(), "recipe".to_owned()], - recursive: false, - name_glob: None, - cursor: None, - limit: 10, - max_files: None, - max_bytes: None, + service + .publish_artifact(PublishArtifact { + parent: fork.root, + name: dname(b"c"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:ccc".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "fork/c".to_owned(), + bytes: vec![b'C'; 4096], + mode: 0o644, + uid: 1000, + gid: 1000, }) .unwrap(); - let lines = result - .matches - .iter() - .map(|entry| entry.line_number) - .collect::>(); - assert_eq!(lines, vec![1, 3, 4]); - // The echo reports the request fields verbatim. - assert_eq!(result.pattern, "食谱"); - assert_eq!(result.patterns, vec!["食材", "recipe"]); -} - -/// The workbench pipe-split forwards `pattern: "a|b"` together with -/// `patterns: ["a", "b"]`. Any line containing the literal "a|b" also contains -/// each split alternative, so union semantics must return the same match set -/// as the split alternatives alone. -#[test] -fn namespace_grep_piped_pattern_union_matches_split_alternatives() { - let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact( - &service, - "/runs/mixed.log", - "runs/mixed.log", - b"alpha metric\nliteral alpha|beta row\nbeta metric\nnothing\n", + // 4a. Fork now sees a="ZZZ..", b="BBB..", c present. + assert_eq!( + service.read_artifact(fork.root, &dname(b"a")).unwrap(), + vec![b'Z'; 4096] ); - let request = |pattern: &str, patterns: Vec| NamespaceGrepRequest { - path: "/runs/mixed.log".to_owned(), - pattern: pattern.to_owned(), - patterns, - recursive: false, - name_glob: None, - cursor: None, - limit: 10, - max_files: None, - max_bytes: None, - }; - - let split_only = service - .grep_paths(request("", vec!["alpha".to_owned(), "beta".to_owned()])) - .unwrap(); - let piped_union = service - .grep_paths(request( - "alpha|beta", - vec!["alpha".to_owned(), "beta".to_owned()], - )) - .unwrap(); + assert_eq!( + service.read_artifact(fork.root, &dname(b"b")).unwrap(), + vec![b'B'; 4096] + ); + assert_eq!( + service.read_artifact(fork.root, &dname(b"c")).unwrap(), + vec![b'C'; 4096] + ); + // 4b. Base is unchanged: a="AAA..", no c. + assert_eq!( + service + .read_artifact(base.attr.inode, &dname(b"a")) + .unwrap(), + vec![b'A'; 4096] + ); + assert!(service + .lookup_plus(base.attr.inode, &dname(b"c")) + .unwrap() + .is_none()); - let lines = |result: &NamespaceGrepResult| { - result - .matches - .iter() - .map(|entry| (entry.path.clone(), entry.line_number)) - .collect::>() - }; - assert_eq!(lines(&piped_union), lines(&split_only)); + // 6. Diff reports exactly { modified: a, added: c }; b (shared) is not reported. + let mut diff = service.diff_subtrees(base.attr.inode, fork.root).unwrap(); + diff.sort_by(|left, right| left.path.cmp(&right.path)); + let summary: Vec<(&str, SubtreeDeltaKind)> = + diff.iter().map(|d| (d.path.as_str(), d.kind)).collect(); assert_eq!( - lines(&split_only), + summary, vec![ - ("/runs/mixed.log".to_owned(), 1), - ("/runs/mixed.log".to_owned(), 2), - ("/runs/mixed.log".to_owned(), 3), + ("/a", SubtreeDeltaKind::Modified), + ("/c", SubtreeDeltaKind::Added), ] ); -} + // The enriched diff carries the changed file's content digest. + assert!(diff + .iter() + .find(|d| d.path == "/a") + .unwrap() + .digest + .is_some()); -#[test] -fn namespace_grep_name_glob_skips_unmatched_files_without_reading() { - let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/runs/a.md", "runs/a.md", b"needle in md\n"); - publish_path_artifact(&service, "/runs/b.log", "runs/b.log", b"needle in log\n"); + // 5. GC safety: reclaim must NOT touch base blocks the fork's divergent write + // abandoned but the base still references; they are owned by the base inode and + // protected by the fork's retained snapshot pin. + let reclaim = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(reclaim.deleted, 0, "no base block may be reclaimed yet"); + assert!(objects.head(&a_block).unwrap().is_some()); + assert!(objects.head(&b_block).unwrap().is_some()); + assert_eq!( + service + .read_artifact(base.attr.inode, &dname(b"a")) + .unwrap(), + vec![b'A'; 4096] + ); - let result = service - .grep_paths(NamespaceGrepRequest { - path: "/runs".to_owned(), - pattern: "needle".to_owned(), - patterns: Vec::new(), - recursive: false, - name_glob: Some("*.md".to_owned()), - cursor: None, - limit: 10, - max_files: None, - max_bytes: None, - }) + // Drop the fork: remove its files and retire its snapshot pin. The fork-only + // blocks (the divergent a' and the new c) then become reclaimable, while the + // base's blocks remain because the base still references them. + let fork_a_diverged = service + .lookup_plus(fork.root, &dname(b"a")) + .unwrap() .unwrap(); - - assert_eq!(result.matches.len(), 1); - assert_eq!(result.matches[0].path, "/runs/a.md"); - assert_eq!(result.files_scanned, 1); - assert_eq!(result.bytes_read, b"needle in md\n".len()); + let fork_c = service + .lookup_plus(fork.root, &dname(b"c")) + .unwrap() + .unwrap(); + let fork_a_block = block_key( + fork_a_diverged.attr.inode, + fork_a_diverged.body.as_ref().unwrap().generation, + 0, + 0, + ); + let fork_c_block = block_key( + fork_c.attr.inode, + fork_c.body.as_ref().unwrap().generation, + 0, + 0, + ); + service.remove_file(fork.root, &dname(b"a")).unwrap(); + service.remove_file(fork.root, &dname(b"b")).unwrap(); + service.remove_file(fork.root, &dname(b"c")).unwrap(); + assert!(service.retire_snapshot(fork.snapshot_id).unwrap()); + let reclaim = service.cleanup_pending_objects(100).unwrap(); + assert!(reclaim.deleted >= 2, "fork-only blocks must be reclaimable"); + assert!(objects.head(&fork_a_block).unwrap().is_none()); + assert!(objects.head(&fork_c_block).unwrap().is_none()); + // Base remains fully intact and readable. + assert!(objects.head(&a_block).unwrap().is_some()); + assert!(objects.head(&b_block).unwrap().is_some()); + assert_eq!( + service + .read_artifact(base.attr.inode, &dname(b"a")) + .unwrap(), + vec![b'A'; 4096] + ); + assert_eq!( + service + .read_artifact(base.attr.inode, &dname(b"b")) + .unwrap(), + vec![b'B'; 4096] + ); + assert_eq!(block_count_for(&objects, a.attr.inode, a_gen), 1); } #[test] -fn namespace_grep_name_glob_matches_cjk_substring() { +fn clone_subtree_copies_nested_dirs_and_diff_reports_removed() { let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact( - &service, - "/runs/营养日志.txt", - "runs/nutrition.txt", - "记录一\n".as_bytes(), - ); - publish_path_artifact( - &service, - "/runs/训练日志.txt", - "runs/training.txt", - "记录二\n".as_bytes(), - ); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/base/sub", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact(&service, "/base/sub/deep", "base/deep", b"deep-bytes"); + publish_path_artifact(&service, "/base/top", "base/top", b"top-bytes"); - let result = service - .grep_paths(NamespaceGrepRequest { - path: "/runs".to_owned(), - pattern: "记录".to_owned(), - patterns: Vec::new(), - recursive: false, - name_glob: Some("*营养*".to_owned()), - cursor: None, - limit: 10, - max_files: None, - max_bytes: None, - }) + let fork = service.clone_subtree_path("/base").unwrap(); + // Nested structure is reproduced under fresh inodes. + let sub = service + .lookup_plus(fork.root, &dname(b"sub")) + .unwrap() .unwrap(); + assert_eq!(sub.attr.file_type, FileType::Directory); + assert_eq!( + service + .read_artifact(sub.attr.inode, &dname(b"deep")) + .unwrap(), + b"deep-bytes" + ); - assert_eq!(result.matches.len(), 1); - assert_eq!(result.matches[0].path, "/runs/营养日志.txt"); - assert_eq!(result.files_scanned, 1); + // Identical subtree => no deltas. + let base = service.resolve_directory_path("/base").unwrap(); + assert!(service.diff_subtrees(base, fork.root).unwrap().is_empty()); + + // Remove a nested file in the fork => Removed delta at the nested path, + // direction base -> fork. + service + .remove_file(sub.attr.inode, &dname(b"deep")) + .unwrap(); + let removed = service.diff_subtrees(base, fork.root).unwrap(); + assert_eq!(removed.len(), 1); + assert_eq!(removed[0].path, "/sub/deep"); + assert_eq!(removed[0].kind, SubtreeDeltaKind::Removed); + assert_eq!(removed[0].size_delta, -(b"deep-bytes".len() as i64)); + // Reversed direction reports it as Added, with the net size flipped. + let added = service.diff_subtrees(fork.root, base).unwrap(); + assert_eq!(added.len(), 1); + assert_eq!(added[0].path, "/sub/deep"); + assert_eq!(added[0].kind, SubtreeDeltaKind::Added); + assert_eq!(added[0].size_delta, b"deep-bytes".len() as i64); } #[test] -fn namespace_grep_rejects_more_than_sixteen_patterns() { +fn clone_subtree_path_rejects_non_directory() { let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - - let err = service - .grep_paths(NamespaceGrepRequest { - path: "/runs".to_owned(), - pattern: String::new(), - patterns: (0..17).map(|index| format!("p{index}")).collect(), - recursive: false, - name_glob: None, - cursor: None, - limit: 10, - max_files: None, - max_bytes: None, - }) - .unwrap_err(); - - assert!(matches!(err, MetadError::InvalidQuery(message) if message.contains("16"))); + publish_path_artifact(&service, "/file.bin", "file", b"bytes"); + assert!(matches!( + service.clone_subtree_path("/file.bin"), + Err(MetadError::NotDirectory) + )); } #[test] -fn namespace_grep_rejects_empty_pattern_entry() { - let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); +fn clone_link_rejects_an_expired_reaped_pin_before_exposing_the_fork() { + let store = SnapshotCommitBarrierStore::new(CommandKind::CreateDir, 1, 2) + .matching_request_prefix(b"clone-subtree-link"); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let published = service + .publish_artifact(PublishArtifact { + parent: base.attr.inode, + name: dname(b"data.bin"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:clone-link-race".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "clone-link-race".to_owned(), + bytes: b"payload".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); - let err = service - .grep_paths(NamespaceGrepRequest { - path: "/runs".to_owned(), - pattern: String::new(), - patterns: vec!["ok".to_owned(), String::new()], - recursive: false, - name_glob: None, - cursor: None, - limit: 10, - max_files: None, - max_bytes: None, + let clone_service = Arc::clone(&service); + let clone = std::thread::spawn(move || clone_service.clone_subtree_path_into("/base", "/fork")); + store.wait_until_blocked(); + assert!(service.object_gc_gate.try_lock().is_err()); + + let pins = store + .scan(ScanRequest { + family: RecordFamily::Snapshot, + prefix: snapshot_pin_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, }) - .unwrap_err(); + .unwrap(); + assert_eq!(pins.len(), 1); + let pin = decode_snapshot_pin(&pins[0].value.0).unwrap(); - assert!(matches!(err, MetadError::InvalidQuery(message) if message.contains("empty"))); + service + .remove_file(base.attr.inode, &dname(b"data.bin")) + .unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + + let cleanup_started = Arc::new(Barrier::new(2)); + let cleanup_service = Arc::clone(&service); + let cleanup_thread_started = Arc::clone(&cleanup_started); + let cleanup = std::thread::spawn(move || { + cleanup_thread_started.wait(); + cleanup_service.cleanup_pending_objects(100) + }); + cleanup_started.wait(); + store.release_blocked(); + + assert!(matches!( + clone.join().unwrap(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + let cleanup = cleanup.join().unwrap().unwrap(); + assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&object).unwrap().is_none()); + assert!(service.lookup_path("/fork").unwrap().is_none()); } #[test] -fn namespace_grep_rejects_missing_pattern_and_patterns() { - let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); +fn fork_binding_survives_pin_reaping_and_a_hardlink_escaping_the_fork_root() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( + &service, + "/base/data.bin", + "fork-retention/source", + b"shared bytes", + ); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); - let err = service - .grep_paths(NamespaceGrepRequest { - path: "/runs".to_owned(), - pattern: String::new(), - patterns: Vec::new(), - recursive: false, - name_glob: None, - cursor: None, - limit: 10, - max_files: None, - max_bytes: None, - }) - .unwrap_err(); + let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); + let fork_file = service + .lookup_plus(fork.root, &dname(b"data.bin")) + .unwrap() + .unwrap(); + service + .link(fork_file.attr.inode, InodeId::root(), dname(b"escaped.bin")) + .unwrap(); + service.remove_file(fork.root, &dname(b"data.bin")).unwrap(); + service.remove_empty_dir_path("/fork").unwrap(); - assert!(matches!(err, MetadError::InvalidQuery(message) if message.contains("pattern"))); -} + // The source can stop naming its object and the construction pin can expire; + // neither event retires the durable binding. The escaped hardlink remains a + // current borrowed reference even though the fork root itself is gone. + service.remove_file_path("/base/data.bin").unwrap(); + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + assert!(service.snapshot_pin(fork.snapshot_id).unwrap().is_none()); + assert_eq!( + service + .metadata + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: crate::layout::fork_binding_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); + assert!(cleanup.blocked_by_snapshots >= 1); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!( + service + .read_artifact(InodeId::root(), &dname(b"escaped.bin")) + .unwrap(), + b"shared bytes" + ); -#[test] -fn namespace_read_structured_offset_without_cursor_skips_items() { - let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/runs/a.json", "runs/a.json", br#"["a","b","c"]"#); + let error = service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap_err(); + assert!( + matches!( + error, + MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + } if snapshot_id == fork.snapshot_id + && fork_root == fork.root + && borrower == fork_file.attr.inode + ), + "unexpected retirement error: {error:?}" + ); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!( + service + .read_artifact(InodeId::root(), &dname(b"escaped.bin")) + .unwrap(), + b"shared bytes" + ); - let page = service - .read_page( - "/runs/a.json", - NamespaceReadOptions { - format: NamespaceReadFormat::Structured, - cursor: None, - offset: 1, - limit: 1, - expected_generation: None, - }, - ) + // Retirement becomes valid after every borrowed fork reference, including + // links outside the original fork root, has been removed. + service + .remove_file(InodeId::root(), &dname(b"escaped.bin")) .unwrap(); - - assert_eq!(page.items.len(), 1); - assert_eq!(page.items[0].index, 1); - assert_eq!(page.items[0].value_json, r#""b""#); - assert!(page.truncated); + assert!(service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&source_block).unwrap().is_none()); } #[test] -fn namespace_read_bytes_honors_returned_cursor() { +fn detached_fork_binding_is_a_legal_link_and_rename_root() { let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/runs/a.bin", "runs/a.bin", b"abcdef"); - - let first = service - .read_page( - "/runs/a.bin", - NamespaceReadOptions { - format: NamespaceReadFormat::Bytes, - cursor: None, - offset: 0, - limit: 2, - expected_generation: None, - }, - ) + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/base/data.bin", + "detached-namespace/source", + b"shared bytes", + ); + let fork = service.clone_subtree_path("/base").unwrap(); + assert!( + !service.materialization_orphan_slow_path_enabled(), + "a durable detached ForkBinding must restore the healthy fast path" + ); + let reachability_scans = service.namespace_reachability_scan_count(); + let fork_file = service + .lookup_plus(fork.root, &dname(b"data.bin")) + .unwrap() .unwrap(); - assert_eq!(first.bytes.as_deref(), Some(b"ab".as_slice())); - let second = service - .read_page( - "/runs/a.bin", - NamespaceReadOptions { - format: NamespaceReadFormat::Bytes, - cursor: first.next_cursor, - offset: 0, - limit: 2, - expected_generation: None, - }, + let renamed = service + .rename( + fork.root, + &dname(b"data.bin"), + fork.root, + dname(b"renamed.bin"), ) .unwrap(); - assert_eq!(second.bytes.as_deref(), Some(b"cd".as_slice())); + assert_eq!(renamed.attr.inode, fork_file.attr.inode); + let escaped = service + .link(fork_file.attr.inode, InodeId::root(), dname(b"escaped.bin")) + .unwrap(); + assert_eq!(escaped.attr.inode, fork_file.attr.inode); + assert_eq!( + service + .read_artifact(InodeId::root(), &dname(b"escaped.bin")) + .unwrap(), + b"shared bytes" + ); + assert_eq!( + service.namespace_reachability_scan_count(), + reachability_scans, + "healthy detached-root rename/link must not scan namespace reachability" + ); } #[test] -fn register_namespace_index_rejects_rows_outside_registered_path() { - let service = service(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - - let err = service - .register_namespace_index(NamespaceIndexRegistration { - path: "/runs".to_owned(), - fields: vec![NamespaceIndexField { - field: NamespaceFindField::new("run.status"), - operators: vec![NamespacePredicateOp::Eq], - sortable: false, - facetable: true, - }], - rows: vec![NamespaceIndexRow { - path: "/archive/a.json".to_owned(), - values: Vec::new(), - }], - }) - .unwrap_err(); +fn reopen_recovers_an_unbound_materialization_into_the_slow_path() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/base/data.bin", + "reopen-orphan/source", + b"borrowed", + ); + let pin = service.snapshot_subtree(base.attr.inode).unwrap(); + let orphan_root = { + let _object_gc_gate = service.object_gc_gate.lock().unwrap(); + service + .materialize_subtree_at(base.attr.inode, Version::new(pin.read_version).unwrap()) + .unwrap() + }; + let orphan = service + .lookup_plus(orphan_root, &dname(b"data.bin")) + .unwrap() + .unwrap(); + assert!(service.materialization_orphan_slow_path_enabled()); + let reopened = + NoKvFs::open_existing(service.mount, metadata, objects, service.shard_index()).unwrap(); assert!( - matches!(err, MetadError::InvalidQuery(message) if message.contains("outside registered namespace")) + reopened.materialization_orphan_slow_path_enabled(), + "reopen must recover an unbound detached tree before serving" ); + assert_eq!(reopened.namespace_reachability_scan_count(), 1); + assert!(matches!( + reopened.link(orphan.attr.inode, InodeId::root(), dname(b"reopened-link")), + Err(MetadError::NotFound) + )); + assert!(matches!( + reopened.rename( + orphan_root, + &dname(b"data.bin"), + InodeId::root(), + dname(b"reopened-rename") + ), + Err(MetadError::NotFound) + )); + assert!(reopened + .lookup_plus(InodeId::root(), &dname(b"reopened-link")) + .unwrap() + .is_none()); + assert!(reopened + .lookup_plus(InodeId::root(), &dname(b"reopened-rename")) + .unwrap() + .is_none()); } #[test] -fn register_namespace_index_uses_metadata_predicate_fence() { +fn open_existing_allows_empty_namespace_until_bootstrap_then_uses_fast_path() { let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), + let objects = MemoryObjectStore::new(); + let service = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + + assert!( + service.materialization_orphan_slow_path_enabled(), + "an empty namespace remains fail-closed until its root is bootstrapped" ); + assert_eq!(service.namespace_reachability_scan_count(), 0); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + assert!( + !service.materialization_orphan_slow_path_enabled(), + "root bootstrap must prove the pristine namespace healthy" + ); + assert_eq!(service.namespace_reachability_scan_count(), 1); - let before = metadata.metadata_store_stats(); + let source = service + .create_file(InodeId::root(), dname(b"source.bin"), 0o644, 1000, 1000) + .unwrap(); + let scans_after_bootstrap = service.namespace_reachability_scan_count(); service - .register_namespace_index(NamespaceIndexRegistration { - path: "/runs".to_owned(), - fields: vec![NamespaceIndexField { - field: NamespaceFindField::new("run.status"), - operators: vec![NamespacePredicateOp::Eq], - sortable: false, - facetable: true, - }], - rows: vec![NamespaceIndexRow { - path: "/runs/a.json".to_owned(), - values: vec![NamespaceIndexValue { - field: NamespaceFindField::new("run.status"), - value: NamespacePredicateValue::String("completed".to_owned()), - }], + .link(source.attr.inode, InodeId::root(), dname(b"linked.bin")) + .unwrap(); + assert_eq!( + service.namespace_reachability_scan_count(), + scans_after_bootstrap, + "healthy link after bootstrap must not run namespace reachability" + ); +} + +#[test] +fn open_existing_rejects_missing_root_with_unbound_materialization_records() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let writer = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + let orphan = InodeId::new(InodeId::ROOT_RAW + 1).unwrap(); + let version = writer.next_version().unwrap(); + let attr = directory_attr(orphan, 0o755, 1000, 1000, version.get()); + writer + .commit_metadata(MetadataCommand { + request_id: request_id(b"test-rootless-orphan", writer.mount, orphan, version), + kind: CommandKind::CreateDir, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Inode, + primary_key: inode_key(writer.mount, orphan), + predicates: Vec::new(), + mutations: vec![Mutation { + family: RecordFamily::Inode, + key: inode_key(writer.mount, orphan), + op: MutationOp::Put, + value: Some(Value(encode_inode_attr(&attr))), }], + watch: Vec::new(), }) .unwrap(); - let after = metadata.metadata_store_stats(); - assert_eq!(after.predicate_total - before.predicate_total, 1); + let reopened = NoKvFs::open_existing(writer.mount, metadata, objects, writer.shard_index()); + assert!(matches!( + reopened, + Err(MetadError::Codec(message)) + if message == "mount root is missing while namespace records still exist" + )); } #[test] -fn stale_path_index_falls_back_to_canonical_namespace() { - let service = service(); - let runs = service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let archive = service - .create_dir_path("/archive", 0o755, 1000, 1000) - .unwrap(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let prepared = service - .prepare_artifact_create_path("/runs/checkpoint.bin") - .unwrap(); - let artifact = service - .publish_prepared_artifact( - prepared.clone(), - body_descriptor(prepared.generation, 6), - vec![one_chunk_manifest(prepared.inode, prepared.generation, 6)], - 0o644, +fn fork_binding_retains_object_backed_symlink_after_source_deletion() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = service + .create_symlink( + base.attr.inode, + dname(b"latest"), + b"runs/42/checkpoint.bin".to_vec(), + 0o777, 1000, 1000, ) + .unwrap(); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); + let fork_symlink = service + .lookup_plus(fork.root, &dname(b"latest")) .unwrap() - .entry; - - assert!(service.stat_path("/runs/checkpoint.bin").unwrap().is_some()); - - service - .rename(runs.attr.inode, &name, archive.attr.inode, name.clone()) .unwrap(); - let before = service.metadata_service_stats(); - assert_eq!(service.stat_path("/runs/checkpoint.bin").unwrap(), None); - let after = service.metadata_service_stats(); + service.remove_file_path("/base/latest").unwrap(); + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); assert_eq!( - after.path_index_lookup_total - before.path_index_lookup_total, + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, 1 ); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&source_block).unwrap().is_some()); assert_eq!( - after.path_index_stale_total - before.path_index_stale_total, - 1 + service.read_symlink(fork_symlink.attr.inode).unwrap(), + b"runs/42/checkpoint.bin" ); + + assert!(matches!( + service.retire_snapshot_path("/base", fork.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + }) if snapshot_id == fork.snapshot_id + && fork_root == fork.root + && borrower == fork_symlink.attr.inode + )); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&source_block).unwrap().is_some()); assert_eq!( - after.path_index_fallback_total - before.path_index_fallback_total, - 1 + service.read_symlink(fork_symlink.attr.inode).unwrap(), + b"runs/42/checkpoint.bin" ); - let mut moved_artifact = artifact; - moved_artifact.dentry.parent = archive.attr.inode; + service.remove_file(fork.root, &dname(b"latest")).unwrap(); + assert!(service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap()); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 1); + assert!(objects.head(&source_block).unwrap().is_none()); +} - let before = service.metadata_service_stats(); - let metadata = service - .stat_path("/archive/checkpoint.bin") - .unwrap() - .expect("moved artifact stat"); - assert_eq!(metadata.attr, moved_artifact.attr); - assert_eq!(metadata.body, moved_artifact.body); - let after = service.metadata_service_stats(); +#[test] +fn fork_binding_can_retire_after_the_fork_rewrites_onto_self_owned_blocks() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( + &service, + "/base/data.bin", + "fork-rewrite/source", + b"borrowed bytes", + ); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); + + let rewritten = service + .replace_artifact(PublishArtifact { + parent: fork.root, + name: dname(b"data.bin"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:fork-rewrite".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "fork-rewrite/self-owned".to_owned(), + bytes: b"self-owned bytes".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let rewritten_body = rewritten.entry.body.as_ref().unwrap(); + let rewritten_block = block_key(rewritten.entry.attr.inode, rewritten_body.generation, 0, 0); + service + .create_file( + InodeId::root(), + dname(b"unrelated-empty"), + 0o644, + 1000, + 1000, + ) + .unwrap(); + service.remove_file_path("/base/data.bin").unwrap(); + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); assert_eq!( - after.path_index_miss_total - before.path_index_miss_total, + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, 1 ); + + assert!(service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap()); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 1); + assert!(objects.head(&source_block).unwrap().is_none()); + assert!(objects.head(&rewritten_block).unwrap().is_some()); assert_eq!( - after.path_index_fallback_total - before.path_index_fallback_total, - 1 + read_artifact_at_path(&service, "/fork/data.bin"), + b"self-owned bytes" ); } #[test] -fn path_index_page_lists_immediate_indexed_children_with_holt_delimiter() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), +fn unrelated_cross_shard_graft_does_not_block_safe_fork_retirement() { + let service = service(); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); + let foreign = InodeId::compose(1, 42).unwrap(); + service + .create_graft( + InodeId::root(), + dname(b"foreign"), + foreign, + 0o755, + 1000, + 1000, + ) + .unwrap(); + + assert!(service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap()); + assert!(!service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap()); + assert_eq!( + service + .lookup_plus(InodeId::root(), &dname(b"foreign")) + .unwrap() + .unwrap() + .attr + .inode, + foreign ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let epoch = service - .create_dir_path("/runs/epoch", 0o755, 1000, 1000) +} + +#[test] +fn fork_binding_cannot_retire_when_append_still_inherits_borrowed_blocks() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( + &service, + "/base/data.bin", + "fork-append/source", + b"borrowed base", + ); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); + let fork_file = service + .lookup_plus(fork.root, &dname(b"data.bin")) + .unwrap() + .unwrap(); + + let prepared = service + .prepare_artifact_replace(fork.root, dname(b"data.bin")) + .unwrap(); + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "fork-append/delta", + &[PublishArtifactRange { + offset: source_body.size, + bytes: b" + self-owned delta".to_vec(), + }], + 0, + ) .unwrap(); service - .create_file_path("/runs/plain.txt", 0o644, 1000, 1000) + .publish_prepared_artifact_staged_session( + prepared, + PublishArtifactStagedSession { + parent: fork.root, + name: dname(b"data.bin"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:fork-append".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "fork-append/delta".to_owned(), + size: source_body.size + b" + self-owned delta".len() as u64, + chunks: written.chunk_manifests(), + staged: written.staged_objects().unwrap(), + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) .unwrap(); - publish_path_artifact(&service, "/runs/top.bin", "runs/top.bin", b"top"); - publish_path_artifact( - &service, - "/runs/epoch/ckpt.bin", - "runs/epoch/ckpt.bin", - b"ckpt", - ); - let before = metadata.metadata_store_stats(); - let first = service.list_indexed_path_page("/runs", None, 1).unwrap(); - let after_first = metadata.metadata_store_stats(); - assert_eq!(first.entries, vec![epoch]); + service.remove_file_path("/base/data.bin").unwrap(); + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); assert_eq!( - first.next_cursor.as_ref().map(DentryName::as_bytes), - Some(b"epoch".as_slice()) + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 ); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&source_block).unwrap().is_some()); assert_eq!( - after_first.scan_key_returned_total - before.scan_key_returned_total, - 2 + read_artifact_at_path(&service, "/fork/data.bin"), + b"borrowed base + self-owned delta" ); - let second = service - .list_indexed_path_page("/runs", first.next_cursor.as_ref(), 10) - .unwrap(); - assert_eq!(second.entries.len(), 1); - assert_eq!(second.entries[0].dentry.name.as_bytes(), b"top.bin"); - assert_eq!(second.entries[0].attr.file_type, FileType::File); - assert_eq!(second.next_cursor, None); + assert!(matches!( + service.retire_snapshot_path("/base", fork.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + }) if snapshot_id == fork.snapshot_id + && fork_root == fork.root + && borrower == fork_file.attr.inode + )); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!( + read_artifact_at_path(&service, "/fork/data.bin"), + b"borrowed base + self-owned delta" + ); } #[test] -fn path_index_page_skips_stale_rows_without_truncating_visible_children() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service - .create_dir_path("/archive", 0o755, 1000, 1000) - .unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - service - .create_dir_path("/runs/aaa", 0o755, 1000, 1000) - .unwrap(); - publish_path_artifact( +fn fork_binding_retirement_fails_closed_on_corrupt_dentry_projection_body() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( &service, - "/runs/aaa/stale.bin", - "runs/aaa/stale.bin", - b"stale", + "/base/data.bin", + "fork-corruption/source", + b"must remain retained", ); - service.rename_path("/runs/aaa", "/archive/aaa").unwrap(); - let first_valid = publish_path_artifact(&service, "/runs/bbb.bin", "runs/bbb.bin", b"bbb"); - let second_valid = publish_path_artifact(&service, "/runs/ccc.bin", "runs/ccc.bin", b"ccc"); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); - let before_store = metadata.metadata_store_stats(); - let before_service = service.metadata_service_stats(); - let first = service.list_indexed_path_page("/runs", None, 1).unwrap(); - let after_first_store = metadata.metadata_store_stats(); - let after_first_service = service.metadata_service_stats(); - assert_eq!(first.entries, vec![first_valid]); - assert_eq!( - first.next_cursor.as_ref().map(DentryName::as_bytes), - Some(b"bbb.bin".as_slice()) - ); - assert!( - after_first_store.scan_key_returned_total - before_store.scan_key_returned_total > 2, - "stale index row should force an extra delimiter scan page" - ); - assert_eq!( - after_first_service.read_dir_plus_entry_total - before_service.read_dir_plus_entry_total, - 1 - ); + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); assert_eq!( - after_first_service.read_dir_plus_projection_hit_total - - before_service.read_dir_plus_projection_hit_total, + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, 1 ); - assert!( - after_first_service.path_index_scan_stale_total - - before_service.path_index_scan_stale_total - >= 1, - "stale derived path-index rows should be reported separately from live entries" - ); - let second = service - .list_indexed_path_page("/runs", first.next_cursor.as_ref(), 1) - .unwrap(); - assert_eq!(second.entries, vec![second_valid]); - assert_eq!(second.next_cursor, None); -} - -#[test] -fn directory_rename_leaves_descendant_path_index_as_derived_stale_cache() { - let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let prepared = service - .prepare_artifact_create_path("/runs/checkpoint.bin") - .unwrap(); - let artifact = service - .publish_prepared_artifact( - prepared.clone(), - body_descriptor(prepared.generation, 6), - vec![one_chunk_manifest(prepared.inode, prepared.generation, 6)], - 0o644, - 1000, - 1000, + let dentry = dentry_key(service.mount, fork.root, &dname(b"data.bin")); + let current = service.read_version().unwrap(); + let row = service + .metadata + .get_versioned( + RecordFamily::Dentry, + &dentry, + current, + ReadPurpose::WritePlanLocal, ) .unwrap() - .entry; - let old_components = parse_absolute_path("/runs/checkpoint.bin").unwrap(); - let old_key = path_index_key(MountId::new(1).unwrap(), &old_components); - assert!(metadata - .get( - RecordFamily::PathIndex, - &old_key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) + .unwrap(); + let mut projection = decode_dentry_projection(&row.value.0).unwrap(); + projection + .body + .as_mut() .unwrap() - .is_some()); + .manifest_id + .push_str("/forged"); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"inject-corrupt-dentry-projection", + service.mount, + projection.attr.inode, + version, + ), + kind: CommandKind::UpdateAttr, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: dentry.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::Dentry, + key: dentry.clone(), + predicate: Predicate::VersionEquals(row.version), + }], + mutations: vec![Mutation { + family: RecordFamily::Dentry, + key: dentry, + op: MutationOp::Put, + value: Some(Value(encode_dentry_projection(&projection))), + }], + watch: Vec::new(), + }) + .unwrap(); + service.remove_file_path("/base/data.bin").unwrap(); - service.rename_path("/runs", "/archive").unwrap(); + let error = service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap_err(); + assert!( + matches!(&error, MetadError::Codec(message) if message.contains("body descriptor")), + "unexpected retirement error: {error:?}" + ); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&source_block).unwrap().is_some()); +} - let renamed_dir_key = path_index_key( - MountId::new(1).unwrap(), - &parse_absolute_path("/runs").unwrap(), +#[test] +fn detached_fork_binding_protects_after_source_deletion_and_pin_reaping() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( + &service, + "/base/data.bin", + "detached-retention/source", + b"detached bytes", ); - assert!(metadata - .get( - RecordFamily::PathIndex, - &renamed_dir_key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .is_none()); - assert!(metadata - .get( - RecordFamily::PathIndex, - &old_key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .is_some()); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let fork = service.clone_subtree_path("/base").unwrap(); + + service.remove_file_path("/base/data.bin").unwrap(); + service.remove_empty_dir_path("/base").unwrap(); + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!( + service + .read_artifact(fork.root, &dname(b"data.bin")) + .unwrap(), + b"detached bytes" + ); + + // A deleted source path cannot accidentally release the retention root. + // The unbound service primitive remains able to retire it after the detached + // fork has dropped its last borrowed reference. assert!(matches!( - service.lookup_path("/runs/checkpoint.bin"), + service.retire_snapshot_path("/base", fork.snapshot_id), Err(MetadError::NotFound) )); - assert_eq!( - service.lookup_path("/archive/checkpoint.bin").unwrap(), - Some(artifact) - ); + service.remove_file(fork.root, &dname(b"data.bin")).unwrap(); + assert!(service.retire_snapshot(fork.snapshot_id).unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&source_block).unwrap().is_none()); } #[test] -fn create_file_publishes_metadata_without_body_descriptor() { - let service = service(); - let name = DentryName::new(b"empty.txt".to_vec()).unwrap(); - let created = service - .create_file(InodeId::root(), name.clone(), 0o644, 1000, 1000) - .unwrap(); - assert_eq!(created.attr.file_type, FileType::File); - assert_eq!(created.attr.size, 0); - assert!(created.body.is_none()); +fn fork_binding_can_be_retired_through_the_sources_new_path_after_rename() { + let (service, _objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/base/data.bin", + "renamed-retention/source", + b"renamed bytes", + ); + let fork = service.clone_subtree_path("/base").unwrap(); + service + .rename_path("/base", "/renamed") + .expect("source directory rename"); + service.remove_file(fork.root, &dname(b"data.bin")).unwrap(); + + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); assert_eq!( - service.lookup_plus(InodeId::root(), &name).unwrap(), - Some(created) + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 ); + assert!(matches!( + service.retire_snapshot_path("/base", fork.snapshot_id), + Err(MetadError::NotFound) + )); + assert!(service + .retire_snapshot_path("/renamed", fork.snapshot_id) + .unwrap()); + assert!(!service.retire_snapshot(fork.snapshot_id).unwrap()); } -#[test] -fn new_file_attrs_use_wall_clock_timestamps() { - let service = service(); - let before = current_time_ms().saturating_sub(1_000); - - let created = service - .create_file( - InodeId::root(), - DentryName::new(b"empty.txt".to_vec()).unwrap(), - 0o644, - 1000, - 1000, - ) - .unwrap(); - assert!(created.attr.mtime_ms >= before); - assert!(created.attr.ctime_ms >= before); - assert!(created.attr.mtime_ms > created.attr.generation); - - let published = service - .publish_artifact(artifact_request( - DentryName::new(b"artifact.bin".to_vec()).unwrap(), - "artifact", - b"body", - )) - .unwrap(); - assert!(published.attr.mtime_ms >= before); - assert!(published.attr.ctime_ms >= before); - assert!(published.attr.mtime_ms > published.attr.generation); +fn read_artifact_at_path( + service: &NoKvFs, + path: &str, +) -> Vec { + let (parent, name) = service.resolve_parent_path(path).unwrap(); + service.read_artifact(parent, &name).unwrap() } #[test] -fn create_symlink_round_trips_target_and_unlinks_like_file() { - let service = service(); - let name = DentryName::new(b"latest".to_vec()).unwrap(); - let created = service - .create_symlink( - InodeId::root(), - name.clone(), - b"runs/42/checkpoint.bin".to_vec(), - 0o777, - 1000, - 1000, - ) +fn failed_rollback_orphan_does_not_block_unrelated_fork_retirement() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RenameReplace, 1, 2) + .matching_request_prefix(b"rollback-subtree-swap"); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + service + .publish_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a1".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-orphan/a1".to_owned(), + bytes: b"A1".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) .unwrap(); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + let ws_inode = ws.attr.inode; + let snapshot_id = snapshot.snapshot_id; - assert_eq!(created.attr.file_type, FileType::Symlink); - assert_eq!(created.attr.size, 22); - assert_eq!( - service.read_symlink(created.attr.inode).unwrap(), - b"runs/42/checkpoint.bin" - ); - assert_eq!( - created.body.as_ref().unwrap().digest_uri, - "sha256:15a533489b90109ab69bd64dabcc260602c854b6b4a472b20aefa0eabcee3a24" - ); - assert_eq!( - service.lookup_plus(InodeId::root(), &name).unwrap(), - Some(created.clone()) + let rollback_service = Arc::clone(&service); + let rollback = + std::thread::spawn(move || rollback_service.rollback_subtree(ws_inode, snapshot_id)); + store.wait_until_blocked(); + assert!( + service.materialization_orphan_slow_path_enabled(), + "materialization must enter slow mode before its detached tree can commit" ); - let removed = service.remove_file(InodeId::root(), &name).unwrap(); - assert_eq!(removed.attr.file_type, FileType::Symlink); - assert_eq!(service.lookup_plus(InodeId::root(), &name).unwrap(), None); -} - -#[test] -fn create_special_node_persists_type_and_rdev_without_body() { - let service = service(); - let fifo_name = DentryName::new(b"events.fifo".to_vec()).unwrap(); - let fifo = service - .create_special_node( - InodeId::root(), - fifo_name.clone(), - SpecialNodeSpec { - file_type: FileType::NamedPipe, - mode: 0o644, - rdev: 0, - uid: 1000, - gid: 1000, - }, - ) + // Change a swap-guarded dentry after rollback materialized its detached tree. + // The graft must fail, leaving that tree unreachable and without a binding. + // The rollback commit is deliberately paused while the service-local + // commit-to-log ordering gate is held. Model the conflicting write from a + // separately recovered owner so it can reach the shared store and invalidate + // the already-planned swap without deadlocking on that process-local gate. + let concurrent = + NoKvFs::open_existing(service.mount, store.clone(), objects, service.shard_index()) + .unwrap(); + concurrent + .replace_artifact(PublishArtifact { + parent: ws_inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a2".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-orphan/a2".to_owned(), + bytes: b"A2".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) .unwrap(); - assert_eq!(fifo.attr.file_type, FileType::NamedPipe); - assert_eq!(fifo.attr.rdev, 0); - assert_eq!(fifo.attr.size, 0); - assert!(fifo.body.is_none()); + store.release_blocked(); + assert!(matches!( + rollback.join().unwrap(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + service.refresh_allocator_state().unwrap(); assert_eq!( - service.lookup_plus(InodeId::root(), &fifo_name).unwrap(), - Some(fifo.clone()) + service.read_artifact(ws_inode, &dname(b"a")).unwrap(), + b"A2" ); - - let char_name = DentryName::new(b"accelerator0".to_vec()).unwrap(); - let char_device = service - .create_special_node( - InodeId::root(), - char_name.clone(), - SpecialNodeSpec { - file_type: FileType::CharDevice, - mode: 0o660, - rdev: 0x1234, - uid: 0, - gid: 44, - }, - ) - .unwrap(); - assert_eq!(char_device.attr.file_type, FileType::CharDevice); - assert_eq!(char_device.attr.rdev, 0x1234); - assert!(char_device.body.is_none()); assert!(service - .read_dir_plus(InodeId::root()) + .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) .unwrap() .iter() - .any(|entry| entry.attr == char_device.attr)); + .all(|binding| binding.binding.snapshot_id != snapshot_id)); - let removed = service.remove_file(InodeId::root(), &char_name).unwrap(); - assert_eq!(removed.attr.file_type, FileType::CharDevice); - assert_eq!( - service.lookup_plus(InodeId::root(), &char_name).unwrap(), - None - ); + // A real detached clone remains a retention root through its binding. Once + // its borrower is removed, the failed rollback's unbound orphan must not keep + // this otherwise-unrelated binding alive forever. + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + service + .publish_artifact(PublishArtifact { + parent: base.attr.inode, + name: dname(b"data"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:base".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-orphan/base".to_owned(), + bytes: b"base".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let fork = service.clone_subtree_path("/base").unwrap(); + let fork_file = service + .lookup_plus(fork.root, &dname(b"data")) + .unwrap() + .unwrap(); + assert!(matches!( + service.retire_snapshot(fork.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + }) if snapshot_id == fork.snapshot_id + && fork_root == fork.root + && borrower == fork_file.attr.inode + )); + service.remove_file(fork.root, &dname(b"data")).unwrap(); + assert!(service.retire_snapshot(fork.snapshot_id).unwrap()); } #[test] -fn advisory_locks_detect_conflicts_and_support_partial_unlock() { - let service = service(); - let name = DentryName::new(b"locked.bin".to_vec()).unwrap(); - let file = service - .create_file(InodeId::root(), name, 0o644, 1000, 1000) - .unwrap(); - let inode = file.attr.inode; - let read_owner = 11; - let write_owner = 22; +fn failed_rollback_orphan_cannot_race_retirement_or_be_resurrected() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RetireSnapshot, 1, 2) + .rejecting(CommandKind::RenameReplace); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service - .set_advisory_lock(AdvisoryLockRequest { - inode, - owner: read_owner, - start: 0, - end: 99, - kind: AdvisoryLockKind::Read, - pid: 1100, - wait: false, + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let original = service + .publish_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:orphan-a1".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-resurrection/a1".to_owned(), + bytes: b"A1".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, }) .unwrap(); + let original_body = original.body.as_ref().unwrap(); + let original_block = block_key(original.attr.inode, original_body.generation, 0, 0); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); service - .set_advisory_lock(AdvisoryLockRequest { - inode, - owner: 33, - start: 20, - end: 30, - kind: AdvisoryLockKind::Read, - pid: 3300, - wait: false, + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:orphan-a2".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-resurrection/a2".to_owned(), + bytes: b"A2".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, }) .unwrap(); - let conflict = service - .get_advisory_lock(AdvisoryLockRequest { - inode, - owner: write_owner, - start: 50, - end: 60, - kind: AdvisoryLockKind::Write, - pid: 2200, - wait: false, + // Reject only the atomic graft. All prior materialization commits remain, + // leaving an inode + dentry tree with neither a mount path nor ForkBinding. + assert!(matches!( + service.rollback_subtree(ws.attr.inode, snapshot.snapshot_id), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!( + service.materialization_orphan_slow_path_enabled(), + "failed materialization must leave inode exposure fail-closed" + ); + let (orphan_row, orphan) = service + .metadata + .scan(ScanRequest { + family: RecordFamily::Dentry, + prefix: dentry_mount_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .into_iter() + .find_map(|row| { + let projection = decode_dentry_projection(&row.value.0).unwrap(); + projection + .body + .as_ref() + .is_some_and(|body| body.manifest_id == "rollback-resurrection/a1") + .then_some((row, projection)) }) + .expect("failed rollback leaves its materialized child"); + let orphan_parent = orphan.dentry.parent; + let orphan_inode = orphan.attr.inode; + let orphan_name = orphan.dentry.name.clone(); + assert_ne!(orphan_parent, ws.attr.inode); + assert!(service + .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) + .unwrap() + .iter() + .all(|binding| binding.binding.snapshot_id != snapshot.snapshot_id)); + + // Hold an unrelated retirement after its mount-wide proof but before its + // binding CAS. A racing hardlink must wait on object_gc_gate; after the CAS + // removes the last legal detached root, it must re-prove and reject the + // unbound rollback orphan instead of exposing it under the mount root. + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + service.as_ref(), + "/base/data", + "rollback-resurrection/base", + b"base", + ); + let fork = service.clone_subtree_path("/base").unwrap(); + service.remove_file(fork.root, &dname(b"data")).unwrap(); + + let retire_service = Arc::clone(&service); + let retire = std::thread::spawn(move || retire_service.retire_snapshot(fork.snapshot_id)); + store.wait_until_blocked(); + assert!(service.object_gc_gate.try_lock().is_err()); + + let (link_tx, link_rx) = std::sync::mpsc::channel(); + let link_service = Arc::clone(&service); + let racing_link = std::thread::spawn(move || { + let result = + link_service.link(orphan_inode, InodeId::root(), dname(b"racing-resurrection")); + link_tx.send(result).unwrap(); + }); + let early_link = link_rx.recv_timeout(Duration::from_millis(100)); + store.release_blocked(); + assert!(retire.join().unwrap().unwrap()); + let link_result = match early_link { + Ok(result) => result, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => link_rx.recv().unwrap(), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + panic!("racing hardlink thread disconnected") + } + }; + racing_link.join().unwrap(); + assert!( + matches!(link_result, Err(MetadError::NotFound)), + "unbound orphan hardlink result: {link_result:?}" + ); + assert!(service + .lookup_plus(InodeId::root(), &dname(b"racing-resurrection")) + .unwrap() + .is_none()); + + // Release the failed rollback's construction snapshot as well, allowing A1 + // to be reclaimed. The orphan metadata is now a dangling borrower; neither + // inode-addressed operation may make it reachable again. + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert!(cleanup.deleted >= 1, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&original_block).unwrap().is_none()); + + let orphan_inode_key = inode_key(service.mount, orphan_inode); + let before_inode = service + .metadata + .get_versioned( + RecordFamily::Inode, + &orphan_inode_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) .unwrap() .unwrap(); - assert_eq!(conflict.owner, read_owner); - assert_eq!(conflict.kind, AdvisoryLockKind::Read); assert!(matches!( - service.set_advisory_lock(AdvisoryLockRequest { - inode, - owner: write_owner, - start: 50, - end: 60, - kind: AdvisoryLockKind::Write, - pid: 2200, - wait: false, - }), - Err(MetadError::LockConflict(_)) + service.link( + orphan_inode, + InodeId::root(), + dname(b"hardlink-resurrection") + ), + Err(MetadError::NotFound) + )); + assert!(matches!( + service.rename( + orphan_parent, + &orphan_name, + InodeId::root(), + dname(b"rename-resurrection") + ), + Err(MetadError::NotFound) )); - service - .set_advisory_lock(AdvisoryLockRequest { - inode, - owner: read_owner, - start: 40, - end: 70, - kind: AdvisoryLockKind::Unlock, - pid: 1100, - wait: false, - }) + let live = service + .lookup_plus(ws.attr.inode, &dname(b"a")) + .unwrap() + .unwrap(); + assert!(matches!( + service.link( + live.attr.inode, + orphan_parent, + dname(b"orphan-destination-link") + ), + Err(MetadError::NotFound) + )); + assert!(matches!( + service.rename( + ws.attr.inode, + &dname(b"a"), + orphan_parent, + dname(b"orphan-destination-rename") + ), + Err(MetadError::NotFound) + )); + + let after_dentry = service + .metadata + .get_versioned( + RecordFamily::Dentry, + &orphan_row.key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let after_inode = service + .metadata + .get_versioned( + RecordFamily::Inode, + &orphan_inode_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() .unwrap(); + assert_eq!(after_dentry.version, orphan_row.version); + assert_eq!(after_dentry.value, orphan_row.value); + assert_eq!(after_inode.version, before_inode.version); + assert_eq!(after_inode.value, before_inode.value); assert!(service - .get_advisory_lock(AdvisoryLockRequest { - inode, - owner: write_owner, - start: 50, - end: 60, - kind: AdvisoryLockKind::Write, - pid: 2200, - wait: false, - }) + .lookup_plus(InodeId::root(), &dname(b"hardlink-resurrection")) .unwrap() .is_none()); assert!(service - .get_advisory_lock(AdvisoryLockRequest { - inode, - owner: write_owner, - start: 10, - end: 20, - kind: AdvisoryLockKind::Write, - pid: 2200, - wait: false, - }) + .lookup_plus(InodeId::root(), &dname(b"rename-resurrection")) .unwrap() - .is_some()); -} - -#[test] -fn snapshot_preserves_symlink_target() { - let service = service(); - let name = DentryName::new(b"latest".to_vec()).unwrap(); - service - .create_symlink( - InodeId::root(), - name.clone(), - b"runs/old".to_vec(), - 0o777, - 1000, - 1000, - ) - .unwrap(); - let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); - service.remove_file(InodeId::root(), &name).unwrap(); - service - .create_symlink( - InodeId::root(), - name.clone(), - b"runs/new".to_vec(), - 0o777, - 1000, - 1000, - ) - .unwrap(); - + .is_none()); + assert!(service + .lookup_plus(orphan_parent, &dname(b"orphan-destination-link")) + .unwrap() + .is_none()); + assert!(service + .lookup_plus(orphan_parent, &dname(b"orphan-destination-rename")) + .unwrap() + .is_none()); + assert!(objects.head(&original_block).unwrap().is_none()); assert_eq!( - service - .read_symlink_at_snapshot("/", snapshot.snapshot_id, std::slice::from_ref(&name)) - .unwrap(), - b"runs/old" + service.read_artifact(ws.attr.inode, &dname(b"a")).unwrap(), + b"A2" ); } #[test] -fn update_attrs_truncates_and_extends_sparse_file() { - let service = service(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request(name.clone(), "checkpoint-v1", b"abcdef")) - .unwrap(); - - let shrunk = service - .update_attrs( - InodeId::root(), - &name, - UpdateAttr { - size: Some(3), - ..UpdateAttr::default() - }, - ) - .unwrap(); - assert_eq!(shrunk.attr.inode, published.attr.inode); - assert_eq!(shrunk.attr.size, 3); - assert_eq!(service.read_file(shrunk.attr.inode, 0, 8).unwrap(), b"abc"); - assert_eq!( - shrunk.body.as_ref().unwrap().digest_uri, - "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" +fn rollback_binding_survives_pin_reaping_without_an_auxiliary_clone() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let original = publish_path_artifact(&service, "/ws/a", "rollback-hold/a1", b"A1"); + let original_body = original.body.as_ref().unwrap(); + let original_block = block_key(original.attr.inode, original_body.generation, 0, 0); + let snap = service.snapshot_subtree_path("/ws").unwrap(); + let diverged = service + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a2".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-hold/a2".to_owned(), + bytes: b"A2".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap() + .entry; + let diverged_block = block_key( + diverged.attr.inode, + diverged.body.as_ref().unwrap().generation, + 0, + 0, ); - let grown = service - .update_attrs( - InodeId::root(), - &name, - UpdateAttr { - size: Some(6), - ..UpdateAttr::default() - }, - ) + service + .rollback_subtree_path("/ws", snap.snapshot_id) .unwrap(); - assert_eq!(grown.attr.size, 6); + let restored = service + .lookup_plus(ws.attr.inode, &dname(b"a")) + .unwrap() + .unwrap(); + assert_ne!(restored.attr.inode, original.attr.inode); + let binding = service + .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) + .unwrap() + .into_iter() + .find(|binding| binding.binding.snapshot_id == snap.snapshot_id) + .expect("rollback installs durable retention"); + assert_eq!(binding.binding.source_root, ws.attr.inode); + assert_ne!(binding.binding.fork_root, ws.attr.inode); + assert!(service + .get_attr(binding.binding.fork_root) + .unwrap() + .is_none()); + + let pin = service.snapshot_pin(snap.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); assert_eq!( - service.read_file(grown.attr.inode, 0, 8).unwrap(), - b"abc\0\0\0" + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 ); - assert_eq!( - grown.body.as_ref().unwrap().digest_uri, - "sha256:dd0b251b2bf91037a1e4fc8416a24ae00bcb9a8c252dc7e2361f2fc015f51c16" + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); + assert!(cleanup.blocked_by_snapshots >= 1); + assert!(objects.head(&original_block).unwrap().is_some()); + assert!(objects.head(&diverged_block).unwrap().is_some()); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"A1"); + assert!(matches!( + service.retire_snapshot_path("/ws", snap.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + }) if snapshot_id == snap.snapshot_id + && fork_root == binding.binding.fork_root + && borrower == restored.attr.inode + )); + + let current = service + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a3".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-hold/a3".to_owned(), + bytes: b"A3".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap() + .entry; + let current_block = block_key( + current.attr.inode, + current.body.as_ref().unwrap().generation, + 0, + 0, ); + assert!(service + .retire_snapshot_path("/ws", snap.snapshot_id) + .unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert!(cleanup.deleted >= 2, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&original_block).unwrap().is_none()); + assert!(objects.head(&diverged_block).unwrap().is_none()); + assert!(objects.head(¤t_block).unwrap().is_some()); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"A3"); } #[test] -fn attr_only_update_preserves_body_generation_and_readability() { - // `cp` preserves metadata, so it chmods a file it just wrote. An attribute- - // only `update_attrs` (no size change) must not advance `attr.generation`: - // the body summary / chunk manifests are keyed by generation and reads - // resolve the body via `attr.generation`, so bumping it would point the - // dentry at a generation that has no body, surfacing as MissingBodyDescriptor - // on the next read (the cp corruption this regression guards). - let service = service(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request(name.clone(), "checkpoint-v1", b"abcdef")) - .unwrap(); - let body_generation = published.body.as_ref().unwrap().generation; - assert_eq!(published.attr.generation, body_generation); - - let chmodded = service - .update_attrs( - InodeId::root(), - &name, - UpdateAttr { - mode: Some(0o600), - ..UpdateAttr::default() - }, - ) - .unwrap(); - - assert_eq!(chmodded.attr.mode, 0o600); - assert_eq!(chmodded.attr.size, published.attr.size); - // Generation is the content version; an attribute-only change keeps it. - assert_eq!(chmodded.attr.generation, body_generation); - assert_eq!(chmodded.body.as_ref().unwrap().generation, body_generation); - // The body is still resolvable and intact after the metadata-only update. - assert_eq!( - service.read_file(chmodded.attr.inode, 0, 6).unwrap(), - b"abcdef" - ); +fn rollback_binding_protects_an_owner_gc_row_enqueued_after_the_swap() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact(&service, "/ws/a", "rollback-late-owner", b"shared"); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let snap = service.snapshot_subtree_path("/ws").unwrap(); + service + .link(source.attr.inode, InodeId::root(), dname(b"outside")) + .unwrap(); + service.remove_file(ws.attr.inode, &dname(b"a")).unwrap(); + assert_eq!(service.cleanup_pending_objects(100).unwrap().attempted, 0); - // A size change still advances the generation (new body content). - let resized = service - .update_attrs( - InodeId::root(), - &name, - UpdateAttr { - size: Some(3), - ..UpdateAttr::default() - }, - ) + service + .rollback_subtree_path("/ws", snap.snapshot_id) .unwrap(); - assert!(resized.attr.generation > body_generation); + let restored = service + .lookup_plus(ws.attr.inode, &dname(b"a")) + .unwrap() + .unwrap(); + let pin = service.snapshot_pin(snap.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); assert_eq!( - resized.attr.generation, - resized.body.as_ref().unwrap().generation + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 ); - assert_eq!(service.read_file(resized.attr.inode, 0, 8).unwrap(), b"abc"); -} -#[test] -fn replace_publish_refreshes_stale_dentry_version_after_attr_update() { - // Reproduces the cp setattr-mid-write -> release publish CAS: a write handle - // prepares an artifact-replace (pinning the dentry version), then a `setattr` - // (here a chmod via update_attrs) advances the dentry version out-of-band. - // Publishing with the stale pinned version must fail the CAS (PredicateFailed - // -> EIO), and re-reading the live version via `current_dentry_version` (what - // publish_handle now does) before publishing must make it succeed without - // losing the body. - let service = service(); - let name = DentryName::new(b"y.bin".to_vec()).unwrap(); service - .publish_artifact(artifact_request(name.clone(), "y-v1", b"abcdef")) + .remove_file(InodeId::root(), &dname(b"outside")) .unwrap(); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"shared"); + assert!(matches!( + service.retire_snapshot_path("/ws", snap.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + borrower, + .. + }) if snapshot_id == snap.snapshot_id && borrower == restored.attr.inode + )); - // The write handle's prepared-replace, capturing the current dentry version. - let mut prepared = service - .prepare_artifact_replace(InodeId::root(), name.clone()) + service + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:self-owned".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-late-owner/self-owned".to_owned(), + bytes: b"self-owned".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) .unwrap(); - let pinned_version = prepared.dentry_version.unwrap(); + assert!(service + .retire_snapshot_path("/ws", snap.snapshot_id) + .unwrap()); + assert!(service.cleanup_pending_objects(100).unwrap().deleted >= 1); + assert!(objects.head(&source_block).unwrap().is_none()); +} - // An intervening chmod advances the dentry version, stranding `prepared`. +#[test] +fn rollback_preserves_the_gc_row_for_a_restored_delta_base() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let original = publish_path_artifact(&service, "/ws/a", "rollback-delta/a1", b"A1"); + let original_body = original.body.as_ref().unwrap(); + let original_generation = original_body.generation; + let original_block = block_key(original.attr.inode, original_generation, 0, 0); + let snap = service.snapshot_subtree_path("/ws").unwrap(); + + let prepared = service + .prepare_artifact_replace(ws.attr.inode, dname(b"a")) + .unwrap(); + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "rollback-delta/a2", + &[PublishArtifactRange { + offset: original_body.size, + bytes: b"-delta".to_vec(), + }], + 0, + ) + .unwrap(); + let staged = written.staged_objects().unwrap(); + let chunks = written.chunk_manifests(); service - .update_attrs( - InodeId::root(), - &name, - UpdateAttr { - mode: Some(0o600), - ..UpdateAttr::default() + .publish_prepared_artifact_staged_session( + prepared, + PublishArtifactStagedSession { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a2".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-delta/a2".to_owned(), + size: original_body.size + 6, + chunks, + staged, + mode: 0o644, + uid: 1000, + gid: 1000, }, ) .unwrap(); - let current_version = service - .current_dentry_version(InodeId::root(), &name) - .unwrap() - .unwrap(); - assert_ne!( - current_version, pinned_version, - "chmod must advance the dentry version" + let appended = service.lookup_path("/ws/a").unwrap().unwrap(); + assert_eq!( + appended.body.as_ref().unwrap().base_generation, + original_generation ); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"A1-delta"); - let new_body = body_descriptor(prepared.generation, 3); - let new_chunks = vec![one_chunk_manifest(prepared.inode, prepared.generation, 3)]; + service.remove_file(ws.attr.inode, &dname(b"a")).unwrap(); + let gc_records = || { + service + .metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .into_iter() + .map(|row| decode_object_gc_record(&row.value.0).unwrap()) + .collect::>() + }; + let original_candidates_before_rollback = gc_records() + .iter() + .filter(|record| record.object_key == original_block.as_str()) + .count(); + assert_eq!( + original_candidates_before_rollback, 1, + "final unlink must enqueue every owner object in the delta generation chain" + ); - // Publishing with the stale pinned version fails the CAS, exactly the cp EIO. - let stale = service.publish_prepared_artifact( - prepared.clone(), - new_body.clone(), - new_chunks.clone(), - 0o600, - 1000, - 1000, + service + .rollback_subtree_path("/ws", snap.snapshot_id) + .unwrap(); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"A1"); + let original_candidates_after_rollback = gc_records() + .iter() + .filter(|record| record.object_key == original_block.as_str()) + .count(); + assert_eq!( + original_candidates_after_rollback, + original_candidates_before_rollback + 1, + "rollback must proactively make every restored owner object reclaimable" ); - assert!( - matches!( - stale, - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - ), - "stale dentry version must fail the replace CAS, got {stale:?}" + + let pin = service.snapshot_pin(snap.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 ); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&original_block).unwrap().is_some()); - // Rebinding the guard to the live version (the publish_handle refresh) lets the - // replace CAS pass and commit the new body. - prepared.dentry_version = Some(current_version); - let published = service - .publish_prepared_artifact(prepared, new_body, new_chunks, 0o600, 1000, 1000) - .unwrap() - .entry; - assert_eq!(published.attr.size, 3); - assert_eq!(published.attr.mode, 0o600); - let committed = service - .stat_path("/y.bin") - .unwrap() - .expect("artifact still resolvable after refreshed publish"); - assert_eq!(committed.attr.inode, published.attr.inode); - assert_eq!(committed.attr.size, 3); - assert_eq!(committed.body.as_ref().unwrap().size, 3); + service.remove_file(ws.attr.inode, &dname(b"a")).unwrap(); + assert!(service + .retire_snapshot_path("/ws", snap.snapshot_id) + .unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert!(cleanup.deleted >= 2, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&original_block).unwrap().is_none()); } #[test] -fn update_root_attrs_changes_root_inode_without_dentry_projection() { +fn rollback_on_a_clone_root_keeps_both_retention_bindings() { let service = service(); - let updated = service - .update_root_attrs(UpdateAttr { - mode: Some(0o700), - uid: Some(42), - gid: Some(43), - ..UpdateAttr::default() - }) - .unwrap(); - - assert_eq!(updated.mode, 0o700); - assert_eq!(updated.uid, 42); - assert_eq!(updated.gid, 43); - assert_eq!(service.get_attr(InodeId::root()).unwrap().unwrap(), updated); -} - -#[test] -fn history_writes_are_snapshot_retention_driven() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - - let before_hot = metadata.metadata_store_stats(); - service - .update_root_attrs(UpdateAttr { - mode: Some(0o700), - ..UpdateAttr::default() + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/base/a", "layered/base", b"base"); + let clone = service.clone_subtree_path_into("/base", "/fork").unwrap(); + let snap = service.snapshot_subtree_path("/fork").unwrap(); + service + .replace_artifact(PublishArtifact { + parent: clone.root, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:diverged".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "layered/diverged".to_owned(), + bytes: b"diverged".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, }) .unwrap(); - let after_hot = metadata.metadata_store_stats(); - assert_eq!( - after_hot.history_write_total - before_hot.history_write_total, - 0 - ); + service + .rollback_subtree_path("/fork", snap.snapshot_id) + .unwrap(); - let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); - assert_eq!(metadata.metadata_store_stats().active_snapshot_pin_total, 1); - let snapshot_attr = service - .get_attr_at_snapshot("/", snapshot.snapshot_id, &[]) - .unwrap() + let bindings = service + .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) .unwrap(); - let before_retained = metadata.metadata_store_stats(); - service - .update_root_attrs(UpdateAttr { - mode: Some(0o750), - ..UpdateAttr::default() - }) + let clone_binding = bindings + .iter() + .find(|binding| binding.binding.snapshot_id == clone.snapshot_id) .unwrap(); - let after_retained = metadata.metadata_store_stats(); + let rollback_binding = bindings + .iter() + .find(|binding| binding.binding.snapshot_id == snap.snapshot_id) + .unwrap(); + assert_eq!(clone_binding.binding.fork_root, clone.root); + assert_eq!(rollback_binding.binding.source_root, clone.root); + assert_ne!(rollback_binding.binding.fork_root, clone.root); + assert_ne!(clone_binding.key, rollback_binding.key); + assert!(matches!( + service.retire_snapshot(clone.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + .. + }) if snapshot_id == clone.snapshot_id + )); + assert!(matches!( + service.retire_snapshot(snap.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + .. + }) if snapshot_id == snap.snapshot_id + )); + assert_eq!(read_artifact_at_path(&service, "/fork/a"), b"base"); +} +#[test] +fn rollback_rejects_hardlinks_in_snapshot_or_current_tree() { + let snapshot_linked = service(); + snapshot_linked + .create_dir_path("/ws", 0o755, 1000, 1000) + .unwrap(); + let file = publish_path_artifact(&snapshot_linked, "/ws/a", "hardlink-snapshot", b"body"); + snapshot_linked + .link(file.attr.inode, InodeId::root(), dname(b"outside")) + .unwrap(); + let snap = snapshot_linked.snapshot_subtree_path("/ws").unwrap(); + let error = snapshot_linked + .rollback_subtree_path("/ws", snap.snapshot_id) + .unwrap_err(); + assert!(matches!( + &error, + MetadError::InvalidPath(message) if message.contains("hardlink-free") + )); assert_eq!( - after_retained.history_write_total - before_retained.history_write_total, - 1 - ); - assert_eq!( - service - .get_attr_at_snapshot("/", snapshot.snapshot_id, &[]) - .unwrap() + snapshot_linked + .read_artifact(InodeId::root(), &dname(b"outside")) .unwrap(), - snapshot_attr - ); - assert_eq!( - service.get_attr(InodeId::root()).unwrap().unwrap().mode, - 0o750 + b"body" ); - assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); - assert_eq!(metadata.metadata_store_stats().active_snapshot_pin_total, 0); - let before_retired_hot = metadata.metadata_store_stats(); - service - .update_root_attrs(UpdateAttr { - mode: Some(0o755), - ..UpdateAttr::default() - }) + let current_linked = service(); + let ws = current_linked + .create_dir_path("/ws", 0o755, 1000, 1000) .unwrap(); - let after_retired_hot = metadata.metadata_store_stats(); + let file = publish_path_artifact(¤t_linked, "/ws/a", "hardlink-current", b"body"); + let snap = current_linked.snapshot_subtree_path("/ws").unwrap(); + current_linked + .link(file.attr.inode, InodeId::root(), dname(b"outside")) + .unwrap(); + let error = current_linked + .rollback_subtree(ws.attr.inode, snap.snapshot_id) + .unwrap_err(); + assert!(matches!( + &error, + MetadError::InvalidPath(message) if message.contains("hardlink-free") + )); + assert_eq!(read_artifact_at_path(¤t_linked, "/ws/a"), b"body"); assert_eq!( - after_retired_hot.history_write_total - before_retired_hot.history_write_total, - 0 + current_linked + .read_artifact(InodeId::root(), &dname(b"outside")) + .unwrap(), + b"body" ); } #[test] -fn create_file_hot_path_write_attribution_is_bounded() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let before = metadata.metadata_store_stats(); - +fn rollback_subtree_restores_snapshot_shares_blocks_and_reclaims_delta() { + let (service, objects) = service_with_objects(); + // 1. Build /ws with files a="A1", b="B1", sub/c="C1" (real object data). + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); service - .create_file( - InodeId::root(), - DentryName::new(b"empty.txt".to_vec()).unwrap(), - 0o644, - 1000, - 1000, - ) + .create_dir_path("/ws/sub", 0o755, 1000, 1000) .unwrap(); + let a = publish_path_artifact(&service, "/ws/a", "ws/a", &vec![b'1'; 4096]); + let b = publish_path_artifact(&service, "/ws/b", "ws/b", &vec![b'2'; 4096]); + let c = publish_path_artifact(&service, "/ws/sub/c", "ws/sub/c", &vec![b'3'; 4096]); + let a_gen = a.body.as_ref().unwrap().generation; + let b_gen = b.body.as_ref().unwrap().generation; + let c_gen = c.body.as_ref().unwrap().generation; + let a1_block = block_key(a.attr.inode, a_gen, 0, 0); + let b1_block = block_key(b.attr.inode, b_gen, 0, 0); + let c1_block = block_key(c.attr.inode, c_gen, 0, 0); + assert!(objects.head(&a1_block).unwrap().is_some()); + assert!(objects.head(&b1_block).unwrap().is_some()); + assert!(objects.head(&c1_block).unwrap().is_some()); - let after = metadata.metadata_store_stats(); - assert_eq!(after.commit_total - before.commit_total, 1); - assert_eq!(after.current_put_total - before.current_put_total, 2); - assert_eq!(after.current_delete_total - before.current_delete_total, 0); - assert_eq!(after.history_write_total - before.history_write_total, 0); - assert_eq!(after.watch_write_total - before.watch_write_total, 0); - assert_eq!(after.dedupe_write_total - before.dedupe_write_total, 1); -} + // 2. Snapshot /ws. + let snap = service.snapshot_subtree_path("/ws").unwrap(); -#[test] -fn create_files_in_dir_coalesces_into_one_metadata_command() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), + // 3. Diverge /ws: rewrite a->"A2", add d="D1", delete b. + service + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a2".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "ws/a2".to_owned(), + bytes: vec![b'4'; 4096], + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let d = publish_path_artifact(&service, "/ws/d", "ws/d", &vec![b'5'; 4096]); + service.remove_file(ws.attr.inode, &dname(b"b")).unwrap(); + // Capture the delta's private blocks so we can assert their fate. + let a_diverged = service + .lookup_plus(ws.attr.inode, &dname(b"a")) + .unwrap() + .unwrap(); + let a2_block = block_key( + a_diverged.attr.inode, + a_diverged.body.as_ref().unwrap().generation, + 0, + 0, ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let before = metadata.metadata_store_stats(); - let before_service = service.metadata_service_stats(); + let d1_block = block_key(d.attr.inode, d.body.as_ref().unwrap().generation, 0, 0); + assert!(objects.head(&a2_block).unwrap().is_some()); + assert!(objects.head(&d1_block).unwrap().is_some()); + // Pre-rollback /ws is the diverged state. + assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'4'; 4096]); + assert!(service + .lookup_plus(ws.attr.inode, &dname(b"b")) + .unwrap() + .is_none()); - let entries = service - .create_files_in_dir_path( - "/runs", - vec![ - DentryName::new(b"a.bin".to_vec()).unwrap(), - DentryName::new(b"b.bin".to_vec()).unwrap(), - ], - 0o644, - 1000, - 1000, - ) + // 4. Roll /ws back to the snapshot. + service + .rollback_subtree_path("/ws", snap.snapshot_id) .unwrap(); - let after = metadata.metadata_store_stats(); - let after_service = service.metadata_service_stats(); - assert_eq!(entries.len(), 2); - assert_eq!(after.commit_total - before.commit_total, 1); - assert_eq!(after.current_put_total - before.current_put_total, 4); - assert_eq!(after.current_delete_total - before.current_delete_total, 0); - assert_eq!(after.history_write_total - before.history_write_total, 0); - assert_eq!(after.watch_write_total - before.watch_write_total, 0); - assert_eq!(after.dedupe_write_total - before.dedupe_write_total, 1); + // 5. /ws now exactly matches the snapshot: a="A1", b="B1" (restored), sub/c="C1", + // and d is gone. The target keeps its inode identity. assert_eq!( - after_service.create_files_batch_total - before_service.create_files_batch_total, - 1 + service.resolve_directory_path("/ws").unwrap(), + ws.attr.inode, + "rollback keeps the target root's identity" ); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'1'; 4096]); + assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'2'; 4096]); assert_eq!( - after_service.create_files_entry_total - before_service.create_files_entry_total, - 2 + read_artifact_at_path(&service, "/ws/sub/c"), + vec![b'3'; 4096] + ); + assert!( + service + .lookup_plus(ws.attr.inode, &dname(b"d")) + .unwrap() + .is_none(), + "the delta-only file d must be gone after rollback" ); - let listed = service.read_dir_plus_path("/runs").unwrap(); - assert_eq!(listed.len(), 2); -} -#[test] -fn create_dirs_in_dir_coalesces_into_one_metadata_command() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let before = metadata.metadata_store_stats(); - let before_service = service.metadata_service_stats(); + // 6. The rolled-back /ws is identical to a fresh clone of the snapshot: an empty + // diff in both directions. + let reference = service + .clone_subtree_path_into("/ws", "/reference") + .unwrap(); + assert!(service + .diff_subtrees(ws.attr.inode, reference.root) + .unwrap() + .is_empty()); + assert!(service + .diff_subtrees(reference.root, ws.attr.inode) + .unwrap() + .is_empty()); - let entries = service - .create_dirs_in_dir_path( - "/runs", - vec![ - DentryName::new(b"a".to_vec()).unwrap(), - DentryName::new(b"b".to_vec()).unwrap(), - ], - 0o755, - 1000, - 1000, - ) + // Remove the reference fork itself. Its binding still cannot be retired: + // rollback propagated the same borrowed keys onto fresh /ws inodes, which + // the mount-global retirement proof must discover even though they were + // never descendants of `reference.root`. + let reference_sub = service + .lookup_plus(reference.root, &dname(b"sub")) + .unwrap() + .unwrap(); + service + .remove_file(reference_sub.attr.inode, &dname(b"c")) .unwrap(); + service + .remove_empty_dir(reference.root, &dname(b"sub")) + .unwrap(); + service.remove_file(reference.root, &dname(b"a")).unwrap(); + service.remove_file(reference.root, &dname(b"b")).unwrap(); + service.remove_empty_dir_path("/reference").unwrap(); - let after = metadata.metadata_store_stats(); - let after_service = service.metadata_service_stats(); - assert_eq!(entries.len(), 2); - assert!(entries - .iter() - .all(|entry| entry.attr.file_type == FileType::Directory)); - assert_eq!(after.commit_total - before.commit_total, 1); - assert_eq!(after.current_put_total - before.current_put_total, 4); - assert_eq!(after.current_delete_total - before.current_delete_total, 0); - assert_eq!(after.history_write_total - before.history_write_total, 0); - assert_eq!(after.watch_write_total - before.watch_write_total, 0); - assert_eq!(after.dedupe_write_total - before.dedupe_write_total, 1); + // 7. Both durable bindings fail closed while /ws still borrows the restored + // blocks. The rollback binding is intentionally conservative: its old + // mount-wide floor also delays the discarded delta until the borrowers + // are rewritten. + assert!(matches!( + service.retire_snapshot(snap.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + .. + }) if snapshot_id == snap.snapshot_id + )); + assert!(matches!( + service.retire_snapshot(reference.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + .. + }) if snapshot_id == reference.snapshot_id && fork_root == reference.root + )); + let reclaim = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(reclaim.deleted, 0, "cleanup outcome: {reclaim:?}"); + assert!(objects.head(&a2_block).unwrap().is_some()); + assert!(objects.head(&d1_block).unwrap().is_some()); + assert!( + objects.head(&a1_block).unwrap().is_some(), + "A1 must survive" + ); + assert!( + objects.head(&b1_block).unwrap().is_some(), + "B1 must survive" + ); + assert!( + objects.head(&c1_block).unwrap().is_some(), + "C1 must survive" + ); + // Restored content is still readable from the shared blocks after reclaim. + assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'1'; 4096]); + assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'2'; 4096]); assert_eq!( - after_service.create_dirs_batch_total - before_service.create_dirs_batch_total, - 1 + read_artifact_at_path(&service, "/ws/sub/c"), + vec![b'3'; 4096] ); + + // Once every rollback borrower owns a fresh generation, each binding can be + // retired independently and both the old snapshot blocks and discarded delta + // become reclaimable. + let restored_sub = service + .lookup_plus(ws.attr.inode, &dname(b"sub")) + .unwrap() + .unwrap(); + let rewrite = |parent: InodeId, name: &[u8], manifest: &str, byte: u8| { + service + .replace_artifact(PublishArtifact { + parent, + name: dname(name), + producer: "unit-test".to_owned(), + digest_uri: format!("sha256:{manifest}"), + content_type: "application/octet-stream".to_owned(), + manifest_id: manifest.to_owned(), + bytes: vec![byte; 4096], + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap() + .entry + }; + let a3 = rewrite(ws.attr.inode, b"a", "ws/a3", b'6'); + let b3 = rewrite(ws.attr.inode, b"b", "ws/b3", b'7'); + let c3 = rewrite(restored_sub.attr.inode, b"c", "ws/c3", b'8'); + let a3_block = block_key(a3.attr.inode, a3.body.as_ref().unwrap().generation, 0, 0); + let b3_block = block_key(b3.attr.inode, b3.body.as_ref().unwrap().generation, 0, 0); + let c3_block = block_key(c3.attr.inode, c3.body.as_ref().unwrap().generation, 0, 0); + assert!(service.retire_snapshot(snap.snapshot_id).unwrap()); + assert!(service.retire_snapshot(reference.snapshot_id).unwrap()); + let reclaim = service.cleanup_pending_objects(100).unwrap(); + assert!(reclaim.deleted >= 5, "cleanup outcome: {reclaim:?}"); + for old in [&a1_block, &b1_block, &c1_block, &a2_block, &d1_block] { + assert!(objects.head(old).unwrap().is_none(), "old block {old:?}"); + } + for current in [&a3_block, &b3_block, &c3_block] { + assert!( + objects.head(current).unwrap().is_some(), + "current block {current:?}" + ); + } + assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'6'; 4096]); + assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'7'; 4096]); assert_eq!( - after_service.create_dirs_entry_total - before_service.create_dirs_entry_total, - 2 + read_artifact_at_path(&service, "/ws/sub/c"), + vec![b'8'; 4096] ); - let listed = service.read_dir_plus_path("/runs").unwrap(); - assert_eq!(listed.len(), 2); } #[test] -fn remove_files_in_dir_coalesces_into_one_holt_apply() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - service - .create_files_in_dir_path( - "/runs", - vec![ - DentryName::new(b"a.bin".to_vec()).unwrap(), - DentryName::new(b"b.bin".to_vec()).unwrap(), - DentryName::new(b"keep.bin".to_vec()).unwrap(), - ], - 0o644, - 1000, - 1000, - ) - .unwrap(); - let before = metadata.metadata_store_stats(); - - let removed = service - .remove_files_in_dir_path( - "/runs", - vec![ - DentryName::new(b"a.bin".to_vec()).unwrap(), - DentryName::new(b"b.bin".to_vec()).unwrap(), - ], - ) +fn rollback_subtree_rejects_an_expired_snapshot_before_materializing() { + let service = service(); + service.set_clock_override_ms(1_000); + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/ws/a", "rollback-expired", b"original"); + let pin = service + .snapshot_subtree_with_lease(ws.attr.inode, 500) .unwrap(); - let after = metadata.metadata_store_stats(); - assert_eq!(removed.len(), 2); - assert!(removed.iter().all(Result::is_ok)); - assert_eq!(after.commit_total - before.commit_total, 2); - assert_eq!(after.current_delete_total - before.current_delete_total, 4); - assert_eq!(after.history_write_total - before.history_write_total, 0); - assert_eq!(after.watch_write_total - before.watch_write_total, 0); - assert_eq!(after.dedupe_write_total - before.dedupe_write_total, 2); - assert_eq!(after.atomic_apply_total - before.atomic_apply_total, 1); - assert_eq!( - after.atomic_apply_command_total - before.atomic_apply_command_total, - 2 - ); - let listed = service.read_dir_plus_path("/runs").unwrap(); - assert_eq!(listed.len(), 1); - assert_eq!(listed[0].dentry.name.as_bytes(), b"keep.bin"); + service.set_clock_override_ms(1_500); + assert!(matches!( + service.rollback_subtree(ws.attr.inode, pin.snapshot_id), + Err(MetadError::SnapshotLeaseExpired { + snapshot_id, + lease_expires_unix_ms: 1_500, + now_ms: 1_500, + }) if snapshot_id == pin.snapshot_id + )); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"original"); } #[test] -fn remove_empty_dirs_in_dir_coalesces_into_one_holt_apply() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( +fn rollback_holds_the_gc_gate_until_restored_references_are_live() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RenameReplace, 1, 2); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); + store.clone(), + objects.clone(), + )); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let ws_inode = ws.attr.inode; + let original = service + .publish_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:original".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-race/original".to_owned(), + bytes: b"original".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let original_body = original.body.as_ref().unwrap(); + let original_object = block_key(original.attr.inode, original_body.generation, 0, 0); + let pin = service + .snapshot_subtree_with_lease(ws.attr.inode, 500) + .unwrap(); service - .create_dir_batches_in_dir_path(vec![CreateInDirPathBatch { - parent_path: "/runs".to_owned(), - names: vec![ - DentryName::new(b"a".to_vec()).unwrap(), - DentryName::new(b"b".to_vec()).unwrap(), - DentryName::new(b"keep".to_vec()).unwrap(), - ], - mode: 0o755, + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:delta".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-race/delta".to_owned(), + bytes: b"delta".to_vec(), + mode: 0o644, uid: 1000, gid: 1000, - }]) - .remove(0) + }) .unwrap(); - let before = metadata.metadata_store_stats(); - let removed = service - .remove_empty_dirs_in_dir_path( - "/runs", - vec![ - DentryName::new(b"a".to_vec()).unwrap(), - DentryName::new(b"b".to_vec()).unwrap(), - ], - ) - .unwrap(); + let rollback_service = Arc::clone(&service); + let rollback = + std::thread::spawn(move || rollback_service.rollback_subtree(ws_inode, pin.snapshot_id)); + store.wait_until_blocked(); + assert!(service.object_gc_gate.try_lock().is_err()); - let after = metadata.metadata_store_stats(); - assert_eq!(removed.len(), 2); - assert!(removed[0].is_ok()); - assert!(removed[1].is_ok()); - assert_eq!(after.commit_total - before.commit_total, 2); - assert_eq!(after.current_delete_total - before.current_delete_total, 4); - assert_eq!(after.history_write_total - before.history_write_total, 0); - assert_eq!(after.watch_write_total - before.watch_write_total, 0); - assert_eq!(after.dedupe_write_total - before.dedupe_write_total, 2); - assert_eq!(after.atomic_apply_total - before.atomic_apply_total, 1); + let deadline_ms = start_ms + 500; + service.set_clock_override_ms(deadline_ms); + let cleanup_started = Arc::new(Barrier::new(2)); + let cleanup_service = Arc::clone(&service); + let cleanup_thread_started = Arc::clone(&cleanup_started); + let cleanup = std::thread::spawn(move || { + cleanup_thread_started.wait(); + cleanup_service.cleanup_pending_objects(100) + }); + cleanup_started.wait(); + store.release_blocked(); + + rollback.join().unwrap().unwrap(); + cleanup.join().unwrap().unwrap(); + assert!(objects.head(&original_object).unwrap().is_some()); assert_eq!( - after.atomic_apply_command_total - before.atomic_apply_command_total, - 2 + service.read_artifact(ws_inode, &dname(b"a")).unwrap(), + b"original" ); - let listed = service.read_dir_plus_path("/runs").unwrap(); - let names = listed - .iter() - .map(|entry| entry.dentry.name.as_bytes()) - .collect::>(); - assert_eq!(names, vec![b"keep".as_slice()]); } #[test] -fn read_dir_plus_page_returns_cursor_without_materializing_full_directory() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); +fn rollback_subtree_rejects_foreign_or_missing_snapshot() { + let service = service(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); service - .create_files_in_dir_path( - "/runs", - vec![ - DentryName::new(b"a.bin".to_vec()).unwrap(), - DentryName::new(b"b.bin".to_vec()).unwrap(), - DentryName::new(b"c.bin".to_vec()).unwrap(), - ], - 0o644, - 1000, - 1000, - ) + .create_dir_path("/other", 0o755, 1000, 1000) .unwrap(); - let runs = service.lookup_path("/runs").unwrap().unwrap(); + let other_root = service.resolve_directory_path("/other").unwrap(); + let snap = service.snapshot_subtree_path("/other").unwrap(); - let before_store = metadata.metadata_store_stats(); - let first = service - .read_dir_plus_page(runs.attr.inode, None, 2) - .unwrap(); - let after_first_store = metadata.metadata_store_stats(); - assert_eq!( - first - .entries - .iter() - .map(|entry| entry.dentry.name.as_bytes()) - .collect::>(), - vec![b"a.bin".as_slice(), b"b.bin".as_slice()] - ); - assert_eq!( - first.next_cursor.as_ref().map(DentryName::as_bytes), - Some(b"b.bin".as_slice()) - ); - assert_eq!( - after_first_store.scan_key_returned_total - before_store.scan_key_returned_total, - 3 - ); + // A snapshot of /other cannot roll back /ws. + assert!(matches!( + service.rollback_subtree_path("/ws", snap.snapshot_id), + Err(MetadError::InvalidPath(_)) + )); + // An unknown snapshot id is not found. + assert!(matches!( + service.rollback_subtree_path("/ws", snap.snapshot_id + 9_999), + Err(MetadError::NotFound) + )); + // The rejected target is untouched and the legitimate one still works. + assert!(service + .rollback_subtree(other_root, snap.snapshot_id) + .is_ok()); +} - let before_service = service.metadata_service_stats(); - let second = service - .read_dir_plus_page(runs.attr.inode, first.next_cursor.as_ref(), 2) - .unwrap(); - let after_service = service.metadata_service_stats(); - assert_eq!( - second - .entries - .iter() - .map(|entry| entry.dentry.name.as_bytes()) - .collect::>(), - vec![b"c.bin".as_slice()] - ); - assert_eq!(second.next_cursor, None); - assert_eq!( - after_service.read_dir_plus_entry_total - before_service.read_dir_plus_entry_total, - 1 +#[test] +fn metadata_backup_then_restore_into_fresh_store_recovers_namespace() { + let (service, objects) = service_with_objects(); + // Build a small namespace; file bodies land in the shared object store. + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + service.create_dir_path("/data", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/runs/a.bin", "m-a", b"alpha-body"); + publish_path_artifact(&service, "/data/b.bin", "m-b", b"bravo-body-2"); + let want_runs = service.lookup_path("/runs/a.bin").unwrap(); + let want_data = service.lookup_path("/data/b.bin").unwrap(); + assert!(want_runs.is_some()); + + let config = MetadataArchiveConfig::new("meta/checkpoints", 3); + let backup = service.backup_metadata(&config).unwrap(); + assert!(backup.image_bytes > 0); + assert!(backup.checkpoint_key.starts_with("meta/checkpoints/ckpt/")); + + // Simulate total loss of the metadata node: a brand-new empty Holt store, + // pointed at the SAME object store (the clone shares the backing map). + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), ); + // The fresh node has no namespace at all (not even a root) until restore. + let outcome = recovered.restore_metadata(&config).unwrap(); assert_eq!( - after_service.read_dir_plus_projection_hit_total - - before_service.read_dir_plus_projection_hit_total, - 1 + outcome.as_ref().map(|o| o.checkpoint_key.as_str()), + Some(backup.checkpoint_key.as_str()) ); + + // Namespace entries (dentry + attr + body descriptor) are identical after + // restore, and a subsequent create allocates a fresh, non-colliding inode. + assert_eq!(recovered.lookup_path("/runs/a.bin").unwrap(), want_runs); + assert_eq!(recovered.lookup_path("/data/b.bin").unwrap(), want_data); + let fresh = publish_path_artifact(&recovered, "/runs/c.bin", "m-c", b"charlie"); + assert_ne!(fresh.attr.inode, want_runs.unwrap().attr.inode); } #[test] -fn publish_artifact_stores_body_then_publishes_metadata() { - let service = service(); - let name = DentryName::new(b"checkpoint.json".to_vec()).unwrap(); - let before_publish = service.object_stats(); - let published = service - .publish_artifact(PublishArtifact { - content_type: "application/json".to_owned(), - ..artifact_request(name.clone(), "runs/1/checkpoint.json", b"{\"x\":1}") - }) - .unwrap(); - assert_eq!( - service.object_stats().object_put_bytes, - before_publish.object_put_bytes + 7 - ); - - let lookup = service - .lookup_plus(InodeId::root(), &name) - .unwrap() +fn checkpoint_install_error_permanently_poisons_the_service_instance() { + let (source, objects) = service_with_objects(); + source + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - assert_eq!(lookup, published); - assert_eq!(lookup.attr.size, 7); - assert_eq!( - lookup.body.as_ref().unwrap().manifest_id, - "runs/1/checkpoint.json" - ); + let config = MetadataArchiveConfig::new("meta/poisoned-checkpoint-install", 2); + source.backup_metadata(&config).unwrap(); - let bytes = service - .read_artifact(InodeId::root(), &name) - .expect("read artifact body"); - assert_eq!(bytes, b"{\"x\":1}"); + let metadata = RestorePostApplyFaultStore::new(); + metadata.fail_checkpoint_after_install(); + let recovered = NoKvFs::new(MountId::new(1).unwrap(), metadata, objects); + assert!(matches!( + recovered.restore_metadata(&config), + Err(MetadError::Metadata(MetadataError::Backend(message))) + if message.contains("checkpoint install error after apply") + )); - let body = service - .body_descriptor(published.attr.inode) - .expect("read body descriptor") - .expect("body descriptor exists"); - assert_eq!(body.manifest_id, "runs/1/checkpoint.json"); - assert_eq!(body.generation, published.attr.generation); - let range = service - .read_file(published.attr.inode, 2, 3) - .expect("read file range"); - assert_eq!(range, b"x\":"); - let before_cache = service.object_stats(); - let cached = service - .read_file(published.attr.inode, 2, 3) - .expect("read cached file range"); - assert_eq!(cached, b"x\":"); - let after_cache = service.object_stats(); - assert!(after_cache.cache_hits > before_cache.cache_hits); - assert_eq!( - after_cache.cache_hit_bytes, - before_cache.cache_hit_bytes + 3 - ); + for error in [ + recovered.lookup_path("/source").unwrap_err(), + recovered.restore_metrics().unwrap_err(), + recovered.fsck_restore_state(false).unwrap_err(), + recovered + .create_dir_path("/must-not-serve", 0o755, 1000, 1000) + .unwrap_err(), + recovered.restore_metadata(&config).unwrap_err(), + ] { + assert!(matches!( + error, + MetadError::MetadataCheckpointInstallUncertain + )); + } } #[test] -fn read_file_uses_one_attr_read_for_body_and_manifest_plan() { - let metadata = PurposeTrackingStore::new(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let published = service - .publish_artifact(artifact_request( - DentryName::new(b"checkpoint.bin".to_vec()).unwrap(), - "checkpoint/body", - b"abcdef", - )) - .unwrap(); - - let before = metadata.counts(); - assert_eq!( - service.read_file(published.attr.inode, 1, 3).unwrap(), - b"bcd" - ); - let after = metadata.counts(); - assert_eq!( - after.user_strong_gets - before.user_strong_gets, - 3, - "read_file should read inode, body summary, and one chunk manifest" - ); - assert_eq!(after.write_plan_gets, before.write_plan_gets); +fn restore_metadata_without_archive_returns_none() { + let (service, _objects) = service_with_objects(); + let config = MetadataArchiveConfig::new("meta/empty", 3); + assert!(service.restore_metadata(&config).unwrap().is_none()); } #[test] -fn read_artifact_uses_dentry_projection_body_descriptor() { - let metadata = PurposeTrackingStore::new(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); +fn restore_rejects_a_pre_fence_archive_before_installing_its_image() { + let (service, objects) = service_with_objects(); service - .publish_artifact(artifact_request(name.clone(), "checkpoint/body", b"abcdef")) + .create_dir_path("/unsafe", 0o755, 1000, 1000) .unwrap(); + let config = MetadataArchiveConfig::new("meta/legacy", 3); + let backup = service.backup_metadata(&config).unwrap(); - let before = metadata.counts(); - assert_eq!( - service.read_artifact(InodeId::root(), &name).unwrap(), - b"abcdef" - ); - let after = metadata.counts(); - assert_eq!( - after.user_strong_gets - before.user_strong_gets, - 2, - "read_artifact should read dentry projection and one chunk manifest" + // Downgrade only CURRENT to the legacy format. The referenced image is + // actually valid and fenced, so an empty target proves rejection happened + // from the pointer proof before install_checkpoint_image could mutate it. + let current_key = ObjectKey::new("meta/legacy/CURRENT").unwrap(); + let current = String::from_utf8(objects.get(¤t_key, None).unwrap()).unwrap(); + let legacy = current + .lines() + .enumerate() + .filter_map(|(index, line)| { + if line.starts_with("object_gc_failover_fenced\t") { + None + } else if index == 0 { + Some("nokv-metadata-archive\t1".to_owned()) + } else { + Some(line.to_owned()) + } + }) + .collect::>() + .join("\n") + + "\n"; + objects.put(¤t_key, legacy.into_bytes()).unwrap(); + + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, ); - assert_eq!(after.write_plan_gets, before.write_plan_gets); + assert!(matches!( + recovered.restore_metadata(&config), + Err(MetadError::MetadataArchiveMissingObjectGcFence { checkpoint_key }) + if checkpoint_key == backup.checkpoint_key + )); + assert!(recovered.get_attr(InodeId::root()).unwrap().is_none()); } #[test] -fn open_path_read_plan_uses_dentry_projection_body_descriptor() { - let metadata = PurposeTrackingStore::new(); - let service = NoKvFs::new( +fn metadata_backup_retains_only_keep_last_checkpoints() { + let (service, objects) = service_with_objects(); + let config = MetadataArchiveConfig::new("meta/ck", 2); + let b1 = service.backup_metadata(&config).unwrap(); + let _b2 = service.backup_metadata(&config).unwrap(); + let b3 = service.backup_metadata(&config).unwrap(); + // keep_last=2: the third backup prunes exactly the first checkpoint object. + assert_eq!(b3.pruned, 1); + let pruned = ObjectKey::new(b1.checkpoint_key.clone()).unwrap(); + assert!(objects.head(&pruned).unwrap().is_none()); + let live = ObjectKey::new(b3.checkpoint_key.clone()).unwrap(); + assert!(objects.head(&live).unwrap().is_some()); + // Restore (into a fresh store) always selects the latest checkpoint. + let recovered = NoKvFs::new( MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let published = service - .publish_artifact(artifact_request( - DentryName::new(b"checkpoint.bin".to_vec()).unwrap(), - "checkpoint/body", - b"abcdef", - )) - .unwrap(); + let restored = recovered.restore_metadata(&config).unwrap().unwrap(); + assert_eq!(restored.checkpoint_key, b3.checkpoint_key); +} - let before = metadata.counts(); - let open = service - .open_path_read_plan("/checkpoint.bin", 1, 3, Some(published.attr.generation)) - .unwrap(); - let after = metadata.counts(); - assert_eq!(open.metadata.attr.inode, published.attr.inode); - assert_eq!(open.plan.output_len, 3); - assert_eq!(open.plan.blocks.len(), 1); - assert_eq!( - after.user_strong_gets - before.user_strong_gets, - 2, - "open_path_read_plan should read dentry projection and one chunk manifest" - ); - assert_eq!(after.write_plan_gets, before.write_plan_gets); +fn log_test_command(request_id: &[u8], commit_version: u64) -> MetadataCommand { + MetadataCommand { + request_id: request_id.to_vec(), + kind: CommandKind::CreateFile, + read_version: Version::new(commit_version - 1).unwrap(), + commit_version: Version::new(commit_version).unwrap(), + primary_family: RecordFamily::Dentry, + primary_key: b"log-primary".to_vec(), + predicates: vec![PredicateRef { + family: RecordFamily::Dentry, + key: b"log-primary".to_vec(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::Dentry, + key: b"log-primary".to_vec(), + op: MutationOp::Put, + value: Some(Value(b"log-value".to_vec())), + }], + watch: Vec::new(), + } } -#[test] -fn open_path_read_plan_batch_uses_one_read_version() { - let metadata = PurposeTrackingStore::new(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let first = service - .publish_artifact(artifact_request( - DentryName::new(b"sample-0.bin".to_vec()).unwrap(), - "dataset/sample-0", - b"abcdefgh", - )) - .unwrap(); - let second = service - .publish_artifact(artifact_request( - DentryName::new(b"sample-1.bin".to_vec()).unwrap(), - "dataset/sample-1", - b"uvwxyz", - )) - .unwrap(); +fn log_test_entry( + request_id: &[u8], + lsn: u64, + commit_version: u64, + prev_digest: [u8; 32], +) -> MetadataLogEntry { + MetadataLogEntry::seal( + "mount-1:/runs", + 1, + lsn, + log_test_command(request_id, commit_version), + CommitResult { + commit_version: Version::new(commit_version).unwrap(), + applied_mutations: 1, + watch_events: 0, + }, + prev_digest, + ) + .unwrap() +} - let before = metadata.counts(); - let opens = service - .open_path_read_plan_batch(&[ - OpenPathReadPlanRequest { - path: "/sample-0.bin".to_owned(), - offset: 1, - len: 3, - expected_generation: Some(first.attr.generation), - }, - OpenPathReadPlanRequest { - path: "/sample-1.bin".to_owned(), - offset: 2, - len: 2, - expected_generation: Some(second.attr.generation), - }, - ]) - .unwrap(); - let after = metadata.counts(); +fn snapshot_segment_keys(snapshot: &MetadataLogSyncSnapshot) -> Vec { + snapshot + .segments + .iter() + .map(|segment| segment.segment_key.clone()) + .collect() +} + +fn log_replay_command( + request_id: &[u8], + key: &[u8], + value: &[u8], + read_version: u64, + commit_version: u64, +) -> MetadataCommand { + MetadataCommand { + request_id: request_id.to_vec(), + kind: CommandKind::RegisterNamespaceIndex, + read_version: Version::new(read_version).unwrap(), + commit_version: Version::new(commit_version).unwrap(), + primary_family: RecordFamily::System, + primary_key: key.to_vec(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: key.to_vec(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: key.to_vec(), + op: MutationOp::Put, + value: Some(Value(value.to_vec())), + }], + watch: Vec::new(), + } +} - assert_eq!(opens.len(), 2); - assert_eq!(opens[0].metadata.attr.inode, first.attr.inode); - assert_eq!(opens[1].metadata.attr.inode, second.attr.inode); - assert_eq!(opens[0].lease.read_version, opens[1].lease.read_version); - assert_eq!(opens[0].plan.output_len, 3); - assert_eq!(opens[1].plan.output_len, 2); - assert_eq!( - after.user_strong_gets - before.user_strong_gets, - 4, - "batch open should read each dentry projection and chunk manifest once" - ); - assert_eq!(after.write_plan_gets, before.write_plan_gets); +fn ordered_log_put_command( + request_id: &[u8], + key: &[u8], + value: &[u8], + read_version: u64, + commit_version: u64, +) -> MetadataCommand { + MetadataCommand { + request_id: request_id.to_vec(), + kind: CommandKind::RegisterNamespaceIndex, + read_version: Version::new(read_version).unwrap(), + commit_version: Version::new(commit_version).unwrap(), + primary_family: RecordFamily::System, + primary_key: key.to_vec(), + predicates: Vec::new(), + mutations: vec![Mutation { + family: RecordFamily::System, + key: key.to_vec(), + op: MutationOp::Put, + value: Some(Value(value.to_vec())), + }], + watch: Vec::new(), + } } #[test] -fn open_path_read_plan_returns_zero_write_lease_and_projected_plan() { - let metadata = PurposeTrackingStore::new(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let published = service - .publish_artifact(artifact_request( - DentryName::new(b"checkpoint.bin".to_vec()).unwrap(), - "checkpoint/body", - b"abcdef", - )) - .unwrap(); +fn metadata_log_segment_archive_round_trips_through_object_store() { + let (service, objects) = service_with_objects(); + let first = log_test_entry(b"req-log-1", 1, 11, METADATA_LOG_ZERO_DIGEST); + let second = log_test_entry(b"req-log-2", 2, 12, first.digest); + let segment = MetadataLogSegment::seal(vec![first, second]).unwrap(); + let config = MetadataLogArchiveConfig::new("meta/shared-log"); - let before_counts = metadata.counts(); - let before_commits = service.metadata_store_stats().commit_total; - let open = service - .open_path_read_plan("/checkpoint.bin", 1, 3, Some(published.attr.generation)) + let archived = service + .archive_metadata_log_segment(&config, &segment) .unwrap(); - let after_counts = metadata.counts(); - assert_eq!(open.metadata.attr.inode, published.attr.inode); - assert_eq!(open.lease.inode, published.attr.inode); - assert_eq!(open.lease.generation, published.attr.generation); - assert!(open.lease.read_version >= published.attr.generation); - assert_eq!(open.plan.output_len, 3); - assert_eq!(open.plan.blocks.len(), 1); - assert_eq!( - service.metadata_store_stats().commit_total, - before_commits, - "layout-open must not persist read state" - ); + assert!(archived.segment_key.starts_with("meta/shared-log/log/")); + assert!(archived + .segment_key + .contains("/00000000000000000001-00000000000000000002-")); + assert!(archived.segment_key.ends_with(".segment")); + assert_eq!(archived.first_lsn, 1); + assert_eq!(archived.last_lsn, 2); assert_eq!( - after_counts.user_strong_gets - before_counts.user_strong_gets, - 2, - "layout-open should read dentry projection and one chunk manifest" + archived.encoded_bytes, + segment.encode().unwrap().len() as u64 ); - assert_eq!(after_counts.write_plan_gets, before_counts.write_plan_gets); -} + assert!(objects + .head(&ObjectKey::new(archived.segment_key.clone()).unwrap()) + .unwrap() + .is_some()); -#[test] -fn read_file_plan_returns_ranges_without_fetching_objects() { - let service = service(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request(name, "checkpoint/body", b"hello metadata")) - .unwrap(); - let before = service.object_stats(); - let plan = service - .read_file_plan(published.attr.inode, published.attr.generation, 6, 6) + let loaded = service + .load_metadata_log_segment(&archived.segment_key) .unwrap(); - assert_eq!(plan.output_len, 6); - assert_eq!(plan.blocks.len(), 1); - assert_eq!(plan.blocks[0].object_offset, 6); - assert_eq!(plan.blocks[0].len, 6); - assert_eq!(plan.blocks[0].output_offset, 0); - assert!(plan.blocks[0].digest_uri.starts_with("xxh3-64:")); - assert_eq!(service.object_stats().object_gets, before.object_gets); - - let stale = service - .read_file_plan(published.attr.inode, published.attr.generation - 1, 0, 1) - .unwrap_err(); - assert!( - matches!(stale, MetadError::StaleBodyGeneration { .. }), - "unexpected error: {stale:?}" - ); + assert_eq!(loaded, segment); } #[test] -fn prepared_artifact_publish_commits_manifest_without_object_fetch() { - let service = service(); - let name = DentryName::new(b"metadata.bin".to_vec()).unwrap(); - let prepared = service - .prepare_artifact_create(InodeId::root(), name.clone()) - .unwrap(); - let body = body_descriptor(prepared.generation, 6); - let result = service - .publish_prepared_artifact( - prepared.clone(), - body, - vec![one_chunk_manifest(prepared.inode, prepared.generation, 6)], - 0o644, - 1000, - 1000, - ) - .unwrap(); - assert_eq!(result.replaced, None); - assert_eq!(result.entry.attr.inode, prepared.inode); - let lookup = service - .lookup_plus(InodeId::root(), &name) - .unwrap() - .unwrap(); - assert_eq!(lookup, result.entry); - let plan = service - .read_file_plan(prepared.inode, prepared.generation, 1, 3) - .unwrap(); - assert_eq!(plan.output_len, 3); - assert_eq!(plan.blocks[0].object_offset, 1); -} +fn metadata_log_archive_config_rejects_keys_outside_the_exact_shard_prefix() { + let config = MetadataLogArchiveConfig::new("meta/shared-log/mount_1__"); + let valid = concat!( + "meta/shared-log/mount_1__/log/", + "00000000000000000001-00000000000000000001-", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.segment" + ); + config.validate_segment_key(valid).unwrap(); -#[test] -fn prepared_artifact_publish_rejects_foreign_block_identity_before_commit() { - let service = service(); - let name = DentryName::new(b"foreign-block.bin".to_vec()).unwrap(); - let prepared = service - .prepare_artifact_create(InodeId::root(), name) - .unwrap(); - let body = body_descriptor(prepared.generation, 6); - let foreign_keys = [ - format!( - "blocks/2/{}/{}/0/0", - prepared.inode.get(), - prepared.generation - ), - format!("blocks/1/999/{}/0/0", prepared.generation), - format!( - "blocks/1/{}/{}/0/0", - prepared.inode.get(), - prepared.generation + 1 - ), - format!( - "blocks/1/{}/{}/1/0", - prepared.inode.get(), - prepared.generation + for invalid in [ + concat!( + "meta/shared-log/mount_2__/log/", + "00000000000000000001-00000000000000000001-", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.segment" ), - format!( - "blocks/1/{}/{}/0", - prepared.inode.get(), - prepared.generation + concat!( + "meta/shared-log/mount_1__/log/nested/", + "00000000000000000001-00000000000000000001-", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.segment" ), - ]; - - for object_key in foreign_keys { - let mut chunk = one_chunk_manifest(prepared.inode, prepared.generation, 6); - chunk.slices[0].blocks[0].object_key = object_key; - let err = service - .publish_prepared_artifact( - prepared.clone(), - body.clone(), - vec![chunk], - 0o644, - 1000, - 1000, - ) - .unwrap_err(); - assert!( - matches!(err, MetadError::InvalidPreparedArtifact(_)), - "unexpected error: {err:?}" - ); - assert!(service - .lookup_plus(InodeId::root(), &prepared.name) - .unwrap() - .is_none()); - } - - let mut valid_nonzero_block = one_chunk_manifest(prepared.inode, prepared.generation, 6); - valid_nonzero_block.slices[0].blocks[0].object_key = - block_key(prepared.inode, prepared.generation, 0, 17) - .as_str() - .to_owned(); - service - .publish_prepared_artifact(prepared, body, vec![valid_nonzero_block], 0o644, 1000, 1000) - .unwrap(); -} - -#[test] -fn prepared_artifact_publish_rejects_duplicate_chunk_range() { - let service = service(); - let name = DentryName::new(b"duplicate-chunk.bin".to_vec()).unwrap(); - let prepared = service - .prepare_artifact_create(InodeId::root(), name) - .unwrap(); - let body = body_descriptor(prepared.generation, 12); - let chunks = vec![ - one_chunk_manifest(prepared.inode, prepared.generation, 6), - one_chunk_manifest(prepared.inode, prepared.generation, 6), - ]; - - let err = service - .publish_prepared_artifact(prepared, body, chunks, 0o644, 1000, 1000) - .unwrap_err(); - assert!( - matches!(err, MetadError::InvalidPreparedArtifact(_)), - "unexpected error: {err:?}" - ); + "meta/shared-log/mount_1__/log/", + ] { + assert!(matches!( + config.validate_segment_key(invalid), + Err(MetadError::Codec(_)) + )); + } } #[test] -fn prepared_artifact_publish_rejects_slice_block_gap() { - let service = service(); - let name = DentryName::new(b"slice-gap.bin".to_vec()).unwrap(); - let prepared = service - .prepare_artifact_create(InodeId::root(), name) +fn metadata_log_segment_load_rejects_corrupted_object() { + let (service, objects) = service_with_objects(); + let first = log_test_entry(b"req-log-1", 1, 11, METADATA_LOG_ZERO_DIGEST); + let segment = MetadataLogSegment::seal(vec![first]).unwrap(); + let config = MetadataLogArchiveConfig::new("meta/shared-log"); + let archived = service + .archive_metadata_log_segment(&config, &segment) .unwrap(); - let body = body_descriptor(prepared.generation, 6); - let mut chunk = one_chunk_manifest(prepared.inode, prepared.generation, 6); - chunk.slices[0].blocks[0].len = 3; + let key = ObjectKey::new(archived.segment_key.clone()).unwrap(); + objects.put(&key, b"corrupted-segment".to_vec()).unwrap(); - let err = service - .publish_prepared_artifact(prepared, body, vec![chunk], 0o644, 1000, 1000) - .unwrap_err(); - assert!( - matches!(err, MetadError::InvalidPreparedArtifact(_)), - "unexpected error: {err:?}" - ); + assert!(matches!( + service.load_metadata_log_segment(&archived.segment_key), + Err(MetadError::Codec(_)) + )); } #[test] -fn prepared_artifact_publish_rejects_block_larger_than_manifest_block_size() { - let service = service(); - let name = DentryName::new(b"oversized-block.bin".to_vec()).unwrap(); - let prepared = service - .prepare_artifact_create(InodeId::root(), name) - .unwrap(); - let mut body = body_descriptor(prepared.generation, 6); - body.block_size = 3; - let chunk = one_chunk_manifest(prepared.inode, prepared.generation, 6); +fn restore_metadata_with_archived_log_segments_replays_after_checkpoint() { + let (service, objects) = service_with_objects(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-log-replay", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); - let err = service - .publish_prepared_artifact(prepared, body, vec![chunk], 0o644, 1000, 1000) - .unwrap_err(); - assert!( - matches!(err, MetadError::InvalidPreparedArtifact(_)), - "unexpected error: {err:?}" + let key = b"log-replay-system-key".to_vec(); + let value = b"after-checkpoint".to_vec(); + let commit_version = checkpoint.commit_version + 1; + let command = log_replay_command( + b"req-log-replay", + &key, + &value, + checkpoint.commit_version, + commit_version, ); -} - -#[test] -fn prepared_artifact_replace_rejects_stale_dentry_version() { - let service = service(); - let name = DentryName::new(b"replace-metadata.bin".to_vec()).unwrap(); - service - .publish_artifact(artifact_request(name.clone(), "old", b"old")) - .unwrap(); - let prepared = service - .prepare_artifact_replace(InodeId::root(), name.clone()) - .unwrap(); - service - .replace_artifact(artifact_request(name, "newer", b"newer")) + let result = service.commit_metadata(command.clone()).unwrap(); + let entry = + MetadataLogEntry::seal("mount-1:/", 1, 1, command, result, METADATA_LOG_ZERO_DIGEST) + .unwrap(); + let segment = MetadataLogSegment::seal(vec![entry.clone()]).unwrap(); + let log_config = MetadataLogArchiveConfig::new("meta/shared-log-replay"); + let archived = service + .archive_metadata_log_segment(&log_config, &segment) .unwrap(); - let err = service - .publish_prepared_artifact( - prepared.clone(), - body_descriptor(prepared.generation, 6), - vec![one_chunk_manifest(prepared.inode, prepared.generation, 6)], - 0o644, - 1000, - 1000, - ) - .unwrap_err(); - assert!( - matches!(err, MetadError::Metadata(MetadataError::PredicateFailed)), - "unexpected error: {err:?}" - ); -} -#[test] -fn prepared_artifact_replace_retry_is_idempotent() { - let service = service(); - let name = DentryName::new(b"retry-metadata.bin".to_vec()).unwrap(); - service - .publish_artifact(artifact_request(name.clone(), "old", b"old")) - .unwrap(); - let prepared = service - .prepare_artifact_replace(InodeId::root(), name) - .unwrap(); - let body = body_descriptor(prepared.generation, 6); - let chunks = vec![one_chunk_manifest(prepared.inode, prepared.generation, 6)]; - let first = service - .publish_prepared_artifact( - prepared.clone(), - body.clone(), - chunks.clone(), - 0o644, - 1000, - 1000, + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + let outcome = recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + std::slice::from_ref(&archived.segment_key), + 0, + METADATA_LOG_ZERO_DIGEST, ) + .unwrap() .unwrap(); - assert!(first.replaced.is_some()); - let second = service - .publish_prepared_artifact(prepared, body, chunks, 0o644, 1000, 1000) - .unwrap(); - assert_eq!(second.entry, first.entry); - assert_eq!(second.replaced, None); + + assert_eq!(outcome.checkpoint.checkpoint_key, checkpoint.checkpoint_key); + assert_eq!(outcome.replayed_entries, 1); + assert_eq!(outcome.durable_lsn, 1); + assert_eq!(outcome.last_digest, entry.digest); + assert!(recovered.read_version().unwrap().get() >= commit_version); + assert_eq!( + recovered + .metadata + .get( + RecordFamily::System, + &key, + Version::new(commit_version).unwrap(), + ReadPurpose::UserStrong + ) + .unwrap(), + Some(Value(value)) + ); } #[test] -fn prepared_artifact_session_uploads_only_dirty_ranges_and_reuses_old_blocks() { - let service = service(); - let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request( - name.clone(), - "artifact.bin", - b"abcdefghij", +fn log_replay_recovers_partial_materialization_as_an_unbound_orphan() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata, objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/base/data.bin", + "replay-orphan/source", + b"borrowed", + ); + let pin = service.snapshot_subtree(base.attr.inode).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/replay-orphan-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, )) .unwrap(); - let before = service.object_stats(); - let prepared = service - .prepare_artifact_replace(InodeId::root(), name.clone()) + + // The checkpoint is healthy. Only the subsequent detached materialization + // commits enter the retained log tail; no ForkBinding commit follows them. + let checkpoint_config = MetadataArchiveConfig::new("meta/replay-orphan-checkpoint", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + assert!(!service.materialization_orphan_slow_path_enabled()); + let orphan_root = { + let _object_gc_gate = service.object_gc_gate.lock().unwrap(); + service + .materialize_subtree_at(base.attr.inode, Version::new(pin.read_version).unwrap()) + .unwrap() + }; + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert!(snapshot.durable_lsn > checkpoint.log_lsn); + assert!(!snapshot.segments.is_empty()); + let segments = snapshot + .segments + .iter() + .map(|pointer| service.load_metadata_log_segment(&pointer.segment_key)) + .collect::, _>>() .unwrap(); - let before_scan = service.metadata_store_stats(); - let replaced = service - .publish_prepared_artifact_session( - prepared, - PublishArtifactSession { - parent: InodeId::root(), - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "unknown".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "artifact.bin".to_owned(), - size: 10, - ranges: vec![PublishArtifactRange { - offset: 3, - bytes: b"XYZ".to_vec(), - }], - mode: 0o644, - uid: 1000, - gid: 1000, - }, + assert!(segments + .iter() + .flat_map(|segment| &segment.entries) + .flat_map(|entry| &entry.command.mutations) + .all(|mutation| mutation.family != RecordFamily::ForkBinding)); + + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + let outcome = recovered + .restore_metadata_with_log_segments( + &checkpoint_config, + "mount-1:/", + &segments, + checkpoint.log_lsn, + checkpoint.log_digest, ) + .unwrap() .unwrap(); - let after = service.object_stats(); - let after_scan = service.metadata_store_stats(); - assert_eq!(after.object_puts, before.object_puts + 1); - assert_eq!(after.object_put_bytes, before.object_put_bytes + 3); - assert_eq!( - after_scan.scan_key_visited_total, - before_scan.scan_key_visited_total - ); - assert_eq!( - after_scan.scan_key_returned_total, - before_scan.scan_key_returned_total + assert_eq!(outcome.replayed_entries, 2); + assert!( + recovered.materialization_orphan_slow_path_enabled(), + "partial materialization replay must remain fail-closed" ); - assert_eq!(replaced.entry.attr.inode, published.attr.inode); - assert_eq!( - service.read_file(published.attr.inode, 0, 10).unwrap(), - b"abcXYZghij" + let orphan = recovered + .lookup_plus(orphan_root, &dname(b"data.bin")) + .unwrap() + .unwrap(); + assert!(matches!( + recovered.link(orphan.attr.inode, InodeId::root(), dname(b"replayed-link")), + Err(MetadError::NotFound) + )); + assert!(matches!( + recovered.rename( + orphan_root, + &dname(b"data.bin"), + InodeId::root(), + dname(b"replayed-rename") + ), + Err(MetadError::NotFound) + )); +} + +#[test] +fn restore_metadata_with_log_segments_rejects_chain_gap_before_restore() { + let (service, objects) = service_with_objects(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-log-gap", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + + let command = log_replay_command( + b"req-log-gap", + b"log-gap-system-key", + b"value", + checkpoint.commit_version, + checkpoint.commit_version + 1, ); - let gc = service.cleanup_pending_objects(10).unwrap(); - assert_eq!(gc.attempted, 0); + let result = service.commit_metadata(command.clone()).unwrap(); + let entry = + MetadataLogEntry::seal("mount-1:/", 1, 2, command, result, METADATA_LOG_ZERO_DIGEST) + .unwrap(); + let segment = MetadataLogSegment::seal(vec![entry]).unwrap(); + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + + let err = recovered + .restore_metadata_with_log_segments( + &checkpoint_config, + "mount-1:/", + &[segment], + 0, + METADATA_LOG_ZERO_DIGEST, + ) + .unwrap_err(); + + assert!(matches!(err, MetadError::Codec(message) if message.contains("lsn gap"))); } #[test] -fn prepared_artifact_session_splits_noncontiguous_dirty_blocks() { - let service = service(); - let name = DentryName::new(b"sparse-dirty.bin".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request( - name.clone(), - "sparse-dirty-v1", - b"abcdefghijklmnop", +fn sync_metadata_log_archives_commit_before_recovery_ack() { + let (service, objects) = service_with_objects(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-sync-log", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, )) .unwrap(); - let prepared = service - .prepare_artifact_replace(InodeId::root(), name.clone()) + + let key = b"sync-log-system-key".to_vec(); + let value = b"sync-after-checkpoint".to_vec(); + let commit_version = checkpoint.commit_version + 1; + let command = log_replay_command( + b"req-sync-log", + &key, + &value, + checkpoint.commit_version, + commit_version, + ); + let result = service.commit_metadata(command).unwrap(); + assert_eq!(result.commit_version.get(), commit_version); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 1); + assert_eq!(snapshot.segments.len(), 1); + let segment_pointer = snapshot.segments.last().unwrap(); + assert!(segment_pointer + .segment_key + .starts_with("meta/sync-log/log/")); + + let loaded = service + .load_metadata_log_segment(&segment_pointer.segment_key) .unwrap(); + assert_eq!(loaded.first_lsn, 1); + assert_eq!(loaded.last_lsn, 1); + assert_eq!(loaded.last_digest, snapshot.last_digest); - service - .publish_prepared_artifact_session( - prepared, - PublishArtifactSession { - parent: InodeId::root(), - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "unknown".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "sparse-dirty-v2".to_owned(), - size: 16, - ranges: vec![ - PublishArtifactRange { - offset: 2, - bytes: b"XY".to_vec(), - }, - PublishArtifactRange { - offset: 10, - bytes: b"UV".to_vec(), - }, - ], - mode: 0o644, - uid: 1000, - gid: 1000, - }, + let segment_keys = snapshot_segment_keys(&snapshot); + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &segment_keys, + 0, + METADATA_LOG_ZERO_DIGEST, ) + .unwrap() .unwrap(); assert_eq!( - service.read_file(published.attr.inode, 0, 16).unwrap(), - b"abXYefghijUVmnop" + recovered + .metadata + .get( + RecordFamily::System, + &key, + Version::new(commit_version).unwrap(), + ReadPurpose::UserStrong + ) + .unwrap(), + Some(Value(value)) ); } #[test] -fn prepared_artifact_staged_session_preserves_dirty_slice_overlay() { - let service = service(); - let name = DentryName::new(b"staged-dirty.bin".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request( - name.clone(), - "staged-dirty-v1", - b"abcdefghijklmnop", +fn log_replay_does_not_fold_a_foreign_graft_inode_into_the_local_allocator() { + let (service, objects) = service_with_objects(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-graft-log-replay", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/graft-log-replay", + "mount-1:/", + 1, + checkpoint.log_lsn, + checkpoint.log_digest, )) .unwrap(); - let prepared = service - .prepare_artifact_replace(InodeId::root(), name.clone()) - .unwrap(); - let written = service - .stage_prepared_artifact_ranges( - &prepared, - "staged-dirty-v2", - &[ - PublishArtifactRange { - offset: 2, - bytes: b"XY".to_vec(), - }, - PublishArtifactRange { - offset: 10, - bytes: b"UV".to_vec(), - }, - ], - 0, + + let foreign_inode = InodeId::compose(1, 42).unwrap(); + service + .create_graft( + InodeId::root(), + dname(b"dataset"), + foreign_inode, + 0o755, + 1000, + 1000, ) .unwrap(); - let staged = written.staged_objects().unwrap(); - let chunks = written.chunk_manifests(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.segments.len(), 1); - service - .publish_prepared_artifact_staged_session( - prepared, - PublishArtifactStagedSession { - parent: InodeId::root(), - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "unknown".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "staged-dirty-v2".to_owned(), - size: 16, - chunks, - staged, - mode: 0o644, - uid: 1000, - gid: 1000, - }, + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ) + .with_shard_index(0); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &snapshot_segment_keys(&snapshot), + checkpoint.log_lsn, + checkpoint.log_digest, ) + .unwrap() .unwrap(); - assert_eq!( - service.read_file(published.attr.inode, 0, 16).unwrap(), - b"abXYefghijUVmnop" - ); - let metadata = service.lookup_path("/staged-dirty.bin").unwrap().unwrap(); - let body = metadata.body.as_ref().unwrap(); - let manifests = service - .chunk_manifests_for_body_at_version( - published.attr.inode, - body, - service.read_version().unwrap(), - ReadPurpose::UserStrong, + let graft = recovered + .lookup_plus(InodeId::root(), &dname(b"dataset")) + .unwrap(); + assert_eq!(graft.unwrap().dentry.child, foreign_inode); + let local = recovered + .create_file( + InodeId::root(), + dname(b"local-after-replay"), + 0o644, + 1000, + 1000, ) .unwrap(); - assert_eq!(manifests[0].slices.len(), 3); - assert_eq!(manifests[0].slices[1].logical_offset, 2); - assert_eq!(manifests[0].slices[2].logical_offset, 10); + assert_eq!( + local.attr.inode.shard_index(), + 0, + "a foreign graft target replayed from the log must not poison the local allocator" + ); } #[test] -fn stat_path_sees_append_after_read_during_prepared_publish_window() { - // Regression for the "concurrent read + append silently drops the last - // append" visibility bug: prepare pre-allocates the commit version (the - // clock bump), so a stat between prepare and publish caches the pre-append - // dentry under the exact version the publish then applies at. The publish - // never advances the clock past that version, so without apply-time cache - // purging the poisoned entry is served for the process lifetime. - let service = service(); - service.create_dir_path("/w", 0o755, 1000, 1000).unwrap(); +fn concurrent_metadata_apply_and_log_archive_preserve_one_recovery_order() { + const FIRST_REQUEST: &[u8] = b"req-ordered-log-first"; + const SECOND_REQUEST: &[u8] = b"req-ordered-log-second"; + let store = PostCommitBarrierStore::new(FIRST_REQUEST); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-ordered-sync-log", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/ordered-sync-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); - let staged_session = - |prepared: &PreparedArtifact, written: &ChunkedWrite, manifest_id: &str, size: u64| { - PublishArtifactStagedSession { - parent: prepared.parent, - name: prepared.name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "unknown".to_owned(), - content_type: "text/plain".to_owned(), - manifest_id: manifest_id.to_owned(), - size, - chunks: written.chunk_manifests(), - staged: written.staged_objects().unwrap(), - mode: 0o644, - uid: 1000, - gid: 1000, - } - }; + let key = b"ordered-sync-log-key".to_vec(); + let first = ordered_log_put_command( + FIRST_REQUEST, + &key, + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, + ); + let second = ordered_log_put_command( + SECOND_REQUEST, + &key, + b"second", + checkpoint.commit_version + 1, + checkpoint.commit_version + 2, + ); - let prepared = service.prepare_artifact_create_path("/w/log.txt").unwrap(); - let written = service - .stage_prepared_artifact_ranges( - &prepared, - "log-v1", - &[PublishArtifactRange { - offset: 0, - bytes: b"seg0|".to_vec(), - }], - 0, - ) + let first_service = Arc::clone(&service); + let first_commit = std::thread::spawn(move || first_service.commit_metadata(first)); + store.wait_until_applied(); + + let (second_tx, second_rx) = std::sync::mpsc::sync_channel(1); + let second_service = Arc::clone(&service); + let second_commit = std::thread::spawn(move || { + let result = second_service.commit_metadata(second); + second_tx.send(result).unwrap(); + }); + let early = second_rx.recv_timeout(Duration::from_millis(100)); + store.release_after_apply(); + first_commit.join().unwrap().unwrap(); + assert!(matches!( + early, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) + )); + second_rx + .recv_timeout(Duration::from_secs(5)) + .unwrap() .unwrap(); - let session = staged_session(&prepared, &written, "log-v1", 5); - service - .publish_prepared_artifact_staged_session(prepared, session) + second_commit.join().unwrap(); + + assert_eq!( + service + .metadata + .get( + RecordFamily::System, + &key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"second".to_vec())) + ); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); + assert_eq!(snapshot.segments.len(), 2); + let first_segment = service + .load_metadata_log_segment(&snapshot.segments[0].segment_key) + .unwrap(); + let second_segment = service + .load_metadata_log_segment(&snapshot.segments[1].segment_key) .unwrap(); + assert_eq!(first_segment.entries[0].command.request_id, FIRST_REQUEST); + assert_eq!(second_segment.entries[0].command.request_id, SECOND_REQUEST); + assert_eq!(first_segment.last_digest, second_segment.prev_digest); - // Append: the replace prepare allocates the commit version V. - let prepared = service.prepare_artifact_replace_path("/w/log.txt").unwrap(); - // A read inside the staging window resolves at read_version == V and - // caches the pre-append entry under it. - let before = service.stat_path("/w/log.txt").unwrap().unwrap(); - assert_eq!(before.attr.size, 5); - let written = service - .stage_prepared_artifact_ranges( - &prepared, - "log-v2", - &[PublishArtifactRange { - offset: 5, - bytes: b"seg1|".to_vec(), - }], + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + let outcome = recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &snapshot_segment_keys(&snapshot), 0, + METADATA_LOG_ZERO_DIGEST, ) + .unwrap() .unwrap(); - let session = staged_session(&prepared, &written, "log-v2", 10); - service - .publish_prepared_artifact_staged_session(prepared, session) - .unwrap(); - - // No later write bumps the clock here: the applied publish itself must - // have invalidated the poisoned entry. - let after = service.stat_path("/w/log.txt").unwrap().unwrap(); - assert_eq!(after.attr.size, 10); + assert_eq!(outcome.replayed_entries, 2); assert_eq!( - service - .lookup_path("/w/log.txt") - .unwrap() - .unwrap() - .attr - .size, - 10 + recovered + .metadata + .get( + RecordFamily::System, + &key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"second".to_vec())) ); } #[test] -fn delta_publish_writes_only_dirty_chunks_and_preserves_base() { - let service = service(); - let name = DentryName::new(b"multi.bin".to_vec()).unwrap(); +fn metadata_commits_remain_concurrent_while_sync_log_is_disabled() { + const FIRST_REQUEST: &[u8] = b"req-no-sync-first"; + let store = PostCommitBarrierStore::new(FIRST_REQUEST); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + MemoryObjectStore::new(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let base = service.read_version().unwrap().get(); + let key = b"no-sync-concurrent-key"; + let first = ordered_log_put_command(FIRST_REQUEST, key, b"first", base, base + 1); + let second = ordered_log_put_command(b"req-no-sync-second", key, b"second", base + 1, base + 2); - // Generation 1: a two-chunk file (a few bytes in chunk 0 and chunk 1). - let create = service - .prepare_artifact_create(InodeId::root(), name.clone()) - .unwrap(); - let inode = create.inode; - let g1 = create.generation; - let written = service - .stage_prepared_artifact_ranges( - &create, - "multi-v1", - &[ - PublishArtifactRange { - offset: 0, - bytes: b"aa".to_vec(), - }, - PublishArtifactRange { - offset: DEFAULT_CHUNK_SIZE, - bytes: b"bb".to_vec(), - }, - ], + let first_service = Arc::clone(&service); + let first_commit = std::thread::spawn(move || first_service.commit_metadata(first)); + store.wait_until_applied(); + + let (second_tx, second_rx) = std::sync::mpsc::sync_channel(1); + let second_service = Arc::clone(&service); + let second_commit = std::thread::spawn(move || { + second_tx + .send(second_service.commit_metadata(second)) + .unwrap(); + }); + let second_result = second_rx + .recv_timeout(Duration::from_secs(5)) + .expect("a disabled sync log must not serialize unrelated metadata commits"); + store.release_after_apply(); + + second_result.unwrap(); + second_commit.join().unwrap(); + first_commit.join().unwrap().unwrap(); + assert_eq!( + service + .metadata + .get( + RecordFamily::System, + key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"second".to_vec())) + ); +} + +#[test] +fn sync_metadata_log_snapshot_keeps_durable_tail_after_checkpoint_prune() { + let (service, _objects) = service_with_objects(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-log-prune", + "mount-1:/", + 1, 0, - ) + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - let staged = written.staged_objects().unwrap(); - let chunks = written.chunk_manifests(); + service - .publish_prepared_artifact_staged_session( - create, - PublishArtifactStagedSession { - parent: InodeId::root(), - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "unknown".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "multi-v1".to_owned(), - size: DEFAULT_CHUNK_SIZE + 2, - chunks, - staged, - mode: 0o644, - uid: 1000, - gid: 1000, - }, - ) + .create_dir_path("/before-checkpoint", 0o755, 1000, 1000) .unwrap(); + let first = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(first.segments.len(), 1); + assert_eq!(first.segments[0].last_lsn, first.durable_lsn); + assert_eq!(first.segments[0].last_digest, first.last_digest); - // Generation 2: overwrite only chunk 0 — a delta over generation 1. - let replace = service - .prepare_artifact_replace(InodeId::root(), name.clone()) + service.prune_sync_metadata_log_segments(first.durable_lsn); + let after_first_prune = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(after_first_prune.durable_lsn, first.durable_lsn); + assert_eq!(after_first_prune.last_digest, first.last_digest); + assert!(after_first_prune.segments.is_empty()); + + service + .create_dir_path("/after-checkpoint", 0o755, 1000, 1000) .unwrap(); - let g2 = replace.generation; - assert_eq!(replace.old_generation, Some(g1)); - let written2 = service - .stage_prepared_artifact_ranges( - &replace, - "multi-v2", - &[PublishArtifactRange { - offset: 0, - bytes: b"XY".to_vec(), - }], - 0, - ) + let second = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(second.segments.len(), 1); + assert_eq!(second.segments[0].first_lsn, first.durable_lsn + 1); + assert_eq!(second.segments[0].last_lsn, second.durable_lsn); + assert_eq!(second.segments[0].last_digest, second.last_digest); + let continued = service + .load_metadata_log_segment(&second.segments[0].segment_key) .unwrap(); - let staged2 = written2.staged_objects().unwrap(); - let chunks2 = written2.chunk_manifests(); + assert_eq!(continued.prev_digest, first.last_digest); + + service.prune_sync_metadata_log_segments(second.durable_lsn); + let after_second_prune = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(after_second_prune.durable_lsn, second.durable_lsn); + assert_eq!(after_second_prune.last_digest, second.last_digest); + assert!(after_second_prune.segments.is_empty()); +} + +#[test] +fn sync_metadata_log_prune_never_deletes_pointer_outside_archive_prefix() { + let (service, objects) = service_with_objects(); + let unrelated = ObjectKey::new("unrelated/keep-me").unwrap(); + objects.put(&unrelated, b"keep".to_vec()).unwrap(); service - .publish_prepared_artifact_staged_session( - replace, - PublishArtifactStagedSession { - parent: InodeId::root(), - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "unknown".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "multi-v2".to_owned(), - size: DEFAULT_CHUNK_SIZE + 2, - chunks: chunks2, - staged: staged2, - mode: 0o644, - uid: 1000, - gid: 1000, - }, + .enable_sync_metadata_log( + MetadataLogSyncConfig::new("meta/sync-log-delete-scope", "mount-1:/", 1, 1, [7; 32]) + .with_segments(vec![MetadataLogSegmentPointer { + segment_key: unrelated.as_str().to_owned(), + first_lsn: 1, + last_lsn: 1, + last_digest: [7; 32], + }]), ) .unwrap(); - let version = service.read_version().unwrap(); - let body = service - .lookup_path("/multi.bin") - .unwrap() - .unwrap() - .body - .unwrap(); - assert_eq!(body.generation, g2); - // The delta falls through to generation 1 for untouched chunks. - assert_eq!(body.base_generation, g1); - - // O(write): generation 2 stores ONLY the dirty chunk (chunk 0), not chunk 1. - assert!(service - .chain_chunk_manifest(inode, &[g2], 0, version, ReadPurpose::UserStrong) - .unwrap() - .is_some()); - assert!(service - .chain_chunk_manifest(inode, &[g2], 1, version, ReadPurpose::UserStrong) - .unwrap() - .is_none()); + let outcome = service.prune_sync_metadata_log_segments(1); - // The base generation is preserved intact — not eagerly deleted. - assert!(service - .chain_chunk_manifest(inode, &[g1], 0, version, ReadPurpose::UserStrong) - .unwrap() - .is_some()); + assert_eq!(outcome.pointers_pruned, 1); + assert_eq!(outcome.objects_deleted, 0); + assert_eq!(outcome.objects_missing, 0); + assert_eq!(outcome.delete_failures, 1); + assert!(objects.head(&unrelated).unwrap().is_some()); assert!(service - .chain_chunk_manifest(inode, &[g1], 1, version, ReadPurpose::UserStrong) + .sync_metadata_log_snapshot() .unwrap() - .is_some()); - - // Reads resolve across the chain: chunk 0 from the delta, chunk 1 inherited. - assert_eq!(service.read_file(inode, 0, 2).unwrap(), b"XY"); - assert_eq!( - service.read_file(inode, DEFAULT_CHUNK_SIZE, 2).unwrap(), - b"bb" - ); + .segments + .is_empty()); } -fn overwrite_staged( - service: &NoKvFs, - prepared: PreparedArtifact, - name: &DentryName, - manifest_id: &str, - offset: u64, - bytes: &[u8], - size: u64, -) { - let written = service - .stage_prepared_artifact_ranges( - &prepared, - manifest_id, - &[PublishArtifactRange { - offset, - bytes: bytes.to_vec(), - }], +#[test] +fn sync_metadata_log_rejects_duplicate_enable_without_replacing_tail() { + let (service, _objects) = service_with_objects(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-log-original", + "mount-1:/", + 1, 0, - ) + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - let staged = written.staged_objects().unwrap(); - let chunks = written.chunk_manifests(); service - .publish_prepared_artifact_staged_session( - prepared, - PublishArtifactStagedSession { - parent: InodeId::root(), - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "unknown".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: manifest_id.to_owned(), - size, - chunks, - staged, - mode: 0o644, - uid: 1000, - gid: 1000, - }, - ) + .create_dir_path("/before-duplicate-enable", 0o755, 1000, 1000) + .unwrap(); + let before = service.sync_metadata_log_snapshot().unwrap(); + + let error = service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-log-replacement", + "mount-1:/", + 2, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap_err(); + assert!(matches!( + error, + MetadError::Codec(message) if message.contains("already enabled") + )); + assert_eq!(service.sync_metadata_log_snapshot().unwrap(), before); + + service + .create_dir_path("/after-duplicate-enable", 0o755, 1000, 1000) + .unwrap(); + let after = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(after.epoch, 1); + assert_eq!(after.durable_lsn, before.durable_lsn + 1); + assert!(after + .segments + .last() + .unwrap() + .segment_key + .starts_with("meta/sync-log-original/log/")); +} + +#[test] +fn sync_metadata_log_preflights_lsn_capacity_before_metadata_apply() { + let (service, _objects) = service_with_objects(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-log-exhausted", + "mount-1:/", + 1, + u64::MAX - 1, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); + + let error = service + .create_dir_path("/must-not-apply", 0o755, 1000, 1000) + .unwrap_err(); + assert!(matches!( + error, + MetadError::SyncLogArchiveFailed { + committed: false, + message, + } if message.contains("LSN is exhausted before commit") + )); + assert!(service.lookup_path("/must-not-apply").unwrap().is_none()); + assert_eq!( + service.sync_metadata_log_snapshot().unwrap().durable_lsn, + u64::MAX - 1 + ); } #[test] -fn delta_chain_compacts_to_self_contained_at_depth_threshold() { - let service = service(); - let name = DentryName::new(b"hot.bin".to_vec()).unwrap(); - let create = service - .prepare_artifact_create(InodeId::root(), name.clone()) +fn restore_metadata_with_sync_log_advances_allocator_after_replay() { + let (service, objects) = service_with_objects(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-sync-allocator", 2); + service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-allocator-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - let inode = create.inode; - overwrite_staged(&service, create, &name, "hot-0", 0, b"AAAA", 4); - - // Overwrite the same region many times. Each delta extends the fall-through - // chain by one; at the depth threshold the publish must re-materialize a - // self-contained generation (base_generation == 0) instead of growing the - // chain without bound. Every read must stay correct throughout. - let mut saw_compaction = false; - for i in 1..=12u32 { - let replace = service - .prepare_artifact_replace(InodeId::root(), name.clone()) - .unwrap(); - let byte = b'A' + (i % 16) as u8; - let want = [byte; 4]; - overwrite_staged(&service, replace, &name, &format!("hot-{i}"), 0, &want, 4); - assert_eq!(service.read_file(inode, 0, 4).unwrap(), want.to_vec()); - let body = service - .lookup_path("/hot.bin") - .unwrap() - .unwrap() - .body - .unwrap(); - if body.base_generation == 0 { - saw_compaction = true; - // Compaction must coalesce the hot chunk's accumulated slices, not - // just collapse the chain — otherwise slice count grows unbounded - // across compaction cycles. The fully-overwritten chunk collapses to - // a single newest-wins slice. - let chunk0 = service - .chain_chunk_manifest( - inode, - &[body.generation], - 0, - service.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap(); - assert_eq!(chunk0.slices.len(), 1); - } - } - assert!( - saw_compaction, - "deep delta chain must compact to a self-contained generation" - ); -} -#[test] -fn chain_collapse_gc_is_snapshot_safe() { - let service = service(); - let name = DentryName::new(b"snap.bin".to_vec()).unwrap(); - let create = service - .prepare_artifact_create(InodeId::root(), name.clone()) + let post_checkpoint = service + .create_dir_path("/runs/post-checkpoint", 0o755, 1000, 1000) .unwrap(); - let inode = create.inode; - overwrite_staged(&service, create, &name, "snap-0", 0, b"AAAA", 4); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + let segment_keys = snapshot_segment_keys(&snapshot); - // Pin generation 1, then overwrite enough to trigger a chain-collapse - // compaction. The compaction enqueues the superseded chain blocks for GC. - let pin = service.snapshot_subtree(InodeId::root()).unwrap(); - for i in 1..=10u32 { - let replace = service - .prepare_artifact_replace(InodeId::root(), name.clone()) - .unwrap(); - let byte = b'A' + (i % 16) as u8; - overwrite_staged( - &service, - replace, - &name, - &format!("snap-{i}"), + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &segment_keys, 0, - &[byte; 4], - 4, - ); - } + METADATA_LOG_ZERO_DIGEST, + ) + .unwrap() + .unwrap(); - // The snapshot still resolves generation-1 content, and a GC pass must NOT - // delete any block the snapshot can still reach — the version retention - // floor blocks reclamation of everything enqueued after the snapshot. assert_eq!( - service - .read_file_at_snapshot("/", pin.snapshot_id, std::slice::from_ref(&name), 0, 4) - .unwrap(), - b"AAAA" + recovered + .lookup_path("/runs/post-checkpoint") + .unwrap() + .unwrap() + .attr + .inode, + post_checkpoint.attr.inode ); - let blocked = service.cleanup_pending_objects(1024).unwrap(); + + let after_failover = recovered + .create_dir_path("/after-failover", 0o755, 1000, 1000) + .unwrap(); assert!( - blocked.blocked_by_snapshots > 0, - "snapshot must block reclamation of still-reachable chain blocks" + after_failover.attr.inode.get() > post_checkpoint.attr.inode.get(), + "failover replay must advance allocator past replayed namespace state" ); - assert_eq!(blocked.deleted, 0); - // Snapshot read still works after the (blocked) GC pass. assert_eq!( - service - .read_file_at_snapshot("/", pin.snapshot_id, std::slice::from_ref(&name), 0, 4) - .unwrap(), - b"AAAA" - ); - - // Retiring the snapshot raises the floor; the superseded chain blocks now - // reclaim — proving the whole chain (not just its top) was enqueued. - assert!(service.retire_snapshot(pin.snapshot_id).unwrap()); - let reclaimed = service.cleanup_pending_objects(1024).unwrap(); - assert!( - reclaimed.deleted > 0, - "retiring the snapshot must let superseded chain blocks reclaim" + recovered + .lookup_path("/runs/post-checkpoint") + .unwrap() + .unwrap() + .attr + .inode, + post_checkpoint.attr.inode ); - - // The live file reads correctly throughout (last write was i=10 -> 'K'). - assert_eq!(service.read_file(inode, 0, 4).unwrap(), b"KKKK"); } #[test] -fn replace_artifact_preserves_inode_and_returns_old_body() { - let service = service(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let first = service - .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) +fn prepare_only_allocator_reservation_replays_before_failover_reuse() { + let (service, objects) = service_with_objects(); + // Initialize the durable object-GC claim before the checkpoint so the + // prepare-only workload below changes only allocator reservations. + service + .prepare_artifact_create(InodeId::root(), dname(b"warmup.bin")) .unwrap(); - let replaced = service - .replace_artifact(artifact_request( - name.clone(), - "checkpoint/new", - b"new-body", + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-prepare-allocator", 2); + service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/prepare-allocator-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, )) .unwrap(); - assert_eq!(replaced.entry.attr.inode, first.attr.inode); - assert!(replaced.entry.attr.generation > first.attr.generation); - assert_eq!(replaced.replaced, Some(first.clone())); - assert_eq!( - service.lookup_plus(InodeId::root(), &name).unwrap(), - Some(replaced.entry.clone()) - ); - assert_eq!( - service.read_artifact(InodeId::root(), &name).unwrap(), - b"new-body" - ); - assert_eq!( - replaced.replaced.unwrap().body.unwrap().manifest_id, - "checkpoint/old" - ); -} - -#[test] -fn get_attr_reads_root_inode() { - let service = service(); - let root = service.get_attr(InodeId::root()).unwrap().unwrap(); - assert_eq!(root.inode, InodeId::root()); - assert_eq!(root.file_type, FileType::Directory); -} + let mut max_prepared_inode = 0; + let mut max_prepared_generation = 0; + for _ in 0..(ALLOCATOR_INODE_RESERVATION + 4) { + let prepared = service + .prepare_artifact_create(InodeId::root(), dname(b"never-published.bin")) + .unwrap(); + max_prepared_inode = max_prepared_inode.max(prepared.inode.get()); + max_prepared_generation = max_prepared_generation.max(prepared.generation); + } -#[test] -fn remove_file_deletes_namespace_and_returns_old_body() { - let service = service(); - let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request(name.clone(), "artifact.bin", b"old")) - .unwrap(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert!(snapshot.durable_lsn > 0); + let segments = snapshot + .segments + .iter() + .map(|segment| { + service + .load_metadata_log_segment(&segment.segment_key) + .unwrap() + }) + .collect::>(); + assert!(segments + .iter() + .flat_map(|segment| &segment.entries) + .any(|entry| { entry.command.kind == CommandKind::ReserveAllocator })); - let removed = service.remove_file(InodeId::root(), &name).unwrap(); - assert_eq!(removed, published); - assert_eq!(removed.body.as_ref().unwrap().manifest_id, "artifact.bin"); - assert!(service - .lookup_plus(InodeId::root(), &name) + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &snapshot_segment_keys(&snapshot), + 0, + METADATA_LOG_ZERO_DIGEST, + ) .unwrap() - .is_none()); - assert!(service.get_attr(removed.attr.inode).unwrap().is_none()); + .unwrap(); + + let after_failover = recovered + .prepare_artifact_create(InodeId::root(), dname(b"after-failover.bin")) + .unwrap(); + assert!( + after_failover.inode.get() > max_prepared_inode, + "recovery must not reuse an inode from a prepare-only reservation" + ); + assert!( + after_failover.generation > max_prepared_generation, + "recovery must not reuse a generation from a prepare-only reservation" + ); } #[test] -fn hardlink_updates_link_count_and_defers_body_gc_until_last_unlink() { - let (service, objects) = service_with_objects(); - let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); - let link_name = DentryName::new(b"artifact.link".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request(name.clone(), "artifact.bin", b"old")) +fn failed_allocator_archive_keeps_local_watermark_slow_until_pending_flush() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + // Initialize the object-GC claim before enabling the log so the only logged + // command in this workload is the allocator reservation at the boundary. + service + .prepare_artifact_create(InodeId::root(), dname(b"claim-warmup.bin")) .unwrap(); - let body = published.body.clone().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - - let linked = service - .link(published.attr.inode, InodeId::root(), link_name.clone()) + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/allocator-pending-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - assert_eq!(linked.attr.inode, published.attr.inode); - assert_eq!(linked.attr.nlink, 2); - assert_eq!( + + while service.clock.load(Ordering::Relaxed).saturating_add(1) + <= service.reserved_version.load(Ordering::Relaxed) + && service.next_inode.load(Ordering::Relaxed).saturating_add(1) + <= service.reserved_next_inode.load(Ordering::Relaxed) + { service - .lookup_plus(InodeId::root(), &name) - .unwrap() - .unwrap() - .attr - .nlink, - 2 + .prepare_artifact_create(InodeId::root(), dname(b"boundary.bin")) + .unwrap(); + } + let reserved_version_before = service.reserved_version.load(Ordering::Relaxed); + let reserved_inode_before = service.reserved_next_inode.load(Ordering::Relaxed); + objects.fail_puts_containing("meta/allocator-pending-log/log/"); + + assert!(matches!( + service.prepare_artifact_create(InodeId::root(), dname(b"first-fails.bin")), + Err(MetadError::SyncLogArchiveFailed { + committed: true, + .. + }) + )); + assert_eq!( + service.reserved_version.load(Ordering::Relaxed), + reserved_version_before, + "a locally applied but unarchived reservation must not open the fast path" ); assert_eq!( - service - .lookup_plus(InodeId::root(), &link_name) - .unwrap() - .unwrap() - .attr - .nlink, - 2 + service.reserved_next_inode.load(Ordering::Relaxed), + reserved_inode_before, + "inode reservation watermark must stay behind until pending archive flush" ); + assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); - let removed = service.remove_file(InodeId::root(), &name).unwrap(); - assert_eq!(removed.attr.inode, published.attr.inode); - assert!(service - .lookup_plus(InodeId::root(), &name) - .unwrap() - .is_none()); - let remaining = service - .lookup_plus(InodeId::root(), &link_name) - .unwrap() + objects.clear_faults(); + service + .prepare_artifact_create(InodeId::root(), dname(b"second-flushes.bin")) .unwrap(); - assert_eq!(remaining.attr.nlink, 1); - assert_eq!( - service - .get_attr(published.attr.inode) - .unwrap() - .unwrap() - .nlink, - 1 - ); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); assert_eq!( - service.read_artifact(InodeId::root(), &link_name).unwrap(), - b"old" + snapshot.durable_lsn, 2, + "the retry must archive the prior applied reservation before its own reservation" ); - assert!(objects.head(&object).unwrap().is_some()); + let kinds = snapshot + .segments + .iter() + .map(|segment| { + service + .load_metadata_log_segment(&segment.segment_key) + .unwrap() + }) + .flat_map(|segment| segment.entries.into_iter()) + .map(|entry| entry.command.kind) + .collect::>(); assert_eq!( - service.cleanup_pending_objects(100).unwrap(), - PendingObjectCleanupOutcome::default() + kinds, + vec![CommandKind::ReserveAllocator, CommandKind::ReserveAllocator] ); - - let removed_last = service.remove_file(InodeId::root(), &link_name).unwrap(); - assert_eq!(removed_last.attr.inode, published.attr.inode); - assert!(service.get_attr(published.attr.inode).unwrap().is_none()); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); - assert!(objects.head(&object).unwrap().is_none()); -} - -#[test] -fn hardlink_rejects_directories() { - let service = service(); - let dir = service - .create_dir( - InodeId::root(), - DentryName::new(b"dir".to_vec()).unwrap(), - 0o755, - 1000, - 1000, - ) - .unwrap(); - let err = service - .link( - dir.attr.inode, - InodeId::root(), - DentryName::new(b"dir-link".to_vec()).unwrap(), - ) - .unwrap_err(); - assert!(matches!(err, MetadError::NotFile)); } #[test] -fn remove_file_queues_old_body_for_object_cleanup() { +fn sync_metadata_log_archives_independent_batch_as_one_segment() { let (service, objects) = service_with_objects(); - let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request(name.clone(), "artifact.bin", b"old")) + service.create_dir_path("/left", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/right", 0o755, 1000, 1000) + .unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-sync-batch-log", 2); + service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-batch-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - let body = published.body.clone().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - assert!(objects.head(&object).unwrap().is_some()); - - let removed = service.remove_file(InodeId::root(), &name).unwrap(); - assert_eq!(removed, published); - assert!(objects.head(&object).unwrap().is_some()); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.scanned, 1); - assert_eq!(cleanup.attempted, 1); - assert_eq!(cleanup.deleted, 1); - assert_eq!(cleanup.missing, 0); - assert_eq!(cleanup.records_removed, 1); - assert!(objects.head(&object).unwrap().is_none()); - assert_eq!( - service.cleanup_pending_objects(100).unwrap(), - PendingObjectCleanupOutcome::default() - ); -} + let results = service.create_file_batches_in_dir_path(vec![ + CreateInDirPathBatch { + parent_path: "/left".to_owned(), + names: vec![DentryName::new(b"a.bin".to_vec()).unwrap()], + mode: 0o644, + uid: 1000, + gid: 1000, + }, + CreateInDirPathBatch { + parent_path: "/right".to_owned(), + names: vec![DentryName::new(b"b.bin".to_vec()).unwrap()], + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ]); -#[test] -fn gc_uses_the_canonical_nonzero_object_block_index() { - let (service, objects) = service_with_objects(); - let name = DentryName::new(b"sparse-gc.bin".to_vec()).unwrap(); - let prepared = service - .prepare_artifact_create(InodeId::root(), name.clone()) + assert_eq!(results.len(), 2); + for result in &results { + assert_eq!(result.as_ref().unwrap().len(), 1); + } + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); + assert_eq!(snapshot.segments.len(), 1); + let segment = service + .load_metadata_log_segment(&snapshot.segments.last().unwrap().segment_key) .unwrap(); - let inode = prepared.inode; - let generation = prepared.generation; - let block_index = 7_u64; - let written = service - .stage_prepared_artifact_ranges( - &prepared, - "sparse-gc", - &[PublishArtifactRange { - offset: 0, - bytes: b"sparse".to_vec(), - }], - block_index, + assert_eq!(segment.first_lsn, 1); + assert_eq!(segment.last_lsn, 2); + assert_eq!(segment.entries.len(), 2); + + let segment_keys = snapshot_segment_keys(&snapshot); + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + let outcome = recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &segment_keys, + 0, + METADATA_LOG_ZERO_DIGEST, ) + .unwrap() .unwrap(); - let staged = written.staged_objects().unwrap(); - let chunks = written.chunk_manifests(); + + assert_eq!(outcome.replayed_entries, 2); + assert!(recovered.lookup_path("/left/a.bin").unwrap().is_some()); + assert!(recovered.lookup_path("/right/b.bin").unwrap().is_some()); +} + +#[test] +fn post_commit_backend_readback_preserves_single_commit_log_order() { + let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-post-commit-readback", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); service - .publish_prepared_artifact_staged_session( - prepared, - PublishArtifactStagedSession { - parent: InodeId::root(), - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "unknown".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "sparse-gc".to_owned(), - size: 6, - chunks, - staged, - mode: 0o644, - uid: 1000, - gid: 1000, - }, - ) + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/post-commit-readback-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - let object = block_key(inode, generation, 0, block_index); - assert!(objects.head(&object).unwrap().is_some()); - service.remove_file(InodeId::root(), &name).unwrap(); + store.arm(); + let first = ordered_log_put_command( + b"req-post-commit-first", + b"post-commit-first", + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, + ); + service.commit_metadata(first).unwrap(); + let second = ordered_log_put_command( + b"req-post-commit-second", + b"post-commit-second", + b"second", + checkpoint.commit_version + 1, + checkpoint.commit_version + 2, + ); + service.commit_metadata(second).unwrap(); - let rows = service - .metadata - .scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(service.mount), - start_after: None, - version: service.read_version().unwrap(), - limit: 0, - purpose: ReadPurpose::UserStrong, + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); + let request_ids = snapshot + .segments + .iter() + .map(|pointer| { + service + .load_metadata_log_segment(&pointer.segment_key) + .unwrap() }) - .unwrap(); - assert_eq!(rows.len(), 1); - let key = decode_object_gc_record_key(service.mount, &rows[0].key).unwrap(); - assert_eq!(key.chunk_index, 0); - assert_eq!(key.block_index, block_index); + .flat_map(|segment| segment.entries.into_iter()) + .map(|entry| entry.command.request_id) + .collect::>(); + assert_eq!( + request_ids, + vec![ + b"req-post-commit-first".to_vec(), + b"req-post-commit-second".to_vec() + ] + ); - let cleanup = service.cleanup_pending_objects(10).unwrap(); - assert_eq!(cleanup.deleted, 1); - assert_eq!(cleanup.records_removed, 1); - assert!(objects.head(&object).unwrap().is_none()); - assert!(service - .metadata - .scan_keys(KeyScanRequest { - family: RecordFamily::System, - prefix: object_gc_quarantine_prefix(service.mount), - start_after: None, - limit: 0, - purpose: ReadPurpose::UserStrong, - }) + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &snapshot_segment_keys(&snapshot), + checkpoint.log_lsn, + checkpoint.log_digest, + ) .unwrap() - .is_empty()); + .unwrap(); + assert_eq!( + recovered + .metadata + .get( + RecordFamily::System, + b"post-commit-second", + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"second".to_vec())) + ); } #[test] -fn durable_object_gc_claim_codec_round_trips_and_rejects_trailing_bytes() { - let mount = MountId::new(1).unwrap(); - let deleting = ObjectGcClaim::Deleting { - owner_epoch: 7, - operation_token: 11, - gc_record_key: gc_object_key(mount, 10, InodeId::new(2).unwrap(), 3, 4, 5), - gc_record_version: 13, - }; - assert_eq!( - decode_object_gc_claim( - mount, - &encode_object_gc_claim(&ObjectGcClaim::Open).unwrap() - ) - .unwrap(), - ObjectGcClaim::Open - ); - assert_eq!( - decode_object_gc_claim(mount, &encode_object_gc_claim(&deleting).unwrap()).unwrap(), - deleting +fn readback_error_blocks_later_apply_and_both_checkpoint_publication_paths() { + let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-readback-block", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/readback-block-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + + store.arm(); + store.fail_next_readbacks(4); + let first = ordered_log_put_command( + b"req-readback-block-first", + b"readback-block-first", + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, ); - let mut malformed = encode_object_gc_claim(&deleting).unwrap(); - malformed.push(0); assert!(matches!( - decode_object_gc_claim(mount, &malformed), - Err(MetadError::Codec(_)) + service.commit_metadata(first), + Err(MetadError::Codec(message)) if message.contains("readback failed") )); -} - -#[test] -fn durable_object_gc_claim_codec_rejects_zero_identity_fields() { - let mount = MountId::new(1).unwrap(); - let gc_record_key = gc_object_key(mount, 10, InodeId::new(2).unwrap(), 3, 4, 5); - for invalid in [ - ObjectGcClaim::Deleting { - owner_epoch: 0, - operation_token: 11, - gc_record_key: gc_record_key.clone(), - gc_record_version: 13, - }, - ObjectGcClaim::Deleting { - owner_epoch: 7, - operation_token: 0, - gc_record_key: gc_record_key.clone(), - gc_record_version: 13, - }, - ObjectGcClaim::Deleting { - owner_epoch: 7, - operation_token: 11, - gc_record_key: gc_record_key.clone(), - gc_record_version: 0, - }, - ] { - assert!(matches!( - decode_object_gc_claim(mount, &encode_object_gc_claim(&invalid).unwrap()), - Err(MetadError::Codec(_)) - )); - } -} -#[test] -fn durable_object_gc_claim_codec_rejects_non_local_or_malformed_gc_keys() { - let mount = MountId::new(1).unwrap(); - let valid_key = gc_object_key(mount, 10, InodeId::new(2).unwrap(), 3, 4, 5); - let mut invalid_keys = vec![ - gc_object_key( - MountId::new(2).unwrap(), - 10, - InodeId::new(2).unwrap(), - 3, - 4, - 5, - ), - gc_queue_prefix(mount), - ]; - for field in [1, 2, 3] { - let mut key = valid_key.clone(); - key[field * 8..(field + 1) * 8].fill(0); - invalid_keys.push(key); - } + let blocked = ordered_log_put_command( + b"req-readback-block-second", + b"readback-block-second", + b"second", + checkpoint.commit_version + 1, + checkpoint.commit_version + 2, + ); + assert!(matches!( + service.commit_metadata(blocked), + Err(MetadError::Codec(message)) if message.contains("readback failed") + )); + assert_eq!( + service + .metadata + .get( + RecordFamily::System, + b"readback-block-second", + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + None + ); - for gc_record_key in invalid_keys { - let invalid = ObjectGcClaim::Deleting { - owner_epoch: 7, - operation_token: 11, - gc_record_key, - gc_record_version: 13, - }; - assert!(matches!( - decode_object_gc_claim(mount, &encode_object_gc_claim(&invalid).unwrap()), - Err(MetadError::Codec(_)) - )); - } + assert!(matches!( + service.backup_metadata(&checkpoint_config), + Err(MetadError::Codec(message)) if message.contains("readback failed") + )); + let controlled = MetadataArchiveConfig::new("meta/controlled-readback-block", 2); + assert!(matches!( + service.prepare_immutable_metadata_backup(&controlled), + Err(MetadError::Codec(message)) if message.contains("readback failed") + )); + assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); + + store.clear_readback_failures(); + let third = ordered_log_put_command( + b"req-readback-block-third", + b"readback-block-third", + b"third", + checkpoint.commit_version + 1, + checkpoint.commit_version + 3, + ); + service.commit_metadata(third).unwrap(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); + let request_ids = snapshot + .segments + .iter() + .map(|pointer| { + service + .load_metadata_log_segment(&pointer.segment_key) + .unwrap() + }) + .flat_map(|segment| segment.entries.into_iter()) + .map(|entry| entry.command.request_id) + .collect::>(); + assert_eq!( + request_ids, + vec![ + b"req-readback-block-first".to_vec(), + b"req-readback-block-third".to_vec() + ] + ); } #[test] -fn durable_gc_claim_blocks_new_object_reference_planning_while_deleting() { - let metadata = PausingObjectGcStore::new(); - let objects = MemoryObjectStore::new(); - let service = Arc::new(NoKvFs::new( +fn mismatched_readback_result_blocks_later_apply_until_exact_result_is_visible() { + let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); + let service = NoKvFs::new( MountId::new(1).unwrap(), - metadata.clone(), - objects.clone(), - )); + store.clone(), + MemoryObjectStore::new(), + ); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let published = service - .publish_artifact(artifact_request( - DentryName::new(b"old".to_vec()).unwrap(), - "old", - b"old", - )) - .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - let resize_name = DentryName::new(b"resize".to_vec()).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-readback-mismatch", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); service - .publish_artifact(artifact_request( - resize_name.clone(), - "resize", - b"resize-body", + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/readback-mismatch-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, )) .unwrap(); - service.remove_file_path("/old").unwrap(); - - metadata.arm(); - let cleaner = { - let service = Arc::clone(&service); - std::thread::spawn(move || service.cleanup_pending_objects(1)) - }; - metadata.wait_until_reached(); + store.arm(); + store.mismatch_next_readbacks(2); + let first = ordered_log_put_command( + b"req-readback-mismatch-first", + b"readback-mismatch-first", + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, + ); assert!(matches!( - service.prepare_artifact_create(InodeId::root(), DentryName::new(b"new".to_vec()).unwrap()), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - )); - assert!(matches!( - service.publish_artifact(artifact_request( - DentryName::new(b"new-publish".to_vec()).unwrap(), - "new-publish", - b"new" - )), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - )); - assert!(matches!( - service.create_symlink( - InodeId::root(), - DentryName::new(b"new-link".to_vec()).unwrap(), - b"target".to_vec(), - 0o777, - 1000, - 1000 - ), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) + service.commit_metadata(first), + Err(MetadError::Codec(message)) if message.contains("result mismatch") )); + let blocked = ordered_log_put_command( + b"req-readback-mismatch-blocked", + b"readback-mismatch-blocked", + b"blocked", + checkpoint.commit_version + 1, + checkpoint.commit_version + 2, + ); assert!(matches!( - service.create_symlink( - InodeId::root(), - DentryName::new(b"invalid-link".to_vec()).unwrap(), - Vec::new(), - 0o777, - 1000, - 1000 - ), - Err(MetadError::InvalidPath(_)) + service.commit_metadata(blocked), + Err(MetadError::Codec(message)) if message.contains("result mismatch") )); - assert!(matches!( - service.update_attrs( - InodeId::root(), - &resize_name, - UpdateAttr { - size: Some(1), - ..UpdateAttr::default() - } + assert!(service + .metadata + .get( + RecordFamily::System, + b"readback-mismatch-blocked", + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + + store.clear_readback_mismatches(); + let trigger = ordered_log_put_command( + b"req-readback-mismatch-trigger", + b"readback-mismatch-trigger", + b"trigger", + checkpoint.commit_version + 1, + checkpoint.commit_version + 3, + ); + service.commit_metadata(trigger).unwrap(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); +} + +#[test] +fn batch_post_commit_backend_readback_archives_full_batch_in_input_order() { + let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-batch-readback", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/batch-readback-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + + let commands = vec![ + ordered_log_put_command( + b"req-batch-readback-first", + b"batch-readback-first", + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, ), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - )); - assert!(matches!( - service.publish_checkpoint( - InodeId::root(), - vec![CheckpointShard { - name: DentryName::new(b"checkpoint-shard".to_vec()).unwrap(), - bytes: b"checkpoint".to_vec(), - }], - 1000, - 1000 + ordered_log_put_command( + b"req-batch-readback-second", + b"batch-readback-second", + b"second", + checkpoint.commit_version, + checkpoint.commit_version + 2, ), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - )); - assert!(matches!( - service.snapshot_subtree(InodeId::root()), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - )); - let attr_only = service - .update_attrs( - InodeId::root(), - &resize_name, - UpdateAttr { - mode: Some(0o600), - ..UpdateAttr::default() - }, + ]; + store.fail_next_batch_results(vec![0, 1]); + let results = service.commit_independent_metadata_batch(&commands); + assert!(results.iter().all(Result::is_ok)); + + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); + assert_eq!(snapshot.segments.len(), 1); + let segment = service + .load_metadata_log_segment(&snapshot.segments[0].segment_key) + .unwrap(); + assert_eq!( + segment + .entries + .iter() + .map(|entry| entry.command.request_id.as_slice()) + .collect::>(), + vec![ + b"req-batch-readback-first".as_slice(), + b"req-batch-readback-second".as_slice() + ] + ); + + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + let outcome = recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &snapshot_segment_keys(&snapshot), + checkpoint.log_lsn, + checkpoint.log_digest, ) + .unwrap() .unwrap(); - assert_eq!(attr_only.attr.mode, 0o600); - metadata.release(); - let cleaned = cleaner.join().unwrap().unwrap(); - assert_eq!(cleaned.deleted, 1); - assert!(objects.head(&object).unwrap().is_none()); + assert_eq!(outcome.replayed_entries, 2); + assert_eq!( + recovered + .metadata + .get( + RecordFamily::System, + b"batch-readback-second", + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"second".to_vec())) + ); } #[test] -fn gc_recheck_preserves_an_object_referenced_by_the_current_manifest() { - let (service, objects) = service_with_objects(); - let name = DentryName::new(b"live".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request(name.clone(), "live", b"live")) +fn unresolved_early_batch_subgroup_freezes_later_success_and_mixed_failure() { + let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-subgroup-readback", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/subgroup-readback-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - enqueue_gc_candidate( - &service, - ObjectGcRecord { - inode: published.attr.inode, - generation: body.generation, - object_key: object.as_str().to_owned(), - size: body.size, - digest_uri: body.digest_uri.clone(), - enqueue_version: 0, - enqueue_unix_ms: 0, - }, - ); - - let claim_key = object_gc_claim_key(service.mount); - let claim_version_before = service - .metadata - .get_versioned( - RecordFamily::System, - &claim_key, - service.read_version().unwrap(), - ReadPurpose::WritePlanLocal, - ) - .unwrap() - .unwrap() - .version; - let cleanup = service.cleanup_pending_objects(1).unwrap(); - assert_eq!(cleanup.blocked_by_snapshots, 1); - assert_eq!(cleanup.deleted, 0); - assert_eq!(cleanup.records_removed, 0); - assert!(objects.head(&object).unwrap().is_some()); + let first = ordered_log_put_command( + b"req-subgroup-first", + b"subgroup-shared-key", + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, + ); + // Same key forces Holt to flush the first atomic subgroup before planning + // and applying this later successful subgroup. + let second = ordered_log_put_command( + b"req-subgroup-second", + b"subgroup-shared-key", + b"second", + checkpoint.commit_version + 1, + checkpoint.commit_version + 2, + ); + let mut rejected = ordered_log_put_command( + b"req-subgroup-rejected", + b"subgroup-rejected-key", + b"rejected", + checkpoint.commit_version, + checkpoint.commit_version + 3, + ); + rejected.predicates.push(PredicateRef { + family: RecordFamily::System, + key: b"missing-subgroup-predicate-key".to_vec(), + predicate: Predicate::Exists, + }); + store.fail_next_batch_results(vec![0]); + store.fail_next_readbacks(1); + let results = + service.commit_independent_metadata_batch(&[first.clone(), second.clone(), rejected]); + assert!(results + .iter() + .all(|result| matches!(result, Err(MetadError::Codec(message)) if message.contains("readback failed")))); + assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); assert_eq!( - service.read_artifact(InodeId::root(), &name).unwrap(), - b"live" + service + .metadata + .get( + RecordFamily::System, + b"subgroup-shared-key", + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"second".to_vec())), + "the later subgroup applied but must not be acknowledged or archived early" ); - // The first pass advances the durable cursor, the second reaches the tail - // and resets it, and the third sees the protected row again. None of those - // advisory scans may rotate the global object-reference epoch. - assert_eq!(service.cleanup_pending_objects(1).unwrap().scanned, 0); + store.clear_readback_failures(); + let trigger = ordered_log_put_command( + b"req-subgroup-trigger", + b"subgroup-trigger-key", + b"trigger", + checkpoint.commit_version + 2, + checkpoint.commit_version + 4, + ); + service.commit_metadata(trigger).unwrap(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 3); + assert_eq!(snapshot.segments.len(), 2); + let resolved_group = service + .load_metadata_log_segment(&snapshot.segments[0].segment_key) + .unwrap(); assert_eq!( - service - .cleanup_pending_objects(1) - .unwrap() - .blocked_by_snapshots, - 1 + resolved_group + .entries + .iter() + .map(|entry| entry.command.request_id.as_slice()) + .collect::>(), + vec![first.request_id.as_slice(), second.request_id.as_slice()], + "the true committed subset must archive once in original execution order" ); - let claim_version_after = service - .metadata - .get_versioned( - RecordFamily::System, - &claim_key, - service.read_version().unwrap(), - ReadPurpose::WritePlanLocal, - ) - .unwrap() - .unwrap() - .version; - assert_eq!(claim_version_after, claim_version_before); } -#[test] -fn snapshot_mint_retries_after_an_intervening_object_delete_epoch() { - let store = SnapshotCommitBarrierStore::new(CommandKind::SnapshotSubtree, 1, 2); - let objects = MemoryObjectStore::new(); - let service = Arc::new(NoKvFs::new( - MountId::new(1).unwrap(), - store.clone(), - objects.clone(), - )); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"snapshot-race".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request(name.clone(), "snapshot-race", b"old")) - .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); +use nokv_object::{ObjectInfo, ObjectRange}; +use std::sync::atomic::AtomicUsize; + +/// An [`ObjectStore`] wrapper that injects PUT failures to simulate a crash at a +/// chosen point (e.g. after the checkpoint object is written but before the +/// `CURRENT` pointer is swapped). Reads and deletes always pass through. +#[derive(Clone)] +struct FaultObjectStore { + inner: MemoryObjectStore, + fail_put_substring: Arc>>, + injected_put_failures: Arc, + get_keys: Arc>>, +} + +impl FaultObjectStore { + fn new(inner: MemoryObjectStore) -> Self { + Self { + inner, + fail_put_substring: Arc::new(Mutex::new(None)), + injected_put_failures: Arc::new(AtomicUsize::new(0)), + get_keys: Arc::new(Mutex::new(Vec::new())), + } + } + + fn fail_puts_containing(&self, substring: &str) { + *self.fail_put_substring.lock().unwrap() = Some(substring.to_owned()); + } + + fn clear_faults(&self) { + *self.fail_put_substring.lock().unwrap() = None; + } + + fn injected_put_failures(&self) -> usize { + self.injected_put_failures.load(Ordering::Relaxed) + } + + fn clear_get_keys(&self) { + self.get_keys.lock().unwrap().clear(); + } + + fn got_key(&self, key: &str) -> bool { + self.get_keys.lock().unwrap().iter().any(|got| got == key) + } +} + +/// Records successful physical PUTs and can report one of them as failed only +/// after the bytes reached the backing store. This models process loss at the +/// restore initialization PUT-after barrier without changing production code. +#[derive(Clone)] +struct RestorePostPutFaultStore { + inner: MemoryObjectStore, + state: Arc>, +} + +#[derive(Default)] +struct RestorePostPutFaultState { + fail_after_matches: Option, + put_keys: Vec, +} + +impl RestorePostPutFaultStore { + fn new(inner: MemoryObjectStore) -> Self { + Self { + inner, + state: Arc::new(Mutex::new(RestorePostPutFaultState::default())), + } + } + + fn fail_after_put(&self, zero_based_match: usize) { + self.state.lock().unwrap().fail_after_matches = Some(zero_based_match); + } - let mint_service = Arc::clone(&service); - let mint = std::thread::spawn(move || mint_service.snapshot_subtree(InodeId::root())); - store.wait_until_blocked(); - service.remove_file(InodeId::root(), &name).unwrap(); - assert_eq!(service.cleanup_pending_objects(1).unwrap().deleted, 1); - assert!(objects.head(&object).unwrap().is_none()); - store.release_blocked(); + fn clear_put_history(&self) { + self.state.lock().unwrap().put_keys.clear(); + } - let snapshot = mint.join().unwrap().unwrap(); - assert!(matches!( - service.read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/snapshot-race"), - Err(MetadError::NotFound) - )); + fn put_keys(&self) -> Vec { + self.state.lock().unwrap().put_keys.clone() + } } -#[test] -fn reopen_resumes_a_durable_object_gc_claim_after_crash() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let published = service - .publish_artifact(artifact_request( - DentryName::new(b"old".to_vec()).unwrap(), - "old", - b"old", - )) - .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - service.remove_file_path("/old").unwrap(); - let gc_row = metadata - .scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(service.mount), - start_after: None, - version: service.read_version().unwrap(), - limit: 1, - purpose: ReadPurpose::WritePlanLocal, - }) - .unwrap() - .pop() - .unwrap(); - let claim_key = leave_object_gc_deleting(&service, &gc_row); - drop(service); +impl ObjectStore for RestorePostPutFaultStore { + fn put( + &self, + key: &ObjectKey, + bytes: impl Into, + ) -> Result { + let info = self.inner.put(key, bytes)?; + let should_fail = { + let mut state = self.state.lock().unwrap(); + state.put_keys.push(key.clone()); + match state.fail_after_matches.as_mut() { + Some(remaining) if *remaining == 0 => { + state.fail_after_matches = None; + true + } + Some(remaining) => { + *remaining -= 1; + false + } + None => false, + } + }; + if should_fail { + return Err(ObjectError::Backend( + "injected restore crash after object PUT".to_owned(), + )); + } + Ok(info) + } - let reopened = NoKvFs::open_existing( - MountId::new(1).unwrap(), - metadata.clone(), - objects.clone(), - 0, - ) - .unwrap(); - assert!(objects.head(&object).unwrap().is_some()); - let deferred_claim = metadata - .get( - RecordFamily::System, - &claim_key, - reopened.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap(); - assert!(matches!( - decode_object_gc_claim(reopened.mount, &deferred_claim.0).unwrap(), - ObjectGcClaim::Deleting { .. } - )); + fn get(&self, key: &ObjectKey, range: Option) -> Result, ObjectError> { + self.inner.get(key, range) + } - let cleanup = reopened.cleanup_pending_objects(1).unwrap(); - assert_eq!(cleanup.deleted, 1); - assert_eq!(cleanup.records_removed, 1); - assert!(objects.head(&object).unwrap().is_none()); - let open_claim = metadata - .get( - RecordFamily::System, - &claim_key, - reopened.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap(); - assert_eq!( - decode_object_gc_claim(reopened.mount, &open_claim.0).unwrap(), - ObjectGcClaim::Open - ); + fn head(&self, key: &ObjectKey) -> Result, ObjectError> { + self.inner.head(key) + } + + fn delete(&self, key: &ObjectKey) -> Result { + self.inner.delete(key) + } } -#[test] -fn history_cleanup_does_not_invalidate_a_prepared_object_upload() { - let (service, _) = service_with_objects(); - let name = DentryName::new(b"upload-during-history-prune.bin".to_vec()).unwrap(); - let prepared = service - .prepare_artifact_create(InodeId::root(), name.clone()) - .unwrap(); +impl ObjectStore for FaultObjectStore { + fn put( + &self, + key: &ObjectKey, + bytes: impl Into, + ) -> Result { + if let Some(substring) = self.fail_put_substring.lock().unwrap().clone() { + if key.as_str().contains(&substring) { + self.injected_put_failures.fetch_add(1, Ordering::Relaxed); + return Err(ObjectError::Backend("injected put fault".to_owned())); + } + } + self.inner.put(key, bytes) + } - service.cleanup_history(100).unwrap(); - assert_eq!( - service - .refresh_prepared_artifact_object_gc_epoch(prepared.clone()) - .unwrap(), - prepared - ); + fn get(&self, key: &ObjectKey, range: Option) -> Result, ObjectError> { + self.get_keys.lock().unwrap().push(key.as_str().to_owned()); + self.inner.get(key, range) + } - let bytes = b"history-prune-independent"; - service - .publish_prepared_artifact_session( - prepared, - PublishArtifactSession { - parent: InodeId::root(), - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "sha256:history-prune-independent".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "history-prune-independent".to_owned(), - size: bytes.len() as u64, - ranges: vec![PublishArtifactRange { - offset: 0, - bytes: bytes.to_vec(), - }], - mode: 0o644, - uid: 1000, - gid: 1000, - }, - ) - .unwrap(); - assert_eq!( - service.read_artifact(InodeId::root(), &name).unwrap(), - bytes - ); + fn head(&self, key: &ObjectKey) -> Result, ObjectError> { + self.inner.head(key) + } + + fn delete(&self, key: &ObjectKey) -> Result { + self.inner.delete(key) + } } #[test] -fn prepared_publish_rejects_an_intervening_object_gc_epoch_and_refreshes_identity() { - let (service, objects) = service_with_objects(); - let victim_name = DentryName::new(b"gc-victim".to_vec()).unwrap(); +fn post_commit_backend_with_archive_failure_retains_exact_pending_segment() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing); + let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); + let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-backend-archive-failure", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); service - .publish_artifact(artifact_request( - victim_name.clone(), - "gc-victim", - b"victim", - )) - .unwrap(); - service.remove_file(InodeId::root(), &victim_name).unwrap(); - - let name = DentryName::new(b"late-after-gc".to_vec()).unwrap(); - let prepared = service - .prepare_artifact_create(InodeId::root(), name.clone()) - .unwrap(); - let bytes = b"late-after-gc"; - let written = service - .stage_prepared_artifact_ranges( - &prepared, - "late-after-gc", - &[PublishArtifactRange { - offset: 0, - bytes: bytes.to_vec(), - }], + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/backend-archive-failure-log", + "mount-1:/", + 1, 0, - ) + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - let staged = written.staged_objects().unwrap(); - assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 1); - let error = service - .publish_prepared_artifact_staged_session( - prepared.clone(), - PublishArtifactStagedSession { - parent: InodeId::root(), - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "sha256:late-after-gc".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "late-after-gc".to_owned(), - size: bytes.len() as u64, - chunks: written.chunk_manifests(), - staged, - mode: 0o644, - uid: 1000, - gid: 1000, - }, - ) - .unwrap_err(); - let staged = match error { - MetadError::PublishArtifactFailed { source, staged } => { - assert!(matches!( - *source, - MetadError::StalePreparedArtifactObjectGcEpoch { .. } - )); - staged - } - error => panic!("unexpected prepared publish error: {error:?}"), - }; - assert!(service.lookup_path("/late-after-gc").unwrap().is_none()); - let cleanup = service.cleanup_staged_objects(&staged).unwrap(); - assert_eq!(cleanup.deleted, staged.len()); - for object in staged.objects() { - assert!(objects.head(&object.key).unwrap().is_none()); - } + objects.fail_puts_containing("meta/backend-archive-failure-log/log/"); + store.arm(); + let first = ordered_log_put_command( + b"req-backend-archive-first", + b"backend-archive-first", + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, + ); + assert!(matches!( + service.commit_metadata(first), + Err(MetadError::SyncLogArchiveFailed { + committed: true, + .. + }) + )); + assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); - let refreshed = service - .refresh_prepared_artifact_object_gc_epoch(prepared.clone()) - .unwrap(); - assert!(refreshed.generation > prepared.generation); - assert_ne!( - refreshed.object_gc_claim_version, - prepared.object_gc_claim_version + objects.clear_faults(); + let second = ordered_log_put_command( + b"req-backend-archive-second", + b"backend-archive-second", + b"second", + checkpoint.commit_version + 1, + checkpoint.commit_version + 2, + ); + service.commit_metadata(second).unwrap(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); + let request_ids = snapshot + .segments + .iter() + .map(|pointer| { + service + .load_metadata_log_segment(&pointer.segment_key) + .unwrap() + }) + .flat_map(|segment| segment.entries.into_iter()) + .map(|entry| entry.command.request_id) + .collect::>(); + assert_eq!( + request_ids, + vec![ + b"req-backend-archive-first".to_vec(), + b"req-backend-archive-second".to_vec() + ] ); } #[test] -fn failover_claim_rotation_rejects_old_prepared_upload_and_allows_cleanup() { - let (source, objects) = service_with_objects(); - let name = dname(b"prepared-before-failover.bin"); - let prepared = source - .prepare_artifact_create(InodeId::root(), name.clone()) - .unwrap(); - let bytes = b"staged by the failed owner"; - let written = source - .stage_prepared_artifact_ranges( - &prepared, - "prepared-before-failover", - &[PublishArtifactRange { - offset: 0, - bytes: bytes.to_vec(), - }], - 0, - ) - .unwrap(); - let staged_before_failover = written.staged_objects().unwrap(); - let staged_key = staged_before_failover.objects()[0].key.clone(); - assert!(objects.head(&staged_key).unwrap().is_some()); - - let checkpoint = MetadataArchiveConfig::new("meta/prepared-failover", 2); - source.backup_metadata(&checkpoint).unwrap(); - let recovered = NoKvFs::new( +fn publish_remains_readable_when_sync_log_ack_fails_after_metadata_commit() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing.clone()); + let service = NoKvFs::new( MountId::new(1).unwrap(), HoltMetadataStore::open_memory().unwrap(), objects.clone(), ); - // Match controlled startup: install the acquired epoch before restoring the - // old owner's checkpoint, recover any crash-left deletion, then rotate Open. - recovered.install_owner_epoch(2).unwrap(); - recovered.restore_metadata(&checkpoint).unwrap().unwrap(); - recovered.recover_object_gc_claim().unwrap(); - recovered.rotate_object_gc_claim_for_failover().unwrap(); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-log-ack", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + objects.fail_puts_containing("meta/sync-log-ack/log/"); - let error = recovered - .publish_prepared_artifact_staged_session( - prepared, - PublishArtifactStagedSession { - parent: InodeId::root(), - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "sha256:prepared-before-failover".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "prepared-before-failover".to_owned(), - size: bytes.len() as u64, - chunks: written.chunk_manifests(), - staged: staged_before_failover, - mode: 0o644, - uid: 1000, - gid: 1000, - }, - ) + let name = dname(b"committed.bin"); + let payload = b"metadata committed before the archive acknowledgement"; + let error = service + .publish_artifact(artifact_request(name.clone(), "committed", payload)) .unwrap_err(); let staged = match error { MetadError::PublishArtifactFailed { source, staged } => { assert!(matches!( *source, - MetadError::StalePreparedArtifactObjectGcEpoch { .. } - | MetadError::Metadata(MetadataError::PredicateFailed) + MetadError::SyncLogArchiveFailed { + committed: true, + .. + } )); staged } - other => panic!("unexpected old prepared publish error: {other:?}"), + other => panic!("unexpected publish error: {other:?}"), }; - assert!(recovered + + let committed = service .lookup_plus(InodeId::root(), &name) .unwrap() - .is_none()); - recovered.cleanup_staged_objects(&staged).unwrap(); - assert!(objects.head(&staged_key).unwrap().is_none()); + .expect("the metadata transaction committed"); + assert_eq!( + service.read_artifact(InodeId::root(), &name).unwrap(), + payload + ); + assert_eq!(staged.len(), 1); + assert_eq!( + staged.objects()[0].key, + block_key( + committed.attr.inode, + committed.body.as_ref().unwrap().generation, + 0, + 0, + ) + ); + assert!(backing.head(&staged.objects()[0].key).unwrap().is_some()); } #[test] -fn failover_claim_rotation_confirms_a_lost_backend_ack_by_exact_readback() { - let metadata = PostCommitErrorStore::new_disarmed(CommandKind::CleanupObjects); +fn checkpoint_keeps_referenced_blocks_after_committed_sync_log_failure() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing.clone()); let service = NoKvFs::new( MountId::new(1).unwrap(), - metadata.clone(), - MemoryObjectStore::new(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), ); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let prepared = service - .prepare_artifact_create(InodeId::root(), dname(b"old-token.bin")) - .unwrap(); - let old_claim_version = prepared.object_gc_claim_version; - - metadata.arm(); - service.rotate_object_gc_claim_for_failover().unwrap(); - let refreshed = service - .refresh_prepared_artifact_object_gc_epoch(prepared) - .unwrap(); - assert_ne!(refreshed.object_gc_claim_version, old_claim_version); -} - -#[test] -fn blocked_head_gc_row_does_not_starve_later_reclaimable_candidate() { - let (service, objects) = service_with_objects(); - let live_name = DentryName::new(b"live-head".to_vec()).unwrap(); - let live = service - .publish_artifact(artifact_request(live_name, "live-head", b"live")) + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/checkpoint-sync-log-ack", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - let live_body = live.body.as_ref().unwrap(); - let live_object = block_key(live.attr.inode, live_body.generation, 0, 0); - enqueue_gc_candidate( - &service, - ObjectGcRecord { - inode: live.attr.inode, - generation: live_body.generation, - object_key: live_object.as_str().to_owned(), - size: live_body.size, - digest_uri: live_body.digest_uri.clone(), - enqueue_version: 0, - enqueue_unix_ms: 0, - }, - ); - - let reclaimable = ObjectKey::new("blocks/1/900/1/0/0").unwrap(); - objects.put(&reclaimable, b"stale".to_vec()).unwrap(); - enqueue_gc_candidate( - &service, - ObjectGcRecord { - inode: InodeId::new(900).unwrap(), - generation: 1, - object_key: reclaimable.as_str().to_owned(), - size: 5, - digest_uri: "sha256:stale".to_owned(), - enqueue_version: 0, - enqueue_unix_ms: 0, - }, - ); - - let first = service.cleanup_pending_objects(1).unwrap(); - assert_eq!(first.scanned, 1); - assert_eq!(first.blocked_by_snapshots, 1); - assert_eq!(first.deleted, 0); - assert!(objects.head(&live_object).unwrap().is_some()); - assert!(objects.head(&reclaimable).unwrap().is_some()); - - let second = service.cleanup_pending_objects(1).unwrap(); - assert_eq!(second.scanned, 1); - assert_eq!(second.deleted, 1); - assert!(objects.head(&live_object).unwrap().is_some()); - assert!(objects.head(&reclaimable).unwrap().is_none()); -} - -#[test] -fn local_gc_row_key_cannot_delete_an_object_owned_by_another_mount() { - let (service, objects) = service_with_objects(); - let foreign_object = ObjectKey::new("blocks/2/902/1/0/0").unwrap(); - objects.put(&foreign_object, b"foreign".to_vec()).unwrap(); - let row_key = enqueue_gc_candidate( - &service, - ObjectGcRecord { - inode: InodeId::new(902).unwrap(), - generation: 1, - object_key: foreign_object.as_str().to_owned(), - size: 7, - digest_uri: "sha256:foreign".to_owned(), - enqueue_version: 0, - enqueue_unix_ms: 0, - }, - ); - assert_eq!(row_key.len(), 48); - assert!(row_key.starts_with(&gc_queue_prefix(service.mount))); + objects.fail_puts_containing("meta/checkpoint-sync-log-ack/log/"); - let outcome = service.cleanup_pending_objects(1).unwrap(); - assert_eq!(outcome.attempted, 0); - assert_eq!(outcome.deleted, 0); - assert_eq!(outcome.records_removed, 1); - assert!(objects.head(&foreign_object).unwrap().is_some()); - assert!(service - .metadata - .get( - RecordFamily::Gc, - &row_key, - service.read_version().unwrap(), - ReadPurpose::UserStrong, + let payload = b"checkpoint shard remains reachable after log ACK failure".to_vec(); + let error = service + .publish_checkpoint( + InodeId::root(), + vec![CheckpointShard { + name: dname(b"rank-0.ckpt"), + bytes: payload.clone(), + }], + 1000, + 1000, ) + .unwrap_err(); + assert!(matches!( + error, + MetadError::SyncLogArchiveFailed { + committed: true, + .. + } + )); + + let committed = service + .lookup_path("/rank-0.ckpt") .unwrap() - .is_none()); + .expect("checkpoint metadata was atomically committed"); assert_eq!( service - .metadata - .scan_keys(KeyScanRequest { - family: RecordFamily::System, - prefix: object_gc_quarantine_prefix(service.mount), - start_after: None, - limit: 0, - purpose: ReadPurpose::UserStrong, - }) - .unwrap() - .len(), - 1 + .read_artifact(InodeId::root(), &dname(b"rank-0.ckpt")) + .unwrap(), + payload ); + let body = committed.body.expect("checkpoint has a staged object body"); + let object = block_key(committed.attr.inode, body.generation, 0, 0); + assert!(backing.head(&object).unwrap().is_some()); } #[test] -fn gc_row_key_cannot_delete_a_different_block_of_the_same_generation() { - let (service, objects) = service_with_objects(); - let inode = InodeId::new(904).unwrap(); - let object = ObjectKey::new("blocks/1/904/1/0/0").unwrap(); - objects.put(&object, b"protected".to_vec()).unwrap(); +fn checkpoint_keeps_blocks_after_uncertain_backend_ack_failure() { + let metadata = PostCommitErrorStore::new(CommandKind::PublishArtifact); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata, objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let payload = b"checkpoint visible despite a lost backend acknowledgement".to_vec(); - let version = service.next_version().unwrap(); - let row_key = gc_object_key(service.mount, version.get(), inode, 1, 0, 1); - let record = ObjectGcRecord { - inode, - generation: 1, - object_key: object.as_str().to_owned(), - size: 9, - digest_uri: "sha256:protected".to_owned(), - enqueue_version: version.get(), - enqueue_unix_ms: service.now_ms(), - }; - service - .commit_metadata(MetadataCommand { - request_id: b"mismatched-block-gc-row".to_vec(), - kind: CommandKind::CleanupObjects, - read_version: predecessor(version).unwrap(), - commit_version: version, - primary_family: RecordFamily::Gc, - primary_key: row_key.clone(), - predicates: vec![PredicateRef { - family: RecordFamily::Gc, - key: row_key.clone(), - predicate: Predicate::NotExists, - }], - mutations: vec![Mutation { - family: RecordFamily::Gc, - key: row_key.clone(), - op: MutationOp::Put, - value: Some(Value(encode_object_gc_record(&record))), + let error = service + .publish_checkpoint( + InodeId::root(), + vec![CheckpointShard { + name: dname(b"uncertain-rank.ckpt"), + bytes: payload.clone(), }], - watch: Vec::new(), - }) - .unwrap(); - - let outcome = service.cleanup_pending_objects(1).unwrap(); - assert_eq!(outcome.attempted, 0); - assert_eq!(outcome.deleted, 0); - assert_eq!(outcome.records_removed, 1); - assert!(objects.head(&object).unwrap().is_some()); - assert!(service - .metadata - .get( - RecordFamily::Gc, - &row_key, - service.read_version().unwrap(), - ReadPurpose::UserStrong, + 1000, + 1000, ) + .unwrap_err(); + assert!(matches!( + error, + MetadError::Metadata(MetadataError::Backend(message)) + if message.contains("journal acknowledgement") + )); + + let committed = service + .lookup_path("/uncertain-rank.ckpt") .unwrap() - .is_none()); + .expect("the injected backend applied before losing its ACK"); assert_eq!( service - .metadata - .scan_keys(KeyScanRequest { - family: RecordFamily::System, - prefix: object_gc_quarantine_prefix(service.mount), - start_after: None, - limit: 0, - purpose: ReadPurpose::UserStrong, - }) - .unwrap() - .len(), - 1 + .read_artifact(InodeId::root(), &dname(b"uncertain-rank.ckpt")) + .unwrap(), + payload ); + let body = committed.body.expect("checkpoint has an object body"); + assert!(objects + .head(&block_key(committed.attr.inode, body.generation, 0, 0,)) + .unwrap() + .is_some()); } #[test] -fn crash_left_gc_claim_quarantines_a_foreign_object_key_without_deleting_it() { - let (service, objects) = service_with_objects(); - let metadata = service.metadata.clone(); - let foreign_object = ObjectKey::new("blocks/2/903/1/0/0").unwrap(); - objects.put(&foreign_object, b"foreign".to_vec()).unwrap(); - let row_key = enqueue_gc_candidate( - &service, - ObjectGcRecord { - inode: InodeId::new(903).unwrap(), - generation: 1, - object_key: foreign_object.as_str().to_owned(), - size: 7, - digest_uri: "sha256:foreign".to_owned(), - enqueue_version: 0, - enqueue_unix_ms: 0, - }, +fn pending_sync_log_blocks_single_and_batch_apply_until_prior_commit_is_archived() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), ); - let row = metadata - .get_versioned( - RecordFamily::Gc, - &row_key, - service.read_version().unwrap(), - ReadPurpose::WritePlanLocal, - ) - .unwrap() + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/batch", 0o755, 1000, 1000) .unwrap(); - leave_object_gc_deleting( - &service, - &ScanItem { - key: row_key.clone(), - value: row.value, - version: row.version, - }, - ); - drop(service); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-pending-sync-log", 2); + service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/pending-sync-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + objects.fail_puts_containing("meta/pending-sync-log/log/"); + + let first_error = service + .create_dir_path("/first", 0o755, 1000, 1000) + .unwrap_err(); + assert!(matches!( + first_error, + MetadError::SyncLogArchiveFailed { + committed: true, + .. + } + )); + assert!(service.lookup_path("/first").unwrap().is_some()); + + let blocked_single = service + .create_dir_path("/second", 0o755, 1000, 1000) + .unwrap_err(); + assert!(matches!( + blocked_single, + MetadError::SyncLogArchiveFailed { + committed: false, + .. + } + )); + assert!(service.lookup_path("/second").unwrap().is_none()); + + let blocked = service.create_file_batches_in_dir_path(vec![CreateInDirPathBatch { + parent_path: "/batch".to_owned(), + names: vec![dname(b"third.bin")], + mode: 0o644, + uid: 1000, + gid: 1000, + }]); + assert!(matches!( + &blocked[0], + Err(MetadError::SyncLogArchiveFailed { + committed: false, + .. + }) + )); + assert!(service.lookup_path("/batch/third.bin").unwrap().is_none()); + assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); + + objects.clear_faults(); + service + .create_dir_path("/second", 0o755, 1000, 1000) + .unwrap(); + let retried = service.create_file_batches_in_dir_path(vec![CreateInDirPathBatch { + parent_path: "/batch".to_owned(), + names: vec![dname(b"third.bin")], + mode: 0o644, + uid: 1000, + gid: 1000, + }]); + assert_eq!(retried[0].as_ref().unwrap().len(), 1); + + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 3); + assert_eq!(snapshot.segments.len(), 3); + let first_segment = service + .load_metadata_log_segment(&snapshot.segments[0].segment_key) + .unwrap(); + let second_segment = service + .load_metadata_log_segment(&snapshot.segments[1].segment_key) + .unwrap(); + let third_segment = service + .load_metadata_log_segment(&snapshot.segments[2].segment_key) + .unwrap(); + assert_eq!((first_segment.first_lsn, first_segment.last_lsn), (1, 1)); + assert_eq!((second_segment.first_lsn, second_segment.last_lsn), (2, 2)); + assert_eq!(first_segment.last_digest, second_segment.prev_digest); + assert_eq!((third_segment.first_lsn, third_segment.last_lsn), (3, 3)); + assert_eq!(second_segment.last_digest, third_segment.prev_digest); - let reopened = NoKvFs::open_existing( + let recovered = NoKvFs::new( MountId::new(1).unwrap(), - metadata.clone(), - objects.clone(), - 0, - ) - .unwrap(); - let outcome = reopened.recover_object_gc_claim().unwrap(); - assert_eq!(outcome.attempted, 0); - assert_eq!(outcome.deleted, 0); - assert_eq!(outcome.records_removed, 1); - assert!(objects.head(&foreign_object).unwrap().is_some()); - assert!(metadata - .get( - RecordFamily::Gc, - &row_key, - reopened.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .is_none()); - assert_eq!( - metadata - .scan_keys(KeyScanRequest { - family: RecordFamily::System, - prefix: object_gc_quarantine_prefix(reopened.mount), - start_after: None, - limit: 0, - purpose: ReadPurpose::UserStrong, - }) - .unwrap() - .len(), - 1 + HoltMetadataStore::open_memory().unwrap(), + objects, ); - let claim = metadata - .get( - RecordFamily::System, - &object_gc_claim_key(reopened.mount), - reopened.read_version().unwrap(), - ReadPurpose::UserStrong, + let outcome = recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &snapshot_segment_keys(&snapshot), + 0, + METADATA_LOG_ZERO_DIGEST, ) .unwrap() .unwrap(); - assert_eq!( - decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), - ObjectGcClaim::Open - ); + assert_eq!(outcome.replayed_entries, 3); + assert!(recovered.lookup_path("/first").unwrap().is_some()); + assert!(recovered.lookup_path("/second").unwrap().is_some()); + assert!(recovered.lookup_path("/batch/third.bin").unwrap().is_some()); } #[test] -fn delete_error_keeps_the_durable_claim_closed_until_same_claim_recovery() { +fn immutable_checkpoint_restores_only_its_exact_control_identity() { let backing = MemoryObjectStore::new(); - let objects = DeleteAckLostObjectStore::new(backing.clone()); - let metadata = HoltMetadataStore::open_memory().unwrap(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let published = service - .publish_artifact(artifact_request( - DentryName::new(b"lost-delete-ack.bin".to_vec()).unwrap(), - "lost-delete-ack", - b"payload", - )) - .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - service - .remove_file( - InodeId::root(), - &DentryName::new(b"lost-delete-ack.bin".to_vec()).unwrap(), - ) + let objects = FaultObjectStore::new(backing.clone()); + let source = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + source.bootstrap_root(0o755, 1000, 1000).unwrap(); + source + .create_dir_path("/controlled", 0o755, 1000, 1000) .unwrap(); - - assert!(matches!( - service.cleanup_pending_objects(1), - Err(MetadError::Object(ObjectError::Backend(message))) - if message == "injected lost DELETE acknowledgement" - )); - assert_eq!(objects.delete_calls(), 1); - assert!(backing.head(&object).unwrap().is_none()); - let claim_key = object_gc_claim_key(service.mount); - let deleting = metadata - .get( - RecordFamily::System, - &claim_key, - service.read_version().unwrap(), - ReadPurpose::UserStrong, - ) + let config = MetadataArchiveConfig::new("meta/exact", 4); + let prepared = source.prepare_immutable_metadata_backup(&config).unwrap(); + let identity = MetadataCheckpointIdentity { + checkpoint_key: prepared.checkpoint_key.clone(), + image_bytes: prepared.image_bytes, + image_digest: prepared.image_digest.clone(), + }; + assert!(backing + .head(&ObjectKey::new("meta/exact/CURRENT").unwrap()) .unwrap() - .unwrap(); - assert!(matches!( - decode_object_gc_claim(service.mount, &deleting.0).unwrap(), - ObjectGcClaim::Deleting { .. } - )); - assert!(matches!( - service.begin_object_reference_mutation(), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - )); - assert_eq!( - metadata - .scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(service.mount), - start_after: None, - version: service.read_version().unwrap(), - limit: 0, - purpose: ReadPurpose::UserStrong, - }) - .unwrap() - .len(), - 1 - ); + .is_none()); - let recovered = service.recover_object_gc_claim().unwrap(); - assert_eq!(objects.delete_calls(), 2); - assert_eq!(recovered.attempted, 1); - assert_eq!(recovered.missing, 1); - assert_eq!(recovered.records_removed, 1); - let open = metadata - .get( - RecordFamily::System, - &claim_key, - service.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap(); - assert_eq!( - decode_object_gc_claim(service.mount, &open.0).unwrap(), - ObjectGcClaim::Open + // Drift standalone CURRENT to a different image. Controlled restore must + // still install only the exact control identity above. + source.create_dir_path("/drift", 0o755, 1000, 1000).unwrap(); + source.backup_metadata(&config).unwrap(); + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, ); - assert!(metadata - .scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(service.mount), - start_after: None, - version: service.read_version().unwrap(), - limit: 0, - purpose: ReadPurpose::UserStrong, - }) - .unwrap() - .is_empty()); + let restored = recovered + .restore_metadata_checkpoint(&config, &identity) + .unwrap(); + assert_eq!(restored.checkpoint_key, prepared.checkpoint_key); + assert!(recovered.lookup_path("/controlled").unwrap().is_some()); + assert!(recovered.lookup_path("/drift").unwrap().is_none()); } #[test] -fn malformed_gc_row_is_quarantined_without_starving_a_valid_candidate() { - let (service, objects) = service_with_objects(); - let mut malformed_key = gc_queue_prefix(service.mount); - malformed_key.extend_from_slice(&0_u64.to_be_bytes()); - let version = service.next_version().unwrap(); - service - .commit_metadata(MetadataCommand { - request_id: b"malformed-gc-row".to_vec(), - kind: CommandKind::CleanupObjects, - read_version: predecessor(version).unwrap(), - commit_version: version, - primary_family: RecordFamily::Gc, - primary_key: malformed_key.clone(), - predicates: vec![PredicateRef { - family: RecordFamily::Gc, - key: malformed_key.clone(), - predicate: Predicate::NotExists, - }], - mutations: vec![Mutation { - family: RecordFamily::Gc, - key: malformed_key, - op: MutationOp::Put, - value: Some(Value(b"not-an-object-gc-record".to_vec())), - }], - watch: Vec::new(), - }) - .unwrap(); - - let object = ObjectKey::new("blocks/1/901/1/0/0").unwrap(); - objects.put(&object, b"stale".to_vec()).unwrap(); - enqueue_gc_candidate( - &service, - ObjectGcRecord { - inode: InodeId::new(901).unwrap(), - generation: 1, - object_key: object.as_str().to_owned(), - size: 5, - digest_uri: "sha256:stale".to_owned(), - enqueue_version: 0, - enqueue_unix_ms: 0, - }, +fn exact_checkpoint_rejects_missing_proof_before_fetching_image() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing.clone()); + let source = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), ); + source.bootstrap_root(0o755, 1000, 1000).unwrap(); + let config = MetadataArchiveConfig::new("meta/proof", 4); + let prepared = source.prepare_immutable_metadata_backup(&config).unwrap(); + let identity = MetadataCheckpointIdentity { + checkpoint_key: prepared.checkpoint_key.clone(), + image_bytes: prepared.image_bytes, + image_digest: prepared.image_digest, + }; + let proof_key = ObjectKey::new(format!("{}.proof", identity.checkpoint_key)).unwrap(); + assert!(backing.delete(&proof_key).unwrap()); + objects.clear_get_keys(); - let outcome = service.cleanup_pending_objects(10).unwrap(); - assert_eq!(outcome.scanned, 2); - assert_eq!(outcome.records_removed, 2); - assert_eq!(outcome.deleted, 1); - assert!(objects.head(&object).unwrap().is_none()); - assert_eq!( - service - .metadata - .scan_keys(KeyScanRequest { - family: RecordFamily::System, - prefix: object_gc_quarantine_prefix(service.mount), - start_after: None, - limit: 0, - purpose: ReadPurpose::UserStrong, - }) - .unwrap() - .len(), - 1 + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), ); + assert!(matches!( + recovered.restore_metadata_checkpoint(&config, &identity), + Err(MetadError::MetadataArchiveMissingObjectGcFence { checkpoint_key }) + if checkpoint_key == identity.checkpoint_key + )); + assert!(!objects.got_key(&identity.checkpoint_key)); + assert!(recovered.get_attr(InodeId::root()).unwrap().is_none()); } #[test] -fn failover_durability_marker_survives_disable_and_reopen_and_blocks_object_gc() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"ha-victim".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request(name.clone(), "ha-victim", b"victim")) - .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - service.remove_file(InodeId::root(), &name).unwrap(); - - service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "metadata-log", - "shard-0", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) - .unwrap(); - service.disable_sync_metadata_log().unwrap(); - let marker_key = failover_durability_required_key(service.mount); - assert_eq!( - metadata - .get( - RecordFamily::System, - &marker_key, - service.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap() - .0, - FAILOVER_DURABILITY_REQUIRED_MARKER +fn exact_checkpoint_rejects_cross_prefix_or_same_key_digest_mismatch() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing); + let source = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), ); - drop(service); - - let reopened = NoKvFs::open_existing( + source.bootstrap_root(0o755, 1000, 1000).unwrap(); + let config = MetadataArchiveConfig::new("meta/owned", 4); + let prepared = source.prepare_immutable_metadata_backup(&config).unwrap(); + let identity = MetadataCheckpointIdentity { + checkpoint_key: prepared.checkpoint_key.clone(), + image_bytes: prepared.image_bytes, + image_digest: prepared.image_digest, + }; + let recovered = NoKvFs::new( MountId::new(1).unwrap(), - metadata.clone(), + HoltMetadataStore::open_memory().unwrap(), objects.clone(), - 0, - ) - .unwrap(); - let outcome = reopened.cleanup_pending_objects(1).unwrap(); - assert_eq!(outcome.scanned, 1); - assert_eq!(outcome.blocked_by_failover_durability, 1); - assert_eq!(outcome.attempted, 0); - assert_eq!(outcome.records_removed, 0); - assert!(objects.head(&object).unwrap().is_some()); - assert_eq!( - metadata - .scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(reopened.mount), - start_after: None, - version: reopened.read_version().unwrap(), - limit: 1, - purpose: ReadPurpose::UserStrong, - }) - .unwrap() - .len(), - 1 ); + assert!(matches!( + recovered.restore_metadata_checkpoint( + &MetadataArchiveConfig::new("meta/other", 4), + &identity, + ), + Err(MetadError::Codec(message)) if message.contains("does not match archive prefix") + )); + + let mismatched = MetadataCheckpointIdentity { + checkpoint_key: identity.checkpoint_key.clone(), + image_bytes: identity.image_bytes, + image_digest: format!("sha256:{}", "0".repeat(64)), + }; + objects.clear_get_keys(); + assert!(matches!( + recovered.restore_metadata_checkpoint(&config, &mismatched), + Err(MetadError::Codec(message)) if message.contains("does not match archive prefix") + )); + assert!(!objects.got_key(&identity.checkpoint_key)); } #[test] -fn failover_marker_committing_first_invalidates_shared_gc_claim_without_delete() { - let metadata = SnapshotCommitBarrierStore::new(CommandKind::CleanupObjects, 0, 2); - let memory = MemoryObjectStore::new(); - let objects = DeleteAckLostObjectStore::new(memory.clone()); - let cleaner = Arc::new(NoKvFs::new( +fn exact_checkpoint_rejects_tampered_same_key_image_before_install() { + let backing = MemoryObjectStore::new(); + let source = NoKvFs::new( MountId::new(1).unwrap(), - metadata.clone(), - objects.clone(), - )); - cleaner.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"marker-first".to_vec()).unwrap(); - let published = cleaner - .publish_artifact(artifact_request(name.clone(), "marker-first", b"victim")) + HoltMetadataStore::open_memory().unwrap(), + backing.clone(), + ); + source.bootstrap_root(0o755, 1000, 1000).unwrap(); + source + .create_dir_path("/must-not-install", 0o755, 1000, 1000) .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - cleaner.remove_file(InodeId::root(), &name).unwrap(); - let marker = NoKvFs::open_existing( + let config = MetadataArchiveConfig::new("meta/tamper", 4); + let prepared = source.prepare_immutable_metadata_backup(&config).unwrap(); + let identity = MetadataCheckpointIdentity { + checkpoint_key: prepared.checkpoint_key.clone(), + image_bytes: prepared.image_bytes, + image_digest: prepared.image_digest, + }; + let image_key = ObjectKey::new(identity.checkpoint_key.clone()).unwrap(); + let mut tampered = backing.get(&image_key, None).unwrap(); + tampered[0] ^= 0xff; + assert_eq!(tampered.len() as u64, identity.image_bytes); + backing.put(&image_key, tampered).unwrap(); + + let recovered = NoKvFs::new( MountId::new(1).unwrap(), - metadata.clone(), + HoltMetadataStore::open_memory().unwrap(), + backing, + ); + assert!(matches!( + recovered.restore_metadata_checkpoint(&config, &identity), + Err(MetadError::Codec(message)) if message.contains("image digest mismatch") + )); + assert!(recovered.get_attr(InodeId::root()).unwrap().is_none()); +} + +#[test] +fn backup_archive_crash_between_checkpoint_and_pointer_is_consistent() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing.clone()); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), objects.clone(), - 0, - ) - .unwrap(); + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/runs/a.bin", "m-a", b"alpha"); - metadata.arm(1); - let cleanup = { - let cleaner = Arc::clone(&cleaner); - std::thread::spawn(move || cleaner.cleanup_pending_objects(1)) - }; - metadata.wait_until_blocked(); - marker.require_failover_durability().unwrap(); - metadata.release_blocked(); + let config = MetadataArchiveConfig::new("meta/ck", 4); + // First backup completes: CURRENT -> checkpoint #1 (captures only /runs/a.bin). + let good = service.backup_metadata(&config).unwrap(); - let outcome = cleanup.join().unwrap().unwrap(); - assert_eq!(outcome.attempted, 0); - assert_eq!(objects.delete_calls(), 0); - assert!(memory.head(&object).unwrap().is_some()); - assert_eq!(metadata.predicate_failures(), 1); - cleaner.refresh_allocator_state().unwrap(); - let blocked = cleaner.cleanup_pending_objects(1).unwrap(); - assert_eq!(blocked.blocked_by_failover_durability, 1); - assert_eq!(objects.delete_calls(), 0); - let marker_key = failover_durability_required_key(cleaner.mount); + // Add /runs/b.bin, then crash the second backup at the pointer swap: the + // checkpoint object is written, but the CURRENT manifest PUT fails. + publish_path_artifact(&service, "/runs/b.bin", "m-b", b"bravo"); + objects.fail_puts_containing("/CURRENT"); + let err = service.backup_metadata(&config).unwrap_err(); + assert!(matches!(err, MetadError::Object(_))); + assert_eq!(objects.injected_put_failures(), 1); + objects.clear_faults(); + + // CURRENT still names the first, complete checkpoint — never the orphaned + // second one. Restore into a fresh node recovers the pre-crash state. + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + backing.clone(), + ); + let restored = recovered.restore_metadata(&config).unwrap().unwrap(); + assert_eq!(restored.checkpoint_key, good.checkpoint_key); + assert!(recovered.lookup_path("/runs/a.bin").unwrap().is_some()); + assert!( + recovered.lookup_path("/runs/b.bin").unwrap().is_none(), + "restore must not expose the torn (uncommitted) checkpoint" + ); + + // With the fault cleared, the archive recovers forward cleanly. + publish_path_artifact(&service, "/runs/c.bin", "m-c", b"charlie"); + let next = service.backup_metadata(&config).unwrap(); + assert_ne!(next.checkpoint_key, good.checkpoint_key); +} + +#[test] +fn object_gc_converges_under_create_delete_churn() { + let (service, objects) = service_with_objects(); + // Churn: create many small files; delete the even rounds (their blocks must + // be reclaimed) and keep the odd rounds (their blocks must never be deleted). + let mut live_keys = Vec::new(); + for round in 0..20u32 { + let name = DentryName::new(format!("churn-{round}.bin").into_bytes()).unwrap(); + let published = service + .publish_artifact(artifact_request( + name.clone(), + &format!("m{round}"), + b"payload", + )) + .unwrap(); + let body = published.body.clone().unwrap(); + let key = block_key(published.attr.inode, body.generation, 0, 0); + if round % 2 == 0 { + service.remove_file(InodeId::root(), &name).unwrap(); + } else { + live_keys.push(key); + } + } + + // Drive GC to convergence with a small per-iteration limit so the queue is + // drained across several batches rather than one sweep. + let mut total_deleted = 0; + let mut guard = 0; + loop { + let outcome = service.cleanup_pending_objects(4).unwrap(); + total_deleted += outcome.deleted; + if outcome.scanned == 0 { + break; + } + guard += 1; + assert!(guard < 1000, "object GC did not converge"); + } + + // Exactly the 10 deleted files were reclaimed, and the queue is now empty. + assert_eq!(total_deleted, 10); assert_eq!( - metadata - .get( - RecordFamily::System, - &marker_key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap() - .0, - FAILOVER_DURABILITY_REQUIRED_MARKER + service.cleanup_pending_objects(100).unwrap(), + PendingObjectCleanupOutcome::default() ); + // Every kept file's block survived: owns_block_object_key never over-deleted. + for key in &live_keys { + assert!( + objects.head(key).unwrap().is_some(), + "live block was wrongly GC'd: {}", + key.as_str() + ); + } } #[test] -fn shared_gc_claim_committing_first_invalidates_preplanned_marker_cas() { - let metadata = SnapshotCommitBarrierStore::new(CommandKind::CleanupObjects, 0, 2); - let objects = MemoryObjectStore::new(); - let cleaner = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - cleaner.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"claim-first".to_vec()).unwrap(); - let published = cleaner - .publish_artifact(artifact_request(name.clone(), "claim-first", b"victim")) - .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - cleaner.remove_file(InodeId::root(), &name).unwrap(); - let marker = Arc::new( - NoKvFs::open_existing( - MountId::new(1).unwrap(), - metadata.clone(), - objects.clone(), - 0, - ) - .unwrap(), +fn fsck_detects_dangling_block_after_out_of_band_object_loss() { + let (service, objects) = service_with_objects(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let a = publish_path_artifact(&service, "/runs/a.bin", "m-a", b"alpha-body"); + publish_path_artifact(&service, "/runs/b.bin", "m-b", b"bravo-body"); + + // A healthy namespace has no dangling references. + let clean = service.fsck_dangling_blocks(0).unwrap(); + assert!( + clean.is_consistent(), + "unexpected dangling: {:?}", + clean.dangling ); + assert_eq!(clean.files_scanned, 2); + assert!(clean.blocks_checked >= 2); - metadata.arm(1); - let marker_commit = { - let marker = Arc::clone(&marker); - std::thread::spawn(move || marker.require_failover_durability()) - }; - metadata.wait_until_blocked(); - let outcome = cleaner.cleanup_pending_objects(1).unwrap(); - assert_eq!(outcome.deleted, 1); - assert!(objects.head(&object).unwrap().is_none()); - metadata.release_blocked(); - marker_commit.join().unwrap().unwrap(); + // Delete one file's backing object out-of-band: drift that object-first + // ordering cannot prevent once the metadata is already committed. + let body = a.body.clone().unwrap(); + let lost = block_key(a.attr.inode, body.generation, 0, 0); + assert!(objects.delete(&lost).unwrap()); - assert_eq!( - metadata.predicate_failures(), - 1, - "the marker command planned against the old Open claim must not cross the Deleting transition" - ); - assert_eq!( - metadata - .get( - RecordFamily::System, - &failover_durability_required_key(cleaner.mount), - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap() - .0, - FAILOVER_DURABILITY_REQUIRED_MARKER - ); + // fsck flags exactly that reference, and nothing else. + let report = service.fsck_dangling_blocks(0).unwrap(); + assert!(!report.is_consistent()); + assert_eq!(report.dangling.len(), 1); + assert_eq!(report.dangling[0].inode, a.attr.inode.get()); + assert_eq!(report.dangling[0].object_key, lost.as_str()); +} + +/// Set up `/runs/a.bin`, snapshot `/runs` with `lease_ms`, then free the block so +/// it is GC-enqueued *after* the snapshot's read version (i.e. protected while +/// the pin is live). Returns the freed block's object key. +fn snapshot_then_free_block( + service: &NoKvFs, + lease_ms: u64, +) -> (SnapshotPin, ObjectKey) { + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let published = publish_path_artifact(service, "/runs/a.bin", "m-a", b"payload"); + let body = published.body.clone().unwrap(); + let block = block_key(published.attr.inode, body.generation, 0, 0); + let runs = service.resolve_directory_path("/runs").unwrap(); + let pin = service.snapshot_subtree_with_lease(runs, lease_ms).unwrap(); + service.remove_file_path("/runs/a.bin").unwrap(); + (pin, block) } #[test] -fn startup_recovery_keeps_interrupted_claim_closed_under_failover_marker() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"interrupted-ha-victim".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request( - name.clone(), - "interrupted-ha-victim", - b"victim", - )) - .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - service.remove_file(InodeId::root(), &name).unwrap(); - service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "metadata-log", - "shard-0", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) - .unwrap(); - service.disable_sync_metadata_log().unwrap(); - let gc_row = metadata - .scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(service.mount), - start_after: None, - version: service.read_version().unwrap(), - limit: 1, - purpose: ReadPurpose::WritePlanLocal, - }) - .unwrap() - .pop() - .unwrap(); - let claim_key = leave_object_gc_deleting(&service, &gc_row); - drop(service); +fn expired_snapshot_pin_does_not_block_object_gc() { + let (service, objects) = service_with_objects(); + // Lease of 0 ms: the pin is expired the moment GC inspects it. + let (_pin, block) = snapshot_then_free_block(&service, 0); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.blocked_by_snapshots, 0); + assert_eq!(cleanup.deleted, 1); + assert!(objects.head(&block).unwrap().is_none()); +} - let reopened = NoKvFs::open_existing( - MountId::new(1).unwrap(), - metadata.clone(), - objects.clone(), - 0, - ) - .unwrap(); - assert!(matches!( - reopened.recover_object_gc_claim(), - Err(MetadError::ObjectGcRecoveryRequiresIntervention { .. }) - )); - let claim = metadata - .get( - RecordFamily::System, - &claim_key, - reopened.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap(); +#[test] +fn live_snapshot_pin_blocks_object_gc_until_retired() { + let (service, objects) = service_with_objects(); + let (pin, block) = snapshot_then_free_block(&service, 3_600_000); + + // A live pin protects the freed block. + let blocked = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(blocked.blocked_by_snapshots, 1); + assert_eq!(blocked.deleted, 0); + assert!(objects.head(&block).unwrap().is_some()); + + // Retiring it releases the protection. + assert!(service.retire_snapshot(pin.snapshot_id).unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1); + assert!(objects.head(&block).unwrap().is_none()); +} + +#[test] +fn renew_snapshot_rejects_expiry_at_the_deadline() { + let (service, _objects) = service_with_objects(); + service.set_clock_override_ms(1_000); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let runs = service.resolve_directory_path("/runs").unwrap(); + let pin = service.snapshot_subtree_with_lease(runs, 500).unwrap(); + + service.set_clock_override_ms(1_500); assert!(matches!( - decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), - ObjectGcClaim::Deleting { .. } + service.renew_snapshot(pin.snapshot_id, 3_600_000), + Err(MetadError::SnapshotLeaseExpired { + snapshot_id, + lease_expires_unix_ms: 1_500, + now_ms: 1_500, + }) if snapshot_id == pin.snapshot_id )); - assert!(objects.head(&object).unwrap().is_some()); + assert_eq!( - metadata - .scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(reopened.mount), - start_after: None, - version: reopened.read_version().unwrap(), - limit: 1, - purpose: ReadPurpose::UserStrong, - }) - .unwrap() - .len(), - 1 + service + .renew_snapshot(pin.snapshot_id + 9_999, 1_000) + .unwrap(), + SnapshotRenewOutcome::Missing { + snapshot_id: pin.snapshot_id + 9_999, + } ); } #[test] -fn enabling_failover_durability_fences_a_crash_left_deleting_claim_before_recovery() { - let metadata = HoltMetadataStore::open_memory().unwrap(); +fn renew_snapshot_cannot_revive_a_pin_after_gc_crosses_its_expiry() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RenewSnapshot, 1, 2) + .rejecting(CommandKind::RetireSnapshot); let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"pre-marker-interrupted".to_vec()).unwrap(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let name = DentryName::new(b"renew-gc-race.bin".to_vec()).unwrap(); let published = service - .publish_artifact(artifact_request( - name.clone(), - "pre-marker-interrupted", - b"victim", - )) + .publish_artifact(artifact_request(name.clone(), "renew-gc-race", b"payload")) .unwrap(); let body = published.body.as_ref().unwrap(); let object = block_key(published.attr.inode, body.generation, 0, 0); - service.remove_file(InodeId::root(), &name).unwrap(); - let gc_row = metadata - .scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(service.mount), - start_after: None, - version: service.read_version().unwrap(), - limit: 1, - purpose: ReadPurpose::WritePlanLocal, - }) - .unwrap() - .pop() + let pin = service + .snapshot_subtree_with_lease(InodeId::root(), 500) .unwrap(); - let claim_key = leave_object_gc_deleting(&service, &gc_row); - drop(service); + let snapshot_id = pin.snapshot_id; + service.remove_file(InodeId::root(), &name).unwrap(); - let reopened = NoKvFs::open_existing( - MountId::new(1).unwrap(), - metadata.clone(), - objects.clone(), - 0, - ) - .unwrap(); - reopened - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "metadata-log", - "shard-0", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) - .unwrap(); - reopened.disable_sync_metadata_log().unwrap(); - let deleting = metadata - .get( - RecordFamily::System, - &claim_key, - reopened.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap(); - assert!(matches!( - decode_object_gc_claim(reopened.mount, &deleting.0).unwrap(), - ObjectGcClaim::Deleting { .. } - )); + let renew_service = Arc::clone(&service); + let renew = std::thread::spawn(move || renew_service.renew_snapshot(snapshot_id, 10_000)); + store.wait_until_blocked(); + + let deadline_ms = start_ms + 500; + service.set_clock_override_ms(deadline_ms); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.snapshot_reap.conflicted, 1); + assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&object).unwrap().is_none()); + store.release_blocked(); assert!(matches!( - reopened.recover_object_gc_claim(), - Err(MetadError::ObjectGcRecoveryRequiresIntervention { .. }) - )); - let still_deleting = metadata - .get( - RecordFamily::System, - &claim_key, - reopened.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap(); - assert!(matches!( - decode_object_gc_claim(reopened.mount, &still_deleting.0).unwrap(), - ObjectGcClaim::Deleting { .. } + renew.join().unwrap(), + Err(MetadError::SnapshotLeaseExpired { + snapshot_id, + lease_expires_unix_ms, + now_ms, + }) if snapshot_id == pin.snapshot_id + && lease_expires_unix_ms == deadline_ms + && now_ms == deadline_ms )); - assert!(objects.head(&object).unwrap().is_some()); assert_eq!( - metadata - .scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(reopened.mount), - start_after: None, - version: reopened.read_version().unwrap(), - limit: 1, - purpose: ReadPurpose::UserStrong, - }) + service + .snapshot_pin(pin.snapshot_id) .unwrap() - .len(), - 1 + .unwrap() + .lease_expires_unix_ms, + deadline_ms ); } #[test] -fn failover_recovery_does_not_reopen_after_external_delete_completed() { - let metadata = HoltMetadataStore::open_memory().unwrap(); +fn renew_snapshot_is_extend_only_and_never_shortens_protection() { + let (service, _objects) = service_with_objects(); + service.set_clock_override_ms(1_000); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let runs = service.resolve_directory_path("/runs").unwrap(); + let pin = service.snapshot_subtree_with_lease(runs, 1_000).unwrap(); + + // Grant a long lease. + let SnapshotRenewOutcome::Renewed { + pin: long, + extended: true, + } = service.renew_snapshot(pin.snapshot_id, 3_600_000).unwrap() + else { + panic!("expected an extended live pin") + }; + + // A shorter renew must NOT shorten the protection already granted: renew is + // extend-only (Iceberg / S3 Object Lock semantics). Shrinking protection is + // expressed by `retire`, never by a shorter renew silently dropping it. + let SnapshotRenewOutcome::Renewed { + pin: after_short, + extended: false, + } = service.renew_snapshot(pin.snapshot_id, 1_000).unwrap() + else { + panic!("expected an unchanged live pin") + }; + assert_eq!( + after_short.lease_expires_unix_ms, long.lease_expires_unix_ms, + "a shorter renew must never shorten protection" + ); + + // A longer renew still extends protection. + let SnapshotRenewOutcome::Renewed { + pin: after_long, + extended: true, + } = service.renew_snapshot(pin.snapshot_id, 7_200_000).unwrap() + else { + panic!("expected an extended live pin") + }; + assert!( + after_long.lease_expires_unix_ms > long.lease_expires_unix_ms, + "a longer renew extends protection" + ); +} + +#[test] +fn concurrent_snapshot_renewals_preserve_the_longest_successful_lease() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RenewSnapshot, 2, 2); let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + let service = Arc::new(NoKvFs::new(MountId::new(1).unwrap(), store, objects)); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"deleted-before-crash".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request( - name.clone(), - "deleted-before-crash", - b"victim", - )) - .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - service.remove_file(InodeId::root(), &name).unwrap(); - let gc_row = metadata - .scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(service.mount), - start_after: None, - version: service.read_version().unwrap(), - limit: 1, - purpose: ReadPurpose::WritePlanLocal, - }) - .unwrap() - .pop() - .unwrap(); - let claim_key = leave_object_gc_deleting(&service, &gc_row); - assert!(objects.delete(&object).unwrap()); - service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "metadata-log", - "shard-0", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) - .unwrap(); - service.disable_sync_metadata_log().unwrap(); - drop(service); + service.set_clock_override_ms(1_000); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let root = service.resolve_directory_path("/runs").unwrap(); + let pin = service.snapshot_subtree_with_lease(root, 1_000).unwrap(); - let reopened = NoKvFs::open_existing( + let short_service = Arc::clone(&service); + let short = std::thread::spawn(move || short_service.renew_snapshot(pin.snapshot_id, 5_000)); + let long_service = Arc::clone(&service); + let long = std::thread::spawn(move || long_service.renew_snapshot(pin.snapshot_id, 10_000)); + + let short = short.join().unwrap().unwrap(); + let long = long.join().unwrap().unwrap(); + assert!(matches!(short, SnapshotRenewOutcome::Renewed { .. })); + assert!(matches!(long, SnapshotRenewOutcome::Renewed { .. })); + assert_eq!( + service + .snapshot_pin(pin.snapshot_id) + .unwrap() + .unwrap() + .lease_expires_unix_ms, + 11_000 + ); +} + +#[test] +fn sixteen_concurrent_snapshot_renewals_converge_on_the_longest_lease() { + const WRITERS: usize = 16; + let store = + SnapshotCommitBarrierStore::new(CommandKind::RenewSnapshot, WRITERS as u64, WRITERS); + let service = Arc::new(NoKvFs::new( MountId::new(1).unwrap(), - metadata.clone(), - objects.clone(), - 0, - ) - .unwrap(); - assert!(matches!( - reopened.recover_object_gc_claim(), - Err(MetadError::ObjectGcRecoveryRequiresIntervention { .. }) - )); - assert!(objects.head(&object).unwrap().is_none()); - let claim = metadata - .get( - RecordFamily::System, - &claim_key, - reopened.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap(); - assert!(matches!( - decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), - ObjectGcClaim::Deleting { .. } + store, + MemoryObjectStore::new(), )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.set_clock_override_ms(1_000); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let root = service.resolve_directory_path("/runs").unwrap(); + let pin = service.snapshot_subtree_with_lease(root, 1_000).unwrap(); + + let mut writers = Vec::new(); + for index in 0..WRITERS { + let service = Arc::clone(&service); + let lease_ms = 2_000 + index as u64 * 1_000; + writers.push(std::thread::spawn(move || { + service.renew_snapshot(pin.snapshot_id, lease_ms) + })); + } + for writer in writers { + assert!(matches!( + writer.join().unwrap().unwrap(), + SnapshotRenewOutcome::Renewed { .. } + )); + } assert_eq!( - metadata - .scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(reopened.mount), - start_after: None, - version: reopened.read_version().unwrap(), - limit: 1, - purpose: ReadPurpose::UserStrong, - }) + service + .snapshot_pin(pin.snapshot_id) .unwrap() - .len(), - 1 + .unwrap() + .lease_expires_unix_ms, + 18_000 ); } #[test] -fn reopen_missing_claim_is_initialized_before_enabling_failover_durability() { - let metadata = HoltMetadataStore::open_memory().unwrap(); +fn stale_reaper_scan_cannot_delete_a_newer_pin_version() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RetireSnapshot, 1, 2); let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects, + )); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - delete_object_gc_claim(&service); - drop(service); + service.set_clock_override_ms(1_000); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + let root = service.resolve_directory_path("/runs").unwrap(); + let pin = service.snapshot_subtree_with_lease(root, 500).unwrap(); - let reopened = - NoKvFs::open_existing(MountId::new(1).unwrap(), metadata.clone(), objects, 0).unwrap(); - assert!(metadata - .get( - RecordFamily::System, - &object_gc_claim_key(reopened.mount), - reopened.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .is_none()); - reopened - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "metadata-log", - "shard-0", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) - .unwrap(); - let claim = metadata - .get( - RecordFamily::System, - &object_gc_claim_key(reopened.mount), - reopened.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap(); - assert_eq!( - decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), - ObjectGcClaim::Open - ); - assert!(metadata - .get( - RecordFamily::System, - &failover_durability_required_key(reopened.mount), - reopened.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .is_some()); + service.set_clock_override_ms(1_500); + let reaper_service = Arc::clone(&service); + let reaper = std::thread::spawn(move || reaper_service.reclaim_expired_snapshot_pins(100)); + store.wait_until_blocked(); + + // A deterministic clock rewind models a stale reaper candidate whose record + // was replaced before its delete applied. The version fence, not wall time, + // is the invariant under test. + service.set_clock_override_ms(1_400); + assert!(matches!( + service.renew_snapshot(pin.snapshot_id, 10_000).unwrap(), + SnapshotRenewOutcome::Renewed { extended: true, .. } + )); + store.release_blocked(); + + let outcome = reaper.join().unwrap().unwrap(); + assert_eq!(outcome.expired_candidates, 1); + assert_eq!(outcome.reaped, 0); + assert_eq!(outcome.conflicted, 1); + assert!(service.snapshot_pin(pin.snapshot_id).unwrap().is_some()); } #[test] -fn reopen_missing_claim_is_initialized_by_cleanup_recovery() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); +fn one_reaper_conflict_does_not_block_other_expired_pins() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RetireSnapshot, 1, 2); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + MemoryObjectStore::new(), + )); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"legacy-gc-victim".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request( - name.clone(), - "legacy-gc-victim", - b"victim", - )) - .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - service.remove_file(InodeId::root(), &name).unwrap(); - delete_object_gc_claim(&service); - drop(service); + service.set_clock_override_ms(1_000); + service.create_dir_path("/a", 0o755, 1000, 1000).unwrap(); + service.create_dir_path("/b", 0o755, 1000, 1000).unwrap(); + let a = service.resolve_directory_path("/a").unwrap(); + let b = service.resolve_directory_path("/b").unwrap(); + let renewed = service.snapshot_subtree_with_lease(a, 500).unwrap(); + let expired = service.snapshot_subtree_with_lease(b, 500).unwrap(); - let reopened = NoKvFs::open_existing( - MountId::new(1).unwrap(), - metadata.clone(), - objects.clone(), - 0, - ) - .unwrap(); - let outcome = reopened.cleanup_pending_objects(1).unwrap(); - assert_eq!(outcome.deleted, 1); - assert!(objects.head(&object).unwrap().is_none()); - let claim = metadata - .get( - RecordFamily::System, - &object_gc_claim_key(reopened.mount), - reopened.read_version().unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap() - .unwrap(); - assert_eq!( - decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), - ObjectGcClaim::Open - ); + service.set_clock_override_ms(1_500); + let reaper_service = Arc::clone(&service); + let reaper = std::thread::spawn(move || reaper_service.reclaim_expired_snapshot_pins(100)); + store.wait_until_blocked(); + service.set_clock_override_ms(1_400); + assert!(matches!( + service.renew_snapshot(renewed.snapshot_id, 10_000).unwrap(), + SnapshotRenewOutcome::Renewed { extended: true, .. } + )); + store.release_blocked(); + + let outcome = reaper.join().unwrap().unwrap(); + assert_eq!(outcome.expired_candidates, 2); + assert_eq!(outcome.reaped, 1); + assert_eq!(outcome.conflicted, 1); + assert!(service.snapshot_pin(renewed.snapshot_id).unwrap().is_some()); + assert!(service.snapshot_pin(expired.snapshot_id).unwrap().is_none()); } #[test] -fn read_lease_grace_blocks_recent_object_gc_records() { - let (service, objects) = service_with_objects(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let first = service - .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) - .unwrap(); - let old_body = first.body.clone().unwrap(); - let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); - let replaced = service - .replace_artifact(artifact_request( - name.clone(), - "checkpoint/new", - b"new-body", - )) - .unwrap(); - let new_body = replaced.entry.body.clone().unwrap(); - let new_object = block_key(replaced.entry.attr.inode, new_body.generation, 0, 0); - - let blocked = service - .cleanup_pending_objects_with_grace(100, std::time::Duration::from_secs(3_600)) - .unwrap(); - assert_eq!(blocked.scanned, 1); - assert_eq!(blocked.blocked_by_snapshots, 0); - assert_eq!(blocked.blocked_by_read_leases, 1); - assert_eq!(blocked.attempted, 0); - assert_eq!(blocked.records_removed, 0); - assert!(objects.head(&old_object).unwrap().is_some()); - assert!(objects.head(&new_object).unwrap().is_some()); +fn uncontended_reaper_page_uses_one_atomic_commit() { + let (service, _objects) = service_with_objects(); + service.set_clock_override_ms(1_000); + service.create_dir_path("/a", 0o755, 1000, 1000).unwrap(); + service.create_dir_path("/b", 0o755, 1000, 1000).unwrap(); + let a = service.resolve_directory_path("/a").unwrap(); + let b = service.resolve_directory_path("/b").unwrap(); + service.snapshot_subtree_with_lease(a, 500).unwrap(); + service.snapshot_subtree_with_lease(b, 500).unwrap(); + service.set_clock_override_ms(1_500); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 1); - assert_eq!(cleanup.records_removed, 1); - assert!(objects.head(&old_object).unwrap().is_none()); - assert!(objects.head(&new_object).unwrap().is_some()); + let before = service.metadata_store_stats().commit_total; + let outcome = service.reclaim_expired_snapshot_pins(100).unwrap(); + let commits = service.metadata_store_stats().commit_total - before; + assert_eq!(outcome.expired_candidates, 2); + assert_eq!(outcome.reaped, 2); + assert_eq!(outcome.conflicted, 0); + assert_eq!(commits, 1); } #[test] -fn replace_artifact_cleanup_deletes_only_old_generation() { - let (service, objects) = service_with_objects(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let first = service - .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) - .unwrap(); - let old_body = first.body.clone().unwrap(); - let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); - let replaced = service - .replace_artifact(artifact_request( - name.clone(), - "checkpoint/new", - b"new-body", - )) - .unwrap(); - let new_body = replaced.entry.body.clone().unwrap(); - let new_object = block_key(replaced.entry.attr.inode, new_body.generation, 0, 0); - assert!(objects.head(&old_object).unwrap().is_some()); - assert!(objects.head(&new_object).unwrap().is_some()); +fn snapshot_path_operations_reject_a_different_root() { + let (service, _objects) = service_with_objects(); + service.set_clock_override_ms(1_000); + service.create_dir_path("/a", 0o755, 1000, 1000).unwrap(); + service.create_dir_path("/b", 0o755, 1000, 1000).unwrap(); + let root = service.resolve_directory_path("/a").unwrap(); + let pin = service.snapshot_subtree_with_lease(root, 10_000).unwrap(); + + assert!(matches!( + service.stat_path_at_snapshot("/b", pin.snapshot_id, "/"), + Err(MetadError::SnapshotRootMismatch { + snapshot_id, + expected_root, + actual_root, + .. + }) if snapshot_id == pin.snapshot_id && expected_root != root && actual_root == Some(root) + )); + assert!(matches!( + service.renew_snapshot_path("/b", pin.snapshot_id, 20_000), + Err(MetadError::SnapshotRootMismatch { .. }) + )); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 1); - assert_eq!(cleanup.records_removed, 1); - assert!(objects.head(&old_object).unwrap().is_none()); - assert!(objects.head(&new_object).unwrap().is_some()); - assert_eq!( - service.read_artifact(InodeId::root(), &name).unwrap(), - b"new-body" - ); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert!(matches!( + service.stat_path_at_snapshot("/a", pin.snapshot_id, "/"), + Err(MetadError::SnapshotLeaseExpired { + snapshot_id, + lease_expires_unix_ms, + now_ms, + }) if snapshot_id == pin.snapshot_id + && lease_expires_unix_ms == pin.lease_expires_unix_ms + && now_ms == pin.lease_expires_unix_ms + )); } #[test] -fn snapshot_after_replace_does_not_block_old_object_cleanup() { - let (service, objects) = service_with_objects(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let first = service - .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) +fn snapshot_component_reads_are_root_bound_even_when_empty_or_zero_length() { + let (service, _objects) = service_with_objects(); + service.set_clock_override_ms(1_000); + service.create_dir_path("/a", 0o755, 1000, 1000).unwrap(); + service.create_dir_path("/b", 0o755, 1000, 1000).unwrap(); + service + .create_file_path("/a/inside", 0o644, 1000, 1000) .unwrap(); - let old_body = first.body.clone().unwrap(); - let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); - let replaced = service - .replace_artifact(artifact_request( - name.clone(), - "checkpoint/new", - b"new-body", - )) + service + .create_file_path("/b/outside", 0o644, 1000, 1000) .unwrap(); - let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); - - assert_eq!( - service - .read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/checkpoint.bin") - .unwrap(), - b"new-body" - ); - assert!(objects.head(&old_object).unwrap().is_some()); + let pin = service + .snapshot_subtree_path_with_lease("/a", 10_000) + .unwrap(); + let inside = DentryName::new("inside").unwrap(); + let outside = DentryName::new("outside").unwrap(); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.scanned, 1); - assert_eq!(cleanup.blocked_by_snapshots, 0); - assert_eq!(cleanup.deleted, 1); - assert_eq!(cleanup.records_removed, 1); - assert!(objects.head(&old_object).unwrap().is_none()); - assert_eq!( - service.read_artifact(InodeId::root(), &name).unwrap(), - b"new-body" - ); - assert_eq!( - replaced.entry.body.unwrap().generation, - snapshot.read_version - ); + assert!(service + .get_attr_at_snapshot("/a", pin.snapshot_id, std::slice::from_ref(&inside)) + .unwrap() + .is_some()); + assert!(service + .get_attr_at_snapshot("/a", pin.snapshot_id, std::slice::from_ref(&outside)) + .unwrap() + .is_none()); + assert!(matches!( + service.get_attr_at_snapshot("/b", pin.snapshot_id, &[]), + Err(MetadError::SnapshotRootMismatch { .. }) + )); + assert!(matches!( + service.read_file_at_snapshot("/b", pin.snapshot_id, std::slice::from_ref(&outside), 0, 0,), + Err(MetadError::SnapshotRootMismatch { .. }) + )); } #[test] -fn snapshot_preserves_old_artifact_and_blocks_object_gc_until_retired() { - let (service, objects) = service_with_objects(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let first = service - .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) +fn snapshot_ids_are_shard_qualified_and_foreign_ids_fail_as_root_mismatch() { + let source = service().with_shard_index(1); + source + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let old_body = first.body.clone().unwrap(); - let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); - let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + let pin = source.snapshot_subtree_path("/source").unwrap(); + assert_eq!(pin.snapshot_id >> 48, 1); - let replaced = service - .replace_artifact(artifact_request( - name.clone(), - "checkpoint/new", - b"new-body", - )) + let destination = service().with_shard_index(2); + destination + .create_dir_path("/destination", 0o755, 1000, 1000) .unwrap(); - let new_body = replaced.entry.body.clone().unwrap(); - let new_object = block_key(replaced.entry.attr.inode, new_body.generation, 0, 0); - - assert_eq!( - service - .read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/checkpoint.bin") - .unwrap(), - b"old" - ); - assert_eq!( - service - .get_attr_at_snapshot("/", snapshot.snapshot_id, std::slice::from_ref(&name)) - .unwrap(), - Some(first.attr.clone()) - ); - assert_eq!( - service - .read_file_at_snapshot("/", snapshot.snapshot_id, std::slice::from_ref(&name), 0, 3,) - .unwrap(), - b"old" - ); - assert_eq!( - service - .read_dir_plus_at_snapshot("/", snapshot.snapshot_id, &[]) - .unwrap(), - vec![first.clone()] - ); - assert_eq!( - service.read_artifact(InodeId::root(), &name).unwrap(), - b"new-body" - ); - let blocked = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(blocked.scanned, 1); - assert_eq!(blocked.blocked_by_snapshots, 1); - assert_eq!(blocked.attempted, 0); - assert!(objects.head(&old_object).unwrap().is_some()); - assert!(objects.head(&new_object).unwrap().is_some()); - - assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); - assert!(!service.retire_snapshot(snapshot.snapshot_id).unwrap()); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 1); - assert_eq!(cleanup.records_removed, 1); - assert!(objects.head(&old_object).unwrap().is_none()); - assert!(objects.head(&new_object).unwrap().is_some()); + assert!(matches!( + destination.stat_path_at_snapshot("/destination", pin.snapshot_id, "/"), + Err(MetadError::SnapshotRootMismatch { + snapshot_id, + actual_root: None, + actual_shard: 1, + .. + }) if snapshot_id == pin.snapshot_id + )); } #[test] -fn snapshot_path_reads_are_rooted_at_snapshot_subtree_and_support_ranges() { +fn restore_to_fork_reinitializes_a_missing_object_gc_claim_before_its_hold_cas() { let service = service(); - let scope = service - .create_dir_path("/scope", 0o755, 1000, 1000) - .unwrap(); - let nested = service - .create_dir_path("/scope/nested", 0o755, 1000, 1000) - .unwrap(); - let outside = service - .create_dir_path("/outside", 0o755, 1000, 1000) - .unwrap(); - let name = DentryName::new(b"model.bin".to_vec()).unwrap(); - let inside_old = service - .publish_artifact(PublishArtifact { - parent: nested.attr.inode, - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "sha256:inside-old".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "inside-old".to_owned(), - bytes: b"inside-old".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) - .unwrap(); service - .publish_artifact(PublishArtifact { - parent: outside.attr.inode, - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "sha256:outside".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "outside".to_owned(), - bytes: b"outside".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let snapshot = service.snapshot_subtree_path("/scope").unwrap(); service - .replace_artifact(PublishArtifact { - parent: nested.attr.inode, - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "sha256:inside-new".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "inside-new".to_owned(), - bytes: b"inside-new".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) + .create_file_path("/source/empty.txt", 0o644, 1000, 1000) .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); - let root = service - .stat_path_at_snapshot("/scope", snapshot.snapshot_id, "/") - .unwrap() + // Model an upgraded metadata image whose root and snapshot predate the + // durable object-GC claim record. + delete_object_gc_claim(&service); + + let restored = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") .unwrap(); - assert_eq!(root.attr.inode, scope.attr.inode); + assert_eq!(restored.state, RestoreState::Complete); assert_eq!( service - .read_dir_plus_path_at_snapshot("/scope", snapshot.snapshot_id, "/") - .unwrap(), - vec![nested.clone()] + .lookup_path("/destination/empty.txt") + .unwrap() + .unwrap() + .attr + .file_type, + FileType::File ); - let file = service - .stat_path_at_snapshot("/scope", snapshot.snapshot_id, "/nested/model.bin") - .unwrap() + + let retry = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") .unwrap(); - assert_eq!(file.attr.generation, inside_old.attr.generation); - assert_eq!(file.body, inside_old.body); - assert_eq!( - service - .read_file_path_at_snapshot("/scope", snapshot.snapshot_id, "/nested/model.bin", 7, 3,) - .unwrap(), - b"old" + assert_eq!(retry, restored); +} + +#[test] +fn restore_rejects_a_source_path_that_did_not_bind_the_pin_root_at_snapshot_version() { + let metadata = RestorePostApplyFaultStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact(&service, "/source/item.bin", "source/item", b"payload"); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let source_root = service.resolve_directory_path("/source").unwrap(); + service.rename_path("/source", "/renamed").unwrap(); + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/renamed", + snapshot.snapshot_id, + "/destination", ); + assert!(matches!( - service.read_file_path_at_snapshot( - "/scope", + service.restore_subtree_path_to_fork( + "/renamed", snapshot.snapshot_id, - "/outside/model.bin", - 0, - 7, + "/destination", ), - Err(MetadError::NotFound) + Err(MetadError::SnapshotRootMismatch { + snapshot_id, + expected_root, + actual_root: None, + .. + }) if snapshot_id == snapshot.snapshot_id && expected_root == source_root )); -} + assert_eq!(metadata.applied_with_prefix(b"restore-install-hold"), 0); + assert!(service.lookup_path("/destination").unwrap().is_none()); -#[test] -fn snapshot_path_list_pages_include_entries_deleted_after_snapshot() { - let service = service(); - for name in [ - b"a.txt".as_slice(), - b"b.txt".as_slice(), - b"c.txt".as_slice(), + let version = service.read_version().unwrap(); + for key in [ + restore::restore_active_key(service.mount_id()), + restore::restore_operation_key(service.mount_id(), &operation_digest), + restore::restore_destination_claim_key(service.mount_id(), "/destination"), ] { - service - .create_file( - InodeId::root(), - DentryName::new(name.to_vec()).unwrap(), - 0o644, - 1000, - 1000, + assert!(service + .metadata_store() + .get( + RecordFamily::System, + &key, + version, + ReadPurpose::WritePlanLocal, ) - .unwrap(); + .unwrap() + .is_none()); } - let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); - service - .rename( - InodeId::root(), - &DentryName::new(b"b.txt".to_vec()).unwrap(), - InodeId::root(), - DentryName::new(b"z.txt".to_vec()).unwrap(), - ) - .unwrap(); + assert!(service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: Vec::new(), + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .is_empty()); +} + +#[test] +fn restore_to_fork_records_success_failure_retry_and_latency_metrics() { + let service = service(); service - .remove_file( - InodeId::root(), - &DentryName::new(b"c.txt".to_vec()).unwrap(), - ) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); - let first = service - .read_dir_plus_path_at_snapshot_page("/", snapshot.snapshot_id, "/", None, 2) + let restored = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") .unwrap(); - assert_eq!( - first - .entries - .iter() - .map(|entry| entry.dentry.name.as_bytes()) - .collect::>(), - vec![b"a.txt".as_slice(), b"b.txt".as_slice()] - ); - let cursor = first.next_cursor.unwrap(); - assert_eq!(cursor.as_bytes(), b"b.txt"); - - let second = service - .read_dir_plus_path_at_snapshot_page("/", snapshot.snapshot_id, "/", Some(&cursor), 2) + let retry = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") .unwrap(); - assert_eq!(second.entries.len(), 1); - assert_eq!(second.entries[0].dentry.name.as_bytes(), b"c.txt"); - assert!(second.next_cursor.is_none()); + assert_eq!(retry, restored); + assert!(matches!( + service.restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/source",), + Err(MetadError::RestoreDestinationConflict { .. }) + )); + + let stats = service.metadata_service_stats(); + assert_eq!(stats.restore_to_fork_requests_total, 3); + assert_eq!(stats.restore_to_fork_success_total, 2); + assert_eq!(stats.restore_to_fork_failure_total, 1); + assert!(stats.restore_to_fork_elapsed_ns_total >= stats.restore_to_fork_elapsed_ns_max); } #[test] -fn history_cleanup_keeps_snapshot_reads_until_snapshot_retired() { +fn restore_to_fork_applies_initialization_while_the_destination_is_detached() { let service = service(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); service - .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); service - .replace_artifact(artifact_request( - name.clone(), - "checkpoint/new", - b"new-body", - )) + .create_dir_path("/source/metadata", 0o755, 1000, 1000) .unwrap(); - - let retained = service.cleanup_history(100).unwrap(); - assert!(retained.retained_by_snapshots > 0); - assert_eq!( - service - .read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/checkpoint.bin") - .unwrap(), - b"old" + publish_path_artifact( + &service, + "/source/metadata/checkpoints.jsonl", + "source-checkpoints", + b"inherited\n", ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let manifest = br#"{"schema":"nokv.workbench.restore_manifest.v1"}"#.to_vec(); - assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); - let pruned = service.cleanup_history(100).unwrap(); - assert!(pruned.removed > 0); + service + .restore_subtree_path_to_fork_initialized( + "/source", + snapshot.snapshot_id, + "/destination", + RestoreInitialization { + remove_relative_paths: vec!["metadata/checkpoints.jsonl".to_owned()], + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: manifest.clone(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, + ) + .unwrap(); + + assert!(service + .lookup_path("/destination/metadata/checkpoints.jsonl") + .unwrap() + .is_none()); + let metadata = service + .lookup_path("/destination/metadata") + .unwrap() + .unwrap(); assert_eq!( service - .metadata - .get( - RecordFamily::Dentry, - &dentry_key(service.mount, InodeId::root(), &name), - Version::new(snapshot.read_version).unwrap(), - ReadPurpose::Snapshot, + .read_artifact( + metadata.attr.inode, + &DentryName::new("restore_manifest.json").unwrap(), ) .unwrap(), - None + manifest ); } #[test] -fn remove_empty_dir_rejects_non_empty_directory() { +fn restore_resource_preflight_rejects_oversized_initialization_command_without_a_hold() { let service = service(); - let dir = DentryName::new(b"runs".to_vec()).unwrap(); - let child = DentryName::new(b"1".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), dir.clone(), 0o755, 1000, 1000) + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); service - .create_dir(created.attr.inode, child, 0o755, 1000, 1000) + .create_dir_path("/source/metadata", 0o755, 1000, 1000) .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/source", + snapshot.snapshot_id, + "/destination", + ); - let err = service.remove_empty_dir(InodeId::root(), &dir).unwrap_err(); - assert!(matches!(err, MetadError::DirectoryNotEmpty)); - assert!(service - .lookup_plus(InodeId::root(), &dir) - .unwrap() - .is_some()); -} - -#[test] -fn remove_empty_dir_deletes_empty_directory() { - let service = service(); - let dir = DentryName::new(b"runs".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), dir.clone(), 0o755, 1000, 1000) - .unwrap(); + // The canonical initialization is below its 8 MiB request limit, but the + // content type is encoded in both the dentry projection and body summary, + // making the single publish MetadataCommand exceed 8 MiB. + let error = service + .restore_subtree_path_to_fork_initialized( + "/source", + snapshot.snapshot_id, + "/destination", + RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: Vec::new(), + content_type: "x".repeat(4_300_000), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, + ) + .unwrap_err(); + assert!(matches!( + error, + MetadError::RestoreResourceLimit { resource, .. } + if resource == "restore initialization publish bytes" + )); - let removed = service.remove_empty_dir(InodeId::root(), &dir).unwrap(); - assert_eq!(removed, created); + let version = service.read_version().unwrap(); + for key in [ + restore::restore_active_key(service.mount_id()), + restore::restore_operation_key(service.mount_id(), &operation_digest), + restore::restore_destination_claim_key(service.mount_id(), "/destination"), + ] { + assert!( + service + .metadata_store() + .get( + RecordFamily::System, + &key, + version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none(), + "resource preflight must not leave durable restore state" + ); + } assert!(service - .lookup_plus(InodeId::root(), &dir) + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: Vec::new(), + start_after: None, + version, + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap() - .is_none()); - assert!(service.get_attr(created.attr.inode).unwrap().is_none()); + .is_empty()); + assert!(service.lookup_path("/destination").unwrap().is_none()); } #[test] -fn remove_empty_dir_allows_directory_after_last_child_removed() { - let service = service(); - let dir = DentryName::new(b"runs".to_vec()).unwrap(); - let child = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), dir.clone(), 0o755, 1000, 1000) +fn restore_custom_index_command_shape_is_rejected_before_the_durable_hold() { + const OVERSIZED_VALUE_BYTES: usize = 3_000_000; + + let metadata = OversizedPathIndexStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); + publish_path_artifact(&service, "/source/item.bin", "preflight/source", b"x"); service - .publish_artifact(PublishArtifact { - parent: created.attr.inode, - name: child.clone(), - producer: "unit-test".to_owned(), - digest_uri: "sha256:test".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "runs/checkpoint.bin".to_owned(), - bytes: b"payload".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, + .register_namespace_index(NamespaceIndexRegistration { + path: "/source".to_owned(), + fields: vec![NamespaceIndexField { + field: NamespaceFindField::new("budget.value"), + operators: vec![NamespacePredicateOp::Eq], + sortable: false, + facetable: false, + }], + rows: vec![NamespaceIndexRow { + path: "/source/item.bin".to_owned(), + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("budget.value"), + value: NamespacePredicateValue::String("small".to_owned()), + }], + }], }) .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let row_key = path_index_row_key(service.mount_id(), "/source", "/source/item.bin"); + let row = service + .metadata_store() + .get_versioned( + RecordFamily::PathIndex, + &row_key, + service.read_version().unwrap(), + ReadPurpose::Snapshot, + ) + .unwrap() + .unwrap(); + metadata.install_row(ScanItem { + key: row_key, + value: Value(encode_path_index_row(&PathIndexRowRecord { + path: "/source/item.bin".to_owned(), + values: vec![( + "budget.value".to_owned(), + PathIndexValueRecord::String("v".repeat(OVERSIZED_VALUE_BYTES)), + )], + })), + version: row.version, + }); + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/source", + snapshot.snapshot_id, + "/destination", + ); - service.remove_file(created.attr.inode, &child).unwrap(); - let removed = service.remove_empty_dir(InodeId::root(), &dir).unwrap(); - - assert_eq!(removed, created); + for _ in 0..2 { + let error = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap_err(); + assert!(matches!( + error, + MetadError::RestoreResourceLimit { + resource, + limit: 8_388_608, + actual, + } if resource == "restore index materialization batch bytes" && actual > 8_388_608 + )); + } + assert_eq!(metadata.applied_with_prefix(b"restore-install-hold"), 0); + assert_eq!(metadata.applied_with_prefix(b"restore-begin-discard"), 0); + let version = service.read_version().unwrap(); + for key in [ + restore::restore_active_key(service.mount_id()), + restore::restore_operation_key(service.mount_id(), &operation_digest), + restore::restore_destination_claim_key(service.mount_id(), "/destination"), + ] { + assert!(service + .metadata_store() + .get( + RecordFamily::System, + &key, + version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + } assert!(service - .lookup_plus(InodeId::root(), &dir) + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: Vec::new(), + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap() - .is_none()); + .is_empty()); + assert!(service.lookup_path("/destination").unwrap().is_none()); } #[test] -fn rename_moves_dentry_without_changing_inode() { - let service = service(); - let old_name = DentryName::new(b"old".to_vec()).unwrap(); - let new_name = DentryName::new(b"new".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), old_name.clone(), 0o755, 1000, 1000) - .unwrap(); +fn restore_canonical_index_command_shape_is_rejected_before_the_durable_hold() { + const LARGE_BODY_FIELD_BYTES: usize = 4_193_000; - let renamed = service - .rename( - InodeId::root(), - &old_name, - InodeId::root(), - new_name.clone(), - ) + let metadata = OversizedPathIndexStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - assert_eq!(renamed.attr.inode, created.attr.inode); - assert!(service - .lookup_plus(InodeId::root(), &old_name) + let published = publish_path_artifact( + &service, + "/source/item.bin", + "canonical-preflight/source", + b"x", + ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let version = service.read_version().unwrap(); + let source = service.resolve_directory_path("/source").unwrap(); + let name = DentryName::new("item.bin").unwrap(); + let dentry_key = dentry_key(service.mount_id(), source, &name); + let path_components = parse_absolute_path("/source/item.bin").unwrap(); + let path_key = path_index_key(service.mount_id(), &path_components); + let generation = published.body.as_ref().unwrap().generation; + let body_key = chunk_manifest_key( + service.mount_id(), + published.attr.inode, + generation, + BODY_SUMMARY_CHUNK_INDEX, + ); + let dentry_version = service + .metadata_store() + .get_versioned( + RecordFamily::Dentry, + &dentry_key, + version, + ReadPurpose::Snapshot, + ) .unwrap() - .is_none()); - assert_eq!( - service.lookup_plus(InodeId::root(), &new_name).unwrap(), - Some(renamed) + .unwrap() + .version; + let path_version = service + .metadata_store() + .get_versioned( + RecordFamily::PathIndex, + &path_key, + version, + ReadPurpose::Snapshot, + ) + .unwrap() + .unwrap() + .version; + let body_version = service + .metadata_store() + .get_versioned( + RecordFamily::ChunkManifest, + &body_key, + version, + ReadPurpose::Snapshot, + ) + .unwrap() + .unwrap() + .version; + let mut body = published.body.clone().unwrap(); + body.producer = "p".repeat(LARGE_BODY_FIELD_BYTES); + let projection = DentryProjection { + dentry: published.dentry, + attr: published.attr, + body: Some(body.clone()), + }; + for (family, key, value, row_version) in [ + ( + RecordFamily::Dentry, + dentry_key, + encode_dentry_projection(&projection), + dentry_version, + ), + ( + RecordFamily::PathIndex, + path_key, + encode_dentry_projection(&projection), + path_version, + ), + ( + RecordFamily::ChunkManifest, + body_key, + encode_body_descriptor(&body), + body_version, + ), + ] { + metadata.install_override( + family, + ScanItem { + key, + value: Value(value), + version: row_version, + }, + ); + } + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/source", + snapshot.snapshot_id, + "/destination", ); + + let error = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap_err(); + assert!(matches!( + error, + MetadError::RestoreResourceLimit { + resource, + limit: 8_388_608, + actual, + } if resource == "restore index materialization batch bytes" && actual > 8_388_608 + )); + assert_eq!(metadata.applied_with_prefix(b"restore-install-hold"), 0); + let version = service.read_version().unwrap(); + for key in [ + restore::restore_active_key(service.mount_id()), + restore::restore_operation_key(service.mount_id(), &operation_digest), + restore::restore_destination_claim_key(service.mount_id(), "/destination"), + ] { + assert!(service + .metadata_store() + .get( + RecordFamily::System, + &key, + version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + } + assert!(service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: Vec::new(), + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .is_empty()); } #[test] -fn rename_replace_returns_replaced_file_body() { - let service = service(); - let source_name = DentryName::new(b"stage".to_vec()).unwrap(); - let final_name = DentryName::new(b"final".to_vec()).unwrap(); - let source = service - .publish_artifact(artifact_request(source_name.clone(), "stage", b"new")) - .unwrap(); - let old = service - .publish_artifact(artifact_request(final_name.clone(), "final-old", b"old")) +fn restore_initialization_member_is_packed_into_canonical_index_preflight() { + const SOURCE_FILES: usize = 62; + const LARGE_CONTENT_TYPE_BYTES: usize = 4_190_000; + + let metadata = RestorePostApplyFaultStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); + for index in 0..SOURCE_FILES { + publish_path_artifact( + &service, + &format!("/source/item-{index:02}.bin"), + &format!("canonical-init/{index:02}"), + b"x", + ); + } + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/source", + snapshot.snapshot_id, + "/destination", + ); - let result = service - .rename_replace( - InodeId::root(), - &source_name, - InodeId::root(), - final_name.clone(), + let error = service + .restore_subtree_path_to_fork_initialized( + "/source", + snapshot.snapshot_id, + "/destination", + RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![RestoreInitializationFile { + relative_path: "new.bin".to_owned(), + bytes: Vec::new(), + content_type: "x".repeat(LARGE_CONTENT_TYPE_BYTES), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, ) - .unwrap(); - assert_eq!(result.entry.attr.inode, source.attr.inode); - assert_eq!(result.replaced, Some(old.clone())); + .unwrap_err(); + assert!(matches!( + error, + MetadError::RestoreResourceLimit { + resource, + limit: 8_388_608, + actual, + } if resource == "restore index materialization batch bytes" && actual > 8_388_608 + )); + assert_eq!(metadata.applied_with_prefix(b"restore-install-hold"), 0); + let version = service.read_version().unwrap(); + for key in [ + restore::restore_active_key(service.mount_id()), + restore::restore_operation_key(service.mount_id(), &operation_digest), + restore::restore_destination_claim_key(service.mount_id(), "/destination"), + ] { + assert!(service + .metadata_store() + .get( + RecordFamily::System, + &key, + version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + } assert!(service - .lookup_plus(InodeId::root(), &source_name) + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: Vec::new(), + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap() - .is_none()); - assert_eq!( - service.lookup_plus(InodeId::root(), &final_name).unwrap(), - Some(result.entry) - ); - assert!(service.get_attr(old.attr.inode).unwrap().is_none()); + .is_empty()); } #[test] -fn watch_replay_returns_typed_events_after_cursor() { - let service = service(); - let cursor = service.watch_subtree(InodeId::root()).unwrap(); - let name = DentryName::new(b"runs".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), name.clone(), 0o755, 1000, 1000) - .unwrap(); - - let events = service.replay_watch(InodeId::root(), cursor, 100).unwrap(); - assert_eq!(events.len(), 1); - assert_eq!(events[0].event.kind, WatchEventKind::Create); - assert_eq!(events[0].event.parent, Some(InodeId::root())); - assert_eq!(events[0].event.name, Some(name.clone())); - assert_eq!(events[0].event.inode, created.attr.inode); +fn restore_large_initialization_projection_passes_individual_command_preflight() { + const LARGE_CONTENT_TYPE_BYTES: usize = 4_190_000; - let next_name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let metadata = RestorePostApplyFaultStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); service - .publish_artifact(artifact_request( - next_name.clone(), - "checkpoint.bin", - b"body", - )) - .unwrap(); - let resumed = service - .replay_watch(InodeId::root(), events[0].cursor, 100) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - assert_eq!(resumed.len(), 1); - assert_eq!(resumed[0].event.kind, WatchEventKind::PublishArtifact); - assert_eq!(resumed[0].event.name, Some(next_name)); -} - -#[test] -fn watch_replay_resumes_from_cursor_without_scanning_old_events() { - let service = service(); - let cursor = service.watch_subtree(InodeId::root()).unwrap(); - for name in ["a", "b", "c"] { - service - .create_dir( - InodeId::root(), - DentryName::new(name.as_bytes().to_vec()).unwrap(), - 0o755, - 1000, - 1000, - ) - .unwrap(); - } + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + metadata.fail_after_apply(b"restore-install-hold", 0); - let before_first = service.metadata_store_stats(); - let first = service.replay_watch(InodeId::root(), cursor, 1).unwrap(); - let after_first = service.metadata_store_stats(); - assert_eq!(first.len(), 1); - assert_eq!( - first[0].event.name.as_ref().map(DentryName::as_bytes), - Some(b"a".as_slice()) - ); - assert_eq!( - after_first.scan_key_visited_total - before_first.scan_key_visited_total, - 1 - ); - assert_eq!( - after_first.scan_key_returned_total - before_first.scan_key_returned_total, - 1 - ); + let error = service + .restore_subtree_path_to_fork_initialized( + "/source", + snapshot.snapshot_id, + "/destination", + RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![RestoreInitializationFile { + relative_path: "new.bin".to_owned(), + bytes: Vec::new(), + content_type: "x".repeat(LARGE_CONTENT_TYPE_BYTES), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, + ) + .unwrap_err(); - let before_second = service.metadata_store_stats(); - let second = service - .replay_watch(InodeId::root(), first[0].cursor, 1) - .unwrap(); - let after_second = service.metadata_store_stats(); - assert_eq!(second.len(), 1); - assert_eq!( - second[0].event.name.as_ref().map(DentryName::as_bytes), - Some(b"b".as_slice()) - ); - assert_eq!( - after_second.scan_key_visited_total - before_second.scan_key_visited_total, - 1 - ); - assert_eq!( - after_second.scan_key_returned_total - before_second.scan_key_returned_total, - 1 - ); + assert!(matches!( + error, + MetadError::Metadata(MetadataError::Backend(message)) + if message == "injected restore crash after durable apply" + )); + assert_eq!(metadata.applied_with_prefix(b"restore-install-hold"), 1); } #[test] -fn rename_replay_notifies_old_and_new_parent_scopes() { +fn restore_indexes_preserve_historical_snapshot_across_publish_rename_delete_and_release() { let service = service(); - let old_parent_name = DentryName::new(b"old-parent".to_vec()).unwrap(); - let new_parent_name = DentryName::new(b"new-parent".to_vec()).unwrap(); - let old_parent = service - .create_dir(InodeId::root(), old_parent_name, 0o755, 1000, 1000) + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let new_parent = service - .create_dir(InodeId::root(), new_parent_name, 0o755, 1000, 1000) + publish_path_artifact(&service, "/source/a.bin", "source/a", b"a-v1"); + publish_path_artifact(&service, "/source/b.bin", "source/b", b"b-v1"); + service + .register_namespace_index(NamespaceIndexRegistration { + path: "/source".to_owned(), + fields: vec![NamespaceIndexField { + field: NamespaceFindField::new("run.kind"), + operators: vec![NamespacePredicateOp::Eq], + sortable: true, + facetable: true, + }], + rows: vec![ + NamespaceIndexRow { + path: "/source/a.bin".to_owned(), + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("run.kind"), + value: NamespacePredicateValue::String("a".to_owned()), + }], + }, + NamespaceIndexRow { + path: "/source/b.bin".to_owned(), + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("run.kind"), + value: NamespacePredicateValue::String("b".to_owned()), + }], + }, + ], + }) .unwrap(); - let file_name = DentryName::new(b"artifact".to_vec()).unwrap(); - let source = service - .create_file(old_parent.attr.inode, file_name.clone(), 0o644, 1000, 1000) + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/restored") .unwrap(); - let old_cursor = service.watch_subtree(old_parent.attr.inode).unwrap(); - let new_cursor = service.watch_subtree(new_parent.attr.inode).unwrap(); + let (restored_operation, _) = load_completed_restore_operation_and_seal( + &service, + "/source", + source_snapshot.snapshot_id, + "/restored", + ); - service - .rename( - old_parent.attr.inode, - &file_name, - new_parent.attr.inode, - file_name.clone(), - ) + let restored_root = service.resolve_directory_path("/restored").unwrap(); + let indexed = service + .list_indexed_path_page("/restored", None, 10) + .unwrap(); + assert_eq!( + indexed + .entries + .iter() + .map(|entry| entry.dentry.name.as_bytes().to_vec()) + .collect::>(), + vec![b"a.bin".to_vec(), b"b.bin".to_vec()] + ); + let found = service + .find_paths(NamespaceFindRequest { + path: "/restored".to_owned(), + predicates: vec![NamespacePredicate { + field: NamespaceFindField::new("run.kind"), + op: NamespacePredicateOp::Eq, + value: Some(NamespacePredicateValue::String("a".to_owned())), + }], + sort: Vec::new(), + include: Vec::new(), + facets: Vec::new(), + cursor: None, + limit: 10, + }) .unwrap(); + assert_eq!(found.match_count, 1); + assert_eq!(found.matches[0].path, "/restored/a.bin"); - let old_events = service - .replay_watch(old_parent.attr.inode, old_cursor, 100) + let historical = service.snapshot_subtree_path("/restored").unwrap(); + republish_path_artifact(&service, restored_root, "a.bin", "restored/a-v2", b"a-v2"); + service + .rename_path("/restored/a.bin", "/restored/renamed.bin") .unwrap(); - assert_eq!(old_events.len(), 1); - assert_eq!(old_events[0].event.kind, WatchEventKind::Remove); - assert_eq!(old_events[0].event.inode, source.attr.inode); + service.remove_file_path("/restored/b.bin").unwrap(); - let new_events = service - .replay_watch(new_parent.attr.inode, new_cursor, 100) + service + .restore_subtree_path_to_fork("/restored", historical.snapshot_id, "/historical") .unwrap(); - assert_eq!(new_events.len(), 1); - assert_eq!(new_events[0].event.kind, WatchEventKind::Rename); - assert_eq!(new_events[0].event.name, Some(file_name)); - assert_eq!(new_events[0].event.inode, source.attr.inode); -} + assert!(service.lookup_path("/historical/a.bin").unwrap().is_some()); + assert!(service.lookup_path("/historical/b.bin").unwrap().is_some()); + assert!(service + .lookup_path("/historical/renamed.bin") + .unwrap() + .is_none()); + let found = service + .find_paths(NamespaceFindRequest { + path: "/historical".to_owned(), + predicates: vec![NamespacePredicate { + field: NamespaceFindField::new("run.kind"), + op: NamespacePredicateOp::Eq, + value: Some(NamespacePredicateValue::String("b".to_owned())), + }], + sort: Vec::new(), + include: Vec::new(), + facets: Vec::new(), + cursor: None, + limit: 10, + }) + .unwrap(); + assert_eq!(found.match_count, 1); + assert_eq!(found.matches[0].path, "/historical/b.bin"); -#[test] -fn watch_replay_survives_service_reopen() { - let dir = tempfile::tempdir().unwrap(); - let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let cursor = service.watch_subtree(InodeId::root()).unwrap(); - let name = DentryName::new(b"runs".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), name.clone(), 0o755, 1000, 1000) + service.remove_file_path("/restored/renamed.bin").unwrap(); + service.remove_empty_dir_path("/restored").unwrap(); + assert!(service + .restore_indexed_children_page( + restored_root, + None, + 10, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .entries + .is_empty()); + let snapshot_version = Version::new(historical.read_version).unwrap(); + let historical_indexed = service + .restore_indexed_children_page( + restored_root, + None, + 10, + snapshot_version, + ReadPurpose::Snapshot, + ) .unwrap(); - drop(service); + assert_eq!(historical_indexed.entries.len(), 2); + let historical_custom = service + .restore_custom_index_at_path( + "/restored", + restored_root, + snapshot_version, + ReadPurpose::Snapshot, + ) + .unwrap() + .unwrap(); + assert_eq!(historical_custom.rows.len(), 2); - let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - let events = reopened.replay_watch(InodeId::root(), cursor, 100).unwrap(); - assert_eq!(events.len(), 1); - assert_eq!(events[0].event.kind, WatchEventKind::Create); - assert_eq!(events[0].event.name, Some(name)); - assert_eq!(events[0].event.inode, created.attr.inode); + assert!(service.retire_snapshot(historical.snapshot_id).unwrap()); + assert!(service + .retire_snapshot(source_snapshot.snapshot_id) + .unwrap()); + for _ in 0..128 { + service.cleanup_restore_releases(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + } + let metrics = service.restore_metrics().unwrap(); + assert_eq!(metrics.releasing, 0); + for keyspace in ["index_source_member", "index_mvcc_source_member"] { + let mut prefix = restore_index::restore_index_private_keyspaces(service.mount_id()) + .into_iter() + .find_map(|(name, prefix)| (name == keyspace).then_some(prefix)) + .unwrap(); + prefix.extend_from_slice(&restored_operation.ref_set_id.to_be_bytes()); + assert!( + service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix, + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .is_empty(), + "released restore left {keyspace} rows" + ); + } } #[test] -fn open_existing_recovers_inode_and_version_allocators() { - let dir = tempfile::tempdir().unwrap(); - let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let first = service - .create_dir( - InodeId::root(), - DentryName::new(b"first".to_vec()).unwrap(), - 0o755, - 1000, - 1000, - ) +fn restore_release_discovers_and_reclaims_post_attach_namespace() { + let (service, objects) = service_with_objects(); + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - drop(service); + publish_path_artifact( + &service, + "/source/original.bin", + "restore/original", + b"original bytes", + ); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + let restored = service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/destination") + .unwrap(); + assert!(service + .retire_snapshot(source_snapshot.snapshot_id) + .unwrap()); - let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - let second = reopened - .create_dir( - InodeId::root(), - DentryName::new(b"second".to_vec()).unwrap(), - 0o755, + let dynamic_dir = service + .create_dir_path("/destination/new", 0o755, 1000, 1000) + .unwrap(); + let dynamic_file = publish_path_artifact( + &service, + "/destination/new/payload.bin", + "restore/dynamic-payload", + b"post-attach bytes", + ); + let checkpoint = service + .publish_checkpoint( + dynamic_dir.attr.inode, + vec![CheckpointShard { + name: DentryName::new("checkpoint.bin").unwrap(), + bytes: b"post-attach checkpoint bytes".to_vec(), + }], 1000, 1000, ) .unwrap(); - assert!(second.attr.inode > first.attr.inode); - assert!(second.attr.generation > first.attr.generation); -} - -#[test] -fn refresh_allocator_state_advances_live_read_version_after_external_commit() { - let dir = tempfile::tempdir().unwrap(); - let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); - let original = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - original.bootstrap_root(0o755, 1000, 1000).unwrap(); - - let external = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - let external_file = external - .create_file_path("/external.bin", 0o644, 1000, 1000) - .unwrap(); - assert!(original.stat_path("/external.bin").unwrap().is_none()); - - original.refresh_allocator_state().unwrap(); - let visible = original - .stat_path("/external.bin") + let checkpoint_file = service + .lookup_path("/destination/new/checkpoint.bin") .unwrap() - .expect("external commit should be visible after refresh"); - assert_eq!(visible.attr, external_file.attr); - let local_file = original - .create_file_path("/after-refresh.bin", 0o644, 1000, 1000) .unwrap(); - assert!(local_file.attr.inode > external_file.attr.inode); - assert!(local_file.attr.generation > external_file.attr.generation); -} + assert_eq!(checkpoint.shards[0].1, checkpoint_file.attr.inode); + service + .set_xattr( + dynamic_file.attr.inode, + b"user.restore-test", + b"present".to_vec(), + XattrSetMode::Create, + ) + .unwrap(); + let dynamic_body = dynamic_file.body.as_ref().unwrap(); + let dynamic_object = block_key(dynamic_file.attr.inode, dynamic_body.generation, 0, 0); + let checkpoint_body = checkpoint_file.body.as_ref().unwrap(); + let checkpoint_object = block_key(checkpoint_file.attr.inode, checkpoint_body.generation, 0, 0); + assert!(objects.head(&dynamic_object).unwrap().is_some()); + assert!(objects.head(&checkpoint_object).unwrap().is_some()); -#[test] -fn open_existing_recovers_after_dentry_only_rename() { - let dir = tempfile::tempdir().unwrap(); - let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let old_name = DentryName::new(b"old".to_vec()).unwrap(); - let new_name = DentryName::new(b"new".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), old_name.clone(), 0o755, 1000, 1000) + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/source", + source_snapshot.snapshot_id, + "/destination", + ); + let operation = restore::decode_restore_operation( + &service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_operation_key(service.mount_id(), &operation_digest), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap() + .0, + ) + .unwrap(); + for inode in [ + dynamic_dir.attr.inode, + dynamic_file.attr.inode, + checkpoint_file.attr.inode, + ] { + let member = service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_staging_member_key( + service.mount_id(), + operation.ref_set_id, + inode, + ), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let member = restore::decode_restore_staging_member(&member.0).unwrap(); + assert!(member.source_inode.is_none()); + assert!(!member.canonical_index_complete); + } + + service + .create_dir_path("/replacement", 0o755, 1000, 1000) .unwrap(); - let renamed = service - .rename( - InodeId::root(), - &old_name, - InodeId::root(), - new_name.clone(), + service + .rename_replace_path("/replacement", "/destination") + .unwrap(); + assert_eq!(service.restore_metrics().unwrap().releasing, 1); + assert!(service.restore_staging_slow_path_enabled()); + assert!(service + .get_attr(restored.destination_root) + .unwrap() + .is_none()); + assert!(service.get_attr(dynamic_file.attr.inode).unwrap().is_none()); + for _ in 0..512 { + service.cleanup_restore_releases(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); + assert!(!service.restore_staging_slow_path_enabled()); + assert!(service + .metadata_store() + .get( + RecordFamily::Inode, + &inode_key(service.mount_id(), dynamic_dir.attr.inode), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + assert!(service + .metadata_store() + .get( + RecordFamily::Inode, + &inode_key(service.mount_id(), dynamic_file.attr.inode), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + assert!(service + .metadata_store() + .get( + RecordFamily::Inode, + &inode_key(service.mount_id(), checkpoint_file.attr.inode), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, ) - .unwrap(); - assert_eq!(renamed.attr.inode, created.attr.inode); - drop(service); - - let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - assert!(reopened - .lookup_plus(InodeId::root(), &old_name) .unwrap() .is_none()); - assert_eq!( - reopened.lookup_plus(InodeId::root(), &new_name).unwrap(), - Some(renamed) - ); - assert_eq!(reopened.read_dir_plus(InodeId::root()).unwrap().len(), 1); -} - -#[test] -fn open_existing_does_not_reuse_removed_inode() { - let dir = tempfile::tempdir().unwrap(); - let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let first_name = DentryName::new(b"first".to_vec()).unwrap(); - let second_name = DentryName::new(b"second".to_vec()).unwrap(); - let first = service - .create_file(InodeId::root(), first_name.clone(), 0o644, 1000, 1000) - .unwrap(); - service.remove_file(InodeId::root(), &first_name).unwrap(); - drop(service); - - let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - let second = reopened - .create_file(InodeId::root(), second_name.clone(), 0o644, 1000, 1000) - .unwrap(); - assert!(second.attr.inode > first.attr.inode); - assert!(second.attr.generation > first.attr.generation); - assert!(reopened - .lookup_plus(InodeId::root(), &first_name) + assert!(service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Dentry, + prefix: dentry_prefix(service.mount_id(), dynamic_dir.attr.inode), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .is_empty()); + assert!(service + .metadata_store() + .get( + RecordFamily::PathIndex, + &path_index_key( + service.mount_id(), + &parse_absolute_path("/destination/new/payload.bin").unwrap(), + ), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + assert!(service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Xattr, + prefix: xattr_prefix(service.mount_id(), dynamic_file.attr.inode), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .is_empty()); + assert!(service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_staging_inode_key(service.mount_id(), dynamic_file.attr.inode,), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) .unwrap() .is_none()); - assert_eq!( - reopened.lookup_plus(InodeId::root(), &second_name).unwrap(), - Some(second) - ); -} - -#[test] -fn pending_object_gc_survives_service_reopen() { - let dir = tempfile::tempdir().unwrap(); - let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request(name.clone(), "artifact.bin", b"old")) - .unwrap(); - let body = published.body.clone().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - service.remove_file(InodeId::root(), &name).unwrap(); - drop(service); - let reopened = - NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects.clone(), 0).unwrap(); - let cleanup = reopened.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 1); - assert_eq!(cleanup.records_removed, 1); - assert!(objects.head(&object).unwrap().is_none()); + for _ in 0..64 { + service.cleanup_pending_objects(64).unwrap(); + if objects.head(&dynamic_object).unwrap().is_none() + && objects.head(&checkpoint_object).unwrap().is_none() + { + break; + } + } + assert!(objects.head(&dynamic_object).unwrap().is_none()); + assert!(objects.head(&checkpoint_object).unwrap().is_none()); + assert_ne!(restored.destination_root, dynamic_dir.attr.inode); } #[test] -fn snapshot_pin_survives_service_reopen() { - let dir = tempfile::tempdir().unwrap(); - let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); - drop(service); - - let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - assert_eq!( - reopened.snapshot_pin(snapshot.snapshot_id).unwrap(), - Some(snapshot) - ); - assert_eq!(reopened.metadata_store_stats().active_snapshot_pin_total, 1); -} +fn restore_release_pages_fragmented_manifest_across_byte_budget_ack_loss_and_reopen() { + const BLOCK_COUNT: usize = 4_200; + const ITEM_BUDGET_BLOCK_CAP: u64 = 4_087; -#[test] -fn failed_publish_returns_staged_objects_for_cleanup_and_does_not_reuse_identity() { - let dir = tempfile::tempdir().unwrap(); + let metadata = OversizedManifestStore::new(); let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); - let first = service - .publish_artifact(artifact_request(name.clone(), "first", b"first")) + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let err = service - .publish_artifact(artifact_request(name.clone(), "duplicate", b"duplicate")) - .unwrap_err(); - let staged = match err { - MetadError::PublishArtifactFailed { source, staged } => { - assert!(matches!( - *source, - MetadError::Metadata(MetadataError::PredicateFailed) - )); - staged - } - err => panic!("unexpected publish error: {err:?}"), - }; - assert_eq!(staged.len(), 1); - for object in staged.objects() { - assert!(objects.head(&object.key).unwrap().is_some()); - } - assert_eq!( - service.lookup_plus(InodeId::root(), &name).unwrap(), - Some(first.clone()) + let source_file = publish_path_artifact( + &service, + "/source/fragments.bin", + "restore/fragmented-release", + b"source", ); - - let cleanup = service.cleanup_staged_objects(&staged).unwrap(); - assert_eq!(cleanup.attempted, staged.len()); - assert_eq!(cleanup.deleted, staged.len()); - assert_eq!(cleanup.missing, 0); - for object in staged.objects() { - assert!(objects.head(&object.key).unwrap().is_none()); - } - drop(service); - - let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - let next_name = DentryName::new(b"next.bin".to_vec()).unwrap(); - let next = reopened - .publish_artifact(artifact_request(next_name, "next", b"next")) + let source_body = source_file.body.as_ref().unwrap(); + let source_object = block_key(source_file.attr.inode, source_body.generation, 0, 0); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") .unwrap(); - - assert!(next.attr.inode.get() > first.attr.inode.get() + 1); - assert!(next.attr.generation > first.attr.generation + 1); -} - -fn dname(raw: &[u8]) -> DentryName { - DentryName::new(raw.to_vec()).unwrap() -} - -fn block_count_for(objects: &MemoryObjectStore, inode: InodeId, generation: u64) -> usize { - // Count the published blocks the base file owns under its (inode, generation). - let mut count = 0; - let mut block = 0; - while objects - .head(&block_key(inode, generation, 0, block)) - .unwrap() - .is_some() - { - count += 1; - block += 1; - } - count -} - -#[test] -fn clone_subtree_shares_base_blocks_diverges_on_write_and_keeps_gc_safe() { - let (service, objects) = service_with_objects(); - // 1. Base namespace: /base with files a ("AAA..") and b ("BBB.."). - let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); - let a = publish_path_artifact(&service, "/base/a", "base/a", &vec![b'A'; 4096]); - let b = publish_path_artifact(&service, "/base/b", "base/b", &vec![b'B'; 4096]); - let a_gen = a.body.as_ref().unwrap().generation; - let b_gen = b.body.as_ref().unwrap().generation; - let a_block = block_key(a.attr.inode, a_gen, 0, 0); - let b_block = block_key(b.attr.inode, b_gen, 0, 0); - assert!(objects.head(&a_block).unwrap().is_some()); - assert!(objects.head(&b_block).unwrap().is_some()); - let objects_after_base = objects.object_count(); - - // 2. Writable O(1)-ish fork of /base. - let fork = service.clone_subtree_path("/base").unwrap(); - assert_ne!(fork.root, base.attr.inode); - - // 3. Sharing: the fork sees the base content, with NO duplicate blocks. - let fork_a = service - .lookup_plus(fork.root, &dname(b"a")) + let operation = durable_restore_operation(&service, snapshot.snapshot_id); + let destination_file = service + .lookup_path("/destination/fragments.bin") .unwrap() .unwrap(); - let fork_b = service - .lookup_plus(fork.root, &dname(b"b")) + let generation = destination_file.body.as_ref().unwrap().generation; + let manifest_key = chunk_manifest_key( + service.mount_id(), + destination_file.attr.inode, + generation, + 0, + ); + let manifest_item = service + .metadata_store() + .get_versioned( + RecordFamily::ChunkManifest, + &manifest_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) .unwrap() .unwrap(); - assert_ne!( - fork_a.attr.inode, a.attr.inode, - "fork must use a fresh inode" - ); - // Shared files keep the source's content generation (the CoW sharing signal). - assert_eq!(fork_a.attr.generation, a_gen); - assert_eq!(fork_b.attr.generation, b_gen); - assert_eq!(fork_b.body.as_ref().unwrap().generation, b_gen); - assert_eq!( - service.read_artifact(fork.root, &dname(b"a")).unwrap(), - vec![b'A'; 4096] - ); - assert_eq!( - service.read_artifact(fork.root, &dname(b"b")).unwrap(), - vec![b'B'; 4096] - ); - // Zero-copy: clone added metadata only, not object blocks. - assert_eq!( - objects.object_count(), - objects_after_base, - "clone must share base blocks, not copy them" - ); - // The fork's a/b manifests reference the SAME object keys as the base. - assert_eq!( - service - .read_file_plan(fork_a.attr.inode, fork_a.attr.generation, 0, 4096) - .unwrap() - .blocks[0] - .object_key, - a_block.as_str() - ); + let long_digest = format!("sha256:{}", "a".repeat(2_300)); + let mut object_keys = Vec::with_capacity(BLOCK_COUNT); + let mut blocks = Vec::with_capacity(BLOCK_COUNT); + for index in 0..BLOCK_COUNT { + let object_key = block_key(destination_file.attr.inode, generation, 0, index as u64); + let bytes = if index == 0 { + vec![(index % 251) as u8; 4] + } else { + vec![(index % 251) as u8] + }; + objects.put(&object_key, bytes).unwrap(); + blocks.push(BlockDescriptor { + object_key: object_key.as_str().to_owned(), + logical_offset: index as u64, + object_offset: 0, + len: 1, + digest_uri: long_digest.clone(), + }); + object_keys.push(object_key); + } + let duplicate_object_key = object_keys[0].clone(); + let fragmented = ChunkManifest { + chunk_index: 0, + logical_offset: 0, + len: BLOCK_COUNT as u64, + slices: vec![ + SliceManifest { + slice_id: 1, + logical_offset: 0, + len: BLOCK_COUNT as u64, + blocks, + }, + // Slice overlays may legally reuse one immutable object through a + // different logical/object subrange. Release identity is the + // canonical key plus digest, not the range-specific descriptor. + SliceManifest { + slice_id: 2, + logical_offset: 1, + len: 2, + blocks: vec![BlockDescriptor { + object_key: duplicate_object_key.as_str().to_owned(), + logical_offset: 1, + object_offset: 1, + len: 2, + digest_uri: long_digest.clone(), + }], + }, + ], + }; + plan_chunk_manifest_reads(std::slice::from_ref(&fragmented), 0, BLOCK_COUNT) + .expect("range-distinct duplicate descriptors are a valid slice overlay"); + metadata.install_manifest(ScanItem { + key: manifest_key.clone(), + value: Value(encode_chunk_manifest(&fragmented)), + version: manifest_item.version, + }); - // 4. Divergence: rewrite a in the fork and add a new file c. + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + service.remove_file_path("/source/fragments.bin").unwrap(); + service.remove_empty_dir_path("/source").unwrap(); service - .replace_artifact(PublishArtifact { - parent: fork.root, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:zzz".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "fork/a".to_owned(), - bytes: vec![b'Z'; 4096], - mode: 0o644, - uid: 1000, - gid: 1000, - }) + .create_dir_path("/replacement", 0o755, 1000, 1000) .unwrap(); service - .publish_artifact(PublishArtifact { - parent: fork.root, - name: dname(b"c"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:ccc".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "fork/c".to_owned(), - bytes: vec![b'C'; 4096], - mode: 0o644, - uid: 1000, - gid: 1000, - }) + .rename_replace_path("/replacement", "/destination") .unwrap(); + metadata.fail_after_apply(b"restore-release-member-manifests", 0); - // 4a. Fork now sees a="ZZZ..", b="BBB..", c present. - assert_eq!( - service.read_artifact(fork.root, &dname(b"a")).unwrap(), - vec![b'Z'; 4096] + let member_key = restore::restore_staging_member_key( + service.mount_id(), + operation.ref_set_id, + destination_file.attr.inode, ); + let first_cursor = loop { + service.cleanup_restore_releases(1).unwrap(); + let member = service + .metadata_store() + .get( + RecordFamily::System, + &member_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .map(|value| restore::decode_restore_staging_member(&value.0).unwrap()); + if let Some(member) = member.filter(|member| !member.manifest_cursor.is_empty()) { + break member; + } + }; assert_eq!( - service.read_artifact(fork.root, &dname(b"b")).unwrap(), - vec![b'B'; 4096] + metadata.applied_with_prefix(b"restore-release-member-manifests"), + 1 ); - assert_eq!( - service.read_artifact(fork.root, &dname(b"c")).unwrap(), - vec![b'C'; 4096] + assert_eq!(first_cursor.manifest_cursor, manifest_key); + assert!(first_cursor.manifest_block_cursor > 0); + assert!(first_cursor.manifest_block_cursor < BLOCK_COUNT as u64); + assert!( + first_cursor.manifest_block_cursor < ITEM_BUDGET_BLOCK_CAP, + "the 8 MiB byte bound, not only the 4096-item bound, must split this manifest" ); - // 4b. Base is unchanged: a="AAA..", no c. - assert_eq!( - service - .read_artifact(base.attr.inode, &dname(b"a")) - .unwrap(), - vec![b'A'; 4096] + assert!(service + .metadata_store() + .get( + RecordFamily::ChunkManifest, + &manifest_key, + service.read_version().unwrap(), + ReadPurpose::RestoreStaging, + ) + .unwrap() + .is_some()); + let report = service.fsck_restore_state(true).unwrap(); + assert!( + report.is_consistent(), + "mid-cursor restore fsck report: {report:#?}" ); assert!(service - .lookup_plus(base.attr.inode, &dname(b"c")) + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_release_quarantine_prefix(service.mount_id()), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap() - .is_none()); + .is_empty()); + drop(service); - // 6. Diff reports exactly { modified: a, added: c }; b (shared) is not reported. - let mut diff = service.diff_subtrees(base.attr.inode, fork.root).unwrap(); - diff.sort_by(|left, right| left.path.cmp(&right.path)); - let summary: Vec<(&str, SubtreeDeltaKind)> = - diff.iter().map(|d| (d.path.as_str(), d.kind)).collect(); + metadata.clear_fault(); + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + let reopened_cursor = restore::decode_restore_staging_member( + &reopened + .metadata_store() + .get( + RecordFamily::System, + &member_key, + reopened.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap() + .0, + ) + .unwrap(); assert_eq!( - summary, - vec![ - ("/a", SubtreeDeltaKind::Modified), - ("/c", SubtreeDeltaKind::Added), - ] + reopened_cursor.manifest_cursor, + first_cursor.manifest_cursor ); - // The enriched diff carries the changed file's content digest. - assert!(diff - .iter() - .find(|d| d.path == "/a") - .unwrap() - .digest - .is_some()); - - // 5. GC safety: reclaim must NOT touch base blocks the fork's divergent write - // abandoned but the base still references; they are owned by the base inode and - // protected by the fork's retained snapshot pin. - let reclaim = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(reclaim.deleted, 0, "no base block may be reclaimed yet"); - assert!(objects.head(&a_block).unwrap().is_some()); - assert!(objects.head(&b_block).unwrap().is_some()); assert_eq!( - service - .read_artifact(base.attr.inode, &dname(b"a")) - .unwrap(), - vec![b'A'; 4096] + reopened_cursor.manifest_block_cursor, + first_cursor.manifest_block_cursor + ); + let report = reopened.fsck_restore_state(true).unwrap(); + assert!( + report.is_consistent(), + "reopened mid-cursor restore fsck report: {report:#?}" ); - // Drop the fork: remove its files and retire its snapshot pin. The fork-only - // blocks (the divergent a' and the new c) then become reclaimable, while the - // base's blocks remain because the base still references them. - let fork_a_diverged = service - .lookup_plus(fork.root, &dname(b"a")) + for _ in 0..512 { + reopened.cleanup_restore_releases(1).unwrap(); + if reopened.restore_metrics().unwrap().releasing == 0 { + break; + } + } + let metrics = reopened.restore_metrics().unwrap(); + assert_eq!(metrics.releasing, 0); + assert_eq!(metrics.operation_count(), 0); + assert_eq!(metrics.staging_rows, 0); + assert_eq!(metrics.exact_reference_rows, 0); + assert_eq!(metrics.index_rows, 0); + assert_eq!(metrics.release_backlog, 0); + assert!(reopened + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_release_quarantine_prefix(reopened.mount_id()), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap() - .unwrap(); - let fork_c = service - .lookup_plus(fork.root, &dname(b"c")) + .is_empty()); + + let destination_prefix = format!( + "blocks/{}/{}/{}/0/", + reopened.mount_id().get(), + destination_file.attr.inode.get(), + generation, + ); + let mut queued_by_object = BTreeMap::::new(); + for row in reopened + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount_id()), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap() + { + let record = decode_object_gc_record(&row.value.0).unwrap(); + if record.object_key.starts_with(&destination_prefix) { + *queued_by_object.entry(record.object_key).or_default() += 1; + } + } + assert_eq!(queued_by_object.len(), BLOCK_COUNT); + assert!(queued_by_object.values().all(|count| *count == 1)); + assert_eq!( + queued_by_object.get(duplicate_object_key.as_str()), + Some(&1), + "range-specific duplicate descriptors must produce one canonical GC candidate" + ); + + // The object-GC worker deliberately uses a two-transaction durable claim + // per object. Its claim/reopen behavior is covered independently; running + // 8,400 claim transitions here would dominate the restore suite. After + // proving every fragmented block is queued exactly once, compact the + // test queue in bounded CAS batches and leave one row for the real worker + // to drain, so the final no-leak assertion still crosses the worker path. + loop { + let rows = reopened + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount_id()), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 256, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + if rows.len() <= 1 { + break; + } + for row in &rows[1..] { + let record = decode_object_gc_record(&row.value.0).unwrap(); + objects + .delete(&ObjectKey::new(record.object_key).unwrap()) + .unwrap(); + } + let version = reopened.next_version().unwrap(); + reopened + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-compact-fragmented-release-gc", + reopened.mount_id(), + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Gc, + primary_key: rows[1].key.clone(), + predicates: rows[1..] + .iter() + .map(|row| PredicateRef { + family: RecordFamily::Gc, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }) + .collect(), + mutations: rows[1..] + .iter() + .map(|row| delete_mutation(RecordFamily::Gc, row.key.clone())) + .collect(), + watch: Vec::new(), + }) + .unwrap(); + } + for _ in 0..8 { + reopened.cleanup_pending_objects(1).unwrap(); + if reopened + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount_id()), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .is_empty() + { + break; + } + } + let remaining_gc = reopened + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount_id()), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 16, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap(); - let fork_a_block = block_key( - fork_a_diverged.attr.inode, - fork_a_diverged.body.as_ref().unwrap().generation, - 0, - 0, + assert!( + remaining_gc.is_empty(), + "remaining GC records: {:#?}", + remaining_gc + .iter() + .map(|row| decode_object_gc_record(&row.value.0)) + .collect::>() ); - let fork_c_block = block_key( - fork_c.attr.inode, - fork_c.body.as_ref().unwrap().generation, - 0, - 0, + assert!(objects.head(&source_object).unwrap().is_none()); + assert!(object_keys + .iter() + .all(|object_key| objects.head(object_key).unwrap().is_none())); + let report = reopened.fsck_restore_state(true).unwrap(); + assert!( + report.is_consistent(), + "final restore fsck report: {report:#?}" ); - service.remove_file(fork.root, &dname(b"a")).unwrap(); - service.remove_file(fork.root, &dname(b"b")).unwrap(); - service.remove_file(fork.root, &dname(b"c")).unwrap(); - assert!(service.retire_snapshot(fork.snapshot_id).unwrap()); - let reclaim = service.cleanup_pending_objects(100).unwrap(); - assert!(reclaim.deleted >= 2, "fork-only blocks must be reclaimable"); - assert!(objects.head(&fork_a_block).unwrap().is_none()); - assert!(objects.head(&fork_c_block).unwrap().is_none()); - // Base remains fully intact and readable. - assert!(objects.head(&a_block).unwrap().is_some()); - assert!(objects.head(&b_block).unwrap().is_some()); - assert_eq!( - service - .read_artifact(base.attr.inode, &dname(b"a")) - .unwrap(), - vec![b'A'; 4096] +} + +#[test] +fn restore_release_finds_a_live_snapshot_and_borrower_beyond_first_pages() { + let service = service(); + service + .create_dir_path("/pin-padding", 0o755, 1000, 1000) + .unwrap(); + for _ in 0..64 { + service.snapshot_subtree_path("/pin-padding").unwrap(); + } + + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/shared.bin", + "restore/paged-pin", + b"paged snapshot retention", + ); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + let restored = service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/destination") + .unwrap(); + + let holder = service + .create_dir_path("/snapshot-holder", 0o755, 1000, 1000) + .unwrap(); + let padding_names = (0..64) + .map(|index| DentryName::new(format!("aa-padding-{index:03}").into_bytes()).unwrap()) + .collect::>(); + service + .create_files_in_dir_path("/snapshot-holder", padding_names, 0o644, 1000, 1000) + .unwrap(); + service + .rename_path( + "/destination/shared.bin", + "/snapshot-holder/zz-borrower.bin", + ) + .unwrap(); + let holder_snapshot = service.snapshot_subtree_path("/snapshot-holder").unwrap(); + assert!(service + .retire_snapshot(source_snapshot.snapshot_id) + .unwrap()); + service.remove_file_path("/source/shared.bin").unwrap(); + service.remove_empty_dir_path("/source").unwrap(); + service.cleanup_pending_objects(64).unwrap(); + + service + .remove_file_path("/snapshot-holder/zz-borrower.bin") + .unwrap(); + service.remove_empty_dir_path("/destination").unwrap(); + for _ in 0..256 { + service.cleanup_restore_releases(64).unwrap(); + } + assert_eq!( + service.restore_metrics().unwrap().releasing, + 1, + "a historical escaped borrower beyond the first pin and dentry pages must retain the ref-set" ); + let historical_entries = service + .read_dir_plus_at_snapshot("/snapshot-holder", holder_snapshot.snapshot_id, &[]) + .unwrap(); + assert!(historical_entries + .iter() + .any(|entry| entry.dentry.name.as_bytes() == b"zz-borrower.bin")); + let borrower_name = DentryName::new("zz-borrower.bin").unwrap(); assert_eq!( service - .read_artifact(base.attr.inode, &dname(b"b")) + .read_file_at_snapshot( + "/snapshot-holder", + holder_snapshot.snapshot_id, + std::slice::from_ref(&borrower_name), + 0, + b"paged snapshot retention".len(), + ) .unwrap(), - vec![b'B'; 4096] - ); - assert_eq!(block_count_for(&objects, a.attr.inode, a_gen), 1); + b"paged snapshot retention" + ); + let historical_indexed = service + .restore_indexed_children_page( + holder.attr.inode, + None, + 128, + Version::new(holder_snapshot.read_version).unwrap(), + ReadPurpose::Snapshot, + ) + .unwrap(); + assert!(historical_indexed + .entries + .iter() + .any(|entry| entry.dentry.name.as_bytes() == b"zz-borrower.bin")); + + assert!(service + .retire_snapshot(holder_snapshot.snapshot_id) + .unwrap()); + for _ in 0..256 { + service.cleanup_restore_releases(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); + assert!(service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_staging_inode_key(service.mount_id(), restored.destination_root), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); } #[test] -fn clone_subtree_copies_nested_dirs_and_diff_reports_removed() { +fn restore_release_finds_a_live_fork_binding_beyond_the_first_page() { let service = service(); - service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); service - .create_dir_path("/base/sub", 0o755, 1000, 1000) + .create_dir_path("/binding-padding-source", 0o755, 1000, 1000) .unwrap(); - publish_path_artifact(&service, "/base/sub/deep", "base/deep", b"deep-bytes"); - publish_path_artifact(&service, "/base/top", "base/top", b"top-bytes"); + for index in 0..64 { + service + .clone_subtree_path_into( + "/binding-padding-source", + &format!("/binding-padding-{index:03}"), + ) + .unwrap(); + } - let fork = service.clone_subtree_path("/base").unwrap(); - // Nested structure is reproduced under fresh inodes. - let sub = service - .lookup_plus(fork.root, &dname(b"sub")) - .unwrap() + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - assert_eq!(sub.attr.file_type, FileType::Directory); - assert_eq!( - service - .read_artifact(sub.attr.inode, &dname(b"deep")) - .unwrap(), - b"deep-bytes" + publish_path_artifact( + &service, + "/source/shared.bin", + "restore/paged-binding", + b"paged binding retention", ); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/destination") + .unwrap(); + let holder = service + .clone_subtree_path_into("/destination", "/generic-holder") + .unwrap(); - // Identical subtree => no deltas. - let base = service.resolve_directory_path("/base").unwrap(); - assert!(service.diff_subtrees(base, fork.root).unwrap().is_empty()); + let holder_pin = service.snapshot_pin(holder.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(holder_pin.lease_expires_unix_ms); + service.reclaim_expired_snapshot_pins(1_024).unwrap(); + assert!(service.snapshot_pin(holder.snapshot_id).unwrap().is_none()); + + service.remove_file_path("/destination/shared.bin").unwrap(); + service.remove_empty_dir_path("/destination").unwrap(); + for _ in 0..4 { + service.cleanup_restore_releases(64).unwrap(); + } + assert_eq!( + service.restore_metrics().unwrap().releasing, + 1, + "the live ForkBinding beyond the first 64-row page must retain the ref-set" + ); - // Remove a nested file in the fork => Removed delta at the nested path, - // direction base -> fork. service - .remove_file(sub.attr.inode, &dname(b"deep")) + .remove_file_path("/generic-holder/shared.bin") .unwrap(); - let removed = service.diff_subtrees(base, fork.root).unwrap(); - assert_eq!(removed.len(), 1); - assert_eq!(removed[0].path, "/sub/deep"); - assert_eq!(removed[0].kind, SubtreeDeltaKind::Removed); - assert_eq!(removed[0].size_delta, -(b"deep-bytes".len() as i64)); - // Reversed direction reports it as Added, with the net size flipped. - let added = service.diff_subtrees(fork.root, base).unwrap(); - assert_eq!(added.len(), 1); - assert_eq!(added[0].path, "/sub/deep"); - assert_eq!(added[0].kind, SubtreeDeltaKind::Added); - assert_eq!(added[0].size_delta, b"deep-bytes".len() as i64); + service.remove_empty_dir_path("/generic-holder").unwrap(); + assert!(service.retire_snapshot(holder.snapshot_id).unwrap()); + for _ in 0..256 { + service.cleanup_restore_releases(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); } #[test] -fn clone_subtree_path_rejects_non_directory() { +fn complete_restore_can_replace_another_complete_restore_and_release_both() { let service = service(); - publish_path_artifact(&service, "/file.bin", "file", b"bytes"); - assert!(matches!( - service.clone_subtree_path("/file.bin"), - Err(MetadError::NotDirectory) - )); -} - -#[test] -fn clone_link_rejects_an_expired_reaped_pin_before_exposing_the_fork() { - let store = SnapshotCommitBarrierStore::new(CommandKind::CreateDir, 1, 2) - .matching_request_prefix(b"clone-subtree-link"); - let objects = MemoryObjectStore::new(); - let service = Arc::new(NoKvFs::new( - MountId::new(1).unwrap(), - store.clone(), - objects.clone(), - )); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); - let published = service - .publish_artifact(PublishArtifact { - parent: base.attr.inode, - name: dname(b"data.bin"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:clone-link-race".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "clone-link-race".to_owned(), - bytes: b"payload".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) + for source in ["/outer-source", "/nested-source"] { + service.create_dir_path(source, 0o755, 1000, 1000).unwrap(); + } + publish_path_artifact( + &service, + "/outer-source/outer.bin", + "restore/replace-complete-outer", + b"outer bytes", + ); + publish_path_artifact( + &service, + "/nested-source/nested.bin", + "restore/replace-complete-nested", + b"nested bytes", + ); + let outer_snapshot = service.snapshot_subtree_path("/outer-source").unwrap(); + let outer = service + .restore_subtree_path_to_fork("/outer-source", outer_snapshot.snapshot_id, "/outer") .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - - let clone_service = Arc::clone(&service); - let clone = std::thread::spawn(move || clone_service.clone_subtree_path_into("/base", "/fork")); - store.wait_until_blocked(); - assert!(service.object_gc_gate.try_lock().is_err()); - - let pins = store - .scan(ScanRequest { - family: RecordFamily::Snapshot, - prefix: snapshot_pin_prefix(service.mount), - start_after: None, - version: service.read_version().unwrap(), - limit: 0, - purpose: ReadPurpose::UserStrong, - }) + let nested_snapshot = service.snapshot_subtree_path("/nested-source").unwrap(); + let nested = service + .restore_subtree_path_to_fork("/nested-source", nested_snapshot.snapshot_id, "/nested") .unwrap(); - assert_eq!(pins.len(), 1); - let pin = decode_snapshot_pin(&pins[0].value.0).unwrap(); + assert!(service.retire_snapshot(outer_snapshot.snapshot_id).unwrap()); + assert!(service + .retire_snapshot(nested_snapshot.snapshot_id) + .unwrap()); - service - .remove_file(base.attr.inode, &dname(b"data.bin")) - .unwrap(); - service.set_clock_override_ms(pin.lease_expires_unix_ms); + service.rename_replace_path("/nested", "/outer").unwrap(); + assert_eq!(service.restore_metrics().unwrap().releasing, 1); + assert_eq!(service.restore_metrics().unwrap().complete, 1); + let surviving_root = service.resolve_directory_path("/outer").unwrap(); assert_eq!( - service.reclaim_expired_snapshot_pins(100).unwrap().reaped, - 1 + service + .read_artifact( + surviving_root, + &DentryName::new(b"nested.bin".to_vec()).unwrap(), + ) + .unwrap(), + b"nested bytes" ); + assert!(service.lookup_path("/outer/outer.bin").unwrap().is_none()); - let cleanup_started = Arc::new(Barrier::new(2)); - let cleanup_service = Arc::clone(&service); - let cleanup_thread_started = Arc::clone(&cleanup_started); - let cleanup = std::thread::spawn(move || { - cleanup_thread_started.wait(); - cleanup_service.cleanup_pending_objects(100) - }); - cleanup_started.wait(); - store.release_blocked(); + for _ in 0..256 { + service.cleanup_restore_releases(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); + assert_eq!(service.restore_metrics().unwrap().complete, 1); + assert!(service + .metadata_store() + .get( + RecordFamily::Inode, + &inode_key(service.mount_id(), outer.destination_root), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + assert!(service + .metadata_store() + .get( + RecordFamily::Inode, + &inode_key(service.mount_id(), nested.destination_root), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_some()); - assert!(matches!( - clone.join().unwrap(), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - )); - let cleanup = cleanup.join().unwrap().unwrap(); - assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); - assert!(objects.head(&object).unwrap().is_none()); - assert!(service.lookup_path("/fork").unwrap().is_none()); + service.remove_file_path("/outer/nested.bin").unwrap(); + service.remove_empty_dir_path("/outer").unwrap(); + for _ in 0..256 { + service.cleanup_restore_releases(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + } + assert_eq!(service.restore_metrics().unwrap().complete, 0); + assert_eq!(service.restore_metrics().unwrap().releasing, 0); + let report = service.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); } #[test] -fn fork_binding_survives_pin_reaping_and_a_hardlink_escaping_the_fork_root() { +fn nested_restore_replace_releases_shared_object_after_escaped_borrower_is_removed() { let (service, objects) = service_with_objects(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .create_dir_path("/source/metadata", 0o755, 1000, 1000) + .unwrap(); let source = publish_path_artifact( &service, - "/base/data.bin", - "fork-retention/source", - b"shared bytes", + "/source/shared.bin", + "restore/nested-replace-escaped", + b"shared restore bytes", ); let source_body = source.body.as_ref().unwrap(); - let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let source_object = block_key(source.attr.inode, source_body.generation, 0, 0); + let inherited_checkpoint = publish_path_artifact( + &service, + "/source/metadata/checkpoints.jsonl", + "restore/nested-replace-checkpoint-alias", + b"source checkpoint alias", + ); + let inherited_checkpoint_body = inherited_checkpoint.body.as_ref().unwrap(); + let inherited_checkpoint_object = block_key( + inherited_checkpoint.attr.inode, + inherited_checkpoint_body.generation, + 0, + 0, + ); - let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); - let fork_file = service - .lookup_plus(fork.root, &dname(b"data.bin")) + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork_initialized( + "/source", + source_snapshot.snapshot_id, + "/destination", + RestoreInitialization { + remove_relative_paths: vec!["metadata/checkpoints.jsonl".to_owned()], + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: b"outer restore manifest".to_vec(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, + ) + .unwrap(); + assert!(service + .lookup_path("/destination/metadata/checkpoints.jsonl") + .unwrap() + .is_none()); + let outer_manifest = service + .lookup_path("/destination/metadata/restore_manifest.json") .unwrap() .unwrap(); + let outer_manifest_body = outer_manifest.body.as_ref().unwrap(); + let outer_manifest_object = block_key( + outer_manifest.attr.inode, + outer_manifest_body.generation, + 0, + 0, + ); + let nested_snapshot = service.snapshot_subtree_path("/destination").unwrap(); service - .link(fork_file.attr.inode, InodeId::root(), dname(b"escaped.bin")) + .restore_subtree_path_to_fork_initialized( + "/destination", + nested_snapshot.snapshot_id, + "/nested", + RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: b"nested restore manifest".to_vec(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, + ) .unwrap(); - service.remove_file(fork.root, &dname(b"data.bin")).unwrap(); - service.remove_empty_dir_path("/fork").unwrap(); + assert!(service + .retire_snapshot(source_snapshot.snapshot_id) + .unwrap()); + assert!(service + .retire_snapshot(nested_snapshot.snapshot_id) + .unwrap()); - // The source can stop naming its object and the construction pin can expire; - // neither event retires the durable binding. The escaped hardlink remains a - // current borrowed reference even though the fork root itself is gone. - service.remove_file_path("/base/data.bin").unwrap(); - let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); - service.set_clock_override_ms(pin.lease_expires_unix_ms); - assert_eq!( - service.reclaim_expired_snapshot_pins(100).unwrap().reaped, - 1 + service.remove_file_path("/source/shared.bin").unwrap(); + service + .remove_file_path("/source/metadata/checkpoints.jsonl") + .unwrap(); + service.remove_empty_dir_path("/source/metadata").unwrap(); + service.remove_empty_dir_path("/source").unwrap(); + service + .rename_path("/destination/shared.bin", "/escaped.bin") + .unwrap(); + service + .rename_replace_path("/nested", "/destination") + .unwrap(); + service.rename_path("/destination", "/replacement").unwrap(); + service.remove_file_path("/replacement/shared.bin").unwrap(); + service + .remove_file_path("/replacement/metadata/restore_manifest.json") + .unwrap(); + service + .remove_empty_dir_path("/replacement/metadata") + .unwrap(); + service.remove_empty_dir_path("/replacement").unwrap(); + + for _ in 0..512 { + service.cleanup_restore_releases(64).unwrap(); + service.cleanup_pending_objects(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 1 + && objects.head(&outer_manifest_object).unwrap().is_none() + { + break; + } + } + assert_eq!(service.restore_metrics().unwrap().releasing, 1); + assert!(objects.head(&outer_manifest_object).unwrap().is_none()); + let intermediate_restore_fsck = service.fsck_restore_state(true).unwrap(); + assert!( + intermediate_restore_fsck.is_consistent(), + "intermediate restore fsck: {intermediate_restore_fsck:?}" ); - assert!(service.snapshot_pin(fork.snapshot_id).unwrap().is_none()); - assert_eq!( - service - .metadata - .scan(ScanRequest { - family: RecordFamily::ForkBinding, - prefix: crate::layout::fork_binding_prefix(service.mount), - start_after: None, - version: service.read_version().unwrap(), - limit: 0, - purpose: ReadPurpose::UserStrong, - }) - .unwrap() - .len(), - 1 + let intermediate_fsck = service.fsck_object_references(FsckMode::Full, 0).unwrap(); + assert!( + intermediate_fsck.is_consistent(), + "intermediate object fsck: {intermediate_fsck:?}" ); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); - assert!(cleanup.blocked_by_snapshots >= 1); - assert!(objects.head(&source_block).unwrap().is_some()); + assert!(objects.head(&source_object).unwrap().is_some()); assert_eq!( - service - .read_artifact(InodeId::root(), &dname(b"escaped.bin")) - .unwrap(), - b"shared bytes" + read_artifact_at_path(&service, "/escaped.bin"), + b"shared restore bytes" ); - let error = service - .retire_snapshot_path("/base", fork.snapshot_id) - .unwrap_err(); + let metadata = service.metadata_store().clone(); + drop(service); + let service = + NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects.clone(), 0).unwrap(); + let reopened_restore_fsck = service.fsck_restore_state(true).unwrap(); assert!( - matches!( - error, - MetadError::ForkRetentionActive { - snapshot_id, - fork_root, - borrower, - } if snapshot_id == fork.snapshot_id - && fork_root == fork.root - && borrower == fork_file.attr.inode - ), - "unexpected retirement error: {error:?}" + reopened_restore_fsck.is_consistent(), + "reopened restore fsck: {reopened_restore_fsck:?}" ); - assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); - assert!(objects.head(&source_block).unwrap().is_some()); - assert_eq!( - service - .read_artifact(InodeId::root(), &dname(b"escaped.bin")) - .unwrap(), - b"shared bytes" + let reopened_object_fsck = service.fsck_object_references(FsckMode::Full, 0).unwrap(); + assert!( + reopened_object_fsck.is_consistent(), + "reopened object fsck: {reopened_object_fsck:?}" ); - // Retirement becomes valid after every borrowed fork reference, including - // links outside the original fork root, has been removed. - service - .remove_file(InodeId::root(), &dname(b"escaped.bin")) - .unwrap(); - assert!(service - .retire_snapshot_path("/base", fork.snapshot_id) - .unwrap()); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); - assert!(objects.head(&source_block).unwrap().is_none()); + service.remove_file_path("/escaped.bin").unwrap(); + for _ in 0..1_024 { + service.cleanup_restore_releases(64).unwrap(); + service.cleanup_pending_objects(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 + && objects.head(&source_object).unwrap().is_none() + { + break; + } + } + let final_metrics = service.restore_metrics().unwrap(); + assert_eq!(final_metrics.releasing, 0); + assert_eq!(final_metrics.staging_rows, 0); + assert_eq!(final_metrics.exact_reference_rows, 0); + assert_eq!(final_metrics.index_rows, 0); + assert_eq!(final_metrics.release_backlog, 0); + assert!(objects.head(&source_object).unwrap().is_none()); + assert!(objects + .head(&inherited_checkpoint_object) + .unwrap() + .is_none()); + let report = service.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); + let object_report = service.fsck_object_references(FsckMode::Full, 0).unwrap(); + assert!( + object_report.is_consistent(), + "final object fsck report: {object_report:#?}" + ); } #[test] -fn detached_fork_binding_is_a_legal_link_and_rename_root() { +fn outer_restore_release_cascades_into_a_nested_complete_restore() { let service = service(); - service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/outer-source", 0o755, 1000, 1000) + .unwrap(); + service + .create_dir_path("/nested-source", 0o755, 1000, 1000) + .unwrap(); publish_path_artifact( &service, - "/base/data.bin", - "detached-namespace/source", - b"shared bytes", - ); - let fork = service.clone_subtree_path("/base").unwrap(); - assert!( - !service.materialization_orphan_slow_path_enabled(), - "a durable detached ForkBinding must restore the healthy fast path" + "/nested-source/payload.bin", + "restore/nested-release", + b"nested restore bytes", ); - let reachability_scans = service.namespace_reachability_scan_count(); - let fork_file = service - .lookup_plus(fork.root, &dname(b"data.bin")) - .unwrap() + let outer_snapshot = service.snapshot_subtree_path("/outer-source").unwrap(); + let outer = service + .restore_subtree_path_to_fork("/outer-source", outer_snapshot.snapshot_id, "/outer") .unwrap(); - - let renamed = service - .rename( - fork.root, - &dname(b"data.bin"), - fork.root, - dname(b"renamed.bin"), + let nested_snapshot = service.snapshot_subtree_path("/nested-source").unwrap(); + let nested = service + .restore_subtree_path_to_fork( + "/nested-source", + nested_snapshot.snapshot_id, + "/outer/nested", ) .unwrap(); - assert_eq!(renamed.attr.inode, fork_file.attr.inode); - let escaped = service - .link(fork_file.attr.inode, InodeId::root(), dname(b"escaped.bin")) + assert!(service.retire_snapshot(outer_snapshot.snapshot_id).unwrap()); + assert!(service + .retire_snapshot(nested_snapshot.snapshot_id) + .unwrap()); + + service + .create_dir_path("/replacement", 0o755, 1000, 1000) .unwrap(); - assert_eq!(escaped.attr.inode, fork_file.attr.inode); - assert_eq!( - service - .read_artifact(InodeId::root(), &dname(b"escaped.bin")) - .unwrap(), - b"shared bytes" + service + .rename_replace_path("/replacement", "/outer") + .unwrap(); + assert_eq!(service.restore_metrics().unwrap().releasing, 1); + assert_eq!(service.restore_metrics().unwrap().complete, 1); + assert!( + service.get_attr(nested.destination_root).unwrap().is_none(), + "slow-path visibility must hide the nested Complete root as soon as its outer root detaches" ); + + for _ in 0..32 { + service.cleanup_restore_releases(1).unwrap(); + if service.restore_metrics().unwrap().releasing == 2 { + break; + } + } assert_eq!( - service.namespace_reachability_scan_count(), - reachability_scans, - "healthy detached-root rename/link must not scan namespace reachability" + service.restore_metrics().unwrap().releasing, + 2, + "outer release must durably cascade the nested Complete root" ); + assert_eq!(service.restore_metrics().unwrap().complete, 0); + assert!(service.lookup_path("/outer/nested").unwrap().is_none()); + + for _ in 0..512 { + service.cleanup_restore_releases(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); + for inode in [outer.destination_root, nested.destination_root] { + assert!(service + .metadata_store() + .get( + RecordFamily::Inode, + &inode_key(service.mount_id(), inode), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + } } #[test] -fn reopen_recovers_an_unbound_materialization_into_the_slow_path() { - let metadata = HoltMetadataStore::open_memory().unwrap(); +fn outer_restore_release_invalidates_a_preplanned_nested_restore_write() { + let store = SnapshotCommitBarrierStore::new(CommandKind::SetXattr, 0, 2); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + MemoryObjectStore::new(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/outer-source", 0o755, 1000, 1000) + .unwrap(); + service + .create_dir_path("/nested-source", 0o755, 1000, 1000) + .unwrap(); + let outer_snapshot = service.snapshot_subtree_path("/outer-source").unwrap(); + service + .restore_subtree_path_to_fork("/outer-source", outer_snapshot.snapshot_id, "/outer") + .unwrap(); + let nested_snapshot = service.snapshot_subtree_path("/nested-source").unwrap(); + let nested = service + .restore_subtree_path_to_fork( + "/nested-source", + nested_snapshot.snapshot_id, + "/outer/nested", + ) + .unwrap(); + store.arm(1); + + let writer = Arc::clone(&service); + let write = std::thread::spawn(move || { + writer.set_xattr( + nested.destination_root, + b"user.preplanned-nested", + b"must not commit".to_vec(), + XattrSetMode::Create, + ) + }); + store.wait_until_blocked(); + + service + .create_dir_path("/replacement", 0o755, 1000, 1000) + .unwrap(); + service + .rename_replace_path("/replacement", "/outer") + .unwrap(); + assert!(service.get_attr(nested.destination_root).unwrap().is_none()); + store.release_blocked(); + + assert!(matches!( + write.join().unwrap(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!(service + .metadata_store() + .get( + RecordFamily::Xattr, + &xattr_key( + service.mount_id(), + nested.destination_root, + b"user.preplanned-nested", + ), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); +} + +#[test] +fn detached_restore_rejects_guessed_writes_at_the_attach_boundary() { + let store = SnapshotCommitBarrierStore::new(CommandKind::CreateDir, 1, 2) + .matching_request_prefix(b"restore-attach-destination"); let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); publish_path_artifact( - &service, - "/base/data.bin", - "reopen-orphan/source", - b"borrowed", + service.as_ref(), + "/source/original.bin", + "restore/attach-race", + b"original bytes", + ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/source", + snapshot.snapshot_id, + "/destination", ); - let pin = service.snapshot_subtree(base.attr.inode).unwrap(); - let orphan_root = { - let _object_gc_gate = service.object_gc_gate.lock().unwrap(); - service - .materialize_subtree_at(base.attr.inode, Version::new(pin.read_version).unwrap()) - .unwrap() - }; - let orphan = service - .lookup_plus(orphan_root, &dname(b"data.bin")) - .unwrap() - .unwrap(); - assert!(service.materialization_orphan_slow_path_enabled()); - let reopened = - NoKvFs::open_existing(service.mount, metadata, objects, service.shard_index()).unwrap(); - assert!( - reopened.materialization_orphan_slow_path_enabled(), - "reopen must recover an unbound detached tree before serving" + let restoring = Arc::clone(&service); + let restore_thread = std::thread::spawn(move || { + restoring.restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + }); + store.wait_until_blocked(); + + let operation = restore::decode_restore_operation( + &service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_operation_key(service.mount_id(), &operation_digest), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap() + .0, + ) + .unwrap(); + assert_eq!( + operation.state, + restore::RestoreOperationState::ReadyToAttach ); - assert_eq!(reopened.namespace_reachability_scan_count(), 1); + let guessed_name = DentryName::new("guessed.txt").unwrap(); assert!(matches!( - reopened.link(orphan.attr.inode, InodeId::root(), dname(b"reopened-link")), - Err(MetadError::NotFound) + service.create_file( + operation.destination_root, + guessed_name.clone(), + 0o644, + 1000, + 1000, + ), + Err(MetadError::RestoreInProgress) )); assert!(matches!( - reopened.rename( - orphan_root, - &dname(b"data.bin"), - InodeId::root(), - dname(b"reopened-rename") + service.set_xattr( + operation.destination_root, + b"user.guessed", + b"forbidden".to_vec(), + XattrSetMode::Create, + ), + Err(MetadError::RestoreInProgress) + )); + let objects_before_publish = objects.object_count(); + assert!(matches!( + service.publish_artifact(PublishArtifact { + parent: operation.destination_root, + name: DentryName::new("guessed.bin").unwrap(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:test".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "restore/forbidden".to_owned(), + bytes: b"must not upload".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }), + Err(MetadError::RestoreInProgress) + )); + assert_eq!(objects.object_count(), objects_before_publish); + assert!(matches!( + service.publish_checkpoint( + operation.destination_root, + vec![CheckpointShard { + name: DentryName::new("guessed-checkpoint.bin").unwrap(), + bytes: b"must not upload".to_vec(), + }], + 1000, + 1000, ), - Err(MetadError::NotFound) + Err(MetadError::RestoreInProgress) )); - assert!(reopened - .lookup_plus(InodeId::root(), &dname(b"reopened-link")) - .unwrap() - .is_none()); - assert!(reopened - .lookup_plus(InodeId::root(), &dname(b"reopened-rename")) + assert_eq!(objects.object_count(), objects_before_publish); + + store.release_blocked(); + let outcome = restore_thread.join().unwrap().unwrap(); + assert_eq!(outcome.destination_root, operation.destination_root); + assert!(service + .lookup_plus(outcome.destination_root, &guessed_name) .unwrap() .is_none()); } #[test] -fn open_existing_allows_empty_namespace_until_bootstrap_then_uses_fast_path() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let objects = MemoryObjectStore::new(); - let service = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - - assert!( - service.materialization_orphan_slow_path_enabled(), - "an empty namespace remains fail-closed until its root is bootstrapped" - ); - assert_eq!(service.namespace_reachability_scan_count(), 0); - +fn restore_attach_reproves_a_concurrently_occupied_destination_as_a_conflict() { + let store = SnapshotCommitBarrierStore::new(CommandKind::CreateDir, 1, 2) + .matching_request_prefix(b"restore-attach-destination"); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + MemoryObjectStore::new(), + )); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - assert!( - !service.materialization_orphan_slow_path_enabled(), - "root bootstrap must prove the pristine namespace healthy" - ); - assert_eq!(service.namespace_reachability_scan_count(), 1); - - let source = service - .create_file(InodeId::root(), dname(b"source.bin"), 0o644, 1000, 1000) + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let scans_after_bootstrap = service.namespace_reachability_scan_count(); service - .link(source.attr.inode, InodeId::root(), dname(b"linked.bin")) + .create_file_path("/source/empty.txt", 0o644, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let snapshot_id = snapshot.snapshot_id; + + let restoring = Arc::clone(&service); + let restore_thread = std::thread::spawn(move || { + restoring.restore_subtree_path_to_fork("/source", snapshot_id, "/destination") + }); + store.wait_until_blocked(); + + service + .create_dir_path("/destination", 0o755, 1000, 1000) .unwrap(); + store.release_blocked(); + assert!(matches!( + restore_thread.join().unwrap(), + Err(MetadError::RestoreDestinationConflict { destination }) + if destination == "/destination" + )); + assert_eq!(store.predicate_failures(), 1); + + let interrupted = durable_restore_operation(&service, snapshot_id); assert_eq!( - service.namespace_reachability_scan_count(), - scans_after_bootstrap, - "healthy link after bootstrap must not run namespace reachability" + interrupted.state, + restore::RestoreOperationState::ReadyToAttach ); + service.remove_empty_dir_path("/destination").unwrap(); + + let outcome = service + .restore_subtree_path_to_fork("/source", snapshot_id, "/destination") + .unwrap(); + assert_eq!(outcome.destination_root, interrupted.destination_root); + assert_eq!(outcome.state, RestoreState::Complete); + assert!(service.fsck_restore_state(false).unwrap().is_consistent()); } #[test] -fn open_existing_rejects_missing_root_with_unbound_materialization_records() { - let metadata = HoltMetadataStore::open_memory().unwrap(); - let objects = MemoryObjectStore::new(); - let writer = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - let orphan = InodeId::new(InodeId::ROOT_RAW + 1).unwrap(); - let version = writer.next_version().unwrap(); - let attr = directory_attr(orphan, 0o755, 1000, 1000, version.get()); - writer - .commit_metadata(MetadataCommand { - request_id: request_id(b"test-rootless-orphan", writer.mount, orphan, version), - kind: CommandKind::CreateDir, - read_version: predecessor(version).unwrap(), - commit_version: version, - primary_family: RecordFamily::Inode, - primary_key: inode_key(writer.mount, orphan), - predicates: Vec::new(), - mutations: vec![Mutation { - family: RecordFamily::Inode, - key: inode_key(writer.mount, orphan), - op: MutationOp::Put, - value: Some(Value(encode_inode_attr(&attr))), - }], - watch: Vec::new(), - }) +fn restore_release_cas_rejects_a_preplanned_complete_write() { + let store = SnapshotCommitBarrierStore::new(CommandKind::CreateFile, 0, 2); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + MemoryObjectStore::new(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); + publish_path_artifact( + service.as_ref(), + "/source/original.bin", + "restore/release-race", + b"original bytes", + ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let restored = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + store.arm(1); + + let writer = Arc::clone(&service); + let create = std::thread::spawn(move || { + writer.create_file( + restored.destination_root, + DentryName::new("raced.txt").unwrap(), + 0o644, + 1000, + 1000, + ) + }); + store.wait_until_blocked(); + service + .create_dir_path("/replacement", 0o755, 1000, 1000) + .unwrap(); + service + .rename_replace_path("/replacement", "/destination") + .unwrap(); + store.release_blocked(); - let reopened = NoKvFs::open_existing(writer.mount, metadata, objects, writer.shard_index()); assert!(matches!( - reopened, - Err(MetadError::Codec(message)) - if message == "mount root is missing while namespace records still exist" + create.join().unwrap(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) )); + assert!(service + .metadata_store() + .get( + RecordFamily::Dentry, + &dentry_key( + service.mount_id(), + restored.destination_root, + &DentryName::new("raced.txt").unwrap(), + ), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); } #[test] -fn fork_binding_retains_object_backed_symlink_after_source_deletion() { - let (service, objects) = service_with_objects(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); - let source = service - .create_symlink( - base.attr.inode, - dname(b"latest"), - b"runs/42/checkpoint.bin".to_vec(), - 0o777, - 1000, - 1000, - ) +fn restore_enrollment_invalidates_an_ordinary_preplanned_xattr_write() { + let store = SnapshotCommitBarrierStore::new(CommandKind::SetXattr, 0, 2); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + MemoryObjectStore::new(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let ordinary = service + .create_file_path("/ordinary.bin", 0o644, 1000, 1000) .unwrap(); - let source_body = source.body.as_ref().unwrap(); - let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); - let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); - let fork_symlink = service - .lookup_plus(fork.root, &dname(b"latest")) - .unwrap() + let ordinary_inode = ordinary.attr.inode; + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + store.arm(1); - service.remove_file_path("/base/latest").unwrap(); - let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); - service.set_clock_override_ms(pin.lease_expires_unix_ms); - assert_eq!( - service.reclaim_expired_snapshot_pins(100).unwrap().reaped, - 1 - ); - assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); - assert!(objects.head(&source_block).unwrap().is_some()); - assert_eq!( - service.read_symlink(fork_symlink.attr.inode).unwrap(), - b"runs/42/checkpoint.bin" - ); + let writer = Arc::clone(&service); + let write = std::thread::spawn(move || { + writer.set_xattr( + ordinary_inode, + b"user.preplanned", + b"must not commit".to_vec(), + XattrSetMode::Create, + ) + }); + store.wait_until_blocked(); + + service + .rename_path("/ordinary.bin", "/destination/ordinary.bin") + .unwrap(); + service + .create_dir_path("/replacement", 0o755, 1000, 1000) + .unwrap(); + service + .rename_replace_path("/replacement", "/destination") + .unwrap(); + assert_eq!(service.restore_metrics().unwrap().releasing, 1); + store.release_blocked(); assert!(matches!( - service.retire_snapshot_path("/base", fork.snapshot_id), - Err(MetadError::ForkRetentionActive { - snapshot_id, - fork_root, - borrower, - }) if snapshot_id == fork.snapshot_id - && fork_root == fork.root - && borrower == fork_symlink.attr.inode + write.join().unwrap(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) )); - assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); - assert!(objects.head(&source_block).unwrap().is_some()); - assert_eq!( - service.read_symlink(fork_symlink.attr.inode).unwrap(), - b"runs/42/checkpoint.bin" - ); - - service.remove_file(fork.root, &dname(b"latest")).unwrap(); assert!(service - .retire_snapshot_path("/base", fork.snapshot_id) - .unwrap()); - assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 1); - assert!(objects.head(&source_block).unwrap().is_none()); + .metadata_store() + .get( + RecordFamily::Xattr, + &xattr_key(service.mount_id(), ordinary_inode, b"user.preplanned",), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); } #[test] -fn fork_binding_can_retire_after_the_fork_rewrites_onto_self_owned_blocks() { - let (service, objects) = service_with_objects(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); - let source = publish_path_artifact( - &service, - "/base/data.bin", - "fork-rewrite/source", - b"borrowed bytes", - ); - let source_body = source.body.as_ref().unwrap(); - let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); - let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); - - let rewritten = service - .replace_artifact(PublishArtifact { - parent: fork.root, - name: dname(b"data.bin"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:fork-rewrite".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "fork-rewrite/self-owned".to_owned(), - bytes: b"self-owned bytes".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) +fn allocator_reservation_does_not_invalidate_a_preplanned_ordinary_write() { + let store = SnapshotCommitBarrierStore::new(CommandKind::SetXattr, 0, 2); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + MemoryObjectStore::new(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let ordinary = service + .create_file_path("/ordinary-before-reservation.bin", 0o644, 1000, 1000) .unwrap(); - let rewritten_body = rewritten.entry.body.as_ref().unwrap(); - let rewritten_block = block_key(rewritten.entry.attr.inode, rewritten_body.generation, 0, 0); - service - .create_file( - InodeId::root(), - dname(b"unrelated-empty"), - 0o644, - 1000, - 1000, + let activation_key = restore::restore_activation_fence_key(service.mount_id()); + let activation_before = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &activation_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let allocator = allocator_key(service.mount_id()); + let allocator_before = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &allocator, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, ) + .unwrap() .unwrap(); - service.remove_file_path("/base/data.bin").unwrap(); - let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); - service.set_clock_override_ms(pin.lease_expires_unix_ms); - assert_eq!( - service.reclaim_expired_snapshot_pins(100).unwrap().reaped, - 1 - ); - assert!(service - .retire_snapshot_path("/base", fork.snapshot_id) - .unwrap()); - assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 1); - assert!(objects.head(&source_block).unwrap().is_none()); - assert!(objects.head(&rewritten_block).unwrap().is_some()); - assert_eq!( - read_artifact_at_path(&service, "/fork/data.bin"), - b"self-owned bytes" - ); -} + store.arm(1); + let writer = Arc::clone(&service); + let inode = ordinary.attr.inode; + let write = std::thread::spawn(move || { + writer.set_xattr( + inode, + b"user.preplanned-before-reservation", + b"must commit".to_vec(), + XattrSetMode::Create, + ) + }); + store.wait_until_blocked(); -#[test] -fn unrelated_cross_shard_graft_does_not_block_safe_fork_retirement() { - let service = service(); - service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); - let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); - let foreign = InodeId::compose(1, 42).unwrap(); - service - .create_graft( - InodeId::root(), - dname(b"foreign"), - foreign, - 0o755, - 1000, - 1000, + for _ in 0..=ALLOCATOR_VERSION_RESERVATION { + service.next_version().unwrap(); + let current = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &allocator, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + if current.version != allocator_before.version { + break; + } + } + let allocator_after = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &allocator, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + assert_ne!(allocator_after.version, allocator_before.version); + let activation_after = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &activation_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, ) + .unwrap() .unwrap(); + assert_eq!(activation_after.version, activation_before.version); - assert!(service - .retire_snapshot_path("/base", fork.snapshot_id) - .unwrap()); - assert!(!service - .retire_snapshot_path("/base", fork.snapshot_id) - .unwrap()); + store.release_blocked(); + write.join().unwrap().unwrap(); assert_eq!( service - .lookup_plus(InodeId::root(), &dname(b"foreign")) - .unwrap() - .unwrap() - .attr - .inode, - foreign + .get_xattr(inode, b"user.preplanned-before-reservation") + .unwrap(), + Some(b"must commit".to_vec()) ); } #[test] -fn fork_binding_cannot_retire_when_append_still_inherits_borrowed_blocks() { - let (service, objects) = service_with_objects(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); - let source = publish_path_artifact( - &service, - "/base/data.bin", - "fork-append/source", - b"borrowed base", - ); - let source_body = source.body.as_ref().unwrap(); - let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); - let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); - let fork_file = service - .lookup_plus(fork.root, &dname(b"data.bin")) - .unwrap() +fn first_restore_hold_invalidates_a_preplanned_ordinary_write() { + let store = SnapshotCommitBarrierStore::new(CommandKind::SetXattr, 0, 2); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + MemoryObjectStore::new(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let ordinary = service + .create_file_path("/ordinary-before-restore.bin", 0o644, 1000, 1000) .unwrap(); - - let prepared = service - .prepare_artifact_replace(fork.root, dname(b"data.bin")) + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let written = service - .stage_prepared_artifact_ranges( - &prepared, - "fork-append/delta", - &[PublishArtifactRange { - offset: source_body.size, - bytes: b" + self-owned delta".to_vec(), - }], - 0, + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + store.arm(1); + + let writer = Arc::clone(&service); + let ordinary_inode = ordinary.attr.inode; + let write = std::thread::spawn(move || { + writer.set_xattr( + ordinary_inode, + b"user.preplanned-before-restore", + b"must not commit".to_vec(), + XattrSetMode::Create, ) - .unwrap(); + }); + store.wait_until_blocked(); + service - .publish_prepared_artifact_staged_session( - prepared, - PublishArtifactStagedSession { - parent: fork.root, - name: dname(b"data.bin"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:fork-append".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "fork-append/delta".to_owned(), - size: source_body.size + b" + self-owned delta".len() as u64, - chunks: written.chunk_manifests(), - staged: written.staged_objects().unwrap(), - mode: 0o644, - uid: 1000, - gid: 1000, - }, - ) + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") .unwrap(); - - service.remove_file_path("/base/data.bin").unwrap(); - let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); - service.set_clock_override_ms(pin.lease_expires_unix_ms); - assert_eq!( - service.reclaim_expired_snapshot_pins(100).unwrap().reaped, - 1 - ); - assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); - assert!(objects.head(&source_block).unwrap().is_some()); - assert_eq!( - read_artifact_at_path(&service, "/fork/data.bin"), - b"borrowed base + self-owned delta" - ); + store.release_blocked(); assert!(matches!( - service.retire_snapshot_path("/base", fork.snapshot_id), - Err(MetadError::ForkRetentionActive { - snapshot_id, - fork_root, - borrower, - }) if snapshot_id == fork.snapshot_id - && fork_root == fork.root - && borrower == fork_file.attr.inode + write.join().unwrap(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) )); - assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); - assert!(objects.head(&source_block).unwrap().is_some()); - assert_eq!( - read_artifact_at_path(&service, "/fork/data.bin"), - b"borrowed base + self-owned delta" - ); + assert!(service + .metadata_store() + .get( + RecordFamily::Xattr, + &xattr_key( + service.mount_id(), + ordinary_inode, + b"user.preplanned-before-restore", + ), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); } #[test] -fn fork_binding_retirement_fails_closed_on_corrupt_dentry_projection_body() { - let (service, objects) = service_with_objects(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); - let source = publish_path_artifact( - &service, - "/base/data.bin", - "fork-corruption/source", - b"must remain retained", - ); - let source_body = source.body.as_ref().unwrap(); - let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); - let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); - - let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); - service.set_clock_override_ms(pin.lease_expires_unix_ms); - assert_eq!( - service.reclaim_expired_snapshot_pins(100).unwrap().reaped, - 1 - ); +fn restore_release_cas_rejects_a_preplanned_checkpoint_publish() { + let store = SnapshotCommitBarrierStore::new(CommandKind::PublishArtifact, 0, 2) + .matching_request_prefix(b"publish-checkpoint"); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let restored = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + store.arm(1); - let dentry = dentry_key(service.mount, fork.root, &dname(b"data.bin")); - let current = service.read_version().unwrap(); - let row = service - .metadata - .get_versioned( - RecordFamily::Dentry, - &dentry, - current, - ReadPurpose::WritePlanLocal, + let writer = Arc::clone(&service); + let publish = std::thread::spawn(move || { + writer.publish_checkpoint( + restored.destination_root, + vec![CheckpointShard { + name: DentryName::new("raced-checkpoint.bin").unwrap(), + bytes: b"raced checkpoint bytes".to_vec(), + }], + 1000, + 1000, ) - .unwrap() + }); + store.wait_until_blocked(); + + service + .create_dir_path("/replacement", 0o755, 1000, 1000) .unwrap(); - let mut projection = decode_dentry_projection(&row.value.0).unwrap(); - projection - .body - .as_mut() - .unwrap() - .manifest_id - .push_str("/forged"); - let version = service.next_version().unwrap(); service - .commit_metadata(MetadataCommand { - request_id: request_id( - b"inject-corrupt-dentry-projection", - service.mount, - projection.attr.inode, - version, - ), - kind: CommandKind::UpdateAttr, - read_version: predecessor(version).unwrap(), - commit_version: version, - primary_family: RecordFamily::Dentry, - primary_key: dentry.clone(), - predicates: vec![PredicateRef { - family: RecordFamily::Dentry, - key: dentry.clone(), - predicate: Predicate::VersionEquals(row.version), - }], - mutations: vec![Mutation { - family: RecordFamily::Dentry, - key: dentry, - op: MutationOp::Put, - value: Some(Value(encode_dentry_projection(&projection))), - }], - watch: Vec::new(), - }) + .rename_replace_path("/replacement", "/destination") .unwrap(); - service.remove_file_path("/base/data.bin").unwrap(); + store.release_blocked(); - let error = service - .retire_snapshot_path("/base", fork.snapshot_id) - .unwrap_err(); - assert!( - matches!(&error, MetadError::Codec(message) if message.contains("body descriptor")), - "unexpected retirement error: {error:?}" - ); - assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); - assert!(objects.head(&source_block).unwrap().is_some()); + assert!(matches!( + publish.join().unwrap(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!(service + .metadata_store() + .get( + RecordFamily::Dentry, + &dentry_key( + service.mount_id(), + restored.destination_root, + &DentryName::new("raced-checkpoint.bin").unwrap(), + ), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + assert_eq!(objects.object_count(), 0); } #[test] -fn detached_fork_binding_protects_after_source_deletion_and_pin_reaping() { - let (service, objects) = service_with_objects(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); - let source = publish_path_artifact( - &service, - "/base/data.bin", - "detached-retention/source", - b"detached bytes", - ); - let source_body = source.body.as_ref().unwrap(); - let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); - let fork = service.clone_subtree_path("/base").unwrap(); +fn restore_release_cas_rejects_a_preplanned_dynamic_catalog_registration() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RegisterNamespaceIndex, 0, 2) + .matching_request_prefix(b"register-namespace-index"); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + MemoryObjectStore::new(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + service + .create_dir_path("/destination/dynamic", 0o755, 1000, 1000) + .unwrap(); + store.arm(1); - service.remove_file_path("/base/data.bin").unwrap(); - service.remove_empty_dir_path("/base").unwrap(); - let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); - service.set_clock_override_ms(pin.lease_expires_unix_ms); - assert_eq!( - service.reclaim_expired_snapshot_pins(100).unwrap().reaped, - 1 - ); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); - assert!(objects.head(&source_block).unwrap().is_some()); - assert_eq!( - service - .read_artifact(fork.root, &dname(b"data.bin")) - .unwrap(), - b"detached bytes" - ); + let writer = Arc::clone(&service); + let registration = std::thread::spawn(move || { + writer.register_namespace_index(NamespaceIndexRegistration { + path: "/destination/dynamic".to_owned(), + fields: vec![NamespaceIndexField { + field: NamespaceFindField::new("run.kind"), + operators: vec![NamespacePredicateOp::Eq], + sortable: true, + facetable: true, + }], + rows: Vec::new(), + }) + }); + store.wait_until_blocked(); + + service + .create_dir_path("/replacement", 0o755, 1000, 1000) + .unwrap(); + service + .rename_replace_path("/replacement", "/destination") + .unwrap(); + store.release_blocked(); - // A deleted source path cannot accidentally release the retention root. - // The unbound service primitive remains able to retire it after the detached - // fork has dropped its last borrowed reference. assert!(matches!( - service.retire_snapshot_path("/base", fork.snapshot_id), - Err(MetadError::NotFound) + registration.join().unwrap(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) )); - service.remove_file(fork.root, &dname(b"data.bin")).unwrap(); - assert!(service.retire_snapshot(fork.snapshot_id).unwrap()); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); - assert!(objects.head(&source_block).unwrap().is_none()); + assert!(service + .metadata_store() + .get( + RecordFamily::PathIndex, + &path_index_catalog_key(service.mount_id(), "/destination/dynamic"), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); } #[test] -fn fork_binding_can_be_retired_through_the_sources_new_path_after_rename() { - let (service, _objects) = service_with_objects(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); +fn complete_restore_root_can_replace_an_empty_ordinary_directory() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); publish_path_artifact( &service, - "/base/data.bin", - "renamed-retention/source", - b"renamed bytes", + "/source/original.bin", + "restore/move-original", + b"original bytes", ); - let fork = service.clone_subtree_path("/base").unwrap(); - service - .rename_path("/base", "/renamed") - .expect("source directory rename"); - service.remove_file(fork.root, &dname(b"data.bin")).unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let restored = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + let victim = service + .create_dir_path("/ordinary-victim", 0o755, 1000, 1000) + .unwrap(); - let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); - service.set_clock_override_ms(pin.lease_expires_unix_ms); + let moved = service + .rename_replace_path("/destination", "/ordinary-victim") + .unwrap(); + assert_eq!(moved.entry.attr.inode, restored.destination_root); + assert!(service.lookup_path("/destination").unwrap().is_none()); assert_eq!( - service.reclaim_expired_snapshot_pins(100).unwrap().reaped, - 1 + service.resolve_directory_path("/ordinary-victim").unwrap(), + restored.destination_root ); - assert!(matches!( - service.retire_snapshot_path("/base", fork.snapshot_id), - Err(MetadError::NotFound) - )); assert!(service - .retire_snapshot_path("/renamed", fork.snapshot_id) - .unwrap()); - assert!(!service.retire_snapshot(fork.snapshot_id).unwrap()); -} - -fn read_artifact_at_path( - service: &NoKvFs, - path: &str, -) -> Vec { - let (parent, name) = service.resolve_parent_path(path).unwrap(); - service.read_artifact(parent, &name).unwrap() + .metadata_store() + .get( + RecordFamily::Inode, + &inode_key(service.mount_id(), victim.attr.inode), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + assert_eq!(service.restore_metrics().unwrap().complete, 1); + assert_eq!(service.restore_metrics().unwrap().releasing, 0); + service + .create_file_path("/ordinary-victim/after-move", 0o644, 1000, 1000) + .unwrap(); } #[test] -fn failed_rollback_orphan_does_not_block_unrelated_fork_retirement() { - let store = SnapshotCommitBarrierStore::new(CommandKind::RenameReplace, 1, 2) - .matching_request_prefix(b"rollback-subtree-swap"); - let objects = MemoryObjectStore::new(); - let service = Arc::new(NoKvFs::new( - MountId::new(1).unwrap(), - store.clone(), - objects.clone(), - )); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); +fn restore_index_pagination_and_mvcc_folding_cross_internal_scan_pages() { + const ENTRY_COUNT: usize = 273; + const API_PAGE: usize = 13; - let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let service = service(); service - .publish_artifact(PublishArtifact { - parent: ws.attr.inode, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:a1".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "rollback-orphan/a1".to_owned(), - bytes: b"A1".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let mut custom_rows = Vec::with_capacity(ENTRY_COUNT); + for index in 0..ENTRY_COUNT { + let path = format!("/source/item-{index:03}.bin"); + publish_path_artifact( + &service, + &path, + &format!("paged-index/{index:03}"), + &[index as u8], + ); + custom_rows.push(NamespaceIndexRow { + path, + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("run.ordinal"), + value: NamespacePredicateValue::U64(index as u64), + }], + }); + } + service + .register_namespace_index(NamespaceIndexRegistration { + path: "/source".to_owned(), + fields: vec![NamespaceIndexField { + field: NamespaceFindField::new("run.ordinal"), + operators: vec![NamespacePredicateOp::Eq], + sortable: true, + facetable: true, + }], + rows: custom_rows, }) .unwrap(); - let snapshot = service.snapshot_subtree_path("/ws").unwrap(); - let ws_inode = ws.attr.inode; - let snapshot_id = snapshot.snapshot_id; + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/restored") + .unwrap(); + let restored_root = service.resolve_directory_path("/restored").unwrap(); - let rollback_service = Arc::clone(&service); - let rollback = - std::thread::spawn(move || rollback_service.rollback_subtree(ws_inode, snapshot_id)); - store.wait_until_blocked(); - assert!( - service.materialization_orphan_slow_path_enabled(), - "materialization must enter slow mode before its detached tree can commit" - ); + let mut current_names = Vec::new(); + let mut cursor = None; + loop { + let page = service + .list_indexed_path_page("/restored", cursor.as_ref(), API_PAGE) + .unwrap(); + assert!(page.entries.len() <= API_PAGE); + current_names.extend( + page.entries + .iter() + .map(|entry| entry.dentry.name.as_bytes().to_vec()), + ); + let Some(next) = page.next_cursor else { + break; + }; + assert!(cursor + .as_ref() + .is_none_or(|previous: &DentryName| previous.as_bytes() < next.as_bytes())); + cursor = Some(next); + } + assert_eq!(current_names.len(), ENTRY_COUNT); + assert!(current_names.windows(2).all(|pair| pair[0] < pair[1])); - // Change a swap-guarded dentry after rollback materialized its detached tree. - // The graft must fail, leaving that tree unreachable and without a binding. - // The rollback commit is deliberately paused while the service-local - // commit-to-log ordering gate is held. Model the conflicting write from a - // separately recovered owner so it can reach the shared store and invalidate - // the already-planned swap without deadlocking on that process-local gate. - let concurrent = - NoKvFs::open_existing(service.mount, store.clone(), objects, service.shard_index()) + let historical = service.snapshot_subtree_path("/restored").unwrap(); + for revision in 0..ENTRY_COUNT { + republish_path_artifact( + &service, + restored_root, + "item-000.bin", + &format!("paged-history/{revision:03}"), + &[revision as u8], + ); + } + let snapshot_version = Version::new(historical.read_version).unwrap(); + let mut historical_names = Vec::new(); + let mut cursor = None; + loop { + let page = service + .restore_indexed_children_page( + restored_root, + cursor.as_ref(), + API_PAGE, + snapshot_version, + ReadPurpose::Snapshot, + ) .unwrap(); - concurrent - .replace_artifact(PublishArtifact { - parent: ws_inode, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:a2".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "rollback-orphan/a2".to_owned(), - bytes: b"A2".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, + historical_names.extend( + page.entries + .iter() + .map(|entry| entry.dentry.name.as_bytes().to_vec()), + ); + let Some(next) = page.next_cursor else { + break; + }; + cursor = Some(next); + } + assert_eq!(historical_names, current_names); + let historical_custom = service + .restore_custom_index_at_path( + "/restored", + restored_root, + snapshot_version, + ReadPurpose::Snapshot, + ) + .unwrap() + .unwrap(); + assert_eq!(historical_custom.rows.len(), ENTRY_COUNT); +} + +#[test] +fn restore_index_materialization_and_seal_never_request_unbounded_pages() { + const ENTRY_COUNT: usize = 273; + + let metadata = PurposeTrackingStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let mut rows = Vec::with_capacity(ENTRY_COUNT); + for index in 0..ENTRY_COUNT { + let path = format!("/source/item-{index:03}.bin"); + publish_path_artifact( + &service, + &path, + &format!("bounded-index/{index:03}"), + &[index as u8], + ); + rows.push(NamespaceIndexRow { + path, + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("run.ordinal"), + value: NamespacePredicateValue::U64(index as u64), + }], + }); + } + service + .register_namespace_index(NamespaceIndexRegistration { + path: "/source".to_owned(), + fields: vec![NamespaceIndexField { + field: NamespaceFindField::new("run.ordinal"), + operators: vec![NamespacePredicateOp::Eq], + sortable: true, + facetable: true, + }], + rows, }) .unwrap(); - store.release_blocked(); - assert!(matches!( - rollback.join().unwrap(), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - )); - service.refresh_allocator_state().unwrap(); - assert_eq!( - service.read_artifact(ws_inode, &dname(b"a")).unwrap(), - b"A2" - ); - assert!(service - .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) - .unwrap() - .iter() - .all(|binding| binding.binding.snapshot_id != snapshot_id)); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); - // A real detached clone remains a retention root through its binding. Once - // its borrower is removed, the failed rollback's unbound orphan must not keep - // this otherwise-unrelated binding alive forever. - let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + metadata.reset_scan_bounds(); service - .publish_artifact(PublishArtifact { - parent: base.attr.inode, - name: dname(b"data"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:base".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "rollback-orphan/base".to_owned(), - bytes: b"base".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") .unwrap(); - let fork = service.clone_subtree_path("/base").unwrap(); - let fork_file = service - .lookup_plus(fork.root, &dname(b"data")) + + let counts = metadata.counts(); + assert_eq!(counts.restore_index_unbounded_scans, 0); + assert!( + counts.largest_restore_index_scan <= 256, + "restore index requested a {}-row page", + counts.largest_restore_index_scan + ); + let restored = service.resolve_directory_path("/restored").unwrap(); + let custom = service + .restore_custom_index_at_path( + "/restored", + restored, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) .unwrap() .unwrap(); - assert!(matches!( - service.retire_snapshot(fork.snapshot_id), - Err(MetadError::ForkRetentionActive { - snapshot_id, - fork_root, - borrower, - }) if snapshot_id == fork.snapshot_id - && fork_root == fork.root - && borrower == fork_file.attr.inode - )); - service.remove_file(fork.root, &dname(b"data")).unwrap(); - assert!(service.retire_snapshot(fork.snapshot_id).unwrap()); + assert_eq!(custom.rows.len(), ENTRY_COUNT); + let counts = metadata.counts(); + assert_eq!(counts.restore_index_unbounded_scans, 0); + assert!( + counts.largest_restore_index_scan <= 256, + "restore index query requested a {}-row physical page", + counts.largest_restore_index_scan + ); +} + +fn query_custom_index_without_dentry_scan( + service: &NoKvFs, + metadata: &PurposeTrackingStore, + path: &str, +) -> Option { + let root = service.resolve_directory_path(path).unwrap(); + let before = metadata.counts(); + let index = service + .restore_custom_index_at_path( + path, + root, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(); + let after = metadata.counts(); + assert_eq!( + after.dentry_scans, before.dentry_scans, + "custom index query for {path} enumerated the namespace" + ); + index } #[test] -fn failed_rollback_orphan_cannot_race_retirement_or_be_resurrected() { - let store = SnapshotCommitBarrierStore::new(CommandKind::RetireSnapshot, 1, 2) - .rejecting(CommandKind::RenameReplace); - let objects = MemoryObjectStore::new(); - let service = Arc::new(NoKvFs::new( +fn custom_index_query_fast_paths_do_not_enumerate_unindexed_or_canonical_namespaces() { + let metadata = PurposeTrackingStore::new(); + let service = NoKvFs::new( MountId::new(1).unwrap(), - store.clone(), - objects.clone(), - )); + metadata.clone(), + MemoryObjectStore::new(), + ); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); - let original = service - .publish_artifact(PublishArtifact { - parent: ws.attr.inode, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:orphan-a1".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "rollback-resurrection/a1".to_owned(), - bytes: b"A1".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) + service + .create_dir_path("/unindexed", 0o755, 1000, 1000) .unwrap(); - let original_body = original.body.as_ref().unwrap(); - let original_block = block_key(original.attr.inode, original_body.generation, 0, 0); - let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + let unindexed = service.resolve_directory_path("/unindexed").unwrap(); + let names = (0..1_025) + .map(|index| DentryName::new(format!("item-{index:04}").into_bytes()).unwrap()) + .collect::>(); service - .replace_artifact(PublishArtifact { - parent: ws.attr.inode, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:orphan-a2".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "rollback-resurrection/a2".to_owned(), - bytes: b"A2".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) + .create_files_in_dir(unindexed, names, 0o644, 1000, 1000) .unwrap(); + assert!(query_custom_index_without_dentry_scan(&service, &metadata, "/unindexed").is_none()); - // Reject only the atomic graft. All prior materialization commits remain, - // leaving an inode + dentry tree with neither a mount path nor ForkBinding. - assert!(matches!( - service.rollback_subtree(ws.attr.inode, snapshot.snapshot_id), - Err(MetadError::Metadata(MetadataError::PredicateFailed)) - )); - assert!( - service.materialization_orphan_slow_path_enabled(), - "failed materialization must leave inode exposure fail-closed" - ); - let (orphan_row, orphan) = service - .metadata - .scan(ScanRequest { - family: RecordFamily::Dentry, - prefix: dentry_mount_prefix(service.mount), - start_after: None, - version: service.read_version().unwrap(), - limit: 0, - purpose: ReadPurpose::WritePlanLocal, - }) - .unwrap() - .into_iter() - .find_map(|row| { - let projection = decode_dentry_projection(&row.value.0).unwrap(); - projection - .body - .as_ref() - .is_some_and(|body| body.manifest_id == "rollback-resurrection/a1") - .then_some((row, projection)) - }) - .expect("failed rollback leaves its materialized child"); - let orphan_parent = orphan.dentry.parent; - let orphan_inode = orphan.attr.inode; - let orphan_name = orphan.dentry.name.clone(); - assert_ne!(orphan_parent, ws.attr.inode); - assert!(service - .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) - .unwrap() - .iter() - .all(|binding| binding.binding.snapshot_id != snapshot.snapshot_id)); - - // Hold an unrelated retirement after its mount-wide proof but before its - // binding CAS. A racing hardlink must wait on object_gc_gate; after the CAS - // removes the last legal detached root, it must re-prove and reject the - // unbound rollback orphan instead of exposing it under the mount root. - service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/canonical", 0o755, 1000, 1000) + .unwrap(); publish_path_artifact( - service.as_ref(), - "/base/data", - "rollback-resurrection/base", - b"base", + &service, + "/canonical/indexed.bin", + "canonical/indexed", + b"indexed", ); - let fork = service.clone_subtree_path("/base").unwrap(); - service.remove_file(fork.root, &dname(b"data")).unwrap(); - - let retire_service = Arc::clone(&service); - let retire = std::thread::spawn(move || retire_service.retire_snapshot(fork.snapshot_id)); - store.wait_until_blocked(); - assert!(service.object_gc_gate.try_lock().is_err()); + service + .register_namespace_index(NamespaceIndexRegistration { + path: "/canonical".to_owned(), + fields: vec![NamespaceIndexField { + field: NamespaceFindField::new("run.kind"), + operators: vec![NamespacePredicateOp::Eq], + sortable: true, + facetable: true, + }], + rows: vec![NamespaceIndexRow { + path: "/canonical/indexed.bin".to_owned(), + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("run.kind"), + value: NamespacePredicateValue::String("canonical".to_owned()), + }], + }], + }) + .unwrap(); + let canonical = + query_custom_index_without_dentry_scan(&service, &metadata, "/canonical").unwrap(); + assert_eq!(canonical.rows.len(), 1); + assert_eq!(canonical.rows[0].path, "/canonical/indexed.bin"); +} - let (link_tx, link_rx) = std::sync::mpsc::channel(); - let link_service = Arc::clone(&service); - let racing_link = std::thread::spawn(move || { - let result = - link_service.link(orphan_inode, InodeId::root(), dname(b"racing-resurrection")); - link_tx.send(result).unwrap(); - }); - let early_link = link_rx.recv_timeout(Duration::from_millis(100)); - store.release_blocked(); - assert!(retire.join().unwrap().unwrap()); - let link_result = match early_link { - Ok(result) => result, - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => link_rx.recv().unwrap(), - Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { - panic!("racing hardlink thread disconnected") - } - }; - racing_link.join().unwrap(); - assert!( - matches!(link_result, Err(MetadError::NotFound)), - "unbound orphan hardlink result: {link_result:?}" +#[test] +fn restore_overlay_queries_are_index_row_driven_across_nested_and_generic_forks() { + let metadata = PurposeTrackingStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), ); - assert!(service - .lookup_plus(InodeId::root(), &dname(b"racing-resurrection")) - .unwrap() - .is_none()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact(&service, "/source/a.bin", "row-driven/a", b"a"); + publish_path_artifact(&service, "/source/b.bin", "row-driven/b", b"b"); + service + .register_namespace_index(NamespaceIndexRegistration { + path: "/source".to_owned(), + fields: vec![NamespaceIndexField { + field: NamespaceFindField::new("run.kind"), + operators: vec![NamespacePredicateOp::Eq], + sortable: true, + facetable: true, + }], + rows: vec![ + NamespaceIndexRow { + path: "/source/a.bin".to_owned(), + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("run.kind"), + value: NamespacePredicateValue::String("a".to_owned()), + }], + }, + NamespaceIndexRow { + path: "/source/b.bin".to_owned(), + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("run.kind"), + value: NamespacePredicateValue::String("b".to_owned()), + }], + }, + ], + }) + .unwrap(); - // Release the failed rollback's construction snapshot as well, allowing A1 - // to be reclaimed. The orphan metadata is now a dangling borrower; neither - // inode-addressed operation may make it reachable again. - assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert!(cleanup.deleted >= 1, "cleanup outcome: {cleanup:?}"); - assert!(objects.head(&original_block).unwrap().is_none()); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/restored") + .unwrap(); + let restored = + query_custom_index_without_dentry_scan(&service, &metadata, "/restored").unwrap(); + assert_eq!(restored.rows.len(), 2); + + // Post-restore namespace growth is deliberately unindexed. The scan + // assertion proves query cost is independent of that growth, so the + // creation-time one-million-entry restore limit cannot become a read-time + // failure after the destination grows beyond it. + let restored_root = service.resolve_directory_path("/restored").unwrap(); + let names = (0..1_025) + .map(|index| DentryName::new(format!("post-restore-{index:04}").into_bytes()).unwrap()) + .collect::>(); + for batch in names.chunks(128) { + service + .create_files_in_dir(restored_root, batch.to_vec(), 0o644, 1000, 1000) + .unwrap(); + } + let grown = query_custom_index_without_dentry_scan(&service, &metadata, "/restored").unwrap(); + assert_eq!(grown.rows.len(), 2); - let orphan_inode_key = inode_key(service.mount, orphan_inode); - let before_inode = service - .metadata - .get_versioned( - RecordFamily::Inode, - &orphan_inode_key, - service.read_version().unwrap(), - ReadPurpose::WritePlanLocal, - ) - .unwrap() + let restored_snapshot = service.snapshot_subtree_path("/restored").unwrap(); + service + .restore_subtree_path_to_fork("/restored", restored_snapshot.snapshot_id, "/nested") .unwrap(); + let nested = query_custom_index_without_dentry_scan(&service, &metadata, "/nested").unwrap(); + assert_eq!(nested.rows.len(), 2); + + service + .clone_subtree_path_into("/restored", "/generic") + .unwrap(); + let generic = query_custom_index_without_dentry_scan(&service, &metadata, "/generic").unwrap(); + assert_eq!(generic.rows.len(), 2); +} + +#[test] +fn restore_index_catalog_replacement_over_command_budget_is_atomic() { + const ENTRY_COUNT: usize = 520; + + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let mut rows = Vec::with_capacity(ENTRY_COUNT); + for index in 0..ENTRY_COUNT { + let path = format!("/source/item-{index:03}.bin"); + publish_path_artifact( + &service, + &path, + &format!("budget-index/{index:03}"), + &[index as u8], + ); + rows.push(NamespaceIndexRow { + path, + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("run.ordinal"), + value: NamespacePredicateValue::U64(index as u64), + }], + }); + } + let fields = vec![NamespaceIndexField { + field: NamespaceFindField::new("run.ordinal"), + operators: vec![NamespacePredicateOp::Eq], + sortable: true, + facetable: true, + }]; + service + .register_namespace_index(NamespaceIndexRegistration { + path: "/source".to_owned(), + fields: fields.clone(), + rows, + }) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + + let error = service + .register_namespace_index(NamespaceIndexRegistration { + path: "/restored".to_owned(), + fields, + rows: Vec::new(), + }) + .unwrap_err(); assert!(matches!( - service.link( - orphan_inode, - InodeId::root(), - dname(b"hardlink-resurrection") - ), - Err(MetadError::NotFound) - )); - assert!(matches!( - service.rename( - orphan_parent, - &orphan_name, - InodeId::root(), - dname(b"rename-resurrection") - ), - Err(MetadError::NotFound) + error, + MetadError::RestoreResourceLimit { + resource, + limit: 4096, + actual, + } if resource.contains("restore index") && actual > 4096 )); - let live = service - .lookup_plus(ws.attr.inode, &dname(b"a")) + let read_version = service.read_version().unwrap(); + assert!(service + .metadata_store() + .get( + RecordFamily::PathIndex, + &path_index_catalog_key(service.mount_id(), "/restored"), + read_version, + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + let restored = service.resolve_directory_path("/restored").unwrap(); + let inherited = service + .restore_custom_index_at_path("/restored", restored, read_version, ReadPurpose::UserStrong) .unwrap() .unwrap(); - assert!(matches!( - service.link( - live.attr.inode, - orphan_parent, - dname(b"orphan-destination-link") - ), - Err(MetadError::NotFound) - )); - assert!(matches!( - service.rename( - ws.attr.inode, - &dname(b"a"), - orphan_parent, - dname(b"orphan-destination-rename") - ), - Err(MetadError::NotFound) - )); + assert_eq!(inherited.rows.len(), ENTRY_COUNT); +} - let after_dentry = service - .metadata +fn assert_restore_command_limit(error: MetadError, expected_resource: &str, expected_limit: u64) { + match error { + MetadError::RestoreResourceLimit { + resource, + limit, + actual, + } => { + assert_eq!(resource, expected_resource); + assert_eq!(limit, expected_limit); + assert!(actual > limit, "resource limit must report an overflow"); + } + other => panic!("expected restore resource limit, got {other}"), + } +} + +fn seed_restored_hardlink_fanout( + service: &NoKvFs, + parent: InodeId, + name: &DentryName, + additional_links: usize, +) -> DentryProjection { + let version = service.next_version().unwrap(); + let read_version = predecessor(version).unwrap(); + let dentry = dentry_key(service.mount_id(), parent, name); + let dentry_item = service + .metadata_store() .get_versioned( RecordFamily::Dentry, - &orphan_row.key, - service.read_version().unwrap(), + &dentry, + read_version, ReadPurpose::WritePlanLocal, ) .unwrap() .unwrap(); - let after_inode = service - .metadata + let original = decode_dentry_projection(&dentry_item.value.0).unwrap(); + let inode_key = inode_key(service.mount_id(), original.attr.inode); + let inode_item = service + .metadata_store() .get_versioned( RecordFamily::Inode, - &orphan_inode_key, - service.read_version().unwrap(), + &inode_key, + read_version, ReadPurpose::WritePlanLocal, ) .unwrap() .unwrap(); - assert_eq!(after_dentry.version, orphan_row.version); - assert_eq!(after_dentry.value, orphan_row.value); - assert_eq!(after_inode.version, before_inode.version); - assert_eq!(after_inode.value, before_inode.value); - assert!(service - .lookup_plus(InodeId::root(), &dname(b"hardlink-resurrection")) - .unwrap() - .is_none()); - assert!(service - .lookup_plus(InodeId::root(), &dname(b"rename-resurrection")) - .unwrap() - .is_none()); - assert!(service - .lookup_plus(orphan_parent, &dname(b"orphan-destination-link")) - .unwrap() - .is_none()); - assert!(service - .lookup_plus(orphan_parent, &dname(b"orphan-destination-rename")) - .unwrap() - .is_none()); - assert!(objects.head(&original_block).unwrap().is_none()); - assert_eq!( - service.read_artifact(ws.attr.inode, &dname(b"a")).unwrap(), - b"A2" - ); + let mut attr = decode_inode_attr(&inode_item.value.0).unwrap(); + attr.nlink = u32::try_from(additional_links + 1).unwrap(); + attr.generation = version.get(); + attr.ctime_ms = current_time_ms(); + + let mut updated_original = original.clone(); + updated_original.attr = attr.clone(); + updated_original.dentry.attr_generation = attr.generation; + let restore_index = service + .restore_index_link_plan(&[(original.clone(), updated_original.clone())], version) + .unwrap(); + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::Dentry, + key: dentry.clone(), + predicate: Predicate::VersionEquals(dentry_item.version), + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode_key.clone(), + predicate: Predicate::VersionEquals(inode_item.version), + }, + ]; + predicates.extend(restore_index.predicates); + let mut mutations = vec![ + Mutation { + family: RecordFamily::Inode, + key: inode_key, + op: MutationOp::Put, + value: Some(Value(encode_inode_attr(&attr))), + }, + put_projection_mutation(RecordFamily::Dentry, dentry.clone(), &updated_original), + ]; + for index in 0..additional_links { + let link_name = DentryName::new(format!("fanout-{index:04}").into_bytes()).unwrap(); + let linked = projection( + parent, + link_name.clone(), + attr.clone(), + original.body.clone(), + ); + mutations.push(put_projection_mutation( + RecordFamily::Dentry, + dentry_key(service.mount_id(), parent, &link_name), + &linked, + )); + } + mutations.extend(restore_index.mutations); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-seed-restore-hardlink-fanout", + service.mount_id(), + original.attr.inode, + version, + ), + kind: CommandKind::Link, + read_version, + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: dentry, + predicates, + mutations, + watch: Vec::new(), + }) + .unwrap(); + updated_original } #[test] -fn rollback_binding_survives_pin_reaping_without_an_auxiliary_clone() { - let (service, objects) = service_with_objects(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); - let original = publish_path_artifact(&service, "/ws/a", "rollback-hold/a1", b"A1"); - let original_body = original.body.as_ref().unwrap(); - let original_block = block_key(original.attr.inode, original_body.generation, 0, 0); - let snap = service.snapshot_subtree_path("/ws").unwrap(); - let diverged = service - .replace_artifact(PublishArtifact { - parent: ws.attr.inode, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:a2".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "rollback-hold/a2".to_owned(), - bytes: b"A2".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) - .unwrap() - .entry; - let diverged_block = block_key( - diverged.attr.inode, - diverged.body.as_ref().unwrap().generation, - 0, - 0, - ); +fn restore_namespace_hardlink_fanout_item_limits_are_atomic() { + const ADDITIONAL_LINKS: usize = 2_050; + let service = service(); service - .rollback_subtree_path("/ws", snap.snapshot_id) - .unwrap(); - let restored = service - .lookup_plus(ws.attr.inode, &dname(b"a")) - .unwrap() + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - assert_ne!(restored.attr.inode, original.attr.inode); - let binding = service - .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) - .unwrap() - .into_iter() - .find(|binding| binding.binding.snapshot_id == snap.snapshot_id) - .expect("rollback installs durable retention"); - assert_eq!(binding.binding.source_root, ws.attr.inode); - assert_ne!(binding.binding.fork_root, ws.attr.inode); - assert!(service - .get_attr(binding.binding.fork_root) - .unwrap() - .is_none()); - - let pin = service.snapshot_pin(snap.snapshot_id).unwrap().unwrap(); - service.set_clock_override_ms(pin.lease_expires_unix_ms); - assert_eq!( - service.reclaim_expired_snapshot_pins(100).unwrap().reaped, - 1 + publish_path_artifact( + &service, + "/source/primary.bin", + "budget/primary", + b"primary", ); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); - assert!(cleanup.blocked_by_snapshots >= 1); - assert!(objects.head(&original_block).unwrap().is_some()); - assert!(objects.head(&diverged_block).unwrap().is_some()); - assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"A1"); - assert!(matches!( - service.retire_snapshot_path("/ws", snap.snapshot_id), - Err(MetadError::ForkRetentionActive { - snapshot_id, - fork_root, - borrower, - }) if snapshot_id == snap.snapshot_id - && fork_root == binding.binding.fork_root - && borrower == restored.attr.inode - )); - - let current = service - .replace_artifact(PublishArtifact { - parent: ws.attr.inode, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:a3".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "rollback-hold/a3".to_owned(), - bytes: b"A3".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) - .unwrap() - .entry; - let current_block = block_key( - current.attr.inode, - current.body.as_ref().unwrap().generation, - 0, - 0, + publish_path_artifact( + &service, + "/source/replacement.bin", + "budget/replacement", + b"replacement", ); - assert!(service - .retire_snapshot_path("/ws", snap.snapshot_id) - .unwrap()); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert!(cleanup.deleted >= 2, "cleanup outcome: {cleanup:?}"); - assert!(objects.head(&original_block).unwrap().is_none()); - assert!(objects.head(&diverged_block).unwrap().is_none()); - assert!(objects.head(¤t_block).unwrap().is_some()); - assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"A3"); -} - -#[test] -fn rollback_binding_protects_an_owner_gc_row_enqueued_after_the_swap() { - let (service, objects) = service_with_objects(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); - let source = publish_path_artifact(&service, "/ws/a", "rollback-late-owner", b"shared"); - let source_body = source.body.as_ref().unwrap(); - let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); - let snap = service.snapshot_subtree_path("/ws").unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); service - .link(source.attr.inode, InodeId::root(), dname(b"outside")) + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + let restored_root = service.resolve_directory_path("/restored").unwrap(); + let primary_name = DentryName::new("primary.bin").unwrap(); + let seeded = + seed_restored_hardlink_fanout(&service, restored_root, &primary_name, ADDITIONAL_LINKS); + let before_primary = service + .lookup_path("/restored/primary.bin") + .unwrap() .unwrap(); - service.remove_file(ws.attr.inode, &dname(b"a")).unwrap(); - assert_eq!(service.cleanup_pending_objects(100).unwrap().attempted, 0); - - service - .rollback_subtree_path("/ws", snap.snapshot_id) + let before_replacement = service + .lookup_path("/restored/replacement.bin") + .unwrap() .unwrap(); - let restored = service - .lookup_plus(ws.attr.inode, &dname(b"a")) + let before_sample = service + .lookup_path("/restored/fanout-2049") .unwrap() .unwrap(); - let pin = service.snapshot_pin(snap.snapshot_id).unwrap().unwrap(); - service.set_clock_override_ms(pin.lease_expires_unix_ms); + let before_index = service + .list_indexed_path_page("/restored", None, 10) + .unwrap(); + assert_eq!(before_primary.attr.nlink, (ADDITIONAL_LINKS + 1) as u32); + + let error = service + .link( + seeded.attr.inode, + restored_root, + DentryName::new("overflow-link.bin").unwrap(), + ) + .unwrap_err(); + assert_restore_command_limit(error, "restore namespace link items", 4_096); + assert!(service + .lookup_path("/restored/overflow-link.bin") + .unwrap() + .is_none()); assert_eq!( - service.reclaim_expired_snapshot_pins(100).unwrap().reaped, - 1 + service.lookup_path("/restored/primary.bin").unwrap(), + Some(before_primary.clone()) + ); + assert_eq!( + service.lookup_path("/restored/fanout-2049").unwrap(), + Some(before_sample.clone()) + ); + assert_eq!( + service.get_attr(seeded.attr.inode).unwrap(), + Some(seeded.attr.clone()) + ); + assert_eq!( + service + .list_indexed_path_page("/restored", None, 10) + .unwrap(), + before_index ); - service - .remove_file(InodeId::root(), &dname(b"outside")) + let error = service + .remove_file_path("/restored/primary.bin") + .unwrap_err(); + assert_restore_command_limit(error, "restore namespace remove file items", 4_096); + assert_eq!( + service.lookup_path("/restored/primary.bin").unwrap(), + Some(before_primary.clone()) + ); + assert_eq!( + service.lookup_path("/restored/fanout-2049").unwrap(), + Some(before_sample.clone()) + ); + + let (existing, dentry_version) = service + .lookup_plus_for_write_plan(restored_root, &primary_name) + .unwrap() .unwrap(); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); - assert!(objects.head(&source_block).unwrap().is_some()); - assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"shared"); - assert!(matches!( - service.retire_snapshot_path("/ws", snap.snapshot_id), - Err(MetadError::ForkRetentionActive { - snapshot_id, - borrower, - .. - }) if snapshot_id == snap.snapshot_id && borrower == restored.attr.inode - )); + let version = service.next_version().unwrap(); + let mut published_attr = existing.attr.clone(); + published_attr.generation = version.get(); + published_attr.ctime_ms = current_time_ms(); + published_attr.mtime_ms = published_attr.ctime_ms; + let mut published_body = existing.body.clone().unwrap(); + published_body.generation = version.get(); + published_body.base_generation = 0; + published_body.manifest_id = "budget/replaced".to_owned(); + let published = projection( + restored_root, + primary_name.clone(), + published_attr, + Some(published_body), + ); + let error = service + .commit_replace_projection_with_chunks(ReplaceProjectionCommit { + request_id: None, + kind: CommandKind::ReplaceArtifact, + projection: &published, + chunks: &[], + old_chunks: &[], + dentry_version, + old_generation: None, + version, + path_index: Some(path_index_key( + service.mount_id(), + &parse_absolute_path("/restored/primary.bin").unwrap(), + )), + object_reference: None, + }) + .unwrap_err(); + assert_restore_command_limit(error, "restore namespace publish replace items", 4_096); + assert_eq!( + service.lookup_path("/restored/primary.bin").unwrap(), + Some(before_primary.clone()) + ); + assert_eq!( + service + .list_indexed_path_page("/restored", None, 10) + .unwrap(), + before_index + ); + + let error = service + .rename_replace_path("/restored/replacement.bin", "/restored/primary.bin") + .unwrap_err(); + assert_restore_command_limit(error, "restore namespace rename items", 4_096); + assert_eq!( + service.lookup_path("/restored/primary.bin").unwrap(), + Some(before_primary) + ); + assert_eq!( + service.lookup_path("/restored/replacement.bin").unwrap(), + Some(before_replacement) + ); + assert_eq!( + service.lookup_path("/restored/fanout-2049").unwrap(), + Some(before_sample) + ); + assert_eq!( + service + .list_indexed_path_page("/restored", None, 10) + .unwrap(), + before_index + ); +} +fn install_large_body_for_restore_budget( + service: &NoKvFs, + path: &str, + producer_bytes: usize, +) { + let components = parse_absolute_path(path).unwrap(); + let (name, parent_components) = components.split_last().unwrap(); + let parent = service + .resolve_directory_path(&canonical_path(parent_components).unwrap()) + .unwrap(); + let version = service.next_version().unwrap(); + let read_version = predecessor(version).unwrap(); + let dentry_key = dentry_key(service.mount_id(), parent, name); + let dentry_item = service + .metadata_store() + .get_versioned( + RecordFamily::Dentry, + &dentry_key, + read_version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let old_projection = decode_dentry_projection(&dentry_item.value.0).unwrap(); + let inode_key = inode_key(service.mount_id(), old_projection.attr.inode); + let inode_item = service + .metadata_store() + .get_versioned( + RecordFamily::Inode, + &inode_key, + read_version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let index_key = path_index_key(service.mount_id(), &components); + let index_item = service + .metadata_store() + .get_versioned( + RecordFamily::PathIndex, + &index_key, + read_version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let mut attr = old_projection.attr.clone(); + attr.size = 0; + attr.generation = version.get(); + attr.mtime_ms = current_time_ms(); + attr.ctime_ms = attr.mtime_ms; + let body = BodyDescriptor { + producer: "x".repeat(producer_bytes), + digest_uri: "sha256:large-command-budget".to_owned(), + size: 0, + content_type: "application/octet-stream".to_owned(), + manifest_id: "large-command-budget".to_owned(), + generation: version.get(), + base_generation: 0, + chunk_size: DEFAULT_CHUNK_SIZE, + block_size: DEFAULT_BLOCK_SIZE as u64, + }; + let updated = projection(parent, name.clone(), attr.clone(), Some(body.clone())); service - .replace_artifact(PublishArtifact { - parent: ws.attr.inode, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:self-owned".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "rollback-late-owner/self-owned".to_owned(), - bytes: b"self-owned".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-install-large-restore-body", + service.mount_id(), + attr.inode, + version, + ), + kind: CommandKind::ReplaceArtifact, + read_version, + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: dentry_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::Dentry, + key: dentry_key.clone(), + predicate: Predicate::VersionEquals(dentry_item.version), + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode_key.clone(), + predicate: Predicate::VersionEquals(inode_item.version), + }, + PredicateRef { + family: RecordFamily::PathIndex, + key: index_key.clone(), + predicate: Predicate::VersionEquals(index_item.version), + }, + ], + mutations: vec![ + Mutation { + family: RecordFamily::Inode, + key: inode_key, + op: MutationOp::Put, + value: Some(Value(encode_inode_attr(&attr))), + }, + put_projection_mutation(RecordFamily::Dentry, dentry_key, &updated), + put_projection_mutation(RecordFamily::PathIndex, index_key, &updated), + Mutation { + family: RecordFamily::ChunkManifest, + key: chunk_manifest_key( + service.mount_id(), + attr.inode, + body.generation, + BODY_SUMMARY_CHUNK_INDEX, + ), + op: MutationOp::Put, + value: Some(Value(encode_body_descriptor(&body))), + }, + ], + watch: Vec::new(), }) .unwrap(); - assert!(service - .retire_snapshot_path("/ws", snap.snapshot_id) - .unwrap()); - assert!(service.cleanup_pending_objects(100).unwrap().deleted >= 1); - assert!(objects.head(&source_block).unwrap().is_none()); } #[test] -fn rollback_enqueues_a_missing_gc_row_for_a_restored_delta_base() { - let (service, objects) = service_with_objects(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); - let original = publish_path_artifact(&service, "/ws/a", "rollback-delta/a1", b"A1"); - let original_body = original.body.as_ref().unwrap(); - let original_generation = original_body.generation; - let original_block = block_key(original.attr.inode, original_generation, 0, 0); - let snap = service.snapshot_subtree_path("/ws").unwrap(); +fn restore_namespace_rename_full_command_byte_limit_is_atomic() { + const CATALOG_COUNT: usize = 27; + const LARGE_VALUE_BYTES: usize = 50_000; - let prepared = service - .prepare_artifact_replace(ws.attr.inode, dname(b"a")) - .unwrap(); - let written = service - .stage_prepared_artifact_ranges( - &prepared, - "rollback-delta/a2", - &[PublishArtifactRange { - offset: original_body.size, - bytes: b"-delta".to_vec(), - }], - 0, - ) + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let staged = written.staged_objects().unwrap(); - let chunks = written.chunk_manifests(); + let mut catalog_paths = vec!["/source".to_owned()]; + let mut parent_path = "/source".to_owned(); + for index in 1..CATALOG_COUNT { + parent_path.push_str(&format!("/d{index:02}")); + service + .create_dir_path(&parent_path, 0o755, 1000, 1000) + .unwrap(); + catalog_paths.push(parent_path.clone()); + } + let source_path = format!("{parent_path}/large.bin"); + publish_path_artifact(&service, &source_path, "budget/large", b"x"); + install_large_body_for_restore_budget(&service, &source_path, LARGE_VALUE_BYTES); + for catalog_path in catalog_paths { + service + .register_namespace_index(NamespaceIndexRegistration { + path: catalog_path, + fields: vec![NamespaceIndexField { + field: NamespaceFindField::new("budget.value"), + operators: vec![NamespacePredicateOp::Eq], + sortable: false, + facetable: false, + }], + rows: vec![NamespaceIndexRow { + path: source_path.clone(), + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("budget.value"), + value: NamespacePredicateValue::String("v".repeat(LARGE_VALUE_BYTES)), + }], + }], + }) + .unwrap(); + } + let snapshot = service.snapshot_subtree_path("/source").unwrap(); service - .publish_prepared_artifact_staged_session( - prepared, - PublishArtifactStagedSession { - parent: ws.attr.inode, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:a2".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "rollback-delta/a2".to_owned(), - size: original_body.size + 6, - chunks, - staged, - mode: 0o644, - uid: 1000, - gid: 1000, - }, + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + let restored_source_path = source_path.replacen("/source", "/restored", 1); + let restored_destination_path = format!("{restored_source_path}-renamed"); + let before = service.lookup_path(&restored_source_path).unwrap().unwrap(); + let restored_root = service.resolve_directory_path("/restored").unwrap(); + let before_index = service + .restore_custom_index_at_path( + "/restored", + restored_root, + service.read_version().unwrap(), + ReadPurpose::UserStrong, ) .unwrap(); - let appended = service.lookup_path("/ws/a").unwrap().unwrap(); + + let error = service + .rename_path(&restored_source_path, &restored_destination_path) + .unwrap_err(); + assert_restore_command_limit(error, "restore namespace rename bytes", 8 * 1024 * 1024); assert_eq!( - appended.body.as_ref().unwrap().base_generation, - original_generation + service.lookup_path(&restored_source_path).unwrap(), + Some(before) ); - assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"A1-delta"); - - service.remove_file(ws.attr.inode, &dname(b"a")).unwrap(); - let gc_records = || { + assert!(service + .lookup_path(&restored_destination_path) + .unwrap() + .is_none()); + assert_eq!( service - .metadata - .scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(service.mount), - start_after: None, - version: service.read_version().unwrap(), - limit: 0, - purpose: ReadPurpose::UserStrong, - }) - .unwrap() - .into_iter() - .map(|row| decode_object_gc_record(&row.value.0).unwrap()) - .collect::>() + .restore_custom_index_at_path( + "/restored", + restored_root, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + before_index + ); +} + +#[test] +fn ordinary_namespace_commands_keep_their_existing_budget_and_scan_behavior() { + const ENTRY_COUNT: usize = 1_400; + + let metadata = PurposeTrackingStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let ordinary = service + .create_dir_path("/ordinary", 0o755, 1000, 1000) + .unwrap(); + let names = (0..ENTRY_COUNT) + .map(|index| DentryName::new(format!("item-{index:04}").into_bytes()).unwrap()) + .collect::>(); + let before = metadata.counts(); + + let created = service + .create_files_in_dir(ordinary.attr.inode, names, 0o644, 1000, 1000) + .unwrap(); + + assert_eq!(created.len(), ENTRY_COUNT); + let after = metadata.counts(); + assert_eq!(after.user_strong_scans, before.user_strong_scans); + assert_eq!(after.write_plan_scans, before.write_plan_scans); + assert_eq!(after.snapshot_scans, before.snapshot_scans); + assert!(service + .lookup_plus(ordinary.attr.inode, &DentryName::new("item-1399").unwrap()) + .unwrap() + .is_some()); +} + +fn replace_path_artifact( + service: &NoKvFs, + path: &str, + manifest_id: &str, + bytes: &[u8], +) -> DentryWithAttr { + let prepared = service.prepare_artifact_replace_path(path).unwrap(); + let session = PublishArtifactSession { + parent: prepared.parent, + name: prepared.name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:test".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: manifest_id.to_owned(), + size: bytes.len() as u64, + ranges: vec![PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + mode: 0o644, + uid: 1000, + gid: 1000, }; - assert!( - gc_records() - .iter() - .all(|record| record.object_key != original_block.as_str()), - "deleting a delta top alone does not discover its historical base object" - ); + service + .publish_prepared_artifact_session(prepared, session) + .unwrap() + .entry +} +#[test] +fn path_replace_of_inherited_restore_entry_does_not_leak_a_canonical_index_row() { + let service = service(); service - .rollback_subtree_path("/ws", snap.snapshot_id) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"A1"); - assert!( - gc_records() - .iter() - .any(|record| record.object_key == original_block.as_str()), - "rollback must proactively make every restored owner object reclaimable" + publish_path_artifact( + &service, + "/source/inherited.bin", + "path-replace/source", + b"source", ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + let destination_components = parse_absolute_path("/destination/inherited.bin").unwrap(); + let destination_index = path_index_key(service.mount_id(), &destination_components); + assert!(service + .metadata_store() + .get( + RecordFamily::PathIndex, + &destination_index, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); - let pin = service.snapshot_pin(snap.snapshot_id).unwrap().unwrap(); - service.set_clock_override_ms(pin.lease_expires_unix_ms); + replace_path_artifact( + &service, + "/destination/inherited.bin", + "path-replace/destination", + b"destination-v2", + ); assert_eq!( - service.reclaim_expired_snapshot_pins(100).unwrap().reaped, - 1 + read_artifact_at_path(&service, "/destination/inherited.bin"), + b"destination-v2" ); - assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); - assert!(objects.head(&original_block).unwrap().is_some()); + assert!(service + .metadata_store() + .get( + RecordFamily::PathIndex, + &destination_index, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); - service.remove_file(ws.attr.inode, &dname(b"a")).unwrap(); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + service + .create_dir_path("/replacement", 0o755, 1000, 1000) + .unwrap(); + service + .rename_replace_path("/replacement", "/destination") + .unwrap(); + for _ in 0..512 { + service.cleanup_restore_releases(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); assert!(service - .retire_snapshot_path("/ws", snap.snapshot_id) - .unwrap()); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert!(cleanup.deleted >= 2, "cleanup outcome: {cleanup:?}"); - assert!(objects.head(&original_block).unwrap().is_none()); + .metadata_store() + .get( + RecordFamily::PathIndex, + &destination_index, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + assert!(service.fsck_restore_state(true).unwrap().is_consistent()); } #[test] -fn rollback_on_a_clone_root_keeps_both_retention_bindings() { +fn path_replace_of_a_later_hardlink_keeps_its_canonical_index_row() { let service = service(); - service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/base/a", "layered/base", b"base"); - let clone = service.clone_subtree_path_into("/base", "/fork").unwrap(); - let snap = service.snapshot_subtree_path("/fork").unwrap(); service - .replace_artifact(PublishArtifact { - parent: clone.root, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:diverged".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "layered/diverged".to_owned(), - bytes: b"diverged".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); + publish_path_artifact( + &service, + "/source/inherited.bin", + "path-replace-link/source", + b"source", + ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); service - .rollback_subtree_path("/fork", snap.snapshot_id) - .unwrap(); - - let bindings = service - .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") .unwrap(); - let clone_binding = bindings - .iter() - .find(|binding| binding.binding.snapshot_id == clone.snapshot_id) + let destination = service.resolve_directory_path("/destination").unwrap(); + let inherited = service + .lookup_path("/destination/inherited.bin") + .unwrap() .unwrap(); - let rollback_binding = bindings - .iter() - .find(|binding| binding.binding.snapshot_id == snap.snapshot_id) + service + .link( + inherited.attr.inode, + destination, + DentryName::new("later-link.bin").unwrap(), + ) .unwrap(); - assert_eq!(clone_binding.binding.fork_root, clone.root); - assert_eq!(rollback_binding.binding.source_root, clone.root); - assert_ne!(rollback_binding.binding.fork_root, clone.root); - assert_ne!(clone_binding.key, rollback_binding.key); - assert!(matches!( - service.retire_snapshot(clone.snapshot_id), - Err(MetadError::ForkRetentionActive { - snapshot_id, - .. - }) if snapshot_id == clone.snapshot_id - )); - assert!(matches!( - service.retire_snapshot(snap.snapshot_id), - Err(MetadError::ForkRetentionActive { - snapshot_id, - .. - }) if snapshot_id == snap.snapshot_id - )); - assert_eq!(read_artifact_at_path(&service, "/fork/a"), b"base"); -} + let link_components = parse_absolute_path("/destination/later-link.bin").unwrap(); + let link_index = path_index_key(service.mount_id(), &link_components); + assert!(service + .metadata_store() + .get( + RecordFamily::PathIndex, + &link_index, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); -#[test] -fn rollback_rejects_hardlinks_in_snapshot_or_current_tree() { - let snapshot_linked = service(); - snapshot_linked - .create_dir_path("/ws", 0o755, 1000, 1000) - .unwrap(); - let file = publish_path_artifact(&snapshot_linked, "/ws/a", "hardlink-snapshot", b"body"); - snapshot_linked - .link(file.attr.inode, InodeId::root(), dname(b"outside")) - .unwrap(); - let snap = snapshot_linked.snapshot_subtree_path("/ws").unwrap(); - let error = snapshot_linked - .rollback_subtree_path("/ws", snap.snapshot_id) - .unwrap_err(); - assert!(matches!( - &error, - MetadError::InvalidPath(message) if message.contains("hardlink-free") - )); + let replaced = replace_path_artifact( + &service, + "/destination/later-link.bin", + "path-replace-link/destination", + b"hardlink-v2", + ); + let canonical = service + .metadata_store() + .get( + RecordFamily::PathIndex, + &link_index, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .expect("later canonical hardlink must receive a PathIndex row"); + let canonical = decode_dentry_projection(&canonical.0).unwrap(); assert_eq!( - snapshot_linked - .read_artifact(InodeId::root(), &dname(b"outside")) - .unwrap(), - b"body" + canonical, + DentryProjection { + dentry: replaced.dentry.clone(), + attr: replaced.attr.clone(), + body: replaced.body.clone(), + } ); - - let current_linked = service(); - let ws = current_linked - .create_dir_path("/ws", 0o755, 1000, 1000) - .unwrap(); - let file = publish_path_artifact(¤t_linked, "/ws/a", "hardlink-current", b"body"); - let snap = current_linked.snapshot_subtree_path("/ws").unwrap(); - current_linked - .link(file.attr.inode, InodeId::root(), dname(b"outside")) - .unwrap(); - let error = current_linked - .rollback_subtree(ws.attr.inode, snap.snapshot_id) - .unwrap_err(); - assert!(matches!( - &error, - MetadError::InvalidPath(message) if message.contains("hardlink-free") - )); - assert_eq!(read_artifact_at_path(¤t_linked, "/ws/a"), b"body"); assert_eq!( - current_linked - .read_artifact(InodeId::root(), &dname(b"outside")) - .unwrap(), - b"body" + read_artifact_at_path(&service, "/destination/inherited.bin"), + b"hardlink-v2" ); + let indexed = service + .list_indexed_path_page("/destination", None, 10) + .unwrap(); + assert_eq!(indexed.entries.len(), 2); + assert!(indexed + .entries + .iter() + .any(|entry| entry.dentry.name.as_bytes() == b"later-link.bin")); + assert!(service.fsck_restore_state(true).unwrap().is_consistent()); } #[test] -fn rollback_subtree_restores_snapshot_shares_blocks_and_reclaims_delta() { - let (service, objects) = service_with_objects(); - // 1. Build /ws with files a="A1", b="B1", sub/c="C1" (real object data). - let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); +fn complete_restore_prepared_terminal_retries_survive_prune_epoch_and_log_replay() { + let metadata = RestorePostApplyFaultStore::new(); + let objects = FaultObjectStore::new(MemoryObjectStore::new()); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); service - .create_dir_path("/ws/sub", 0o755, 1000, 1000) + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/inherited.bin", + "prepared-terminal/source", + b"source-v1", + ); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/prepared-terminal-checkpoint", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/prepared-terminal-log", + "mount-1:/", + 1, + checkpoint.log_lsn, + checkpoint.log_digest, + )) + .unwrap(); + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/destination") .unwrap(); - let a = publish_path_artifact(&service, "/ws/a", "ws/a", &vec![b'1'; 4096]); - let b = publish_path_artifact(&service, "/ws/b", "ws/b", &vec![b'2'; 4096]); - let c = publish_path_artifact(&service, "/ws/sub/c", "ws/sub/c", &vec![b'3'; 4096]); - let a_gen = a.body.as_ref().unwrap().generation; - let b_gen = b.body.as_ref().unwrap().generation; - let c_gen = c.body.as_ref().unwrap().generation; - let a1_block = block_key(a.attr.inode, a_gen, 0, 0); - let b1_block = block_key(b.attr.inode, b_gen, 0, 0); - let c1_block = block_key(c.attr.inode, c_gen, 0, 0); - assert!(objects.head(&a1_block).unwrap().is_some()); - assert!(objects.head(&b1_block).unwrap().is_some()); - assert!(objects.head(&c1_block).unwrap().is_some()); - // 2. Snapshot /ws. - let snap = service.snapshot_subtree_path("/ws").unwrap(); + let prepared_create = service + .prepare_artifact_create_path("/destination/created.bin") + .unwrap(); + let create_bytes = b"created-after-restore"; + let create_written = service + .stage_prepared_artifact_ranges( + &prepared_create, + "prepared-terminal/create", + &[PublishArtifactRange { + offset: 0, + bytes: create_bytes.to_vec(), + }], + 0, + ) + .unwrap(); + let create_session = PublishArtifactStagedSession { + parent: prepared_create.parent, + name: prepared_create.name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:prepared-terminal-create".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "prepared-terminal/create".to_owned(), + size: create_bytes.len() as u64, + chunks: create_written.chunk_manifests(), + staged: create_written.staged_objects().unwrap(), + mode: 0o644, + uid: 1000, + gid: 1000, + }; + + let prepared_replace = service + .prepare_artifact_replace_path("/destination/inherited.bin") + .unwrap(); + let replace_bytes = b"destination-v2"; + let replace_written = service + .stage_prepared_artifact_ranges( + &prepared_replace, + "prepared-terminal/replace", + &[PublishArtifactRange { + offset: 0, + bytes: replace_bytes.to_vec(), + }], + 0, + ) + .unwrap(); + let replace_session = PublishArtifactStagedSession { + parent: prepared_replace.parent, + name: prepared_replace.name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:prepared-terminal-replace".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "prepared-terminal/replace".to_owned(), + size: replace_bytes.len() as u64, + chunks: replace_written.chunk_manifests(), + staged: replace_written.staged_objects().unwrap(), + mode: 0o644, + uid: 1000, + gid: 1000, + }; - // 3. Diverge /ws: rewrite a->"A2", add d="D1", delete b. - service - .replace_artifact(PublishArtifact { - parent: ws.attr.inode, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:a2".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "ws/a2".to_owned(), - bytes: vec![b'4'; 4096], - mode: 0o644, - uid: 1000, - gid: 1000, - }) + let publish_before = metadata.applied_with_prefix(b"publish-artifact"); + metadata.fail_after_apply(b"publish-artifact", 0); + let first_create = service + .publish_prepared_artifact_staged_session(prepared_create.clone(), create_session.clone()) .unwrap(); - let d = publish_path_artifact(&service, "/ws/d", "ws/d", &vec![b'5'; 4096]); - service.remove_file(ws.attr.inode, &dname(b"b")).unwrap(); - // Capture the delta's private blocks so we can assert their fate. - let a_diverged = service - .lookup_plus(ws.attr.inode, &dname(b"a")) - .unwrap() + assert_eq!( + metadata.applied_with_prefix(b"publish-artifact"), + publish_before + 1 + ); + + let replace_before = metadata.applied_with_prefix(b"replace-artifact"); + metadata.fail_after_apply(b"replace-artifact", 0); + let first_replace = service + .publish_prepared_artifact_staged_session(prepared_replace.clone(), replace_session.clone()) .unwrap(); - let a2_block = block_key( - a_diverged.attr.inode, - a_diverged.body.as_ref().unwrap().generation, - 0, - 0, + assert_eq!( + metadata.applied_with_prefix(b"replace-artifact"), + replace_before + 1 ); - let d1_block = block_key(d.attr.inode, d.body.as_ref().unwrap().generation, 0, 0); - assert!(objects.head(&a2_block).unwrap().is_some()); - assert!(objects.head(&d1_block).unwrap().is_some()); - // Pre-rollback /ws is the diverged state. - assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'4'; 4096]); + assert!(service - .lookup_plus(ws.attr.inode, &dname(b"b")) + .retire_snapshot(source_snapshot.snapshot_id) + .unwrap()); + assert_eq!(service.history_retention_floor().unwrap(), None); + let historical_dentry = dentry_key( + service.mount_id(), + prepared_replace.parent, + &prepared_replace.name, + ); + let prepared_read_version = predecessor(Version::new(prepared_replace.generation).unwrap()) + .expect("prepared generation has a predecessor"); + for _ in 0..128 { + service.cleanup_history(4_096).unwrap(); + if metadata + .get_versioned( + RecordFamily::Dentry, + &historical_dentry, + prepared_read_version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none() + { + break; + } + } + assert!(metadata + .get_versioned( + RecordFamily::Dentry, + &historical_dentry, + prepared_read_version, + ReadPurpose::WritePlanLocal, + ) .unwrap() .is_none()); + service.rotate_object_gc_claim_for_failover().unwrap(); - // 4. Roll /ws back to the snapshot. - service - .rollback_subtree_path("/ws", snap.snapshot_id) + let log_before_retry = service.sync_metadata_log_snapshot().unwrap(); + let stats_before_retry = service.metadata_store_stats(); + let restore_before_retry = service.restore_metrics().unwrap(); + let path_cache_epoch_before_retry = service.path_cache_epoch.load(Ordering::Acquire); + let retry_create = service + .publish_prepared_artifact_staged_session(prepared_create.clone(), create_session.clone()) .unwrap(); - - // 5. /ws now exactly matches the snapshot: a="A1", b="B1" (restored), sub/c="C1", - // and d is gone. The target keeps its inode identity. + let retry_replace = service + .publish_prepared_artifact_staged_session(prepared_replace.clone(), replace_session.clone()) + .unwrap(); + assert_eq!(retry_create.entry, first_create.entry); + assert_eq!(retry_replace.entry, first_replace.entry); + assert!(retry_create.replaced.is_none()); + assert!(retry_replace.replaced.is_none()); assert_eq!( - service.resolve_directory_path("/ws").unwrap(), - ws.attr.inode, - "rollback keeps the target root's identity" + metadata.applied_with_prefix(b"publish-artifact"), + publish_before + 1 ); - assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'1'; 4096]); - assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'2'; 4096]); assert_eq!( - read_artifact_at_path(&service, "/ws/sub/c"), - vec![b'3'; 4096] + metadata.applied_with_prefix(b"replace-artifact"), + replace_before + 1 ); - assert!( - service - .lookup_plus(ws.attr.inode, &dname(b"d")) - .unwrap() - .is_none(), - "the delta-only file d must be gone after rollback" + assert_eq!( + service.sync_metadata_log_snapshot().unwrap(), + log_before_retry + ); + let stats_after_retry = service.metadata_store_stats(); + assert_eq!( + stats_after_retry.dedupe_write_total, + stats_before_retry.dedupe_write_total + ); + assert_eq!( + stats_after_retry.current_put_total, + stats_before_retry.current_put_total + ); + assert_eq!( + stats_after_retry.current_delete_total, + stats_before_retry.current_delete_total + ); + assert_eq!( + stats_after_retry.history_write_total, + stats_before_retry.history_write_total + ); + assert_eq!( + stats_after_retry.watch_write_total, + stats_before_retry.watch_write_total + ); + assert_eq!(service.restore_metrics().unwrap(), restore_before_retry); + assert_eq!( + service.path_cache_epoch.load(Ordering::Acquire), + path_cache_epoch_before_retry, + "terminal readback without metadata apply must not invalidate path caches" + ); + assert_eq!( + read_artifact_at_path(&service, "/destination/created.bin"), + create_bytes + ); + assert_eq!( + read_artifact_at_path(&service, "/destination/inherited.bin"), + replace_bytes ); - // 6. The rolled-back /ws is identical to a fresh clone of the snapshot: an empty - // diff in both directions. - let reference = service - .clone_subtree_path_into("/ws", "/reference") - .unwrap(); - assert!(service - .diff_subtrees(ws.attr.inode, reference.root) - .unwrap() - .is_empty()); - assert!(service - .diff_subtrees(reference.root, ws.attr.inode) + let inherited_index = path_index_key( + service.mount_id(), + &parse_absolute_path("/destination/inherited.bin").unwrap(), + ); + let created_index = path_index_key( + service.mount_id(), + &parse_absolute_path("/destination/created.bin").unwrap(), + ); + assert!(metadata + .get( + RecordFamily::PathIndex, + &inherited_index, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) .unwrap() - .is_empty()); - - // Remove the reference fork itself. Its binding still cannot be retired: - // rollback propagated the same borrowed keys onto fresh /ws inodes, which - // the mount-global retirement proof must discover even though they were - // never descendants of `reference.root`. - let reference_sub = service - .lookup_plus(reference.root, &dname(b"sub")) + .is_none()); + assert!(metadata + .get( + RecordFamily::PathIndex, + &created_index, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) .unwrap() + .is_some()); + let indexed = service + .list_indexed_path_page("/destination", None, 16) .unwrap(); - service - .remove_file(reference_sub.attr.inode, &dname(b"c")) - .unwrap(); - service - .remove_empty_dir(reference.root, &dname(b"sub")) - .unwrap(); - service.remove_file(reference.root, &dname(b"a")).unwrap(); - service.remove_file(reference.root, &dname(b"b")).unwrap(); - service.remove_empty_dir_path("/reference").unwrap(); + assert_eq!(indexed.entries.len(), 2); + assert!(service.fsck_restore_state(true).unwrap().is_consistent()); - // 7. Both durable bindings fail closed while /ws still borrows the restored - // blocks. The rollback binding is intentionally conservative: its old - // mount-wide floor also delays the discarded delta until the borrowers - // are rewritten. - assert!(matches!( - service.retire_snapshot(snap.snapshot_id), - Err(MetadError::ForkRetentionActive { - snapshot_id, - .. - }) if snapshot_id == snap.snapshot_id - )); + let mut different_payload = create_session.clone(); + different_payload.mode = 0o600; assert!(matches!( - service.retire_snapshot(reference.snapshot_id), - Err(MetadError::ForkRetentionActive { - snapshot_id, - fork_root, - .. - }) if snapshot_id == reference.snapshot_id && fork_root == reference.root + service.publish_prepared_artifact_staged_session( + prepared_create.clone(), + different_payload, + ), + Err(MetadError::PublishArtifactFailed { source, .. }) + if matches!(*source, MetadError::StalePreparedArtifactObjectGcEpoch { .. }) )); - let reclaim = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(reclaim.deleted, 0, "cleanup outcome: {reclaim:?}"); - assert!(objects.head(&a2_block).unwrap().is_some()); - assert!(objects.head(&d1_block).unwrap().is_some()); - assert!( - objects.head(&a1_block).unwrap().is_some(), - "A1 must survive" - ); - assert!( - objects.head(&b1_block).unwrap().is_some(), - "B1 must survive" - ); - assert!( - objects.head(&c1_block).unwrap().is_some(), - "C1 must survive" - ); - // Restored content is still readable from the shared blocks after reclaim. - assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'1'; 4096]); - assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'2'; 4096]); assert_eq!( - read_artifact_at_path(&service, "/ws/sub/c"), - vec![b'3'; 4096] + service.sync_metadata_log_snapshot().unwrap(), + log_before_retry ); - // Once every rollback borrower owns a fresh generation, each binding can be - // retired independently and both the old snapshot blocks and discarded delta - // become reclaimable. - let restored_sub = service - .lookup_plus(ws.attr.inode, &dname(b"sub")) + let log = service.sync_metadata_log_snapshot().unwrap(); + let segment_keys = snapshot_segment_keys(&log); + let prepared_log_entries = log + .segments + .iter() + .map(|pointer| { + service + .load_metadata_log_segment(&pointer.segment_key) + .unwrap() + }) + .flat_map(|segment| segment.entries) + .filter(|entry| { + entry.command.commit_version.get() == prepared_create.generation + || entry.command.commit_version.get() == prepared_replace.generation + }) + .collect::>(); + assert_eq!(prepared_log_entries.len(), 2); + assert!(prepared_log_entries.iter().all(|entry| { + is_prepared_artifact_request_id(entry.command.kind, &entry.command.request_id) + })); + drop(service); + + let recovered_metadata = HoltMetadataStore::open_memory().unwrap(); + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + recovered_metadata, + objects.clone(), + ); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &segment_keys, + checkpoint.log_lsn, + checkpoint.log_digest, + ) .unwrap() .unwrap(); - let rewrite = |parent: InodeId, name: &[u8], manifest: &str, byte: u8| { - service - .replace_artifact(PublishArtifact { - parent, - name: dname(name), - producer: "unit-test".to_owned(), - digest_uri: format!("sha256:{manifest}"), - content_type: "application/octet-stream".to_owned(), - manifest_id: manifest.to_owned(), - bytes: vec![byte; 4096], - mode: 0o644, - uid: 1000, - gid: 1000, - }) + recovered + .enable_sync_metadata_log( + MetadataLogSyncConfig::new( + "meta/prepared-terminal-log", + "mount-1:/", + 2, + log.durable_lsn, + log.last_digest, + ) + .with_segments(log.segments.clone()) + .with_durable_recovery_baseline(checkpoint.log_lsn, checkpoint.log_digest) + .with_authoritative_recovery_baseline(), + ) + .unwrap(); + let recovered_log_before = recovered.sync_metadata_log_snapshot().unwrap(); + assert_eq!( + recovered + .publish_prepared_artifact_staged_session(prepared_create, create_session) .unwrap() - .entry - }; - let a3 = rewrite(ws.attr.inode, b"a", "ws/a3", b'6'); - let b3 = rewrite(ws.attr.inode, b"b", "ws/b3", b'7'); - let c3 = rewrite(restored_sub.attr.inode, b"c", "ws/c3", b'8'); - let a3_block = block_key(a3.attr.inode, a3.body.as_ref().unwrap().generation, 0, 0); - let b3_block = block_key(b3.attr.inode, b3.body.as_ref().unwrap().generation, 0, 0); - let c3_block = block_key(c3.attr.inode, c3.body.as_ref().unwrap().generation, 0, 0); - assert!(service.retire_snapshot(snap.snapshot_id).unwrap()); - assert!(service.retire_snapshot(reference.snapshot_id).unwrap()); - let reclaim = service.cleanup_pending_objects(100).unwrap(); - assert!(reclaim.deleted >= 5, "cleanup outcome: {reclaim:?}"); - for old in [&a1_block, &b1_block, &c1_block, &a2_block, &d1_block] { - assert!(objects.head(old).unwrap().is_none(), "old block {old:?}"); - } - for current in [&a3_block, &b3_block, &c3_block] { - assert!( - objects.head(current).unwrap().is_some(), - "current block {current:?}" - ); - } - assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'6'; 4096]); - assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'7'; 4096]); + .entry, + first_create.entry + ); assert_eq!( - read_artifact_at_path(&service, "/ws/sub/c"), - vec![b'8'; 4096] + recovered + .publish_prepared_artifact_staged_session(prepared_replace, replace_session) + .unwrap() + .entry, + first_replace.entry + ); + assert_eq!( + recovered.sync_metadata_log_snapshot().unwrap(), + recovered_log_before + ); + assert_eq!( + read_artifact_at_path(&recovered, "/destination/created.bin"), + create_bytes + ); + assert_eq!( + read_artifact_at_path(&recovered, "/destination/inherited.bin"), + replace_bytes ); + let report = recovered.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); } #[test] -fn rollback_subtree_rejects_an_expired_snapshot_before_materializing() { - let service = service(); - service.set_clock_override_ms(1_000); - let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/ws/a", "rollback-expired", b"original"); - let pin = service - .snapshot_subtree_with_lease(ws.attr.inode, 500) +fn complete_restore_prepared_retry_archives_pending_log_without_reapply() { + let metadata = RestorePostApplyFaultStore::new(); + let objects = FaultObjectStore::new(MemoryObjectStore::new()); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/inherited.bin", + "prepared-pending/source", + b"source", + ); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/prepared-pending-checkpoint", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/prepared-pending-log", + "mount-1:/", + 1, + checkpoint.log_lsn, + checkpoint.log_digest, + )) + .unwrap(); + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/destination") + .unwrap(); + let prepared = service + .prepare_artifact_create_path("/destination/pending.bin") + .unwrap(); + let bytes = b"pending-log-terminal"; + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "prepared-pending/create", + &[PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + 0, + ) .unwrap(); + let session = PublishArtifactStagedSession { + parent: prepared.parent, + name: prepared.name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:prepared-pending".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "prepared-pending/create".to_owned(), + size: bytes.len() as u64, + chunks: written.chunk_manifests(), + staged: written.staged_objects().unwrap(), + mode: 0o644, + uid: 1000, + gid: 1000, + }; + let log_before = service.sync_metadata_log_snapshot().unwrap(); + let applies_before = metadata.applied_with_prefix(b"publish-artifact"); - service.set_clock_override_ms(1_500); + objects.fail_puts_containing("meta/prepared-pending-log/log/"); assert!(matches!( - service.rollback_subtree(ws.attr.inode, pin.snapshot_id), - Err(MetadError::SnapshotLeaseExpired { - snapshot_id, - lease_expires_unix_ms: 1_500, - now_ms: 1_500, - }) if snapshot_id == pin.snapshot_id + service.publish_prepared_artifact_staged_session(prepared.clone(), session.clone()), + Err(MetadError::PublishArtifactFailed { source, .. }) + if matches!(*source, MetadError::SyncLogArchiveFailed { committed: true, .. }) )); - assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"original"); -} + assert_eq!( + metadata.applied_with_prefix(b"publish-artifact"), + applies_before + 1 + ); + assert_eq!( + service.sync_metadata_log_snapshot().unwrap().durable_lsn, + log_before.durable_lsn + ); -#[test] -fn rollback_holds_the_gc_gate_until_restored_references_are_live() { - let store = SnapshotCommitBarrierStore::new(CommandKind::RenameReplace, 1, 2); - let objects = MemoryObjectStore::new(); - let service = Arc::new(NoKvFs::new( - MountId::new(1).unwrap(), - store.clone(), - objects.clone(), - )); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); - let ws_inode = ws.attr.inode; - let original = service - .publish_artifact(PublishArtifact { - parent: ws.attr.inode, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:original".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "rollback-race/original".to_owned(), - bytes: b"original".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) - .unwrap(); - let original_body = original.body.as_ref().unwrap(); - let original_object = block_key(original.attr.inode, original_body.generation, 0, 0); - let pin = service - .snapshot_subtree_with_lease(ws.attr.inode, 500) + objects.clear_faults(); + let retry = service + .publish_prepared_artifact_staged_session(prepared.clone(), session.clone()) .unwrap(); - service - .replace_artifact(PublishArtifact { - parent: ws.attr.inode, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:delta".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "rollback-race/delta".to_owned(), - bytes: b"delta".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, + assert_eq!(retry.entry.attr.generation, prepared.generation); + assert_eq!( + metadata.applied_with_prefix(b"publish-artifact"), + applies_before + 1, + "retry must archive the pending command, not apply it again" + ); + let archived = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(archived.durable_lsn, log_before.durable_lsn + 1); + let matching_entries = archived + .segments + .iter() + .map(|pointer| { + service + .load_metadata_log_segment(&pointer.segment_key) + .unwrap() }) + .flat_map(|segment| segment.entries) + .filter(|entry| entry.command.commit_version.get() == prepared.generation) + .collect::>(); + assert_eq!(matching_entries.len(), 1); + assert!(is_prepared_artifact_request_id( + matching_entries[0].command.kind, + &matching_entries[0].command.request_id, + )); + let stable_log = archived.clone(); + service + .publish_prepared_artifact_staged_session(prepared, session) .unwrap(); + assert_eq!(service.sync_metadata_log_snapshot().unwrap(), stable_log); + assert_eq!( + read_artifact_at_path(&service, "/destination/pending.bin"), + bytes + ); + assert!(service.fsck_restore_state(true).unwrap().is_consistent()); - let rollback_service = Arc::clone(&service); - let rollback = - std::thread::spawn(move || rollback_service.rollback_subtree(ws_inode, pin.snapshot_id)); - store.wait_until_blocked(); - assert!(service.object_gc_gate.try_lock().is_err()); - - let deadline_ms = start_ms + 500; - service.set_clock_override_ms(deadline_ms); - let cleanup_started = Arc::new(Barrier::new(2)); - let cleanup_service = Arc::clone(&service); - let cleanup_thread_started = Arc::clone(&cleanup_started); - let cleanup = std::thread::spawn(move || { - cleanup_thread_started.wait(); - cleanup_service.cleanup_pending_objects(100) - }); - cleanup_started.wait(); - store.release_blocked(); - - rollback.join().unwrap().unwrap(); - cleanup.join().unwrap().unwrap(); - assert!(objects.head(&original_object).unwrap().is_some()); + let segment_keys = snapshot_segment_keys(&archived); + drop(service); + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &segment_keys, + checkpoint.log_lsn, + checkpoint.log_digest, + ) + .unwrap() + .unwrap(); assert_eq!( - service.read_artifact(ws_inode, &dname(b"a")).unwrap(), - b"original" + read_artifact_at_path(&recovered, "/destination/pending.bin"), + bytes ); + let report = recovered.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); } #[test] -fn rollback_subtree_rejects_foreign_or_missing_snapshot() { - let service = service(); - service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); +fn late_prepared_terminal_requires_complete_lsn_chain_after_checkpoint() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata, objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); service - .create_dir_path("/other", 0o755, 1000, 1000) + .create_dir_path("/destination", 0o755, 1000, 1000) .unwrap(); - let other_root = service.resolve_directory_path("/other").unwrap(); - let snap = service.snapshot_subtree_path("/other").unwrap(); - // A snapshot of /other cannot roll back /ws. - assert!(matches!( - service.rollback_subtree_path("/ws", snap.snapshot_id), - Err(MetadError::InvalidPath(_)) - )); - // An unknown snapshot id is not found. - assert!(matches!( - service.rollback_subtree_path("/ws", snap.snapshot_id + 9_999), - Err(MetadError::NotFound) - )); - // The rejected target is untouched and the legitimate one still works. - assert!(service - .rollback_subtree(other_root, snap.snapshot_id) - .is_ok()); -} + let checkpoint_config = MetadataArchiveConfig::new("meta/late-prepared-checkpoint", 3); + let initial = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/late-prepared-log", + "mount-1:/", + 1, + initial.log_lsn, + initial.log_digest, + )) + .unwrap(); -#[test] -fn metadata_backup_then_restore_into_fresh_store_recovers_namespace() { - let (service, objects) = service_with_objects(); - // Build a small namespace; file bodies land in the shared object store. - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - service.create_dir_path("/data", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/runs/a.bin", "m-a", b"alpha-body"); - publish_path_artifact(&service, "/data/b.bin", "m-b", b"bravo-body-2"); - let want_runs = service.lookup_path("/runs/a.bin").unwrap(); - let want_data = service.lookup_path("/data/b.bin").unwrap(); - assert!(want_runs.is_some()); + let prepared = service + .prepare_artifact_create_path("/destination/late.bin") + .unwrap(); + let bytes = b"late-prepared-payload"; + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "late-prepared", + &[PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + 0, + ) + .unwrap(); + let session = PublishArtifactStagedSession { + parent: prepared.parent, + name: prepared.name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:late-prepared".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "late-prepared".to_owned(), + size: bytes.len() as u64, + chunks: written.chunk_manifests(), + staged: written.staged_objects().unwrap(), + mode: 0o644, + uid: 1000, + gid: 1000, + }; - let config = MetadataArchiveConfig::new("meta/checkpoints", 3); - let backup = service.backup_metadata(&config).unwrap(); - assert!(backup.image_bytes > 0); - assert!(backup.checkpoint_key.starts_with("meta/checkpoints/ckpt/")); + for index in 0..4 { + service + .create_dir_path(&format!("/advance-{index}"), 0o755, 1000, 1000) + .unwrap(); + } + let baseline = service.backup_metadata(&checkpoint_config).unwrap(); + assert!(prepared.generation < baseline.commit_version); - // Simulate total loss of the metadata node: a brand-new empty Holt store, - // pointed at the SAME object store (the clone shares the backing map). - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), - ); - // The fresh node has no namespace at all (not even a root) until restore. - let outcome = recovered.restore_metadata(&config).unwrap(); + service + .create_dir_path("/after-baseline", 0o755, 1000, 1000) + .unwrap(); + let first = service + .publish_prepared_artifact_staged_session(prepared.clone(), session.clone()) + .unwrap(); + let published_log = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(service.prepared_terminal_proof_cache_len(), Some(1)); + + let prepared_pointer = published_log + .segments + .iter() + .find(|pointer| { + service + .load_metadata_log_segment(&pointer.segment_key) + .unwrap() + .entries + .iter() + .any(|entry| { + entry.command.commit_version.get() == prepared.generation + && is_prepared_artifact_request_id( + entry.command.kind, + &entry.command.request_id, + ) + }) + }) + .cloned() + .expect("prepared publish segment"); + let prior_active_pointer = published_log + .segments + .iter() + .find(|pointer| { + pointer.first_lsn > baseline.log_lsn && pointer.last_lsn < prepared_pointer.first_lsn + }) + .cloned() + .expect("active segment before prepared publish"); + + for pointer in [&prior_active_pointer, &prepared_pointer] { + let key = ObjectKey::new(pointer.segment_key.clone()).unwrap(); + let original = objects.get(&key, None).unwrap(); + objects.put(&key, b"corrupt-log-segment".to_vec()).unwrap(); + assert!(service + .publish_prepared_artifact_staged_session(prepared.clone(), session.clone()) + .is_err()); + assert_eq!(service.prepared_terminal_proof_cache_len(), Some(1)); + objects.put(&key, original).unwrap(); + } + + let log_before_bad_ack = service.sync_metadata_log_snapshot().unwrap(); + let proof_count_before_bad_ack = service.prepared_terminal_proof_cache_len(); + let mut wrong_digest = log_before_bad_ack.last_digest; + wrong_digest[0] ^= 0xff; + assert!(service + .acknowledge_published_metadata_checkpoint(log_before_bad_ack.durable_lsn, wrong_digest,) + .is_err()); + assert_eq!( + service.sync_metadata_log_snapshot().unwrap(), + log_before_bad_ack + ); assert_eq!( - outcome.as_ref().map(|o| o.checkpoint_key.as_str()), - Some(backup.checkpoint_key.as_str()) + service.prepared_terminal_proof_cache_len(), + proof_count_before_bad_ack ); - // Namespace entries (dentry + attr + body descriptor) are identical after - // restore, and a subsequent create allocates a fresh, non-colliding inode. - assert_eq!(recovered.lookup_path("/runs/a.bin").unwrap(), want_runs); - assert_eq!(recovered.lookup_path("/data/b.bin").unwrap(), want_data); - let fresh = publish_path_artifact(&recovered, "/runs/c.bin", "m-c", b"charlie"); - assert_ne!(fresh.attr.inode, want_runs.unwrap().attr.inode); -} - -#[test] -fn restore_metadata_without_archive_returns_none() { - let (service, _objects) = service_with_objects(); - let config = MetadataArchiveConfig::new("meta/empty", 3); - assert!(service.restore_metadata(&config).unwrap().is_none()); -} - -#[test] -fn restore_rejects_a_pre_fence_archive_before_installing_its_image() { - let (service, objects) = service_with_objects(); service - .create_dir_path("/unsafe", 0o755, 1000, 1000) + .acknowledge_published_metadata_checkpoint(baseline.log_lsn, baseline.log_digest) .unwrap(); - let config = MetadataArchiveConfig::new("meta/legacy", 3); - let backup = service.backup_metadata(&config).unwrap(); - - // Downgrade only CURRENT to the legacy format. The referenced image is - // actually valid and fenced, so an empty target proves rejection happened - // from the pointer proof before install_checkpoint_image could mutate it. - let current_key = ObjectKey::new("meta/legacy/CURRENT").unwrap(); - let current = String::from_utf8(objects.get(¤t_key, None).unwrap()).unwrap(); - let legacy = current - .lines() - .enumerate() - .filter_map(|(index, line)| { - if line.starts_with("object_gc_failover_fenced\t") { - None - } else if index == 0 { - Some("nokv-metadata-archive\t1".to_owned()) - } else { - Some(line.to_owned()) - } - }) - .collect::>() - .join("\n") - + "\n"; - objects.put(¤t_key, legacy.into_bytes()).unwrap(); + assert_eq!(service.prepared_terminal_proof_cache_len(), Some(1)); - let recovered = NoKvFs::new( + let checkpoint_only = NoKvFs::new( MountId::new(1).unwrap(), HoltMetadataStore::open_memory().unwrap(), - objects, + objects.clone(), ); - assert!(matches!( - recovered.restore_metadata(&config), - Err(MetadError::MetadataArchiveMissingObjectGcFence { checkpoint_key }) - if checkpoint_key == backup.checkpoint_key - )); - assert!(recovered.get_attr(InodeId::root()).unwrap().is_none()); -} + checkpoint_only + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &[], + baseline.log_lsn, + baseline.log_digest, + ) + .unwrap() + .unwrap(); + assert!(checkpoint_only + .lookup_path("/destination/late.bin") + .unwrap() + .is_none()); -#[test] -fn metadata_backup_retains_only_keep_last_checkpoints() { - let (service, objects) = service_with_objects(); - let config = MetadataArchiveConfig::new("meta/ck", 2); - let b1 = service.backup_metadata(&config).unwrap(); - let _b2 = service.backup_metadata(&config).unwrap(); - let b3 = service.backup_metadata(&config).unwrap(); - // keep_last=2: the third backup prunes exactly the first checkpoint object. - assert_eq!(b3.pruned, 1); - let pruned = ObjectKey::new(b1.checkpoint_key.clone()).unwrap(); - assert!(objects.head(&pruned).unwrap().is_none()); - let live = ObjectKey::new(b3.checkpoint_key.clone()).unwrap(); - assert!(objects.head(&live).unwrap().is_some()); - // Restore (into a fresh store) always selects the latest checkpoint. + let tail_segment_keys = published_log + .segments + .iter() + .filter(|pointer| pointer.first_lsn > baseline.log_lsn) + .map(|pointer| pointer.segment_key.clone()) + .collect::>(); let recovered = NoKvFs::new( MountId::new(1).unwrap(), HoltMetadataStore::open_memory().unwrap(), objects.clone(), ); - let restored = recovered.restore_metadata(&config).unwrap().unwrap(); - assert_eq!(restored.checkpoint_key, b3.checkpoint_key); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &tail_segment_keys, + baseline.log_lsn, + baseline.log_digest, + ) + .unwrap() + .unwrap(); + assert_eq!( + read_artifact_at_path(&recovered, "/destination/late.bin"), + bytes + ); + + service.disable_sync_metadata_log().unwrap(); + service + .enable_sync_metadata_log( + MetadataLogSyncConfig::new( + "meta/late-prepared-log", + "mount-1:/", + 2, + published_log.durable_lsn, + published_log.last_digest, + ) + .with_segments(published_log.segments.clone()) + .with_durable_recovery_baseline(baseline.log_lsn, baseline.log_digest) + .with_authoritative_recovery_baseline(), + ) + .unwrap(); + assert_eq!(service.prepared_terminal_proof_cache_len(), Some(0)); + assert_eq!( + service + .publish_prepared_artifact_staged_session(prepared.clone(), session.clone()) + .unwrap() + .entry, + first.entry + ); + + service.disable_sync_metadata_log().unwrap(); + objects + .delete(&ObjectKey::new(prepared_pointer.segment_key).unwrap()) + .unwrap(); + service + .enable_sync_metadata_log( + MetadataLogSyncConfig::new( + "meta/late-prepared-log", + "mount-1:/", + 3, + published_log.durable_lsn, + published_log.last_digest, + ) + .with_segments(published_log.segments) + .with_durable_recovery_baseline(baseline.log_lsn, baseline.log_digest) + .with_authoritative_recovery_baseline(), + ) + .unwrap(); + assert!(service + .publish_prepared_artifact_staged_session(prepared, session) + .is_err()); } -fn log_test_command(request_id: &[u8], commit_version: u64) -> MetadataCommand { - MetadataCommand { - request_id: request_id.to_vec(), - kind: CommandKind::CreateFile, - read_version: Version::new(commit_version - 1).unwrap(), - commit_version: Version::new(commit_version).unwrap(), - primary_family: RecordFamily::Dentry, - primary_key: b"log-primary".to_vec(), - predicates: vec![PredicateRef { - family: RecordFamily::Dentry, - key: b"log-primary".to_vec(), - predicate: Predicate::NotExists, - }], - mutations: vec![Mutation { - family: RecordFamily::Dentry, - key: b"log-primary".to_vec(), - op: MutationOp::Put, - value: Some(Value(b"log-value".to_vec())), +fn restore_test_initialization() -> RestoreInitialization { + RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: br#"{"schema":"nokv.workbench.restore_manifest.v1"}"#.to_vec(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, }], - watch: Vec::new(), } } -fn log_test_entry( - request_id: &[u8], - lsn: u64, - commit_version: u64, - prev_digest: [u8; 32], -) -> MetadataLogEntry { - MetadataLogEntry::seal( - "mount-1:/runs", - 1, - lsn, - log_test_command(request_id, commit_version), - CommitResult { - commit_version: Version::new(commit_version).unwrap(), - applied_mutations: 1, - watch_events: 0, - }, - prev_digest, - ) - .unwrap() -} - -fn snapshot_segment_keys(snapshot: &MetadataLogSyncSnapshot) -> Vec { - snapshot - .segments - .iter() - .map(|segment| segment.segment_key.clone()) - .collect() +fn retry_restore_until_complete( + service: &NoKvFs, + snapshot_id: u64, + initialization: &RestoreInitialization, +) -> RestoreOutcome +where + M: MetadataStore, + O: ObjectStore, +{ + for _ in 0..4096 { + match service.restore_subtree_path_to_fork_initialized( + "/source", + snapshot_id, + "/destination", + initialization.clone(), + ) { + Ok(outcome) => return outcome, + Err(MetadError::RestoreInProgress) => {} + Err(error) => panic!("restore retry failed: {error}"), + } + } + panic!("bounded restore retry did not converge") } -fn log_replay_command( - request_id: &[u8], - key: &[u8], - value: &[u8], - read_version: u64, - commit_version: u64, -) -> MetadataCommand { - MetadataCommand { - request_id: request_id.to_vec(), - kind: CommandKind::RegisterNamespaceIndex, - read_version: Version::new(read_version).unwrap(), - commit_version: Version::new(commit_version).unwrap(), - primary_family: RecordFamily::System, - primary_key: key.to_vec(), - predicates: vec![PredicateRef { - family: RecordFamily::System, - key: key.to_vec(), - predicate: Predicate::NotExists, - }], - mutations: vec![Mutation { - family: RecordFamily::System, - key: key.to_vec(), - op: MutationOp::Put, - value: Some(Value(value.to_vec())), - }], - watch: Vec::new(), - } +fn durable_restore_operation( + service: &NoKvFs, + snapshot_id: u64, +) -> restore::RestoreOperation +where + M: MetadataStore, + O: ObjectStore, +{ + let digest = restore::restore_operation_digest( + service.mount_id(), + "/source", + snapshot_id, + "/destination", + ); + restore::decode_restore_operation( + &service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_operation_key(service.mount_id(), &digest), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .expect("restore operation must be durable") + .0, + ) + .unwrap() } -fn ordered_log_put_command( - request_id: &[u8], - key: &[u8], - value: &[u8], - read_version: u64, - commit_version: u64, -) -> MetadataCommand { - MetadataCommand { - request_id: request_id.to_vec(), - kind: CommandKind::RegisterNamespaceIndex, - read_version: Version::new(read_version).unwrap(), - commit_version: Version::new(commit_version).unwrap(), - primary_family: RecordFamily::System, - primary_key: key.to_vec(), - predicates: Vec::new(), - mutations: vec![Mutation { - family: RecordFamily::System, - key: key.to_vec(), - op: MutationOp::Put, - value: Some(Value(value.to_vec())), - }], - watch: Vec::new(), - } +fn replace_durable_restore_operation( + service: &NoKvFs, + durable_key_digest: [u8; 32], + operation: &restore::RestoreOperation, +) where + M: MetadataStore, + O: ObjectStore, +{ + let operation_key = restore::restore_operation_key(service.mount_id(), &durable_key_digest); + let operation_item = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &operation_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .expect("restore operation must be durable"); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-replace-restore-operation", + service.mount_id(), + operation.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: operation_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::VersionEquals(operation_item.version), + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: operation_key, + op: MutationOp::Put, + value: Some(Value(restore::encode_restore_operation(operation).unwrap())), + }], + watch: Vec::new(), + }) + .unwrap(); } #[test] -fn metadata_log_segment_archive_round_trips_through_object_store() { - let (service, objects) = service_with_objects(); - let first = log_test_entry(b"req-log-1", 1, 11, METADATA_LOG_ZERO_DIGEST); - let second = log_test_entry(b"req-log-2", 2, 12, first.digest); - let segment = MetadataLogSegment::seal(vec![first, second]).unwrap(); - let config = MetadataLogArchiveConfig::new("meta/shared-log"); - - let archived = service - .archive_metadata_log_segment(&config, &segment) +fn restore_operation_identity_rejects_a_corrupted_snapshot_id_on_durable_read_paths() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - - assert!(archived.segment_key.starts_with("meta/shared-log/log/")); - assert!(archived - .segment_key - .contains("/00000000000000000001-00000000000000000002-")); - assert!(archived.segment_key.ends_with(".segment")); - assert_eq!(archived.first_lsn, 1); - assert_eq!(archived.last_lsn, 2); - assert_eq!( - archived.encoded_bytes, - segment.encode().unwrap().len() as u64 + service + .create_file_path("/source/empty.txt", 0o644, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let restored = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/source", + snapshot.snapshot_id, + "/destination", ); - assert!(objects - .head(&ObjectKey::new(archived.segment_key.clone()).unwrap()) - .unwrap() - .is_some()); + let mut corrupted = durable_restore_operation(&service, snapshot.snapshot_id); + corrupted.snapshot_id = corrupted.snapshot_id.checked_add(1).unwrap(); + replace_durable_restore_operation(&service, operation_digest, &corrupted); - let loaded = service - .load_metadata_log_segment(&archived.segment_key) - .unwrap(); - assert_eq!(loaded, segment); + assert!(matches!( + service.restore_subtree_path_to_fork( + "/source", + snapshot.snapshot_id, + "/destination", + ), + Err(MetadError::Codec(message)) + if message.contains("request identity") + )); + assert!(matches!( + service.is_complete_restore_root(restored.destination_root), + Err(MetadError::Codec(message)) + if message.contains("request identity") + )); + assert!(matches!( + service.recover_restore_staging_visibility(), + Err(MetadError::Codec(message)) + if message.contains("request identity") + )); + + let report = service.fsck_restore_state(false).unwrap(); + assert!(report + .issues + .iter() + .any(|issue| issue.code == "operation_identity_mismatch")); } #[test] -fn metadata_log_archive_config_rejects_keys_outside_the_exact_shard_prefix() { - let config = MetadataLogArchiveConfig::new("meta/shared-log/mount_1__"); - let valid = concat!( - "meta/shared-log/mount_1__/log/", - "00000000000000000001-00000000000000000001-", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.segment" +fn restore_operation_identity_rejects_a_corrupted_value_digest() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .create_file_path("/source/empty.txt", 0o644, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let restored = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/source", + snapshot.snapshot_id, + "/destination", ); - config.validate_segment_key(valid).unwrap(); + let mut corrupted = durable_restore_operation(&service, snapshot.snapshot_id); + corrupted.operation_digest = [0xa5; 32]; + assert_ne!(corrupted.operation_digest, operation_digest); + replace_durable_restore_operation(&service, operation_digest, &corrupted); - for invalid in [ - concat!( - "meta/shared-log/mount_2__/log/", - "00000000000000000001-00000000000000000001-", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.segment" - ), - concat!( - "meta/shared-log/mount_1__/log/nested/", - "00000000000000000001-00000000000000000001-", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.segment" + assert!(matches!( + service.restore_subtree_path_to_fork( + "/source", + snapshot.snapshot_id, + "/destination", ), - "meta/shared-log/mount_1__/log/", - ] { - assert!(matches!( - config.validate_segment_key(invalid), - Err(MetadError::Codec(_)) - )); - } + Err(MetadError::Codec(message)) + if message.contains("durable key") + )); + assert!(matches!( + service.is_complete_restore_root(restored.destination_root), + Err(MetadError::Codec(message)) + if message.contains("durable key") + )); + assert!(matches!( + service.recover_restore_staging_visibility(), + Err(MetadError::Codec(message)) + if message.contains("durable key") + )); + + let report = service.fsck_restore_state(false).unwrap(); + assert!(report + .issues + .iter() + .any(|issue| issue.code == "operation_identity_mismatch")); } #[test] -fn metadata_log_segment_load_rejects_corrupted_object() { - let (service, objects) = service_with_objects(); - let first = log_test_entry(b"req-log-1", 1, 11, METADATA_LOG_ZERO_DIGEST); - let segment = MetadataLogSegment::seal(vec![first]).unwrap(); - let config = MetadataLogArchiveConfig::new("meta/shared-log"); - let archived = service - .archive_metadata_log_segment(&config, &segment) +fn concurrent_identical_restore_requests_create_one_root_ref_set_and_manifest_put() { + let metadata = RestorePostApplyFaultStore::new(); + let objects = RestorePostPutFaultStore::new(MemoryObjectStore::new()); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let key = ObjectKey::new(archived.segment_key.clone()).unwrap(); - objects.put(&key, b"corrupted-segment".to_vec()).unwrap(); + service + .create_dir_path("/source/metadata", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let initialization = restore_test_initialization(); + objects.clear_put_history(); + let start = Arc::new(Barrier::new(17)); + let mut calls = Vec::new(); + for _ in 0..16 { + let service = Arc::clone(&service); + let start = Arc::clone(&start); + let initialization = initialization.clone(); + calls.push(std::thread::spawn(move || { + start.wait(); + service.restore_subtree_path_to_fork_initialized( + "/source", + snapshot.snapshot_id, + "/destination", + initialization, + ) + })); + } + start.wait(); + let outcomes = calls + .into_iter() + .map(|call| call.join().unwrap().unwrap()) + .collect::>(); + assert!(outcomes.windows(2).all(|pair| pair[0] == pair[1])); + assert_eq!(outcomes[0].state, RestoreState::Complete); + assert_eq!( + metadata.applied_with_prefix(b"restore-install-hold"), + 1, + "sixteen simultaneous callers must install one durable hold/root" + ); + assert_eq!( + metadata.applied_with_prefix(b"restore-start-base-refs"), + 1, + "one ref-set build must own the terminal operation" + ); + assert_eq!(metadata.applied_with_prefix(b"restore-seal-base-refs"), 1); + assert_eq!( + metadata.applied_with_prefix(b"restore-init-upload-intent"), + 1 + ); + assert_eq!(metadata.applied_with_prefix(b"restore-init-publish"), 1); + let put_keys = objects.put_keys(); + assert_eq!( + put_keys.len(), + 1, + "the deterministic restore manifest must be uploaded once" + ); + assert!(put_keys[0].as_str().starts_with("blocks/")); - assert!(matches!( - service.load_metadata_log_segment(&archived.segment_key), - Err(MetadError::Codec(_)) - )); + let metrics = service.restore_metrics().unwrap(); + assert_eq!(metrics.complete, 1); + for keyspace in ["operation", "destination_claim", "root_index", "base_seal"] { + assert_eq!( + metrics.control_rows.get(keyspace), + Some(&1), + "restore must leave exactly one {keyspace} row" + ); + } + assert!(service.fsck_restore_state(true).unwrap().is_consistent()); } -#[test] -fn restore_metadata_with_archived_log_segments_replays_after_checkpoint() { - let (service, objects) = service_with_objects(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-log-replay", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); +fn exercise_restore_post_apply_crash( + request_prefix: &[u8], + zero_based_match: usize, + empty_entry_count: usize, + artifact_count: usize, + expected_state: restore::RestoreOperationState, + rebuild_expected: bool, +) { + let metadata = RestorePostApplyFaultStore::new(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + if empty_entry_count > 0 { + let names = (0..empty_entry_count) + .map(|index| DentryName::new(format!("empty-{index:04}.bin")).unwrap()) + .collect(); + service + .create_files_in_dir_path("/source", names, 0o644, 1000, 1000) + .unwrap(); + } + for index in 0..artifact_count { + publish_path_artifact( + &service, + &format!("/source/artifact-{index:03}.bin"), + &format!("restore/crash-reference-{index:03}"), + format!("payload-{index:03}").as_bytes(), + ); + } + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + metadata.fail_after_apply(request_prefix, zero_based_match); - let key = b"log-replay-system-key".to_vec(); - let value = b"after-checkpoint".to_vec(); - let commit_version = checkpoint.commit_version + 1; - let command = log_replay_command( - b"req-log-replay", - &key, - &value, - checkpoint.commit_version, - commit_version, + let error = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap_err(); + assert!( + matches!( + &error, + MetadError::Metadata(MetadataError::Backend(message)) + if message == "injected restore crash after durable apply" + ), + "unexpected crash result for {} occurrence {zero_based_match}: {error:?}", + String::from_utf8_lossy(request_prefix), ); - let result = service.commit_metadata(command.clone()).unwrap(); - let entry = - MetadataLogEntry::seal("mount-1:/", 1, 1, command, result, METADATA_LOG_ZERO_DIGEST) - .unwrap(); - let segment = MetadataLogSegment::seal(vec![entry.clone()]).unwrap(); - let log_config = MetadataLogArchiveConfig::new("meta/shared-log-replay"); - let archived = service - .archive_metadata_log_segment(&log_config, &segment) - .unwrap(); + assert_eq!( + metadata.applied_with_prefix(request_prefix), + zero_based_match + 1 + ); + let crashed = durable_restore_operation(&service, snapshot.snapshot_id); + assert_eq!(crashed.state, expected_state); + if expected_state == restore::RestoreOperationState::Complete { + assert!(service.lookup_path("/destination").unwrap().is_some()); + } else { + assert!(service.lookup_path("/destination").unwrap().is_none()); + assert!(service + .get_attr(crashed.destination_root) + .unwrap() + .is_none()); + } + drop(service); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), + metadata.clear_fault(); + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let reopened_operation = durable_restore_operation(&reopened, snapshot.snapshot_id); + assert_eq!( + reopened_operation.state, expected_state, + "reopen must not advance the restore state machine" ); - let outcome = recovered - .restore_metadata_with_archived_log_segments( - &checkpoint_config, - "mount-1:/", - std::slice::from_ref(&archived.segment_key), + if expected_state != restore::RestoreOperationState::Complete { + assert!(reopened.lookup_path("/destination").unwrap().is_none()); + } + let outcome = retry_restore_until_complete( + &reopened, + snapshot.snapshot_id, + &RestoreInitialization::default(), + ); + assert_eq!(outcome.state, RestoreState::Complete); + if rebuild_expected { + assert_ne!( + outcome.destination_root, crashed.destination_root, + "Preparing recovery must discard and rebuild its detached root" + ); + } else { + assert_eq!( + outcome.destination_root, crashed.destination_root, + "sealed ReadyToAttach/Complete recovery must retain its durable root" + ); + } + assert!(reopened.lookup_path("/destination").unwrap().is_some()); + let report = reopened.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); +} + +#[test] +fn restore_post_apply_crashes_rebuild_hold_and_four_materialization_batches() { + exercise_restore_post_apply_crash( + b"restore-install-hold", + 0, + 1, + 0, + restore::RestoreOperationState::Preparing, + true, + ); + for batch in 0..4 { + exercise_restore_post_apply_crash( + b"restore-materialize-batch", + batch, + 257, 0, - METADATA_LOG_ZERO_DIGEST, - ) - .unwrap() - .unwrap(); + restore::RestoreOperationState::Preparing, + true, + ); + } +} - assert_eq!(outcome.checkpoint.checkpoint_key, checkpoint.checkpoint_key); - assert_eq!(outcome.replayed_entries, 1); - assert_eq!(outcome.durable_lsn, 1); - assert_eq!(outcome.last_digest, entry.digest); - assert!(recovered.read_version().unwrap().get() >= commit_version); - assert_eq!( - recovered - .metadata - .get( - RecordFamily::System, - &key, - Version::new(commit_version).unwrap(), - ReadPurpose::UserStrong - ) - .unwrap(), - Some(Value(value)) +#[test] +fn restore_post_apply_crashes_rebuild_index_and_reference_seals() { + // Reference-page apply-then-ACK loss is authoritatively reconciled from + // the durable build cursor; the adaptive byte-budget test below covers + // that path through reopen. Seal commits still surface the injected error + // and require the explicit Preparing cleanup/rebuild contract. + exercise_restore_post_apply_crash( + b"restore-seal-index", + 0, + 3, + 0, + restore::RestoreOperationState::Preparing, + true, + ); + exercise_restore_post_apply_crash( + b"restore-seal-base-refs", + 0, + 0, + 130, + restore::RestoreOperationState::Preparing, + true, + ); +} + +#[test] +fn restore_reopen_only_attaches_sealed_ready_state_on_explicit_retry() { + exercise_restore_post_apply_crash( + b"restore-ready-to-attach", + 0, + 3, + 0, + restore::RestoreOperationState::ReadyToAttach, + false, + ); +} + +#[test] +fn restore_attach_ack_loss_reopens_at_the_exact_terminal_outcome() { + exercise_restore_post_apply_crash( + b"restore-attach-destination", + 0, + 3, + 0, + restore::RestoreOperationState::Complete, + false, ); } #[test] -fn log_replay_recovers_partial_materialization_as_an_unbound_orphan() { +fn restore_attach_publication_failure_stays_committed_until_exact_retry() { let metadata = HoltMetadataStore::open_memory().unwrap(); - let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata, objects.clone()); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); - publish_path_artifact( - &service, - "/base/data.bin", - "replay-orphan/source", - b"borrowed", + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/source", + snapshot.snapshot_id, + "/destination", ); - let pin = service.snapshot_subtree(base.attr.inode).unwrap(); service .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/replay-orphan-log", + "metadata/restore-attach-publication-failure", "mount-1:/", 1, 0, METADATA_LOG_ZERO_DIGEST, )) .unwrap(); - - // The checkpoint is healthy. Only the subsequent detached materialization - // commits enter the retained log tail; no ForkBinding commit follows them. - let checkpoint_config = MetadataArchiveConfig::new("meta/replay-orphan-checkpoint", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); - assert!(!service.materialization_orphan_slow_path_enabled()); - let orphan_root = { - let _object_gc_gate = service.object_gc_gate.lock().unwrap(); - service - .materialize_subtree_at(base.attr.inode, Version::new(pin.read_version).unwrap()) - .unwrap() - }; - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert!(snapshot.durable_lsn > checkpoint.log_lsn); - assert!(!snapshot.segments.is_empty()); - let segments = snapshot - .segments - .iter() - .map(|pointer| service.load_metadata_log_segment(&pointer.segment_key)) - .collect::, _>>() + let attach_publication_failures = Arc::new(AtomicU64::new(0)); + let observed_failures = Arc::clone(&attach_publication_failures); + service + .install_sync_metadata_log_publication_hook(move |_| { + let operation = metadata + .get( + RecordFamily::System, + &restore::restore_operation_key(MountId::new(1).unwrap(), &operation_digest), + Version::new(u64::MAX).unwrap(), + ReadPurpose::WritePlanLocal, + ) + .map_err(|error| error.to_string())? + .map(|value| restore::decode_restore_operation(&value.0)) + .transpose() + .map_err(|error| error.to_string())?; + if operation.is_some_and(|operation| { + operation.state == restore::RestoreOperationState::Complete + }) { + observed_failures.fetch_add(1, Ordering::SeqCst); + return Err("injected attach control publication failure".to_owned()); + } + Ok(()) + }) .unwrap(); - assert!(segments - .iter() - .flat_map(|segment| &segment.entries) - .flat_map(|entry| &entry.command.mutations) - .all(|mutation| mutation.family != RecordFamily::ForkBinding)); + live_test_barrier::reset_test_attach_applied_calls(); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects, + let error = service + .with_immediate_sync_metadata_log_publication(|| { + service.restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + }) + .unwrap_err(); + assert!(matches!( + &error, + MetadError::SyncLogArchiveFailed { + committed: true, + message, + } if message.contains("injected attach control publication failure") + )); + assert_eq!(attach_publication_failures.load(Ordering::SeqCst), 1); + assert_eq!(live_test_barrier::test_attach_applied_calls(), 0); + assert_eq!( + durable_restore_operation(&service, snapshot.snapshot_id).state, + restore::RestoreOperationState::Complete ); - let outcome = recovered - .restore_metadata_with_log_segments( - &checkpoint_config, - "mount-1:/", - &segments, - checkpoint.log_lsn, - checkpoint.log_digest, - ) - .unwrap() + assert!(service.lookup_path("/destination").unwrap().is_some()); + + let retry = service + .with_immediate_sync_metadata_log_publication(|| { + service.restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + }) .unwrap(); - assert_eq!(outcome.replayed_entries, 2); + assert_eq!(retry.state, RestoreState::Complete); + assert_eq!(attach_publication_failures.load(Ordering::SeqCst), 1); + assert_eq!(live_test_barrier::test_attach_applied_calls(), 0); +} + +#[test] +fn restore_initialization_post_put_crash_rebuilds_without_reusing_the_orphan() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = RestorePostPutFaultStore::new(MemoryObjectStore::new()); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .create_dir_path("/source/metadata", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let initialization = restore_test_initialization(); + objects.clear_put_history(); + objects.fail_after_put(0); + + let error = service + .restore_subtree_path_to_fork_initialized( + "/source", + snapshot.snapshot_id, + "/destination", + initialization.clone(), + ) + .unwrap_err(); assert!( - recovered.materialization_orphan_slow_path_enabled(), - "partial materialization replay must remain fail-closed" + matches!( + &error, + MetadError::Object(ObjectError::StagedWriteFailed { source, staged }) + if source.contains("injected restore crash after object PUT") && staged.is_empty() + ), + "unexpected initialization crash result: {error:?}" ); - let orphan = recovered - .lookup_plus(orphan_root, &dname(b"data.bin")) - .unwrap() - .unwrap(); - assert!(matches!( - recovered.link(orphan.attr.inode, InodeId::root(), dname(b"replayed-link")), - Err(MetadError::NotFound) + let crashed = durable_restore_operation(&service, snapshot.snapshot_id); + assert_eq!(crashed.state, restore::RestoreOperationState::Preparing); + let orphan = objects.put_keys().into_iter().next().unwrap(); + assert!(objects.head(&orphan).unwrap().is_some()); + drop(service); + + let reopened = + NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects.clone(), 0).unwrap(); + assert_eq!( + durable_restore_operation(&reopened, snapshot.snapshot_id).state, + restore::RestoreOperationState::Preparing + ); + assert!(reopened.lookup_path("/destination").unwrap().is_none()); + let outcome = retry_restore_until_complete(&reopened, snapshot.snapshot_id, &initialization); + assert_ne!(outcome.destination_root, crashed.destination_root); + let puts = objects.put_keys(); + assert_eq!(puts.len(), 2); + assert_ne!(puts[0], puts[1]); + assert!(objects.head(&orphan).unwrap().is_none()); + assert!(objects.head(&puts[1]).unwrap().is_some()); + assert_eq!( + read_artifact_at_path(&reopened, "/destination/metadata/restore_manifest.json"), + br#"{"schema":"nokv.workbench.restore_manifest.v1"}"# + ); + assert!(reopened.fsck_restore_state(true).unwrap().is_consistent()); +} + +#[test] +fn restore_materialization_rejects_gc_epoch_rotation_then_rebuilds_exact_refs() { + let metadata = SnapshotCommitBarrierStore::new(CommandKind::CreateFiles, 1, 2) + .matching_request_prefix(b"restore-materialize-batch"); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let source = publish_path_artifact( + &service, + "/source/shared.bin", + "restore/gc-epoch-race", + b"borrowed across GC epoch", + ); + let source_body = source.body.as_ref().unwrap(); + let shared_object = block_key(source.attr.inode, source_body.generation, 0, 0); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + + let restore = { + let service = Arc::clone(&service); + std::thread::spawn(move || { + service.restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + }) + }; + metadata.wait_until_blocked(); + service.rotate_object_gc_claim_for_failover().unwrap(); + metadata.release_blocked(); assert!(matches!( - recovered.rename( - orphan_root, - &dname(b"data.bin"), - InodeId::root(), - dname(b"replayed-rename") - ), - Err(MetadError::NotFound) + restore.join().unwrap(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) )); + let interrupted = durable_restore_operation(&service, snapshot.snapshot_id); + assert_eq!(interrupted.state, restore::RestoreOperationState::Preparing); + assert!(service.lookup_path("/destination").unwrap().is_none()); + + let completed = retry_restore_until_complete( + &service, + snapshot.snapshot_id, + &RestoreInitialization::default(), + ); + assert_ne!(completed.destination_root, interrupted.destination_root); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + service.remove_file_path("/source/shared.bin").unwrap(); + service.remove_empty_dir_path("/source").unwrap(); + for _ in 0..64 { + service.cleanup_pending_objects(1).unwrap(); + } + assert!(objects.head(&shared_object).unwrap().is_some()); + assert_eq!( + read_artifact_at_path(&service, "/destination/shared.bin"), + b"borrowed across GC epoch" + ); + + service.remove_file_path("/destination/shared.bin").unwrap(); + service.remove_empty_dir_path("/destination").unwrap(); + for _ in 0..256 { + service.cleanup_restore_releases(1).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); + for _ in 0..256 { + service.cleanup_pending_objects(1).unwrap(); + if objects.head(&shared_object).unwrap().is_none() { + break; + } + } + assert!(objects.head(&shared_object).unwrap().is_none()); + assert!(service.fsck_restore_state(true).unwrap().is_consistent()); +} + +fn load_completed_restore_operation_and_seal( + service: &NoKvFs, + source_path: &str, + snapshot_id: u64, + destination_path: &str, +) -> (restore::RestoreOperation, restore_gc::RestoreBaseSeal) { + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + source_path, + snapshot_id, + destination_path, + ); + let version = service.read_version().unwrap(); + let operation = restore::decode_restore_operation( + &service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_operation_key(service.mount_id(), &operation_digest), + version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .expect("completed restore operation must be durable") + .0, + ) + .unwrap(); + assert_eq!(operation.state, restore::RestoreOperationState::Complete); + let seal = restore_gc::decode_restore_base_seal( + &service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_base_seal_key(service.mount_id(), operation.ref_set_id), + version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .expect("completed restore must have a sealed exact-reference set") + .0, + ) + .unwrap(); + (operation, seal) } #[test] -fn restore_metadata_with_log_segments_rejects_chain_gap_before_restore() { - let (service, objects) = service_with_objects(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-log-gap", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); +fn restore_exact_references_cross_multiple_durable_batches() { + const REFERENCE_COUNT: usize = 130; + + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + for index in 0..REFERENCE_COUNT { + publish_path_artifact( + &service, + &format!("/source/artifact-{index:03}.bin"), + &format!("restore/reference-batch-{index:03}"), + format!("payload-{index:03}").as_bytes(), + ); + } + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + + let (_, seal) = load_completed_restore_operation_and_seal( + &service, + "/source", + snapshot.snapshot_id, + "/restored", + ); + assert_eq!(seal.reference_count, REFERENCE_COUNT as u64); + let metrics = service.restore_metrics().unwrap(); + for keyspace in ["base_owner", "base_inverse", "base_inverse_owner"] { + assert_eq!( + metrics.control_rows.get(keyspace), + Some(&REFERENCE_COUNT), + "every exact-reference projection must cross the 64-row batch boundary" + ); + } + assert_eq!( + read_artifact_at_path(&service, "/restored/artifact-129.bin"), + b"payload-129" + ); + let report = service.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); +} + +#[test] +fn restore_release_batches_retained_reference_cursor_across_pages() { + const REFERENCE_COUNT: usize = 65; + + let metadata = RestorePostApplyFaultStore::new(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let mut source_objects = Vec::with_capacity(REFERENCE_COUNT); + for index in 0..REFERENCE_COUNT { + let artifact = publish_path_artifact( + &service, + &format!("/source/artifact-{index:03}.bin"), + &format!("restore/retained-reference-{index:03}"), + format!("payload-{index:03}").as_bytes(), + ); + let body = artifact.body.as_ref().unwrap(); + source_objects.push(block_key(artifact.attr.inode, body.generation, 0, 0)); + } + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + let operation = durable_restore_operation(&service, snapshot.snapshot_id); + let seal = restore_gc::decode_restore_base_seal( + &service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_base_seal_key(service.mount_id(), operation.ref_set_id), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .expect("completed restore must have a sealed exact-reference set") + .0, + ) + .unwrap(); + assert_eq!(seal.reference_count, REFERENCE_COUNT as u64); + + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + service + .create_dir_path("/escaped", 0o755, 1000, 1000) + .unwrap(); + for index in 0..REFERENCE_COUNT { + service + .rename_path( + &format!("/destination/artifact-{index:03}.bin"), + &format!("/escaped/artifact-{index:03}.bin"), + ) + .unwrap(); + service + .remove_file_path(&format!("/source/artifact-{index:03}.bin")) + .unwrap(); + } + service.remove_empty_dir_path("/source").unwrap(); + for _ in 0..4 { + service.cleanup_pending_objects(REFERENCE_COUNT).unwrap(); + } + assert!(source_objects + .iter() + .all(|object_key| objects.head(object_key).unwrap().is_some())); + + service.remove_empty_dir_path("/destination").unwrap(); + let mount = service.mount_id(); + let release_key = restore::restore_release_job_key(mount, operation.ref_set_id); + let load_job = || { + service + .metadata_store() + .get_versioned( + RecordFamily::System, + &release_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .map(|item| { + ( + restore::decode_restore_release_job(&item.value.0).unwrap(), + item.version, + ) + }) + .expect("detached restore root must create a release job") + }; + let owner_rows = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_base_owner_prefix(mount, operation.ref_set_id), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + assert_eq!(owner_rows.len(), REFERENCE_COUNT); + let (initial_job, initial_job_version) = load_job(); + assert_eq!( + initial_job.phase, + restore::RestoreReleasePhase::ExactReferences + ); + assert!(initial_job.cursor.is_empty()); - let command = log_replay_command( - b"req-log-gap", - b"log-gap-system-key", - b"value", - checkpoint.commit_version, - checkpoint.commit_version + 1, + let update_before = metadata.applied_with_prefix(b"restore-update-release-job"); + let reference_release_before = metadata.applied_with_prefix(b"restore-release-reference"); + let commits_before = service.metadata_store_stats().commit_total; + assert_eq!(service.cleanup_restore_releases(1).unwrap(), 1); + assert_eq!( + metadata.applied_with_prefix(b"restore-update-release-job") - update_before, + 1, + "one full page of retained references must persist one release-job cursor" ); - let result = service.commit_metadata(command.clone()).unwrap(); - let entry = - MetadataLogEntry::seal("mount-1:/", 1, 2, command, result, METADATA_LOG_ZERO_DIGEST) - .unwrap(); - let segment = MetadataLogSegment::seal(vec![entry]).unwrap(); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), + assert_eq!( + metadata.applied_with_prefix(b"restore-release-reference"), + reference_release_before, + "retained rows must not issue per-object release commands" + ); + assert_eq!( + service.metadata_store_stats().commit_total - commits_before, + 2, + "one retained-reference page writes one job cursor and one mount worker cursor" ); + let (first_page_job, first_page_version) = load_job(); + assert_eq!( + first_page_job.phase, + restore::RestoreReleasePhase::ExactReferences + ); + assert_eq!(first_page_job.cursor, owner_rows[63].key); + assert!(first_page_version > initial_job_version); - let err = recovered - .restore_metadata_with_log_segments( - &checkpoint_config, - "mount-1:/", - &[segment], - 0, - METADATA_LOG_ZERO_DIGEST, - ) - .unwrap_err(); + let commits_before = service.metadata_store_stats().commit_total; + assert_eq!(service.cleanup_restore_releases(1).unwrap(), 1); + assert_eq!( + metadata.applied_with_prefix(b"restore-update-release-job") - update_before, + 2, + "the retained reference beyond the first page must advance the release job" + ); + assert_eq!( + metadata.applied_with_prefix(b"restore-release-reference"), + reference_release_before + ); + assert_eq!( + service.metadata_store_stats().commit_total - commits_before, + 2 + ); + let (second_page_job, second_page_version) = load_job(); + assert_eq!(second_page_job.phase, restore::RestoreReleasePhase::Members); + assert!(second_page_job.cursor.is_empty()); + assert!(second_page_version > first_page_version); - assert!(matches!(err, MetadError::Codec(message) if message.contains("lsn gap"))); + let retained_owner_rows = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_base_owner_prefix(mount, operation.ref_set_id), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + assert_eq!(retained_owner_rows.len(), REFERENCE_COUNT); + assert!(source_objects + .iter() + .all(|object_key| objects.head(object_key).unwrap().is_some())); + for index in 0..REFERENCE_COUNT { + assert_eq!( + read_artifact_at_path(&service, &format!("/escaped/artifact-{index:03}.bin")), + format!("payload-{index:03}").as_bytes() + ); + } } #[test] -fn sync_metadata_log_archives_commit_before_recovery_ack() { - let (service, objects) = service_with_objects(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-sync-log", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); +fn restore_release_batches_nonretained_references_across_pages() { + const REFERENCE_COUNT: usize = 65; + + let metadata = RestorePostApplyFaultStore::new(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/sync-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - - let key = b"sync-log-system-key".to_vec(); - let value = b"sync-after-checkpoint".to_vec(); - let commit_version = checkpoint.commit_version + 1; - let command = log_replay_command( - b"req-sync-log", - &key, - &value, - checkpoint.commit_version, - commit_version, - ); - let result = service.commit_metadata(command).unwrap(); - assert_eq!(result.commit_version.get(), commit_version); - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.durable_lsn, 1); - assert_eq!(snapshot.segments.len(), 1); - let segment_pointer = snapshot.segments.last().unwrap(); - assert!(segment_pointer - .segment_key - .starts_with("meta/sync-log/log/")); - - let loaded = service - .load_metadata_log_segment(&segment_pointer.segment_key) + let mut source_objects = Vec::with_capacity(REFERENCE_COUNT); + for index in 0..REFERENCE_COUNT { + let artifact = publish_path_artifact( + &service, + &format!("/source/artifact-{index:03}.bin"), + &format!("restore/nonretained-reference-{index:03}"), + format!("payload-{index:03}").as_bytes(), + ); + let body = artifact.body.as_ref().unwrap(); + source_objects.push(block_key(artifact.attr.inode, body.generation, 0, 0)); + } + let expected_objects = source_objects + .iter() + .map(|key| key.as_str().to_owned()) + .collect::>(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") .unwrap(); - assert_eq!(loaded.first_lsn, 1); - assert_eq!(loaded.last_lsn, 1); - assert_eq!(loaded.last_digest, snapshot.last_digest); + let operation = durable_restore_operation(&service, snapshot.snapshot_id); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); - let segment_keys = snapshot_segment_keys(&snapshot); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), - ); - recovered - .restore_metadata_with_archived_log_segments( - &checkpoint_config, - "mount-1:/", - &segment_keys, - 0, - METADATA_LOG_ZERO_DIGEST, + service + .create_dir_path("/replacement", 0o755, 1000, 1000) + .unwrap(); + service + .rename_replace_path("/replacement", "/destination") + .unwrap(); + assert_eq!(service.restore_metrics().unwrap().releasing, 1); + + let mount = service.mount_id(); + let release_key = restore::restore_release_job_key(mount, operation.ref_set_id); + let load_job = || { + restore::decode_restore_release_job( + &service + .metadata_store() + .get( + RecordFamily::System, + &release_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .expect("detached restore root must create a release job") + .0, ) .unwrap() - .unwrap(); + }; + let scan_owner_rows = || { + service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_base_owner_prefix(mount, operation.ref_set_id), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + }; + let queued_objects = || { + service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .into_iter() + .map(|row| decode_object_gc_record(&row.value.0).unwrap().object_key) + .filter(|object_key| expected_objects.contains(object_key)) + .collect::>() + }; + let owner_rows = scan_owner_rows(); + assert_eq!(owner_rows.len(), REFERENCE_COUNT); + let apply_before = metadata.applied_with_prefix(b"restore-release-reference-batch"); + let mutation_counts_before = metadata + .applied_mutation_counts_with_prefix(b"restore-release-reference-batch") + .len(); + assert_eq!(service.cleanup_restore_releases(1).unwrap(), 1); assert_eq!( - recovered - .metadata - .get( - RecordFamily::System, - &key, - Version::new(commit_version).unwrap(), - ReadPurpose::UserStrong - ) - .unwrap(), - Some(Value(value)) + metadata.applied_with_prefix(b"restore-release-reference-batch") - apply_before, + 1, + "one cleanup pass must release the first 64 references in one command" + ); + let mutation_counts = + metadata.applied_mutation_counts_with_prefix(b"restore-release-reference-batch"); + assert_eq!( + mutation_counts[mutation_counts_before], + 1 + 3 * restore::RESTORE_BATCH_ENTRIES + restore::RESTORE_BATCH_ENTRIES, + "the first page contains one job cursor, 64 exact-reference triples, and 64 GC rows" + ); + let remaining = scan_owner_rows(); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].key, owner_rows[64].key); + let first_job = load_job(); + assert_eq!( + first_job.phase, + restore::RestoreReleasePhase::ExactReferences ); + assert_eq!(first_job.cursor, owner_rows[63].key); + let first_queued = queued_objects(); + assert_eq!(first_queued.len(), restore::RESTORE_BATCH_ENTRIES); + assert_eq!( + first_queued.iter().collect::>().len(), + restore::RESTORE_BATCH_ENTRIES, + "the first page must enqueue each canonical object once" + ); + + assert_eq!(service.cleanup_restore_releases(1).unwrap(), 1); + assert_eq!( + metadata.applied_with_prefix(b"restore-release-reference-batch") - apply_before, + 2, + "the final reference must be released by one second batch" + ); + let mutation_counts = + metadata.applied_mutation_counts_with_prefix(b"restore-release-reference-batch"); + assert_eq!( + mutation_counts[mutation_counts_before + 1], + 1 + 3 + 1, + "the tail page contains one job cursor, one exact-reference triple, and one GC row" + ); + assert!(scan_owner_rows().is_empty()); + let final_job = load_job(); + assert_eq!(final_job.phase, restore::RestoreReleasePhase::Members); + assert!(final_job.cursor.is_empty()); + let queued = queued_objects(); + assert_eq!(queued.len(), REFERENCE_COUNT); + assert_eq!(queued.iter().collect::>().len(), REFERENCE_COUNT); + assert!(source_objects + .iter() + .all(|object_key| objects.head(object_key).unwrap().is_some())); } #[test] -fn log_replay_does_not_fold_a_foreign_graft_inode_into_the_local_allocator() { - let (service, objects) = service_with_objects(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-graft-log-replay", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); +fn restore_release_retries_a_borrower_manifest_backend_read_without_quarantine() { + let metadata = RestoreBorrowerManifestReadFaultStore::new(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/graft-log-replay", - "mount-1:/", - 1, - checkpoint.log_lsn, - checkpoint.log_digest, - )) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - - let foreign_inode = InodeId::compose(1, 42).unwrap(); + let source = publish_path_artifact( + &service, + "/source/artifact.bin", + "restore/retry-borrower-manifest-source", + b"source generation", + ); + let source_body = source.body.as_ref().unwrap(); + let source_object = block_key(source.attr.inode, source_body.generation, 0, 0); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); service - .create_graft( - InodeId::root(), - dname(b"dataset"), - foreign_inode, - 0o755, - 1000, - 1000, - ) + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") .unwrap(); - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.segments.len(), 1); + let operation = durable_restore_operation(&service, snapshot.snapshot_id); + let borrower = service + .lookup_path("/destination/artifact.bin") + .unwrap() + .expect("restored borrower must be visible"); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects, - ) - .with_shard_index(0); - recovered - .restore_metadata_with_archived_log_segments( - &checkpoint_config, - "mount-1:/", - &snapshot_segment_keys(&snapshot), - checkpoint.log_lsn, - checkpoint.log_digest, + // Keep the borrower inode reachable after detaching the restore root, but + // move it to a new generation that no longer references the restored base + // object. Release must read this manifest before it can drop the exact ref. + let prepared = service + .prepare_artifact_replace_path("/destination/artifact.bin") + .unwrap(); + let replacement = service + .publish_prepared_artifact_session( + prepared.clone(), + PublishArtifactSession { + parent: prepared.parent, + name: prepared.name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:test".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "restore/retry-borrower-manifest-replacement".to_owned(), + size: b"replacement generation".len() as u64, + ranges: vec![PublishArtifactRange { + offset: 0, + bytes: b"replacement generation".to_vec(), + }], + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap() + .entry; + assert_eq!(replacement.attr.inode, borrower.attr.inode); + let replacement_body = replacement.body.as_ref().unwrap(); + let borrower_manifest_key = chunk_manifest_key( + service.mount_id(), + replacement.attr.inode, + replacement_body.generation, + 0, + ); + assert!(service + .metadata_store() + .get( + RecordFamily::ChunkManifest, + &borrower_manifest_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, ) .unwrap() + .is_some()); + + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + service + .create_dir_path("/escaped", 0o755, 1000, 1000) .unwrap(); + service + .rename_path("/destination/artifact.bin", "/escaped/artifact.bin") + .unwrap(); + service.remove_empty_dir_path("/destination").unwrap(); + + let mount = service.mount_id(); + let release_key = restore::restore_release_job_key(mount, operation.ref_set_id); + let load_job = || { + service + .metadata_store() + .get_versioned( + RecordFamily::System, + &release_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .map(|item| { + ( + restore::decode_restore_release_job(&item.value.0).unwrap(), + item.version, + ) + }) + .expect("detached restore root must create a release job") + }; + let owner_rows = || { + service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_base_owner_prefix(mount, operation.ref_set_id), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + }; + let quarantines = || { + service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_release_quarantine_prefix(mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + }; - let graft = recovered - .lookup_plus(InodeId::root(), &dname(b"dataset")) - .unwrap(); - assert_eq!(graft.unwrap().dentry.child, foreign_inode); - let local = recovered - .create_file( - InodeId::root(), - dname(b"local-after-replay"), - 0o644, - 1000, - 1000, - ) + let initial_owners = owner_rows(); + assert_eq!(initial_owners.len(), 1); + let (initial_job, initial_job_version) = load_job(); + assert_eq!( + initial_job.phase, + restore::RestoreReleasePhase::ExactReferences + ); + assert!(initial_job.cursor.is_empty()); + assert!(quarantines().is_empty()); + + metadata.fail_next_restore_manifest_read(borrower_manifest_key); + assert_eq!(service.cleanup_restore_releases(1).unwrap(), 1); + assert_eq!(metadata.failures(), 1); + let (failed_job, failed_job_version) = load_job(); + assert_eq!(failed_job.phase, initial_job.phase); + assert_eq!(failed_job.cursor, initial_job.cursor); + assert_eq!(failed_job_version, initial_job_version); + let failed_owners = owner_rows(); + assert_eq!(failed_owners.len(), 1); + assert_eq!(failed_owners[0].key, initial_owners[0].key); + assert_eq!(failed_owners[0].version, initial_owners[0].version); + assert!(quarantines().is_empty()); + + assert_eq!(service.cleanup_restore_releases(1).unwrap(), 1); + assert_eq!(metadata.failures(), 1); + assert!(owner_rows().is_empty()); + let (released_job, released_job_version) = load_job(); + assert_eq!(released_job.phase, restore::RestoreReleasePhase::Members); + assert!(released_job.cursor.is_empty()); + assert!(released_job_version > failed_job_version); + assert!(quarantines().is_empty()); + let queued = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap(); + assert!(queued.into_iter().any(|row| { + decode_object_gc_record(&row.value.0) + .is_ok_and(|record| record.object_key == source_object.as_str()) + })); assert_eq!( - local.attr.inode.shard_index(), - 0, - "a foreign graft target replayed from the log must not poison the local allocator" + read_artifact_at_path(&service, "/escaped/artifact.bin"), + b"replacement generation" ); + assert!(objects.head(&source_object).unwrap().is_some()); } #[test] -fn concurrent_metadata_apply_and_log_archive_preserve_one_recovery_order() { - const FIRST_REQUEST: &[u8] = b"req-ordered-log-first"; - const SECOND_REQUEST: &[u8] = b"req-ordered-log-second"; - let store = PostCommitBarrierStore::new(FIRST_REQUEST); +fn restore_release_deduplicates_gc_for_distinct_borrowers_of_one_object() { + let metadata = RestorePostApplyFaultStore::new(); let objects = MemoryObjectStore::new(); - let service = Arc::new(NoKvFs::new( - MountId::new(1).unwrap(), - store.clone(), - objects.clone(), - )); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-ordered-sync-log", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( + &service, + "/base/shared.bin", + "restore/release-shared-object", + b"shared object bytes", + ); + let source_body = source.body.as_ref().unwrap(); + let source_object = block_key(source.attr.inode, source_body.generation, 0, 0); + service.create_dir_path("/both", 0o755, 1000, 1000).unwrap(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/ordered-sync-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + .clone_subtree_path_into("/base", "/both/one") .unwrap(); - - let key = b"ordered-sync-log-key".to_vec(); - let first = ordered_log_put_command( - FIRST_REQUEST, - &key, - b"first", - checkpoint.commit_version, - checkpoint.commit_version + 1, - ); - let second = ordered_log_put_command( - SECOND_REQUEST, - &key, - b"second", - checkpoint.commit_version + 1, - checkpoint.commit_version + 2, - ); - - let first_service = Arc::clone(&service); - let first_commit = std::thread::spawn(move || first_service.commit_metadata(first)); - store.wait_until_applied(); - - let (second_tx, second_rx) = std::sync::mpsc::sync_channel(1); - let second_service = Arc::clone(&service); - let second_commit = std::thread::spawn(move || { - let result = second_service.commit_metadata(second); - second_tx.send(result).unwrap(); - }); - let early = second_rx.recv_timeout(Duration::from_millis(100)); - store.release_after_apply(); - first_commit.join().unwrap().unwrap(); - assert!(matches!( - early, - Err(std::sync::mpsc::RecvTimeoutError::Timeout) - )); - second_rx - .recv_timeout(Duration::from_secs(5)) - .unwrap() + service + .clone_subtree_path_into("/base", "/both/two") + .unwrap(); + let snapshot = service.snapshot_subtree_path("/both").unwrap(); + service + .restore_subtree_path_to_fork("/both", snapshot.snapshot_id, "/restored") .unwrap(); - second_commit.join().unwrap(); - assert_eq!( - service - .metadata - .get( - RecordFamily::System, - &key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap(), - Some(Value(b"second".to_vec())) + service.restore_metrics().unwrap().control_rows["base_owner"], + 2 ); - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.durable_lsn, 2); - assert_eq!(snapshot.segments.len(), 2); - let first_segment = service - .load_metadata_log_segment(&snapshot.segments[0].segment_key) - .unwrap(); - let second_segment = service - .load_metadata_log_segment(&snapshot.segments[1].segment_key) - .unwrap(); - assert_eq!(first_segment.entries[0].command.request_id, FIRST_REQUEST); - assert_eq!(second_segment.entries[0].command.request_id, SECOND_REQUEST); - assert_eq!(first_segment.last_digest, second_segment.prev_digest); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects, + service + .create_dir_path("/replacement", 0o755, 1000, 1000) + .unwrap(); + service + .rename_replace_path("/replacement", "/restored") + .unwrap(); + let mutation_counts_before = metadata + .applied_mutation_counts_with_prefix(b"restore-release-reference-batch") + .len(); + assert_eq!(service.cleanup_restore_releases(1).unwrap(), 1); + let mutation_counts = + metadata.applied_mutation_counts_with_prefix(b"restore-release-reference-batch"); + assert_eq!( + mutation_counts[mutation_counts_before], + 1 + 3 * 2 + 1, + "two borrower triples sharing one canonical object need one GC mutation" ); - let outcome = recovered - .restore_metadata_with_archived_log_segments( - &checkpoint_config, - "mount-1:/", - &snapshot_segment_keys(&snapshot), - 0, - METADATA_LOG_ZERO_DIGEST, - ) + let metrics = service.restore_metrics().unwrap(); + for keyspace in ["base_owner", "base_inverse", "base_inverse_owner"] { + assert_eq!(metrics.control_rows.get(keyspace), Some(&0)); + } + let queued = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount_id()), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap() - .unwrap(); - assert_eq!(outcome.replayed_entries, 2); + .into_iter() + .filter_map(|row| { + decode_object_gc_record(&row.value.0) + .ok() + .filter(|record| record.object_key == source_object.as_str()) + }) + .collect::>(); + assert_eq!(queued.len(), 1); + assert!(objects.head(&source_object).unwrap().is_some()); assert_eq!( - recovered - .metadata - .get( - RecordFamily::System, - &key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap(), - Some(Value(b"second".to_vec())) + read_artifact_at_path(&service, "/both/one/shared.bin"), + b"shared object bytes" + ); + assert_eq!( + read_artifact_at_path(&service, "/both/two/shared.bin"), + b"shared object bytes" ); } #[test] -fn metadata_commits_remain_concurrent_while_sync_log_is_disabled() { - const FIRST_REQUEST: &[u8] = b"req-no-sync-first"; - let store = PostCommitBarrierStore::new(FIRST_REQUEST); - let service = Arc::new(NoKvFs::new( - MountId::new(1).unwrap(), - store.clone(), - MemoryObjectStore::new(), - )); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let base = service.read_version().unwrap().get(); - let key = b"no-sync-concurrent-key"; - let first = ordered_log_put_command(FIRST_REQUEST, key, b"first", base, base + 1); - let second = ordered_log_put_command(b"req-no-sync-second", key, b"second", base + 1, base + 2); - - let first_service = Arc::clone(&service); - let first_commit = std::thread::spawn(move || first_service.commit_metadata(first)); - store.wait_until_applied(); - - let (second_tx, second_rx) = std::sync::mpsc::sync_channel(1); - let second_service = Arc::clone(&service); - let second_commit = std::thread::spawn(move || { - second_tx - .send(second_service.commit_metadata(second)) - .unwrap(); - }); - let second_result = second_rx - .recv_timeout(Duration::from_secs(5)) - .expect("a disabled sync log must not serialize unrelated metadata commits"); - store.release_after_apply(); +fn restore_release_defers_gc_until_a_cross_page_object_is_validated() { + const BORROWER_COUNT: usize = 65; - second_result.unwrap(); - second_commit.join().unwrap(); - first_commit.join().unwrap().unwrap(); - assert_eq!( + let metadata = HoltMetadataStore::open_memory().unwrap(); + let backing = MemoryObjectStore::new(); + let objects = DeleteAckLostObjectStore::new(backing); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata, objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( + &service, + "/base/shared.bin", + "restore/cross-page-shared-object", + b"shared across an exact-reference page boundary", + ); + let source_body = source.body.as_ref().unwrap(); + let source_object = block_key(source.attr.inode, source_body.generation, 0, 0); + enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: source.attr.inode, + generation: source_body.generation, + object_key: source_object.as_str().to_owned(), + size: source_body.size, + digest_uri: source_body.digest_uri.clone(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + service.create_dir_path("/both", 0o755, 1000, 1000).unwrap(); + for index in 0..BORROWER_COUNT { service - .metadata + .clone_subtree_path_into("/base", &format!("/both/borrower-{index:03}")) + .unwrap(); + } + let snapshot = service.snapshot_subtree_path("/both").unwrap(); + service + .restore_subtree_path_to_fork("/both", snapshot.snapshot_id, "/restored") + .unwrap(); + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/both", + snapshot.snapshot_id, + "/restored", + ); + let operation = restore::decode_restore_operation( + &service + .metadata_store() .get( RecordFamily::System, - key, - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, + &restore::restore_operation_key(service.mount_id(), &operation_digest), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, ) - .unwrap(), - Some(Value(b"second".to_vec())) + .unwrap() + .expect("restore operation must be durable") + .0, + ) + .unwrap(); + assert_eq!( + service.restore_metrics().unwrap().control_rows["base_owner"], + BORROWER_COUNT ); -} - -#[test] -fn sync_metadata_log_snapshot_keeps_durable_tail_after_checkpoint_prune() { - let (service, _objects) = service_with_objects(); - service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/sync-log-prune", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + service + .create_dir_path("/replacement", 0o755, 1000, 1000) .unwrap(); - service - .create_dir_path("/before-checkpoint", 0o755, 1000, 1000) + .rename_replace_path("/replacement", "/restored") .unwrap(); - let first = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(first.segments.len(), 1); - assert_eq!(first.segments[0].last_lsn, first.durable_lsn); - assert_eq!(first.segments[0].last_digest, first.last_digest); - - service.prune_sync_metadata_log_segments(first.durable_lsn); - let after_first_prune = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(after_first_prune.durable_lsn, first.durable_lsn); - assert_eq!(after_first_prune.last_digest, first.last_digest); - assert!(after_first_prune.segments.is_empty()); - service - .create_dir_path("/after-checkpoint", 0o755, 1000, 1000) + // Remove every ordinary namespace holder before corrupting the final exact + // inverse. The pre-existing candidate predates every clone retention floor, + // so the same cleanup call must rely on the exact-reference continuation + // proof when it inspects that candidate immediately after the release page. + for index in 0..BORROWER_COUNT { + service + .remove_file_path(&format!("/both/borrower-{index:03}/shared.bin")) + .unwrap(); + service + .remove_empty_dir_path(&format!("/both/borrower-{index:03}")) + .unwrap(); + } + service.remove_empty_dir_path("/both").unwrap(); + service.remove_file_path("/base/shared.bin").unwrap(); + service.remove_empty_dir_path("/base").unwrap(); + let mount = service.mount_id(); + let owner_prefix = restore::restore_base_owner_prefix(mount, operation.ref_set_id); + let owner_rows = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: owner_prefix.clone(), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap(); - let second = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(second.segments.len(), 1); - assert_eq!(second.segments[0].first_lsn, first.durable_lsn + 1); - assert_eq!(second.segments[0].last_lsn, second.durable_lsn); - assert_eq!(second.segments[0].last_digest, second.last_digest); - let continued = service - .load_metadata_log_segment(&second.segments[0].segment_key) + assert_eq!(owner_rows.len(), BORROWER_COUNT); + let last_reference = + restore_gc::decode_restore_base_reference(&owner_rows[BORROWER_COUNT - 1].value.0).unwrap(); + assert!(owner_rows.iter().all(|row| { + restore_gc::decode_restore_base_reference(&row.value.0) + .is_ok_and(|reference| reference.object_key == source_object.as_str()) + })); + let object_digest: [u8; 32] = Sha256::digest(source_object.as_str().as_bytes()).into(); + let last_inverse_key = restore::restore_base_inverse_key( + mount, + &object_digest, + operation.ref_set_id, + last_reference.borrower_inode, + last_reference.borrower_generation, + ); + let last_inverse = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &last_inverse_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .expect("the final borrower must have an inverse before corruption"); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-delete-cross-page-restore-inverse", + mount, + last_reference.borrower_inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: last_inverse_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: last_inverse_key.clone(), + predicate: Predicate::VersionEquals(last_inverse.version), + }], + mutations: vec![delete_mutation(RecordFamily::System, last_inverse_key)], + watch: Vec::new(), + }) .unwrap(); - assert_eq!(continued.prev_digest, first.last_digest); - service.prune_sync_metadata_log_segments(second.durable_lsn); - let after_second_prune = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(after_second_prune.durable_lsn, second.durable_lsn); - assert_eq!(after_second_prune.last_digest, second.last_digest); - assert!(after_second_prune.segments.is_empty()); -} + let queued_for_object = || { + service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .into_iter() + .filter(|row| { + decode_object_gc_record(&row.value.0) + .is_ok_and(|record| record.object_key == source_object.as_str()) + }) + .count() + }; -#[test] -fn sync_metadata_log_prune_never_deletes_pointer_outside_archive_prefix() { - let (service, objects) = service_with_objects(); - let unrelated = ObjectKey::new("unrelated/keep-me").unwrap(); - objects.put(&unrelated, b"keep".to_vec()).unwrap(); - service - .enable_sync_metadata_log( - MetadataLogSyncConfig::new("meta/sync-log-delete-scope", "mount-1:/", 1, 1, [7; 32]) - .with_segments(vec![MetadataLogSegmentPointer { - segment_key: unrelated.as_str().to_owned(), - first_lsn: 1, - last_lsn: 1, - last_digest: [7; 32], - }]), - ) + assert!(queued_for_object() >= 1); + let first = service.cleanup_pending_objects(64).unwrap(); + assert_eq!(first.restore_release_jobs_processed, 1); + assert_eq!(first.deleted, 0, "first cleanup outcome: {first:?}"); + assert_eq!(objects.delete_calls(), 0); + assert!(first.scanned >= 1, "first cleanup outcome: {first:?}"); + assert!( + first.blocked_by_snapshots >= 1, + "first cleanup outcome: {first:?}" + ); + assert!(queued_for_object() >= 1); + assert!(objects.head(&source_object).unwrap().is_some()); + let remaining = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: owner_prefix, + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap(); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].key, owner_rows[BORROWER_COUNT - 1].key); - let outcome = service.prune_sync_metadata_log_segments(1); - - assert_eq!(outcome.pointers_pruned, 1); - assert_eq!(outcome.objects_deleted, 0); - assert_eq!(outcome.objects_missing, 0); - assert_eq!(outcome.delete_failures, 1); - assert!(objects.head(&unrelated).unwrap().is_some()); - assert!(service - .sync_metadata_log_snapshot() - .unwrap() - .segments - .is_empty()); + let second = service.cleanup_pending_objects(64).unwrap(); + assert_eq!(second.restore_release_jobs_processed, 1); + assert_eq!(second.deleted, 0, "second cleanup outcome: {second:?}"); + assert_eq!(second.restore_release_quarantine, 1); + assert_eq!(second.restore_release_mount_wide_quarantine, 0); + assert_eq!(objects.delete_calls(), 0); + assert!(second.scanned >= 1, "second cleanup outcome: {second:?}"); + assert!( + second.blocked_by_snapshots >= 1, + "second cleanup outcome: {second:?}" + ); + assert!(queued_for_object() >= 1); + assert!(objects.head(&source_object).unwrap().is_some()); + let quarantines = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_release_object_quarantine_prefix(mount, &object_digest), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + assert_eq!(quarantines.len(), 1); + let quarantine = + restore::validate_restore_release_quarantine_row(mount, &quarantines[0]).unwrap(); + assert_eq!( + quarantine.scope, + restore::RestoreReleaseQuarantineScope::Object(object_digest) + ); + assert_eq!(quarantine.original_key, owner_rows[BORROWER_COUNT - 1].key); + assert!(quarantine.reason.contains("no inverse")); } #[test] -fn sync_metadata_log_rejects_duplicate_enable_without_replacing_tail() { - let (service, _objects) = service_with_objects(); - service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/sync-log-original", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) - .unwrap(); - service - .create_dir_path("/before-duplicate-enable", 0o755, 1000, 1000) - .unwrap(); - let before = service.sync_metadata_log_snapshot().unwrap(); - - let error = service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/sync-log-replacement", - "mount-1:/", - 2, - 0, - METADATA_LOG_ZERO_DIGEST, - )) - .unwrap_err(); - assert!(matches!( - error, - MetadError::Codec(message) if message.contains("already enabled") - )); - assert_eq!(service.sync_metadata_log_snapshot().unwrap(), before); +fn restore_release_mount_wide_quarantines_a_rekeyed_cross_page_owner_before_gc() { + const BORROWER_COUNT: usize = 65; + let metadata = HoltMetadataStore::open_memory().unwrap(); + let backing = MemoryObjectStore::new(); + let objects = DeleteAckLostObjectStore::new(backing); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata, objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( + &service, + "/base/shared.bin", + "restore/rekeyed-cross-page-owner", + b"shared across a rekeyed exact-reference page boundary", + ); + let source_body = source.body.as_ref().unwrap(); + let source_object = block_key(source.attr.inode, source_body.generation, 0, 0); + enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: source.attr.inode, + generation: source_body.generation, + object_key: source_object.as_str().to_owned(), + size: source_body.size, + digest_uri: source_body.digest_uri.clone(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); service - .create_dir_path("/after-duplicate-enable", 0o755, 1000, 1000) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let after = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(after.epoch, 1); - assert_eq!(after.durable_lsn, before.durable_lsn + 1); - assert!(after - .segments - .last() - .unwrap() - .segment_key - .starts_with("meta/sync-log-original/log/")); -} - -#[test] -fn sync_metadata_log_preflights_lsn_capacity_before_metadata_apply() { - let (service, _objects) = service_with_objects(); + for index in 0..BORROWER_COUNT { + service + .clone_subtree_path_into("/base", &format!("/source/borrower-{index:03}")) + .unwrap(); + } + let snapshot = service.snapshot_subtree_path("/source").unwrap(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/sync-log-exhausted", - "mount-1:/", - 1, - u64::MAX - 1, - METADATA_LOG_ZERO_DIGEST, - )) + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") .unwrap(); - - let error = service - .create_dir_path("/must-not-apply", 0o755, 1000, 1000) - .unwrap_err(); - assert!(matches!( - error, - MetadError::SyncLogArchiveFailed { - committed: false, - message, - } if message.contains("LSN is exhausted before commit") - )); - assert!(service.lookup_path("/must-not-apply").unwrap().is_none()); + let operation = durable_restore_operation(&service, snapshot.snapshot_id); assert_eq!( - service.sync_metadata_log_snapshot().unwrap().durable_lsn, - u64::MAX - 1 + service.restore_metrics().unwrap().control_rows["base_owner"], + BORROWER_COUNT ); -} - -#[test] -fn restore_metadata_with_sync_log_advances_allocator_after_replay() { - let (service, objects) = service_with_objects(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-sync-allocator", 2); - service.backup_metadata(&checkpoint_config).unwrap(); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/sync-allocator-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + .create_dir_path("/replacement", 0o755, 1000, 1000) .unwrap(); - - let post_checkpoint = service - .create_dir_path("/runs/post-checkpoint", 0o755, 1000, 1000) + service + .rename_replace_path("/replacement", "/destination") .unwrap(); - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - let segment_keys = snapshot_segment_keys(&snapshot); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects, - ); - recovered - .restore_metadata_with_archived_log_segments( - &checkpoint_config, - "mount-1:/", - &segment_keys, - 0, - METADATA_LOG_ZERO_DIGEST, + for index in 0..BORROWER_COUNT { + service + .remove_file_path(&format!("/source/borrower-{index:03}/shared.bin")) + .unwrap(); + service + .remove_empty_dir_path(&format!("/source/borrower-{index:03}")) + .unwrap(); + } + service.remove_empty_dir_path("/source").unwrap(); + service.remove_file_path("/base/shared.bin").unwrap(); + service.remove_empty_dir_path("/base").unwrap(); + + let mount = service.mount_id(); + let owner_prefix = restore::restore_base_owner_prefix(mount, operation.ref_set_id); + let owner_rows = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: owner_prefix.clone(), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + assert_eq!(owner_rows.len(), BORROWER_COUNT); + let last_owner = &owner_rows[BORROWER_COUNT - 1]; + let last_reference = restore_gc::decode_restore_base_reference(&last_owner.value.0).unwrap(); + let source_object_digest: [u8; 32] = Sha256::digest(source_object.as_str().as_bytes()).into(); + assert!(owner_rows.iter().all(|row| { + restore_gc::decode_restore_base_reference(&row.value.0) + .is_ok_and(|reference| reference.object_key == source_object.as_str()) + })); + + // Move the 65th owner to a syntactically valid, larger object digest while + // retaining its encoded reference to object A. Also remove A's matching + // target-first inverse, so only mount-wide fail-closed quarantine can keep + // the pre-existing A candidate from being deleted after the first 64 rows. + let rekeyed_object_digest = [u8::MAX; 32]; + assert!(source_object_digest < rekeyed_object_digest); + let rekeyed_owner_key = restore::restore_base_owner_key( + mount, + operation.ref_set_id, + &rekeyed_object_digest, + last_reference.borrower_inode, + last_reference.borrower_generation, + ); + assert!(rekeyed_owner_key > last_owner.key); + let inverse_key = restore::restore_base_inverse_key( + mount, + &source_object_digest, + operation.ref_set_id, + last_reference.borrower_inode, + last_reference.borrower_generation, + ); + let inverse = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &inverse_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, ) .unwrap() + .expect("the rekeyed owner must begin with an A inverse"); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-rekey-cross-page-restore-owner", + mount, + last_reference.borrower_inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: rekeyed_owner_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: last_owner.key.clone(), + predicate: Predicate::VersionEquals(last_owner.version), + }, + PredicateRef { + family: RecordFamily::System, + key: rekeyed_owner_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.clone(), + predicate: Predicate::VersionEquals(inverse.version), + }, + ], + mutations: vec![ + delete_mutation(RecordFamily::System, last_owner.key.clone()), + Mutation { + family: RecordFamily::System, + key: rekeyed_owner_key.clone(), + op: MutationOp::Put, + value: Some(last_owner.value.clone()), + }, + delete_mutation(RecordFamily::System, inverse_key), + ], + watch: Vec::new(), + }) .unwrap(); - assert_eq!( - recovered - .lookup_path("/runs/post-checkpoint") - .unwrap() - .unwrap() - .attr - .inode, - post_checkpoint.attr.inode - ); - - let after_failover = recovered - .create_dir_path("/after-failover", 0o755, 1000, 1000) + let boundary_rows = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: owner_prefix.clone(), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap(); + assert_eq!(boundary_rows.len(), BORROWER_COUNT); + assert_eq!(boundary_rows[BORROWER_COUNT - 1].key, rekeyed_owner_key); + + let outcome = service.cleanup_pending_objects(64).unwrap(); + assert_eq!(outcome.restore_release_jobs_processed, 1); + assert_eq!(outcome.restore_release_quarantine, 1); + assert_eq!(outcome.restore_release_mount_wide_quarantine, 1); + assert_eq!(outcome.deleted, 0, "cleanup outcome: {outcome:?}"); + assert_eq!(objects.delete_calls(), 0); assert!( - after_failover.attr.inode.get() > post_checkpoint.attr.inode.get(), - "failover replay must advance allocator past replayed namespace state" + outcome.blocked_by_snapshots >= 1, + "cleanup outcome: {outcome:?}" ); + assert!(objects.head(&source_object).unwrap().is_some()); + + let remaining = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: owner_prefix, + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].key, rekeyed_owner_key); + let quarantines = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_release_mount_wide_quarantine_prefix(mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + assert_eq!(quarantines.len(), 1); + let quarantine = + restore::validate_restore_release_quarantine_row(mount, &quarantines[0]).unwrap(); assert_eq!( - recovered - .lookup_path("/runs/post-checkpoint") - .unwrap() - .unwrap() - .attr - .inode, - post_checkpoint.attr.inode + quarantine.scope, + restore::RestoreReleaseQuarantineScope::MountWide ); + assert_eq!(quarantine.original_key, rekeyed_owner_key); + assert!(quarantine.reason.contains("changed identity")); } #[test] -fn prepare_only_allocator_reservation_replays_before_failover_reuse() { - let (service, objects) = service_with_objects(); - // Initialize the durable object-GC claim before the checkpoint so the - // prepare-only workload below changes only allocator reservations. - service - .prepare_artifact_create(InodeId::root(), dname(b"warmup.bin")) - .unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-prepare-allocator", 2); - service.backup_metadata(&checkpoint_config).unwrap(); +fn restore_exact_references_adapt_to_byte_budget_ack_loss_and_reopen() { + const REFERENCE_COUNT: usize = 64; + const REFERENCES_PER_DIRECTORY: usize = 32; + // 134 KiB is deliberately close to the production boundary: 61 such + // references plus owner/inverse/inverse-owner proof exceed 8 MiB, while + // either 32-entry materialization command remains comfortably bounded. + const DIGEST_PADDING: usize = 134 * 1024; + + let metadata = OversizedManifestStore::new(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/prepare-allocator-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - - let mut max_prepared_inode = 0; - let mut max_prepared_generation = 0; - for _ in 0..(ALLOCATOR_INODE_RESERVATION + 4) { - let prepared = service - .prepare_artifact_create(InodeId::root(), dname(b"never-published.bin")) + for directory in 0..2 { + service + .create_dir_path(&format!("/source/group-{directory}"), 0o755, 1000, 1000) .unwrap(); - max_prepared_inode = max_prepared_inode.max(prepared.inode.get()); - max_prepared_generation = max_prepared_generation.max(prepared.generation); } - - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert!(snapshot.durable_lsn > 0); - let segments = snapshot - .segments + let large_digest = format!("sha256:{}", "d".repeat(DIGEST_PADDING)); + for index in 0..REFERENCE_COUNT { + let directory = index / REFERENCES_PER_DIRECTORY; + let path = format!("/source/group-{directory}/artifact-{index:02}.bin"); + let prepared = service.prepare_artifact_create_path(&path).unwrap(); + let written = service + .stage_prepared_artifact_ranges( + &prepared, + &format!("restore/adaptive-reference-{index:02}"), + &[PublishArtifactRange { + offset: 0, + bytes: vec![index as u8], + }], + 0, + ) + .unwrap(); + let staged = written.staged_objects().unwrap(); + let mut chunks = written.chunk_manifests(); + chunks[0].slices[0].blocks[0].digest_uri = large_digest.clone(); + service + .publish_prepared_artifact_staged_session( + prepared.clone(), + PublishArtifactStagedSession { + parent: prepared.parent, + name: prepared.name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:body".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: format!("restore/adaptive-reference-{index:02}"), + size: 1, + chunks, + staged, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + } + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + + // The first persisted page only advances across empty staging members; + // lose the ACK for the following, byte-packed reference page. + metadata.fail_after_apply(b"restore-persist-base-refs", 1); + let outcome = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + assert_eq!(outcome.state, RestoreState::Complete); + let mutation_counts = + metadata.applied_mutation_counts_with_prefix(b"restore-persist-base-refs"); + let reference_counts = mutation_counts .iter() - .map(|segment| { - service - .load_metadata_log_segment(&segment.segment_key) - .unwrap() - }) + .map(|count| count.saturating_sub(1) / 3) .collect::>(); - assert!(segments - .iter() - .flat_map(|segment| &segment.entries) - .any(|entry| { entry.command.kind == CommandKind::ReserveAllocator })); - - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects, - ); - recovered - .restore_metadata_with_archived_log_segments( - &checkpoint_config, - "mount-1:/", - &snapshot_segment_keys(&snapshot), - 0, - METADATA_LOG_ZERO_DIGEST, - ) - .unwrap() - .unwrap(); - - let after_failover = recovered - .prepare_artifact_create(InodeId::root(), dname(b"after-failover.bin")) - .unwrap(); + assert_eq!(reference_counts.iter().sum::(), REFERENCE_COUNT); assert!( - after_failover.inode.get() > max_prepared_inode, - "recovery must not reuse an inode from a prepare-only reservation" + reference_counts.iter().copied().max().unwrap_or(0) < 61, + "the 61-reference runtime page must be shortened by the 8 MiB packer: {reference_counts:?}" ); assert!( - after_failover.generation > max_prepared_generation, - "recovery must not reuse a generation from a prepare-only reservation" + reference_counts.iter().filter(|count| **count > 0).count() >= 2, + "large references must span multiple durable pages: {reference_counts:?}" + ); + assert_eq!( + metadata.applied_with_prefix(b"restore-persist-base-refs"), + mutation_counts.len(), + "an apply-then-ACK-loss page must be recognized exactly once" + ); + let operation = durable_restore_operation(&service, snapshot.snapshot_id); + let destination_root = operation.destination_root; + drop(service); + + metadata.clear_fault(); + let reopened = + NoKvFs::open_existing(MountId::new(1).unwrap(), metadata.clone(), objects, 0).unwrap(); + let reopened_operation = durable_restore_operation(&reopened, snapshot.snapshot_id); + assert_eq!(reopened_operation.destination_root, destination_root); + assert_eq!( + reopened_operation.state, + restore::RestoreOperationState::Complete + ); + let version = reopened.read_version().unwrap(); + let seal = restore_gc::decode_restore_base_seal( + &reopened + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_base_seal_key(reopened.mount_id(), reopened_operation.ref_set_id), + version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .expect("adaptive reference set must be sealed") + .0, + ) + .unwrap(); + assert_eq!(seal.reference_count, REFERENCE_COUNT as u64); + let inverse_prefix = restore::restore_control_keyspaces(reopened.mount_id()) + .into_iter() + .find_map(|(name, prefix)| (name == "base_inverse").then_some(prefix)) + .expect("base inverse keyspace is registered"); + for prefix in [ + restore::restore_base_owner_prefix(reopened.mount_id(), reopened_operation.ref_set_id), + inverse_prefix, + restore::restore_base_inverse_owner_prefix( + reopened.mount_id(), + reopened_operation.ref_set_id, + ), + ] { + let rows = reopened + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix, + start_after: None, + version, + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + assert_eq!(rows.len(), REFERENCE_COUNT); + assert_eq!( + rows.iter() + .map(|row| row.key.as_slice()) + .collect::>() + .len(), + REFERENCE_COUNT, + "exact-reference projection contains a duplicate key" + ); + } + assert_eq!( + read_artifact_at_path(&reopened, "/destination/group-1/artifact-63.bin"), + vec![63] ); + let report = reopened.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); } #[test] -fn failed_allocator_archive_keeps_local_watermark_slow_until_pending_flush() { - let backing = MemoryObjectStore::new(); - let objects = FaultObjectStore::new(backing); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), - ); +fn restore_release_exact_reference_batch_adapts_to_byte_budget_ack_loss_and_reopen() { + const REFERENCE_COUNT: usize = 64; + const REFERENCES_PER_DIRECTORY: usize = 32; + const DIGEST_PADDING: usize = 134 * 1024; + const RELEASE_REQUEST_PREFIX: &[u8] = b"restore-release-reference-batch"; + + let metadata = OversizedManifestStore::new(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - // Initialize the object-GC claim before enabling the log so the only logged - // command in this workload is the allocator reservation at the boundary. service - .prepare_artifact_create(InodeId::root(), dname(b"claim-warmup.bin")) + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + for directory in 0..2 { + service + .create_dir_path(&format!("/source/group-{directory}"), 0o755, 1000, 1000) + .unwrap(); + } + let large_digest = format!("sha256:{}", "r".repeat(DIGEST_PADDING)); + let mut expected_objects = HashSet::with_capacity(REFERENCE_COUNT); + for index in 0..REFERENCE_COUNT { + let directory = index / REFERENCES_PER_DIRECTORY; + let path = format!("/source/group-{directory}/artifact-{index:02}.bin"); + let prepared = service.prepare_artifact_create_path(&path).unwrap(); + let written = service + .stage_prepared_artifact_ranges( + &prepared, + &format!("restore/adaptive-release-reference-{index:02}"), + &[PublishArtifactRange { + offset: 0, + bytes: vec![index as u8], + }], + 0, + ) + .unwrap(); + let staged = written.staged_objects().unwrap(); + let mut chunks = written.chunk_manifests(); + chunks[0].slices[0].blocks[0].digest_uri = large_digest.clone(); + service + .publish_prepared_artifact_staged_session( + prepared.clone(), + PublishArtifactStagedSession { + parent: prepared.parent, + name: prepared.name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:body".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: format!("restore/adaptive-release-reference-{index:02}"), + size: 1, + chunks, + staged, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + let artifact = service.lookup_path(&path).unwrap().unwrap(); + let body = artifact.body.as_ref().unwrap(); + assert!(expected_objects.insert( + block_key(artifact.attr.inode, body.generation, 0, 0) + .as_str() + .to_owned() + )); + } + + let projection_counts = |service: &NoKvFs, + ref_set_id: u64| { + let version = service.read_version().unwrap(); + let count_prefix = |prefix| { + service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix, + start_after: None, + version, + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .len() + }; + let inverse_prefix = restore::restore_control_keyspaces(service.mount_id()) + .into_iter() + .find_map(|(name, prefix)| (name == "base_inverse").then_some(prefix)) + .expect("base inverse keyspace is registered"); + let inverse_count = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: inverse_prefix, + start_after: None, + version, + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .into_iter() + .filter(|row| { + restore_gc::decode_restore_base_inverse(&row.value.0) + .is_ok_and(|inverse| inverse.ref_set_id == ref_set_id) + }) + .count(); + [ + count_prefix(restore::restore_base_owner_prefix( + service.mount_id(), + ref_set_id, + )), + inverse_count, + count_prefix(restore::restore_base_inverse_owner_prefix( + service.mount_id(), + ref_set_id, + )), + ] + }; + let queued_exact_objects = |service: &NoKvFs| { + let mut queued = BTreeMap::::new(); + for row in service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount_id()), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + { + let record = decode_object_gc_record(&row.value.0).unwrap(); + if expected_objects.contains(&record.object_key) { + *queued.entry(record.object_key).or_default() += 1; + } + } + queued + }; + + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") .unwrap(); + let operation = durable_restore_operation(&service, snapshot.snapshot_id); + let ref_set_id = operation.ref_set_id; + assert_eq!( + projection_counts(&service, ref_set_id), + [REFERENCE_COUNT; 3] + ); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/allocator-pending-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + .create_dir_path("/replacement", 0o755, 1000, 1000) .unwrap(); - - while service.clock.load(Ordering::Relaxed).saturating_add(1) - <= service.reserved_version.load(Ordering::Relaxed) - && service.next_inode.load(Ordering::Relaxed).saturating_add(1) - <= service.reserved_next_inode.load(Ordering::Relaxed) - { - service - .prepare_artifact_create(InodeId::root(), dname(b"boundary.bin")) - .unwrap(); - } - let reserved_version_before = service.reserved_version.load(Ordering::Relaxed); - let reserved_inode_before = service.reserved_next_inode.load(Ordering::Relaxed); - objects.fail_puts_containing("meta/allocator-pending-log/log/"); - - assert!(matches!( - service.prepare_artifact_create(InodeId::root(), dname(b"first-fails.bin")), - Err(MetadError::SyncLogArchiveFailed { - committed: true, - .. - }) - )); - assert_eq!( - service.reserved_version.load(Ordering::Relaxed), - reserved_version_before, - "a locally applied but unarchived reservation must not open the fast path" + service + .rename_replace_path("/replacement", "/destination") + .unwrap(); + assert_eq!(service.restore_metrics().unwrap().releasing, 1); + + metadata.fail_after_apply(RELEASE_REQUEST_PREFIX, 0); + assert_eq!(service.cleanup_restore_releases(1).unwrap(), 1); + let first_mutation_counts = + metadata.applied_mutation_counts_with_prefix(RELEASE_REQUEST_PREFIX); + assert_eq!(first_mutation_counts.len(), 1); + let first_mutation_count = first_mutation_counts[0]; + assert_eq!(first_mutation_count.saturating_sub(1) % 4, 0); + let first_reference_count = first_mutation_count.saturating_sub(1) / 4; + assert!(first_reference_count > 0); + assert!( + first_reference_count < 61, + "the 61-reference release page must be shortened by the 8 MiB packer" ); + assert!(first_reference_count < REFERENCE_COUNT); assert_eq!( - service.reserved_next_inode.load(Ordering::Relaxed), - reserved_inode_before, - "inode reservation watermark must stay behind until pending archive flush" + metadata.applied_request_counts_with_prefix(RELEASE_REQUEST_PREFIX), + vec![1], + "the apply-then-ACK-loss release request must not be applied twice" ); - assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); + let expected_remaining = REFERENCE_COUNT - first_reference_count; + assert_eq!( + projection_counts(&service, ref_set_id), + [expected_remaining; 3] + ); + let first_queued = queued_exact_objects(&service); + assert_eq!(first_queued.len(), first_reference_count); + assert!(first_queued.values().all(|count| *count == 1)); + assert!(expected_objects.iter().all(|object_key| { + objects + .head(&ObjectKey::new(object_key.clone()).unwrap()) + .unwrap() + .is_some() + })); + drop(service); - objects.clear_faults(); - service - .prepare_artifact_create(InodeId::root(), dname(b"second-flushes.bin")) - .unwrap(); - let snapshot = service.sync_metadata_log_snapshot().unwrap(); + metadata.clear_fault(); + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + assert_eq!(reopened.restore_metrics().unwrap().releasing, 1); assert_eq!( - snapshot.durable_lsn, 2, - "the retry must archive the prior applied reservation before its own reservation" + projection_counts(&reopened, ref_set_id), + [expected_remaining; 3], + "reopen must resume from the ACK-reconciled release frontier" ); - let kinds = snapshot - .segments + for _ in 0..1_024 { + if reopened.restore_metrics().unwrap().releasing == 0 { + break; + } + reopened.cleanup_restore_releases(1).unwrap(); + } + let final_metrics = reopened.restore_metrics().unwrap(); + assert_eq!(final_metrics.releasing, 0); + assert_eq!(final_metrics.operation_count(), 0); + assert_eq!(projection_counts(&reopened, ref_set_id), [0; 3]); + + let mutation_counts = metadata.applied_mutation_counts_with_prefix(RELEASE_REQUEST_PREFIX); + let released_reference_counts = mutation_counts .iter() - .map(|segment| { - service - .load_metadata_log_segment(&segment.segment_key) - .unwrap() + .map(|count| { + assert_eq!(count.saturating_sub(1) % 4, 0); + count.saturating_sub(1) / 4 }) - .flat_map(|segment| segment.entries.into_iter()) - .map(|entry| entry.command.kind) .collect::>(); assert_eq!( - kinds, - vec![CommandKind::ReserveAllocator, CommandKind::ReserveAllocator] + released_reference_counts.iter().sum::(), + REFERENCE_COUNT + ); + assert!( + released_reference_counts.len() >= 2, + "134 KiB exact references must span multiple release batches: {released_reference_counts:?}" + ); + assert!( + released_reference_counts.iter().copied().max().unwrap_or(0) < 61, + "every release batch must remain under the 8 MiB bound: {released_reference_counts:?}" + ); + let request_apply_counts = metadata.applied_request_counts_with_prefix(RELEASE_REQUEST_PREFIX); + assert_eq!(request_apply_counts.len(), mutation_counts.len()); + assert!( + request_apply_counts.iter().all(|count| *count == 1), + "every release request id must apply exactly once: {request_apply_counts:?}" ); + assert_eq!( + metadata.applied_with_prefix(RELEASE_REQUEST_PREFIX), + mutation_counts.len() + ); + let final_queued = queued_exact_objects(&reopened); + assert_eq!(final_queued.len(), REFERENCE_COUNT); + assert!(final_queued.values().all(|count| *count == 1)); + assert!(expected_objects.iter().all(|object_key| { + objects + .head(&ObjectKey::new(object_key.clone()).unwrap()) + .unwrap() + .is_some() + })); + let report = reopened.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); } #[test] -fn sync_metadata_log_archives_independent_batch_as_one_segment() { - let (service, objects) = service_with_objects(); - service.create_dir_path("/left", 0o755, 1000, 1000).unwrap(); +fn restore_reference_page_serializes_inverse_validation_with_release() { + let mount = MountId::new(1).unwrap(); + let metadata = RestoreBaseInverseScanBarrierStore::new(mount); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new(mount, metadata.clone(), objects.clone())); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); service - .create_dir_path("/right", 0o755, 1000, 1000) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-sync-batch-log", 2); - service.backup_metadata(&checkpoint_config).unwrap(); + publish_path_artifact( + &service, + "/source/shared.bin", + "restore/release-validation-race", + b"shared across the release race", + ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/sync-batch-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) - .unwrap(); - - let results = service.create_file_batches_in_dir_path(vec![ - CreateInDirPathBatch { - parent_path: "/left".to_owned(), - names: vec![DentryName::new(b"a.bin".to_vec()).unwrap()], - mode: 0o644, - uid: 1000, - gid: 1000, - }, - CreateInDirPathBatch { - parent_path: "/right".to_owned(), - names: vec![DentryName::new(b"b.bin".to_vec()).unwrap()], - mode: 0o644, - uid: 1000, - gid: 1000, - }, - ]); - - assert_eq!(results.len(), 2); - for result in &results { - assert_eq!(result.as_ref().unwrap().len(), 1); - } - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.durable_lsn, 2); - assert_eq!(snapshot.segments.len(), 1); - let segment = service - .load_metadata_log_segment(&snapshot.segments.last().unwrap().segment_key) + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/first") .unwrap(); - assert_eq!(segment.first_lsn, 1); - assert_eq!(segment.last_lsn, 2); - assert_eq!(segment.entries.len(), 2); + service.remove_file_path("/first/shared.bin").unwrap(); + service.remove_empty_dir_path("/first").unwrap(); + assert_eq!(service.restore_metrics().unwrap().releasing, 1); - let segment_keys = snapshot_segment_keys(&snapshot); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), + metadata.arm(); + let restore_service = Arc::clone(&service); + let snapshot_id = snapshot.snapshot_id; + let restore = std::thread::spawn(move || { + restore_service.restore_subtree_path_to_fork("/source", snapshot_id, "/second") + }); + metadata.wait_until_blocked(); + assert!( + service.object_gc_gate.try_lock().is_err(), + "reference validation must hold object_gc_gate before reading an inverse" + ); + + let release_service = Arc::clone(&service); + let release_started = Arc::new(Barrier::new(2)); + let release_thread_started = Arc::clone(&release_started); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let release = std::thread::spawn(move || { + release_thread_started.wait(); + let result = release_service.cleanup_restore_releases(64); + release_tx.send(result).unwrap(); + }); + release_started.wait(); + assert!( + release_rx.recv_timeout(Duration::from_millis(100)).is_err(), + "release must not delete an inverse between validation's scan and owner read" ); - let outcome = recovered - .restore_metadata_with_archived_log_segments( - &checkpoint_config, - "mount-1:/", - &segment_keys, - 0, - METADATA_LOG_ZERO_DIGEST, - ) + + metadata.release_blocked(); + let outcome = restore.join().unwrap().unwrap(); + assert_eq!(outcome.state, RestoreState::Complete); + assert!(service.lookup_path("/second").unwrap().is_some()); + release_rx + .recv_timeout(Duration::from_secs(5)) .unwrap() .unwrap(); + release.join().unwrap(); - assert_eq!(outcome.replayed_entries, 2); - assert!(recovered.lookup_path("/left/a.bin").unwrap().is_some()); - assert!(recovered.lookup_path("/right/b.bin").unwrap().is_some()); + for _ in 0..256 { + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + service.cleanup_restore_releases(64).unwrap(); + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); + drop(service); + + let reopened = NoKvFs::open_existing(mount, metadata, objects, 0).unwrap(); + assert_eq!( + read_artifact_at_path(&reopened, "/second/shared.bin"), + b"shared across the release race" + ); + let report = reopened.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); } #[test] -fn post_commit_backend_readback_preserves_single_commit_log_order() { - let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); - let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-post-commit-readback", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); +fn restore_reference_validation_rejects_a_durable_owner_orphan() { + let service = service(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/post-commit-readback-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - - store.arm(); - let first = ordered_log_put_command( - b"req-post-commit-first", - b"post-commit-first", - b"first", - checkpoint.commit_version, - checkpoint.commit_version + 1, + publish_path_artifact( + &service, + "/source/shared.bin", + "restore/durable-owner-orphan", + b"fail closed on a real owner orphan", ); - service.commit_metadata(first).unwrap(); - let second = ordered_log_put_command( - b"req-post-commit-second", - b"post-commit-second", - b"second", - checkpoint.commit_version + 1, - checkpoint.commit_version + 2, + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/first") + .unwrap(); + let (operation, _) = load_completed_restore_operation_and_seal( + &service, + "/source", + snapshot.snapshot_id, + "/first", ); - service.commit_metadata(second).unwrap(); - - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.durable_lsn, 2); - let request_ids = snapshot - .segments - .iter() - .map(|pointer| { - service - .load_metadata_log_segment(&pointer.segment_key) - .unwrap() + let owner = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_base_owner_prefix(service.mount_id(), operation.ref_set_id), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, }) - .flat_map(|segment| segment.entries.into_iter()) - .map(|entry| entry.command.request_id) - .collect::>(); - assert_eq!( - request_ids, - vec![ - b"req-post-commit-first".to_vec(), - b"req-post-commit-second".to_vec() - ] - ); - - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects, - ); - recovered - .restore_metadata_with_archived_log_segments( - &checkpoint_config, - "mount-1:/", - &snapshot_segment_keys(&snapshot), - checkpoint.log_lsn, - checkpoint.log_digest, - ) .unwrap() + .into_iter() + .next() + .unwrap(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: b"test-delete-restore-base-owner".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: owner.key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: owner.key.clone(), + predicate: Predicate::VersionEquals(owner.version), + }], + mutations: vec![delete_mutation(RecordFamily::System, owner.key)], + watch: Vec::new(), + }) .unwrap(); + + assert!(matches!( + service.restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/second"), + Err(MetadError::Codec(message)) + if message == "restore base inverse has no canonical owner" + )); assert_eq!( - recovered - .metadata - .get( - RecordFamily::System, - b"post-commit-second", - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap(), - Some(Value(b"second".to_vec())) + read_artifact_at_path(&service, "/first/shared.bin"), + b"fail closed on a real owner orphan" ); + let report = service.fsck_restore_state(false).unwrap(); + assert!(!report.is_consistent()); + assert!(report.issues.iter().any(|issue| { + issue.code == "orphan_or_mismatched_base_inverse" && issue.message.contains("owner") + })); } #[test] -fn readback_error_blocks_later_apply_and_both_checkpoint_publication_paths() { - let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); +fn restore_overlay_release_does_not_repeat_historical_retention_walks() { + const FILE_COUNT: usize = 65; + const OVERLAY_TRANSITION_REQUEST_PREFIX: &[u8] = b"restore-enter-release-overlay"; + + let mount = MountId::new(1).unwrap(); + let purpose_tracking = PurposeTrackingStore::new(); + let metadata = RestorePostApplyFaultStore::wrapping(purpose_tracking.clone()); let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects); + let service = NoKvFs::new(mount, metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-readback-block", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/readback-block-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let names = (0..FILE_COUNT) + .map(|index| DentryName::new(format!("indexed-{index:03}.csv")).unwrap()) + .collect::>(); + service + .create_files_in_dir_path("/source", names, 0o644, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") .unwrap(); + let operation_digest = + restore::restore_operation_digest(mount, "/source", snapshot.snapshot_id, "/restored"); + let operation = restore::decode_restore_operation( + &service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_operation_key(mount, &operation_digest), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap() + .0, + ) + .unwrap(); + for index in 0..FILE_COUNT { + service + .remove_file_path(&format!("/restored/indexed-{index:03}.csv")) + .unwrap(); + } + service.remove_empty_dir_path("/restored").unwrap(); + metadata.fail_after_apply(OVERLAY_TRANSITION_REQUEST_PREFIX, 0); - store.arm(); - store.fail_next_readbacks(4); - let first = ordered_log_put_command( - b"req-readback-block-first", - b"readback-block-first", - b"first", - checkpoint.commit_version, - checkpoint.commit_version + 1, - ); - assert!(matches!( - service.commit_metadata(first), - Err(MetadError::Codec(message)) if message.contains("readback failed") - )); + { + let load_job = || { + service + .metadata_store() + .get_versioned( + RecordFamily::System, + &restore::restore_release_job_key(mount, operation.ref_set_id), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .map(|item| { + ( + restore::decode_restore_release_job(&item.value.0).unwrap(), + item.version, + ) + }) + }; + for _ in 0..4096 { + if load_job() + .as_ref() + .is_some_and(|(job, _)| job.phase == restore::RestoreReleasePhase::Overlay) + { + break; + } + service.cleanup_restore_releases(1).unwrap(); + } + let (overlay, _) = load_job().expect("release must reach the durable Overlay phase"); + assert_eq!(overlay.phase, restore::RestoreReleasePhase::Overlay); + assert!( + service.restore_metrics().unwrap().index_rows > FILE_COUNT, + "the overlay regression must cross a 64-row release page" + ); + assert_eq!( + metadata.applied_with_prefix(OVERLAY_TRANSITION_REQUEST_PREFIX), + 1, + "the durable Overlay transition must survive its lost acknowledgement" + ); + } + drop(service); - let blocked = ordered_log_put_command( - b"req-readback-block-second", - b"readback-block-second", - b"second", - checkpoint.commit_version + 1, - checkpoint.commit_version + 2, - ); - assert!(matches!( - service.commit_metadata(blocked), - Err(MetadError::Codec(message)) if message.contains("readback failed") - )); - assert_eq!( + metadata.clear_fault(); + let service = NoKvFs::open_existing(mount, metadata.clone(), objects, 0).unwrap(); + let load_job = || { service - .metadata - .get( + .metadata_store() + .get_versioned( RecordFamily::System, - b"readback-block-second", - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, + &restore::restore_release_job_key(mount, operation.ref_set_id), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, ) - .unwrap(), - None - ); + .unwrap() + .map(|item| { + ( + restore::decode_restore_release_job(&item.value.0).unwrap(), + item.version, + ) + }) + }; + let (overlay, _) = load_job().expect("Overlay phase must survive reopen"); + assert_eq!(overlay.phase, restore::RestoreReleasePhase::Overlay); + let before = purpose_tracking.counts(); + let initial_cursor = overlay.cursor; + let mut advanced = false; + for _ in 0..4 { + service.cleanup_restore_releases(1).unwrap(); + match load_job() { + None => { + advanced = true; + break; + } + Some((job, _)) if job.cursor != initial_cursor => { + advanced = true; + break; + } + Some(_) => {} + } + } + assert!(advanced, "the overlay release cursor did not advance"); + let after = purpose_tracking.counts(); + assert_eq!(after.snapshot_gets, before.snapshot_gets); + assert_eq!(after.snapshot_scans, before.snapshot_scans); - assert!(matches!( - service.backup_metadata(&checkpoint_config), - Err(MetadError::Codec(message)) if message.contains("readback failed") - )); - let controlled = MetadataArchiveConfig::new("meta/controlled-readback-block", 2); - assert!(matches!( - service.prepare_immutable_metadata_backup(&controlled), - Err(MetadError::Codec(message)) if message.contains("readback failed") - )); - assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); + for _ in 0..256 { + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + service.cleanup_restore_releases(1).unwrap(); + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); + let after_finish = purpose_tracking.counts(); + assert_eq!(after_finish.snapshot_gets, before.snapshot_gets); + assert_eq!(after_finish.snapshot_scans, before.snapshot_scans); + let report = service.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); +} - store.clear_readback_failures(); - let third = ordered_log_put_command( - b"req-readback-block-third", - b"readback-block-third", - b"third", - checkpoint.commit_version + 1, - checkpoint.commit_version + 3, +#[test] +fn restore_fsck_keeps_one_version_and_reports_damage_across_concurrent_repair() { + let objects = HeadBarrierObjectStore::new(MemoryObjectStore::new()); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let source = publish_path_artifact( + service.as_ref(), + "/source/shared.bin", + "restore/fsck-point-in-time", + b"point-in-time restore fsck", + ); + let source_body = source.body.as_ref().unwrap(); + let object_key = block_key(source.attr.inode, source_body.generation, 0, 0); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + + assert!(objects.delete(&object_key).unwrap()); + let damaged_version = service.read_version().unwrap(); + let graph_sequence = service.restore_graph_sequence.load(Ordering::Acquire); + + objects.arm(); + let fsck_service = Arc::clone(&service); + let fsck = std::thread::spawn(move || fsck_service.fsck_restore_state(true)); + objects.wait_until_head(); + + // Repair the physical object after HEAD captured its absence, then commit + // unrelated metadata before the restore-private scans finish. A correct + // point-in-time proof must return the captured damage instead of rejecting + // the concurrent commit through the old mount-global epoch check. + objects + .put(&object_key, b"point-in-time restore fsck".to_vec()) + .unwrap(); + let epoch_before_repair = service.path_cache_epoch.load(Ordering::Acquire); + let cursor_key = restore::restore_init_upload_tombstone_cursor_key(service.mount_id()); + let cursor_prefix = restore::restore_init_upload_tombstone_prefix(service.mount_id()); + for index in 0..16 { + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: format!("test-rotate-restore-scheduler-cursor-{index}").into_bytes(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: cursor_key.clone(), + predicates: Vec::new(), + mutations: vec![Mutation { + family: RecordFamily::System, + key: cursor_key.clone(), + op: MutationOp::Put, + value: Some(Value(if index % 2 == 0 { + cursor_prefix.clone() + } else { + Vec::new() + })), + }], + watch: Vec::new(), + }) + .unwrap(); + } + service + .create_dir_path("/unrelated", 0o755, 1000, 1000) + .unwrap(); + let repaired_version = service.read_version().unwrap(); + assert!( + service.path_cache_epoch.load(Ordering::Acquire) > epoch_before_repair, + "concurrent commit did not exercise the old global-epoch rejection path" ); - service.commit_metadata(third).unwrap(); - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.durable_lsn, 2); - let request_ids = snapshot - .segments - .iter() - .map(|pointer| { - service - .load_metadata_log_segment(&pointer.segment_key) - .unwrap() - }) - .flat_map(|segment| segment.entries.into_iter()) - .map(|entry| entry.command.request_id) - .collect::>(); assert_eq!( - request_ids, - vec![ - b"req-readback-block-first".to_vec(), - b"req-readback-block-third".to_vec() - ] + service.restore_graph_sequence.load(Ordering::Acquire), + graph_sequence, + "unrelated writes and scheduler cursor movement changed the restore graph sequence" + ); + + objects.release_head(); + let report = fsck.join().unwrap().unwrap(); + assert_eq!(report.metrics.read_version, damaged_version.get()); + assert!(!report.is_consistent()); + assert!(report + .dangling_borrowed_objects + .iter() + .any(|dangling| dangling.object_key == object_key.as_str())); + + let repaired = service.fsck_restore_state(true).unwrap(); + assert!( + repaired.is_consistent(), + "repaired restore fsck: {repaired:#?}" ); + assert!(repaired.metrics.read_version >= repaired_version.get()); } #[test] -fn mismatched_readback_result_blocks_later_apply_until_exact_result_is_visible() { - let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - store.clone(), +fn restore_metrics_samples_allocator_fence_after_concurrent_reservation() { + let mount = MountId::new(1).unwrap(); + let metadata = AllocatorFenceBarrierStore::new(mount); + let service = Arc::new(NoKvFs::new( + mount, + metadata.clone(), MemoryObjectStore::new(), - ); + )); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-readback-mismatch", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/readback-mismatch-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") .unwrap(); - store.arm(); - store.mismatch_next_readbacks(2); - let first = ordered_log_put_command( - b"req-readback-mismatch-first", - b"readback-mismatch-first", - b"first", - checkpoint.commit_version, - checkpoint.commit_version + 1, + let allocator_key = allocator_key(mount); + let max_version = Version::new(u64::MAX).unwrap(); + let before = metadata + .get_versioned( + RecordFamily::System, + &allocator_key, + max_version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let graph_sequence = service.restore_graph_sequence.load(Ordering::SeqCst); + + metadata.arm(); + let metrics_service = Arc::clone(&service); + let metrics = std::thread::spawn(move || metrics_service.restore_metrics()); + metadata.wait_until_active_read(); + + let reservation_boundary = service.reserved_version.load(Ordering::Relaxed); + while service.clock.load(Ordering::Relaxed) <= reservation_boundary { + service.next_version().unwrap(); + } + let after = metadata + .get_versioned( + RecordFamily::System, + &allocator_key, + max_version, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + assert!(after.version > before.version); + assert_eq!( + service.restore_graph_sequence.load(Ordering::SeqCst), + graph_sequence, + "ordinary allocator high-water reservation changed the restore graph sequence" ); - assert!(matches!( - service.commit_metadata(first), - Err(MetadError::Codec(message)) if message.contains("result mismatch") + + metadata.release_active_read(); + let metrics = metrics.join().unwrap().unwrap(); + assert_eq!(metrics.complete, 1); + assert!(metrics.active_marker); + assert!(metrics.allocator_v2_fenced); +} + +#[test] +fn restore_fsck_retries_when_the_graph_changes_after_a_scan_page() { + let mount = MountId::new(1).unwrap(); + let metadata = RestoreBaseInverseScanBarrierStore::new(mount); + let service = Arc::new(NoKvFs::new( + mount, + metadata.clone(), + MemoryObjectStore::new(), )); - let blocked = ordered_log_put_command( - b"req-readback-mismatch-blocked", - b"readback-mismatch-blocked", - b"blocked", - checkpoint.commit_version + 1, - checkpoint.commit_version + 2, + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + service.as_ref(), + "/source/artifact.bin", + "restore/fsck-reader-first", + b"reader-first graph rewrite", ); - assert!(matches!( - service.commit_metadata(blocked), - Err(MetadError::Codec(message)) if message.contains("result mismatch") - )); - assert!(service - .metadata - .get( + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + + let activation_key = restore::restore_activation_fence_key(mount); + let activation = metadata + .get_versioned( RecordFamily::System, - b"readback-mismatch-blocked", + &activation_key, Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, + ReadPurpose::WritePlanLocal, ) .unwrap() - .is_none()); - - store.clear_readback_mismatches(); - let trigger = ordered_log_put_command( - b"req-readback-mismatch-trigger", - b"readback-mismatch-trigger", - b"trigger", - checkpoint.commit_version + 1, - checkpoint.commit_version + 3, - ); - service.commit_metadata(trigger).unwrap(); - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.durable_lsn, 2); -} + .unwrap(); + metadata.arm(); + let fsck_service = Arc::clone(&service); + let fsck = std::thread::spawn(move || fsck_service.fsck_restore_state(false)); + metadata.wait_until_blocked(); -#[test] -fn batch_post_commit_backend_readback_archives_full_batch_in_input_order() { - let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); - let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-batch-readback", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + let rewrite_version = service.next_version().unwrap(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/batch-readback-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + .commit_metadata(MetadataCommand { + request_id: b"test-rewrite-restore-activation-during-fsck".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(rewrite_version).unwrap(), + commit_version: rewrite_version, + primary_family: RecordFamily::System, + primary_key: activation_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: activation_key.clone(), + predicate: Predicate::VersionEquals(activation.version), + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: activation_key, + op: MutationOp::Put, + value: Some(activation.value), + }], + watch: Vec::new(), + }) .unwrap(); + metadata.release_blocked(); - let commands = vec![ - ordered_log_put_command( - b"req-batch-readback-first", - b"batch-readback-first", - b"first", - checkpoint.commit_version, - checkpoint.commit_version + 1, - ), - ordered_log_put_command( - b"req-batch-readback-second", - b"batch-readback-second", - b"second", - checkpoint.commit_version, - checkpoint.commit_version + 2, - ), - ]; - store.fail_next_batch_results(vec![0, 1]); - let results = service.commit_independent_metadata_batch(&commands); - assert!(results.iter().all(Result::is_ok)); + let report = fsck.join().unwrap().unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); + assert!(report.metrics.read_version >= rewrite_version.get()); +} - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.durable_lsn, 2); - assert_eq!(snapshot.segments.len(), 1); - let segment = service - .load_metadata_log_segment(&snapshot.segments[0].segment_key) +#[test] +fn restore_fsck_point_checks_missing_index_inverse_rows() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - assert_eq!( - segment - .entries - .iter() - .map(|entry| entry.command.request_id.as_slice()) - .collect::>(), - vec![ - b"req-batch-readback-first".as_slice(), - b"req-batch-readback-second".as_slice() - ] + publish_path_artifact( + &service, + "/source/artifact.bin", + "restore/fsck-missing-index-inverse", + b"indexed artifact", ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects, - ); - let outcome = recovered - .restore_metadata_with_archived_log_segments( - &checkpoint_config, - "mount-1:/", - &snapshot_segment_keys(&snapshot), - checkpoint.log_lsn, - checkpoint.log_digest, - ) + let inverse_prefix = restore_index::restore_index_private_keyspaces(service.mount_id()) + .into_iter() + .find_map(|(name, prefix)| (name == "index_parent_inverse").then_some(prefix)) + .unwrap(); + let inverse = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: inverse_prefix, + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap() + .into_iter() + .next() .unwrap(); - assert_eq!(outcome.replayed_entries, 2); - assert_eq!( - recovered - .metadata - .get( - RecordFamily::System, - b"batch-readback-second", - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap(), - Some(Value(b"second".to_vec())) - ); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: b"test-delete-restore-index-inverse-before-fsck".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: inverse.key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: inverse.key.clone(), + predicate: Predicate::VersionEquals(inverse.version), + }], + mutations: vec![delete_mutation(RecordFamily::System, inverse.key)], + watch: Vec::new(), + }) + .unwrap(); + + let report = service.fsck_restore_state(false).unwrap(); + assert!(!report.is_consistent()); + assert!(report + .issues + .iter() + .any(|issue| issue.code == "index_closure_mismatch")); } #[test] -fn unresolved_early_batch_subgroup_freezes_later_success_and_mixed_failure() { - let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); - let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects); +fn restore_fsck_rejects_a_graph_apply_before_the_writer_returns() { + const REQUEST_ID: &[u8] = b"test-restore-graph-post-apply-fence"; + + let metadata = PostCommitBarrierStore::new(REQUEST_ID); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + )); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-subgroup-readback", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/subgroup-readback-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - - let first = ordered_log_put_command( - b"req-subgroup-first", - b"subgroup-shared-key", - b"first", - checkpoint.commit_version, - checkpoint.commit_version + 1, - ); - // Same key forces Holt to flush the first atomic subgroup before planning - // and applying this later successful subgroup. - let second = ordered_log_put_command( - b"req-subgroup-second", - b"subgroup-shared-key", - b"second", - checkpoint.commit_version + 1, - checkpoint.commit_version + 2, - ); - let mut rejected = ordered_log_put_command( - b"req-subgroup-rejected", - b"subgroup-rejected-key", - b"rejected", - checkpoint.commit_version, - checkpoint.commit_version + 3, - ); - rejected.predicates.push(PredicateRef { - family: RecordFamily::System, - key: b"missing-subgroup-predicate-key".to_vec(), - predicate: Predicate::Exists, - }); - store.fail_next_batch_results(vec![0]); - store.fail_next_readbacks(1); - let results = - service.commit_independent_metadata_batch(&[first.clone(), second.clone(), rejected]); - assert!(results - .iter() - .all(|result| matches!(result, Err(MetadError::Codec(message)) if message.contains("readback failed")))); - assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); - assert_eq!( - service - .metadata - .get( - RecordFamily::System, - b"subgroup-shared-key", - Version::new(u64::MAX).unwrap(), - ReadPurpose::UserStrong, - ) - .unwrap(), - Some(Value(b"second".to_vec())), - "the later subgroup applied but must not be acknowledged or archived early" + publish_path_artifact( + service.as_ref(), + "/source/artifact.bin", + "restore/fsck-post-apply-fence", + b"indexed artifact", ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); - store.clear_readback_failures(); - let trigger = ordered_log_put_command( - b"req-subgroup-trigger", - b"subgroup-trigger-key", - b"trigger", - checkpoint.commit_version + 2, - checkpoint.commit_version + 4, - ); - service.commit_metadata(trigger).unwrap(); - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.durable_lsn, 3); - assert_eq!(snapshot.segments.len(), 2); - let resolved_group = service - .load_metadata_log_segment(&snapshot.segments[0].segment_key) + let inverse_prefix = restore_index::restore_index_private_keyspaces(service.mount_id()) + .into_iter() + .find_map(|(name, prefix)| (name == "index_parent_inverse").then_some(prefix)) .unwrap(); - assert_eq!( - resolved_group - .entries - .iter() - .map(|entry| entry.command.request_id.as_slice()) - .collect::>(), - vec![first.request_id.as_slice(), second.request_id.as_slice()], - "the true committed subset must archive once in original execution order" - ); -} + let inverse = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: inverse_prefix, + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .into_iter() + .next() + .unwrap(); + let version = service.next_version().unwrap(); + let command = MetadataCommand { + request_id: REQUEST_ID.to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: inverse.key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: inverse.key.clone(), + predicate: Predicate::VersionEquals(inverse.version), + }], + mutations: vec![delete_mutation(RecordFamily::System, inverse.key)], + watch: Vec::new(), + }; + let writer_service = Arc::clone(&service); + let writer = std::thread::spawn(move || writer_service.commit_metadata(command)); + metadata.wait_until_applied(); -use nokv_object::{ObjectInfo, ObjectRange}; -use std::sync::atomic::AtomicUsize; + assert_eq!(service.restore_graph_writers.load(Ordering::Acquire), 1); + let error = service.fsck_restore_state(false).unwrap_err(); + assert!( + error + .to_string() + .contains("restore fsck raced a restore graph write"), + "unexpected restore fsck error: {error}" + ); + let error = service.restore_metrics().unwrap_err(); + assert!( + error + .to_string() + .contains("restore metrics raced a restore graph write"), + "unexpected restore metrics error: {error}" + ); -/// An [`ObjectStore`] wrapper that injects PUT failures to simulate a crash at a -/// chosen point (e.g. after the checkpoint object is written but before the -/// `CURRENT` pointer is swapped). Reads and deletes always pass through. -#[derive(Clone)] -struct FaultObjectStore { - inner: MemoryObjectStore, - fail_put_substring: Arc>>, - injected_put_failures: Arc, - get_keys: Arc>>, + metadata.release_after_apply(); + writer.join().unwrap().unwrap(); + assert_eq!(service.restore_graph_writers.load(Ordering::Acquire), 0); + let report = service.fsck_restore_state(false).unwrap(); + assert!(!report.is_consistent()); + assert!(report + .issues + .iter() + .any(|issue| issue.code == "index_closure_mismatch")); } -impl FaultObjectStore { - fn new(inner: MemoryObjectStore) -> Self { - Self { - inner, - fail_put_substring: Arc::new(Mutex::new(None)), - injected_put_failures: Arc::new(AtomicUsize::new(0)), - get_keys: Arc::new(Mutex::new(Vec::new())), - } - } +#[test] +fn restore_fsck_uses_bounded_pages_for_large_private_graphs() { + const ENTRY_COUNT: usize = 300; + const FSCK_PAGE_ROWS: u64 = 256; - fn fail_puts_containing(&self, substring: &str) { - *self.fail_put_substring.lock().unwrap() = Some(substring.to_owned()); - } + let metadata = PurposeTrackingStore::new(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let names = (0..ENTRY_COUNT) + .map(|index| DentryName::new(format!("empty-{index:03}.bin")).unwrap()) + .collect::>(); + service + .create_files_in_dir_path("/source", names, 0o644, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); - fn clear_faults(&self) { - *self.fail_put_substring.lock().unwrap() = None; - } + metadata.reset_scan_bounds(); + let report = service.fsck_restore_state(false).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); + let counts = metadata.counts(); + assert_eq!(counts.unbounded_scans, 0); + assert!( + counts.largest_bounded_scan <= FSCK_PAGE_ROWS, + "restore fsck requested a {}-row page", + counts.largest_bounded_scan + ); + assert!(counts.write_plan_scans > 1); +} - fn injected_put_failures(&self) -> usize { - self.injected_put_failures.load(Ordering::Relaxed) - } +#[test] +fn restore_fsck_point_checks_missing_index_source_members() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .create_file_path("/source/artifact.bin", 0o644, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); - fn clear_get_keys(&self) { - self.get_keys.lock().unwrap().clear(); - } + let source_prefix = restore_index::restore_index_private_keyspaces(service.mount_id()) + .into_iter() + .find_map(|(name, prefix)| (name == "index_source_member").then_some(prefix)) + .unwrap(); + let source = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: source_prefix, + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .into_iter() + .next() + .unwrap(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: b"test-delete-restore-index-source-member".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: source.key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: source.key.clone(), + predicate: Predicate::VersionEquals(source.version), + }], + mutations: vec![delete_mutation(RecordFamily::System, source.key)], + watch: Vec::new(), + }) + .unwrap(); - fn got_key(&self, key: &str) -> bool { - self.get_keys.lock().unwrap().iter().any(|got| got == key) - } + let report = service.fsck_restore_state(false).unwrap(); + assert!(!report.is_consistent()); + assert!(report.issues.iter().any(|issue| { + issue.code == "index_closure_mismatch" && issue.message.contains("source member") + })); } -impl ObjectStore for FaultObjectStore { - fn put( - &self, - key: &ObjectKey, - bytes: impl Into, - ) -> Result { - if let Some(substring) = self.fail_put_substring.lock().unwrap().clone() { - if key.as_str().contains(&substring) { - self.injected_put_failures.fetch_add(1, Ordering::Relaxed); - return Err(ObjectError::Backend("injected put fault".to_owned())); - } - } - self.inner.put(key, bytes) - } - - fn get(&self, key: &ObjectKey, range: Option) -> Result, ObjectError> { - self.get_keys.lock().unwrap().push(key.as_str().to_owned()); - self.inner.get(key, range) - } +#[test] +fn restore_exact_references_allow_distinct_borrowers_of_one_canonical_object() { + let service = service(); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/base/shared.bin", + "shared-base", + b"shared object bytes", + ); + service.create_dir_path("/both", 0o755, 1000, 1000).unwrap(); + service + .clone_subtree_path_into("/base", "/both/one") + .unwrap(); + service + .clone_subtree_path_into("/base", "/both/two") + .unwrap(); + let snapshot = service.snapshot_subtree_path("/both").unwrap(); - fn head(&self, key: &ObjectKey) -> Result, ObjectError> { - self.inner.head(key) - } + service + .restore_subtree_path_to_fork("/both", snapshot.snapshot_id, "/restored") + .unwrap(); - fn delete(&self, key: &ObjectKey) -> Result { - self.inner.delete(key) + let one = service.lookup_path("/restored/one").unwrap().unwrap(); + let two = service.lookup_path("/restored/two").unwrap().unwrap(); + assert_eq!( + service + .read_artifact(one.attr.inode, &DentryName::new("shared.bin").unwrap()) + .unwrap(), + b"shared object bytes" + ); + assert_eq!( + service + .read_artifact(two.attr.inode, &DentryName::new("shared.bin").unwrap()) + .unwrap(), + b"shared object bytes" + ); + let (_, seal) = load_completed_restore_operation_and_seal( + &service, + "/both", + snapshot.snapshot_id, + "/restored", + ); + assert_eq!(seal.reference_count, 2); + let metrics = service.restore_metrics().unwrap(); + for keyspace in ["base_owner", "base_inverse", "base_inverse_owner"] { + assert_eq!(metrics.control_rows.get(keyspace), Some(&2)); } } #[test] -fn post_commit_backend_with_archive_failure_retains_exact_pending_segment() { - let backing = MemoryObjectStore::new(); - let objects = FaultObjectStore::new(backing); - let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); - let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-backend-archive-failure", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); +fn restore_exact_references_reject_inconsistent_identity_for_one_object_key() { + let service = service(); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/base/shared.bin", + "restore/inconsistent-object-base", + b"shared object bytes", + ); + service.create_dir_path("/both", 0o755, 1000, 1000).unwrap(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/backend-archive-failure-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + .clone_subtree_path_into("/base", "/both/one") + .unwrap(); + service + .clone_subtree_path_into("/base", "/both/two") .unwrap(); - objects.fail_puts_containing("meta/backend-archive-failure-log/log/"); - store.arm(); - let first = ordered_log_put_command( - b"req-backend-archive-first", - b"backend-archive-first", - b"first", - checkpoint.commit_version, - checkpoint.commit_version + 1, + let second = service + .lookup_path("/both/two/shared.bin") + .unwrap() + .unwrap(); + let second_body = second.body.as_ref().unwrap(); + let manifest_key = chunk_manifest_key( + service.mount_id(), + second.attr.inode, + second_body.generation, + 0, ); - assert!(matches!( - service.commit_metadata(first), - Err(MetadError::SyncLogArchiveFailed { - committed: true, - .. + let manifest_item = service + .metadata_store() + .get_versioned( + RecordFamily::ChunkManifest, + &manifest_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let mut manifest = decode_chunk_manifest(&manifest_item.value.0).unwrap(); + manifest.slices[0].blocks[0].digest_uri = "sha256:inconsistent".to_owned(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: b"test-corrupt-shared-object-identity".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::ChunkManifest, + primary_key: manifest_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::ChunkManifest, + key: manifest_key.clone(), + predicate: Predicate::VersionEquals(manifest_item.version), + }], + mutations: vec![Mutation { + family: RecordFamily::ChunkManifest, + key: manifest_key, + op: MutationOp::Put, + value: Some(Value(encode_chunk_manifest(&manifest))), + }], + watch: Vec::new(), }) - )); - assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); + .unwrap(); - objects.clear_faults(); - let second = ordered_log_put_command( - b"req-backend-archive-second", - b"backend-archive-second", - b"second", - checkpoint.commit_version + 1, - checkpoint.commit_version + 2, - ); - service.commit_metadata(second).unwrap(); - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.durable_lsn, 2); - let request_ids = snapshot - .segments - .iter() - .map(|pointer| { - service - .load_metadata_log_segment(&pointer.segment_key) - .unwrap() - }) - .flat_map(|segment| segment.entries.into_iter()) - .map(|entry| entry.command.request_id) - .collect::>(); - assert_eq!( - request_ids, - vec![ - b"req-backend-archive-first".to_vec(), - b"req-backend-archive-second".to_vec() - ] - ); + let snapshot = service.snapshot_subtree_path("/both").unwrap(); + assert!(matches!( + service.restore_subtree_path_to_fork( + "/both", + snapshot.snapshot_id, + "/destination", + ), + Err(MetadError::Codec(message)) + if message.contains("inconsistent object identity") + )); + assert!(service.lookup_path("/destination").unwrap().is_none()); } #[test] -fn publish_remains_readable_when_sync_log_ack_fails_after_metadata_commit() { - let backing = MemoryObjectStore::new(); - let objects = FaultObjectStore::new(backing.clone()); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/sync-log-ack", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) +fn restore_exact_references_outlive_source_for_delta_sparse_and_symlink_borrowers() { + let (service, objects) = service_with_objects(); + let source = service + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - objects.fail_puts_containing("meta/sync-log-ack/log/"); - - let name = dname(b"committed.bin"); - let payload = b"metadata committed before the archive acknowledgement"; - let error = service - .publish_artifact(artifact_request(name.clone(), "committed", payload)) - .unwrap_err(); - let staged = match error { - MetadError::PublishArtifactFailed { source, staged } => { - assert!(matches!( - *source, - MetadError::SyncLogArchiveFailed { - committed: true, - .. - } - )); - staged - } - other => panic!("unexpected publish error: {other:?}"), - }; - let committed = service - .lookup_plus(InodeId::root(), &name) - .unwrap() - .expect("the metadata transaction committed"); - assert_eq!( - service.read_artifact(InodeId::root(), &name).unwrap(), - payload + let append_base = publish_path_artifact( + &service, + "/source/append.bin", + "restore/append-base", + b"append-base", ); - assert_eq!(staged.len(), 1); - assert_eq!( - staged.objects()[0].key, - block_key( - committed.attr.inode, - committed.body.as_ref().unwrap().generation, - 0, + let append_base_body = append_base.body.as_ref().unwrap(); + let append_base_object = block_key(append_base.attr.inode, append_base_body.generation, 0, 0); + let append_prepared = service + .prepare_artifact_replace(source.attr.inode, dname(b"append.bin")) + .unwrap(); + let append_generation = append_prepared.generation; + let append_written = service + .stage_prepared_artifact_ranges( + &append_prepared, + "restore/append-delta", + &[PublishArtifactRange { + offset: append_base_body.size, + bytes: b"+delta".to_vec(), + }], 0, ) + .unwrap(); + service + .publish_prepared_artifact_staged_session( + append_prepared, + PublishArtifactStagedSession { + parent: source.attr.inode, + name: dname(b"append.bin"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:restore-append".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "restore/append-delta".to_owned(), + size: append_base_body.size + 6, + chunks: append_written.chunk_manifests(), + staged: append_written.staged_objects().unwrap(), + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + let append = service.lookup_path("/source/append.bin").unwrap().unwrap(); + assert_eq!( + append.body.as_ref().unwrap().base_generation, + append_base_body.generation ); - assert!(backing.head(&staged.objects()[0].key).unwrap().is_some()); -} + let append_delta_object = block_key(append.attr.inode, append_generation, 0, 0); -#[test] -fn checkpoint_keeps_referenced_blocks_after_committed_sync_log_failure() { - let backing = MemoryObjectStore::new(); - let objects = FaultObjectStore::new(backing.clone()); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/checkpoint-sync-log-ack", - "mount-1:/", - 1, + let sparse_base = publish_path_artifact( + &service, + "/source/sparse.bin", + "restore/sparse-base", + b"sparse-base", + ); + let sparse_base_body = sparse_base.body.as_ref().unwrap(); + let sparse_base_object = block_key(sparse_base.attr.inode, sparse_base_body.generation, 0, 0); + let sparse_tail_offset = DEFAULT_CHUNK_SIZE + 5; + let sparse_prepared = service + .prepare_artifact_replace(source.attr.inode, dname(b"sparse.bin")) + .unwrap(); + let sparse_generation = sparse_prepared.generation; + let sparse_written = service + .stage_prepared_artifact_ranges( + &sparse_prepared, + "restore/sparse-delta", + &[PublishArtifactRange { + offset: sparse_tail_offset, + bytes: b"tail".to_vec(), + }], 0, - METADATA_LOG_ZERO_DIGEST, - )) + ) .unwrap(); - objects.fail_puts_containing("meta/checkpoint-sync-log-ack/log/"); + service + .publish_prepared_artifact_staged_session( + sparse_prepared, + PublishArtifactStagedSession { + parent: source.attr.inode, + name: dname(b"sparse.bin"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:restore-sparse".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "restore/sparse-delta".to_owned(), + size: sparse_tail_offset + 4, + chunks: sparse_written.chunk_manifests(), + staged: sparse_written.staged_objects().unwrap(), + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + let sparse = service.lookup_path("/source/sparse.bin").unwrap().unwrap(); + assert_eq!( + sparse.body.as_ref().unwrap().base_generation, + sparse_base_body.generation + ); + let sparse_delta_object = block_key(sparse.attr.inode, sparse_generation, 1, 0); - let payload = b"checkpoint shard remains reachable after log ACK failure".to_vec(); - let error = service - .publish_checkpoint( - InodeId::root(), - vec![CheckpointShard { - name: dname(b"rank-0.ckpt"), - bytes: payload.clone(), - }], + let symlink_target = vec![b'x'; 8 * 1024]; + let symlink = service + .create_symlink( + source.attr.inode, + dname(b"latest"), + symlink_target.clone(), + 0o777, 1000, 1000, ) - .unwrap_err(); - assert!(matches!( - error, - MetadError::SyncLogArchiveFailed { - committed: true, - .. - } - )); + .unwrap(); + let symlink_body = symlink.body.as_ref().unwrap(); + let symlink_object = block_key(symlink.attr.inode, symlink_body.generation, 0, 0); + let source_objects = [ + append_base_object.clone(), + append_delta_object.clone(), + sparse_base_object.clone(), + sparse_delta_object.clone(), + symlink_object.clone(), + ]; + assert!(source_objects + .iter() + .all(|object| objects.head(object).unwrap().is_some())); - let committed = service - .lookup_path("/rank-0.ckpt") + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let outcome = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + let (_, seal) = load_completed_restore_operation_and_seal( + &service, + "/source", + snapshot.snapshot_id, + "/destination", + ); + assert_eq!(seal.reference_count, source_objects.len() as u64); + let metrics = service.restore_metrics().unwrap(); + for keyspace in ["base_owner", "base_inverse", "base_inverse_owner"] { + assert_eq!( + metrics.control_rows.get(keyspace), + Some(&source_objects.len()) + ); + } + assert!(service + .metadata_store() + .get( + RecordFamily::ForkBinding, + &fork_binding_key(service.mount_id(), outcome.destination_root), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) .unwrap() - .expect("checkpoint metadata was atomically committed"); + .is_none()); + + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + assert_eq!(service.history_retention_floor().unwrap(), None); + service.remove_file_path("/source/append.bin").unwrap(); + service.remove_file_path("/source/sparse.bin").unwrap(); + service.remove_file_path("/source/latest").unwrap(); + service.remove_empty_dir_path("/source").unwrap(); + for _ in 0..64 { + service.cleanup_pending_objects(64).unwrap(); + } + assert!(source_objects + .iter() + .all(|object| objects.head(object).unwrap().is_some())); + + assert_eq!( + read_artifact_at_path(&service, "/destination/append.bin"), + b"append-base+delta" + ); + let destination_sparse = service + .lookup_path("/destination/sparse.bin") + .unwrap() + .unwrap(); assert_eq!( service - .read_artifact(InodeId::root(), &dname(b"rank-0.ckpt")) + .read_file(destination_sparse.attr.inode, 0, b"sparse-base".len()) .unwrap(), - payload + b"sparse-base" ); - let body = committed.body.expect("checkpoint has a staged object body"); - let object = block_key(committed.attr.inode, body.generation, 0, 0); - assert!(backing.head(&object).unwrap().is_some()); -} - -#[test] -fn checkpoint_keeps_blocks_after_uncertain_backend_ack_failure() { - let metadata = PostCommitErrorStore::new(CommandKind::PublishArtifact); - let objects = MemoryObjectStore::new(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata, objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let payload = b"checkpoint visible despite a lost backend acknowledgement".to_vec(); - - let error = service - .publish_checkpoint( - InodeId::root(), - vec![CheckpointShard { - name: dname(b"uncertain-rank.ckpt"), - bytes: payload.clone(), - }], - 1000, - 1000, - ) - .unwrap_err(); - assert!(matches!( - error, - MetadError::Metadata(MetadataError::Backend(message)) - if message.contains("journal acknowledgement") - )); - - let committed = service - .lookup_path("/uncertain-rank.ckpt") - .unwrap() - .expect("the injected backend applied before losing its ACK"); assert_eq!( service - .read_artifact(InodeId::root(), &dname(b"uncertain-rank.ckpt")) + .read_file(destination_sparse.attr.inode, sparse_tail_offset, 4) .unwrap(), - payload + b"tail" ); - let body = committed.body.expect("checkpoint has an object body"); - assert!(objects - .head(&block_key(committed.attr.inode, body.generation, 0, 0,)) - .unwrap() - .is_some()); -} - -#[test] -fn pending_sync_log_blocks_single_and_batch_apply_until_prior_commit_is_archived() { - let backing = MemoryObjectStore::new(); - let objects = FaultObjectStore::new(backing); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), + let destination_symlink = service.lookup_path("/destination/latest").unwrap().unwrap(); + assert_eq!( + service + .read_symlink(destination_symlink.attr.inode) + .unwrap(), + symlink_target ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service - .create_dir_path("/batch", 0o755, 1000, 1000) + + let destination_append = service + .lookup_path("/destination/append.bin") + .unwrap() .unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-pending-sync-log", 2); - service.backup_metadata(&checkpoint_config).unwrap(); service - .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/pending-sync-log", - "mount-1:/", - 1, - 0, - METADATA_LOG_ZERO_DIGEST, - )) + .link( + destination_append.attr.inode, + InodeId::root(), + dname(b"escaped-append.bin"), + ) .unwrap(); - objects.fail_puts_containing("meta/pending-sync-log/log/"); + service.remove_file_path("/destination/append.bin").unwrap(); + service.remove_file_path("/destination/sparse.bin").unwrap(); + service.remove_file_path("/destination/latest").unwrap(); + service.remove_empty_dir_path("/destination").unwrap(); + for _ in 0..32 { + service.cleanup_restore_releases(64).unwrap(); + } + assert_eq!(service.restore_metrics().unwrap().releasing, 1); + for _ in 0..64 { + service.cleanup_pending_objects(64).unwrap(); + } + assert!(objects.head(&append_base_object).unwrap().is_some()); + assert!(objects.head(&append_delta_object).unwrap().is_some()); + assert_eq!( + read_artifact_at_path(&service, "/escaped-append.bin"), + b"append-base+delta" + ); - let first_error = service - .create_dir_path("/first", 0o755, 1000, 1000) - .unwrap_err(); - assert!(matches!( - first_error, - MetadError::SyncLogArchiveFailed { - committed: true, - .. + service.remove_file_path("/escaped-append.bin").unwrap(); + for _ in 0..256 { + service.cleanup_restore_releases(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; } - )); - assert!(service.lookup_path("/first").unwrap().is_some()); - - let blocked_single = service - .create_dir_path("/second", 0o755, 1000, 1000) - .unwrap_err(); - assert!(matches!( - blocked_single, - MetadError::SyncLogArchiveFailed { - committed: false, - .. + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); + for _ in 0..256 { + service.cleanup_pending_objects(64).unwrap(); + if source_objects + .iter() + .all(|object| objects.head(object).unwrap().is_none()) + { + break; } - )); - assert!(service.lookup_path("/second").unwrap().is_none()); - - let blocked = service.create_file_batches_in_dir_path(vec![CreateInDirPathBatch { - parent_path: "/batch".to_owned(), - names: vec![dname(b"third.bin")], - mode: 0o644, - uid: 1000, - gid: 1000, - }]); - assert!(matches!( - &blocked[0], - Err(MetadError::SyncLogArchiveFailed { - committed: false, - .. - }) - )); - assert!(service.lookup_path("/batch/third.bin").unwrap().is_none()); - assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); + } + assert!(source_objects + .iter() + .all(|object| objects.head(object).unwrap().is_none())); +} - objects.clear_faults(); +#[test] +fn restore_exact_reference_graph_replays_from_checkpoint_log_and_releases_after_reopen() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata, objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); service - .create_dir_path("/second", 0o755, 1000, 1000) - .unwrap(); - let retried = service.create_file_batches_in_dir_path(vec![CreateInDirPathBatch { - parent_path: "/batch".to_owned(), - names: vec![dname(b"third.bin")], - mode: 0o644, - uid: 1000, - gid: 1000, - }]); - assert_eq!(retried[0].as_ref().unwrap().len(), 1); - - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.durable_lsn, 3); - assert_eq!(snapshot.segments.len(), 3); - let first_segment = service - .load_metadata_log_segment(&snapshot.segments[0].segment_key) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let second_segment = service - .load_metadata_log_segment(&snapshot.segments[1].segment_key) + let source_file = publish_path_artifact( + &service, + "/source/shared.bin", + "restore/replay-exact-reference", + b"checkpoint-log exact reference", + ); + let source_body = source_file.body.as_ref().unwrap(); + let shared_object = block_key(source_file.attr.inode, source_body.generation, 0, 0); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/restore-ref-replay-checkpoint", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/restore-ref-replay-log", + "mount-1:/", + 1, + checkpoint.log_lsn, + checkpoint.log_digest, + )) .unwrap(); - let third_segment = service - .load_metadata_log_segment(&snapshot.segments[2].segment_key) + + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/destination") .unwrap(); - assert_eq!((first_segment.first_lsn, first_segment.last_lsn), (1, 1)); - assert_eq!((second_segment.first_lsn, second_segment.last_lsn), (2, 2)); - assert_eq!(first_segment.last_digest, second_segment.prev_digest); - assert_eq!((third_segment.first_lsn, third_segment.last_lsn), (3, 3)); - assert_eq!(second_segment.last_digest, third_segment.prev_digest); + assert!(service + .retire_snapshot(source_snapshot.snapshot_id) + .unwrap()); + assert_eq!(service.history_retention_floor().unwrap(), None); + service.remove_file_path("/source/shared.bin").unwrap(); + service.remove_empty_dir_path("/source").unwrap(); + for _ in 0..8 { + service.cleanup_pending_objects(1).unwrap(); + } + assert!( + objects.head(&shared_object).unwrap().is_some(), + "the Complete exact-reference graph must protect the deleted source object" + ); + let log = service.sync_metadata_log_snapshot().unwrap(); + assert!(log.durable_lsn > checkpoint.log_lsn); + assert!(!log.segments.is_empty()); + let segment_keys = snapshot_segment_keys(&log); + drop(service); + let recovered_metadata = HoltMetadataStore::open_memory().unwrap(); let recovered = NoKvFs::new( MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects, + recovered_metadata.clone(), + objects.clone(), ); - let outcome = recovered + recovered .restore_metadata_with_archived_log_segments( &checkpoint_config, "mount-1:/", - &snapshot_segment_keys(&snapshot), - 0, - METADATA_LOG_ZERO_DIGEST, + &segment_keys, + checkpoint.log_lsn, + checkpoint.log_digest, ) .unwrap() .unwrap(); - assert_eq!(outcome.replayed_entries, 3); - assert!(recovered.lookup_path("/first").unwrap().is_some()); - assert!(recovered.lookup_path("/second").unwrap().is_some()); - assert!(recovered.lookup_path("/batch/third.bin").unwrap().is_some()); -} + recovered + .enable_sync_metadata_log( + MetadataLogSyncConfig::new( + "meta/restore-ref-replay-log", + "mount-1:/", + 2, + log.durable_lsn, + log.last_digest, + ) + .with_segments(log.segments.clone()), + ) + .unwrap(); + assert!(recovered.lookup_path("/source").unwrap().is_none()); + assert_eq!( + read_artifact_at_path(&recovered, "/destination/shared.bin"), + b"checkpoint-log exact reference" + ); + assert_eq!(recovered.history_retention_floor().unwrap(), None); + let report = recovered.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); + let recovered_metrics = recovered.restore_metrics().unwrap(); + assert_eq!(recovered_metrics.complete, 1); + for keyspace in ["base_owner", "base_inverse", "base_inverse_owner"] { + assert_eq!(recovered_metrics.control_rows.get(keyspace), Some(&1)); + } + assert!(objects.head(&shared_object).unwrap().is_some()); -#[test] -fn immutable_checkpoint_restores_only_its_exact_control_identity() { - let backing = MemoryObjectStore::new(); - let objects = FaultObjectStore::new(backing.clone()); - let source = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), + let moved = recovered + .rename_path("/destination", "/moved-destination") + .unwrap(); + assert_eq!( + moved.attr.inode, + recovered + .resolve_directory_path("/moved-destination") + .unwrap() ); - source.bootstrap_root(0o755, 1000, 1000).unwrap(); - source - .create_dir_path("/controlled", 0o755, 1000, 1000) + assert_eq!(recovered.restore_metrics().unwrap().complete, 1); + recovered + .remove_file_path("/moved-destination/shared.bin") .unwrap(); - let config = MetadataArchiveConfig::new("meta/exact", 4); - let prepared = source.prepare_immutable_metadata_backup(&config).unwrap(); - let identity = MetadataCheckpointIdentity { - checkpoint_key: prepared.checkpoint_key.clone(), - image_bytes: prepared.image_bytes, - image_digest: prepared.image_digest.clone(), - }; - assert!(backing - .head(&ObjectKey::new("meta/exact/CURRENT").unwrap()) - .unwrap() - .is_none()); + recovered + .remove_empty_dir_path("/moved-destination") + .unwrap(); + assert_eq!(recovered.restore_metrics().unwrap().releasing, 1); + assert_eq!(recovered.cleanup_restore_releases(1).unwrap(), 1); + assert_eq!( + recovered.restore_metrics().unwrap().releasing, + 1, + "one bounded release page must leave resumable durable work" + ); + let release_log = recovered.sync_metadata_log_snapshot().unwrap(); + drop(recovered); - // Drift standalone CURRENT to a different image. Controlled restore must - // still install only the exact control identity above. - source.create_dir_path("/drift", 0o755, 1000, 1000).unwrap(); - source.backup_metadata(&config).unwrap(); - let recovered = NoKvFs::new( + let reopened = NoKvFs::open_existing( MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects, + recovered_metadata, + objects.clone(), + 0, + ) + .unwrap(); + reopened + .enable_sync_metadata_log( + MetadataLogSyncConfig::new( + "meta/restore-ref-replay-log", + "mount-1:/", + 2, + release_log.durable_lsn, + release_log.last_digest, + ) + .with_segments(release_log.segments), + ) + .unwrap(); + for _ in 0..512 { + reopened.cleanup_restore_releases(1).unwrap(); + if reopened.restore_metrics().unwrap().releasing == 0 { + break; + } + } + assert_eq!(reopened.restore_metrics().unwrap().releasing, 0); + let cleanup = reopened.cleanup_pending_objects(1).unwrap(); + assert_eq!(cleanup.blocked_by_failover_durability, 1); + assert!( + objects.head(&shared_object).unwrap().is_some(), + "checkpoint-recoverable metadata must keep external DELETE fail-closed" ); - let restored = recovered - .restore_metadata_checkpoint(&config, &identity) + let queued = reopened + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount_id()), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 64, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap(); - assert_eq!(restored.checkpoint_key, prepared.checkpoint_key); - assert!(recovered.lookup_path("/controlled").unwrap().is_some()); - assert!(recovered.lookup_path("/drift").unwrap().is_none()); + assert!(queued.iter().any(|row| { + decode_object_gc_record(&row.value.0) + .is_ok_and(|record| record.object_key == shared_object.as_str()) + })); + let report = reopened.fsck_restore_state(true).unwrap(); + assert!(report.is_consistent(), "restore fsck report: {report:#?}"); } #[test] -fn exact_checkpoint_rejects_missing_proof_before_fetching_image() { - let backing = MemoryObjectStore::new(); - let objects = FaultObjectStore::new(backing.clone()); - let source = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), +fn restore_release_waits_for_a_live_snapshot_of_the_restored_namespace() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/shared.bin", + "restore-snapshot-holder", + b"snapshot-retained bytes", ); - source.bootstrap_root(0o755, 1000, 1000).unwrap(); - let config = MetadataArchiveConfig::new("meta/proof", 4); - let prepared = source.prepare_immutable_metadata_backup(&config).unwrap(); - let identity = MetadataCheckpointIdentity { - checkpoint_key: prepared.checkpoint_key.clone(), - image_bytes: prepared.image_bytes, - image_digest: prepared.image_digest, - }; - let proof_key = ObjectKey::new(format!("{}.proof", identity.checkpoint_key)).unwrap(); - assert!(backing.delete(&proof_key).unwrap()); - objects.clear_get_keys(); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/destination") + .unwrap(); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), + let destination_snapshot = service.snapshot_subtree_path("/destination").unwrap(); + service.remove_file_path("/destination/shared.bin").unwrap(); + service.remove_empty_dir_path("/destination").unwrap(); + + for _ in 0..4 { + service.cleanup_restore_releases(64).unwrap(); + } + assert_eq!( + service.restore_metrics().unwrap().releasing, + 1, + "a live snapshot must retain the complete restore ref-set after the live root disappears" ); - assert!(matches!( - recovered.restore_metadata_checkpoint(&config, &identity), - Err(MetadError::MetadataArchiveMissingObjectGcFence { checkpoint_key }) - if checkpoint_key == identity.checkpoint_key - )); - assert!(!objects.got_key(&identity.checkpoint_key)); - assert!(recovered.get_attr(InodeId::root()).unwrap().is_none()); + let cursor_key = restore::restore_release_cursor_key(service.mount_id()); + let before_idle = service.metadata_store_stats(); + let cursor_version = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &cursor_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap() + .version; + assert_eq!(service.cleanup_restore_releases(64).unwrap(), 1); + assert_eq!(service.cleanup_restore_releases(64).unwrap(), 1); + assert_eq!( + service.metadata_store_stats().commit_total, + before_idle.commit_total, + "a snapshot-blocked release must not rewrite an unchanged worker cursor" + ); + assert_eq!( + service + .metadata_store() + .get_versioned( + RecordFamily::System, + &cursor_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap() + .version, + cursor_version + ); + + assert!(service + .retire_snapshot(destination_snapshot.snapshot_id) + .unwrap()); + for _ in 0..128 { + service.cleanup_restore_releases(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); } #[test] -fn exact_checkpoint_rejects_cross_prefix_or_same_key_digest_mismatch() { - let backing = MemoryObjectStore::new(); - let objects = FaultObjectStore::new(backing); - let source = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), - ); - source.bootstrap_root(0o755, 1000, 1000).unwrap(); - let config = MetadataArchiveConfig::new("meta/owned", 4); - let prepared = source.prepare_immutable_metadata_backup(&config).unwrap(); - let identity = MetadataCheckpointIdentity { - checkpoint_key: prepared.checkpoint_key.clone(), - image_bytes: prepared.image_bytes, - image_digest: prepared.image_digest, - }; - let recovered = NoKvFs::new( +fn restore_release_serializes_with_snapshot_renewal_across_the_old_expiry() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RenewSnapshot, 1, 2); + let service = Arc::new(NoKvFs::new( MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), - ); - assert!(matches!( - recovered.restore_metadata_checkpoint( - &MetadataArchiveConfig::new("meta/other", 4), - &identity, - ), - Err(MetadError::Codec(message)) if message.contains("does not match archive prefix") + store.clone(), + MemoryObjectStore::new(), )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.set_clock_override_ms(1_000); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/shared.bin", + "restore-renew-release-race", + b"renewed snapshot bytes", + ); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/destination") + .unwrap(); + let destination_snapshot = service + .snapshot_subtree_path_with_lease("/destination", 500) + .unwrap(); + assert!(service + .retire_snapshot(source_snapshot.snapshot_id) + .unwrap()); + service.remove_file_path("/destination/shared.bin").unwrap(); + service.remove_empty_dir_path("/destination").unwrap(); - let mismatched = MetadataCheckpointIdentity { - checkpoint_key: identity.checkpoint_key.clone(), - image_bytes: identity.image_bytes, - image_digest: format!("sha256:{}", "0".repeat(64)), - }; - objects.clear_get_keys(); + let renew_service = Arc::clone(&service); + let renew = std::thread::spawn(move || { + renew_service.renew_snapshot(destination_snapshot.snapshot_id, 10_000) + }); + store.wait_until_blocked(); + service.set_clock_override_ms(destination_snapshot.lease_expires_unix_ms); assert!(matches!( - recovered.restore_metadata_checkpoint(&config, &mismatched), - Err(MetadError::Codec(message)) if message.contains("does not match archive prefix") + service.restore_snapshot_gate.try_write(), + Err(std::sync::TryLockError::WouldBlock) )); - assert!(!objects.got_key(&identity.checkpoint_key)); + + let (release_started_tx, release_started_rx) = std::sync::mpsc::channel(); + let (release_done_tx, release_done_rx) = std::sync::mpsc::channel(); + let release_service = Arc::clone(&service); + let release = std::thread::spawn(move || { + release_started_tx.send(()).unwrap(); + let outcome = release_service.cleanup_restore_releases(64); + release_done_tx.send(outcome).unwrap(); + }); + release_started_rx.recv().unwrap(); + assert!( + release_done_rx + .recv_timeout(Duration::from_millis(250)) + .is_err(), + "restore release must wait for the in-flight renewal proof and CAS" + ); + + store.release_blocked(); + let SnapshotRenewOutcome::Renewed { + pin: renewed, + extended: true, + } = renew.join().unwrap().unwrap() + else { + panic!("expected the pre-expiry renewal to commit") + }; + assert_eq!(renewed.lease_expires_unix_ms, 11_000); + release_done_rx + .recv_timeout(Duration::from_secs(5)) + .unwrap() + .unwrap(); + release.join().unwrap(); + assert_eq!( + service.restore_metrics().unwrap().releasing, + 1, + "release must observe the successfully renewed pin after it acquires the gate" + ); + + assert!(service + .retire_snapshot(destination_snapshot.snapshot_id) + .unwrap()); + for _ in 0..256 { + service.cleanup_restore_releases(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); } #[test] -fn exact_checkpoint_rejects_tampered_same_key_image_before_install() { - let backing = MemoryObjectStore::new(); - let source = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - backing.clone(), +fn restore_release_waits_for_a_generic_fork_binding_after_its_pin_expires() { + let service = service(); + service.set_clock_override_ms(1_000); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/shared.bin", + "restore-fork-binding-holder", + b"fork-binding-retained bytes", ); - source.bootstrap_root(0o755, 1000, 1000).unwrap(); - source - .create_dir_path("/must-not-install", 0o755, 1000, 1000) + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + let restored = service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/destination") + .unwrap(); + let fork = service + .clone_subtree_path_into("/destination", "/generic-holder") .unwrap(); - let config = MetadataArchiveConfig::new("meta/tamper", 4); - let prepared = source.prepare_immutable_metadata_backup(&config).unwrap(); - let identity = MetadataCheckpointIdentity { - checkpoint_key: prepared.checkpoint_key.clone(), - image_bytes: prepared.image_bytes, - image_digest: prepared.image_digest, - }; - let image_key = ObjectKey::new(identity.checkpoint_key.clone()).unwrap(); - let mut tampered = backing.get(&image_key, None).unwrap(); - tampered[0] ^= 0xff; - assert_eq!(tampered.len() as u64, identity.image_bytes); - backing.put(&image_key, tampered).unwrap(); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - backing, + let fork_pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(fork_pin.lease_expires_unix_ms); + service.reclaim_expired_snapshot_pins(100).unwrap(); + assert!(service.snapshot_pin(fork.snapshot_id).unwrap().is_none()); + assert!(service + .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) + .unwrap() + .iter() + .any(|binding| { + binding.binding.snapshot_id == fork.snapshot_id + && binding.binding.source_root == restored.destination_root + && binding.binding.fork_root == fork.root + })); + + service.remove_file_path("/destination/shared.bin").unwrap(); + service.remove_empty_dir_path("/destination").unwrap(); + for _ in 0..4 { + service.cleanup_restore_releases(64).unwrap(); + } + assert_eq!( + service.restore_metrics().unwrap().releasing, + 1, + "the durable ForkBinding must retain the ref-set after its construction pin expires" ); - assert!(matches!( - recovered.restore_metadata_checkpoint(&config, &identity), - Err(MetadError::Codec(message)) if message.contains("image digest mismatch") - )); - assert!(recovered.get_attr(InodeId::root()).unwrap().is_none()); + + service + .remove_file_path("/generic-holder/shared.bin") + .unwrap(); + service.remove_empty_dir_path("/generic-holder").unwrap(); + assert!(service.retire_snapshot(fork.snapshot_id).unwrap()); + for _ in 0..128 { + service.cleanup_restore_releases(64).unwrap(); + if service.restore_metrics().unwrap().releasing == 0 { + break; + } + } + assert_eq!(service.restore_metrics().unwrap().releasing, 0); } #[test] -fn backup_archive_crash_between_checkpoint_and_pointer_is_consistent() { - let backing = MemoryObjectStore::new(); - let objects = FaultObjectStore::new(backing.clone()); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), +fn restore_indexes_flow_through_generic_fork_after_source_delete_and_nested_restore() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact(&service, "/source/a.bin", "generic-index/a", b"a"); + publish_path_artifact(&service, "/source/b.bin", "generic-index/b", b"b"); + service + .register_namespace_index(NamespaceIndexRegistration { + path: "/source".to_owned(), + fields: vec![NamespaceIndexField { + field: NamespaceFindField::new("run.kind"), + operators: vec![NamespacePredicateOp::Eq], + sortable: true, + facetable: true, + }], + rows: vec![ + NamespaceIndexRow { + path: "/source/a.bin".to_owned(), + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("run.kind"), + value: NamespacePredicateValue::String("a".to_owned()), + }], + }, + NamespaceIndexRow { + path: "/source/b.bin".to_owned(), + values: vec![NamespaceIndexValue { + field: NamespaceFindField::new("run.kind"), + value: NamespacePredicateValue::String("b".to_owned()), + }], + }, + ], + }) + .unwrap(); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/restored") + .unwrap(); + let generic = service + .clone_subtree_path_into("/restored", "/generic") + .unwrap(); + + service.remove_file_path("/restored/a.bin").unwrap(); + service.remove_file_path("/restored/b.bin").unwrap(); + service.remove_empty_dir_path("/restored").unwrap(); + for _ in 0..4 { + service.cleanup_restore_releases(64).unwrap(); + } + assert_eq!(service.restore_metrics().unwrap().releasing, 1); + + let indexed = service + .list_indexed_path_page("/generic", None, 10) + .unwrap(); + assert_eq!( + indexed + .entries + .iter() + .map(|entry| entry.dentry.name.as_bytes().to_vec()) + .collect::>(), + vec![b"a.bin".to_vec(), b"b.bin".to_vec()] ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/runs/a.bin", "m-a", b"alpha"); + let found = service + .find_paths(NamespaceFindRequest { + path: "/generic".to_owned(), + predicates: vec![NamespacePredicate { + field: NamespaceFindField::new("run.kind"), + op: NamespacePredicateOp::Eq, + value: Some(NamespacePredicateValue::String("b".to_owned())), + }], + sort: Vec::new(), + include: Vec::new(), + facets: Vec::new(), + cursor: None, + limit: 10, + }) + .unwrap(); + assert_eq!(found.match_count, 1); + assert_eq!(found.matches[0].path, "/generic/b.bin"); - let config = MetadataArchiveConfig::new("meta/ck", 4); - // First backup completes: CURRENT -> checkpoint #1 (captures only /runs/a.bin). - let good = service.backup_metadata(&config).unwrap(); + let generic_snapshot = service.snapshot_subtree_path("/generic").unwrap(); + service + .restore_subtree_path_to_fork("/generic", generic_snapshot.snapshot_id, "/nested") + .unwrap(); + let nested_indexed = service.list_indexed_path_page("/nested", None, 10).unwrap(); + assert_eq!(nested_indexed.entries.len(), 2); + let nested_found = service + .find_paths(NamespaceFindRequest { + path: "/nested".to_owned(), + predicates: vec![NamespacePredicate { + field: NamespaceFindField::new("run.kind"), + op: NamespacePredicateOp::Eq, + value: Some(NamespacePredicateValue::String("a".to_owned())), + }], + sort: Vec::new(), + include: Vec::new(), + facets: Vec::new(), + cursor: None, + limit: 10, + }) + .unwrap(); + assert_eq!(nested_found.match_count, 1); + assert_eq!(nested_found.matches[0].path, "/nested/a.bin"); - // Add /runs/b.bin, then crash the second backup at the pointer swap: the - // checkpoint object is written, but the CURRENT manifest PUT fails. - publish_path_artifact(&service, "/runs/b.bin", "m-b", b"bravo"); - objects.fail_puts_containing("/CURRENT"); - let err = service.backup_metadata(&config).unwrap_err(); - assert!(matches!(err, MetadError::Object(_))); - assert_eq!(objects.injected_put_failures(), 1); - objects.clear_faults(); + service.remove_file_path("/nested/a.bin").unwrap(); + service.remove_file_path("/nested/b.bin").unwrap(); + service.remove_empty_dir_path("/nested").unwrap(); + service.remove_file_path("/generic/a.bin").unwrap(); + service.remove_file_path("/generic/b.bin").unwrap(); + service.remove_empty_dir_path("/generic").unwrap(); + assert!(service.retire_snapshot(generic.snapshot_id).unwrap()); + assert!(service + .retire_snapshot(generic_snapshot.snapshot_id) + .unwrap()); + assert!(service + .retire_snapshot(source_snapshot.snapshot_id) + .unwrap()); +} - // CURRENT still names the first, complete checkpoint — never the orphaned - // second one. Restore into a fresh node recovers the pre-crash state. - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - backing.clone(), - ); - let restored = recovered.restore_metadata(&config).unwrap().unwrap(); - assert_eq!(restored.checkpoint_key, good.checkpoint_key); - assert!(recovered.lookup_path("/runs/a.bin").unwrap().is_some()); - assert!( - recovered.lookup_path("/runs/b.bin").unwrap().is_none(), - "restore must not expose the torn (uncommitted) checkpoint" +#[test] +fn snapshot_retirement_cannot_remove_a_preparing_restore_history_floor() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RegisterNamespaceIndex, 0, 1) + .matching_request_prefix(b"restore-init-publish") + .rejecting(CommandKind::PublishArtifact); + let service = NoKvFs::new(MountId::new(1).unwrap(), store, MemoryObjectStore::new()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .create_dir_path("/source/metadata", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let restore = service.restore_subtree_path_to_fork_initialized( + "/source", + snapshot.snapshot_id, + "/destination", + RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: b"{}".to_vec(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, ); - - // With the fault cleared, the archive recovers forward cleanly. - publish_path_artifact(&service, "/runs/c.bin", "m-c", b"charlie"); - let next = service.backup_metadata(&config).unwrap(); - assert_ne!(next.checkpoint_key, good.checkpoint_key); + assert!(matches!( + restore, + Err(MetadError::PublishArtifactFailed { .. }) + )); + assert!(matches!( + service.retire_snapshot(snapshot.snapshot_id), + Err(MetadError::RestoreInProgress) + )); } #[test] -fn object_gc_converges_under_create_delete_churn() { - let (service, objects) = service_with_objects(); - // Churn: create many small files; delete the even rounds (their blocks must - // be reclaimed) and keep the odd rounds (their blocks must never be deleted). - let mut live_keys = Vec::new(); - for round in 0..20u32 { - let name = DentryName::new(format!("churn-{round}.bin").into_bytes()).unwrap(); - let published = service - .publish_artifact(artifact_request( - name.clone(), - &format!("m{round}"), - b"payload", - )) - .unwrap(); - let body = published.body.clone().unwrap(); - let key = block_key(published.attr.inode, body.generation, 0, 0); - if round % 2 == 0 { - service.remove_file(InodeId::root(), &name).unwrap(); - } else { - live_keys.push(key); - } - } +fn ordinary_and_complete_restore_reads_do_not_probe_the_system_keyspace() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let source = service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact(&service, "/source/file.txt", "visibility/read", b"read"); - // Drive GC to convergence with a small per-iteration limit so the queue is - // drained across several batches rather than one sweep. - let mut total_deleted = 0; - let mut guard = 0; - loop { - let outcome = service.cleanup_pending_objects(4).unwrap(); - total_deleted += outcome.deleted; - if outcome.scanned == 0 { - break; - } - guard += 1; - assert!(guard < 1000, "object GC did not converge"); - } + assert!(!service.restore_staging_slow_path_enabled()); + let before = service.metadata_store_stats(); + assert!(service.get_attr(source.attr.inode).unwrap().is_some()); + let after = service.metadata_store_stats(); + assert_eq!(after.get_total - before.get_total, 1); + assert_eq!( + after.get_write_plan_local_total - before.get_write_plan_local_total, + 0, + "an ordinary inode read must not probe the restore staging inverse" + ); - // Exactly the 10 deleted files were reclaimed, and the queue is now empty. - assert_eq!(total_deleted, 10); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let restored = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + assert!(!service.restore_staging_slow_path_enabled()); + let before_retry = service.metadata_store_stats(); + let retry = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + let after_retry = service.metadata_store_stats(); + assert_eq!(retry, restored); + assert_eq!( + after_retry.scan_total - before_retry.scan_total, + 0, + "an exact terminal retry must not rescan restore-private keyspaces" + ); + let before = service.metadata_store_stats(); + assert!(service + .get_attr(restored.destination_root) + .unwrap() + .is_some()); + let after = service.metadata_store_stats(); + assert_eq!(after.get_total - before.get_total, 1); + assert_eq!( + after.get_write_plan_local_total - before.get_write_plan_local_total, + 0, + "a completed restore must use the ordinary inode read hot path" + ); + + let reopened = NoKvFs::open_existing(service.mount, metadata, objects, 0).unwrap(); + assert!(!reopened.restore_staging_slow_path_enabled()); + reopened.observe_required_owner_epoch(2).unwrap(); + assert!(!reopened.restore_staging_slow_path_enabled()); + reopened.install_owner_epoch(2).unwrap(); + assert!(!reopened.restore_staging_slow_path_enabled()); + let before = reopened.metadata_store_stats(); + assert!(reopened + .get_attr(restored.destination_root) + .unwrap() + .is_some()); + let after = reopened.metadata_store_stats(); + assert_eq!(after.get_total - before.get_total, 1); assert_eq!( - service.cleanup_pending_objects(100).unwrap(), - PendingObjectCleanupOutcome::default() + after.get_write_plan_local_total - before.get_write_plan_local_total, + 0, + "reopen/failover recovery must restore the completed fast path" ); - // Every kept file's block survived: owns_block_object_key never over-deleted. - for key in &live_keys { - assert!( - objects.head(key).unwrap().is_some(), - "live block was wrongly GC'd: {}", - key.as_str() - ); - } } #[test] -fn fsck_detects_dangling_block_after_out_of_band_object_loss() { - let (service, objects) = service_with_objects(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let a = publish_path_artifact(&service, "/runs/a.bin", "m-a", b"alpha-body"); - publish_path_artifact(&service, "/runs/b.bin", "m-b", b"bravo-body"); +fn concurrent_ordinary_writes_before_restore_do_not_probe_staging_inverses() { + const WRITERS: usize = 8; + const WRITES_PER_WRITER: usize = 32; - // A healthy namespace has no dangling references. - let clean = service.fsck_dangling_blocks(0).unwrap(); - assert!( - clean.is_consistent(), - "unexpected dangling: {:?}", - clean.dangling + let mount = MountId::new(1).unwrap(); + let metadata = RestoreInverseProbeStore::new(mount); + let service = Arc::new(NoKvFs::new( + mount, + metadata.clone(), + MemoryObjectStore::new(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let start = Arc::new(Barrier::new(WRITERS)); + let writers = (0..WRITERS) + .map(|writer| { + let service = Arc::clone(&service); + let start = Arc::clone(&start); + std::thread::spawn(move || { + start.wait(); + for index in 0..WRITES_PER_WRITER { + service + .set_xattr( + InodeId::root(), + format!("user.concurrent-{writer}-{index}").as_bytes(), + vec![writer as u8, index as u8], + XattrSetMode::Any, + ) + .unwrap(); + } + }) + }) + .collect::>(); + for writer in writers { + writer.join().unwrap(); + } + + assert_eq!( + metadata.inverse_gets(), + 0, + "a mount which has never activated restore must use the stable activation fence", ); - assert_eq!(clean.files_scanned, 2); - assert!(clean.blocks_checked >= 2); + assert!(metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: metadata.inverse_prefix.clone(), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .is_empty()); +} - // Delete one file's backing object out-of-band: drift that object-first - // ordering cannot prevent once the metadata is already committed. - let body = a.body.clone().unwrap(); - let lost = block_key(a.attr.inode, body.generation, 0, 0); - assert!(objects.delete(&lost).unwrap()); +#[test] +fn concurrent_non_member_writes_after_restore_never_observe_inverse_sentinels() { + const WRITERS: usize = 8; + const WRITES_PER_WRITER: usize = 32; - // fsck flags exactly that reference, and nothing else. - let report = service.fsck_dangling_blocks(0).unwrap(); - assert!(!report.is_consistent()); - assert_eq!(report.dangling.len(), 1); - assert_eq!(report.dangling[0].inode, a.attr.inode.get()); - assert_eq!(report.dangling[0].object_key, lost.as_str()); -} + let service = Arc::new(service()); + let ordinary = service + .create_file_path("/ordinary.bin", 0o644, 1000, 1000) + .unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + + let start = Arc::new(Barrier::new(WRITERS)); + let writers = (0..WRITERS) + .map(|writer| { + let service = Arc::clone(&service); + let start = Arc::clone(&start); + let inode = ordinary.attr.inode; + std::thread::spawn(move || { + start.wait(); + for index in 0..WRITES_PER_WRITER { + service + .set_xattr( + inode, + format!("user.post-restore-{writer}-{index}").as_bytes(), + vec![writer as u8, index as u8], + XattrSetMode::Any, + ) + .unwrap(); + } + }) + }) + .collect::>(); + for writer in writers { + writer.join().unwrap(); + } -/// Set up `/runs/a.bin`, snapshot `/runs` with `lease_ms`, then free the block so -/// it is GC-enqueued *after* the snapshot's read version (i.e. protected while -/// the pin is live). Returns the freed block's object key. -fn snapshot_then_free_block( - service: &NoKvFs, - lease_ms: u64, -) -> (SnapshotPin, ObjectKey) { - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let published = publish_path_artifact(service, "/runs/a.bin", "m-a", b"payload"); - let body = published.body.clone().unwrap(); - let block = block_key(published.attr.inode, body.generation, 0, 0); - let runs = service.resolve_directory_path("/runs").unwrap(); - let pin = service.snapshot_subtree_with_lease(runs, lease_ms).unwrap(); - service.remove_file_path("/runs/a.bin").unwrap(); - (pin, block) + assert!(service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_staging_inode_key(service.mount_id(), ordinary.attr.inode), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); } #[test] -fn expired_snapshot_pin_does_not_block_object_gc() { - let (service, objects) = service_with_objects(); - // Lease of 0 ms: the pin is expired the moment GC inspects it. - let (_pin, block) = snapshot_then_free_block(&service, 0); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.blocked_by_snapshots, 0); - assert_eq!(cleanup.deleted, 1); - assert!(objects.head(&block).unwrap().is_none()); +fn restore_visibility_recovery_fails_closed_without_a_complete_marker() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let restored = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") + .unwrap(); + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/source", + snapshot.snapshot_id, + "/destination", + ); + let operation = service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_operation_key(service.mount_id(), &operation_digest), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .map(|value| restore::decode_restore_operation(&value.0).unwrap()) + .unwrap(); + let marker_key = + restore_index::restore_index_complete_key(service.mount_id(), operation.ref_set_id); + let marker = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &marker_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + + // Model a corrupt/replaced image without creating an exposure window in + // the test process itself. + service.mark_restore_staging_possible(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-remove-restore-complete-marker", + service.mount_id(), + restored.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: marker_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: marker_key.clone(), + predicate: Predicate::VersionEquals(marker.version), + }], + mutations: vec![delete_mutation(RecordFamily::System, marker_key)], + watch: Vec::new(), + }) + .unwrap(); + + assert!(matches!( + service.recover_restore_staging_visibility(), + Err(MetadError::Codec(message)) + if message == "complete restore operation has no visibility marker" + )); + assert!(service.restore_staging_slow_path_enabled()); + assert!(service + .get_attr(restored.destination_root) + .unwrap() + .is_none()); } #[test] -fn live_snapshot_pin_blocks_object_gc_until_retired() { - let (service, objects) = service_with_objects(); - let (pin, block) = snapshot_then_free_block(&service, 3_600_000); +fn reopen_and_owner_failover_keep_crash_left_restore_staging_hidden() { + let metadata = SnapshotCommitBarrierStore::new(CommandKind::RegisterNamespaceIndex, 0, 1) + .matching_request_prefix(b"restore-init-publish") + .rejecting(CommandKind::PublishArtifact); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .create_dir_path("/source/metadata", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let operation_digest = restore::restore_operation_digest( + service.mount_id(), + "/source", + snapshot.snapshot_id, + "/destination", + ); + let restore = service.restore_subtree_path_to_fork_initialized( + "/source", + snapshot.snapshot_id, + "/destination", + RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: b"{}".to_vec(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, + ); + assert!(matches!( + restore, + Err(MetadError::PublishArtifactFailed { .. }) + )); + let operation = service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_operation_key(service.mount_id(), &operation_digest), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .map(|value| restore::decode_restore_operation(&value.0).unwrap()) + .unwrap(); + assert_eq!(operation.state, restore::RestoreOperationState::Preparing); + assert!(service.restore_staging_slow_path_enabled()); + assert!(service + .get_attr(operation.destination_root) + .unwrap() + .is_none()); - // A live pin protects the freed block. - let blocked = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(blocked.blocked_by_snapshots, 1); - assert_eq!(blocked.deleted, 0); - assert!(objects.head(&block).unwrap().is_some()); + let reopened = NoKvFs::open_existing(service.mount, metadata, objects, 0).unwrap(); + assert!(reopened.restore_staging_slow_path_enabled()); + assert!(reopened + .get_attr(operation.destination_root) + .unwrap() + .is_none()); + reopened.observe_required_owner_epoch(2).unwrap(); + assert!(reopened.restore_staging_slow_path_enabled()); + reopened.install_owner_epoch(2).unwrap(); + assert!(reopened.restore_staging_slow_path_enabled()); + assert!(reopened + .get_attr(operation.destination_root) + .unwrap() + .is_none()); +} - // Retiring it releases the protection. - assert!(service.retire_snapshot(pin.snapshot_id).unwrap()); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 1); - assert!(objects.head(&block).unwrap().is_none()); +#[test] +fn identical_retry_discards_a_preparing_restore_and_rebuilds_it() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .create_dir_path("/source/metadata", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let initialization = RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: b"durable manifest".to_vec(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }; + objects.fail_puts_containing("blocks/"); + assert!(matches!( + service.restore_subtree_path_to_fork_initialized( + "/source", + snapshot.snapshot_id, + "/destination", + initialization.clone(), + ), + Err(MetadError::Object(_)) + )); + + objects.clear_faults(); + let mut cleanup_pages = 0_usize; + let restored = loop { + match service.restore_subtree_path_to_fork_initialized( + "/source", + snapshot.snapshot_id, + "/destination", + initialization.clone(), + ) { + Ok(outcome) => break outcome, + Err(MetadError::RestoreInProgress) => { + cleanup_pages += 1; + assert!( + cleanup_pages <= 64, + "bounded restore cleanup failed to converge" + ); + } + Err(error) => panic!("identical restore retry failed: {error}"), + } + }; + assert!(cleanup_pages > 0); + assert_eq!(restored.state, RestoreState::Complete); + assert!(service + .lookup_path("/destination/metadata/restore_manifest.json") + .unwrap() + .is_some()); } #[test] -fn renew_snapshot_rejects_expiry_at_the_deadline() { - let (service, _objects) = service_with_objects(); - service.set_clock_override_ms(1_000); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let runs = service.resolve_directory_path("/runs").unwrap(); - let pin = service.snapshot_subtree_with_lease(runs, 500).unwrap(); +fn restore_init_tombstone_cursor_reaches_rows_beyond_one_page() { + let (service, objects) = service_with_objects(); + let mount = service.mount_id(); + let mut tombstone_rows = Vec::new(); + let mut object_keys = Vec::new(); + for index in 0..3_u64 { + let inode = InodeId::new(100 + index).unwrap(); + let generation = 200 + index; + let mut operation_digest = [0_u8; 32]; + operation_digest[31] = index as u8 + 1; + let tombstone = restore::RestoreInitUploadTombstone { + operation_digest, + initialization_digest: [index as u8 + 10; 32], + inode, + generation, + size: 1, + relative_path: format!("metadata/restore-{index}.json"), + }; + let key = + restore::restore_init_upload_tombstone_key(mount, &operation_digest, inode, generation); + tombstone_rows.push(( + key, + restore::encode_restore_init_upload_tombstone(&tombstone).unwrap(), + )); + let object_key = block_key(inode, generation, 0, 0); + objects.put(&object_key, vec![index as u8]).unwrap(); + object_keys.push(object_key); + } + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-restore-init-tombstones", + mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: tombstone_rows[0].0.clone(), + predicates: tombstone_rows + .iter() + .map(|(key, _)| PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }) + .collect(), + mutations: tombstone_rows + .iter() + .map(|(key, value)| Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(value.clone())), + }) + .collect(), + watch: Vec::new(), + }) + .unwrap(); - service.set_clock_override_ms(1_500); - assert!(matches!( - service.renew_snapshot(pin.snapshot_id, 3_600_000), - Err(MetadError::SnapshotLeaseExpired { - snapshot_id, - lease_expires_unix_ms: 1_500, - now_ms: 1_500, - }) if snapshot_id == pin.snapshot_id - )); + let first = service.cleanup_pending_objects(2).unwrap(); + assert_eq!(first.restore_init_tombstones_scanned, 2); + assert!(objects.head(&object_keys[0]).unwrap().is_none()); + assert!(objects.head(&object_keys[1]).unwrap().is_none()); + assert!(objects.head(&object_keys[2]).unwrap().is_some()); + let second = service.cleanup_pending_objects(2).unwrap(); + assert_eq!(second.restore_init_tombstones_scanned, 1); + assert!(objects.head(&object_keys[2]).unwrap().is_none()); assert_eq!( service - .renew_snapshot(pin.snapshot_id + 9_999, 1_000) - .unwrap(), - SnapshotRenewOutcome::Missing { - snapshot_id: pin.snapshot_id + 9_999, - } + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_init_upload_tombstone_prefix(mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .len(), + 3, + "automatic cleanup must retain the permanent late-PUT ledger" ); } #[test] -fn renew_snapshot_cannot_revive_a_pin_after_gc_crosses_its_expiry() { - let store = SnapshotCommitBarrierStore::new(CommandKind::RenewSnapshot, 1, 2) - .rejecting(CommandKind::RetireSnapshot); - let objects = MemoryObjectStore::new(); - let service = Arc::new(NoKvFs::new( - MountId::new(1).unwrap(), - store.clone(), - objects.clone(), - )); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let start_ms = service.now_ms(); - service.set_clock_override_ms(start_ms); - let name = DentryName::new(b"renew-gc-race.bin".to_vec()).unwrap(); - let published = service - .publish_artifact(artifact_request(name.clone(), "renew-gc-race", b"payload")) - .unwrap(); - let body = published.body.as_ref().unwrap(); - let object = block_key(published.attr.inode, body.generation, 0, 0); - let pin = service - .snapshot_subtree_with_lease(InodeId::root(), 500) +fn restore_release_cursor_does_not_starve_jobs_beyond_one_page() { + let service = service(); + let mount = service.mount_id(); + let job_keys = (1..=3_u64) + .map(|ref_set_id| restore::restore_release_job_key(mount, ref_set_id)) + .collect::>(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-malformed-restore-release-jobs", + mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: job_keys[0].clone(), + predicates: job_keys + .iter() + .map(|key| PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }) + .collect(), + mutations: job_keys + .iter() + .enumerate() + .map(|(index, key)| Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(vec![index as u8])), + }) + .collect(), + watch: Vec::new(), + }) .unwrap(); - let snapshot_id = pin.snapshot_id; - service.remove_file(InodeId::root(), &name).unwrap(); - let renew_service = Arc::clone(&service); - let renew = std::thread::spawn(move || renew_service.renew_snapshot(snapshot_id, 10_000)); - store.wait_until_blocked(); + let append_malformed_job = || { + let version = service.next_version().unwrap(); + let key = restore::restore_release_job_key(mount, version.get()); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-append-malformed-restore-release-job", + mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(vec![0xff])), + }], + watch: Vec::new(), + }) + .unwrap(); + key + }; - let deadline_ms = start_ms + 500; - service.set_clock_override_ms(deadline_ms); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.snapshot_reap.conflicted, 1); - assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); - assert!(objects.head(&object).unwrap().is_none()); - store.release_blocked(); + assert_eq!(service.cleanup_restore_releases(2).unwrap(), 2); + let first_quarantine = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_release_quarantine_prefix(mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + assert_eq!(first_quarantine.len(), 2); + let first_appended = append_malformed_job(); - assert!(matches!( - renew.join().unwrap(), - Err(MetadError::SnapshotLeaseExpired { - snapshot_id, - lease_expires_unix_ms, - now_ms, - }) if snapshot_id == pin.snapshot_id - && lease_expires_unix_ms == deadline_ms - && now_ms == deadline_ms - )); + assert_eq!(service.cleanup_restore_releases(2).unwrap(), 1); + let quarantine = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_release_quarantine_prefix(mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + assert_eq!(quarantine.len(), 3); + let second_appended = append_malformed_job(); + assert_eq!( + service.cleanup_restore_releases(2).unwrap(), + 2, + "a frozen cycle must revisit old jobs while newer job ids keep arriving" + ); + assert!(first_appended > job_keys[2]); + assert!(second_appended > first_appended); + assert_eq!( + service.cleanup_restore_releases(0).unwrap(), + 1, + "a zero caller limit has an explicit one-job effective budget" + ); + for row in &quarantine { + let decoded = restore::validate_restore_release_quarantine_row(mount, row).unwrap(); + assert_eq!(decoded.family, RecordFamily::System); + assert_eq!( + decoded.scope, + restore::RestoreReleaseQuarantineScope::Diagnostic + ); + assert!(job_keys.contains(&decoded.original_key)); + } assert_eq!( service - .snapshot_pin(pin.snapshot_id) - .unwrap() + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_release_job_prefix(mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) .unwrap() - .lease_expires_unix_ms, - deadline_ms + .len(), + 5, + "malformed active jobs stay durable for fsck repair" ); } #[test] -fn renew_snapshot_is_extend_only_and_never_shortens_protection() { - let (service, _objects) = service_with_objects(); - service.set_clock_override_ms(1_000); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let runs = service.resolve_directory_path("/runs").unwrap(); - let pin = service.snapshot_subtree_with_lease(runs, 1_000).unwrap(); +fn restore_release_cursor_ack_loss_reopens_with_frozen_cycle() { + let mount = MountId::new(1).unwrap(); + let metadata = RestorePostApplyFaultStore::new(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(mount, metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let job_keys = (1..=3_u64) + .map(|ref_set_id| restore::restore_release_job_key(mount, ref_set_id)) + .collect::>(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-release-cursor-ack-loss-jobs", + mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: job_keys[0].clone(), + predicates: job_keys + .iter() + .map(|key| PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }) + .collect(), + mutations: job_keys + .iter() + .map(|key| Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(vec![0xfe])), + }) + .collect(), + watch: Vec::new(), + }) + .unwrap(); - // Grant a long lease. - let SnapshotRenewOutcome::Renewed { - pin: long, - extended: true, - } = service.renew_snapshot(pin.snapshot_id, 3_600_000).unwrap() - else { - panic!("expected an extended live pin") - }; + metadata.fail_after_apply(b"restore-release-worker-cursor", 0); + assert!(matches!( + service.cleanup_restore_releases(2), + Err(MetadError::Metadata(MetadataError::Backend(message))) + if message == "injected restore crash after durable apply" + )); + let cursor_key = restore::restore_release_cursor_key(mount); + let cursor_before_reopen = metadata + .get_versioned( + RecordFamily::System, + &cursor_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let frozen = + restore::decode_restore_release_worker_cursor(mount, &cursor_before_reopen.value.0) + .unwrap(); + assert_eq!(frozen.start_after, job_keys[1]); - // A shorter renew must NOT shorten the protection already granted: renew is - // extend-only (Iceberg / S3 Object Lock semantics). Shrinking protection is - // expressed by `retire`, never by a shorter renew silently dropping it. - let SnapshotRenewOutcome::Renewed { - pin: after_short, - extended: false, - } = service.renew_snapshot(pin.snapshot_id, 1_000).unwrap() - else { - panic!("expected an unchanged live pin") - }; - assert_eq!( - after_short.lease_expires_unix_ms, long.lease_expires_unix_ms, - "a shorter renew must never shorten protection" - ); + metadata.clear_fault(); + drop(service); + let reopened = NoKvFs::open_existing(mount, metadata.clone(), objects, 0).unwrap(); + let append_version = reopened.next_version().unwrap(); + let appended_key = restore::restore_release_job_key(mount, append_version.get()); + assert!(append_version.get() > frozen.cycle_high_water); + reopened + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-release-cursor-post-reopen-append", + mount, + InodeId::root(), + append_version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(append_version).unwrap(), + commit_version: append_version, + primary_family: RecordFamily::System, + primary_key: appended_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: appended_key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: appended_key, + op: MutationOp::Put, + value: Some(Value(vec![0xfd])), + }], + watch: Vec::new(), + }) + .unwrap(); - // A longer renew still extends protection. - let SnapshotRenewOutcome::Renewed { - pin: after_long, - extended: true, - } = service.renew_snapshot(pin.snapshot_id, 7_200_000).unwrap() - else { - panic!("expected an extended live pin") - }; - assert!( - after_long.lease_expires_unix_ms > long.lease_expires_unix_ms, - "a longer renew extends protection" + assert_eq!(reopened.cleanup_restore_releases(2).unwrap(), 1); + assert_eq!( + reopened.cleanup_restore_releases(2).unwrap(), + 2, + "reopen must preserve the old cycle boundary and revisit its first page" ); + assert!(metadata.applied_with_prefix(b"restore-release-worker-cursor") >= 3); } #[test] -fn concurrent_snapshot_renewals_preserve_the_longest_successful_lease() { - let store = SnapshotCommitBarrierStore::new(CommandKind::RenewSnapshot, 2, 2); - let objects = MemoryObjectStore::new(); - let service = Arc::new(NoKvFs::new(MountId::new(1).unwrap(), store, objects)); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.set_clock_override_ms(1_000); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let root = service.resolve_directory_path("/runs").unwrap(); - let pin = service.snapshot_subtree_with_lease(root, 1_000).unwrap(); +fn restore_release_cursor_wraps_before_late_old_ref_sets() { + let service = service(); + let mount = service.mount_id(); + for index in 0..32_u8 { + service + .set_xattr( + InodeId::root(), + format!("user.release-frontier-{index}").as_bytes(), + vec![index], + XattrSetMode::Any, + ) + .unwrap(); + } + let job_keys = [10_u64, 20, 30] + .into_iter() + .map(|ref_set_id| restore::restore_release_job_key(mount, ref_set_id)) + .collect::>(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-old-ref-set-release-jobs", + mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: job_keys[0].clone(), + predicates: job_keys + .iter() + .map(|key| PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }) + .collect(), + mutations: job_keys + .iter() + .map(|key| Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(vec![0xfc])), + }) + .collect(), + watch: Vec::new(), + }) + .unwrap(); + assert_eq!(service.cleanup_restore_releases(2).unwrap(), 2); + let cursor_item = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &restore::restore_release_cursor_key(mount), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let frozen = + restore::decode_restore_release_worker_cursor(mount, &cursor_item.value.0).unwrap(); + assert!(frozen.cycle_high_water > 22); + assert_eq!(frozen.start_after, job_keys[1]); - let short_service = Arc::clone(&service); - let short = std::thread::spawn(move || short_service.renew_snapshot(pin.snapshot_id, 5_000)); - let long_service = Arc::clone(&service); - let long = std::thread::spawn(move || long_service.renew_snapshot(pin.snapshot_id, 10_000)); + let late_keys = [21_u64, 22] + .into_iter() + .map(|ref_set_id| restore::restore_release_job_key(mount, ref_set_id)) + .collect::>(); + let late_version = service.next_version().unwrap(); + assert!(late_version.get() > frozen.cycle_high_water); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-late-old-ref-set-release-jobs", + mount, + InodeId::root(), + late_version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(late_version).unwrap(), + commit_version: late_version, + primary_family: RecordFamily::System, + primary_key: late_keys[0].clone(), + predicates: late_keys + .iter() + .map(|key| PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }) + .collect(), + mutations: late_keys + .iter() + .map(|key| Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(vec![0xfb])), + }) + .collect(), + watch: Vec::new(), + }) + .unwrap(); - let short = short.join().unwrap().unwrap(); - let long = long.join().unwrap().unwrap(); - assert!(matches!(short, SnapshotRenewOutcome::Renewed { .. })); - assert!(matches!(long, SnapshotRenewOutcome::Renewed { .. })); assert_eq!( - service - .snapshot_pin(pin.snapshot_id) - .unwrap() - .unwrap() - .lease_expires_unix_ms, - 11_000 + service.cleanup_restore_releases(2).unwrap(), + 2, + "new rows for old ref-set ids must end the old cycle before its cursor chases them" ); + let quarantine = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_release_quarantine_prefix(mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + assert_eq!(quarantine.len(), 2, "the first page was revisited promptly"); } #[test] -fn sixteen_concurrent_snapshot_renewals_converge_on_the_longest_lease() { - const WRITERS: usize = 16; - let store = - SnapshotCommitBarrierStore::new(CommandKind::RenewSnapshot, WRITERS as u64, WRITERS); - let service = Arc::new(NoKvFs::new( - MountId::new(1).unwrap(), - store, - MemoryObjectStore::new(), - )); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.set_clock_override_ms(1_000); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let root = service.resolve_directory_path("/runs").unwrap(); - let pin = service.snapshot_subtree_with_lease(root, 1_000).unwrap(); - - let mut writers = Vec::new(); - for index in 0..WRITERS { - let service = Arc::clone(&service); - let lease_ms = 2_000 + index as u64 * 1_000; - writers.push(std::thread::spawn(move || { - service.renew_snapshot(pin.snapshot_id, lease_ms) - })); - } - for writer in writers { - assert!(matches!( - writer.join().unwrap().unwrap(), - SnapshotRenewOutcome::Renewed { .. } - )); +fn restore_release_cursor_raises_frontier_after_blocked_row_before_boundary() { + let service = service(); + let mount = service.mount_id(); + for index in 0..24_u8 { + service + .set_xattr( + InodeId::root(), + format!("user.release-mixed-{index}").as_bytes(), + vec![index], + XattrSetMode::Any, + ) + .unwrap(); } + let blocked_key = restore::restore_release_job_key(mount, 10); + let blocked_version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-release-mixed-blocked-row", + mount, + InodeId::root(), + blocked_version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(blocked_version).unwrap(), + commit_version: blocked_version, + primary_family: RecordFamily::System, + primary_key: blocked_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: blocked_key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: blocked_key.clone(), + op: MutationOp::Put, + value: Some(Value(vec![0xfa])), + }], + watch: Vec::new(), + }) + .unwrap(); + let cursor_key = restore::restore_release_cursor_key(mount); + let cursor_version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-release-mixed-cursor", + mount, + InodeId::root(), + cursor_version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(cursor_version).unwrap(), + commit_version: cursor_version, + primary_family: RecordFamily::System, + primary_key: cursor_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: cursor_key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: cursor_key.clone(), + op: MutationOp::Put, + value: Some(Value( + restore::encode_restore_release_worker_cursor( + mount, + &restore::RestoreReleaseWorkerCursor { + cycle_high_water: blocked_version.get(), + start_after: Vec::new(), + }, + ) + .unwrap(), + )), + }], + watch: Vec::new(), + }) + .unwrap(); + let changed_key = restore::restore_release_job_key(mount, 20); + let changed_version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-release-mixed-changed-row", + mount, + InodeId::root(), + changed_version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(changed_version).unwrap(), + commit_version: changed_version, + primary_family: RecordFamily::System, + primary_key: changed_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: changed_key, + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: restore::restore_release_job_key(mount, 20), + op: MutationOp::Put, + value: Some(Value(vec![0xf9])), + }], + watch: Vec::new(), + }) + .unwrap(); + + assert_eq!(service.cleanup_restore_releases(64).unwrap(), 1); + let raised = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &cursor_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let raised = restore::decode_restore_release_worker_cursor(mount, &raised.value.0).unwrap(); + assert!(raised.cycle_high_water >= changed_version.get()); + assert_eq!(raised.start_after, Vec::::new()); assert_eq!( - service - .snapshot_pin(pin.snapshot_id) - .unwrap() - .unwrap() - .lease_expires_unix_ms, - 18_000 + service.cleanup_restore_releases(64).unwrap(), + 2, + "a blocked older row must not pin a newer row-version boundary" ); } #[test] -fn stale_reaper_scan_cannot_delete_a_newer_pin_version() { - let store = SnapshotCommitBarrierStore::new(CommandKind::RetireSnapshot, 1, 2); - let objects = MemoryObjectStore::new(); - let service = Arc::new(NoKvFs::new( - MountId::new(1).unwrap(), - store.clone(), - objects, - )); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.set_clock_override_ms(1_000); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let root = service.resolve_directory_path("/runs").unwrap(); - let pin = service.snapshot_subtree_with_lease(root, 500).unwrap(); - - service.set_clock_override_ms(1_500); - let reaper_service = Arc::clone(&service); - let reaper = std::thread::spawn(move || reaper_service.reclaim_expired_snapshot_pins(100)); - store.wait_until_blocked(); - - // A deterministic clock rewind models a stale reaper candidate whose record - // was replaced before its delete applied. The version fence, not wall time, - // is the invariant under test. - service.set_clock_override_ms(1_400); - assert!(matches!( - service.renew_snapshot(pin.snapshot_id, 10_000).unwrap(), - SnapshotRenewOutcome::Renewed { extended: true, .. } - )); - store.release_blocked(); - - let outcome = reaper.join().unwrap().unwrap(); - assert_eq!(outcome.expired_candidates, 1); - assert_eq!(outcome.reaped, 0); - assert_eq!(outcome.conflicted, 1); - assert!(service.snapshot_pin(pin.snapshot_id).unwrap().is_some()); -} - -#[test] -fn one_reaper_conflict_does_not_block_other_expired_pins() { - let store = SnapshotCommitBarrierStore::new(CommandKind::RetireSnapshot, 1, 2); - let service = Arc::new(NoKvFs::new( - MountId::new(1).unwrap(), - store.clone(), - MemoryObjectStore::new(), - )); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - service.set_clock_override_ms(1_000); - service.create_dir_path("/a", 0o755, 1000, 1000).unwrap(); - service.create_dir_path("/b", 0o755, 1000, 1000).unwrap(); - let a = service.resolve_directory_path("/a").unwrap(); - let b = service.resolve_directory_path("/b").unwrap(); - let renewed = service.snapshot_subtree_with_lease(a, 500).unwrap(); - let expired = service.snapshot_subtree_with_lease(b, 500).unwrap(); - - service.set_clock_override_ms(1_500); - let reaper_service = Arc::clone(&service); - let reaper = std::thread::spawn(move || reaper_service.reclaim_expired_snapshot_pins(100)); - store.wait_until_blocked(); - service.set_clock_override_ms(1_400); - assert!(matches!( - service.renew_snapshot(renewed.snapshot_id, 10_000).unwrap(), - SnapshotRenewOutcome::Renewed { extended: true, .. } - )); - store.release_blocked(); +fn restore_release_worker_moves_noncanonical_job_key_out_of_scan_path() { + let service = service(); + let mount = service.mount_id(); + let mut invalid_key = restore::restore_release_job_prefix(mount); + invalid_key.extend_from_slice(&0_u64.to_be_bytes()); + let valid_key = restore::restore_release_job_key(mount, 1); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-noncanonical-release-job-key", + mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: invalid_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: invalid_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: valid_key.clone(), + predicate: Predicate::NotExists, + }, + ], + mutations: vec![ + Mutation { + family: RecordFamily::System, + key: invalid_key.clone(), + op: MutationOp::Put, + value: Some(Value(vec![0xf8])), + }, + Mutation { + family: RecordFamily::System, + key: valid_key.clone(), + op: MutationOp::Put, + value: Some(Value(vec![0xf7])), + }, + ], + watch: Vec::new(), + }) + .unwrap(); - let outcome = reaper.join().unwrap().unwrap(); - assert_eq!(outcome.expired_candidates, 2); - assert_eq!(outcome.reaped, 1); - assert_eq!(outcome.conflicted, 1); - assert!(service.snapshot_pin(renewed.snapshot_id).unwrap().is_some()); - assert!(service.snapshot_pin(expired.snapshot_id).unwrap().is_none()); + assert_eq!(service.cleanup_restore_releases(1).unwrap(), 1); + assert!(service + .metadata_store() + .get_versioned( + RecordFamily::System, + &invalid_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_none()); + let cursor = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &restore::restore_release_cursor_key(mount), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + restore::decode_restore_release_worker_cursor_at_version( + mount, + &cursor.value.0, + service.read_version().unwrap(), + ) + .unwrap(); + assert_eq!(service.cleanup_restore_releases(1).unwrap(), 1); + assert!(service + .metadata_store() + .get_versioned( + RecordFamily::System, + &valid_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_some()); } #[test] -fn uncontended_reaper_page_uses_one_atomic_commit() { - let (service, _objects) = service_with_objects(); - service.set_clock_override_ms(1_000); - service.create_dir_path("/a", 0o755, 1000, 1000).unwrap(); - service.create_dir_path("/b", 0o755, 1000, 1000).unwrap(); - let a = service.resolve_directory_path("/a").unwrap(); - let b = service.resolve_directory_path("/b").unwrap(); - service.snapshot_subtree_with_lease(a, 500).unwrap(); - service.snapshot_subtree_with_lease(b, 500).unwrap(); - service.set_clock_override_ms(1_500); - - let before = service.metadata_store_stats().commit_total; - let outcome = service.reclaim_expired_snapshot_pins(100).unwrap(); - let commits = service.metadata_store_stats().commit_total - before; - assert_eq!(outcome.expired_candidates, 2); - assert_eq!(outcome.reaped, 2); - assert_eq!(outcome.conflicted, 0); - assert_eq!(commits, 1); +fn idle_restore_release_cursor_does_not_write_metadata() { + let service = service(); + let mount = service.mount_id(); + let cursor_key = restore::restore_release_cursor_key(mount); + let high_water = service.read_version().unwrap().get(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-install-idle-release-cursor", + mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: cursor_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: cursor_key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: cursor_key.clone(), + op: MutationOp::Put, + value: Some(Value( + restore::encode_restore_release_worker_cursor( + mount, + &restore::RestoreReleaseWorkerCursor { + cycle_high_water: high_water, + start_after: Vec::new(), + }, + ) + .unwrap(), + )), + }], + watch: Vec::new(), + }) + .unwrap(); + let before = service.read_version().unwrap(); + let cursor_version = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &cursor_key, + before, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap() + .version; + assert_eq!(service.cleanup_restore_releases(64).unwrap(), 0); + assert_eq!(service.cleanup_restore_releases(64).unwrap(), 0); + assert_eq!(service.read_version().unwrap(), before); + assert_eq!( + service + .metadata_store() + .get_versioned( + RecordFamily::System, + &cursor_key, + before, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap() + .version, + cursor_version + ); } #[test] -fn snapshot_path_operations_reject_a_different_root() { - let (service, _objects) = service_with_objects(); - service.set_clock_override_ms(1_000); - service.create_dir_path("/a", 0o755, 1000, 1000).unwrap(); - service.create_dir_path("/b", 0o755, 1000, 1000).unwrap(); - let root = service.resolve_directory_path("/a").unwrap(); - let pin = service.snapshot_subtree_with_lease(root, 10_000).unwrap(); +fn future_restore_release_cursor_fails_fsck_cleanup_and_downgrade() { + let service = service(); + let mount = service.mount_id(); + let cursor_key = restore::restore_release_cursor_key(mount); + let read_version = service.read_version().unwrap(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-install-future-release-cursor", + mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: cursor_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: cursor_key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: cursor_key, + op: MutationOp::Put, + value: Some(Value( + restore::encode_restore_release_worker_cursor( + mount, + &restore::RestoreReleaseWorkerCursor { + cycle_high_water: read_version.get() + 1_000, + start_after: Vec::new(), + }, + ) + .unwrap(), + )), + }], + watch: Vec::new(), + }) + .unwrap(); + let report = service.fsck_restore_state(false).unwrap(); + assert!(report + .issues + .iter() + .any(|issue| issue.code == "restore_cursor_invalid")); assert!(matches!( - service.stat_path_at_snapshot("/b", pin.snapshot_id, "/"), - Err(MetadError::SnapshotRootMismatch { - snapshot_id, - expected_root, - actual_root, - .. - }) if snapshot_id == pin.snapshot_id && expected_root != root && actual_root == Some(root) - )); - assert!(matches!( - service.renew_snapshot_path("/b", pin.snapshot_id, 20_000), - Err(MetadError::SnapshotRootMismatch { .. }) + service.cleanup_restore_releases(1), + Err(MetadError::Codec(message)) + if message == "restore release worker cursor is ahead of metadata" )); - - service.set_clock_override_ms(pin.lease_expires_unix_ms); assert!(matches!( - service.stat_path_at_snapshot("/a", pin.snapshot_id, "/"), - Err(MetadError::SnapshotLeaseExpired { - snapshot_id, - lease_expires_unix_ms, - now_ms, - }) if snapshot_id == pin.snapshot_id - && lease_expires_unix_ms == pin.lease_expires_unix_ms - && now_ms == pin.lease_expires_unix_ms + service.drain_restore_to_fork_v1( + RestoreCapabilityDisabled::acknowledged(), + RestoreWritersQuiesced::acknowledged(), + ), + Err(RestoreDowngradeError::Metadata(MetadError::Codec(message))) + if message == "restore release worker cursor is ahead of metadata" )); } #[test] -fn snapshot_component_reads_are_root_bound_even_when_empty_or_zero_length() { - let (service, _objects) = service_with_objects(); - service.set_clock_override_ms(1_000); - service.create_dir_path("/a", 0o755, 1000, 1000).unwrap(); - service.create_dir_path("/b", 0o755, 1000, 1000).unwrap(); +fn restore_inverse_owner_registry_prevents_target_first_orphan_finalization() { + let service = service(); service - .create_file_path("/a/inside", 0o644, 1000, 1000) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - service - .create_file_path("/b/outside", 0o644, 1000, 1000) + publish_path_artifact( + &service, + "/source/shared.bin", + "inverse-closure", + b"borrowed bytes", + ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + let outcome = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/destination") .unwrap(); - let pin = service - .snapshot_subtree_path_with_lease("/a", 10_000) + let mount = service.mount_id(); + let operation_digest = + restore::restore_operation_digest(mount, "/source", snapshot.snapshot_id, "/destination"); + let operation_item = service + .metadata_store() + .get_versioned( + RecordFamily::System, + &restore::restore_operation_key(mount, &operation_digest), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() .unwrap(); - let inside = DentryName::new("inside").unwrap(); - let outside = DentryName::new("outside").unwrap(); + let operation = restore::decode_restore_operation(&operation_item.value.0).unwrap(); + assert_eq!(operation.destination_root, outcome.destination_root); - assert!(service - .get_attr_at_snapshot("/a", pin.snapshot_id, std::slice::from_ref(&inside)) - .unwrap() - .is_some()); - assert!(service - .get_attr_at_snapshot("/a", pin.snapshot_id, std::slice::from_ref(&outside)) - .unwrap() - .is_none()); - assert!(matches!( - service.get_attr_at_snapshot("/b", pin.snapshot_id, &[]), - Err(MetadError::SnapshotRootMismatch { .. }) - )); - assert!(matches!( - service.read_file_at_snapshot("/b", pin.snapshot_id, std::slice::from_ref(&outside), 0, 0,), - Err(MetadError::SnapshotRootMismatch { .. }) - )); -} + let owner_rows = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_base_owner_prefix(mount, operation.ref_set_id), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + assert_eq!(owner_rows.len(), 1); + let inverse_owner_rows = service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_base_inverse_owner_prefix(mount, operation.ref_set_id), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + assert_eq!(inverse_owner_rows.len(), 1); -#[test] -fn snapshot_ids_are_shard_qualified_and_foreign_ids_fail_as_root_mismatch() { - let source = service().with_shard_index(1); - source - .create_dir_path("/source", 0o755, 1000, 1000) + // Simulate a damaged owner-first row while leaving the target-first inverse + // and its ref-set registry. The registry must keep the operation durable; + // otherwise finalization would erase the only identity fsck can use to + // repair the orphan inverse. + let owner = &owner_rows[0]; + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-drop-restore-base-owner", + mount, + outcome.destination_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: owner.key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: owner.key.clone(), + predicate: Predicate::VersionEquals(owner.version), + }], + mutations: vec![delete_mutation(RecordFamily::System, owner.key.clone())], + watch: Vec::new(), + }) .unwrap(); - let pin = source.snapshot_subtree_path("/source").unwrap(); - assert_eq!(pin.snapshot_id >> 48, 1); - let destination = service().with_shard_index(2); - destination - .create_dir_path("/destination", 0o755, 1000, 1000) + service + .create_dir_path("/replacement", 0o755, 1000, 1000) .unwrap(); - assert!(matches!( - destination.stat_path_at_snapshot("/destination", pin.snapshot_id, "/"), - Err(MetadError::SnapshotRootMismatch { - snapshot_id, - actual_root: None, - actual_shard: 1, - .. - }) if snapshot_id == pin.snapshot_id - )); + service + .rename_replace_path("/replacement", "/destination") + .unwrap(); + for _ in 0..4 { + service.cleanup_restore_releases(1).unwrap(); + } + assert_eq!(service.restore_metrics().unwrap().releasing, 1); + assert!(service + .metadata_store() + .get( + RecordFamily::System, + &restore::restore_operation_key(mount, &operation_digest), + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .is_some()); + assert_eq!( + service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore::restore_base_inverse_owner_prefix(mount, operation.ref_set_id,), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .len(), + 1 + ); } #[test] diff --git a/crates/nokv-protocol/src/lib.rs b/crates/nokv-protocol/src/lib.rs index 08452b7b6..d39af5b8c 100644 --- a/crates/nokv-protocol/src/lib.rs +++ b/crates/nokv-protocol/src/lib.rs @@ -15,6 +15,11 @@ use serde::{Deserialize, Serialize}; const BINARY_CODEC_LIMIT: u64 = 16 * 1024 * 1024; +/// Capability name advertised by metadata owners that implement the durable +/// restore-to-fork contract. Keep this string stable: deployment preflight +/// treats it as a wire-level feature gate during rolling upgrades. +pub const RESTORE_TO_FORK_V1_CAPABILITY: &str = "restore_to_fork_v1"; + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] pub struct WireOpenPathReadPlanRequest { pub path: String, @@ -285,6 +290,17 @@ pub enum MetadataRpcRequest { src_path: String, dst_path: String, }, + /// Query capabilities from the owner selected by `path`. The path is a + /// routing key only; it need not exist in the namespace. + MetadataCapabilities { + path: String, + }, + RestoreSubtreePathToFork { + source_path: String, + snapshot_id: u64, + destination_path: String, + initialization: WireRestoreInitialization, + }, DiffSubtrees { a_path: String, b_path: String, @@ -452,6 +468,7 @@ pub enum WireMetadataError { owner_epoch: u64, required_epoch: u64, }, + InvalidOwnerEpoch, LeaseExpired { now_ms: u64, deadline_ms: u64, @@ -472,6 +489,18 @@ pub enum WireMetadataError { /// The target dentry is a cross-shard graft point; remove/rename of it must /// go through the graft lifecycle. The client maps this to `EBUSY`. GraftPoint, + RestoreInProgress, + RestoreRootChanged { + root: u64, + }, + RestoreBindingChanged { + root: u64, + }, + RestoreResourceLimit { + resource: String, + limit: u64, + actual: u64, + }, StalePreparedArtifactObjectGcEpoch { expected: u64, current: u64, @@ -499,6 +528,15 @@ pub enum WireMetadataError { snapshot_id: u64, attempts: usize, }, + RestoreHardlinkUnsupported { + inode: u64, + }, + RestoreCrossShardUnsupported { + inode: u64, + }, + RestoreDestinationConflict { + destination: String, + }, SyncLogArchiveFailed { committed: bool, message: String, @@ -599,6 +637,12 @@ pub enum MetadataRpcResult { root: u64, snapshot_id: u64, }, + MetadataCapabilities { + capabilities: WireMetadataCapabilities, + }, + Restore { + outcome: WireRestoreOutcome, + }, SubtreeDeltas { deltas: Vec, }, @@ -1144,6 +1188,32 @@ pub struct WirePreparedArtifact { pub object_gc_claim_version: u64, } +/// Capabilities of the metadata owner that handled a path-routed probe. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct WireMetadataCapabilities { + pub mount_id: u64, + pub restore_to_fork_v1: bool, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct WireRestoreInitialization { + pub remove_relative_paths: Vec, + pub files: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct WireRestoreInitializationFile { + pub relative_path: String, + pub bytes: Vec, + pub content_type: String, + pub mode: u32, + pub uid: u32, + pub gid: u32, +} + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] pub struct WireChunkManifest { pub chunk_index: u64, @@ -1205,6 +1275,24 @@ pub struct WireSnapshotPin { pub lease_expires_unix_ms: u64, } +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum WireRestoreState { + Complete, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct WireRestoreOutcome { + pub operation_id: String, + pub state: WireRestoreState, + pub source_root: u64, + pub destination_root: u64, + pub snapshot_id: u64, + pub read_version: u64, + pub cleanup_pending: bool, +} + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(tag = "state", rename_all = "snake_case")] pub enum WireSnapshotRenewOutcome { @@ -1696,6 +1784,13 @@ pub fn request_routing_key(request: &MetadataRpcRequest) -> RoutingKey<'_> { MetadataRpcRequest::RenamePath { source, .. } => RoutingKey::Path(source), MetadataRpcRequest::RenameReplacePath { source, .. } => RoutingKey::Path(source), MetadataRpcRequest::CloneSubtreePath { src_path, .. } => RoutingKey::Path(src_path), + MetadataRpcRequest::MetadataCapabilities { path } => RoutingKey::Path(path), + // Durable restore state is owned by the destination shard. The client + // rejects cross-shard pairs before sending, while destination routing + // keeps rolling-upgrade capability and private-state ownership aligned. + MetadataRpcRequest::RestoreSubtreePathToFork { + destination_path, .. + } => RoutingKey::Path(destination_path), MetadataRpcRequest::DiffSubtrees { a_path, .. } => RoutingKey::Path(a_path), MetadataRpcRequest::RollbackSubtreePath { target_path, .. } => { RoutingKey::Path(target_path) @@ -1750,6 +1845,117 @@ pub fn request_routing_key(request: &MetadataRpcRequest) -> RoutingKey<'_> { mod tests { use super::*; + #[test] + fn restore_request_and_outcome_codec_are_typed_and_path_routed() { + let request = MetadataRpcRequest::RestoreSubtreePathToFork { + source_path: "/workbenches/source".to_owned(), + snapshot_id: 17, + destination_path: "/workbenches/restored".to_owned(), + initialization: WireRestoreInitialization { + remove_relative_paths: vec!["metadata/checkpoints.jsonl".to_owned()], + files: vec![WireRestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: br#"{"schema":"nokv.workbench.restore_manifest.v1"}"#.to_vec(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, + }; + assert!(matches!( + request_routing_key(&request), + RoutingKey::Path("/workbenches/restored") + )); + assert_eq!( + decode_request(&encode_request(&request).unwrap()).unwrap(), + request + ); + + let envelope = MetadataRpcEnvelope { + ok: true, + result: Some(MetadataRpcResult::Restore { + outcome: WireRestoreOutcome { + operation_id: "restore-17".to_owned(), + state: WireRestoreState::Complete, + source_root: 7, + destination_root: 8, + snapshot_id: 17, + read_version: 42, + cleanup_pending: false, + }, + }), + error: None, + error_kind: None, + }; + assert_eq!( + decode_envelope(&encode_envelope(&envelope).unwrap()).unwrap(), + envelope + ); + } + + #[test] + fn capability_probe_routes_on_its_owner_path() { + let request = MetadataRpcRequest::MetadataCapabilities { + path: "/agents/a/workbenches".to_owned(), + }; + assert!(matches!( + request_routing_key(&request), + RoutingKey::Path("/agents/a/workbenches") + )); + assert_eq!( + decode_request(&encode_request(&request).unwrap()).unwrap(), + request + ); + } + + #[test] + fn legacy_wire_bytes_remain_stable() { + fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + bytes.iter().fold(String::new(), |mut out, byte| { + write!(&mut out, "{byte:02x}").unwrap(); + out + }) + } + + let request = MetadataRpcRequest::DiffSubtrees { + a_path: "/a".to_owned(), + b_path: "/b".to_owned(), + }; + let result = MetadataRpcEnvelope { + ok: true, + result: Some(MetadataRpcResult::CloneSubtree { + root: 99, + snapshot_id: 7, + }), + error: None, + error_kind: None, + }; + let error = MetadataRpcEnvelope { + ok: false, + result: None, + error: Some("expired".to_owned()), + error_kind: Some(WireMetadataError::SnapshotLeaseExpired { + snapshot_id: 7, + lease_expires_unix_ms: 11, + now_ms: 12, + }), + }; + assert_eq!( + hex(&encode_request(&request).unwrap()), + "83a26f70ad646966665f7375627472656573a6615f70617468a22f61a6625f70617468a22f62" + ); + assert_eq!( + hex(&encode_envelope(&result).unwrap()), + "82a26f6bc3a6726573756c7483a474797065ad636c6f6e655f73756274726565a4726f6f7463ab736e617073686f745f696407" + ); + assert_eq!( + hex(&encode_envelope(&error).unwrap()), + "83a26f6bc2a56572726f72a765787069726564aa6572726f725f6b696e6484a474797065b6736e617073686f745f6c656173655f65787069726564ab736e617073686f745f696407b56c656173655f657870697265735f756e69785f6d730ba66e6f775f6d730c" + ); + } + #[test] fn prepared_artifact_object_gc_epoch_round_trips() { let prepared = WirePreparedArtifact { @@ -2207,4 +2413,16 @@ mod tests { let encoded = encode_envelope(&envelope).unwrap(); assert_eq!(decode_envelope(&encoded).unwrap(), envelope); } + + #[test] + fn binary_codec_preserves_invalid_owner_epoch_error_kind() { + let envelope = MetadataRpcEnvelope { + ok: false, + result: None, + error: Some("owner epoch must be non-zero".to_owned()), + error_kind: Some(WireMetadataError::InvalidOwnerEpoch), + }; + let encoded = encode_envelope(&envelope).unwrap(); + assert_eq!(decode_envelope(&encoded).unwrap(), envelope); + } } diff --git a/crates/nokv-server/src/control.rs b/crates/nokv-server/src/control.rs index 52036e781..b08d9eb97 100644 --- a/crates/nokv-server/src/control.rs +++ b/crates/nokv-server/src/control.rs @@ -1,5 +1,5 @@ use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; use std::time::Duration; use nokv_control::{ @@ -72,6 +72,11 @@ pub(crate) struct ServerShardOwner { store: Arc, lease: ShardLease, recovery_publish_gate: Arc>, + /// Serializes exact recovery-reference CAS operations with lease renewal. + /// The visibility gate may be held across a multi-command RPC, while this + /// narrower gate is intentionally re-acquired by each post-commit log-tail + /// publication from inside that RPC. + recovery_control_gate: Arc>, published_recovery_state: Arc>>, recovery_dirty: Arc, /// Lease TTL (ms) for the wall-clock self-fence; `None` when auto-renewal is @@ -223,6 +228,7 @@ impl ServerShardOwner { store, lease, recovery_publish_gate: Arc::new(RwLock::new(())), + recovery_control_gate: Arc::new(Mutex::new(())), published_recovery_state: Arc::new(RwLock::new(None)), recovery_dirty: Arc::new(AtomicBool::new(false)), lease_ttl_ms, @@ -329,9 +335,11 @@ impl ServerShardOwner { self.mark_serving_with_recovery_refs_locked(service, checkpoint, log, durable_lsn) } - /// Publish while the caller already holds `recovery_publish_gate`. This is - /// used when the protected operation begins before the control CAS (for - /// example, snapshotting the sync-log tail or preparing a checkpoint). + /// Publish an exact recovery identity while serializing on the narrower + /// control gate. Ordinary callers also hold `recovery_publish_gate` to + /// protect namespace visibility; synchronous post-commit publication from + /// a multi-command RPC already runs inside that outer critical section and + /// deliberately acquires only the control gate here. pub(crate) fn mark_serving_with_recovery_refs_locked( &self, service: &NoKvFs, @@ -343,6 +351,10 @@ impl ServerShardOwner { M: MetadataStore, O: ObjectStore, { + let _control = self + .recovery_control_gate + .lock() + .unwrap_or_else(|error| error.into_inner()); // Local deadline/epoch fencing is checked immediately before the // control-plane lease CAS. An owner that self-fenced must not revive // itself by publishing a recovery reference. @@ -369,13 +381,16 @@ impl ServerShardOwner { record } Ok(record) => { - service.observe_required_owner_epoch(record.epoch)?; + service.fence_required_owner_epoch(record.epoch)?; return Err(mark_err.into()); } Err(_) => return Err(mark_err.into()), }, }; - service.install_owner_epoch(record.epoch)?; + // `validate_record_lease` guarantees this is the same immutable epoch + // installed during acquisition. Re-installing it here would attempt + // restore-visibility recovery while a restore commit still owns that + // fence, deadlocking synchronous post-commit publication. // `mark_serving` is a control-record CAS, not a lease keepalive. Never // extend the local deadline here: only a successful `renew` round trip // proves that the control plane extended the real lease. @@ -392,12 +407,18 @@ impl ServerShardOwner { M: MetadataStore, O: ObjectStore, { + let _control = self + .recovery_control_gate + .lock() + .unwrap_or_else(|error| error.into_inner()); // Capture the deadline basis BEFORE the round-trip so a slow renew never // pushes the local deadline past the control plane's real lease expiry. let basis_ms = service.now_ms(); match self.store.renew(&self.lease) { Ok(record) => { - service.install_owner_epoch(record.epoch)?; + // The lease epoch is immutable and was installed by acquire. + // Avoid re-entering restore visibility while a post-commit + // publication is waiting on this control gate. if let Some(ttl) = self.lease_ttl_ms { service.set_lease_deadline(basis_ms.saturating_add(ttl)); } @@ -411,7 +432,7 @@ impl ServerShardOwner { // the last successful renew still fences this owner once it // passes, so a partitioned owner cannot keep committing. if let Ok(record) = self.store.get_shard(&self.lease.shard_id) { - service.observe_required_owner_epoch(record.epoch)?; + service.fence_required_owner_epoch(record.epoch)?; } Err(err.into()) } @@ -426,6 +447,10 @@ impl ServerShardOwner { /// Relinquish ownership so a standby can acquire immediately instead of /// waiting out the lease TTL. Used on graceful shutdown. pub(crate) fn release(&self) -> Result<(), ServerError> { + let _control = self + .recovery_control_gate + .lock() + .unwrap_or_else(|error| error.into_inner()); self.store.release(&self.lease)?; Ok(()) } diff --git a/crates/nokv-server/src/rpc/mod.rs b/crates/nokv-server/src/rpc/mod.rs index 9deadcfc1..b7fab4a43 100644 --- a/crates/nokv-server/src/rpc/mod.rs +++ b/crates/nokv-server/src/rpc/mod.rs @@ -17,12 +17,16 @@ pub(crate) use transport::{ read_frame, write_frame, MAX_FRAMED_RPC_WORKERS, MIN_FRAMED_RPC_WORKERS, }; -use nokv_meta::{MetadError, OpenPathReadPlanRequest, PublishArtifactStagedSession}; +use nokv_meta::{ + MetadError, OpenPathReadPlanRequest, PublishArtifactStagedSession, RestoreInitialization, + RestoreInitializationFile, RestoreState, +}; use nokv_protocol::{ decode_advisory_lock_kind, decode_file_type, decode_name_cursor, decode_request, decode_xattr_name, encode_envelope, encode_name_cursor, encode_xattr_name, MetadataRpcEnvelope, - MetadataRpcRequest, MetadataRpcResult, WireAdvisoryLock, WireMetadataError, - WireOpenPathReadPlanRequest, WirePathMetadata, + MetadataRpcRequest, MetadataRpcResult, WireAdvisoryLock, WireMetadataCapabilities, + WireMetadataError, WireOpenPathReadPlanRequest, WirePathMetadata, WireRestoreOutcome, + WireRestoreState, }; use nokv_types::{AdvisoryLockRequest, SpecialNodeSpec}; @@ -689,6 +693,58 @@ fn execute_unfenced( snapshot_id: handle.snapshot_id, }) } + MetadataRpcRequest::MetadataCapabilities { .. } => { + Ok(MetadataRpcResult::MetadataCapabilities { + capabilities: WireMetadataCapabilities { + mount_id: slot.service().mount_id().get(), + restore_to_fork_v1: true, + }, + }) + } + MetadataRpcRequest::RestoreSubtreePathToFork { + source_path, + snapshot_id, + destination_path, + initialization, + } => { + let outcome = slot + .service() + .with_immediate_sync_metadata_log_publication(|| { + slot.service().restore_subtree_path_to_fork_initialized( + &source_path, + snapshot_id, + &destination_path, + RestoreInitialization { + remove_relative_paths: initialization.remove_relative_paths, + files: initialization + .files + .into_iter() + .map(|file| RestoreInitializationFile { + relative_path: file.relative_path, + bytes: file.bytes, + content_type: file.content_type, + mode: file.mode, + uid: file.uid, + gid: file.gid, + }) + .collect(), + }, + ) + })?; + Ok(MetadataRpcResult::Restore { + outcome: WireRestoreOutcome { + operation_id: outcome.operation_id, + state: match outcome.state { + RestoreState::Complete => WireRestoreState::Complete, + }, + source_root: outcome.source_root.get(), + destination_root: outcome.destination_root.get(), + snapshot_id: outcome.snapshot_id, + read_version: outcome.read_version, + cleanup_pending: outcome.cleanup_pending, + }, + }) + } MetadataRpcRequest::DiffSubtrees { a_path, b_path } => { let deltas = slot.service().diff_subtrees_path(&a_path, &b_path)?; Ok(MetadataRpcResult::SubtreeDeltas { @@ -1083,6 +1139,7 @@ fn refreshes_metadata_view(request: &MetadataRpcRequest) -> bool { | MetadataRpcRequest::ReadArtifactPathAtSnapshot { .. } | MetadataRpcRequest::DiffSubtrees { .. } | MetadataRpcRequest::SnapshotPin { .. } => true, + MetadataRpcRequest::MetadataCapabilities { .. } => true, MetadataRpcRequest::BootstrapRoot { .. } | MetadataRpcRequest::CreateDir { .. } | MetadataRpcRequest::CreateGraft { .. } @@ -1111,6 +1168,7 @@ fn refreshes_metadata_view(request: &MetadataRpcRequest) -> bool { | MetadataRpcRequest::RenameReplacePath { .. } | MetadataRpcRequest::SnapshotSubtreePath { .. } | MetadataRpcRequest::CloneSubtreePath { .. } + | MetadataRpcRequest::RestoreSubtreePathToFork { .. } | MetadataRpcRequest::RollbackSubtreePath { .. } | MetadataRpcRequest::RetireSnapshot { .. } | MetadataRpcRequest::RenewSnapshot { .. } diff --git a/crates/nokv-server/src/rpc/tests.rs b/crates/nokv-server/src/rpc/tests.rs index 5e049a218..69d33ffc1 100644 --- a/crates/nokv-server/src/rpc/tests.rs +++ b/crates/nokv-server/src/rpc/tests.rs @@ -79,6 +79,25 @@ fn rpc_creates_and_lists_directory() { assert_eq!(entries[0].dentry.name_hex, "72756e73"); } +#[test] +fn rpc_capability_probe_is_path_routed_and_advertises_restore_v1() { + let server = test_server(); + let envelope = request_envelope( + &server, + MetadataRpcRequest::MetadataCapabilities { + path: "/workbenches/not-yet-created".to_owned(), + }, + ); + assert!(envelope.ok, "unexpected capability error: {envelope:?}"); + match envelope.result.unwrap() { + MetadataRpcResult::MetadataCapabilities { capabilities } => { + assert_eq!(capabilities.mount_id, server.service().mount_id().get()); + assert!(capabilities.restore_to_fork_v1); + } + other => panic!("unexpected capability result: {other:?}"), + } +} + #[test] fn rpc_write_publishes_sync_shared_log_before_ack() { let dir = tempdir().unwrap(); @@ -259,8 +278,10 @@ fn rpc_publish_control_failure_does_not_hide_committed_metadata_or_local_log() { assert!(!envelope.ok, "control publication must gate the RPC ACK"); assert!(matches!( envelope.error_kind, - Some(WireMetadataError::Metadata { message }) - if message.contains("injected checkpoint publish failure") + Some(WireMetadataError::SyncLogArchiveFailed { + committed: true, + message, + }) if message.contains("injected checkpoint publish failure") )); let committed = server @@ -282,6 +303,79 @@ fn rpc_publish_control_failure_does_not_hide_committed_metadata_or_local_log() { assert_eq!(control_after.log, control_before.log); } +#[test] +fn prior_dirty_tail_does_not_mark_a_later_rejected_mutation_committed() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("meta/rpc-prior-dirty-checkpoint".to_owned()); + let control = Arc::new(FailingCheckpointPublishControl::new()); + let server = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(ServerSharedLogOptions::new( + "meta/rpc-prior-dirty-log", + )))], + )), + ) + .unwrap(); + let control_before = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + + control.fail_next_marks(2); + let committed = request_envelope( + &server, + MetadataRpcRequest::CreateDirPath { + path: "/prior-dirty".to_owned(), + mode: 0o755, + uid: 1000, + gid: 1000, + }, + ); + assert!(matches!( + committed.error_kind, + Some(WireMetadataError::SyncLogArchiveFailed { + committed: true, + .. + }) + )); + let dirty_lsn = server + .service() + .sync_metadata_log_snapshot() + .unwrap() + .durable_lsn; + assert!(dirty_lsn > control_before.durable_lsn); + + control.fail_next_marks(2); + let rejected = request_envelope( + &server, + MetadataRpcRequest::RemoveFilePath { + path: "/prior-dirty".to_owned(), + }, + ); + assert!(!rejected.ok); + assert!(matches!( + rejected.error_kind, + Some(WireMetadataError::NotFile) + )); + assert_eq!( + server + .service() + .sync_metadata_log_snapshot() + .unwrap() + .durable_lsn, + dirty_lsn, + "the rejected request must not advance the local recovery tail" + ); + assert_eq!( + control.get_shard(&ShardId::new("mount-1:/")).unwrap(), + control_before, + "the prior dirty tail must remain unpublished while control fails" + ); +} + #[test] fn controlled_read_waits_for_inflight_write_control_publication() { let dir = tempdir().unwrap(); diff --git a/crates/nokv-server/src/rpc/wire.rs b/crates/nokv-server/src/rpc/wire.rs index 153b5c2eb..3d0ecc406 100644 --- a/crates/nokv-server/src/rpc/wire.rs +++ b/crates/nokv-server/src/rpc/wire.rs @@ -1,3 +1,4 @@ +use nokv_control::ControlError; use nokv_meta::{ DentryWithAttr, MetadError, NamespaceAggregateGroup, NamespaceAggregateMeasure, NamespaceAggregateOp, NamespaceAggregateOutputMeasure, NamespaceAggregateRequest, @@ -63,6 +64,14 @@ pub(super) fn wire_server_error(err: &ServerError) -> WireMetadataError { ServerError::Object(err) => WireMetadataError::Object { message: err.to_string(), }, + ServerError::Control(ControlError::NotOwner { shard_id }) + | ServerError::Control(ControlError::StaleEpoch { shard_id, .. }) + | ServerError::Control(ControlError::StaleLease { shard_id, .. }) => { + WireMetadataError::NotOwner { + shard_id: shard_id.as_str().to_owned(), + endpoint: None, + } + } ServerError::Control(err) => WireMetadataError::Metadata { message: err.to_string(), }, @@ -99,6 +108,7 @@ fn wire_metad_error(err: &MetadError) -> WireMetadataError { owner_epoch: *owner_epoch, required_epoch: *required_epoch, }, + MetadError::InvalidOwnerEpoch => WireMetadataError::InvalidOwnerEpoch, MetadError::LeaseExpired { now_ms, deadline_ms, @@ -106,6 +116,10 @@ fn wire_metad_error(err: &MetadError) -> WireMetadataError { now_ms: *now_ms, deadline_ms: *deadline_ms, }, + MetadError::NotOwner { shard_id, endpoint } => WireMetadataError::NotOwner { + shard_id: shard_id.clone(), + endpoint: endpoint.clone(), + }, MetadError::SyncLogArchiveFailed { committed, message } => { WireMetadataError::SyncLogArchiveFailed { committed: *committed, @@ -134,6 +148,22 @@ fn wire_metad_error(err: &MetadError) -> WireMetadataError { dest_shard: *dest_shard, }, MetadError::GraftPoint => WireMetadataError::GraftPoint, + MetadError::RestoreInProgress => WireMetadataError::RestoreInProgress, + MetadError::RestoreRootChanged { root } => { + WireMetadataError::RestoreRootChanged { root: root.get() } + } + MetadError::RestoreBindingChanged { root } => { + WireMetadataError::RestoreBindingChanged { root: root.get() } + } + MetadError::RestoreResourceLimit { + resource, + limit, + actual, + } => WireMetadataError::RestoreResourceLimit { + resource: resource.clone(), + limit: *limit, + actual: *actual, + }, MetadError::StalePreparedArtifactObjectGcEpoch { expected, current } => { WireMetadataError::StalePreparedArtifactObjectGcEpoch { expected: *expected, @@ -181,6 +211,17 @@ fn wire_metad_error(err: &MetadError) -> WireMetadataError { snapshot_id: *snapshot_id, attempts: *attempts, }, + MetadError::RestoreHardlinkUnsupported { inode } => { + WireMetadataError::RestoreHardlinkUnsupported { inode: inode.get() } + } + MetadError::RestoreCrossShardUnsupported { inode } => { + WireMetadataError::RestoreCrossShardUnsupported { inode: inode.get() } + } + MetadError::RestoreDestinationConflict { destination } => { + WireMetadataError::RestoreDestinationConflict { + destination: destination.clone(), + } + } other => WireMetadataError::Metadata { message: other.to_string(), }, @@ -947,3 +988,68 @@ fn wire_object_read_block(block: &ObjectReadBlock) -> WireObjectReadBlock { pub(super) fn protocol_error(err: MetadataProtocolError) -> MetadError { MetadError::Codec(err.to_string()) } + +#[cfg(test)] +mod tests { + use super::*; + use nokv_control::ShardId; + + fn assert_control_owner_error_is_typed(error: ControlError, shard: &str) { + assert_eq!( + wire_server_error(&ServerError::Control(error)), + WireMetadataError::NotOwner { + shard_id: shard.to_owned(), + endpoint: None, + } + ); + } + + #[test] + fn invalid_owner_epoch_stays_typed_on_the_wire() { + assert_eq!( + wire_metad_error(&MetadError::InvalidOwnerEpoch), + WireMetadataError::InvalidOwnerEpoch + ); + } + + #[test] + fn control_owner_errors_stay_typed_on_the_wire() { + let shard = ShardId::new("mount-1:/"); + assert_control_owner_error_is_typed( + ControlError::NotOwner { + shard_id: shard.clone(), + }, + shard.as_str(), + ); + assert_control_owner_error_is_typed( + ControlError::StaleEpoch { + shard_id: shard.clone(), + expected: 2, + actual: 3, + }, + shard.as_str(), + ); + assert_control_owner_error_is_typed( + ControlError::StaleLease { + shard_id: shard.clone(), + epoch: 2, + lease_id: 7, + }, + shard.as_str(), + ); + } + + #[test] + fn metadata_not_owner_stays_typed_on_the_wire() { + assert_eq!( + wire_metad_error(&MetadError::NotOwner { + shard_id: "mount-1:/workbenches".to_owned(), + endpoint: Some("127.0.0.1:7731".to_owned()), + }), + WireMetadataError::NotOwner { + shard_id: "mount-1:/workbenches".to_owned(), + endpoint: Some("127.0.0.1:7731".to_owned()), + } + ); + } +} diff --git a/crates/nokv-server/src/server.rs b/crates/nokv-server/src/server.rs index c53674aee..b3522c024 100644 --- a/crates/nokv-server/src/server.rs +++ b/crates/nokv-server/src/server.rs @@ -6,7 +6,7 @@ use std::net::{TcpListener, TcpStream}; use std::path::{Path, PathBuf}; use std::sync::{mpsc, Arc, Condvar, Mutex}; use std::thread::{self, JoinHandle}; -use std::time::Duration; +use std::time::{Duration, Instant}; use nokv_control::{CheckpointRef, ControlError, ControlStore, LogRef, LogSegmentRef, ShardId}; use nokv_meta::holtstore::HoltMetadataStore; @@ -36,9 +36,16 @@ const DEFAULT_ROOT_MODE: u32 = 0o755; const SERVER_CONNECTION_WORKERS: usize = 256; const SERVER_CONNECTION_QUEUE: usize = 1024; const DEFAULT_ARCHIVE_KEEP_LAST: usize = 8; +const RESTORE_METRICS_CACHE_TTL: Duration = Duration::from_secs(15); type OpenedControlStore = (Arc, Vec); +#[derive(Clone)] +struct RestoreMetricsCache { + sampled_at: Instant, + metrics: nokv_meta::RestoreMetrics, +} + enum ServerMetadataBackupWorker { Standalone(MetadataBackupWorker), Controlled(ControlledMetadataBackupWorker), @@ -107,6 +114,10 @@ pub struct Server { /// single-node dev path (no control plane, no cross-shard grafts). control: Option>, framed_rpc_workers: rpc::RpcWorkerPool, + /// Restore gauges are exact but O(private rows). Cache the exact snapshot + /// briefly; its embedded read_version makes staleness explicit while + /// preventing every monitoring scrape from walking a million-row restore. + restore_metrics_cache: Mutex>, #[cfg(test)] _test_meta_dir: Option, } @@ -289,6 +300,7 @@ impl Server { mount: options.mount, control: control_handle, framed_rpc_workers, + restore_metrics_cache: Mutex::new(None), #[cfg(test)] _test_meta_dir: None, }) @@ -567,15 +579,35 @@ impl Server { if commits_metadata { let _visibility = owner.lock_recovery_publication(); + let before = slot.service().sync_metadata_log_publication_state(); // Set before raw execution so a partial commit, returned error, or // panic cannot release the gate looking clean. owner.mark_recovery_dirty(); let result = execute(); + let after = slot.service().sync_metadata_log_publication_state(); + let request_may_have_committed = + metadata_publication_advanced(before.as_ref(), after.as_ref()); // A clone/rollback or any other multi-stage mutator may have // committed even when its final business result is an error. The // publication barrier therefore runs on both result paths and its // failure takes precedence, keeping later reads fail-closed. - Self::publish_owner_latest_log_ref_locked(slot.service(), owner)?; + if let Err(publication_error) = + Self::publish_owner_latest_log_ref_locked(slot.service(), owner) + { + if result + .as_ref() + .is_err_and(preserve_mutation_error_over_publication) + { + return result; + } + if !request_may_have_committed && result.is_err() { + return result; + } + return Err(mutation_publication_error( + publication_error, + request_may_have_committed, + )); + } return result; } @@ -627,7 +659,7 @@ impl Server { }; if let Some(cached) = owner.published_recovery_state() { - prune_control_covered_log_segments(service, &publication, &cached)?; + prune_control_covered_log_segments(service, &cached)?; publication = service .sync_metadata_log_publication_state() .expect("enabled metadata log remains enabled under publication gate"); @@ -645,7 +677,7 @@ impl Server { // authoritative checkpoint while local covered pointers remain. // Dirty repair alone may perform this remote validation. let authoritative = owner.renew(service)?; - prune_control_covered_log_segments(service, &publication, &authoritative)?; + prune_control_covered_log_segments(service, &authoritative)?; let repaired = service .sync_metadata_log_publication_state() .expect("enabled metadata log remains enabled under publication gate"); @@ -691,6 +723,29 @@ impl Server { Ok(()) } + fn cached_restore_metrics( + &self, + service: &NoKvFs, + ) -> Result { + let mut cache = self + .restore_metrics_cache + .lock() + .unwrap_or_else(|error| error.into_inner()); + if let Some(cached) = cache.as_ref() { + if cached.sampled_at.elapsed() < RESTORE_METRICS_CACHE_TTL { + return Ok(cached.metrics.clone()); + } + } + // Keep the cache gate through collection: concurrent scrapes share one + // bounded exact scan instead of multiplying its I/O and CPU cost. + let metrics = service.restore_metrics()?; + *cache = Some(RestoreMetricsCache { + sampled_at: Instant::now(), + metrics: metrics.clone(), + }); + Ok(metrics) + } + pub fn stats_json(&self) -> String { let ready = self.check_readiness().is_ok(); let slot = self.default_slot(); @@ -700,13 +755,14 @@ impl Server { let metadata_service = service.metadata_service_stats(); let object_gc = slot.object_gc.state(); let history_gc = slot.history_gc.state(); + let restore = self.cached_restore_metrics(service); // Stats are an operational safety surface. If the persisted marker // cannot be read or decoded, report the failover requirement as active // rather than presenting an unsafe deployment as unconstrained. let failover_durability_required = service.failover_durability_is_required().unwrap_or(true); format!( - "{{\"ready\":{},\"block_cache_enabled\":{},\"object_puts\":{},\"object_put_bytes\":{},\"object_gets\":{},\"object_get_bytes\":{},\"coalesced_gets\":{},\"coalesced_get_bytes\":{},\"cache_hits\":{},\"cache_hit_bytes\":{},\"prefetch_enqueued\":{},\"prefetch_dropped\":{},\"prefetch_completed\":{},\"prefetch_failed\":{},\"prefetch_object_gets\":{},\"prefetch_object_get_bytes\":{},\"prefetch_cache_hits\":{},\"prefetch_cache_hit_bytes\":{},\"read_plan_cache_hits\":{},\"read_plan_cache_misses\":{},\"object_writeback_enqueued\":{},\"object_writeback_inline\":{},\"object_writeback_completed\":{},\"object_writeback_failed\":{},\"object_writeback_staged_bytes\":{},\"object_writeback_uploaded_bytes\":{},\"object_writeback_queue_wait_ns\":{},\"object_writeback_queue_max_wait_ns\":{},\"object_writeback_upload_ns\":{},\"object_writeback_upload_max_ns\":{},\"object_writeback_collect_ns\":{},\"object_writeback_digest_ns\":{},\"object_writeback_store_put_ns\":{},\"object_writeback_cache_put_ns\":{},\"manifest_chunks\":{},\"manifest_blocks\":{},\"metadata_store\":{},\"metadata_service\":{},\"shard_owner\":{},\"object_gc\":{},\"history_gc\":{},\"metadata_backup\":{}}}\n", + "{{\"ready\":{},\"block_cache_enabled\":{},\"object_puts\":{},\"object_put_bytes\":{},\"object_gets\":{},\"object_get_bytes\":{},\"coalesced_gets\":{},\"coalesced_get_bytes\":{},\"cache_hits\":{},\"cache_hit_bytes\":{},\"prefetch_enqueued\":{},\"prefetch_dropped\":{},\"prefetch_completed\":{},\"prefetch_failed\":{},\"prefetch_object_gets\":{},\"prefetch_object_get_bytes\":{},\"prefetch_cache_hits\":{},\"prefetch_cache_hit_bytes\":{},\"read_plan_cache_hits\":{},\"read_plan_cache_misses\":{},\"object_writeback_enqueued\":{},\"object_writeback_inline\":{},\"object_writeback_completed\":{},\"object_writeback_failed\":{},\"object_writeback_staged_bytes\":{},\"object_writeback_uploaded_bytes\":{},\"object_writeback_queue_wait_ns\":{},\"object_writeback_queue_max_wait_ns\":{},\"object_writeback_upload_ns\":{},\"object_writeback_upload_max_ns\":{},\"object_writeback_collect_ns\":{},\"object_writeback_digest_ns\":{},\"object_writeback_store_put_ns\":{},\"object_writeback_cache_put_ns\":{},\"manifest_chunks\":{},\"manifest_blocks\":{},\"metadata_store\":{},\"metadata_service\":{},\"restore\":{},\"shard_owner\":{},\"object_gc\":{},\"history_gc\":{},\"metadata_backup\":{}}}\n", ready, service.block_cache_enabled(), objects.object_puts, @@ -745,6 +801,7 @@ impl Server { objects.manifest_blocks, metadata_store_json(&metadata), metadata_service_json(&metadata_service), + restore_metrics_result_json(restore.as_ref()), self.shard_owner_json(), object_gc_json(&object_gc, failover_durability_required), history_gc_json(&history_gc), @@ -760,6 +817,13 @@ impl Server { let service = slot.service(); let object_outcome = service.cleanup_pending_objects(limit)?; let history_outcome = service.cleanup_history(limit)?; + object.restore_release_jobs_processed += object_outcome.restore_release_jobs_processed; + object.restore_release_backlog += object_outcome.restore_release_backlog; + object.restore_release_quarantine += object_outcome.restore_release_quarantine; + object.restore_release_mount_wide_quarantine += + object_outcome.restore_release_mount_wide_quarantine; + object.restore_init_tombstones_scanned += + object_outcome.restore_init_tombstones_scanned; object.scanned += object_outcome.scanned; object.blocked_by_snapshots += object_outcome.blocked_by_snapshots; object.blocked_by_read_leases += object_outcome.blocked_by_read_leases; @@ -778,8 +842,13 @@ impl Server { history.retained_by_snapshots += history_outcome.retained_by_snapshots; } Ok(format!( - r#"{{"object_gc":{{"scanned":{},"blocked_by_snapshots":{},"blocked_by_read_leases":{},"blocked_by_failover_durability":{},"attempted":{},"deleted":{},"missing":{},"records_removed":{},"snapshot_reap":{{"scanned":{},"expired_candidates":{},"reaped":{},"conflicted":{}}}}},"history_gc":{{"scanned":{},"removed":{},"retained_by_snapshots":{}}}}} + r#"{{"object_gc":{{"restore_release_jobs_processed":{},"restore_release_backlog":{},"restore_release_quarantine":{},"restore_release_mount_wide_quarantine":{},"restore_init_tombstones_scanned":{},"scanned":{},"blocked_by_snapshots":{},"blocked_by_read_leases":{},"blocked_by_failover_durability":{},"attempted":{},"deleted":{},"missing":{},"records_removed":{},"snapshot_reap":{{"scanned":{},"expired_candidates":{},"reaped":{},"conflicted":{}}}}},"history_gc":{{"scanned":{},"removed":{},"retained_by_snapshots":{}}}}} "#, + object.restore_release_jobs_processed, + object.restore_release_backlog, + object.restore_release_quarantine, + object.restore_release_mount_wide_quarantine, + object.restore_init_tombstones_scanned, object.scanned, object.blocked_by_snapshots, object.blocked_by_read_leases, @@ -842,13 +911,30 @@ impl Server { pub fn run_fsck(&self) -> Result { let mut inodes_scanned = 0_usize; let mut files_scanned = 0_usize; + let mut symlinks_scanned = 0_usize; + let mut snapshot_pins_scanned = 0_usize; + let mut fork_bindings_scanned = 0_usize; + let mut historical_bodies_scanned = 0_usize; let mut blocks_checked = 0_usize; + let mut consistent = true; let mut dangling_entries = Vec::new(); + let mut size_mismatch_entries = Vec::new(); + let mut restore_reports = Vec::new(); for slot in self.shards.values() { - let report = slot.service().fsck_dangling_blocks(0)?; - inodes_scanned += report.inodes_scanned; - files_scanned += report.files_scanned; - blocks_checked += report.blocks_checked; + let report = slot + .service() + .fsck_object_references(nokv_meta::FsckMode::Full, 0)?; + inodes_scanned = inodes_scanned.saturating_add(report.inodes_scanned); + files_scanned = files_scanned.saturating_add(report.files_scanned); + symlinks_scanned = symlinks_scanned.saturating_add(report.symlinks_scanned); + snapshot_pins_scanned = + snapshot_pins_scanned.saturating_add(report.snapshot_pins_scanned); + fork_bindings_scanned = + fork_bindings_scanned.saturating_add(report.fork_bindings_scanned); + historical_bodies_scanned = + historical_bodies_scanned.saturating_add(report.historical_bodies_scanned); + blocks_checked = blocks_checked.saturating_add(report.blocks_checked); + consistent &= report.is_consistent(); for entry in &report.dangling { dangling_entries.push(format!( "{{\"inode\":{},\"generation\":{},\"object_key\":\"{}\"}}", @@ -857,13 +943,45 @@ impl Server { escape_json_string(&entry.object_key) )); } + for entry in &report.size_mismatches { + size_mismatch_entries.push(format!( + "{{\"inode\":{},\"generation\":{},\"object_key\":\"{}\",\"expected_size\":{},\"actual_size\":{}}}", + entry.inode, + entry.generation, + escape_json_string(&entry.object_key), + entry.expected_size, + entry.actual_size, + )); + } + let restore = slot.service().fsck_restore_state(true)?; + consistent &= restore.is_consistent(); + restore_reports.push(format!( + "{{\"mount_id\":{},\"report\":{}}}", + slot.service().mount_id().get(), + restore_fsck_json(&restore), + )); } let dangling_count = dangling_entries.len(); let dangling = dangling_entries.join(","); + let size_mismatch_count = size_mismatch_entries.len(); + let size_mismatches = size_mismatch_entries.join(","); + let restore_reports = restore_reports.join(","); Ok(format!( - r#"{{"inodes_scanned":{},"files_scanned":{},"blocks_checked":{},"dangling_count":{},"dangling":[{}]}} + r#"{{"consistent":{},"inodes_scanned":{},"files_scanned":{},"symlinks_scanned":{},"snapshot_pins_scanned":{},"fork_bindings_scanned":{},"historical_bodies_scanned":{},"blocks_checked":{},"dangling_count":{},"dangling":[{}],"size_mismatch_count":{},"size_mismatches":[{}],"restore_shards":[{}]}} "#, - inodes_scanned, files_scanned, blocks_checked, dangling_count, dangling, + consistent, + inodes_scanned, + files_scanned, + symlinks_scanned, + snapshot_pins_scanned, + fork_bindings_scanned, + historical_bodies_scanned, + blocks_checked, + dangling_count, + dangling, + size_mismatch_count, + size_mismatches, + restore_reports, )) } @@ -934,6 +1052,118 @@ fn metadata_log_ref(snapshot: &nokv_meta::MetadataLogSyncSnapshot) -> Option callback -> service +/// ownership cycle; loss of the service is a hard publication failure rather +/// than permission to acknowledge a locally applied command. +fn install_owner_sync_log_publication_hook( + service: &Arc>, + owner: &ServerShardOwner, +) -> Result<(), ServerError> +where + M: nokv_meta::MetadataStore + Send + Sync + 'static, + O: nokv_object::ObjectStore + Send + Sync + 'static, +{ + let weak_service = Arc::downgrade(service); + let owner = owner.clone(); + service.install_sync_metadata_log_publication_hook(move |snapshot| { + let service = weak_service + .upgrade() + .ok_or_else(|| "metadata service was dropped before log publication".to_owned())?; + publish_archived_metadata_log_snapshot(service.as_ref(), &owner, snapshot) + .map(|_| ()) + .map_err(|error| error.to_string()) + })?; + Ok(()) +} + +/// Publish one already-archived tail without re-entering the outer namespace +/// visibility gate or the metadata commit/log gate. `ServerShardOwner` still +/// serializes the CAS with renewal and performs the final lease/epoch fence. +fn publish_archived_metadata_log_snapshot( + service: &NoKvFs, + owner: &ServerShardOwner, + snapshot: &nokv_meta::MetadataLogSyncSnapshot, +) -> Result +where + M: nokv_meta::MetadataStore, + O: nokv_object::ObjectStore, +{ + let expected_log = metadata_log_ref(snapshot); + owner.mark_recovery_dirty(); + let state = owner.mark_serving_with_recovery_refs_locked( + service, + None, + expected_log.clone(), + snapshot.durable_lsn, + )?; + if state.durable_lsn != snapshot.durable_lsn || state.log != expected_log { + return Err(ServerError::Metadata(MetadError::Codec( + "post-commit control publication changed the exact metadata log tail".to_owned(), + ))); + } + owner.mark_recovery_clean(); + Ok(state) +} + +fn preserve_mutation_error_over_publication(error: &ServerError) -> bool { + match error { + ServerError::NotOwner { .. } => true, + ServerError::Control( + ControlError::NotOwner { .. } + | ControlError::StaleEpoch { .. } + | ControlError::StaleLease { .. }, + ) => true, + ServerError::Metadata(error) => preserve_metad_error_over_publication(error), + ServerError::Io(_) | ServerError::Control(_) | ServerError::Object(_) => false, + } +} + +fn metadata_publication_advanced( + before: Option<&MetadataLogPublicationState>, + after: Option<&MetadataLogPublicationState>, +) -> bool { + match (before, after) { + (Some(before), Some(after)) => { + after.snapshot.durable_lsn > before.snapshot.durable_lsn + || (!before.has_pending_segment && after.has_pending_segment) + || (!before.has_unresolved_commit_group && after.has_unresolved_commit_group) + } + (None, Some(after)) => { + after.snapshot.durable_lsn > 0 + || after.has_pending_segment + || after.has_unresolved_commit_group + } + (Some(_), None) | (None, None) => false, + } +} + +fn preserve_metad_error_over_publication(error: &MetadError) -> bool { + match error { + MetadError::PublishArtifactFailed { source, .. } => { + preserve_metad_error_over_publication(source) + } + MetadError::SyncLogArchiveFailed { .. } + | MetadError::MetadataCheckpointInstallUncertain + | MetadError::StaleOwnerEpoch { .. } + | MetadError::LeaseExpired { .. } + | MetadError::NotOwner { .. } => true, + _ => false, + } +} + +fn mutation_publication_error(error: ServerError, committed: bool) -> ServerError { + match error { + ServerError::Control(ControlError::Backend(message)) => { + ServerError::Metadata(MetadError::SyncLogArchiveFailed { + committed, + message: format!("metadata control publication failed: {message}"), + }) + } + error => error, + } +} + fn metadata_log_publication_matches_control( publication: &MetadataLogPublicationState, control: &ServerShardOwnerState, @@ -1008,7 +1238,6 @@ where fn prune_control_covered_log_segments( service: &NoKvFs, - publication: &MetadataLogPublicationState, control: &ServerShardOwnerState, ) -> Result<(), ServerError> where @@ -1018,22 +1247,12 @@ where let Some(checkpoint) = control.checkpoint.as_ref() else { return Ok(()); }; - let Some(last_covered) = publication - .snapshot - .segments - .iter() - .take_while(|segment| segment.last_lsn <= checkpoint.lsn) - .last() - else { - return Ok(()); - }; let expected_digest = parse_hex_digest(&checkpoint.digest)?; - if last_covered.last_lsn != checkpoint.lsn || last_covered.last_digest != expected_digest { - return Err(ServerError::Metadata(MetadError::Codec(format!( - "authoritative checkpoint tail {}:{} does not match the local covered log boundary", - checkpoint.lsn, checkpoint.digest - )))); - } + // The control checkpoint may have won its publication CAS while the local + // acknowledgement was lost. Advance the meta-side baseline first; only + // then can deleting the now-covered active segment objects preserve exact + // prepared-terminal retry durability. + service.acknowledge_published_metadata_checkpoint(checkpoint.lsn, expected_digest)?; let outcome = service.prune_sync_metadata_log_segments(checkpoint.lsn); if outcome.delete_failures > 0 { eprintln!( @@ -1053,14 +1272,7 @@ where M: nokv_meta::MetadataStore, O: nokv_object::ObjectStore, { - let publication = service - .sync_metadata_log_publication_state() - .ok_or_else(|| { - ServerError::Metadata(MetadError::Codec( - "synchronous metadata log disappeared during publication".to_owned(), - )) - })?; - prune_control_covered_log_segments(service, &publication, &control)?; + prune_control_covered_log_segments(service, &control)?; let final_publication = service .sync_metadata_log_publication_state() .expect("enabled metadata log remains enabled under publication gate"); @@ -1223,6 +1435,7 @@ where None, outcome.log_lsn, )?; + service.acknowledge_published_metadata_checkpoint(outcome.log_lsn, outcome.log_digest)?; let log_prune = service.prune_sync_metadata_log_segments(outcome.log_lsn); outcome.log_segments_pruned = log_prune.pointers_pruned; outcome.log_segment_objects_deleted = log_prune.objects_deleted; @@ -1431,16 +1644,29 @@ fn open_shard_slot( let last_digest = control_recovery_digest(&state)?; let inherited_segments = inherited_log_segments(&state)?; // Isolate each shard's shared-log archive under its own prefix. - service.enable_sync_metadata_log( - MetadataLogSyncConfig::new( - shared_log_archive.prefix.clone(), - state.shard_id.as_str(), - state.epoch, - state.durable_lsn, - last_digest, - ) - .with_segments(inherited_segments), - )?; + let mut log_config = MetadataLogSyncConfig::new( + shared_log_archive.prefix.clone(), + state.shard_id.as_str(), + state.epoch, + state.durable_lsn, + last_digest, + ) + .with_segments(inherited_segments); + if restored_from_control { + let checkpoint = state.checkpoint.as_ref().ok_or_else(|| { + ServerError::Metadata(MetadError::Codec( + "restored control state lost its checkpoint baseline".to_owned(), + )) + })?; + log_config = log_config + .with_durable_recovery_baseline( + checkpoint.lsn, + parse_hex_digest(&checkpoint.digest)?, + ) + .with_authoritative_recovery_baseline(); + } + service.enable_sync_metadata_log(log_config)?; + install_owner_sync_log_publication_hook(&service, &owner)?; // A fresh archive supersedes any historical CURRENT that could retain // object references from before the deletion fence. The immutable image // is prepared first; only the owner-lease CAS makes it authoritative. @@ -1792,7 +2018,7 @@ fn metadata_store_json(stats: &nokv_meta::MetadataStoreStats) -> String { fn metadata_service_json(stats: &nokv_meta::MetadataServiceStats) -> String { format!( - "{{\"path_index_lookup_total\":{},\"path_index_hit_total\":{},\"path_index_miss_total\":{},\"path_index_stale_total\":{},\"path_index_scan_stale_total\":{},\"path_index_fallback_total\":{},\"create_files_batch_total\":{},\"create_files_entry_total\":{},\"create_dirs_batch_total\":{},\"create_dirs_entry_total\":{},\"read_dir_plus_total\":{},\"read_dir_plus_entry_total\":{},\"read_dir_plus_projection_hit_total\":{},\"metadata_log_segments_archived_total\":{},\"metadata_log_entries_archived_total\":{},\"metadata_log_archive_bytes_total\":{}}}", + "{{\"path_index_lookup_total\":{},\"path_index_hit_total\":{},\"path_index_miss_total\":{},\"path_index_stale_total\":{},\"path_index_scan_stale_total\":{},\"path_index_fallback_total\":{},\"create_files_batch_total\":{},\"create_files_entry_total\":{},\"create_dirs_batch_total\":{},\"create_dirs_entry_total\":{},\"read_dir_plus_total\":{},\"read_dir_plus_entry_total\":{},\"read_dir_plus_projection_hit_total\":{},\"metadata_log_segments_archived_total\":{},\"metadata_log_entries_archived_total\":{},\"metadata_log_archive_bytes_total\":{},\"restore_to_fork_requests_total\":{},\"restore_to_fork_success_total\":{},\"restore_to_fork_failure_total\":{},\"restore_to_fork_elapsed_ns_total\":{},\"restore_to_fork_elapsed_ns_max\":{}}}", stats.path_index_lookup_total, stats.path_index_hit_total, stats.path_index_miss_total, @@ -1809,6 +2035,11 @@ fn metadata_service_json(stats: &nokv_meta::MetadataServiceStats) -> String { stats.metadata_log_segments_archived_total, stats.metadata_log_entries_archived_total, stats.metadata_log_archive_bytes_total, + stats.restore_to_fork_requests_total, + stats.restore_to_fork_success_total, + stats.restore_to_fork_failure_total, + stats.restore_to_fork_elapsed_ns_total, + stats.restore_to_fork_elapsed_ns_max, ) } @@ -1898,12 +2129,119 @@ fn validate_control_log_segment_identity( Ok(()) } +fn restore_metrics_result_json(result: Result<&nokv_meta::RestoreMetrics, &MetadError>) -> String { + match result { + Ok(metrics) => format!( + "{{\"available\":true,{}}}", + restore_metrics_fields_json(metrics) + ), + Err(error) => format!( + "{{\"available\":false,\"error\":\"{}\"}}", + escape_json_string(&error.to_string()) + ), + } +} + +fn restore_metrics_fields_json(metrics: &nokv_meta::RestoreMetrics) -> String { + let control_rows = metrics + .control_rows + .iter() + .map(|(name, count)| format!("\"{}\":{}", escape_json_string(name), count)) + .collect::>() + .join(","); + format!( + "\"read_version\":{},\"active_marker\":{},\"allocator_v2_fenced\":{},\"operations\":{{\"preparing\":{},\"ready_to_attach\":{},\"complete\":{},\"cleaning\":{},\"discarding\":{},\"releasing\":{}}},\"max_preparing_version_age\":{},\"max_releasing_version_age\":{},\"staging_rows\":{},\"exact_reference_rows\":{},\"index_rows\":{},\"cleanup_backlog\":{},\"release_backlog\":{},\"quarantine_rows\":{},\"control_rows\":{{{}}}", + metrics.read_version, + metrics.active_marker, + metrics.allocator_v2_fenced, + metrics.preparing, + metrics.ready_to_attach, + metrics.complete, + metrics.cleaning, + metrics.discarding, + metrics.releasing, + metrics.max_preparing_version_age, + metrics.max_releasing_version_age, + metrics.staging_rows, + metrics.exact_reference_rows, + metrics.index_rows, + metrics.cleanup_backlog, + metrics.release_backlog, + metrics.quarantine_rows, + control_rows, + ) +} + +fn restore_fsck_json(report: &nokv_meta::RestoreFsckReport) -> String { + let issues = report + .issues + .iter() + .map(|issue| { + format!( + "{{\"code\":\"{}\",\"message\":\"{}\",\"operation_id\":{},\"ref_set_id\":{}}}", + escape_json_string(&issue.code), + escape_json_string(&issue.message), + issue.operation_id.as_ref().map_or_else( + || "null".to_owned(), + |operation_id| format!("\"{}\"", escape_json_string(operation_id)), + ), + issue + .ref_set_id + .map_or_else(|| "null".to_owned(), |ref_set_id| ref_set_id.to_string()), + ) + }) + .collect::>() + .join(","); + let dangling = report + .dangling_borrowed_objects + .iter() + .map(|entry| { + format!( + "{{\"inode\":{},\"generation\":{},\"object_key\":\"{}\"}}", + entry.inode, + entry.generation, + escape_json_string(&entry.object_key), + ) + }) + .collect::>() + .join(","); + let mismatches = report + .borrowed_object_size_mismatches + .iter() + .map(|entry| { + format!( + "{{\"inode\":{},\"generation\":{},\"object_key\":\"{}\",\"expected_size\":{},\"actual_size\":{}}}", + entry.inode, + entry.generation, + escape_json_string(&entry.object_key), + entry.expected_size, + entry.actual_size, + ) + }) + .collect::>() + .join(","); + format!( + "{{\"consistent\":{},\"metrics\":{{{}}},\"borrowed_objects_checked\":{},\"dangling_borrowed_objects\":[{}],\"borrowed_object_size_mismatches\":[{}],\"issues\":[{}]}}", + report.is_consistent(), + restore_metrics_fields_json(&report.metrics), + report.borrowed_objects_checked, + dangling, + mismatches, + issues, + ) +} + fn object_gc_json(state: &ObjectGcWorkerState, failover_durability_required: bool) -> String { let outcome = state.last_outcome.map_or_else( || "null".to_owned(), |outcome| { format!( - "{{\"scanned\":{},\"blocked_by_snapshots\":{},\"blocked_by_read_leases\":{},\"blocked_by_failover_durability\":{},\"attempted\":{},\"deleted\":{},\"missing\":{},\"records_removed\":{},\"snapshot_reap\":{{\"scanned\":{},\"expired_candidates\":{},\"reaped\":{},\"conflicted\":{}}}}}", + "{{\"restore_release_jobs_processed\":{},\"restore_release_backlog\":{},\"restore_release_quarantine\":{},\"restore_release_mount_wide_quarantine\":{},\"restore_init_tombstones_scanned\":{},\"scanned\":{},\"blocked_by_snapshots\":{},\"blocked_by_read_leases\":{},\"blocked_by_failover_durability\":{},\"attempted\":{},\"deleted\":{},\"missing\":{},\"records_removed\":{},\"snapshot_reap\":{{\"scanned\":{},\"expired_candidates\":{},\"reaped\":{},\"conflicted\":{}}}}}", + outcome.restore_release_jobs_processed, + outcome.restore_release_backlog, + outcome.restore_release_quarantine, + outcome.restore_release_mount_wide_quarantine, + outcome.restore_init_tombstones_scanned, outcome.scanned, outcome.blocked_by_snapshots, outcome.blocked_by_read_leases, @@ -2138,8 +2476,8 @@ pub(crate) mod tests { }; use nokv_meta::{ CommandKind, CommitResult, HistoryGcOptions, MetadataCommand, MetadataLogEntry, - MetadataStore, Mutation, MutationOp, ObjectGcOptions, Predicate, PredicateRef, ReadPurpose, - ScanRequest, Value, Version, + MetadataStore, Mutation, MutationOp, ObjectGcOptions, Predicate, PredicateRef, + PublishArtifactRange, PublishArtifactSession, ReadPurpose, ScanRequest, Value, Version, }; use nokv_object::{ MemoryObjectStore, ObjectKey, ObjectStore, ObjectStoreConfig, S3ObjectStoreOptions, @@ -2218,6 +2556,7 @@ pub(crate) mod tests { pub(crate) struct FailingCheckpointPublishControl { inner: InMemoryControlStore, fail_marks: AtomicUsize, + lose_mark_acks: AtomicUsize, fail_renews: AtomicUsize, mark_calls: AtomicUsize, fail_on_call: AtomicUsize, @@ -2228,6 +2567,7 @@ pub(crate) mod tests { Self { inner: InMemoryControlStore::new(), fail_marks: AtomicUsize::new(0), + lose_mark_acks: AtomicUsize::new(0), fail_renews: AtomicUsize::new(0), mark_calls: AtomicUsize::new(0), fail_on_call: AtomicUsize::new(0), @@ -2238,6 +2578,10 @@ pub(crate) mod tests { self.fail_marks.store(count, AtomicOrdering::SeqCst); } + fn lose_next_mark_acks(&self, count: usize) { + self.lose_mark_acks.store(count, AtomicOrdering::SeqCst); + } + pub(crate) fn fail_next_renews(&self, count: usize) { self.fail_renews.store(count, AtomicOrdering::SeqCst); } @@ -2340,6 +2684,21 @@ pub(crate) mod tests { "injected checkpoint publish failure".to_owned(), )); } + if self + .lose_mark_acks + .fetch_update( + AtomicOrdering::SeqCst, + AtomicOrdering::SeqCst, + |remaining| remaining.checked_sub(1), + ) + .is_ok() + { + self.inner + .mark_serving(lease, checkpoint, log, durable_lsn)?; + return Err(ControlError::Backend( + "injected mark-serving acknowledgement loss".to_owned(), + )); + } self.inner.mark_serving(lease, checkpoint, log, durable_lsn) } @@ -2902,6 +3261,237 @@ pub(crate) mod tests { owner.release().unwrap(); } + #[test] + fn immediate_log_publication_blocks_command_return_until_control_cas() { + let objects = BlockingPutStore::new(); + let control = Arc::new(BlockingServingControl::new()); + let (service, owner, _archive, _initial) = controlled_sync_log_backup_fixture( + Arc::clone(&control) as Arc, + objects, + "metadata/immediate-log-publication", + ); + install_owner_sync_log_publication_hook(&service, &owner).unwrap(); + let before = control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .durable_lsn; + + control.arm_mark(); + let (sent, received) = std::sync::mpsc::channel(); + let worker_service = Arc::clone(&service); + let worker = thread::spawn(move || { + let result = worker_service.with_immediate_sync_metadata_log_publication(|| { + worker_service.create_dir_path("/in-flight", 0o755, 1000, 1000) + }); + sent.send(result).unwrap(); + }); + control.wait_until_mark_reached(); + + let local = service.sync_metadata_log_snapshot().unwrap(); + assert!(local.durable_lsn > before); + assert_eq!( + control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .durable_lsn, + before, + "the blocked control CAS has not made the archived tail recoverable yet" + ); + assert!( + received.recv_timeout(Duration::from_millis(100)).is_err(), + "an applied metadata command returned before its control LogRef CAS" + ); + + control.release_mark(); + received + .recv_timeout(Duration::from_secs(2)) + .expect("metadata command should return after publication") + .unwrap(); + worker.join().unwrap(); + let durable = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!(durable.durable_lsn, local.durable_lsn); + assert_eq!(durable.log.unwrap().durable_lsn, local.durable_lsn); + owner.release().unwrap(); + } + + #[test] + fn immediate_log_publication_failure_reports_committed_and_repairs_exact_tail() { + let objects = BlockingPutStore::new(); + let control = Arc::new(FailingCheckpointPublishControl::new()); + let (service, owner, _archive, _initial) = controlled_sync_log_backup_fixture( + Arc::clone(&control) as Arc, + objects, + "metadata/immediate-log-publication-failure", + ); + install_owner_sync_log_publication_hook(&service, &owner).unwrap(); + let before = control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .durable_lsn; + + control.fail_next_marks(1); + let error = service + .with_immediate_sync_metadata_log_publication(|| { + service.create_dir_path("/committed-before-control", 0o755, 1000, 1000) + }) + .unwrap_err(); + assert!(matches!( + error, + MetadError::SyncLogArchiveFailed { + committed: true, + message, + } if message.contains("injected checkpoint publish failure") + )); + assert!(service + .lookup_path("/committed-before-control") + .unwrap() + .is_some()); + let local = service.sync_metadata_log_snapshot().unwrap(); + assert!(local.durable_lsn > before); + assert_eq!( + control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .durable_lsn, + before + ); + + let repaired = Server::publish_owner_latest_log_ref(service.as_ref(), &owner) + .unwrap() + .unwrap(); + assert_eq!(repaired.durable_lsn, local.durable_lsn); + assert_eq!(repaired.log, metadata_log_ref(&local)); + owner.release().unwrap(); + } + + #[test] + fn immediate_log_publication_accepts_exact_lost_ack_readback() { + let objects = BlockingPutStore::new(); + let control = Arc::new(FailingCheckpointPublishControl::new()); + let (service, owner, _archive, _initial) = controlled_sync_log_backup_fixture( + Arc::clone(&control) as Arc, + objects, + "metadata/immediate-log-publication-lost-ack", + ); + install_owner_sync_log_publication_hook(&service, &owner).unwrap(); + + control.lose_next_mark_acks(1); + service + .with_immediate_sync_metadata_log_publication(|| { + service.create_dir_path("/lost-ack", 0o755, 1000, 1000) + }) + .unwrap(); + let local = service.sync_metadata_log_snapshot().unwrap(); + let durable = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!(durable.durable_lsn, local.durable_lsn); + assert_eq!(durable.log, metadata_log_ref(&local)); + assert!(!owner.recovery_is_dirty()); + owner.release().unwrap(); + } + + #[test] + fn mutation_publication_error_preserves_committed_and_owner_retry_identity() { + let committed = ServerError::Metadata(MetadError::SyncLogArchiveFailed { + committed: true, + message: "inner publication failed".to_owned(), + }); + assert!(preserve_mutation_error_over_publication(&committed)); + + let stale_owner = ServerError::Control(ControlError::StaleEpoch { + shard_id: ShardId::new("mount-1:/"), + expected: 1, + actual: 2, + }); + assert!(preserve_mutation_error_over_publication(&stale_owner)); + + let mapped = mutation_publication_error( + ServerError::Control(ControlError::Backend("etcd unavailable".to_owned())), + true, + ); + assert!(matches!( + mapped, + ServerError::Metadata(MetadError::SyncLogArchiveFailed { + committed: true, + message, + }) if message.contains("etcd unavailable") + )); + } + + #[test] + fn checkpoint_publication_repair_acks_prepared_baseline_before_pruning_log() { + let objects = BlockingPutStore::new(); + let control = Arc::new(InMemoryControlStore::new()); + let (service, owner, archive, _initial) = controlled_sync_log_backup_fixture( + Arc::clone(&control) as Arc, + objects.clone(), + "metadata/prepared-checkpoint-ack-repair", + ); + let prepared = service + .prepare_artifact_create_path("/prepared.bin") + .unwrap(); + let bytes = b"prepared checkpoint repair"; + let session = PublishArtifactSession { + parent: prepared.parent, + name: prepared.name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:prepared-checkpoint-repair".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "prepared-checkpoint-repair".to_owned(), + size: bytes.len() as u64, + ranges: vec![PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + mode: 0o644, + uid: 1000, + gid: 1000, + }; + let first = service + .publish_prepared_artifact_session(prepared.clone(), session.clone()) + .unwrap(); + let before_checkpoint = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(before_checkpoint.segments.len(), 1); + let prepared_log_key = + ObjectKey::new(before_checkpoint.segments[0].segment_key.clone()).unwrap(); + + // Simulate cancellation immediately after the control checkpoint CAS: + // the checkpoint is authoritative, but the caller never executes the + // local baseline acknowledgement or segment pruning. + let checkpoint = service.prepare_immutable_metadata_backup(&archive).unwrap(); + owner + .mark_serving_with_recovery_refs( + service.as_ref(), + Some(checkpoint_ref_for_backup(&checkpoint)), + None, + checkpoint.log_lsn, + ) + .unwrap(); + assert_eq!( + service.sync_metadata_log_snapshot().unwrap().segments.len(), + 1 + ); + assert!(objects.head(&prepared_log_key).unwrap().is_some()); + + let repaired = Server::publish_owner_latest_log_ref(service.as_ref(), &owner) + .unwrap() + .unwrap(); + assert_eq!(repaired.checkpoint.unwrap().lsn, checkpoint.log_lsn); + assert!(service + .sync_metadata_log_snapshot() + .unwrap() + .segments + .is_empty()); + assert!(objects.head(&prepared_log_key).unwrap().is_none()); + assert_eq!( + service + .publish_prepared_artifact_session(prepared, session) + .unwrap() + .entry, + first.entry + ); + owner.release().unwrap(); + } + #[test] fn controlled_checkpoint_deletes_only_segments_at_or_before_captured_lsn() { let objects = BlockingPutStore::new(); @@ -3648,6 +4238,33 @@ pub(crate) mod tests { assert!(body.contains("\"history_gc\"")); } + #[test] + fn stats_cache_shares_one_exact_restore_metrics_scan() { + let server = test_server(); + let before = server.service().metadata_store_stats().scan_total; + assert!(server + .stats_json() + .contains("\"restore\":{\"available\":true")); + let after_first = server.service().metadata_store_stats().scan_total; + assert!(after_first > before); + + assert!(server + .stats_json() + .contains("\"restore\":{\"available\":true")); + let after_second = server.service().metadata_store_stats().scan_total; + assert_eq!(after_second, after_first); + } + + #[test] + fn manual_fsck_reports_object_and_restore_consistency() { + let server = test_server(); + let report = server.run_fsck().unwrap(); + assert!(report.contains("\"consistent\":true")); + assert!(report.contains("\"snapshot_pins_scanned\":0")); + assert!(report.contains("\"restore_shards\":[{")); + assert!(report.contains("\"borrowed_objects_checked\":0")); + } + #[test] fn controlled_stats_report_persisted_failover_fence_with_empty_gc_queue() { let dir = tempdir().unwrap(); diff --git a/crates/nokv/Cargo.toml b/crates/nokv/Cargo.toml index 562ae7ee9..1a028aaa6 100644 --- a/crates/nokv/Cargo.toml +++ b/crates/nokv/Cargo.toml @@ -20,6 +20,7 @@ nokv-control.workspace = true nokv-fuse.workspace = true nokv-meta.workspace = true nokv-object.workspace = true +nokv-protocol.workspace = true nokv-server.workspace = true nokv-agent.workspace = true nokv-types.workspace = true diff --git a/crates/nokv/src/bin/nokv.rs b/crates/nokv/src/bin/nokv.rs index 9d9f4b473..d2e8d3a3d 100644 --- a/crates/nokv/src/bin/nokv.rs +++ b/crates/nokv/src/bin/nokv.rs @@ -1986,7 +1986,28 @@ where nokv_agent::agent_tool_definitions() } McpProfile::Workbench => { - workbench_mcp::tool_definitions() + // tools/list has no destination arguments, so + // fleet advertisement must cover every owner + // that can win below the configured root. Any + // enumeration or probe failure is unsupported. + let restore_to_fork_v1 = workbench_options + .as_ref() + .and_then(|workbench| { + client + .metadata() + .metadata_capabilities_for_subtree_owners( + &workbench.root, + ) + .ok() + }) + .is_some_and(|owners| { + owners.iter().all(|capabilities| { + capabilities.restore_to_fork_v1 + }) + }); + workbench_mcp::tool_definitions_for_capabilities( + restore_to_fork_v1, + ) } }; let tools: Vec = definitions @@ -2217,9 +2238,11 @@ impl Error for CliError {} mod tests { use super::*; use std::net::TcpListener; + use std::sync::Arc; use std::thread; use nokv_client::NoKvFsClient; + use nokv_control::{ControlStore, InMemoryControlStore, NodeId, ShardId}; use nokv_object::{LocalObjectStore, MemoryObjectStore, ObjectKey, ObjectStore}; use tempfile::tempdir; @@ -2282,6 +2305,22 @@ mod tests { bind } + fn register_test_fleet_shard( + store: &dyn ControlStore, + prefix: &str, + shard_index: u16, + endpoint: &str, + ) { + let shard_id = ShardId::new(format!("mount-1:{prefix}")); + store + .register_shard(shard_id.clone(), prefix.to_owned(), shard_index) + .unwrap(); + let lease = store + .acquire_unassigned(shard_id, NodeId::new(endpoint)) + .unwrap(); + store.mark_serving(&lease, None, None, 0).unwrap(); + } + #[test] fn parse_defaults_to_rustfs() { let (config, command) = parse(vec![s("ls"), s("/")]).unwrap(); @@ -3195,6 +3234,46 @@ mod tests { run_workbench_mcp_requests_on_client(client, workbench_mcp_options(), requests) } + /// Restore initialization is uploaded by the metadata owner, unlike the + /// ordinary MCP write path where the client stages blocks. Keep a local + /// hot tier for this contract test so it does not depend on an external S3 + /// service while still exercising the real server-side object PUT path. + fn restore_workbench_mcp_server() -> &'static (SocketAddr, PathBuf) { + static SERVER: std::sync::OnceLock<(SocketAddr, PathBuf)> = std::sync::OnceLock::new(); + SERVER.get_or_init(|| { + let object_dir = tempdir().unwrap(); + let object_root = object_dir.path().join("objects"); + let object = ObjectStoreConfig::tiered_local_with_options( + LocalObjectStoreOptions::new(object_root.clone()), + fake_s3_options(), + TieredObjectStoreOptions { + put_policy: nokv_object::TieredPutPolicy::HotThenBackgroundCold, + ..TieredObjectStoreOptions::default() + }, + ); + let addr = spawn_test_server_with_object_config(object); + std::mem::forget(object_dir); + (addr, object_root) + }) + } + + fn run_restore_workbench_mcp_requests( + requests: Vec, + ) -> Vec { + let (addr, object_root) = restore_workbench_mcp_server(); + let store = + LocalObjectStore::new(LocalObjectStoreOptions::new(object_root.clone())).unwrap(); + let client = NoKvFsClient::connect(*addr, store); + run_workbench_mcp_requests_on_client(client, workbench_mcp_options(), requests) + } + + fn restore_direct_client() -> NoKvFsClient { + let (addr, object_root) = restore_workbench_mcp_server(); + let store = + LocalObjectStore::new(LocalObjectStoreOptions::new(object_root.clone())).unwrap(); + NoKvFsClient::connect(*addr, store) + } + fn run_workbench_mcp_requests_on_client( client: NoKvFsClient, options: McpCliOptions, @@ -3297,6 +3376,7 @@ mod tests { "workbench_snapshot", "workbench_snapshot_renew", "workbench_snapshot_list", + "workbench_restore", ] ); assert!(names.iter().all(|name| name.starts_with("workbench_"))); @@ -3305,6 +3385,41 @@ mod tests { assert!(!names.contains(&"grep")); } + #[test] + fn workbench_mcp_omits_restore_when_a_descendant_owner_cannot_confirm_capability() { + let fallback_owner = spawn_test_server(); + let control: Arc = Arc::new(InMemoryControlStore::new()); + register_test_fleet_shard(control.as_ref(), "/", 0, &fallback_owner.to_string()); + control + .register_shard( + ShardId::new("mount-1:/workbenches/special/deep"), + "/workbenches/special/deep".to_owned(), + 1, + ) + .unwrap(); + + let object_dir = tempdir().unwrap(); + let object_store = LocalObjectStore::new(LocalObjectStoreOptions::new( + object_dir.path().join("objects"), + )) + .unwrap(); + let client = + NoKvFsClient::connect_fleet(control, MountId::new(1).unwrap(), object_store).unwrap(); + let responses = run_workbench_mcp_requests_on_client( + client, + workbench_mcp_options(), + vec![serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/list" + })], + ); + + let tools = responses[0]["result"]["tools"].as_array().unwrap(); + assert!(tools.iter().any(|tool| tool["name"] == "workbench_create")); + assert!(tools.iter().all(|tool| tool["name"] != "workbench_restore")); + } + #[test] fn workbench_mcp_tool_schemas_are_closed() { let responses = run_shared_workbench_mcp_requests(vec![serde_json::json!({ @@ -3318,6 +3433,28 @@ mod tests { assert_eq!(tool["inputSchema"]["additionalProperties"], false); assert!(tool["inputSchema"]["properties"].is_object()); } + let restore = tools + .iter() + .find(|tool| tool["name"] == "workbench_restore") + .expect("restore-capable owner must advertise workbench_restore"); + assert_eq!( + restore["inputSchema"], + serde_json::json!({ + "type": "object", + "required": ["id", "at_snapshot", "destination_id"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "at_snapshot": { + "anyOf": [ + {"type": "integer", "minimum": 0}, + {"type": "string", "minLength": 1} + ] + }, + "destination_id": {"type": "string", "minLength": 1} + }, + "additionalProperties": false + }) + ); } #[test] @@ -5261,6 +5398,103 @@ mod tests { } } + #[test] + fn workbench_mcp_restore_returns_only_terminal_state_and_writes_manifest() { + let source_id = "restore-contract-source-001"; + let destination_id = "restore-contract-destination-001"; + let mut source_requests = commit_snapshot_prelude(source_id); + source_requests.push(workbench_tool_call( + 4, + "workbench_snapshot", + serde_json::json!({"id": source_id, "name": "durable", "ttl_days": 7}), + )); + let source = run_restore_workbench_mcp_requests(source_requests); + let snapshot_id = source[3]["result"]["structuredContent"]["snapshot_id"] + .as_u64() + .unwrap(); + + let responses = run_restore_workbench_mcp_requests(vec![ + workbench_tool_call( + 1, + "workbench_restore", + serde_json::json!({ + "id": source_id, + "at_snapshot": snapshot_id, + "destination_id": destination_id, + }), + ), + workbench_tool_call( + 2, + "workbench_restore", + serde_json::json!({ + "id": source_id, + "at_snapshot": snapshot_id, + "destination_id": destination_id, + }), + ), + workbench_tool_call( + 3, + "workbench_snapshot_list", + serde_json::json!({"id": destination_id}), + ), + ]); + for response in &responses[..2] { + assert_ne!(response["result"]["isError"], true, "response: {response}"); + let outcome = &response["result"]["structuredContent"]; + assert_eq!(outcome["status"], "success"); + assert_eq!(outcome["state"], "complete"); + assert_eq!(outcome["snapshot_id"], snapshot_id); + assert_eq!(outcome["source_workbench_id"], source_id); + assert_eq!(outcome["destination_workbench_id"], destination_id); + assert_eq!(outcome["cleanup_pending"], false); + assert!(outcome["operation_id"] + .as_str() + .is_some_and(|operation_id| operation_id.starts_with("restore-"))); + assert_eq!( + outcome["restore_manifest"], + format!("/workbenches/{destination_id}/metadata/restore_manifest.json") + ); + } + assert_eq!( + responses[0]["result"]["structuredContent"]["operation_id"], + responses[1]["result"]["structuredContent"]["operation_id"] + ); + assert_eq!( + responses[0]["result"]["structuredContent"]["destination_root"], + responses[1]["result"]["structuredContent"]["destination_root"] + ); + + let destination_checkpoints = &responses[2]["result"]["structuredContent"]; + assert_ne!(responses[2]["result"]["isError"], true); + assert_eq!(destination_checkpoints["checkpoint_count"], 0); + assert_eq!( + destination_checkpoints["checkpoints"], + serde_json::json!([]) + ); + + let client = restore_direct_client(); + let manifest_bytes = client + .cat(&format!( + "/workbenches/{destination_id}/metadata/restore_manifest.json" + )) + .unwrap(); + let manifest: serde_json::Value = serde_json::from_slice(&manifest_bytes).unwrap(); + assert_eq!(manifest["schema"], "nokv.workbench.restore_manifest.v1"); + assert_eq!(manifest["source_workbench_id"], source_id); + assert_eq!(manifest["destination_workbench_id"], destination_id); + assert_eq!(manifest["snapshot_id"], snapshot_id); + assert_eq!( + manifest["operation_id"], + responses[0]["result"]["structuredContent"]["operation_id"] + ); + assert_eq!( + client + .cat(&format!("/workbenches/{destination_id}/outputs/result.txt")) + .unwrap(), + b"done\n" + ); + } + #[test] fn workbench_mcp_snapshot_renew_reports_reaped_snapshot() { let id = "ckpt-reaped-007"; diff --git a/crates/nokv/src/bin/nokv/workbench_mcp.rs b/crates/nokv/src/bin/nokv/workbench_mcp.rs index c1433ff75..f2be1deb5 100644 --- a/crates/nokv/src/bin/nokv/workbench_mcp.rs +++ b/crates/nokv/src/bin/nokv/workbench_mcp.rs @@ -5,10 +5,13 @@ use base64::Engine; use nokv_agent::AgentToolDefinition; use nokv_client::{ decode_name_cursor, encode_name_cursor, is_artifact_write_conflict, is_metadata_not_found, - ArtifactMetadata, ClientError, NoKvFsClient, + ArtifactMetadata, ClientError, NoKvFsClient, RestoreState, +}; +use nokv_meta::{ + restore_operation_id, MetadError, RestoreInitialization, RestoreInitializationFile, }; -use nokv_meta::MetadError; use nokv_object::ObjectStore; +use nokv_protocol::RESTORE_TO_FORK_V1_CAPABILITY; use nokv_types::{FileType, PathMetadata}; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; @@ -118,6 +121,7 @@ impl WorkbenchToolError { pub fn as_value(&self) -> Value { json!({ + "status": "error", "code": self.code, "message": self.message, "retryable": self.retryable, @@ -142,8 +146,11 @@ pub fn normalize_workbench_root(raw: &str) -> Result { Ok(normalized) } -pub fn tool_definitions() -> Vec { - vec![ +/// Build the workbench surface after capability probing. Restore is omitted, +/// rather than advertised optimistically, unless the caller confirms the +/// durable v1 contract for every metadata owner that can own a target. +pub fn tool_definitions_for_capabilities(restore_to_fork_v1: bool) -> Vec { + let mut tools = vec![ AgentToolDefinition { name: "workbench_create", description: @@ -440,7 +447,25 @@ pub fn tool_definitions() -> Vec { "additionalProperties": false }), }, - ] + ]; + if restore_to_fork_v1 { + tools.push(AgentToolDefinition { + name: "workbench_restore", + description: + "Restore a live checkpoint into a new workbench using a durable COW fork. The source remains unchanged, the destination must be absent, and exact retries are idempotent.", + parameters: json!({ + "type": "object", + "required": ["id", "at_snapshot", "destination_id"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "at_snapshot": restore_at_snapshot_parameter_schema(), + "destination_id": {"type": "string", "minLength": 1} + }, + "additionalProperties": false + }), + }); + } + tools } /// Shared schema for the `at_snapshot` argument: either a numeric snapshot id or @@ -457,6 +482,15 @@ fn at_snapshot_parameter_schema() -> Value { }) } +fn restore_at_snapshot_parameter_schema() -> Value { + json!({ + "anyOf": [ + {"type": "integer", "minimum": 0}, + {"type": "string", "minLength": 1} + ] + }) +} + // Mirrors of the agent find/aggregate sub-schemas (nokv-agent // agent_tool_definitions); keep them in sync when the agent schemas change. fn predicates_parameter_schema() -> Value { @@ -531,6 +565,7 @@ where "workbench_snapshot" => snapshot_workbench(client, options, args), "workbench_snapshot_renew" => renew_snapshot_workbench(client, options, args), "workbench_snapshot_list" => list_snapshots_workbench(client, options, args), + "workbench_restore" => restore_workbench(client, options, args), other => Err(WorkbenchToolError::new(format!( "unknown workbench tool {other}" ))), @@ -1561,6 +1596,128 @@ where })) } +fn restore_workbench( + client: &NoKvFsClient, + options: &WorkbenchMcpOptions, + args: &Value, +) -> Result +where + O: ObjectStore + Send + Sync + 'static, +{ + validate_exact_arguments(args, &["id", "at_snapshot", "destination_id"])?; + let id = required_workbench_id(args)?; + let destination_id = required_string(args, "destination_id")?.to_owned(); + validate_workbench_id(&destination_id)?; + if destination_id == id { + return Err(WorkbenchToolError::new( + "destination_id must differ from the source workbench id", + )); + } + let at_snapshot = args + .get("at_snapshot") + .ok_or_else(|| WorkbenchToolError::new("missing required argument at_snapshot"))?; + let snapshot_id = resolve_at_snapshot(client, options, &id, at_snapshot)?; + + let source_path = workbench_path(options, &id); + let destination_path = workbench_path(options, &destination_id); + let manifest_path = section_path( + options, + &destination_id, + "metadata", + Some("restore_manifest.json"), + ); + // The destination owner is the authority that must understand and retain + // the private restore state. Probing only the source can advertise a tool + // across a rolling-upgrade shard boundary that the attach owner cannot + // safely execute. + let capabilities = match client.metadata().metadata_capabilities(&destination_path) { + Ok(capabilities) => capabilities, + Err(ClientError::Protocol(_)) => { + return Err(restore_capability_mismatch(&destination_path)); + } + Err(err) => return Err(client_error(err)), + }; + if !capabilities.restore_to_fork_v1 { + return Err(restore_capability_mismatch(&destination_path)); + } + let expected_operation_id = restore_operation_id( + capabilities.mount_id, + &source_path, + snapshot_id, + &destination_path, + ) + .map_err(|err| WorkbenchToolError::new(err.to_string()))?; + let manifest = json!({ + "schema": "nokv.workbench.restore_manifest.v1", + "operation_id": expected_operation_id, + "restored_from": { + "workbench_id": id, + "path": source_path, + "snapshot_id": snapshot_id, + }, + "source_workbench_id": id, + "source_path": source_path, + "destination_workbench_id": destination_id, + "destination_path": destination_path, + "snapshot_id": snapshot_id, + }); + let manifest_bytes = serde_json::to_vec(&manifest).map_err(|err| { + WorkbenchToolError::new(format!("failed to encode restore manifest: {err}")) + })?; + let outcome = client + .metadata() + .restore_subtree_path_to_fork_initialized( + &source_path, + snapshot_id, + &destination_path, + RestoreInitialization { + remove_relative_paths: vec![format!("metadata/{CHECKPOINT_REGISTRY_RELPATH}")], + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: manifest_bytes, + content_type: "application/json".to_owned(), + mode: DEFAULT_MODE_FILE, + uid: options.uid, + gid: options.gid, + }], + }, + ) + .map_err(|err| restore_client_error(snapshot_id, &destination_path, err))?; + + if outcome.state != RestoreState::Complete + || outcome.operation_id != expected_operation_id + || outcome.snapshot_id != snapshot_id + || outcome.cleanup_pending + { + return Err(WorkbenchToolError::typed( + "RestoreProtocolMismatch", + "metadata owner returned a restore outcome that does not match the durable request", + true, + json!({ + "expected_operation_id": expected_operation_id, + "actual_operation_id": outcome.operation_id, + "expected_snapshot_id": snapshot_id, + "actual_snapshot_id": outcome.snapshot_id, + "cleanup_pending": outcome.cleanup_pending, + }), + )); + } + + Ok(json!({ + "status": "success", + "state": "complete", + "operation_id": outcome.operation_id, + "source_workbench_id": id, + "destination_workbench_id": destination_id, + "snapshot_id": outcome.snapshot_id, + "read_version": outcome.read_version, + "source_root": outcome.source_root.get(), + "destination_root": outcome.destination_root.get(), + "cleanup_pending": false, + "restore_manifest": manifest_path, + })) +} + /// At-snapshot read/stat/list. The lease is checked *before* any bytes are read /// so a caller never silently observes a half-dead snapshot (unchanged files /// still resolving while overwritten ones vanish); expiry is a loud error. @@ -2759,6 +2916,29 @@ fn required_string<'a>(args: &'a Value, name: &'static str) -> Result<&'a str, W .ok_or_else(|| WorkbenchToolError::new(format!("missing required string argument {name}"))) } +fn validate_exact_arguments(args: &Value, required: &[&str]) -> Result<(), WorkbenchToolError> { + let object = args + .as_object() + .ok_or_else(|| WorkbenchToolError::new("tool arguments must be a JSON object"))?; + for field in required { + if !object.contains_key(*field) { + return Err(WorkbenchToolError::new(format!( + "missing required argument {field}" + ))); + } + } + if let Some(field) = object + .keys() + .find(|field| !required.contains(&field.as_str())) + { + return Err(WorkbenchToolError::new(format!( + "unknown argument {field}; expected exactly {}", + required.join(", ") + ))); + } + Ok(()) +} + fn optional_string<'a>( args: &'a Value, name: &'static str, @@ -2805,6 +2985,35 @@ fn optional_usize(args: &Value, name: &'static str) -> Result, Wor fn client_error(err: ClientError) -> WorkbenchToolError { match &err { + ClientError::Metadata(MetadError::NotOwner { shard_id, endpoint }) => { + WorkbenchToolError::typed( + "NotOwner", + err.to_string(), + true, + json!({"shard_id": shard_id, "endpoint": endpoint}), + ) + } + ClientError::Metadata(MetadError::StaleOwnerEpoch { + owner_epoch, + required_epoch, + }) => WorkbenchToolError::typed( + "StaleOwnerEpoch", + err.to_string(), + true, + json!({"owner_epoch": owner_epoch, "required_epoch": required_epoch}), + ), + ClientError::Metadata(MetadError::InvalidOwnerEpoch) => { + WorkbenchToolError::typed("InvalidOwnerEpoch", err.to_string(), false, json!({})) + } + ClientError::Metadata(MetadError::LeaseExpired { + now_ms, + deadline_ms, + }) => WorkbenchToolError::typed( + "LeaseExpired", + err.to_string(), + true, + json!({"now_ms": now_ms, "deadline_ms": deadline_ms}), + ), ClientError::Metadata(MetadError::SnapshotLeaseExpired { snapshot_id, lease_expires_unix_ms, @@ -2866,10 +3075,128 @@ fn client_error(err: ClientError) -> WorkbenchToolError { true, json!({"snapshot_id": snapshot_id, "attempts": attempts}), ), + ClientError::Metadata(MetadError::RestoreInProgress) => { + WorkbenchToolError::typed("RestoreInProgress", err.to_string(), true, json!({})) + } + ClientError::Metadata(MetadError::RestoreRootChanged { root }) => { + WorkbenchToolError::typed( + "RestoreRootChanged", + err.to_string(), + false, + json!({"root": root.get()}), + ) + } + ClientError::Metadata(MetadError::RestoreBindingChanged { root }) => { + WorkbenchToolError::typed( + "RestoreBindingChanged", + err.to_string(), + false, + json!({"root": root.get()}), + ) + } + ClientError::Metadata(MetadError::RestoreDestinationConflict { destination }) => { + WorkbenchToolError::typed( + "RestoreDestinationConflict", + err.to_string(), + false, + json!({"destination": destination}), + ) + } + ClientError::Metadata(MetadError::RestoreResourceLimit { + resource, + limit, + actual, + }) => WorkbenchToolError::typed( + "RestoreResourceLimit", + err.to_string(), + false, + json!({"resource": resource, "limit": limit, "actual": actual}), + ), + ClientError::Metadata(MetadError::RestoreHardlinkUnsupported { inode }) => { + WorkbenchToolError::typed( + "RestoreHardlinkUnsupported", + err.to_string(), + false, + json!({"inode": inode.get()}), + ) + } + ClientError::Metadata(MetadError::RestoreCrossShardUnsupported { inode }) => { + WorkbenchToolError::typed( + "RestoreCrossShardUnsupported", + err.to_string(), + false, + json!({"inode": inode.get()}), + ) + } + ClientError::Metadata(MetadError::StalePreparedArtifactObjectGcEpoch { + expected, + current, + }) => WorkbenchToolError::typed( + "StalePreparedArtifactObjectGcEpoch", + err.to_string(), + true, + json!({"expected": expected, "current": current}), + ), + ClientError::Metadata(MetadError::SyncLogArchiveFailed { committed, message }) => { + WorkbenchToolError::typed( + "SyncLogArchiveFailed", + err.to_string(), + true, + json!({"committed": committed, "archive_error": message}), + ) + } _ => WorkbenchToolError::typed("NoKvClientError", err.to_string(), false, json!({})), } } +fn restore_client_error( + snapshot_id: u64, + owner_path: &str, + err: ClientError, +) -> WorkbenchToolError { + if let ClientError::Io(message) = &err { + return WorkbenchToolError::typed( + "RestoreTransportUnavailable", + err.to_string(), + true, + json!({ + "snapshot_id": snapshot_id, + "owner_path": owner_path, + "transport_error": message, + }), + ); + } + if matches!(&err, ClientError::Protocol(message) + if message.contains(RESTORE_TO_FORK_V1_CAPABILITY)) + { + return restore_capability_mismatch(owner_path); + } + if let ClientError::Metadata(MetadError::CrossShard { + source_shard, + dest_shard, + }) = &err + { + return WorkbenchToolError::typed( + "RestoreCrossShardUnsupported", + err.to_string(), + false, + json!({"source_shard": source_shard, "dest_shard": dest_shard}), + ); + } + snapshot_client_error(snapshot_id, err) +} + +fn restore_capability_mismatch(path: &str) -> WorkbenchToolError { + WorkbenchToolError::typed( + "CapabilityMismatch", + format!( + "metadata owner for {path} does not advertise required capability {RESTORE_TO_FORK_V1_CAPABILITY}" + ), + false, + json!({"capability": RESTORE_TO_FORK_V1_CAPABILITY, "path": path}), + ) +} + fn snapshot_client_error(snapshot_id: u64, err: ClientError) -> WorkbenchToolError { if matches!(&err, ClientError::Metadata(MetadError::NotFound)) { return WorkbenchToolError::typed( @@ -2886,6 +3213,157 @@ fn snapshot_client_error(snapshot_id: u64, err: ClientError) -> WorkbenchToolErr mod tests { use super::*; + fn disconnected_client() -> NoKvFsClient { + NoKvFsClient::connect( + "127.0.0.1:1".parse().unwrap(), + nokv_object::MemoryObjectStore::new(), + ) + } + + fn workbench_options() -> WorkbenchMcpOptions { + WorkbenchMcpOptions { + root: "/workbenches".to_owned(), + max_bytes: DEFAULT_WORKBENCH_MAX_BYTES, + uid: 1000, + gid: 1000, + } + } + + #[test] + fn restore_schema_is_exact_and_capability_gated() { + let enabled = tool_definitions_for_capabilities(true); + let restore = enabled + .iter() + .find(|tool| tool.name == "workbench_restore") + .expect("restore capability should advertise the tool"); + assert_eq!( + restore.parameters, + json!({ + "type": "object", + "required": ["id", "at_snapshot", "destination_id"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "at_snapshot": { + "anyOf": [ + {"type": "integer", "minimum": 0}, + {"type": "string", "minLength": 1} + ] + }, + "destination_id": {"type": "string", "minLength": 1} + }, + "additionalProperties": false + }) + ); + assert!(tool_definitions_for_capabilities(false) + .iter() + .all(|tool| tool.name != "workbench_restore")); + } + + #[test] + fn restore_rejects_null_negative_empty_extra_and_same_id_before_rpc() { + let client = disconnected_client(); + let options = workbench_options(); + for args in [ + json!({"id": "source", "at_snapshot": null, "destination_id": "target"}), + json!({"id": "source", "at_snapshot": -1, "destination_id": "target"}), + json!({"id": "source", "at_snapshot": "", "destination_id": "target"}), + json!({"id": "source", "at_snapshot": 1, "destination_id": "target", "extra": true}), + json!({"id": "same", "at_snapshot": 1, "destination_id": "same"}), + ] { + let error = restore_workbench(&client, &options, &args).unwrap_err(); + assert_eq!(error.as_value()["status"], "error"); + assert_eq!(error.as_value()["retryable"], false); + assert!( + !error.to_string().contains("Connection refused"), + "validation must fail before capability/RPC: {error}" + ); + } + } + + #[test] + fn capability_mismatch_is_a_structured_fail_closed_error() { + let error = restore_capability_mismatch("/workbenches/source"); + let value = error.as_value(); + assert_eq!(value["status"], "error"); + assert_eq!(value["code"], "CapabilityMismatch"); + assert_eq!(value["retryable"], false); + assert_eq!( + value["details"]["capability"], + RESTORE_TO_FORK_V1_CAPABILITY + ); + assert_eq!(value["details"]["path"], "/workbenches/source"); + } + + #[test] + fn restore_retry_boundaries_are_preserved_as_structured_errors() { + let transport = restore_client_error( + 41, + "/workbenches/destination", + ClientError::Io("metadata rpc response timed out".to_owned()), + ) + .as_value(); + assert_eq!(transport["code"], "RestoreTransportUnavailable"); + assert_eq!(transport["retryable"], true); + assert_eq!(transport["details"]["snapshot_id"], 41); + assert_eq!( + transport["details"]["owner_path"], + "/workbenches/destination" + ); + assert_eq!( + transport["details"]["transport_error"], + "metadata rpc response timed out" + ); + + let not_owner = client_error(ClientError::Metadata(MetadError::NotOwner { + shard_id: "mount-1:/workbenches".to_owned(), + endpoint: Some("127.0.0.1:7731".to_owned()), + })) + .as_value(); + assert_eq!(not_owner["code"], "NotOwner"); + assert_eq!(not_owner["retryable"], true); + assert_eq!(not_owner["details"]["shard_id"], "mount-1:/workbenches"); + assert_eq!(not_owner["details"]["endpoint"], "127.0.0.1:7731"); + + let stale_epoch = client_error(ClientError::Metadata(MetadError::StaleOwnerEpoch { + owner_epoch: 7, + required_epoch: 8, + })) + .as_value(); + assert_eq!(stale_epoch["code"], "StaleOwnerEpoch"); + assert_eq!(stale_epoch["retryable"], true); + assert_eq!(stale_epoch["details"]["owner_epoch"], 7); + assert_eq!(stale_epoch["details"]["required_epoch"], 8); + + let invalid_epoch = + client_error(ClientError::Metadata(MetadError::InvalidOwnerEpoch)).as_value(); + assert_eq!(invalid_epoch["code"], "InvalidOwnerEpoch"); + assert_eq!(invalid_epoch["retryable"], false); + + let object_epoch = client_error(ClientError::Metadata( + MetadError::StalePreparedArtifactObjectGcEpoch { + expected: 11, + current: 12, + }, + )) + .as_value(); + assert_eq!(object_epoch["code"], "StalePreparedArtifactObjectGcEpoch"); + assert_eq!(object_epoch["retryable"], true); + assert_eq!(object_epoch["details"]["expected"], 11); + assert_eq!(object_epoch["details"]["current"], 12); + + for committed in [false, true] { + let archive = client_error(ClientError::Metadata(MetadError::SyncLogArchiveFailed { + committed, + message: "archive unavailable".to_owned(), + })) + .as_value(); + assert_eq!(archive["code"], "SyncLogArchiveFailed"); + assert_eq!(archive["retryable"], true); + assert_eq!(archive["details"]["committed"], committed); + assert_eq!(archive["details"]["archive_error"], "archive unavailable"); + } + } + #[test] fn normalizes_workbench_root() { assert_eq!( diff --git a/docs/architecture.md b/docs/architecture.md index ac2d242d7..a49822e0f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -190,18 +190,42 @@ objective is the worker interval; the bodies were always safe in the object stor ## Consistency Checking -`nokv fsck` verifies the live namespace against the object store: it walks every -live file at its current body generation and confirms each referenced block -still exists (`head`). This is the read-side complement to the object-first write -ordering — the ordering guarantees metadata never references a missing object, -and fsck detects any drift after the fact (an out-of-band deletion, an -eventual-consistency anomaly in external storage, or a latent bug), reporting -each dangling reference as `(inode, generation, object_key)`. Superseded and -snapshot-pinned generations are not mistaken for drift (the scan uses each -inode's current body generation), and a clone's borrowed block keys resolve -against the source objects that still exist. Reclaiming the opposite drift — -orphan objects written but never referenced — is a planned extension that needs -an object-store `list`. +`nokv fsck` performs the explicit, full consistency scan. It walks current file +and object-backed symlink bodies plus every body reachable from an unexpired +snapshot pin or durable `ForkBinding`. Sparse/append bodies are resolved through +their complete `base_generation` chain. Every referenced block is checked with +`head` for both existence and exact length, so the report catches out-of-band +deletion, truncated replacement, object-store anomalies, and latent metadata +bugs without treating retained history as orphaned. + +Durable workbench restore adds a second graph to this check. Fsck validates the +operation/claim/root graph, temporary history binding, staging member inverses, +exact object owner/inverse/seal rows, initialization tombstones, cleanup and +release jobs, quarantine envelopes, and the private index overlay (including +its physical MVCC envelope and owner/inverse closure). Optional object +verification confirms each exact borrowed reference still has both a backing +object and an effective borrower manifest. A metadata write concurrent with +either scan invalidates the report instead of returning a mixed-version result; +the operator retries. + +Full fsck is intentionally expensive. `/stats` does not run it: restore gauges +count private keyspaces in bounded pages, strictly decode only operation rows, +and cache the last exact, version-labelled sample briefly at the server. This +keeps monitoring memory bounded without presenting a sampled count as exact. +Process-local metadata-service counters separately expose restore request, +success/failure, total elapsed nanoseconds, and maximum elapsed nanoseconds; +they include lock wait and exact terminal retries but reset on owner restart. +Reclaiming the opposite drift—objects written but never referenced and absent +from a restore initialization tombstone—still requires an object-store `list` +and remains a separate extension. + +The `restore_to_fork_v1` downgrade fence may be removed only through the typed +drain API after deployment routing has disabled the capability and every restore +writer has been globally stopped or fenced. The drain re-deletes and CAS-removes +late-PUT tombstones, requires every registered restore control/index prefix to +be empty, then atomically rewrites allocator v2 to v1, removes the active marker, +and rotates the open object-GC claim. A clean full fsck and a fresh metadata +checkpoint are still mandatory before starting a pre-restore metadata binary. ## Distributed Direction diff --git a/docs/development/workbench-checkpoint-lifecycle.md b/docs/development/workbench-checkpoint-lifecycle.md index 9ee3ff3b3..b76f1335b 100644 --- a/docs/development/workbench-checkpoint-lifecycle.md +++ b/docs/development/workbench-checkpoint-lifecycle.md @@ -130,16 +130,19 @@ Replace the bare "cite snapshot_id" instruction with "cite name + snapshot_id and state the expiry". Keep the four test-pinned anchor strings; bump to v0.3.0. -## 4. Phase 2/3 (follow-up PRs, designed now, not built) +## 4. Phase 2 durable restore and later follow-ups - **L1 named refs as GC roots**: named checkpoints move from registry-file convention to pinned records the retention floor respects without a lease; explicit delete only; `min-checkpoints-to-keep` guard. -- **Restore = fork**: `workbench_restore {snapshot, to_workbench}` built on - `materialize_subtree_at` (pub(super), clone.rs:88) + `link_clone_root`; - verified feasible (~40 service lines + one RPC). In-place rollback stays - CLI-only behind an explicit flag (`rollback_subtree` already enforces - same-root). +- **Durable restore-to-fork**: `workbench_restore` is not a thin wrapper around + generic clone. It owns a private, replayable operation/claim/root graph, + detached materialization, deterministic initialization, exact borrowed-object + references, a Complete-gated index overlay, and bounded cleanup/release jobs. + The temporary existing-format `ForkBinding` protects history only until the + exact-reference seal and final attach CAS; `Complete` does not retain a global + history floor. Generic clone/rollback encodings and behavior remain unchanged, + and in-place rollback stays CLI-only behind its existing explicit flag. - Touch-on-use renewal, structured at-snapshot reads, first-class `SnapshotExpired` error, 14-day reflog trail for reaped anonymous pins, expiry watch events, L2 archival export. diff --git a/scripts/lingtai-workbench/README.md b/scripts/lingtai-workbench/README.md index c7c6fd8d8..1764bb63d 100644 --- a/scripts/lingtai-workbench/README.md +++ b/scripts/lingtai-workbench/README.md @@ -181,13 +181,21 @@ The MCP server exposes workbench tools with the `workbench_` prefix: ```text workbench_create workbench_put_file +workbench_append +workbench_edit workbench_list workbench_stat workbench_read workbench_grep +workbench_search +workbench_aggregate +workbench_catalog workbench_find workbench_commit workbench_snapshot +workbench_snapshot_renew +workbench_snapshot_list +workbench_restore ``` The server registration name remains `nokv-workbench`; that is the MCP server @@ -199,5 +207,95 @@ Run the installer tests with: ```bash python3 ./scripts/lingtai-workbench/install_workbench_mcp_test.py +python3 ./scripts/lingtai-workbench/durable_restore_live_e2e_test.py +python3 -m py_compile ./scripts/lingtai-workbench/durable_restore_live_e2e.py bash -n ./scripts/lingtai-workbench/up.sh ``` + +## Durable Restore Live Gate + +`durable_restore_live_e2e.py` is the merge gate for durable workbench +restore-to-fork. It starts an isolated RustFS container, a request-counting S3 +proxy, `nokv serve`, and disposable LingTai Agent registrations. The Agent MCP +is deliberately disconnected and recovered before use. The concurrency gate +then launches 16 independent MCP processes plus a seventeenth observer from the +same Agent-resolved launch contract. + +Run the complete gate from the LingTai companion checkout's uv environment: + +```bash +uv run --project /path/to/lingtai-kernel \ + python /path/to/NoKV/scripts/lingtai-workbench/durable_restore_live_e2e.py \ + --lingtai-kernel-dir /path/to/lingtai-kernel \ + --profile full \ + --require-all +``` + +The full gate always uses a 1 GiB sparse-but-real binary fixture. Every 4 MiB +block contains a distinct deterministic marker; the upload must expose exactly +256 independent 4 MiB RustFS objects and the final remote digest must match. It validates +the capability-gated raw restore schema, numeric exact retry, first-visible +manifest and checkpoint-registry removal, 16-way idempotency, COW PUT counts, +durable crash recovery after hold, every dynamically discovered materialization +and exact-reference batch (including the first absent phase as a bounded +termination proof), initialization PUT-before and PUT-after, reference seal, +index seal, and attach apply-before-ACK. Every crash after the initialization +PUT but before attach must observe the old incarnation, +publish its durable cleanup tombstone, delete the old-incarnation object, and +only then PUT the rebuilt manifest under a fresh incarnation key. The permanent +tombstone stays eligible for repeated sweeps so an arbitrarily late old-owner +PUT cannot become untracked or delete the rebuilt manifest. It also crashes +during bounded cleanup and paged release, +then validates server +kill/replay, Agent reconnect, `search`/`aggregate`/`catalog`, nested restore, +source pin retirement and deletion, root move and rename-replace, and +escaped-borrower retention followed by final exact-reference object release. +Search is consumed through every cursor page; rename/delete/publish and final +release must leave no query ghosts. A deterministic pre-attach barrier proves +that stat and all three query surfaces remain hidden before the visibility +pointer flips. The live Agent's actual restore handler resends the same numeric +request after MCP reconnect. Restore metrics and strict object/restore fsck must +show one private Complete graph after the 16-way call; after release its +operation/member/exact-reference/index/release rows must return to the measured +durable-ledger baseline with no backlog or quarantine. All transient graph rows +must be zero; permanent initialization tombstones and their round-robin cursor, +plus the release cursor, must retain exactly their pre-restore row counts. The durable +`restore_to_fork_v1_active` marker and allocator-v2 downgrade fence must remain +present (they are removed only by the explicit downgrade-drain protocol and are +not leaks). Pin retirement races explicit object and history GC, demonstrates +zero remaining pins/ForkBindings, and preserves the borrower. The final RustFS +inventory must equal the exact initial inventory. +`--require-all` has no skip path: missing Docker, AWS CLI, LingTai dependencies, +the restore capability, a stale/unbuilt binary, a changed binary hash, or any +scenario fails the command. The JSON summary records NoKV/LingTai revisions and +the exact launched NoKV binary SHA-256. + +The barrier protocol is explicitly test-only. The gate sets +`NOKV_TEST_RESTORE_BARRIER_DIR` on `nokv serve`, arms +`..arm`, waits for the server's `.ready` marker, sends +SIGKILL, removes all markers, reopens the same metadata WAL and RustFS bucket, +and resends the same numeric request. Every completed create must expose exactly +one root and one live manifest object (a post-PUT crash necessarily records one +discarded old-incarnation PUT as well); released operations must leave no object +references behind. + +The metadata HA smoke also holds a post-checkpoint restore at `index-sealed`, +fails owner A over through etcd, and explicitly redrives the same operation on +owner B. It uses a per-run RustFS bucket (including with external RustFS), +requires strict JSON fsck for both object references and the unique two-Complete +restore graph, and removes the bucket on exit: + +```bash +NOKV_HA_STALE_OWNER_CHAOS=1 ./scripts/run-metadata-ha-smoke.sh +``` + +The live public surface exercises the canonical path index and built-in query +fields. No MCP or CLI API currently registers custom `PathIndexCatalog` rows, +so custom-row overlay and replay coverage belongs in the `nokv-meta` +integration suite; this live gate does not claim to exercise that private +registration interface. + +For local iteration only, use `--profile quick`; it keeps the non-crash +contract, idempotency, restart, indexing, and object-lifecycle assertions with +a 16 MiB binary fixture, but omits the expensive crash matrix. Add +`--keep-state` when diagnosing a failure. diff --git a/scripts/lingtai-workbench/durable_restore_live_e2e.py b/scripts/lingtai-workbench/durable_restore_live_e2e.py new file mode 100755 index 000000000..1692339bb --- /dev/null +++ b/scripts/lingtai-workbench/durable_restore_live_e2e.py @@ -0,0 +1,3768 @@ +#!/usr/bin/env python3 +# Copyright 2024-2026 The NoKV Authors. +# SPDX-License-Identifier: Apache-2.0 + +"""Live acceptance gate for durable NoKV workbench restore-to-fork. + +The gate owns an isolated RustFS container, a counting S3 proxy, a NoKV +metadata server, and disposable LingTai Agent registrations. It drives restore +through independent real LingTai MCP clients; the NoKV CLI is used only for +namespace mutations and streaming the large binary fixture. + +The full profile is the merge gate. ``--profile full --require-all`` always +uses a real 1 GiB COW fixture and treats every missing dependency or unexecuted +assertion as a failure. +""" + +from __future__ import annotations + +import argparse +import collections +import concurrent.futures +import dataclasses +import hashlib +import http.client +import http.server +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, Callable +from unittest.mock import MagicMock + + +WORKBENCH_ROOT_TEMPLATE = "/agents/{agent_id}/wb" +RESTORE_TOOL = "workbench_restore" +SECTIONS = ("input", "scripts", "outputs", "logs", "metadata") +BASE_WORKBENCH_TOOLS = { + "workbench_create", + "workbench_put_file", + "workbench_append", + "workbench_edit", + "workbench_list", + "workbench_stat", + "workbench_read", + "workbench_grep", + "workbench_search", + "workbench_aggregate", + "workbench_catalog", + "workbench_find", + "workbench_commit", + "workbench_snapshot", + "workbench_snapshot_renew", + "workbench_snapshot_list", +} +HOP_BY_HOP_HEADERS = { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "transfer-encoding", + "upgrade", +} +CLEANUP_CRASH_PHASE = "cleanup-batch-000000" +RELEASE_CRASH_PHASE = "release-batch-000000" +RETRYABLE_FSCK_CONFLICT = ( + "object-reference fsck raced a metadata write; retry the scan" +) +DEFAULT_LIVE_MOUNT_ID = 1 +CONCURRENT_RESTORE_CALLS = 16 +# The public workbench_search schema caps one page at 10. Keep this below the +# cap so even the quick workload is forced through multiple real cursor pages. +INDEX_PAGE_LIMIT = 7 +COW_BLOCK_BYTES = 4 << 20 +FULL_COW_BLOCK_COUNT = 256 +MAX_DISCOVERED_BATCH_PHASES = 16_384 +# Manual GC calls exercise the release/retention races deterministically. Keep +# periodic workers far enough apart that the online full fsck can obtain the +# stable metadata epoch its public contract requires. +# Every lifecycle assertion below explicitly drives GC, including a dedicated +# concurrent pin-retirement race. Keep passive ticks outside the acceptance +# window so stable-epoch fsck is not nondeterministically invalidated by a +# timer that provides no additional coverage. +BACKGROUND_GC_INTERVAL_MS = 3_600_000 + + +class AcceptanceError(RuntimeError): + """A deterministic acceptance assertion failed.""" + + +@dataclasses.dataclass(frozen=True) +class WorkloadProfile: + name: str + indexed_files: int + large_bytes: int + + +def workload_profile(name: str) -> WorkloadProfile: + if name == "quick": + return WorkloadProfile(name, indexed_files=24, large_bytes=16 << 20) + if name == "full": + return WorkloadProfile(name, indexed_files=96, large_bytes=1 << 30) + raise AcceptanceError(f"unknown workload profile: {name}") + + +def indexed_restore_phase(kind: str, index: int) -> str: + if kind not in {"materialize", "reference"}: + raise AcceptanceError(f"unknown indexed restore phase kind: {kind!r}") + if index < 0 or index > 999_999: + raise AcceptanceError(f"restore phase index is out of range: {index}") + return f"{kind}-batch-{index:06d}" + + +def create_crash_phases( + materialization_batches: int, reference_batches: int +) -> tuple[str, ...]: + if materialization_batches < 1 or reference_batches < 1: + raise AcceptanceError("restore crash matrix requires non-empty batch phases") + return ( + "hold-applied", + *( + indexed_restore_phase("materialize", index) + for index in range(materialization_batches) + ), + "initialization-put-before-000000", + "initialization-put-after-000000", + "index-sealed", + *( + indexed_restore_phase("reference", index) + for index in range(reference_batches) + ), + "references-sealed", + "attach-applied", + ) + + +def restore_phase_requires_manifest_rebuild(phase: str) -> bool: + """Return whether a crash leaves an uploaded pre-attach manifest behind.""" + return ( + phase == "initialization-put-after-000000" + or phase == "index-sealed" + or phase == "references-sealed" + or phase.startswith("reference-batch-") + ) + + +def fixture_block_count(size: int) -> int: + if size <= 0 or size % COW_BLOCK_BYTES != 0: + raise AcceptanceError("COW fixture size must be a positive multiple of 4 MiB") + return size // COW_BLOCK_BYTES + + +def fixture_block_marker(index: int) -> bytes: + if index < 0: + raise AcceptanceError("COW block index must be non-negative") + identity = index.to_bytes(8, "big") + return ( + f"nokv-durable-restore-v1:block:{index:06d}:".encode() + + hashlib.sha256(b"nokv-cow-block-marker-v1\0" + identity).digest() + + b"\n" + ) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(8 << 20): + digest.update(chunk) + return digest.hexdigest() + + +@dataclasses.dataclass(frozen=True) +class ToolError: + code: str + message: str + retryable: bool + details: dict[str, Any] + + +@dataclasses.dataclass(frozen=True) +class McpLaunch: + command: str + args: list[str] + env: dict[str, str] + root: str + + +@dataclasses.dataclass(frozen=True) +class ObjectFingerprint: + size: int + etag: str + last_modified: str + + +@dataclasses.dataclass(frozen=True) +class ObjectMutation: + method: str + target: str + + +@dataclasses.dataclass(frozen=True) +class BarrierHandle: + directory: Path + stem: str + + @property + def arm_path(self) -> Path: + return self.directory / f"{self.stem}.arm" + + @property + def ready_path(self) -> Path: + return self.directory / f"{self.stem}.ready" + + @property + def continue_path(self) -> Path: + return self.directory / f"{self.stem}.continue" + + def arm(self) -> None: + self.directory.mkdir(parents=True, exist_ok=True) + for path in (self.ready_path, self.continue_path): + path.unlink(missing_ok=True) + self.arm_path.write_text("armed\n", encoding="utf-8") + + def wait_ready(self, timeout: float) -> None: + deadline = time.monotonic() + timeout + while not self.ready_path.exists(): + if time.monotonic() >= deadline: + raise AcceptanceError( + f"restore crash barrier did not become ready: {self.ready_path}" + ) + time.sleep(0.002) + + def wait_ready_or_done( + self, future: concurrent.futures.Future[Any], timeout: float + ) -> bool: + deadline = time.monotonic() + timeout + while True: + if self.ready_path.exists(): + return True + if future.done(): + return False + if time.monotonic() >= deadline: + raise AcceptanceError( + f"restore crash barrier did not become ready: {self.ready_path}" + ) + time.sleep(0.002) + + def disarm_after_crash(self) -> None: + self.arm_path.unlink(missing_ok=True) + self.ready_path.unlink(missing_ok=True) + self.continue_path.unlink(missing_ok=True) + + def release(self) -> None: + self.continue_path.write_text("continue\n", encoding="utf-8") + + +class RestoreBarrierController: + def __init__(self, directory: Path) -> None: + self.directory = directory + + def handle(self, operation_id: str, phase: str) -> BarrierHandle: + if not operation_id.startswith("restore-"): + raise AcceptanceError(f"invalid restore operation id: {operation_id!r}") + if not phase or "/" in phase or "\\" in phase or "\0" in phase: + raise AcceptanceError(f"invalid restore barrier phase: {phase!r}") + return BarrierHandle(self.directory, f"{operation_id}.{phase}") + + def clear(self) -> None: + shutil.rmtree(self.directory, ignore_errors=True) + self.directory.mkdir(parents=True, exist_ok=True) + + +def canonical_absolute_path(path: str) -> str: + if not path.startswith("/"): + raise AcceptanceError(f"restore operation path must be absolute: {path!r}") + components = [component for component in path.split("/") if component] + if any(component in {".", ".."} for component in components): + raise AcceptanceError(f"restore operation path is not canonical: {path!r}") + return "/" + "/".join(components) + + +def restore_operation_id( + mount_id: int, source_path: str, snapshot_id: int, destination_path: str +) -> str: + if mount_id <= 0 or snapshot_id < 0: + raise AcceptanceError("mount id must be positive and snapshot id non-negative") + source = canonical_absolute_path(source_path).encode() + destination = canonical_absolute_path(destination_path).encode() + digest = hashlib.sha256() + digest.update(b"nokv-restore-to-fork-request-v1\0") + digest.update(mount_id.to_bytes(8, "big")) + digest.update(len(source).to_bytes(8, "big")) + digest.update(source) + digest.update(snapshot_id.to_bytes(8, "big")) + digest.update(len(destination).to_bytes(8, "big")) + digest.update(destination) + return f"restore-{digest.hexdigest()}" + + +def changed_objects( + before: dict[str, ObjectFingerprint], + after: dict[str, ObjectFingerprint], +) -> set[str]: + """Return new, deleted, or overwritten S3 keys.""" + return { + key for key in before.keys() | after.keys() if before.get(key) != after.get(key) + } + + +def assert_native_tool_error(result: dict[str, Any], expected_code: str) -> ToolError: + """Require LingTai to preserve the MCP structured error at the top level.""" + if result.get("status") != "error": + raise AcceptanceError(f"MCP error lacks top-level error status: {result!r}") + code = result.get("code") + message = result.get("message") + retryable = result.get("retryable") + details = result.get("details") + if not isinstance(code, str) or not code: + raise AcceptanceError(f"MCP error lacks top-level code: {result!r}") + if not isinstance(message, str) or not message: + raise AcceptanceError(f"MCP error lacks top-level message: {result!r}") + if not isinstance(retryable, bool): + raise AcceptanceError(f"MCP error lacks top-level retryable: {result!r}") + if not isinstance(details, dict): + raise AcceptanceError(f"MCP error lacks top-level details: {result!r}") + if code != expected_code: + raise AcceptanceError( + f"MCP error code differs: expected {expected_code!r}, observed {code!r}" + ) + return ToolError(code, message, retryable, details) + + +def decode_tool_error(result: dict[str, Any]) -> ToolError: + payload: Any = result + if not isinstance(payload.get("code"), str): + payload = result.get("message") + if isinstance(payload, str): + try: + payload = json.loads(payload) + except json.JSONDecodeError as exc: + raise AcceptanceError( + f"MCP error is not structured JSON: {payload!r}" + ) from exc + if not isinstance(payload, dict): + raise AcceptanceError(f"MCP error payload is not an object: {payload!r}") + code = payload.get("code") + message = payload.get("message") + retryable = payload.get("retryable") + details = payload.get("details") + if not isinstance(code, str) or not isinstance(message, str): + raise AcceptanceError(f"typed MCP error lacks code/message: {payload!r}") + if not isinstance(retryable, bool) or not isinstance(details, dict): + raise AcceptanceError(f"typed MCP error lacks retryable/details: {payload!r}") + return ToolError(code, message, retryable, details) + + +def validate_restore_manifest( + manifest: dict[str, Any], + *, + operation_id: str, + source_workbench_id: str, + source_path: str, + destination_workbench_id: str, + destination_path: str, + snapshot_id: int, +) -> None: + restored_from = manifest.get("restored_from") + if ( + manifest.get("schema") != "nokv.workbench.restore_manifest.v1" + or manifest.get("operation_id") != operation_id + or manifest.get("source_workbench_id") != source_workbench_id + or manifest.get("source_path") != source_path + or manifest.get("destination_workbench_id") != destination_workbench_id + or manifest.get("destination_path") != destination_path + or manifest.get("snapshot_id") != snapshot_id + or not isinstance(restored_from, dict) + or restored_from.get("workbench_id") != source_workbench_id + or restored_from.get("path") != source_path + or restored_from.get("snapshot_id") != snapshot_id + ): + raise AcceptanceError(f"restore manifest is malformed: {manifest!r}") + + +def error_text(result: dict[str, Any]) -> str: + message = result.get("message") + if isinstance(message, str): + return message + return json.dumps(result, sort_keys=True) + + +def assert_tool_error(result: dict[str, Any], context: str) -> None: + if result.get("status") != "error" and "code" not in result: + raise AcceptanceError(f"{context} unexpectedly succeeded: {result!r}") + + +def validate_tool_contract(tools: list[dict[str, Any]]) -> None: + """Validate the capability-gated 17-tool surface and raw restore schema.""" + by_name = {tool.get("name"): tool for tool in tools} + expected = BASE_WORKBENCH_TOOLS | {RESTORE_TOOL} + if set(by_name) != expected: + raise AcceptanceError( + "unexpected workbench tool surface; " + f"missing={sorted(expected - set(by_name))}, " + f"extra={sorted(set(by_name) - expected)}" + ) + schema = by_name[RESTORE_TOOL].get("schema") + if not isinstance(schema, dict): + raise AcceptanceError("workbench_restore lacks inputSchema") + expected_fields = {"id", "at_snapshot", "destination_id"} + if schema.get("type") != "object": + raise AcceptanceError("workbench_restore schema must be an object") + if set(schema.get("required", [])) != expected_fields: + raise AcceptanceError("workbench_restore must require exactly three fields") + properties = schema.get("properties") + if not isinstance(properties, dict) or set(properties) != expected_fields: + raise AcceptanceError("workbench_restore schema contains wrong properties") + if schema.get("additionalProperties") is not False: + raise AcceptanceError("workbench_restore must reject additional properties") + for field in ("id", "destination_id"): + field_schema = properties.get(field) + if not isinstance(field_schema, dict) or field_schema.get("type") != "string": + raise AcceptanceError(f"workbench_restore {field} must be a string") + if field_schema.get("minLength") != 1: + raise AcceptanceError(f"workbench_restore {field} must be non-empty") + alternatives = properties.get("at_snapshot", {}).get("anyOf") + if not isinstance(alternatives, list) or len(alternatives) != 2: + raise AcceptanceError("at_snapshot must have exactly two alternatives") + by_type = { + item.get("type"): item for item in alternatives if isinstance(item, dict) + } + if set(by_type) != {"integer", "string"}: + raise AcceptanceError("at_snapshot accepts a type other than integer/string") + if by_type["integer"].get("minimum") != 0: + raise AcceptanceError("numeric at_snapshot must be non-negative") + if by_type["string"].get("minLength") != 1: + raise AcceptanceError("string at_snapshot must be non-empty") + + +def read_json_object(result: dict[str, Any]) -> dict[str, Any]: + if ( + result.get("format") != "structured" + or result.get("record_type") != "json_object" + ): + raise AcceptanceError(f"expected structured JSON object: {result!r}") + if result.get("truncated") is True or result.get("next_cursor") is not None: + raise AcceptanceError("manifest read was truncated") + value: dict[str, Any] = {} + items = result.get("items") + if not isinstance(items, list): + raise AcceptanceError("structured JSON response lacks items") + for item in items: + record = item.get("value") if isinstance(item, dict) else None + key = record.get("key") if isinstance(record, dict) else None + if not isinstance(key, str) or "value" not in record: + raise AcceptanceError(f"malformed JSON record: {item!r}") + value[key] = record["value"] + return value + + +RESTORE_OPERATION_STATES = ( + "preparing", + "ready_to_attach", + "complete", + "cleaning", + "discarding", + "releasing", +) +RESTORE_DURABLE_LEDGER_ROWS = { + "init_upload_tombstone", + "init_upload_tombstone_cursor", + "release_cursor", +} + + +def nonnegative_int(value: Any, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise AcceptanceError(f"{field} must be a non-negative integer: {value!r}") + return value + + +def restore_release_graph_drained(metrics: Any) -> bool: + if not isinstance(metrics, dict): + raise AcceptanceError(f"restore metrics are not an object: {metrics!r}") + operations = metrics.get("operations") + if not isinstance(operations, dict) or set(operations) != set( + RESTORE_OPERATION_STATES + ): + raise AcceptanceError(f"restore operation metrics are malformed: {metrics!r}") + operation_rows = sum( + nonnegative_int(operations[state], f"restore.operations.{state}") + for state in RESTORE_OPERATION_STATES + ) + private_rows = sum( + nonnegative_int(metrics.get(field), f"restore.{field}") + for field in ( + "staging_rows", + "exact_reference_rows", + "index_rows", + "cleanup_backlog", + "release_backlog", + ) + ) + if nonnegative_int(metrics.get("quarantine_rows"), "restore.quarantine_rows"): + raise AcceptanceError(f"restore release entered quarantine: {metrics!r}") + return operation_rows == 0 and private_rows == 0 + + +def validate_restore_metrics_object( + metrics: Any, *, expected_complete: int, expect_empty: bool +) -> dict[str, Any]: + if not isinstance(metrics, dict): + raise AcceptanceError(f"restore metrics are not an object: {metrics!r}") + if metrics.get("active_marker") is not True: + raise AcceptanceError(f"restore active marker is not durable: {metrics!r}") + if metrics.get("allocator_v2_fenced") is not True: + raise AcceptanceError( + f"restore allocator downgrade fence is absent: {metrics!r}" + ) + operations = metrics.get("operations") + if not isinstance(operations, dict) or set(operations) != set( + RESTORE_OPERATION_STATES + ): + raise AcceptanceError(f"restore operation metrics are malformed: {metrics!r}") + counts = { + state: nonnegative_int(operations[state], f"restore.operations.{state}") + for state in RESTORE_OPERATION_STATES + } + if counts["complete"] != expected_complete or any( + count != 0 for state, count in counts.items() if state != "complete" + ): + raise AcceptanceError( + "restore operation graph is not uniquely terminal: " + f"expected_complete={expected_complete}, counts={counts!r}" + ) + for field in ( + "staging_rows", + "exact_reference_rows", + "index_rows", + "cleanup_backlog", + "release_backlog", + "quarantine_rows", + ): + nonnegative_int(metrics.get(field), f"restore.{field}") + control_rows = metrics.get("control_rows") + if not isinstance(control_rows, dict) or not control_rows: + raise AcceptanceError(f"restore control row metrics are malformed: {metrics!r}") + for name, count in control_rows.items(): + if not isinstance(name, str) or not name: + raise AcceptanceError(f"restore control row name is malformed: {name!r}") + nonnegative_int(count, f"restore.control_rows.{name}") + if nonnegative_int( + control_rows.get("operation"), "restore.control_rows.operation" + ) != sum(counts.values()): + raise AcceptanceError("restore operation row count disagrees with state counts") + if any( + nonnegative_int(metrics[field], f"restore.{field}") != 0 + for field in ("cleanup_backlog", "release_backlog", "quarantine_rows") + ): + raise AcceptanceError(f"restore backlog/quarantine is non-zero: {metrics!r}") + if expect_empty: + leaked_rows = { + name: nonnegative_int(count, f"restore.control_rows.{name}") + for name, count in control_rows.items() + if name not in RESTORE_DURABLE_LEDGER_ROWS and count != 0 + } + if leaked_rows or any( + nonnegative_int(metrics[field], f"restore.{field}") != 0 + for field in ("staging_rows", "exact_reference_rows", "index_rows") + ): + raise AcceptanceError( + "released restore graph rows did not drain to the durable-ledger " + f"baseline: leaked={leaked_rows!r}, metrics={metrics!r}" + ) + elif expected_complete > 0 and any( + nonnegative_int(metrics[field], f"restore.{field}") == 0 + for field in ("staging_rows", "exact_reference_rows", "index_rows") + ): + raise AcceptanceError(f"complete restore graph lacks private rows: {metrics!r}") + return metrics + + +def validate_restore_metrics( + stats: Any, *, expected_complete: int, expect_empty: bool +) -> dict[str, Any]: + if not isinstance(stats, dict): + raise AcceptanceError(f"server stats are not an object: {stats!r}") + metrics = stats.get("restore") + if not isinstance(metrics, dict) or metrics.get("available") is not True: + raise AcceptanceError(f"restore metrics are unavailable: {metrics!r}") + return validate_restore_metrics_object( + metrics, expected_complete=expected_complete, expect_empty=expect_empty + ) + + +def validate_fsck_report( + report: Any, + *, + expected_complete: int, + expected_snapshot_pins: int, + expected_fork_bindings: int, +) -> dict[str, Any]: + if not isinstance(report, dict) or report.get("consistent") is not True: + raise AcceptanceError(f"fsck is not consistent: {report!r}") + for count_field, rows_field in ( + ("dangling_count", "dangling"), + ("size_mismatch_count", "size_mismatches"), + ): + if nonnegative_int(report.get(count_field), count_field) != 0: + raise AcceptanceError(f"fsck {count_field} is non-zero: {report!r}") + if report.get(rows_field) != []: + raise AcceptanceError(f"fsck {rows_field} is not empty: {report!r}") + if ( + nonnegative_int(report.get("snapshot_pins_scanned"), "snapshot_pins_scanned") + != expected_snapshot_pins + ): + raise AcceptanceError(f"fsck snapshot pin count changed: {report!r}") + if ( + nonnegative_int(report.get("fork_bindings_scanned"), "fork_bindings_scanned") + != expected_fork_bindings + ): + raise AcceptanceError(f"fsck ForkBinding count changed: {report!r}") + shards = report.get("restore_shards") + if not isinstance(shards, list) or len(shards) != 1: + raise AcceptanceError(f"fsck must report exactly one restore shard: {shards!r}") + shard = shards[0] + if not isinstance(shard, dict) or shard.get("mount_id") != DEFAULT_LIVE_MOUNT_ID: + raise AcceptanceError(f"fsck restore shard identity is wrong: {shard!r}") + restore = shard.get("report") + if not isinstance(restore, dict) or restore.get("consistent") is not True: + raise AcceptanceError(f"restore fsck is not consistent: {restore!r}") + for field in ( + "issues", + "dangling_borrowed_objects", + "borrowed_object_size_mismatches", + ): + if restore.get(field) != []: + raise AcceptanceError(f"restore fsck {field} is not empty: {restore!r}") + validate_restore_metrics_object( + restore.get("metrics"), + expected_complete=expected_complete, + expect_empty=expected_complete == 0, + ) + borrowed = nonnegative_int( + restore.get("borrowed_objects_checked"), "borrowed_objects_checked" + ) + if expected_complete > 0 and borrowed == 0: + raise AcceptanceError("restore fsck checked no borrowed objects") + return report + + +def free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def tail(path: Path, lines: int = 120) -> str: + if not path.exists(): + return "" + return "\n".join( + path.read_text(encoding="utf-8", errors="replace").splitlines()[-lines:] + ) + + +class CountingProxyServer(http.server.ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, bind: tuple[str, int], backend_port: int) -> None: + super().__init__(bind, CountingProxyHandler) + self.backend_port = backend_port + self._lock = threading.Lock() + self._successful_puts: list[str] = [] + self._successful_mutations: list[ObjectMutation] = [] + + def record_put(self, target: str) -> None: + with self._lock: + self._successful_puts.append(target) + self._successful_mutations.append(ObjectMutation("PUT", target)) + + def record_delete(self, target: str) -> None: + with self._lock: + self._successful_mutations.append(ObjectMutation("DELETE", target)) + + def put_records(self) -> list[str]: + with self._lock: + return list(self._successful_puts) + + def mutation_records(self) -> list[ObjectMutation]: + with self._lock: + return list(self._successful_mutations) + + +class CountingProxyHandler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server: CountingProxyServer + + def log_message(self, _format: str, *_args: Any) -> None: + return + + def do_DELETE(self) -> None: + self._forward() + + def do_GET(self) -> None: + self._forward() + + def do_HEAD(self) -> None: + self._forward() + + def do_POST(self) -> None: + self._forward() + + def do_PUT(self) -> None: + self._forward() + + def _forward(self) -> None: + transfer_encoding = self.headers.get("Transfer-Encoding", "").lower() + if transfer_encoding and transfer_encoding != "identity": + self.send_error( + 502, + "counting proxy requires Content-Length; chunked S3 requests are unsupported", + ) + return + raw_length = self.headers.get("Content-Length") + try: + length = int(raw_length) if raw_length is not None else 0 + except ValueError: + self.send_error(400, "invalid Content-Length") + return + backend = http.client.HTTPConnection( + "127.0.0.1", self.server.backend_port, timeout=300 + ) + try: + backend.putrequest( + self.command, + self.path, + skip_host=True, + skip_accept_encoding=True, + ) + for name, value in self.headers.items(): + if name.lower() not in HOP_BY_HOP_HEADERS: + backend.putheader(name, value) + backend.putheader("Connection", "close") + backend.endheaders() + remaining = length + while remaining: + chunk = self.rfile.read(min(1 << 20, remaining)) + if not chunk: + raise ConnectionError("client closed before request body completed") + backend.send(chunk) + remaining -= len(chunk) + response = backend.getresponse() + if self.command == "PUT" and 200 <= response.status < 300: + self.server.record_put(self.path) + elif self.command == "DELETE" and 200 <= response.status < 300: + self.server.record_delete(self.path) + self.send_response(response.status, response.reason) + for name, value in response.getheaders(): + if name.lower() not in HOP_BY_HOP_HEADERS: + self.send_header(name, value) + self.send_header("Connection", "close") + self.end_headers() + if self.command != "HEAD": + while True: + chunk = response.read(1 << 20) + if not chunk: + break + self.wfile.write(chunk) + self.close_connection = True + except Exception as exc: + if not self.wfile.closed: + try: + self.send_error(502, f"S3 counting proxy failure: {exc}") + except (BrokenPipeError, ConnectionError): + pass + finally: + backend.close() + + +@dataclasses.dataclass +class LiveConfig: + repo_root: Path + cargo_bin: Path + nokv_bin: Path + lingtai_kernel_dir: Path + state_dir: Path + server_port: int + rustfs_port: int + rustfs_console_port: int + proxy_port: int + rustfs_image: str + bucket: str + container: str + profile: WorkloadProfile + command_timeout: float + tool_timeout: float + startup_timeout: float + gc_deadline: float + build: bool + keep_state: bool + require_all: bool + + +class LiveEnvironment: + def __init__(self, config: LiveConfig) -> None: + self.config = config + self.server_bind = f"127.0.0.1:{config.server_port}" + self.rustfs_endpoint = f"http://127.0.0.1:{config.rustfs_port}" + self.s3_endpoint = f"http://127.0.0.1:{config.proxy_port}" + self.server_log = config.state_dir / "nokv-server.log" + self.restore_barrier_dir = config.state_dir / "restore-barriers" + self._server: subprocess.Popen[str] | None = None + self._server_log_handle: Any = None + self._container_started = False + self._proxy: CountingProxyServer | None = None + self._proxy_thread: threading.Thread | None = None + self.nokv_binary_sha256 = "" + self.repo_revision = "" + self.lingtai_revision = "" + + @property + def aws_env(self) -> dict[str, str]: + env = os.environ.copy() + env.update( + { + "AWS_ACCESS_KEY_ID": "rustfsadmin", + "AWS_SECRET_ACCESS_KEY": "rustfsadmin", + "AWS_DEFAULT_REGION": "us-east-1", + "AWS_EC2_METADATA_DISABLED": "true", + "NOKV_TEST_RESTORE_BARRIER_DIR": str(self.restore_barrier_dir), + "NOKV_TEST_BARRIER_TIMEOUT_MS": str( + max(60_000, int(self.config.tool_timeout * 1000)) + ), + } + ) + return env + + def common_nokv_args(self) -> list[str]: + return [ + "--server-bind", + self.server_bind, + "--object-backend", + "rustfs", + "--s3-endpoint", + self.s3_endpoint, + "--s3-bucket", + self.config.bucket, + "--s3-access-key-id", + "rustfsadmin", + "--s3-secret-access-key", + "rustfsadmin", + "--no-metadata-checkpoint-archive", + ] + + def mcp_args(self, root: str) -> list[str]: + return self.common_nokv_args() + [ + "mcp", + "--profile", + "workbench", + "--workbench-root", + root, + ] + + def run( + self, + args: list[str], + *, + timeout: float | None = None, + env: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + try: + result = subprocess.run( + args, + cwd=self.config.repo_root, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout or self.config.command_timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise AcceptanceError(f"command timed out: {args!r}") from exc + if result.returncode != 0: + raise AcceptanceError( + f"command failed ({result.returncode}): {args!r}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result + + def build(self) -> None: + if self.config.build: + self.run( + [ + str(self.config.cargo_bin), + "build", + "-p", + "nokv", + "--bin", + "nokv", + "--target-dir", + str(self.config.repo_root / "target"), + ], + timeout=max(1200, self.config.startup_timeout), + ) + if not self.config.nokv_bin.is_file(): + raise AcceptanceError(f"NoKV binary not found: {self.config.nokv_bin}") + if not os.access(self.config.nokv_bin, os.X_OK): + raise AcceptanceError( + f"NoKV binary is not executable: {self.config.nokv_bin}" + ) + self.nokv_binary_sha256 = sha256_file(self.config.nokv_bin) + self.repo_revision = self.git_revision(self.config.repo_root) + self.lingtai_revision = self.git_revision(self.config.lingtai_kernel_dir) + + def git_revision(self, directory: Path) -> str: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=directory, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=30, + check=False, + ) + if result.returncode != 0: + raise AcceptanceError( + f"failed to resolve git revision for {directory}: {result.stderr}" + ) + revision = result.stdout.strip() + if len(revision) != 40 or any( + character not in "0123456789abcdef" for character in revision + ): + raise AcceptanceError(f"invalid git revision for {directory}: {revision!r}") + return revision + + def assert_binary_unchanged(self) -> None: + current = sha256_file(self.config.nokv_bin) + if not self.nokv_binary_sha256 or current != self.nokv_binary_sha256: + raise AcceptanceError( + "NoKV binary changed during acceptance: " + f"started={self.nokv_binary_sha256!r}, current={current!r}" + ) + + def start_rustfs(self) -> None: + env = self.aws_env + env.update( + { + "LINGTAI_WORKBENCH_DATA_ROOT": str(self.config.state_dir), + "LINGTAI_WORKBENCH_RUSTFS_CONTAINER": self.config.container, + "LINGTAI_WORKBENCH_RUSTFS_IMAGE": self.config.rustfs_image, + "LINGTAI_WORKBENCH_RUSTFS_PORT": str(self.config.rustfs_port), + "LINGTAI_WORKBENCH_RUSTFS_CONSOLE_PORT": str( + self.config.rustfs_console_port + ), + "LINGTAI_WORKBENCH_S3_ENDPOINT": self.rustfs_endpoint, + "LINGTAI_WORKBENCH_S3_BUCKET": self.config.bucket, + "LINGTAI_WORKBENCH_RUSTFS_DATA_DIR": str( + self.config.state_dir / "rustfs" + ), + } + ) + self.run( + [str(self.config.repo_root / "scripts/lingtai-workbench/start_rustfs.sh")], + timeout=self.config.startup_timeout, + env=env, + ) + self._container_started = True + + def start_proxy(self) -> None: + if self._proxy is not None: + raise AcceptanceError("S3 proxy is already running") + self._proxy = CountingProxyServer( + ("127.0.0.1", self.config.proxy_port), self.config.rustfs_port + ) + self._proxy_thread = threading.Thread( + target=self._proxy.serve_forever, name="s3-counting-proxy", daemon=True + ) + self._proxy_thread.start() + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + try: + self.run( + [ + "aws", + "--endpoint-url", + self.s3_endpoint, + "s3api", + "head-bucket", + "--bucket", + self.config.bucket, + ], + timeout=2, + env=self.aws_env, + ) + return + except AcceptanceError: + time.sleep(0.05) + raise AcceptanceError("S3 counting proxy did not become ready") + + def successful_puts(self) -> list[str]: + if self._proxy is None: + raise AcceptanceError("S3 proxy is not running") + return self._proxy.put_records() + + def successful_mutations(self) -> list[ObjectMutation]: + if self._proxy is None: + raise AcceptanceError("S3 proxy is not running") + return self._proxy.mutation_records() + + def start_server(self) -> None: + if self._server is not None: + raise AcceptanceError("NoKV server is already running") + self._server_log_handle = self.server_log.open("a", encoding="utf-8") + args = ( + [str(self.config.nokv_bin)] + + self.common_nokv_args() + + [ + "--meta", + str(self.config.state_dir / "meta"), + "--object-gc-interval-ms", + str(BACKGROUND_GC_INTERVAL_MS), + "--object-gc-limit", + "4096", + "--history-gc-interval-ms", + str(BACKGROUND_GC_INTERVAL_MS), + "--history-gc-limit", + "4096", + "serve", + ] + ) + self._server = subprocess.Popen( + args, + cwd=self.config.repo_root, + env=self.aws_env, + text=True, + stdout=self._server_log_handle, + stderr=subprocess.STDOUT, + ) + deadline = time.monotonic() + self.config.startup_timeout + while time.monotonic() < deadline: + if self._server.poll() is not None: + raise AcceptanceError( + f"NoKV server exited during startup\n{tail(self.server_log)}" + ) + try: + if self.http_text("/readyz", timeout=1).strip() == "ready": + return + except (OSError, urllib.error.URLError): + pass + time.sleep(0.1) + raise AcceptanceError(f"NoKV server startup timed out\n{tail(self.server_log)}") + + def stop_server(self, *, kill: bool = False) -> None: + process = self._server + self._server = None + if process is not None and process.poll() is None: + if kill: + process.kill() + else: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + if self._server_log_handle is not None: + self._server_log_handle.close() + self._server_log_handle = None + + def cleanup(self) -> None: + self.stop_server() + if self._proxy is not None: + self._proxy.shutdown() + self._proxy.server_close() + self._proxy = None + if self._proxy_thread is not None: + self._proxy_thread.join(timeout=5) + self._proxy_thread = None + if self._container_started or shutil.which("docker"): + subprocess.run( + ["docker", "rm", "-f", self.config.container], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + check=False, + ) + if not self.config.keep_state: + shutil.rmtree(self.config.state_dir, ignore_errors=True) + + def cli(self, *command: str) -> str: + return self.run( + [str(self.config.nokv_bin)] + self.common_nokv_args() + list(command), + env=self.aws_env, + ).stdout + + def hash_remote_file(self, path: str) -> tuple[str, int]: + args = [str(self.config.nokv_bin)] + self.common_nokv_args() + ["cat", path] + process = subprocess.Popen( + args, + cwd=self.config.repo_root, + env=self.aws_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert process.stdout is not None + digest = hashlib.sha256() + size = 0 + deadline = time.monotonic() + max(self.config.command_timeout, 900) + while True: + if time.monotonic() >= deadline: + process.kill() + raise AcceptanceError(f"hashing {path} exceeded deadline") + chunk = process.stdout.read(8 << 20) + if not chunk: + break + digest.update(chunk) + size += len(chunk) + stderr = ( + process.stderr.read().decode("utf-8", errors="replace") + if process.stderr + else "" + ) + if process.wait(timeout=10) != 0: + raise AcceptanceError(f"cat {path} failed: {stderr}") + return digest.hexdigest(), size + + def inventory(self) -> dict[str, ObjectFingerprint]: + result = self.run( + [ + "aws", + "--endpoint-url", + self.s3_endpoint, + "s3api", + "list-objects-v2", + "--bucket", + self.config.bucket, + "--output", + "json", + ], + env=self.aws_env, + ) + payload = json.loads(result.stdout) + inventory: dict[str, ObjectFingerprint] = {} + for item in payload.get("Contents", []): + inventory[item["Key"]] = ObjectFingerprint( + size=int(item["Size"]), + etag=str(item["ETag"]), + last_modified=str(item["LastModified"]), + ) + return inventory + + def http_text(self, path: str, *, timeout: float = 5, method: str = "GET") -> str: + request = urllib.request.Request( + f"http://{self.server_bind}{path}", method=method + ) + started = time.monotonic() + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read().decode("utf-8") + except TimeoutError as error: + elapsed = time.monotonic() - started + raise AcceptanceError( + f"NoKV {method} {path} timed out after {elapsed:.2f}s " + f"(limit={timeout:.2f}s)" + ) from error + except urllib.error.HTTPError as error: + body = error.read().decode("utf-8", errors="replace") + raise AcceptanceError( + f"NoKV {method} {path} returned HTTP {error.code}: {body}" + ) from error + + def manual_gc(self, *, limit: int = 64) -> dict[str, Any]: + if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 4096: + raise AcceptanceError(f"manual GC limit is invalid: {limit!r}") + raw = self.http_text(f"/gc?limit={limit}", timeout=60, method="POST") + value = json.loads(raw) + if not isinstance(value, dict): + raise AcceptanceError(f"manual GC returned non-object: {value!r}") + return value + + def stats(self) -> dict[str, Any]: + value = json.loads(self.http_text("/stats", timeout=60)) + if not isinstance(value, dict): + raise AcceptanceError(f"stats returned non-object: {value!r}") + return value + + def fsck(self) -> dict[str, Any]: + deadline = time.monotonic() + min( + self.config.tool_timeout, self.config.gc_deadline + ) + while True: + try: + value = json.loads(self.http_text("/fsck", timeout=300)) + except AcceptanceError as exc: + if ( + RETRYABLE_FSCK_CONFLICT not in str(exc) + or time.monotonic() >= deadline + ): + raise + time.sleep(0.05) + continue + if not isinstance(value, dict): + raise AcceptanceError(f"fsck returned non-object: {value!r}") + return value + + def raw_mcp_tools(self, root: str) -> tuple[list[dict[str, Any]], str]: + request = json.dumps( + {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}, + separators=(",", ":"), + ) + try: + result = subprocess.run( + [str(self.config.nokv_bin), *self.mcp_args(root)], + cwd=self.config.repo_root, + env=self.aws_env, + input=request + "\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=min(60, self.config.tool_timeout), + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise AcceptanceError("raw MCP tools/list preflight timed out") from exc + if result.returncode != 0: + raise AcceptanceError( + "raw MCP tools/list preflight failed: " + f"stdout={result.stdout!r}, stderr={result.stderr!r}" + ) + lines = [line for line in result.stdout.splitlines() if line.strip()] + if len(lines) != 1: + raise AcceptanceError( + f"raw MCP tools/list returned {len(lines)} responses: {lines!r}" + ) + try: + response = json.loads(lines[0]) + except json.JSONDecodeError as exc: + raise AcceptanceError( + f"raw MCP tools/list returned invalid JSON: {lines[0]!r}" + ) from exc + if ( + not isinstance(response, dict) + or response.get("id") != 1 + or "error" in response + ): + raise AcceptanceError(f"raw MCP tools/list protocol failure: {response!r}") + raw_tools = response.get("result", {}).get("tools") + if not isinstance(raw_tools, list): + raise AcceptanceError(f"raw MCP tools/list lacks tools: {response!r}") + tools: list[dict[str, Any]] = [] + for raw in raw_tools: + if not isinstance(raw, dict) or not isinstance( + raw.get("inputSchema"), dict + ): + raise AcceptanceError(f"raw MCP tool lacks inputSchema: {raw!r}") + tools.append( + { + "name": raw.get("name"), + "description": raw.get("description"), + "schema": raw["inputSchema"], + } + ) + canonical = json.dumps(raw_tools, separators=(",", ":"), sort_keys=True) + return tools, hashlib.sha256(canonical.encode()).hexdigest() + + +class WorkbenchClient: + def __init__(self, client_class: type, launch: McpLaunch, timeout: float) -> None: + self.root = launch.root + self.timeout = timeout + self._client = client_class( + command=launch.command, args=launch.args, env=launch.env + ) + self._client.start() + + def close(self) -> None: + self._client.close() + + def tools(self) -> list[dict[str, Any]]: + return self._client.list_tools(timeout=min(30, self.timeout)) + + def raw_call(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = self._client.call_tool(name, arguments, timeout=self.timeout) + if not isinstance(result, dict): + raise AcceptanceError(f"{name} returned non-object: {result!r}") + return result + + def call(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = self.raw_call(name, arguments) + if result.get("status") == "error" or "code" in result: + raise AcceptanceError(f"{name} failed: {result!r}") + if result.get("status") != "success": + raise AcceptanceError(f"{name} returned malformed success: {result!r}") + return result + + +class AcceptanceSuite: + def __init__( + self, env: LiveEnvironment, agent_class: type, client_class: type + ) -> None: + self.env = env + self.agent_class = agent_class + self.client_class = client_class + self.clients: list[WorkbenchClient] = [] + self.root = "" + self.results: dict[str, dict[str, Any]] = {} + self.source = "durable-source" + self.destination = "durable-fork" + self.moved = "durable-fork-moved" + self.nested = "durable-nested" + self.replaced = "durable-replaced" + self.snapshot_id = 0 + self.snapshot_version = 0 + self.operation_id = "" + self.nested_snapshot_id = 0 + self.nested_operation_id = "" + self.expected_digest = "" + self.fork_owned_keys: set[str] = set() + self.large_object_keys: set[str] = set() + self.restore_manifest_keys: set[str] = set() + self.snapshot_deferred_release_keys: set[str] = set() + self.initial_inventory: dict[str, ObjectFingerprint] = {} + self.baseline_index: dict[str, Any] = {} + self.mutated_index: dict[str, Any] = {} + self.launch: McpLaunch | None = None + self.fixture_block_count = 0 + self.fixture_marker_digest = "" + self.raw_preflight_schema_sha256 = "" + self.restore_control_baseline: dict[str, int] = {} + self.registration_generation = 0 + self.barriers = RestoreBarrierController(env.restore_barrier_dir) + self.barriers.clear() + + @property + def client_a(self) -> WorkbenchClient: + return self.clients[0] + + @property + def client_b(self) -> WorkbenchClient: + return self.clients[1] + + def scenario(self, name: str, function: Callable[[], dict[str, Any]]) -> None: + print(f"[restore-live-e2e] START {name}", flush=True) + started = time.monotonic() + try: + details = function() + except Exception as exc: + self.results[name] = { + "status": "failed", + "duration_seconds": round(time.monotonic() - started, 3), + "details": {"error": str(exc)}, + } + print(f"[restore-live-e2e] FAIL {name}: {exc}", flush=True) + raise + self.results[name] = { + "status": "passed", + "duration_seconds": round(time.monotonic() - started, 3), + "details": details, + } + print(f"[restore-live-e2e] PASS {name}", flush=True) + + @staticmethod + def mock_service() -> MagicMock: + service = MagicMock() + service.get_adapter.return_value = MagicMock() + service.provider = "gemini" + service.model = "nokv-restore-live-e2e" + return service + + @staticmethod + def resolved_launch(client: Any) -> McpLaunch: + command = getattr(client, "_command", None) + args = getattr(client, "_args", None) + env = getattr(client, "_env", None) + if not isinstance(command, str) or not isinstance(args, list): + raise AcceptanceError("LingTai MCP client lacks resolved launch config") + if not isinstance(env, dict): + raise AcceptanceError("LingTai MCP client lacks resolved environment") + try: + root = args[args.index("--workbench-root") + 1] + except (ValueError, IndexError) as exc: + raise AcceptanceError("resolved MCP args lack workbench root") from exc + if not isinstance(root, str) or not root.startswith("/") or "{" in root: + raise AcceptanceError(f"LingTai left an unresolved root: {root!r}") + return McpLaunch(command, list(args), dict(env), root) + + def registered_launch(self) -> McpLaunch: + if not self.raw_preflight_schema_sha256: + preflight_root = "/agents/deployment-preflight/wb" + preflight_launch = McpLaunch( + str(self.env.config.nokv_bin), + self.env.mcp_args(preflight_root), + self.env.aws_env, + preflight_root, + ) + self.validate_launch(preflight_launch) + raw_tools, schema_digest = self.env.raw_mcp_tools(preflight_root) + validate_tool_contract(raw_tools) + self.raw_preflight_schema_sha256 = schema_digest + agent_name = "durable-restore-agent" + workdir = self.env.config.state_dir / "agents" / agent_name + workdir.mkdir(parents=True, exist_ok=True) + registry = { + "name": "nokv-workbench", + "summary": "NoKV durable restore live E2E.", + "transport": "stdio", + "command": str(self.env.config.nokv_bin), + "args": self.env.mcp_args(WORKBENCH_ROOT_TEMPLATE), + "source": "durable-restore-live-e2e", + } + (workdir / "mcp_registry.jsonl").write_text( + json.dumps(registry, separators=(",", ":")) + "\n", encoding="utf-8" + ) + (workdir / "init.json").write_text( + json.dumps( + { + "mcp": { + "nokv-workbench": { + "type": "stdio", + "command": str(self.env.config.nokv_bin), + "args": self.env.mcp_args(WORKBENCH_ROOT_TEMPLATE), + "env": self.env.aws_env, + } + } + }, + separators=(",", ":"), + ), + encoding="utf-8", + ) + agent: Any | None = None + try: + agent = self.agent_class( + service=self.mock_service(), + agent_name=agent_name, + working_dir=workdir, + capabilities={"mcp": {}}, + ) + spec = agent._mcp_init_specs.get("nokv-workbench") + initial = spec.get("client") if isinstance(spec, dict) else None + if initial is None: + raise AcceptanceError("LingTai Agent did not register NoKV MCP") + before = self.resolved_launch(initial) + self.validate_launch(before) + validate_tool_contract(initial.list_tools(timeout=30)) + initial.close() + retry = agent._retry_failed_mcps() + if retry.get("recovered") != ["nokv-workbench"]: + raise AcceptanceError(f"Agent MCP reconnect failed: {retry!r}") + retried = agent._mcp_init_specs["nokv-workbench"].get("client") + if retried is None: + raise AcceptanceError("Agent reconnect produced no MCP client") + after = self.resolved_launch(retried) + self.validate_launch(after) + validate_tool_contract(retried.list_tools(timeout=30)) + if before.root != after.root: + raise AcceptanceError("workbench root changed across Agent reconnect") + handler = getattr(agent, "_tool_handlers", {}).get("workbench_create") + if not callable(handler): + raise AcceptanceError("Agent did not install workbench tool handler") + self.registration_generation += 1 + probe = handler( + {"id": f"agent-registration-probe-{self.registration_generation}"} + ) + if not isinstance(probe, dict) or probe.get("status") != "success": + raise AcceptanceError(f"Agent MCP handler failed: {probe!r}") + if not callable(getattr(agent, "_tool_handlers", {}).get(RESTORE_TOOL)): + raise AcceptanceError("Agent did not install workbench_restore handler") + return after + finally: + if agent is not None: + agent.stop(timeout=5.0) + + def validate_launch(self, launch: McpLaunch) -> None: + expected_binary = self.env.config.nokv_bin.resolve() + actual_binary = Path(launch.command).expanduser().resolve() + if actual_binary != expected_binary: + raise AcceptanceError( + f"LingTai launched stale/wrong NoKV binary: {actual_binary} != {expected_binary}" + ) + self.env.assert_binary_unchanged() + if sha256_file(actual_binary) != self.env.nokv_binary_sha256: + raise AcceptanceError( + "LingTai launch binary hash differs from the built binary" + ) + expected_args = self.env.mcp_args(launch.root) + if launch.args != expected_args: + raise AcceptanceError( + "LingTai resolved MCP launch differs from deployment contract: " + f"actual={launch.args!r}, expected={expected_args!r}" + ) + if any("{" in argument or "}" in argument for argument in launch.args): + raise AcceptanceError( + f"LingTai launch retained a placeholder: {launch.args!r}" + ) + for name, expected in self.env.aws_env.items(): + if ( + name.startswith(("AWS_", "NOKV_TEST_")) + and launch.env.get(name) != expected + ): + raise AcceptanceError( + f"LingTai launch environment changed {name}: {launch.env.get(name)!r}" + ) + + def connect_clients(self) -> None: + launch = self.registered_launch() + if self.root and self.root != launch.root: + raise AcceptanceError("Agent workbench root changed after restart") + self.root = launch.root + self.launch = launch + self.clients = [ + WorkbenchClient(self.client_class, launch, self.env.config.tool_timeout), + WorkbenchClient(self.client_class, launch, self.env.config.tool_timeout), + ] + + def agent_restore_redrive(self, conflicting_snapshot_id: int) -> dict[str, Any]: + """Use the Agent handler to redrive and preserve a native MCP error.""" + agent_name = "durable-restore-agent" + workdir = self.env.config.state_dir / "agents" / agent_name + request = self.restore_request(self.destination) + canonical_request = json.dumps(request, separators=(",", ":"), sort_keys=True) + agent: Any | None = None + try: + agent = self.agent_class( + service=self.mock_service(), + agent_name=agent_name, + working_dir=workdir, + capabilities={"mcp": {}}, + ) + initial_client = agent._mcp_init_specs["nokv-workbench"].get("client") + if initial_client is None: + raise AcceptanceError("Agent redrive has no initial NoKV MCP client") + initial_launch = self.resolved_launch(initial_client) + self.validate_launch(initial_launch) + validate_tool_contract(initial_client.list_tools(timeout=30)) + handler = getattr(agent, "_tool_handlers", {}).get(RESTORE_TOOL) + if not callable(handler): + raise AcceptanceError("Agent redrive lacks workbench_restore handler") + first = handler(dict(request)) + self.assert_complete_outcome(first, self.destination, self.operation_id) + initial_client.close() + retry = agent._retry_failed_mcps() + if retry.get("recovered") != ["nokv-workbench"]: + raise AcceptanceError( + f"Agent restore redrive reconnect failed: {retry!r}" + ) + retried_client = agent._mcp_init_specs["nokv-workbench"].get("client") + if retried_client is None: + raise AcceptanceError("Agent restore redrive produced no MCP client") + retried_launch = self.resolved_launch(retried_client) + self.validate_launch(retried_launch) + if retried_launch != initial_launch: + raise AcceptanceError( + "Agent restore redrive changed the resolved MCP launch" + ) + validate_tool_contract(retried_client.list_tools(timeout=30)) + retried_handler = getattr(agent, "_tool_handlers", {}).get(RESTORE_TOOL) + if not callable(retried_handler): + raise AcceptanceError( + "Agent lost workbench_restore handler after reconnect" + ) + second_request = json.loads(canonical_request) + if ( + json.dumps(second_request, separators=(",", ":"), sort_keys=True) + != canonical_request + ): + raise AcceptanceError( + "Agent redrive changed the persisted restore request" + ) + second = retried_handler(second_request) + self.assert_complete_outcome(second, self.destination, self.operation_id) + if first != second: + raise AcceptanceError( + f"Agent exact terminal redrive changed outcome: {first!r} != {second!r}" + ) + native_error = assert_native_tool_error( + retried_handler( + { + "id": self.source, + "at_snapshot": conflicting_snapshot_id, + "destination_id": self.destination, + } + ), + "RestoreDestinationConflict", + ) + return { + "operation_id": self.operation_id, + "snapshot_id": self.snapshot_id, + "request": second_request, + "reconnect": retry, + "native_structured_error": { + "code": native_error.code, + "message": native_error.message, + "retryable": native_error.retryable, + "details": native_error.details, + }, + } + finally: + if agent is not None: + agent.stop(timeout=5.0) + + def close_clients(self) -> None: + for client in self.clients: + try: + client.close() + except Exception: + pass + self.clients = [] + + def reconnect_after_server_kill(self) -> None: + self.env.stop_server(kill=True) + self.close_clients() + self.env.start_server() + self.connect_clients() + + def absolute( + self, workbench_id: str, section: str | None = None, path: str = "" + ) -> str: + value = f"{self.root}/{workbench_id}" + if section is not None: + value += f"/{section}" + if path: + value += f"/{path}" + return value + + def restore_request(self, destination_id: str) -> dict[str, Any]: + return { + "id": self.source, + "at_snapshot": self.snapshot_id, + "destination_id": destination_id, + } + + def expected_restore_operation_id(self, destination_id: str) -> str: + return restore_operation_id( + DEFAULT_LIVE_MOUNT_ID, + self.absolute(self.source), + self.snapshot_id, + self.absolute(destination_id), + ) + + def assert_complete_outcome( + self, + outcome: dict[str, Any], + destination_id: str, + operation_id: str, + ) -> None: + if ( + outcome.get("status") != "success" + or outcome.get("state") != "complete" + or outcome.get("operation_id") != operation_id + or outcome.get("source_workbench_id") != self.source + or outcome.get("destination_workbench_id") != destination_id + or outcome.get("snapshot_id") != self.snapshot_id + or outcome.get("read_version") != self.snapshot_version + or outcome.get("cleanup_pending") is not False + or not isinstance(outcome.get("source_root"), int) + or not isinstance(outcome.get("destination_root"), int) + ): + raise AcceptanceError(f"malformed durable restore outcome: {outcome!r}") + + def retry_restore_until_complete( + self, destination_id: str, operation_id: str + ) -> tuple[dict[str, Any], int]: + request = self.restore_request(destination_id) + deadline = time.monotonic() + self.env.config.tool_timeout + attempts = 0 + while time.monotonic() < deadline: + attempts += 1 + outcome = self.client_a.raw_call(RESTORE_TOOL, request) + if outcome.get("status") == "success": + self.assert_complete_outcome(outcome, destination_id, operation_id) + return outcome, attempts + error = decode_tool_error(outcome) + if error.code != "RestoreInProgress" or not error.retryable: + raise AcceptanceError( + "exact restore retry returned a non-retryable intermediate error: " + f"{outcome!r}" + ) + time.sleep(0.02) + raise AcceptanceError( + f"exact restore retry did not complete before deadline: {operation_id}" + ) + + def wait_for_restore_error( + self, destination_id: str, expected_code: str + ) -> tuple[ToolError, int]: + request = self.restore_request(destination_id) + deadline = time.monotonic() + self.env.config.tool_timeout + attempts = 0 + release_drained = False + while time.monotonic() < deadline: + attempts += 1 + outcome = self.client_a.raw_call(RESTORE_TOOL, request) + if outcome.get("status") == "success": + raise AcceptanceError( + "released restore operation unexpectedly remained terminal: " + f"{outcome!r}" + ) + error = decode_tool_error(outcome) + if error.code == expected_code: + return error, attempts + if error.code != "RestoreInProgress" or not error.retryable: + raise AcceptanceError( + f"waiting for {expected_code} observed {outcome!r}" + ) + if release_drained: + raise AcceptanceError( + "restore remained in progress after the release backlog drained: " + f"{outcome!r}" + ) + # The release worker is deliberately paged and passive workers stay + # outside this acceptance window. A full restore subtree can + # require hundreds of durable member/index subpages, so drive one + # bounded worker page per exact retry. + gc_outcome = self.env.manual_gc(limit=64) + object_gc = gc_outcome.get("object_gc") + if not isinstance(object_gc, dict) or any( + not isinstance(object_gc.get(field), int) + for field in ( + "restore_release_jobs_processed", + "restore_release_backlog", + "restore_release_quarantine", + "restore_release_mount_wide_quarantine", + ) + ): + raise AcceptanceError( + f"manual release GC returned malformed evidence: {gc_outcome!r}" + ) + if ( + object_gc["restore_release_quarantine"] != 0 + or object_gc["restore_release_mount_wide_quarantine"] != 0 + ): + raise AcceptanceError( + f"manual release GC quarantined durable state: {gc_outcome!r}" + ) + release_drained = object_gc["restore_release_backlog"] == 0 + time.sleep(0.02) + raise AcceptanceError( + f"restore did not converge to {expected_code}: {destination_id}" + ) + + def crash_at_restore_barrier( + self, + operation_id: str, + phase: str, + operation: Callable[[], Any], + *, + require_interrupted_result: bool, + allow_missing_phase: bool = False, + ) -> dict[str, Any]: + handle = self.barriers.handle(operation_id, phase) + handle.arm() + executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) + future = executor.submit(operation) + server_crashed = False + try: + try: + reached = handle.wait_ready_or_done( + future, self.env.config.tool_timeout + ) + except Exception as barrier_error: + handle.release() + try: + observation: Any = future.result(timeout=30) + except Exception as operation_error: + observation = f"{type(operation_error).__name__}: {operation_error}" + raise AcceptanceError( + f"restore crash phase {phase!r} was not reached; " + f"operation observation={observation!r}" + ) from barrier_error + + if not reached: + result = future.result(timeout=30) + if not allow_missing_phase: + raise AcceptanceError( + f"restore crash phase {phase!r} was not reached; " + f"operation completed as {result!r}" + ) + return {"kind": "phase-absent", "value": result} + + self.env.stop_server(kill=True) + server_crashed = True + handle.disarm_after_crash() + try: + result = future.result(timeout=min(30, self.env.config.tool_timeout)) + observation = {"kind": "result", "value": result} + except concurrent.futures.TimeoutError: + self.close_clients() + try: + result = future.result(timeout=10) + observation = {"kind": "result", "value": result} + except Exception as exc: + observation = { + "kind": "exception", + "value": f"{type(exc).__name__}: {exc}", + } + except Exception as exc: + observation = { + "kind": "exception", + "value": f"{type(exc).__name__}: {exc}", + } + finally: + handle.disarm_after_crash() + executor.shutdown(wait=True, cancel_futures=True) + + if server_crashed: + self.close_clients() + self.env.start_server() + self.connect_clients() + value = observation.get("value") + if ( + require_interrupted_result + and observation.get("kind") == "result" + and isinstance(value, dict) + and value.get("status") == "success" + ): + raise AcceptanceError( + f"restore crash phase {phase!r} returned success before SIGKILL" + ) + return observation + + def put_text( + self, + workbench_id: str, + path: str, + text: str, + *, + content_type: str = "text/plain", + replace: bool = False, + ) -> dict[str, Any]: + return self.client_a.call( + "workbench_put_file", + { + "id": workbench_id, + "section": "outputs", + "path": path, + "text": text, + "content_type": content_type, + "replace": replace, + }, + ) + + def create_large_fixture(self) -> Path: + path = self.env.config.state_dir / "cow-fixture.bin" + size = self.env.config.profile.large_bytes + self.fixture_block_count = fixture_block_count(size) + marker_digest = hashlib.sha256() + with path.open("wb") as output: + for index in range(self.fixture_block_count): + marker = fixture_block_marker(index) + marker_digest.update(marker) + output.seek(index * COW_BLOCK_BYTES) + output.write(marker) + output.truncate(size) + self.fixture_marker_digest = marker_digest.hexdigest() + self.expected_digest = sha256_file(path) + return path + + def index_signature( + self, workbench_id: str, client: WorkbenchClient | None = None + ) -> dict[str, Any]: + client = client or self.client_a + search_arguments: dict[str, Any] = { + "id": workbench_id, + "section": "outputs", + "predicates": [{"field": "name", "op": "suffix", "value": ".csv"}], + "fields": ["name", "body.content_type"], + "limit": INDEX_PAGE_LIMIT, + } + matches: list[dict[str, Any]] = [] + cursor: str | None = None + pages = 0 + while True: + if cursor is not None: + search_arguments["cursor"] = cursor + search = client.call("workbench_search", search_arguments) + page_matches = search.get("matches") + if not isinstance(page_matches, list): + raise AcceptanceError(f"search lacks matches: {search!r}") + for match_ in page_matches: + if not isinstance(match_, dict): + raise AcceptanceError( + f"search returned malformed match: {match_!r}" + ) + matches.append(match_) + pages += 1 + if search.get("truncated") is not True: + if search.get("next_cursor") is not None: + raise AcceptanceError( + "untruncated search unexpectedly returned a cursor" + ) + break + cursor = search.get("next_cursor") + if not isinstance(cursor, str) or not cursor: + raise AcceptanceError("truncated search lacks next_cursor") + if pages > self.env.config.profile.indexed_files + 16: + raise AcceptanceError("search pagination did not converge") + paths = [match_.get("path") for match_ in matches] + if not all(isinstance(path, str) for path in paths) or len(set(paths)) != len( + paths + ): + raise AcceptanceError( + f"search returned duplicate/malformed paths: {paths!r}" + ) + relative_paths = [match_.get("relative_path") for match_ in matches] + if not all(isinstance(path, str) for path in relative_paths) or len( + set(relative_paths) + ) != len(relative_paths): + raise AcceptanceError( + f"search returned duplicate/malformed relative paths: {relative_paths!r}" + ) + for match_ in matches: + if ( + match_.get("workbench_id") != workbench_id + or match_.get("section") != "outputs" + ): + raise AcceptanceError(f"search escaped workbench scope: {match_!r}") + aggregate = client.call( + "workbench_aggregate", + { + "id": workbench_id, + "section": "outputs", + "predicates": [{"field": "kind", "op": "eq", "value": "file"}], + "measures": [{"name": "files", "op": "count"}], + }, + ) + catalog = client.call("workbench_catalog", {"id": workbench_id}) + groups = aggregate.get("groups") + if not isinstance(groups, list) or not groups: + raise AcceptanceError(f"aggregate returned no groups: {aggregate!r}") + filterable = catalog.get("catalog", {}).get("filterable") + if not isinstance(filterable, list): + raise AcceptanceError(f"catalog lacks filterable fields: {catalog!r}") + fields = sorted( + field + for group in filterable + for field in group.get("fields", []) + if isinstance(field, str) + ) + return { + "csv_match_count": len(matches), + "csv_names": sorted( + match_.get("values", {}).get("name") + for match_ in matches + if isinstance(match_.get("values", {}).get("name"), str) + ), + "csv_paths": sorted( + path for path in relative_paths if isinstance(path, str) + ), + "search_pages": pages, + "file_count": groups[0].get("values", {}).get("files"), + "catalog_fields": fields, + } + + def assert_manifest( + self, + workbench_id: str, + operation_id: str, + source: str, + snapshot_id: int, + client: WorkbenchClient | None = None, + ) -> None: + client = client or self.client_b + checkpoints = client.call("workbench_snapshot_list", {"id": workbench_id}) + if checkpoints.get("checkpoint_count") != 0: + raise AcceptanceError("restored workbench inherited checkpoint aliases") + inherited = client.raw_call( + "workbench_stat", + { + "id": workbench_id, + "section": "metadata", + "path": "checkpoints.jsonl", + }, + ) + assert_tool_error(inherited, "inherited checkpoint registry") + manifest = read_json_object( + client.call( + "workbench_read", + { + "id": workbench_id, + "section": "metadata", + "path": "restore_manifest.json", + }, + ) + ) + validate_restore_manifest( + manifest, + operation_id=operation_id, + source_workbench_id=source, + source_path=self.absolute(source), + destination_workbench_id=workbench_id, + destination_path=self.absolute(workbench_id), + snapshot_id=snapshot_id, + ) + + def scenario_contract_and_fixture(self) -> dict[str, Any]: + validate_tool_contract(self.client_a.tools()) + validate_tool_contract(self.client_b.tools()) + self.initial_inventory = self.env.inventory() + if self.initial_inventory: + raise AcceptanceError( + "isolated RustFS bucket is not initially empty: " + f"{sorted(self.initial_inventory)!r}" + ) + before_source = dict(self.initial_inventory) + self.client_a.call("workbench_create", {"id": self.source}) + for index in range(self.env.config.profile.indexed_files): + self.put_text( + self.source, + f"indexed-{index:03d}.csv", + f"index,value\n{index},{index * 2}\n", + content_type="text/csv", + ) + self.put_text( + self.source, + "state.json", + json.dumps({"state": "checkpoint", "rank": 7}), + content_type="application/json", + ) + before_large = self.env.inventory() + large = self.create_large_fixture() + self.env.cli( + "put-artifact", + self.absolute(self.source, "outputs", "cow-large.bin"), + str(large), + ) + after_large = self.env.inventory() + self.large_object_keys = set(after_large) - set(before_large) + if len(self.large_object_keys) != self.fixture_block_count: + raise AcceptanceError( + "large fixture did not upload exactly one object per 4 MiB block: " + f"expected={self.fixture_block_count}, keys={sorted(self.large_object_keys)!r}" + ) + wrong_block_sizes = { + key: after_large[key].size + for key in self.large_object_keys + if after_large[key].size != COW_BLOCK_BYTES + } + if wrong_block_sizes: + raise AcceptanceError( + f"large fixture block objects have wrong sizes: {wrong_block_sizes!r}" + ) + if ( + self.env.config.profile.name == "full" + and self.fixture_block_count != FULL_COW_BLOCK_COUNT + ): + raise AcceptanceError( + f"full COW fixture has {self.fixture_block_count} blocks, expected 256" + ) + self.client_a.call( + "workbench_commit", + {"id": self.source, "manifest": {"acceptance": "durable-restore"}}, + ) + after_source = self.env.inventory() + self.fork_owned_keys = set(after_source) - set(before_source) + if not self.fork_owned_keys: + raise AcceptanceError("source fixture uploaded no RustFS objects") + snapshot = self.client_a.call( + "workbench_snapshot", + {"id": self.source, "name": "durable-point", "ttl_days": 7}, + ) + self.snapshot_id = int(snapshot["snapshot_id"]) + self.snapshot_version = int(snapshot["read_version"]) + self.baseline_index = self.index_signature(self.source) + expected_csv_names = [ + f"indexed-{index:03d}.csv" + for index in range(self.env.config.profile.indexed_files) + ] + if ( + self.baseline_index["csv_match_count"] + != self.env.config.profile.indexed_files + or self.baseline_index["csv_names"] != expected_csv_names + or len(self.baseline_index["csv_paths"]) != len(expected_csv_names) + ): + raise AcceptanceError( + f"source search index incomplete: {self.baseline_index!r}" + ) + if ( + self.env.config.profile.name == "full" + and self.baseline_index["search_pages"] < 2 + ): + raise AcceptanceError("full source index did not exercise pagination") + + invalid_requests = [ + {"id": self.source, "at_snapshot": None, "destination_id": "invalid-null"}, + {"id": self.source, "at_snapshot": "", "destination_id": "invalid-empty"}, + { + "id": self.source, + "at_snapshot": -1, + "destination_id": "invalid-negative", + }, + { + "id": self.source, + "at_snapshot": self.snapshot_id, + "destination_id": "invalid-extra", + "extra": True, + }, + { + "id": self.source, + "at_snapshot": self.snapshot_id, + "destination_id": self.source, + }, + ] + for request in invalid_requests: + assert_tool_error( + self.client_a.raw_call(RESTORE_TOOL, request), + f"strict schema {request!r}", + ) + + self.put_text( + self.source, + "indexed-000.csv", + "index,value\n0,current\n", + content_type="text/csv", + replace=True, + ) + self.env.cli("rm", self.absolute(self.source, "outputs", "indexed-001.csv")) + self.put_text(self.source, "current-only.txt", "not in checkpoint\n") + return { + "tool_count": len(self.client_a.tools()), + "snapshot_id": self.snapshot_id, + "fixture_bytes": self.env.config.profile.large_bytes, + "fixture_4mib_blocks": self.fixture_block_count, + "fixture_marker_digest": self.fixture_marker_digest, + "source_object_keys": len(self.fork_owned_keys), + "nokv_binary_sha256": self.env.nokv_binary_sha256, + "nokv_revision": self.env.repo_revision, + "lingtai_revision": self.env.lingtai_revision, + "raw_tools_schema_sha256": self.raw_preflight_schema_sha256, + } + + def exercise_create_crash_phase( + self, phase: str, destination: str, *, allow_missing_phase: bool + ) -> dict[str, Any] | None: + operation_id = self.expected_restore_operation_id(destination) + request = self.restore_request(destination) + inventory_before = self.env.inventory() + puts_before = len(self.env.successful_puts()) + mutations_before = len(self.env.successful_mutations()) + observation = self.crash_at_restore_barrier( + operation_id, + phase, + lambda request=request: self.client_a.raw_call(RESTORE_TOOL, request), + require_interrupted_result=True, + allow_missing_phase=allow_missing_phase, + ) + if observation["kind"] == "phase-absent": + completed = observation["value"] + self.assert_complete_outcome(completed, destination, operation_id) + changed = changed_objects(inventory_before, self.env.inventory()) + put_records = self.env.successful_puts()[puts_before:] + if len(changed) != 1 or len(put_records) != 1: + raise AcceptanceError( + f"batch discovery probe for {phase!r} was not a clean COW restore: " + f"changed={sorted(changed)!r}, puts={put_records!r}" + ) + self.delete_workbench(self.client_a, destination) + self.record_snapshot_deferred_release(changed) + return None + + inventory_after_crash = self.env.inventory() + crash_created = set(inventory_after_crash) - set(inventory_before) + requires_rebuild = restore_phase_requires_manifest_rebuild(phase) + if requires_rebuild and len(crash_created) != 1: + raise AcceptanceError( + f"post-manifest crash at {phase!r} must expose one old incarnation: " + f"{sorted(crash_created)!r}" + ) + if not requires_rebuild and phase != "attach-applied" and crash_created: + raise AcceptanceError( + f"pre-manifest crash at {phase!r} left object mutations: " + f"{sorted(crash_created)!r}" + ) + if phase == "attach-applied" and len(crash_created) != 1: + raise AcceptanceError( + "post-attach crash must retain exactly the committed manifest: " + f"{sorted(crash_created)!r}" + ) + + outcome, retry_attempts = self.retry_restore_until_complete( + destination, operation_id + ) + self.assert_manifest(destination, operation_id, self.source, self.snapshot_id) + if self.index_signature(destination) != self.baseline_index: + raise AcceptanceError( + f"crash recovery at {phase!r} attached an incomplete index" + ) + + inventory_after = self.env.inventory() + changed = changed_objects(inventory_before, inventory_after) + put_records = self.env.successful_puts()[puts_before:] + expected_puts = 2 if requires_rebuild else 1 + if len(changed) != 1 or len(put_records) != expected_puts: + raise AcceptanceError( + f"crash recovery at {phase!r} published the wrong manifest count: " + f"changed={sorted(changed)!r}, puts={put_records!r}" + ) + if changed & self.fork_owned_keys: + raise AcceptanceError( + f"crash recovery at {phase!r} copied/overwrote source blocks" + ) + + orphan_cleanup_sequence: list[str] = [] + if requires_rebuild: + if len(set(put_records)) != 2: + raise AcceptanceError( + f"recovery at {phase!r} reused the old incarnation key: {put_records!r}" + ) + orphan_target, rebuilt_target = put_records + orphan_key = next(iter(crash_created)) + rebuilt_key = next(iter(changed)) + if not orphan_target.endswith(f"/{orphan_key}"): + raise AcceptanceError( + "counted old PUT does not match crash inventory: " + f"target={orphan_target!r}, key={orphan_key!r}" + ) + if not rebuilt_target.endswith(f"/{rebuilt_key}"): + raise AcceptanceError( + "counted rebuilt PUT does not match final inventory: " + f"target={rebuilt_target!r}, key={rebuilt_key!r}" + ) + relevant = [ + mutation + for mutation in self.env.successful_mutations()[mutations_before:] + if mutation.target in {orphan_target, rebuilt_target} + ] + expected = [ + ObjectMutation("PUT", orphan_target), + ObjectMutation("DELETE", orphan_target), + ObjectMutation("PUT", rebuilt_target), + ] + positions: list[int] = [] + start = 0 + for expected_mutation in expected: + position = next( + ( + index + for index in range(start, len(relevant)) + if relevant[index] == expected_mutation + ), + None, + ) + if position is None: + raise AcceptanceError( + f"old PUT was not deleted before rebuild at {phase!r}: {relevant!r}" + ) + positions.append(position) + start = position + 1 + orphan_cleanup_sequence = ["PUT(old)", "DELETE(old)", "PUT(new)"] + + self.delete_workbench(self.client_a, destination) + release_gc = self.record_snapshot_deferred_release(changed) + return { + "phase": phase, + "operation_id": outcome["operation_id"], + "destination_root": outcome["destination_root"], + "interruption": observation["kind"], + "retry_attempts": retry_attempts, + "manifest_puts": len(put_records), + "orphan_cleanup_sequence": orphan_cleanup_sequence, + "release_gc": release_gc, + } + + def discover_create_batch_crashes( + self, kind: str, records: list[dict[str, Any]] + ) -> int: + for index in range(MAX_DISCOVERED_BATCH_PHASES): + phase = indexed_restore_phase(kind, index) + destination = f"crash-{kind}-{index:06d}" + record = self.exercise_create_crash_phase( + phase, destination, allow_missing_phase=True + ) + if record is None: + if index < 2: + raise AcceptanceError( + f"full fixture exercised only {index} {kind} batch(es)" + ) + return index + records.append(record) + raise AcceptanceError( + f"{kind} batch discovery exceeded {MAX_DISCOVERED_BATCH_PHASES} phases" + ) + + def scenario_create_crash_matrix(self) -> dict[str, Any]: + if self.env.config.profile.name != "full": + raise AcceptanceError("create crash matrix requires the full profile") + if self.env.config.profile.indexed_files <= 64: + raise AcceptanceError( + "full crash fixture must cross the 64-entry materialization/ref batch" + ) + + records: list[dict[str, Any]] = [] + hold = self.exercise_create_crash_phase( + "hold-applied", "crash-hold", allow_missing_phase=False + ) + assert hold is not None + records.append(hold) + materialization_batches = self.discover_create_batch_crashes( + "materialize", records + ) + for phase, suffix in ( + ("initialization-put-before-000000", "init-before"), + ("initialization-put-after-000000", "init-after"), + ("index-sealed", "index-sealed"), + ): + record = self.exercise_create_crash_phase( + phase, f"crash-{suffix}", allow_missing_phase=False + ) + assert record is not None + records.append(record) + reference_batches = self.discover_create_batch_crashes("reference", records) + for phase, suffix in ( + ("references-sealed", "references-sealed"), + ("attach-applied", "attach"), + ): + record = self.exercise_create_crash_phase( + phase, f"crash-{suffix}", allow_missing_phase=False + ) + assert record is not None + records.append(record) + expected_phases = create_crash_phases( + materialization_batches, reference_batches + ) + observed_phases = tuple(record["phase"] for record in records) + if observed_phases != expected_phases: + raise AcceptanceError( + "dynamic crash matrix is incomplete or out of order: " + f"expected={expected_phases!r}, observed={observed_phases!r}" + ) + released_graph = self.private_graph_evidence( + expected_complete=0, + expected_snapshot_pins=1, + expect_empty=True, + ) + self.restore_control_baseline = dict(released_graph["control_rows"]) + return { + "phase_count": len(records), + "materialization_batches": materialization_batches, + "reference_batches": reference_batches, + "phases": records, + "released_graph": released_graph, + "durable_ledger_baseline": self.restore_control_baseline, + } + + def scenario_cleanup_release_crash_recovery(self) -> dict[str, Any]: + if self.env.config.profile.name != "full": + raise AcceptanceError( + "cleanup/release crash recovery requires full profile" + ) + + cleanup_destination = "crash-cleanup" + cleanup_operation_id = self.expected_restore_operation_id(cleanup_destination) + cleanup_request = self.restore_request(cleanup_destination) + cleanup_inventory_before = self.env.inventory() + cleanup_puts_before = len(self.env.successful_puts()) + preparing_observation = self.crash_at_restore_barrier( + cleanup_operation_id, + "materialize-batch-000000", + lambda: self.client_a.raw_call(RESTORE_TOOL, cleanup_request), + require_interrupted_result=True, + ) + cleanup_observation = self.crash_at_restore_barrier( + cleanup_operation_id, + CLEANUP_CRASH_PHASE, + lambda: self.client_a.raw_call(RESTORE_TOOL, cleanup_request), + require_interrupted_result=True, + ) + cleanup_outcome, cleanup_retry_attempts = self.retry_restore_until_complete( + cleanup_destination, cleanup_operation_id + ) + self.assert_manifest( + cleanup_destination, + cleanup_operation_id, + self.source, + self.snapshot_id, + ) + if self.index_signature(cleanup_destination) != self.baseline_index: + raise AcceptanceError("cleanup/rebuild restored an incomplete index") + cleanup_changed = changed_objects( + cleanup_inventory_before, self.env.inventory() + ) + cleanup_puts = self.env.successful_puts()[cleanup_puts_before:] + if len(cleanup_changed) != 1 or len(cleanup_puts) != 1: + raise AcceptanceError( + "cleanup/rebuild must publish one deterministic manifest: " + f"changed={sorted(cleanup_changed)!r}, puts={cleanup_puts!r}" + ) + self.delete_workbench(self.client_a, cleanup_destination) + cleanup_release_gc = self.record_snapshot_deferred_release(cleanup_changed) + + replacement = "crash-release-replacement" + release_destination = "crash-release" + replacement_inventory_before = self.env.inventory() + self.client_a.call("workbench_create", {"id": replacement}) + self.put_text(replacement, "replacement.txt", "replacement survived\n") + self.client_a.call( + "workbench_commit", + {"id": replacement, "manifest": {"release_crash": True}}, + ) + replacement_keys = set(self.env.inventory()) - set(replacement_inventory_before) + if not replacement_keys: + raise AcceptanceError("release replacement uploaded no RustFS objects") + + release_inventory_before = self.env.inventory() + release_puts_before = len(self.env.successful_puts()) + release_outcome = self.client_a.call( + RESTORE_TOOL, self.restore_request(release_destination) + ) + release_operation_id = self.expected_restore_operation_id(release_destination) + self.assert_complete_outcome( + release_outcome, release_destination, release_operation_id + ) + self.assert_manifest( + release_destination, + release_operation_id, + self.source, + self.snapshot_id, + ) + release_changed = changed_objects( + release_inventory_before, self.env.inventory() + ) + release_puts = self.env.successful_puts()[release_puts_before:] + if len(release_changed) != 1 or len(release_puts) != 1: + raise AcceptanceError( + "release crash setup must publish one manifest: " + f"changed={sorted(release_changed)!r}, puts={release_puts!r}" + ) + + def replace_and_drive_release_worker() -> str: + result = self.env.cli( + "rename-replace", + self.absolute(replacement), + self.absolute(release_destination), + ) + # Root replacement only commits the Releasing transition. Drive + # the asynchronous worker explicitly so the release-batch barrier + # cannot lose a race to the CLI returning between background ticks. + self.env.manual_gc() + return result + + release_observation = self.crash_at_restore_barrier( + release_operation_id, + RELEASE_CRASH_PHASE, + replace_and_drive_release_worker, + require_interrupted_result=False, + ) + replacement_read = self.client_b.call( + "workbench_read", + { + "id": release_destination, + "section": "outputs", + "path": "replacement.txt", + }, + ) + replacement_items = replacement_read.get("items") + replacement_text = ( + replacement_items[0].get("value", {}).get("text") + if isinstance(replacement_items, list) and replacement_items + else None + ) + if replacement_text != "replacement survived": + raise AcceptanceError( + "rename-replace was not durable across the release crash" + ) + release_error, release_error_attempts = self.wait_for_restore_error( + release_destination, "RestoreDestinationConflict" + ) + release_gc = self.record_snapshot_deferred_release(release_changed) + self.delete_workbench(self.client_a, release_destination) + replacement_gc = self.record_snapshot_deferred_release(replacement_keys) + released_graph = self.private_graph_evidence( + expected_complete=0, + expected_snapshot_pins=1, + expect_empty=True, + ) + self.restore_control_baseline = dict(released_graph["control_rows"]) + return { + "cleanup_operation_id": cleanup_outcome["operation_id"], + "preparing_interruption": preparing_observation["kind"], + "cleanup_interruption": cleanup_observation["kind"], + "cleanup_retry_attempts": cleanup_retry_attempts, + "cleanup_manifest_puts": len(cleanup_puts), + "cleanup_release_gc": cleanup_release_gc, + "release_operation_id": release_operation_id, + "release_interruption": release_observation["kind"], + "release_terminal_code": release_error.code, + "release_error_attempts": release_error_attempts, + "release_worker_gc_calls": release_error_attempts - 1, + "release_manifest_puts": len(release_puts), + "release_gc": release_gc, + "replacement_gc": replacement_gc, + "released_graph": released_graph, + "durable_ledger_baseline": self.restore_control_baseline, + } + + def scenario_concurrent_restore(self) -> dict[str, Any]: + if self.launch is None: + raise AcceptanceError( + "concurrent restore lacks an Agent-resolved MCP launch" + ) + request = { + "id": self.source, + "at_snapshot": self.snapshot_id, + "destination_id": self.destination, + } + operation_id = self.expected_restore_operation_id(self.destination) + inventory_before = self.env.inventory() + puts_before = len(self.env.successful_puts()) + start_barrier = threading.Barrier(CONCURRENT_RESTORE_CALLS + 1) + hold = self.barriers.handle(operation_id, "references-sealed") + hold.arm() + dedicated_clients: list[WorkbenchClient] = [] + futures: list[concurrent.futures.Future[tuple[dict[str, Any], int]]] = [] + + def restore(client: WorkbenchClient) -> tuple[dict[str, Any], int]: + start_barrier.wait(timeout=self.env.config.tool_timeout) + deadline = time.monotonic() + self.env.config.tool_timeout + attempts = 0 + while time.monotonic() < deadline: + attempts += 1 + outcome = client.raw_call(RESTORE_TOOL, dict(request)) + if outcome.get("status") == "success": + self.assert_complete_outcome( + outcome, self.destination, operation_id + ) + return outcome, attempts + error = decode_tool_error(outcome) + if error.code != "RestoreInProgress" or not error.retryable: + raise AcceptanceError( + f"concurrent exact restore returned {outcome!r}" + ) + time.sleep(0.002) + raise AcceptanceError("concurrent exact restore retry deadline expired") + + def validate_first_visible(observer: WorkbenchClient) -> str: + manifest = read_json_object( + observer.call( + "workbench_read", + { + "id": self.destination, + "section": "metadata", + "path": "restore_manifest.json", + }, + ) + ) + operation_id = manifest.get("operation_id") + if not isinstance(operation_id, str) or not operation_id: + raise AcceptanceError( + f"first-visible manifest lacks operation id: {manifest!r}" + ) + self.assert_manifest( + self.destination, + operation_id, + self.source, + self.snapshot_id, + observer, + ) + first_index = self.index_signature(self.destination, observer) + if first_index != self.baseline_index: + raise AcceptanceError( + "destination became visible before its index overlay was complete: " + f"{first_index!r}" + ) + return operation_id + + def assert_deterministically_hidden(observer: WorkbenchClient) -> None: + observed = observer.raw_call("workbench_stat", {"id": self.destination}) + assert_tool_error(observed, "pre-attach destination stat") + for name, arguments in ( + ( + "workbench_search", + { + "id": self.destination, + "predicates": [], + "fields": ["name"], + "limit": INDEX_PAGE_LIMIT, + }, + ), + ( + "workbench_aggregate", + { + "id": self.destination, + "predicates": [], + "measures": [{"name": "files", "op": "count"}], + }, + ), + ("workbench_catalog", {"id": self.destination}), + ): + result = observer.raw_call(name, arguments) + if result.get("status") == "error" or "code" in result: + continue + if name == "workbench_search" and ( + result.get("matches") != [] or result.get("match_count") != 0 + ): + raise AcceptanceError(f"pre-attach search leaked rows: {result!r}") + if name == "workbench_aggregate" and ( + result.get("groups") != [] + or result.get("input_match_count") not in {0, None} + ): + raise AcceptanceError( + f"pre-attach aggregate leaked rows: {result!r}" + ) + catalog = result.get("catalog") + if name == "workbench_catalog" and not ( + result.get("catalog_empty") is True + or isinstance(catalog, dict) + and catalog.get("filterable") == [] + ): + raise AcceptanceError( + f"pre-attach catalog leaked fields: {result!r}" + ) + global_search = observer.call( + "workbench_search", + { + "predicates": [ + { + "field": "name", + "op": "eq", + "value": f"indexed-{self.env.config.profile.indexed_files - 1:03d}.csv", + } + ], + "fields": ["name"], + "limit": INDEX_PAGE_LIMIT, + }, + ) + if any( + match_.get("workbench_id") == self.destination + for match_ in global_search.get("matches", []) + if isinstance(match_, dict) + ): + raise AcceptanceError("destination index leaked before root attach") + + visible = False + visible_before_all_acks = False + first_visible_operation_id = "" + outcomes: list[dict[str, Any]] = [] + attempts: list[int] = [] + executor = concurrent.futures.ThreadPoolExecutor( + max_workers=CONCURRENT_RESTORE_CALLS + ) + try: + dedicated_clients = [ + WorkbenchClient( + self.client_class, self.launch, self.env.config.tool_timeout + ) + for _ in range(CONCURRENT_RESTORE_CALLS + 1) + ] + for client in dedicated_clients: + validate_tool_contract(client.tools()) + workers = dedicated_clients[:CONCURRENT_RESTORE_CALLS] + observer = dedicated_clients[-1] + futures = [executor.submit(restore, client) for client in workers] + start_barrier.wait(timeout=self.env.config.tool_timeout) + hold.wait_ready(self.env.config.tool_timeout) + if any(future.done() for future in futures): + raise AcceptanceError( + "a concurrent restore returned before the pre-attach hold was released" + ) + assert_deterministically_hidden(observer) + hold.release() + deadline = time.monotonic() + self.env.config.tool_timeout + while time.monotonic() < deadline: + observed = observer.raw_call("workbench_stat", {"id": self.destination}) + if observed.get("status") == "success": + visible = True + visible_before_all_acks = not all( + future.done() for future in futures + ) + first_visible_operation_id = validate_first_visible(observer) + break + assert_tool_error(observed, "destination visibility probe") + if all(future.done() for future in futures): + break + time.sleep(0.005) + failures: list[str] = [] + for future in futures: + try: + outcome, attempt_count = future.result( + timeout=self.env.config.tool_timeout + ) + outcomes.append(outcome) + attempts.append(attempt_count) + except Exception as exc: + failures.append(str(exc)) + if failures: + distribution = collections.Counter(failures) + raise AcceptanceError( + f"{len(failures)}/{CONCURRENT_RESTORE_CALLS} concurrent restore calls failed: " + f"{dict(distribution)!r}" + ) + if not visible: + observed = observer.raw_call("workbench_stat", {"id": self.destination}) + if observed.get("status") != "success": + raise AcceptanceError("destination never became visible") + first_visible_operation_id = validate_first_visible(observer) + finally: + try: + hold.release() + except Exception: + pass + executor.shutdown(wait=True, cancel_futures=True) + hold.disarm_after_crash() + for client in dedicated_clients: + try: + client.close() + except Exception: + pass + operation_ids = {outcome.get("operation_id") for outcome in outcomes} + roots = {outcome.get("destination_root") for outcome in outcomes} + if len(operation_ids) != 1 or None in operation_ids or len(roots) != 1: + raise AcceptanceError("concurrent exact retries did not converge") + self.operation_id = str(outcomes[0]["operation_id"]) + if self.operation_id != operation_id: + raise AcceptanceError( + f"concurrent restore operation id is not deterministic: {self.operation_id}" + ) + if first_visible_operation_id != self.operation_id: + raise AcceptanceError( + "first-visible manifest operation differs from terminal outcomes" + ) + for outcome in outcomes: + if ( + outcome.get("state") != "complete" + or outcome.get("snapshot_id") != self.snapshot_id + or outcome.get("read_version") != self.snapshot_version + or outcome.get("cleanup_pending") is not False + ): + raise AcceptanceError(f"malformed restore outcome: {outcome!r}") + + self.assert_manifest( + self.destination, self.operation_id, self.source, self.snapshot_id + ) + restored_index = self.index_signature(self.destination) + if restored_index != self.baseline_index: + raise AcceptanceError( + f"restored search/aggregate/catalog differ: source={self.baseline_index!r}, " + f"destination={restored_index!r}" + ) + inventory_after = self.env.inventory() + put_records = self.env.successful_puts()[puts_before:] + changed = changed_objects(inventory_before, inventory_after) + if len(put_records) != 1: + raise AcceptanceError( + "16 concurrent restores must issue exactly one manifest PUT; " + f"observed {len(put_records)}: {put_records!r}" + ) + if len(changed) != 1: + raise AcceptanceError( + f"COW restore changed objects other than one manifest: {sorted(changed)!r}" + ) + if self.fork_owned_keys & changed: + raise AcceptanceError("restore overwrote or copied a source data object") + self.restore_manifest_keys.update(changed) + + graph = self.private_graph_evidence( + expected_complete=1, + expected_snapshot_pins=1, + expect_empty=False, + ) + conflict_snapshot = self.client_a.call( + "workbench_snapshot", {"id": self.source, "ttl_days": 7} + ) + conflict_snapshot_id = int(conflict_snapshot["snapshot_id"]) + agent_redrive = self.agent_restore_redrive(conflict_snapshot_id) + conflict = decode_tool_error( + self.client_a.raw_call( + RESTORE_TOOL, + { + "id": self.source, + "at_snapshot": conflict_snapshot_id, + "destination_id": self.destination, + }, + ) + ) + if conflict.code != "RestoreDestinationConflict": + raise AcceptanceError(f"wrong destination conflict: {conflict!r}") + self.env.cli( + "retire-snapshot", + self.absolute(self.source), + str(conflict_snapshot_id), + ) + return { + "operation_id": self.operation_id, + "concurrent_calls": len(outcomes), + "independent_mcp_processes": len(dedicated_clients), + "max_retry_attempts": max(attempts), + "manifest_puts": len(put_records), + "changed_objects": sorted(changed), + "visible_before_all_acks": visible_before_all_acks, + "deterministic_pre_attach_hold": "references-sealed", + "private_graph": graph, + "agent_handler_redrive": agent_redrive, + } + + def scenario_restart_and_nested_restore(self) -> dict[str, Any]: + self.reconnect_after_server_kill() + retry = self.client_a.call( + RESTORE_TOOL, + { + "id": self.source, + "at_snapshot": self.snapshot_id, + "destination_id": self.destination, + }, + ) + if retry.get("operation_id") != self.operation_id: + raise AcceptanceError("operation id changed after server/Agent restart") + self.env.cli( + "rename", + self.absolute(self.destination, "outputs", "indexed-000.csv"), + self.absolute(self.destination, "outputs", "renamed.csv"), + ) + self.env.cli( + "rm", self.absolute(self.destination, "outputs", "indexed-001.csv") + ) + self.put_text( + self.destination, + "published.csv", + "index,value\nnew,1\n", + content_type="text/csv", + ) + self.mutated_index = self.index_signature(self.destination) + expected_names = { + f"indexed-{index:03d}.csv" + for index in range(self.env.config.profile.indexed_files) + } + expected_names.difference_update({"indexed-000.csv", "indexed-001.csv"}) + expected_names.update({"renamed.csv", "published.csv"}) + if self.mutated_index["csv_names"] != sorted(expected_names): + raise AcceptanceError( + "overlay rename/delete/publish left missing or ghost index rows: " + f"{self.mutated_index!r}" + ) + nested_snapshot = self.client_a.call( + "workbench_snapshot", + {"id": self.destination, "name": "nested-point", "ttl_days": 7}, + ) + self.nested_snapshot_id = int(nested_snapshot["snapshot_id"]) + inventory_before_nested = self.env.inventory() + puts_before_nested = len(self.env.successful_puts()) + nested = self.client_b.call( + RESTORE_TOOL, + { + "id": self.destination, + "at_snapshot": self.nested_snapshot_id, + "destination_id": self.nested, + }, + ) + self.nested_operation_id = str(nested["operation_id"]) + self.assert_manifest( + self.nested, + self.nested_operation_id, + self.destination, + self.nested_snapshot_id, + ) + if self.index_signature(self.nested) != self.mutated_index: + raise AcceptanceError( + "nested restore did not preserve completed overlay index" + ) + nested_changed = changed_objects(inventory_before_nested, self.env.inventory()) + nested_puts = self.env.successful_puts()[puts_before_nested:] + if len(nested_changed) != 1 or len(nested_puts) != 1: + raise AcceptanceError( + "nested COW restore must write only its manifest: " + f"changed={sorted(nested_changed)!r}, puts={nested_puts!r}" + ) + self.restore_manifest_keys.update(nested_changed) + self.env.cli( + "retire-snapshot", + self.absolute(self.destination), + str(self.nested_snapshot_id), + ) + return { + "server_restart": "SIGKILL", + "agent_reconnected": True, + "nested_operation_id": self.nested_operation_id, + "nested_manifest_puts": len(nested_puts), + } + + def list_entries( + self, client: WorkbenchClient, workbench_id: str, section: str, path: str = "" + ) -> list[dict[str, Any]]: + entries: list[dict[str, Any]] = [] + cursor: str | None = None + while True: + args: dict[str, Any] = { + "id": workbench_id, + "section": section, + "limit": 100, + } + if path: + args["path"] = path + if cursor is not None: + args["cursor"] = cursor + page = client.call("workbench_list", args) + values = page.get("entries") + if not isinstance(values, list): + raise AcceptanceError(f"list lacks entries: {page!r}") + entries.extend(values) + if page.get("truncated") is not True: + return entries + cursor = page.get("next_cursor") + if not isinstance(cursor, str) or not cursor: + raise AcceptanceError("truncated list lacks cursor") + + def delete_section_tree( + self, client: WorkbenchClient, workbench_id: str, section: str, path: str = "" + ) -> None: + entries = self.list_entries(client, workbench_id, section, path) + directories: list[str] = [] + for entry in entries: + name = entry.get("name") + kind = entry.get("kind") + if not isinstance(name, str): + raise AcceptanceError(f"malformed list entry: {entry!r}") + relative = f"{path}/{name}" if path else name + if kind == "directory": + self.delete_section_tree(client, workbench_id, section, relative) + directories.append(relative) + else: + self.env.cli("rm", self.absolute(workbench_id, section, relative)) + for relative in reversed(directories): + self.env.cli("rmdir", self.absolute(workbench_id, section, relative)) + + def delete_workbench(self, client: WorkbenchClient, workbench_id: str) -> None: + for section in SECTIONS: + self.delete_section_tree(client, workbench_id, section) + self.env.cli("rmdir", self.absolute(workbench_id, section)) + self.env.cli("rmdir", self.absolute(workbench_id)) + + def wait_for_keys(self, keys: set[str], *, present: bool) -> dict[str, Any]: + started = time.monotonic() + stall_deadline = started + self.env.config.gc_deadline + # A full 1 GiB restore has hundreds of exact references plus member and + # index rows. One bounded release page may perform many durable CASes, + # so a fixed wall-clock deadline can expire while every call is making + # measurable progress. Keep the configured deadline as the maximum + # no-progress window and retain a finite overall cap for genuine loops. + hard_deadline = started + self.env.config.gc_deadline * 8 + last_gc: dict[str, Any] = {} + previous_remaining = len(keys) + previous_control_rows: int | None = None + last_control_rows: int | None = None + previous_commit_total: int | None = None + last_commit_total: int | None = None + while time.monotonic() < hard_deadline: + # Keep the object page bounded while release jobs are still being + # paged. Scanning thousands of snapshot-blocked object records on + # every round does not advance a release job any faster and can + # consume the entire lifecycle deadline before its member/index + # pages reach the object-enqueue boundary. + last_gc = self.env.manual_gc(limit=64) + current = set(self.env.inventory()) + if (keys <= current) if present else keys.isdisjoint(current): + return last_gc + remaining = len(keys - current) if present else len(keys & current) + stats = self.env.stats() + restore = stats.get("restore") + if not isinstance(restore, dict): + raise AcceptanceError( + f"restore metrics disappeared while waiting for objects: {restore!r}" + ) + control_rows = restore.get("control_rows") + if not isinstance(control_rows, dict): + raise AcceptanceError( + "restore control-row metrics disappeared while waiting for objects: " + f"{restore!r}" + ) + last_control_rows = sum( + nonnegative_int(value, f"restore.control_rows.{name}") + for name, value in control_rows.items() + ) + metadata = stats.get("metadata_store") + if not isinstance(metadata, dict): + raise AcceptanceError( + "metadata-store metrics disappeared while waiting for objects: " + f"{metadata!r}" + ) + last_commit_total = nonnegative_int( + metadata.get("commit_total"), "metadata_store.commit_total" + ) + object_gc = last_gc.get("object_gc") + if not isinstance(object_gc, dict): + raise AcceptanceError(f"manual GC omitted object outcome: {last_gc!r}") + queue_progress = sum( + nonnegative_int(object_gc.get(field), f"object_gc.{field}") + for field in ("deleted", "missing", "records_removed") + ) + graph_progress = ( + previous_control_rows is not None + and last_control_rows < previous_control_rows + ) + commit_progress = ( + previous_commit_total is not None + and last_commit_total > previous_commit_total + ) + if ( + remaining < previous_remaining + or graph_progress + or commit_progress + or queue_progress > 0 + ): + stall_deadline = time.monotonic() + self.env.config.gc_deadline + elif time.monotonic() >= stall_deadline: + break + previous_remaining = remaining + previous_control_rows = last_control_rows + previous_commit_total = last_commit_total + time.sleep(0.1) + state = sorted(keys & set(self.env.inventory())) + raise AcceptanceError( + f"object lifecycle deadline expired (present={present}, remaining={state!r}, " + f"last_control_rows={last_control_rows!r}, " + f"last_commit_total={last_commit_total!r}, last_gc={last_gc!r})" + ) + + def record_snapshot_deferred_release(self, keys: set[str]) -> dict[str, Any]: + """Record post-snapshot objects that the live retention floor must hold. + + The source checkpoint intentionally remains pinned throughout the crash + matrices. NoKV's mount-global history floor therefore retains objects + enqueued after that checkpoint, including each temporary fork's own + restore manifest. Their private restore rows must still drain now, but + the current mount-global policy intentionally defers physical deletion + until the source pin is retired. + """ + if not keys: + raise AcceptanceError("snapshot-deferred release requires object keys") + deadline = time.monotonic() + min( + self.env.config.gc_deadline, self.env.config.tool_timeout + ) + last_gc: dict[str, Any] = {} + remaining: set[str] = set(keys) + snapshot_block_evidence: dict[str, Any] | None = None + while time.monotonic() < deadline: + last_gc = self.env.manual_gc(limit=64) + remaining = keys & set(self.env.inventory()) + object_gc = last_gc.get("object_gc") + stats = self.env.stats() + metadata = stats.get("metadata_store") + if ( + isinstance(object_gc, dict) + and object_gc.get("blocked_by_snapshots", 0) > 0 + and isinstance(metadata, dict) + and metadata.get("active_snapshot_pin_total") == 1 + ): + snapshot_block_evidence = last_gc + restore_metrics = stats.get("restore") + if not restore_release_graph_drained(restore_metrics): + time.sleep(0.02) + continue + validate_restore_metrics_object( + restore_metrics, expected_complete=0, expect_empty=True + ) + if not remaining: + return { + "released_immediately": len(keys), + "deferred_by_snapshot": [], + "last_gc": last_gc, + } + if snapshot_block_evidence is not None: + self.snapshot_deferred_release_keys.update(remaining) + return { + "released_immediately": len(keys) - len(remaining), + "deferred_by_snapshot": sorted(remaining), + "last_gc": snapshot_block_evidence, + } + time.sleep(0.02) + raise AcceptanceError( + "released objects remained without live-snapshot GC evidence: " + f"remaining={sorted(remaining)!r}, last_gc={last_gc!r}" + ) + + def wait_for_exact_inventory( + self, expected: dict[str, ObjectFingerprint] + ) -> dict[str, ObjectFingerprint]: + deadline = time.monotonic() + self.env.config.gc_deadline + current: dict[str, ObjectFingerprint] = {} + while time.monotonic() < deadline: + self.env.manual_gc() + current = self.env.inventory() + if current == expected: + return current + time.sleep(0.1) + raise AcceptanceError( + "RustFS inventory did not return to its exact baseline: " + f"expected={expected!r}, current={current!r}" + ) + + def wait_for_restore_metrics( + self, *, expected_complete: int, expect_empty: bool + ) -> dict[str, Any]: + deadline = time.monotonic() + self.env.config.gc_deadline + last_error = "metrics were not sampled" + while time.monotonic() < deadline: + self.env.manual_gc() + try: + return validate_restore_metrics( + self.env.stats(), + expected_complete=expected_complete, + expect_empty=expect_empty, + ) + except AcceptanceError as exc: + last_error = str(exc) + time.sleep(0.1) + raise AcceptanceError( + "restore private graph did not converge before deadline: " + last_error + ) + + def private_graph_evidence( + self, + *, + expected_complete: int, + expected_snapshot_pins: int, + expect_empty: bool, + ) -> dict[str, Any]: + metrics = self.wait_for_restore_metrics( + expected_complete=expected_complete, expect_empty=expect_empty + ) + report = validate_fsck_report( + self.env.fsck(), + expected_complete=expected_complete, + expected_snapshot_pins=expected_snapshot_pins, + expected_fork_bindings=0, + ) + restore_report = report["restore_shards"][0]["report"] + return { + "active_marker": metrics["active_marker"], + "allocator_v2_fenced": metrics["allocator_v2_fenced"], + "operations": metrics["operations"], + "staging_rows": metrics["staging_rows"], + "exact_reference_rows": metrics["exact_reference_rows"], + "index_rows": metrics["index_rows"], + "control_rows": metrics["control_rows"], + "borrowed_objects_checked": restore_report["borrowed_objects_checked"], + "snapshot_pins_scanned": report["snapshot_pins_scanned"], + "fork_bindings_scanned": report["fork_bindings_scanned"], + } + + def assert_no_index_ghosts(self, workbench_ids: set[str]) -> None: + cursor: str | None = None + pages = 0 + while True: + arguments: dict[str, Any] = { + "predicates": [{"field": "name", "op": "suffix", "value": ".csv"}], + "fields": ["name"], + "limit": INDEX_PAGE_LIMIT, + } + if cursor is not None: + arguments["cursor"] = cursor + page = self.client_a.call("workbench_search", arguments) + matches = page.get("matches") + if not isinstance(matches, list): + raise AcceptanceError(f"global ghost search is malformed: {page!r}") + ghosts = [ + match_ + for match_ in matches + if isinstance(match_, dict) + and match_.get("workbench_id") in workbench_ids + ] + if ghosts: + raise AcceptanceError( + f"released restore index rows remain visible: {ghosts!r}" + ) + pages += 1 + if page.get("truncated") is not True: + break + cursor = page.get("next_cursor") + if not isinstance(cursor, str) or not cursor: + raise AcceptanceError("global ghost search has no next cursor") + if pages > self.env.config.profile.indexed_files + 32: + raise AcceptanceError("global ghost search pagination did not converge") + for workbench_id in workbench_ids: + for name, arguments in ( + ( + "workbench_search", + { + "id": workbench_id, + "predicates": [], + "fields": ["name"], + "limit": INDEX_PAGE_LIMIT, + }, + ), + ( + "workbench_aggregate", + { + "id": workbench_id, + "predicates": [], + "measures": [{"name": "files", "op": "count"}], + }, + ), + ("workbench_catalog", {"id": workbench_id}), + ): + result = self.client_b.raw_call(name, arguments) + if result.get("status") == "error" or "code" in result: + continue + if name == "workbench_search" and result.get("matches") != []: + raise AcceptanceError( + f"released {workbench_id} search has ghosts: {result!r}" + ) + if name == "workbench_aggregate" and ( + result.get("groups") != [] + or result.get("input_match_count") not in {0, None} + ): + raise AcceptanceError( + f"released {workbench_id} aggregate has ghosts: {result!r}" + ) + catalog = result.get("catalog") + if name == "workbench_catalog" and not ( + result.get("catalog_empty") is True + or isinstance(catalog, dict) + and catalog.get("filterable") == [] + ): + raise AcceptanceError( + f"released {workbench_id} catalog has ghosts: {result!r}" + ) + + def scenario_source_retirement_moves_and_release(self) -> dict[str, Any]: + def progress(step: str) -> None: + print(f"[restore-live-e2e] STEP source-release:{step}", flush=True) + + progress("retention-floor-preflight") + stats_before_floor_lift = self.env.stats() + metadata_before = stats_before_floor_lift.get("metadata_store") + history_before = stats_before_floor_lift.get("history_gc") + if ( + not isinstance(metadata_before, dict) + or metadata_before.get("active_snapshot_pin_total") != 1 + ): + raise AcceptanceError( + f"source snapshot did not hold exactly one retention pin: {metadata_before!r}" + ) + if not isinstance(history_before, dict) or not isinstance( + history_before.get("iterations"), int + ): + raise AcceptanceError(f"history GC stats are malformed: {history_before!r}") + history_iterations_before = int(history_before["iterations"]) + retained_floor_gc = self.env.manual_gc() + retained_floor_history = retained_floor_gc.get("history_gc") + if ( + not isinstance(retained_floor_history, dict) + or nonnegative_int( + retained_floor_history.get("retained_by_snapshots"), + "history_gc.retained_by_snapshots", + ) + == 0 + ): + raise AcceptanceError( + "source snapshot did not retain any metadata history before retirement: " + f"{retained_floor_gc!r}" + ) + gc_stop = threading.Event() + gc_started = threading.Event() + gc_outcomes: list[dict[str, Any]] = [] + + def race_gc_with_pin_retirement() -> None: + while not gc_stop.is_set(): + gc_outcomes.append(self.env.manual_gc()) + gc_started.set() + time.sleep(0.002) + + gc_executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) + gc_future = gc_executor.submit(race_gc_with_pin_retirement) + try: + if not gc_started.wait(timeout=self.env.config.tool_timeout): + raise AcceptanceError( + "concurrent GC did not start before pin retirement" + ) + post_retirement_gc_start = len(gc_outcomes) + self.env.cli( + "retire-snapshot", self.absolute(self.source), str(self.snapshot_id) + ) + self.env.cli( + "rename", self.absolute(self.destination), self.absolute(self.moved) + ) + self.delete_workbench(self.client_a, self.source) + finally: + gc_stop.set() + gc_future.result(timeout=self.env.config.tool_timeout) + gc_executor.shutdown(wait=True, cancel_futures=True) + progress("pin-retirement-race-complete") + if not gc_outcomes: + raise AcceptanceError( + "pin/history/object GC concurrency produced no outcome" + ) + # Periodic workers are deliberately outside the acceptance window so + # full fsck can prove a stable metadata epoch. Prove the floor lift from + # the bounded manual-GC outcomes instead of waiting for the passive + # worker's iteration counter. The slice also includes a call that may + # have started before retirement but completed after the pin commit. + floor_lift_gc = self.env.manual_gc() + post_retirement_gc_outcomes = gc_outcomes[post_retirement_gc_start:] + [ + floor_lift_gc + ] + history_scanned_after_retirement = 0 + history_removed_after_retirement = 0 + for outcome in post_retirement_gc_outcomes: + history = outcome.get("history_gc") + if not isinstance(history, dict): + raise AcceptanceError( + f"manual GC omitted history outcome: {outcome!r}" + ) + history_scanned_after_retirement += nonnegative_int( + history.get("scanned"), "history_gc.scanned" + ) + history_removed_after_retirement += nonnegative_int( + history.get("removed"), "history_gc.removed" + ) + if history_removed_after_retirement == 0: + raise AcceptanceError( + "snapshot retirement did not release any retained metadata history: " + f"before={retained_floor_gc!r}, " + f"after={post_retirement_gc_outcomes!r}" + ) + progress("history-floor-lifted") + deferred_release_gc = self.wait_for_keys( + self.snapshot_deferred_release_keys, present=False + ) + deferred_release_count = len(self.snapshot_deferred_release_keys) + self.snapshot_deferred_release_keys.clear() + progress("snapshot-deferred-objects-released") + # indexed-001.csv was deliberately removed from the first restore + # before the nested snapshot. Once the source pin and namespace retire, + # its source-owned object is allowed to disappear immediately. The COW + # fixture is the object set that both surviving restore roots must keep. + self.wait_for_keys(self.large_object_keys, present=True) + + floor_graph = self.private_graph_evidence( + expected_complete=2, + expected_snapshot_pins=0, + expect_empty=False, + ) + progress("surviving-restore-graph-verified") + floor_deadline = time.monotonic() + self.env.config.gc_deadline + stats_after_floor_lift: dict[str, Any] = {} + while time.monotonic() < floor_deadline: + stats_after_floor_lift = self.env.stats() + metadata_after = stats_after_floor_lift.get("metadata_store") + history_after = stats_after_floor_lift.get("history_gc") + if ( + isinstance(metadata_after, dict) + and metadata_after.get("active_snapshot_pin_total") == 0 + and isinstance(history_after, dict) + and isinstance(history_after.get("iterations"), int) + ): + break + time.sleep(0.1) + else: + raise AcceptanceError( + "snapshot retirement did not lift the retention floor/history GC: " + f"before={stats_before_floor_lift!r}, after={stats_after_floor_lift!r}" + ) + + retry = self.client_a.call( + RESTORE_TOOL, + { + "id": self.source, + "at_snapshot": self.snapshot_id, + "destination_id": self.destination, + }, + ) + if retry.get("operation_id") != self.operation_id: + raise AcceptanceError("terminal retry failed after pin/source deletion") + progress("terminal-retry-after-source-delete") + digest, size = self.env.hash_remote_file( + self.absolute(self.moved, "outputs", "cow-large.bin") + ) + if ( + digest != self.expected_digest + or size != self.env.config.profile.large_bytes + ): + raise AcceptanceError( + f"restored COW digest mismatch: digest={digest}, size={size}" + ) + if self.index_signature(self.moved) != self.mutated_index: + raise AcceptanceError("root move lost overlay index visibility") + progress("moved-root-content-and-index-verified") + + escaped_path = f"{self.root}/escaped-cow-large.bin" + self.env.cli( + "rename", + self.absolute(self.moved, "outputs", "cow-large.bin"), + escaped_path, + ) + progress("borrower-escaped-restore-root") + + # Replace one Complete restore root with another. Replacing a non-empty + # ordinary directory would violate POSIX and strand an unowned subtree; + # this path instead exercises the PR2 contract directly: the victim + # restore enters Releasing in the same command while the nested restore + # remains Complete and moves to the destination. + self.env.cli( + "rename-replace", self.absolute(self.nested), self.absolute(self.moved) + ) + self.env.cli("rename", self.absolute(self.moved), self.absolute(self.replaced)) + nested_read = self.client_a.call( + "workbench_read", + { + "id": self.replaced, + "section": "outputs", + "path": "renamed.csv", + }, + ) + if nested_read.get("total_size_bytes") is None: + raise AcceptanceError("rename-replace lost restored borrower content") + progress("rename-replace-verified") + + self.wait_for_keys(self.large_object_keys, present=True) + self.delete_workbench(self.client_a, self.replaced) + progress("replacement-root-deleted") + all_restore_keys = self.fork_owned_keys | self.restore_manifest_keys + release_without_escaped = all_restore_keys - self.large_object_keys + partial_release_gc = self.wait_for_keys(release_without_escaped, present=False) + progress("partial-release-complete") + self.wait_for_keys(self.large_object_keys, present=True) + escaped_digest, escaped_size = self.env.hash_remote_file(escaped_path) + if escaped_digest != self.expected_digest or escaped_size != size: + raise AcceptanceError( + "escaped borrower became unreadable after restore root deletion" + ) + self.env.cli("rm", escaped_path) + progress("escaped-borrower-deleted") + final_release_gc = self.wait_for_keys(all_restore_keys, present=False) + progress("final-object-release-complete") + final_graph = self.private_graph_evidence( + expected_complete=0, + expected_snapshot_pins=0, + expect_empty=True, + ) + control_rows_returned_to_baseline: bool | None = None + if self.restore_control_baseline: + if final_graph["control_rows"] != self.restore_control_baseline: + raise AcceptanceError( + "released restore control rows did not return to the measured " + "durable-ledger baseline: " + f"expected={self.restore_control_baseline!r}, " + f"current={final_graph['control_rows']!r}" + ) + control_rows_returned_to_baseline = True + self.assert_no_index_ghosts( + { + self.source, + self.destination, + self.moved, + self.nested, + self.replaced, + } + ) + progress("private-graph-and-index-clean") + self.wait_for_exact_inventory(self.initial_inventory) + progress("inventory-returned-to-baseline") + self.env.assert_binary_unchanged() + return { + "large_digest": digest, + "large_size": size, + "escaped_borrower_digest": escaped_digest, + "source_deleted": True, + "root_move": True, + "rename_replace": True, + "escaped_rename": True, + "released_object_keys": len(all_restore_keys), + "restore_manifest_keys_released": len(self.restore_manifest_keys), + "partial_release_gc": partial_release_gc, + "last_gc": final_release_gc, + "concurrent_gc_calls": len(gc_outcomes), + "snapshot_deferred_release_keys": deferred_release_count, + "snapshot_deferred_release_gc": deferred_release_gc, + "history_gc_iterations_before": history_iterations_before, + "history_gc_iterations_after": stats_after_floor_lift["history_gc"][ + "iterations" + ], + "history_gc_retained_before_retirement": retained_floor_history[ + "retained_by_snapshots" + ], + "history_gc_scanned_after_retirement": history_scanned_after_retirement, + "history_gc_removed_after_retirement": history_removed_after_retirement, + "floor_lift_graph": floor_graph, + "final_private_graph": final_graph, + "durable_ledger_baseline": self.restore_control_baseline, + "control_rows_returned_to_baseline": control_rows_returned_to_baseline, + "final_inventory_matches_initial": True, + } + + def run(self) -> dict[str, dict[str, Any]]: + self.connect_clients() + self.scenario( + "contract_agent_reconnect_and_1g_fixture", + self.scenario_contract_and_fixture, + ) + if self.env.config.profile.name == "full": + self.scenario( + "durable_create_crash_barrier_matrix", + self.scenario_create_crash_matrix, + ) + self.scenario( + "durable_cleanup_release_crash_recovery", + self.scenario_cleanup_release_crash_recovery, + ) + self.scenario( + "first_visibility_16way_idempotent_cow_restore", + self.scenario_concurrent_restore, + ) + self.scenario( + "server_restart_index_mutation_and_nested_restore", + self.scenario_restart_and_nested_restore, + ) + self.scenario( + "source_retirement_root_moves_and_exact_ref_release", + self.scenario_source_retirement_moves_and_release, + ) + return self.results + + +def load_lingtai(lingtai_kernel_dir: Path) -> tuple[type, type]: + source = lingtai_kernel_dir / "src" + if not source.is_dir(): + raise AcceptanceError(f"LingTai source directory not found: {source}") + sys.path.insert(0, str(source)) + try: + from lingtai.agent import Agent + from lingtai.services.mcp import MCPClient + except Exception as exc: + raise AcceptanceError( + "failed to import LingTai Agent/MCPClient; run this script inside the " + "LingTai uv environment" + ) from exc + module_path = Path(sys.modules[Agent.__module__].__file__).resolve() + try: + module_path.relative_to(source.resolve()) + except ValueError as exc: + raise AcceptanceError( + f"imported LingTai from {module_path}, expected {source.resolve()}" + ) from exc + return Agent, MCPClient + + +def require_commands(names: tuple[str, ...]) -> None: + missing = [name for name in names if shutil.which(name) is None] + if missing: + raise AcceptanceError(f"required commands are missing: {', '.join(missing)}") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + repo_root = Path(__file__).resolve().parents[2] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", choices=("quick", "full"), default="quick") + parser.add_argument("--require-all", action="store_true") + parser.add_argument("--repo-root", type=Path, default=repo_root) + parser.add_argument( + "--cargo-bin", + type=Path, + default=Path( + os.environ.get("CARGO", shutil.which("cargo") or "~/.cargo/bin/cargo") + ), + ) + parser.add_argument("--nokv-bin", type=Path) + parser.add_argument( + "--lingtai-kernel-dir", + type=Path, + default=Path(os.environ.get("LINGTAI_KERNEL_DIR", "~/lingtai-kernel")), + ) + parser.add_argument("--state-dir", type=Path) + parser.add_argument("--server-port", type=int, default=0) + parser.add_argument("--rustfs-port", type=int, default=0) + parser.add_argument("--rustfs-console-port", type=int, default=0) + parser.add_argument("--proxy-port", type=int, default=0) + parser.add_argument("--rustfs-image", default="rustfs/rustfs:latest") + parser.add_argument("--command-timeout", type=float, default=300) + parser.add_argument("--tool-timeout", type=float, default=300) + parser.add_argument("--startup-timeout", type=float, default=180) + parser.add_argument("--gc-deadline", type=float, default=120) + parser.add_argument("--no-build", action="store_true") + parser.add_argument("--keep-state", action="store_true") + args = parser.parse_args(argv) + if args.require_all and args.profile != "full": + parser.error("--require-all requires --profile full") + if args.require_all and args.no_build: + parser.error("--require-all forbids --no-build to prevent stale NoKV binaries") + return args + + +def live_config(args: argparse.Namespace) -> LiveConfig: + repo_root = args.repo_root.expanduser().resolve() + state_dir: Path + if args.state_dir: + state_dir = args.state_dir.expanduser().resolve() + if state_dir.exists() and any(state_dir.iterdir()): + raise AcceptanceError(f"state directory must be empty: {state_dir}") + state_dir.mkdir(parents=True, exist_ok=True) + else: + (repo_root / "target").mkdir(parents=True, exist_ok=True) + state_dir = Path( + tempfile.mkdtemp(prefix="durable-restore-live-", dir=repo_root / "target") + ) + ports = [ + args.server_port or free_port(), + args.rustfs_port or free_port(), + args.rustfs_console_port or free_port(), + args.proxy_port or free_port(), + ] + if len(set(ports)) != len(ports): + raise AcceptanceError("server, RustFS, console, and proxy ports must differ") + suffix = f"{os.getpid()}-{int(time.time())}" + nokv_bin = ( + args.nokv_bin.expanduser().resolve() + if args.nokv_bin + else repo_root / "target/debug/nokv" + ) + if args.require_all and nokv_bin != (repo_root / "target/debug/nokv").resolve(): + raise AcceptanceError( + "--require-all must build and execute this checkout's target/debug/nokv" + ) + return LiveConfig( + repo_root=repo_root, + cargo_bin=args.cargo_bin.expanduser().absolute(), + nokv_bin=nokv_bin, + lingtai_kernel_dir=args.lingtai_kernel_dir.expanduser().resolve(), + state_dir=state_dir, + server_port=ports[0], + rustfs_port=ports[1], + rustfs_console_port=ports[2], + proxy_port=ports[3], + rustfs_image=args.rustfs_image, + bucket=f"nokv-durable-restore-{suffix}", + container=f"nokv-durable-restore-{suffix}", + profile=workload_profile(args.profile), + command_timeout=args.command_timeout, + tool_timeout=args.tool_timeout, + startup_timeout=args.startup_timeout, + gc_deadline=args.gc_deadline, + build=not args.no_build, + keep_state=args.keep_state, + require_all=args.require_all, + ) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + config = live_config(args) + env = LiveEnvironment(config) + suite: AcceptanceSuite | None = None + exit_code = 0 + try: + require_commands(("aws", "docker", "git")) + if config.build and not config.cargo_bin.is_file(): + raise AcceptanceError(f"cargo executable not found: {config.cargo_bin}") + agent_class, client_class = load_lingtai(config.lingtai_kernel_dir) + env.build() + env.start_rustfs() + env.start_proxy() + env.start_server() + suite = AcceptanceSuite(env, agent_class, client_class) + suite.run() + if config.require_all: + expected = { + "contract_agent_reconnect_and_1g_fixture", + "durable_create_crash_barrier_matrix", + "durable_cleanup_release_crash_recovery", + "first_visibility_16way_idempotent_cow_restore", + "server_restart_index_mutation_and_nested_restore", + "source_retirement_root_moves_and_exact_ref_release", + } + if set(suite.results) != expected or any( + result.get("status") != "passed" for result in suite.results.values() + ): + raise AcceptanceError( + "--require-all rejected an incomplete scenario set" + ) + if config.profile.large_bytes != 1 << 30: + raise AcceptanceError("--require-all did not execute the 1 GiB fixture") + contract = suite.results["contract_agent_reconnect_and_1g_fixture"][ + "details" + ] + crash = suite.results["durable_create_crash_barrier_matrix"]["details"] + cleanup_release = suite.results[ + "durable_cleanup_release_crash_recovery" + ]["details"] + concurrent = suite.results["first_visibility_16way_idempotent_cow_restore"][ + "details" + ] + release = suite.results[ + "source_retirement_root_moves_and_exact_ref_release" + ]["details"] + if ( + contract.get("fixture_4mib_blocks") != FULL_COW_BLOCK_COUNT + or not isinstance(contract.get("fixture_marker_digest"), str) + or len(contract["fixture_marker_digest"]) != 64 + or not isinstance(contract.get("raw_tools_schema_sha256"), str) + or len(contract["raw_tools_schema_sha256"]) != 64 + ): + raise AcceptanceError( + "--require-all lacks block or raw tools/list preflight evidence" + ) + if ( + crash.get("materialization_batches", 0) < 2 + or crash.get("reference_batches", 0) < 2 + ): + raise AcceptanceError( + "--require-all did not discover every restore batch" + ) + if ( + concurrent.get("independent_mcp_processes") + != CONCURRENT_RESTORE_CALLS + 1 + ): + raise AcceptanceError( + "--require-all did not launch 16 callers plus observer" + ) + if cleanup_release.get("release_worker_gc_calls", 0) <= 1: + raise AcceptanceError( + "--require-all did not prove paged release-worker recovery" + ) + if ( + release.get("final_inventory_matches_initial") is not True + or release.get("control_rows_returned_to_baseline") is not True + or release.get("final_private_graph", {}) + .get("operations", {}) + .get("complete") + != 0 + or release.get("final_private_graph", {}).get("active_marker") + is not True + or release.get("final_private_graph", {}).get("allocator_v2_fenced") + is not True + ): + raise AcceptanceError( + "--require-all did not prove final graph/inventory drain" + ) + env.assert_binary_unchanged() + except Exception as exc: + exit_code = 1 + print(f"[restore-live-e2e] acceptance failed: {exc}", file=sys.stderr) + if env.server_log.exists(): + print(tail(env.server_log), file=sys.stderr) + finally: + if suite is not None: + suite.close_clients() + summary = { + "status": "passed" if exit_code == 0 else "failed", + "profile": args.profile, + "require_all": args.require_all, + "fixture_bytes": config.profile.large_bytes, + "server_bind": env.server_bind, + "s3_endpoint": env.s3_endpoint, + "rustfs_endpoint": env.rustfs_endpoint, + "bucket": config.bucket, + "state_dir": str(config.state_dir), + "provenance": { + "nokv_binary": str(config.nokv_bin), + "nokv_binary_sha256": env.nokv_binary_sha256, + "nokv_revision": env.repo_revision, + "lingtai_revision": env.lingtai_revision, + }, + "results": suite.results if suite is not None else {}, + } + print(json.dumps(summary, indent=2, sort_keys=True), flush=True) + env.cleanup() + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/lingtai-workbench/durable_restore_live_e2e_test.py b/scripts/lingtai-workbench/durable_restore_live_e2e_test.py new file mode 100644 index 000000000..e12af543b --- /dev/null +++ b/scripts/lingtai-workbench/durable_restore_live_e2e_test.py @@ -0,0 +1,637 @@ +#!/usr/bin/env python3 +# Copyright 2024-2026 The NoKV Authors. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import contextlib +import concurrent.futures +import copy +import hashlib +import http.client +import http.server +import importlib.util +import io +import sys +import tempfile +import threading +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).with_name("durable_restore_live_e2e.py") +SPEC = importlib.util.spec_from_file_location("durable_restore_live_e2e", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def restore_tool(schema: dict) -> list[dict]: + tools = [ + {"name": name, "description": "", "schema": {}} + for name in sorted(MODULE.BASE_WORKBENCH_TOOLS) + ] + tools.append({"name": MODULE.RESTORE_TOOL, "description": "", "schema": schema}) + return tools + + +def valid_schema() -> dict: + return { + "type": "object", + "required": ["id", "at_snapshot", "destination_id"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "at_snapshot": { + "anyOf": [ + {"type": "integer", "minimum": 0}, + {"type": "string", "minLength": 1}, + ] + }, + "destination_id": {"type": "string", "minLength": 1}, + }, + "additionalProperties": False, + } + + +class ToolContractTests(unittest.TestCase): + def test_accepts_exact_restore_contract(self): + MODULE.validate_tool_contract(restore_tool(valid_schema())) + + def test_rejects_null_restore_snapshot(self): + schema = valid_schema() + schema["properties"]["at_snapshot"]["anyOf"].append({"type": "null"}) + with self.assertRaisesRegex(MODULE.AcceptanceError, "exactly two"): + MODULE.validate_tool_contract(restore_tool(schema)) + + def test_rejects_additional_properties(self): + schema = valid_schema() + schema["additionalProperties"] = True + with self.assertRaisesRegex(MODULE.AcceptanceError, "additional"): + MODULE.validate_tool_contract(restore_tool(schema)) + + def test_rejects_empty_destination_identifier_schema(self): + schema = valid_schema() + del schema["properties"]["destination_id"]["minLength"] + with self.assertRaisesRegex(MODULE.AcceptanceError, "non-empty"): + MODULE.validate_tool_contract(restore_tool(schema)) + + def test_rejects_missing_capability_gated_tool(self): + tools = restore_tool(valid_schema())[:-1] + with self.assertRaisesRegex(MODULE.AcceptanceError, "missing"): + MODULE.validate_tool_contract(tools) + + +class HelpersTests(unittest.TestCase): + def test_manual_gc_defaults_to_production_sized_pages(self): + environment = object.__new__(MODULE.LiveEnvironment) + requests = [] + + def http_text(path, *, timeout, method): + requests.append((path, timeout, method)) + return "{}" + + environment.http_text = http_text + + self.assertEqual(environment.manual_gc(), {}) + self.assertEqual(requests, [("/gc?limit=64", 60, "POST")]) + + def test_index_gate_uses_a_valid_multi_page_search_limit(self): + self.assertGreaterEqual(MODULE.INDEX_PAGE_LIMIT, 1) + self.assertLessEqual(MODULE.INDEX_PAGE_LIMIT, 10) + self.assertLess( + MODULE.INDEX_PAGE_LIMIT, + MODULE.workload_profile("quick").indexed_files, + ) + + def test_full_profile_is_exactly_one_gibibyte(self): + profile = MODULE.workload_profile("full") + self.assertEqual(profile.large_bytes, 1 << 30) + self.assertGreater(profile.indexed_files, 64) + self.assertEqual(MODULE.fixture_block_count(profile.large_bytes), 256) + + def test_each_cow_block_has_a_deterministic_distinct_marker(self): + markers = [MODULE.fixture_block_marker(index) for index in range(256)] + self.assertEqual(len(set(markers)), 256) + self.assertTrue( + all(len(marker) <= MODULE.COW_BLOCK_BYTES for marker in markers) + ) + self.assertEqual(markers[0], MODULE.fixture_block_marker(0)) + with self.assertRaisesRegex(MODULE.AcceptanceError, "4 MiB"): + MODULE.fixture_block_count(MODULE.COW_BLOCK_BYTES + 1) + + def test_binary_digest_hashes_the_exact_file(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "nokv" + path.write_bytes(b"current binary bytes") + self.assertEqual( + MODULE.sha256_file(path), + hashlib.sha256(b"current binary bytes").hexdigest(), + ) + + def test_object_diff_detects_put_overwrite_and_delete(self): + first = MODULE.ObjectFingerprint(10, "one", "t1") + overwritten = MODULE.ObjectFingerprint(10, "one", "t2") + before = {"stable": first, "overwritten": first, "deleted": first} + after = {"stable": first, "overwritten": overwritten, "new": first} + self.assertEqual( + MODULE.changed_objects(before, after), + {"overwritten", "deleted", "new"}, + ) + + def test_structured_error_preserves_retry_contract(self): + error = MODULE.decode_tool_error( + { + "status": "error", + "code": "RestoreInProgress", + "message": "retry", + "retryable": True, + "details": {"operation_id": "restore-1"}, + } + ) + self.assertEqual(error.code, "RestoreInProgress") + self.assertTrue(error.retryable) + + def test_native_structured_error_requires_top_level_fields(self): + native = { + "status": "error", + "code": "RestoreDestinationConflict", + "message": "destination is already claimed", + "retryable": False, + "details": {"destination": "/workbenches/restored"}, + } + error = MODULE.assert_native_tool_error( + native, "RestoreDestinationConflict" + ) + self.assertEqual(error.details["destination"], "/workbenches/restored") + stringified = { + "status": "error", + "message": ( + '{"code":"RestoreDestinationConflict",' + '"message":"destination is already claimed",' + '"retryable":false,"details":{}}' + ), + } + with self.assertRaisesRegex(MODULE.AcceptanceError, "top-level code"): + MODULE.assert_native_tool_error( + stringified, "RestoreDestinationConflict" + ) + + def test_restore_manifest_requires_all_source_and_destination_identity_fields(self): + manifest = { + "schema": "nokv.workbench.restore_manifest.v1", + "operation_id": "restore-1", + "restored_from": { + "workbench_id": "source", + "path": "/workbenches/source", + "snapshot_id": 7, + }, + "source_workbench_id": "source", + "source_path": "/workbenches/source", + "destination_workbench_id": "restored", + "destination_path": "/workbenches/restored", + "snapshot_id": 7, + } + + def validate(candidate): + MODULE.validate_restore_manifest( + candidate, + operation_id="restore-1", + source_workbench_id="source", + source_path="/workbenches/source", + destination_workbench_id="restored", + destination_path="/workbenches/restored", + snapshot_id=7, + ) + + validate(manifest) + for field_path in ( + ("restored_from", "workbench_id"), + ("restored_from", "path"), + ("source_path",), + ("destination_path",), + ): + with self.subTest(field_path=field_path): + malformed = copy.deepcopy(manifest) + target = malformed + for component in field_path[:-1]: + target = target[component] + target[field_path[-1]] = "wrong" + with self.assertRaisesRegex( + MODULE.AcceptanceError, "restore manifest is malformed" + ): + validate(malformed) + + def test_require_all_rejects_quick_profile(self): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + MODULE.parse_args(["--profile", "quick", "--require-all"]) + + def test_require_all_rejects_stale_no_build_mode(self): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + MODULE.parse_args(["--profile", "full", "--require-all", "--no-build"]) + + def test_restore_operation_id_matches_rust_codec_vector(self): + operation_id = MODULE.restore_operation_id( + 1, "/agents/a/wb/source/", 123, "/agents/a/wb/dest" + ) + self.assertEqual( + operation_id, + "restore-6cf0e554391351b42e765d841664687bc0c5fc99af34a84bcdef22f71c27245c", + ) + self.assertNotEqual( + operation_id, + MODULE.restore_operation_id( + 2, "/agents/a/wb/source", 123, "/agents/a/wb/dest" + ), + ) + + def test_restore_barrier_controller_owns_marker_lifecycle(self): + with tempfile.TemporaryDirectory() as temporary: + controller = MODULE.RestoreBarrierController(Path(temporary)) + handle = controller.handle( + "restore-0123456789abcdef", "materialize-batch-000000" + ) + handle.arm() + self.assertTrue(handle.arm_path.exists()) + handle.ready_path.write_text("ready\n", encoding="utf-8") + handle.wait_ready(0.1) + handle.release() + self.assertTrue(handle.continue_path.exists()) + handle.disarm_after_crash() + self.assertFalse(handle.arm_path.exists()) + self.assertFalse(handle.ready_path.exists()) + self.assertFalse(handle.continue_path.exists()) + + def test_optional_barrier_discovery_stops_when_operation_finishes(self): + with tempfile.TemporaryDirectory() as temporary: + controller = MODULE.RestoreBarrierController(Path(temporary)) + handle = controller.handle( + "restore-0123456789abcdef", "reference-batch-000099" + ) + handle.arm() + future = concurrent.futures.Future() + future.set_result({"status": "success"}) + self.assertFalse(handle.wait_ready_or_done(future, 0.1)) + handle.ready_path.write_text("ready\n", encoding="utf-8") + self.assertTrue(handle.wait_ready_or_done(future, 0.1)) + + def test_restore_crash_matrix_covers_cleanup_and_release(self): + phases = MODULE.create_crash_phases(3, 4) + self.assertEqual( + [phase for phase in phases if phase.startswith("materialize-batch-")], + [ + "materialize-batch-000000", + "materialize-batch-000001", + "materialize-batch-000002", + ], + ) + self.assertEqual( + [phase for phase in phases if phase.startswith("reference-batch-")], + [ + "reference-batch-000000", + "reference-batch-000001", + "reference-batch-000002", + "reference-batch-000003", + ], + ) + self.assertEqual(phases[0], "hold-applied") + self.assertEqual(phases[-1], "attach-applied") + self.assertTrue( + MODULE.restore_phase_requires_manifest_rebuild( + "initialization-put-after-000000" + ) + ) + self.assertTrue( + MODULE.restore_phase_requires_manifest_rebuild("reference-batch-000012") + ) + self.assertTrue(MODULE.restore_phase_requires_manifest_rebuild("index-sealed")) + self.assertFalse( + MODULE.restore_phase_requires_manifest_rebuild("attach-applied") + ) + self.assertEqual(MODULE.CLEANUP_CRASH_PHASE, "cleanup-batch-000000") + self.assertEqual(MODULE.RELEASE_CRASH_PHASE, "release-batch-000000") + + def test_restore_stats_and_fsck_validation_fail_closed(self): + metrics = { + "available": True, + "active_marker": True, + "allocator_v2_fenced": True, + "operations": { + "preparing": 0, + "ready_to_attach": 0, + "complete": 1, + "cleaning": 0, + "discarding": 0, + "releasing": 0, + }, + "staging_rows": 8, + "exact_reference_rows": 9, + "index_rows": 10, + "cleanup_backlog": 0, + "release_backlog": 0, + "quarantine_rows": 0, + "control_rows": {"operation": 1}, + } + stats = {"restore": metrics} + MODULE.validate_restore_metrics(stats, expected_complete=1, expect_empty=False) + report = { + "consistent": True, + "dangling_count": 0, + "dangling": [], + "size_mismatch_count": 0, + "size_mismatches": [], + "snapshot_pins_scanned": 0, + "fork_bindings_scanned": 0, + "restore_shards": [ + { + "mount_id": 1, + "report": { + "consistent": True, + "metrics": { + key: value + for key, value in metrics.items() + if key != "available" + }, + "borrowed_objects_checked": 3, + "dangling_borrowed_objects": [], + "borrowed_object_size_mismatches": [], + "issues": [], + }, + } + ], + } + MODULE.validate_fsck_report( + report, + expected_complete=1, + expected_snapshot_pins=0, + expected_fork_bindings=0, + ) + report["restore_shards"][0]["report"]["issues"] = [{"code": "bad"}] + with self.assertRaisesRegex(MODULE.AcceptanceError, "issues"): + MODULE.validate_fsck_report( + report, + expected_complete=1, + expected_snapshot_pins=0, + expected_fork_bindings=0, + ) + + def test_live_fsck_retries_only_the_declared_metadata_race(self): + environment = object.__new__(MODULE.LiveEnvironment) + environment.config = type( + "Config", + (), + {"command_timeout": 1.0, "tool_timeout": 1.0, "gc_deadline": 1.0}, + )() + responses = iter( + [ + MODULE.AcceptanceError( + "NoKV GET /fsck returned HTTP 500: " + + MODULE.RETRYABLE_FSCK_CONFLICT + ), + '{"consistent":true}', + ] + ) + + def retryable_http_text(*_args, **_kwargs): + response = next(responses) + if isinstance(response, Exception): + raise response + return response + + environment.http_text = retryable_http_text + self.assertEqual(environment.fsck(), {"consistent": True}) + + def terminal_http_text(*_args, **_kwargs): + raise MODULE.AcceptanceError( + "NoKV GET /fsck returned HTTP 500: durable corruption" + ) + + environment.http_text = terminal_http_text + with self.assertRaisesRegex(MODULE.AcceptanceError, "durable corruption"): + environment.fsck() + + def test_release_terminal_wait_drives_bounded_gc_pages(self): + suite = object.__new__(MODULE.AcceptanceSuite) + suite.source = "source" + suite.snapshot_id = 7 + suite.env = type( + "Environment", + (), + { + "config": type("Config", (), {"tool_timeout": 1.0})(), + }, + )() + suite.env.gc_limits = [] + release_backlogs = iter([1, 0]) + + def manual_gc(*, limit): + suite.env.gc_limits.append(limit) + return { + "object_gc": { + "restore_release_jobs_processed": 1, + "restore_release_backlog": next(release_backlogs), + "restore_release_quarantine": 0, + "restore_release_mount_wide_quarantine": 0, + } + } + + suite.env.manual_gc = manual_gc + outcomes = iter( + [ + { + "status": "error", + "code": "RestoreInProgress", + "message": "release is pending", + "retryable": True, + "details": {}, + }, + { + "status": "error", + "code": "RestoreInProgress", + "message": "release is pending", + "retryable": True, + "details": {}, + }, + { + "status": "error", + "code": "RestoreDestinationConflict", + "message": "destination now belongs to another workbench", + "retryable": False, + "details": {}, + }, + ] + ) + suite.clients = [ + type( + "Client", + (), + {"raw_call": staticmethod(lambda _name, _arguments: next(outcomes))}, + )() + ] + + error, attempts = suite.wait_for_restore_error( + "destination", "RestoreDestinationConflict" + ) + + self.assertEqual(error.code, "RestoreDestinationConflict") + self.assertEqual(attempts, 3) + self.assertEqual(suite.env.gc_limits, [64, 64]) + + def test_release_terminal_wait_fails_closed_on_quarantine(self): + suite = object.__new__(MODULE.AcceptanceSuite) + suite.source = "source" + suite.snapshot_id = 7 + suite.env = type( + "Environment", + (), + { + "config": type("Config", (), {"tool_timeout": 1.0})(), + }, + )() + suite.env.manual_gc = lambda *, limit: { + "object_gc": { + "restore_release_jobs_processed": 1, + "restore_release_backlog": 1, + "restore_release_quarantine": 1, + "restore_release_mount_wide_quarantine": 0, + } + } + suite.clients = [ + type( + "Client", + (), + { + "raw_call": staticmethod( + lambda _name, _arguments: { + "status": "error", + "code": "RestoreInProgress", + "message": "release is pending", + "retryable": True, + "details": {}, + } + ) + }, + )() + ] + + with self.assertRaisesRegex(MODULE.AcceptanceError, "quarantined"): + suite.wait_for_restore_error( + "destination", "RestoreDestinationConflict" + ) + + def test_released_restore_metrics_allow_only_durable_ledger_rows(self): + metrics = { + "active_marker": True, + "allocator_v2_fenced": True, + "operations": { + "preparing": 0, + "ready_to_attach": 0, + "complete": 0, + "cleaning": 0, + "discarding": 0, + "releasing": 0, + }, + "staging_rows": 0, + "exact_reference_rows": 0, + "index_rows": 0, + "cleanup_backlog": 0, + "release_backlog": 0, + "quarantine_rows": 0, + "control_rows": { + "operation": 0, + "init_upload_tombstone": 2, + "init_upload_tombstone_cursor": 1, + "release_cursor": 1, + }, + } + MODULE.validate_restore_metrics_object( + metrics, expected_complete=0, expect_empty=True + ) + self.assertTrue(MODULE.restore_release_graph_drained(metrics)) + releasing = copy.deepcopy(metrics) + releasing["operations"]["releasing"] = 1 + releasing["control_rows"]["operation"] = 1 + releasing["release_backlog"] = 1 + self.assertFalse(MODULE.restore_release_graph_drained(releasing)) + for leaked_row in ("base_owner", "release_job"): + with self.subTest(leaked_row=leaked_row): + leaked = { + **metrics, + "control_rows": { + **metrics["control_rows"], + leaked_row: 1, + }, + } + with self.assertRaisesRegex(MODULE.AcceptanceError, "did not drain"): + MODULE.validate_restore_metrics_object( + leaked, expected_complete=0, expect_empty=True + ) + + def test_counting_proxy_forwards_and_counts_successful_put(self): + class BackendHandler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, _format, *_args): + return + + def do_PUT(self): + length = int(self.headers["Content-Length"]) + self.server.payload = self.rfile.read(length) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def do_DELETE(self): + self.send_response(204) + self.send_header("Content-Length", "0") + self.end_headers() + + backend = http.server.ThreadingHTTPServer( + ("127.0.0.1", MODULE.free_port()), BackendHandler + ) + proxy = MODULE.CountingProxyServer( + ("127.0.0.1", MODULE.free_port()), backend.server_port + ) + backend_thread = threading.Thread(target=backend.serve_forever, daemon=True) + proxy_thread = threading.Thread(target=proxy.serve_forever, daemon=True) + backend_thread.start() + proxy_thread.start() + try: + connection = http.client.HTTPConnection( + "127.0.0.1", proxy.server_port, timeout=5 + ) + connection.request("PUT", "/bucket/object", body=b"payload") + response = connection.getresponse() + response.read() + connection.close() + self.assertEqual(response.status, 200) + self.assertEqual(backend.payload, b"payload") + self.assertEqual(proxy.put_records(), ["/bucket/object"]) + connection = http.client.HTTPConnection( + "127.0.0.1", proxy.server_port, timeout=5 + ) + connection.request("DELETE", "/bucket/object") + delete_response = connection.getresponse() + delete_response.read() + connection.close() + self.assertEqual(delete_response.status, 204) + self.assertEqual( + proxy.mutation_records(), + [ + MODULE.ObjectMutation("PUT", "/bucket/object"), + MODULE.ObjectMutation("DELETE", "/bucket/object"), + ], + ) + finally: + proxy.shutdown() + backend.shutdown() + proxy.server_close() + backend.server_close() + proxy_thread.join(timeout=5) + backend_thread.join(timeout=5) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/run-metadata-ha-smoke.sh b/scripts/run-metadata-ha-smoke.sh index c2b2fd316..51d84866b 100755 --- a/scripts/run-metadata-ha-smoke.sh +++ b/scripts/run-metadata-ha-smoke.sh @@ -15,14 +15,17 @@ RUSTFS_BIN="${NOKV_RUSTFS_BIN:-rustfs}" ETCD_BIN="${NOKV_ETCD_BIN:-etcd}" AWS_BIN="${NOKV_AWS_BIN:-aws}" CURL_BIN="${NOKV_CURL_BIN:-curl}" +PYTHON_BIN="${NOKV_PYTHON_BIN:-python3}" RUSTFS_ADDRESS="${NOKV_HA_RUSTFS_ADDRESS:-127.0.0.1:9030}" RUSTFS_CONSOLE_ADDRESS="${NOKV_HA_RUSTFS_CONSOLE_ADDRESS:-127.0.0.1:9031}" RUSTFS_ENDPOINT="${NOKV_HA_RUSTFS_ENDPOINT:-http://${RUSTFS_ADDRESS}}" -RUSTFS_BUCKET="${NOKV_HA_RUSTFS_BUCKET:-nokv-ha-smoke}" +RUSTFS_BUCKET_BASE="${NOKV_HA_RUSTFS_BUCKET:-nokv-ha-smoke}" +RUSTFS_BUCKET="" RUSTFS_ACCESS_KEY="${NOKV_HA_RUSTFS_ACCESS_KEY:-rustfsadmin}" RUSTFS_SECRET_KEY="${NOKV_HA_RUSTFS_SECRET_KEY:-rustfsadmin}" RUSTFS_BUFFER_PROFILE="${NOKV_HA_RUSTFS_BUFFER_PROFILE:-AiTraining}" +EXTERNAL_RUSTFS="${NOKV_HA_EXTERNAL_RUSTFS:-0}" ETCD_CLIENT_ADDRESS="${NOKV_HA_ETCD_CLIENT_ADDRESS:-127.0.0.1:12379}" ETCD_PEER_ADDRESS="${NOKV_HA_ETCD_PEER_ADDRESS:-127.0.0.1:12380}" @@ -53,6 +56,13 @@ PRE_INODE=0 POST_INODE=0 AFTER_FAILOVER_INODE=0 CHECKPOINT_COMMIT_VERSION=0 +RESTORE_SNAPSHOT_ID=0 +RESTORE_CHECKPOINT_OPERATION="" +RESTORE_LOG_OPERATION="" +RESTORE_CALL_PID="" +RESTORE_BARRIER_PHASE="index-sealed" +RESTORE_REDRIVE_MAX_ATTEMPTS="${NOKV_HA_RESTORE_REDRIVE_MAX_ATTEMPTS:-256}" +BUCKET_CREATED=0 usage() { cat <&2 + if [[ -n "$last_in_progress" ]]; then + echo " $last_in_progress" >&2 + fi + return 1 +} + +assert_restore_complete() { + local payload="$1" expected_operation="$2" expected_destination="$3" + printf '%s\n' "$payload" | "$PYTHON_BIN" -c ' +import json +import sys + +expected_operation, expected_destination = sys.argv[1:] +payload = json.load(sys.stdin) +if payload.get("status") != "success" or payload.get("state") != "complete": + raise SystemExit(f"restore did not reach terminal Complete: {payload!r}") +if payload.get("cleanup_pending") is not False: + raise SystemExit(f"restore reported cleanup_pending: {payload!r}") +if payload.get("operation_id") != expected_operation: + raise SystemExit( + "restore operation changed across recovery: {!r}".format( + payload.get("operation_id") + ) + ) +if payload.get("destination_workbench_id") != expected_destination: + raise SystemExit(f"restore destination changed: {payload!r}") +' "$expected_operation" "$expected_destination" } cleanup() { local status=$? + if [[ -n "$RESTORE_CALL_PID" ]] && kill -0 "$RESTORE_CALL_PID" >/dev/null 2>&1; then + kill "$RESTORE_CALL_PID" >/dev/null 2>&1 || true + wait "$RESTORE_CALL_PID" >/dev/null 2>&1 || true + fi if [[ -n "$SERVER_B_PID" ]] && kill -0 "$SERVER_B_PID" >/dev/null 2>&1; then kill "$SERVER_B_PID" >/dev/null 2>&1 || true wait "$SERVER_B_PID" >/dev/null 2>&1 || true @@ -121,6 +346,16 @@ cleanup() { kill "$SERVER_A_PID" >/dev/null 2>&1 || true wait "$SERVER_A_PID" >/dev/null 2>&1 || true fi + if [[ "$BUCKET_CREATED" -eq 1 && -n "$RUSTFS_BUCKET" ]]; then + AWS_ACCESS_KEY_ID="$RUSTFS_ACCESS_KEY" \ + AWS_SECRET_ACCESS_KEY="$RUSTFS_SECRET_KEY" \ + "$AWS_BIN" --endpoint-url "$RUSTFS_ENDPOINT" \ + s3 rm "s3://${RUSTFS_BUCKET}" --recursive >/dev/null 2>&1 || true + AWS_ACCESS_KEY_ID="$RUSTFS_ACCESS_KEY" \ + AWS_SECRET_ACCESS_KEY="$RUSTFS_SECRET_KEY" \ + "$AWS_BIN" --endpoint-url "$RUSTFS_ENDPOINT" \ + s3api delete-bucket --bucket "$RUSTFS_BUCKET" >/dev/null 2>&1 || true + fi if [[ "$OWN_ETCD" -eq 1 && -n "$ETCD_PID" ]] && kill -0 "$ETCD_PID" >/dev/null 2>&1; then kill "$ETCD_PID" >/dev/null 2>&1 || true wait "$ETCD_PID" >/dev/null 2>&1 || true @@ -157,6 +392,20 @@ wait_for_url() { return 1 } +wait_for_path() { + local path="$1" pid="$2" name="$3" deadline=$((SECONDS + 120)) + while ((SECONDS < deadline)); do + [[ -f "$path" ]] && return 0 + if [[ -n "$pid" ]] && ! kill -0 "$pid" >/dev/null 2>&1; then + echo "error: $name exited before publishing $path" >&2 + return 1 + fi + sleep 0.05 + done + echo "error: timed out waiting for $name at $path" >&2 + return 1 +} + wait_for_server() { local bind="$1" pid="$2" name="$3" deadline=$((SECONDS + 30)) while ((SECONDS < deadline)); do @@ -191,17 +440,19 @@ wait_for_stale_owner_fence() { create_bucket() { local deadline=$((SECONDS + 30)) + if AWS_ACCESS_KEY_ID="$RUSTFS_ACCESS_KEY" \ + AWS_SECRET_ACCESS_KEY="$RUSTFS_SECRET_KEY" \ + "$AWS_BIN" --endpoint-url "$RUSTFS_ENDPOINT" \ + s3api head-bucket --bucket "$RUSTFS_BUCKET" >/dev/null 2>&1; then + echo "error: supposedly unique HA bucket already exists: $RUSTFS_BUCKET" >&2 + return 1 + fi while ((SECONDS < deadline)); do if AWS_ACCESS_KEY_ID="$RUSTFS_ACCESS_KEY" \ AWS_SECRET_ACCESS_KEY="$RUSTFS_SECRET_KEY" \ "$AWS_BIN" --endpoint-url "$RUSTFS_ENDPOINT" \ s3api create-bucket --bucket "$RUSTFS_BUCKET" >/dev/null 2>&1; then - return 0 - fi - if AWS_ACCESS_KEY_ID="$RUSTFS_ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$RUSTFS_SECRET_KEY" \ - "$AWS_BIN" --endpoint-url "$RUSTFS_ENDPOINT" \ - s3api head-bucket --bucket "$RUSTFS_BUCKET" >/dev/null 2>&1; then + BUCKET_CREATED=1 return 0 fi sleep 0.5 @@ -218,6 +469,14 @@ if [[ "$STALE_OWNER_CHAOS" != "0" && "$STALE_OWNER_CHAOS" != "1" ]]; then echo "error: NOKV_HA_STALE_OWNER_CHAOS must be 0 or 1" >&2 exit 2 fi +if [[ "$EXTERNAL_RUSTFS" != "0" && "$EXTERNAL_RUSTFS" != "1" ]]; then + echo "error: NOKV_HA_EXTERNAL_RUSTFS must be 0 or 1" >&2 + exit 2 +fi +if ! [[ "$RESTORE_REDRIVE_MAX_ATTEMPTS" =~ ^[1-9][0-9]*$ ]]; then + echo "error: NOKV_HA_RESTORE_REDRIVE_MAX_ATTEMPTS must be a positive integer" >&2 + exit 2 +fi OWNER_A_BIND="${NOKV_HA_OWNER_A_BIND:-$SERVER_BIND}" OWNER_B_BIND="$SERVER_BIND" @@ -225,9 +484,12 @@ if [[ "$STALE_OWNER_CHAOS" == "1" ]]; then OWNER_B_BIND="${NOKV_HA_OWNER_B_BIND:-127.0.0.1:7731}" fi -require_cmd "$RUSTFS_BIN" +if [[ "$EXTERNAL_RUSTFS" == "0" ]]; then + require_cmd "$RUSTFS_BIN" +fi require_cmd "$AWS_BIN" require_cmd "$CURL_BIN" +require_cmd "$PYTHON_BIN" if [[ -z "${NOKV_HA_ETCD_ENDPOINTS:-}" ]]; then require_cmd "$ETCD_BIN" OWN_ETCD=1 @@ -237,9 +499,12 @@ WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/nokv-ha-smoke.XXXXXX")" mkdir -p "$WORK_DIR/rustfs" "$WORK_DIR/etcd" "$WORK_DIR/meta-a" "$WORK_DIR/meta-b" UNIQUE="$(date +%s)-$$" +RUSTFS_BUCKET="${RUSTFS_BUCKET_BASE:0:40}-${UNIQUE}" ETCD_PREFIX="${NOKV_HA_ETCD_PREFIX:-/nokv/ha-smoke/${UNIQUE}}" CHECKPOINT_PREFIX="${NOKV_HA_CHECKPOINT_PREFIX:-metadata/ha-smoke/${UNIQUE}/checkpoints}" SHARED_LOG_PREFIX="${NOKV_HA_SHARED_LOG_PREFIX:-metadata/ha-smoke/${UNIQUE}/shared-log}" +RESTORE_BARRIER_DIR="$WORK_DIR/restore-barriers" +mkdir -p "$RESTORE_BARRIER_DIR" S3_ARGS=( --object-backend rustfs @@ -268,16 +533,20 @@ echo "==> building nokv with etcd feature" ) NOKV="$HA_CARGO_TARGET_DIR/debug/nokv" -echo "==> starting RustFS at $RUSTFS_ENDPOINT" -RUSTFS_ACCESS_KEY="$RUSTFS_ACCESS_KEY" \ - RUSTFS_SECRET_KEY="$RUSTFS_SECRET_KEY" \ - "$RUSTFS_BIN" server \ - --address "$RUSTFS_ADDRESS" \ - --console-enable \ - --console-address "$RUSTFS_CONSOLE_ADDRESS" \ - --buffer-profile "$RUSTFS_BUFFER_PROFILE" \ - "$WORK_DIR/rustfs" >"$WORK_DIR/rustfs.log" 2>&1 & -RUSTFS_PID=$! +if [[ "$EXTERNAL_RUSTFS" == "0" ]]; then + echo "==> starting RustFS at $RUSTFS_ENDPOINT" + RUSTFS_ACCESS_KEY="$RUSTFS_ACCESS_KEY" \ + RUSTFS_SECRET_KEY="$RUSTFS_SECRET_KEY" \ + "$RUSTFS_BIN" server \ + --address "$RUSTFS_ADDRESS" \ + --console-enable \ + --console-address "$RUSTFS_CONSOLE_ADDRESS" \ + --buffer-profile "$RUSTFS_BUFFER_PROFILE" \ + "$WORK_DIR/rustfs" >"$WORK_DIR/rustfs.log" 2>&1 & + RUSTFS_PID=$! +else + echo "==> using external RustFS at $RUSTFS_ENDPOINT" +fi wait_for_url "$RUSTFS_ENDPOINT" RustFS create_bucket @@ -300,6 +569,8 @@ FIRST_ETCD_ENDPOINT="${ETCD_ENDPOINTS%%,*}" wait_for_url "${FIRST_ETCD_ENDPOINT%/}/health" etcd echo "==> starting owner A with etcd lease and sync shared-log" +NOKV_TEST_RESTORE_BARRIER_DIR="$RESTORE_BARRIER_DIR" \ +NOKV_TEST_BARRIER_TIMEOUT_MS=300000 \ "$NOKV" \ --meta "$WORK_DIR/meta-a" \ --server-bind "$OWNER_A_BIND" \ @@ -321,11 +592,27 @@ echo "$PRE_OUT" PRE_INODE="$(printf '%s\n' "$PRE_OUT" | extract_inode)" "$NOKV" "${CLIENT_A[@]}" ls /runs | grep -q "pre.bin" +echo "==> creating a durable workbench restore before checkpoint" +mcp_call "$OWNER_A_BIND" workbench_create \ + '{"id":"ha-restore-source"}' >/dev/null +mcp_call "$OWNER_A_BIND" workbench_put_file \ + '{"id":"ha-restore-source","section":"outputs","path":"result.txt","text":"ha-restore-source\n"}' >/dev/null +mcp_call "$OWNER_A_BIND" workbench_commit \ + '{"id":"ha-restore-source","manifest":{"test":"metadata-ha-smoke"}}' >/dev/null +RESTORE_SNAPSHOT_JSON="$(mcp_call "$OWNER_A_BIND" workbench_snapshot \ + '{"id":"ha-restore-source","name":"failover","ttl_days":7}')" +RESTORE_SNAPSHOT_ID="$(printf '%s\n' "$RESTORE_SNAPSHOT_JSON" | json_field snapshot_id)" +RESTORE_CHECKPOINT_JSON="$(mcp_call "$OWNER_A_BIND" workbench_restore \ + "{\"id\":\"ha-restore-source\",\"at_snapshot\":${RESTORE_SNAPSHOT_ID},\"destination_id\":\"ha-restore-checkpoint\"}")" +RESTORE_CHECKPOINT_OPERATION="$(printf '%s\n' "$RESTORE_CHECKPOINT_JSON" | json_field operation_id)" +assert_restore_complete \ + "$RESTORE_CHECKPOINT_JSON" "$RESTORE_CHECKPOINT_OPERATION" "ha-restore-checkpoint" + echo "==> publishing checkpoint ref through owner A" BACKUP_JSON="$("$NOKV" "${CLIENT_A[@]}" backup)" echo " $BACKUP_JSON" echo "$BACKUP_JSON" | grep -q '"checkpoint_key"' -CHECKPOINT_COMMIT_VERSION="$(printf '%s\n' "$BACKUP_JSON" | extract_json_number commit_version)" +CHECKPOINT_COMMIT_VERSION="$(printf '%s\n' "$BACKUP_JSON" | json_field commit_version)" echo "==> writing data after checkpoint so failover must replay shared log" POST_OUT="$("$NOKV" "${CLIENT_A[@]}" put-artifact /runs/post.bin "$WORK_DIR/post.bin")" @@ -333,6 +620,27 @@ echo "$POST_OUT" POST_INODE="$(printf '%s\n' "$POST_OUT" | extract_inode)" "$NOKV" "${CLIENT_A[@]}" cat /runs/post.bin | grep -q "ha-smoke-post-checkpoint" +echo "==> holding a second durable restore in-flight in the post-checkpoint shared log" +RESTORE_LOG_OPERATION="$(restore_operation_id "$RESTORE_SNAPSHOT_ID" ha-restore-log)" +RESTORE_BARRIER_STEM="${RESTORE_LOG_OPERATION}.${RESTORE_BARRIER_PHASE}" +RESTORE_BARRIER_ARM="$RESTORE_BARRIER_DIR/${RESTORE_BARRIER_STEM}.arm" +RESTORE_BARRIER_READY="$RESTORE_BARRIER_DIR/${RESTORE_BARRIER_STEM}.ready" +RESTORE_BARRIER_CONTINUE="$RESTORE_BARRIER_DIR/${RESTORE_BARRIER_STEM}.continue" +rm -f "$RESTORE_BARRIER_READY" "$RESTORE_BARRIER_CONTINUE" +printf 'armed\n' >"$RESTORE_BARRIER_ARM" +( + mcp_call "$OWNER_A_BIND" workbench_restore \ + "{\"id\":\"ha-restore-source\",\"at_snapshot\":${RESTORE_SNAPSHOT_ID},\"destination_id\":\"ha-restore-log\"}" \ + >"$WORK_DIR/inflight-restore.out" 2>"$WORK_DIR/inflight-restore.err" +) & +RESTORE_CALL_PID=$! +wait_for_path "$RESTORE_BARRIER_READY" "$RESTORE_CALL_PID" "in-flight restore" +if [[ -s "$WORK_DIR/inflight-restore.out" ]]; then + echo "error: in-flight restore returned before failover" >&2 + cat "$WORK_DIR/inflight-restore.out" >&2 + exit 1 +fi + if [[ "$STALE_OWNER_CHAOS" == "1" ]]; then echo "==> pausing owner A and waiting for etcd lease expiry" else @@ -346,10 +654,19 @@ else wait "$SERVER_A_PID" >/dev/null 2>&1 || true SERVER_A_PID="" fi +if [[ -n "$RESTORE_CALL_PID" ]] && kill -0 "$RESTORE_CALL_PID" >/dev/null 2>&1; then + kill "$RESTORE_CALL_PID" >/dev/null 2>&1 || true + wait "$RESTORE_CALL_PID" >/dev/null 2>&1 || true +fi +RESTORE_CALL_PID="" +rm -f "$RESTORE_BARRIER_ARM" "$RESTORE_BARRIER_READY" +printf 'continue\n' >"$RESTORE_BARRIER_CONTINUE" sleep "$((ETCD_TTL_SECONDS + 2))" echo "==> starting owner B as failover from epoch 1" OWNER_B_START_MS="$(now_ms)" +NOKV_TEST_RESTORE_BARRIER_DIR="$RESTORE_BARRIER_DIR" \ +NOKV_TEST_BARRIER_TIMEOUT_MS=300000 \ "$NOKV" \ --meta "$WORK_DIR/meta-b" \ --server-bind "$OWNER_B_BIND" \ @@ -363,13 +680,36 @@ wait_for_server "$OWNER_B_BIND" "$SERVER_B_PID" "nokv owner B" OWNER_B_READY_MS="$(now_ms)" STATS="$("$CURL_BIN" -fsS "http://${OWNER_B_BIND}/stats")" -echo "$STATS" | grep -q '"node_id":"node-b"' -echo "$STATS" | grep -q '"epoch":2' +OWNER_B_RECOVERY_SUMMARY="$(printf '%s\n' "$STATS" | assert_owner_b_recovered_inflight)" +echo " $OWNER_B_RECOVERY_SUMMARY" echo "==> verifying checkpoint restore and shared-log replay" "$NOKV" "${CLIENT_B[@]}" cat /runs/pre.bin | grep -q "ha-smoke-pre-checkpoint" "$NOKV" "${CLIENT_B[@]}" cat /runs/post.bin | grep -q "ha-smoke-post-checkpoint" +echo "==> verifying Complete restore from checkpoint and redriving in-flight log restore" +RECOVERED_CHECKPOINT_JSON="$(mcp_call "$OWNER_B_BIND" workbench_restore \ + "{\"id\":\"ha-restore-source\",\"at_snapshot\":${RESTORE_SNAPSHOT_ID},\"destination_id\":\"ha-restore-checkpoint\"}")" +assert_restore_complete \ + "$RECOVERED_CHECKPOINT_JSON" "$RESTORE_CHECKPOINT_OPERATION" "ha-restore-checkpoint" +RECOVERED_LOG_JSON="$(mcp_restore_until_complete "$OWNER_B_BIND" \ + "{\"id\":\"ha-restore-source\",\"at_snapshot\":${RESTORE_SNAPSHOT_ID},\"destination_id\":\"ha-restore-log\"}")" +assert_restore_complete "$RECOVERED_LOG_JSON" "$RESTORE_LOG_OPERATION" "ha-restore-log" +"$NOKV" "${CLIENT_B[@]}" \ + cat /workbenches/ha-restore-checkpoint/outputs/result.txt \ + | grep -q "ha-restore-source" +"$NOKV" "${CLIENT_B[@]}" \ + cat /workbenches/ha-restore-log/outputs/result.txt \ + | grep -q "ha-restore-source" + +echo "==> retiring the source pin and verifying terminal restore redrive" +"$NOKV" "${CLIENT_B[@]}" retire-snapshot \ + /workbenches/ha-restore-source "$RESTORE_SNAPSHOT_ID" \ + | grep -q 'retired=true' +REDRIVEN_LOG_JSON="$(mcp_call "$OWNER_B_BIND" workbench_restore \ + "{\"id\":\"ha-restore-source\",\"at_snapshot\":${RESTORE_SNAPSHOT_ID},\"destination_id\":\"ha-restore-log\"}")" +assert_restore_complete "$REDRIVEN_LOG_JSON" "$RESTORE_LOG_OPERATION" "ha-restore-log" + echo "==> verifying owner B accepts new writes without clobbering replayed data" AFTER_OUT="$("$NOKV" "${CLIENT_B[@]}" mkdir /after-failover)" echo "$AFTER_OUT" @@ -385,7 +725,8 @@ fi echo "==> running fsck after failover" FSCK_JSON="$("$NOKV" "${CLIENT_B[@]}" fsck)" echo " $FSCK_JSON" -echo "$FSCK_JSON" | grep -q '"dangling_count":0' +FSCK_SUMMARY="$(printf '%s\n' "$FSCK_JSON" | assert_fsck_clean 2 0 0)" +echo " strict $FSCK_SUMMARY" if [[ "$STALE_OWNER_CHAOS" == "1" ]]; then echo "==> resuming owner A and verifying stale-owner fencing" @@ -406,6 +747,7 @@ if [[ "$STALE_OWNER_CHAOS" == "1" ]]; then STALE_OWNER_FENCE_MS="$(now_ms)" fi VERIFY_DONE_MS="$(now_ms)" +rm -f "$RESTORE_BARRIER_ARM" "$RESTORE_BARRIER_READY" "$RESTORE_BARRIER_CONTINUE" FAILOVER_OBSERVED_MS="$((OWNER_B_READY_MS - OWNER_A_KILL_MS))" LEASE_WAIT_MS="$((OWNER_B_START_MS - OWNER_A_KILL_MS))" @@ -417,7 +759,7 @@ if [[ "$STALE_OWNER_CHAOS" == "1" ]]; then STALE_OWNER_DETECT_AFTER_RESUME_MS="$((STALE_OWNER_DETECT_MS - OWNER_A_RESUME_MS))" STALE_OWNER_FENCE_AFTER_DETECT_MS="$((STALE_OWNER_FENCE_MS - STALE_OWNER_DETECT_MS))" fi -METRICS_JSON="{\"lease_ttl_seconds\":${ETCD_TTL_SECONDS},\"stale_owner_chaos\":${STALE_OWNER_CHAOS},\"owner_a_ready_ms\":${OWNER_A_READY_MS},\"owner_a_kill_ms\":${OWNER_A_KILL_MS},\"owner_a_resume_ms\":${OWNER_A_RESUME_MS},\"owner_b_start_ms\":${OWNER_B_START_MS},\"owner_b_ready_ms\":${OWNER_B_READY_MS},\"failover_observed_ms\":${FAILOVER_OBSERVED_MS},\"lease_wait_ms\":${LEASE_WAIT_MS},\"owner_b_startup_ms\":${OWNER_B_STARTUP_MS},\"verify_after_ready_ms\":${VERIFY_AFTER_READY_MS},\"stale_owner_detect_after_resume_ms\":${STALE_OWNER_DETECT_AFTER_RESUME_MS},\"stale_owner_fence_after_detect_ms\":${STALE_OWNER_FENCE_AFTER_DETECT_MS},\"checkpoint_commit_version\":${CHECKPOINT_COMMIT_VERSION:-0},\"pre_inode\":${PRE_INODE:-0},\"post_checkpoint_inode\":${POST_INODE:-0},\"after_failover_inode\":${AFTER_FAILOVER_INODE:-0}}" +METRICS_JSON="{\"lease_ttl_seconds\":${ETCD_TTL_SECONDS},\"stale_owner_chaos\":${STALE_OWNER_CHAOS},\"owner_a_ready_ms\":${OWNER_A_READY_MS},\"owner_a_kill_ms\":${OWNER_A_KILL_MS},\"owner_a_resume_ms\":${OWNER_A_RESUME_MS},\"owner_b_start_ms\":${OWNER_B_START_MS},\"owner_b_ready_ms\":${OWNER_B_READY_MS},\"failover_observed_ms\":${FAILOVER_OBSERVED_MS},\"lease_wait_ms\":${LEASE_WAIT_MS},\"owner_b_startup_ms\":${OWNER_B_STARTUP_MS},\"verify_after_ready_ms\":${VERIFY_AFTER_READY_MS},\"stale_owner_detect_after_resume_ms\":${STALE_OWNER_DETECT_AFTER_RESUME_MS},\"stale_owner_fence_after_detect_ms\":${STALE_OWNER_FENCE_AFTER_DETECT_MS},\"checkpoint_commit_version\":${CHECKPOINT_COMMIT_VERSION:-0},\"restore_snapshot_id\":${RESTORE_SNAPSHOT_ID:-0},\"restore_inflight_recovered\":true,\"restore_inflight_phase\":\"${RESTORE_BARRIER_PHASE}\",\"restore_inflight_operation\":\"${RESTORE_LOG_OPERATION}\",\"rustfs_bucket\":\"${RUSTFS_BUCKET}\",\"fsck_consistent\":true,\"pre_inode\":${PRE_INODE:-0},\"post_checkpoint_inode\":${POST_INODE:-0},\"after_failover_inode\":${AFTER_FAILOVER_INODE:-0}}" if [[ -n "${NOKV_HA_METRICS_JSON:-}" ]]; then printf '%s\n' "$METRICS_JSON" >"$NOKV_HA_METRICS_JSON" fi @@ -427,4 +769,4 @@ echo "HA_SMOKE_METRICS $METRICS_JSON" if [[ "$STALE_OWNER_CHAOS" == "1" ]]; then echo "HA_STALE_OWNER_OK: resumed epoch-1 owner observed epoch 2 and rejected a stale write" fi -echo "HA_SMOKE_OK: etcd owner failover restored checkpoint, replayed shared log, and served epoch 2" +echo "HA_SMOKE_OK: etcd owner failover restored checkpoint/log state, redrove an in-flight durable workbench restore, passed strict fsck, and served epoch 2"