From eb8cf499a677a08fdd829979731e799a8814211c Mon Sep 17 00:00:00 2001 From: Rajib Dutta Date: Wed, 5 Aug 2026 01:21:06 +0000 Subject: [PATCH 1/7] Adding dma io zeroization. --- fw/plat/uno/fw/pal/src/alloc.rs | 18 ++++ fw/plat/uno/fw/pal/src/gdma.rs | 104 ++++++++++++++++++++++++ fw/plat/uno/fw/pal/src/io.rs | 15 +++- fw/plat/uno/fw/reg/soc/src/dummy_mem.rs | 23 ++++++ fw/plat/uno/fw/reg/soc/src/lib.rs | 1 + fw/plat/uno/rdl/soc/dummy_mem.rdl | 41 ++++++++++ fw/plat/uno/rdl/soc/uno.rdl | 3 + 7 files changed, 202 insertions(+), 3 deletions(-) create mode 100644 fw/plat/uno/fw/reg/soc/src/dummy_mem.rs create mode 100644 fw/plat/uno/rdl/soc/dummy_mem.rdl diff --git a/fw/plat/uno/fw/pal/src/alloc.rs b/fw/plat/uno/fw/pal/src/alloc.rs index 2f43d200e..b8cfabf62 100644 --- a/fw/plat/uno/fw/pal/src/alloc.rs +++ b/fw/plat/uno/fw/pal/src/alloc.rs @@ -159,6 +159,24 @@ fn heap_base_cap(io_index: u16, heap: usize) -> (*mut u8, usize) { } } +/// Per-IO SRAM (DMA heap) buffer region as `(base_ptr, capacity)`. +/// +/// Returns the whole slot-local `SRAM_IO_BUF[io_index]` region (not just +/// the used watermark) so it can be scrubbed in full on IO teardown; see +/// [`UnoHsmPal::scrub_io_slot`](crate::UnoHsmPal::scrub_io_slot). +#[inline(always)] +pub(crate) fn io_slot_dma_region(io_index: u16) -> (*mut u8, usize) { + heap_base_cap(io_index, DMA) +} + +/// Per-IO DTCM (NonDma heap) buffer region as `(base_ptr, capacity)`. +/// +/// Companion to [`io_slot_dma_region`] for the fast/NonDma scratch heap. +#[inline(always)] +pub(crate) fn io_slot_nondma_region(io_index: u16) -> (*mut u8, usize) { + heap_base_cap(io_index, NONDMA) +} + /// Access the watermark cell for one `(io_index, heap)` pair. /// /// # Parameters diff --git a/fw/plat/uno/fw/pal/src/gdma.rs b/fw/plat/uno/fw/pal/src/gdma.rs index 537f6a2ea..d975f4f29 100644 --- a/fw/plat/uno/fw/pal/src/gdma.rs +++ b/fw/plat/uno/fw/pal/src/gdma.rs @@ -35,6 +35,9 @@ use azihsm_fw_hsm_pal_traits::HsmResult; use azihsm_fw_uno_drivers_gdma::GdmaAddr; use azihsm_fw_uno_drivers_gdma::GdmaBuf; use azihsm_fw_uno_drivers_gdma::MemInterface; +use azihsm_fw_uno_reg_soc::dummy_mem::DUMMY_MEM_BASE; +use azihsm_fw_uno_reg_soc::dummy_mem::DUMMY_MEM_SIZE; +use azihsm_fw_uno_trace::tracing::*; use crate::UnoHsmPal; @@ -218,3 +221,104 @@ impl HsmGdmaController for UnoHsmPal { .await } } + +impl UnoHsmPal { + /// GDMA-zero a GDMA-reachable device region `[dst_ptr, dst_ptr + len)`. + /// + /// Copies zeros from the [`DUMMY_MEM_BASE`] window, chunked to its + /// [`DUMMY_MEM_SIZE`] so an arbitrarily large region can be wiped + /// with a fixed 16 KiB zero source. Both operands use the device + /// interface. Only valid for GDMA-reachable memory (GSRAM); the M7 TCM + /// is not on the GDMA fabric and must be wiped by the CPU. + async fn gdma_zero_region(&self, dst_ptr: *mut u8, len: usize) -> HsmResult<()> { + let mut off = 0usize; + while off < len { + let chunk = core::cmp::min(len - off, DUMMY_MEM_SIZE as usize); + let src = device_dma_buf(DUMMY_MEM_BASE as *const u8, chunk as u32); + // SAFETY: `off < len`, so `dst_ptr + off` stays within the + // caller-provided `[dst_ptr, dst_ptr + len)` region. + let dst = device_dma_buf(unsafe { dst_ptr.add(off) }, chunk as u32); + self.gdma + .copy_mem( + src, + MemInterface::Device, + chunk as u32, + dst, + MemInterface::Device, + chunk as u32, + )? + .await?; + off += chunk; + } + Ok(()) + } + + /// Securely scrub both per-IO scratch buffers for `io_index`. + /// + /// Called on IO teardown so no key material from one IO survives into + /// the next IO that reuses the slot. On Uno every crypto operand lives + /// in these bump-allocated buffers (no heap), so wiping the full slot is + /// what protects private keys / plaintext — the CPU-visible arena is the + /// only place they exist. + /// + /// The SRAM (DMA) buffer is wiped off-CPU with the GDMA engine; the DTCM + /// (NonDma) buffer sits in the M7 TCM, which GDMA cannot reach, so it is + /// wiped by the CPU. If the GDMA wipe fails (e.g. no free tags under + /// heavy concurrency), the SRAM buffer is scrubbed by the CPU as a + /// fallback so secrets are never left resident. + pub(crate) async fn scrub_io_slot(&self, io_index: u16) { + let (dma_ptr, dma_len) = crate::alloc::io_slot_dma_region(io_index); + if self.gdma_zero_region(dma_ptr, dma_len).await.is_err() { + // Surface the fallback: a persistent GDMA fault would otherwise + // silently degrade every wipe into a byte-wise CPU loop with no + // signal. Logged via the trace facade (as the iic/oic drivers do + // for unexpected conditions); compiled out when no trace backend + // is selected. Correctness is unaffected — the CPU wipe below + // still fully scrubs the buffer. + warn!( + "gdma", + "SRAM scrub for slot {} fell back to CPU (GDMA wipe failed)", io_index + ); + // SAFETY: `(dma_ptr, dma_len)` describes the slot's SRAM buffer, + // valid for writes over its whole capacity. + unsafe { cpu_zeroize(dma_ptr, dma_len) }; + } + // DTCM (NonDma) heap — not GDMA-reachable, so wipe with the CPU. + let (nd_ptr, nd_len) = crate::alloc::io_slot_nondma_region(io_index); + // SAFETY: `(nd_ptr, nd_len)` describes the slot's DTCM buffer, valid + // for writes over its whole capacity. + unsafe { cpu_zeroize(nd_ptr, nd_len) }; + } +} + +/// Volatile CPU zeroization of `[ptr, ptr + len)` that the optimizer cannot +/// elide, mirroring [`DmaBuf::zeroize`]. Used for the M7 TCM (which GDMA +/// cannot reach) and as the GDMA-failure fallback for SRAM. +/// +/// Writes 32-bit words when the region is word-aligned, and byte-wise +/// otherwise (or for a sub-word tail). Both per-IO regions are word-aligned +/// with word-multiple sizes (`DTCM_IO_BUF` 0x600, `SRAM_IO_BUF` 0x4800), so +/// in practice this is a pure word loop — 4x fewer stores than a byte-wise +/// wipe on the IO teardown path. +/// +/// # Safety +/// +/// `ptr` must be valid for writes of `len` bytes. +#[inline] +unsafe fn cpu_zeroize(ptr: *mut u8, len: usize) { + let mut i = 0usize; + // Word body: 4 bytes per store (both per-IO regions hit this path). + while i + 4 <= len && (ptr as usize + i) % 4 == 0 { + // SAFETY: `i + 4 <= len` and the address is 4-byte aligned, so this + // writes wholly within the caller's region at a valid alignment. + unsafe { core::ptr::write_volatile(ptr.add(i) as *mut u32, 0) }; + i += 4; + } + // Misaligned region or sub-word tail. + while i < len { + // SAFETY: `i < len`, so `ptr + i` is within the caller's region. + unsafe { core::ptr::write_volatile(ptr.add(i), 0) }; + i += 1; + } + core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); +} diff --git a/fw/plat/uno/fw/pal/src/io.rs b/fw/plat/uno/fw/pal/src/io.rs index ad41199ab..55c1b4062 100644 --- a/fw/plat/uno/fw/pal/src/io.rs +++ b/fw/plat/uno/fw/pal/src/io.rs @@ -18,7 +18,7 @@ //! | `IO_CQ[index]` | 16B CQE | Completion queue entry (write) | //! | `IO_META[index]` | 8B metadata | Controller/queue IDs from IIC recv | //! | `DTCM_IO_BUF[index]` | 1.5KB fmem | Fast DTCM workspace buffer | -//! | `SRAM_IO_BUF[index]` | 8KB smem | Large SRAM workspace buffer | +//! | `SRAM_IO_BUF[index]` | 18KB smem | Large SRAM workspace buffer | //! //! The IIC controller DMAs incoming SQE data directly into `IO_SQ[index]` //! (configured via `io_pool_base`). The firmware reads the SQE in-place @@ -166,10 +166,19 @@ impl HsmIoController for UnoHsmPal { } /// Drops an IO without sending a completion (e.g. for disabled - /// partitions). Returns the IO_SQ slot to the ISQ. - #[allow(clippy::unused_async)] + /// partitions). Scrubs the slot's scratch buffers, then returns the + /// IO_SQ slot to the ISQ. + /// + /// This is the universal IO teardown point (the core dispatch loop + /// calls it on both the completed and the dropped paths), so the + /// per-IO buffer scrub lives here. See + /// [`scrub_io_slot`](UnoHsmPal::scrub_io_slot). async fn drop_io(&self, io: Self::Io) -> HsmResult<()> { let queue_id = io.queue_id(); + // Scrub both per-IO scratch buffers *before* returning the slot to + // the ISQ, so no key material from this IO can be observed by the + // next IO that reuses the slot. + self.scrub_io_slot(io.index).await; self.iic.free_io(io.index, queue_id); Ok(()) } diff --git a/fw/plat/uno/fw/reg/soc/src/dummy_mem.rs b/fw/plat/uno/fw/reg/soc/src/dummy_mem.rs new file mode 100644 index 000000000..d274e3ab2 --- /dev/null +++ b/fw/plat/uno/fw/reg/soc/src/dummy_mem.rs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// AUTO-GENERATED by azihsm_reggen — do not edit. + +//! Register definitions for dummy_mem. +//! +//! 'Passive read-as-zero region (16 KB at 0xA0B0_0000), used as the constant zero source for GDMA-based buffer wiping.' + +pub const DUMMY_MEM_BASE: u32 = 0xA0B00000; +pub const DUMMY_MEM_OFFSET: u32 = 0x00; +pub const DUMMY_MEM_COUNT: u32 = 1; +pub const DUMMY_MEM_STRIDE: u32 = 0x4000; +pub const DUMMY_MEM_SIZE: u32 = 0x4000; + +pub mod regs { + //! MMIO register struct for firmware use. + tock_registers::register_structs! { + pub DummyMemRegs { + (0x0 => pub dummy_mem: [u8; 16384]), + (0x4000 => @END), + } + } +} diff --git a/fw/plat/uno/fw/reg/soc/src/lib.rs b/fw/plat/uno/fw/reg/soc/src/lib.rs index 44aa63366..41b45f5a3 100644 --- a/fw/plat/uno/fw/reg/soc/src/lib.rs +++ b/fw/plat/uno/fw/reg/soc/src/lib.rs @@ -28,6 +28,7 @@ mod access { pub use access::*; pub mod aes; pub mod dual_cp_m7; +pub mod dummy_mem; pub mod gdma; pub mod hsm_dtcm; pub mod iic; diff --git a/fw/plat/uno/rdl/soc/dummy_mem.rdl b/fw/plat/uno/rdl/soc/dummy_mem.rdl new file mode 100644 index 000000000..37174d930 --- /dev/null +++ b/fw/plat/uno/rdl/soc/dummy_mem.rdl @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// +// Dummy Memory address map (dummy_mem). +// +// A passive, register-less AXI target at 0xA0B0_0000 (16 KiB). Reads +// return zero, so it is used as the constant zero source for GDMA-based +// buffer wiping: the GDMA engine has no native fill/memset — an SQ entry's +// OP_CODE only encodes read-src / write-dst copy — so scrubbing a buffer +// means DMA-copying zeros out of this region. Per the LionMS SoC memory +// map ("Dummy Memory", 16 KiB @ 0xA0B0_0000), the region is reserved for +// exactly this purpose. + + +// ══════════════════════════════════════════════════════════════ +// Read-as-zero window +// ══════════════════════════════════════════════════════════════ + +mem dummy_mem_t { + desc = "16KB read-as-zero AXI target. The constant zero source for GDMA + buffer zeroization; GDMA has no fill/memset opcode, so a wipe is + a DMA copy whose source is this region."; + mementries = 4096; // 4096 × 32 bits = 16384 B = 16 KB + memwidth = 32; +}; + + +// ══════════════════════════════════════════════════════════════ +// Address Map +// ══════════════════════════════════════════════════════════════ + +addrmap dummy_mem { + name = "Dummy Memory"; + desc = "Passive read-as-zero region (16 KB at 0xA0B0_0000), used as the + constant zero source for GDMA-based buffer wiping."; + + default hw = na; + default sw = r; + + dummy_mem_t DUMMY_MEM @ 0x0; +}; diff --git a/fw/plat/uno/rdl/soc/uno.rdl b/fw/plat/uno/rdl/soc/uno.rdl index 611c3b40e..dfd59f9c9 100644 --- a/fw/plat/uno/rdl/soc/uno.rdl +++ b/fw/plat/uno/rdl/soc/uno.rdl @@ -13,6 +13,7 @@ // 0xA020_0000 AES 4 KB // 0xA030_0000 RNG 4 KB // 0xA040_0000 SHA 4 KB +// 0xA0B0_0000 DummyMemory 16 KB (read-as-zero; GDMA zero source) // 0xA128_0000 IIC 16 KB (controller 0) // 0xA12C_0000 OIC 16 KB (controller 0) // 0xB000_6000 INTC 272 B (IPC interrupt controller) @@ -27,6 +28,7 @@ `include "gdma.rdl" `include "aes.rdl" `include "sha.rdl" +`include "dummy_mem.rdl" `include "upka.rdl" `include "rng.rdl" `include "intc.rdl" @@ -51,6 +53,7 @@ addrmap uno { aes AES @ 0xA020_0000; rng RNG @ 0xA030_0000; sha SHA @ 0xA040_0000; + dummy_mem DUMMY_MEM @ 0xA0B0_0000; iic IIC @ 0xA128_0000; oic OIC @ 0xA12C_0000; intc INTC @ 0xB000_6000; From a84b02a7a2741e464fe5e12bef026d9fc4eea8bb Mon Sep 17 00:00:00 2001 From: Rajib Dutta Date: Wed, 5 Aug 2026 01:51:36 +0000 Subject: [PATCH 2/7] Scrub the admin IO slot via a with_admin_io session Internal provisioning (partition identity, enable-time keygen, boot-key masking, vault teardown) runs on the reserved admin IO slot rather than a host IO, so it never reaches drop_io. Its scratch was only watermark- rewound, never wiped, leaving raw private-key material (e.g. the P-384 identity scalar) resident in the slot indefinitely. Add UnoHsmPal::with_admin_io, the admin-side counterpart of drop_io: it opens an admin IO plus a rewound scoped allocator, runs the caller's async closure, then scrubs the slot. Binding the scrub to the session rather than to each caller means no admin path can forget it, and the higher-ranked bound stops any scratch from escaping past the scrub. ensure_unwrapping_key_imported is left as-is: it is a synchronous fn whose atomicity against the cooperative scheduler is load-bearing, and it feeds the &'static GSRAM blob straight into the vault with no DMA-scratch copy. Also fix a clippy manual_is_multiple_of warning in cpu_zeroize. --- fw/plat/uno/fw/pal/src/gdma.rs | 2 +- fw/plat/uno/fw/pal/src/io.rs | 47 +++++++++++ fw/plat/uno/fw/pal/src/part.rs | 138 +++++++++++++++++---------------- 3 files changed, 118 insertions(+), 69 deletions(-) diff --git a/fw/plat/uno/fw/pal/src/gdma.rs b/fw/plat/uno/fw/pal/src/gdma.rs index d975f4f29..9243364dc 100644 --- a/fw/plat/uno/fw/pal/src/gdma.rs +++ b/fw/plat/uno/fw/pal/src/gdma.rs @@ -308,7 +308,7 @@ impl UnoHsmPal { unsafe fn cpu_zeroize(ptr: *mut u8, len: usize) { let mut i = 0usize; // Word body: 4 bytes per store (both per-IO regions hit this path). - while i + 4 <= len && (ptr as usize + i) % 4 == 0 { + while i + 4 <= len && (ptr as usize + i).is_multiple_of(4) { // SAFETY: `i + 4 <= len` and the address is 4-byte aligned, so this // writes wholly within the caller's region at a valid alignment. unsafe { core::ptr::write_volatile(ptr.add(i) as *mut u32, 0) }; diff --git a/fw/plat/uno/fw/pal/src/io.rs b/fw/plat/uno/fw/pal/src/io.rs index 55c1b4062..6382f1c81 100644 --- a/fw/plat/uno/fw/pal/src/io.rs +++ b/fw/plat/uno/fw/pal/src/io.rs @@ -25,6 +25,7 @@ //! and writes the CQE into `IO_CQ[index]` for OIC to transmit. use core::mem; +use core::ops::AsyncFnOnce; use azihsm_fw_hsm_pal_traits::HsmCqe; use azihsm_fw_hsm_pal_traits::HsmIo; @@ -45,6 +46,7 @@ use tock_registers::interfaces::Writeable; use crate::UnoHsmPal; use crate::alloc::ADMIN_IO_INDEX; +use crate::alloc::UnoScopedAlloc; use crate::alloc::reset_io_alloc; /// Typed overlay of the IO GSRAM region. @@ -183,3 +185,48 @@ impl HsmIoController for UnoHsmPal { Ok(()) } } + +impl UnoHsmPal { + /// Runs `f` as an *admin session* over the dedicated admin IO slot + /// ([`ADMIN_IO_INDEX`]), scrubbing that slot when `f` completes. + /// + /// This is the admin-side counterpart of + /// [`drop_io`](HsmIoController::drop_io): internal provisioning + /// (partition identity, enable-time keygen, boot-key masking, vault + /// teardown) runs without a host IO, so it never reaches `drop_io` and + /// its scratch would otherwise only be watermark-rewound, never wiped — + /// leaving raw private-key material (e.g. the P-384 identity scalar) + /// resident in the admin slot indefinitely. + /// + /// Binding the scrub to the session rather than to each caller means no + /// admin path can forget it: obtaining an admin [`UnoHsmIo`] *is* + /// entering a scrubbed scope. `f` also receives a + /// [`UnoScopedAlloc`] rewound to the slot's base, so every admin + /// sequence starts from a clean bump heap. + /// + /// Sessions must not nest — [`UnoScopedAlloc::for_admin`] rewinds the + /// slot's watermarks, so an inner session would alias an outer one's + /// live buffers. All current callers are strictly sequential. + /// + /// # Parameters + /// - `pid`: partition the session targets; written into the admin + /// slot's `IO_META` so [`pid`](HsmIo::pid) resolves correctly. + /// - `f`: async closure run with the admin IO handle and a rewound + /// scoped allocator. + /// + /// # Returns + /// Whatever `f` returned. `R` cannot borrow from the session (enforced + /// by the higher-ranked bound), so no scratch outlives the scrub. + pub(crate) async fn with_admin_io(&self, pid: HsmPartId, f: F) -> R + where + F: for<'a> AsyncFnOnce(&'a UnoHsmIo, &'a UnoScopedAlloc<'a>) -> R, + { + let io = UnoHsmIo::admin(pid); + let result = { + let alloc = UnoScopedAlloc::for_admin(self); + f(&io, &alloc).await + }; + self.scrub_io_slot(ADMIN_IO_INDEX).await; + result + } +} diff --git a/fw/plat/uno/fw/pal/src/part.rs b/fw/plat/uno/fw/pal/src/part.rs index c00a734bc..1b155e5a6 100644 --- a/fw/plat/uno/fw/pal/src/part.rs +++ b/fw/plat/uno/fw/pal/src/part.rs @@ -170,11 +170,11 @@ impl UnoHsmPal { /// (survives enable/disable; cleared on free). async fn provision_masked_bk_boot(&self, pid: HsmPartId) -> HsmResult<()> { let part = PartStore::partition(pid)?; - let admin_io = UnoHsmIo::admin(pid); - // Rewind the admin slot's bump heap before the masking sequence. - let _alloc = UnoScopedAlloc::for_admin(self); - let masked = azihsm_fw_core_crypto_key_derive::mask_bk_boot(self, &admin_io).await?; - part.set_masked_bk_boot(masked) + self.with_admin_io(pid, async |admin_io, _alloc| { + let masked = azihsm_fw_core_crypto_key_derive::mask_bk_boot(self, admin_io).await?; + part.set_masked_bk_boot(masked) + }) + .await } /// Frees partition `pid` (mirrors `part_free`): @@ -224,65 +224,65 @@ impl UnoHsmPal { pct: HsmEccPct, ) -> HsmResult { let part = PartStore::partition(pid)?; - let admin_io = UnoHsmIo::admin(pid); - let alloc = UnoScopedAlloc::for_admin(self); - - // Generate the key pair into transient admin-slot DMA scratch - // buffers. The public key is *not* written straight into its - // part_store field: doing so would hold a `&mut` borrow into the - // GSRAM-backed PartStore across the keygen `.await`, which the - // PartStore driver forbids (no yielding while a slot is mutably - // borrowed). The store is updated synchronously after the await. - let priv_buf = alloc.dma_alloc(P384_PRIV_LEN)?; - let pub_buf = alloc.dma_alloc(ID_PUB_KEY_LEN)?; - let (_priv_len, pub_len) = self - .ecc_gen_keypair( - &admin_io, - &alloc, - HsmEccCurve::P384, - Some((priv_buf, pub_buf)), - pct, - ) - .await?; - - if pub_len != ID_PUB_KEY_LEN { - return Err(HsmError::InternalError); - } - - // Persist the freshly generated public key into its part_store - // field (selected by `kind`). This borrow of the PartStore slot is - // strictly synchronous — no `.await` is reached while it is held. - match kind { - HsmVaultKeyKind::Ecc384Private => { - // The PKA emits the public key little-endian; store the identity - // key big-endian (natural SEC1/DER order) so every host-facing - // consumer (PartInfo, POTA verify, X.509 leaf, session HPKE) - // reads `part_id_pub_key` directly without per-handler swaps. - pub_buf[..ID_PUB_KEY_LEN / 2].reverse(); - pub_buf[ID_PUB_KEY_LEN / 2..].reverse(); - part.set_id_pub_key(pub_buf)? + self.with_admin_io(pid, async |admin_io, alloc| { + // Generate the key pair into transient admin-slot DMA scratch + // buffers. The public key is *not* written straight into its + // part_store field: doing so would hold a `&mut` borrow into the + // GSRAM-backed PartStore across the keygen `.await`, which the + // PartStore driver forbids (no yielding while a slot is mutably + // borrowed). The store is updated synchronously after the await. + let priv_buf = alloc.dma_alloc(P384_PRIV_LEN)?; + let pub_buf = alloc.dma_alloc(ID_PUB_KEY_LEN)?; + let (_priv_len, pub_len) = self + .ecc_gen_keypair( + admin_io, + alloc, + HsmEccCurve::P384, + Some((priv_buf, pub_buf)), + pct, + ) + .await?; + + if pub_len != ID_PUB_KEY_LEN { + return Err(HsmError::InternalError); } - HsmVaultKeyKind::EstablishCred => part.set_ec_pub_key(pub_buf)?, - HsmVaultKeyKind::SessionEncryption => part.set_se_pub_key(pub_buf)?, - _ => return Err(HsmError::InternalError), - } - // Assemble the stored blob to the format the vault expects for - // `kind`: the identity key stores the bare 48-byte private scalar, - // while the establish-credential and session-encryption keys store - // the 144-byte `pub(96) ‖ priv(48)` blob (matching the reference - // firmware's on-storage layout), using the scratch public key. - let key_buf: &DmaBuf = match kind { - HsmVaultKeyKind::Ecc384Private => priv_buf, - HsmVaultKeyKind::EstablishCred | HsmVaultKeyKind::SessionEncryption => { - self.build_enable_key_blob(&alloc, pub_buf, priv_buf)? + // Persist the freshly generated public key into its part_store + // field (selected by `kind`). This borrow of the PartStore slot is + // strictly synchronous — no `.await` is reached while it is held. + match kind { + HsmVaultKeyKind::Ecc384Private => { + // The PKA emits the public key little-endian; store the identity + // key big-endian (natural SEC1/DER order) so every host-facing + // consumer (PartInfo, POTA verify, X.509 leaf, session HPKE) + // reads `part_id_pub_key` directly without per-handler swaps. + pub_buf[..ID_PUB_KEY_LEN / 2].reverse(); + pub_buf[ID_PUB_KEY_LEN / 2..].reverse(); + part.set_id_pub_key(pub_buf)? + } + HsmVaultKeyKind::EstablishCred => part.set_ec_pub_key(pub_buf)?, + HsmVaultKeyKind::SessionEncryption => part.set_se_pub_key(pub_buf)?, + _ => return Err(HsmError::InternalError), } - _ => return Err(HsmError::InternalError), - }; - crate::vault::vault(&admin_io) - .create(self, &admin_io, u8::from(pid), key_buf, kind, None, attrs) - .await + // Assemble the stored blob to the format the vault expects for + // `kind`: the identity key stores the bare 48-byte private scalar, + // while the establish-credential and session-encryption keys store + // the 144-byte `pub(96) ‖ priv(48)` blob (matching the reference + // firmware's on-storage layout), using the scratch public key. + let key_buf: &DmaBuf = match kind { + HsmVaultKeyKind::Ecc384Private => priv_buf, + HsmVaultKeyKind::EstablishCred | HsmVaultKeyKind::SessionEncryption => { + self.build_enable_key_blob(alloc, pub_buf, priv_buf)? + } + _ => return Err(HsmError::InternalError), + }; + + crate::vault::vault(admin_io) + .create(self, admin_io, u8::from(pid), key_buf, kind, None, attrs) + .await + }) + .await } /// Builds the enable-key blob (`pub(96) ‖ priv(48)`, the @@ -373,10 +373,12 @@ impl UnoHsmPal { /// Best-effort deletion of one vault key for partition `pid`. async fn delete_key(&self, pid: HsmPartId, key_id: HsmKeyId) { - let admin_io = UnoHsmIo::admin(pid); - let _ = crate::vault::vault(&admin_io) - .delete(self, &admin_io, key_id) - .await; + self.with_admin_io(pid, async |admin_io, _alloc| { + let _ = crate::vault::vault(admin_io) + .delete(self, admin_io, key_id) + .await; + }) + .await; } /// Clears partition `pid`'s per-tenant state — deletes every @@ -511,10 +513,10 @@ impl UnoHsmPal { PartState::Enabled => { // Wipe every vault key (app + session + internal) so no prior // tenant key material survives the reset. - let admin_io = UnoHsmIo::admin(pid); - crate::vault::vault(&admin_io) - .clear(self, &admin_io) - .await?; + self.with_admin_io(pid, async |admin_io, _alloc| { + crate::vault::vault(admin_io).clear(self, admin_io).await + }) + .await?; // Clear per-tenant persistent state, preserving provisioning. part.clear_state(PartResetKind::Migrate); // The `vault.clear()` above also deleted the identity private From 8c9afcc85bb8f053a072d9247f5aa5438e37c360 Mon Sep 17 00:00:00 2001 From: Rajib Dutta Date: Wed, 5 Aug 2026 21:45:34 +0000 Subject: [PATCH 3/7] uno/io: make the admin-session scrub guarantee enforceable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `with_admin_io`'s higher-ranked bound stops the closure returning anything borrowed from the session's scoped allocator, and the doc claimed that meant no scratch could outlive the scrub. That was not quite true: the closure captures the PAL, and the PAL-level `dma_alloc` hands back a buffer whose lifetime is tied to the PAL, not to the session. A closure could return one and the caller would silently receive memory the scrub had just zeroed. Add `R: 'static`. Every caller already returns an owned value, so this costs nothing, and it turns the doc claim into something the compiler checks: an attempt to return a PAL-level buffer out of the session now fails to compile with "returning this value requires that `'1` must outlive `'static`". Also record why `ensure_unwrapping_key_imported` still takes a raw admin IO instead of a session: it is synchronous, so it cannot await a scrub, and it allocates no scratch — `create_sync` copies from the `&'static` GSRAM slot straight into vault storage — so there is nothing to wipe. Without the note it reads like an omission. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2444329e-909e-4ced-b5df-5a40fb65bb6c --- fw/plat/uno/fw/pal/src/io.rs | 9 +++++++-- fw/plat/uno/fw/pal/src/part.rs | 6 ++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/fw/plat/uno/fw/pal/src/io.rs b/fw/plat/uno/fw/pal/src/io.rs index 6382f1c81..1e080e6f3 100644 --- a/fw/plat/uno/fw/pal/src/io.rs +++ b/fw/plat/uno/fw/pal/src/io.rs @@ -215,10 +215,15 @@ impl UnoHsmPal { /// scoped allocator. /// /// # Returns - /// Whatever `f` returned. `R` cannot borrow from the session (enforced - /// by the higher-ranked bound), so no scratch outlives the scrub. + /// Whatever `f` returned. `R` cannot borrow anything the scrub will wipe: + /// the higher-ranked bound stops it borrowing from the session's scoped + /// allocator, and `R: 'static` additionally stops it returning a buffer + /// obtained from the PAL-level allocator, whose lifetime is tied to the + /// PAL and so outlives the session. Every current caller returns an owned + /// value, so the bound costs nothing. pub(crate) async fn with_admin_io(&self, pid: HsmPartId, f: F) -> R where + R: 'static, F: for<'a> AsyncFnOnce(&'a UnoHsmIo, &'a UnoScopedAlloc<'a>) -> R, { let io = UnoHsmIo::admin(pid); diff --git a/fw/plat/uno/fw/pal/src/part.rs b/fw/plat/uno/fw/pal/src/part.rs index 1b155e5a6..9b3f7be57 100644 --- a/fw/plat/uno/fw/pal/src/part.rs +++ b/fw/plat/uno/fw/pal/src/part.rs @@ -576,6 +576,12 @@ impl UnoHsmPal { .with_internal(true) .with_local(true) .with_unwrap(true); + // Raw admin IO rather than a `with_admin_io` session: this path is + // synchronous (it must not yield, so it cannot await a scrub) and it + // allocates no scratch — `create_sync` copies straight from the + // `&'static` GSRAM slot into vault storage, so the admin slot is never + // dirtied and there is nothing to wipe. The handle is only used to + // select the partition's vault. let admin_io = UnoHsmIo::admin(pid); let kid = crate::vault::vault(&admin_io).create_sync( u8::from(pid), From 9a30fdea35aefbdaf73ada83d72682e40e5c8af1 Mon Sep 17 00:00:00 2001 From: Rajib Dutta Date: Thu, 6 Aug 2026 01:33:31 +0000 Subject: [PATCH 4/7] uno: size the IO-slot scrub to what was actually written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The teardown scrub wiped each slot's full capacity — 18 KiB of SRAM and 1.5 KiB of DTCM — however little the command touched. The live bump watermark cannot bound that, because it rewinds on every scope exit, so track a per-(slot, heap) peak that only grows and wipe to that instead. The peak is cleared by the scrub rather than by the allocator reset, so it always means "bytes written since the last scrub". Everything past it is already zero from that scrub, and an IO that ends without one keeps its peak, so the next scrub still covers it. `dma_alloc_var*` need care: they reserve the whole remaining heap so the callback need not know its encoded length up front, which would pin the peak at capacity on every IO. They now snapshot the peak before the reservation and restore it to the trimmed length, and document the contract this rests on — the callback must not write past the length it returns. On the error path the peak is left at the full reservation, since a partial write before the failure is unbounded. Measured on hardware over ~1.7k IOs: the DMA peak averages ~933 B of the 18 KiB slot with a maximum of ~3.3 KiB, so the wipe now fits in a single DUMMY_MEM-sized GDMA transfer instead of two. Instrumenting the scrub to scan each slot's whole capacity afterwards found no non-zero byte past the recorded peak, on any slot, in any run. Also in this change: - `zeroize_mem` wipes through the GDMA engine rather than the CPU, falling back to a volatile CPU wipe if the engine has no free tag. Every `DmaBuf` is GSRAM-backed by construction — the DTCM heap hands out plain `&mut [u8]` — so the destination is always GDMA-reachable. - `UnoHsmIo::admin` becomes `admin_no_scrub`, so a path that opts out of the scrubbed session has to say so. Its one caller is the synchronous unwrapping-key import, which cannot await a scrub and writes nothing to either bump heap. - `delete_key` takes the caller's admin IO instead of opening a session per key. `KeyVault::delete` takes no allocator, so it cannot dirty the heaps, and `clear_enabled_state` deletes one key per provisioning slot and per live session — a session each would have scrubbed the slot every time for no benefit. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2444329e-909e-4ced-b5df-5a40fb65bb6c --- fw/plat/uno/fw/pal/src/alloc.rs | 110 ++++++++++++++++++++++++++++---- fw/plat/uno/fw/pal/src/gdma.rs | 63 ++++++++++++------ fw/plat/uno/fw/pal/src/io.rs | 30 ++++++--- fw/plat/uno/fw/pal/src/pal.rs | 15 +++++ fw/plat/uno/fw/pal/src/part.rs | 60 +++++++++++------ 5 files changed, 218 insertions(+), 60 deletions(-) diff --git a/fw/plat/uno/fw/pal/src/alloc.rs b/fw/plat/uno/fw/pal/src/alloc.rs index b8cfabf62..9243c4e6e 100644 --- a/fw/plat/uno/fw/pal/src/alloc.rs +++ b/fw/plat/uno/fw/pal/src/alloc.rs @@ -159,22 +159,27 @@ fn heap_base_cap(io_index: u16, heap: usize) -> (*mut u8, usize) { } } -/// Per-IO SRAM (DMA heap) buffer region as `(base_ptr, capacity)`. +/// Per-IO SRAM (DMA heap) region that may hold IO data, as +/// `(base_ptr, dirty_len)`. /// -/// Returns the whole slot-local `SRAM_IO_BUF[io_index]` region (not just -/// the used watermark) so it can be scrubbed in full on IO teardown; see -/// [`UnoHsmPal::scrub_io_slot`](crate::UnoHsmPal::scrub_io_slot). +/// `dirty_len` is the slot's peak watermark (see [`pk`]) — every byte +/// written since the last scrub, and nothing beyond it. Used by +/// [`UnoHsmPal::scrub_io_slot`](crate::UnoHsmPal::scrub_io_slot) so a +/// command that touches a few hundred bytes does not pay an 18 KiB wipe. #[inline(always)] -pub(crate) fn io_slot_dma_region(io_index: u16) -> (*mut u8, usize) { - heap_base_cap(io_index, DMA) +pub(crate) fn io_slot_dma_dirty(pal: &UnoHsmPal, io_index: u16) -> (*mut u8, usize) { + let (base, cap) = heap_base_cap(io_index, DMA); + (base, pk(pal, io_index, DMA).with(|v| *v).min(cap)) } -/// Per-IO DTCM (NonDma heap) buffer region as `(base_ptr, capacity)`. +/// Per-IO DTCM (NonDma heap) region that may hold IO data, as +/// `(base_ptr, dirty_len)`. /// -/// Companion to [`io_slot_dma_region`] for the fast/NonDma scratch heap. +/// Companion to [`io_slot_dma_dirty`] for the fast/NonDma scratch heap. #[inline(always)] -pub(crate) fn io_slot_nondma_region(io_index: u16) -> (*mut u8, usize) { - heap_base_cap(io_index, NONDMA) +pub(crate) fn io_slot_nondma_dirty(pal: &UnoHsmPal, io_index: u16) -> (*mut u8, usize) { + let (base, cap) = heap_base_cap(io_index, NONDMA); + (base, pk(pal, io_index, NONDMA).with(|v| *v).min(cap)) } /// Access the watermark cell for one `(io_index, heap)` pair. @@ -191,6 +196,43 @@ fn wm(pal: &UnoHsmPal, io_index: u16, heap: usize) -> &SingleCell { &pal.io_alloc[io_index as usize][heap] } +/// Access the *peak* (high-water) watermark cell for one +/// `(io_index, heap)` pair. +/// +/// [`wm`] rewinds whenever a scope drops, so by IO teardown it says nothing +/// about how much of the slot was written. This cell only ever grows until +/// the slot is scrubbed, so it bounds the region that may still hold key +/// material — it is what the teardown wipe is sized from. +/// +/// # Parameters +/// - `pal`: PAL instance owning the peak table. +/// - `io_index`: IO slot index. +/// - `heap`: Heap selector (`NONDMA` or `DMA`). +/// +/// # Returns +/// Shared reference to the corresponding [`SingleCell`] peak watermark. +#[inline(always)] +fn pk(pal: &UnoHsmPal, io_index: u16, heap: usize) -> &SingleCell { + &pal.io_peak[io_index as usize][heap] +} + +/// Clear both peak watermarks for `io_index`, marking the slot as scrubbed. +/// +/// Called by [`UnoHsmPal::scrub_io_slot`](crate::UnoHsmPal::scrub_io_slot) +/// *after* the wipe, so a peak always means "bytes written since the last +/// scrub" — independent of where IOs happen to reset their watermarks. That +/// keeps allocations made outside a [`reset_io_alloc`] / scrub pair covered +/// by the next wipe. +/// +/// # Parameters +/// - `pal`: PAL instance owning the peak table. +/// - `io_index`: IO slot index whose peaks are cleared. +pub(crate) fn clear_io_peak(pal: &UnoHsmPal, io_index: u16) { + for cell in &pal.io_peak[io_index as usize] { + cell.with(|v| *v = 0); + } +} + /// Bump-allocate `size` bytes with given alignment, return (start_offset, slice). /// /// # Parameters @@ -228,6 +270,11 @@ fn bump( } w.with(|v| *v = end); + // Raise the slot's high-water mark. Unlike `w` this never rewinds, so it + // still bounds the written region once every scope has dropped — that is + // what the teardown scrub wipes. `dma_alloc_var*` reserve the whole + // remaining heap here and correct this back down to the trimmed length. + pk(pal, io_index, heap).with(|v| *v = (*v).max(end)); // SAFETY: start..end is within bounds and does not overlap any prior // live allocation (the watermark only advances within a scope). Ok((start, unsafe { @@ -418,6 +465,22 @@ impl HsmAlloc for UnoHsmPal { /// DMA capacity from current aligned watermark. It returns the number of /// bytes logically consumed. /// + /// # Contract + /// + /// **`f` must not write past the `len` it returns.** The teardown scrub is + /// sized from the slot's high-water mark, and this method reports only + /// `start + len` as its contribution — anything written beyond `len` would + /// be left unwiped and could survive into the next IO on this slot. `f` + /// receives the whole remaining heap purely so the encoded length need not + /// be known up front. + /// + /// Callers encode through either `MborEncoder` or a TBOR response frame + /// builder (`Tbor*Resp::encode(...).finish()`); both append sequentially + /// and report their final position, so both write a strict prefix. This + /// was checked on hardware: instrumenting the teardown scrub to scan each + /// slot's whole capacity found no non-zero byte past the recorded peak + /// over ~1.7k IOs. + /// /// # Returns /// Mutable [`DmaBuf`] of length selected by `f`, or error from `f`/allocator. fn dma_alloc_var(&self, io: &impl HsmIo, f: F) -> HsmResult<&mut DmaBuf> @@ -432,15 +495,27 @@ impl HsmAlloc for UnoHsmPal { if aligned >= cap { return Err(HsmError::NotEnoughSpace); } + // Snapshot the peak *before* `bump`, which is about to inflate it to + // full capacity because this reserves the whole remaining heap. The + // trim below restores it to what `f` actually wrote (see `# Contract`); + // without this the scrub would wipe the whole slot on every IO. + let peak_before = pk(self, io_index, DMA).with(|v| *v); let (start, buf) = bump(self, io_index, DMA, cap - aligned, 4)?; match f(buf) { Ok(len) => { - w.with(|v| *v = start + len.min(buf.len())); + let end = start + len.min(buf.len()); + w.with(|v| *v = end); + pk(self, io_index, DMA).with(|v| *v = peak_before.max(end)); // SAFETY: `buf` came from the SRAM Dma pool. Ok(unsafe { DmaBuf::from_raw_mut(&mut buf[..len]) }) } Err(e) => { w.with(|v| *v = start); + // Leave the peak at the full reservation `bump` just set. `f` + // may have written a partial encoding before failing and there + // is no `len` to bound it by, so the whole reserved span must + // be treated as dirty. This costs one full-slot wipe on a + // rejected request — rare, and the safe direction. Err(e) } } @@ -453,6 +528,11 @@ impl HsmAlloc for UnoHsmPal { /// - `f`: Callback receiving temporary writable remaining DMA span and /// returning `(len, extra)` where `len` is consumed bytes. /// + /// # Contract + /// + /// Same as [`Self::dma_alloc_var`]: **`f` must not write past `len`**, or + /// the teardown scrub will miss those bytes. + /// /// # Returns /// Tuple `(&mut DmaBuf, T)` on success, or error from `f`/allocator. fn dma_alloc_var_with(&self, io: &impl HsmIo, f: F) -> HsmResult<(&mut DmaBuf, T)> @@ -467,15 +547,21 @@ impl HsmAlloc for UnoHsmPal { if aligned >= cap { return Err(HsmError::NotEnoughSpace); } + // See `dma_alloc_var`: snapshot before `bump` inflates the peak to cap. + let peak_before = pk(self, io_index, DMA).with(|v| *v); let (start, buf) = bump(self, io_index, DMA, cap - aligned, 4)?; match f(buf) { Ok((len, extra)) => { - w.with(|v| *v = start + len.min(buf.len())); + let end = start + len.min(buf.len()); + w.with(|v| *v = end); + pk(self, io_index, DMA).with(|v| *v = peak_before.max(end)); // SAFETY: `buf` came from the SRAM Dma pool. Ok((unsafe { DmaBuf::from_raw_mut(&mut buf[..len]) }, extra)) } Err(e) => { w.with(|v| *v = start); + // See `dma_alloc_var`: leave the peak at the full reservation, + // since a partial write before the failure is unbounded. Err(e) } } diff --git a/fw/plat/uno/fw/pal/src/gdma.rs b/fw/plat/uno/fw/pal/src/gdma.rs index 9243364dc..e8b5b667c 100644 --- a/fw/plat/uno/fw/pal/src/gdma.rs +++ b/fw/plat/uno/fw/pal/src/gdma.rs @@ -135,12 +135,25 @@ impl HsmGdmaController for UnoHsmPal { /// Zero an HSM-local buffer. /// - /// Software volatile wipe for now; a hardware GDMA memset will replace - /// this on uno later (the async signature is kept so callers are - /// unaffected). [`DmaBuf::zeroize`] guarantees the writes are not - /// elided so key material is actually scrubbed. + /// Wipes off-CPU with the GDMA engine via + /// [`gdma_zero_region`](UnoHsmPal::gdma_zero_region), which copies from + /// the read-as-zero `DUMMY_MEM` window — GDMA has no fill/memset opcode, + /// so a wipe is a device-to-device copy out of that region. Every + /// [`DmaBuf`] is GSRAM-backed by construction (the DTCM heap hands out + /// plain `&mut [u8]`, never a `DmaBuf`), so the destination is always + /// GDMA-reachable. + /// + /// Falls back to [`DmaBuf::zeroize`] if the engine has no free tag or the + /// transfer fails, so the buffer is scrubbed either way; both paths use + /// writes that cannot be optimized away. async fn zeroize_mem(&self, _io: &impl HsmIo, dst: &mut DmaBuf) -> HsmResult<()> { - dst.zeroize(); + let len = dst.len(); + if len == 0 { + return Ok(()); + } + if self.gdma_zero_region(dst.as_mut_ptr(), len).await.is_err() { + dst.zeroize(); + } Ok(()) } @@ -257,8 +270,8 @@ impl UnoHsmPal { /// /// Called on IO teardown so no key material from one IO survives into /// the next IO that reuses the slot. On Uno every crypto operand lives - /// in these bump-allocated buffers (no heap), so wiping the full slot is - /// what protects private keys / plaintext — the CPU-visible arena is the + /// in these bump-allocated buffers (no heap), so wiping them is what + /// protects private keys / plaintext — the CPU-visible arena is the /// only place they exist. /// /// The SRAM (DMA) buffer is wiped off-CPU with the GDMA engine; the DTCM @@ -266,28 +279,40 @@ impl UnoHsmPal { /// wiped by the CPU. If the GDMA wipe fails (e.g. no free tags under /// heavy concurrency), the SRAM buffer is scrubbed by the CPU as a /// fallback so secrets are never left resident. + /// + /// Only each heap's *high-water* region is wiped, not its full capacity. + /// The bump watermark rewinds on every scope exit, but the peak bounds + /// every byte written since the previous scrub, and everything past it is + /// already zero from that scrub. Measured over ~1.7k host IOs the DMA peak + /// averaged ~933 B of the 18 KiB slot (max ~3.3 KiB), so this also keeps + /// the wipe inside a single `DUMMY_MEM`-sized GDMA transfer instead of two. + /// A heap that was never touched is skipped entirely. pub(crate) async fn scrub_io_slot(&self, io_index: u16) { - let (dma_ptr, dma_len) = crate::alloc::io_slot_dma_region(io_index); - if self.gdma_zero_region(dma_ptr, dma_len).await.is_err() { + let (dma_ptr, dma_len) = crate::alloc::io_slot_dma_dirty(self, io_index); + if dma_len != 0 && self.gdma_zero_region(dma_ptr, dma_len).await.is_err() { // Surface the fallback: a persistent GDMA fault would otherwise - // silently degrade every wipe into a byte-wise CPU loop with no - // signal. Logged via the trace facade (as the iic/oic drivers do - // for unexpected conditions); compiled out when no trace backend - // is selected. Correctness is unaffected — the CPU wipe below - // still fully scrubs the buffer. + // silently degrade every wipe into a CPU loop with no signal. + // Logged via the trace facade (as the iic/oic drivers do for + // unexpected conditions); compiled out when no trace backend is + // selected. Correctness is unaffected — the CPU wipe below still + // fully scrubs the buffer. warn!( "gdma", "SRAM scrub for slot {} fell back to CPU (GDMA wipe failed)", io_index ); - // SAFETY: `(dma_ptr, dma_len)` describes the slot's SRAM buffer, - // valid for writes over its whole capacity. + // SAFETY: `(dma_ptr, dma_len)` is a prefix of the slot's SRAM + // buffer, valid for writes over its whole length. unsafe { cpu_zeroize(dma_ptr, dma_len) }; } // DTCM (NonDma) heap — not GDMA-reachable, so wipe with the CPU. - let (nd_ptr, nd_len) = crate::alloc::io_slot_nondma_region(io_index); - // SAFETY: `(nd_ptr, nd_len)` describes the slot's DTCM buffer, valid - // for writes over its whole capacity. + let (nd_ptr, nd_len) = crate::alloc::io_slot_nondma_dirty(self, io_index); + // SAFETY: `(nd_ptr, nd_len)` is a prefix of the slot's DTCM buffer, + // valid for writes over its whole length. unsafe { cpu_zeroize(nd_ptr, nd_len) }; + + // Both heaps are now zero up to their peaks; drop the high-water marks + // so the next scrub covers only writes made after this point. + crate::alloc::clear_io_peak(self, io_index); } } diff --git a/fw/plat/uno/fw/pal/src/io.rs b/fw/plat/uno/fw/pal/src/io.rs index 1e080e6f3..fa6059ecb 100644 --- a/fw/plat/uno/fw/pal/src/io.rs +++ b/fw/plat/uno/fw/pal/src/io.rs @@ -67,8 +67,8 @@ pub struct UnoHsmIo { } impl UnoHsmIo { - /// Constructs an IO handle over the dedicated admin slot - /// ([`ADMIN_IO_INDEX`]), targeting partition `pid`. + /// Constructs a **bare, unscrubbed** IO handle over the dedicated admin + /// slot ([`ADMIN_IO_INDEX`]), targeting partition `pid`. /// /// Internal provisioning (partition identity and enable-time keygen) /// runs without a host IO. Reusing the concrete [`UnoHsmIo`] / @@ -77,9 +77,21 @@ impl UnoHsmIo { /// The target `pid` is written into the admin slot's `IO_META` so /// [`pid`](HsmIo::pid) resolves correctly. /// + /// # Prefer [`with_admin_io`](UnoHsmPal::with_admin_io) + /// + /// This constructor performs **no scrub**: anything the caller writes to + /// the admin slot's bump heaps stays resident until some later scrub. + /// Use it only on a path that provably dirties neither heap — today just + /// the synchronous unwrapping-key import, which cannot `await` a scrub + /// and copies straight from `&'static` GSRAM into vault storage. Every + /// other admin path must go through + /// [`with_admin_io`](UnoHsmPal::with_admin_io), which wipes the slot on + /// completion. The name is deliberately blunt so a new call site has to + /// opt into the hazard explicitly. + /// /// [`ADMIN_IO_INDEX`]: crate::alloc::ADMIN_IO_INDEX /// [`UnoScopedAlloc`]: crate::alloc::UnoScopedAlloc - pub(crate) fn admin(pid: HsmPartId) -> Self { + pub(crate) fn admin_no_scrub(pid: HsmPartId) -> Self { let io = Self { index: ADMIN_IO_INDEX, }; @@ -199,10 +211,12 @@ impl UnoHsmPal { /// resident in the admin slot indefinitely. /// /// Binding the scrub to the session rather than to each caller means no - /// admin path can forget it: obtaining an admin [`UnoHsmIo`] *is* - /// entering a scrubbed scope. `f` also receives a - /// [`UnoScopedAlloc`] rewound to the slot's base, so every admin - /// sequence starts from a clean bump heap. + /// admin path that dirties the slot can forget it. The one deliberate + /// exception is [`UnoHsmIo::admin_no_scrub`], whose name states that it + /// opts out; it is reserved for paths that provably write nothing to + /// either bump heap. `f` also receives a [`UnoScopedAlloc`] rewound to + /// the slot's base, so every admin sequence starts from a clean bump + /// heap. /// /// Sessions must not nest — [`UnoScopedAlloc::for_admin`] rewinds the /// slot's watermarks, so an inner session would alias an outer one's @@ -226,7 +240,7 @@ impl UnoHsmPal { R: 'static, F: for<'a> AsyncFnOnce(&'a UnoHsmIo, &'a UnoScopedAlloc<'a>) -> R, { - let io = UnoHsmIo::admin(pid); + let io = UnoHsmIo::admin_no_scrub(pid); let result = { let alloc = UnoScopedAlloc::for_admin(self); f(&io, &alloc).await diff --git a/fw/plat/uno/fw/pal/src/pal.rs b/fw/plat/uno/fw/pal/src/pal.rs index cd0ae716e..cd0409ef9 100644 --- a/fw/plat/uno/fw/pal/src/pal.rs +++ b/fw/plat/uno/fw/pal/src/pal.rs @@ -279,6 +279,20 @@ pub struct UnoHsmPal { /// Per-IO bump allocator state (watermarks for Local + Global heaps). pub(crate) io_alloc: IoAllocTable, + + /// Per-IO *peak* (high-water) allocator state, same shape as + /// [`io_alloc`](Self::io_alloc). `io_alloc` rewinds on every scope exit, + /// so this is what bounds the region the teardown scrub must wipe. + /// + /// Starts at zero, so each slot's first scrub after boot covers only what + /// this boot wrote. That relies on the IO heaps arriving clean: on a warm + /// boot 1SP wipes GSRAM apart from the regions it must preserve to restore + /// the IO queues and the persistent store, neither of which overlaps these + /// heaps. A 1SP change that stopped clearing them would leave the previous + /// incarnation's scratch unwiped until an allocation happened to reach past + /// it; initialise this table saturated (any value `>= ` capacity) to make + /// the first scrub cover the whole slot instead. + pub(crate) io_peak: IoAllocTable, } // SAFETY: UnoHsmPal is only accessed from a single-threaded Embassy @@ -366,6 +380,7 @@ impl Default for UnoHsmPal { ipc: unsafe { static_init!(Ipc, Ipc::new(ipc_config)) }, boot_phase: Cell::new(BootPhase::WaitNormalBoot), io_alloc: IO_ALLOC_INIT, + io_peak: IO_ALLOC_INIT, } } } diff --git a/fw/plat/uno/fw/pal/src/part.rs b/fw/plat/uno/fw/pal/src/part.rs index 9b3f7be57..83dc8af99 100644 --- a/fw/plat/uno/fw/pal/src/part.rs +++ b/fw/plat/uno/fw/pal/src/part.rs @@ -148,7 +148,10 @@ impl UnoHsmPal { return; }; if let Some(key_id) = part.id_key_id() { - self.delete_key(pid, key_id).await; + self.with_admin_io(pid, async |admin_io, _alloc| { + self.delete_key(admin_io, key_id).await; + }) + .await; } part.clear_identity(); part.clear_masked_bk_boot(); @@ -191,13 +194,17 @@ impl UnoHsmPal { return Ok(()); } - // Disable: clear enable-time keys/state (no-op if not enabled). - self.clear_enabled_state(pid).await; + // Disable: clear enable-time keys/state (no-op if not enabled), then + // delete the identity key. One admin session covers every vault + // delete below, so the slot is scrubbed once instead of once per key. + self.with_admin_io(pid, async |admin_io, _alloc| { + self.clear_enabled_state(admin_io, pid).await; + if let Some(key_id) = part.id_key_id() { + self.delete_key(admin_io, key_id).await; + } + }) + .await; - // Dealloc: delete the identity key and zeroize identity material. - if let Some(key_id) = part.id_key_id() { - self.delete_key(pid, key_id).await; - } part.clear_identity(); // The masked boot key persists across enable/disable; it is wiped // only here, on free. @@ -364,21 +371,29 @@ impl UnoHsmPal { } Err(e) => { // Roll back the establish-credential key. - self.delete_key(pid, ec_id).await; + self.with_admin_io(pid, async |admin_io, _alloc| { + self.delete_key(admin_io, ec_id).await; + }) + .await; part.clear_enabled_keys(); Err(e) } } } - /// Best-effort deletion of one vault key for partition `pid`. - async fn delete_key(&self, pid: HsmPartId, key_id: HsmKeyId) { - self.with_admin_io(pid, async |admin_io, _alloc| { - let _ = crate::vault::vault(admin_io) - .delete(self, admin_io, key_id) - .await; - }) - .await; + /// Best-effort deletion of one vault key, on the caller's admin session. + /// + /// Takes the caller's `admin_io` instead of opening its own session: + /// [`KeyVault::delete`](azihsm_fw_uno_key_vault::KeyVault::delete) takes + /// only a GDMA controller and an IO — no allocator — so it writes nothing + /// to the admin slot's bump heaps. A session per key would scrub the full + /// 18 KiB slot once per deletion for no benefit, and + /// [`clear_enabled_state`](Self::clear_enabled_state) deletes one key per + /// provisioning slot *and* per live session. + async fn delete_key(&self, admin_io: &UnoHsmIo, key_id: HsmKeyId) { + let _ = crate::vault::vault(admin_io) + .delete(self, admin_io, key_id) + .await; } /// Clears partition `pid`'s per-tenant state — deletes every @@ -392,7 +407,7 @@ impl UnoHsmPal { /// are torn down only on free. Best-effort and idempotent: keys are /// deleted only if present, so it is safe to call regardless of the /// current lifecycle state. - async fn clear_enabled_state(&self, pid: HsmPartId) { + async fn clear_enabled_state(&self, admin_io: &UnoHsmIo, pid: HsmPartId) { let Ok(part) = PartStore::partition(pid) else { return; }; @@ -411,14 +426,14 @@ impl UnoHsmPal { .into_iter() .flatten() { - self.delete_key(pid, key_id).await; + self.delete_key(admin_io, key_id).await; } // Delete every session-blob vault key (Active, NeedsRenegotiation, // or Pending) mapped by the session table, so none are orphaned in // the vault when the table is zeroized below. if let Ok(sessions) = SessionStore::partition(pid) { for key_id in sessions.occupied_physical_ids().into_iter().flatten() { - self.delete_key(pid, key_id).await; + self.delete_key(admin_io, key_id).await; } } part.clear_state(PartResetKind::Disable); @@ -474,7 +489,10 @@ impl UnoHsmPal { let part = PartStore::partition(pid)?; match part.state()? { PartState::Enabled => { - self.clear_enabled_state(pid).await; + self.with_admin_io(pid, async |admin_io, _alloc| { + self.clear_enabled_state(admin_io, pid).await; + }) + .await; part.set_state(PartState::Disabled); Ok(()) } @@ -582,7 +600,7 @@ impl UnoHsmPal { // `&'static` GSRAM slot into vault storage, so the admin slot is never // dirtied and there is nothing to wipe. The handle is only used to // select the partition's vault. - let admin_io = UnoHsmIo::admin(pid); + let admin_io = UnoHsmIo::admin_no_scrub(pid); let kid = crate::vault::vault(&admin_io).create_sync( u8::from(pid), bk, From 819d6136e4b61d69ec197bf5f76774114c541b9c Mon Sep 17 00:00:00 2001 From: Rajib Dutta Date: Thu, 6 Aug 2026 01:54:44 +0000 Subject: [PATCH 5/7] uno/alloc: reject an over-reported dma_alloc_var length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `dma_alloc_var*` clamped the callback's `len` when advancing the watermark but then sliced `buf[..len]` unclamped, so the two disagreed and a callback that reported more than it was handed would panic — taking the firmware down rather than failing the one command. Reject it with `InvalidArg` instead, as the std PAL already does: refuse to hand back a longer slice than we own. The watermark rewinds, but the peak stays at the full reservation, since a callback that over-reports wrote an unknown amount and the teardown scrub has to cover it. Reported by the Copilot reviewer on #632. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2444329e-909e-4ced-b5df-5a40fb65bb6c --- fw/plat/uno/fw/pal/src/alloc.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/fw/plat/uno/fw/pal/src/alloc.rs b/fw/plat/uno/fw/pal/src/alloc.rs index 9243c4e6e..b6dfb5f9b 100644 --- a/fw/plat/uno/fw/pal/src/alloc.rs +++ b/fw/plat/uno/fw/pal/src/alloc.rs @@ -474,6 +474,11 @@ impl HsmAlloc for UnoHsmPal { /// receives the whole remaining heap purely so the encoded length need not /// be known up front. /// + /// A `len` larger than the buffer is rejected with + /// [`HsmError::InvalidArg`] rather than panicking on the slice, matching + /// the std PAL; the peak is left at the full reservation in that case, + /// since such a callback wrote an unknown amount. + /// /// Callers encode through either `MborEncoder` or a TBOR response frame /// builder (`Tbor*Resp::encode(...).finish()`); both append sequentially /// and report their final position, so both write a strict prefix. This @@ -503,7 +508,16 @@ impl HsmAlloc for UnoHsmPal { let (start, buf) = bump(self, io_index, DMA, cap - aligned, 4)?; match f(buf) { Ok(len) => { - let end = start + len.min(buf.len()); + if len > buf.len() { + // Callback overran the buffer it was handed; refuse to + // expose a longer slice than we own, matching the std PAL, + // rather than panicking on the slice below. Rewind the + // watermark but leave the peak at the full reservation: + // the callback wrote an unknown amount. + w.with(|v| *v = start); + return Err(HsmError::InvalidArg); + } + let end = start + len; w.with(|v| *v = end); pk(self, io_index, DMA).with(|v| *v = peak_before.max(end)); // SAFETY: `buf` came from the SRAM Dma pool. @@ -552,7 +566,13 @@ impl HsmAlloc for UnoHsmPal { let (start, buf) = bump(self, io_index, DMA, cap - aligned, 4)?; match f(buf) { Ok((len, extra)) => { - let end = start + len.min(buf.len()); + if len > buf.len() { + // See `dma_alloc_var`: refuse the over-report instead of + // panicking, and leave the peak at the full reservation. + w.with(|v| *v = start); + return Err(HsmError::InvalidArg); + } + let end = start + len; w.with(|v| *v = end); pk(self, io_index, DMA).with(|v| *v = peak_before.max(end)); // SAFETY: `buf` came from the SRAM Dma pool. From c14ce4bcb611cdb54435e52a3a1eea11dd1b7c4f Mon Sep 17 00:00:00 2001 From: Rajib Dutta Date: Thu, 6 Aug 2026 02:08:26 +0000 Subject: [PATCH 6/7] pal: give the dma_alloc_var overrun its own status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `InvalidArg` was the wrong status for a callback that reports writing more than the buffer it was handed. The enum's own docs reserve `InvalidArg` for a caller bug — "unknown id, kind mismatch", "a malformed request" — but here the request is fine and the fault is ours: an encoder mis-reporting its output length. Returning `InvalidArg` blames the host for a firmware bug. Add `DmaAllocLenOverrun`, following the `UndoLogFull` precedent of a dedicated, documented status for a should-never-happen firmware bug, and use it from both PALs so they report the same thing for the same condition. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2444329e-909e-4ced-b5df-5a40fb65bb6c --- fw/pal/traits/src/error.rs | 14 ++++++++++++++ fw/plat/std/pal/src/alloc.rs | 10 +++++++--- fw/plat/uno/fw/pal/src/alloc.rs | 10 +++++----- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/fw/pal/traits/src/error.rs b/fw/pal/traits/src/error.rs index 79c8972c8..01e6cc4cc 100644 --- a/fw/pal/traits/src/error.rs +++ b/fw/pal/traits/src/error.rs @@ -436,6 +436,20 @@ pub enum HsmError { /// `x963_kdf` / `sp800_56a_kdf`. ConcatKdfError = 0x0870010B, + /// A variable-length DMA allocation callback + /// ([`HsmAlloc::dma_alloc_var`](crate::HsmAlloc::dma_alloc_var) / + /// [`dma_alloc_var_with`](crate::HsmAlloc::dma_alloc_var_with)) reported + /// having written more bytes than the buffer it was handed. The + /// allocator refuses to expose a slice longer than it owns, so the + /// command fails instead of over-reading the heap. Like + /// [`UndoLogFull`](Self::UndoLogFull) this indicates a **firmware bug** — + /// an encoder mis-reporting its output length — and should never occur; + /// it is distinct from [`InvalidArg`](Self::InvalidArg), which blames a + /// malformed request, and from + /// [`NotEnoughSpace`](Self::NotEnoughSpace), which signals genuine + /// exhaustion. + DmaAllocLenOverrun = 0x0870010C, + // Firmware-internal diagnostic codes logged by the CPU fault and panic // exception handlers (`azihsm_fw_uno_fault`). These are not DDI protocol // statuses: they use the PAL diagnostic facility (`0x08F`) to stay clear of diff --git a/fw/plat/std/pal/src/alloc.rs b/fw/plat/std/pal/src/alloc.rs index 0db315e75..7320f6173 100644 --- a/fw/plat/std/pal/src/alloc.rs +++ b/fw/plat/std/pal/src/alloc.rs @@ -241,9 +241,11 @@ impl HsmAlloc for StdHsmPal { Ok(len) => { if len > buf.len() { // Closure overran the buffer it was handed; refuse to - // expose a longer slice than we actually own. + // expose a longer slice than we actually own. This is a + // firmware bug (an encoder mis-reporting its length), not + // a malformed request, hence the dedicated status. mark_cell.set(saved_mark); - return Err(HsmError::InvalidArg); + return Err(HsmError::DmaAllocLenOverrun); } let final_end = aligned + len; mark_cell.set(final_end); @@ -274,8 +276,10 @@ impl HsmAlloc for StdHsmPal { match f(buf) { Ok((len, extra)) => { if len > buf.len() { + // See `dma_alloc_var`: a firmware bug, not a malformed + // request. mark_cell.set(saved_mark); - return Err(HsmError::InvalidArg); + return Err(HsmError::DmaAllocLenOverrun); } let final_end = aligned + len; mark_cell.set(final_end); diff --git a/fw/plat/uno/fw/pal/src/alloc.rs b/fw/plat/uno/fw/pal/src/alloc.rs index b6dfb5f9b..876c96234 100644 --- a/fw/plat/uno/fw/pal/src/alloc.rs +++ b/fw/plat/uno/fw/pal/src/alloc.rs @@ -475,9 +475,9 @@ impl HsmAlloc for UnoHsmPal { /// be known up front. /// /// A `len` larger than the buffer is rejected with - /// [`HsmError::InvalidArg`] rather than panicking on the slice, matching - /// the std PAL; the peak is left at the full reservation in that case, - /// since such a callback wrote an unknown amount. + /// [`HsmError::DmaAllocLenOverrun`] rather than panicking on the slice, + /// matching the std PAL; the peak is left at the full reservation in that + /// case, since such a callback wrote an unknown amount. /// /// Callers encode through either `MborEncoder` or a TBOR response frame /// builder (`Tbor*Resp::encode(...).finish()`); both append sequentially @@ -515,7 +515,7 @@ impl HsmAlloc for UnoHsmPal { // watermark but leave the peak at the full reservation: // the callback wrote an unknown amount. w.with(|v| *v = start); - return Err(HsmError::InvalidArg); + return Err(HsmError::DmaAllocLenOverrun); } let end = start + len; w.with(|v| *v = end); @@ -570,7 +570,7 @@ impl HsmAlloc for UnoHsmPal { // See `dma_alloc_var`: refuse the over-report instead of // panicking, and leave the peak at the full reservation. w.with(|v| *v = start); - return Err(HsmError::InvalidArg); + return Err(HsmError::DmaAllocLenOverrun); } let end = start + len; w.with(|v| *v = end); From 443ccf94108c9405d37776518896e250e13a32e2 Mon Sep 17 00:00:00 2001 From: Rajib Dutta Date: Fri, 7 Aug 2026 23:41:49 +0000 Subject: [PATCH 7/7] uno/gdma: wipe the CPU scrub path with the zeroize crate (QWORD) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review feedback: use a vetted zeroization primitive and widen the CPU store from 32-bit to 64-bit. `cpu_zeroize` was a hand-rolled 32-bit volatile-store loop. Replace it with the audited `zeroize` crate applied to a `[u64]` view of the region: an 8-byte-aligned QWORD body framed by a byte-wise head/tail for any unaligned edges. Both per-IO regions (`DTCM_IO_BUF` 0x600, `SRAM_IO_BUF` 0x4800) are 8-byte aligned and 8-multiples, so in practice it is a pure `u64` wipe with no head or tail. On Cortex-M7 `[u64]::zeroize()` lowers to `STRD` (64-bit doubleword stores), verified in the release disassembly — the wipe loop is a 4x unrolled `strd rZero, rZero, [ptr, #off]`. That halves the store count versus the previous word loop and quarters a byte-wise wipe, on the CPU fallback and the small DTCM path (the 18 KiB SRAM slot is still wiped by GDMA). `zeroize` emits the volatile writes plus an atomic fence, so the wipe still cannot be elided. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2444329e-909e-4ced-b5df-5a40fb65bb6c --- fw/plat/uno/fw/Cargo.toml | 1 + fw/plat/uno/fw/pal/Cargo.toml | 1 + fw/plat/uno/fw/pal/src/gdma.rs | 51 +++++++++++++++++++++------------- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/fw/plat/uno/fw/Cargo.toml b/fw/plat/uno/fw/Cargo.toml index 5f1fd7992..b34a355fb 100644 --- a/fw/plat/uno/fw/Cargo.toml +++ b/fw/plat/uno/fw/Cargo.toml @@ -112,6 +112,7 @@ static_assertions = "1.1.0" syn = { features = ["full"], version = "2" } tock-registers = { git = "https://github.com/tock/tock.git", rev = "release-2.2" } zerocopy = "0.8.48" +zeroize = { default-features = false, version = "1.8.1" } [workspace.lints.rust] future_incompatible = { level = "deny", priority = -1 } diff --git a/fw/plat/uno/fw/pal/Cargo.toml b/fw/plat/uno/fw/pal/Cargo.toml index 5dbf7b7f9..d2db54b70 100644 --- a/fw/plat/uno/fw/pal/Cargo.toml +++ b/fw/plat/uno/fw/pal/Cargo.toml @@ -48,6 +48,7 @@ tock-registers = { workspace = true } bitfield-struct = { workspace = true } open-enum = { workspace = true } zerocopy = { workspace = true } +zeroize = { workspace = true } [features] default = [] diff --git a/fw/plat/uno/fw/pal/src/gdma.rs b/fw/plat/uno/fw/pal/src/gdma.rs index e8b5b667c..5c5557541 100644 --- a/fw/plat/uno/fw/pal/src/gdma.rs +++ b/fw/plat/uno/fw/pal/src/gdma.rs @@ -38,6 +38,7 @@ use azihsm_fw_uno_drivers_gdma::MemInterface; use azihsm_fw_uno_reg_soc::dummy_mem::DUMMY_MEM_BASE; use azihsm_fw_uno_reg_soc::dummy_mem::DUMMY_MEM_SIZE; use azihsm_fw_uno_trace::tracing::*; +use zeroize::Zeroize; use crate::UnoHsmPal; @@ -317,33 +318,45 @@ impl UnoHsmPal { } /// Volatile CPU zeroization of `[ptr, ptr + len)` that the optimizer cannot -/// elide, mirroring [`DmaBuf::zeroize`]. Used for the M7 TCM (which GDMA +/// elide, via the vetted [`zeroize`] crate. Used for the M7 TCM (which GDMA /// cannot reach) and as the GDMA-failure fallback for SRAM. /// -/// Writes 32-bit words when the region is word-aligned, and byte-wise -/// otherwise (or for a sub-word tail). Both per-IO regions are word-aligned -/// with word-multiple sizes (`DTCM_IO_BUF` 0x600, `SRAM_IO_BUF` 0x4800), so -/// in practice this is a pure word loop — 4x fewer stores than a byte-wise -/// wipe on the IO teardown path. +/// Zeroes the widest naturally-aligned element the region supports: an +/// 8-byte-aligned `u64` (QWORD) body, framed by a byte-wise head/tail for +/// any unaligned edges. `[u64]::zeroize()` lowers to 64-bit volatile stores +/// (a single `STRD`/64-bit AXI transfer on Cortex-M7), so it issues half as +/// many stores as a 32-bit wipe and a quarter of a byte-wise one. Both +/// per-IO regions are 8-byte aligned with 8-multiple sizes (`DTCM_IO_BUF` +/// 0x600, `SRAM_IO_BUF` 0x4800), so in practice this is a pure `u64` body +/// with no head or tail. `zeroize` emits the volatile writes plus a +/// compiler+atomic fence, so the wipe cannot be elided. /// /// # Safety /// /// `ptr` must be valid for writes of `len` bytes. #[inline] unsafe fn cpu_zeroize(ptr: *mut u8, len: usize) { - let mut i = 0usize; - // Word body: 4 bytes per store (both per-IO regions hit this path). - while i + 4 <= len && (ptr as usize + i).is_multiple_of(4) { - // SAFETY: `i + 4 <= len` and the address is 4-byte aligned, so this - // writes wholly within the caller's region at a valid alignment. - unsafe { core::ptr::write_volatile(ptr.add(i) as *mut u32, 0) }; - i += 4; + // Byte-wise head up to the first 8-byte boundary (empty for the aligned + // per-IO regions). + let head = ((8 - (ptr as usize & 7)) & 7).min(len); + if head != 0 { + // SAFETY: `head <= len`, so this stays within the caller's region. + unsafe { core::slice::from_raw_parts_mut(ptr, head) }.zeroize(); } - // Misaligned region or sub-word tail. - while i < len { - // SAFETY: `i < len`, so `ptr + i` is within the caller's region. - unsafe { core::ptr::write_volatile(ptr.add(i), 0) }; - i += 1; + // QWORD body: a whole number of `u64`s at an 8-byte-aligned address. + let body_len = (len - head) & !7usize; + if body_len != 0 { + // SAFETY: `ptr + head` is 8-byte aligned by construction and + // `body_len` is a `u64` multiple within the region, so the cast and + // slice are valid for writes. + let body_ptr = unsafe { ptr.add(head) } as *mut u64; + let body = unsafe { core::slice::from_raw_parts_mut(body_ptr, body_len / 8) }; + body.zeroize(); + } + // Byte-wise tail (sub-`u64` remainder; empty for the per-IO regions). + let tail_off = head + body_len; + if tail_off < len { + // SAFETY: `tail_off < len`, so `ptr + tail_off` is within the region. + unsafe { core::slice::from_raw_parts_mut(ptr.add(tail_off), len - tail_off) }.zeroize(); } - core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); }