diff --git a/.github/workflows/dev-release.yaml b/.github/workflows/dev-release.yaml new file mode 100644 index 000000000..0cfea5c76 --- /dev/null +++ b/.github/workflows/dev-release.yaml @@ -0,0 +1,85 @@ +name: Build and Release Dev + +on: + push: + branches: [dev] + +permissions: + contents: write + +concurrency: + group: dev-release + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install pinned Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.89.0" + + - name: Show toolchain + run: | + rustc --version --verbose + cargo --version --verbose + + - name: Build cloud-hypervisor (release) + run: | + cargo build --release + + - name: Generate checksum and build metadata + run: | + artifact=target/release/cloud-hypervisor + sha256sum "$artifact" > "${artifact}.sha256" + cat > target/release/build-info.json < Ok(Self::Copy), "ondemand" => Ok(Self::OnDemand), + "mmap" => Ok(Self::Mmap), _ => Err(MemoryRestoreModeParseError::InvalidValue(s.to_owned())), } } @@ -2817,11 +2821,11 @@ pub struct RestoreConfig { impl RestoreConfig { pub const SYNTAX: &'static str = "Restore from a VM snapshot. \ - \nRestore parameters \"source_url=,prefault=on|off,memory_restore_mode=copy|ondemand,\ + \nRestore parameters \"source_url=,prefault=on|off,memory_restore_mode=copy|ondemand|mmap,\ net_fds=,resume=true|false\" \ \n`source_url` should be a valid URL (e.g file:///foo/bar or tcp://192.168.1.10/foo) \ \n`prefault` controls eager prefaulting for the copy-based restore path (disabled by default) \ - \n`memory_restore_mode=copy` preserves the existing eager read-copy restore behavior, while `memory_restore_mode=ondemand` enables lazy demand paging and fails restore if userfaultfd support is unavailable \ + \n`memory_restore_mode=copy` preserves the existing eager read-copy restore behavior, `memory_restore_mode=ondemand` enables lazy demand paging and fails restore if userfaultfd support is unavailable, and `memory_restore_mode=mmap` maps the snapshot file copy-on-write (plain private RAM only; falls back to copy otherwise) \ \n`net_fds` is a list of net ids with new file descriptors. \ Only net devices backed by FDs directly are needed as input.\ \n `resume` controls whether the VM will be directly resumed after restore "; @@ -2880,7 +2884,7 @@ impl RestoreConfig { // corresponding 'RestoreNetConfig' with a matched 'id' and expected // number of FDs. pub fn validate(&self, vm_config: &VmConfig) -> ValidationResult<()> { - if self.memory_restore_mode == MemoryRestoreMode::OnDemand && self.prefault { + if self.memory_restore_mode != MemoryRestoreMode::Copy && self.prefault { return Err(ValidationError::InvalidRestorePrefaultWithOnDemand); } @@ -5333,6 +5337,18 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" invalid_restore_mode.validate(&snapshot_vm_config), Err(ValidationError::InvalidRestorePrefaultWithOnDemand) ); + + let invalid_mmap_prefault = RestoreConfig { + source_url: PathBuf::from("/path/to/snapshot"), + prefault: true, + memory_restore_mode: MemoryRestoreMode::Mmap, + net_fds: None, + resume: false, + }; + assert_eq!( + invalid_mmap_prefault.validate(&snapshot_vm_config), + Err(ValidationError::InvalidRestorePrefaultWithOnDemand) + ); } fn platform_fixture() -> PlatformConfig { diff --git a/vmm/src/memory_manager.rs b/vmm/src/memory_manager.rs index d04fcbb35..1a864b96d 100644 --- a/vmm/src/memory_manager.rs +++ b/vmm/src/memory_manager.rs @@ -395,6 +395,10 @@ pub enum Error { #[error("Error copying snapshot into region")] SnapshotCopy(#[source] GuestMemoryError), + /// Error mapping snapshot file over guest RAM + #[error("Error mapping snapshot file over guest RAM")] + SnapshotMmap(#[source] io::Error), + /// Failed to allocate MMIO address #[error("Failed to allocate MMIO address")] AllocateMmioAddress, @@ -909,6 +913,47 @@ impl MemoryManager { Ok(()) } + /// Restore guest memory by mapping the snapshot file copy-on-write over + /// the anonymous guest mappings — before any KVM memslot or device + /// consumes them, so overlay identity concerns do not apply. Falls back + /// to the eager copy whenever a range cannot be mapped safely; the + /// snapshot file must remain on disk for the VM lifetime. + fn mmap_saved_regions( + &mut self, + file_path: PathBuf, + saved_regions: &MemoryRangeTable, + ) -> Result<(), Error> { + if saved_regions.is_empty() { + return Ok(()); + } + let guest_memory = self.guest_memory.memory(); + if !mmap_restore_compatible(&guest_memory, saved_regions) { + info!("guest RAM unsuitable for mmap restore; falling back to copy"); + drop(guest_memory); + return self.fill_saved_regions(file_path, saved_regions); + } + let memory_file = OpenOptions::new() + .read(true) + .open(file_path) + .map_err(Error::SnapshotOpen)?; + // A range mapped past EOF faults SIGBUS at run time, not restore time. + let mapped_len: u64 = saved_regions.regions().iter().map(|r| r.length).sum(); + let file_len = memory_file.metadata().map_err(Error::SnapshotOpen)?.len(); + if file_len < mapped_len { + return Err(Error::SnapshotMmap(io::Error::new( + io::ErrorKind::UnexpectedEof, + "snapshot memory file is shorter than the saved ranges", + ))); + } + mmap_saved_ranges( + &guest_memory, + &memory_file, + saved_regions, + self.reserve, + self.thp, + ) + } + /// Restore guest memory using userfaultfd for lazy demand paging. /// /// Instead of reading the entire snapshot into guest RAM upfront (which @@ -1905,16 +1950,20 @@ impl MemoryManager { Default::default(), )?; - if memory_restore_mode == MemoryRestoreMode::OnDemand { - mm.lock().unwrap().restore_by_uffd( + match memory_restore_mode { + MemoryRestoreMode::OnDemand => mm.lock().unwrap().restore_by_uffd( &memory_file_path, &mem_snapshot.memory_ranges, exit_evt, - )?; - } else { - mm.lock() + )?, + MemoryRestoreMode::Mmap => mm + .lock() + .unwrap() + .mmap_saved_regions(memory_file_path, &mem_snapshot.memory_ranges)?, + MemoryRestoreMode::Copy => mm + .lock() .unwrap() - .fill_saved_regions(memory_file_path, &mem_snapshot.memory_ranges)?; + .fill_saved_regions(memory_file_path, &mem_snapshot.memory_ranges)?, } Ok(mm) @@ -3391,3 +3440,197 @@ impl Migratable for MemoryManager { Ok(table) } } + +/// Reports whether every saved range is page-aligned and lies wholly inside a +/// single plain private-anonymous guest region — the MAP_FIXED overlay +/// preconditions. File-backed regions are rejected: their stale mapping +/// metadata would misdirect a later snapshot or fd consumer. +fn mmap_restore_compatible( + guest_memory: &GuestMemoryMmap, + saved_regions: &MemoryRangeTable, +) -> bool { + // SAFETY: sysconf(_SC_PAGESIZE) has no failure mode relevant here. + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as u64; + let mut file_offset: u64 = 0; + for range in saved_regions.regions() { + if !file_offset.is_multiple_of(page_size) + || !range.gpa.is_multiple_of(page_size) + || !range.length.is_multiple_of(page_size) + { + return false; + } + let Some(end) = range.gpa.checked_add(range.length) else { + return false; + }; + let ok = guest_memory + .find_region(GuestAddress(range.gpa)) + .is_some_and(|r| { + r.file_offset().is_none() && end <= r.start_addr().raw_value() + r.len() + }); + if !ok { + return false; + } + let Some(next) = file_offset.checked_add(range.length) else { + return false; + }; + file_offset = next; + } + true +} + +/// Maps each saved range over its guest RAM window, re-applying reserve and +/// THP policy; ranges must have passed [`mmap_restore_compatible`]. +fn mmap_saved_ranges( + guest_memory: &GuestMemoryMmap, + memory_file: &File, + saved_regions: &MemoryRangeTable, + reserve: bool, + thp: bool, +) -> Result<(), Error> { + let reserve_flag = if reserve { 0 } else { libc::MAP_NORESERVE }; + let mut file_offset: u64 = 0; + for range in saved_regions.regions() { + let host_addr = guest_memory + .get_host_address(GuestAddress(range.gpa)) + .map_err(|e| Error::SnapshotMmap(io::Error::other(e)))?; + let length = range.length as usize; + // SAFETY: the window is page-aligned, wholly inside a live private + // anonymous region nothing consumes yet, so the MAP_FIXED replacement + // cannot clobber foreign mappings; the fd stays valid for the call. + let ret = unsafe { + libc::mmap( + host_addr.cast(), + length, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_FIXED | reserve_flag, + memory_file.as_raw_fd(), + file_offset as libc::off_t, + ) + }; + if ret == libc::MAP_FAILED { + return Err(Error::SnapshotMmap(io::Error::last_os_error())); + } + if thp { + // SAFETY: ret/length name the private mapping just installed above. + let adv = unsafe { libc::madvise(ret, length, libc::MADV_HUGEPAGE) }; + if adv != 0 { + warn!( + "mmap restore: MADV_HUGEPAGE failed: {}", + io::Error::last_os_error() + ); + } + } + file_offset = file_offset.checked_add(range.length).ok_or_else(|| { + Error::SnapshotMmap(io::Error::other("snapshot range file offset overflow")) + })?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::io::{Read, Seek, SeekFrom, Write}; + + use vm_migration::protocol::{MemoryRange, MemoryRangeTable}; + + use super::*; + + fn page_size() -> u64 { + // SAFETY: sysconf(_SC_PAGESIZE) has no failure mode relevant here. + unsafe { libc::sysconf(libc::_SC_PAGESIZE) as u64 } + } + + #[test] + fn mmap_restore_maps_data_and_reads_holes_as_zero() { + let page = page_size(); + let gm = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), (2 * page) as usize)]).unwrap(); + + let mut file = tempfile::tempfile().unwrap(); + file.write_all(&vec![0xabu8; page as usize]).unwrap(); + file.set_len(2 * page).unwrap(); + file.seek(SeekFrom::Start(0)).unwrap(); + + let mut table = MemoryRangeTable::default(); + table.push(MemoryRange { + gpa: 0, + length: 2 * page, + }); + + assert!(mmap_restore_compatible(&gm, &table)); + mmap_saved_ranges(&gm, &file, &table, false, false).unwrap(); + + assert_eq!(gm.read_obj::(GuestAddress(0)).unwrap(), 0xab); + assert_eq!(gm.read_obj::(GuestAddress(page - 1)).unwrap(), 0xab); + assert_eq!(gm.read_obj::(GuestAddress(page)).unwrap(), 0); + assert_eq!(gm.read_obj::(GuestAddress(2 * page - 1)).unwrap(), 0); + + // CoW: guest writes must not reach the file. + gm.write_obj::(0x5a, GuestAddress(0)).unwrap(); + let mut back = [0u8; 1]; + file.seek(SeekFrom::Start(0)).unwrap(); + file.read_exact(&mut back).unwrap(); + assert_eq!(back[0], 0xab); + } + + #[test] + fn mmap_restore_rejects_unaligned_or_out_of_region_ranges() { + let page = page_size(); + let gm = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), page as usize)]).unwrap(); + + let mut unaligned = MemoryRangeTable::default(); + unaligned.push(MemoryRange { + gpa: 1, + length: page, + }); + assert!(!mmap_restore_compatible(&gm, &unaligned)); + + let mut spanning = MemoryRangeTable::default(); + spanning.push(MemoryRange { + gpa: 0, + length: 2 * page, + }); + assert!(!mmap_restore_compatible(&gm, &spanning)); + } + + #[test] + fn mmap_restore_rejects_file_backed_region() { + let page = page_size(); + let backing = tempfile::tempfile().unwrap(); + backing.set_len(page).unwrap(); + let region = MmapRegion::build( + Some(FileOffset::new(backing, 0)), + page as usize, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + ) + .unwrap(); + let gm = GuestMemoryMmap::from_regions(vec![ + GuestRegionMmap::new(region, GuestAddress(0)).unwrap(), + ]) + .unwrap(); + + let mut table = MemoryRangeTable::default(); + table.push(MemoryRange { + gpa: 0, + length: page, + }); + assert!(!mmap_restore_compatible(&gm, &table)); + } + + #[test] + fn mmap_restore_honors_reserve_and_thp() { + let page = page_size(); + let gm = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), page as usize)]).unwrap(); + let mut file = tempfile::tempfile().unwrap(); + file.write_all(&vec![0xcdu8; page as usize]).unwrap(); + file.seek(SeekFrom::Start(0)).unwrap(); + + let mut table = MemoryRangeTable::default(); + table.push(MemoryRange { + gpa: 0, + length: page, + }); + mmap_saved_ranges(&gm, &file, &table, true, true).unwrap(); + assert_eq!(gm.read_obj::(GuestAddress(0)).unwrap(), 0xcd); + } +} diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 3bcb5a34d..8305f8a6f 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -1406,6 +1406,42 @@ impl Vm { vm_config.lock().unwrap().cpus.max_phys_bits, ); + let mut memory_restore_mode = memory_restore_mode.unwrap_or_default(); + if memory_restore_mode == MemoryRestoreMode::Mmap { + let config = vm_config.lock().unwrap(); + // Consumers that rely on guest RAM staying anonymous (DMA pinning, + // MADV_DONTNEED zeroing, KSM, per-zone reserve/NUMA replay) keep the + // eager copy; shared/hugepage zones are already rejected per region. + let mem = &config.memory; + let passthrough = config.devices.as_ref().is_some_and(|d| !d.is_empty()) + || config.user_devices.as_ref().is_some_and(|d| !d.is_empty()); + // pvmemcontrol madvises guest RAM as if anonymous. + #[cfg(feature = "pvmemcontrol")] + let pvmemcontrol = config.pvmemcontrol.is_some(); + #[cfg(not(feature = "pvmemcontrol"))] + let pvmemcontrol = false; + let unsupported_zone = mem.zones.as_ref().is_some_and(|zones| { + zones.iter().any(|z| { + z.host_numa_node.is_some() + || z.hotplug_size.is_some() + || z.hotplugged_size.is_some() + || z.reserve + || z.mergeable + }) + }); + if passthrough + || pvmemcontrol + || mem.mergeable + || mem.hotplug_size.is_some() + || unsupported_zone + { + warn!( + "memory_restore_mode=mmap needs non-resizable private RAM without passthrough, NUMA binding or KSM; using copy" + ); + memory_restore_mode = MemoryRestoreMode::Copy; + } + } + let memory_manager = if let Some(snapshot) = snapshot_from_id(snapshot, MEMORY_MANAGER_SNAPSHOT_ID) { MemoryManager::new_from_snapshot( @@ -1414,7 +1450,7 @@ impl Vm { &vm_config.lock().unwrap().memory.clone(), source_url, prefault.unwrap_or(false), - memory_restore_mode.unwrap_or_default(), + memory_restore_mode, phys_bits, &exit_evt, )