From 9e3c2edfd89dd6d22837f10543dae60d60e089a3 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 9 Dec 2025 11:52:03 +0100 Subject: [PATCH 01/11] vmm: silence some messages in http_endpoint On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/http/http_endpoint.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vmm/src/api/http/http_endpoint.rs b/vmm/src/api/http/http_endpoint.rs index 371692b8a6..1b6954005c 100644 --- a/vmm/src/api/http/http_endpoint.rs +++ b/vmm/src/api/http/http_endpoint.rs @@ -38,7 +38,7 @@ use std::fs::File; use std::sync::mpsc::{Receiver, Sender, SyncSender}; use std::sync::{LazyLock, Mutex}; -use log::info; +use log::debug; use micro_http::{Body, Method, Request, Response, StatusCode, Version}; use vmm_sys_util::eventfd::EventFd; @@ -515,13 +515,13 @@ impl PutHandler for VmSendMigration { ) .map_err(HttpError::ApiError)?; - info!("live migration started"); + debug!("live migration started"); let (_, receiver) = &*ONGOING_LIVEMIGRATION; - info!("waiting for live migration result"); + debug!("waiting for live migration result"); let mig_res = receiver.lock().unwrap().recv().unwrap(); - info!("received live migration result"); + debug!("received live migration result"); // We forward the migration error here to the guest mig_res From 51edb7cd6359a3d9d68504a018c987e5e797ccfa Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 29 Jan 2026 15:06:13 +0100 Subject: [PATCH 02/11] vmm: improve logging of ongoing live-migrations The logging is not very spammy nor costly (iterations take seconds to dozens of minutes) and is clearly a win for us to debug things. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index d8d013822d..64dd7c10cd 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -2240,13 +2240,16 @@ impl Vmm { s.iteration_duration = s.iteration_start_time.elapsed(); info!( - "iteration:{},cost={}ms,throttle={}%,transmitted={}MiB,dirty_rate={}pps,Mebibyte/s={:.2}", + "iter={},dur={}ms,overhead={}ms,throttle={}%,size={}MiB,dirtyrate={},bandwidth={:.2}MiBs,downtime={}ms", s.iteration, s.iteration_duration.as_millis(), + (s.iteration_duration - s.transmit_duration).as_millis(), vm.throttle_percent(), - s.bytes_to_transmit / 1024 / 1024, + s.bytes_to_transmit.div_ceil(1024).div_ceil(1024), s.dirty_rate_pps, - s.bytes_per_sec / 1024.0 / 1024.0 + s.bytes_per_sec / 1024.0 / 1024.0, + s.calculated_downtime_duration + .map_or(migrate_downtime_limit.as_millis(), |d| d.as_millis()), ); // Increment iteration counter From 0ccfecfdc6e00fe404360670673981748b2315ae Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 13 Jan 2026 11:26:55 +0100 Subject: [PATCH 03/11] vm-migration: prepare progress types for new API endpoint This is the first commit in a series of commits to introduce a new API endpoint in Cloud Hypervisor to report progress and live-insights about an ongoing live migration. Having live and frequently refreshing statistics/metrics about an ongoing live migration is especially interesting for debugging and monitoring. For the first time, we will be able to see how live-migrations behave and create benchmarking infrastructure around it. The ch driver in libvirt will use these information to populate its `virsh domjobinfo` information. We will add a new API endpoint to query information. Further, the endpoint will be interesting to query information about a previously failed or canceled live migration. Specifically interesting about this API endpoint is that it will be the first endpoint that needs the "asynchronization" of the API: more than one API request in parallel. This needs support at least in the HTTP API and the internal API. The "SendMigration" call is long-running and active even if someone is querying the new endpoint. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vm-migration/src/lib.rs | 4 +- vm-migration/src/progress.rs | 424 +++++++++++++++++++++++++++++++++++ 2 files changed, 427 insertions(+), 1 deletion(-) create mode 100644 vm-migration/src/progress.rs diff --git a/vm-migration/src/lib.rs b/vm-migration/src/lib.rs index 7ae78eaf24..832531af5a 100644 --- a/vm-migration/src/lib.rs +++ b/vm-migration/src/lib.rs @@ -9,10 +9,12 @@ use thiserror::Error; use crate::protocol::MemoryRangeTable; -mod bitpos_iterator; +pub mod progress; pub mod protocol; pub mod tls; +mod bitpos_iterator; + #[derive(Error, Debug)] pub enum MigratableError { #[error("Failed to pause migratable component")] diff --git a/vm-migration/src/progress.rs b/vm-migration/src/progress.rs new file mode 100644 index 0000000000..39895b09df --- /dev/null +++ b/vm-migration/src/progress.rs @@ -0,0 +1,424 @@ +// Copyright © 2025 Cyberus Technology GmbH +// +// SPDX-License-Identifier: Apache-2.0 + +//! Module for reporting of the live-migration progress. +//! +//! The main export is [`MigrationProgress`]. +//! +//! # Motivation +//! +//! Monitoring a live-migration is important for debugging of cloud deployments, +//! for cloud monitoring in general, and for network optimization, such as +//! verifying the throughput for the migration is as high as expected. +//! +//! It also helps to analyze the downtime of VMs and see how much pressure a +//! guest is putting on its memory (by writing), which is slowing down +//! migrations. + +use std::error::Error; +use std::fmt; +use std::fmt::Display; +use std::num::NonZeroU32; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +#[derive( + Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub enum TransportationMode { + Local, + Tcp { connections: NonZeroU32, tls: bool }, +} + +/// Carries information about the transmission of the VM's memory. +#[derive( + Clone, + Copy, + Debug, + Default, + PartialOrd, + Ord, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, +)] +pub struct MemoryTransmissionInfo { + /// The memory iteration (only in precopy mode). + pub memory_iteration: u64, + /// Memory bytes per second. + pub memory_transmission_bps: u64, + /// The total size of the VMs memory in bytes. + pub memory_bytes_total: u64, + /// The total size of transmitted bytes. + pub memory_bytes_transmitted: u64, + /// The amount of remaining bytes for this iteration. + pub memory_bytes_remaining_iteration: u64, + /// The amount of transmitted 4k pages. + pub memory_pages_4k_transmitted: u64, + /// The amount of remaining 4k pages for this iteration. + pub memory_pages_4k_remaining_iteration: u64, + /// The amount of constant pages for that we could take a shortcut. + /// Pages where all bits are either zero or one. + pub memory_pages_constant_count: u64, + /// Current memory dirty rate in pages per seconds (pps). + pub memory_dirty_rate_pps: u64, +} + +/// The different phases of an ongoing ([`MigrationState::Ongoing`]) migration +/// (good case). +/// +/// The states correspond to the [live-migration protocol]. +/// +/// [live-migration protocol]: super::protocol +#[derive( + Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub enum MigrationStateOngoingPhase { + /// The migration starts. Handshake and transfer of VM config. + Starting, + /// Transfer of memory FDs. + /// + /// Only used for local migrations. + MemoryFds, + /// Transfer of VM memory in precopy mode. + /// + /// Not used for local migrations. + MemoryPrecopy, + // TODO eventually add MemoryPostcopy here + /// The VM migration is completing. This means the last chunks of memory + /// are transmitted as well as the final VM state (vCPUs, devices). + Completing, +} + +impl Display for MigrationStateOngoingPhase { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Starting => write!(f, "starting"), + Self::MemoryFds => write!(f, "memory FDs"), + Self::MemoryPrecopy => write!(f, "memory (precopy)"), + Self::Completing => write!(f, "completing"), + } + } +} + +/// The different states of a migration, covering steady progress and failure. +#[derive( + Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub enum MigrationState { + /// The migration has been cancelled. + Cancelled {}, + /// The migration has failed. + Failed { + /// Stringified error. + error_msg: String, + /// Debug-stringified error. + error_msg_debug: String, + // TODO this is very tricky because I need clone() + // error: Box, + }, + /// The migration has finished successfully. + Finished {}, + /// The migration is ongoing. + Ongoing { + phase: MigrationStateOngoingPhase, + /// Percent in range `0..=100`. + vcpu_throttle_percent: u8, + }, +} + +impl Display for MigrationState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MigrationState::Cancelled { .. } => write!(f, "{}", self.state_name()), + MigrationState::Failed { error_msg, .. } => { + write!(f, "{}: {error_msg}", self.state_name()) + } + MigrationState::Finished { .. } => write!(f, "{}", self.state_name()), + MigrationState::Ongoing { + phase, + vcpu_throttle_percent, + } => write!( + f, + "{}: phase={phase}, vcpu_throttle={vcpu_throttle_percent}", + self.state_name() + ), + } + } +} + +impl MigrationState { + fn state_name(&self) -> &'static str { + match self { + MigrationState::Cancelled { .. } => "cancelled", + MigrationState::Failed { .. } => "failed", + MigrationState::Finished { .. } => "finished", + MigrationState::Ongoing { .. } => "ongoing", + } + } +} + +/// Returns the current UNIX timestamp in ms. +fn current_unix_timestamp_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should be valid duration") + .as_millis() as u64 +} + +/// Holds a snapshot of progress and status information for an ongoing live +/// migration, or the last snapshot of a canceled or aborted migration. +/// +/// This type carries insightful information for every step of the +/// [live-migration protocol] in a way that makes it easy for API users to +/// parse the data with ease while retaining all important information. +/// +/// [live-migration protocol]: super::protocol +#[derive( + Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub struct MigrationProgress { + /// UNIX timestamp of the start of the live-migration process in ms. + pub timestamp_begin_ms: u64, + /// UNIX timestamp of the current snapshot in ms. + pub timestamp_snapshot_ms: u64, + /// Relative timestamp since the beginning of the migration in ms. + pub timestamp_snapshot_relative_ms: u64, + /// Configured target downtime. + pub downtime_configured_ms: u64, + /// Currently estimated (computed) downtime given the remaining + /// transmissions and the bandwidth. + /// + /// If this is `0`, the downtime could not yet be calculated. + pub downtime_estimated_ms: u64, + /// Requested transportation mode. + pub transportation_mode: TransportationMode, + /// Snapshot of the current phase. + pub state: MigrationState, + /// Latest [`MemoryTransmissionInfo`] info, if any. + /// + /// The most interesting phase is when current state is + /// [`MigrationState::Ongoing`] and [`MigrationStateOngoingPhase::MemoryPrecopy`] + /// as this value will be updated frequently. + pub memory_transmission_info: MemoryTransmissionInfo, +} + +impl MigrationProgress { + /// Creates new progress in a valid init state. + /// + /// This progress must be updated using any of: + /// - [`Self::update`] + /// - [`Self::mark_as_finished`] + /// - [`Self::mark_as_failed`] + /// - [`Self::mark_as_cancelled`] + pub fn new(transportation_mode: TransportationMode, target_downtime: Duration) -> Self { + let timestamp = current_unix_timestamp_ms(); + Self { + timestamp_begin_ms: timestamp, + timestamp_snapshot_ms: timestamp, + timestamp_snapshot_relative_ms: 0, + downtime_configured_ms: target_downtime.as_millis() as u64, + downtime_estimated_ms: 0, + transportation_mode, + state: MigrationState::Ongoing { + phase: MigrationStateOngoingPhase::Starting, + vcpu_throttle_percent: 0, + }, + memory_transmission_info: MemoryTransmissionInfo::default(), + } + } + + /// Updates the state of an ongoing migration. + /// + /// Only updates new values that are provided via `Some`. + /// + /// # Arguments + /// + /// - `new_phase`: The current [`MigrationStateOngoingPhase`]. + /// - `new_memory_transmission_info`: If `Some`, the current [`MemoryTransmissionInfo`]. + /// - `new_cpu_throttle_percent`: If `Some`, the current value of the vCPU throttle percentage. + /// Must be in range `0..=100`. + /// - `new_estimated_downtime`: If `Some`, the latest expected (calculated) downtime. + pub fn update( + &mut self, + new_phase: MigrationStateOngoingPhase, + new_memory_transmission_info: Option, + new_cpu_throttle_percent: Option, + new_estimated_downtime: Option, + ) { + if let Some(percent) = new_cpu_throttle_percent { + assert!(percent <= 100); + } + + if let Some(downtime) = new_estimated_downtime { + self.downtime_estimated_ms = u64::try_from(downtime.as_millis()).unwrap(); + } else { + // This is better than showing `0` and it is likely close to the final actual downtime. + self.downtime_estimated_ms = self.downtime_configured_ms; + } + + match &self.state { + MigrationState::Ongoing { + phase: _old_phase, + vcpu_throttle_percent: old_vcpu_throttle_percent, + } => { + self.timestamp_snapshot_ms = current_unix_timestamp_ms(); + self.timestamp_snapshot_relative_ms = + self.timestamp_snapshot_ms - self.timestamp_begin_ms; + + self.memory_transmission_info = + new_memory_transmission_info.unwrap_or(self.memory_transmission_info); + self.state = MigrationState::Ongoing { + phase: new_phase, + vcpu_throttle_percent: new_cpu_throttle_percent + .unwrap_or(*old_vcpu_throttle_percent), + }; + } + illegal => { + // panic is fine as we have a logic error here, nothing that was caused by a user. + panic!( + "illegal state transition: {} -> ongoing", + illegal.state_name(), + ); + } + } + } + + /// Sets the underlying state to [`MigrationState::Cancelled`] and + /// updates all corresponding metadata. + /// + /// After this state change, the object is supposed to be handled as immutable. + pub fn mark_as_cancelled(&mut self) { + if !matches!(self.state, MigrationState::Ongoing { .. }) { + panic!( + "illegal state transition: {} -> cancelled", + self.state.state_name() + ); + } + self.timestamp_snapshot_ms = current_unix_timestamp_ms(); + self.timestamp_snapshot_relative_ms = self.timestamp_snapshot_ms - self.timestamp_begin_ms; + self.state = MigrationState::Cancelled {}; + } + + /// Sets the underlying state to [`MigrationState::Failed`] and + /// updates all corresponding metadata. + /// + /// After this state change, the object is supposed to be handled as immutable. + pub fn mark_as_failed(&mut self, error: &dyn Error) { + if !matches!(self.state, MigrationState::Ongoing { .. }) { + panic!( + "illegal state transition: {} -> failed", + self.state.state_name() + ); + } + self.timestamp_snapshot_ms = current_unix_timestamp_ms(); + self.timestamp_snapshot_relative_ms = self.timestamp_snapshot_ms - self.timestamp_begin_ms; + self.state = MigrationState::Failed { + error_msg: format!("{error}",), + error_msg_debug: format!("{error:?}",), + }; + } + + /// Sets the underlying state to [`MigrationState::Finished`] and + /// updates all corresponding metadata. + /// + /// After this state change, the object is supposed to be handled as immutable. + pub fn mark_as_finished(&mut self) { + if !matches!(self.state, MigrationState::Ongoing { .. }) { + panic!( + "illegal state transition: {} -> finished", + self.state.state_name() + ); + } + self.timestamp_snapshot_ms = current_unix_timestamp_ms(); + self.timestamp_snapshot_relative_ms = self.timestamp_snapshot_ms - self.timestamp_begin_ms; + self.state = MigrationState::Finished {}; + } +} + +#[cfg(test)] +mod tests { + use anyhow::anyhow; + + use super::*; + + // Helpful to see what the API will look like. + #[test] + fn print_json() { + let starting = MigrationProgress::new( + TransportationMode::Tcp { + connections: NonZeroU32::new(1).unwrap(), + tls: false, + }, + Duration::from_millis(100), + ); + let memory_precopy = { + let mut state = starting.clone(); + state.update( + MigrationStateOngoingPhase::MemoryPrecopy, + Some(MemoryTransmissionInfo { + memory_iteration: 7, + memory_transmission_bps: 0, + memory_bytes_total: 0x1337, + memory_bytes_transmitted: 0x1337, + memory_pages_4k_transmitted: 42, + memory_pages_4k_remaining_iteration: 42, + memory_bytes_remaining_iteration: 124, + memory_dirty_rate_pps: 42, + memory_pages_constant_count: 0, + }), + Some(42), + Some(Duration::from_millis(200)), + ); + state + }; + let completing = { + let mut state = memory_precopy.clone(); + state.update( + MigrationStateOngoingPhase::Completing, + None, + Some(99), + Some(Duration::from_millis(25)), + ); + state + }; + let completed = { + let mut state = completing.clone(); + state.mark_as_finished(); + state + }; + let failed = { + let mut state = completing.clone(); + let error = anyhow!("Some very bad error".to_string()); + let error: &dyn Error = error.as_ref(); + state.mark_as_failed(error); + state + }; + let cancelled = { + let mut state = completing.clone(); + state.mark_as_cancelled(); + state + }; + + let vals = [ + starting, + memory_precopy, + completing, + completed, + failed, + cancelled, + ]; + for val in vals { + println!("state: {}", val.state.state_name()); + println!("Rust: {val:?}"); + println!( + "serde_json: {}", + serde_json::to_string_pretty(&val).unwrap() + ); + println!(); + println!("================="); + } + } +} From 35f66510bd37dc0168401c054a0e96950b57e1b1 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 13 Jan 2026 11:27:01 +0100 Subject: [PATCH 04/11] vmm: add migration-progress API endpoint This is part of the commit series to enable live updates about an ongoing live migration. See the first commit for an introduction. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/mod.rs | 49 ++++++++++++++++++++++++++++++++++++++++++++++ vmm/src/lib.rs | 5 +++++ 2 files changed, 54 insertions(+) diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index a39fc8a477..cb1f8bac1e 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -43,6 +43,7 @@ use micro_http::Body; use serde::{Deserialize, Serialize}; use thiserror::Error; use vm_migration::MigratableError; +use vm_migration::progress::MigrationProgress; use vmm_sys_util::eventfd::EventFd; #[cfg(feature = "dbus_api")] @@ -203,6 +204,10 @@ pub enum ApiError { /// Error triggering NMI #[error("Error triggering NMI")] VmNmi(#[source] VmError), + + /// Error fetching the migration progress + #[error("Error fetching the migration progress")] + VmMigrationProgress(#[source] VmError), } pub type ApiResult = Result; @@ -314,6 +319,9 @@ pub enum ApiResponsePayload { /// Virtual machine information VmInfo(VmInfoResponse), + /// The progress of a possibly ongoing live migration. + VmMigrationProgress(Box>), + /// Vmm ping response VmmPing(VmmPingResponse), @@ -399,6 +407,8 @@ pub trait RequestHandler { ) -> Result<(), MigratableError>; fn vm_nmi(&mut self) -> Result<(), VmError>; + + fn vm_migration_progress(&mut self) -> Result, VmError>; } /// It would be nice if we could pass around an object like this: @@ -1531,3 +1541,42 @@ impl ApiAction for VmNmi { get_response_body(self, api_evt, api_sender, data) } } + +pub struct VmMigrationProgress; + +impl ApiAction for VmMigrationProgress { + type RequestBody = (); + type ResponseBody = Box>; + + fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { + Box::new(move |vmm| { + info!("API request event: VmMigrationProgress"); + + let response = vmm + .vm_migration_progress() + .map(Box::new) + .map(ApiResponsePayload::VmMigrationProgress) + .map_err(ApiError::VmMigrationProgress); + + response_sender + .send(response) + .map_err(VmmError::ApiResponseSend)?; + + Ok(false) + }) + } + + fn send( + &self, + api_evt: EventFd, + api_sender: Sender, + data: Self::RequestBody, + ) -> ApiResult { + let info = get_response(self, api_evt, api_sender, data)?; + + match info { + ApiResponsePayload::VmMigrationProgress(info) => Ok(info), + _ => Err(ApiError::ResponsePayloadType), + } + } +} diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 64dd7c10cd..d4e2dc9a2d 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -56,6 +56,7 @@ use vm_memory::{ GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryAtomic, ReadVolatile, VolatileMemoryError, VolatileSlice, WriteVolatile, }; +use vm_migration::progress::MigrationProgress; use vm_migration::protocol::*; use vm_migration::tls::{TlsConnectionWrapper, TlsStream, TlsStreamWrapper}; use vm_migration::{ @@ -3632,6 +3633,10 @@ impl RequestHandler for Vmm { Ok(()) } + + fn vm_migration_progress(&mut self) -> result::Result, VmError> { + Ok(None) + } } const CPU_MANAGER_SNAPSHOT_ID: &str = "cpu-manager"; From 440039ca679e25ffb688db3ff17e1a0f4d53f315 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Mon, 12 Jan 2026 17:39:49 +0100 Subject: [PATCH 05/11] vmm: add migration-progress HTTP endpoint This is part of the commit series to enable live updates about an ongoing live migration. See the first commit for an introduction. In this commit, we add the HTTP endpoint to export ongoing VM live-migration progress. This work was made possible because of the following fundamental prerequisites: - internal API was made async - http thread was made async This way, one can send requests to fetch the latest state without blocking in any code path of the API. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/http/http_endpoint.rs | 33 ++++++++++++++++++++++++++++--- vmm/src/api/http/mod.rs | 10 +++++++--- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/vmm/src/api/http/http_endpoint.rs b/vmm/src/api/http/http_endpoint.rs index 1b6954005c..b840b93af1 100644 --- a/vmm/src/api/http/http_endpoint.rs +++ b/vmm/src/api/http/http_endpoint.rs @@ -58,9 +58,10 @@ use crate::api::http::http_endpoint::fds_helper::{attach_fds_to_cfg, attach_fds_ use crate::api::http::{EndpointHandler, HttpError, error_response}; use crate::api::{ AddDisk, ApiAction, ApiError, ApiRequest, NetConfig, VmAddDevice, VmAddFs, VmAddNet, VmAddPmem, - VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, VmConfig, VmCounters, VmDelete, VmNmi, VmPause, - VmPowerButton, VmReboot, VmReceiveMigration, VmReceiveMigrationData, VmRemoveDevice, VmResize, - VmResizeDisk, VmResizeZone, VmRestore, VmResume, VmSendMigration, VmShutdown, VmSnapshot, + VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, VmConfig, VmCounters, VmDelete, + VmMigrationProgress, VmNmi, VmPause, VmPowerButton, VmReboot, VmReceiveMigration, + VmReceiveMigrationData, VmRemoveDevice, VmResize, VmResizeDisk, VmResizeZone, VmRestore, + VmResume, VmSendMigration, VmShutdown, VmSnapshot, }; use crate::config::RestoreConfig; use crate::cpu::Error as CpuError; @@ -708,6 +709,32 @@ impl EndpointHandler for VmmShutdown { } } +impl EndpointHandler for VmMigrationProgress { + fn handle_request( + &self, + req: &Request, + api_notifier: EventFd, + api_sender: Sender, + ) -> Response { + match req.method() { + Method::Get => match crate::api::VmMigrationProgress + .send(api_notifier, api_sender, ()) + .map_err(HttpError::ApiError) + { + Ok(info) => { + let mut response = Response::new(Version::Http11, StatusCode::OK); + let info_serialized = serde_json::to_string(&info).unwrap(); + + response.set_body(Body::new(info_serialized)); + response + } + Err(e) => error_response(e, StatusCode::InternalServerError), + }, + _ => error_response(HttpError::BadRequest, StatusCode::BadRequest), + } + } +} + #[cfg(test)] mod external_fds_tests { use super::*; diff --git a/vmm/src/api/http/mod.rs b/vmm/src/api/http/mod.rs index 2aa52e8e37..764bc608db 100644 --- a/vmm/src/api/http/mod.rs +++ b/vmm/src/api/http/mod.rs @@ -29,9 +29,9 @@ use self::http_endpoint::{VmActionHandler, VmCreate, VmInfo, VmmPing, VmmShutdow use crate::api::VmCoredump; use crate::api::{ AddDisk, ApiError, ApiRequest, VmAddDevice, VmAddFs, VmAddNet, VmAddPmem, VmAddUserDevice, - VmAddVdpa, VmAddVsock, VmBoot, VmCounters, VmDelete, VmNmi, VmPause, VmPowerButton, VmReboot, - VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeDisk, VmResizeZone, VmRestore, VmResume, - VmSendMigration, VmShutdown, VmSnapshot, + VmAddVdpa, VmAddVsock, VmBoot, VmCounters, VmDelete, VmMigrationProgress, VmNmi, VmPause, + VmPowerButton, VmReboot, VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeDisk, + VmResizeZone, VmRestore, VmResume, VmSendMigration, VmShutdown, VmSnapshot, }; use crate::landlock::Landlock; use crate::seccomp_filters::{Thread, get_seccomp_filter}; @@ -275,6 +275,10 @@ pub static HTTP_ROUTES: LazyLock = LazyLock::new(|| { endpoint!("/vm.shutdown"), Box::new(VmActionHandler::new(&VmShutdown)), ); + r.routes.insert( + endpoint!("/vm.migration-progress"), + Box::new(VmMigrationProgress {}), + ); r.routes.insert( endpoint!("/vm.snapshot"), Box::new(VmActionHandler::new(&VmSnapshot)), From af9eb959737cf96e313b81825ece5bece1040d9f Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 22 Jan 2026 13:11:18 +0100 Subject: [PATCH 06/11] vmm: prepare migration code for migration progress reporting This is part of the commit series to enable live updates about an ongoing live migration. See the first commit for an introduction. This commit prepares the avoidance of naming clashes in the following. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index d4e2dc9a2d..c0e8a65028 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -697,7 +697,7 @@ impl VmmVersionInfo { /// /// Is supposed to be updated on the fly. #[derive(Debug, Clone)] -struct MigrationState { +struct MigrationStateInternal { /* ---------------------------------------------- */ /* Properties that are updated before the first iteration */ /// The instant where the actual downtime of the VM began. @@ -743,7 +743,7 @@ struct MigrationState { migration_duration: Duration, } -impl MigrationState { +impl MigrationStateInternal { pub fn new() -> Self { Self { // Field will be overwritten later. @@ -2111,7 +2111,7 @@ impl Vmm { Ok(()) } - fn can_increase_autoconverge_step(s: &MigrationState) -> bool { + fn can_increase_autoconverge_step(s: &MigrationStateInternal) -> bool { if s.iteration < AUTO_CONVERGE_ITERATION_DELAY { false } else { @@ -2128,7 +2128,7 @@ impl Vmm { vm: &mut Vm, mem_send: &SendAdditionalConnections, socket: &mut SocketStream, - s: &mut MigrationState, + s: &mut MigrationStateInternal, migration_timeout: Duration, migrate_downtime_limit: Duration, ) -> result::Result { @@ -2263,7 +2263,7 @@ impl Vmm { fn do_memory_migration( vm: &mut Vm, socket: &mut SocketStream, - s: &mut MigrationState, + s: &mut MigrationStateInternal, send_data_migration: &VmSendMigrationData, ) -> result::Result<(), MigratableError> { let mem_send = SendAdditionalConnections::new(send_data_migration, &vm.guest_memory())?; @@ -2346,7 +2346,7 @@ impl Vmm { hypervisor: &dyn hypervisor::Hypervisor, send_data_migration: &VmSendMigrationData, ) -> result::Result<(), MigratableError> { - let mut s = MigrationState::new(); + let mut s = MigrationStateInternal::new(); // Set up the socket connection let mut socket = send_migration_socket(send_data_migration)?; From 3ea3e7d919bf621a8e7eb0c8fb24f62c2ab6d869 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 22 Jan 2026 13:13:33 +0100 Subject: [PATCH 07/11] vmm: actually populate migration progress This is part of the commit series to enable live updates about an ongoing live migration. See the first commit for an introduction. This commit actually brings all the functionality together. The first version has the limitation that we populate the latest snapshot once per memory iteration, although this is the most interesting part by far. In a follow-up, we can make this more fine-grained. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 110 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 2 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index c0e8a65028..db5cc891df 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -56,7 +56,10 @@ use vm_memory::{ GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryAtomic, ReadVolatile, VolatileMemoryError, VolatileSlice, WriteVolatile, }; -use vm_migration::progress::MigrationProgress; +use vm_migration::progress::{ + MemoryTransmissionInfo, MigrationProgress, MigrationState, MigrationStateOngoingPhase, + TransportationMode, +}; use vm_migration::protocol::*; use vm_migration::tls::{TlsConnectionWrapper, TlsStream, TlsStreamWrapper}; use vm_migration::{ @@ -282,6 +285,9 @@ impl From for EpollDispatch { } } +// TODO make this a member of Vmm? +static MIGRATION_PROGRESS_SNAPSHOT: Mutex> = Mutex::new(None); + enum SocketStream { Unix(UnixStream), Tcp(TcpStream), @@ -2133,6 +2139,34 @@ impl Vmm { migrate_downtime_limit: Duration, ) -> result::Result { let mut iteration_table; + let total_memory_size_bytes = vm + .memory_range_table()? + .regions() + .iter() + .map(|range| range.length) + .sum::(); + + let update_migration_progress = |s: &mut MigrationStateInternal, vm: &Vm| { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .update( + MigrationStateOngoingPhase::MemoryPrecopy, + Some(MemoryTransmissionInfo { + memory_iteration: s.iteration, + memory_transmission_bps: s.bytes_per_sec as u64, + memory_bytes_total: total_memory_size_bytes, + memory_bytes_transmitted: s.total_transferred_bytes, + memory_pages_4k_transmitted: s.total_transferred_pages, + memory_pages_4k_remaining_iteration: s.pages_to_transmit, + memory_bytes_remaining_iteration: s.bytes_to_transmit, + memory_dirty_rate_pps: s.dirty_rate_pps, + memory_pages_constant_count: 0, /* TODO */ + }), + Some(vm.throttle_percent()), + s.calculated_downtime_duration, + ); + }; // We loop until we converge (target downtime is achievable). loop { @@ -2179,6 +2213,9 @@ impl Vmm { .sum(); s.pages_to_transmit = s.bytes_to_transmit.div_ceil(PAGE_SIZE as u64); + // Update before we might exit the loop. + update_migration_progress(s, vm); + // Unlikely happy-path. if s.bytes_to_transmit == 0 { break; @@ -2226,6 +2263,9 @@ impl Vmm { } } + // Update with new metrics before transmission. + update_migration_progress(s, vm); + // Send the current dirty pages s.transmit_start_time = Instant::now(); mem_send.send_memory(&iteration_table, socket)?; @@ -2346,6 +2386,31 @@ impl Vmm { hypervisor: &dyn hypervisor::Hypervisor, send_data_migration: &VmSendMigrationData, ) -> result::Result<(), MigratableError> { + // Update migration progress snapshot + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + if lock + .as_ref() + .map(|p| &p.state) + .is_some_and(|snapshot| matches!(snapshot, MigrationState::Ongoing { .. })) + { + // If this panic triggers, we made a programming error in our state handling. + panic!("migration already ongoing"); + } + let transportation_mode = if send_data_migration.local { + TransportationMode::Local + } else { + TransportationMode::Tcp { + connections: send_data_migration.connections, + tls: send_data_migration.tls_dir.is_some(), + } + }; + lock.replace(MigrationProgress::new( + transportation_mode, + Duration::from_millis(send_data_migration.downtime), + )); + } + let mut s = MigrationStateInternal::new(); // Set up the socket connection @@ -2400,6 +2465,11 @@ impl Vmm { if send_data_migration.local { match &mut socket { SocketStream::Unix(unix_socket) => { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .update(MigrationStateOngoingPhase::MemoryFds, None, None, None); + // Proceed with sending memory file descriptors over UNIX socket vm.send_memory_fds(unix_socket)?; } @@ -2442,6 +2512,14 @@ impl Vmm { Self::do_memory_migration(vm, &mut socket, &mut s, send_data_migration)?; } + // Update migration progress snapshot + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .update(MigrationStateOngoingPhase::Completing, None, None, None); + } + // We release the locks early to enable locking them on the destination host. // The VM is already stopped. vm.release_disk_locks() @@ -2492,6 +2570,22 @@ impl Vmm { s.iteration, ); + // Update migration progress snapshot + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .mark_as_finished(); + } + + // Give management software a chance to fetch the migration state. + // The VMM already executes on the other side and keeping Cloud Hypervisor running for a + // couple of more seconds is fine. + // + // We do the sleep in the migration thread to keep the internal API unblocked. + info!("Sleeping five seconds before shutting off."); + thread::sleep(Duration::from_secs(5)); + // Let every Migratable object know about the migration being complete vm.complete_migration() } @@ -2652,6 +2746,14 @@ impl Vmm { } } Err(e) => { + // Update migration progress snapshot + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .mark_as_failed(&e); + } + error!("Migration failed: {e}"); { info!("Sending Receiver in HTTP thread that migration failed"); @@ -3635,7 +3737,11 @@ impl RequestHandler for Vmm { } fn vm_migration_progress(&mut self) -> result::Result, VmError> { - Ok(None) + // We explicitly do not check here for `is VM running?` to always + // enable querying the state of the last failed migration. + let lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + let maybe_snapshot = lock.clone(); + Ok(maybe_snapshot) } } From 056e53cbcc3c44883f89364dbacff10ae4d33d14 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 13 Jan 2026 11:41:42 +0100 Subject: [PATCH 08/11] vmm: remove unnecessary Result This is part of the commit series to enable live updates about an ongoing live-migration. See the first commit for an introduction. There isn't really an error that can happen when we query this endpoint. A previous snapshot may either be there or not. It also doesn't make sense here to check if the current VM is running, as users should always be able to query information about the past (failed or canceled) live migration. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/mod.rs | 8 +++++--- vmm/src/lib.rs | 7 ++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index cb1f8bac1e..a3bf3a148d 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -408,7 +408,9 @@ pub trait RequestHandler { fn vm_nmi(&mut self) -> Result<(), VmError>; - fn vm_migration_progress(&mut self) -> Result, VmError>; + /// Returns the progress of the currently active migration or any previous + /// failed or canceled migration. + fn vm_migration_progress(&mut self) -> Option; } /// It would be nice if we could pass around an object like this: @@ -1552,8 +1554,8 @@ impl ApiAction for VmMigrationProgress { Box::new(move |vmm| { info!("API request event: VmMigrationProgress"); - let response = vmm - .vm_migration_progress() + let snapshot = Ok(vmm.vm_migration_progress()); + let response = snapshot .map(Box::new) .map(ApiResponsePayload::VmMigrationProgress) .map_err(ApiError::VmMigrationProgress); diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index db5cc891df..082f575aad 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -3736,12 +3736,9 @@ impl RequestHandler for Vmm { Ok(()) } - fn vm_migration_progress(&mut self) -> result::Result, VmError> { - // We explicitly do not check here for `is VM running?` to always - // enable querying the state of the last failed migration. + fn vm_migration_progress(&mut self) -> Option { let lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - let maybe_snapshot = lock.clone(); - Ok(maybe_snapshot) + lock.clone() } } From d86cd810cd3940a84766fd7681e4bfa2a8fdb202 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 9 Dec 2025 08:57:26 +0100 Subject: [PATCH 09/11] ch-remote: add migration-progress command This is part of the commit series to enable live updates about an ongoing live migration. See the first commit for an introduction. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- cloud-hypervisor/src/bin/ch-remote.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloud-hypervisor/src/bin/ch-remote.rs b/cloud-hypervisor/src/bin/ch-remote.rs index 42b6e05476..0e544b9ade 100644 --- a/cloud-hypervisor/src/bin/ch-remote.rs +++ b/cloud-hypervisor/src/bin/ch-remote.rs @@ -303,6 +303,8 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu Some("shutdown") => { simple_api_command(socket, "PUT", "shutdown", None).map_err(Error::HttpApiClient) } + Some("migration-progress") => simple_api_command(socket, "GET", "migration-progress", None) + .map_err(Error::HttpApiClient), Some("nmi") => simple_api_command(socket, "PUT", "nmi", None).map_err(Error::HttpApiClient), Some("resize") => { let resize = resize_config( @@ -1072,6 +1074,7 @@ fn get_cli_commands_sorted() -> Box<[Command]> { .arg(Arg::new("path").index(1).default_value("-")), Command::new("delete").about("Delete a VM"), Command::new("info").about("Info on the VM"), + Command::new("migration-progress"), Command::new("nmi").about("Trigger NMI"), Command::new("pause").about("Pause the VM"), Command::new("ping").about("Ping the VMM to check for API server availability"), From 66bcec8dbf02d094118a4aa275283003cb1a1a48 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 5 Feb 2026 14:26:46 +0100 Subject: [PATCH 10/11] vmm: migration: switch to non-blocking SendMigration call Time has proven that the previous design was not optimal. Now, the SendMigration call is not blocking for the duration of the migration. Instead, it just triggers the migration. Using the new MigrationProgress endpoint, management software can trigger the state of the migration and also find information for failed migrations. A new `keep_alive` parameter for SendMigration will keep the VMM alive and usable after the migration to ensure management software can fetch the final state. The management software is then supposed to send a ShutdownVmm command. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- cloud-hypervisor/src/bin/ch-remote.rs | 1 + vmm/src/api/http/http_endpoint.rs | 44 +++++--------------- vmm/src/api/mod.rs | 2 + vmm/src/lib.rs | 60 ++++++++++----------------- 4 files changed, 35 insertions(+), 72 deletions(-) diff --git a/cloud-hypervisor/src/bin/ch-remote.rs b/cloud-hypervisor/src/bin/ch-remote.rs index 0e544b9ade..3fc5d35a15 100644 --- a/cloud-hypervisor/src/bin/ch-remote.rs +++ b/cloud-hypervisor/src/bin/ch-remote.rs @@ -973,6 +973,7 @@ fn send_migration_data( migration_timeout, connections, tls_dir, + keep_alive: true, }; serde_json::to_string(&send_migration_data).unwrap() diff --git a/vmm/src/api/http/http_endpoint.rs b/vmm/src/api/http/http_endpoint.rs index b840b93af1..27bf18c5bf 100644 --- a/vmm/src/api/http/http_endpoint.rs +++ b/vmm/src/api/http/http_endpoint.rs @@ -35,23 +35,12 @@ //! [special HTTP library]: https://github.com/firecracker-microvm/micro-http use std::fs::File; -use std::sync::mpsc::{Receiver, Sender, SyncSender}; -use std::sync::{LazyLock, Mutex}; +use std::sync::mpsc::Sender; -use log::debug; +use log::info; use micro_http::{Body, Method, Request, Response, StatusCode, Version}; use vmm_sys_util::eventfd::EventFd; -/// Helper to make the VmSendMigration call blocking as long as a migration is ongoing. -#[allow(clippy::type_complexity)] -pub static ONGOING_LIVEMIGRATION: LazyLock<( - SyncSender>, - Mutex>>, -)> = LazyLock::new(|| { - let (sender, receiver) = std::sync::mpsc::sync_channel(0); - (sender, Mutex::new(receiver)) -}); - #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))] use crate::api::VmCoredump; use crate::api::http::http_endpoint::fds_helper::{attach_fds_to_cfg, attach_fds_to_cfgs}; @@ -508,26 +497,15 @@ impl PutHandler for VmSendMigration { _files: Vec, ) -> std::result::Result, HttpError> { if let Some(body) = body { - let res = self - .send( - api_notifier, - api_sender, - serde_json::from_slice(body.raw())?, - ) - .map_err(HttpError::ApiError)?; - - debug!("live migration started"); - - let (_, receiver) = &*ONGOING_LIVEMIGRATION; - - debug!("waiting for live migration result"); - let mig_res = receiver.lock().unwrap().recv().unwrap(); - debug!("received live migration result"); - - // We forward the migration error here to the guest - mig_res - .map(|_| res) - .map_err(|e| HttpError::ApiError(ApiError::VmSendMigration(e))) + self.send( + api_notifier, + api_sender, + serde_json::from_slice(body.raw())?, + ) + .inspect(|_| { + info!("live migration started (in background)"); + }) + .map_err(HttpError::ApiError) } else { Err(HttpError::BadRequest) } diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index a3bf3a148d..4af2823616 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -300,6 +300,8 @@ pub struct VmSendMigrationData { /// Directory containing the TLS root CA certificate (ca-cert.pem) #[serde(default)] pub tls_dir: Option, + /// Keep the VMM alive. + pub keep_alive: bool, } // Default value for downtime the same as qemu. diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 082f575aad..aa35c530a3 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -69,7 +69,6 @@ use vmm_sys_util::eventfd::EventFd; use vmm_sys_util::signal::unblock_signal; use vmm_sys_util::sock_ctrl_msg::ScmSocket; -use crate::api::http::http_endpoint::ONGOING_LIVEMIGRATION; use crate::api::{ ApiRequest, ApiResponse, RequestHandler, VmInfoResponse, VmReceiveMigrationData, VmSendMigrationData, VmmPingResponse, @@ -821,7 +820,7 @@ impl MigrationWorker { } /// Perform the migration and communicate with the [`Vmm`] thread. - fn run(mut self) -> (Vm, result::Result<(), MigratableError>) { + fn run(mut self) -> (Vm, result::Result<(), MigratableError>, VmSendMigrationData) { debug!("migration thread is starting"); let res = self.migrate().inspect_err(|e| error!("migrate error: {e}")); @@ -830,7 +829,7 @@ impl MigrationWorker { self.check_migration_evt.write(1).unwrap(); debug!("migration thread is finished"); - (self.vm, res) + (self.vm, res, self.config) } } @@ -900,7 +899,9 @@ pub struct Vmm { /// Handle to the [`MigrationWorker`] thread. /// /// The handle will return the [`Vm`] back in any case. Further, the underlying error (if any) is returned. - migration_thread_handle: Option)>>, + #[allow(clippy::type_complexity)] + migration_thread_handle: + Option, VmSendMigrationData)>>, } /// Wait for a file descriptor to become readable. In this case, we return @@ -2570,22 +2571,6 @@ impl Vmm { s.iteration, ); - // Update migration progress snapshot - { - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.as_mut() - .expect("live migration should be ongoing") - .mark_as_finished(); - } - - // Give management software a chance to fetch the migration state. - // The VMM already executes on the other side and keeping Cloud Hypervisor running for a - // couple of more seconds is fine. - // - // We do the sleep in the migration thread to keep the internal API unblocked. - info!("Sleeping five seconds before shutting off."); - thread::sleep(Duration::from_secs(5)); - // Let every Migratable object know about the migration being complete vm.complete_migration() } @@ -2721,31 +2706,36 @@ impl Vmm { fn check_migration_result(&mut self) { // At this point, the thread must be finished. // If we fail here, we have lost anyway. Just panic. - let (vm, migration_res) = self + let (vm, migration_res, migration_cfg) = self .migration_thread_handle .take() .expect("should have thread") .join() .expect("should have joined"); - // Give VMM back control. - self.vm = MaybeVmOwnership::Vmm(vm); - match migration_res { Ok(()) => { { - info!("Sending Receiver in HTTP thread that migration succeeded"); - let (sender, _) = &*ONGOING_LIVEMIGRATION; - // unblock API call; propagate migration result - sender.send(Ok(())).unwrap(); + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .mark_as_finished(); } - // Shutdown the VM after the migration succeeded - if let Err(e) = self.exit_evt.write(1) { - error!("Failed shutting down the VM after migration: {e}"); + if migration_cfg.keep_alive { + // API users can still query live-migration statistics + info!("Keeping VMM alive as requested"); + } else { + // Shutdown the VM after the migration succeeded + if let Err(e) = self.exit_evt.write(1) { + error!("Failed shutting down the VM after migration: {e}"); + } } + drop(vm); } Err(e) => { + // Give VMM back control. + self.vm = MaybeVmOwnership::Vmm(vm); // Update migration progress snapshot { let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); @@ -2753,14 +2743,6 @@ impl Vmm { .expect("live migration should be ongoing") .mark_as_failed(&e); } - - error!("Migration failed: {e}"); - { - info!("Sending Receiver in HTTP thread that migration failed"); - let (sender, _) = &*ONGOING_LIVEMIGRATION; - // unblock API call; propagate migration result - sender.send(Err(e)).unwrap(); - } // we don't fail the VMM here, it just continues running its VM } } From f66f9cac12e93feb1a2ec2192f427072cd0686fc Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 5 Feb 2026 14:27:09 +0100 Subject: [PATCH 11/11] ch-remote: wait for migration to finish by querying migration progress On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- Cargo.lock | 1 + cloud-hypervisor/Cargo.toml | 1 + cloud-hypervisor/src/bin/ch-remote.rs | 69 +++++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04bce118bb..f5fe91053d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -521,6 +521,7 @@ dependencies = [ "tpm", "tracer", "vm-memory", + "vm-migration", "vmm", "vmm-sys-util", "wait-timeout", diff --git a/cloud-hypervisor/Cargo.toml b/cloud-hypervisor/Cargo.toml index 426a522635..6f58723fda 100644 --- a/cloud-hypervisor/Cargo.toml +++ b/cloud-hypervisor/Cargo.toml @@ -36,6 +36,7 @@ thiserror = { workspace = true } tpm = { path = "../tpm" } tracer = { path = "../tracer" } vm-memory = { workspace = true } +vm-migration = { path = "../vm-migration" } vmm = { path = "../vmm" } vmm-sys-util = { workspace = true } zbus = { version = "5.7.1", optional = true } diff --git a/cloud-hypervisor/src/bin/ch-remote.rs b/cloud-hypervisor/src/bin/ch-remote.rs index 3fc5d35a15..6af932536e 100644 --- a/cloud-hypervisor/src/bin/ch-remote.rs +++ b/cloud-hypervisor/src/bin/ch-remote.rs @@ -13,15 +13,18 @@ use std::num::NonZeroU32; use std::os::unix::net::UnixStream; use std::path::PathBuf; use std::process; +use std::thread::sleep; +use std::time::Duration; use api_client::{ - Error as ApiClientError, simple_api_command, simple_api_command_with_fds, - simple_api_full_command, + Error as ApiClientError, StatusCode, simple_api_command, simple_api_command_with_fds, + simple_api_full_command, simple_api_full_command_and_response, }; use clap::{Arg, ArgAction, ArgMatches, Command}; -use log::error; +use log::{error, info}; use option_parser::{ByteSized, ByteSizedParseError}; use thiserror::Error; +use vm_migration::progress::{MigrationProgress, MigrationState}; use vmm::config::RestoreConfig; use vmm::vm_config::{ DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, UserDeviceConfig, VdpaConfig, @@ -524,8 +527,65 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu .unwrap() .get_one::("tls-dir") .cloned(), + true, ); + simple_api_command(socket, "PUT", "send-migration", Some(&send_migration_data)) + .map_err(Error::HttpApiClient)?; + + // Wait for migration to finish + loop { + let response = simple_api_full_command_and_response( + socket, + "GET", + "vm.migration-progress", + None, + ) + .map_err(Error::HttpApiClient)? + // should have response + .ok_or(Error::HttpApiClient(ApiClientError::ServerResponse( + StatusCode::Ok, + None, + )))?; + + assert_ne!( + response, "null", + "migration progress should be there immediately when the migration was dispatched" + ); + + let progress = serde_json::from_slice::(response.as_bytes()) + .map_err(|e| { + error!("failed to parse response as MigrationProgress: {e}"); + Error::HttpApiClient(ApiClientError::ServerResponse( + StatusCode::Ok, + Some(response), + )) + })?; + + match progress.state { + MigrationState::Cancelled { .. } => { + info!("Migration was cancelled"); + break; + } + MigrationState::Failed { + error_msg, + error_msg_debug, + } => { + error!("Migration failed! {error_msg}\n{error_msg_debug}"); + break; + } + MigrationState::Finished { .. } => { + info!("Migration finished successfully"); + break; + } + MigrationState::Ongoing { .. } => { + sleep(Duration::from_millis(50)); + continue; + } + } + } + + simple_api_full_command(socket, "PUT", "vmm.shutdown", None) .map_err(Error::HttpApiClient) } Some("receive-migration") => { @@ -965,6 +1025,7 @@ fn send_migration_data( migration_timeout: u64, connections: NonZeroU32, tls_dir: Option, + keep_alive: bool, ) -> String { let send_migration_data = vmm::api::VmSendMigrationData { destination_url: url, @@ -973,7 +1034,7 @@ fn send_migration_data( migration_timeout, connections, tls_dir, - keep_alive: true, + keep_alive, }; serde_json::to_string(&send_migration_data).unwrap()