Skip to content

Fix HA-enabled VMs stuck in Running after out-of-band stop (OOM-kill) #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
poddm wants to merge 1 commit into
base: att/4.22.0.0
Choose a base branch
from
Open

Fix HA-enabled VMs stuck in Running after out-of-band stop (OOM-kill) #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 40 additions & 2 deletions engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,8 +209,10 @@
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.exception.StorageAccessException;
import com.cloud.exception.StorageUnavailableException;
import com.cloud.ha.HaWorkVO;
import com.cloud.ha.HighAvailabilityManager;
import com.cloud.ha.HighAvailabilityManager.WorkType;
import com.cloud.ha.dao.HighAvailabilityDao;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
Expand Down Expand Up @@ -361,6 +363,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
@Inject
private HighAvailabilityManager _haMgr;
@Inject
private HighAvailabilityDao _haDao;
@Inject
private HostPodDao _podDao;
@Inject
private DataCenterDao _dcDao;
Expand Down Expand Up @@ -5395,9 +5399,43 @@ private void handlePowerOffReportWithNoPendingJobsOnVM(final VMInstanceVO vm) {
&& HaVmRestartHostUp.value()
&& vm.getHypervisorType() != HypervisorType.VMware
&& vm.getHypervisorType() != HypervisorType.Hyperv) {
logger.info("Detected out-of-band stop of a HA enabled VM {}, will schedule restart.", vm);
logger.info("Detected out-of-band stop of HA enabled VM {}, transitioning to Stopped and scheduling HA restart (investigate=false).", vm);
if (!_haMgr.hasPendingHaWork(vm.getId())) {
_haMgr.scheduleRestart(vm, true);
// FLXCLD-2814 FIX: The power state report already confirmed the VM is off
// (e.g. OOM-killed QEMU process). We cannot use _haMgr.scheduleRestart() because it
// calls advanceStop(), which submits a VM work job and blocks waiting for completion.
// This code runs on the AgentManager-Handler thread, and the VM job queue does not
// dispatch jobs from this context — causing an indefinite block that leaves the VM
// stuck in Running. Instead we release resources, transition to Stopped, and insert
// an HA work item directly into op_ha_work (Step.Scheduled) for the HA worker to pick up.
final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
releaseVmResources(profile, true);

// Save lastHostId before transition (stateTransitTo sets hostId=null)
final Long lastHostId = vm.getHostId();
try {
stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOffReport, null);
} catch (final NoTransitionException e) {
logger.warn("Failed to transition VM {} to Stopped state: {}", vm, e.getMessage());
return;
}

_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(),
VM_SYNC_ALERT_SUBJECT, String.format("VM %s(%s) stopped out-of-band (OOM-killed or crashed). HA restart scheduled.",
vm.getHostName(), vm.getInstanceName()));

// Insert HA work item directly — bypasses advanceStop/job queue entirely
final VMInstanceVO refreshedVm = _vmDao.findByUuid(vm.getUuid());
final Long haHostId = lastHostId != null ? lastHostId : refreshedVm.getLastHostId();
if (haHostId == null || haHostId == 0L) {
logger.warn("Cannot schedule HA for VM {} — no valid host_id available (would violate FK constraint).", vm);
} else {
final HaWorkVO work = new HaWorkVO(refreshedVm.getId(), refreshedVm.getType(),
WorkType.HA, HighAvailabilityManager.Step.Scheduled, haHostId,
refreshedVm.getState(), 0, refreshedVm.getUpdated(), null);
_haDao.persist(work);
logger.info("Scheduled VM for HA: {}", refreshedVm);
}
} else {
logger.info("VM {} already has a pending HA task working on it.", vm);
}
Expand Down
3 changes: 3 additions & 0 deletions ...rvisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -533,6 +533,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_BLOCKED, PowerState.PowerOn);
POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_NOSTATE, PowerState.PowerUnknown);
POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_SHUTDOWN, PowerState.PowerOff);
// Report crashed domains as PowerOff immediately instead of omitting them
// from the report (which delays detection via the "missing VM" threshold).
POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_CRASHED, PowerState.PowerOff);
}

public VirtualRoutingResource virtRouterResource;
Expand Down