diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java index 3fbb5340fcce..456d88bbadb1 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java @@ -52,4 +52,11 @@ public interface KVMHAConfig { ConfigKey KvmHAFenceTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.fence.timeout", "60", "The maximum length of time, in seconds, expected for a fence operation to complete.", true, ConfigKey.Scope.Cluster); + + ConfigKey KvmHAFenceOnStorageHeartbeat = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat", "false", + "Whether to consider a host fenced when the out-of-band management fence operation fails (e.g. a complete power failure that takes down the BMC together with the host) " + + "but the neighbouring hosts in the cluster positively report the host's storage heartbeat as expired. The host is never considered fenced when any check still " + + "sees storage activity from it or when the heartbeat check itself fails. The heartbeat semantics follow the existing checks, including " + + "'kvm.ha.fence.host.if.heartbeat.fails.on.storage'. When disabled, fencing relies solely on out-of-band management.", + true, ConfigKey.Scope.Cluster); } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index f0b5cfc337de..533276bf8744 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -21,6 +21,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; +import com.cloud.host.Status; import com.cloud.hypervisor.Hypervisor; import org.apache.cloudstack.api.response.OutOfBandManagementResponse; @@ -89,17 +90,53 @@ public boolean fence(Host r) throws HAFenceException { try { if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.OFF, null); - return resp.getSuccess(); + if (resp.getSuccess()) { + return true; + } + logger.warn("OOBM fence operation failed for the host {}", r); + return fenceHostViaStorageHeartbeat(r); } else { logger.warn("OOBM fence operation failed for this host {}", r); - return false; + return fenceHostViaStorageHeartbeat(r); } } catch (Exception e){ logger.warn("OOBM service is not configured or enabled for this host {} error is {}", r, e.getMessage()); + if (fenceHostViaStorageHeartbeat(r)) { + return true; + } throw new HAFenceException(String.format("OBM service is not configured or enabled for this host %s", r.getName()), e); } } + /** + * Fallback fencing for when out-of-band management cannot power off the host, e.g. a + * complete power failure that takes down the BMC (IPMI/iDRAC/iLO) together with the host. + * The host is considered fenced only when the neighbouring hosts in the cluster positively + * report its storage heartbeat as expired; it is never considered fenced when any check + * still sees it alive or when the heartbeat check itself fails. + */ + protected boolean fenceHostViaStorageHeartbeat(final Host host) { + if (!isStorageHeartbeatFencingEnabled(host)) { + return false; + } + try { + final Status hostStatus = hostActivityChecker.getHostAgentStatus(host); + if (hostStatus == Status.Down) { + logger.warn("Neighbouring hosts report the storage heartbeat of the host {} as expired, considering it fenced as [{}] is enabled", + host, KVMHAConfig.KvmHAFenceOnStorageHeartbeat.key()); + return true; + } + logger.warn("Not considering the host {} fenced via storage heartbeat as its reported status is [{}]", host, hostStatus); + } catch (Exception e) { + logger.warn("Storage heartbeat fencing check failed for the host {}", host, e); + } + return false; + } + + protected boolean isStorageHeartbeatFencingEnabled(final Host host) { + return KVMHAConfig.KvmHAFenceOnStorageHeartbeat.valueIn(host.getClusterId()); + } + @Override public HAResource.ResourceSubType resourceSubType() { return HAResource.ResourceSubType.KVM; @@ -152,6 +189,7 @@ public ConfigKey[] getConfigKeys() { KVMHAConfig.KvmHADegradedMaxPeriod, KVMHAConfig.KvmHARecoverWaitPeriod, KVMHAConfig.KvmHARecoverAttemptThreshold, + KVMHAConfig.KvmHAFenceOnStorageHeartbeat, }; } } diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/kvm/ha/KVMHostHATest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/kvm/ha/KVMHostHATest.java index a94fb01475a3..4c2dd8d2f8d1 100644 --- a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/kvm/ha/KVMHostHATest.java +++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/kvm/ha/KVMHostHATest.java @@ -20,10 +20,18 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.apache.cloudstack.api.response.OutOfBandManagementResponse; import org.apache.cloudstack.ha.provider.HACheckerException; +import org.apache.cloudstack.ha.provider.HAFenceException; +import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement.PowerOperation; +import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService; import org.joda.time.DateTime; import org.junit.Before; import org.junit.Test; @@ -33,6 +41,7 @@ import com.cloud.exception.StorageUnavailableException; import com.cloud.host.Host; +import com.cloud.host.Status; import com.cloud.hypervisor.Hypervisor.HypervisorType; @RunWith(MockitoJUnitRunner.class) @@ -42,12 +51,21 @@ public class KVMHostHATest { private Host host; @Mock private KVMHostActivityChecker kvmHostActivityChecker; + @Mock + private OutOfBandManagementService outOfBandManagementService; private KVMHAProvider kvmHAProvider; @Before public void setup() { - kvmHAProvider = new KVMHAProvider(); + kvmHAProvider = spy(new KVMHAProvider()); kvmHAProvider.hostActivityChecker = kvmHostActivityChecker; + kvmHAProvider.outOfBandManagementService = outOfBandManagementService; + } + + private OutOfBandManagementResponse oobmResponse(final boolean success) { + final OutOfBandManagementResponse response = new OutOfBandManagementResponse(); + response.setSuccess(success); + return response; } @Test @@ -80,4 +98,64 @@ public void testHostActivityForDownHost() throws HACheckerException, StorageUnav assertFalse(kvmHAProvider.hasActivity(host, dt)); } + @Test + public void testFenceSucceedsViaOobm() throws HAFenceException { + when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true); + when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenReturn(oobmResponse(true)); + + assertTrue(kvmHAProvider.fence(host)); + + verify(kvmHostActivityChecker, never()).getHostAgentStatus(host); + } + + @Test + public void testFenceOobmFailedFallbackDisabled() throws HAFenceException { + when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true); + when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenReturn(oobmResponse(false)); + doReturn(false).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host); + + assertFalse(kvmHAProvider.fence(host)); + + verify(kvmHostActivityChecker, never()).getHostAgentStatus(host); + } + + @Test + public void testFenceOobmFailedStorageHeartbeatReportsHostDown() throws HAFenceException { + when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true); + when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenReturn(oobmResponse(false)); + doReturn(true).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host); + when(kvmHostActivityChecker.getHostAgentStatus(host)).thenReturn(Status.Down); + + assertTrue(kvmHAProvider.fence(host)); + } + + @Test + public void testFenceOobmFailedStorageHeartbeatSeesHostAlive() throws HAFenceException { + when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true); + when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenReturn(oobmResponse(false)); + doReturn(true).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host); + when(kvmHostActivityChecker.getHostAgentStatus(host)).thenReturn(Status.Disconnected); + + assertFalse(kvmHAProvider.fence(host)); + } + + @Test + public void testFenceOobmExceptionStorageHeartbeatReportsHostDown() throws HAFenceException { + when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true); + when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenThrow(new RuntimeException("BMC unreachable")); + doReturn(true).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host); + when(kvmHostActivityChecker.getHostAgentStatus(host)).thenReturn(Status.Down); + + assertTrue(kvmHAProvider.fence(host)); + } + + @Test(expected = HAFenceException.class) + public void testFenceOobmExceptionFallbackDisabledThrows() throws HAFenceException { + when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true); + when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenThrow(new RuntimeException("BMC unreachable")); + doReturn(false).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host); + + kvmHAProvider.fence(host); + } + }