KVM: add storage-heartbeat fencing fallback when OOBM fence fails (host stuck in Fencing on total power loss)

plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java

@@ -52,4 +52,11 @@ public interface KVMHAConfig {
 
     ConfigKey<Long> KvmHAFenceTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.fence.timeout", "60",
             "The maximum length of time, in seconds, expected for a fence operation to complete.", true, ConfigKey.Scope.Cluster);
+
+    ConfigKey<Boolean> KvmHAFenceOnStorageHeartbeat = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat", "false",
+            "Whether to consider a host fenced when the out-of-band management fence operation fails (e.g. a complete power failure that takes down the BMC together with the host) "
+                    + "but the neighbouring hosts in the cluster positively report the host's storage heartbeat as expired. The host is never considered fenced when any check still "
+                    + "sees storage activity from it or when the heartbeat check itself fails. The heartbeat semantics follow the existing checks, including "
+                    + "'kvm.ha.fence.host.if.heartbeat.fails.on.storage'. When disabled, fencing relies solely on out-of-band management.",
+            true, ConfigKey.Scope.Cluster);
 }

plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java

@@ -21,6 +21,7 @@
 
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.host.Host;
+import com.cloud.host.Status;
 import com.cloud.hypervisor.Hypervisor;
 
 import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
@@ -89,17 +90,53 @@ public boolean fence(Host r) throws HAFenceException {
         try {
             if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){
                 final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.OFF, null);
-                return resp.getSuccess();
+                if (resp.getSuccess()) {
+                    return true;
+                }
+                logger.warn("OOBM fence operation failed for the host {}", r);
+                return fenceHostViaStorageHeartbeat(r);
             } else {
                 logger.warn("OOBM fence operation failed for this host {}", r);
-                return false;
+                return fenceHostViaStorageHeartbeat(r);
             }
         } catch (Exception e){
             logger.warn("OOBM service is not configured or enabled for this host {} error is {}", r, e.getMessage());
+            if (fenceHostViaStorageHeartbeat(r)) {
+                return true;
+            }
             throw new HAFenceException(String.format("OBM service is not configured or enabled for this host %s", r.getName()), e);
         }
     }
 
+    /**
+     * Fallback fencing for when out-of-band management cannot power off the host, e.g. a
+     * complete power failure that takes down the BMC (IPMI/iDRAC/iLO) together with the host.
+     * The host is considered fenced only when the neighbouring hosts in the cluster positively
+     * report its storage heartbeat as expired; it is never considered fenced when any check
+     * still sees it alive or when the heartbeat check itself fails.
+     */
+    protected boolean fenceHostViaStorageHeartbeat(final Host host) {
+        if (!isStorageHeartbeatFencingEnabled(host)) {
+            return false;
+        }
+        try {
+            final Status hostStatus = hostActivityChecker.getHostAgentStatus(host);
+            if (hostStatus == Status.Down) {
+                logger.warn("Neighbouring hosts report the storage heartbeat of the host {} as expired, considering it fenced as [{}] is enabled",
+                        host, KVMHAConfig.KvmHAFenceOnStorageHeartbeat.key());
+                return true;
+            }
+            logger.warn("Not considering the host {} fenced via storage heartbeat as its reported status is [{}]", host, hostStatus);
+        } catch (Exception e) {
+            logger.warn("Storage heartbeat fencing check failed for the host {}", host, e);
+        }
+        return false;
+    }
+
+    protected boolean isStorageHeartbeatFencingEnabled(final Host host) {
+        return KVMHAConfig.KvmHAFenceOnStorageHeartbeat.valueIn(host.getClusterId());
+    }
+
     @Override
     public HAResource.ResourceSubType resourceSubType() {
         return HAResource.ResourceSubType.KVM;
@@ -152,6 +189,7 @@ public ConfigKey<?>[] getConfigKeys() {
             KVMHAConfig.KvmHADegradedMaxPeriod,
             KVMHAConfig.KvmHARecoverWaitPeriod,
             KVMHAConfig.KvmHARecoverAttemptThreshold,
+            KVMHAConfig.KvmHAFenceOnStorageHeartbeat,
         };
     }
 }

plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/kvm/ha/KVMHostHATest.java

@@ -20,10 +20,18 @@
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
 import org.apache.cloudstack.ha.provider.HACheckerException;
+import org.apache.cloudstack.ha.provider.HAFenceException;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement.PowerOperation;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
 import org.joda.time.DateTime;
 import org.junit.Before;
 import org.junit.Test;
@@ -33,6 +41,7 @@
 
 import com.cloud.exception.StorageUnavailableException;
 import com.cloud.host.Host;
+import com.cloud.host.Status;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 
 @RunWith(MockitoJUnitRunner.class)
@@ -42,12 +51,21 @@ public class KVMHostHATest {
     private Host host;
     @Mock
     private KVMHostActivityChecker kvmHostActivityChecker;
+    @Mock
+    private OutOfBandManagementService outOfBandManagementService;
     private KVMHAProvider kvmHAProvider;
 
     @Before
     public void setup() {
-        kvmHAProvider = new KVMHAProvider();
+        kvmHAProvider = spy(new KVMHAProvider());
         kvmHAProvider.hostActivityChecker = kvmHostActivityChecker;
+        kvmHAProvider.outOfBandManagementService = outOfBandManagementService;
+    }
+
+    private OutOfBandManagementResponse oobmResponse(final boolean success) {
+        final OutOfBandManagementResponse response = new OutOfBandManagementResponse();
+        response.setSuccess(success);
+        return response;
     }
 
     @Test
@@ -80,4 +98,64 @@ public void testHostActivityForDownHost() throws HACheckerException, StorageUnav
         assertFalse(kvmHAProvider.hasActivity(host, dt));
     }
 
+    @Test
+    public void testFenceSucceedsViaOobm() throws HAFenceException {
+        when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true);
+        when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenReturn(oobmResponse(true));
+
+        assertTrue(kvmHAProvider.fence(host));
+
+        verify(kvmHostActivityChecker, never()).getHostAgentStatus(host);
+    }
+
+    @Test
+    public void testFenceOobmFailedFallbackDisabled() throws HAFenceException {
+        when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true);
+        when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenReturn(oobmResponse(false));
+        doReturn(false).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host);
+
+        assertFalse(kvmHAProvider.fence(host));
+
+        verify(kvmHostActivityChecker, never()).getHostAgentStatus(host);
+    }
+
+    @Test
+    public void testFenceOobmFailedStorageHeartbeatReportsHostDown() throws HAFenceException {
+        when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true);
+        when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenReturn(oobmResponse(false));
+        doReturn(true).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host);
+        when(kvmHostActivityChecker.getHostAgentStatus(host)).thenReturn(Status.Down);
+
+        assertTrue(kvmHAProvider.fence(host));
+    }
+
+    @Test
+    public void testFenceOobmFailedStorageHeartbeatSeesHostAlive() throws HAFenceException {
+        when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true);
+        when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenReturn(oobmResponse(false));
+        doReturn(true).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host);
+        when(kvmHostActivityChecker.getHostAgentStatus(host)).thenReturn(Status.Disconnected);
+
+        assertFalse(kvmHAProvider.fence(host));
+    }
+
+    @Test
+    public void testFenceOobmExceptionStorageHeartbeatReportsHostDown() throws HAFenceException {
+        when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true);
+        when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenThrow(new RuntimeException("BMC unreachable"));
+        doReturn(true).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host);
+        when(kvmHostActivityChecker.getHostAgentStatus(host)).thenReturn(Status.Down);
+
+        assertTrue(kvmHAProvider.fence(host));
+    }
+
+    @Test(expected = HAFenceException.class)
+    public void testFenceOobmExceptionFallbackDisabledThrows() throws HAFenceException {
+        when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true);
+        when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenThrow(new RuntimeException("BMC unreachable"));
+        doReturn(false).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host);
+
+        kvmHAProvider.fence(host);
+    }
+
 }