Add button for manual leader failover

SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java

@@ -61,7 +61,8 @@ public enum AbortReason {
     ERROR_IN_LEADER_ONLY_POLLER,
     TEST_ABORT,
     MESOS_ERROR,
-    LOST_MESOS_CONNECTION
+    LOST_MESOS_CONNECTION,
+    MANUAL
   }
 
   public void abort(AbortReason abortReason, Optional<Throwable> throwable) {

SingularityService/src/main/java/com/hubspot/singularity/resources/DisastersResource.java

@@ -1,6 +1,10 @@
 package com.hubspot.singularity.resources;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.inject.Inject;
+import com.hubspot.singularity.Singularity;
+import com.hubspot.singularity.SingularityAbort;
+import com.hubspot.singularity.SingularityAbort.AbortReason;
 import com.hubspot.singularity.SingularityAction;
 import com.hubspot.singularity.SingularityDisabledAction;
 import com.hubspot.singularity.SingularityDisasterType;
@@ -10,6 +14,7 @@
 import com.hubspot.singularity.auth.SingularityAuthorizer;
 import com.hubspot.singularity.config.ApiPaths;
 import com.hubspot.singularity.data.DisasterManager;
+import com.ning.http.client.AsyncHttpClient;
 import io.dropwizard.auth.Auth;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
@@ -20,29 +25,46 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.DELETE;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import org.apache.curator.framework.recipes.leader.LeaderLatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Path(ApiPaths.DISASTERS_RESOURCE_PATH)
 @Produces(MediaType.APPLICATION_JSON)
 @Schema(title = "Manages active disasters and disabled actions")
 @Tags({ @Tag(name = "Disasters") })
-public class DisastersResource {
+public class DisastersResource extends AbstractLeaderAwareResource {
+  private static final Logger LOG = LoggerFactory.getLogger(DisastersResource.class);
+
   private final DisasterManager disasterManager;
   private final SingularityAuthorizer authorizationHelper;
+  private final SingularityAbort abort;
 
   @Inject
   public DisastersResource(
     DisasterManager disasterManager,
-    SingularityAuthorizer authorizationHelper
+    SingularityAuthorizer authorizationHelper,
+    LeaderLatch leaderLatch,
+    AsyncHttpClient httpClient,
+    @Singularity ObjectMapper objectMapper,
+    SingularityAbort abort
   ) {
+    super(httpClient, leaderLatch, objectMapper);
     this.disasterManager = disasterManager;
     this.authorizationHelper = authorizationHelper;
+    this.abort = abort;
   }
 
   @GET
@@ -162,4 +184,38 @@ public void enableAction(
     authorizationHelper.checkAdminAuthorization(user);
     disasterManager.enable(action);
   }
+
+  @POST
+  @Path("/failover")
+  @Operation(
+    summary = "Force the leading Singularity instance to restart and give up leadership"
+  )
+  public Response forceFailover(
+    @Parameter(hidden = true) @Auth SingularityUser user,
+    @Context HttpServletRequest requestContext
+  ) {
+    authorizationHelper.checkAdminAuthorization(user);
+    return maybeProxyToLeader(
+      requestContext,
+      Response.class,
+      null,
+      () -> this.runFailover(user)
+    );
+  }
+
+  private Response runFailover(SingularityUser user) {
+    CompletableFuture.runAsync(
+      () -> {
+        LOG.warn("Failover triggered by {}", user.getId());
+        abort.abort(
+          AbortReason.MANUAL,
+          Optional.of(
+            new RuntimeException(String.format("Forced failover by %s", user.getId()))
+          )
+        );
+      },
+      Executors.newSingleThreadExecutor()
+    );
+    return Response.ok().build();
+  }
 }

SingularityUI/app/actions/api/disasters.es6

@@ -81,3 +81,9 @@ export const NewPriorityFreeze = buildJsonApiAction(
     body: message == null ? {minimumPriorityLevel: minPriority, killTasks: killTasks} : {minimumPriorityLevel: minPriority, killTasks: killTasks, message: message}
   })
 );
+
+export const ForceFailover = buildJsonApiAction(
+  'FORCE_FAILOVER',
+  'POST',
+  {url: '/disasters/failover'}
+);

SingularityUI/app/components/disasters/ForceFailoverButton.jsx

@@ -0,0 +1,40 @@
+import React, { PropTypes } from 'react';
+import { Glyphicon } from 'react-bootstrap';
+import OverlayTrigger from 'react-bootstrap/lib/OverlayTrigger';
+import ToolTip from 'react-bootstrap/lib/Tooltip';
+import { getClickComponent } from '../common/modal/ModalWrapper';
+import ForceFailoverModal from './ForceFailoverModal';
+
+const forceFailoverTooltip = (
+  <ToolTip id="edit-freeze">
+    Force the leading Singularity instance to restart
+  </ToolTip>
+);
+
+const ForceFailoverButton = ({children, user, freeze}) => {
+  const clickComponentData = {props: {children}};
+  return (
+    <span>
+      {getClickComponent(clickComponentData)}
+      <ForceFailoverModal
+        ref={(modal) => {clickComponentData.refs = {modal};}}
+      />
+    </span>
+  );
+};
+
+ForceFailoverButton.propTypes = {
+  children: PropTypes.node,
+};
+
+ForceFailoverButton.defaultProps = {
+  children: (
+    <OverlayTrigger placement="top" id="view-bounce-overlay" overlay={forceFailoverTooltip}>
+      <a>
+        <Glyphicon glyph="plus" />
+      </a>
+    </OverlayTrigger>
+  )
+};
+
+export default ForceFailoverButton;

SingularityUI/app/components/disasters/ForceFailoverModal.jsx

@@ -0,0 +1,41 @@
+import React, { Component, PropTypes } from 'react';
+import { connect } from 'react-redux';
+import { ForceFailover } from '../../actions/api/disasters';
+import FormModal from '../common/modal/FormModal';
+
+class ForceFailoverModal extends Component {
+  static propTypes = {
+    user: PropTypes.string,
+    forceFailover: PropTypes.func.isRequired,
+  };
+
+  show() {
+    this.refs.forceFailoverModal.show();
+  }
+
+  render() {
+    return (
+      <FormModal
+        ref="forceFailoverModal"
+        name="Force Failover"
+        action="Failover"
+        buttonStyle="danger"
+        onConfirm={(data) => this.props.forceFailover()}
+        formElements={[]}
+      >
+        <p>Are you sure you want to force the leading singularity instance to restart?</p>
+      </FormModal>
+    );
+  }
+}
+
+const mapDispatchToProps = (dispatch) => ({
+  forceFailover: () => dispatch(ForceFailover.trigger()),
+});
+
+export default connect(
+  null,
+  mapDispatchToProps,
+  null,
+  { withRef: true }
+)(ForceFailoverModal);

SingularityUI/app/components/disasters/ManageDisasters.jsx

@@ -8,6 +8,7 @@ import AutomatedActionsButton from './AutomatedActionsButton';
 import DeletePriorityFreezeButton from './DeletePriorityFreezeButton';
 import NewPriorityFreezeButton from './NewPriorityFreezeButton';
 import EditPriorityFreezeButton from './EditPriorityFreezeButton';
+import ForceFailoverButton from './ForceFailoverButton';
 import Utils from '../../utils';
 
 const DISASTER_TYPES = ['EXCESSIVE_TASK_LAG', 'LOST_SLAVES', 'LOST_TASKS', 'USER_INITIATED']
@@ -85,11 +86,22 @@ function ManageDisasters (props) {
   return (
     <Section title="Manage">
       <div className="row">
-        <div className="col-md-6">
+        <div className="col-md-2">
+          <h3>Leader Failover</h3>
+          <ForceFailoverButton>
+            <button
+              className="btn btn-danger"
+              alt="Force Failover"
+              title="forceFailover">
+              Force Failover
+            </button>
+          </ForceFailoverButton>
+        </div>
+        <div className="col-md-5">
           <h3>Priority Freeze</h3>
           {priority}
         </div>
-        <div className="col-md-6">
+        <div className="col-md-5">
           <h3>Disasters</h3>
           <div className="row">
             <AutomatedActionsButton