redis cluster del node: forget + shutdown

carlosfu · carlosfu · commit d281d118a57d · 2017-06-27T16:33:05.000+08:00
diff --git a/cachecloud-open-web/src/main/java/com/sohu/cache/constant/ClusterOperateResult.java b/cachecloud-open-web/src/main/java/com/sohu/cache/constant/ClusterOperateResult.java
@@ -0,0 +1,46 @@
+package com.sohu.cache.constant;
+
+
+/**
+ * Cluster Operate Result
+ * @author leifu
+ * @Date 2017年6月27日
+ * @Time 上午8:43:10
+ */
+public class ClusterOperateResult {
+
+    private int status;
+
+    private String message;
+
+    public ClusterOperateResult(int status, String message) {
+        this.status = status;
+        this.message = message;
+    }
+
+    public static ClusterOperateResult success() {
+        return new ClusterOperateResult(1, "");
+    }
+
+    public static ClusterOperateResult fail(String message) {
+        return new ClusterOperateResult(0, message);
+    }
+    
+    public boolean isSuccess() {
+        return status == 1;
+    }
+
+    public int getStatus() {
+        return status;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    @Override
+    public String toString() {
+        return "ClusterOperateResult [status=" + status + ", message=" + message + "]";
+    }
+
+}
diff --git a/cachecloud-open-web/src/main/java/com/sohu/cache/redis/RedisCenter.java b/cachecloud-open-web/src/main/java/com/sohu/cache/redis/RedisCenter.java
@@ -107,6 +107,14 @@ public Map<RedisConstant, Map<String, Object>> collectRedisInfo(long appId, long
      * @return 主返回true，从返回false；
      */
     public Boolean isMaster(long appId, String ip, int port);
+    
+    /**
+     * 根据ip和port判断redis实例当前是否有从节点
+     * @param ip   ip
+     * @param port port
+     * @return 主返回true，从返回false；
+     */
+    public Boolean hasSlaves(long appId, String ip, int port);
 
     /**
      * 获取从节点的主节点地址
diff --git a/cachecloud-open-web/src/main/java/com/sohu/cache/redis/RedisDeployCenter.java b/cachecloud-open-web/src/main/java/com/sohu/cache/redis/RedisDeployCenter.java
@@ -2,6 +2,7 @@
 
 import java.util.List;
 
+import com.sohu.cache.constant.ClusterOperateResult;
 import com.sohu.cache.entity.AppDesc;
 import com.sohu.cache.web.enums.RedisOperateEnum;
 
@@ -122,6 +123,22 @@ public interface RedisDeployCenter {
      */
     public boolean clusterFailover(long appId, int slaveInstanceId, String failoverParam) throws Exception;
 
+    /**
+     * 检查是否具备forget的条件
+     * @param appId
+     * @param forgetInstanceId
+     * @return
+     */
+    public ClusterOperateResult checkClusterForget(Long appId, int forgetInstanceId);
+
+    /**
+     * 删除节点
+     * @param appId
+     * @param delNodeInstanceId
+     * @return
+     */
+    public ClusterOperateResult delNode(Long appId, int delNodeInstanceId);
+
 
 
 }
diff --git a/cachecloud-open-web/src/main/java/com/sohu/cache/redis/impl/RedisCenterImpl.java b/cachecloud-open-web/src/main/java/com/sohu/cache/redis/impl/RedisCenterImpl.java
@@ -703,6 +703,27 @@ private Map<RedisConstant, Map<String, Object>> processRedisStats(String statRes
         }
         return redisStatMap;
     }
+    
+    /**
+     * 根据infoMap的结果判断实例的主从
+     *
+     * @param infoMap
+     * @return
+     */
+    private Boolean hasSlaves(Map<RedisConstant, Map<String, Object>> infoMap) {
+        Map<String, Object> replicationMap = infoMap.get(RedisConstant.Replication);
+        if (MapUtils.isEmpty(replicationMap)) {
+            return null;
+        }
+        for (Entry<String, Object> entry : replicationMap.entrySet()) {
+            String key = entry.getKey();
+            //判断一个即可
+            if (key != null && key.contains("slave0")) {
+                return true;
+            }
+        }
+        return false;
+    }
 
     /**
      * 根据infoMap的结果判断实例的主从
@@ -742,6 +763,26 @@ public Boolean isMaster(long appId, String ip, int port) {
             jedis.close();
         }
     }
+    
+    /**
+     * 根据ip和port判断redis实例当前是否有从节点
+     * @param ip   ip
+     * @param port port
+     * @return 主返回true，从返回false；
+     */
+    public Boolean hasSlaves(long appId, String ip, int port) {
+        Jedis jedis = getJedis(appId, ip, port, REDIS_DEFAULT_TIME, REDIS_DEFAULT_TIME);
+        try {
+            String info = jedis.info("all");
+            Map<RedisConstant, Map<String, Object>> infoMap = processRedisStats(info);
+            return hasSlaves(infoMap);
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            return null;
+        } finally {
+            jedis.close();
+        }
+    }   
 
     @Override
     public HostAndPort getMaster(String ip, int port, String password) {
diff --git a/cachecloud-open-web/src/main/java/com/sohu/cache/redis/impl/RedisDeployCenterImpl.java b/cachecloud-open-web/src/main/java/com/sohu/cache/redis/impl/RedisDeployCenterImpl.java
@@ -1,11 +1,13 @@
 package com.sohu.cache.redis.impl;
 
+import com.sohu.cache.constant.ClusterOperateResult;
 import com.sohu.cache.constant.InstanceStatusEnum;
 import com.sohu.cache.dao.AppDao;
 import com.sohu.cache.dao.InstanceDao;
 import com.sohu.cache.dao.MachineDao;
 import com.sohu.cache.entity.AppDesc;
 import com.sohu.cache.entity.InstanceInfo;
+import com.sohu.cache.entity.InstanceSlotModel;
 import com.sohu.cache.entity.MachineInfo;
 import com.sohu.cache.machine.MachineCenter;
 import com.sohu.cache.protocol.MachineProtocol;
@@ -15,6 +17,7 @@
 import com.sohu.cache.redis.RedisConfigTemplateService;
 import com.sohu.cache.redis.RedisDeployCenter;
 import com.sohu.cache.redis.enums.RedisConfigEnum;
+import com.sohu.cache.stats.instance.InstanceDeployCenter;
 import com.sohu.cache.util.ConstUtils;
 import com.sohu.cache.util.IdempotentConfirmer;
 import com.sohu.cache.util.TypeUtil;
@@ -51,6 +54,8 @@ public class RedisDeployCenterImpl implements RedisDeployCenter {
     
     private RedisConfigTemplateService redisConfigTemplateService;
     
+    private InstanceDeployCenter instanceDeployCenter;
+    
     @Override
     public boolean deployClusterInstance(long appId, List<RedisClusterNode> clusterNodes, int maxMemory) {
         if (!isExist(appId)) {
@@ -975,7 +980,7 @@ public boolean clusterFailover(final long appId, int slaveInstanceId, final Stri
         Assert.isTrue(appDesc != null);
         int type = appDesc.getType();
         if (!TypeUtil.isRedisCluster(type)) {
-            logger.error("{} is not redis type", appDesc);
+            logger.error("{} is not redis cluster type", appDesc);
             return false;
         }
         InstanceInfo instanceInfo = instanceDao.getInstanceInfoById(slaveInstanceId);
@@ -1007,6 +1012,115 @@ public boolean execute() {
         }
         return true;
     }
+    
+    @Override
+    public ClusterOperateResult delNode(final Long appId, int delNodeInstanceId) {
+        final InstanceInfo forgetInstanceInfo = instanceDao.getInstanceInfoById(delNodeInstanceId);
+        final String forgetNodeId = redisCenter.getNodeId(appId, forgetInstanceInfo.getIp(),
+                forgetInstanceInfo.getPort());
+        if (StringUtils.isBlank(forgetNodeId)) {
+            logger.warn("{} nodeId is null", forgetInstanceInfo.getHostPort());
+            return ClusterOperateResult.fail(String.format("%s nodeId is null", forgetInstanceInfo.getHostPort()));
+        }
+        List<InstanceInfo> instanceInfos = instanceDao.getInstListByAppId(appId);
+        for (InstanceInfo instanceInfo : instanceInfos) {
+            if (instanceInfo == null) {
+                continue;
+            }
+            if (instanceInfo.isOffline()) {
+                continue;
+            }
+            // 过滤当前节点
+            if (forgetInstanceInfo.getHostPort().equals(instanceInfo.getHostPort())) {
+                continue;
+            }
+            final String instanceHost = instanceInfo.getIp();
+            final int instancePort = instanceInfo.getPort();
+            boolean isForget = new IdempotentConfirmer() {
+                @Override
+                public boolean execute() {
+                    String response = null;
+                    Jedis jedis = null;
+                    try {
+                        jedis = redisCenter.getJedis(appId, instanceHost, instancePort);
+                        logger.warn("{}:{} is forgetting {}", instanceHost, instancePort, forgetNodeId);
+                        response = jedis.clusterForget(forgetNodeId);
+                        boolean success = response != null && response.equalsIgnoreCase("OK");
+                        logger.warn("{}:{} is forgetting {} result is {}", instanceHost, instancePort, forgetNodeId,
+                                success);
+                        return success;
+                    } catch (Exception e) {
+                        logger.error(e.getMessage());
+                    } finally {
+                        if (jedis != null) {
+                            jedis.close();
+                        }
+                    }
+                    return response != null && response.equalsIgnoreCase("OK");
+                }
+            }.run();
+            if (!isForget) {
+                logger.warn("{}:{} forget {} failed", instanceHost, instancePort, forgetNodeId);
+                return ClusterOperateResult.fail(String.format("%s:%s forget %s failed", instanceHost, instancePort, forgetNodeId));
+            }
+        }
+        
+        // shutdown
+        boolean isShutdown = instanceDeployCenter.shutdownExistInstance(appId, delNodeInstanceId);
+        if (!isShutdown) {
+            logger.warn("{} shutdown failed", forgetInstanceInfo.getHostPort());
+            return ClusterOperateResult.fail(String.format("%s shutdown failed", forgetInstanceInfo.getHostPort()));
+        }
+        
+        return ClusterOperateResult.success();
+    }
+    
+    
+    /**
+     * 1. 被forget的节点必须在线(这个条件有待验证) 
+     * 2. 被forget的节点不能有从节点 
+     * 3. 被forget的节点不能有slots
+     */
+    @Override
+    public ClusterOperateResult checkClusterForget(Long appId, int forgetInstanceId) {
+        // 0.各种验证
+        Assert.isTrue(appId > 0);
+        Assert.isTrue(forgetInstanceId > 0);
+        AppDesc appDesc = appDao.getAppDescById(appId);
+        Assert.isTrue(appDesc != null);
+        int type = appDesc.getType();
+        if (!TypeUtil.isRedisCluster(type)) {
+            logger.error("{} is not redis cluster type", appDesc);
+            return ClusterOperateResult.fail(String.format("instanceId: %s must be cluster type", forgetInstanceId));
+        }
+        InstanceInfo instanceInfo = instanceDao.getInstanceInfoById(forgetInstanceId);
+        Assert.isTrue(instanceInfo != null);
+        String forgetHost = instanceInfo.getIp();
+        int forgetPort = instanceInfo.getPort();
+        // 1.是否在线
+        boolean isRun = redisCenter.isRun(appId, forgetHost, forgetPort);
+        if (!isRun) {
+            logger.warn("{}:{} is not run", forgetHost, forgetPort);
+            return ClusterOperateResult.fail(String.format("被forget的节点(%s:%s)必须在线", forgetHost, forgetPort));
+        }
+        // 2.被forget的节点不能有从节点
+        Boolean hasSlaves = redisCenter.hasSlaves(appId, forgetHost, forgetPort);
+        if (hasSlaves == null || hasSlaves) {
+            logger.warn("{}:{} has slave", forgetHost, forgetPort);
+            return ClusterOperateResult.fail(String.format("被forget的节点(%s:%s)不能有从节点", forgetHost, forgetPort));
+        }
+
+        // 3.被forget的节点不能有slots
+        Map<String, InstanceSlotModel> clusterSlotsMap = redisCenter.getClusterSlotsMap(appId);
+        InstanceSlotModel instanceSlotModel = clusterSlotsMap.get(instanceInfo.getHostPort());
+        if (instanceSlotModel != null && instanceSlotModel.getSlotList() != null
+                && instanceSlotModel.getSlotList().size() > 0) {
+            logger.warn("{}:{} has slots", forgetHost, forgetPort);
+            return ClusterOperateResult.fail(String.format("被forget的节点(%s:%s)不能持有slot", forgetHost, forgetPort));
+        }
+
+        return ClusterOperateResult.success();
+    }
 
     /**
      * 拷贝redis配置
diff --git a/cachecloud-open-web/src/main/java/com/sohu/cache/web/controller/AppManageController.java b/cachecloud-open-web/src/main/java/com/sohu/cache/web/controller/AppManageController.java
@@ -2,6 +2,7 @@
 
 import com.sohu.cache.web.enums.RedisOperateEnum;
 import com.sohu.cache.constant.AppCheckEnum;
+import com.sohu.cache.constant.ClusterOperateResult;
 import com.sohu.cache.constant.DataFormatCheckResult;
 import com.sohu.cache.constant.ErrorMessageEnum;
 import com.sohu.cache.constant.HorizontalResult;
@@ -625,6 +626,46 @@ public ModelAndView appInfoAndAudit(HttpServletRequest request, HttpServletRespo
 		}
 		return new ModelAndView("manage/appOps/appInfoAndAudit");
 	}
+	
+	/**
+     * redisCluster节点删除: forget + shutdown
+     * 
+     * @param appId 应用id
+     * @param forgetInstanceId 需要被forget的节点
+     * @return
+     */
+    @RequestMapping("/clusterDelNode")
+    public ModelAndView clusterDelNode(HttpServletRequest request, HttpServletResponse response, Model model, Long appId,
+            int delNodeInstanceId) {
+        AppUser appUser = getUserInfo(request);
+        logger.warn("user {}, clusterForget: appId:{}, instanceId:{}", appUser.getName(), appId, delNodeInstanceId);
+        // 检测forget条件
+        ClusterOperateResult checkClusterForgetResult = null;
+        try {
+            checkClusterForgetResult = redisDeployCenter.checkClusterForget(appId, delNodeInstanceId);
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+        if (checkClusterForgetResult == null || !checkClusterForgetResult.isSuccess()) {
+            model.addAttribute("success", checkClusterForgetResult.getStatus());
+            model.addAttribute("message", checkClusterForgetResult.getMessage());
+            return new ModelAndView("");
+        }
+        
+        // 执行delnode:forget + shutdown
+        ClusterOperateResult delNodeResult = null;
+        try {
+            delNodeResult = redisDeployCenter.delNode(appId, delNodeInstanceId);
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+        model.addAttribute("success", delNodeResult.getStatus());
+        model.addAttribute("message", delNodeResult.getMessage());
+        logger.warn("user {}, clusterForget: appId:{}, instanceId:{}, result is {}", appUser.getName(), appId, delNodeInstanceId, delNodeResult.getStatus());
+        
+        return new ModelAndView("");
+        
+    }
 
 	/**
 	 * redisCluster从节点failover
diff --git a/cachecloud-open-web/src/main/webapp/WEB-INF/jsp/manage/appOps/appInstance.jsp b/cachecloud-open-web/src/main/webapp/WEB-INF/jsp/manage/appOps/appInstance.jsp
