CLOUDSTACK-8733: Host stuck in rebalancing state during agent LB

This is happening as ClusterServiceServletAdapter is started after ClusteredAgentManagerImpl.
Fix is to start ClusterServiceServletAdapter before ClusteredAgentManagerImpl.

(cherry-picked from c989921)

Author: koushik-das

framework/cluster/src/com/cloud/cluster/ClusterServiceServletAdapter.java

@@ -24,12 +24,12 @@
 import javax.naming.ConfigurationException;
 
 import org.apache.log4j.Logger;
-
 import org.apache.cloudstack.framework.config.ConfigDepot;
 
 import com.cloud.cluster.dao.ManagementServerHostDao;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.component.AdapterBase;
+import com.cloud.utils.component.ComponentLifecycle;
 import com.cloud.utils.db.DbProperties;
 
 public class ClusterServiceServletAdapter extends AdapterBase implements ClusterServiceAdapter {
@@ -50,6 +50,10 @@ public class ClusterServiceServletAdapter extends AdapterBase implements Cluster
 
     private int _clusterServicePort = DEFAULT_SERVICE_PORT;
 
+    public ClusterServiceServletAdapter() {
+        setRunLevel(ComponentLifecycle.RUN_LEVEL_FRAMEWORK);
+    }
+
     @Override
     public ClusterService getPeerService(String strPeer) throws RemoteException {
         try {

framework/cluster/test/com/cloud/cluster/ClusterServiceServletAdapterTest.java

@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.cluster;
+
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.cloud.utils.component.ComponentLifecycle;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ClusterServiceServletAdapterTest {
+
+    ClusterServiceServletAdapter clusterServiceServletAdapter;
+    ClusterManagerImpl clusterManagerImpl;
+
+    @Before
+    public void setup() throws IllegalArgumentException,
+            IllegalAccessException, NoSuchFieldException, SecurityException {
+        clusterServiceServletAdapter = new ClusterServiceServletAdapter();
+        clusterManagerImpl = new ClusterManagerImpl();
+    }
+
+    @Test
+    public void testRunLevel() {
+        int runLevel = clusterServiceServletAdapter.getRunLevel();
+        assertTrue(runLevel == ComponentLifecycle.RUN_LEVEL_FRAMEWORK);
+        assertTrue(runLevel == clusterManagerImpl.getRunLevel());
+    }
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java

@@ -207,6 +207,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
     @Override
     @DB
     public void returnItem(final long queueItemId) {
+        s_logger.info("Returning queue item " + queueItemId + " back to queue for second try in case of DB deadlock");
         try {
             Transaction.execute(new TransactionCallbackNoReturn() {
                 @Override