enable canHandle and implement detached volume migration for vmware

Author: DaanHoogland

api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java

@@ -87,10 +87,11 @@ public interface HypervisorGuru extends Adapter {
     Map<String, String> getClusterSettings(long vmId);
 
     /**
-     * will migrate a vm to a pool. For now this will ony work for stopped VMs on Vmware.
+     * will generate commands to migrate a vm to a pool. For now this will ony work for stopped VMs on Vmware.
+     *
      * @param vm the stopped vm to migrate
      * @param destination the primary storage pool to migrate to
-     * @return all data pertaining to the 'new' vm
+     * @return a list of commands to perform for a succesful migration
      */
     List<Command> migrate(VirtualMachine vm, StoragePool destination);
 }

core/src/main/java/com/cloud/agent/api/MigrateToPoolCommand.java

@@ -1,32 +1,61 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
 package com.cloud.agent.api;
 
+import com.cloud.agent.api.to.VolumeTO;
+
+import java.util.Collection;
+
 /**
  * used to tell the agent to migrate a vm to a different primare storage pool.
  * It is for now only mplemented on Vmware and is supposed to work irrespective of whether the VM is started or not.
  *
  */
 public class MigrateToPoolCommand extends Command {
-    String vmName;
-    String destPool;
-    boolean executeInSequence = false;
+    private Collection<VolumeTO> volumes;
+    private String vmName;
+    private String destinationPool;
+    private boolean executeInSequence = false;
 
     protected MigrateToPoolCommand() {
     }
 
     /**
      *
      * @param vmName the name of the VM to migrate
-     * @param destPool the primare storage pool to migrate the VM to
+     * @param volumes used to suplly feedback on vmware generated names
+     * @param destinationPool the primare storage pool to migrate the VM to
      * @param executeInSequence
      */
-    public MigrateToPoolCommand(String vmName, String destPool, boolean executeInSequence) {
+    public MigrateToPoolCommand(String vmName, Collection<VolumeTO> volumes, String destinationPool, boolean executeInSequence) {
         this.vmName = vmName;
-        this.destPool = destPool;
+        this.volumes = volumes;
+        this.destinationPool = destinationPool;
         this.executeInSequence = executeInSequence;
     }
 
+    public Collection<VolumeTO> getVolumes() {
+        return volumes;
+    }
+
     public String getDestinationPool() {
-        return destPool;
+        return destinationPool;
     }
 
     public String getVmName() {

core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java

@@ -31,6 +31,7 @@ public class MigrateVolumeCommand extends Command {
     long volumeId;
     String volumePath;
     StorageFilerTO pool;
+    StorageFilerTO sourcePool;
     String attachedVmName;
     Volume.Type volumeType;
 
@@ -47,14 +48,17 @@ public MigrateVolumeCommand(long volumeId, String volumePath, StoragePool pool,
     }
 
     public MigrateVolumeCommand(long volumeId, String volumePath, StoragePool pool, String attachedVmName, Volume.Type volumeType, int timeout) {
-        this.volumeId = volumeId;
-        this.volumePath = volumePath;
-        this.pool = new StorageFilerTO(pool);
+        this(volumeId,volumePath,pool,timeout);
         this.attachedVmName = attachedVmName;
         this.volumeType = volumeType;
         this.setWait(timeout);
     }
 
+    public MigrateVolumeCommand(long volumeId, String volumePath, StoragePool sourcePool, StoragePool targetPool) {
+        this(volumeId,volumePath,targetPool, null, Volume.Type.UNKNOWN, -1);
+        this.sourcePool = new StorageFilerTO(sourcePool);
+    }
+
     public MigrateVolumeCommand(DataTO srcData, DataTO destData, Map<String, String> srcDetails, Map<String, String> destDetails, int timeout) {
         this.srcData = srcData;
         this.destData = destData;
@@ -81,6 +85,14 @@ public StorageFilerTO getPool() {
         return pool;
     }
 
+    public StorageFilerTO getSourcePool() {
+        return sourcePool;
+    }
+
+    public StorageFilerTO getTargetPool() {
+        return pool;
+    }
+
     public String getAttachedVmName() {
         return attachedVmName;
     }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataMotionStrategy.java

@@ -25,11 +25,28 @@
 import com.cloud.agent.api.to.VirtualMachineTO;
 import com.cloud.host.Host;
 
+/**
+ * interface to query how to move data around and to commision the moving
+ */
 public interface DataMotionStrategy {
+    /**
+     * reports whether this instance can do a move from source to destination
+     * @param srcData object to move
+     * @param destData location to move it to
+     * @return the expertise level with which this instance knows how to handle the move
+     */
     StrategyPriority canHandle(DataObject srcData, DataObject destData);
 
     StrategyPriority canHandle(Map<VolumeInfo, DataStore> volumeMap, Host srcHost, Host destHost);
 
+    /**
+     * copy the source volume to its destination (on a host if not null)
+     *
+     * @param srcData volume to move
+     * @param destData volume description as intended after the move
+     * @param destHost if not null destData should be reachable from here
+     * @param callback where to report completion or failure to
+     */
     void copyAsync(DataObject srcData, DataObject destData, Host destHost, AsyncCompletionCallback<CopyCommandResult> callback);
 
     void copyAsync(Map<VolumeInfo, DataStore> volumeMap, VirtualMachineTO vmTo, Host srcHost, Host destHost, AsyncCompletionCallback<CopyCommandResult> callback);

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java

@@ -1408,6 +1408,13 @@ protected Void copyVolumeFromPrimaryToImageCallback(AsyncCallbackDispatcher<Volu
 
     @Override
     public AsyncCallFuture<VolumeApiResult> copyVolume(VolumeInfo srcVolume, DataStore destStore) {
+        if (s_logger.isDebugEnabled()) {
+            DataStoreRole srcRole = srcVolume.getDataStore().getRole();
+            DataStoreRole destRole = destStore.getRole();
+
+            String msg = String.format("copying %s(id=%d, role=%s) to %s (id=%d, role=%s)", srcVolume.getName(), srcVolume.getId(), srcRole, destStore.getName(), destStore.getId(), destRole);
+            s_logger.debug(msg);
+        }
 
         if (srcVolume.getState() == Volume.State.Uploaded) {
             return copyVolumeFromImageToPrimary(srcVolume, destStore);
@@ -1417,6 +1424,8 @@ public AsyncCallFuture<VolumeApiResult> copyVolume(VolumeInfo srcVolume, DataSto
             return copyVolumeFromPrimaryToImage(srcVolume, destStore);
         }
 
+        // OfflineVmwareMigration: aren't we missing secondary to secondary in this logic?
+
         AsyncCallFuture<VolumeApiResult> future = new AsyncCallFuture<VolumeApiResult>();
         VolumeApiResult res = new VolumeApiResult(srcVolume);
         try {
@@ -1438,7 +1447,10 @@ public AsyncCallFuture<VolumeApiResult> copyVolume(VolumeInfo srcVolume, DataSto
             caller.setCallback(caller.getTarget().copyVolumeCallBack(null, null)).setContext(context);
             motionSrv.copyAsync(srcVolume, destVolume, caller);
         } catch (Exception e) {
-            s_logger.debug("Failed to copy volume" + e);
+            s_logger.error("Failed to copy volume:" + e);
+            if(s_logger.isDebugEnabled()) {
+                s_logger.debug("Failed to copy volume.", e);
+            }
             res.setResult(e.toString());
             future.complete(res);
         }
@@ -1461,27 +1473,25 @@ protected Void copyVolumeCallBack(AsyncCallbackDispatcher<VolumeServiceImpl, Cop
                 AsyncCallFuture<VolumeApiResult> destroyFuture = expungeVolumeAsync(destVolume);
                 destroyFuture.get();
                 future.complete(res);
-                return null;
-            }
-            srcVolume.processEvent(Event.OperationSuccessed);
-            destVolume.processEvent(Event.MigrationCopySucceeded, result.getAnswer());
-            volDao.updateUuid(srcVolume.getId(), destVolume.getId());
-            _volumeStoreDao.updateVolumeId(srcVolume.getId(), destVolume.getId());
-            try {
-                destroyVolume(srcVolume.getId());
-                srcVolume = volFactory.getVolume(srcVolume.getId());
-                AsyncCallFuture<VolumeApiResult> destroyFuture = expungeVolumeAsync(srcVolume);
-                // If volume destroy fails, this could be because of vdi is still in use state, so wait and retry.
-                if (destroyFuture.get().isFailed()) {
-                    Thread.sleep(5 * 1000);
-                    destroyFuture = expungeVolumeAsync(srcVolume);
-                    destroyFuture.get();
+            } else {
+                srcVolume.processEvent(Event.OperationSuccessed);
+                destVolume.processEvent(Event.MigrationCopySucceeded, result.getAnswer());
+                volDao.updateUuid(srcVolume.getId(), destVolume.getId());
+                try {
+                    destroyVolume(srcVolume.getId());
+                    srcVolume = volFactory.getVolume(srcVolume.getId());
+                    AsyncCallFuture<VolumeApiResult> destroyFuture = expungeVolumeAsync(srcVolume);
+                    // If volume destroy fails, this could be because of vdi is still in use state, so wait and retry.
+                    if (destroyFuture.get().isFailed()) {
+                        Thread.sleep(5 * 1000);
+                        destroyFuture = expungeVolumeAsync(srcVolume);
+                        destroyFuture.get();
+                    }
+                    future.complete(res);
+                } catch (Exception e) {
+                    s_logger.debug("failed to clean up volume on storage", e);
                 }
-                future.complete(res);
-            } catch (Exception e) {
-                s_logger.debug("failed to clean up volume on storage", e);
             }
-            return null;
         } catch (Exception e) {
             s_logger.debug("Failed to process copy volume callback", e);
             res.setResult(e.toString());

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java

@@ -28,6 +28,7 @@
 
 import com.cloud.agent.api.MigrateToPoolCommand;
 import com.cloud.agent.api.UnregisterVMCommand;
+import com.cloud.agent.api.to.VolumeTO;
 import com.cloud.dc.ClusterDetailsDao;
 import com.cloud.storage.StoragePool;
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
@@ -646,12 +647,19 @@ public Map<String, String> getClusterSettings(long vmId) {
         details.put(VmwareReserveMemory.key(), VmwareReserveMemory.valueIn(clusterId).toString());
         return details;
     }
+
     @Override
     public List<Command> migrate(VirtualMachine vm, StoragePool destination) {
         List<Command> commands = new ArrayList<Command>();
 
         // OfflineVmwareMigration: specialised migration command
-        MigrateToPoolCommand migrateToPoolCommand = new MigrateToPoolCommand(vm.getInstanceName(), destination.getUuid(), true);
+        List<VolumeVO> volumes = _volumeDao.findByInstance(vm.getId());
+        List<VolumeTO> vols = new ArrayList<>();
+        for (Volume volume : volumes) {
+            VolumeTO vol = new VolumeTO(volume,destination);
+            vols.add(vol);
+        }
+        MigrateToPoolCommand migrateToPoolCommand = new MigrateToPoolCommand(vm.getInstanceName(), vols, destination.getUuid(), true);
         commands.add(migrateToPoolCommand);
 
         // OfflineVmwareMigration: cleanup if needed