Skip to content

Commit e5fbb61

Browse files
nvazqueznvazquez
authored
FR04 - DPDK live migrations (#3)
* Fix unit test * Send dpdk capability when available and enhance findHostsForMigration API * Send extra DPDK configurations on live migrations * Fix findHostsForMigration and add unit tests * Check VM details and service offerings when determining if VM is DPDK enabled * Fix for DPDK enabled VM * Allow non-DPDK user VMs on DPDK enabled hosts * Replace interfaces using DPDK ports before migration * Move nics DPDK enablement to implement TO * Remove DPDK created ports if VM migration fails or prepare migration fails
1 parent b5d71df commit e5fbb61

File tree

23 files changed

+735
-93
lines changed

23 files changed

+735
-93
lines changed

‎api/src/com/cloud/agent/api/to/DPDKTO.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.agent.api.to;
18+
19+
public class DPDKTO {
20+
21+
private String path;
22+
private String port;
23+
private String mode;
24+
25+
public DPDKTO() {
26+
}
27+
28+
public DPDKTO(String path, String port, String mode) {
29+
this.path = path;
30+
this.port = port;
31+
this.mode = mode;
32+
}
33+
34+
public String getPath() {
35+
return path;
36+
}
37+
38+
public String getPort() {
39+
return port;
40+
}
41+
42+
public String getMode() {
43+
return mode;
44+
}
45+
}

‎api/src/com/cloud/agent/api/to/NicTO.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public class NicTO extends NetworkTO {
3030
String nicUuid;
3131
List<String> nicSecIps;
3232
Map<NetworkOffering.Detail, String> details;
33-
boolean dpdkDisabled;
33+
boolean dpdkEnabled;
3434

3535
public NicTO() {
3636
super();
@@ -111,11 +111,11 @@ public void setDetails(final Map<NetworkOffering.Detail, String> details) {
111111
this.details = details;
112112
}
113113

114-
public boolean isDpdkDisabled() {
115-
return dpdkDisabled;
114+
public boolean isDpdkEnabled() {
115+
return dpdkEnabled;
116116
}
117117

118-
public void setDpdkDisabled(boolean dpdkDisabled) {
119-
this.dpdkDisabled = dpdkDisabled;
118+
public void setDpdkEnabled(boolean dpdkEnabled) {
119+
this.dpdkEnabled = dpdkEnabled;
120120
}
121121
}

‎core/src/com/cloud/agent/api/MigrateCommand.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.util.HashMap;
2323
import java.util.Map;
2424

25+
import com.cloud.agent.api.to.DPDKTO;
2526
import com.cloud.agent.api.to.VirtualMachineTO;
2627

2728
public class MigrateCommand extends Command {
@@ -33,6 +34,15 @@ public class MigrateCommand extends Command {
3334
private boolean isWindows;
3435
private VirtualMachineTO vmTO;
3536
private boolean executeInSequence = false;
37+
private Map<String, DPDKTO> dpdkInterfaceMapping = new HashMap<>();
38+
39+
public Map<String, DPDKTO> getDpdkInterfaceMapping() {
40+
return dpdkInterfaceMapping;
41+
}
42+
43+
public void setDpdkInterfaceMapping(Map<String, DPDKTO> dpdkInterfaceMapping) {
44+
this.dpdkInterfaceMapping = dpdkInterfaceMapping;
45+
}
3646

3747
protected MigrateCommand() {
3848
}

‎core/src/com/cloud/agent/api/PrepareForMigrationAnswer.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,15 @@
1919

2020
package com.cloud.agent.api;
2121

22+
import com.cloud.agent.api.to.DPDKTO;
23+
24+
import java.util.HashMap;
25+
import java.util.Map;
26+
2227
public class PrepareForMigrationAnswer extends Answer {
28+
29+
private Map<String, DPDKTO> dpdkInterfaceMapping = new HashMap<>();
30+
2331
protected PrepareForMigrationAnswer() {
2432
}
2533

@@ -34,4 +42,12 @@ public PrepareForMigrationAnswer(PrepareForMigrationCommand cmd, Exception ex) {
3442
public PrepareForMigrationAnswer(PrepareForMigrationCommand cmd) {
3543
super(cmd, true, null);
3644
}
45+
46+
public void setDpdkInterfaceMapping(Map<String, DPDKTO> mapping) {
47+
this.dpdkInterfaceMapping = mapping;
48+
}
49+
50+
public Map<String, DPDKTO> getDpdkInterfaceMapping() {
51+
return this.dpdkInterfaceMapping;
52+
}
3753
}

‎core/src/com/cloud/agent/api/StopCommand.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package com.cloud.agent.api;
2121

22+
import com.cloud.agent.api.to.DPDKTO;
2223
import com.cloud.agent.api.to.GPUDeviceTO;
2324
import com.cloud.vm.VirtualMachine;
2425

@@ -34,6 +35,16 @@ public class StopCommand extends RebootCommand {
3435
boolean checkBeforeCleanup = false;
3536
String controlIp = null;
3637
boolean forceStop = false;
38+
private Map<String, DPDKTO> dpdkInterfaceMapping;
39+
40+
public Map<String, DPDKTO> getDpdkInterfaceMapping() {
41+
return dpdkInterfaceMapping;
42+
}
43+
44+
public void setDpdkInterfaceMapping(Map<String, DPDKTO> dpdkInterfaceMapping) {
45+
this.dpdkInterfaceMapping = dpdkInterfaceMapping;
46+
}
47+
3748
/**
3849
* On KVM when using iSCSI-based managed storage, if the user shuts a VM down from the guest OS (as opposed to doing so from CloudStack),
3950
* we need to pass to the KVM agent a list of applicable iSCSI volumes that need to be disconnected.

‎engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,10 @@
3939
import javax.inject.Inject;
4040
import javax.naming.ConfigurationException;
4141

42-
import org.apache.cloudstack.api.ApiConstants;
42+
import com.cloud.agent.api.PrepareForMigrationAnswer;
43+
import com.cloud.agent.api.to.DPDKTO;
4344
import org.apache.commons.collections.CollectionUtils;
45+
import org.apache.commons.collections.MapUtils;
4446
import org.apache.log4j.Logger;
4547

4648
import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
@@ -1132,8 +1134,6 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
11321134

11331135
vmGuru.finalizeDeployment(cmds, vmProfile, dest, ctx);
11341136

1135-
addExtraConfig(vmTO);
1136-
11371137
work = _workDao.findById(work.getId());
11381138
if (work == null || work.getStep() != Step.Prepare) {
11391139
throw new ConcurrentOperationException("Work steps have been changed: " + work);
@@ -1298,15 +1298,6 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
12981298
}
12991299
}
13001300

1301-
private void addExtraConfig(VirtualMachineTO vmTO) {
1302-
Map<String, String> details = vmTO.getDetails();
1303-
for (String key : details.keySet()) {
1304-
if (key.startsWith(ApiConstants.EXTRA_CONFIG)) {
1305-
vmTO.addExtraConfig(key, details.get(key));
1306-
}
1307-
}
1308-
}
1309-
13101301
// for managed storage on KVM, need to make sure the path field of the volume in question is populated with the IQN
13111302
private void handlePath(final DiskTO[] disks, final HypervisorType hypervisorType) {
13121303
if (hypervisorType != HypervisorType.KVM) {
@@ -2231,6 +2222,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
22312222
}
22322223

22332224
boolean migrated = false;
2225+
Map<String, DPDKTO> dpdkInterfaceMapping = null;
22342226
try {
22352227
final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
22362228
final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType()));
@@ -2242,6 +2234,11 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
22422234

22432235
mc.setHostGuid(dest.getHost().getGuid());
22442236

2237+
dpdkInterfaceMapping = ((PrepareForMigrationAnswer) pfma).getDpdkInterfaceMapping();
2238+
if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
2239+
mc.setDpdkInterfaceMapping(dpdkInterfaceMapping);
2240+
}
2241+
22452242
try {
22462243
final Answer ma = _agentMgr.send(vm.getLastHostId(), mc);
22472244
if (ma == null || !ma.getResult()) {
@@ -2268,7 +2265,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
22682265
if (!checkVmOnHost(vm, dstHostId)) {
22692266
s_logger.error("Unable to complete migration for " + vm);
22702267
try {
2271-
_agentMgr.send(srcHostId, new Commands(cleanup(vm)), null);
2268+
_agentMgr.send(srcHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null);
22722269
} catch (final AgentUnavailableException e) {
22732270
s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId);
22742271
}
@@ -2289,7 +2286,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
22892286
"Unable to migrate vm " + vm.getInstanceName() + " from host " + fromHost.getName() + " in zone " + dest.getDataCenter().getName() + " and pod " +
22902287
dest.getPod().getName(), "Migrate Command failed. Please check logs.");
22912288
try {
2292-
_agentMgr.send(dstHostId, new Commands(cleanup(vm)), null);
2289+
_agentMgr.send(dstHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null);
22932290
} catch (final AgentUnavailableException ae) {
22942291
s_logger.info("Looks like the destination Host is unavailable for cleanup");
22952292
}
@@ -2896,9 +2893,12 @@ private void orchestrateReboot(final String vmUuid, final Map<VirtualMachineProf
28962893
}
28972894
}
28982895

2899-
public Command cleanup(final VirtualMachine vm) {
2896+
public Command cleanup(final VirtualMachine vm, Map<String, DPDKTO> dpdkInterfaceMapping) {
29002897
StopCommand cmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
29012898
cmd.setControlIp(getControlNicIpForVM(vm));
2899+
if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
2900+
cmd.setDpdkInterfaceMapping(dpdkInterfaceMapping);
2901+
}
29022902
return cmd;
29032903
}
29042904

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2909,6 +2909,10 @@ protected List<Object> getHostInfo() {
29092909
cap = cap + ",snapshot";
29102910
}
29112911

2912+
if (dpdkSupport) {
2913+
cap += ",dpdk";
2914+
}
2915+
29122916
info.add((int)cpus);
29132917
info.add(speed);
29142918
// Report system's RAM as actual RAM minus host OS reserved RAM

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,9 +184,13 @@ public boolean parseDomainXML(String domXML) {
184184
def.defEthernet(dev, mac, NicModel.valueOf(model.toUpperCase()), scriptPath, networkRateKBps);
185185
} else if (type.equals("vhostuser")) {
186186
String sourcePort = getAttrValue("source", "path", nic);
187-
String[] sourcePathParts = sourcePort.split("/");
188-
String port = sourcePathParts[sourcePathParts.length - 1];
187+
String mode = getAttrValue("source", "mode", nic);
188+
int lastSlashIndex = sourcePort.lastIndexOf("/");
189+
String ovsPath = sourcePort.substring(0,lastSlashIndex);
190+
String port = sourcePort.substring(lastSlashIndex + 1);
189191
def.setDpdkSourcePort(port);
192+
def.setDpdkOvsPath(ovsPath);
193+
def.setInterfaceMode(mode);
190194
}
191195

192196
if (StringUtils.isNotBlank(slot)) {

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -898,7 +898,7 @@ public String toString() {
898898
}
899899

900900
public static class InterfaceDef {
901-
enum GuestNetType {
901+
public enum GuestNetType {
902902
BRIDGE("bridge"), DIRECT("direct"), NETWORK("network"), USER("user"), ETHERNET("ethernet"), INTERNAL("internal"), VHOSTUSER("vhostuser");
903903
String _type;
904904

@@ -1112,10 +1112,24 @@ public void setDpdkSourcePort(String port) {
11121112
_dpdkSourcePort = port;
11131113
}
11141114

1115-
@Override
1116-
public String toString() {
1115+
public String getDpdkOvsPath() {
1116+
return _dpdkSourcePath;
1117+
}
1118+
1119+
public void setDpdkOvsPath(String path) {
1120+
_dpdkSourcePath = path;
1121+
}
1122+
1123+
public String getInterfaceMode() {
1124+
return _interfaceMode;
1125+
}
1126+
1127+
public void setInterfaceMode(String mode) {
1128+
_interfaceMode = mode;
1129+
}
1130+
1131+
public String getContent() {
11171132
StringBuilder netBuilder = new StringBuilder();
1118-
netBuilder.append("<interface type='" + _netType + "'>\n");
11191133
if (_netType == GuestNetType.BRIDGE) {
11201134
netBuilder.append("<source bridge='" + _sourceName + "'/>\n");
11211135
} else if (_netType == GuestNetType.NETWORK) {
@@ -1169,6 +1183,14 @@ public String toString() {
11691183
if (_slot != null) {
11701184
netBuilder.append(String.format("<address type='pci' domain='0x0000' bus='0x00' slot='0x%02x' function='0x0'/>\n", _slot));
11711185
}
1186+
return netBuilder.toString();
1187+
}
1188+
1189+
@Override
1190+
public String toString() {
1191+
StringBuilder netBuilder = new StringBuilder();
1192+
netBuilder.append("<interface type='" + _netType + "'>\n");
1193+
netBuilder.append(getContent());
11721194
netBuilder.append("</interface>\n");
11731195
return netBuilder.toString();
11741196
}

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java‎

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -87,12 +87,34 @@ public void getPifs() {
8787
s_logger.debug("done looking for pifs, no more bridges");
8888
}
8989

90+
/**
91+
* Plug interface with DPDK support:
92+
* - Create a new port with DPDK support for the interface
93+
* - Set the 'intf' path to the new port
94+
*/
95+
protected void plugDPDKInterface(InterfaceDef intf, String trafficLabel, Map<String, String> extraConfig,
96+
String vlanId, String guestOsType, NicTO nic, String nicAdapter) {
97+
s_logger.debug("DPDK support enabled: configuring per traffic label " + trafficLabel);
98+
String dpdkOvsPath = _libvirtComputingResource.dpdkOvsPath;
99+
if (StringUtils.isBlank(dpdkOvsPath)) {
100+
throw new CloudRuntimeException("DPDK is enabled on the host but no OVS path has been provided");
101+
}
102+
String port = dpdkDriver.getNextDpdkPort();
103+
DPDKHelper.VHostUserMode dpdKvHostUserMode = dpdkDriver.getDPDKvHostUserMode(extraConfig);
104+
dpdkDriver.addDpdkPort(_pifs.get(trafficLabel), port, vlanId, dpdKvHostUserMode, dpdkOvsPath);
105+
String interfaceMode = dpdkDriver.getGuestInterfacesModeFromDPDKVhostUserMode(dpdKvHostUserMode);
106+
intf.defDpdkNet(dpdkOvsPath, port, nic.getMac(),
107+
getGuestNicModel(guestOsType, nicAdapter), 0,
108+
dpdkDriver.getExtraDpdkProperties(extraConfig),
109+
interfaceMode);
110+
}
111+
90112
@Override
91113
public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
92114
s_logger.debug("plugging nic=" + nic);
93115

94116
LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef();
95-
if (!_libvirtComputingResource.dpdkSupport || nic.isDpdkDisabled()) {
117+
if (!_libvirtComputingResource.dpdkSupport || !nic.isDpdkEnabled()) {
96118
// Let libvirt handle OVS ports creation when DPDK property is disabled or when it is enabled but disabled for the nic
97119
// For DPDK support, libvirt does not handle ports creation, invoke 'addDpdkPort' method
98120
intf.setVirtualPortType("openvswitch");
@@ -114,20 +136,8 @@ public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<S
114136
if ((nic.getBroadcastType() == Networks.BroadcastDomainType.Vlan || nic.getBroadcastType() == Networks.BroadcastDomainType.Pvlan) &&
115137
!vlanId.equalsIgnoreCase("untagged")) {
116138
if (trafficLabel != null && !trafficLabel.isEmpty()) {
117-
if (_libvirtComputingResource.dpdkSupport && !nic.isDpdkDisabled()) {
118-
s_logger.debug("DPDK support enabled: configuring per traffic label " + trafficLabel);
119-
String dpdkOvsPath = _libvirtComputingResource.dpdkOvsPath;
120-
if (StringUtils.isBlank(dpdkOvsPath)) {
121-
throw new CloudRuntimeException("DPDK is enabled on the host but no OVS path has been provided");
122-
}
123-
String port = dpdkDriver.getNextDpdkPort();
124-
DPDKHelper.VHostUserMode dpdKvHostUserMode = dpdkDriver.getDPDKvHostUserMode(extraConfig);
125-
dpdkDriver.addDpdkPort(_pifs.get(trafficLabel), port, vlanId, dpdKvHostUserMode, dpdkOvsPath);
126-
String interfaceMode = dpdkDriver.getGuestInterfacesModeFromDPDKVhostUserMode(dpdKvHostUserMode);
127-
intf.defDpdkNet(dpdkOvsPath, port, nic.getMac(),
128-
getGuestNicModel(guestOsType, nicAdapter), 0,
129-
dpdkDriver.getExtraDpdkProperties(extraConfig),
130-
interfaceMode);
139+
if (_libvirtComputingResource.dpdkSupport && nic.isDpdkEnabled()) {
140+
plugDPDKInterface(intf, trafficLabel, extraConfig, vlanId, guestOsType, nic, nicAdapter);
131141
} else {
132142
s_logger.debug("creating a vlan dev and bridge for guest traffic per traffic label " + trafficLabel);
133143
intf.defBridgeNet(_pifs.get(trafficLabel), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);
@@ -184,7 +194,7 @@ public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<S
184194
@Override
185195
public void unplug(InterfaceDef iface) {
186196
// Libvirt apparently takes care of this, see BridgeVifDriver unplug
187-
if (_libvirtComputingResource.dpdkSupport) {
197+
if (_libvirtComputingResource.dpdkSupport && StringUtils.isNotBlank(iface.getDpdkSourcePort())) {
188198
// If DPDK is enabled, we'll need to cleanup the port as libvirt won't
189199
String dpdkPort = iface.getDpdkSourcePort();
190200
String cmd = String.format("ovs-vsctl del-port %s", dpdkPort);

0 commit comments

Comments
 (0)