Merge release branch 4.18 to main

* 4.18:
  Storage and volumes statistics tasks for StorPool primary storage (#7404)
  proper storage construction (#6797)
  guarantee MAC uniqueness (#7634)
  server: allow migration of all VMs with local storage on KVM (#7656)
  Add L2 networks to Zones with SG (#7719)

Author: DaanHoogland

api/src/main/java/com/cloud/network/NetworkModel.java

@@ -90,7 +90,7 @@ public interface NetworkModel {
             INSTANCE_ID_FILE, VM_ID_FILE, PUBLIC_KEYS_FILE, CLOUD_IDENTIFIER_FILE, HYPERVISOR_HOST_NAME_FILE));
 
     static final ConfigKey<Integer> MACIdentifier = new ConfigKey<>("Advanced",Integer.class, "mac.identifier", "0",
-            "This value will be used while generating the mac addresses for isolated and shared networks. The hexadecimal equivalent value will be present at the 2nd octet of the mac address. Default value is null which means this feature is disabled.Its scope is global.", true, ConfigKey.Scope.Global);
+            "This value will be used while generating the mac addresses for isolated and shared networks. The hexadecimal equivalent value will be present at the 2nd octet of the mac address. Default value is zero (0) which means that the DB id of the zone will be used.", true, ConfigKey.Scope.Zone);
 
     static final ConfigKey<Boolean> AdminIsAllowedToDeployAnywhere = new ConfigKey<>("Advanced",Boolean.class, "admin.is.allowed.to.deploy.anywhere", "false",
             "This will determine if the root admin is allowed to deploy in networks in subdomains.", true, ConfigKey.Scope.Global);
@@ -114,6 +114,13 @@ public interface NetworkModel {
 
     List<? extends Nic> getNics(long vmId);
 
+    /**
+     * Gets the next available MAC and checks it for global uniqueness in the nics table. It will keep looking until it finds a MAC address that is unique.
+     *
+     * @param networkConfigurationId the id of the network to use the nic in. used for finding the zone
+     * @return a string containing a MAC address
+     * @throws InsufficientAddressCapacityException if no MAC can be returned
+     */
     String getNextAvailableMacAddressInNetwork(long networkConfigurationId) throws InsufficientAddressCapacityException;
 
     PublicIpAddress getPublicIpAddress(long ipAddressId);

api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java

@@ -323,10 +323,10 @@ public Long getPhysicalNetworkId() {
             }
         }
         if (physicalNetworkId != null) {
-            if (offering.getGuestType() == GuestType.Shared) {
+            if ((offering.getGuestType() == GuestType.Shared) || (offering.getGuestType() == GuestType.L2)) {
                 return physicalNetworkId;
             } else {
-                throw new InvalidParameterValueException("Physical network ID can be specified for networks of guest IP type " + GuestType.Shared + " only.");
+                throw new InvalidParameterValueException("Physical network ID can be specified for networks of guest IP type " + GuestType.Shared + " or " + GuestType.L2 + " only.");
             }
         } else {
             if (zoneId == null) {

api/src/test/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmdTest.java

@@ -251,7 +251,23 @@ public void testGetPhysicalNetworkIdForNonSharedNet() {
         try {
             cmd.getPhysicalNetworkId();
         } catch (Exception e) {
-            Assert.assertTrue(e.getMessage().startsWith("Physical network ID can be specified for networks of guest IP type Shared only"));
+            Assert.assertTrue(e.getMessage().startsWith("Physical network ID can be specified for networks of guest IP type Shared or L2 only."));
+        }
+    }
+
+    @Test
+    public void testGetPhysicalNetworkIdForL2Net() {
+        Long physicalNetworkId = 1L;
+        Long networkOfferingId = 1L;
+        ReflectionTestUtils.setField(cmd, "networkOfferingId", networkOfferingId);
+        NetworkOffering networkOffering = Mockito.mock(NetworkOffering.class);
+        ReflectionTestUtils.setField(cmd, "physicalNetworkId", physicalNetworkId);
+        Mockito.when(_entityMgr.findById(NetworkOffering.class, networkOfferingId)).thenReturn(networkOffering);
+        Mockito.when(networkOffering.getGuestType()).thenReturn(Network.GuestType.L2);
+        try {
+            Assert.assertEquals(cmd.getPhysicalNetworkId(), physicalNetworkId);
+        } catch (Exception e) {
+            Assert.fail("Failed to get physical network id");
         }
     }
 

engine/orchestration/src/main/java/org/apache/cloudstack/engine/datacenter/entity/api/db/dao/EngineDataCenterDao.java

@@ -28,14 +28,6 @@ public interface EngineDataCenterDao extends GenericDao<EngineDataCenterVO, Long
         StateDao<DataCenterResourceEntity.State, DataCenterResourceEntity.State.Event, DataCenterResourceEntity> {
     EngineDataCenterVO findByName(String name);
 
-    /**
-     * @param id data center id
-     * @return a pair of mac address strings.  The first one is private and second is public.
-     */
-    String[] getNextAvailableMacAddressPair(long id);
-
-    String[] getNextAvailableMacAddressPair(long id, long mask);
-
     List<EngineDataCenterVO> findZonesByDomainId(Long domainId);
 
     List<EngineDataCenterVO> listPublicZones(String keyword);

engine/orchestration/src/main/java/org/apache/cloudstack/engine/datacenter/entity/api/db/dao/EngineDataCenterDaoImpl.java

@@ -23,7 +23,6 @@
 
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
-import javax.persistence.TableGenerator;
 
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
@@ -39,10 +38,8 @@
 import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
-import com.cloud.utils.db.SequenceFetcher;
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.db.UpdateBuilder;
-import com.cloud.utils.net.NetUtils;
 
 /**
  * @config
@@ -66,7 +63,6 @@ public class EngineDataCenterDaoImpl extends GenericDaoBase<EngineDataCenterVO,
 
     protected long _prefix;
     protected Random _rand = new Random(System.currentTimeMillis());
-    protected TableGenerator _tgMacAddress;
 
     @Inject
     protected DcDetailsDao _detailsDao;
@@ -139,25 +135,6 @@ public List<EngineDataCenterVO> findByKeyword(String keyword) {
         return listBy(ssc);
     }
 
-    @Override
-    public String[] getNextAvailableMacAddressPair(long id) {
-        return getNextAvailableMacAddressPair(id, 0);
-    }
-
-    @Override
-    public String[] getNextAvailableMacAddressPair(long id, long mask) {
-        SequenceFetcher fetch = SequenceFetcher.getInstance();
-
-        long seq = fetch.getNextSequence(Long.class, _tgMacAddress, id);
-        seq = seq | _prefix | ((id & 0x7f) << 32);
-        seq |= mask;
-        seq |= ((_rand.nextInt(Short.MAX_VALUE) << 16) & 0x00000000ffff0000l);
-        String[] pair = new String[2];
-        pair[0] = NetUtils.long2Mac(seq);
-        pair[1] = NetUtils.long2Mac(seq | 0x1l << 39);
-        return pair;
-    }
-
     @Override
     public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
         if (!super.configure(name, params)) {
@@ -204,9 +181,6 @@ protected EngineDataCenterDaoImpl() {
         UUIDSearch = createSearchBuilder();
         UUIDSearch.and("uuid", UUIDSearch.entity().getUuid(), SearchCriteria.Op.EQ);
         UUIDSearch.done();
-
-        _tgMacAddress = _tgs.get("macAddress");
-        assert _tgMacAddress != null : "Couldn't get mac address table generator";
     }
 
     @Override

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -2665,8 +2665,8 @@ private Network createGuestNetwork(final long networkOfferingId, final String na
                 }
                 // Only Account specific Isolated network with sourceNat service disabled are allowed in security group
                 // enabled zone
-                if (ntwkOff.getGuestType() != GuestType.Shared) {
-                    throw new InvalidParameterValueException("Only shared guest network can be created in security group enabled zone");
+                if ((ntwkOff.getGuestType() != GuestType.Shared) && (ntwkOff.getGuestType() != GuestType.L2)) {
+                    throw new InvalidParameterValueException("Only shared or L2 guest network can be created in security group enabled zone");
                 }
                 if (_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)) {
                     throw new InvalidParameterValueException("Service SourceNat is not allowed in security group enabled zone");

engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDao.java

@@ -52,14 +52,6 @@ public Integer getVlan() {
 
     DataCenterVO findByName(String name);
 
-    /**
-     * @param id data center id
-     * @return a pair of mac address strings.  The first one is private and second is public.
-     */
-    String[] getNextAvailableMacAddressPair(long id);
-
-    String[] getNextAvailableMacAddressPair(long id, long mask);
-
     PrivateAllocationData allocatePrivateIpAddress(long id, long podId, long instanceId, String reservationId, boolean forSystemVms);
 
     DataCenterIpAddressVO allocatePrivateIpAddress(long id, String reservationId);

engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDaoImpl.java

@@ -24,7 +24,6 @@
 
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
-import javax.persistence.TableGenerator;
 
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.log4j.Logger;
@@ -45,9 +44,7 @@
 import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
-import com.cloud.utils.db.SequenceFetcher;
 import com.cloud.utils.db.TransactionLegacy;
-import com.cloud.utils.net.NetUtils;
 
 /**
  * @config
@@ -83,7 +80,7 @@ public class DataCenterDaoImpl extends GenericDaoBase<DataCenterVO, Long> implem
 
     protected long _prefix;
     protected Random _rand = new Random(System.currentTimeMillis());
-    protected TableGenerator _tgMacAddress;
+
 
     @Override
     public DataCenterVO findByName(String name) {
@@ -230,25 +227,6 @@ public String allocatePodVlan(long podId, long accountId) {
         return vo.getVlan();
     }
 
-    @Override
-    public String[] getNextAvailableMacAddressPair(long id) {
-        return getNextAvailableMacAddressPair(id, 0);
-    }
-
-    @Override
-    public String[] getNextAvailableMacAddressPair(long id, long mask) {
-        SequenceFetcher fetch = SequenceFetcher.getInstance();
-
-        long seq = fetch.getNextSequence(Long.class, _tgMacAddress, id);
-        seq = seq | _prefix | ((id & 0x7f) << 32);
-        seq |= mask;
-        seq |= ((_rand.nextInt(Short.MAX_VALUE) << 16) & 0x00000000ffff0000l);
-        String[] pair = new String[2];
-        pair[0] = NetUtils.long2Mac(seq);
-        pair[1] = NetUtils.long2Mac(seq | 0x1l << 39);
-        return pair;
-    }
-
     @Override
     public PrivateAllocationData allocatePrivateIpAddress(long dcId, long podId, long instanceId, String reservationId, boolean forSystemVms) {
         _ipAllocDao.releaseIpAddress(instanceId);
@@ -348,9 +326,6 @@ public DataCenterDaoImpl() {
         TokenSearch = createSearchBuilder();
         TokenSearch.and("zoneToken", TokenSearch.entity().getZoneToken(), SearchCriteria.Op.EQ);
         TokenSearch.done();
-
-        _tgMacAddress = _tgs.get("macAddress");
-        assert _tgMacAddress != null : "Couldn't get mac address table generator";
     }
 
     @Override

engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java

@@ -436,9 +436,6 @@ public String getNextAvailableMacAddress(final long networkConfigId, Integer zon
         if(zoneMacIdentifier != null && zoneMacIdentifier.intValue() != 0 ){
             seq = seq | _prefix << 40 | (long)zoneMacIdentifier << 32 | networkConfigId << 16 & 0x00000000ffff0000l;
         }
-        else {
-            seq = seq | _prefix << 40 | _rand.nextInt(Short.MAX_VALUE) << 16 & 0x00000000ffff0000l;
-        }
         return NetUtils.long2Mac(seq);
     }
 

engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java

@@ -87,7 +87,7 @@ public interface NicDao extends GenericDao<NicVO, Long> {
 
     List<NicVO> listByVmIdAndKeyword(long instanceId, String keyword);
 
-    NicVO findByInstanceIdAndMacAddress(long instanceId, String macAddress);
+    NicVO findByMacAddress(String macAddress);
 
     NicVO findByNetworkIdAndMacAddressIncludingRemoved(long networkId, String mac);