marvin and tests python 2to3

.python-version

@@ -1 +1 @@
-system
+3.6

.travis.yml

@@ -25,7 +25,7 @@ jdk:
   - openjdk11
 
 python:
-  - "2.7"
+  - "3.9"
 
 node_js:
   - 12

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java

@@ -1898,7 +1898,7 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
         catch (Exception ex) {
             errMsg = "Copy operation failed in 'StorageSystemDataMotionStrategy.copyAsync': " + ex.getMessage();
 
-            throw new CloudRuntimeException(errMsg);
+            throw new CloudRuntimeException(errMsg, ex);
         }
         finally {
             CopyCmdAnswer copyCmdAnswer = new CopyCmdAnswer(errMsg);

packaging/centos7/cloud.spec

@@ -48,6 +48,9 @@ BuildRequires: /usr/bin/mkisofs
 BuildRequires: mysql-connector-python
 BuildRequires: maven => 3.0.0
 BuildRequires: python-setuptools
+BuildRequires: python3
+BuildRequires: python3-pip
+BuildRequires: python3-setuptools
 BuildRequires: wget
 BuildRequires: nodejs
 
@@ -153,6 +156,9 @@ Apache CloudStack command line interface
 
 %package marvin
 Summary: Apache CloudStack Marvin library
+Requires: python3
+Requires: python3-devel
+Requires: python3-pip
 Requires: python-pip
 Requires: gcc
 Requires: python-devel
@@ -418,6 +424,8 @@ fi
 # Install mysql-connector-python
 pip3 install %{_datadir}/%{name}-management/setup/wheel/six-1.15.0-py2.py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/setuptools-47.3.1-py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/protobuf-3.12.2-cp36-cp36m-manylinux1_x86_64.whl %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.20-cp36-cp36m-manylinux1_x86_64.whl
 
+pip3 install urllib3
+
 /usr/bin/systemctl on cloudstack-management > /dev/null 2>&1 || true
 
 grep -s -q "db.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties"
@@ -506,6 +514,9 @@ fi
 %post marvin
 pip install --upgrade https://files.pythonhosted.org/packages/ca/ea/1e2553b088bad2f9fa8120c2624f797b2d7450d3b61bb492d29c72e3d3c2/mysql_connector_python-8.0.20-cp27-cp27mu-manylinux1_x86_64.whl
 pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
+pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
+pip3 install --upgrade nose
+pip3 install --upgrade urllib3
 
 #No default permission as the permission setup is complex
 %files management

pom.xml

@@ -960,6 +960,7 @@
                             <exclude>**/*.md</exclude>
                             <exclude>.java-version</exclude>
                             <exclude>.python-version</exclude>
+                            <exclude>systemvm/.pythen-version</exclude>
                             <exclude>.idea/</exclude>
                             <exclude>.metadata/**</exclude>
                             <exclude>.git/**</exclude>

systemvm/.pythen-version

@@ -0,0 +1 @@
+2.7.18

systemvm/test/runtests.sh

@@ -22,15 +22,17 @@ export PYTHONPATH="../debian/opt/cloud/bin/"
 export PYTHONDONTWRITEBYTECODE=False
 
 echo "Running pycodestyle to check systemvm/python code for errors"
-pycodestyle --max-line-length=179 *py
-pycodestyle --max-line-length=179 --exclude=monitorServices.py,baremetal-vr.py,passwd_server_ip.py `find ../debian -name \*.py`
+python -m pycodestyle --max-line-length=179 *py
+python -m pycodestyle --max-line-length=179 --exclude=monitorServices.py,baremetal-vr.py,passwd_server_ip.py `find ../debian -name \*.py`
 if [ $? -gt 0 ]
 then
     echo "pycodestyle failed, please check your code"
     exit 1
 fi
 
 echo "Running pylint to check systemvm/python code for errors"
+python --version
+pyenv versions
 pylint --disable=R,C,W *.py
 pylint --disable=R,C,W `find ../debian -name \*.py`
 if [ $? -gt 0 ]

test/integration/component/cpu_limits/__init__.py

@@ -13,4 +13,4 @@
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
-# under the License.
+# under the License.

test/integration/component/find_hosts_for_migration.py

@@ -19,7 +19,8 @@
 """
 #Import Local Modules
 from nose.plugins.attrib import attr
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.cloudstackAPI import (migrateVirtualMachine,
                                   prepareHostForMaintenance,
                                   cancelHostMaintenance,
@@ -270,4 +271,4 @@ def test_01_find_hosts_for_migration(self):
 
         self.assertTrue(notSuitableHost is not None, "notsuitablehost should not be None")
         self.debug("Suitable Hosts: %s" % suitableHost)
-        self.debug("Not suitable Hosts: %s" % notSuitableHost)
+        self.debug("Not suitable Hosts: %s" % notSuitableHost)

test/integration/component/maint/__init__.py

@@ -18,4 +18,4 @@
 """
 Tests that put hosts, zones, resources in to maintenance mode are here.
 These will have to be run sequentiall when resources are available so as not disrupt other tests
-"""
+"""

test/integration/component/maint/test_bugs.py

@@ -16,7 +16,8 @@
 # under the License.
 
 # Import Local Modules
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.cloudstackAPI import (updateStoragePool,
                                   resizeVolume,
                                   listCapacity,

test/integration/component/maint/test_capacity_host_delete.py

@@ -18,7 +18,8 @@
 # Test from the Marvin - Testing in Python wiki
 
 # All tests inherit from cloudstackTestCase
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 
 # Import Integration Libraries
 

test/integration/component/maint/test_dedicate_guest_vlan_ranges.py

@@ -24,7 +24,8 @@
 """
 #Import Local Modules
 from nose.plugins.attrib import attr
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.lib.utils import (validateList,
                               cleanup_resources,
                               random_gen,

test/integration/component/maint/test_dedicate_public_ip_range.py

@@ -26,7 +26,8 @@
 """
 # Import Local Modules
 from nose.plugins.attrib import attr
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.lib.utils import (validateList,
                               cleanup_resources,
                               random_gen)

test/integration/component/maint/test_escalation_templates.py

@@ -15,7 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.lib.base import (Account,
                              Domain, Template, Configurations,VirtualMachine,Snapshot,ServiceOffering
                              )
@@ -161,7 +162,7 @@ def test_01_CS40139_listtemplate_with_different_pagesize(self):
             account=self.account.name,
             domainid=self.account.domainid)
         status = validateList(listfirst500template)
-        self.assertEquals(
+        self.assertEqual(
                 PASS,
                 status[0],
                 "First 500 template list is empty")
@@ -173,7 +174,7 @@ def test_01_CS40139_listtemplate_with_different_pagesize(self):
             account=self.account.name,
             domainid=self.account.domainid)
         status = validateList(listremainingtemplate)
-        self.assertEquals(
+        self.assertEqual(
                 PASS,
                 status[0],
                 "Next 500 template list is empty")
@@ -185,7 +186,7 @@ def test_01_CS40139_listtemplate_with_different_pagesize(self):
             account=self.account.name,
             domainid=self.account.domainid)
         status = validateList(listalltemplate)
-        self.assertEquals(
+        self.assertEqual(
                 PASS,
                 status[0],
                 "entire template list is empty")
@@ -253,7 +254,7 @@ def test_02_template_permissions(self):
             )
             self.updateConfigurAndRestart("allow.public.user.templates", "true")
             self.fail("Template creation passed for user")
-        except CloudstackAPIException  as e:
+        except CloudstackAPIException as e:
             self.assertRaises("Exception Raised : %s" % e)
         # Register new public template as domain admin
         # Exception should be raised for registering public template
@@ -268,7 +269,7 @@ def test_02_template_permissions(self):
             )
             self.updateConfigurAndRestart("allow.public.user.templates", "true")
             self.fail("Template creation passed for domain admin")
-        except CloudstackAPIException  as e:
+        except CloudstackAPIException as e:
             self.assertRaises("Exception Raised : %s" % e)
 
         if self.hypervisor.lower() in ['hyperv', 'lxc']:
@@ -314,7 +315,7 @@ def test_02_template_permissions(self):
                 )
                 self.updateConfigurAndRestart("allow.public.user.templates", "true")
                 self.fail("Template creation passed from snapshot for domain user")
-            except CloudstackAPIException  as e:
+            except CloudstackAPIException as e:
                 self.assertRaises("Exception Raised : %s" % e)
 
             VirtualMachine.stop(user_vm_created, self.user_api_client)
@@ -324,7 +325,7 @@ def test_02_template_permissions(self):
                 domainid=user_account.domainid,
                 state="Stopped")
             status = validateList(list_stopped_vms_after)
-            self.assertEquals(
+            self.assertEqual(
                 PASS,
                 status[0],
                 "Stopped VM is not in Stopped state"
@@ -336,7 +337,7 @@ def test_02_template_permissions(self):
                 )
                 self.updateConfigurAndRestart("allow.public.user.templates", "true")
                 self.fail("Template creation passed from volume for domain user")
-            except CloudstackAPIException  as e:
+            except CloudstackAPIException as e:
                 self.assertRaises("Exception Raised : %s" % e)
 
             admin_vm_created = VirtualMachine.create(
@@ -378,7 +379,7 @@ def test_02_template_permissions(self):
                 )
                 self.updateConfigurAndRestart("allow.public.user.templates", "true")
                 self.fail("Template creation passed from snapshot for domain admin")
-            except CloudstackAPIException  as e:
+            except CloudstackAPIException as e:
                 self.assertRaises("Exception Raised : %s" % e)
 
             VirtualMachine.stop(admin_vm_created, self.admin_api_client)
@@ -388,7 +389,7 @@ def test_02_template_permissions(self):
                 domainid=self.account.domainid,
                 state="Stopped")
             status = validateList(list_stopped_vms_after)
-            self.assertEquals(
+            self.assertEqual(
                 PASS,
                 status[0],
                 "Stopped VM is not in Stopped state"
@@ -400,7 +401,7 @@ def test_02_template_permissions(self):
                 )
                 self.updateConfigurAndRestart("allow.public.user.templates", "true")
                 self.fail("Template creation passed from volume for domain admin")
-            except CloudstackAPIException  as e:
+            except CloudstackAPIException as e:
                 self.assertRaises("Exception Raised : %s" % e)
 
             self.updateConfigurAndRestart("allow.public.user.templates", "true")

test/integration/component/maint/test_escalations_hosts.py

@@ -18,7 +18,8 @@
 # Test from the Marvin - Testing in Python wiki
 
 # All tests inherit from cloudstackTestCase
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 
 # Import Integration Libraries
 

test/integration/component/maint/test_high_availability.py

@@ -20,7 +20,8 @@
 """
 # Import Local Modules
 from nose.plugins.attrib import attr
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.cloudstackAPI import (prepareHostForMaintenance,
                                   cancelHostMaintenance)
 from marvin.lib.utils import cleanup_resources

test/integration/component/maint/test_host_high_availability.py

@@ -19,7 +19,8 @@
 """
 #Import Local Modules
 from nose.plugins.attrib import attr
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.cloudstackAPI import (migrateVirtualMachine,
                                   prepareHostForMaintenance,
                                   cancelHostMaintenance)

test/integration/component/maint/test_ip_resource_count_for_vpc.py

@@ -19,7 +19,8 @@
 """
 #Import Local Modules
 from nose.plugins.attrib import attr
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.lib.base import (VirtualMachine,
                              NetworkOffering,
                              VpcOffering,

test/integration/component/maint/test_multiple_ip_ranges.py

@@ -16,7 +16,8 @@
 # under the License.
 """ Tests for Multiple IP Ranges feature
 """
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.lib.utils import cleanup_resources, get_process_status
 from marvin.lib.base import (Account,
                              DiskOffering,
@@ -203,11 +204,11 @@ def setUp(self):
         )
         self.ip_range = list(
             netaddr.iter_iprange(
-                unicode(
-                    self.testdata["vlan_ip_range"]["startip"]), unicode(
+                str(
+                    self.testdata["vlan_ip_range"]["startip"]), str(
                     self.testdata["vlan_ip_range"]["endip"])))
         self.nic_ip = netaddr.IPAddress(
-            unicode(
+            str(
                 self.vm_response[0].nic[0].ipaddress))
         self.debug("vm got {} as ip address".format(self.nic_ip))
         self.assertIn(
@@ -281,11 +282,11 @@ def test_01_deploy_vm_in_new_cidr(self):
         """
         self.ip_range = list(
             netaddr.iter_iprange(
-                unicode(
-                    self.testdata["vlan_ip_range"]["startip"]), unicode(
+                str(
+                    self.testdata["vlan_ip_range"]["startip"]), str(
                     self.testdata["vlan_ip_range"]["endip"])))
         self.nic_ip = netaddr.IPAddress(
-            unicode(
+            str(
                 self.vm_response[0].nic[0].ipaddress))
         self.debug("vm got {} as ip address".format(self.nic_ip))
         self.assertIn(

test/integration/component/maint/test_vpc_host_maintenance.py

@@ -19,7 +19,8 @@
 """
 #Import Local Modules
 from nose.plugins.attrib import attr
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.cloudstackAPI import migrateSystemVm
 from marvin.lib.utils import cleanup_resources
 from marvin.lib.base import (Host,

test/integration/component/maint/test_vpc_on_host_maintenance.py

@@ -16,7 +16,8 @@
 # under the License.
 
 from nose.plugins.attrib import attr
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.lib.utils import cleanup_resources
 from marvin.lib.base import (Account,
                              Host,

test/integration/component/maint/testpath_disablestoragepool.py

@@ -19,7 +19,8 @@
 # All tests inherit from cloudstack TestCase
 
 from marvin.cloudstackTestCase import cloudstackTestCase
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.codes import FAILED, PASS
 from marvin.lib.base import (Account,
                              VirtualMachine,

test/integration/component/maint/testpath_vMotion_vmware.py

@@ -17,7 +17,8 @@
 """ Test cases for Test Paths Storage Migration
 """
 from nose.plugins.attrib import attr
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.lib.utils import (cleanup_resources,
                               validateList,
                               is_server_ssh_ready