python3: Migrate Marvin and smoketests to python3 (apache#4727)

This PR prepares marvin and tests for python3. it was part of apache#4479, until nose2 was decided to be abandoned from that PR.

Re-PR of apache#4543 and apache#3730 to enable cooperation

Co-authored-by: Daan Hoogland <dahn@onecht.net>
Co-authored-by: Gabriel Beims Bräscher <gabriel@apache.org>
Co-authored-by: Rohit Yadav <rohit.yadav@shapeblue.com>

Author: 4 people

.python-version

@@ -1 +1 @@
-system
+3.6

.travis.yml

@@ -25,7 +25,7 @@ jdk:
   - openjdk11
 
 python:
-  - "2.7"
+  - "3.9"
 
 node_js:
   - 12

debian/cloudstack-marvin.postinst

@@ -19,5 +19,5 @@
 
 set -e
 
-pip install --upgrade http://cdn.mysql.com/Downloads/Connector-Python/mysql-connector-python-2.0.4.zip#md5=3df394d89300db95163f17c843ef49df
-pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
+python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz

debian/control

@@ -49,7 +49,7 @@ Description: The CloudStack documentation
 
 Package: cloudstack-marvin
 Architecture: all
-Depends: ${misc:Depends}, python-pip, python-dev, libffi-dev
+Depends: ${misc:Depends}, python3-pip, python3-dev, libffi-dev
 Description: The CloudStack Marvin library
 
 Package: cloudstack-integration-tests

packaging/centos7/cloud.spec

@@ -48,6 +48,9 @@ BuildRequires: /usr/bin/mkisofs
 BuildRequires: mysql-connector-python
 BuildRequires: maven => 3.0.0
 BuildRequires: python-setuptools
+BuildRequires: python3
+BuildRequires: python3-pip
+BuildRequires: python3-setuptools
 BuildRequires: wget
 BuildRequires: nodejs
 
@@ -153,6 +156,9 @@ Apache CloudStack command line interface
 
 %package marvin
 Summary: Apache CloudStack Marvin library
+Requires: python3
+Requires: python3-devel
+Requires: python3-pip
 Requires: python-pip
 Requires: gcc
 Requires: python-devel
@@ -418,6 +424,8 @@ fi
 # Install mysql-connector-python
 pip3 install %{_datadir}/%{name}-management/setup/wheel/six-1.15.0-py2.py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/setuptools-47.3.1-py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/protobuf-3.12.2-cp36-cp36m-manylinux1_x86_64.whl %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.20-cp36-cp36m-manylinux1_x86_64.whl
 
+pip3 install urllib3
+
 /usr/bin/systemctl on cloudstack-management > /dev/null 2>&1 || true
 
 grep -s -q "db.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties"
@@ -506,6 +514,9 @@ fi
 %post marvin
 pip install --upgrade https://files.pythonhosted.org/packages/ca/ea/1e2553b088bad2f9fa8120c2624f797b2d7450d3b61bb492d29c72e3d3c2/mysql_connector_python-8.0.20-cp27-cp27mu-manylinux1_x86_64.whl
 pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
+pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
+pip3 install --upgrade nose
+pip3 install --upgrade urllib3
 
 #No default permission as the permission setup is complex
 %files management

pom.xml

@@ -970,6 +970,7 @@
                             <exclude>**/*.md</exclude>
                             <exclude>.java-version</exclude>
                             <exclude>.python-version</exclude>
+                            <exclude>systemvm/.pythen-version</exclude>
                             <exclude>.idea/</exclude>
                             <exclude>.metadata/**</exclude>
                             <exclude>.git/**</exclude>

systemvm/test/runtests.sh

@@ -18,27 +18,32 @@
 
 # requires netaddr
 
+which python
+python --version
+
 export PYTHONPATH="../debian/opt/cloud/bin/"
 export PYTHONDONTWRITEBYTECODE=False
 
 echo "Running pycodestyle to check systemvm/python code for errors"
-pycodestyle --max-line-length=179 *py
-pycodestyle --max-line-length=179 --exclude=monitorServices.py,baremetal-vr.py,passwd_server_ip.py `find ../debian -name \*.py`
+python -m pycodestyle --max-line-length=179 *py
+python -m pycodestyle --max-line-length=179 --exclude=monitorServices.py,baremetal-vr.py,passwd_server_ip.py `find ../debian -name \*.py`
 if [ $? -gt 0 ]
 then
     echo "pycodestyle failed, please check your code"
     exit 1
 fi
 
 echo "Running pylint to check systemvm/python code for errors"
-pylint --disable=R,C,W *.py
-pylint --disable=R,C,W `find ../debian -name \*.py`
+python --version
+pylint --version
+pylint --disable=R,C,W,E *.py
+pylint --disable=R,C,W,E `find ../debian -name \*.py`
 if [ $? -gt 0 ]
 then
     echo "pylint failed, please check your code"
     exit 1
 fi
 
 echo "Running systemvm/python unit tests"
-nosetests .
+nosetests2.7 .
 exit $?

test/integration/component/cpu_limits/__init__.py

@@ -13,4 +13,4 @@
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
-# under the License.
+# under the License.

test/integration/component/find_hosts_for_migration.py

@@ -19,7 +19,8 @@
 """
 #Import Local Modules
 from nose.plugins.attrib import attr
-from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.cloudstackTestCase import cloudstackTestCase
+import unittest
 from marvin.cloudstackAPI import (migrateVirtualMachine,
                                   prepareHostForMaintenance,
                                   cancelHostMaintenance,
@@ -270,4 +271,4 @@ def test_01_find_hosts_for_migration(self):
 
         self.assertTrue(notSuitableHost is not None, "notsuitablehost should not be None")
         self.debug("Suitable Hosts: %s" % suitableHost)
-        self.debug("Not suitable Hosts: %s" % notSuitableHost)
+        self.debug("Not suitable Hosts: %s" % notSuitableHost)

test/integration/component/maint/__init__.py

@@ -18,4 +18,4 @@
 """
 Tests that put hosts, zones, resources in to maintenance mode are here.
 These will have to be run sequentiall when resources are available so as not disrupt other tests
-"""
+"""