Update SystemVM scripts from python2 to python3

systemvm/debian/opt/cloud/bin/configure.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -- coding: utf-8 --
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -21,8 +21,9 @@
 import os
 import re
 import sys
-import urllib
-import urllib2
+from urllib.request import *
+from urllib.parse import *
+from urllib.error import *
 import time
 
 from collections import OrderedDict
@@ -77,10 +78,10 @@ def __update(self, vm_ip, password):
             if proc.find():
                 url = "http://%s:8080/" % server_ip
                 payload = {"ip": vm_ip, "password": password, "token": token}
-                data = urllib.urlencode(payload)
-                request = urllib2.Request(url, data=data, headers={"DomU_Request": "save_password"})
+                data = urllib.parse.urlencode(payload)
+                request = urllib.request.Request(url, data=data, headers={"DomU_Request": "save_password"})
                 try:
-                    resp = urllib2.urlopen(request, data)
+                    resp = urlopen(request, data)
                     logging.debug("Update password server result: http:%s, content:%s" % (resp.code, resp.read()))
                 except Exception as e:
                     logging.error("Failed to update password server due to: %s" % e)
@@ -135,15 +136,15 @@ def add_rule(self):
             icmp_type = ''
             rule = self.rule
             icmp_type = "any"
-            if "icmp_type" in self.rule.keys() and self.rule['icmp_type'] != -1:
+            if "icmp_type" in list(self.rule.keys()) and self.rule['icmp_type'] != -1:
                 icmp_type = self.rule['icmp_type']
-            if "icmp_code" in self.rule.keys() and rule['icmp_code'] != -1:
+            if "icmp_code" in list(self.rule.keys()) and rule['icmp_code'] != -1:
                 icmp_type = "%s/%s" % (self.rule['icmp_type'], self.rule['icmp_code'])
             rnge = ''
-            if "first_port" in self.rule.keys() and \
+            if "first_port" in list(self.rule.keys()) and \
                self.rule['first_port'] == self.rule['last_port']:
                 rnge = " --dport %s " % self.rule['first_port']
-            if "first_port" in self.rule.keys() and \
+            if "first_port" in list(self.rule.keys()) and \
                self.rule['first_port'] != self.rule['last_port']:
                 rnge = " --dport %s:%s" % (rule['first_port'], rule['last_port'])
 
@@ -246,9 +247,9 @@ def __init__(self, obj, config):
             self.netmask = obj['nic_netmask']
             self.config = config
             self.cidr = "%s/%s" % (self.ip, self.netmask)
-            if "ingress_rules" in obj.keys():
+            if "ingress_rules" in list(obj.keys()):
                 self.ingress = obj['ingress_rules']
-            if "egress_rules" in obj.keys():
+            if "egress_rules" in list(obj.keys()):
                 self.egress = obj['egress_rules']
             self.fw = config.get_fw()
 
@@ -286,21 +287,21 @@ def init_vpc(self, direction, acl, rule, config):
                 self.type = rule['type']
                 self.icmp_type = "any"
                 self.protocol = self.type
-                if "icmp_type" in rule.keys() and rule['icmp_type'] != -1:
+                if "icmp_type" in list(rule.keys()) and rule['icmp_type'] != -1:
                     self.icmp_type = rule['icmp_type']
-                if "icmp_code" in rule.keys() and rule['icmp_code'] != -1:
+                if "icmp_code" in list(rule.keys()) and rule['icmp_code'] != -1:
                     self.icmp_type = "%s/%s" % (self.icmp_type, rule['icmp_code'])
                 if self.type == "protocol":
                     if rule['protocol'] == 41:
                         rule['protocol'] = "ipv6"
                     self.protocol = rule['protocol']
                 self.action = "DROP"
                 self.dport = ""
-                if 'allowed' in rule.keys() and rule['allowed']:
+                if 'allowed' in list(rule.keys()) and rule['allowed']:
                     self.action = "ACCEPT"
-                if 'first_port' in rule.keys():
+                if 'first_port' in list(rule.keys()):
                     self.dport = "-m %s --dport %s" % (self.protocol, rule['first_port'])
-                if 'last_port' in rule.keys() and self.dport and \
+                if 'last_port' in list(rule.keys()) and self.dport and \
                    rule['last_port'] != rule['first_port']:
                     self.dport = "%s:%s" % (self.dport, rule['last_port'])
 
@@ -385,18 +386,18 @@ def __createfile(self, ip, folder, file, data):
             fh.write("")
         self.__unflock(fh)
         fh.close()
-        os.chmod(dest, 0644)
+        os.chmod(dest, 0o644)
 
         if folder == "metadata" or folder == "meta-data":
             try:
-                os.makedirs(metamanifestdir, 0755)
+                os.makedirs(metamanifestdir, 0o755)
             except OSError as e:
                 # error 17 is already exists, we do it this way for concurrency
                 if e.errno != 17:
-                    print "failed to make directories " + metamanifestdir + " due to :" + e.strerror
+                    print("failed to make directories " + metamanifestdir + " due to :" + e.strerror)
                     sys.exit(1)
             if os.path.exists(metamanifest):
-                fh = open(metamanifest, "r+a")
+                fh = open(metamanifest, "r+")
                 self.__exflock(fh)
                 if file not in fh.read():
                     fh.write(file + '\n')
@@ -410,17 +411,17 @@ def __createfile(self, ip, folder, file, data):
                 fh.close()
 
         if os.path.exists(metamanifest):
-            os.chmod(metamanifest, 0644)
+            os.chmod(metamanifest, 0o644)
 
     def __htaccess(self, ip, folder, file):
         entry = "RewriteRule ^" + file + "$  ../" + folder + "/%{REMOTE_ADDR}/" + file + " [L,NC,QSA]"
         htaccessFolder = "/var/www/html/latest"
         htaccessFile = htaccessFolder + "/.htaccess"
 
-        CsHelper.mkdir(htaccessFolder, 0755, True)
+        CsHelper.mkdir(htaccessFolder, 0o755, True)
 
         if os.path.exists(htaccessFile):
-            fh = open(htaccessFile, "r+a")
+            fh = open(htaccessFile, "r+")
             self.__exflock(fh)
             if entry not in fh.read():
                 fh.write(entry + '\n')
@@ -439,11 +440,11 @@ def __htaccess(self, ip, folder, file):
         htaccessFile = htaccessFolder+"/.htaccess"
 
         try:
-            os.makedirs(htaccessFolder, 0755)
+            os.makedirs(htaccessFolder, 0o755)
         except OSError as e:
             # error 17 is already exists, we do it this way for sake of concurrency
             if e.errno != 17:
-                print "failed to make directories " + htaccessFolder + " due to :" + e.strerror
+                print("failed to make directories " + htaccessFolder + " due to :" + e.strerror)
                 sys.exit(1)
 
         fh = open(htaccessFile, "w")
@@ -457,7 +458,7 @@ def __htaccess(self, ip, folder, file):
             htaccessFolder = "/var/www/html/latest"
             htaccessFile = htaccessFolder + "/.htaccess"
 
-            fh = open(htaccessFile, "r+a")
+            fh = open(htaccessFile, "r+")
             self.__exflock(fh)
             if entry not in fh.read():
                 fh.write(entry + '\n')
@@ -474,15 +475,15 @@ def __exflock(self, file):
         try:
             flock(file, LOCK_EX)
         except IOError as e:
-            print "failed to lock file" + file.name + " due to : " + e.strerror
+            print("failed to lock file" + file.name + " due to : " + e.strerror)
             sys.exit(1)  # FIXME
         return True
 
     def __unflock(self, file):
         try:
             flock(file, LOCK_UN)
         except IOError as e:
-            print "failed to unlock file" + file.name + " due to : " + e.strerror
+            print("failed to unlock file" + file.name + " due to : " + e.strerror)
             sys.exit(1)  # FIXME
         return True
 
@@ -592,9 +593,9 @@ def configure_ipsec(self, obj):
 
         # This will load the new config
         CsHelper.execute("ipsec reload")
-        os.chmod(vpnsecretsfile, 0400)
+        os.chmod(vpnsecretsfile, 0o400)
 
-        for i in xrange(3):
+        for i in range(3):
             result = CsHelper.execute('ipsec status vpn-%s | grep "%s"' % (rightpeer, peerlist.split(",", 1)[0]))
             if len(result) > 0:
                 break
@@ -1060,7 +1061,7 @@ def main(argv):
                                ])
 
     def execDatabag(key, db):
-        if key not in db.keys() or 'executor' not in db[key]:
+        if key not in list(db.keys()) or 'executor' not in db[key]:
             logging.warn("Unable to find config or executor(s) for the databag type %s" % key)
             return
         for executor in db[key]['executor']:
@@ -1074,10 +1075,10 @@ def execIptables(config):
 
     if json_type == "cmd_line":
         logging.debug("cmd_line.json changed. All other files will be processed as well.")
-        for key in databag_map.keys():
+        for key in list(databag_map.keys()):
             execDatabag(key, databag_map)
         execIptables(config)
-    elif json_type in databag_map.keys():
+    elif json_type in list(databag_map.keys()):
         execDatabag(json_type, databag_map)
         if databag_map[json_type]['process_iptables']:
             execIptables(config)

systemvm/debian/opt/cloud/bin/cs/CsAddress.py

@@ -16,14 +16,14 @@
 # specific language governing permissions and limitations
 # under the License.
 import logging
-from netaddr import IPAddress, IPNetwork
+from ipaddress import *
 import subprocess
 import time
-import CsHelper
-from CsDatabag import CsDataBag
-from CsApp import CsApache, CsDnsmasq, CsPasswdSvc
-from CsRoute import CsRoute
-from CsRule import CsRule
+from . import CsHelper
+from .CsDatabag import CsDataBag
+from .CsApp import CsApache, CsDnsmasq, CsPasswdSvc
+from .CsRoute import CsRoute
+from .CsRule import CsRule
 
 VRRP_TYPES = ['guest']
 
@@ -148,8 +148,8 @@ def get_gateway(self):
             return self.config.cmdline().get_guest_gw()
 
     def ip_in_subnet(self, ip):
-        ipo = IPAddress(ip)
-        net = IPNetwork("%s/%s" % (self.get_ip(), self.get_size()))
+        ipo = ip_address(ip)
+        net = ip_network("%s/%s" % (self.get_ip(), self.get_size()))
         return ipo in net
 
     def get_gateway_cidr(self):
@@ -562,7 +562,7 @@ def post_config_change(self, method):
             if self.config.is_vpc():
                 if self.get_type() in ["public"] and "gateway" in self.address and self.address["gateway"] and self.address["gateway"] != "None":
                     route.add_route(self.dev, self.address["gateway"])
-                    for inf, addresses in self.config.address().dbag.iteritems():
+                    for inf, addresses in list(self.config.address().dbag.items()):
                         if not inf.startswith("eth"):
                             continue
                         for address in addresses:
@@ -636,7 +636,7 @@ def list(self):
                 self.iplist[cidr] = self.dev
 
     def configured(self):
-        if self.address['cidr'] in self.iplist.keys():
+        if self.address['cidr'] in list(self.iplist.keys()):
             return True
         return False
 
@@ -665,7 +665,7 @@ def getDevice(self):
         return self.dev
 
     def hasIP(self, ip):
-        return ip in self.address.values()
+        return ip in list(self.address.values())
 
     def arpPing(self):
         cmd = "arping -c 1 -I %s -A -U -s %s %s" % (
@@ -676,7 +676,7 @@ def arpPing(self):
 
     # Delete any ips that are configured but not in the bag
     def compare(self, bag):
-        if len(self.iplist) > 0 and (self.dev not in bag.keys() or len(bag[self.dev]) == 0):
+        if len(self.iplist) > 0 and (self.dev not in list(bag.keys()) or len(bag[self.dev]) == 0):
             # Remove all IPs on this device
             logging.info(
                 "Will remove all configured addresses on device %s", self.dev)
@@ -687,13 +687,13 @@ def compare(self, bag):
         # This condition should not really happen but did :)
         # It means an apache file got orphaned after a guest network address
         # was deleted
-        if len(self.iplist) == 0 and (self.dev not in bag.keys() or len(bag[self.dev]) == 0):
+        if len(self.iplist) == 0 and (self.dev not in list(bag.keys()) or len(bag[self.dev]) == 0):
             app = CsApache(self)
             app.remove()
 
         for ip in self.iplist:
             found = False
-            if self.dev in bag.keys():
+            if self.dev in list(bag.keys()):
                 for address in bag[self.dev]:
                     self.setAddress(address)
                     if (self.hasIP(ip) or self.is_guest_gateway(address, ip)) and address["add"]:
@@ -726,7 +726,7 @@ def delete(self, ip):
         remove = []
         if ip == "all":
             logging.info("Removing addresses from device %s", self.dev)
-            remove = self.iplist.keys()
+            remove = list(self.iplist.keys())
         else:
             remove.append(ip)
         for ip in remove:

systemvm/debian/opt/cloud/bin/cs/CsApp.py

@@ -16,8 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 import os
-from CsFile import CsFile
-import CsHelper
+from .CsFile import CsFile
+from .CsProcess import CsProcess
+from . import CsHelper
 
 
 class CsApp:

systemvm/debian/opt/cloud/bin/cs/CsConfig.py

@@ -16,8 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from CsDatabag import CsCmdLine
-from CsAddress import CsAddress
+from .CsDatabag import CsCmdLine
+from .CsAddress import CsAddress
 import logging
 
 

systemvm/debian/opt/cloud/bin/cs/CsDatabag.py

@@ -33,7 +33,7 @@ def __init__(self, key, config=None):
             self.config = config
 
     def dump(self):
-        print self.dbag
+        print(self.dbag)
 
     def get_bag(self):
         return self.dbag

systemvm/debian/opt/cloud/bin/cs/CsDhcp.py

@@ -14,12 +14,12 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import CsHelper
+from . import CsHelper
 import logging
 import os
-from netaddr import *
+from ipaddress import ip_address
 from random import randint
-from CsGuestNetwork import CsGuestNetwork
+from .CsGuestNetwork import CsGuestNetwork
 from cs.CsDatabag import CsDataBag
 from cs.CsFile import CsFile
 
@@ -207,7 +207,7 @@ def add(self, entry):
                                                                             entry['ipv4_address'],
                                                                             entry['host_name']))
 
-        i = IPAddress(entry['ipv4_address'])
+        i = ip_address(entry['ipv4_address'])
         # Calculate the device
         for v in self.devinfo:
             if i > v['network'].network and i < v['network'].broadcast:

systemvm/debian/opt/cloud/bin/cs/CsFile.py

@@ -70,7 +70,7 @@ def commit(self):
 
     def dump(self):
         for line in self.new_config:
-            print line
+            print(line)
 
     def addeq(self, string):
         """ Update a line in a file of the form token=something
@@ -153,7 +153,7 @@ def searchString(self, search, ignoreLinesStartWith):
         logging.debug("Searching for %s string " % search)
 
         for index, line in enumerate(self.new_config):
-            print ' line = ' + line
+            print(' line = ' + line)
             if line.lstrip().startswith(ignoreLinesStartWith):
                 continue
             if search in line:

systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 from merge import DataBag
-import CsHelper
+from . import CsHelper
 
 
 class CsGuestNetwork:
@@ -27,7 +27,7 @@ def __init__(self, device, config):
         db.load()
         dbag = db.getDataBag()
         self.config = config
-        if device in dbag.keys() and len(dbag[device]) != 0:
+        if device in list(dbag.keys()) and len(dbag[device]) != 0:
             self.data = dbag[device][0]
         else:
             self.guest = False