feat(jans-linux-setup): use builtin libs for tar, zip, wget (#5899)

* feat(jans-linux-setup): use builtin libs for tar, zip, wget

* fix(jans-linux-setup): merge unpack_zip

* fix(jans-linux-setup): code smells

jans-linux-setup/jans_setup/setup_app/config.py

@@ -16,6 +16,7 @@
 class Config:
 
     # we define statics here so that is is acessible without construction
+    opt_dir = '/opt'
     jansOptFolder = '/opt/jans'
     distFolder = '/opt/dist'
     jre_home = '/opt/jre'

jans-linux-setup/jans_setup/setup_app/data/package_list.json

@@ -1,67 +1,67 @@
 {
   "red 7": {
     "optional": "",
-    "mandatory": "httpd httpd-mod_ssl mod_auth_openidc curl wget tar xz unzip rsyslog bzip2",
+    "mandatory": "httpd httpd-mod_ssl mod_auth_openidc rsyslog",
     "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel-yaml", "certifi":"python3-certifi", "pymysql":"python3-PyMySQL", "Crypto": "python3-cryptography", "psycopg2":"python3-psycopg2"}
   },
   "red 8": {
     "optional": "",
-    "mandatory": "httpd mod_ssl mod_auth_openidc curl wget tar xz unzip rsyslog bzip2",
+    "mandatory": "httpd mod_ssl mod_auth_openidc rsyslog",
     "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel-yaml", "certifi":"python3-certifi", "pymysql":"python3-PyMySQL", "Crypto": "python3-cryptography", "psycopg2":"python3-psycopg2"}
   },
   "red 9": {
     "optional": "",
-    "mandatory": "httpd mod_ssl mod_auth_openidc curl wget tar xz unzip  rsyslog bzip2",
+    "mandatory": "httpd mod_ssl mod_auth_openidc rsyslog",
     "python": {"ldap3":"python-ldap3", "requests":"python-requests", "ruamel.yaml":"python-ruamel-yaml", "certifi":"python-certifi", "pymysql":"python-PyMySQL", "Crypto": "python-cryptography", "psycopg2":"python-psycopg2"}
   },
   "centos 7": {
     "optional": "",
-    "mandatory": "httpd mod_ssl curl mod_auth_openidc wget tar xz unzip bzip2 rsyslog  python3-certifi",
+    "mandatory": "httpd mod_ssl mod_auth_openidc  rsyslog  python3-certifi",
     "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel-yaml", "pymysql":"python3-PyMySQL", "Crypto": "python3-cryptography", "psycopg2":"python3-psycopg2"}
   },
   "centos 8": {
     "optional": "",
-    "mandatory": "httpd mod_ssl mod_auth_openidc curl wget tar xz unzip  rsyslog bzip2",
+    "mandatory": "httpd mod_ssl mod_auth_openidc rsyslog",
     "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel-yaml", "certifi":"python3-certifi", "pymysql":"python3-PyMySQL", "Crypto": "python3-cryptography", "psycopg2":"python3-psycopg2"}
   },
   "centos 9": {
     "optional": "",
-    "mandatory": "httpd mod_ssl mod_auth_openidc curl wget tar xz unzip  rsyslog bzip2",
+    "mandatory": "httpd mod_ssl mod_auth_openidc rsyslog",
     "python": {"ldap3":"python-ldap3", "requests":"python-requests", "ruamel.yaml":"python-ruamel-yaml", "certifi":"python-certifi", "pymysql":"python-PyMySQL", "Crypto": "python-cryptography", "psycopg2":"python-psycopg2"}
   },
   "suse 15": {
     "optional": "",
-    "mandatory": "apache2 apache2-mod_auth_openidc curl wget tar xz unzip rsyslog bzip2 openssl",
+    "mandatory": "apache2 apache2-mod_auth_openidc rsyslog openssl",
     "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "certifi":"python3-certifi", "pymysql":"python3-PyMySQL", "cryptography": "python3-cryptography", "psycopg2":"python3-psycopg2"}
   },
   "suse tumbleweed": {
     "optional": "",
-    "mandatory": "apache2 apache2-mod_auth_openidc curl wget tar xz unzip rsyslog bzip2 openssl",
+    "mandatory": "apache2 apache2-mod_auth_openidc  rsyslog openssl",
     "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "certifi":"python3-certifi", "pymysql":"python3-PyMySQL", "cryptography": "python3-cryptography", "psycopg2":"python3-psycopg2"}
   },
     "debian 11": {
     "optional": "",
-    "mandatory": "apache2 curl wget tar xz-utils unzip rsyslog bzip2",
+    "mandatory": "apache2 rsyslog",
     "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "certifi":"python3-certifi", "pymysql":"python3-pymysql", "cryptography":"python3-cryptography", "psycopg2":"python3-psycopg2"}
   },
   "debian 10": {
     "optional": "",
-    "mandatory": "apache2 curl wget tar xz-utils unzip rsyslog bzip2",
+    "mandatory": "apache2 rsyslog",
     "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "certifi":"python3-certifi", "pymysql":"python3-pymysql", "Crypto": "python3-crypto", "psycopg2":"python3-psycopg2"}
   },
   "debian 9": {
     "optional": "",
-    "mandatory": "apache2 curl wget tar xz-utils unzip rsyslog bzip2",
+    "mandatory": "apache2 rsyslog",
     "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "certifi":"python3-certifi", "pymysql":"python3-pymysql", "Crypto": "python3-crypto", "psycopg2":"python3-psycopg2"}
   },
   "ubuntu 20": {
     "optional": "",
-    "mandatory": "apache2 curl wget xz-utils unzip rsyslog bzip2 python3-urllib3 python3-certifi",
+    "mandatory": "apache2 rsyslog python3-urllib3 python3-certifi",
     "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "pymysql":"python3-pymysql", "Crypto": "python3-crypto", "psycopg2":"python3-psycopg2"}
   },
   "ubuntu 22": {
     "optional": "",
-    "mandatory": "apache2 curl wget xz-utils unzip rsyslog bzip2 python3-urllib3 python3-certifi",
+    "mandatory": "apache2 rsyslog python3-urllib3 python3-certifi",
     "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "pymysql":"python3-pymysql", "cryptography":"python3-cryptography", "certifi":"python3-certifi", "psycopg2":"python3-psycopg2"}
   }
 }

jans-linux-setup/jans_setup/setup_app/installers/jetty.py

@@ -66,17 +66,14 @@ def install(self):
         self.addUserToGroup('jans', Config.jetty_user)
         self.run([paths.cmd_mkdir, '-p', self.jetty_user_home_lib])
 
-        jettyArchive, jetty_dist = self.get_jetty_info()
+        jetty_archive, jetty_dist = self.get_jetty_info()
 
         jettyTemp = os.path.join(jetty_dist, 'temp')
         self.run([paths.cmd_mkdir, '-p', jettyTemp])
         self.chown(jettyTemp, Config.jetty_user, Config.jetty_group, recursive=True)
 
-        try:
-            self.logIt("Extracting %s into /opt/jetty" % jettyArchive)
-            self.run(['tar', '-xzf', jettyArchive, '-C', jetty_dist, '--no-xattrs', '--no-same-owner', '--no-same-permissions'])
-        except:
-            self.logIt("Error encountered while extracting archive %s" % jettyArchive)
+        self.logIt(f"Extracting {jetty_archive} into {jetty_dist}")
+        shutil.unpack_archive(jetty_archive, format='gztar', extract_dir=jetty_dist)
 
         jettyDestinationPath = max(glob.glob(os.path.join(jetty_dist, '{}-*'.format(self.jetty_dist_string))))
 
@@ -109,18 +106,18 @@ def get_jetty_info(self):
         if not jetty_archive_list:
             self.logIt("Jetty archive not found in {}. Exiting...".format(Config.dist_app_dir), True, True)
 
-        jettyArchive = max(jetty_archive_list)
+        jetty_archive = max(jetty_archive_list)
 
-        jettyArchive_fn = os.path.basename(jettyArchive)
-        jetty_regex = re.search('{}-(\d*\.\d*)'.format(self.jetty_dist_string), jettyArchive_fn)
+        jetty_archive_fn = os.path.basename(jetty_archive)
+        jetty_regex = re.search('{}-(\d*\.\d*)'.format(self.jetty_dist_string), jetty_archive_fn)
         if not jetty_regex:
             self.logIt("Can't determine Jetty version", True, True)
 
         jetty_dist = '/opt/jetty-' + jetty_regex.groups()[0]
         Config.templateRenderingDict['jetty_dist'] = jetty_dist
         self.jetty_version_string = jetty_regex.groups()[0]
 
-        return jettyArchive, jetty_dist
+        return jetty_archive, jetty_dist
 
     @property
     def web_app_xml_fn(self):
@@ -143,9 +140,6 @@ def installJettyService(self, serviceConfiguration, supportCustomizations=False,
 
         if base.snap:
             Config.templateRenderingDict['jetty_dist'] = self.jetty_base
-        else:
-            # we need this, because this method may be called externally
-            jettyArchive, jetty_dist = self.get_jetty_info()
 
         self.logIt("Preparing %s service base folders" % service_name)
         self.run([paths.cmd_mkdir, '-p', jetty_service_base])

jans-linux-setup/jans_setup/setup_app/installers/jre.py

@@ -39,13 +39,11 @@ def install(self):
         else:
             jre_archive = self.source_files[0][0]
 
-        self.logIt("Installing server JRE {} ...".format(os.path.basename(jre_archive)))
+        jre_basename = os.path.basename(jre_archive)
+        self.logIt(f"Installing server JRE {jre_basename} ...")
 
-        try:
-            self.logIt("Extracting %s into /opt/" % os.path.basename(jre_archive))
-            self.run([paths.cmd_tar, '-xzf', jre_archive, '-C', '/opt/', '--no-xattrs', '--no-same-owner', '--no-same-permissions'])
-        except Exception as e:
-            self.logIt("Error encountered while extracting archive {}".format(e))
+        self.logIt(f"Extracting {jre_basename} into {Config.opt_dir}")
+        shutil.unpack_archive(jre_archive, format='gztar', extract_dir=Config.opt_dir)
 
         if Config.java_type == 'jdk':
             jreDestinationPath = max(glob.glob('/opt/jdk-11*'))

jans-linux-setup/jans_setup/setup_app/installers/node.py

@@ -38,13 +38,10 @@ def install(self):
             self.createUser('node', self.node_user_home)
             self.addUserToGroup('jans', 'node')
 
-        nodeArchive = max(node_archieve_list)
+        node_archive = max(node_archieve_list)
 
-        try:
-            self.logIt("Extracting %s into /opt" % nodeArchive)
-            self.run([paths.cmd_tar, '-xJf', nodeArchive, '-C', '/opt/', '--no-xattrs', '--no-same-owner', '--no-same-permissions'])
-        except Exception as e:
-            self.logIt("Error encountered while extracting archive {}: {}".format(nodeArchive, e))
+        self.logIt(f"Extracting {node_archive} into {Config.opt_dir}")
+        shutil.unpack_archive(node_archive, format='gztar', extract_dir=Config.opt_dir)
 
         nodeDestinationPath = max(glob.glob('/opt/node-*-linux-x64'))
 

jans-linux-setup/jans_setup/setup_app/installers/opendj.py

@@ -1,4 +1,5 @@
 import os
+import stat
 import glob
 import shutil
 import ssl
@@ -95,12 +96,8 @@ def extractOpenDJ(self):
 
         opendj_archive = max(glob.glob(os.path.join(Config.dist_app_dir, 'opendj-server-*4*.zip')))
 
-        try:
-            self.logIt("Unzipping %s in /opt/" % opendj_archive)
-            self.run([paths.cmd_unzip, '-n', '-q', '%s' % (opendj_archive), '-d', '/opt/' ])
-        except:
-            self.logIt("Error encountered while doing unzip %s -d /opt/" % (opendj_archive))
-
+        self.logIt(f"Extracting {opendj_archive} into {Config.opt_dir}")
+        base.unpack_zip(opendj_archive, Config.opt_dir)
         real_ldap_base_dir = os.path.realpath(Config.ldap_base_dir)
         self.chown(real_ldap_base_dir, Config.ldap_user, Config.ldap_group, recursive=True)
 

jans-linux-setup/jans_setup/setup_app/installers/oxd.py

@@ -1,6 +1,8 @@
 import os
 import glob
 import ruamel.yaml
+import shutil
+import tempfile
 
 from setup_app import paths
 from setup_app.static import AppType, InstallOption
@@ -25,9 +27,9 @@ def __init__(self):
 
     def install(self):
         self.logIt("Installing", pbar=self.service_name)
-        self.run(['tar', '-zxf', Config.oxd_package, '--no-same-owner', '--strip-components=1', '-C', self.oxd_root])
+        self.shutil.unpack_archive(Config.oxd_package, self.oxd_root)
         self.run(['chown', '-R', 'jetty:jetty', self.oxd_root])
-        
+
         if base.snap:
             self.log_dir = os.path.join(base.snap_common, 'jans/oxd-server/log/')
         else:
@@ -148,27 +150,18 @@ def download_files(self, force=False):
 
         self.logIt("Downloading {} and preparing package".format(os.path.basename(oxd_url)))
 
-        oxd_zip_fn = '/tmp/oxd-server.zip'
-        oxd_tgz_fn = '/tmp/oxd-server.tgz' if base.snap else os.path.join(Config.dist_jans_dir, 'oxd-server.tgz')
-        tmp_dir = os.path.join('/tmp', os.urandom(5).hex())
-        oxd_tmp_dir = os.path.join(tmp_dir, 'oxd-server')
-
-        self.run([paths.cmd_mkdir, '-p', oxd_tmp_dir])
-        self.download_file(oxd_url, oxd_zip_fn)
-        self.run([paths.cmd_unzip, '-qqo', oxd_zip_fn, '-d', oxd_tmp_dir])
-        self.run([paths.cmd_mkdir, os.path.join(oxd_tmp_dir, 'data')])
-
-        if not base.snap:
-            service_file = 'oxd-server.init.d' if base.deb_sysd_clone else 'oxd-server.service'
-            service_url = 'https://raw.githubusercontent.com/GluuFederation/community-edition-package/master/package/systemd/oxd-server.service'.format(base.current_app.app_info['ox_version'], service_file)
-            self.download_file(service_url, os.path.join(oxd_tmp_dir, service_file))
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            oxd_zip_fn = os.path.join(tmp_dir, 'oxd-server.zip')
+            oxd_tmp_dir = os.path.join(tmp_dir, 'oxd-server')
+            self.download_file(oxd_url, oxd_zip_fn)
+            shutil.unpack_archive(oxd_zip_fn, oxd_tmp_dir)
+            self.createDirs(os.path.join(oxd_tmp_dir, 'data'))
 
-        oxd_server_sh_url = 'https://raw.githubusercontent.com/GluuFederation/oxd/master/debian/oxd-server'
-        self.download_file(oxd_server_sh_url, os.path.join(oxd_tmp_dir, 'bin/oxd-server'))
+            oxd_server_sh_url = 'https://raw.githubusercontent.com/GluuFederation/oxd/master/debian/oxd-server'
+            self.download_file(oxd_server_sh_url, os.path.join(oxd_tmp_dir, 'bin/oxd-server'))
+            Config.oxd_package = shutil.make_archive(os.path.join(Config.dist_jans_dir, 'oxd-server'), "gztar", root_dir=tmp_dir, base_dir="oxd-server")
 
-        self.run(['tar', '-zcf', oxd_tgz_fn, 'oxd-server'], cwd=tmp_dir)
-        #self.run(['rm', '-r', '-f', tmp_dir])
-        Config.oxd_package = oxd_tgz_fn
+            Config.oxd_package = oxd_tgz_fn
 
     def create_folders(self):
         if not os.path.exists(self.oxd_root):

jans-linux-setup/jans_setup/setup_app/paths.py

@@ -24,8 +24,5 @@
 cmd_rm = '/bin/rm'
 cmd_py3 = shutil.which('python3')
 cmd_openssl = shutil.which('openssl')
-cmd_wget = shutil.which('wget')
 cmd_sed = shutil.which('sed')
-cmd_tar = shutil.which('tar')
-cmd_unzip = shutil.which('unzip')
 cmd_update_rc = shutil.which('update-rc.d')

jans-linux-setup/jans_setup/setup_app/test_data_loader.py

@@ -7,6 +7,7 @@
 import ldap3
 import urllib.request
 import base64
+import shutil
 
 from setup_app import paths
 from setup_app import static
@@ -251,7 +252,8 @@ def load_test_data(self):
         # Client keys deployment
         target_jwks_fn = os.path.join(base.current_app.HttpdInstaller.server_root, 'jans_test_client_keys.zip')
         base.download('https://github.com/JanssenProject/jans/raw/main/jans-auth-server/client/src/test/resources/jans_test_client_keys.zip', target_jwks_fn)
-        self.run([paths.cmd_unzip, '-o', target_jwks_fn, '-d', base.current_app.HttpdInstaller.server_root])
+        shutil.unpack_archive(target_jwks_fn, base.current_app.HttpdInstaller.server_root)
+
         self.removeFile(target_jwks_fn)
 
         self.chown(os.path.join(base.current_app.HttpdInstaller.server_root, 'jans-auth-client'), base.current_app.HttpdInstaller.apache_user, base.current_app.HttpdInstaller.apache_group, recursive=True)
@@ -442,9 +444,8 @@ def load_test_data(self):
         if Config.installEleven:
             eleven_tokens_package = os.path.join(Config.staticFolder, 'eleven/jans-eleven-tokens.tar.gz')
             target_dir = '/var/lib/softhsm/tokens/'
-            if not os.path.exists(target_dir):
-                os.makedirs(target_dir)
-            self.run([paths.cmd_tar, '-zxf', eleven_tokens_package, '-C', target_dir])
+            self.logIt(f"Extracting {eleven_tokens_package} into {target_dir}")
+            shutil.unpack_archive(eleven_tokens_package, format='gztar', extract_dir=target_dir)
 
         if Config.install_scim_server:
             self.restart('jans-scim')

jans-linux-setup/jans_setup/setup_app/utils/base.py

@@ -421,6 +421,14 @@ def extract_subdir(zip_fn, sub_dir, target_dir, par_dir=None):
         shutil.unpack_archive(zip_fn, unpack_dir, format='zip')
         shutil.copytree(os.path.join(unpack_dir, par_dir, sub_dir), target_dir)
 
+def unpack_zip(zip_fn, extract_dir):
+    with zipfile.ZipFile(zip_fn, 'r') as zf:
+        for info in zf.infolist():
+            zf.extract(info.filename, path=extract_dir)
+            out_path = os.path.join(extract_dir, info.filename)
+            perm = info.external_attr >> 16
+            os.chmod(out_path, perm)
+
 app_info_fn = os.environ.get('JANS_APP_INFO') or os.path.join(par_dir, 'app_info.json')
 current_app.app_info = readJsonFile(app_info_fn)
 current_app.jans_zip = os.path.join(Config.distFolder, 'jans/jans.zip')

jans-linux-setup/jans_setup/setup_app/utils/package_utils.py

@@ -123,11 +123,6 @@ def check_and_install_packages(self):
             if os.path.exists(default_site):
                 os.remove(default_site)
 
-        #re-new paths
-        paths.cmd_wget = shutil.which('wget')
-        paths.cmd_tar = shutil.which('tar')
-        paths.cmd_unzip = shutil.which('unzip')
-
 
     def installPackage(self, packageName, remote=False):
         base.logIt("Installing " + packageName)