From 860fcd512e85e35067459036439510e874e5a491 Mon Sep 17 00:00:00 2001 From: Devrim Date: Mon, 28 Aug 2023 10:47:35 +0300 Subject: [PATCH] feat(jans-linux-setup): use builtin libs for tar, zip, wget (#5899) * feat(jans-linux-setup): use builtin libs for tar, zip, wget * fix(jans-linux-setup): merge unpack_zip * fix(jans-linux-setup): code smells Signed-off-by: Mustafa Baser --- .../jans_setup/setup_app/config.py | 1 + .../setup_app/data/package_list.json | 26 +++++++------- .../jans_setup/setup_app/installers/jetty.py | 20 ++++------- .../jans_setup/setup_app/installers/jre.py | 10 +++--- .../jans_setup/setup_app/installers/node.py | 9 ++--- .../jans_setup/setup_app/installers/opendj.py | 9 ++--- .../jans_setup/setup_app/installers/oxd.py | 35 ++++++++----------- .../jans_setup/setup_app/paths.py | 3 -- .../jans_setup/setup_app/test_data_loader.py | 9 ++--- .../jans_setup/setup_app/utils/base.py | 8 +++++ .../setup_app/utils/package_utils.py | 5 --- 11 files changed, 58 insertions(+), 77 deletions(-) diff --git a/jans-linux-setup/jans_setup/setup_app/config.py b/jans-linux-setup/jans_setup/setup_app/config.py index 92cca81caff..1d72a4913ab 100644 --- a/jans-linux-setup/jans_setup/setup_app/config.py +++ b/jans-linux-setup/jans_setup/setup_app/config.py @@ -16,6 +16,7 @@ class Config: # we define statics here so that is is acessible without construction + opt_dir = '/opt' jansOptFolder = '/opt/jans' distFolder = '/opt/dist' jre_home = '/opt/jre' diff --git a/jans-linux-setup/jans_setup/setup_app/data/package_list.json b/jans-linux-setup/jans_setup/setup_app/data/package_list.json index 7120c1aaa97..493d1909fd9 100644 --- a/jans-linux-setup/jans_setup/setup_app/data/package_list.json +++ b/jans-linux-setup/jans_setup/setup_app/data/package_list.json @@ -1,67 +1,67 @@ { "red 7": { "optional": "", - "mandatory": "httpd httpd-mod_ssl mod_auth_openidc curl wget tar xz unzip rsyslog bzip2", + "mandatory": "httpd httpd-mod_ssl mod_auth_openidc rsyslog", "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel-yaml", "certifi":"python3-certifi", "pymysql":"python3-PyMySQL", "Crypto": "python3-cryptography", "psycopg2":"python3-psycopg2"} }, "red 8": { "optional": "", - "mandatory": "httpd mod_ssl mod_auth_openidc curl wget tar xz unzip rsyslog bzip2", + "mandatory": "httpd mod_ssl mod_auth_openidc rsyslog", "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel-yaml", "certifi":"python3-certifi", "pymysql":"python3-PyMySQL", "Crypto": "python3-cryptography", "psycopg2":"python3-psycopg2"} }, "red 9": { "optional": "", - "mandatory": "httpd mod_ssl mod_auth_openidc curl wget tar xz unzip rsyslog bzip2", + "mandatory": "httpd mod_ssl mod_auth_openidc rsyslog", "python": {"ldap3":"python-ldap3", "requests":"python-requests", "ruamel.yaml":"python-ruamel-yaml", "certifi":"python-certifi", "pymysql":"python-PyMySQL", "Crypto": "python-cryptography", "psycopg2":"python-psycopg2"} }, "centos 7": { "optional": "", - "mandatory": "httpd mod_ssl curl mod_auth_openidc wget tar xz unzip bzip2 rsyslog python3-certifi", + "mandatory": "httpd mod_ssl mod_auth_openidc rsyslog python3-certifi", "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel-yaml", "pymysql":"python3-PyMySQL", "Crypto": "python3-cryptography", "psycopg2":"python3-psycopg2"} }, "centos 8": { "optional": "", - "mandatory": "httpd mod_ssl mod_auth_openidc curl wget tar xz unzip rsyslog bzip2", + "mandatory": "httpd mod_ssl mod_auth_openidc rsyslog", "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel-yaml", "certifi":"python3-certifi", "pymysql":"python3-PyMySQL", "Crypto": "python3-cryptography", "psycopg2":"python3-psycopg2"} }, "centos 9": { "optional": "", - "mandatory": "httpd mod_ssl mod_auth_openidc curl wget tar xz unzip rsyslog bzip2", + "mandatory": "httpd mod_ssl mod_auth_openidc rsyslog", "python": {"ldap3":"python-ldap3", "requests":"python-requests", "ruamel.yaml":"python-ruamel-yaml", "certifi":"python-certifi", "pymysql":"python-PyMySQL", "Crypto": "python-cryptography", "psycopg2":"python-psycopg2"} }, "suse 15": { "optional": "", - "mandatory": "apache2 apache2-mod_auth_openidc curl wget tar xz unzip rsyslog bzip2 openssl", + "mandatory": "apache2 apache2-mod_auth_openidc rsyslog openssl", "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "certifi":"python3-certifi", "pymysql":"python3-PyMySQL", "cryptography": "python3-cryptography", "psycopg2":"python3-psycopg2"} }, "suse tumbleweed": { "optional": "", - "mandatory": "apache2 apache2-mod_auth_openidc curl wget tar xz unzip rsyslog bzip2 openssl", + "mandatory": "apache2 apache2-mod_auth_openidc rsyslog openssl", "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "certifi":"python3-certifi", "pymysql":"python3-PyMySQL", "cryptography": "python3-cryptography", "psycopg2":"python3-psycopg2"} }, "debian 11": { "optional": "", - "mandatory": "apache2 curl wget tar xz-utils unzip rsyslog bzip2", + "mandatory": "apache2 rsyslog", "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "certifi":"python3-certifi", "pymysql":"python3-pymysql", "cryptography":"python3-cryptography", "psycopg2":"python3-psycopg2"} }, "debian 10": { "optional": "", - "mandatory": "apache2 curl wget tar xz-utils unzip rsyslog bzip2", + "mandatory": "apache2 rsyslog", "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "certifi":"python3-certifi", "pymysql":"python3-pymysql", "Crypto": "python3-crypto", "psycopg2":"python3-psycopg2"} }, "debian 9": { "optional": "", - "mandatory": "apache2 curl wget tar xz-utils unzip rsyslog bzip2", + "mandatory": "apache2 rsyslog", "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "certifi":"python3-certifi", "pymysql":"python3-pymysql", "Crypto": "python3-crypto", "psycopg2":"python3-psycopg2"} }, "ubuntu 20": { "optional": "", - "mandatory": "apache2 curl wget xz-utils unzip rsyslog bzip2 python3-urllib3 python3-certifi", + "mandatory": "apache2 rsyslog python3-urllib3 python3-certifi", "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "pymysql":"python3-pymysql", "Crypto": "python3-crypto", "psycopg2":"python3-psycopg2"} }, "ubuntu 22": { "optional": "", - "mandatory": "apache2 curl wget xz-utils unzip rsyslog bzip2 python3-urllib3 python3-certifi", + "mandatory": "apache2 rsyslog python3-urllib3 python3-certifi", "python": {"ldap3":"python3-ldap3", "requests":"python3-requests", "ruamel.yaml":"python3-ruamel.yaml", "pymysql":"python3-pymysql", "cryptography":"python3-cryptography", "certifi":"python3-certifi", "psycopg2":"python3-psycopg2"} } } diff --git a/jans-linux-setup/jans_setup/setup_app/installers/jetty.py b/jans-linux-setup/jans_setup/setup_app/installers/jetty.py index 213cd614711..2687adb2d4f 100644 --- a/jans-linux-setup/jans_setup/setup_app/installers/jetty.py +++ b/jans-linux-setup/jans_setup/setup_app/installers/jetty.py @@ -66,17 +66,14 @@ def install(self): self.addUserToGroup('jans', Config.jetty_user) self.run([paths.cmd_mkdir, '-p', self.jetty_user_home_lib]) - jettyArchive, jetty_dist = self.get_jetty_info() + jetty_archive, jetty_dist = self.get_jetty_info() jettyTemp = os.path.join(jetty_dist, 'temp') self.run([paths.cmd_mkdir, '-p', jettyTemp]) self.chown(jettyTemp, Config.jetty_user, Config.jetty_group, recursive=True) - try: - self.logIt("Extracting %s into /opt/jetty" % jettyArchive) - self.run(['tar', '-xzf', jettyArchive, '-C', jetty_dist, '--no-xattrs', '--no-same-owner', '--no-same-permissions']) - except: - self.logIt("Error encountered while extracting archive %s" % jettyArchive) + self.logIt(f"Extracting {jetty_archive} into {jetty_dist}") + shutil.unpack_archive(jetty_archive, format='gztar', extract_dir=jetty_dist) jettyDestinationPath = max(glob.glob(os.path.join(jetty_dist, '{}-*'.format(self.jetty_dist_string)))) @@ -109,10 +106,10 @@ def get_jetty_info(self): if not jetty_archive_list: self.logIt("Jetty archive not found in {}. Exiting...".format(Config.dist_app_dir), True, True) - jettyArchive = max(jetty_archive_list) + jetty_archive = max(jetty_archive_list) - jettyArchive_fn = os.path.basename(jettyArchive) - jetty_regex = re.search('{}-(\d*\.\d*)'.format(self.jetty_dist_string), jettyArchive_fn) + jetty_archive_fn = os.path.basename(jetty_archive) + jetty_regex = re.search('{}-(\d*\.\d*)'.format(self.jetty_dist_string), jetty_archive_fn) if not jetty_regex: self.logIt("Can't determine Jetty version", True, True) @@ -120,7 +117,7 @@ def get_jetty_info(self): Config.templateRenderingDict['jetty_dist'] = jetty_dist self.jetty_version_string = jetty_regex.groups()[0] - return jettyArchive, jetty_dist + return jetty_archive, jetty_dist @property def web_app_xml_fn(self): @@ -143,9 +140,6 @@ def installJettyService(self, serviceConfiguration, supportCustomizations=False, if base.snap: Config.templateRenderingDict['jetty_dist'] = self.jetty_base - else: - # we need this, because this method may be called externally - jettyArchive, jetty_dist = self.get_jetty_info() self.logIt("Preparing %s service base folders" % service_name) self.run([paths.cmd_mkdir, '-p', jetty_service_base]) diff --git a/jans-linux-setup/jans_setup/setup_app/installers/jre.py b/jans-linux-setup/jans_setup/setup_app/installers/jre.py index 0a1e5582c3c..0b77a1c291d 100644 --- a/jans-linux-setup/jans_setup/setup_app/installers/jre.py +++ b/jans-linux-setup/jans_setup/setup_app/installers/jre.py @@ -39,13 +39,11 @@ def install(self): else: jre_archive = self.source_files[0][0] - self.logIt("Installing server JRE {} ...".format(os.path.basename(jre_archive))) + jre_basename = os.path.basename(jre_archive) + self.logIt(f"Installing server JRE {jre_basename} ...") - try: - self.logIt("Extracting %s into /opt/" % os.path.basename(jre_archive)) - self.run([paths.cmd_tar, '-xzf', jre_archive, '-C', '/opt/', '--no-xattrs', '--no-same-owner', '--no-same-permissions']) - except Exception as e: - self.logIt("Error encountered while extracting archive {}".format(e)) + self.logIt(f"Extracting {jre_basename} into {Config.opt_dir}") + shutil.unpack_archive(jre_archive, format='gztar', extract_dir=Config.opt_dir) if Config.java_type == 'jdk': jreDestinationPath = max(glob.glob('/opt/jdk-11*')) diff --git a/jans-linux-setup/jans_setup/setup_app/installers/node.py b/jans-linux-setup/jans_setup/setup_app/installers/node.py index b917873694a..7ad38680c76 100644 --- a/jans-linux-setup/jans_setup/setup_app/installers/node.py +++ b/jans-linux-setup/jans_setup/setup_app/installers/node.py @@ -38,13 +38,10 @@ def install(self): self.createUser('node', self.node_user_home) self.addUserToGroup('jans', 'node') - nodeArchive = max(node_archieve_list) + node_archive = max(node_archieve_list) - try: - self.logIt("Extracting %s into /opt" % nodeArchive) - self.run([paths.cmd_tar, '-xJf', nodeArchive, '-C', '/opt/', '--no-xattrs', '--no-same-owner', '--no-same-permissions']) - except Exception as e: - self.logIt("Error encountered while extracting archive {}: {}".format(nodeArchive, e)) + self.logIt(f"Extracting {node_archive} into {Config.opt_dir}") + shutil.unpack_archive(node_archive, format='gztar', extract_dir=Config.opt_dir) nodeDestinationPath = max(glob.glob('/opt/node-*-linux-x64')) diff --git a/jans-linux-setup/jans_setup/setup_app/installers/opendj.py b/jans-linux-setup/jans_setup/setup_app/installers/opendj.py index 4f4413f3ef0..c207f02faba 100644 --- a/jans-linux-setup/jans_setup/setup_app/installers/opendj.py +++ b/jans-linux-setup/jans_setup/setup_app/installers/opendj.py @@ -1,4 +1,5 @@ import os +import stat import glob import shutil import ssl @@ -95,12 +96,8 @@ def extractOpenDJ(self): opendj_archive = max(glob.glob(os.path.join(Config.dist_app_dir, 'opendj-server-*4*.zip'))) - try: - self.logIt("Unzipping %s in /opt/" % opendj_archive) - self.run([paths.cmd_unzip, '-n', '-q', '%s' % (opendj_archive), '-d', '/opt/' ]) - except: - self.logIt("Error encountered while doing unzip %s -d /opt/" % (opendj_archive)) - + self.logIt(f"Extracting {opendj_archive} into {Config.opt_dir}") + base.unpack_zip(opendj_archive, Config.opt_dir) real_ldap_base_dir = os.path.realpath(Config.ldap_base_dir) self.chown(real_ldap_base_dir, Config.ldap_user, Config.ldap_group, recursive=True) diff --git a/jans-linux-setup/jans_setup/setup_app/installers/oxd.py b/jans-linux-setup/jans_setup/setup_app/installers/oxd.py index 5231f3068ef..93467b2165f 100644 --- a/jans-linux-setup/jans_setup/setup_app/installers/oxd.py +++ b/jans-linux-setup/jans_setup/setup_app/installers/oxd.py @@ -1,6 +1,8 @@ import os import glob import ruamel.yaml +import shutil +import tempfile from setup_app import paths from setup_app.static import AppType, InstallOption @@ -25,9 +27,9 @@ def __init__(self): def install(self): self.logIt("Installing", pbar=self.service_name) - self.run(['tar', '-zxf', Config.oxd_package, '--no-same-owner', '--strip-components=1', '-C', self.oxd_root]) + self.shutil.unpack_archive(Config.oxd_package, self.oxd_root) self.run(['chown', '-R', 'jetty:jetty', self.oxd_root]) - + if base.snap: self.log_dir = os.path.join(base.snap_common, 'jans/oxd-server/log/') else: @@ -148,27 +150,18 @@ def download_files(self, force=False): self.logIt("Downloading {} and preparing package".format(os.path.basename(oxd_url))) - oxd_zip_fn = '/tmp/oxd-server.zip' - oxd_tgz_fn = '/tmp/oxd-server.tgz' if base.snap else os.path.join(Config.dist_jans_dir, 'oxd-server.tgz') - tmp_dir = os.path.join('/tmp', os.urandom(5).hex()) - oxd_tmp_dir = os.path.join(tmp_dir, 'oxd-server') - - self.run([paths.cmd_mkdir, '-p', oxd_tmp_dir]) - self.download_file(oxd_url, oxd_zip_fn) - self.run([paths.cmd_unzip, '-qqo', oxd_zip_fn, '-d', oxd_tmp_dir]) - self.run([paths.cmd_mkdir, os.path.join(oxd_tmp_dir, 'data')]) - - if not base.snap: - service_file = 'oxd-server.init.d' if base.deb_sysd_clone else 'oxd-server.service' - service_url = 'https://raw.githubusercontent.com/GluuFederation/community-edition-package/master/package/systemd/oxd-server.service'.format(base.current_app.app_info['ox_version'], service_file) - self.download_file(service_url, os.path.join(oxd_tmp_dir, service_file)) + with tempfile.TemporaryDirectory() as tmp_dir: + oxd_zip_fn = os.path.join(tmp_dir, 'oxd-server.zip') + oxd_tmp_dir = os.path.join(tmp_dir, 'oxd-server') + self.download_file(oxd_url, oxd_zip_fn) + shutil.unpack_archive(oxd_zip_fn, oxd_tmp_dir) + self.createDirs(os.path.join(oxd_tmp_dir, 'data')) - oxd_server_sh_url = 'https://raw.githubusercontent.com/GluuFederation/oxd/master/debian/oxd-server' - self.download_file(oxd_server_sh_url, os.path.join(oxd_tmp_dir, 'bin/oxd-server')) + oxd_server_sh_url = 'https://raw.githubusercontent.com/GluuFederation/oxd/master/debian/oxd-server' + self.download_file(oxd_server_sh_url, os.path.join(oxd_tmp_dir, 'bin/oxd-server')) + Config.oxd_package = shutil.make_archive(os.path.join(Config.dist_jans_dir, 'oxd-server'), "gztar", root_dir=tmp_dir, base_dir="oxd-server") - self.run(['tar', '-zcf', oxd_tgz_fn, 'oxd-server'], cwd=tmp_dir) - #self.run(['rm', '-r', '-f', tmp_dir]) - Config.oxd_package = oxd_tgz_fn + Config.oxd_package = oxd_tgz_fn def create_folders(self): if not os.path.exists(self.oxd_root): diff --git a/jans-linux-setup/jans_setup/setup_app/paths.py b/jans-linux-setup/jans_setup/setup_app/paths.py index c49bdee294c..4823e31f2c4 100644 --- a/jans-linux-setup/jans_setup/setup_app/paths.py +++ b/jans-linux-setup/jans_setup/setup_app/paths.py @@ -24,8 +24,5 @@ cmd_rm = '/bin/rm' cmd_py3 = shutil.which('python3') cmd_openssl = shutil.which('openssl') -cmd_wget = shutil.which('wget') cmd_sed = shutil.which('sed') -cmd_tar = shutil.which('tar') -cmd_unzip = shutil.which('unzip') cmd_update_rc = shutil.which('update-rc.d') diff --git a/jans-linux-setup/jans_setup/setup_app/test_data_loader.py b/jans-linux-setup/jans_setup/setup_app/test_data_loader.py index a31e7c988ed..e8f73b91fe7 100644 --- a/jans-linux-setup/jans_setup/setup_app/test_data_loader.py +++ b/jans-linux-setup/jans_setup/setup_app/test_data_loader.py @@ -7,6 +7,7 @@ import ldap3 import urllib.request import base64 +import shutil from setup_app import paths from setup_app import static @@ -251,7 +252,8 @@ def load_test_data(self): # Client keys deployment target_jwks_fn = os.path.join(base.current_app.HttpdInstaller.server_root, 'jans_test_client_keys.zip') base.download('https://github.com/JanssenProject/jans/raw/main/jans-auth-server/client/src/test/resources/jans_test_client_keys.zip', target_jwks_fn) - self.run([paths.cmd_unzip, '-o', target_jwks_fn, '-d', base.current_app.HttpdInstaller.server_root]) + shutil.unpack_archive(target_jwks_fn, base.current_app.HttpdInstaller.server_root) + self.removeFile(target_jwks_fn) self.chown(os.path.join(base.current_app.HttpdInstaller.server_root, 'jans-auth-client'), base.current_app.HttpdInstaller.apache_user, base.current_app.HttpdInstaller.apache_group, recursive=True) @@ -442,9 +444,8 @@ def load_test_data(self): if Config.installEleven: eleven_tokens_package = os.path.join(Config.staticFolder, 'eleven/jans-eleven-tokens.tar.gz') target_dir = '/var/lib/softhsm/tokens/' - if not os.path.exists(target_dir): - os.makedirs(target_dir) - self.run([paths.cmd_tar, '-zxf', eleven_tokens_package, '-C', target_dir]) + self.logIt(f"Extracting {eleven_tokens_package} into {target_dir}") + shutil.unpack_archive(eleven_tokens_package, format='gztar', extract_dir=target_dir) if Config.install_scim_server: self.restart('jans-scim') diff --git a/jans-linux-setup/jans_setup/setup_app/utils/base.py b/jans-linux-setup/jans_setup/setup_app/utils/base.py index a957a78ca16..e70eee28f45 100644 --- a/jans-linux-setup/jans_setup/setup_app/utils/base.py +++ b/jans-linux-setup/jans_setup/setup_app/utils/base.py @@ -421,6 +421,14 @@ def extract_subdir(zip_fn, sub_dir, target_dir, par_dir=None): shutil.unpack_archive(zip_fn, unpack_dir, format='zip') shutil.copytree(os.path.join(unpack_dir, par_dir, sub_dir), target_dir) +def unpack_zip(zip_fn, extract_dir): + with zipfile.ZipFile(zip_fn, 'r') as zf: + for info in zf.infolist(): + zf.extract(info.filename, path=extract_dir) + out_path = os.path.join(extract_dir, info.filename) + perm = info.external_attr >> 16 + os.chmod(out_path, perm) + app_info_fn = os.environ.get('JANS_APP_INFO') or os.path.join(par_dir, 'app_info.json') current_app.app_info = readJsonFile(app_info_fn) current_app.jans_zip = os.path.join(Config.distFolder, 'jans/jans.zip') diff --git a/jans-linux-setup/jans_setup/setup_app/utils/package_utils.py b/jans-linux-setup/jans_setup/setup_app/utils/package_utils.py index b5e7dd452ec..f5cec014db2 100644 --- a/jans-linux-setup/jans_setup/setup_app/utils/package_utils.py +++ b/jans-linux-setup/jans_setup/setup_app/utils/package_utils.py @@ -123,11 +123,6 @@ def check_and_install_packages(self): if os.path.exists(default_site): os.remove(default_site) - #re-new paths - paths.cmd_wget = shutil.which('wget') - paths.cmd_tar = shutil.which('tar') - paths.cmd_unzip = shutil.which('unzip') - def installPackage(self, packageName, remote=False): base.logIt("Installing " + packageName)