diff --git a/lib/vsc/utils/daemon.py b/lib/vsc/utils/daemon.py
index 66151b46..2ff89174 100644
--- a/lib/vsc/utils/daemon.py
+++ b/lib/vsc/utils/daemon.py
@@ -141,4 +141,3 @@ def run(self):
You should override this method when you subclass Daemon. It will be called after the process has been
daemonized by start() or restart().
"""
- pass
diff --git a/lib/vsc/utils/optcomplete.py b/lib/vsc/utils/optcomplete.py
index d9bb9a51..96d81a3f 100644
--- a/lib/vsc/utils/optcomplete.py
+++ b/lib/vsc/utils/optcomplete.py
@@ -177,7 +177,6 @@ def _call(self, **kwargs): # pylint: disable=unused-argument
class NoneCompleter(Completer):
"""Generates empty completion list. For compatibility reasons."""
- pass
class ListCompleter(Completer):
diff --git a/lib/vsc/utils/py2vs3/__init__.py b/lib/vsc/utils/py2vs3/__init__.py
deleted file mode 100644
index c87c7d66..00000000
--- a/lib/vsc/utils/py2vs3/__init__.py
+++ /dev/null
@@ -1,67 +0,0 @@
-#
-# Copyright 2020-2023 Ghent University
-#
-# This file is part of vsc-base,
-# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
-# with support of Ghent University (http://ugent.be/hpc),
-# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
-# the Flemish Research Foundation (FWO) (http://www.fwo.be/en)
-# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
-#
-# https://github.com/hpcugent/vsc-base
-#
-# vsc-base is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Library General Public License as
-# published by the Free Software Foundation, either version 2 of
-# the License, or (at your option) any later version.
-#
-# vsc-base is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Library General Public License for more details.
-#
-# You should have received a copy of the GNU Library General Public License
-# along with vsc-base. If not, see .
-#
-"""
-Utility functions to help with keeping the codebase compatible with both Python 2 and 3.
-
-@author: Kenneth Hoste (Ghent University)
-"""
-# declare vsc.utils.py2vs3 namespace
-# (must be exactly like this to avoid vsc-install complaining)
-import pkg_resources
-pkg_resources.declare_namespace(__name__)
-
-import logging
-import sys
-
-
-def is_py_ver(maj_ver, min_ver=0):
- """Check whether current Python version matches specified version specs."""
-
- curr_ver = sys.version_info
-
- lower_limit = (maj_ver, min_ver)
- upper_limit = (maj_ver + 1, 0)
-
- return lower_limit <= curr_ver < upper_limit
-
-
-def is_py2():
- """Determine whether we're using Python 3."""
- return is_py_ver(2)
-
-
-def is_py3():
- """Determine whether we're using Python 3."""
- return is_py_ver(3)
-
-
-# all functionality provided by the py2 and py3 modules is made available via the vsc.utils.py2vs3 namespace
-if is_py3():
- logging.warning(
- "DEPRECATED: vsc.utils.py2vs3 is deprecated. Please remote it and use the py3 native modules, functions, ...")
- from vsc.utils.py2vs3.py3 import * # noqa
-else:
- raise ImportError("py2 module unsupported and removed, please stop using it.")
diff --git a/lib/vsc/utils/py2vs3/py3.py b/lib/vsc/utils/py2vs3/py3.py
deleted file mode 100644
index 2a3a506d..00000000
--- a/lib/vsc/utils/py2vs3/py3.py
+++ /dev/null
@@ -1,68 +0,0 @@
-#
-# Copyright 2020-2023 Ghent University
-#
-# This file is part of vsc-base,
-# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
-# with support of Ghent University (http://ugent.be/hpc),
-# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
-# the Flemish Research Foundation (FWO) (http://www.fwo.be/en)
-# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
-#
-# https://github.com/hpcugent/vsc-base
-#
-# vsc-base is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Library General Public License as
-# published by the Free Software Foundation, either version 2 of
-# the License, or (at your option) any later version.
-#
-# vsc-base is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Library General Public License for more details.
-#
-# You should have received a copy of the GNU Library General Public License
-# along with vsc-base. If not, see .
-#
-"""
-Utility functions to help with keeping the codebase compatible with both Python 2 and 3.
-
-@author: Kenneth Hoste (Ghent University)
-"""
-import logging
-import configparser # noqa
-import pickle # noqa
-from io import StringIO # noqa
-from shlex import quote # noqa
-from tempfile import TemporaryDirectory # noqa
-from urllib.parse import urlencode, unquote # noqa
-from urllib.request import HTTPError, HTTPSHandler, Request, build_opener, urlopen # noqa
-from collections.abc import Mapping # noqa
-
-FileExistsErrorExc = FileExistsError # noqa
-FileNotFoundErrorExc = FileNotFoundError # noqa
-
-
-def is_string(value):
- """Check whether specified value is of type string (not bytes)."""
- logging.warning("deprecated: is_string() used, please replace by isinstance()")
- return isinstance(value, str)
-
-
-def ensure_ascii_string(value):
- """
- Convert the provided value to an ASCII string (no Unicode characters).
- """
- msg = "deprecated: vsc.utils.py2vs3.ensure_ascii_string used."
- msg += "Please replace by vsc.utils.missing.ensure_ascii_string."
- logging.warning(msg)
- if isinstance(value, bytes):
- # if we have a bytestring, decode it to a regular string using ASCII encoding,
- # and replace Unicode characters with backslash escaped sequences
- value = value.decode('ascii', 'backslashreplace')
- else:
- # for other values, just convert to a string (which may still include Unicode characters)
- # then convert to bytestring with UTF-8 encoding,
- # which can then be decoded to regular string using ASCII encoding
- value = bytes(str(value), encoding='utf-8').decode(encoding='ascii', errors='backslashreplace')
-
- return value
diff --git a/lib/vsc/utils/rest.py b/lib/vsc/utils/rest.py
index 78f853d6..fe01566f 100644
--- a/lib/vsc/utils/rest.py
+++ b/lib/vsc/utils/rest.py
@@ -67,7 +67,7 @@ class Client:
USER_AGENT = 'vsc-rest-client'
def __init__(self, url, username=None, password=None, token=None, token_type='Token', user_agent=None,
- append_slash=False):
+ append_slash=False, decode=True):
"""
Create a Client object,
this client can consume a REST api hosted at host/endpoint
@@ -81,6 +81,7 @@ def __init__(self, url, username=None, password=None, token=None, token_type='To
self.username = username
self.url = url
self.append_slash = append_slash
+ self.decode = decode
if not user_agent:
self.user_agent = self.USER_AGENT
@@ -193,9 +194,12 @@ def request(self, method, url, body, headers, content_type=None):
else:
body = conn.read()
body = body.decode('utf-8') # byte encoded response
- try:
- pybody = json.loads(body)
- except ValueError:
+ if self.decode:
+ try:
+ pybody = json.loads(body)
+ except ValueError:
+ pybody = body
+ else:
pybody = body
logging.debug('reponse len: %s ', len(pybody))
return status, pybody
diff --git a/lib/vsc/utils/run.py b/lib/vsc/utils/run.py
index a674ff5e..ef303612 100644
--- a/lib/vsc/utils/run.py
+++ b/lib/vsc/utils/run.py
@@ -299,14 +299,20 @@ def _start_in_path(self):
try:
self._cwd_before_startpath = os.getcwd() # store it some one can return to it
os.chdir(self.startpath)
- except OSError:
- self.log.raiseException("_start_in_path: failed to change path from %s to startpath %s" %
- (self._cwd_before_startpath, self.startpath))
+ except OSError as exc:
+ msg = (
+ f"_start_in_path: failed to change path from {self._cwd_before_startpath} "
+ f"to startpath {self.startpath}")
+ self.log.exception(msg)
+ raise OSError(msg) from exc
else:
- self.log.raiseException("_start_in_path: provided startpath %s exists but is no directory" %
- self.startpath)
+ msg = f"_start_in_path: provided startpath {self.startpath} exists but is no directory"
+ self.log.error(msg)
+ raise ValueError(msg)
else:
- self.log.raiseException(f"_start_in_path: startpath {self.startpath} does not exist")
+ msg = f"_start_in_path: startpath {self.startpath} does not exist"
+ self.log.error(msg)
+ raise ValueError(msg)
def _return_to_previous_start_in_path(self):
"""Change to original path before the change to startpath"""
@@ -322,15 +328,22 @@ def _return_to_previous_start_in_path(self):
self.log.warning(("_return_to_previous_start_in_path: current diretory %s does not match "
"startpath %s"), currentpath, self.startpath)
os.chdir(self._cwd_before_startpath)
- except OSError:
- self.log.raiseException(("_return_to_previous_start_in_path: failed to change path from current %s "
- "to previous path %s"), currentpath, self._cwd_before_startpath)
+ except OSError as exc:
+ msg = (
+ f"_return_to_previous_start_in_path: failed to change path from current {currentpath} "
+ f"to previous path {self._cwd_before_startpath}")
+ self.log.exception(msg)
+ raise OSError(msg) from exc
else:
- self.log.raiseException(("_return_to_previous_start_in_path: provided previous cwd path %s exists "
- "but is no directory") % self._cwd_before_startpath)
+ msg = (
+ f"_return_to_previous_start_in_path: provided previous cwd path {self._cwd_before_startpath} "
+ "exists but is not a directory.")
+ self.log.error(msg)
+ raise ValueError(msg)
else:
- self.log.raiseException("_return_to_previous_start_in_path: previous cwd path %s does not exist" %
- self._cwd_before_startpath)
+ msg = f"_return_to_previous_start_in_path: previous cwd path {self._cwd_before_startpath} does not exist"
+ self.log.error(msg)
+ raise ValueError(msg)
def _make_popen_named_args(self, others=None):
"""Create the named args for Popen"""
@@ -397,9 +410,12 @@ def _wait_for_process(self):
try:
self._process_exitcode = self._process.wait()
self._process_output = self._read_process(-1) # -1 is read all
- except Exception:
- self.log.raiseException("_wait_for_process: problem during wait exitcode %s output %s" %
- (self._process_exitcode, self._process_output))
+ except Exception as exc:
+ msg = (
+ f"_wait_for_process: problem during wait exitcode {self._process_exitcode} "
+ f"output {self._process_output}")
+ self.log.exception(msg)
+ raise OSError(msg) from exc
def _cleanup_process(self):
"""Cleanup any leftovers from the process"""
@@ -702,22 +718,27 @@ def _make_popen_named_args(self, others=None):
if os.path.isfile(self.filename):
self.log.warning("_make_popen_named_args: going to overwrite existing file %s", self.filename)
elif os.path.isdir(self.filename):
- self.log.raiseException(("_make_popen_named_args: writing to filename %s impossible. "
- "Path exists and is a directory.") % self.filename)
+ msg = (f"_make_popen_named_args: writing to filename {self.filename} impossible. "
+ "Path exists and is a directory.")
+ self.log.error(msg)
+ raise ValueError(msg)
else:
- self.log.raiseException("_make_popen_named_args: path exists and is not a file or directory %s" %
- self.filename)
+ msg = f"_make_popen_named_args: path exists and is not a file or directory {self.filename}"
+ self.log.error(msg)
+ raise ValueError(msg)
else:
dirname = os.path.dirname(self.filename)
if dirname and not os.path.isdir(dirname):
try:
os.makedirs(dirname)
except OSError:
- self.log.raiseException(("_make_popen_named_args: dirname %s for file %s does not exists. "
- "Creating it failed.") % (dirname, self.filename))
+ msg = (f"_make_popen_named_args: dirname {dirname} for file {self.filename} "
+ f"does not exists. Creating it failed.")
+ self.log.exception(msg)
+ raise OSError(msg) from OSError
try:
- self.filehandle = open(self.filename, 'w') # pylint: disable=consider-using-with
+ self.filehandle = open(self.filename, 'w', encoding='utf8') # pylint: disable=consider-using-with
except OSError:
self.log.raiseException(f"_make_popen_named_args: failed to open filehandle for file {self.filename}")
@@ -829,7 +850,7 @@ def _parse_qa(self, qa, qa_reg, no_qa):
def escape_special(string):
specials = r'.*+?(){}[]|\$^'
- return re.sub(r"([%s])" % ''.join([r'\%s' % x for x in specials]), r"\\\1", string)
+ return re.sub(r"([%s])" % ''.join([rf'\{x}' for x in specials]), r"\\\1", string)
SPLIT = '[\\s\n]+'
REG_SPLIT = re.compile(r"" + SPLIT)
@@ -951,54 +972,36 @@ def _loop_process_output(self, output):
class RunNoShellQA(RunNoShell, RunQA):
"""Question/Answer processing"""
- pass
-
class RunAsyncLoop(RunLoop, RunAsync):
"""Async read in loop"""
- pass
class RunNoShellAsyncLoop(RunNoShellLoop, RunNoShellAsync):
"""Async read in loop"""
- pass
-
class RunAsyncLoopLog(RunLoopLog, RunAsync):
"""Async read, log to logger"""
- pass
-
class RunNoShellAsyncLoopLog(RunNoShellLoopLog, RunNoShellAsync):
"""Async read, log to logger"""
- pass
class RunQALog(RunLoopLog, RunQA):
"""Async loop QA with LoopLog"""
- pass
-
class RunNoShellQALog(RunNoShellLoopLog, RunNoShellQA):
"""Async loop QA with LoopLog"""
- pass
-
class RunQAStdout(RunLoopStdout, RunQA):
"""Async loop QA with LoopLogStdout"""
- pass
-
class RunNoShellQAStdout(RunNoShellLoopStdout, RunNoShellQA):
"""Async loop QA with LoopLogStdout"""
- pass
-
class RunAsyncLoopStdout(RunLoopStdout, RunAsync):
"""Async read, flush to stdout"""
- pass
class RunNoShellAsyncLoopStdout(RunNoShellLoopStdout, RunNoShellAsync):
"""Async read, flush to stdout"""
- pass
# convenient names
diff --git a/setup.py b/setup.py
index d4b0b0ba..8c5bf515 100755
--- a/setup.py
+++ b/setup.py
@@ -33,12 +33,12 @@
"""
import vsc.install.shared_setup as shared_setup
-from vsc.install.shared_setup import ag, kh, jt, sdw
+from vsc.install.shared_setup import ag, kh, jt, sdw, wdp
PACKAGE = {
- 'version': '3.5.9',
+ 'version': '3.6.1',
'author': [sdw, jt, ag, kh],
- 'maintainer': [sdw, jt, ag, kh],
+ 'maintainer': [sdw, jt, ag, kh, wdp],
'install_requires': [
'vsc-install >= 0.17.19',
],
diff --git a/test/py2vs3.py b/test/py2vs3.py
deleted file mode 100644
index e1cb1095..00000000
--- a/test/py2vs3.py
+++ /dev/null
@@ -1,165 +0,0 @@
-#
-# Copyright 2020-2023 Ghent University
-#
-# This file is part of vsc-base,
-# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
-# with support of Ghent University (http://ugent.be/hpc),
-# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
-# the Flemish Research Foundation (FWO) (http://www.fwo.be/en)
-# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
-#
-# https://github.com/hpcugent/vsc-base
-#
-# vsc-base is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Library General Public License as
-# published by the Free Software Foundation, either version 2 of
-# the License, or (at your option) any later version.
-#
-# vsc-base is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Library General Public License for more details.
-#
-# You should have received a copy of the GNU Library General Public License
-# along with vsc-base. If not, see .
-#
-"""
-Tests for the vsc.utils.py2vs3 module.
-
-@author: Kenneth Hoste (Ghent University)
-"""
-import os
-import sys
-
-from vsc.utils.py2vs3 import ensure_ascii_string, is_py_ver, is_py2, is_py3, is_string, pickle, TemporaryDirectory
-from vsc.install.testing import TestCase
-
-
-class TestPy2vs3(TestCase):
- """Test for vsc.utils.py2vs3 module."""
-
- def test_is_py_ver(self):
- """Tests for is_py_ver, is_py2, is_py3 functions."""
-
- maj_ver = sys.version_info[0]
- min_ver = sys.version_info[1]
-
- if maj_ver >= 3:
- self.assertFalse(is_py2())
- self.assertFalse(is_py_ver(2))
- self.assertFalse(is_py_ver(2, 4))
- self.assertFalse(is_py_ver(2, min_ver=6))
- self.assertTrue(is_py3())
- self.assertTrue(is_py_ver(3))
- self.assertTrue(is_py_ver(3, min_ver))
- self.assertTrue(is_py_ver(3, min_ver=min_ver))
- if min_ver >= 6:
- self.assertTrue(is_py_ver(3, 6))
- self.assertTrue(is_py_ver(3, min_ver=6))
- self.assertFalse(is_py_ver(3, 99))
- self.assertFalse(is_py_ver(3, min_ver=99))
- else:
- # must be Python 2.6 or more recent Python 2 nowdays
- self.assertTrue(is_py2())
- self.assertTrue(is_py_ver(2))
- self.assertTrue(is_py_ver(2, 4))
- self.assertTrue(is_py_ver(2, min_ver=6))
- self.assertFalse(is_py3())
- self.assertFalse(is_py_ver(3))
- self.assertFalse(is_py_ver(3, 6))
- self.assertFalse(is_py_ver(3, min_ver=6))
-
- def test_is_string(self):
- """Tests for is_string function."""
- for item in ['foo', 'foo', "hello world", """foo\nbar""", '']:
- self.assertTrue(is_string(item))
-
- for item in [1, None, ['foo'], ('foo',), {'foo': 'bar'}]:
- self.assertFalse(is_string(item))
-
- if is_py3():
- self.assertFalse(is_string(b'bytes_are_not_a_string'))
- else:
- # in Python 2, b'foo' is really just a regular string
- self.assertTrue(is_string(b'foo'))
-
- def test_picke(self):
- """Tests for pickle module provided by py2vs3."""
-
- test_pickle_file = os.path.join(self.tmpdir, 'test.pickle')
-
- test_dict = {1: 'one', 2: 'two'}
-
- fp = open(test_pickle_file, 'wb')
- pickle.dump(test_dict, fp)
- fp.close()
-
- self.assertTrue(os.path.exists(test_pickle_file))
-
- fp = open(test_pickle_file, 'rb')
- loaded_pickle = pickle.load(fp)
- fp.close()
-
- self.assertEqual(test_dict, loaded_pickle)
-
- def test_ensure_ascii_string(self):
- """Tests for ensure_ascii_string function."""
-
- unicode_txt = 'this -> ยข <- is unicode'
-
- test_cases = [
- ('', ''),
- ('foo', 'foo'),
- ([1, 2, 3], "[1, 2, 3]"),
- (['1', '2', '3'], "['1', '2', '3']"),
- ({'one': 1}, "{'one': 1}"),
- # in both Python 2 & 3, Unicode characters that are part of a non-string value get escaped
- ([unicode_txt], "['this -> \\xc2\\xa2 <- is unicode']"),
- ({'foo': unicode_txt}, "{'foo': 'this -> \\xc2\\xa2 <- is unicode'}"),
- ]
- if is_py2():
- test_cases.extend([
- # Unicode characters from regular strings are stripped out in Python 2
- (unicode_txt, 'this -> <- is unicode'),
- # also test with unicode-type values (only exists in Python 2)
- (unicode('foo'), 'foo'),
- (unicode(unicode_txt, encoding='utf-8'), 'this -> \\xa2 <- is unicode'),
- ])
- else:
- # in Python 3, Unicode characters are replaced by backslashed escape sequences in string values
- expected_unicode_out = 'this -> \\xc2\\xa2 <- is unicode'
- test_cases.extend([
- (unicode_txt, expected_unicode_out),
- # also test with bytestring-type values (only exists in Python 3)
- (bytes('foo', encoding='utf-8'), 'foo'),
- (bytes(unicode_txt, encoding='utf-8'), expected_unicode_out),
- ])
-
- for inp, out in test_cases:
- res = ensure_ascii_string(inp)
- self.assertTrue(is_string(res))
- self.assertEqual(res, out)
-
- def test_urllib_imports(self):
- """Test importing urllib* stuff from py2vs3."""
- from vsc.utils.py2vs3 import HTTPError, HTTPSHandler, Request, build_opener, unquote, urlencode, urlopen
-
- def test_temporary_directory(self):
- """Test the class TemporaryDirectory."""
- with TemporaryDirectory() as temp_dir:
- path = temp_dir
- self.assertTrue(os.path.exists(temp_dir), 'Directory created by TemporaryDirectory should work')
- self.assertTrue(os.path.isdir(temp_dir), 'Directory created by TemporaryDirectory should be a directory')
- self.assertFalse(os.path.exists(temp_dir), 'Directory created by TemporaryDirectory should cleanup automagically')
-
- def test_os_exceptions(self):
- """Test importing urllib* stuff from py2vs3."""
- from vsc.utils.py2vs3 import FileNotFoundErrorExc, FileExistsErrorExc
-
- self.assertRaises(FileNotFoundErrorExc, lambda: os.unlink(os.path.join(self.tmpdir, 'notafile')))
-
- afile = os.path.join(self.tmpdir, 'afile')
- with open(afile, 'w') as f:
- f.write("abc")
-
- self.assertRaises(FileExistsErrorExc, lambda: os.open(afile, os.O_CREAT | os.O_WRONLY | os.O_EXCL))
diff --git a/test/rest.py b/test/rest.py
index 398b76c3..78a84873 100644
--- a/test/rest.py
+++ b/test/rest.py
@@ -134,3 +134,40 @@ def test_censor_request(self):
nondict_payload = [payload, 'more_payload']
payload_censored = client.censor_request(['password'], nondict_payload)
self.assertEqual(payload_censored, nondict_payload)
+
+
+class RestClientNoDecodeTest(TestCase):
+ """ small test for The RestClient
+ This should not be to much, since there is an hourly limit of requests for the github api
+ """
+
+ def setUp(self):
+ """setup"""
+ super().setUp()
+ self.client = RestClient('https://api.github.com', username=GITHUB_LOGIN, token=GITHUB_TOKEN, decode=False)
+
+ def test_client(self):
+ """Do a test api call"""
+ if GITHUB_TOKEN is None:
+ print("Skipping test_client, since no GitHub token is available")
+ return
+
+ status, body = self.client.repos[GITHUB_USER][GITHUB_REPO].contents.a_directory['a_file.txt'].get()
+ self.assertEqual(status, 200)
+ # dGhpcyBpcyBhIGxpbmUgb2YgdGV4dAo= == 'this is a line of text' in base64 encoding
+ self.assertEqual(body['content'].strip(), "dGhpcyBpcyBhIGxpbmUgb2YgdGV4dAo=")
+
+ status, body = self.client.repos['hpcugent']['easybuild-framework'].pulls[1].get()
+ self.assertEqual(status, 200)
+ self.assertEqual(body['merge_commit_sha'], 'fba3e13815f3d2a9dfbd2f89f1cf678dd58bb1f1')
+
+ def test_get_method(self):
+ """A quick test of a GET to the github API"""
+
+ status, body = self.client.users['hpcugent'].get()
+ self.assertEqual(status, 200)
+ self.assertTrue('"login":"hpcugent"' in body)
+ self.assertTrue('"id":1515263' in body)
+
+
+