Merge branch 'master' into refactoring/#67-publish-to-pypi

.github/workflows/main.yml

@@ -13,22 +13,30 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Get ODBC driver
+        id: get-odbc-driver
         run: |
-          bash .github/workflows/scripts/download_odbc_driver.sh "$(pwd)/.."
+          FILE=$(bash .github/workflows/scripts/download_odbc_driver.sh "$(pwd)/..")
+          echo so-file="$FILE" >> "$GITHUB_OUTPUT"
       - name: Get EXAPLUS
-        run: |
-          bash .github/workflows/scripts/download_exaplus.sh "$(pwd)/.."
+        run: bash .github/workflows/scripts/download_exaplus.sh "$(pwd)/.."
       - name: Setup Python & Poetry Environment
         uses: exasol/python-toolbox/.github/actions/python-environment@0.12.0
         with:
           python-version: ${{ matrix.python-version }}
           poetry-version: '1.8.2'
       - name: Spawn EXASOL environemnt
         run: |
-          poetry run python3 -m exasol_integration_test_docker_environment.main spawn-test-environment --environment-name test --database-port-forward 8888 --bucketfs-port-forward 6666
+          poetry run python3 -m exasol_integration_test_docker_environment.main \
+              spawn-test-environment \
+              --environment-name test \
+              --database-port-forward 8888 \
+              --bucketfs-port-forward 6666
           sleep 120 # wait for SLC to be unpacked
       - name: Run all tests
-        run: poetry run scripts/test/run_locally.sh --server "localhost:8888" --odbc-driver="../downloads/ODBC/lib/linux/x86_64/libexaodbc-uo2214lv2.so"
+        run: |
+          poetry run scripts/test/run_locally.sh \
+             --server "localhost:8888" \
+             --odbc-driver="${{ steps.get-odbc-driver.outputs.so-file }}"
         env:
           EXAPLUS: ../downloads/EXAPLUS
           TEST_DOCKER_NETWORK_NAME: db_network_test

.github/workflows/release_droid_upload_github_release_assets.yml

@@ -34,22 +34,29 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Get ODBC driver
+        id: get-odbc-driver
         run: |
-          bash .github/workflows/scripts/download_odbc_driver.sh "$(pwd)/.."
+          FILE=$(bash .github/workflows/scripts/download_odbc_driver.sh "$(pwd)/..")
+          echo so-file="$FILE" >> "$GITHUB_OUTPUT"
       - name: Get EXAPLUS
-        run: |
-          bash .github/workflows/scripts/download_exaplus.sh "$(pwd)/.."
+        run: bash .github/workflows/scripts/download_exaplus.sh "$(pwd)/.."
       - name: Setup Python & Poetry Environment
         uses: exasol/python-toolbox/.github/actions/python-environment@0.12.0
         with:
           python-version: ${{ matrix.python-version }}
           poetry-version: '1.8.2'
       - name: Spawn EXASOL environemnt
         run: |
-          poetry run python3 -m exasol_integration_test_docker_environment.main spawn-test-environment --environment-name test --database-port-forward 8888 --bucketfs-port-forward 6666
+          poetry run python3 -m exasol_integration_test_docker_environment.main \
+              spawn-test-environment \
+              --environment-name test \
+              --database-port-forward 8888 \
+              --bucketfs-port-forward 6666
           sleep 120 # wait for SLC to be unpacked
       - name: Run all tests
-        run: poetry run scripts/test/run_locally.sh --server "localhost:8888" --odbc-driver="../downloads/ODBC/lib/linux/x86_64/libexaodbc-uo2214lv2.so"
+        run: |
+          poetry run scripts/test/run_locally.sh --server "localhost:8888" \
+          --odbc-driver="${{ steps.get-odbc-driver.outputs.so-file }}"
         env:
           EXAPLUS: ../downloads/EXAPLUS
           TEST_DOCKER_NETWORK_NAME: db_network_test

.github/workflows/scripts/download_exaplus.sh

@@ -2,5 +2,7 @@
 
 set -euo pipefail
 
-mkdir -p "$1/downloads/EXAPLUS"
-curl -s https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/EXAplus-7.0.11.tar.gz | tar -C "$1/downloads/EXAPLUS" --strip-components 1 -zxf -
+DIR="$1/downloads/EXAPLUS"
+mkdir -p "$DIR"
+curl -s https://x-up.s3.amazonaws.com/7.x/24.1.1/EXAplus-24.1.1.tar.gz \
+    | tar -C "$DIR" --strip-components 2 -zxf -

.github/workflows/scripts/download_odbc_driver.sh

@@ -2,5 +2,8 @@
 
 set -euo pipefail
 
-mkdir -p "$1/downloads/ODBC"
-curl -s https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/EXASOL_ODBC-7.0.11.tar.gz | tar -C "$1/downloads/ODBC" --strip-components 1 -zxf -
+DIR="$1/downloads/ODBC"
+mkdir -p "$DIR"
+curl -s https://x-up.s3.amazonaws.com/7.x/24.1.1/Exasol_ODBC-24.1.1-Linux_x86_64.tar.gz \
+    | tar -C "$DIR" --strip-components 2 -zxf -
+find "$DIR" -name libexaodbc\*.so

.gitignore

@@ -7,4 +7,4 @@ dist
 odbc.ini
 
 # Emacs
-TAGS
+TAGS

doc/changes/changes_0.6.0.md

@@ -4,8 +4,27 @@ Code name: Configure TLS certificate validation
 
 ## Summary
 
+This release adds a CLI option controlling parameter `SSLCERTIFICATE` in file `odbc.ini`.
+
 Starting with version `0.6.0`, EPTF is also available on pypi.
 
-## Refactoring
+Additionally, the release fixes vulnerabilities by updating dependencies:
+* CVE-2024-35195 in dependency `requests` in versions < `2.32.0` caused by requests `Session` object not verifying requests after making first request with `verify=False`
+* CVE-2024-37891 in transitive dependency via `boto3` to `urllib3` in versions < `2.2.2` caused by proxy-authorization request header not to be stripped during cross-origin redirects as no update of notebook-connector is available, yet.
+* GHSA-w235-7p84-xx57 in transitive dependency via `luigi` to `tornado` in versions < `6.4.1` enabling CRLF injection in `CurlAsyncHTTPClient` headers.
+* GHSA-753j-mpmx-qq6g in transitive dependency via `luigi` to `tornado` in versions < `6.4.1` due to inconsistent interpretation of HTTP Requests ('HTTP Request/Response Smuggling')
+
+However, the release ignores the following vulnerabilities
+* GHSA-753j-mpmx-qq6g in dependency `configobj` in versions &le; `5.0.8` being ReDoS exploitable by developers using values in a server-side configuration file as SLCT is used only client side and a patched version is not available, yet.
+
+## Security Fixes
+
+* #70: Fixed vulnerabilities by updating dependencies.
+
+## Features
+
+* #66: Added CLI option controlling parameter `SSLCERTIFICATE` in file `odbc.ini`.
+
+## Refactorings
 
 * #67: Enabled publication on pypi

exasol_python_test_framework/exatest/__init__.py

@@ -247,6 +247,7 @@ def _parse_opts(self):
                 logdir='.',
                 loglevel='warning',
                 odbc_log='off',
+                tls_cert='unverified',
                 connect=None,
                 driver=None,
                 debugger=False,
@@ -282,8 +283,15 @@ def prepare_hook(self):
 
     def _main(self):
         self.opts.log.info('prepare for tests')
-        self._client_setup.prepare_odbc_init(self.opts.logdir, self.opts.server, self.opts.driver,
-                                             self.opts.user, self.opts.password, self.opts.odbc_log)
+        self._client_setup.prepare_odbc_init(
+            self.opts.logdir,
+            self.opts.server,
+            self.opts.driver,
+            self.opts.user,
+            self.opts.password,
+            self.opts.odbc_log,
+            self.opts.tls_cert,
+        )
         prepare_ok = self.prepare_hook()
         self.opts.log.info('starting tests')
         testprogram = unittest.main(

exasol_python_test_framework/exatest/clients/client_setup.py

@@ -2,16 +2,33 @@
 import os
 import socket
 
+from inspect import cleandoc
+from pathlib import Path
+
+
+ENV_VAR = "ODBCINI"
+
+LOG_LEVELS = {
+    "off": None,
+    "error": "ON ERROR ONLY",
+    "normal": "DEFAULT",
+    "verbose": "VERBOSE",
+}
+
+TLS_CERT_OPTIONS = {
+    "verify": None,
+    "unverified": "SSL_VERIFY_NONE",
+}
+
 
 def validate_driver_path(arg: str):
     if not arg:
         raise argparse.ArgumentTypeError("The driver path is empty!")
-    import os.path
-    absolute = os.path.abspath(os.path.expanduser(arg))
-    if not os.path.exists(absolute):
-        raise argparse.ArgumentTypeError(f"The driver path '{absolute}' does not exist!")
+    path = Path(arg).expanduser().absolute()
+    if not path.exists():
+        raise argparse.ArgumentTypeError(f'The driver path "{path}" does not exist!')
     else:
-        return absolute
+        return path
 
 
 def validate_server(arg: str):
@@ -25,47 +42,83 @@ class ClientSetup(object):
     def __init__(self):
         self.dsn = "exatest"
 
-    def odbc_arguments(self, arg_parser):
-        arg_parser.add_argument('--server',
-                          help='connection string',
-                          required=True,
-                          type=lambda x: validate_server(x))
-        arg_parser.add_argument('--user', help='connection user', nargs="?", type=str, default="sys")
-        arg_parser.add_argument('--password', help='connection password', nargs="?", type=str, default="exasol")
-        arg_parser.add_argument('--driver',
-                          help='path to ODBC driver',
-                          required=True,
-                          type=lambda x: validate_driver_path(x))
-        odbcloglevel = ('off', 'error', 'normal', 'verbose')
-        arg_parser.add_argument('--odbc-log', choices=odbcloglevel,
-                          help='activate ODBC driver log (default: %(default)s)')
-
-        return arg_parser
-
-    def _write_odbcini(self, log_path, server, driver, user, password, odbc_log):
-        name = os.path.realpath(os.path.join(log_path, 'odbc.ini'))
-        server=socket.getfqdn(server)
-        with open(name, 'w') as tmp:
-            tmp.write('[ODBC Data Sources]\n')
-            tmp.write('%s=EXASolution\n'%self.dsn)
-            tmp.write('\n')
-            tmp.write('[%s]\n'%self.dsn)
-            tmp.write('Driver = %s\n' % driver)
-            tmp.write('EXAHOST = %s\n' % server)
-            tmp.write('EXAUID = %s\n' % user)
-            tmp.write('EXAPWD = %s\n' % password)
-            tmp.write('CONNECTIONLCCTYPE = en_US.UTF-8\n')      # TODO Maybe make this optional
-            tmp.write('CONNECTIONLCNUMERIC = en_US.UTF-8\n')
-            tmp.write('SSLCERTIFICATE = SSL_VERIFY_NONE\n')
-            if odbc_log != 'off':
-                tmp.write('EXALOGFILE = %s/exaodbc.log\n' % log_path.logdir)
-                tmp.write('LOGMODE = %s\n' % {
-                        'error': 'ON ERROR ONLY',
-                        'normal': 'DEFAULT',
-                        'verbose': 'VERBOSE',
-                        }[odbc_log])
-        return name
-
-    def prepare_odbc_init(self, log_path, server, driver, user, password, odbc_log):
-        name = self._write_odbcini(log_path, server, driver, user, password, odbc_log)
-        os.environ['ODBCINI'] = name
+    def odbc_arguments(self, parser):
+        parser.add_argument(
+            "--server",
+            help="connection string",
+            required=True,
+            type=validate_server)
+        parser.add_argument(
+            "--user", help="connection user", nargs="?", type=str, default="sys")
+        parser.add_argument(
+            "--password", help="connection password", nargs="?", type=str, default="exasol")
+        parser.add_argument(
+            "--driver",
+            help="path to ODBC driver",
+            required=True,
+            type=validate_driver_path)
+        parser.add_argument(
+            "--odbc-log", choices=LOG_LEVELS,
+            help='activate ODBC driver log (default: %(default)s)',
+        )
+        parser.add_argument(
+            "--tls-cert", choices=TLS_CERT_OPTIONS,
+            help='TLS certificate verification (default: %(default)s)',
+        )
+        return parser
+
+    def _write_odbcini(self, log_path, server, driver,
+                       user, password, odbc_log,
+                       tls_cert) -> str:
+        def cleandoc_nl(string):
+            return cleandoc(string) + "\n"
+
+        path = Path(log_path).joinpath("odbc.ini").resolve()
+        server = socket.getfqdn(server)
+        with path.open('w') as file:
+            file.write(cleandoc_nl(
+                    # TODO: Maybe make CONNECTIONLCCTYPE optional
+                    f"""
+                    [ODBC Data Sources]
+                    {self.dsn}=EXASolution
+
+                    [{self.dsn}]
+                    Driver = {driver}
+                    EXAHOST = {server}
+                    EXAUID = {user}
+                    EXAPWD = {password}
+                    CONNECTIONLCCTYPE = en_US.UTF-8
+                    CONNECTIONLCNUMERIC = en_US.UTF-8
+                    """
+            ))
+            if tls_cert != "verify":
+                file.write(f"SSLCERTIFICATE = {TLS_CERT_OPTIONS[tls_cert]}\n")
+            if odbc_log != "off":
+                file.write(cleandoc_nl(
+                        f"""
+                        EXALOGFILE = {log_path}/exaodbc.log
+                        LOG_LEVELS = {LOG_LEVELS[odbc_log]}
+                        """
+                        ))
+        return str(path)
+
+    def prepare_odbc_init(
+            self,
+            log_path,
+            server,
+            driver,
+            user,
+            password,
+            odbc_log,
+            tls_cert = "unverified",
+    ):
+        path = self._write_odbcini(
+            log_path,
+            server,
+            driver,
+            user,
+            password,
+            odbc_log,
+            tls_cert,
+        )
+        os.environ[ENV_VAR] = path