From 6773cc0ce9173860c1f918e5c3537907bdefe3dd Mon Sep 17 00:00:00 2001 From: sgpeter1 Date: Tue, 23 Jul 2024 16:20:52 -0600 Subject: [PATCH] Adds --engine-option to `csvsql`. (#1256) feat(csvsql): adds --engine-option --- CHANGELOG.rst | 1 + csvkit/cli.py | 12 ++++++++++++ csvkit/utilities/csvsql.py | 9 ++++++--- csvkit/utilities/sql2csv.py | 15 +-------------- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9e4a640b..1ed528dc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,7 @@ Unreleased ---------- +- feat: :doc:`/scripts/csvsql` adds a :code:`--engine-option` option. - feat: :doc:`/scripts/sql2csv` adds a :code:`--execution-option` option. - feat: :doc:`/scripts/sql2csv` uses the ``stream_results=True`` execution option, by default, to not load all data into memory at once. diff --git a/csvkit/cli.py b/csvkit/cli.py index d35d7e4c..0dd299ab 100644 --- a/csvkit/cli.py +++ b/csvkit/cli.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import ast import argparse import bz2 import csv @@ -560,6 +561,17 @@ def parse_column_identifiers(ids, column_names, column_offset=1, excluded_column return [c for c in columns if c not in excludes] +def parse_list(pairs): + options = {} + for key, value in pairs: + try: + value = ast.literal_eval(value) + except ValueError: + pass + options[key] = value + return options + + # Adapted from https://github.com/pallets/click/blame/main/src/click/utils.py def _expand_args(args): out = [] diff --git a/csvkit/utilities/csvsql.py b/csvkit/utilities/csvsql.py index 9ac27327..b561c5b7 100644 --- a/csvkit/utilities/csvsql.py +++ b/csvkit/utilities/csvsql.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - import os.path import sys @@ -7,7 +6,7 @@ import agatesql # noqa: F401 from sqlalchemy import create_engine, dialects -from csvkit.cli import CSVKitUtility, isatty +from csvkit.cli import CSVKitUtility, isatty, parse_list try: import importlib_metadata @@ -33,6 +32,10 @@ def add_arguments(self): self.argparser.add_argument( '--db', dest='connection_string', help='If present, a SQLAlchemy connection string to use to directly execute generated SQL on a database.') + self.argparser.add_argument( + '--engine-option', dest='engine_option', nargs=2, action='append', default=[], + help="A keyword argument to SQLAlchemy's create_engine(), as a space-separated pair. " + "This option can be specified multiple times. For example: thick_mode True") self.argparser.add_argument( '--query', dest='queries', action='append', help='Execute one or more SQL queries delimited by ";" and output the result of the last query as CSV. ' @@ -137,7 +140,7 @@ def main(self): # Establish database validity before reading CSV files if self.args.connection_string: try: - engine = create_engine(self.args.connection_string) + engine = create_engine(self.args.connection_string, **parse_list(self.args.engine_option)) except ImportError as e: raise ImportError( "You don't appear to have the necessary database backend installed for connection string you're " diff --git a/csvkit/utilities/sql2csv.py b/csvkit/utilities/sql2csv.py index fc5f92cb..0ff19894 100644 --- a/csvkit/utilities/sql2csv.py +++ b/csvkit/utilities/sql2csv.py @@ -1,21 +1,8 @@ #!/usr/bin/env python -import ast - import agate from sqlalchemy import create_engine -from csvkit.cli import CSVKitUtility - - -def parse_list(pairs): - options = {} - for key, value in pairs: - try: - value = ast.literal_eval(value) - except ValueError: - pass - options[key] = value - return options +from csvkit.cli import CSVKitUtility, parse_list class SQL2CSV(CSVKitUtility):