Skip to content
This repository was archived by the owner on May 17, 2024. It is now read-only.

Fix type handling of key_columns in toml #653

Merged
merged 4 commits into from
Aug 2, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions data_diff/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@
import toml


_ARRAY_FIELDS = (
"key_columns",
"columns",
)


class ConfigParseError(Exception):
pass

Expand Down Expand Up @@ -38,6 +44,11 @@ def _apply_config(config: Dict[str, Any], run_name: str, kw: Dict[str, Any]):
for index in "12":
run_args[index] = {attr: kw.pop(f"{attr}{index}") for attr in ("database", "table")}

# Make sure array fields are decoded as list, since array fields in toml are decoded as list, but TableSegment object requires tuple type.
for field in _ARRAY_FIELDS:
if isinstance(run_args.get(field), list):
run_args[field] = tuple(run_args[field])

# Process databases + tables
for index in "12":
try:
Expand Down
4 changes: 4 additions & 0 deletions tests/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ def test_basic(self):

[run.default]
update_column = "timestamp"
key_columns = ["id"]
columns = ["name", "age"]
verbose = true
threads = 2

Expand All @@ -39,6 +41,8 @@ def test_basic(self):
assert res["table2"] == "rating_del1"
assert res["threads1"] == 11
assert res["threads2"] == 22
assert res["key_columns"] == ("id",)
assert res["columns"] == ("name", "age")

res = apply_config_from_string(config, "pg_pg", {"update_column": "foo", "table2": "bar"})
assert res["update_column"] == "foo"
Expand Down