Skip to content

Commit 09683af

Browse files
committed
Fix for more than 50 fields in Postgres.
Postgres does not allow functions that have more that have more than 100 arguments. When using the concat function, this limits comparisons to less than 50 fields. Using || for concat like the Oracle variant fixes this.
1 parent 2697d3a commit 09683af

File tree

2 files changed

+49
-0
lines changed

2 files changed

+49
-0
lines changed

data_diff/sqeleton/databases/postgresql.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from typing import List
12
from ..abcs.database_types import (
23
DbPath,
34
JSON,
@@ -92,6 +93,10 @@ def quote(self, s: str):
9293
def to_string(self, s: str):
9394
return f"{s}::varchar"
9495

96+
def concat(self, items: List[str]) -> str:
97+
joined_exprs = " || ".join(items)
98+
return f"({joined_exprs})"
99+
95100
def _convert_db_precision_to_digits(self, p: int) -> int:
96101
# Subtracting 2 due to wierd precision issues in PostgreSQL
97102
return super()._convert_db_precision_to_digits(p) - 2

tests/test_postgresql.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,3 +70,47 @@ def test_uuid(self):
7070
self.connection.query(self.table_src.drop(True))
7171
self.connection.query(self.table_dst.drop(True))
7272
mysql_conn.query(self.table_dst.drop(True))
73+
74+
75+
class Test100Fields(unittest.TestCase):
76+
def setUp(self) -> None:
77+
self.connection = get_conn(db.PostgreSQL)
78+
79+
table_suffix = random_table_suffix()
80+
81+
self.table_src_name = f"src{table_suffix}"
82+
self.table_dst_name = f"dst{table_suffix}"
83+
84+
self.table_src = table(self.table_src_name)
85+
self.table_dst = table(self.table_dst_name)
86+
87+
def test_100_fields(self):
88+
self.connection.query('CREATE EXTENSION IF NOT EXISTS "uuid-ossp";', None)
89+
90+
columns = [f"col{i}" for i in range(100)]
91+
fields = " ,".join(f'"{field}" TEXT' for field in columns)
92+
93+
queries = [
94+
self.table_src.drop(True),
95+
self.table_dst.drop(True),
96+
f"CREATE TABLE {self.table_src_name} (id uuid DEFAULT uuid_generate_v4 (), {fields})",
97+
commit,
98+
self.table_src.insert_rows([[f"{x * y}" for x in range(100)] for y in range(10)], columns=columns),
99+
commit,
100+
self.table_dst.create(self.table_src),
101+
commit,
102+
self.table_src.insert_rows([[1 for x in range(100)]], columns=columns),
103+
commit,
104+
]
105+
106+
for query in queries:
107+
self.connection.query(query)
108+
109+
a = TableSegment(self.connection, self.table_src.path, ("id",), extra_columns=tuple(columns))
110+
b = TableSegment(self.connection, self.table_dst.path, ("id",), extra_columns=tuple(columns))
111+
112+
differ = HashDiffer()
113+
diff = list(differ.diff_tables(a, b))
114+
id_ = diff[0][1][0]
115+
result = (id_,) + tuple("1" for x in range(100))
116+
self.assertEqual(diff, [("-", result)])

0 commit comments

Comments
 (0)