Merge pull request #338 from datafold/dec6

Small refactor in __main__ and tests; Better test coverage for CLI

Author: erezsh

data_diff/__main__.py

@@ -307,18 +307,22 @@ def _main(
     else:
         db2 = connect(database2, threads2 or threads)
 
-    now: datetime = db1.query(current_timestamp(), datetime)
-    now = now.replace(tzinfo=None)
-    try:
-        options = dict(
-            min_update=max_age and parse_time_before(now, max_age),
-            max_update=min_age and parse_time_before(now, min_age),
-            case_sensitive=case_sensitive,
-            where=where,
-        )
-    except ParseError as e:
-        logging.error(f"Error while parsing age expression: {e}")
-        return
+    options = dict(
+        case_sensitive=case_sensitive,
+        where=where,
+    )
+
+    if min_age or max_age:
+        now: datetime = db1.query(current_timestamp(), datetime)
+        now = now.replace(tzinfo=None)
+        try:
+            if max_age:
+                options["min_update"] = parse_time_before(now, max_age)
+            if min_age:
+                options["max_update"] = parse_time_before(now, min_age)
+        except ParseError as e:
+            logging.error(f"Error while parsing age expression: {e}")
+            return
 
     dbs = db1, db2
 

data_diff/sqeleton/databases/base.py

@@ -124,7 +124,7 @@ class BaseDialect(AbstractDialect):
     SUPPORTS_INDEXES = False
     TYPE_CLASSES: Dict[str, type] = {}
 
-    PLACEHOLDER_TABLE = None        # Used for Oracle
+    PLACEHOLDER_TABLE = None  # Used for Oracle
 
     def offset_limit(self, offset: Optional[int] = None, limit: Optional[int] = None):
         if offset:

data_diff/sqeleton/databases/duckdb.py

@@ -114,6 +114,7 @@ def set_timezone_to_utc(self) -> str:
     def current_timestamp(self) -> str:
         return "current_timestamp"
 
+
 class DuckDB(Database):
     dialect = Dialect()
     SUPPORTS_UNIQUE_CONSTAINT = False  # Temporary, until we implement it

data_diff/sqeleton/databases/presto.py

@@ -140,6 +140,7 @@ def set_timezone_to_utc(self) -> str:
     def current_timestamp(self) -> str:
         return "current_timestamp"
 
+
 class Presto(Database):
     dialect = Dialect()
     CONNECT_URI_HELP = "presto://<user>@<host>/<catalog>/<schema>"

data_diff/sqeleton/queries/__init__.py

@@ -18,7 +18,7 @@
     or_,
     leftjoin,
     rightjoin,
-    current_timestamp
+    current_timestamp,
 )
 from .ast_classes import Expr, ExprNode, Select, Count, BinOp, Explain, In, Code, Column
 from .extras import Checksum, NormalizeAsString, ApplyFuncAndNormalizeAsString

data_diff/sqeleton/queries/ast_classes.py

@@ -604,7 +604,6 @@ def compile(self, parent_c: Compiler) -> str:
         elif c.dialect.PLACEHOLDER_TABLE:
             select += f" FROM {c.dialect.PLACEHOLDER_TABLE}"
 
-
         if self.where_exprs:
             select += " WHERE " + " AND ".join(map(c.compile, self.where_exprs))
 

pyproject.toml

@@ -39,7 +39,6 @@ clickhouse-driver = {version="*", optional=true}
 duckdb = {version="^0.6.0", optional=true}
 
 [tool.poetry.dev-dependencies]
-arrow = "^1.2.3"
 parameterized = "*"
 unittest-parallel = "*"
 preql = "^0.2.19"

tests/sqeleton/test_database.py

@@ -68,7 +68,6 @@ def test_table_list(self):
 
 @test_each_database
 class TestQueries(unittest.TestCase):
-
     def test_current_timestamp(self):
         db = get_conn(self.db_cls)
         res = db.query(current_timestamp(), datetime)

tests/test_api.py

@@ -1,5 +1,4 @@
-import arrow
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from data_diff import diff_tables, connect_to_table, Algorithm
 from data_diff.databases import MySQL
@@ -17,20 +16,20 @@ def setUp(self) -> None:
 
         self.conn = self.connection
 
-        self.now = now = arrow.get()
+        self.now = now = datetime.now()
 
         rows = [
             (now, "now"),
-            (self.now.shift(seconds=-10), "a"),
-            (self.now.shift(seconds=-7), "b"),
-            (self.now.shift(seconds=-6), "c"),
+            (self.now - timedelta(seconds=10), "a"),
+            (self.now - timedelta(seconds=7), "b"),
+            (self.now - timedelta(seconds=6), "c"),
         ]
 
         self.conn.query(
             [
-                self.src_table.insert_rows((i, ts.datetime, s) for i, (ts, s) in enumerate(rows)),
+                self.src_table.insert_rows((i, ts, s) for i, (ts, s) in enumerate(rows)),
                 self.dst_table.create(self.src_table),
-                self.src_table.insert_row(len(rows), self.now.shift(seconds=-3).datetime, "3 seconds ago"),
+                self.src_table.insert_row(len(rows), self.now - timedelta(seconds=3), "3 seconds ago"),
                 commit,
             ]
         )

tests/test_cli.py

@@ -3,10 +3,10 @@
 import sys
 from datetime import datetime, timedelta
 
-from data_diff.databases import MySQL
-from data_diff.sqeleton.queries import commit
+from data_diff.sqeleton.queries import commit, current_timestamp
 
-from .common import TEST_MYSQL_CONN_STRING, DiffTestCase
+from .common import DiffTestCase, CONN_STRINGS
+from .test_diff_tables import test_each_database
 
 
 def run_datadiff_cli(*args):
@@ -20,14 +20,14 @@ def run_datadiff_cli(*args):
     return stdout.splitlines()
 
 
+@test_each_database
 class TestCLI(DiffTestCase):
-    db_cls = MySQL
     src_schema = {"id": int, "datetime": datetime, "text_comment": str}
 
     def setUp(self) -> None:
         super().setUp()
 
-        now = self.connection.query("select now()", datetime)
+        now = self.connection.query(current_timestamp(), datetime)
 
         rows = [
             (now, "now"),
@@ -46,16 +46,16 @@ def setUp(self) -> None:
         )
 
     def test_basic(self):
-        diff = run_datadiff_cli(
-            TEST_MYSQL_CONN_STRING, self.table_src_name, TEST_MYSQL_CONN_STRING, self.table_dst_name
-        )
+        conn_str = CONN_STRINGS[self.db_cls]
+        diff = run_datadiff_cli(conn_str, self.table_src_name, conn_str, self.table_dst_name)
         assert len(diff) == 1
 
     def test_options(self):
+        conn_str = CONN_STRINGS[self.db_cls]
         diff = run_datadiff_cli(
-            TEST_MYSQL_CONN_STRING,
+            conn_str,
             self.table_src_name,
-            TEST_MYSQL_CONN_STRING,
+            conn_str,
             self.table_dst_name,
             "--bisection-factor",
             "16",
@@ -68,4 +68,4 @@ def test_options(self):
             "--max-age",
             "1h",
         )
-        assert len(diff) == 1
+        assert len(diff) == 1, diff