added migration scripts

superset/migrations/versions/260bf0649a77_migrate_x_dateunit_in_time_range.py

@@ -0,0 +1,105 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""migrate [x dateunit] to [x dateunit ago/later]
+
+Revision ID: 260bf0649a77
+Revises: c878781977c6
+Create Date: 2021-01-23 16:25:14.496774
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '260bf0649a77'
+down_revision = 'c878781977c6'
+
+import json
+import re
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import Column, Integer, Text, or_
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.dialects.mysql.base import MySQLDialect
+from sqlalchemy.dialects.sqlite.base import SQLiteDialect
+
+from superset import db
+from superset.utils.date_parser import DateRangeMigration
+
+Base = declarative_base()
+
+
+class Slice(Base):
+    __tablename__ = "slices"
+
+    id = Column(Integer, primary_key=True)
+    slice_name = Column(Text)
+    params = Column(Text)
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = db.Session(bind=bind)
+    x_dateunit_in_since = DateRangeMigration.x_dateunit_in_since
+    x_dateunit_in_until = DateRangeMigration.x_dateunit_in_until
+
+    if isinstance(bind.dialect, SQLiteDialect):
+        to_lower = sa.func.LOWER
+        where_clause = or_(
+            sa.func.REGEXP(to_lower(Slice.params), x_dateunit_in_since),
+            sa.func.REGEXP(to_lower(Slice.params), x_dateunit_in_until),
+        )
+    elif isinstance(bind.dialect, MySQLDialect):
+        where_clause = or_(
+            sa.func.REGEXP_LIKE(Slice.params, x_dateunit_in_since, 'i'),
+            sa.func.REGEXP_LIKE(Slice.params, x_dateunit_in_until, 'i'),
+        )
+    else:
+        # default metadata is pg, so: isinstance(bind.dialect, PGDialect):
+        where_clause = or_(
+            Slice.params.op("~*")(x_dateunit_in_since),
+            Slice.params.op("~*")(x_dateunit_in_until),
+        )
+
+    slices = (
+        session
+        .query(Slice)
+        .filter(where_clause)
+        .all()
+    )
+
+    sep = " : "
+    pattern = DateRangeMigration.x_dateunit
+    for idx, slc in enumerate(slices):
+        print(f"Upgrading ({idx + 1}/{len(slices)}): {slc.slice_name}#{slc.id}")
+        params = json.loads(slc.params)
+        time_range = params['time_range']
+        if sep in time_range:
+            start, end = time_range.split(sep)
+            if re.match(pattern, start):
+                start = f"{start.strip()} ago"
+            if re.match(pattern, end):
+                end = f"{end.strip()} later"
+            params['time_range'] = f"{start}{sep}{end}"
+
+            slc.params = json.dumps(params, sort_keys=True, indent=4)
+            session.commit()
+
+    session.close()
+
+
+def downgrade():
+    pass

superset/utils/date_parser.py

@@ -82,28 +82,24 @@ def parse_human_datetime(human_readable: str) -> datetime:
             )
         )
 
-    error_msg = ValueError(
-        _(
-            "Couldn't parse date string [%{human_readable}s]",
-            human_readable=human_readable,
-        )
-    )
     try:
         dttm = parse(human_readable)
-    except Exception:  # pylint: disable=broad-except
-        try:
-            cal = parsedatetime.Calendar()
-            parsed_dttm, parsed_flags = cal.parseDT(human_readable)
-            # 0 == not parsed at all
-            if parsed_flags == 0:
-                raise error_msg
-            # when time is not extracted, we 'reset to midnight'
-            if parsed_flags & 2 == 0:
-                parsed_dttm = parsed_dttm.replace(hour=0, minute=0, second=0)
-            dttm = dttm_from_timetuple(parsed_dttm.utctimetuple())
-        except Exception as ex:
+    except (ValueError, OverflowError) as ex:
+        cal = parsedatetime.Calendar()
+        parsed_dttm, parsed_flags = cal.parseDT(human_readable)
+        # 0 == not parsed at all
+        if parsed_flags == 0:
             logger.exception(ex)
-            raise error_msg
+            raise ValueError(
+                _(
+                    "Couldn't parse date string [%(human_readable)s]",
+                    human_readable=human_readable,
+                )
+            )
+        # when time is not extracted, we 'reset to midnight'
+        if parsed_flags & 2 == 0:
+            parsed_dttm = parsed_dttm.replace(hour=0, minute=0, second=0)
+        dttm = dttm_from_timetuple(parsed_dttm.utctimetuple())
     return dttm
 
 
@@ -492,3 +488,13 @@ def datetime_eval(datetime_expression: Optional[str] = None) -> Optional[datetim
         except ParseException as error:
             raise ValueError(error)
     return None
+
+
+class DateRangeMigration:
+    x_dateunit_in_since = (
+        r'"time_range":\s"\s*[0-9]+\s(day|week|month|quarter|year)s?\s*\s:\s'
+    )
+    x_dateunit_in_until = (
+        r'"time_range":\s".*\s:\s\s*[0-9]+\s(day|week|month|quarter|year)s?\s*"'
+    )
+    x_dateunit = r"\s*[0-9]+\s(day|week|month|quarter|year)s?\s*"

tests/utils/date_parser_tests.py

@@ -18,6 +18,7 @@
 from unittest.mock import patch
 
 from superset.utils.date_parser import (
+    DateRangeMigration,
     datetime_eval,
     get_since_until,
     parse_human_datetime,
@@ -272,3 +273,21 @@ def test_parse_human_datetime(self):
 
         with self.assertRaises(ValueError):
             parse_human_datetime("xxxxxxx")
+
+    def test_DateRangeMigration(self):
+        params = '{"time_range": "   8 days     : 2020-03-10T00:00:00"}'
+        self.assertRegex(params, DateRangeMigration.x_dateunit_in_since)
+
+        params = '{"time_range": "2020-03-10T00:00:00 :    8 days    "}'
+        self.assertRegex(params, DateRangeMigration.x_dateunit_in_until)
+
+        params = '{"time_range": "   2 weeks    :    8 days    "}'
+        self.assertRegex(params, DateRangeMigration.x_dateunit_in_since)
+        self.assertRegex(params, DateRangeMigration.x_dateunit_in_until)
+
+        params = '{"time_range": "2 weeks ago : 8 days later"}'
+        self.assertNotRegex(params, DateRangeMigration.x_dateunit_in_since)
+        self.assertNotRegex(params, DateRangeMigration.x_dateunit_in_until)
+
+        field = "   8 days   "
+        self.assertRegex(field, DateRangeMigration.x_dateunit)