Update datetime handling for utcnow deprecation

- All datetime values are now stored internally as timezone aware objects
  with a UTC timezone
- Local datetime values are now converted to timezone aware objects with
  a local timezone rather than using a fixed offset
- Infolabels strip off microseconds and tzinfo as different Kodi getter/setter methods
  handle such values incorrectly
- Old behaviour is retained for backwards compatibility
- Fix (remove) incorrect empty string default fallback for lastplayed
- Fix typo in format string for date infolabel used for Kodi 18
- Remove unnecessary datetime parsing for plugin_created_date

resources/lib/youtube_plugin/kodion/items/video_item.py

@@ -279,7 +279,7 @@ def get_license_key(self):
         return self.license_key
 
     def set_last_played(self, last_played):
-        self._last_played = last_played or ''
+        self._last_played = last_played
 
     def get_last_played(self):
         return self._last_played

resources/lib/youtube_plugin/kodion/sql_store/playback_history.py

@@ -9,7 +9,7 @@
 
 from __future__ import absolute_import, division, unicode_literals
 
-from .storage import Storage
+from .storage import Storage, fromtimestamp
 
 
 class PlaybackHistory(Storage):
@@ -21,8 +21,9 @@ class PlaybackHistory(Storage):
     def __init__(self, filepath):
         super(PlaybackHistory, self).__init__(filepath)
 
-    def _add_last_played(self, value, item):
-        value['last_played'] = self._convert_timestamp(item[1])
+    @staticmethod
+    def _add_last_played(value, item):
+        value['last_played'] = fromtimestamp(item[1])
         return value
 
     def get_items(self, keys=None):

resources/lib/youtube_plugin/kodion/sql_store/storage.py

@@ -14,11 +14,10 @@
 import pickle
 import sqlite3
 import time
-from datetime import datetime
 from traceback import format_stack
 
 from ..logger import log_error
-from ..utils.datetime_parser import since_epoch
+from ..utils.datetime_parser import fromtimestamp, since_epoch
 from ..utils.methods import make_dirs
 
 
@@ -317,7 +316,7 @@ def _set(self, item_id, item):
             self._execute(cursor, self._sql['set'], values=values)
 
     def _set_many(self, items, flatten=False):
-        now = since_epoch(datetime.now())
+        now = since_epoch()
         num_items = len(items)
 
         if flatten:
@@ -368,7 +367,7 @@ def _decode(obj, process=None, item=None):
 
     @staticmethod
     def _encode(key, obj, timestamp=None):
-        timestamp = timestamp or since_epoch(datetime.now())
+        timestamp = timestamp or since_epoch()
         blob = sqlite3.Binary(pickle.dumps(
             obj, protocol=pickle.HIGHEST_PROTOCOL
         ))
@@ -383,7 +382,7 @@ def _get(self, item_id, process=None, seconds=None):
             item = result.fetchone() if result else None
             if not item:
                 return None
-        cut_off = since_epoch(datetime.now()) - seconds if seconds else 0
+        cut_off = since_epoch() - seconds if seconds else 0
         if not cut_off or item[1] >= cut_off:
             return self._decode(item[2], process, item)
         return None
@@ -402,7 +401,7 @@ def _get_by_ids(self, item_ids=None, oldest_first=True, limit=-1,
             query = self._sql['get_by_key'].format('?,' * (num_ids - 1) + '?')
             item_ids = tuple(item_ids)
 
-        cut_off = since_epoch(datetime.now()) - seconds if seconds else 0
+        cut_off = since_epoch() - seconds if seconds else 0
         with self as (db, cursor), db:
             result = self._execute(cursor, query, item_ids)
             if as_dict:
@@ -418,7 +417,7 @@ def _get_by_ids(self, item_ids=None, oldest_first=True, limit=-1,
             else:
                 result = [
                     (item[0],
-                     self._convert_timestamp(item[1]),
+                     fromtimestamp(item[1]),
                      self._decode(item[2], process, item))
                     for item in result if not cut_off or item[1] >= cut_off
                 ]
@@ -434,7 +433,3 @@ def _remove_many(self, item_ids):
         with self as (db, cursor), db:
             self._execute(cursor, query, tuple(item_ids))
             self._execute(cursor, 'VACUUM')
-
-    @classmethod
-    def _convert_timestamp(cls, val):
-        return datetime.fromtimestamp(val)

resources/lib/youtube_plugin/kodion/ui/xbmc/info_labels.py

@@ -20,10 +20,11 @@ def _process_date_value(info_labels, name, param):
 
 
 def _process_datetime_value(info_labels, name, param):
-    if param:
-        info_labels[name] = (param.isoformat('T')
-                             if current_system_version.compatible(19, 0) else
-                             param.strftime('%d.%d.%Y'))
+    if not param:
+        return
+    info_labels[name] = (param.replace(microsecond=0, tzinfo=None).isoformat()
+                         if current_system_version.compatible(19, 0) else
+                         param.strftime('%d.%m.%Y'))
 
 
 def _process_int_value(info_labels, name, param):
@@ -81,14 +82,6 @@ def _process_mediatype(info_labels, name, param):
     info_labels[name] = param
 
 
-def _process_last_played(info_labels, name, param):
-    if param:
-        try:
-            info_labels[name] = param.strftime('%Y-%m-%d %H:%M:%S')
-        except AttributeError:
-            info_labels[name] = param
-
-
 def create_from_item(base_item):
     info_labels = {}
 
@@ -144,7 +137,7 @@ def create_from_item(base_item):
         # 'duration' = '3:18' (string)
         _process_video_duration(info_labels, base_item.get_duration())
 
-        _process_last_played(info_labels, 'lastplayed', base_item.get_last_played())
+        _process_datetime_value(info_labels, 'lastplayed', base_item.get_last_played())
 
         # 'rating' = 4.5 (float)
         _process_video_rating(info_labels, base_item.get_rating())

resources/lib/youtube_plugin/kodion/utils/datetime_parser.py

@@ -17,95 +17,129 @@
 
 from ..exceptions import KodionException
 
+try:
+    from datetime import timezone
+except ImportError:
+    timezone = None
+
+
+__RE_MATCH_TIME_ONLY__ = re.compile(
+    r'^(?P<hour>[0-9]{2})(:?(?P<minute>[0-9]{2})(:?(?P<second>[0-9]{2}))?)?$'
+)
+__RE_MATCH_DATE_ONLY__ = re.compile(
+    r'^(?P<year>[0-9]{4})[-/.]?(?P<month>[0-9]{2})[-/.]?(?P<day>[0-9]{2})$'
+)
+__RE_MATCH_DATETIME__ = re.compile(
+    r'^(?P<year>[0-9]{4})[-/.]?(?P<month>[0-9]{2})[-/.]?(?P<day>[0-9]{2})'
+    r'["T ](?P<hour>[0-9]{2}):?(?P<minute>[0-9]{2}):?(?P<second>[0-9]{2})'
+)
+__RE_MATCH_PERIOD__ = re.compile(
+    r'P((?P<years>\d+)Y)?((?P<months>\d+)M)?((?P<days>\d+)D)?'
+    r'(T((?P<hours>\d+)H)?((?P<minutes>\d+)M)?((?P<seconds>\d+)S)?)?'
+)
+__RE_MATCH_ABBREVIATED__ = re.compile(
+    r'\w+, (?P<day>\d+) (?P<month>\w+) (?P<year>\d+)'
+    r' (?P<hour>\d+):(?P<minute>\d+):(?P<second>\d+)'
+)
+
+__INTERNAL_CONSTANTS__ = {
+    'epoch_dt': (
+        datetime.fromtimestamp(0, tz=timezone.utc) if timezone
+        else datetime.fromtimestamp(0)
+    ),
+    'local_offset': None,
+    'Jan': 1,
+    'Feb': 2,
+    'Mar': 3,
+    'Apr': 4,
+    'May': 5,
+    'June': 6,
+    'Jun': 6,
+    'July': 7,
+    'Jul': 7,
+    'Aug': 8,
+    'Sept': 9,
+    'Sep': 9,
+    'Oct': 10,
+    'Nov': 11,
+    'Dec': 12,
+}
 
 now = datetime.now
-
-__RE_MATCH_TIME_ONLY__ = re.compile(r'^(?P<hour>[0-9]{2})(:?(?P<minute>[0-9]{2})(:?(?P<second>[0-9]{2}))?)?$')
-__RE_MATCH_DATE_ONLY__ = re.compile(r'^(?P<year>[0-9]{4})[-/.]?(?P<month>[0-9]{2})[-/.]?(?P<day>[0-9]{2})$')
-__RE_MATCH_DATETIME__ = re.compile(r'^(?P<year>[0-9]{4})[-/.]?(?P<month>[0-9]{2})[-/.]?(?P<day>[0-9]{2})["T ](?P<hour>[0-9]{2}):?(?P<minute>[0-9]{2}):?(?P<second>[0-9]{2})')
-__RE_MATCH_PERIOD__ = re.compile(r'P((?P<years>\d+)Y)?((?P<months>\d+)M)?((?P<days>\d+)D)?(T((?P<hours>\d+)H)?((?P<minutes>\d+)M)?((?P<seconds>\d+)S)?)?')
-__RE_MATCH_ABBREVIATED__ = re.compile(r'(\w+), (?P<day>\d+) (?P<month>\w+) (?P<year>\d+) (?P<hour>\d+):(?P<minute>\d+):(?P<second>\d+)')
-
-__LOCAL_OFFSET__ = now() - datetime.utcnow()
-
-__EPOCH_DT__ = datetime.fromtimestamp(0)
-
-
-def parse(datetime_string, as_utc=True):
-    offset = 0 if as_utc else None
-
-    def _to_int(value):
-        if value is None:
-            return 0
-        return int(value)
-
-    # match time only '00:45:10'
-    time_only_match = __RE_MATCH_TIME_ONLY__.match(datetime_string)
-    if time_only_match:
-        return utc_to_local(
-            dt=datetime.combine(
-                date.today(),
-                dt_time(hour=_to_int(time_only_match.group('hour')),
-                        minute=_to_int(time_only_match.group('minute')),
-                        second=_to_int(time_only_match.group('second')))
-            ),
-            offset=offset
+fromtimestamp = datetime.fromtimestamp
+
+
+def parse(datetime_string):
+    # match time only "00:45:10"
+    match = __RE_MATCH_TIME_ONLY__.match(datetime_string)
+    if match:
+        match = {
+            group: int(value)
+            for group, value in match.groupdict().items()
+            if value
+        }
+        return datetime.combine(
+            date=date.today(),
+            time=dt_time(**match)
         ).time()
 
     # match date only '2014-11-08'
-    date_only_match = __RE_MATCH_DATE_ONLY__.match(datetime_string)
-    if date_only_match:
-        return utc_to_local(
-            dt=datetime(_to_int(date_only_match.group('year')),
-                        _to_int(date_only_match.group('month')),
-                        _to_int(date_only_match.group('day'))),
-            offset=offset
-        )
+    match = __RE_MATCH_DATE_ONLY__.match(datetime_string)
+    if match:
+        match = {
+            group: int(value)
+            for group, value in match.groupdict().items()
+            if value
+        }
+        return datetime(**match)
 
     # full date time
-    date_time_match = __RE_MATCH_DATETIME__.match(datetime_string)
-    if date_time_match:
-        return utc_to_local(
-            dt=datetime(_to_int(date_time_match.group('year')),
-                        _to_int(date_time_match.group('month')),
-                        _to_int(date_time_match.group('day')),
-                        _to_int(date_time_match.group('hour')),
-                        _to_int(date_time_match.group('minute')),
-                        _to_int(date_time_match.group('second'))),
-            offset=offset
-        )
+    match = __RE_MATCH_DATETIME__.match(datetime_string)
+    if match:
+        match = {
+            group: int(value)
+            for group, value in match.groupdict().items()
+            if value
+        }
+        return datetime(**match)
 
     # period - at the moment we support only hours, minutes and seconds
     # e.g. videos and audio
-    period_match = __RE_MATCH_PERIOD__.match(datetime_string)
-    if period_match:
-        return timedelta(hours=_to_int(period_match.group('hours')),
-                         minutes=_to_int(period_match.group('minutes')),
-                         seconds=_to_int(period_match.group('seconds')))
+    match = __RE_MATCH_PERIOD__.match(datetime_string)
+    if match:
+        match = {
+            group: int(value)
+            for group, value in match.groupdict().items()
+            if value
+        }
+        return timedelta(**match)
 
     # abbreviated match
-    abbreviated_match = __RE_MATCH_ABBREVIATED__.match(datetime_string)
-    if abbreviated_match:
-        month = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'June': 6,
-                 'Jun': 6, 'July': 7, 'Jul': 7, 'Aug': 8, 'Sept': 9, 'Sep': 9,
-                 'Oct': 10, 'Nov': 11, 'Dec': 12}
-        return utc_to_local(
-            dt=datetime(year=_to_int(abbreviated_match.group('year')),
-                        month=month[abbreviated_match.group('month')],
-                        day=_to_int(abbreviated_match.group('day')),
-                        hour=_to_int(abbreviated_match.group('hour')),
-                        minute=_to_int(abbreviated_match.group('minute')),
-                        second=_to_int(abbreviated_match.group('second'))),
-            offset=offset
-        )
+    match = __RE_MATCH_ABBREVIATED__.match(datetime_string)
+    if match:
+        match = {
+            group: (
+                __INTERNAL_CONSTANTS__.get(value, 0) if group == 'month'
+                else int(value)
+            )
+            for group, value in match.groupdict().items()
+            if value
+        }
+        return datetime(**match)
 
     raise KodionException('Could not parse |{datetime}| as ISO 8601'
                           .format(datetime=datetime_string))
 
 
 def get_scheduled_start(context, datetime_object, local=True):
-    _now = now() if local else datetime.utcnow()
-    if datetime_object.date() == _now:
+    if timezone:
+        _now = now(tz=timezone.utc)
+        if local:
+            _now = _now.astimezone(None)
+    else:
+        _now = now() if local else datetime.utcnow()
+
+    if datetime_object.date() == _now.date():
         return '@ {start_time}'.format(
             start_time=context.format_time(datetime_object.time())
         )
@@ -115,13 +149,27 @@ def get_scheduled_start(context, datetime_object, local=True):
     )
 
 
-def utc_to_local(dt, offset=None):
-    offset = __LOCAL_OFFSET__ if offset is None else timedelta(hours=offset)
+def utc_to_local(dt):
+    if timezone:
+        return dt.astimezone(None)
+
+    if __INTERNAL_CONSTANTS__['local_offset']:
+        offset = __INTERNAL_CONSTANTS__['local_offset']
+    else:
+        offset = now() - datetime.utcnow()
+        __INTERNAL_CONSTANTS__['local_offset'] = offset
+
     return dt + offset
 
 
-def datetime_to_since(context, dt):
-    _now = now()
+def datetime_to_since(context, dt, local=True):
+    if timezone:
+        _now = now(tz=timezone.utc)
+        if local:
+            _now = _now.astimezone(None)
+    else:
+        _now = now() if local else datetime.utcnow()
+
     diff = _now - dt
     yesterday = _now - timedelta(days=1)
     yyesterday = _now - timedelta(days=2)
@@ -190,5 +238,7 @@ def strptime(datetime_str, fmt='%Y-%m-%dT%H:%M:%S'):
     return datetime.strptime(datetime_str, fmt)
 
 
-def since_epoch(dt_object):
-    return (dt_object - __EPOCH_DT__).total_seconds()
+def since_epoch(dt_object=None):
+    if dt_object is None:
+        dt_object = now(tz=timezone.utc) if timezone else datetime.utcnow()
+    return (dt_object - __INTERNAL_CONSTANTS__['epoch_dt']).total_seconds()

resources/lib/youtube_plugin/youtube/client/youtube.py

@@ -439,7 +439,7 @@ def helper(video_id, responses):
             vid = candidate['id']['videoId']
             if vid not in seen:
                 seen.append(vid)
-                candidate['plugin_created_date'] = datetime_parser.now().strftime('%Y-%m-%dT%H:%M:%SZ')
+                candidate['plugin_created_date'] = datetime_parser.since_epoch()
                 items.insert(0, candidate)
 
         # Truncate items to keep it manageable, and cache
@@ -448,7 +448,7 @@ def helper(video_id, responses):
 
         # Build the result set
         items.sort(
-            key=lambda a: datetime_parser.parse(a['plugin_created_date']),
+            key=lambda a: a.get('plugin_created_date', 0),
             reverse=True
         )
         sorted_items = []