Convert internal timestamps to TZ-aware, treat user-provided TZ-naive…

… ones as UTC

Signed-off-by: Sergey Vasilyev <[email protected]>

docs/authentication.rst

@@ -54,6 +54,9 @@ or an instance of :class:`kopf.ConnectionInfo`::
             expiration=datetime.datetime(2099, 12, 31, 23, 59, 59),
         )
 
+Both TZ-naive & TZ-aware expiration times are supported.
+The TZ-naive timestamps are always treated as UTC.
+
 As with any other handlers, the login handler can be async if the network
 communication is needed and async mode is supported::
 

kopf/_cogs/structs/credentials.py

@@ -118,7 +118,7 @@ def __init__(
         self._current = {}
         self._invalid = collections.defaultdict(list)
         self._lock = asyncio.Lock()
-        self._next_expiration = datetime.datetime.max
+        self._next_expiration: Optional[datetime.datetime] = None
 
         if __src is not None:
             self._update_converted(__src)
@@ -230,14 +230,18 @@ async def expire(self) -> None:
         and not blocked from reappearing.
         """
         now = datetime.datetime.now(datetime.timezone.utc)
-        if self._next_expiration.tzinfo is None:
-            now = now.replace(tzinfo=None)  # for comparability
-        if now >= self._next_expiration:  # quick & lockless for speed: it is done on every API call
+
+        # Quick & lockless for speed: it is done on every API call, we have no time for locks.
+        if self._next_expiration is None or now >= self._next_expiration:
             async with self._lock:
                 for key, item in list(self._current.items()):
-                    if item.info.expiration is not None and now >= item.info.expiration:
-                        await self._flush_caches(item)
-                        del self._current[key]
+                    expiration = item.info.expiration
+                    if expiration is not None:
+                        if expiration.tzinfo is None:
+                            expiration = expiration.replace(tzinfo=datetime.timezone.utc)
+                        if now >= expiration:
+                            await self._flush_caches(item)
+                            del self._current[key]
                 self._update_expiration()
                 need_reauth = not self._current  # i.e. nothing is left at all
 
@@ -318,10 +322,11 @@ async def populate(
 
     def is_empty(self) -> bool:
         now = datetime.datetime.now(datetime.timezone.utc)
-        return all(
-            item.info.expiration is not None and now >= item.info.expiration  # i.e. expired
-            for key, item in self._current.items()
-        )
+        expirations = [
+            dt if dt is None or dt.tzinfo is not None else dt.replace(tzinfo=datetime.timezone.utc)
+            for dt in (item.info.expiration for item in self._current.values())
+        ]
+        return all(dt is not None and now >= dt for dt in expirations)  # i.e. expired
 
     async def wait_for_readiness(self) -> None:
         await self._ready.wait_for(True)
@@ -383,8 +388,8 @@ def _update_converted(
 
     def _update_expiration(self) -> None:
         expirations = [
-            item.info.expiration
-            for item in self._current.values()
-            if item.info.expiration is not None
+            dt if dt.tzinfo is not None else dt.replace(tzinfo=datetime.timezone.utc)
+            for dt in (item.info.expiration for item in self._current.values())
+            if dt is not None
         ]
-        self._next_expiration = min(expirations + [datetime.datetime.max])
+        self._next_expiration = min(expirations) if expirations else None

kopf/_core/actions/progression.py

@@ -18,6 +18,8 @@
 from typing import Any, Collection, Dict, Iterable, Iterator, \
                    Mapping, NamedTuple, Optional, overload
 
+import iso8601
+
 from kopf._cogs.configs import progress
 from kopf._cogs.structs import bodies, ids, patches
 from kopf._core.actions import execution
@@ -62,9 +64,9 @@ def from_scratch(cls, *, purpose: Optional[str] = None) -> "HandlerState":
     def from_storage(cls, __d: progress.ProgressRecord) -> "HandlerState":
         return cls(
             active=False,
-            started=_datetime_fromisoformat(__d.get('started')) or datetime.datetime.now(datetime.timezone.utc),
-            stopped=_datetime_fromisoformat(__d.get('stopped')),
-            delayed=_datetime_fromisoformat(__d.get('delayed')),
+            started=_parse_iso8601(__d.get('started')) or datetime.datetime.now(datetime.timezone.utc),
+            stopped=_parse_iso8601(__d.get('stopped')),
+            delayed=_parse_iso8601(__d.get('delayed')),
             purpose=__d.get('purpose') if __d.get('purpose') else None,
             retries=__d.get('retries') or 0,
             success=__d.get('success') or False,
@@ -76,9 +78,9 @@ def from_storage(cls, __d: progress.ProgressRecord) -> "HandlerState":
 
     def for_storage(self) -> progress.ProgressRecord:
         return progress.ProgressRecord(
-            started=None if self.started is None else _datetime_toisoformat(self.started),
-            stopped=None if self.stopped is None else _datetime_toisoformat(self.stopped),
-            delayed=None if self.delayed is None else _datetime_toisoformat(self.delayed),
+            started=None if self.started is None else _format_iso8601(self.started),
+            stopped=None if self.stopped is None else _format_iso8601(self.stopped),
+            delayed=None if self.delayed is None else _format_iso8601(self.delayed),
             purpose=None if self.purpose is None else str(self.purpose),
             retries=None if self.retries is None else int(self.retries),
             success=None if self.success is None else bool(self.success),
@@ -355,33 +357,24 @@ def deliver_results(
 
 
 @overload
-def _datetime_toisoformat(val: None) -> None: ...
+def _format_iso8601(val: None) -> None: ...
 
 
 @overload
-def _datetime_toisoformat(val: datetime.datetime) -> str: ...
+def _format_iso8601(val: datetime.datetime) -> str: ...
 
 
-def _datetime_toisoformat(val: Optional[datetime.datetime]) -> Optional[str]:
-    if val is None:
-        return None
-    else:
-        return val.isoformat(timespec='microseconds')
+def _format_iso8601(val: Optional[datetime.datetime]) -> Optional[str]:
+    return None if val is None else val.isoformat(timespec='microseconds')
 
 
 @overload
-def _datetime_fromisoformat(val: None) -> None: ...
+def _parse_iso8601(val: None) -> None: ...
 
 
 @overload
-def _datetime_fromisoformat(val: str) -> datetime.datetime: ...
+def _parse_iso8601(val: str) -> datetime.datetime: ...
 
 
-def _datetime_fromisoformat(val: Optional[str]) -> Optional[datetime.datetime]:
-    if val is None:
-        return None
-    else:
-        dt = datetime.datetime.fromisoformat(val)
-        if dt.tzinfo is None:
-            dt = dt.replace(tzinfo=datetime.timezone.utc)
-        return dt
+def _parse_iso8601(val: Optional[str]) -> Optional[datetime.datetime]:
+    return None if val is None else iso8601.parse_date(val)  # always TZ-aware

kopf/_core/engines/peering.py

@@ -69,7 +69,6 @@ def __init__(
         self.lifetime = datetime.timedelta(seconds=int(lifetime))
         self.lastseen = (iso8601.parse_date(lastseen) if lastseen is not None else
                          datetime.datetime.now(datetime.timezone.utc))
-        self.lastseen = self.lastseen.replace(tzinfo=None)  # only the naive utc -- for comparison
         self.deadline = self.lastseen + self.lifetime
         self.is_dead = self.deadline <= datetime.datetime.now(datetime.timezone.utc)
 

tests/authentication/test_vault.py

@@ -1,6 +1,7 @@
 import datetime
 
 import freezegun
+import iso8601
 import pytest
 
 from kopf._cogs.structs.credentials import ConnectionInfo, LoginError, Vault, VaultKey
@@ -55,7 +56,7 @@ async def test_yielding_after_population(mocker):
 
 @freezegun.freeze_time('2020-01-01T00:00:00')
 async def test_yielding_items_before_expiration(mocker):
-    future = datetime.datetime(2020, 1, 1, 0, 0, 0, 1)
+    future = iso8601.parse_date('2020-01-01T00:00:00.000001')
     key1 = VaultKey('some-key')
     info1 = ConnectionInfo(server='https://expected/', expiration=future)
     vault = Vault()
@@ -74,8 +75,8 @@ async def test_yielding_items_before_expiration(mocker):
 @pytest.mark.parametrize('delta', [0, 1])
 @freezegun.freeze_time('2020-01-01T00:00:00')
 async def test_yielding_ignores_expired_items(mocker, delta):
-    future = datetime.datetime(2020, 1, 1, 0, 0, 0, 1)
-    past = datetime.datetime(2020, 1, 1) - datetime.timedelta(microseconds=delta)
+    future = iso8601.parse_date('2020-01-01T00:00:00.000001')
+    past = iso8601.parse_date('2020-01-01') - datetime.timedelta(microseconds=delta)
     key1 = VaultKey('some-key')
     key2 = VaultKey('other-key')
     info1 = ConnectionInfo(server='https://expected/', expiration=past)
@@ -96,7 +97,7 @@ async def test_yielding_ignores_expired_items(mocker, delta):
 @pytest.mark.parametrize('delta', [0, 1])
 @freezegun.freeze_time('2020-01-01T00:00:00')
 async def test_yielding_when_everything_is_expired(mocker, delta):
-    past = datetime.datetime(2020, 1, 1) - datetime.timedelta(microseconds=delta)
+    past = iso8601.parse_date('2020-01-01') - datetime.timedelta(microseconds=delta)
     key1 = VaultKey('some-key')
     info1 = ConnectionInfo(server='https://expected/', expiration=past)
     vault = Vault()

tests/handling/daemons/conftest.py

@@ -104,7 +104,7 @@ def frozen_time():
     A helper to simulate time movements to step over long sleeps/timeouts.
     """
     # TODO LATER: Either freezegun should support the system clock, or find something else.
-    with freezegun.freeze_time("2020-01-01 00:00:00") as frozen:
+    with freezegun.freeze_time("2020-01-01T00:00:00") as frozen:
         # Use freezegun-supported time instead of system clocks -- for testing purposes only.
         # NB: Patch strictly after the time is frozen -- to use fake_time(), not real time().
         with patch('time.monotonic', time.time), patch('time.perf_counter', time.time):

tests/handling/indexing/test_index_exclusion.py

@@ -1,8 +1,8 @@
 import asyncio
-import datetime
 import logging
 
 import freezegun
+import iso8601
 import pytest
 
 from kopf._cogs.aiokits.aiotoggles import Toggle
@@ -76,7 +76,7 @@ async def test_temporary_failures_with_expired_delays_are_reindexed(
         resource, namespace, settings, registry, memories, indexers, index, caplog, event_type, handlers):
     caplog.set_level(logging.DEBUG)
     body = {'metadata': {'namespace': namespace, 'name': 'name1'}}
-    delayed = datetime.datetime(2020, 12, 31, 23, 59, 59, 0)
+    delayed = iso8601.parse_date('2020-12-31T23:59:59')
     memory = await memories.recall(raw_body=body)
     memory.indexing_memory.indexing_state = State({'index_fn': HandlerState(delayed=delayed)})
     await process_resource_event(
@@ -153,9 +153,9 @@ async def test_removed_and_remembered_on_permanent_errors(
 
 @freezegun.freeze_time('2020-12-31T00:00:00')
 @pytest.mark.parametrize('delay_kwargs, expected_delayed', [
-    (dict(), datetime.datetime(2020, 12, 31, 0, 1, 0)),
-    (dict(delay=0), datetime.datetime(2020, 12, 31, 0, 0, 0)),
-    (dict(delay=9), datetime.datetime(2020, 12, 31, 0, 0, 9)),
+    (dict(), iso8601.parse_date('2020-12-31T00:01:00')),
+    (dict(delay=0), iso8601.parse_date('2020-12-31T00:00:00')),
+    (dict(delay=9), iso8601.parse_date('2020-12-31T00:00:09')),
     (dict(delay=None), None),
 ])
 @pytest.mark.usefixtures('indexed_123')

tests/handling/test_cause_logging.py

@@ -1,8 +1,8 @@
 import asyncio
-import datetime
 import logging
 
 import freezegun
+import iso8601
 import pytest
 
 import kopf
@@ -106,7 +106,7 @@ async def test_diffs_not_logged_if_absent(registry, settings, resource, handlers
 
 
 # Timestamps: time zero (0), before (B), after (A), and time zero+1s (1).
-TS0 = datetime.datetime(2020, 12, 31, 23, 59, 59, 123456)
+TS0 = iso8601.parse_date('2020-12-31T23:59:59.123456')
 TS1_ISO = '2021-01-01T00:00:00.123456'
 
 

tests/handling/test_delays.py

@@ -1,8 +1,8 @@
 import asyncio
-import datetime
 import logging
 
 import freezegun
+import iso8601
 import pytest
 
 import kopf
@@ -18,7 +18,7 @@
 
 @pytest.mark.parametrize('cause_reason', HANDLER_REASONS)
 @pytest.mark.parametrize('now, delayed_iso, delay', [
-    ['2020-01-01T00:00:00', '2020-01-01T00:04:56.789000', 4 * 60 + 56.789],
+    ['2020-01-01T00:00:00', '2020-01-01T00:04:56.789000+00:00', 4 * 60 + 56.789],
 ], ids=['fast'])
 async def test_delayed_handlers_progress(
         registry, settings, handlers, resource, cause_mock, cause_reason,
@@ -66,8 +66,8 @@ async def test_delayed_handlers_progress(
 
 @pytest.mark.parametrize('cause_reason', HANDLER_REASONS)
 @pytest.mark.parametrize('now, delayed_iso, delay', [
-    ['2020-01-01T00:00:00', '2020-01-01T00:04:56.789000', 4 * 60 + 56.789],
-    ['2020-01-01T00:00:00', '2099-12-31T23:59:59.000000', WAITING_KEEPALIVE_INTERVAL],
+    ['2020-01-01T00:00:00', '2020-01-01T00:04:56.789000+00:00', 4 * 60 + 56.789],
+    ['2020-01-01T00:00:00', '2099-12-31T23:59:59.000000+00:00', WAITING_KEEPALIVE_INTERVAL],
 ], ids=['fast', 'slow'])
 async def test_delayed_handlers_sleep(
         registry, settings, handlers, resource, cause_mock, cause_reason,
@@ -76,8 +76,8 @@ async def test_delayed_handlers_sleep(
 
     # Simulate the original persisted state of the resource.
     # Make sure the finalizer is added since there are mandatory deletion handlers.
-    started_dt = datetime.datetime.fromisoformat('2000-01-01T00:00:00')  # long time ago is fine.
-    delayed_dt = datetime.datetime.fromisoformat(delayed_iso)
+    started_dt = iso8601.parse_date('2000-01-01T00:00:00')  # long time ago is fine.
+    delayed_dt = iso8601.parse_date(delayed_iso)
     event_type = None if cause_reason == Reason.RESUME else 'irrelevant'
     event_body = {
         'metadata': {'finalizers': [settings.persistence.finalizer]},

tests/handling/test_timing_consistency.py

@@ -2,6 +2,7 @@
 import datetime
 
 import freezegun
+import iso8601
 
 import kopf
 from kopf._cogs.structs.ephemera import Memo
@@ -34,7 +35,7 @@ async def test_consistent_awakening(registry, settings, resource, k8s_mocked, mo
     """
 
     # Simulate that the object is scheduled to be awakened between the watch-event and sleep.
-    ts0 = datetime.datetime(2019, 12, 30, 10, 56, 43)
+    ts0 = iso8601.parse_date('2019-12-30T10:56:43')
     tsA_triggered = "2019-12-30T10:56:42.999999"
     ts0_scheduled = "2019-12-30T10:56:43.000000"
     tsB_delivered = "2019-12-30T10:56:43.000001"

tests/peering/test_peer_patching.py

@@ -46,7 +46,7 @@ async def test_touching_a_peer_stores_it(
     patch = await patch_mock.call_args_list[0][0][0].json()
     assert set(patch['status']) == {'id1'}
     assert patch['status']['id1']['priority'] == 0
-    assert patch['status']['id1']['lastseen'] == '2020-12-31T23:59:59.123456'
+    assert patch['status']['id1']['lastseen'] == '2020-12-31T23:59:59.123456+00:00'
     assert patch['status']['id1']['lifetime'] == 60
 
 

tests/peering/test_peers.py

@@ -1,6 +1,7 @@
 import datetime
 
 import freezegun
+import iso8601
 
 from kopf._core.engines.peering import Peer
 
@@ -10,14 +11,14 @@ def test_defaults():
     peer = Peer(identity='id')
     assert peer.identity == 'id'
     assert peer.lifetime == datetime.timedelta(seconds=60)
-    assert peer.lastseen == datetime.datetime(2020, 12, 31, 23, 59, 59, 123456)
+    assert peer.lastseen == iso8601.parse_date('2020-12-31T23:59:59.123456')
 
 
 @freezegun.freeze_time('2020-12-31T23:59:59.123456')
 def test_repr():
     peer = Peer(identity='some-id')
     text = repr(peer)
-    assert text == "<Peer some-id: priority=0, lifetime=60, lastseen='2020-12-31T23:59:59.123456'>"
+    assert text == "<Peer some-id: priority=0, lifetime=60, lastseen='2020-12-31T23:59:59.123456+00:00'>"
 
 
 @freezegun.freeze_time('2020-12-31T23:59:59.123456')
@@ -47,13 +48,13 @@ def test_creation_with_lifetime_unspecified():
 @freezegun.freeze_time('2020-12-31T23:59:59.123456')
 def test_creation_with_lastseen_as_string():
     peer = Peer(identity='id', lastseen='2020-01-01T12:34:56.789123')
-    assert peer.lastseen == datetime.datetime(2020, 1, 1, 12, 34, 56, 789123)
+    assert peer.lastseen == iso8601.parse_date('2020-01-01T12:34:56.789123')
 
 
 @freezegun.freeze_time('2020-12-31T23:59:59.123456')
 def test_creation_with_lastseen_unspecified():
     peer = Peer(identity='id')
-    assert peer.lastseen == datetime.datetime(2020, 12, 31, 23, 59, 59, 123456)
+    assert peer.lastseen == iso8601.parse_date('2020-12-31T23:59:59.123456')
 
 
 @freezegun.freeze_time('2020-12-31T23:59:59.123456')
@@ -64,8 +65,8 @@ def test_creation_as_alive():
         lastseen='2020-12-31T23:59:50.123456',  # less than 10 seconds before "now"
     )
     assert peer.lifetime == datetime.timedelta(seconds=10)
-    assert peer.lastseen == datetime.datetime(2020, 12, 31, 23, 59, 50, 123456)
-    assert peer.deadline == datetime.datetime(2021, 1, 1, 0, 0, 0, 123456)
+    assert peer.lastseen == iso8601.parse_date('2020-12-31T23:59:50.123456')
+    assert peer.deadline == iso8601.parse_date('2021-01-01T00:00:00.123456')
     assert peer.is_dead is False
 
 
@@ -77,6 +78,6 @@ def test_creation_as_dead():
         lastseen='2020-12-31T23:59:49.123456',  # 10 seconds before "now" sharp
     )
     assert peer.lifetime == datetime.timedelta(seconds=10)
-    assert peer.lastseen == datetime.datetime(2020, 12, 31, 23, 59, 49, 123456)
-    assert peer.deadline == datetime.datetime(2020, 12, 31, 23, 59, 59, 123456)
+    assert peer.lastseen == iso8601.parse_date('2020-12-31T23:59:49.123456')
+    assert peer.deadline == iso8601.parse_date('2020-12-31T23:59:59.123456')
     assert peer.is_dead is True