From 47377de5817be68341d622e70f261de5eb642ffb Mon Sep 17 00:00:00 2001 From: Stefan `Sec` Zehl Date: Sat, 20 Apr 2024 10:34:01 +0200 Subject: [PATCH] [reassembler] improve timestamp handling with timezones --- iridiumtk/reassembler/id_sat_map.py | 10 ++++--- iridiumtk/reassembler/ira.py | 3 +- iridiumtk/reassembler/iratime.py | 4 ++- iridiumtk/reassembler/msg.py | 4 +-- iridiumtk/reassembler/pktstats.py | 5 ++-- iridiumtk/reassembler/ppm.py | 10 +++---- iridiumtk/reassembler/sbd.py | 6 ++-- util.py | 45 +++++++++++++++++++++++++++++ 8 files changed, 69 insertions(+), 18 deletions(-) diff --git a/iridiumtk/reassembler/id_sat_map.py b/iridiumtk/reassembler/id_sat_map.py index 67bd548..7944fe5 100755 --- a/iridiumtk/reassembler/id_sat_map.py +++ b/iridiumtk/reassembler/id_sat_map.py @@ -3,6 +3,7 @@ import sys import datetime +from util import dt from .base import * from .ira import ReassembleIRA @@ -81,8 +82,7 @@ def find_closest_satellite(self, t, xyz, satlist): return closest_satellite, closest_distance def process(self,q): - time = datetime.datetime.utcfromtimestamp(q.time) - time = time.replace(tzinfo=utc) + time = dt.epoch(q.time) t = self.ts.utc(time) if self.first: self.first=False @@ -106,8 +106,10 @@ def process(self,q): def consume(self,q): if config.verbose: -# print("%s: sat %02d beam %02d [%d %4.2f %4.2f %s] matched %-20s @ %5.2f°"%( datetime.datetime.utcfromtimestamp(q.time), q.sat,q.beam,q.time,q.lat,q.lon,q.alt,q.name,q.sep)) - print("%s: sat %02d beam %02d [%d %8.4f %8.4f %s] matched %-20s @ %5fkm"%( datetime.datetime.utcfromtimestamp(q.time), q.sat,q.beam,q.time,q.lat,q.lon,q.alt,q.name,q.sep)) + #print("%s: sat %02d beam %02d [%d %8.2f %8.2f %s] matched %-20s @ %5.2f°" % + print("%s: sat %02d beam %02d [%d %8.4f %8.4f %s] matched %-20s @ %5fkm" % + (dt.epoch(q.time), q.sat, q.beam, q.time, q.lat, q.lon, q.alt, q.name, q.sep)) + if q.sep > self.MAX_DIST: q.name="NONE" if not q.sat in self.sats: diff --git a/iridiumtk/reassembler/ira.py b/iridiumtk/reassembler/ira.py index 8bcfb24..1266d72 100755 --- a/iridiumtk/reassembler/ira.py +++ b/iridiumtk/reassembler/ira.py @@ -3,6 +3,7 @@ import sys import re +from util import dt from .base import * from ..config import config, outfile @@ -35,7 +36,7 @@ def filter(self,line): return q def process(self,q): q.enrich() - strtime=datetime.datetime.fromtimestamp(q.time,tz=Z).strftime("%Y-%m-%dT%H:%M:%S.{:02.0f}Z".format(int((q.time%1)*100))) + strtime = dt.epoch(q.time).isoformat(timespec='centiseconds') for x in q.pages: return ["%s %03d %02d %6.2f %6.2f %03d : %s %s"%(strtime, q.sat,q.beam,q.lat,q.lon,q.alt,x[0],x[1])] def consume(self,q): diff --git a/iridiumtk/reassembler/iratime.py b/iridiumtk/reassembler/iratime.py index 8e2a07c..412730b 100755 --- a/iridiumtk/reassembler/iratime.py +++ b/iridiumtk/reassembler/iratime.py @@ -4,11 +4,13 @@ import sys import re import collections +from util import dt from .base import * from ..config import config, outfile class ReassembleIRATime(Reassemble): + """Check if there are IRA for the same beam quicker than 4.2 seconds""" def __init__(self): pass def filter(self,line): @@ -32,7 +34,7 @@ def filter(self,line): def process(self,q): if q.beam in self.buf[q.sat]: if q.time-self.buf[q.sat][q.beam] < 4.2: - strtime=datetime.datetime.fromtimestamp(q.time,tz=Z).strftime("%Y-%m-%dT%H:%M:%S") + strtime = dt.epoch(q.time).isoformat(timespec='seconds') print("%3d %3d: %s %f"%(q.beam,q.sat,strtime,q.time-self.buf[q.sat][q.beam])) self.buf[q.sat][q.beam]=q.time diff --git a/iridiumtk/reassembler/msg.py b/iridiumtk/reassembler/msg.py index 7430eaf..6b1099a 100755 --- a/iridiumtk/reassembler/msg.py +++ b/iridiumtk/reassembler/msg.py @@ -4,7 +4,7 @@ import sys import datetime import re -from util import to_ascii, slice_extra +from util import to_ascii, slice_extra, dt from .base import * from ..config import config, outfile @@ -179,7 +179,7 @@ def consume(self, msg): if not msg.correct: return - date= datetime.datetime.fromtimestamp(msg.time).strftime("%Y-%m-%dT%H:%M:%S") + date = dt.epoch_local(msg.time).isoformat(timespec='seconds') str="Message %07d %02d @%s (len:%d)"%(msg.ric, msg.seq, date, msg.pcnt) txt= msg.content if 'noburst' in config.args: diff --git a/iridiumtk/reassembler/pktstats.py b/iridiumtk/reassembler/pktstats.py index 69e25a0..3674198 100755 --- a/iridiumtk/reassembler/pktstats.py +++ b/iridiumtk/reassembler/pktstats.py @@ -4,6 +4,7 @@ import sys import datetime from copy import deepcopy +from util import dt from .base import * from ..config import config, outfile, state @@ -81,9 +82,9 @@ def printstats(self, timeslot, stats, skip=False): comment='' if skip: comment='#!' - print("#!@ %s L:"%(datetime.datetime.fromtimestamp(ts)), file=sys.stderr) + print("#!@ %s L:"%(dt.epoch_local(ts)), file=sys.stderr) else: - print("# @ %s L:"%(datetime.datetime.fromtimestamp(ts)), file=sys.stderr) + print("# @ %s L:"%(dt.epoch_local(ts)), file=sys.stderr) for k in stats: for t in stats[k]: print("%siridium.parsed.%s.%s %7d %8d"%(comment,k,t,stats[k][t],ts)) diff --git a/iridiumtk/reassembler/ppm.py b/iridiumtk/reassembler/ppm.py index b505055..133113c 100755 --- a/iridiumtk/reassembler/ppm.py +++ b/iridiumtk/reassembler/ppm.py @@ -9,7 +9,7 @@ import os import socket from copy import deepcopy -from util import fmt_iritime, to_ascii, slice_extra +from util import fmt_iritime, to_ascii, slice_extra, dt from .base import * from ..config import config, outfile @@ -40,13 +40,13 @@ def filter(self,line): m=self.r2.match(q.data) if not m: return if m.group(2): - q.itime = datetime.datetime.strptime(m.group(1), '%Y-%m-%dT%H:%M:%S.%f') + q.itime = dt.strptime(m.group(1), '%Y-%m-%dT%H:%M:%S.%f').replace(tzinfo=datetime.timezone.utc) else: - q.itime = datetime.datetime.strptime(m.group(1), '%Y-%m-%dT%H:%M:%S') + q.itime = dt.strptime(m.group(1), '%Y-%m-%dT%H:%M:%S').replace(tzinfo=datetime.timezone.utc) return q def process(self,q): - q.uxtime=datetime.datetime.utcfromtimestamp(q.time) + q.uxtime = dt.epoch(q.time) # correct for slot: # 1st vs. 4th slot is 3 * (downlink + guard) @@ -93,7 +93,7 @@ def consume(self, data): if (data[1]-self.cur[1]).total_seconds() > 600: (irun,toff,ppm)=self.onedelta(self.cur,data, verbose=False) if 'grafana' in config.args: - print("iridium.live.ppm %.5f %d"%(ppm,(data[1]-datetime.datetime.fromtimestamp(0)).total_seconds())) + print("iridium.live.ppm %.5f %d" % (ppm, data[1].timestamp())) sys.stdout.flush() else: print("@ %s: ppm: % 6.3f ds: % 8.5f "%(data[1],ppm,(data[1]-data[0]).total_seconds())) diff --git a/iridiumtk/reassembler/sbd.py b/iridiumtk/reassembler/sbd.py index 103c1c0..d642647 100755 --- a/iridiumtk/reassembler/sbd.py +++ b/iridiumtk/reassembler/sbd.py @@ -4,7 +4,7 @@ import sys import datetime import re -from util import to_ascii +from util import to_ascii, dt from .base import * from .ida import ReassembleIDA @@ -210,11 +210,11 @@ def consume_l2(self,q): if len(q.data)>0: print("%s %-99s %s | %s"%( - datetime.datetime.fromtimestamp(q.time).strftime("%Y-%m-%dT%H:%M:%S"), + dt.epoch_local(int(q.time)).isoformat(), hdr,q.data.hex(" "),to_ascii(q.data, dot=True)), file=outfile) else: print("%s %s"%( - datetime.datetime.fromtimestamp(q.time).strftime("%Y-%m-%dT%H:%M:%S"), + dt.epoch_local(int(q.time)).isoformat(), hdr), file=outfile) acars_labels={ # ref. http://www.hoka.it/oldweb/tech_info/systems/acarslabel.htm diff --git a/util.py b/util.py index 8d420a2..aa0a41f 100755 --- a/util.py +++ b/util.py @@ -189,3 +189,48 @@ def parse_channel(fstr): else: frequency=int(fstr) return int(frequency) + +class dt(datetime.datetime): + def isoformat(self, sep='T', timespec='auto'): + """Return the time formatted according to ISO. + + The full format looks like 'YYYY-MM-DD HH:MM:SS.mmmmmm'. + By default, the fractional part is omitted if self.microsecond == 0 + or shortened to 'YYYY-MM-DD HH:MM:SS.mmm' if possible. + + If self.tzinfo is not None, the UTC offset is also attached, giving + giving a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HHMM'. + + if the UTC offset is 0, a 'Z' will be used, so the format is + shortened to 'YYYY-MM-DD HH:MM:SS.mmmmmmZ'. + + Optional arguments exist for compability but should not be used. + """ + + if self.microsecond != 0 and self.microsecond % 1000 == 0 and timespec == 'auto': + timespec = 'milliseconds' + + trunc = False + if timespec == 'centiseconds': + timespec = 'milliseconds' + trunc = True + + _iso = super().isoformat(sep, timespec) + + if trunc and _iso[-6] == '+' and _iso[-10] == '.': + _iso = _iso[:-7] + _iso[-6:] + if _iso[-6:] == '+00:00': + _iso = _iso[:-6] + 'Z' + if (_iso[-6] == '+' or _iso[-6] == '-') and _iso[-3] == ':': + _iso = _iso[:-3] + _iso[-2:] + return _iso + + @classmethod + def epoch(cls, t): + """Construct a correct UTC datetime from a POSIX timestamp.""" + return cls.fromtimestamp(t, datetime.timezone.utc) + + @classmethod + def epoch_local(cls, t): + """Construct a datetime from a POSIX timestamp in the local timezone.""" + return cls.fromtimestamp(t, datetime.timezone.utc).astimezone()