From 5374fbade46171d7514c56ea35ae043c03e58848 Mon Sep 17 00:00:00 2001 From: Rodrigo Fuentes Date: Wed, 7 Aug 2024 12:45:02 -0300 Subject: [PATCH] Discard also records without lat or lon --- .../transforms/test_discard_zero_lat_lon.py | 8 +++++++ .../transforms/discard_zero_lat_lon.py | 24 +++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/pipe-vms-ingestion/tests/vms_ingestion/normalization/transforms/test_discard_zero_lat_lon.py b/packages/pipe-vms-ingestion/tests/vms_ingestion/normalization/transforms/test_discard_zero_lat_lon.py index 1f5bc69..37519c1 100644 --- a/packages/pipe-vms-ingestion/tests/vms_ingestion/normalization/transforms/test_discard_zero_lat_lon.py +++ b/packages/pipe-vms-ingestion/tests/vms_ingestion/normalization/transforms/test_discard_zero_lat_lon.py @@ -90,6 +90,14 @@ def test_convert_speed_kph_to_kt(self): "received_at": None, "ssvid": "1", }, + { + "timestamp": datetime.strptime( + "2024-01-01T00:05:00", "%Y-%m-%dT%H:%M:%S" + ), + "shipname": "SANTA MARIA", + "received_at": None, + "ssvid": "1", + }, ] ) diff --git a/packages/pipe-vms-ingestion/vms_ingestion/normalization/transforms/discard_zero_lat_lon.py b/packages/pipe-vms-ingestion/vms_ingestion/normalization/transforms/discard_zero_lat_lon.py index 422ce1d..edde22c 100644 --- a/packages/pipe-vms-ingestion/vms_ingestion/normalization/transforms/discard_zero_lat_lon.py +++ b/packages/pipe-vms-ingestion/vms_ingestion/normalization/transforms/discard_zero_lat_lon.py @@ -8,21 +8,13 @@ def expand(self, pcoll): def discard_zero_lat_lon(self): return Filter( lambda x: not ( - ( - "lat" in x - and "lon" in x - and x.get("lat") == 0 - and x.get("lon") == 0 - or x.get("lat") is None - or x.get("lon") is None - ) - or ( - "LATITUDE" in x - and "LONGITUDE" in x - and x.get("LATITUDE") == 0 - and x.get("LONGITUDE") == 0 - or x.get("LATITUDE") is None - or x.get("LONGITUDE") is None - ) + # exclude when lat, lon is 0, 0 + x.get("lat") == 0 + and x.get("lon") == 0 + # exclude also if lat or lon is missing + or x.get("lat") is None + or x.get("lon") is None ) + # include only records that have lat and lon fields + and ("lat" in x and "lon" in x) )