Discard also records without lat or lon

GlobalFishingWatch · Aug 7, 2024 · 5374fba · 5374fba
1 parent f883eec
commit 5374fba
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 16 deletions.
diff --git a/...e-vms-ingestion/tests/vms_ingestion/normalization/transforms/test_discard_zero_lat_lon.py b/...e-vms-ingestion/tests/vms_ingestion/normalization/transforms/test_discard_zero_lat_lon.py
@@ -90,6 +90,14 @@ def test_convert_speed_kph_to_kt(self):
                         "received_at": None,
                         "ssvid": "1",
                     },
+                    {
+                        "timestamp": datetime.strptime(
+                            "2024-01-01T00:05:00", "%Y-%m-%dT%H:%M:%S"
+                        ),
+                        "shipname": "SANTA MARIA",
+                        "received_at": None,
+                        "ssvid": "1",
+                    },
                 ]
             )
 

diff --git a/packages/pipe-vms-ingestion/vms_ingestion/normalization/transforms/discard_zero_lat_lon.py b/packages/pipe-vms-ingestion/vms_ingestion/normalization/transforms/discard_zero_lat_lon.py
@@ -8,21 +8,13 @@ def expand(self, pcoll):
     def discard_zero_lat_lon(self):
         return Filter(
             lambda x: not (
-                (
-                    "lat" in x
-                    and "lon" in x
-                    and x.get("lat") == 0
-                    and x.get("lon") == 0
-                    or x.get("lat") is None
-                    or x.get("lon") is None
-                )
-                or (
-                    "LATITUDE" in x
-                    and "LONGITUDE" in x
-                    and x.get("LATITUDE") == 0
-                    and x.get("LONGITUDE") == 0
-                    or x.get("LATITUDE") is None
-                    or x.get("LONGITUDE") is None
-                )
+                # exclude when lat, lon is  0, 0
+                x.get("lat") == 0
+                and x.get("lon") == 0
+                #  exclude also if lat or lon is missing
+                or x.get("lat") is None
+                or x.get("lon") is None
             )
+            # include only records that have lat and lon fields
+            and ("lat" in x and "lon" in x)
         )