Fix per_normalize and per_map_source_message tests

GlobalFishingWatch · Nov 29, 2024 · 307935e · 307935e
1 parent 474b196
commit 307935e
Show file tree

Hide file tree

Showing 3 changed files with 94 additions and 9 deletions.
diff --git a/packages/pipe-vms-ingestion/tests/vms_ingestion/normalization/feeds/data/raw_peru.json b/packages/pipe-vms-ingestion/tests/vms_ingestion/normalization/feeds/data/raw_peru.json
@@ -66,5 +66,22 @@
     "ESPECIESCHI": "",
     "ESPECIESCHD": "ANGL/SAR/R.H./ANCH/B",
     "DESC_REGIMEN": "DECRETO LEGISLATIVO N 1392"
+  },
+  {
+    "Correlativo": "41",
+    "ID": "2.4073014212795151e+18",
+    "DATETRANSMISSION": "2024-07-30 14:21:38",
+    "COASTLENGHT": "53.85904",
+    "LATITUDE": "-10.15091",
+    "LONGITUDE": "-79.12178",
+    "SPEED": "4.3",
+    "COURSE": "271.0",
+    "EVENTDESCRIPTION": "NO DEFINIDO",
+    "PLATE": "412412412",
+    "nickname": "NAUTILUS CHN II",
+    "SOURCE": "CLS",
+    "ESPECIESCHI": "",
+    "ESPECIESCHD": "ANGL/SAR/R.H./ANCH/B",
+    "DESC_REGIMEN": "DECRETO LEGISLATIVO N 1392"
   }
 ]
diff --git a/packages/pipe-vms-ingestion/tests/vms_ingestion/normalization/feeds/test_per_normalize.py b/packages/pipe-vms-ingestion/tests/vms_ingestion/normalization/feeds/test_per_normalize.py
@@ -51,6 +51,7 @@ class TestPERNormalize(unittest.TestCase):
             "lat": -11.71153,
             "length": None,
             "lon": -78.28806,
+            "mmsi": None,
             "msgid": "a27fd65ad5d2993887701e80d13dd7aa528f2442105b1e157952b9e30612e312",
             "received_at": None,
             "receiver": None,
@@ -86,6 +87,7 @@ class TestPERNormalize(unittest.TestCase):
             "lat": -5.81764,
             "length": None,
             "lon": -81.03528,
+            "mmsi": None,
             "msgid": "af314b1b519a8f075d09c7dbccba002733701a18348c7373a400611fbba2456b",
             "received_at": None,
             "receiver": None,
@@ -121,6 +123,7 @@ class TestPERNormalize(unittest.TestCase):
             "lat": -10.12852,
             "length": None,
             "lon": -79.17246,
+            "mmsi": None,
             "msgid": "bd33418b076e4707082a958d182a1a0d896393b6479be61f025e81283efaca41",
             "received_at": None,
             "receiver": None,
@@ -156,6 +159,7 @@ class TestPERNormalize(unittest.TestCase):
             "lat": -10.15091,
             "length": None,
             "lon": -79.12178,
+            "mmsi": None,
             "msgid": "189384014029369382bcf37b6e0468d7c84b024b08ccd201f1bf80d8527d427b",
             "received_at": None,
             "receiver": None,
@@ -178,6 +182,42 @@ class TestPERNormalize(unittest.TestCase):
             "updated_at": FAKE_TIME,
             "width": None,
         },
+        {
+            "callsign": None,
+            "class_b_cs_flag": None,
+            "course": 271.0,
+            "destination": None,
+            "flag": "CHN",
+            "fleet": "artisanal",
+            "heading": None,
+            "imo": None,
+            "ingested_at": None,
+            "lat": -10.15091,
+            "length": None,
+            "lon": -79.12178,
+            "mmsi": "412412412",
+            "msgid": "894ac86a79cc0f9008f7bea013e60f7fc36dc7fb178e6e7a2ab184abcfb4fd59",
+            "received_at": None,
+            "receiver": None,
+            "receiver_type": None,
+            "registry_number": "412412412",
+            "shipname": "NAUTILUS CHN II",
+            "shiptype": None,
+            "source": "PERU_VMS",
+            "source_fleet": "ARTISANAL",
+            "source_provider": "DIRNEA",
+            "source_ssvid": None,
+            "source_tenant": "PER",
+            "source_type": "VMS",
+            "speed": 4.3,
+            "ssvid": "2d2fb6e498322431a04d52ab1d39ab1a1008524882bdbf2c200db107484295f4",
+            "status": None,
+            "timestamp": datetime(2024, 7, 30, 19, 21, 38),
+            "timestamp_date": date(2024, 7, 30),
+            "type": "VMS",
+            "updated_at": datetime(2020, 2, 3, 17, 5, 55),
+            "width": None,
+        },
     ]
 
     # Example test that tests the pipeline's transforms.
@@ -195,6 +235,4 @@ def test_normalize(self, mock_now):
             output: pvalue.PCollection = input | PERNormalize(feed="PEr")
 
             # Assert that the output PCollection matches the EXPECTED data.
-            assert_that(
-                output, pcol_equal_to(TestPERNormalize.EXPECTED), label="CheckOutput"
-            )
+            assert_that(output, pcol_equal_to(TestPERNormalize.EXPECTED), label="CheckOutput")
diff --git a/...vms-ingestion/tests/vms_ingestion/normalization/transforms/test_per_map_source_message.py b/...vms-ingestion/tests/vms_ingestion/normalization/transforms/test_per_map_source_message.py
@@ -8,10 +8,7 @@
 from apache_beam.testing.util import assert_that
 from tests.util import pcol_equal_to
 from vms_ingestion.normalization import build_pipeline_options_with_defaults
-from vms_ingestion.normalization.transforms.per_map_source_message import (
-    PERMapSourceMessage,
-    per_infer_fleet,
-)
+from vms_ingestion.normalization.transforms.per_map_source_message import PERMapSourceMessage, per_infer_fleet
 
 
 class TestPERMapSourceMessage(unittest.TestCase):
@@ -42,23 +39,56 @@ class TestPERMapSourceMessage(unittest.TestCase):
             "ESPECIESCHI": "",
             "ESPECIESCHD": "PER/POT",
             "DESC_REGIMEN": "DS. 006-2016-PRODUCE",
-        }
+        },
+        {
+            "Correlativo": "41",
+            "ID": "2.4073014212795151e+18",
+            "DATETRANSMISSION": datetime.fromisoformat("2024-07-30 14:21:38"),
+            "COASTLENGHT": "53.85904",
+            "LATITUDE": "-10.15091",
+            "LONGITUDE": "-79.12178",
+            "SPEED": "4.3",
+            "COURSE": "271.0",
+            "EVENTDESCRIPTION": "NO DEFINIDO",
+            "PLATE": "412412412",
+            "nickname": "NAUTILUS CHN II",
+            "SOURCE": "CLS",
+            "ESPECIESCHI": "",
+            "ESPECIESCHD": "ANGL/SAR/R.H./ANCH/B",
+            "DESC_REGIMEN": "DECRETO LEGISLATIVO N 1392",
+        },
     ]
 
     # Our output data, which is the expected data that the final PCollection must match.
     EXPECTED = [
         {
             "callsign": None,
             "course": 245.0,
+            "flag": None,
             "fleet": "artisanal",
             "lat": -11.71153,
             "lon": -78.28806,
+            "mmsi": None,
             "registry_number": "TA-12345-OP",
             "shipname": "SANTA MARIA",
             "speed": 0.5,
             "ssvid": "TA-12345-OP",
             "timestamp": datetime.fromisoformat("2024-07-31 05:00:00+00:00"),
-        }
+        },
+        {
+            "callsign": None,
+            "course": 271.0,
+            "flag": "CHN",
+            "fleet": "artisanal",
+            "lat": -10.15091,
+            "lon": -79.12178,
+            "mmsi": "412412412",
+            "registry_number": "412412412",
+            "shipname": "NAUTILUS CHN II",
+            "speed": 4.3,
+            "ssvid": "412412412",
+            "timestamp": datetime(2024, 7, 30, 19, 21, 38),
+        },
     ]
 
     # Tests the transform.