fix: make sure '#N/A' CMPG is retained

Phaeton · Mar 3, 2023 · a4ac931 · a4ac931
1 parent 6615c23
commit a4ac931
Show file tree

Hide file tree

Showing 25 changed files with 122,116 additions and 89,968 deletions.
diff --git a/.github/workflows/create_db_derivatives.yaml b/.github/workflows/create_db_derivatives.yaml
@@ -1,14 +1,15 @@
-name: Create and commit category and image database files
+name: Create and commit category, images and extended database files
 
 on:
   push:
+    branches: ["automation"]
     paths:
       - "plane-alert-db.csv"
       - "plane_images.txt"
       - "blacklist.txt"
 
 jobs:
-  create-categories-and-images:
+  createDerivativeDatabases:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -17,10 +18,10 @@ jobs:
           python-version: "3.10"
       - run: pip install -r ./scripts/requirements.txt
 
-      - name: Create category, extended and images csv files
+      - name: Create category, images and extended database CSV files
         run: python ./scripts/create_db_derivatives.py
 
-      - name: Commit category, extended and images csv files
+      - name: Commit category, images and extended database CSV files
         uses: stefanzweifel/git-auto-commit-action@v4
         with:
-          commit_message: "refactor: update 'plane-alert-db' derivative databases."
+          commit_message: "refactor: update derivative databases."
diff --git a/_bin/README.md b/_bin/README.md
@@ -0,0 +1,3 @@
+# Readme
+
+This folder contains the old database files used before the derivative database creation GitHub action was added to the repository. This folder will be removed when we ensure the GitHub action is working correctly.
diff --git a/_bin/badgers-best-images.csv b/_bin/badgers-best-images.csv
diff --git a/_bin/badgers-best.csv b/_bin/badgers-best.csv
diff --git a/_bin/plane-alert-db-images.csv b/_bin/plane-alert-db-images.csv
diff --git a/_bin/plane-alert-db.csv b/_bin/plane-alert-db.csv
diff --git a/_bin/plane-alert-pia.csv b/_bin/plane-alert-pia.csv
@@ -0,0 +1,49 @@
+$ICAO,$Registration,$Operator,$Type,$ICAO Type,#CMPG,$Tag 1,$#Tag 2,$#Tag 3,Category,$#Link,#ImageLink,#ImageLink2,#ImageLink3
+A0FDDB,N50KL
+A0FDF9,N9XJ
+A0FDFB,N650HC
+A0FE01,N658HC
+A0FE02,N113CS
+A0FE30,N279PH
+A0FE68,N935EF
+A0FE84,N952DA
+A0FE85,N83EP
+A0FE2E,N439PW
+A0FE3B,N16DJ
+A0FE3F,N725DT
+A0FE6E,N437JD
+A0FE9F,N17513
+A0FEAB,N302AK
+A0FEAF,N68KP
+A0FEB4,N676JM
+A0FEBA,N977V
+A0FECB,N628TS
+A0FED0,N68885
+A0FED7,N650JR
+A0FEE1,N6JR
+A0FEEA,N8800E
+A0FEED,N681HS
+A0FEEE,N51TE
+A0FEEF,N682HS
+A0FEE7,N98AC
+A0FEF0,N8100E
+A0FEF1,N8000E
+A0FEF2,N8200E
+A0FEF4,N998PB
+A0FEF6,N928SZ
+A0FEF7,N634BE
+A0FEFB,N32MJ
+A0FEFC,N1980K
+A0FF05,N542TP
+A0FF09,N414KU
+A0FF0B,N271DV
+A0FF17,N311JX
+A0FF20,N898TS
+A0FF21,N68KJ
+A0FF22,N621MM
+A0FF24,N14KL
+A0FF25,N758PB
+A0FF27,N2E
+A102F8,N586GA
+A1DC4A,N711PV
+A4E954,N88BK
diff --git a/_bin/plane-alert-search-terms-to-do.csv b/_bin/plane-alert-search-terms-to-do.csv
@@ -0,0 +1,12 @@
+Ministry,216 added
+National Guard,6 added
+Force,4670 added (not added United States Air Force and Royal Air Force)
+Navy,1460 added
+Coast Guard,398 added
+Army,2071 added
+Marine Corps,185 added
+Administration,64 added
+Guardia,41 added
+Policja,
+Aero Flite,
+Coulson,
diff --git a/_bin/plane-alert-wip.csv b/_bin/plane-alert-wip.csv
diff --git a/planepix.txt → _bin/planepix.txt b/planepix.txt → _bin/planepix.txt
diff --git a/plane-alert-civ-images.csv b/plane-alert-civ-images.csv
diff --git a/plane-alert-civ.csv b/plane-alert-civ.csv
diff --git a/plane-alert-db-extended-images.csv b/plane-alert-db-extended-images.csv
diff --git a/plane-alert-db-extended.csv b/plane-alert-db-extended.csv
diff --git a/plane-alert-db-images.csv b/plane-alert-db-images.csv
diff --git a/plane-alert-gov-images.csv b/plane-alert-gov-images.csv
diff --git a/plane-alert-gov.csv b/plane-alert-gov.csv
diff --git a/plane-alert-mil-images.csv b/plane-alert-mil-images.csv
diff --git a/plane-alert-mil.csv b/plane-alert-mil.csv
diff --git a/plane-alert-pol-images.csv b/plane-alert-pol-images.csv
diff --git a/plane-alert-pol.csv b/plane-alert-pol.csv
diff --git a/plane_images.txt b/plane_images.txt
diff --git a/scripts/README.md b/scripts/README.md
@@ -0,0 +1,6 @@
+# Readme
+
+This folder contains several scripts used in the GitHub actions:
+
+- `create_db_derivatives`: A script that can be used to create the derivative databases based on the `plane-alert-db.csv`, `plane_images.txt` and `blacklist.txt` files.
+- `create_images_reference`: A tiny little script that I used to create the new `plane_images.txt` file. This file will be removed when we are sure the file of the new image is correct.
diff --git a/scripts/create_db_derivatives.py b/scripts/create_db_derivatives.py
@@ -53,5 +53,6 @@
 
     logging.info("Creating the main database images csv file...")
     main_images_df = pd.merge(df, images_df, how="left", on="$ICAO")
+    main_images_df["#CMPG"] = main_images_df["#CMPG"].fillna("#N/A")
     main_images_df.to_csv("plane-alert-db-images.csv", index=False)
     logging.info("Category and images csv files created successfully.")
diff --git a/scripts/create_images_reference.py b/scripts/create_images_reference.py
@@ -1,9 +1,8 @@
-"""This script retrieves the plane images from the 'plane-alert-db-images.csv' databases 
-file and stores them into the 'plane_images.txt' reference file to use later to create 
-imagesCSV database files for each category.
+"""This script retrieves the plane images in the 'plane-alert-db-images.csv' database 
+and 'planepix.txt' file. It stores these images in a new 'plane_images.txt' reference 
+file to use later to create the 'images' CSV database files.
 
-This script can be removed if we know for sure that the results of the new GitHub action
-are correct.
+This script can be removed if we know that the new GitHub action results are correct.
 """
 import logging
 
@@ -33,7 +32,9 @@
     plane_alert_db_images = pd.merge(
         plane_alert_db_images, planepix_df, how="outer", on="$ICAO"
     )
-    plane_alert_db_images = plane_alert_db_images.replace("", np.nan)
+    plane_alert_db_images = plane_alert_db_images.replace(
+        "", np.nan
+    )  # Replace empty strings with NaN.
     logging.info(f"Images merged ({plane_alert_db_images.shape[0]}).")
 
     logging.info("Remove duplicates from the merged images...")
@@ -46,6 +47,25 @@
     )
     logging.info(f"Images without duplicates ({plane_alert_db_images.shape[0]}).")
 
+    logging.info("Make sure that the image urls have the correct format...")
+    plane_alert_db_images[
+        ["#ImageLink", "#ImageLink2", "#ImageLink3", "#ImageLink4"]
+    ] = plane_alert_db_images[
+        ["#ImageLink", "#ImageLink2", "#ImageLink3", "#ImageLink4"]
+    ].apply(
+        lambda row: row.apply(
+            lambda x: x
+            if (isinstance(x, float) and np.isnan(x)) or x.startswith("https://")
+            else (
+                x.replace("http://", "https://")
+                if x.startswith("http://")
+                else "https://" + x
+            )
+        ),
+        axis=1,
+    )
+    logging.info(f"Images with correct format ({plane_alert_db_images.shape[0]}).")
+
     # Print new images.
     logging.info("Check if there were new images in the 'planepix.txt' file...")
     new_image_links = plane_alert_db_images[
@@ -67,6 +87,8 @@
     logging.info("Adding extra 'ImageLink' column if needed...")
     if columns.shape[0] > plane_alert_db_images.columns.shape[0]:
         logging.info("No extra 'ImageLink' column needed to be added.")
+    else:
+        logging.info("Extra '#ImageLink4' column added.")
     plane_alert_db_images.columns = columns[: plane_alert_db_images.columns.shape[0]]
 
     logging.info("Saving found images in 'plane_images.txt' file...")
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Readme

		This folder contains the old database files used before the derivative database creation GitHub action was added to the repository. This folder will be removed when we ensure the GitHub action is working correctly.