From 4958e4e78590c4b234c5d742081c0987fc2b9dbe Mon Sep 17 00:00:00 2001 From: Phaeton Date: Tue, 13 Aug 2024 19:51:29 +0000 Subject: [PATCH 1/2] refactor: update derivative databases. --- plane-alert-civ-images.csv | 2 +- plane-alert-db-images.csv | 2 +- plane_images.csv | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/plane-alert-civ-images.csv b/plane-alert-civ-images.csv index b78ac6eb..2e5723ce 100644 --- a/plane-alert-civ-images.csv +++ b/plane-alert-civ-images.csv @@ -3413,7 +3413,7 @@ AC1D80,N88WR,Wynn Resorts,Gulfstream G550,GLF5,Civ,The Gambler,House Always Wins AC1F1D,N880PC,Promega Corp,Dassault Falcon 2000EX,F2TH,Civ,Bizjet,Biotechnology,Enzymes,As Seen on TV,https://en.wikipedia.org/wiki/Promega,https://cdn.jetphotos.com/full/5/66476_1528922673.jpg,https://cdn.jetphotos.com/full/5/47873_1435519384.jpg,https://cdn.jetphotos.com/full/6/85844_1411574321.jpg, AC225D,N881JJ,Five Guys Enterprises,Gulfstream G450,G450,Civ,Bizjet,Burgers and Fries,Fast Food Fast Plane,As Seen on TV,https://w.wiki/Ara8,,,, AC2323,N881SG,Ekspres Air,Boeing 737NG-BBJ,B737,Civ,Man Made Climate Change,Boeing Business Jet,Climate Crisis,Climate Crisis,https://xkcd.com/1732/,https://cdn.jetphotos.com/full/6/19102_1638981282.jpg,https://cdn.jetphotos.com/full/6/29920_1638697973.jpg,https://cdn.jetphotos.com/full/6/55462_1636059899.jpg, -AC2A05,N883LS,Las Vegas Sands Corp,Boeing 737-BBJ,B737,Civ,Man Made Climate Change,Boeing Business Jet,Snake Eyes,As Seen on TV,https://youtu.be/-XpiVnv3-n4,https://cdn.jetphotos.com/full/6/19074_1634819113.jpg,https://cdn.jetphotos.com/full/6/52153_1634145940.jpg,https://cdn.jetphotos.com/full/5/43981_1620961145.jpg, +AC2A05,N883LS,Las Vegas Sands Corp,Boeing 737-BBJ,B737,Civ,Man Made Climate Change,Boeing Business Jet,Snake Eyes,As Seen on TV,https://youtu.be/-XpiVnv3-n4,,,, AC2AA4,N883TA,Top Aces Inc,General Dynamics F-16A Netz,F16,Civ,Aggressor Squadron,Callsign Viper,Broke The Hard Deck,Hired Gun,https://www.topaces.com,,,, AC2DC7,N884MC,Chevron USA,Embraer Praetor 600,E550,Civ,Oil Spill,Man Made Climate Change,Climate Crisis,Climate Crisis,https://xkcd.com/1732/,https://cdn.jetphotos.com/full/5/11654_1582667655.jpg,https://cdn.jetphotos.com/full/5/11254_1579661825.jpg,, AC3173,N885LS,Las Vegas Sands Corp,Boeing 737-BBJ,B737,Civ,Man Made Climate Change,Boeing Business Jet,Snake Eyes,As Seen on TV,https://youtu.be/vTlrLl2XseY,https://cdn.jetphotos.com/full/5/68205_1633657360.jpg,https://cdn.jetphotos.com/full/5/75130_1630644632.jpg,https://cdn.jetphotos.com/full/5/62487_1629343354.jpg, diff --git a/plane-alert-db-images.csv b/plane-alert-db-images.csv index f4981b68..8be2adae 100644 --- a/plane-alert-db-images.csv +++ b/plane-alert-db-images.csv @@ -9065,7 +9065,7 @@ AC1D80,N88WR,Wynn Resorts,Gulfstream G550,GLF5,Civ,The Gambler,House Always Wins AC1F1D,N880PC,Promega Corp,Dassault Falcon 2000EX,F2TH,Civ,Bizjet,Biotechnology,Enzymes,As Seen on TV,https://en.wikipedia.org/wiki/Promega,https://cdn.jetphotos.com/full/5/66476_1528922673.jpg,https://cdn.jetphotos.com/full/5/47873_1435519384.jpg,https://cdn.jetphotos.com/full/6/85844_1411574321.jpg, AC225D,N881JJ,Five Guys Enterprises,Gulfstream G450,G450,Civ,Bizjet,Burgers and Fries,Fast Food Fast Plane,As Seen on TV,https://w.wiki/Ara8,,,, AC2323,N881SG,Ekspres Air,Boeing 737NG-BBJ,B737,Civ,Man Made Climate Change,Boeing Business Jet,Climate Crisis,Climate Crisis,https://xkcd.com/1732/,https://cdn.jetphotos.com/full/6/19102_1638981282.jpg,https://cdn.jetphotos.com/full/6/29920_1638697973.jpg,https://cdn.jetphotos.com/full/6/55462_1636059899.jpg, -AC2A05,N883LS,Las Vegas Sands Corp,Boeing 737-BBJ,B737,Civ,Man Made Climate Change,Boeing Business Jet,Snake Eyes,As Seen on TV,https://youtu.be/-XpiVnv3-n4,https://cdn.jetphotos.com/full/6/19074_1634819113.jpg,https://cdn.jetphotos.com/full/6/52153_1634145940.jpg,https://cdn.jetphotos.com/full/5/43981_1620961145.jpg, +AC2A05,N883LS,Las Vegas Sands Corp,Boeing 737-BBJ,B737,Civ,Man Made Climate Change,Boeing Business Jet,Snake Eyes,As Seen on TV,https://youtu.be/-XpiVnv3-n4,,,, AC2AA4,N883TA,Top Aces Inc,General Dynamics F-16A Netz,F16,Civ,Aggressor Squadron,Callsign Viper,Broke The Hard Deck,Hired Gun,https://www.topaces.com,,,, AC2CCB,N884BA,USAF,Boeing KC-46A Pegasus,B762,Mil,Air2Air,Shiny,To Fly Fight And Win,USAF,https://w.wiki/8ocC,https://cdn.jetphotos.com/full/6/76043_1614612172.jpg,https://cdn.jetphotos.com/full/5/34743_1614612082.jpg,https://cdn.jetphotos.com/full/5/56354_1552036416.jpg, AC2DC7,N884MC,Chevron USA,Embraer Praetor 600,E550,Civ,Oil Spill,Man Made Climate Change,Climate Crisis,Climate Crisis,https://xkcd.com/1732/,https://cdn.jetphotos.com/full/5/11654_1582667655.jpg,https://cdn.jetphotos.com/full/5/11254_1579661825.jpg,, diff --git a/plane_images.csv b/plane_images.csv index a046bfcd..d762beed 100644 --- a/plane_images.csv +++ b/plane_images.csv @@ -9099,6 +9099,7 @@ AC1F1D,https://cdn.jetphotos.com/full/5/66476_1528922673.jpg,https://cdn.jetphot AC2259,https://media.abpic.co.uk/pictures/full_size_0578/1880633-large.jpg,,, AC225D,,,, AC2323,https://cdn.jetphotos.com/full/6/19102_1638981282.jpg,https://cdn.jetphotos.com/full/6/29920_1638697973.jpg,https://cdn.jetphotos.com/full/6/55462_1636059899.jpg, +AC2A05,,,, AC2AA4,,,, AC2CCB,https://cdn.jetphotos.com/full/6/76043_1614612172.jpg,https://cdn.jetphotos.com/full/5/34743_1614612082.jpg,https://cdn.jetphotos.com/full/5/56354_1552036416.jpg, AC2DC7,https://cdn.jetphotos.com/full/5/11654_1582667655.jpg,https://cdn.jetphotos.com/full/5/11254_1579661825.jpg,, From 9d08d1381c1e10fbc384fe7b1811abcbe97730bd Mon Sep 17 00:00:00 2001 From: Phaeton <808865+Phaeton@users.noreply.github.com> Date: Tue, 13 Aug 2024 16:08:33 -0400 Subject: [PATCH 2/2] Add check for category to flag new/invalid submissions. --- .github/workflows/check_categories.yaml | 25 ++++++++++ .github/workflows/create_db_derivatives.yaml | 3 ++ plane-alert-categories.csv | 52 ++++++++++++++++++++ scripts/check_categories.py | 46 +++++++++++++++++ scripts/export_categories.py | 31 ++++++++++++ 5 files changed, 157 insertions(+) create mode 100644 .github/workflows/check_categories.yaml create mode 100644 plane-alert-categories.csv create mode 100644 scripts/check_categories.py create mode 100644 scripts/export_categories.py diff --git a/.github/workflows/check_categories.yaml b/.github/workflows/check_categories.yaml new file mode 100644 index 00000000..8b0817f8 --- /dev/null +++ b/.github/workflows/check_categories.yaml @@ -0,0 +1,25 @@ +name: Check categories + +on: + pull_request: + paths: + - "plane-alert-db.csv" + - "plane-alert-pia.csv" + - "plane-alert-ukraine.csv" + - "plane_images.csv" + +jobs: + checkCategories: + runs-on: ubuntu-latest + name: Check if main databases have valid Categories + steps: + - name: Checkout + uses: actions/checkout@v4.1.7 + - uses: actions/setup-python@v5.1.1 + with: + python-version: "3.10" + cache: "pip" + - run: pip install -r ./scripts/requirements.txt + + - name: Run main database checks + run: python ./scripts/check_categories.py diff --git a/.github/workflows/create_db_derivatives.yaml b/.github/workflows/create_db_derivatives.yaml index 31448e0c..0e309d75 100644 --- a/.github/workflows/create_db_derivatives.yaml +++ b/.github/workflows/create_db_derivatives.yaml @@ -31,6 +31,9 @@ jobs: - name: Create category and images derivative CSV files run: python ./scripts/create_db_derivatives.py + - name: Create valid category listing + run: python ./scripts/export_categories.py + - name: Update README.md to include changes run: python ./scripts/update_readme.py diff --git a/plane-alert-categories.csv b/plane-alert-categories.csv new file mode 100644 index 00000000..3a00faae --- /dev/null +++ b/plane-alert-categories.csv @@ -0,0 +1,52 @@ +Category +Other Air Forces +Toy Soldiers +Flying Doctors +Dictator Alert +Governments +Ptolemy would be proud +Police Forces +Dogs with Jobs +Quango +Jump Johnny Jump +Aerial Firefighter +Climate Crisis +Jesus he Knows me +Big Hello +Other Navies +United States Navy +Coastguard +Bizjets +Da Comrade +Gunship +As Seen on TV +Oligarch +Oxcart +Distinctive +Joe Cool +Don't you know who I am? +Historic +You came here in that thing? +Hired Gun +Perfectly Serviceable Aircraft +Gas Bags +Vanity Plate +Watch Me Fly +Zoomies +GAF +Special Forces +Aerobatic Teams +UAV +UK National Police Air Service +RAF +Royal Navy Fleet Air Arm +Royal Aircraft +Army Air Corps +Football +Radiohead +Nuclear +USAF +PIA +CAP +United States Marine Corps +Ukraine diff --git a/scripts/check_categories.py b/scripts/check_categories.py new file mode 100644 index 00000000..7e3840a6 --- /dev/null +++ b/scripts/check_categories.py @@ -0,0 +1,46 @@ +""""script to flag new/invalid categories""" + +import logging +import pandas as pd +import sys + +logging.basicConfig( + format="%(asctime)s %(levelname)-8s [%(name)s] %(message)s", level=logging.INFO +) + +if __name__ == "__main__": + logging.info("Reading the main csv file...") + df = pd.read_csv("plane-alert-db.csv") + + logging.info("Reading the Ukraine csv file...") + ukraine_df = pd.read_csv("plane-alert-ukraine.csv") + + category_unique_df = ( + pd.concat([df["Category"], ukraine_df["Category"]]) + .drop_duplicates() + .reset_index(drop=False) + ) + category_unique_df = category_unique_df.drop('index', axis=1) + logging.info(f"Total Categories in PR Count: ({category_unique_df.shape[0]}).") + + logging.info("Reading the export category csv file...") + valid_df = pd.read_csv("plane-alert-categories.csv") + + if not valid_df.equals(category_unique_df): + logging.info("Invalid category used!") + merged_df = valid_df.merge(category_unique_df, indicator=True, how='outer') + changed_df = merged_df[merged_df['_merge'] == 'right_only'] + changed_df = changed_df.drop('_merge', axis=1) + + logging.info( + "New Categories found ({}):\n{}".format( + changed_df.shape[0], + changed_df.to_string(header=False, index=False), + ) + ) + + sys.stdout.write( + f"The files contain invalid or new Categories:\n" + ) + sys.exit(1) + logging.info("Categories check good!") \ No newline at end of file diff --git a/scripts/export_categories.py b/scripts/export_categories.py new file mode 100644 index 00000000..99024143 --- /dev/null +++ b/scripts/export_categories.py @@ -0,0 +1,31 @@ +""""script to export current categories""" + +import logging +import pandas as pd + +logging.basicConfig( + format="%(asctime)s %(levelname)-8s [%(name)s] %(message)s", level=logging.INFO +) + +if __name__ == "__main__": + logging.info("Reading the main csv file...") + df = pd.read_csv("plane-alert-db.csv") + + logging.info("Reading the Ukraine csv file...") + ukraine_df = pd.read_csv("plane-alert-ukraine.csv") + + category_unique_df = ( + pd.concat([df["Category"], ukraine_df["Category"]]) + .drop_duplicates() + .reset_index(drop=True) + ) + logging.info(f"Total Categories Count: ({category_unique_df.shape[0]}).") + + category_unique_df.to_csv( + "plane-alert-categories.csv", + mode="wb", + index=False, + header=True, + encoding="utf8", + lineterminator="\n", + ) \ No newline at end of file