diff --git a/.github/labeler.yml b/.github/labeler.yml index 14aff6ba..a6eadde1 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -6,10 +6,6 @@ images: - any: - changed-files: - any-glob-to-any-file: ['plane_images.csv'] -ukraine: -- any: - - changed-files: - - any-glob-to-any-file: ['plane-alert-ukraine.csv'] planes: - all: - changed-files: diff --git a/.github/workflows/check_categories.yaml b/.github/workflows/check_categories.yaml index 66a3b865..60e3a11a 100644 --- a/.github/workflows/check_categories.yaml +++ b/.github/workflows/check_categories.yaml @@ -5,7 +5,6 @@ on: paths: - "plane-alert-db.csv" - "plane-alert-pia.csv" - - "plane-alert-ukraine.csv" jobs: checkCategories: diff --git a/.github/workflows/check_derivatives_change.yml b/.github/workflows/check_derivatives_change.yml index 031d5a83..ec46d181 100644 --- a/.github/workflows/check_derivatives_change.yml +++ b/.github/workflows/check_derivatives_change.yml @@ -25,7 +25,6 @@ jobs: *.csv !plane-alert-db.csv !plane-alert-pia.csv - !plane-alert-ukraine.csv !plane_images.csv separator: "," diff --git a/.github/workflows/check_main_databases.yml b/.github/workflows/check_main_databases.yml index 250d9792..a8bec539 100644 --- a/.github/workflows/check_main_databases.yml +++ b/.github/workflows/check_main_databases.yml @@ -5,7 +5,6 @@ on: paths: - "plane-alert-db.csv" - "plane-alert-pia.csv" - - "plane-alert-ukraine.csv" - "plane_images.csv" jobs: diff --git a/.github/workflows/create_db_derivatives.yaml b/.github/workflows/create_db_derivatives.yaml index 63dfb55c..910a95a3 100644 --- a/.github/workflows/create_db_derivatives.yaml +++ b/.github/workflows/create_db_derivatives.yaml @@ -8,7 +8,6 @@ on: branches: ["main"] paths: - "plane-alert-db.csv" - - "plane-alert-ukraine.csv" - "plane_images.csv" - "scripts/create_db_derivatives.py" diff --git a/plane-alert-ukraine-images.csv b/plane-alert-ukraine-images.csv deleted file mode 100644 index f3f48f2e..00000000 --- a/plane-alert-ukraine-images.csv +++ /dev/null @@ -1 +0,0 @@ -$ICAO,$Registration,$Operator,$Type,$ICAO Type,#CMPG,$Tag 1,$#Tag 2,$#Tag 3,Category,$#Link,#ImageLink,#ImageLink2,#ImageLink3,#ImageLink4 diff --git a/plane-alert-ukraine.csv b/plane-alert-ukraine.csv deleted file mode 100644 index 232a5c36..00000000 --- a/plane-alert-ukraine.csv +++ /dev/null @@ -1 +0,0 @@ -$ICAO,$Registration,$Operator,$Type,$ICAO Type,#CMPG,$Tag 1,$#Tag 2,$#Tag 3,Category,$#Link diff --git a/readme.mustache b/readme.mustache index b48ab0ad..ff9b0486 100644 --- a/readme.mustache +++ b/readme.mustache @@ -11,7 +11,7 @@ This README is dynamically generated by the `update_readme.yml` action using the This project consists of lists of 'interesting' aircraft, formatted as CSV files. The list is designed to work with the excellent ****. > **Warning** -> Please only suggest/make any changes to the [plane-alert-db.csv](plane-alert-db.csv), [plane-alert-pia.csv](plane-alert-pia.csv), [plane-alert-ukraine.csv](plane-alert-ukraine.csv) and [plane_images.csv](plane_images.csv) files on GitHub - all other files (except PIA) are generated from this file, and if you do not make your changes there, they will be overwritten and lost. Additionally, it is **not recommended** to edit the CSV files in Microsoft Excel, as Excel will attempt to "fix" some ICAO hexes and other fields. It's better to use a code editor such as VS Studio Code—you can access the web version of Code by pressing the period key . when viewing the file you wish to edit. +> Please only suggest/make any changes to the [plane-alert-db.csv](plane-alert-db.csv), [plane-alert-pia.csv](plane-alert-pia.csv), and [plane_images.csv](plane_images.csv) files on GitHub - all other files (except PIA) are generated from this file, and if you do not make your changes there, they will be overwritten and lost. Additionally, it is **not recommended** to edit the CSV files in Microsoft Excel, as Excel will attempt to "fix" some ICAO hexes and other fields. It's better to use a code editor such as VS Studio Code—you can access the web version of Code by pressing the period key . when viewing the file you wish to edit. ## TOC @@ -29,7 +29,6 @@ These aircraft are divided into four main databases: - [plane-alert-db.csv](plane-alert-db.csv) - A list of interesting aircraft with tags, categories and links. ({{plane_alert_db}}) - [plane-alert-pia.csv](plane-alert-pia.csv): A list that contains PIA planes. ({{plane_alert_pia}}) -- [plane-alert-ukraine.csv](plane-alert-ukraine.csv): A list with Ukrainian planes. ({{plane_alert_ukraine}}) - [plane_images.csv](plane_images.csv): A accompanying list that contains aircraft images. ({{plane_alert_images}}) Based on these main databases, several derivative databases are created using a [GitHub action](https://github.com/sdr-enthusiasts/plane-alert-db/actions/workflows/create_db_derivatives.yaml): @@ -42,7 +41,6 @@ Based on these main databases, several derivative databases are created using a A second version of each of the above lists contains up to 4 image links per aircraft. These lists are created in [GitHub action](https://github.com/sdr-enthusiasts/plane-alert-db/actions/workflows/create_db_derivatives.yaml) using the [plane_images.csv](plane_images.csv) database. **Please consider this experimental, do not come to rely on any of the image links** - [plane-alert-db-images.csv](plane-alert-db-images.csv) -- [plane-alert-ukraine-images.csv](plane-alert-ukraine.csv) - [plane-alert-civ-images.csv](plane-alert-civ-images.csv) - [plane-alert-mil-images.csv](plane-alert-mil-images.csv) - [plane-alert-pol-images.csv](plane-alert-pol-images.csv) @@ -50,7 +48,7 @@ A second version of each of the above lists contains up to 4 image links per air Note, we used to create a seperate list, `plane-alert-twitter-blocked.csv`, for use with Planefence's Twitter posting functionality. This list would prevent certain aircraft from being posted to Twitter in an attempt to keep the posting bot account from being banned. Since Twitter has now made it all but impossible for users to make bots for free, we've stopped creating this list. -This [Dashboard](https://lookerstudio.google.com/reporting/46ff4328-09d3-4e65-ab5a-bd2ba27a18fd) contains details of the [main](https://github.com/sdr-enthusiasts/plane-alert-db/blob/main/plane-alert-db.csv) and [Ukraine](https://github.com/sdr-enthusiasts/plane-alert-db/blob/main/plane-alert-ukraine.csv) lists. +This [Dashboard](https://lookerstudio.google.com/reporting/46ff4328-09d3-4e65-ab5a-bd2ba27a18fd) contains details of the [main](https://github.com/sdr-enthusiasts/plane-alert-db/blob/main/plane-alert-db.csv) list. ## Description of Categories diff --git a/scripts/check_categories.py b/scripts/check_categories.py index 7e3840a6..86c36d04 100644 --- a/scripts/check_categories.py +++ b/scripts/check_categories.py @@ -12,11 +12,8 @@ logging.info("Reading the main csv file...") df = pd.read_csv("plane-alert-db.csv") - logging.info("Reading the Ukraine csv file...") - ukraine_df = pd.read_csv("plane-alert-ukraine.csv") - category_unique_df = ( - pd.concat([df["Category"], ukraine_df["Category"]]) + df["Category"] .drop_duplicates() .reset_index(drop=False) ) diff --git a/scripts/check_invalid_derivatives.py b/scripts/check_invalid_derivatives.py index c5e4b386..ae58e9db 100644 --- a/scripts/check_invalid_derivatives.py +++ b/scripts/check_invalid_derivatives.py @@ -15,7 +15,6 @@ MAIN_DB_FILES = [ "plane-alert-db.csv", "plane-alert-pia.csv", - "plane-alert-ukraine.csv", "plane_images.csv", ] OUTPUT_FILE = os.getenv("GITHUB_OUTPUT") diff --git a/scripts/check_james_planes.py b/scripts/check_james_planes.py index 8a361229..7dd8f07e 100644 --- a/scripts/check_james_planes.py +++ b/scripts/check_james_planes.py @@ -17,17 +17,6 @@ main_df = pd.read_csv("plane-alert-db.csv") logging.info("'plane-alert-db' csv file read successfully.") - logging.info("Reading the 'plane-alert-twitter-blocked' csv file...") - twitter_blocker_df = pd.read_csv("plane-alert-twitter-blocked.csv") - logging.info("'plane-alert-twitter-blocked' csv file read successfully.") - - logging.info("Reading 'plane-alert-ukraine' csv file...") - ukraine_df = pd.read_csv("plane-alert-ukraine.csv") - logging.info("'plane-alert-ukraine' csv file read successfully.") - - logging.info("Concatenating all the dataframes...") - main_df = pd.concat([main_df, twitter_blocker_df, ukraine_df]) - logging.info("Concatenation done.") logging.info( f"Get new items in 'james-planes' database ({james_planes_df.shape[0]})..." diff --git a/scripts/check_main_databases.py b/scripts/check_main_databases.py index ecaab433..69b3bb00 100644 --- a/scripts/check_main_databases.py +++ b/scripts/check_main_databases.py @@ -12,7 +12,6 @@ ) MAIN_DATABASE_NAME = "plane-alert-db.csv" -UKRAINE_DATABASE_NAME = "plane-alert-ukraine.csv" PLANE_IMAGES_DATABASE_NAME = "plane_images.csv" @@ -170,26 +169,6 @@ def contains_valid_ICAO_hexes(df): logging.info("The main database is valid.") - ########################################## - # Check 'plane-alert-ukraine' db. # - ########################################## - logging.info("Checking the 'plane-alert-ukraine' database...") - try: - ukraine_df = pd.read_csv(UKRAINE_DATABASE_NAME) - ukraine_df.name = UKRAINE_DATABASE_NAME - except Exception as e: - logging.error(f"The '{UKRAINE_DATABASE_NAME}' database is not a valid CSV.") - sys.stdout.write( - f"The '{UKRAINE_DATABASE_NAME}' database is not a valid CSV: {e}\n" - ) - sys.exit(1) - - # Preform database checks. - contains_duplicate_ICAOs(ukraine_df) - contains_duplicate_regs(ukraine_df) - contains_bad_links(ukraine_df) - logging.info("The 'plane-alert-ukraine' database is valid.") - ########################################## # Check 'plane_images.csv' db. # ########################################## diff --git a/scripts/create_db_derivatives.py b/scripts/create_db_derivatives.py index f8bfbc33..8a233dab 100644 --- a/scripts/create_db_derivatives.py +++ b/scripts/create_db_derivatives.py @@ -1,7 +1,6 @@ """This script creates (derivative) category and images CSV database files from the main 'plane-alert-db.csv' database file. The categories are created based on the 'CMPG' -column, while images are added using the 'plane_images.csv' reference file. It also -creates the 'plane-alert-ukraine-images.csv' database file. Lastly, based on the +column, while images are added using the 'plane_images.csv' reference file. Based on the 'plane-alert-db.csv' database file, missing records are added and/or extra records are removed from the 'plane_images.csv' reference file. """ @@ -69,28 +68,6 @@ ) logging.info("Category and category images CSV files created successfully.") - logging.info("Reading the Ukraine csv file...") - unsort_ukraine_df = pd.read_csv("plane-alert-ukraine.csv") - ukraine_df = unsort_ukraine_df.sort_values(by=["$ICAO"], ascending=True) - ukraine_df.to_csv( - "plane-alert-ukraine.csv", - mode="wb", - index=False, - header=True, - encoding="utf8", - lineterminator="\n", - ) - logging.info("Ukraine csv file read and sorted successfully.") - logging.info("Creating the Ukraine database images CSV file...") - ukraine_df_images = pd.merge(ukraine_df, images_df, how="left", on="$ICAO") - ukraine_df_images.to_csv( - "plane-alert-ukraine-images.csv", - index=False, - mode="wb", - encoding="utf8", - lineterminator="\n", - ) - logging.info("Ukraine database images CSV file created successfully.") logging.info("Creating the main database images csv file...") main_images_df = pd.merge(df, images_df, how="left", on="$ICAO") @@ -105,14 +82,14 @@ logging.info("Category and images CSV files created successfully.") logging.info( - "Check for new ICAOs in DB files and add them to the images reference file..." + "Check for new ICAOs in DB file and add them to the images reference file..." ) plane_alert_df = ( - pd.concat([df["$ICAO"], ukraine_df["$ICAO"]]) + df["$ICAO"] .drop_duplicates() .reset_index(drop=True) ) - logging.info(f"ICAOs retrieved from DB files: ({plane_alert_df.shape[0]}).") + logging.info(f"ICAOs retrieved from DB file: ({plane_alert_df.shape[0]}).") logging.info(f"ICAOs retrieved from 'plane_images.csv ({images_df.shape[0]}).") new_ICAOs_df = plane_alert_df[~plane_alert_df.isin(images_df["$ICAO"])] if new_ICAOs_df.shape[0] > 0: diff --git a/scripts/export_categories.py b/scripts/export_categories.py index a8234e49..23ae08de 100644 --- a/scripts/export_categories.py +++ b/scripts/export_categories.py @@ -11,11 +11,8 @@ logging.info("Reading the main csv file...") df = pd.read_csv("plane-alert-db.csv") - logging.info("Reading the Ukraine csv file...") - ukraine_df = pd.read_csv("plane-alert-ukraine.csv") - category_unique_df = ( - pd.concat([df["Category"], ukraine_df["Category"]]) + df["Category"] .drop_duplicates() .reset_index(drop=True) ) diff --git a/scripts/get_unique_bangers_best_items.py b/scripts/get_unique_bangers_best_items.py index 524c3d16..ecbbabe0 100644 --- a/scripts/get_unique_bangers_best_items.py +++ b/scripts/get_unique_bangers_best_items.py @@ -14,18 +14,6 @@ main_df = pd.read_csv("plane-alert-db.csv") logging.info("'plane-alert-db' csv file read successfully.") - logging.info("Reading the 'plane-alert-twitter-blocked' csv file...") - twitter_blocker_df = pd.read_csv("plane-alert-twitter-blocked.csv") - logging.info("'plane-alert-twitter-blocked' csv file read successfully.") - - logging.info("Reading 'plane-alert-ukraine' csv file...") - ukraine_df = pd.read_csv("plane-alert-ukraine.csv") - logging.info("'plane-alert-ukraine' csv file read successfully.") - - logging.info("Concatenating all the dataframes...") - main_df = pd.concat([main_df, twitter_blocker_df, ukraine_df]) - logging.info("Concatenation done.") - logging.info("Reading 'badgers-best' csv file...") bangers = pd.read_csv("badgers-best.csv") logging.info("'badgers-best' csv file read successfully.") @@ -36,4 +24,4 @@ logging.info("No new items found.") else: logging.info("New items found.") - logging.info(new_items) + logging.info(new_items) \ No newline at end of file diff --git a/scripts/update_readme.py b/scripts/update_readme.py index 78d23978..518751f9 100644 --- a/scripts/update_readme.py +++ b/scripts/update_readme.py @@ -13,9 +13,6 @@ logging.info("Reading the main csv file...") df = pd.read_csv("plane-alert-db.csv") - logging.info("Reading the Ukraine csv file...") - ukraine_df = pd.read_csv("plane-alert-ukraine.csv") - logging.info("Reading the PIA csv file...") pia_df = pd.read_csv("plane-alert-pia.csv") @@ -24,20 +21,20 @@ logging.info("All csv files read successfully.") plane_count_df = ( - pd.concat([df["$ICAO"], ukraine_df["$ICAO"]]) + df["$ICAO"] .drop_duplicates() .reset_index(drop=True) ) logging.info(f"Total Planes Count: ({plane_count_df.shape[0]}).") category_unique_df = ( - pd.concat([df["Category"], ukraine_df["Category"]]) + df["Category"] .drop_duplicates() .reset_index(drop=True) ) logging.info(f"Total Categories Count: ({category_unique_df.shape[0]}).") - category_df = pd.concat([df["Category"], ukraine_df["Category"]]).reset_index( + category_df = df["Category"].reset_index( drop=False ) @@ -53,7 +50,6 @@ "categories": category_unique_df.shape[0], "plane_alert_db": df.shape[0], "plane_alert_pia": pia_df.shape[0], - "plane_alert_ukraine": ukraine_df.shape[0], "plane_alert_images": images_df.dropna( subset=["#ImageLink"], inplace=False ).shape[0], @@ -203,9 +199,6 @@ "cap_count": category_df[ category_df["Category"] == "CAP" ].shape[0], - "ukraine_count": category_df[ - category_df["Category"] == "Ukraine" - ].shape[0], }, ) )