diff --git a/.coverage b/.coverage new file mode 100644 index 0000000..6849e4c Binary files /dev/null and b/.coverage differ diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..db9e9a9 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,18 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/codespaces-linux/.devcontainer/base.Dockerfile + +FROM mcr.microsoft.com/vscode/devcontainers/universal:2-focal + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends +RUN apt-get update && apt-get -y install --no-install-recommends \ + python3.8-venv \ + gcc + +ARG USER="codespace" +ARG VENV_PATH="/home/${USER}/venv" +COPY requirements.txt /tmp/ +COPY Makefile /tmp/ +RUN su $USER -c "/usr/bin/python3 -m venv /home/${USER}/venv" \ + && su $USER -c "${VENV_PATH}/bin/pip --disable-pip-version-check --no-cache-dir install -r /tmp/requirements.txt" \ + && rm -rf /tmp/requirements.txt \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..b422ac6 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,79 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/codespaces-linux +{ + "name": "GitHub Codespaces (Default)", + + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + "features": { + "ghcr.io/devcontainers/features/nvidia-cuda:1": { + "installCudnn": true + } + }, + + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "go.toolsManagement.checkForUpdates": "local", + "go.useLanguageServer": true, + "go.gopath": "/go", + "python.defaultInterpreterPath": "/home/codespace/venv/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/home/codespace/venv/bin/autopep8", + "python.formatting.blackPath": "/home/codespace/venv/bin/black", + "python.formatting.yapfPath": "/home/codespace/venv/bin/yapf", + "python.linting.banditPath": "/home/codespace/venv/bin/bandit", + "python.linting.flake8Path": "/home/codespace/venv/bin/flake8", + "python.linting.mypyPath": "/home/codespace/venv/bin/mypy", + "python.linting.pycodestylePath": "/home/codespace/venv/bin/pycodestyle", + "python.linting.pydocstylePath": "/home/codespace/venv/bin/pydocstyle", + "python.linting.pylintPath": "/home/codespace/venv/bin/pylint", + "lldb.executable": "/usr/bin/lldb", + "files.watcherExclude": { + "**/target/**": true + } + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "GitHub.vscode-pull-request-github", + "GitHub.copilot-nightly", + "GitHub.copilot-labs", + "ms-azuretools.vscode-docker", + "ms-toolsai.jupyter", + "ms-toolsai.jupyter-keymap", + "ms-toolsai.jupyter-renderers", + "ms-python.vscode-pylance", + "ms-python.python", + "ms-vscode.makefile-tools" + ] + } + }, + + "remoteUser": "codespace", + + "overrideCommand": false, + + "mounts": ["source=codespaces-linux-var-lib-docker,target=/var/lib/docker,type=volume"], + + "runArgs": [ + "--cap-add=SYS_PTRACE", + "--security-opt", + "seccomp=unconfined", + "--privileged", + "--init" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // "oryx build" will automatically install your dependencies and attempt to build your project + //"postCreateCommand": "oryx build -p virtualenv_name=.venv --log-file /tmp/oryx-build.log --manifest-dir /tmp || echo 'Could not auto-build. Skipping.'" + "postCreateCommand": "bash setup.sh" +} \ No newline at end of file diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml new file mode 100644 index 0000000..4f2441d --- /dev/null +++ b/.github/workflows/cicd.yml @@ -0,0 +1,28 @@ +name: CI +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: install packages + run: make install + - name: lint + run: make lint + - name: extract + run: make extract + - name: load + run: make load + - name: format + run: make format + - name: test + run: make test + - name: generate_and_push + run: make generate_and_push + \ No newline at end of file diff --git a/AirlineSafetyDB.db b/AirlineSafetyDB.db new file mode 100644 index 0000000..220856c Binary files /dev/null and b/AirlineSafetyDB.db differ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c55d399 --- /dev/null +++ b/Makefile @@ -0,0 +1,34 @@ +install: + pip install --upgrade pip && pip install -r requirements.txt + +format: + black *.py + +lint: + ruff check *.py + +test: + python -m pytest -vv --cov=main --cov=myLib test_*.py + +all: install format lint test + +generate_and_push: + # Create the markdown file + python test_main.py + + # Add, commit, and push the generated files to GitHub + @if [ -n "$$(git status --porcelain)" ]; then \ + git config --local user.email "action@github.com"; \ + git config --local user.name "GitHub Action"; \ + git add .; \ + git commit -m "Add SQL log as query_output.md"; \ + git push; \ + else \ + echo "No changes to commit. Skipping commit and push."; \ + fi + +extract: + python main.py extract + +load: + python main.py transform_load \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..8ebc8b2 --- /dev/null +++ b/README.md @@ -0,0 +1,107 @@ +[![CI](https://github.com/nogibjj/Mobasserul_Haque_MiniProject5/actions/workflows/cicd.yml/badge.svg)](https://github.com/nogibjj/Mobasserul_Haque_MiniProject5/actions/workflows/cicd.yml) + +# Airline Safety Database ETL and Query Tool + +This project provides an ETL (Extract, Transform, Load) and querying tool for managing and analyzing the Airline Safety Database. It is built using Python and SQLite, enabling users to perform various operations on airline safety records, including extraction, loading, updating, deleting, creating, and querying records. + +## Features + +- **ETL Operations**: + - Extract data from a source. + - Transform and load data into the SQLite database. + +- **Query Operations**: + - Update existing records in the database. + - Delete records based on a unique identifier. + - Create new records in the database. + - Execute custom SQL queries. + - Read a limited number of records from the database. + +- **Logging and Output**: + - All executed queries are logged in a markdown file for reference. + - Query results are outputted in a formatted markdown file for easier readability. + +## Directory Structure + +``` +├── .devcontainer/ +│ ├── devcontainer.json +│ └── Dockerfile +├── .github/ +│ └── workflows/cicd.yml +├── data/ +│ └── airline_safety.csv +├── myLib/ +│ ├── __init__.py +│ ├── __pycache__/ +│ ├── extract.py +│ ├── query.py +│ └── transform_load.py +├── AirlineSafetyDB.db +├── main.py +├── Makefile +├── query_log.md +├── query_output.md +├── README.md +├── ServeTimesDB.db requirements.txt +└── test_main.py +``` +## CRUD operations : + +## Usage + +Run the script using the following command: + +```python +python main.py [arguments] +``` +## Arguments: + +`record_id`, `airline`, `avail_seat_km_per_week`, `incidents_85_99`, `fatal_accidents_85_99`, `fatalities_85_99`, `incidents_00_14`, `fatal_accidents_00_14`, `fatalities_00_14` + +## Actions: + +extract: Extract data from the source. + +```python +python main.py extract +``` +transform_load: Transform and load data into the database. + +```python +python main.py transform_load +``` +update_record: Update an existing record in the database. + +```python +python main.py update_record(1, "Air Canada", 2000000000, 3, 1, 5, 2, 0, 0) +``` +create_record: Create a new record in the database + +```python +python main.py create_record("Air Canada", 1865253802, 2, 0, 0, 2, 0, 0) +``` +delete_record: delete an existing record in the database. + +```python +python main.py delete_record(1) +``` +read_data: Read and display the top N rows from the database. + +```python +python main.py read_data(10) # Reads the top 10 rows +``` +general_query: Run a custom SQL query on the database. + +```python +python main.py "SELECT * FROM AirlineSafety WHERE airline = 'Aeroflot*'" +``` + +## Testing +To run the test suite, use: + +```python +python -m pytest -vv --cov=main --cov=myLib test_*.py +``` +## References +https://github.com/nogibjj/sqlite-lab \ No newline at end of file diff --git a/__pycache__/main.cpython-312.pyc b/__pycache__/main.cpython-312.pyc new file mode 100644 index 0000000..460d822 Binary files /dev/null and b/__pycache__/main.cpython-312.pyc differ diff --git a/__pycache__/test_main.cpython-310-pytest-7.1.3.pyc b/__pycache__/test_main.cpython-310-pytest-7.1.3.pyc new file mode 100644 index 0000000..2685271 Binary files /dev/null and b/__pycache__/test_main.cpython-310-pytest-7.1.3.pyc differ diff --git a/data/airline-safety.csv b/data/airline-safety.csv new file mode 100644 index 0000000..1ae0712 --- /dev/null +++ b/data/airline-safety.csv @@ -0,0 +1,57 @@ +airline,avail_seat_km_per_week,incidents_85_99,fatal_accidents_85_99,fatalities_85_99,incidents_00_14,fatal_accidents_00_14,fatalities_00_14 +Aer Lingus,320906734,2,0,0,0,0,0 +Aeroflot*,1197672318,76,14,128,6,1,88 +Aerolineas Argentinas,385803648,6,0,0,1,0,0 +Aeromexico*,596871813,3,1,64,5,0,0 +Air Canada,1865253802,2,0,0,2,0,0 +Air France,3004002661,14,4,79,6,2,337 +Air India*,869253552,2,1,329,4,1,158 +Air New Zealand*,710174817,3,0,0,5,1,7 +Alaska Airlines*,965346773,5,0,0,5,1,88 +Alitalia,698012498,7,2,50,4,0,0 +All Nippon Airways,1841234177,3,1,1,7,0,0 +American*,5228357340,21,5,101,17,3,416 +Austrian Airlines,358239823,1,0,0,1,0,0 +Avianca,396922563,5,3,323,0,0,0 +British Airways*,3179760952,4,0,0,6,0,0 +Cathay Pacific*,2582459303,0,0,0,2,0,0 +China Airlines,813216487,12,6,535,2,1,225 +Condor,417982610,2,1,16,0,0,0 +COPA,550491507,3,1,47,0,0,0 +Delta / Northwest*,6525658894,24,12,407,24,2,51 +Egyptair,557699891,8,3,282,4,1,14 +El Al,335448023,1,1,4,1,0,0 +Ethiopian Airlines,488560643,25,5,167,5,2,92 +Finnair,506464950,1,0,0,0,0,0 +Garuda Indonesia,613356665,10,3,260,4,2,22 +Gulf Air,301379762,1,0,0,3,1,143 +Hawaiian Airlines,493877795,0,0,0,1,0,0 +Iberia,1173203126,4,1,148,5,0,0 +Japan Airlines,1574217531,3,1,520,0,0,0 +Kenya Airways,277414794,2,0,0,2,2,283 +KLM*,1874561773,7,1,3,1,0,0 +Korean Air,1734522605,12,5,425,1,0,0 +LAN Airlines,1001965891,3,2,21,0,0,0 +Lufthansa*,3426529504,6,1,2,3,0,0 +Malaysia Airlines,1039171244,3,1,34,3,2,537 +Pakistan International,348563137,8,3,234,10,2,46 +Philippine Airlines,413007158,7,4,74,2,1,1 +Qantas*,1917428984,1,0,0,5,0,0 +Royal Air Maroc,295705339,5,3,51,3,0,0 +SAS*,682971852,5,0,0,6,1,110 +Saudi Arabian,859673901,7,2,313,11,0,0 +Singapore Airlines,2376857805,2,2,6,2,1,83 +South African,651502442,2,1,159,1,0,0 +Southwest Airlines,3276525770,1,0,0,8,0,0 +Sri Lankan / AirLanka,325582976,2,1,14,4,0,0 +SWISS*,792601299,2,1,229,3,0,0 +TACA,259373346,3,1,3,1,1,3 +TAM,1509195646,8,3,98,7,2,188 +TAP - Air Portugal,619130754,0,0,0,0,0,0 +Thai Airways,1702802250,8,4,308,2,1,1 +Turkish Airlines,1946098294,8,3,64,8,2,84 +United / Continental*,7139291291,19,8,319,14,2,109 +US Airways / America West*,2455687887,16,7,224,11,2,23 +Vietnam Airlines,625084918,7,3,171,1,0,0 +Virgin Atlantic,1005248585,1,0,0,0,0,0 +Xiamen Airlines,430462962,9,1,82,2,0,0 \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..84931f0 --- /dev/null +++ b/main.py @@ -0,0 +1,108 @@ +import sys +import argparse +from myLib.extract import extract +from myLib.transform_load import load +from myLib.query import ( + update_record, + delete_record, + create_record, + general_query, + read_data, +) + + +def handle_arguments(args): + """Add action based on initial calls""" + parser = argparse.ArgumentParser(description="ETL-Query script for AirlineSafetyDB") + parser.add_argument( + "action", + choices=[ + "extract", + "transform_load", + "update_record", + "delete_record", + "create_record", + "general_query", + "read_data", + ], + ) + args = parser.parse_args(args[:1]) + print(args.action) + + if args.action == "update_record": + parser.add_argument("record_id", type=int) + parser.add_argument("airline") + parser.add_argument("avail_seat_km_per_week", type=int) + parser.add_argument("incidents_85_99", type=int) + parser.add_argument("fatal_accidents_85_99", type=int) + parser.add_argument("fatalities_85_99", type=int) + parser.add_argument("incidents_00_14", type=int) + parser.add_argument("fatal_accidents_00_14", type=int) + parser.add_argument("fatalities_00_14", type=int) + + if args.action == "create_record": + parser.add_argument("airline") + parser.add_argument("avail_seat_km_per_week", type=int) + parser.add_argument("incidents_85_99", type=int) + parser.add_argument("fatal_accidents_85_99", type=int) + parser.add_argument("fatalities_85_99", type=int) + parser.add_argument("incidents_00_14", type=int) + parser.add_argument("fatal_accidents_00_14", type=int) + parser.add_argument("fatalities_00_14", type=int) + + if args.action == "general_query": + parser.add_argument("query") + + if args.action == "delete_record": + parser.add_argument("record_id", type=int) + + # Parse again with every argument + return parser.parse_args(sys.argv[1:]) + + +def main(): + """Handles all the CLI commands""" + args = handle_arguments(sys.argv[1:]) + + if args.action == "extract": + print("Extracting data...") + extract() + elif args.action == "transform_load": + print("Transforming data...") + load() + elif args.action == "update_record": + update_record( + args.record_id, + args.airline, + args.avail_seat_km_per_week, + args.incidents_85_99, + args.fatal_accidents_85_99, + args.fatalities_85_99, + args.incidents_00_14, + args.fatal_accidents_00_14, + args.fatalities_00_14, + ) + elif args.action == "delete_record": + delete_record(args.record_id) + elif args.action == "create_record": + create_record( + args.airline, + args.avail_seat_km_per_week, + args.incidents_85_99, + args.fatal_accidents_85_99, + args.fatalities_85_99, + args.incidents_00_14, + args.fatal_accidents_00_14, + args.fatalities_00_14, + ) + elif args.action == "general_query": + general_query(args.query) + elif args.action == "read_data": + data = read_data() + print(data) + else: + print(f"Unknown action: {args.action}") + + +if __name__ == "__main__": + main() diff --git a/myLib/__init__.py b/myLib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/myLib/__pycache__/__init__.cpython-310.pyc b/myLib/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..4e3ef1c Binary files /dev/null and b/myLib/__pycache__/__init__.cpython-310.pyc differ diff --git a/myLib/__pycache__/__init__.cpython-312.pyc b/myLib/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..c7cbc13 Binary files /dev/null and b/myLib/__pycache__/__init__.cpython-312.pyc differ diff --git a/myLib/__pycache__/extract.cpython-310.pyc b/myLib/__pycache__/extract.cpython-310.pyc new file mode 100644 index 0000000..8bae1a4 Binary files /dev/null and b/myLib/__pycache__/extract.cpython-310.pyc differ diff --git a/myLib/__pycache__/extract.cpython-312.pyc b/myLib/__pycache__/extract.cpython-312.pyc new file mode 100644 index 0000000..f165182 Binary files /dev/null and b/myLib/__pycache__/extract.cpython-312.pyc differ diff --git a/myLib/__pycache__/query.cpython-310.pyc b/myLib/__pycache__/query.cpython-310.pyc new file mode 100644 index 0000000..b208712 Binary files /dev/null and b/myLib/__pycache__/query.cpython-310.pyc differ diff --git a/myLib/__pycache__/query.cpython-312.pyc b/myLib/__pycache__/query.cpython-312.pyc new file mode 100644 index 0000000..75a71fc Binary files /dev/null and b/myLib/__pycache__/query.cpython-312.pyc differ diff --git a/myLib/__pycache__/transform_load.cpython-310.pyc b/myLib/__pycache__/transform_load.cpython-310.pyc new file mode 100644 index 0000000..781b66b Binary files /dev/null and b/myLib/__pycache__/transform_load.cpython-310.pyc differ diff --git a/myLib/__pycache__/transform_load.cpython-312.pyc b/myLib/__pycache__/transform_load.cpython-312.pyc new file mode 100644 index 0000000..84dc616 Binary files /dev/null and b/myLib/__pycache__/transform_load.cpython-312.pyc differ diff --git a/myLib/extract.py b/myLib/extract.py new file mode 100644 index 0000000..b8aa454 --- /dev/null +++ b/myLib/extract.py @@ -0,0 +1,32 @@ +import requests +import os + +def extract(url="https://raw.githubusercontent.com/fivethirtyeight/data/master/airline-safety/airline-safety.csv", + file_path="data/airline-safety.csv"): + print("Starting extraction process...") + + if not os.path.exists("data"): + print("Creating directory 'data'") + os.makedirs("data") + else: + print("Directory 'data' already exists") + + print(f"Sending request to {url}...") + r = requests.get(url) + + # status code of the response + print(f"HTTP Status Code: {r.status_code}") + + # Check if the request was successful + if r.status_code == 200: + print(f"Request successful! Saving content to {file_path}...") + + with open(file_path, 'wb') as f: + f.write(r.content) + print(f"File saved successfully at {file_path}") + else: + print(f"Failed to download file. Status code: {r.status_code}") + + return file_path + +extract() diff --git a/myLib/query.py b/myLib/query.py new file mode 100644 index 0000000..a34b159 --- /dev/null +++ b/myLib/query.py @@ -0,0 +1,130 @@ +"""Query the AirlineSafetyDB database""" + +import sqlite3 +from tabulate import tabulate + +# Define a global variable for the log file and output file +LOG_FILE = "query_log.md" +OUTPUT_FILE = "query_output.md" + +def log_query(query): + """Adds to a query markdown file""" + with open(LOG_FILE, "a") as file: + file.write(f"```sql\n{query}\n```\n\n") + +def write_to_output(data, headers): + """Writes query results to output.md""" + with open(OUTPUT_FILE, "a") as file: + if data: + file.write(tabulate(data, headers=headers, tablefmt="grid")) + file.write("\n\n") # Add space between different queries + else: + file.write("No data found.\n\n") # In case of no data + +def general_query(query): + """Runs a query the user inputs""" + # Connect to the SQLite database + conn = sqlite3.connect("AirlineSafetyDB.db") + cursor = conn.cursor() + + # Execute the query + cursor.execute(query) + + # If the query modifies the database, commit the changes + if query.strip().lower().startswith(("insert", "update", "delete")): + conn.commit() + + # Log the query + log_query(f"{query}") + + # Close the cursor and connection + cursor.close() + conn.close() + +def create_record(airline, avail_seat_km_per_week, incidents_85_99, fatal_accidents_85_99, + fatalities_85_99, incidents_00_14, fatal_accidents_00_14, fatalities_00_14): + """Create a new record in the AirlineSafetyDB""" + conn = sqlite3.connect("AirlineSafetyDB.db") + cursor = conn.cursor() + cursor.execute( + """ + INSERT INTO AirlineSafety + (airline, avail_seat_km_per_week, incidents_85_99, fatal_accidents_85_99, fatalities_85_99, + incidents_00_14, fatal_accidents_00_14, fatalities_00_14) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + (airline, avail_seat_km_per_week, incidents_85_99, fatal_accidents_85_99, fatalities_85_99, + incidents_00_14, fatal_accidents_00_14, fatalities_00_14), + ) + conn.commit() + conn.close() + + # Log the query + log_query( + f"""INSERT INTO AirlineSafety VALUES ( + {airline}, {avail_seat_km_per_week}, {incidents_85_99}, {fatal_accidents_85_99}, {fatalities_85_99}, + {incidents_00_14}, {fatal_accidents_00_14}, {fatalities_00_14});""" + ) + + +def update_record(record_id, airline, avail_seat_km_per_week, incidents_85_99, fatal_accidents_85_99, + fatalities_85_99, incidents_00_14, fatal_accidents_00_14, fatalities_00_14): + """Update a record in the AirlineSafetyDB""" + conn = sqlite3.connect("AirlineSafetyDB.db") + cursor = conn.cursor() + cursor.execute( + """ + UPDATE AirlineSafety + SET airline=?, avail_seat_km_per_week=?, incidents_85_99=?, fatal_accidents_85_99=?, + fatalities_85_99=?, incidents_00_14=?, fatal_accidents_00_14=?, fatalities_00_14=? + WHERE id=? + """, + ( + airline, avail_seat_km_per_week, incidents_85_99, fatal_accidents_85_99, fatalities_85_99, + incidents_00_14, fatal_accidents_00_14, fatalities_00_14, record_id, + ), + ) + conn.commit() + conn.close() + + # Log the query + log_query( + f"""UPDATE AirlineSafety SET + airline={airline}, avail_seat_km_per_week={avail_seat_km_per_week}, + incidents_85_99={incidents_85_99}, fatal_accidents_85_99={fatal_accidents_85_99}, + fatalities_85_99={fatalities_85_99}, incidents_00_14={incidents_00_14}, + fatal_accidents_00_14={fatal_accidents_00_14}, fatalities_00_14={fatalities_00_14} + WHERE id={record_id};""" + ) + + +def delete_record(record_id): + """Delete a record from the AirlineSafetyDB""" + conn = sqlite3.connect("AirlineSafetyDB.db") + cursor = conn.cursor() + cursor.execute("DELETE FROM AirlineSafety WHERE id=?", (record_id,)) + conn.commit() + conn.close() + + # Log the query + log_query(f"DELETE FROM AirlineSafety WHERE id={record_id};") + + +def read_data(limit=10): + """Read the top N rows from the AirlineSafety table""" + conn = sqlite3.connect("AirlineSafetyDB.db") + cursor = conn.cursor() + cursor.execute(f"SELECT * FROM AirlineSafety LIMIT {limit}") + data = cursor.fetchall() + + # Get column names + col_names = [description[0] for description in cursor.description] + + # Log the query + log_query(f"SELECT * FROM AirlineSafety LIMIT {limit};") + + write_to_output(data, col_names) # Write results to query_output.md + + conn.close() + return data + diff --git a/myLib/transform_load.py b/myLib/transform_load.py new file mode 100644 index 0000000..c70883c --- /dev/null +++ b/myLib/transform_load.py @@ -0,0 +1,62 @@ +""" +Transforms and Loads airline safety data into a local SQLite3 database +""" +import sqlite3 +import csv + + +# Load the csv file and insert data into a new SQLite3 database +def load(dataset="data/airline-safety.csv"): + """Transforms and Loads airline safety data into the local SQLite3 database""" + + # Open the dataset and skip the header + with open(dataset, newline="") as csvfile: + payload = csv.reader(csvfile, delimiter=",") + next(payload) # Skips the header row + + # Connect to the SQLite database (or create it if it doesn't exist) + conn = sqlite3.connect("AirlineSafetyDB.db") + c = conn.cursor() + + # Drop the table if it already exists to avoid duplicate data + c.execute("DROP TABLE IF EXISTS AirlineSafety") + + # Create a new table for airline safety data + c.execute( + """ + CREATE TABLE AirlineSafety ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + airline TEXT, + avail_seat_km_per_week INTEGER, + incidents_85_99 INTEGER, + fatal_accidents_85_99 INTEGER, + fatalities_85_99 INTEGER, + incidents_00_14 INTEGER, + fatal_accidents_00_14 INTEGER, + fatalities_00_14 INTEGER + ) + """ + ) + + # Insert data into the table + c.executemany( + """ + INSERT INTO AirlineSafety ( + airline, + avail_seat_km_per_week, + incidents_85_99, + fatal_accidents_85_99, + fatalities_85_99, + incidents_00_14, + fatal_accidents_00_14, + fatalities_00_14 + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + payload + ) + + conn.commit() + conn.close() + + return "AirlineSafetyDB.db" diff --git a/query_log.md b/query_log.md new file mode 100644 index 0000000..1bcc730 --- /dev/null +++ b/query_log.md @@ -0,0 +1,234 @@ +```sql +SELECT * FROM AirlineSafety LIMIT 10; +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + Air Canada, 1865253802, 2, 0, 0, + 2, 0, 0); +``` + +```sql +UPDATE AirlineSafety SET + airline=Air Canada, avail_seat_km_per_week=2000000000, + incidents_85_99=3, fatal_accidents_85_99=1, + fatalities_85_99=5, incidents_00_14=2, + fatal_accidents_00_14=0, fatalities_00_14=0 + WHERE id=1; +``` + +```sql +DELETE FROM AirlineSafety WHERE id=1; +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 5; +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 10; +``` + +```sql +SELECT * FROM AirlineSafety WHERE airline = 'Air Canada'; +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + Air Canada, 1865253802, 2, 0, 0, + 2, 0, 0); +``` + +```sql +UPDATE AirlineSafety SET + airline=Air Canada, avail_seat_km_per_week=1900000000, + incidents_85_99=3, fatal_accidents_85_99=1, + fatalities_85_99=5, incidents_00_14=1, + fatal_accidents_00_14=0, fatalities_00_14=0 + WHERE id=1; +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + Airline A, 500000, 5, 0, 0, + 2, 0, 0); +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + Airline B, 700000, 8, 1, 10, + 3, 0, 0); +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + Airline C, 800000, 10, 2, 20, + 5, 1, 1); +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 5; +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + Airline A, 500000, 5, 0, 0, + 2, 0, 0); +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + Airline B, 700000, 8, 1, 10, + 3, 0, 0); +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + Airline C, 800000, 10, 2, 20, + 5, 1, 1); +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 5; +``` + +```sql +UPDATE AirlineSafety SET + airline=Updated Airline A, avail_seat_km_per_week=600000, + incidents_85_99=3, fatal_accidents_85_99=0, + fatalities_85_99=1, incidents_00_14=0, + fatal_accidents_00_14=0, fatalities_00_14=0 + WHERE id=1; +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 5; +``` + +```sql +DELETE FROM AirlineSafety WHERE id=2; +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 5; +``` + +```sql +SELECT * FROM AirlineSafety WHERE avail_seat_km_per_week > 600000; +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 5; +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + Airline A, 500000, 5, 0, 0, + 2, 0, 0); +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + Airline B, 700000, 8, 1, 10, + 3, 0, 0); +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + Airline C, 800000, 10, 2, 20, + 5, 1, 1); +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 5; +``` + +```sql +UPDATE AirlineSafety SET + airline=Updated Airline A, avail_seat_km_per_week=600000, + incidents_85_99=3, fatal_accidents_85_99=0, + fatalities_85_99=1, incidents_00_14=0, + fatal_accidents_00_14=0, fatalities_00_14=0 + WHERE id=1; +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 5; +``` + +```sql +DELETE FROM AirlineSafety WHERE id=2; +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 5; +``` + +```sql +SELECT * FROM AirlineSafety WHERE avail_seat_km_per_week > 600000; +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 5; +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + New Airline, 1000000, 2, 0, 0, + 1, 0, 0); +``` + +```sql +UPDATE AirlineSafety SET + airline=Updated Airline A, avail_seat_km_per_week=600000, + incidents_85_99=3, fatal_accidents_85_99=0, + fatalities_85_99=1, incidents_00_14=0, + fatal_accidents_00_14=1, fatalities_00_14=0 + WHERE id=1; +``` + +```sql +DELETE FROM AirlineSafety WHERE id=1; +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + New Airline, 1000000, 2, 0, 0, + 1, 0, 0); +``` + +```sql +SELECT * FROM AirlineSafety WHERE airline = 'Aeroflot*' +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 10; +``` + +```sql +INSERT INTO AirlineSafety VALUES ( + New Airline, 1000000, 2, 0, 0, + 1, 0, 0); +``` + +```sql +SELECT * FROM AirlineSafety LIMIT 10; +``` + +```sql +UPDATE AirlineSafety SET + airline=Updated Airline A, avail_seat_km_per_week=600000, + incidents_85_99=3, fatal_accidents_85_99=0, + fatalities_85_99=1, incidents_00_14=0, + fatal_accidents_00_14=1, fatalities_00_14=0 + WHERE id=1; +``` + +```sql +DELETE FROM AirlineSafety WHERE id=1; +``` + +```sql +SELECT * FROM AirlineSafety WHERE airline = 'Aeroflot*' +``` + diff --git a/query_output.md b/query_output.md new file mode 100644 index 0000000..b20f68c --- /dev/null +++ b/query_output.md @@ -0,0 +1,86 @@ + +# Airline Safety Statistics + +## Table 1 +| id | airline | avail_seat_km_per_week | incidents_85_99 | fatal_accidents_85_99 | fatalities_85_99 | incidents_00_14 | fatal_accidents_00_14 | fatalities_00_14 | +|------|-----------------------|--------------------------|-------------------|-------------------------|--------------------|-------------------|-------------------------|--------------------| +| 2 | Aeroflot* | 1197672318 | 76 | 14 | 128 | 6 | 1 | 88 | +| 3 | Aerolineas Argentinas | 385803648 | 6 | 0 | 0 | 1 | 0 | 0 | +| 4 | Aeromexico* | 596871813 | 3 | 1 | 64 | 5 | 0 | 0 | +| 5 | Air Canada | 1865253802 | 2 | 0 | 0 | 2 | 0 | 0 | +| 6 | Air France | 3004002661 | 14 | 4 | 79 | 6 | 2 | 337 | + +## Table 2 +| id | airline | avail_seat_km_per_week | incidents_85_99 | fatal_accidents_85_99 | fatalities_85_99 | incidents_00_14 | fatal_accidents_00_14 | fatalities_00_14 | +|------|-----------------------|--------------------------|-------------------|-------------------------|--------------------|-------------------|-------------------------|--------------------| +| 2 | Aeroflot* | 1197672318 | 76 | 14 | 128 | 6 | 1 | 88 | +| 3 | Aerolineas Argentinas | 385803648 | 6 | 0 | 0 | 1 | 0 | 0 | +| 4 | Aeromexico* | 596871813 | 3 | 1 | 64 | 5 | 0 | 0 | +| 5 | Air Canada | 1865253802 | 2 | 0 | 0 | 2 | 0 | 0 | +| 6 | Air France | 3004002661 | 14 | 4 | 79 | 6 | 2 | 337 | + +## Table 3 +| id | airline | avail_seat_km_per_week | incidents_85_99 | fatal_accidents_85_99 | fatalities_85_99 | incidents_00_14 | fatal_accidents_00_14 | fatalities_00_14 | +|------|-----------------------|--------------------------|-------------------|-------------------------|--------------------|-------------------|-------------------------|--------------------| +| 2 | Aeroflot* | 1197672318 | 76 | 14 | 128 | 6 | 1 | 88 | +| 3 | Aerolineas Argentinas | 385803648 | 6 | 0 | 0 | 1 | 0 | 0 | +| 4 | Aeromexico* | 596871813 | 3 | 1 | 64 | 5 | 0 | 0 | +| 5 | Air Canada | 1865253802 | 2 | 0 | 0 | 2 | 0 | 0 | +| 6 | Air France | 3004002661 | 14 | 4 | 79 | 6 | 2 | 337 | + +## Table 4 +| id | airline | avail_seat_km_per_week | incidents_85_99 | fatal_accidents_85_99 | fatalities_85_99 | incidents_00_14 | fatal_accidents_00_14 | fatalities_00_14 | +|------|-----------------------|--------------------------|-------------------|-------------------------|--------------------|-------------------|-------------------------|--------------------| +| 3 | Aerolineas Argentinas | 385803648 | 6 | 0 | 0 | 1 | 0 | 0 | +| 4 | Aeromexico* | 596871813 | 3 | 1 | 64 | 5 | 0 | 0 | +| 5 | Air Canada | 1865253802 | 2 | 0 | 0 | 2 | 0 | 0 | +| 6 | Air France | 3004002661 | 14 | 4 | 79 | 6 | 2 | 337 | +| 7 | Air India* | 869253552 | 2 | 1 | 329 | 4 | 1 | 158 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| id | airline | avail_seat_km_per_week | incidents_85_99 | fatal_accidents_85_99 | fatalities_85_99 | incidents_00_14 | fatal_accidents_00_14 | fatalities_00_14 | ++======+=======================+==========================+===================+=========================+====================+===================+=========================+====================+ +| 2 | Aeroflot* | 1197672318 | 76 | 14 | 128 | 6 | 1 | 88 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 3 | Aerolineas Argentinas | 385803648 | 6 | 0 | 0 | 1 | 0 | 0 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 4 | Aeromexico* | 596871813 | 3 | 1 | 64 | 5 | 0 | 0 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 5 | Air Canada | 1865253802 | 2 | 0 | 0 | 2 | 0 | 0 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 6 | Air France | 3004002661 | 14 | 4 | 79 | 6 | 2 | 337 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 7 | Air India* | 869253552 | 2 | 1 | 329 | 4 | 1 | 158 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 8 | Air New Zealand* | 710174817 | 3 | 0 | 0 | 5 | 1 | 7 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 9 | Alaska Airlines* | 965346773 | 5 | 0 | 0 | 5 | 1 | 88 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 10 | Alitalia | 698012498 | 7 | 2 | 50 | 4 | 0 | 0 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 11 | All Nippon Airways | 1841234177 | 3 | 1 | 1 | 7 | 0 | 0 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ + ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| id | airline | avail_seat_km_per_week | incidents_85_99 | fatal_accidents_85_99 | fatalities_85_99 | incidents_00_14 | fatal_accidents_00_14 | fatalities_00_14 | ++======+=======================+==========================+===================+=========================+====================+===================+=========================+====================+ +| 1 | Aer Lingus | 320906734 | 2 | 0 | 0 | 0 | 0 | 0 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 2 | Aeroflot* | 1197672318 | 76 | 14 | 128 | 6 | 1 | 88 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 3 | Aerolineas Argentinas | 385803648 | 6 | 0 | 0 | 1 | 0 | 0 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 4 | Aeromexico* | 596871813 | 3 | 1 | 64 | 5 | 0 | 0 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 5 | Air Canada | 1865253802 | 2 | 0 | 0 | 2 | 0 | 0 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 6 | Air France | 3004002661 | 14 | 4 | 79 | 6 | 2 | 337 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 7 | Air India* | 869253552 | 2 | 1 | 329 | 4 | 1 | 158 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 8 | Air New Zealand* | 710174817 | 3 | 0 | 0 | 5 | 1 | 7 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 9 | Alaska Airlines* | 965346773 | 5 | 0 | 0 | 5 | 1 | 88 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ +| 10 | Alitalia | 698012498 | 7 | 2 | 50 | 4 | 0 | 0 | ++------+-----------------------+--------------------------+-------------------+-------------------------+--------------------+-------------------+-------------------------+--------------------+ + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..19a1c84 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +#devops +black==22.3.0 +pytest==7.1.3 +pytest-cov==4.0.0 +#ruff and tabulate +ruff==0.6.5 +tabulate==0.9.0 + diff --git a/test_main.py b/test_main.py new file mode 100644 index 0000000..e1fda87 --- /dev/null +++ b/test_main.py @@ -0,0 +1,125 @@ +""" +Test cases for AirlineSafetyDB + +""" + +import subprocess + + +def test_extract(): + """Tests extract()""" + result = subprocess.run( + ["python", "main.py", "extract"], + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0 + assert "Extracting data..." in result.stdout + + +def test_transform_load(): + """Tests transform_load()""" + result = subprocess.run( + ["python", "main.py", "transform_load"], + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0 + assert "Transforming data..." in result.stdout + + +def test_update_record(): + """Tests update_record()""" + result = subprocess.run( + [ + "python", + "main.py", + "update_record", + "1", # Record ID to update + "Updated Airline A", # Airline name + "600000", # Avail seat km per week + "3", # Incidents 85-99 + "0", # Fatal accidents 85-99 + "1", # Fatalities 85-99 + "0", # Incidents 00-14 + "1", # Fatal accidents 00-14 + "0", # Fatalities 00-14 + ], + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0 + + +def test_delete_record(): + """Tests delete_record()""" + result = subprocess.run( + ["python", "main.py", "delete_record", "1"], # Record ID to delete + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0 + + +def test_create_record(): + """Tests create_record()""" + result = subprocess.run( + [ + "python", + "main.py", + "create_record", + "New Airline", # Airline name + "1000000", # Avail seat km per week + "2", # Incidents 85-99 + "0", # Fatal accidents 85-99 + "0", # Fatalities 85-99 + "1", # Incidents 00-14 + "0", # Fatal accidents 00-14 + "0", # Fatalities 00-14 + ], + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0 + + +def test_general_query(): + """Tests general_query()""" + result = subprocess.run( + [ + "python", + "main.py", + "general_query", + "SELECT * FROM AirlineSafety WHERE airline = 'Aeroflot*'", + ], + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0 + + +def test_read_data(): + """Tests read_data()""" + result = subprocess.run( + ["python", "main.py", "read_data"], + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0 + + +if __name__ == "__main__": + test_extract() + test_transform_load() + test_create_record() + test_read_data() + test_update_record() + test_delete_record() + test_general_query()