Initial commit

.devcontainer/Dockerfile

@@ -0,0 +1,18 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/codespaces-linux/.devcontainer/base.Dockerfile
+
+FROM mcr.microsoft.com/vscode/devcontainers/universal:2-focal
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+RUN apt-get update && apt-get -y install --no-install-recommends \
+   python3.8-venv \
+   gcc 
+
+ARG USER="codespace"
+ARG VENV_PATH="/home/${USER}/venv"
+COPY requirements.txt /tmp/
+COPY Makefile /tmp/
+RUN su $USER -c "/usr/bin/python3 -m venv /home/${USER}/venv" \
+   && su $USER -c "${VENV_PATH}/bin/pip --disable-pip-version-check --no-cache-dir install -r /tmp/requirements.txt" \
+   && rm -rf /tmp/requirements.txt

.devcontainer/devcontainer.json

@@ -0,0 +1,79 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/codespaces-linux
+{
+	"name": "GitHub Codespaces (Default)",
+
+	"build": {
+		"dockerfile": "Dockerfile",
+		"context": ".."
+	},
+	"features": {
+		"ghcr.io/devcontainers/features/nvidia-cuda:1": { 
+		  "installCudnn": true
+		}
+	  },
+
+	// Configure tool-specific properties.
+	"customizations": {
+		// Configure properties specific to VS Code.
+		"vscode": {
+			// Set *default* container specific settings.json values on container create.
+			"settings": { 
+				"go.toolsManagement.checkForUpdates": "local",
+				"go.useLanguageServer": true,
+				"go.gopath": "/go",
+				"python.defaultInterpreterPath": "/home/codespace/venv/bin/python",
+				"python.linting.enabled": true,
+				"python.linting.pylintEnabled": true,
+				"python.formatting.autopep8Path": "/home/codespace/venv/bin/autopep8",
+				"python.formatting.blackPath": "/home/codespace/venv/bin/black",
+				"python.formatting.yapfPath": "/home/codespace/venv/bin/yapf",
+				"python.linting.banditPath": "/home/codespace/venv/bin/bandit",
+				"python.linting.flake8Path": "/home/codespace/venv/bin/flake8",
+				"python.linting.mypyPath": "/home/codespace/venv/bin/mypy",
+				"python.linting.pycodestylePath": "/home/codespace/venv/bin/pycodestyle",
+				"python.linting.pydocstylePath": "/home/codespace/venv/bin/pydocstyle",
+				"python.linting.pylintPath": "/home/codespace/venv/bin/pylint",
+				"lldb.executable": "/usr/bin/lldb",
+				"files.watcherExclude": {
+					"**/target/**": true
+				}
+			},
+
+			// Add the IDs of extensions you want installed when the container is created.
+			"extensions": [
+				"GitHub.vscode-pull-request-github",
+				"GitHub.copilot-nightly",
+				"GitHub.copilot-labs",
+				"ms-azuretools.vscode-docker",
+				"ms-toolsai.jupyter",
+				"ms-toolsai.jupyter-keymap",
+				"ms-toolsai.jupyter-renderers",
+				"ms-python.vscode-pylance",
+				"ms-python.python",
+				"ms-vscode.makefile-tools"
+			]
+		}
+	},
+
+	"remoteUser": "codespace",
+
+	"overrideCommand": false,
+
+	"mounts": ["source=codespaces-linux-var-lib-docker,target=/var/lib/docker,type=volume"],
+
+	"runArgs": [
+		"--cap-add=SYS_PTRACE",
+		"--security-opt",
+		"seccomp=unconfined",
+		"--privileged",
+		"--init"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// "oryx build" will automatically install your dependencies and attempt to build your project
+	//"postCreateCommand": "oryx build -p virtualenv_name=.venv --log-file /tmp/oryx-build.log --manifest-dir /tmp || echo 'Could not auto-build. Skipping.'"
+	 "postCreateCommand": "bash setup.sh"
+}

.github/workflows/cicd.yml

@@ -0,0 +1,28 @@
+name: CI
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: install packages
+        run: make install
+      - name: lint
+        run: make lint
+      - name: extract 
+        run: make extract 
+      - name: load
+        run: make load 
+      - name: format
+        run: make format
+      - name: test
+        run: make test
+      - name: generate_and_push
+        run: make generate_and_push
+

Makefile

@@ -0,0 +1,34 @@
+install:
+	pip install --upgrade pip && pip install -r requirements.txt
+
+format:
+	black *.py
+
+lint:
+	ruff check *.py
+
+test:
+	python -m pytest -vv --cov=main --cov=myLib test_*.py	
+
+all: install format lint test
+
+generate_and_push:
+	# Create the markdown file 
+	python test_main.py  
+
+	# Add, commit, and push the generated files to GitHub
+	@if [ -n "$$(git status --porcelain)" ]; then \
+		git config --local user.email "[email protected]"; \
+		git config --local user.name "GitHub Action"; \
+		git add .; \
+		git commit -m "Add SQL log as query_output.md"; \
+		git push; \
+	else \
+		echo "No changes to commit. Skipping commit and push."; \
+	fi
+
+extract:
+	python main.py extract
+
+load: 
+	python main.py transform_load	

README.md

@@ -0,0 +1,107 @@
+[![CI](https://github.com/nogibjj/Mobasserul_Haque_MiniProject5/actions/workflows/cicd.yml/badge.svg)](https://github.com/nogibjj/Mobasserul_Haque_MiniProject5/actions/workflows/cicd.yml)
+
+# Airline Safety Database ETL and Query Tool
+
+This project provides an ETL (Extract, Transform, Load) and querying tool for managing and analyzing the Airline Safety Database. It is built using Python and SQLite, enabling users to perform various operations on airline safety records, including extraction, loading, updating, deleting, creating, and querying records.
+
+## Features
+
+- **ETL Operations**: 
+  - Extract data from a source.
+  - Transform and load data into the SQLite database.
+
+- **Query Operations**:
+  - Update existing records in the database.
+  - Delete records based on a unique identifier.
+  - Create new records in the database.
+  - Execute custom SQL queries.
+  - Read a limited number of records from the database.
+
+- **Logging and Output**:
+  - All executed queries are logged in a markdown file for reference.
+  - Query results are outputted in a formatted markdown file for easier readability.
+
+## Directory Structure
+
+```
+├── .devcontainer/
+│   ├── devcontainer.json
+│   └── Dockerfile
+├── .github/
+│   └── workflows/cicd.yml
+├── data/
+│   └── airline_safety.csv
+├── myLib/
+│   ├── __init__.py
+│   ├── __pycache__/
+│   ├── extract.py
+│   ├── query.py
+│   └── transform_load.py
+├── AirlineSafetyDB.db
+├── main.py
+├── Makefile
+├── query_log.md
+├── query_output.md
+├── README.md  
+├── ServeTimesDB.db  requirements.txt
+└── test_main.py
+```
+## CRUD operations : 
+
+## Usage
+
+Run the script using the following command:
+
+```python
+python main.py <action> [arguments]
+```
+## Arguments: 
+
+`record_id`, `airline`, `avail_seat_km_per_week`, `incidents_85_99`, `fatal_accidents_85_99`, `fatalities_85_99`, `incidents_00_14`, `fatal_accidents_00_14`, `fatalities_00_14`
+
+## Actions:
+
+extract: Extract data from the source.
+
+```python
+python main.py extract
+```
+transform_load: Transform and load data into the database.
+
+```python
+python main.py transform_load
+```
+update_record: Update an existing record in the database.
+
+```python
+python main.py update_record(1, "Air Canada", 2000000000, 3, 1, 5, 2, 0, 0)
+```
+create_record: Create a new record in the database
+
+```python
+python main.py create_record("Air Canada", 1865253802, 2, 0, 0, 2, 0, 0)
+```
+delete_record: delete an existing record in the database.
+
+```python
+python main.py delete_record(1)
+```
+read_data: Read and display the top N rows from the database.
+
+```python
+python main.py read_data(10)  # Reads the top 10 rows
+```
+general_query: Run a custom SQL query on the database.
+
+```python
+python main.py "SELECT * FROM AirlineSafety WHERE airline = 'Aeroflot*'"
+```
+
+## Testing
+To run the test suite, use:
+
+```python
+python -m pytest -vv --cov=main --cov=myLib test_*.py
+```
+## References 
+https://github.com/nogibjj/sqlite-lab

data/airline-safety.csv

@@ -0,0 +1,57 @@
+airline,avail_seat_km_per_week,incidents_85_99,fatal_accidents_85_99,fatalities_85_99,incidents_00_14,fatal_accidents_00_14,fatalities_00_14
+Aer Lingus,320906734,2,0,0,0,0,0
+Aeroflot*,1197672318,76,14,128,6,1,88
+Aerolineas Argentinas,385803648,6,0,0,1,0,0
+Aeromexico*,596871813,3,1,64,5,0,0
+Air Canada,1865253802,2,0,0,2,0,0
+Air France,3004002661,14,4,79,6,2,337
+Air India*,869253552,2,1,329,4,1,158
+Air New Zealand*,710174817,3,0,0,5,1,7
+Alaska Airlines*,965346773,5,0,0,5,1,88
+Alitalia,698012498,7,2,50,4,0,0
+All Nippon Airways,1841234177,3,1,1,7,0,0
+American*,5228357340,21,5,101,17,3,416
+Austrian Airlines,358239823,1,0,0,1,0,0
+Avianca,396922563,5,3,323,0,0,0
+British Airways*,3179760952,4,0,0,6,0,0
+Cathay Pacific*,2582459303,0,0,0,2,0,0
+China Airlines,813216487,12,6,535,2,1,225
+Condor,417982610,2,1,16,0,0,0
+COPA,550491507,3,1,47,0,0,0
+Delta / Northwest*,6525658894,24,12,407,24,2,51
+Egyptair,557699891,8,3,282,4,1,14
+El Al,335448023,1,1,4,1,0,0
+Ethiopian Airlines,488560643,25,5,167,5,2,92
+Finnair,506464950,1,0,0,0,0,0
+Garuda Indonesia,613356665,10,3,260,4,2,22
+Gulf Air,301379762,1,0,0,3,1,143
+Hawaiian Airlines,493877795,0,0,0,1,0,0
+Iberia,1173203126,4,1,148,5,0,0
+Japan Airlines,1574217531,3,1,520,0,0,0
+Kenya Airways,277414794,2,0,0,2,2,283
+KLM*,1874561773,7,1,3,1,0,0
+Korean Air,1734522605,12,5,425,1,0,0
+LAN Airlines,1001965891,3,2,21,0,0,0
+Lufthansa*,3426529504,6,1,2,3,0,0
+Malaysia Airlines,1039171244,3,1,34,3,2,537
+Pakistan International,348563137,8,3,234,10,2,46
+Philippine Airlines,413007158,7,4,74,2,1,1
+Qantas*,1917428984,1,0,0,5,0,0
+Royal Air Maroc,295705339,5,3,51,3,0,0
+SAS*,682971852,5,0,0,6,1,110
+Saudi Arabian,859673901,7,2,313,11,0,0
+Singapore Airlines,2376857805,2,2,6,2,1,83
+South African,651502442,2,1,159,1,0,0
+Southwest Airlines,3276525770,1,0,0,8,0,0
+Sri Lankan / AirLanka,325582976,2,1,14,4,0,0
+SWISS*,792601299,2,1,229,3,0,0
+TACA,259373346,3,1,3,1,1,3
+TAM,1509195646,8,3,98,7,2,188
+TAP - Air Portugal,619130754,0,0,0,0,0,0
+Thai Airways,1702802250,8,4,308,2,1,1
+Turkish Airlines,1946098294,8,3,64,8,2,84
+United / Continental*,7139291291,19,8,319,14,2,109
+US Airways / America West*,2455687887,16,7,224,11,2,23
+Vietnam Airlines,625084918,7,3,171,1,0,0
+Virgin Atlantic,1005248585,1,0,0,0,0,0
+Xiamen Airlines,430462962,9,1,82,2,0,0