Resolving pyspark issue

.devcontainer/Dockerfile

@@ -15,4 +15,4 @@ COPY requirements.txt /tmp/
 COPY Makefile /tmp/
 RUN su $USER -c "/usr/bin/python3 -m venv /home/${USER}/venv" \
    && su $USER -c "${VENV_PATH}/bin/pip --disable-pip-version-check --no-cache-dir install -r /tmp/requirements.txt" \
-   && rm -rf /tmp/requirements.txt 
+   && rm -rf /tmp/requirements.txt 

.github/workflows/workflow.yml → .github/workflows/cicd.yml

@@ -1,28 +1,23 @@
 name: CI
 on:
   push:
-    branches: ["main"]
+    branches: [ "main" ]
   pull_request:
-    branches: ["main"]
+    branches: [ "main" ]
   workflow_dispatch:
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: install packages
         run: make install
-      - name: format
-        run: make format
       - name: lint
         run: make lint
       - name: test
         run: make test
-      - name: extract 
-        run: make extract 
-      - name: transfer and load
-        run: make transform_and_load 
-      - name: query
-        run: make query
+      - name: format
+        run: make format
+      - name: deploy
+        run: make deploy

Makefile

@@ -0,0 +1,25 @@
+install:
+	pip install --upgrade pip &&\
+		pip install -r requirements.txt
+
+test:
+	python -m pytest -vv --cov=main --cov=mylib test_*.py
+
+format:	
+	black *.py 
+
+lint:
+	#disable comment to test speed
+	#pylint --disable=R,C --ignore-patterns=test_.*?py *.py mylib/*.py
+	#ruff linting is 10-100X faster than pylint
+	ruff check *.py mylib/*.py
+
+container-lint:
+	docker run --rm -i hadolint/hadolint < Dockerfile
+
+refactor: format lint
+
+deploy:
+	#deploy goes here
+
+all: install lint test format deploy

requirements.txt

@@ -1,3 +1,5 @@
+# testing was saying i needed pandas...
+pandas
 #devops
 black==22.3.0
 click==8.1.3 
@@ -10,9 +12,9 @@ boto3==1.24.87
 #web
 fastapi==0.85.0
 uvicorn==0.18.3
-# other dependencies
-databricks-sql-connector
-databricks
-pyspark
+#other dependencies
 requests
-tabulate
+pyspark
+tabulate
+
+

setup.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
 source /home/codespace/venv/bin/activate
 #append it to bash so every shell launches with it 
-echo 'source /home/codespace/venv/bin/activate' >> ~/.bashrc
+echo 'source /home/codespace/venv/bin/activate' >> ~/.bashrc
+make install-tensorflow