diff --git a/.github/manifests/disk-usage-report-job.yaml b/.github/manifests/disk-usage-report-job.yaml new file mode 100644 index 0000000..161f177 --- /dev/null +++ b/.github/manifests/disk-usage-report-job.yaml @@ -0,0 +1,35 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: disk-usage-report-job + namespace: jupyterhub +spec: + backoffLimit: 0 # No retry on failure + template: + metadata: + labels: + app: disk-usage-report + spec: + containers: + - name: disk-usage-report + image: dandiarchive/dandihub-report-generator:latest + args: + - "/home/" + volumeMounts: + - name: persistent-storage + mountPath: "/home" + subPath: "home" + restartPolicy: Never + nodeSelector: + NodeGroupType: default + NodePool: default + hub.jupyter.org/node-purpose: user + tolerations: + - key: "hub.jupyter.org/dedicated" + operator: "Equal" + value: "user" + effect: "NoSchedule" + volumes: + - name: persistent-storage + persistentVolumeClaim: + claimName: efs-persist diff --git a/.github/manifests/hello-world-pod.yaml b/.github/manifests/hello-world-pod.yaml new file mode 100644 index 0000000..1977f33 --- /dev/null +++ b/.github/manifests/hello-world-pod.yaml @@ -0,0 +1,20 @@ +# manifests/hello-world-pod.yaml +apiVersion: v1 +kind: Pod +metadata: + name: hello-world-pod +spec: + containers: + - name: hello + image: busybox + command: ['sh', '-c', 'echo Hello, World! && sleep 30'] + nodeSelector: + NodeGroupType: default + NodePool: default + hub.jupyter.org/node-purpose: user + tolerations: + - key: "hub.jupyter.org/dedicated" + operator: "Equal" + value: "user" + effect: "NoSchedule" + diff --git a/.github/scripts/du.py b/.github/scripts/du.py new file mode 100755 index 0000000..e626769 --- /dev/null +++ b/.github/scripts/du.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys +import json +from datetime import date + +OUTPUT_DIR = "/home/asmacdo/du_reports/" +SIZE_THRESHOLD_GB = 1 +SIZE_THRESHOLD_BYTES = SIZE_THRESHOLD_GB * 1024 * 1024 * 1024 + +# Function to calculate disk usage of a directory in bytes +def get_disk_usage_bytes(path): + result = subprocess.run(['du', '-sb', path], capture_output=True, text=True) + size_str = result.stdout.split()[0] # Get the size in bytes (du -sb gives size in bytes) + return int(size_str) + +# Function to convert bytes to a human-readable format (e.g., KB, MB, GB) +def bytes_to_human_readable(size_in_bytes): + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if size_in_bytes < 1024: + return f"{size_in_bytes:.2f} {unit}" + size_in_bytes /= 1024 + +def prepare_report(directory): + report = {} + # List user home dirs in the directory and calculate disk usage + for user_dir in os.listdir(directory): + user_path = os.path.join(directory, user_dir) + if os.path.isdir(user_path): + disk_usage_bytes = get_disk_usage_bytes(user_path) + report[user_dir] = { + "disk_usage_bytes": disk_usage_bytes + } + if disk_usage_bytes > SIZE_THRESHOLD_BYTES: + # TODO: Placeholder for other actions + report[user_dir]["action"] = f"Directory size exceeds {SIZE_THRESHOLD_BYTES / (1024**3):.2f}GB, further action taken." + else: + report[user_dir]["action"] = "No action required." + + for user, data in report.items(): + data["disk_usage_human_readable"] = bytes_to_human_readable(data["disk_usage_bytes"]) + + os.makedirs(os.path.dirname(OUTPUT_DIR), exist_ok=True) + current_date = date.today().strftime('%Y-%m-%d') + with open(f"{OUTPUT_DIR}/{current_date}.json", "w") as f: + json.dump(report, f, indent=4) + print(f"Disk usage report generated at {os.path.abspath(OUTPUT_FILE)}") + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: du.py ") + else: + path = sys.argv[1] + prepare_report(path) diff --git a/.github/workflows/report.yaml b/.github/workflows/report.yaml new file mode 100644 index 0000000..8424fd6 --- /dev/null +++ b/.github/workflows/report.yaml @@ -0,0 +1,104 @@ +# name: Generate Data Usage Report +# +# on: +# pull_request: +# branches: +# - main +# +# jobs: +# generate_data_usage_report: +# runs-on: ubuntu-latest +# +# steps: +# - name: Checkout code +# uses: actions/checkout@v3 +# +# - name: Log in to DockerHub +# uses: docker/login-action@v2 +# with: +# username: ${{ secrets.DOCKERHUB_USERNAME }} +# password: ${{ secrets.DOCKERHUB_TOKEN }} +# +# - name: Build and push Docker image +# uses: docker/build-push-action@v3 +# with: +# context: . +# file: images/Dockerfile.dandihub_report_generator +# push: true +# tags: ${{ secrets.DOCKERHUB_USERNAME }}/dandihub-report-generator:latest +# +# - name: Configure AWS credentials +# uses: aws-actions/configure-aws-credentials@v3 +# with: +# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# aws-region: us-east-2 +# +# - name: Assume ProvisioningRole +# run: | +# CREDS=$(aws sts assume-role --role-arn ${{ secrets.AWS_PROVISIONING_ROLE_ARN }} --role-session-name "GitHubActionsSession") +# export AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r '.Credentials.AccessKeyId') +# export AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r '.Credentials.SecretAccessKey') +# export AWS_SESSION_TOKEN=$(echo $CREDS | jq -r '.Credentials.SessionToken') +# +# - name: Configure kubectl with AWS EKS +# run: | +# aws eks update-kubeconfig --name eks-dandihub --region us-east-2 --role-arn ${{ secrets.AWS_PROVISIONING_ROLE_ARN }} +# +# # TODO remove +# - name: Sanity check +# run: | +# kubectl get pods -n jupyterhub +# +# # - name: Deploy Hello World Pod +# # run: | +# # kubectl apply -f .github/manifests/hello-world-pod.yaml +# # +# # - name: Wait for Hello World Pod to complete +# # run: | +# # kubectl wait --for=condition=Ready pod/hello-world-pod --timeout=300s # 5 minutes +# # continue-on-error: true # Allow the workflow to continue even if this step fails +# # +# # - name: Get Hello World Pod logs +# # run: | +# # kubectl logs hello-world-pod +# # if: ${{ success() }} # Only run this step if the previous step was successful +# # +# # - name: Delete Hello World Pod +# # run: | +# # kubectl delete pod hello-world-pod +# # if: ${{ always() }} # Always run this step, even if other steps fail +# # +# - name: Replace image placeholder in manifest +# run: | +# sed -i 's|IMAGE_PLACEHOLDER|'"${{ secrets.DOCKERHUB_USERNAME }}/dandihub-report-generator:latest"'|' .github/manifests/disk-usage-report-job.yaml +# +# - name: Deploy Disk Usage Report Job +# run: | +# kubectl apply -f .github/manifests/disk-usage-report-job.yaml +# +# # TODO should timeout be longer? +# - name: Wait for Disk Usage Report Job to complete +# run: | +# kubectl wait --for=condition=complete job/disk-usage-report-job --timeout=360s -n jupyterhub +# continue-on-error: true +# +# # continue-on-error for previous steps so we delete the job +# - name: Delete Disk Usage Report Job +# run: | +# kubectl delete job disk-usage-report-job -n jupyterhub +# +# # - name: Clone dandi-hub-usage-reports repository +# # run: | +# # git clone https://github.com/dandi/dandi-hub-usage-reports.git +# # +# # - name: Copy report file to repository, commit and push report +# # run: | +# # cd dandi-hub-usage-reports +# # DATE=$(date +'%Y-%m-%d') +# # mv ../du_report.json $DATE_du_report.json +# # git config --global user.name "GitHub Actions" +# # git config --global user.email "actions@github.com" +# # git add $DATE_du_report.json +# # git commit -m "Add disk usage report for $DATE" +# # git push https://${{ secrets.GITHUB_TOKEN }}@github.com/dandi/dandi-hub-usage-reports.git diff --git a/NEXTSTEPS b/NEXTSTEPS new file mode 100644 index 0000000..44d0177 --- /dev/null +++ b/NEXTSTEPS @@ -0,0 +1,32 @@ +DONE + - Set AWS_ROLE ARN secret + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + +TODO: + - Create Dockerhub Service account + - set username & token as secrets + - Create Github CI account + - Docker Image Tagging: + - The Docker image is tagged with latest. For better version control, consider using commit SHA or version numbers. + - Log Retrieval: + - The logs from the pod are retrieved to help you verify the script's output. + - Cleanup: + - Deleting the Job ensures that no resources are left running after the workflow completes. + +By making these updates, your workflow will now: + + Include your du.py script in a Docker image. + Build and push this image to DockerHub. + Deploy a Kubernetes Job to your EKS cluster that runs the script. + Wait for the Job to complete and retrieve logs. + Clean up resources after execution. + +Feel free to ask if you need further assistance or clarification on any of these steps! + + +- Get image pushing +- create private gh repository under dandi org for reports + + + diff --git a/images/Dockerfile.dandihub_report_generator b/images/Dockerfile.dandihub_report_generator new file mode 100644 index 0000000..5f46008 --- /dev/null +++ b/images/Dockerfile.dandihub_report_generator @@ -0,0 +1,15 @@ +FROM python:3.9-slim + +# Set the working directory +WORKDIR /app + +# Copy the du.py script into the container +COPY .github/scripts/du.py /app/du.py + +# Install required packages +RUN apt-get update \ + && apt-get install -y coreutils \ + && rm -rf /var/lib/apt/lists/* + +# Set the entrypoint to the script +ENTRYPOINT ["python3", "/app/du.py"]