Create sanity.yml

Signed-off-by: Mark Saroufim <[email protected]>

.github/workflows/sanity.yml

@@ -0,0 +1,43 @@
+name: Check Sample Submission with HELM
+
+on: push
+
+jobs:
+  build-and-check:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    # - name: Setup Docker with GPU support
+    #   run: |
+    #     sudo apt-get install -y nvidia-container-toolkit
+    #     sudo systemctl restart docker
+
+    - name: Build Docker Image
+      run: |
+        cd neurips_llm_efficiency_challenge/sample-submissions/lit-gpt
+        docker build -t sample_submission .
+
+    - name: Run Docker Container
+      run: docker run -p 8080:80 -d sample_submission
+
+    - name: Wait for a bit to let server start
+      run: sleep 20
+
+    - name: Send HTTP request
+      run: |
+        curl -X POST -H "Content-Type: application/json" -d '{"prompt": "The capital of france is "}' http://localhost:8080/process
+
+    - name: Install NeurIPS client (HELM)
+      run: |
+        pip install git+https://github.com/stanford-crfm/helm.git
+
+    # Use the correct path for the conf file.
+    - name: Run HELM Tests
+      run: |
+        helm-run --conf-paths neurips_llm_efficiency_challenge/run_specs_full_coarse_600_budget.conf --suite v1 --max-eval-instances 10
+        helm-summarize --suite v1