update evaluate

Azure-Samples · Sep 6, 2024 · a3057ff · a3057ff
1 parent 0d35236
commit a3057ff
Show file tree

Hide file tree

Showing 16 changed files with 67,134 additions and 231 deletions.
diff --git a/.github/workflows/evaluations.yaml b/.github/workflows/evaluations.yaml
@@ -63,16 +63,16 @@ jobs:
           inlineScript: |
             az account set --subscription ${{env.AZURE_SUBSCRIPTION_ID}}
 
-      - name: evaluate orchestrator
+      - name: evaluate
         working-directory: ./src/api
         run: |
-          python -m evaluate.evaluate
+          python -m evaluators.evaluate
       
       - name: Upload eval results as build artifact
         uses: actions/upload-artifact@v4
         with:
           name: eval_result
-          path: ./src/api/evaluate/eval_results.jsonl
+          path: ./src/api/evaluators/eval_results.jsonl
 
       - name: GitHub Summary Step
         if: ${{ success() }}

diff --git a/.gitignore b/.gitignore
@@ -11,14 +11,20 @@ deployment/push_and_deploy_pf.py
 endpoint.json
 principal.txt
 .azure
-evaluations/process_log/
-evaluations/flow.flex.yaml
-evaluations/eval_result.**
-evaluations/studio_url.txt
+evaluators/process_log/
+evaluators/flow.flex.yaml
+evaluators/eval_result.**
+evaluators/studio_url.txt
 src/contoso_chat/.runs/*
 src/api/contoso_chat/.runs/*
 src/api/.runs/*
 .runs/*
-evaluations/.runs/*
-evaluations/result_evaluated.jsonl
-evaluations/result.jsonl
+evaluators/.runs/*
+evaluators/result_evaluated.jsonl
+evaluators/result.jsonl
+src/api/evaluators/.runs/*
+src/api/evaluators/result_evaluated.jsonl
+src/api/evaluators/result.jsonl
+src/api/evaluators/eval_results.jsonl
+src/api/evaluators/eval_results.md
+src/api/evaluators/.runs/*
diff --git a/evaluations/.promptflow/relevance/flow.tools.json b/evaluations/.promptflow/relevance/flow.tools.json
diff --git a/evaluations/evaluate-chat-flow.ipynb b/evaluations/evaluate-chat-flow.ipynb
diff --git a/evaluations/evaluate.py b/evaluations/evaluate.py
diff --git a/evaluations/custom_evals/coherence.prompty → ...evaluators/custom_evals/coherence.prompty b/evaluations/custom_evals/coherence.prompty → ...evaluators/custom_evals/coherence.prompty
diff --git a/evaluations/custom_evals/coherence.py → src/api/evaluators/custom_evals/coherence.py b/evaluations/custom_evals/coherence.py → src/api/evaluators/custom_evals/coherence.py
diff --git a/evaluations/custom_evals/fluency.prompty → ...i/evaluators/custom_evals/fluency.prompty b/evaluations/custom_evals/fluency.prompty → ...i/evaluators/custom_evals/fluency.prompty
diff --git a/evaluations/custom_evals/fluency.py → src/api/evaluators/custom_evals/fluency.py b/evaluations/custom_evals/fluency.py → src/api/evaluators/custom_evals/fluency.py
diff --git a/...uations/custom_evals/groundedness.prompty → ...luators/custom_evals/groundedness.prompty b/...uations/custom_evals/groundedness.prompty → ...luators/custom_evals/groundedness.prompty
diff --git a/evaluations/custom_evals/groundedness.py → ...i/evaluators/custom_evals/groundedness.py b/evaluations/custom_evals/groundedness.py → ...i/evaluators/custom_evals/groundedness.py
diff --git a/evaluations/custom_evals/relevance.prompty → ...evaluators/custom_evals/relevance.prompty b/evaluations/custom_evals/relevance.prompty → ...evaluators/custom_evals/relevance.prompty
diff --git a/evaluations/custom_evals/relevance.py → src/api/evaluators/custom_evals/relevance.py b/evaluations/custom_evals/relevance.py → src/api/evaluators/custom_evals/relevance.py
diff --git a/data/data.jsonl → src/api/evaluators/data.jsonl b/data/data.jsonl → src/api/evaluators/data.jsonl