Merge remote-tracking branch 'origin/develop' into develop

# Conflicts:
#	modules/programming/module_programming_llm/module_programming_llm/helpers/models/openai.py

.gitignore

@@ -10,3 +10,4 @@ data/*
 !.idea/runConfigurations/
 .env
 **/.env
+**/data/*

.vscode/launch.json

@@ -26,7 +26,7 @@
             "request": "launch",
             "cwd": "${workspaceFolder}/modules/programming/module_programming_llm",
             "module": "module_programming_llm",
-            "justMyCode": true
+            "justMyCode": false
         },
         {
             "name": "Module Programming ThemisML",
@@ -62,7 +62,7 @@
             "type": "python",
             "request": "launch",
             "cwd": "${workspaceFolder}/modules/modeling/module_modeling_llm",
-            "module": "module_text_cofee"
+            "module": "module_modeling_llm"
         }
     ]
 }

assessment_module_manager/modules.docker.ini

@@ -44,6 +44,6 @@ supports_graded_feedback_requests = false
 url = http://module-modeling-llm:5008
 type = modeling
 supports_evaluation = false
-supports_non_graded_feedback_requests = false
+supports_non_graded_feedback_requests = true
 supports_graded_feedback_requests = true
 

assessment_module_manager/modules.ini

@@ -44,6 +44,6 @@ supports_graded_feedback_requests = false
 url = http://localhost:5008
 type = modeling
 supports_evaluation = false
-supports_non_graded_feedback_requests = false
+supports_non_graded_feedback_requests = true
 supports_graded_feedback_requests = true
 

modules/modeling/module_modeling_llm/.env.example

@@ -12,7 +12,7 @@ DATABASE_URL=sqlite:///../data/data.sqlite
 
 # Default model to use
 # See below for options, available models are also logged on startup
-LLM_DEFAULT_MODEL="azure_openai_gpt-35"
+LLM_DEFAULT_MODEL="azure_openai_gpt-4o"
 
 # Enable LLM-as-a-judge approach 0 = disabled, 1 = enabled
 LLM_ENABLE_LLM_AS_A_JUDGE=1
@@ -23,13 +23,13 @@ LLM_EVALUATION_MODEL="azure_openai_gpt-4"
 # Standard OpenAI (Non-Azure) [leave blank if not used]
 # Model names prefixed with `openai_` followed by the model name, e.g. `openai_text-davinci-003`
 # A list of models can be found in `module_text_llm/helpers/models/openai.py` (openai_models)
-LLM_OPENAI_API_KEY="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+OPENAI_API_KEY="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 
 # Azure OpenAI [leave blank if not used]
 # Model names prefixed with `azure_openai_` followed by the deployment id, e.g. `azure_openai_gpt-35`
-LLM_AZURE_OPENAI_API_KEY="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-LLM_AZURE_OPENAI_API_BASE="https://ase-eu01.openai.azure.com/" # change base if needed
-LLM_AZURE_OPENAI_API_VERSION="2023-07-01-preview" # change base if needed
+AZURE_OPENAI_API_KEY="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+AZURE_OPENAI_ENDPOINT="https://ase-eu01.openai.azure.com/" # change base if needed
+OPENAI_API_VERSION="2023-07-01-preview" # change base if needed
 
 # Replicate [leave blank if not used]
 # See https://replicate.com and adjust model config options in `module_text_llm/helpers/models/replicate.py`

modules/modeling/module_modeling_llm/module_modeling_llm/__main__.py

@@ -31,7 +31,7 @@ def process_incoming_feedback(exercise: Exercise, submission: Submission, feedba
 async def suggest_feedback(exercise: Exercise, submission: Submission, is_graded: bool, module_config: Configuration) -> List[Feedback]:
     logger.info("suggest_feedback: Suggestions for submission %d of exercise %d were requested", submission.id,
                 exercise.id)
-    return await generate_suggestions(exercise, submission, module_config.approach, module_config.debug)
+    return await generate_suggestions(exercise, submission, is_graded, module_config.approach, module_config.debug)
 
 
 if __name__ == "__main__":

modules/modeling/module_modeling_llm/module_modeling_llm/config.py

@@ -3,8 +3,10 @@
 from athena import config_schema_provider
 from module_modeling_llm.helpers.models import ModelConfigType, DefaultModelConfig
 from module_modeling_llm.prompts.generate_suggestions import (
-    system_message as generate_suggestions_system_message,
-    human_message as generate_suggestions_human_message
+    graded_feedback_system_message as default_graded_feedback_system_message,
+    graded_feedback_human_message as default_graded_feedback_human_message,
+    filter_feedback_system_message as default_filter_feedback_system_message,
+    filter_feedback_human_message as default_filter_feedback_human_message
 )
 
 
@@ -16,10 +18,15 @@ class GenerateSuggestionsPrompt(BaseModel):
     _Note: **{problem_statement}**, **{example_solution}**, or **{grading_instructions}** might be omitted if the input
     is too long._
     """
-    system_message: str = Field(default=generate_suggestions_system_message,
-                                description="Message for priming AI behavior and instructing it what to do.")
-    human_message: str = Field(default=generate_suggestions_human_message,
-                               description="Message from a human. The input on which the AI is supposed to act.")
+    graded_feedback_system_message: str = Field(default=default_graded_feedback_system_message,
+        description="Message for priming AI behavior and instructing it what to do.")
+    graded_feedback_human_message: str = Field(default=default_graded_feedback_human_message,
+        description="Message from a human. The input on which the AI is supposed to act.")
+    filter_feedback_system_message: str = Field(default=default_filter_feedback_system_message,
+        description="Message for priming AI behavior for filtering ungraded feedback.")
+    filter_feedback_human_message: str = Field(default=default_filter_feedback_human_message,
+        description="Message for instructing AI to filter ungraded feedback.")
+
 
 
 class BasicApproachConfig(BaseModel):