Merge pull request #12722 from empirical-org/develop

Evidence Label Feedback - More explicit error handling (#12691)

services/QuillLMS/engines/evidence/app/models/evidence/labeled_entry.rb

@@ -67,7 +67,7 @@ def self.closest_prompt_texts(entry:, prompt_id:, label:, limit: DEFAULT_RAG_LIM
       end
     end
 
-    # This method doesn't store the embedding of the entry, used for feedback API
+    # This method does not store the embedding of the entry, used for feedback API
     def self.find_nearest_neighbors(prompt_id:, entry:, limit: DEFAULT_RAG_LIMIT)
       nearest_neighbors(:embedding, embedding_for(entry:), distance: DISTANCE_METRIC)
         .where(prompt_id:)

services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/label_feedback/retriever.rb

@@ -10,6 +10,8 @@ class Retriever < ApplicationService
         KEY_LABEL = 'label'
         MATCH_THRESHOLD = 0.03
 
+        OPTIMAL_LABEL = Evidence::LabeledEntry::COLLAPSED_OPTIMAL_LABEL
+
         DEFAULT_MISC_THRESHOLD = 0.5
         MISC_LABEL = Evidence::Label::MISC_LABEL
         MISC_LIMITS = Evidence::GenAI::LabelFeedback::MiscThreshold::PROMPT_HASH
@@ -21,7 +23,8 @@ class Retriever < ApplicationService
           SOURCE_ALL_SAME = 'all_same_label',
           SOURCE_CLOSE_MATCH = 'close_match',
           SOURCE_MISC_FALLBACK = 'misc_fallback',
-          SOURCE_LLM = 'llm'
+          SOURCE_LLM = 'llm',
+          SOURCE_NO_DATA = 'no_data'
         ]
 
         Response = Data.define(:label_text, :closest_distance, :source, :limit, :misc_threshold)
@@ -40,6 +43,7 @@ def run = Response.new(label_text:, closest_distance:, source:, limit:, misc_thr
         private def source = label_and_source.source
 
         private def label_and_source
+          return optimal_no_data if closest_example.nil?
           return closest_label(SOURCE_ALL_SAME) if all_examples_have_same_label?
           return closest_label(SOURCE_CLOSE_MATCH) if close_match?
           return misc_label if no_close_examples_and_has_misc_label?
@@ -53,6 +57,8 @@ def run = Response.new(label_text:, closest_distance:, source:, limit:, misc_thr
           LabelAndSource.new(label_text: response[KEY_LABEL], source: SOURCE_LLM)
         end
 
+        private def optimal_no_data = LabelAndSource.new(label_text: OPTIMAL_LABEL, source: SOURCE_NO_DATA)
+
         private def closest_examples
           @closest_examples ||= Evidence::LabeledEntry
             .find_nearest_neighbors(prompt_id:, entry:, limit:)
@@ -61,7 +67,7 @@ def run = Response.new(label_text:, closest_distance:, source:, limit:, misc_thr
         private def closest_example = closest_examples.first
         private def closest_label(source) = LabelAndSource.new(label_text: closest_example.label_transformed, source:)
         private def closest_distance
-          @closest_distance ||= closest_example.neighbor_distance.round(6)
+          @closest_distance ||= closest_example&.neighbor_distance&.round(6)
         end
 
         private def misc_label = LabelAndSource.new(label_text: MISC_LABEL, source: SOURCE_MISC_FALLBACK)

services/QuillLMS/engines/evidence/lib/evidence/open_ai/concerns/api.rb

@@ -36,9 +36,11 @@ def run
 
           cleaned_results
         rescue *Evidence::HTTP_TIMEOUT_ERRORS
-          []
+          timeout_response
         end
 
+        private def timeout_response = []
+
         private def headers
           {
             'Content-Type' => 'application/json',

services/QuillLMS/engines/evidence/lib/evidence/open_ai/embedding_fetcher.rb

@@ -25,6 +25,8 @@ def cleaned_results
           &.[]('embedding')
       end
 
+      private def timeout_response = nil
+
       def endpoint = ENDPOINT
 
       # https://platform.openai.com/docs/api-reference/embeddings/create

services/QuillLMS/engines/evidence/spec/lib/evidence/open_ai/embedding_fetcher_spec.rb

@@ -44,10 +44,17 @@ module OpenAI
         end
       end
 
-      context 'error handling' do
-        it 'returns an empty array when encountering a timeout' do
+      context 'error handling for timeout' do
+        it 'returns nil when encountering a timeout' do
           stub_request(:post, endpoint).to_timeout
-          expect(subject).to eq([])
+          expect(subject).to be_nil
+        end
+      end
+
+      context 'error handling' do
+        it 'returns nil when an error is hit' do
+          stub_request(:post, endpoint).to_return(status: 500, body: 'Internal Error')
+          expect(subject).to be_nil
         end
       end
 

services/QuillLMS/engines/evidence/spec/models/evidence/labeled_entry_spec.rb

@@ -172,6 +172,31 @@ module Evidence
       end
     end
 
+    context '#find_nearest_neighbors' do
+      subject { described_class.find_nearest_neighbors(prompt_id:, entry:) }
+
+      let(:prompt) { create(:evidence_prompt) }
+      let(:prompt_id) { prompt.id }
+      let(:entry) { '' }
+      let(:labeled_entry) { create(factory, prompt:) }
+
+      context 'API response' do
+        before do
+          allow(Evidence::OpenAI::EmbeddingFetcher).to receive(:run).and_return(embedding)
+        end
+
+        it { expect(subject).to eq [labeled_entry] }
+      end
+
+      context 'API error' do
+        before do
+          allow(Evidence::OpenAI::EmbeddingFetcher).to receive(:run).and_return(nil)
+        end
+
+        it { expect(subject).to eq [] }
+      end
+    end
+
     context 'benchmarking', :benchmarking do
       let(:num_iterations) { 1000 }
       let(:labeled_entries) { create_list(factory, num_iterations) }

services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/label_feedback/retriever_spec.rb

@@ -23,6 +23,7 @@
   let(:all_same_response_object) { described_class::Response.new(label_text: closest_label, closest_distance:, source: described_class::SOURCE_ALL_SAME, misc_threshold:, limit:) }
   let(:misc_response_object) { described_class::Response.new(label_text: Evidence::Label::MISC_LABEL, closest_distance:, source: described_class::SOURCE_MISC_FALLBACK, misc_threshold:, limit:) }
   let(:generated_response_object) { described_class::Response.new(label_text: generated_label, closest_distance: closest_distance2, source: described_class::SOURCE_LLM, misc_threshold:, limit:) }
+  let(:no_data_response_object) { described_class::Response.new(label_text: described_class::OPTIMAL_LABEL, closest_distance: nil, source: described_class::SOURCE_NO_DATA, misc_threshold:, limit:) }
 
   let(:api_response) { { 'label' => generated_label } }
 
@@ -46,6 +47,16 @@
       end
     end
 
+    context 'no nearest neighbors - API error' do
+      before do
+        allow(Evidence::LabeledEntry).to receive(:find_nearest_neighbors).and_return([])
+      end
+
+      it 'returns the optimal label and no_data source' do
+        expect(subject).to eq(no_data_response_object)
+      end
+    end
+
     context 'when closest example is a close match' do
       it 'returns the closest label' do
         expect(subject).to eq(close_response_object)