Introduce feedbacks hit tracking for testcases (AFLplusplus#2248)

* introduce feedbacks hit tracking for testcases

* make Testcase::hit_feedbacks into Cow<&str> instead of String
rename get_hit_feedbacks to append_hit_feedbacks
update documentation

* simplify ConstFeedback

* rename Feedback::last_result to prev_result

* impl TODO prev_result for NewHashFeedback, ListFeedback, TransferredFeedback, NautilusFeedback

* rename prev_result to last_result

* add docs

* introduce Objectives hit tracking

* update docs

* update Cargo.toml docs

* update docs

* track Feedbacks & Objectives hit in Fuzzer::add_input

* fmt

* clippy

* fix type error in OomFeedback::last_result

* impl last_result for AsanErrorsFeedback

* add track_hit_feedbacks as a feature to libafl_libfuzzer_runtime

* fix clippy

* change return type of Feedback::last_result to a Result

* remove expect in NewHashFeedback::is_interesting

* move Error::premature_last_result to libafl from libafl_bolts

libafl/Cargo.toml

@@ -26,6 +26,9 @@ document-features = ["dep:document-features"]
 ## Enables features that need rust's `std` lib to work, like print, env, ... support
 std = ["serde_json", "serde_json/std", "nix", "serde/std", "bincode", "wait-timeout", "uuid", "backtrace", "serial_test", "libafl_bolts/std", "typed-builder"]
 
+## Tracks the Feedbacks and the Objectives that were interesting for a Testcase
+track_hit_feedbacks = ["std"]
+
 ## Collects performance statistics of the fuzzing pipeline and displays it on `Monitor` components
 introspection = []
 

libafl/src/corpus/testcase.rs

@@ -2,6 +2,8 @@
 //! It will contain a respective input, and metadata.
 
 use alloc::string::String;
+#[cfg(feature = "track_hit_feedbacks")]
+use alloc::{borrow::Cow, vec::Vec};
 use core::{
     cell::{Ref, RefMut},
     time::Duration,
@@ -67,6 +69,12 @@ where
     disabled: bool,
     /// has found crash (or timeout) or not
     objectives_found: usize,
+    /// Vector of `Feedback` names that deemed this `Testcase` as corpus worthy
+    #[cfg(feature = "track_hit_feedbacks")]
+    hit_feedbacks: Vec<Cow<'static, str>>,
+    /// Vector of `Feedback` names that deemed this `Testcase` as solution worthy
+    #[cfg(feature = "track_hit_feedbacks")]
+    hit_objectives: Vec<Cow<'static, str>>,
 }
 
 impl<I> HasMetadata for Testcase<I>
@@ -211,6 +219,34 @@ where
         self.disabled = disabled;
     }
 
+    /// Get the hit feedbacks
+    #[inline]
+    #[cfg(feature = "track_hit_feedbacks")]
+    pub fn hit_feedbacks(&self) -> &Vec<Cow<'static, str>> {
+        &self.hit_feedbacks
+    }
+
+    /// Get the hit feedbacks (mutable)
+    #[inline]
+    #[cfg(feature = "track_hit_feedbacks")]
+    pub fn hit_feedbacks_mut(&mut self) -> &mut Vec<Cow<'static, str>> {
+        &mut self.hit_feedbacks
+    }
+
+    /// Get the hit objectives
+    #[inline]
+    #[cfg(feature = "track_hit_feedbacks")]
+    pub fn hit_objectives(&self) -> &Vec<Cow<'static, str>> {
+        &self.hit_objectives
+    }
+
+    /// Get the hit objectives (mutable)
+    #[inline]
+    #[cfg(feature = "track_hit_feedbacks")]
+    pub fn hit_objectives_mut(&mut self) -> &mut Vec<Cow<'static, str>> {
+        &mut self.hit_objectives
+    }
+
     /// Create a new Testcase instance given an input
     #[inline]
     pub fn new(mut input: I) -> Self {
@@ -230,6 +266,10 @@ where
             parent_id: None,
             disabled: false,
             objectives_found: 0,
+            #[cfg(feature = "track_hit_feedbacks")]
+            hit_feedbacks: Vec::new(),
+            #[cfg(feature = "track_hit_feedbacks")]
+            hit_objectives: Vec::new(),
         }
     }
 
@@ -252,6 +292,10 @@ where
             parent_id: Some(parent_id),
             disabled: false,
             objectives_found: 0,
+            #[cfg(feature = "track_hit_feedbacks")]
+            hit_feedbacks: Vec::new(),
+            #[cfg(feature = "track_hit_feedbacks")]
+            hit_objectives: Vec::new(),
         }
     }
 
@@ -274,6 +318,10 @@ where
             parent_id: None,
             disabled: false,
             objectives_found: 0,
+            #[cfg(feature = "track_hit_feedbacks")]
+            hit_feedbacks: Vec::new(),
+            #[cfg(feature = "track_hit_feedbacks")]
+            hit_objectives: Vec::new(),
         }
     }
 
@@ -296,6 +344,10 @@ where
             parent_id: None,
             disabled: false,
             objectives_found: 0,
+            #[cfg(feature = "track_hit_feedbacks")]
+            hit_feedbacks: Vec::new(),
+            #[cfg(feature = "track_hit_feedbacks")]
+            hit_objectives: Vec::new(),
         }
     }
 
@@ -348,6 +400,10 @@ where
             metadata_path: None,
             disabled: false,
             objectives_found: 0,
+            #[cfg(feature = "track_hit_feedbacks")]
+            hit_feedbacks: Vec::new(),
+            #[cfg(feature = "track_hit_feedbacks")]
+            hit_objectives: Vec::new(),
         }
     }
 }

libafl/src/feedbacks/concolic.rs

@@ -97,4 +97,9 @@ where
     ) -> Result<(), Error> {
         Ok(())
     }
+
+    #[cfg(feature = "track_hit_feedbacks")]
+    fn last_result(&self) -> Result<bool, Error> {
+        Ok(false)
+    }
 }

libafl/src/feedbacks/differential.rs

@@ -13,6 +13,8 @@ use libafl_bolts::{
 };
 use serde::{Deserialize, Serialize};
 
+#[cfg(feature = "track_hit_feedbacks")]
+use crate::feedbacks::premature_last_result_err;
 use crate::{
     events::EventFirer,
     executors::ExitKind,
@@ -61,6 +63,9 @@ where
     o1_ref: Handle<O1>,
     /// The second observer to compare against
     o2_ref: Handle<O2>,
+    // The previous run's result of `Self::is_interesting`
+    #[cfg(feature = "track_hit_feedbacks")]
+    last_result: Option<bool>,
     /// The function used to compare the two observers
     compare_fn: F,
     phantomm: PhantomData<(I, S)>,
@@ -86,6 +91,8 @@ where
                 o1_ref,
                 o2_ref,
                 name: Cow::from(name),
+                #[cfg(feature = "track_hit_feedbacks")]
+                last_result: None,
                 compare_fn,
                 phantomm: PhantomData,
             })
@@ -108,6 +115,8 @@ where
             o1_ref: self.o1_ref.clone(),
             o2_ref: self.o2_ref.clone(),
             compare_fn: self.compare_fn.clone(),
+            #[cfg(feature = "track_hit_feedbacks")]
+            last_result: None,
             phantomm: self.phantomm,
         }
     }
@@ -169,8 +178,17 @@ where
         let o2: &O2 = observers
             .get(&self.o2_ref)
             .ok_or_else(|| err(self.o2_ref.name()))?;
+        let res = (self.compare_fn)(o1, o2) == DiffResult::Diff;
+        #[cfg(feature = "track_hit_feedbacks")]
+        {
+            self.last_result = Some(res);
+        }
+        Ok(res)
+    }
 
-        Ok((self.compare_fn)(o1, o2) == DiffResult::Diff)
+    #[cfg(feature = "track_hit_feedbacks")]
+    fn last_result(&self) -> Result<bool, Error> {
+        self.last_result.ok_or(premature_last_result_err())
     }
 }
 

libafl/src/feedbacks/list.rs

@@ -143,6 +143,10 @@ where
         }
         Ok(())
     }
+    #[cfg(feature = "track_hit_feedbacks")]
+    fn last_result(&self) -> Result<bool, Error> {
+        Ok(!self.novelty.is_empty())
+    }
 }
 
 impl<T> Named for ListFeedback<T>

libafl/src/feedbacks/map.rs

@@ -18,6 +18,8 @@ use libafl_bolts::{
 use num_traits::PrimInt;
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 
+#[cfg(feature = "track_hit_feedbacks")]
+use crate::feedbacks::premature_last_result_err;
 use crate::{
     corpus::Testcase,
     events::{Event, EventFirer},
@@ -391,6 +393,9 @@ pub struct MapFeedback<C, N, O, R, T> {
     map_ref: Handle<C>,
     /// Name of the feedback as shown in the `UserStats`
     stats_name: Cow<'static, str>,
+    // The previous run's result of [`Self::is_interesting`]
+    #[cfg(feature = "track_hit_feedbacks")]
+    last_result: Option<bool>,
     /// Phantom Data of Reducer
     phantom: PhantomData<(C, N, O, R, T)>,
 }
@@ -424,7 +429,12 @@ where
         EM: EventFirer<State = S>,
         OT: ObserversTuple<S>,
     {
-        Ok(self.is_interesting_default(state, manager, input, observers, exit_kind))
+        let res = self.is_interesting_default(state, manager, input, observers, exit_kind);
+        #[cfg(feature = "track_hit_feedbacks")]
+        {
+            self.last_result = Some(res);
+        }
+        Ok(res)
     }
 
     #[rustversion::not(nightly)]
@@ -440,7 +450,13 @@ where
         EM: EventFirer<State = S>,
         OT: ObserversTuple<S>,
     {
-        Ok(self.is_interesting_default(state, manager, input, observers, exit_kind))
+        let res = self.is_interesting_default(state, manager, input, observers, exit_kind);
+
+        #[cfg(feature = "track_hit_feedbacks")]
+        {
+            self.last_result = Some(res);
+        }
+        Ok(res)
     }
 
     fn append_metadata<EM, OT>(
@@ -533,6 +549,11 @@ where
 
         Ok(())
     }
+
+    #[cfg(feature = "track_hit_feedbacks")]
+    fn last_result(&self) -> Result<bool, Error> {
+        self.last_result.ok_or(premature_last_result_err())
+    }
 }
 
 /// Specialize for the common coverage map size, maximization of u8s
@@ -648,7 +669,10 @@ where
                 }
             }
         }
-
+        #[cfg(feature = "track_hit_feedbacks")]
+        {
+            self.last_result = Some(interesting);
+        }
         Ok(interesting)
     }
 }
@@ -699,6 +723,8 @@ where
             name: map_observer.name().clone(),
             map_ref: map_observer.handle(),
             stats_name: create_stats_name(map_observer.name()),
+            #[cfg(feature = "track_hit_feedbacks")]
+            last_result: None,
             phantom: PhantomData,
         }
     }
@@ -714,6 +740,8 @@ where
             map_ref: map_observer.handle(),
             stats_name: create_stats_name(&name),
             name,
+            #[cfg(feature = "track_hit_feedbacks")]
+            last_result: None,
             phantom: PhantomData,
         }
     }