Transforming input before reporting

src/config.rs

@@ -54,8 +54,12 @@ pub struct Config {
     #[serde(skip)]
     /// adds a closure for each function, e.g., print the result
     pub for_each_fn: Option<fn(LanguageType, Report)>,
+    #[serde(skip)]
+    /// function to transform the file content before analysis
+    pub transform_fn: Option<fn(&[u8], &LanguageType) -> String>,
 }
 
+
 impl Config {
     /// Constructs a new `Config` from either `$base/tokei.toml` or
     /// `$base/.tokeirc`. `tokei.toml` takes precedence over `.tokeirc`

src/language/language_type.rs

@@ -25,8 +25,9 @@ include!(concat!(env!("OUT_DIR"), "/language_type.rs"));
 
 impl Serialize for LanguageType {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-        where
-            S: serde::Serializer {
+    where
+        S: serde::Serializer,
+    {
         serializer.serialize_str(self.name())
     }
 }
@@ -63,7 +64,16 @@ impl LanguageType {
 
     /// Parses the bytes provided as the given [`LanguageType`].
     pub fn parse_from_slice<A: AsRef<[u8]>>(self, text: A, config: &Config) -> CodeStats {
-        let text = text.as_ref();
+        let mut text = text.as_ref();
+        let func = config.transform_fn;
+        let transformed: String;
+        text = match func {
+            Some(f) => {
+                transformed = f(text, &self);
+                transformed.as_bytes()
+            }
+            _ => text,
+        };
 
         if self == LanguageType::Jupyter {
             return self
@@ -73,21 +83,16 @@ impl LanguageType {
 
         let syntax = SyntaxCounter::new(self);
 
-        if let Some(end) = syntax
-            .shared
-            .important_syntax
-            .find(text)
-            .and_then(|m| {
-                // Get the position of the last line before the important
-                // syntax.
-                text[..=m.start()]
-                    .iter()
-                    .rev()
-                    .position(|&c| c == b'\n')
-                    .filter(|&p| p != 0)
-                    .map(|p| m.start() - p)
-            })
-        {
+        if let Some(end) = syntax.shared.important_syntax.find(text).and_then(|m| {
+            // Get the position of the last line before the important
+            // syntax.
+            text[..=m.start()]
+                .iter()
+                .rev()
+                .position(|&c| c == b'\n')
+                .filter(|&p| p != 0)
+                .map(|p| m.start() - p)
+        }) {
             let (skippable_text, rest) = text.split_at(end + 1);
             let is_fortran = syntax.shared.is_fortran;
             let is_literate = syntax.shared.is_literate;