Add tests for rule level confidence filtering

microsoft · Dec 9, 2024 · dfb2d4f · dfb2d4f
1 parent b0d4412
commit dfb2d4f
Show file tree

Hide file tree

Showing 2 changed files with 170 additions and 0 deletions.
diff --git a/DevSkim-DotNet/Microsoft.DevSkim.Tests/OptionsTests.cs b/DevSkim-DotNet/Microsoft.DevSkim.Tests/OptionsTests.cs
@@ -362,4 +362,169 @@ public void TestParsingJsonOptions()
         // This should be 0, because the globs exclude js files
         Assert.AreEqual(0, analyzerWithSerialized.Run());
     }
+
+    DevSkimRule highConfidenceRule = new DevSkimRule()
+    {
+        Name = "Weak/Broken Hash Algorithm",
+        Id = "HighConfidence",
+        Description = "Confidence Filter Tests",
+        Tags = new List<string>() { "Tests.ConfidenceFilter" },
+        Severity = Severity.Critical,
+        Confidence = Confidence.High,
+        Patterns = new[]
+        {
+            new SearchPattern()
+            {
+                Pattern = "Hello",
+                PatternType = PatternType.Regex,
+                Scopes = new[]
+                {
+                    PatternScope.All
+                }
+            }
+        }
+    };
+
+    DevSkimRule mediumConfidenceRule = new DevSkimRule()
+    {
+        Name = "Weak/Broken Hash Algorithm",
+        Id = "MediumConfidence",
+        Description = "Confidence Filter Tests",
+        Tags = new List<string>() { "Tests.ConfidenceFilter" },
+        Severity = Severity.Critical,
+        Confidence = Confidence.Medium,
+        Patterns = new[]
+        {
+            new SearchPattern()
+            {
+                Pattern = "Hello",
+                PatternType = PatternType.Regex,
+                Scopes = new[]
+                {
+                    PatternScope.All
+                }
+            }
+        }
+    };
+
+    DevSkimRule lowConfidenceRule = new DevSkimRule()
+    {
+        Name = "Weak/Broken Hash Algorithm",
+        Id = "LowConfidence",
+        Description = "Confidence Filter Tests",
+        Tags = new List<string>() { "Tests.ConfidenceFilter" },
+        Severity = Severity.Critical,
+        Confidence = Confidence.Low,
+        Patterns = new[]
+        {
+            new SearchPattern()
+            {
+                Pattern = "Hello",
+                PatternType = PatternType.Regex,
+                Scopes = new[]
+                {
+                    PatternScope.All
+                }
+            }
+        }
+    };
+
+    DevSkimRule unspecifiedConfidenceRule = new DevSkimRule()
+    {
+        Name = "Weak/Broken Hash Algorithm",
+        Id = "UnspecifiedConfidence",
+        Description = "Confidence Filter Tests",
+        Tags = new List<string>() { "Tests.ConfidenceFilter" },
+        Severity = Severity.Critical,
+        Confidence = Confidence.Unspecified,
+        Patterns = new[]
+        {
+            new SearchPattern()
+            {
+                Pattern = "Hello",
+                PatternType = PatternType.Regex,
+                Scopes = new[]
+                {
+                    PatternScope.All
+                }
+            }
+        }
+    };
+
+    [TestMethod]
+    public void TestConfidenceFiltering()
+    {
+
+        var ruleSet = new DevSkimRuleSet();
+        ruleSet.AddRule(highConfidenceRule);
+        ruleSet.AddRule(mediumConfidenceRule);
+        ruleSet.AddRule(lowConfidenceRule);
+        ruleSet.AddRule(unspecifiedConfidenceRule);
+        Assert.AreEqual(4, ruleSet.Count());
+        Assert.AreEqual(1, 
+            ruleSet.WithConfidenceFilter(Confidence.High)
+                .Count(x => x.Confidence.HasFlag(Confidence.High)));
+        Assert.AreEqual(1, 
+            ruleSet.WithConfidenceFilter(Confidence.Medium)
+                .Count(x => x.Confidence.HasFlag(Confidence.Medium)));
+        Assert.AreEqual(1, 
+            ruleSet.WithConfidenceFilter(Confidence.Low)
+                .Count(x => x.Confidence.HasFlag(Confidence.Low)));
+        Assert.AreEqual(1, 
+            ruleSet.WithConfidenceFilter(Confidence.Unspecified)
+                .Count(x => x.Confidence.HasFlag(Confidence.Unspecified)));
+    }
+
+    [TestMethod]
+    public void TestFullFlowWithConfidenceFiltering()
+    {
+        var rulesContent = new List<DevSkimRule>()
+        {
+            unspecifiedConfidenceRule,
+            lowConfidenceRule,
+            mediumConfidenceRule,
+            highConfidenceRule
+        };
+        var testContent = "Hello";
+        var rulesPath = PathHelper.GetRandomTempFile("json");
+        var csharpTestPath = PathHelper.GetRandomTempFile("cs");
+        {
+            using var csharpStream = File.Create(csharpTestPath);
+            JsonSerializer.Serialize(csharpStream, testContent);
+            File.WriteAllText(rulesPath, JsonSerializer.Serialize(rulesContent));
+        }
+
+        var confidenceOptions = new AnalyzeCommandOptions()
+        {
+            Path = csharpTestPath,
+            Rules = new[] { rulesPath },
+            Confidences = new [] { Confidence.High, Confidence.Medium, Confidence.Low, Confidence.Unspecified },
+            ExitCodeIsNumIssues = true
+        };
+
+        var analyzerWithSerialized = new AnalyzeCommand(confidenceOptions);
+        Assert.AreEqual(4, analyzerWithSerialized.Run());
+
+        confidenceOptions.Confidences = new[] { Confidence.High };
+
+        // Unspecified confidence rules are not filtered out because confidence may have been
+        //  (and should be set) at pattern level
+        analyzerWithSerialized = new AnalyzeCommand(confidenceOptions);
+        Assert.AreEqual(2, analyzerWithSerialized.Run());
+
+        confidenceOptions.Confidences = new[] { Confidence.Medium };
+
+        analyzerWithSerialized = new AnalyzeCommand(confidenceOptions);
+        Assert.AreEqual(2, analyzerWithSerialized.Run());
+
+        confidenceOptions.Confidences = new[] { Confidence.Low };
+
+        analyzerWithSerialized = new AnalyzeCommand(confidenceOptions);
+        Assert.AreEqual(2, analyzerWithSerialized.Run());
+
+        confidenceOptions.Confidences = new[] { Confidence.Unspecified };
+
+        analyzerWithSerialized = new AnalyzeCommand(confidenceOptions);
+        Assert.AreEqual(1, analyzerWithSerialized.Run());
+    }
 }
diff --git a/DevSkim-DotNet/Microsoft.DevSkim/DevSkimRuleSet.cs b/DevSkim-DotNet/Microsoft.DevSkim/DevSkimRuleSet.cs
@@ -40,6 +40,11 @@ public static DevSkimRuleSet GetDefaultRuleSet()
             return ruleSet;
         }
 
+        /// <summary>
+        /// Return a new RuleSet containing only rules that have one of the flags of the specified confidence enum
+        /// </summary>
+        /// <param name="filter">The Enum with flags set for which Confidence rules to use</param>
+        /// <returns>A new DevSkimRuleSet with only rules that have the specified confidence set at the Rule level</returns>
         public DevSkimRuleSet WithConfidenceFilter(Confidence filter)
         {
             DevSkimRuleSet newSet = new DevSkimRuleSet();