Merge pull request #105 from JoshKeegan/skipAndSpecflowIgnore

Implement dynamic Skip and Specflow ignore

README.md

@@ -85,6 +85,22 @@ public void Default_Reaches3(int id)
 ```
 The same optional arguments (max retries and delay between each retry) are supported as for facts, and can be used in the same way.
 
+### Skipping tests at Runtime
+In addition to retries, `RetryFact` and `RetryTheory` both support dynamically skipping tests at runtime. To make a test skip just use `Skip.Always()`
+within your test code.  
+It also supports custom exception types so you can skip a test if a type of exception gets thrown. You do this by specifying the exception type to the 
+attribute above your test, e.g.
+```cs
+[RetryFact(typeof(TestException))]
+public void CustomException_SkipsAtRuntime()
+{
+    throw new TestException();
+}
+```
+This functionality also allows for skipping to work when you are already using another library for dynamically skipping tests by specifying the exception
+type that is used by that library to the `RetryFact`. e.g. if you are using the popular Xunit.SkippableFact nuget package and want to add retries, converting the 
+test is as simple as replacing `[SkippableFact]` with `[RetryFact(typeof(Xunit.SkipException))]` above the test and you don't need to change the test itself.
+
 ## Viewing retry logs
 By default, you won't see whether your tests are being retried as we make this information available 
 via the xunit diagnostic logs but test runners will hide these detailed logs by default.  

build/Makefile

@@ -1,4 +1,4 @@
-VERSION=1.6.0#
+VERSION=1.7.0#
 
 clean:
 	rm -r ../artefacts || true

src/xRetry.SpecFlow/SkipException.cs

@@ -0,0 +1,14 @@
+using System;
+
+// ReSharper disable once CheckNamespace
+// ReSharper disable once IdentifierTypo
+namespace Xunit
+{
+    /// <summary>
+    /// Do not use.
+    /// Exists purely as a marker to replicate the exception thrown by Xunit.SkippableFact that SpecFlow.xUnit
+    /// makes use of. That way we can intercept the exception that is throwing without also having our own runtime
+    /// plugin, or adding a direct dependency on either of these other libraries.
+    /// </summary>
+    internal class SkipException : Exception {  }
+}

src/xRetry.SpecFlow/TestGeneratorProvider.cs

@@ -59,17 +59,21 @@ public override void SetTestMethodCategories(TestClassGenerationContext generati
             CodeAttributeDeclaration retryAttribute = CodeDomHelper.AddAttribute(testMethod,
                 "xRetry.Retry" + (originalAttribute.Name == FACT_ATTRIBUTE ? "Fact" : "Theory"));
 
-            if (retryTag.MaxRetries != null)
-            {
-                retryAttribute.Arguments.Add(
-                    new CodeAttributeArgument(new CodePrimitiveExpression(retryTag.MaxRetries)));
-
-                if(retryTag.DelayBetweenRetriesMs != null)
-                {
-                    retryAttribute.Arguments.Add(
-                        new CodeAttributeArgument(new CodePrimitiveExpression(retryTag.DelayBetweenRetriesMs)));
-                }
-            }
+            retryAttribute.Arguments.Add(new CodeAttributeArgument(
+                new CodePrimitiveExpression(retryTag.MaxRetries ?? RetryFactAttribute.DEFAULT_MAX_RETRIES)));
+            retryAttribute.Arguments.Add(new CodeAttributeArgument(
+                new CodePrimitiveExpression(retryTag.DelayBetweenRetriesMs ??
+                                            RetryFactAttribute.DEFAULT_DELAY_BETWEEN_RETRIES_MS)));
+
+            // Always skip on Xunit.SkipException (from Xunit.SkippableFact) which is used by SpecFlow.xUnit to implement
+            //  dynamic test skipping. This way we can intercept the exception that is already thrown without also having
+            //  our own runtime plugin.
+            retryAttribute.Arguments.Add(new CodeAttributeArgument(
+                new CodeArrayCreateExpression(new CodeTypeReference(typeof(Type)),
+                    new CodeExpression[]
+                    {
+                        new CodeTypeOfExpression(typeof(Xunit.SkipException))
+                    })));
 
             // Copy arguments from the original attribute
             for (int i = 0; i < originalAttribute.Arguments.Count; i++)

src/xRetry.SpecFlow/xRetry.SpecFlow.csproj

@@ -12,4 +12,8 @@
     <PackageReference Include="SpecFlow.CustomPlugin" Version="3.9.22" />
   </ItemGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\xRetry\xRetry.csproj" />
+  </ItemGroup>
+
 </Project>

src/xRetry/BlockingMessageBus.cs

@@ -10,16 +10,20 @@ namespace xRetry
     public class BlockingMessageBus : IMessageBus
     {
         private readonly IMessageBus underlyingMessageBus;
+        private readonly MessageTransformer messageTransformer;
         private ConcurrentQueue<IMessageSinkMessage> messageQueue = new ConcurrentQueue<IMessageSinkMessage>();
 
-        public BlockingMessageBus(IMessageBus underlyingMessageBus)
+        public BlockingMessageBus(IMessageBus underlyingMessageBus, MessageTransformer messageTransformer)
         {
             this.underlyingMessageBus = underlyingMessageBus;
+            this.messageTransformer = messageTransformer;
         }
 
-        public bool QueueMessage(IMessageSinkMessage message)
+        public bool QueueMessage(IMessageSinkMessage rawMessage)
         {
-            messageQueue.Enqueue(message);
+            // Transform the message to apply any additional functionality, then intercept & store it for replay later
+            IMessageSinkMessage transformedMessage = messageTransformer.Transform(rawMessage);
+            messageQueue.Enqueue(transformedMessage);
 
             // Returns if execution should continue. Since we are intercepting the message, we
             //  have no way of checking this so always continue...

src/xRetry/Exceptions/SkipTestException.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Runtime.Serialization;
+
+namespace xRetry.Exceptions
+{
+    [Serializable]
+    public class SkipTestException : Exception
+    {
+        public readonly string Reason;
+
+        public SkipTestException(string reason)
+            : base("Test skipped. Reason: " + reason)
+        {
+            Reason = reason;
+        }
+
+        protected SkipTestException(SerializationInfo info, StreamingContext context)
+            : base(info, context)
+        {
+            Reason = info.GetString(nameof(Reason));
+        }
+
+        public override void GetObjectData(SerializationInfo info, StreamingContext context)
+        {
+            info.AddValue(nameof(Reason), Reason);
+
+            base.GetObjectData(info, context);
+        }
+    }
+}

src/xRetry/Extensions/EnumerableExtensions.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace xRetry.Extensions
+{
+    public static class EnumerableExtensions
+    {
+        public static bool ContainsAny<T>(this IEnumerable<T> values, T[] searchFor, IEqualityComparer<T> comparer = null)
+        {
+            if (searchFor == null)
+            {
+                throw new ArgumentNullException(nameof(searchFor));
+            }
+            if (comparer == null)
+            {
+                comparer = EqualityComparer<T>.Default;
+            }
+
+            return searchFor.Length != 0 && 
+                   values.Any(val => searchFor.Any(search => comparer.Equals(val, search)));
+        }
+    }
+}

src/xRetry/IRetryableTestCase.cs

@@ -6,5 +6,6 @@ public interface IRetryableTestCase : IXunitTestCase
     {
         int MaxRetries { get; }
         int DelayBetweenRetriesMs { get; }
+        string[] SkipOnExceptionFullNames { get; }
     }
 }

src/xRetry/MessageTransformer.cs

@@ -0,0 +1,39 @@
+using System.Linq;
+using xRetry.Extensions;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace xRetry
+{
+    public class MessageTransformer
+    {
+        private readonly string[] skipOnExceptionFullNames;
+
+        public bool Skipped { get; private set; }
+
+        public MessageTransformer(string[] skipOnExceptionFullNames)
+        {
+            this.skipOnExceptionFullNames = skipOnExceptionFullNames;
+        }
+
+        /// <summary>
+        /// Transforms a message received from an xUnit test into another message, replacing it
+        /// where necessary to add additional functionality, e.g. dynamic skipping
+        /// </summary>
+        /// <param name="message"></param>
+        /// <returns></returns>
+        public IMessageSinkMessage Transform(IMessageSinkMessage message)
+        {
+            // If this is a message saying that the test has been skipped, replace the message with skipping the test
+            if (message is TestFailed failed && failed.ExceptionTypes.ContainsAny(skipOnExceptionFullNames))
+            {
+                string reason = failed.Messages?.FirstOrDefault();
+                Skipped = true;
+                return new TestSkipped(failed.Test, reason);
+            }
+
+            // Otherwise this isn't a message saying the test is skipped, follow usual intercept for replay later behaviour
+            return message;
+        }
+    }
+}

src/xRetry/RetryFactAttribute.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using Xunit;
 using Xunit.Sdk;
 
@@ -12,15 +13,38 @@ namespace xRetry
     [AttributeUsage(AttributeTargets.Method)]
     public class RetryFactAttribute : FactAttribute
     {
-        public readonly int MaxRetries;
-        public readonly int DelayBetweenRetriesMs;
+        public const int DEFAULT_MAX_RETRIES = 3;
+        public const int DEFAULT_DELAY_BETWEEN_RETRIES_MS = 0;
+
+        public readonly int MaxRetries = DEFAULT_MAX_RETRIES;
+        public readonly int DelayBetweenRetriesMs = DEFAULT_DELAY_BETWEEN_RETRIES_MS;
+        public readonly Type[] SkipOnExceptions;
+
+        /// <summary>
+        /// Ctor (just skip on exceptions)
+        /// </summary>
+        /// <param name="skipOnExceptions">Mark the test as skipped when this type of exception is encountered</param>
+        public RetryFactAttribute(params Type[] skipOnExceptions)
+        {
+            SkipOnExceptions = skipOnExceptions ?? Type.EmptyTypes;
+
+            if (SkipOnExceptions.Any(t => !t.IsSubclassOf(typeof(Exception))))
+            {
+                throw new ArgumentException("Specified type must be an exception", nameof(skipOnExceptions));
+            }
+        }
 
         /// <summary>
-        /// Ctor
+        /// Ctor (full)
         /// </summary>
         /// <param name="maxRetries">The number of times to run a test for until it succeeds</param>
         /// <param name="delayBetweenRetriesMs">The amount of time (in ms) to wait between each test run attempt</param>
-        public RetryFactAttribute(int maxRetries = 3, int delayBetweenRetriesMs = 0)
+        /// <param name="skipOnExceptions">Mark the test as skipped when this type of exception is encountered</param>
+        public RetryFactAttribute(
+            int maxRetries = DEFAULT_MAX_RETRIES, 
+            int delayBetweenRetriesMs = DEFAULT_DELAY_BETWEEN_RETRIES_MS, 
+            params Type[] skipOnExceptions)
+            : this(skipOnExceptions)
         {
             if (maxRetries < 1)
             {

src/xRetry/RetryFactDiscoverer.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using Xunit.Abstractions;
@@ -36,8 +37,12 @@ public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions disco
                 int maxRetries = factAttribute.GetNamedArgument<int>(nameof(RetryFactAttribute.MaxRetries));
                 int delayBetweenRetriesMs =
                     factAttribute.GetNamedArgument<int>(nameof(RetryFactAttribute.DelayBetweenRetriesMs));
+                Type[] skipOnExceptions =
+                    factAttribute.GetNamedArgument<Type[]>(nameof(RetryTheoryAttribute.SkipOnExceptions));
+
                 testCase = new RetryTestCase(messageSink, discoveryOptions.MethodDisplayOrDefault(),
-                    discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs);
+                    discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs,
+                    skipOnExceptions);
             }
 
             return new[] { testCase };

src/xRetry/RetryTestCase.cs

@@ -2,6 +2,7 @@
 using System.ComponentModel;
 using System.Threading;
 using System.Threading.Tasks;
+using xRetry.Exceptions;
 using Xunit.Abstractions;
 using Xunit.Sdk;
 
@@ -12,6 +13,7 @@ public class RetryTestCase : XunitTestCase, IRetryableTestCase
     {
         public int MaxRetries { get; private set; }
         public int DelayBetweenRetriesMs { get; private set; }
+        public string[] SkipOnExceptionFullNames { get; private set; }
 
         [EditorBrowsable(EditorBrowsableState.Never)]
         [Obsolete(
@@ -25,12 +27,14 @@ public RetryTestCase(
             ITestMethod testMethod,
             int maxRetries,
             int delayBetweenRetriesMs,
+            Type[] skipOnExceptions,
             object[] testMethodArguments = null)
             : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod,
                 testMethodArguments)
         {
             MaxRetries = maxRetries;
             DelayBetweenRetriesMs = delayBetweenRetriesMs;
+            SkipOnExceptionFullNames = GetSkipOnExceptionFullNames(skipOnExceptions);
         }
 
         public override Task<RunSummary> RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus,
@@ -47,6 +51,7 @@ public override void Serialize(IXunitSerializationInfo data)
 
             data.AddValue("MaxRetries", MaxRetries);
             data.AddValue("DelayBetweenRetriesMs", DelayBetweenRetriesMs);
+            data.AddValue("SkipOnExceptionFullNames", SkipOnExceptionFullNames);
         }
 
         public override void Deserialize(IXunitSerializationInfo data)
@@ -55,6 +60,18 @@ public override void Deserialize(IXunitSerializationInfo data)
 
             MaxRetries = data.GetValue<int>("MaxRetries");
             DelayBetweenRetriesMs = data.GetValue<int>("DelayBetweenRetriesMs");
+            SkipOnExceptionFullNames = data.GetValue<string[]>("SkipOnExceptionFullNames");
+        }
+
+        public static string[] GetSkipOnExceptionFullNames(Type[] customSkipOnExceptions)
+        {
+            string[] toRet = new string[customSkipOnExceptions.Length + 1];
+            for (int i = 0; i < customSkipOnExceptions.Length; i++)
+            {
+                toRet[i] = customSkipOnExceptions[i].FullName;
+            }
+            toRet[toRet.Length - 1] = typeof(SkipTestException).FullName;
+            return toRet;
         }
     }
 }

src/xRetry/RetryTestCaseRunner.cs

@@ -28,14 +28,21 @@ public static async Task<RunSummary> RunAsync(
             {
                 // Prevent messages from the test run from being passed through, as we don't want 
                 //  a message to mark the test as failed when we're going to retry it
-                using (BlockingMessageBus blockingMessageBus = new BlockingMessageBus(messageBus))
+                MessageTransformer messageTransformer = new MessageTransformer(testCase.SkipOnExceptionFullNames);
+                using (BlockingMessageBus blockingMessageBus = new BlockingMessageBus(messageBus, messageTransformer))
                 {
                     diagnosticMessageSink.OnMessage(new DiagnosticMessage("Running test \"{0}\" attempt ({1}/{2})",
                         testCase.DisplayName, i, testCase.MaxRetries));
 
                     RunSummary summary = await fnRunSingle(blockingMessageBus);
 
-                    // If we succeeded, or we've reached the max retries return the result
+                    if (messageTransformer.Skipped)
+                    {
+                        summary.Failed = 0;
+                        summary.Skipped = 1;
+                    }
+
+                    // If we succeeded, skipped, or we've reached the max retries return the result
                     if (summary.Failed == 0 || i == testCase.MaxRetries)
                     {
                         // If we have failed (after all retries, log that)

src/xRetry/RetryTheoryAttribute.cs

@@ -12,7 +12,14 @@ namespace xRetry
     public class RetryTheoryAttribute : RetryFactAttribute
     {
         /// <inheritdoc/>
-        public RetryTheoryAttribute(int maxRetries = 3, int delayBetweenRetriesMs = 0)
-            : base(maxRetries, delayBetweenRetriesMs) {  }
+        public RetryTheoryAttribute(params Type[] skipOnExceptions)
+            : base(skipOnExceptions) {  }
+
+        /// <inheritdoc/>
+        public RetryTheoryAttribute(
+            int maxRetries = DEFAULT_MAX_RETRIES, 
+            int delayBetweenRetriesMs = DEFAULT_DELAY_BETWEEN_RETRIES_MS, 
+            params Type[] skipOnExceptions)
+            : base(maxRetries, delayBetweenRetriesMs, skipOnExceptions) {  }
     }
 }