From ea04a1b9d77462f6194c60a99c480579a110b14e Mon Sep 17 00:00:00 2001 From: Josh Keegan Date: Thu, 16 Sep 2021 18:08:19 +0100 Subject: [PATCH 01/10] Tests for skip/ignore --- .../Exceptions/SkipTestExceptionTests.cs | 37 +++++++++++++++++ .../Facts/RetryFactRuntimeSkipTests.cs | 31 ++++++++++++++ .../RetryRuntimeIgnoreScenarios.feature | 19 +++++++++ .../SpecFlow/Steps/RuntimeIgnoreSteps.cs | 22 ++++++++++ .../Theories/RetryTheoryRuntimeSkipTests.cs | 41 +++++++++++++++++++ test/UnitTests/UnitTests.csproj | 2 + 6 files changed, 152 insertions(+) create mode 100644 test/UnitTests/Exceptions/SkipTestExceptionTests.cs create mode 100644 test/UnitTests/Facts/RetryFactRuntimeSkipTests.cs create mode 100644 test/UnitTests/SpecFlow/Features/RetryRuntimeIgnoreScenarios.feature create mode 100644 test/UnitTests/SpecFlow/Steps/RuntimeIgnoreSteps.cs create mode 100644 test/UnitTests/Theories/RetryTheoryRuntimeSkipTests.cs diff --git a/test/UnitTests/Exceptions/SkipTestExceptionTests.cs b/test/UnitTests/Exceptions/SkipTestExceptionTests.cs new file mode 100644 index 0000000..c7d4f91 --- /dev/null +++ b/test/UnitTests/Exceptions/SkipTestExceptionTests.cs @@ -0,0 +1,37 @@ +using System.IO; +using System.Runtime.Serialization.Formatters.Binary; +using AutoFixture; +using FluentAssertions; +using xRetry.Exceptions; +using Xunit; + +namespace UnitTests.Exceptions +{ + public class SkipTestExceptionTests + { + [Fact] + public void Serialisation_RoundTrip_RetainsData() + { + // Arrange + Fixture fixture = new Fixture(); + SkipTestException expected = new SkipTestException(fixture.Create()); + + // Act + SkipTestException actual; + BinaryFormatter formatter = new BinaryFormatter(); + using (MemoryStream s = new MemoryStream()) + { +#pragma warning disable SYSLIB0011 // Type or member is obsolete +#pragma warning disable 618 + formatter.Serialize(s, expected); + s.Position = 0; + actual = (SkipTestException) formatter.Deserialize(s); +#pragma warning restore SYSLIB0011 // Type or member is obsolete +#pragma warning restore 618 + } + + // Assert + actual.Should().BeEquivalentTo(expected); + } + } +} diff --git a/test/UnitTests/Facts/RetryFactRuntimeSkipTests.cs b/test/UnitTests/Facts/RetryFactRuntimeSkipTests.cs new file mode 100644 index 0000000..c4e2ff6 --- /dev/null +++ b/test/UnitTests/Facts/RetryFactRuntimeSkipTests.cs @@ -0,0 +1,31 @@ +using xRetry; +using Xunit; +using Skip = xRetry.Skip; + +namespace UnitTests.Facts +{ + public class RetryFactRuntimeSkipTests + { + [RetryFact] + public void SkipAtRuntime() + { + // Note: All we're doing with this test is checking that the rest of the test doesn't get run + // checking it's skipped (and doesn't pass) would need to be done manually. + Skip.Always(); + + Assert.True(false, "Should have been skipped . . ."); + } + + private static int skippedNumCalls = 0; + + [RetryFact] + public void Skip_DoesNotRetry() + { + // Assertion would fail on subsequent attempts, before reaching the skip + skippedNumCalls++; + Assert.Equal(1, skippedNumCalls); + + Skip.Always(); + } + } +} diff --git a/test/UnitTests/SpecFlow/Features/RetryRuntimeIgnoreScenarios.feature b/test/UnitTests/SpecFlow/Features/RetryRuntimeIgnoreScenarios.feature new file mode 100644 index 0000000..75e4dc7 --- /dev/null +++ b/test/UnitTests/SpecFlow/Features/RetryRuntimeIgnoreScenarios.feature @@ -0,0 +1,19 @@ +Feature: Retry Ignore Scenarios + In order to allow for tests to be ignored/skipped at runtime + So that the full feature set of SpecFlow is still available with xRetry (IUnitTestRuntimeProvider.TestIgnore) + As a QA engineer + I want to be able to ignore/skip tests + +@retry +Scenario: Test is ignored at runtime + When I ignore this test + Then fail because this test should have been skipped + +@retry +Scenario Outline: Test (outline) is ignored at runtime + When I ignore this test + Then fail because this test should have been skipped + Examples: + | n | + | 1 | + | 2 | \ No newline at end of file diff --git a/test/UnitTests/SpecFlow/Steps/RuntimeIgnoreSteps.cs b/test/UnitTests/SpecFlow/Steps/RuntimeIgnoreSteps.cs new file mode 100644 index 0000000..8e47d36 --- /dev/null +++ b/test/UnitTests/SpecFlow/Steps/RuntimeIgnoreSteps.cs @@ -0,0 +1,22 @@ +using TechTalk.SpecFlow; +using TechTalk.SpecFlow.UnitTestProvider; + +namespace UnitTests.SpecFlow.Steps +{ + [Binding] + public class RuntimeIgnoreSteps + { + private readonly IUnitTestRuntimeProvider unitTestRuntimeProvider; + + public RuntimeIgnoreSteps(IUnitTestRuntimeProvider unitTestRuntimeProvider) + { + this.unitTestRuntimeProvider = unitTestRuntimeProvider; + } + + [When(@"I ignore this test")] + public void WhenIIgnoreThisTest() + { + unitTestRuntimeProvider.TestIgnore("Ignored at runtime"); + } + } +} diff --git a/test/UnitTests/Theories/RetryTheoryRuntimeSkipTests.cs b/test/UnitTests/Theories/RetryTheoryRuntimeSkipTests.cs new file mode 100644 index 0000000..aaf09d1 --- /dev/null +++ b/test/UnitTests/Theories/RetryTheoryRuntimeSkipTests.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; +using xRetry; +using Xunit; +using Skip = xRetry.Skip; + +namespace UnitTests.Theories +{ + public class RetryTheoryRuntimeSkipTests + { + [RetryTheory] + [InlineData(0)] + [InlineData(1)] + public void SkipAtRuntime(int _) + { + // Note: All we're doing with this test is checking that the rest of the test doesn't get run + // checking it's skipped (and doesn't pass) would need to be done manually. + Skip.Always(); + + Assert.True(false, "Should have been skipped . . ."); + } + + // testId => numCalls + private static readonly Dictionary skippedNumCalls = new Dictionary() + { + { 0, 0 }, + { 1, 0 } + }; + + [RetryTheory] + [InlineData(0)] + [InlineData(1)] + public void Skip_DoesNotRetry(int id) + { + // Assertion would fail on subsequent attempts, before reaching the skip + skippedNumCalls[id]++; + Assert.Equal(1, skippedNumCalls[id]); + + Skip.Always(); + } + } +} diff --git a/test/UnitTests/UnitTests.csproj b/test/UnitTests/UnitTests.csproj index e257ea8..e6f34b1 100644 --- a/test/UnitTests/UnitTests.csproj +++ b/test/UnitTests/UnitTests.csproj @@ -17,6 +17,8 @@ + + From 6ebf30791b3bf311231f35c8fe5319509065356a Mon Sep 17 00:00:00 2001 From: Josh Keegan Date: Thu, 16 Sep 2021 18:09:17 +0100 Subject: [PATCH 02/10] Base classes required for tests to compile. Implementation time... --- src/xRetry/Exceptions/SkipTestException.cs | 32 ++++++++++++++++++++++ src/xRetry/Skip.cs | 16 +++++++++++ 2 files changed, 48 insertions(+) create mode 100644 src/xRetry/Exceptions/SkipTestException.cs create mode 100644 src/xRetry/Skip.cs diff --git a/src/xRetry/Exceptions/SkipTestException.cs b/src/xRetry/Exceptions/SkipTestException.cs new file mode 100644 index 0000000..d0fe720 --- /dev/null +++ b/src/xRetry/Exceptions/SkipTestException.cs @@ -0,0 +1,32 @@ +using System; +using System.Runtime.Serialization; + +namespace xRetry.Exceptions +{ + // TODO: Is serialisation implementation good enough? + // Maybe json etc... would fail. See RetryTestCase for example + [Serializable] + public class SkipTestException : Exception + { + public readonly string Reason; + + public SkipTestException(string reason) + : base("Test skipped. Reason: " + reason) + { + Reason = reason; + } + + protected SkipTestException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + Reason = info.GetString(nameof(Reason)); + } + + public override void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue(nameof(Reason), Reason); + + base.GetObjectData(info, context); + } + } +} diff --git a/src/xRetry/Skip.cs b/src/xRetry/Skip.cs new file mode 100644 index 0000000..b24e5f9 --- /dev/null +++ b/src/xRetry/Skip.cs @@ -0,0 +1,16 @@ +using xRetry.Exceptions; + +namespace xRetry +{ + public static class Skip + { + /// + /// Throws an exception that results in a "Skipped" result for the test. + /// + /// Reason for the test needing to be skipped + public static void Always(string reason = null) + { + throw new SkipTestException(reason); + } + } +} From ca819bbbd28f6bbffbbaae68d8c1244e01b3f785 Mon Sep 17 00:00:00 2001 From: Josh Keegan Date: Thu, 16 Sep 2021 18:20:38 +0100 Subject: [PATCH 03/10] Implement xunit retry skip behaviour (tests now pass). Specflow still TODO --- src/xRetry/BlockingMessageBus.cs | 16 +++++++++++++++- src/xRetry/RetryTestCaseRunner.cs | 8 +++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/xRetry/BlockingMessageBus.cs b/src/xRetry/BlockingMessageBus.cs index 829d287..30c3215 100644 --- a/src/xRetry/BlockingMessageBus.cs +++ b/src/xRetry/BlockingMessageBus.cs @@ -1,4 +1,6 @@ using System.Collections.Concurrent; +using System.Linq; +using xRetry.Exceptions; using Xunit.Abstractions; using Xunit.Sdk; @@ -11,6 +13,7 @@ public class BlockingMessageBus : IMessageBus { private readonly IMessageBus underlyingMessageBus; private ConcurrentQueue messageQueue = new ConcurrentQueue(); + public bool Skipped { get; private set; } = false; public BlockingMessageBus(IMessageBus underlyingMessageBus) { @@ -19,7 +22,18 @@ public BlockingMessageBus(IMessageBus underlyingMessageBus) public bool QueueMessage(IMessageSinkMessage message) { - messageQueue.Enqueue(message); + // If this is a message saying that the test has been skipped, we can interrupt execution at this point + if (message is TestFailed failed && failed.ExceptionTypes.Contains(typeof(SkipTestException).FullName)) + { + string reason = failed.Messages?.FirstOrDefault(); + messageQueue.Enqueue(new TestSkipped(failed.Test, reason)); + Skipped = true; + } + else + { + // Otherwise this isn't a message saying the test is skipped, follow usual intercept & replay later behaviour + messageQueue.Enqueue(message); + } // Returns if execution should continue. Since we are intercepting the message, we // have no way of checking this so always continue... diff --git a/src/xRetry/RetryTestCaseRunner.cs b/src/xRetry/RetryTestCaseRunner.cs index 58ffe89..fafef9d 100644 --- a/src/xRetry/RetryTestCaseRunner.cs +++ b/src/xRetry/RetryTestCaseRunner.cs @@ -35,7 +35,13 @@ public static async Task RunAsync( RunSummary summary = await fnRunSingle(blockingMessageBus); - // If we succeeded, or we've reached the max retries return the result + if (blockingMessageBus.Skipped) + { + summary.Failed = 0; + summary.Skipped = 1; + } + + // If we succeeded, skipped, or we've reached the max retries return the result if (summary.Failed == 0 || i == testCase.MaxRetries) { // If we have failed (after all retries, log that) From 7ad647b162fc79d60e745e5671c5d4464565753d Mon Sep 17 00:00:00 2001 From: Josh Keegan Date: Thu, 16 Sep 2021 19:10:14 +0100 Subject: [PATCH 04/10] Implement runtime skip for specflow (tests green). Due to not being able to reuse the xunit plugin though, inconclusive and pending will also need implementing & testing --- src/xRetry.SpecFlow/RuntimePlugin.cs | 21 ++++++++++++++++++ src/xRetry.SpecFlow/TestRuntimeProvider.cs | 25 ++++++++++++++++++++++ src/xRetry.SpecFlow/xRetry.SpecFlow.csproj | 4 ++++ 3 files changed, 50 insertions(+) create mode 100644 src/xRetry.SpecFlow/RuntimePlugin.cs create mode 100644 src/xRetry.SpecFlow/TestRuntimeProvider.cs diff --git a/src/xRetry.SpecFlow/RuntimePlugin.cs b/src/xRetry.SpecFlow/RuntimePlugin.cs new file mode 100644 index 0000000..661b51b --- /dev/null +++ b/src/xRetry.SpecFlow/RuntimePlugin.cs @@ -0,0 +1,21 @@ +using TechTalk.SpecFlow.Plugins; +using TechTalk.SpecFlow.UnitTestProvider; +using xRetry.SpecFlow; + +[assembly: RuntimePlugin(typeof(RuntimePlugin))] +namespace xRetry.SpecFlow +{ + public class RuntimePlugin : IRuntimePlugin + { + public void Initialize(RuntimePluginEvents runtimePluginEvents, RuntimePluginParameters runtimePluginParameters, + UnitTestProviderConfiguration unitTestProviderConfiguration) + { + runtimePluginEvents.CustomizeGlobalDependencies += customiseGlobalDependencies; + } + + private void customiseGlobalDependencies(object sender, CustomizeGlobalDependenciesEventArgs eventArgs) + { + eventArgs.ObjectContainer.RegisterTypeAs(); + } + } +} diff --git a/src/xRetry.SpecFlow/TestRuntimeProvider.cs b/src/xRetry.SpecFlow/TestRuntimeProvider.cs new file mode 100644 index 0000000..33a1b7c --- /dev/null +++ b/src/xRetry.SpecFlow/TestRuntimeProvider.cs @@ -0,0 +1,25 @@ +using TechTalk.SpecFlow.UnitTestProvider; + +namespace xRetry.SpecFlow +{ + public class TestRuntimeProvider : IUnitTestRuntimeProvider + { + public bool DelayedFixtureTearDown => false; + + public void TestIgnore(string message) + { + Skip.Always(message); + } + + // TODO: Implement to match existing Specflow (& also test I guess...) + public void TestInconclusive(string message) + { + throw new System.NotImplementedException(); + } + + public void TestPending(string message) + { + throw new System.NotImplementedException(); + } + } +} diff --git a/src/xRetry.SpecFlow/xRetry.SpecFlow.csproj b/src/xRetry.SpecFlow/xRetry.SpecFlow.csproj index 51f0bdb..9257d48 100644 --- a/src/xRetry.SpecFlow/xRetry.SpecFlow.csproj +++ b/src/xRetry.SpecFlow/xRetry.SpecFlow.csproj @@ -12,4 +12,8 @@ + + + + From ab86a0280f833e633f898bcc371c851236b21c22 Mon Sep 17 00:00:00 2001 From: Josh Keegan Date: Thu, 16 Sep 2021 19:20:26 +0100 Subject: [PATCH 05/10] Update comments to reflect behaviour change (original uncommitted implementation had a bug which the comment shows!) --- src/xRetry/BlockingMessageBus.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/xRetry/BlockingMessageBus.cs b/src/xRetry/BlockingMessageBus.cs index 30c3215..466e5b5 100644 --- a/src/xRetry/BlockingMessageBus.cs +++ b/src/xRetry/BlockingMessageBus.cs @@ -22,7 +22,7 @@ public BlockingMessageBus(IMessageBus underlyingMessageBus) public bool QueueMessage(IMessageSinkMessage message) { - // If this is a message saying that the test has been skipped, we can interrupt execution at this point + // If this is a message saying that the test has been skipped, replace the message with skipping the test if (message is TestFailed failed && failed.ExceptionTypes.Contains(typeof(SkipTestException).FullName)) { string reason = failed.Messages?.FirstOrDefault(); @@ -31,7 +31,7 @@ public bool QueueMessage(IMessageSinkMessage message) } else { - // Otherwise this isn't a message saying the test is skipped, follow usual intercept & replay later behaviour + // Otherwise this isn't a message saying the test is skipped, follow usual intercept for replay later behaviour messageQueue.Enqueue(message); } From 2900f0f41b68d2e74407988ff5572e54d5083736 Mon Sep 17 00:00:00 2001 From: Josh Keegan Date: Mon, 20 Sep 2021 18:57:52 +0100 Subject: [PATCH 06/10] Implement Specflow ignore functionality without requiring runtime plugin, so we don't also affect the behaviour of inconclusive or pending specflow test outcomes --- src/xRetry.SpecFlow/RuntimePlugin.cs | 21 ------- src/xRetry.SpecFlow/SkipException.cs | 14 +++++ src/xRetry.SpecFlow/TestGeneratorProvider.cs | 26 ++++---- src/xRetry.SpecFlow/TestRuntimeProvider.cs | 25 -------- src/xRetry/BlockingMessageBus.cs | 8 ++- src/xRetry/Extensions/EnumerableExtensions.cs | 24 ++++++++ src/xRetry/IRetryableTestCase.cs | 1 + src/xRetry/RetryFactAttribute.cs | 32 ++++++++-- src/xRetry/RetryFactDiscoverer.cs | 7 ++- src/xRetry/RetryTestCase.cs | 17 ++++++ src/xRetry/RetryTestCaseRunner.cs | 2 +- src/xRetry/RetryTheoryAttribute.cs | 11 +++- src/xRetry/RetryTheoryDiscoverer.cs | 8 ++- .../RetryTheoryDiscoveryAtRuntimeCase.cs | 7 ++- .../Facts/RetryFactRuntimeSkipTests.cs | 7 +++ test/UnitTests/RetryFactAttributeTests.cs | 60 +++++++++++++++++++ .../TestClasses/NonSerializableTestData.cs | 12 ++++ test/UnitTests/TestClasses/TestException.cs | 6 ++ .../RetryTheoryNonSerializableDataTests.cs | 11 +--- ...eoryRuntimeSkipNonSerializableDataTests.cs | 53 ++++++++++++++++ .../Theories/RetryTheoryRuntimeSkipTests.cs | 9 +++ 21 files changed, 281 insertions(+), 80 deletions(-) delete mode 100644 src/xRetry.SpecFlow/RuntimePlugin.cs create mode 100644 src/xRetry.SpecFlow/SkipException.cs delete mode 100644 src/xRetry.SpecFlow/TestRuntimeProvider.cs create mode 100644 src/xRetry/Extensions/EnumerableExtensions.cs create mode 100644 test/UnitTests/RetryFactAttributeTests.cs create mode 100644 test/UnitTests/TestClasses/NonSerializableTestData.cs create mode 100644 test/UnitTests/TestClasses/TestException.cs create mode 100644 test/UnitTests/Theories/RetryTheoryRuntimeSkipNonSerializableDataTests.cs diff --git a/src/xRetry.SpecFlow/RuntimePlugin.cs b/src/xRetry.SpecFlow/RuntimePlugin.cs deleted file mode 100644 index 661b51b..0000000 --- a/src/xRetry.SpecFlow/RuntimePlugin.cs +++ /dev/null @@ -1,21 +0,0 @@ -using TechTalk.SpecFlow.Plugins; -using TechTalk.SpecFlow.UnitTestProvider; -using xRetry.SpecFlow; - -[assembly: RuntimePlugin(typeof(RuntimePlugin))] -namespace xRetry.SpecFlow -{ - public class RuntimePlugin : IRuntimePlugin - { - public void Initialize(RuntimePluginEvents runtimePluginEvents, RuntimePluginParameters runtimePluginParameters, - UnitTestProviderConfiguration unitTestProviderConfiguration) - { - runtimePluginEvents.CustomizeGlobalDependencies += customiseGlobalDependencies; - } - - private void customiseGlobalDependencies(object sender, CustomizeGlobalDependenciesEventArgs eventArgs) - { - eventArgs.ObjectContainer.RegisterTypeAs(); - } - } -} diff --git a/src/xRetry.SpecFlow/SkipException.cs b/src/xRetry.SpecFlow/SkipException.cs new file mode 100644 index 0000000..68bf5a7 --- /dev/null +++ b/src/xRetry.SpecFlow/SkipException.cs @@ -0,0 +1,14 @@ +using System; + +// ReSharper disable once CheckNamespace +// ReSharper disable once IdentifierTypo +namespace Xunit +{ + /// + /// Do not use. + /// Exists purely as a marker to replicate the exception thrown by Xunit.SkippableFact that SpecFlow.xUnit + /// makes use of. That way we can intercept the exception that is throwing without also having our own runtime + /// plugin, or adding a direct dependency on either of these other libraries. + /// + internal class SkipException : Exception { } +} diff --git a/src/xRetry.SpecFlow/TestGeneratorProvider.cs b/src/xRetry.SpecFlow/TestGeneratorProvider.cs index 25ffd90..7ba7b8b 100644 --- a/src/xRetry.SpecFlow/TestGeneratorProvider.cs +++ b/src/xRetry.SpecFlow/TestGeneratorProvider.cs @@ -59,17 +59,21 @@ public override void SetTestMethodCategories(TestClassGenerationContext generati CodeAttributeDeclaration retryAttribute = CodeDomHelper.AddAttribute(testMethod, "xRetry.Retry" + (originalAttribute.Name == FACT_ATTRIBUTE ? "Fact" : "Theory")); - if (retryTag.MaxRetries != null) - { - retryAttribute.Arguments.Add( - new CodeAttributeArgument(new CodePrimitiveExpression(retryTag.MaxRetries))); - - if(retryTag.DelayBetweenRetriesMs != null) - { - retryAttribute.Arguments.Add( - new CodeAttributeArgument(new CodePrimitiveExpression(retryTag.DelayBetweenRetriesMs))); - } - } + retryAttribute.Arguments.Add(new CodeAttributeArgument( + new CodePrimitiveExpression(retryTag.MaxRetries ?? RetryFactAttribute.DEFAULT_MAX_RETRIES))); + retryAttribute.Arguments.Add(new CodeAttributeArgument( + new CodePrimitiveExpression(retryTag.DelayBetweenRetriesMs ?? + RetryFactAttribute.DEFAULT_DELAY_BETWEEN_RETRIES_MS))); + + // Always skip on Xunit.SkipException (from Xunit.SkippableFact) which is used by SpecFlow.xUnit to implement + // dynamic test skipping. This way we can intercept the exception that is already thrown without also having + // our own runtime plugin. + retryAttribute.Arguments.Add(new CodeAttributeArgument( + new CodeArrayCreateExpression(new CodeTypeReference(typeof(Type)), + new CodeExpression[] + { + new CodeTypeOfExpression(typeof(Xunit.SkipException)) + }))); // Copy arguments from the original attribute for (int i = 0; i < originalAttribute.Arguments.Count; i++) diff --git a/src/xRetry.SpecFlow/TestRuntimeProvider.cs b/src/xRetry.SpecFlow/TestRuntimeProvider.cs deleted file mode 100644 index 33a1b7c..0000000 --- a/src/xRetry.SpecFlow/TestRuntimeProvider.cs +++ /dev/null @@ -1,25 +0,0 @@ -using TechTalk.SpecFlow.UnitTestProvider; - -namespace xRetry.SpecFlow -{ - public class TestRuntimeProvider : IUnitTestRuntimeProvider - { - public bool DelayedFixtureTearDown => false; - - public void TestIgnore(string message) - { - Skip.Always(message); - } - - // TODO: Implement to match existing Specflow (& also test I guess...) - public void TestInconclusive(string message) - { - throw new System.NotImplementedException(); - } - - public void TestPending(string message) - { - throw new System.NotImplementedException(); - } - } -} diff --git a/src/xRetry/BlockingMessageBus.cs b/src/xRetry/BlockingMessageBus.cs index 466e5b5..fc87277 100644 --- a/src/xRetry/BlockingMessageBus.cs +++ b/src/xRetry/BlockingMessageBus.cs @@ -1,6 +1,6 @@ using System.Collections.Concurrent; using System.Linq; -using xRetry.Exceptions; +using xRetry.Extensions; using Xunit.Abstractions; using Xunit.Sdk; @@ -12,18 +12,20 @@ namespace xRetry public class BlockingMessageBus : IMessageBus { private readonly IMessageBus underlyingMessageBus; + private readonly string[] skipOnExceptionFullNames; private ConcurrentQueue messageQueue = new ConcurrentQueue(); public bool Skipped { get; private set; } = false; - public BlockingMessageBus(IMessageBus underlyingMessageBus) + public BlockingMessageBus(IMessageBus underlyingMessageBus, string[] skipOnExceptionFullNames) { this.underlyingMessageBus = underlyingMessageBus; + this.skipOnExceptionFullNames = skipOnExceptionFullNames; } public bool QueueMessage(IMessageSinkMessage message) { // If this is a message saying that the test has been skipped, replace the message with skipping the test - if (message is TestFailed failed && failed.ExceptionTypes.Contains(typeof(SkipTestException).FullName)) + if (message is TestFailed failed && failed.ExceptionTypes.ContainsAny(skipOnExceptionFullNames)) { string reason = failed.Messages?.FirstOrDefault(); messageQueue.Enqueue(new TestSkipped(failed.Test, reason)); diff --git a/src/xRetry/Extensions/EnumerableExtensions.cs b/src/xRetry/Extensions/EnumerableExtensions.cs new file mode 100644 index 0000000..6d51872 --- /dev/null +++ b/src/xRetry/Extensions/EnumerableExtensions.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace xRetry.Extensions +{ + public static class EnumerableExtensions + { + public static bool ContainsAny(this IEnumerable values, T[] searchFor, IEqualityComparer comparer = null) + { + if (searchFor == null) + { + throw new ArgumentNullException(nameof(searchFor)); + } + if (comparer == null) + { + comparer = EqualityComparer.Default; + } + + return searchFor.Length != 0 && + values.Any(val => searchFor.Any(search => comparer.Equals(val, search))); + } + } +} diff --git a/src/xRetry/IRetryableTestCase.cs b/src/xRetry/IRetryableTestCase.cs index 366486b..7832b8f 100644 --- a/src/xRetry/IRetryableTestCase.cs +++ b/src/xRetry/IRetryableTestCase.cs @@ -6,5 +6,6 @@ public interface IRetryableTestCase : IXunitTestCase { int MaxRetries { get; } int DelayBetweenRetriesMs { get; } + string[] SkipOnExceptionFullNames { get; } } } diff --git a/src/xRetry/RetryFactAttribute.cs b/src/xRetry/RetryFactAttribute.cs index 6819e30..fa15f8d 100644 --- a/src/xRetry/RetryFactAttribute.cs +++ b/src/xRetry/RetryFactAttribute.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using Xunit; using Xunit.Sdk; @@ -12,15 +13,38 @@ namespace xRetry [AttributeUsage(AttributeTargets.Method)] public class RetryFactAttribute : FactAttribute { - public readonly int MaxRetries; - public readonly int DelayBetweenRetriesMs; + public const int DEFAULT_MAX_RETRIES = 3; + public const int DEFAULT_DELAY_BETWEEN_RETRIES_MS = 0; + + public readonly int MaxRetries = DEFAULT_MAX_RETRIES; + public readonly int DelayBetweenRetriesMs = DEFAULT_DELAY_BETWEEN_RETRIES_MS; + public readonly Type[] SkipOnExceptions; + + /// + /// Ctor (just skip on exceptions) + /// + /// Mark the test as skipped when this type of exception is encountered + public RetryFactAttribute(params Type[] skipOnExceptions) + { + SkipOnExceptions = skipOnExceptions ?? Type.EmptyTypes; + + if (SkipOnExceptions.Any(t => !t.IsSubclassOf(typeof(Exception)))) + { + throw new ArgumentException("Specified type must be an exception", nameof(skipOnExceptions)); + } + } /// - /// Ctor + /// Ctor (full) /// /// The number of times to run a test for until it succeeds /// The amount of time (in ms) to wait between each test run attempt - public RetryFactAttribute(int maxRetries = 3, int delayBetweenRetriesMs = 0) + /// Mark the test as skipped when this type of exception is encountered + public RetryFactAttribute( + int maxRetries = DEFAULT_MAX_RETRIES, + int delayBetweenRetriesMs = DEFAULT_DELAY_BETWEEN_RETRIES_MS, + params Type[] skipOnExceptions) + : this(skipOnExceptions) { if (maxRetries < 1) { diff --git a/src/xRetry/RetryFactDiscoverer.cs b/src/xRetry/RetryFactDiscoverer.cs index 1884c91..67b6966 100644 --- a/src/xRetry/RetryFactDiscoverer.cs +++ b/src/xRetry/RetryFactDiscoverer.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using Xunit.Abstractions; @@ -36,8 +37,12 @@ public IEnumerable Discover(ITestFrameworkDiscoveryOptions disco int maxRetries = factAttribute.GetNamedArgument(nameof(RetryFactAttribute.MaxRetries)); int delayBetweenRetriesMs = factAttribute.GetNamedArgument(nameof(RetryFactAttribute.DelayBetweenRetriesMs)); + Type[] skipOnExceptions = + factAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.SkipOnExceptions)); + testCase = new RetryTestCase(messageSink, discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs); + discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs, + skipOnExceptions); } return new[] { testCase }; diff --git a/src/xRetry/RetryTestCase.cs b/src/xRetry/RetryTestCase.cs index ff0b91a..724601b 100644 --- a/src/xRetry/RetryTestCase.cs +++ b/src/xRetry/RetryTestCase.cs @@ -2,6 +2,7 @@ using System.ComponentModel; using System.Threading; using System.Threading.Tasks; +using xRetry.Exceptions; using Xunit.Abstractions; using Xunit.Sdk; @@ -12,6 +13,7 @@ public class RetryTestCase : XunitTestCase, IRetryableTestCase { public int MaxRetries { get; private set; } public int DelayBetweenRetriesMs { get; private set; } + public string[] SkipOnExceptionFullNames { get; private set; } [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete( @@ -25,12 +27,14 @@ public RetryTestCase( ITestMethod testMethod, int maxRetries, int delayBetweenRetriesMs, + Type[] skipOnExceptions, object[] testMethodArguments = null) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments) { MaxRetries = maxRetries; DelayBetweenRetriesMs = delayBetweenRetriesMs; + SkipOnExceptionFullNames = GetSkipOnExceptionFullNames(skipOnExceptions); } public override Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, @@ -47,6 +51,7 @@ public override void Serialize(IXunitSerializationInfo data) data.AddValue("MaxRetries", MaxRetries); data.AddValue("DelayBetweenRetriesMs", DelayBetweenRetriesMs); + data.AddValue("SkipOnExceptionFullNames", SkipOnExceptionFullNames); } public override void Deserialize(IXunitSerializationInfo data) @@ -55,6 +60,18 @@ public override void Deserialize(IXunitSerializationInfo data) MaxRetries = data.GetValue("MaxRetries"); DelayBetweenRetriesMs = data.GetValue("DelayBetweenRetriesMs"); + SkipOnExceptionFullNames = data.GetValue("SkipOnExceptionFullNames"); + } + + public static string[] GetSkipOnExceptionFullNames(Type[] customSkipOnExceptions) + { + string[] toRet = new string[customSkipOnExceptions.Length + 1]; + for (int i = 0; i < customSkipOnExceptions.Length; i++) + { + toRet[i] = customSkipOnExceptions[i].FullName; + } + toRet[toRet.Length - 1] = typeof(SkipTestException).FullName; + return toRet; } } } diff --git a/src/xRetry/RetryTestCaseRunner.cs b/src/xRetry/RetryTestCaseRunner.cs index fafef9d..f540fcf 100644 --- a/src/xRetry/RetryTestCaseRunner.cs +++ b/src/xRetry/RetryTestCaseRunner.cs @@ -28,7 +28,7 @@ public static async Task RunAsync( { // Prevent messages from the test run from being passed through, as we don't want // a message to mark the test as failed when we're going to retry it - using (BlockingMessageBus blockingMessageBus = new BlockingMessageBus(messageBus)) + using (BlockingMessageBus blockingMessageBus = new BlockingMessageBus(messageBus, testCase.SkipOnExceptionFullNames)) { diagnosticMessageSink.OnMessage(new DiagnosticMessage("Running test \"{0}\" attempt ({1}/{2})", testCase.DisplayName, i, testCase.MaxRetries)); diff --git a/src/xRetry/RetryTheoryAttribute.cs b/src/xRetry/RetryTheoryAttribute.cs index f33c27c..0b24c65 100644 --- a/src/xRetry/RetryTheoryAttribute.cs +++ b/src/xRetry/RetryTheoryAttribute.cs @@ -12,7 +12,14 @@ namespace xRetry public class RetryTheoryAttribute : RetryFactAttribute { /// - public RetryTheoryAttribute(int maxRetries = 3, int delayBetweenRetriesMs = 0) - : base(maxRetries, delayBetweenRetriesMs) { } + public RetryTheoryAttribute(params Type[] skipOnExceptions) + : base(skipOnExceptions) { } + + /// + public RetryTheoryAttribute( + int maxRetries = DEFAULT_MAX_RETRIES, + int delayBetweenRetriesMs = DEFAULT_DELAY_BETWEEN_RETRIES_MS, + params Type[] skipOnExceptions) + : base(maxRetries, delayBetweenRetriesMs, skipOnExceptions) { } } } diff --git a/src/xRetry/RetryTheoryDiscoverer.cs b/src/xRetry/RetryTheoryDiscoverer.cs index f0aa946..3b7e701 100644 --- a/src/xRetry/RetryTheoryDiscoverer.cs +++ b/src/xRetry/RetryTheoryDiscoverer.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using Xunit.Abstractions; using Xunit.Sdk; @@ -18,6 +19,8 @@ protected override IEnumerable CreateTestCasesForDataRow( int maxRetries = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.MaxRetries)); int delayBetweenRetriesMs = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.DelayBetweenRetriesMs)); + Type[] skipOnExceptions = + theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.SkipOnExceptions)); return new[] { new RetryTestCase( @@ -27,6 +30,7 @@ protected override IEnumerable CreateTestCasesForDataRow( testMethod, maxRetries, delayBetweenRetriesMs, + skipOnExceptions, dataRow) }; } @@ -37,11 +41,13 @@ protected override IEnumerable CreateTestCasesForTheory( int maxRetries = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.MaxRetries)); int delayBetweenRetriesMs = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.DelayBetweenRetriesMs)); + Type[] skipOnExceptions = + theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.SkipOnExceptions)); return new[] { new RetryTheoryDiscoveryAtRuntimeCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs) + discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs, skipOnExceptions) }; } } diff --git a/src/xRetry/RetryTheoryDiscoveryAtRuntimeCase.cs b/src/xRetry/RetryTheoryDiscoveryAtRuntimeCase.cs index ed01d29..80d6ca7 100644 --- a/src/xRetry/RetryTheoryDiscoveryAtRuntimeCase.cs +++ b/src/xRetry/RetryTheoryDiscoveryAtRuntimeCase.cs @@ -17,6 +17,7 @@ public class RetryTheoryDiscoveryAtRuntimeCase : XunitTestCase, IRetryableTestCa { public int MaxRetries { get; private set; } public int DelayBetweenRetriesMs { get; private set; } + public string[] SkipOnExceptionFullNames { get; private set; } /// [EditorBrowsable(EditorBrowsableState.Never)] @@ -29,11 +30,13 @@ public RetryTheoryDiscoveryAtRuntimeCase( TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, int maxRetries, - int delayBetweenRetriesMs) + int delayBetweenRetriesMs, + Type[] skipOnExceptions) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod) { MaxRetries = maxRetries; DelayBetweenRetriesMs = delayBetweenRetriesMs; + SkipOnExceptionFullNames = RetryTestCase.GetSkipOnExceptionFullNames(skipOnExceptions); } /// @@ -51,6 +54,7 @@ public override void Serialize(IXunitSerializationInfo data) data.AddValue("MaxRetries", MaxRetries); data.AddValue("DelayBetweenRetriesMs", DelayBetweenRetriesMs); + data.AddValue("SkipOnExceptionFullNames", SkipOnExceptionFullNames); } public override void Deserialize(IXunitSerializationInfo data) @@ -59,6 +63,7 @@ public override void Deserialize(IXunitSerializationInfo data) MaxRetries = data.GetValue("MaxRetries"); DelayBetweenRetriesMs = data.GetValue("DelayBetweenRetriesMs"); + SkipOnExceptionFullNames = data.GetValue("SkipOnExceptionFullNames"); } } } diff --git a/test/UnitTests/Facts/RetryFactRuntimeSkipTests.cs b/test/UnitTests/Facts/RetryFactRuntimeSkipTests.cs index c4e2ff6..1c052e7 100644 --- a/test/UnitTests/Facts/RetryFactRuntimeSkipTests.cs +++ b/test/UnitTests/Facts/RetryFactRuntimeSkipTests.cs @@ -1,3 +1,4 @@ +using UnitTests.TestClasses; using xRetry; using Xunit; using Skip = xRetry.Skip; @@ -16,6 +17,12 @@ public void SkipAtRuntime() Assert.True(false, "Should have been skipped . . ."); } + [RetryFact(typeof(TestException))] + public void CustomException_SkipsAtRuntime() + { + throw new TestException(); + } + private static int skippedNumCalls = 0; [RetryFact] diff --git a/test/UnitTests/RetryFactAttributeTests.cs b/test/UnitTests/RetryFactAttributeTests.cs new file mode 100644 index 0000000..0d59796 --- /dev/null +++ b/test/UnitTests/RetryFactAttributeTests.cs @@ -0,0 +1,60 @@ +using System; +using AutoFixture; +using FluentAssertions; +using xRetry; +using Xunit; + +namespace UnitTests +{ + public class RetryFactAttributeTests + { + [Fact] + public void Ctor_Empty_NoSkipOnExceptions() + { + // Arrange & Act + RetryFactAttribute attr = new RetryFactAttribute(); + + // Assert + attr.SkipOnExceptions.Should().BeEmpty(); + } + + [Fact] + public void SkipOnExceptionsCtor_Exceptions_ShouldSave() + { + // Arrange + Type[] expected = new[] + { + typeof(ArgumentException), + typeof(ArgumentNullException) + }; + + // Act + RetryFactAttribute attr = new RetryFactAttribute(expected); + + // Assert + attr.SkipOnExceptions.Should().BeEquivalentTo(expected); + } + + [Fact] + public void FullCtr_Exceptions_ShouldSave() + { + // Arrange + Fixture fixture = new Fixture(); + Type[] expected = new[] + { + typeof(ArgumentException), + typeof(ArgumentNullException) + }; + + // Act + RetryFactAttribute attr = new RetryFactAttribute(fixture.Create(), fixture.Create(), expected); + + // Assert + attr.SkipOnExceptions.Should().BeEquivalentTo(expected); + } + + [Fact] + public void Ctor_NonExceptionTypes_ShouldThrow() => + Assert.Throws(() => new RetryFactAttribute(typeof(RetryFactAttributeTests))); + } +} diff --git a/test/UnitTests/TestClasses/NonSerializableTestData.cs b/test/UnitTests/TestClasses/NonSerializableTestData.cs new file mode 100644 index 0000000..fc66036 --- /dev/null +++ b/test/UnitTests/TestClasses/NonSerializableTestData.cs @@ -0,0 +1,12 @@ +namespace UnitTests.TestClasses +{ + public class NonSerializableTestData + { + public readonly int Id; + + public NonSerializableTestData(int id) + { + Id = id; + } + } +} diff --git a/test/UnitTests/TestClasses/TestException.cs b/test/UnitTests/TestClasses/TestException.cs new file mode 100644 index 0000000..683f8d8 --- /dev/null +++ b/test/UnitTests/TestClasses/TestException.cs @@ -0,0 +1,6 @@ +using System; + +namespace UnitTests.TestClasses +{ + public class TestException : Exception { } +} diff --git a/test/UnitTests/Theories/RetryTheoryNonSerializableDataTests.cs b/test/UnitTests/Theories/RetryTheoryNonSerializableDataTests.cs index 727aaa1..4ff5b2b 100644 --- a/test/UnitTests/Theories/RetryTheoryNonSerializableDataTests.cs +++ b/test/UnitTests/Theories/RetryTheoryNonSerializableDataTests.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using UnitTests.TestClasses; using xRetry; using Xunit; @@ -32,15 +33,5 @@ public static IEnumerable GetTestData() => new[] new object[] { new NonSerializableTestData(0) }, new object[] { new NonSerializableTestData(1) } }; - - public class NonSerializableTestData - { - public readonly int Id; - - public NonSerializableTestData(int id) - { - Id = id; - } - } } } diff --git a/test/UnitTests/Theories/RetryTheoryRuntimeSkipNonSerializableDataTests.cs b/test/UnitTests/Theories/RetryTheoryRuntimeSkipNonSerializableDataTests.cs new file mode 100644 index 0000000..39717c4 --- /dev/null +++ b/test/UnitTests/Theories/RetryTheoryRuntimeSkipNonSerializableDataTests.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using UnitTests.TestClasses; +using xRetry; +using Xunit; +using Skip = xRetry.Skip; + +namespace UnitTests.Theories +{ + public class RetryTheoryRuntimeSkipNonSerializableDataTests + { + [RetryTheory] + [MemberData(nameof(GetTestData))] + public void SkipAtRuntime(NonSerializableTestData _) + { + // Note: All we're doing with this test is checking that the rest of the test doesn't get run + // checking it's skipped (and doesn't pass) would need to be done manually. + Skip.Always(); + + Assert.True(false, "Should have been skipped . . ."); + } + + [RetryTheory(typeof(TestException))] + [MemberData(nameof(GetTestData))] + public void CustomException_SkipsAtRuntime(NonSerializableTestData _) + { + throw new TestException(); + } + + // testId => numCalls + private static readonly Dictionary skippedNumCalls = new Dictionary() + { + { 0, 0 }, + { 1, 0 } + }; + + [RetryTheory] + [MemberData(nameof(GetTestData))] + public void Skip_DoesNotRetry(NonSerializableTestData nonSerializableWrapper) + { + // Assertion would fail on subsequent attempts, before reaching the skip + skippedNumCalls[nonSerializableWrapper.Id]++; + Assert.Equal(1, skippedNumCalls[nonSerializableWrapper.Id]); + + Skip.Always(); + } + + public static IEnumerable GetTestData() => new[] + { + new object[] { new NonSerializableTestData(0) }, + new object[] { new NonSerializableTestData(1) } + }; + } +} diff --git a/test/UnitTests/Theories/RetryTheoryRuntimeSkipTests.cs b/test/UnitTests/Theories/RetryTheoryRuntimeSkipTests.cs index aaf09d1..5913007 100644 --- a/test/UnitTests/Theories/RetryTheoryRuntimeSkipTests.cs +++ b/test/UnitTests/Theories/RetryTheoryRuntimeSkipTests.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using UnitTests.TestClasses; using xRetry; using Xunit; using Skip = xRetry.Skip; @@ -19,6 +20,14 @@ public void SkipAtRuntime(int _) Assert.True(false, "Should have been skipped . . ."); } + [RetryTheory(typeof(TestException))] + [InlineData(0)] + [InlineData(1)] + public void CustomException_SkipsAtRuntime(int _) + { + throw new TestException(); + } + // testId => numCalls private static readonly Dictionary skippedNumCalls = new Dictionary() { From 024c1f32339d6ff58f702d2ac6276f98c835eb7c Mon Sep 17 00:00:00 2001 From: Josh Keegan Date: Sat, 30 Oct 2021 16:35:07 +0100 Subject: [PATCH 07/10] refactor: extract skip check from the blocking message bus --- src/xRetry/BlockingMessageBus.cs | 26 ++++++--------------- src/xRetry/MessageTransformer.cs | 39 +++++++++++++++++++++++++++++++ src/xRetry/RetryTestCaseRunner.cs | 5 ++-- 3 files changed, 49 insertions(+), 21 deletions(-) create mode 100644 src/xRetry/MessageTransformer.cs diff --git a/src/xRetry/BlockingMessageBus.cs b/src/xRetry/BlockingMessageBus.cs index fc87277..1b7e7d6 100644 --- a/src/xRetry/BlockingMessageBus.cs +++ b/src/xRetry/BlockingMessageBus.cs @@ -1,6 +1,4 @@ using System.Collections.Concurrent; -using System.Linq; -using xRetry.Extensions; using Xunit.Abstractions; using Xunit.Sdk; @@ -12,30 +10,20 @@ namespace xRetry public class BlockingMessageBus : IMessageBus { private readonly IMessageBus underlyingMessageBus; - private readonly string[] skipOnExceptionFullNames; + private readonly MessageTransformer messageTransformer; private ConcurrentQueue messageQueue = new ConcurrentQueue(); - public bool Skipped { get; private set; } = false; - public BlockingMessageBus(IMessageBus underlyingMessageBus, string[] skipOnExceptionFullNames) + public BlockingMessageBus(IMessageBus underlyingMessageBus, MessageTransformer messageTransformer) { this.underlyingMessageBus = underlyingMessageBus; - this.skipOnExceptionFullNames = skipOnExceptionFullNames; + this.messageTransformer = messageTransformer; } - public bool QueueMessage(IMessageSinkMessage message) + public bool QueueMessage(IMessageSinkMessage rawMessage) { - // If this is a message saying that the test has been skipped, replace the message with skipping the test - if (message is TestFailed failed && failed.ExceptionTypes.ContainsAny(skipOnExceptionFullNames)) - { - string reason = failed.Messages?.FirstOrDefault(); - messageQueue.Enqueue(new TestSkipped(failed.Test, reason)); - Skipped = true; - } - else - { - // Otherwise this isn't a message saying the test is skipped, follow usual intercept for replay later behaviour - messageQueue.Enqueue(message); - } + // Transform the message to apply any additional functionality, then intercept & store it for replay later + IMessageSinkMessage transformedMessage = messageTransformer.Transform(rawMessage); + messageQueue.Enqueue(transformedMessage); // Returns if execution should continue. Since we are intercepting the message, we // have no way of checking this so always continue... diff --git a/src/xRetry/MessageTransformer.cs b/src/xRetry/MessageTransformer.cs new file mode 100644 index 0000000..703b0cf --- /dev/null +++ b/src/xRetry/MessageTransformer.cs @@ -0,0 +1,39 @@ +using System.Linq; +using xRetry.Extensions; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace xRetry +{ + public class MessageTransformer + { + private readonly string[] skipOnExceptionFullNames; + + public bool Skipped { get; private set; } + + public MessageTransformer(string[] skipOnExceptionFullNames) + { + this.skipOnExceptionFullNames = skipOnExceptionFullNames; + } + + /// + /// Transforms a message received from an xUnit test into another message, replacing it + /// where necessary to add additional functionality, e.g. dynamic skipping + /// + /// + /// + public IMessageSinkMessage Transform(IMessageSinkMessage message) + { + // If this is a message saying that the test has been skipped, replace the message with skipping the test + if (message is TestFailed failed && failed.ExceptionTypes.ContainsAny(skipOnExceptionFullNames)) + { + string reason = failed.Messages?.FirstOrDefault(); + Skipped = true; + return new TestSkipped(failed.Test, reason); + } + + // Otherwise this isn't a message saying the test is skipped, follow usual intercept for replay later behaviour + return message; + } + } +} diff --git a/src/xRetry/RetryTestCaseRunner.cs b/src/xRetry/RetryTestCaseRunner.cs index f540fcf..759e027 100644 --- a/src/xRetry/RetryTestCaseRunner.cs +++ b/src/xRetry/RetryTestCaseRunner.cs @@ -28,14 +28,15 @@ public static async Task RunAsync( { // Prevent messages from the test run from being passed through, as we don't want // a message to mark the test as failed when we're going to retry it - using (BlockingMessageBus blockingMessageBus = new BlockingMessageBus(messageBus, testCase.SkipOnExceptionFullNames)) + MessageTransformer messageTransformer = new MessageTransformer(testCase.SkipOnExceptionFullNames); + using (BlockingMessageBus blockingMessageBus = new BlockingMessageBus(messageBus, messageTransformer)) { diagnosticMessageSink.OnMessage(new DiagnosticMessage("Running test \"{0}\" attempt ({1}/{2})", testCase.DisplayName, i, testCase.MaxRetries)); RunSummary summary = await fnRunSingle(blockingMessageBus); - if (blockingMessageBus.Skipped) + if (messageTransformer.Skipped) { summary.Failed = 0; summary.Skipped = 1; From 26e14fd568137bae68964bc1fe35f95959581f73 Mon Sep 17 00:00:00 2001 From: Josh Keegan Date: Sat, 30 Oct 2021 16:50:19 +0100 Subject: [PATCH 08/10] docs for new skip functionality --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index c7ed51f..96ee4d7 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,22 @@ public void Default_Reaches3(int id) ``` The same optional arguments (max retries and delay between each retry) are supported as for facts, and can be used in the same way. +### Skipping tests at Runtime +In addition to retries, `RetryFact` and `RetryTheory` both support dynamically skipping tests at runtime. To make a test skip just use `Skip.Always()` +within your test code. +It also supports custom exception types so you can skip a test if a type of exception gets thrown. You do this by specifying the exception type to the +attribute above your test, e.g. +```cs +[RetryFact(typeof(TestException))] +public void CustomException_SkipsAtRuntime() +{ + throw new TestException(); +} +``` +This functionality also allows for skipping to work when you are already using another library for dynamically skipping tests by specifying the exception +type that is used by that library to the `RetryFact`. e.g. if you are using the popular Xunit.SkippableFact nuget package and want to add retries, converting the +test is as simple as replacing `[SkippableFact]` with `[RetryFact(typeof(Xunit.SkipException))]` above the test and you don't need to change the test itself. + ## Viewing retry logs By default, you won't see whether your tests are being retried as we make this information available via the xunit diagnostic logs but test runners will hide these detailed logs by default. From 3e86ddcccabdf957e2584a375f23e88e6367c88e Mon Sep 17 00:00:00 2001 From: Josh Keegan Date: Sat, 30 Oct 2021 16:50:33 +0100 Subject: [PATCH 09/10] release prep 1.7.0: version bump --- build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Makefile b/build/Makefile index c81134c..736603a 100644 --- a/build/Makefile +++ b/build/Makefile @@ -1,4 +1,4 @@ -VERSION=1.6.0# +VERSION=1.7.0# clean: rm -r ../artefacts || true From f4719c5d3c328a370bbe3134c2c2fc389864b6e5 Mon Sep 17 00:00:00 2001 From: Josh Keegan Date: Sat, 30 Oct 2021 17:15:59 +0100 Subject: [PATCH 10/10] remove TODO: Don't need xunit specific serialisation for an exception --- src/xRetry/Exceptions/SkipTestException.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/xRetry/Exceptions/SkipTestException.cs b/src/xRetry/Exceptions/SkipTestException.cs index d0fe720..f923fda 100644 --- a/src/xRetry/Exceptions/SkipTestException.cs +++ b/src/xRetry/Exceptions/SkipTestException.cs @@ -3,8 +3,6 @@ namespace xRetry.Exceptions { - // TODO: Is serialisation implementation good enough? - // Maybe json etc... would fail. See RetryTestCase for example [Serializable] public class SkipTestException : Exception {