Merge pull request #115 from JoshKeegan/retryFeatures

New feature: ability to retry features

README.md

@@ -19,22 +19,38 @@ this library should **not** be used to cover it up!
 ## Usage: SpecFlow 3
 Add the `xRetry.SpecFlow` nuget package to your project.  
 
+### Scenarios (and outlines)
 Above any scenario or scenario outline that should be retried, add a `@retry` tag, e.g:
 ```gherkin
 @retry
 Scenario: Retry three times by default
 	When I increment the default retry count
 	Then the default result should be 3
 ```
-This will retry the test up to 3 times by default. You can optionally specify a number of times 
-to retry the test in brackets, e.g. `@retry(5)`.  
+This will attempt to run the test until it passes, up to 3 times by default. 
+You can optionally specify a number of times to attempt to run the test in brackets, e.g. `@retry(5)`.  
 
 You can also optionally specify a delay between each retry (in milliseconds) as a second 
-parameter, e.g. `@retry(5,100)` will run your test 5 times until it passes, waiting 100ms
-between each attempt.  
+parameter, e.g. `@retry(5,100)` will run your test up to 5 times, waiting 100ms between each attempt.  
 Note that you must not include a space between the parameters, as Cucumber/SpecFlow uses
 a space to separate tags, i.e. `@retry(5, 100)` would not work due to the space after the comma.
 
+
+### Features
+You can also make every test in a feature retryable by adding the `@retry` tag to the feature, e.g:
+```gherkin
+@retry
+Feature: Retryable Feature
+
+Scenario: Retry scenario three times by default
+	When I increment the retry count
+	Then the result should be 3
+```
+All options that can be used against an individual scenario can also be applied like this at the feature level.  
+If a `@retry` tag exists on both the feature and a scenario within that feature, the tag on the scenario will take
+precedent over the one on the feature. This is useful if you wanted all scenarios in a feature to be retried 
+by default but for some cases also wanted to wait some time before each retry attempt. You can also use this to prevent a specific scenario not be retried, even though it is within a feature with a `@retry` tag, by adding `@retry(1)` to the scenario.
+
 ## Usage: xUnit
 Add the `xRetry` nuget package to your project.
 
@@ -53,12 +69,11 @@ public void Default_Reaches3()
 }
 
 ```
-This will retry the test up to 3 times by default. You can optionally specify a number of times
-to retry the test as an argument, e.g. `[RetryFact(5)]`.  
+This will attempt to run the test until it passes, up to 3 times by default. 
+You can optionally specify a number of times to attempt to run the test as an argument, e.g. `[RetryFact(5)]`.  
 
 You can also optionally specify a delay between each retry (in milliseconds) as a second 
-parameter, e.g. `[RetryFact(5, 100)]` will run your test 5 times until it passes, waiting 100ms
-between each attempt.
+parameter, e.g. `[RetryFact(5, 100)]` will run your test up to 5 times, waiting 100ms between each attempt.
 
 
 ### Theories
@@ -82,7 +97,7 @@ public void Default_Reaches3(int id)
     Assert.Equal(3, defaultNumCalls[id]);
 }
 ```
-The same optional arguments (max retries and delay between each retry) are supported as for facts, and can be used in the same way.
+The same optional arguments (max attempts and delay between each retry) are supported as for facts, and can be used in the same way.
 
 ### Skipping tests at Runtime
 In addition to retries, `RetryFact` and `RetryTheory` both support dynamically skipping tests at runtime. To make a test skip just use `Skip.Always()`
@@ -111,12 +126,12 @@ this projects own unit tests which has been set up with this enabled.
 Feel free to open a pull request! If you want to start any sizeable chunk of work, consider 
 opening an issue first to discuss, and make sure nobody else is working on the same problem.  
 
-### Running locally
-To run locally, always build `xRetry.SpecFlowPlugin` before the tests, to ensure MSBuild
-uses the latest version of your changes.  
+### Developing locally
+#### In an IDE
+To build and run locally, always build `xRetry.SpecFlowPlugin` with the Release profile before the tests to ensure MSBuild uses the latest version of your changes when building the UnitTests project.  
 
-If you install `make` and go to the `build` directory, you can run the following from the 
-terminal to build, run tests and create the nuget packages:
+### From the terminal
+If you install `make` and go to the `build` directory, you can run the following to build, run tests and create the nuget packages:
 ```bash
 make ci
 ```

build/Makefile

@@ -1,4 +1,4 @@
-VERSION=1.7.0#
+VERSION=1.8.0#
 
 clean:
 	rm -r ../artefacts || true

src/xRetry.SpecFlow/TestGeneratorProvider.cs

@@ -13,6 +13,8 @@ namespace xRetry.SpecFlow
     public class TestGeneratorProvider : XUnit2TestGeneratorProvider
     {
         private const string IGNORE_TAG = "ignore";
+        private const string RETRY_FACT_ATTRIBUTE = "xRetry.RetryFact";
+        private const string RETRY_THEORY_ATTRIBUTE = "xRetry.RetryTheory";
 
         private readonly IRetryTagParser retryTagParser;
 
@@ -22,6 +24,29 @@ public TestGeneratorProvider(CodeDomHelper codeDomHelper, ProjectSettings projec
             this.retryTagParser = retryTagParser;
         }
 
+        // Called for scenario outlines, even when it has no tags.
+        // We don't yet have access to tags against the scenario at this point, but can handle feature tags now.
+        public override void SetRowTest(TestClassGenerationContext generationContext, CodeMemberMethod testMethod, string scenarioTitle)
+        {
+            base.SetRowTest(generationContext, testMethod, scenarioTitle);
+
+            string[] featureTags = generationContext.Feature.Tags.Select(t => stripLeadingAtSign(t.Name)).ToArray();
+
+            applyRetry(featureTags, Enumerable.Empty<string>(), testMethod);
+        }
+
+        // Called for scenarios, even when it has no tags.
+        // We don't yet have access to tags against the scenario at this point, but can handle feature tags now.
+        public override void SetTestMethod(TestClassGenerationContext generationContext, CodeMemberMethod testMethod, string friendlyTestName)
+        {
+            base.SetTestMethod(generationContext, testMethod, friendlyTestName);
+
+            string[] featureTags = generationContext.Feature.Tags.Select(t => stripLeadingAtSign(t.Name)).ToArray();
+
+            applyRetry(featureTags, Enumerable.Empty<string>(), testMethod);
+        }
+
+        // Called for both scenarios & scenario outlines, but only if it has tags
         public override void SetTestMethodCategories(TestClassGenerationContext generationContext,
             CodeMemberMethod testMethod, IEnumerable<string> scenarioCategories)
         {
@@ -30,15 +55,28 @@ public override void SetTestMethodCategories(TestClassGenerationContext generati
 
             base.SetTestMethodCategories(generationContext, testMethod, scenarioCategories);
 
+            // Feature tags will have already been processed in one of the methods above, which are executed before this
+            IEnumerable<string> featureTags = generationContext.Feature.Tags.Select(t => stripLeadingAtSign(t.Name));
+            applyRetry((string[]) scenarioCategories, featureTags, testMethod);
+        }
+
+        /// <summary>
+        /// Apply retry tags to the current test
+        /// </summary>
+        /// <param name="tags">Tags that haven't yet been processed. If the test has just been created these will be for the feature, otherwise for the scenario</param>
+        /// <param name="processedTags">Tags that have already been processed. If the test has just been created this will be empty, otherwise they will be the feature tags</param>
+        /// <param name="testMethod">Test method we are applying retries for</param>
+        private void applyRetry(IList<string> tags, IEnumerable<string> processedTags, CodeMemberMethod testMethod)
+        {
             // Do not add retries to skipped tests (even if they have the retry attribute) as retrying won't affect the outcome.
             //  This allows for the new (for SpecFlow 3.1.x) implementation that relies on Xunit.SkippableFact to still work, as it
             //  too will replace the attribute for running the test with a custom one.
-            if (isIgnored(generationContext, scenarioCategories))
+            if (tags.Any(isIgnoreTag) || processedTags.Any(isIgnoreTag))
             {
                 return;
             }
 
-            string strRetryTag = getRetryTag(scenarioCategories);
+            string strRetryTag = getRetryTag(tags);
             if (strRetryTag == null)
             {
                 return;
@@ -48,7 +86,11 @@ public override void SetTestMethodCategories(TestClassGenerationContext generati
 
             // Remove the original fact or theory attribute
             CodeAttributeDeclaration originalAttribute = testMethod.CustomAttributes.OfType<CodeAttributeDeclaration>()
-                .FirstOrDefault(a => a.Name == FACT_ATTRIBUTE || a.Name == THEORY_ATTRIBUTE);
+                .FirstOrDefault(a =>
+                    a.Name == FACT_ATTRIBUTE || 
+                    a.Name == THEORY_ATTRIBUTE || 
+                    a.Name == RETRY_FACT_ATTRIBUTE ||
+                    a.Name == RETRY_THEORY_ATTRIBUTE);
             if (originalAttribute == null)
             {
                 return;
@@ -57,7 +99,9 @@ public override void SetTestMethodCategories(TestClassGenerationContext generati
 
             // Add the Retry attribute
             CodeAttributeDeclaration retryAttribute = CodeDomHelper.AddAttribute(testMethod,
-                "xRetry.Retry" + (originalAttribute.Name == FACT_ATTRIBUTE ? "Fact" : "Theory"));
+                originalAttribute.Name == FACT_ATTRIBUTE || originalAttribute.Name == RETRY_FACT_ATTRIBUTE
+                    ? RETRY_FACT_ATTRIBUTE
+                    : RETRY_THEORY_ATTRIBUTE);
 
             retryAttribute.Arguments.Add(new CodeAttributeArgument(
                 new CodePrimitiveExpression(retryTag.MaxRetries ?? RetryFactAttribute.DEFAULT_MAX_RETRIES)));
@@ -75,17 +119,18 @@ public override void SetTestMethodCategories(TestClassGenerationContext generati
                         new CodeTypeOfExpression(typeof(Xunit.SkipException))
                     })));
 
-            // Copy arguments from the original attribute
-            for (int i = 0; i < originalAttribute.Arguments.Count; i++)
+            // Copy arguments from the original attribute. If it's already a retry attribute, don't copy the retry arguments though
+            for (int i = originalAttribute.Name == RETRY_FACT_ATTRIBUTE ||
+                         originalAttribute.Name == RETRY_THEORY_ATTRIBUTE
+                     ? retryAttribute.Arguments.Count
+                     : 0;
+                 i < originalAttribute.Arguments.Count;
+                 i++)
             {
                 retryAttribute.Arguments.Add(originalAttribute.Arguments[i]);
             }
         }
 
-        private static bool isIgnored(TestClassGenerationContext generationContext, IEnumerable<string> tags) =>
-            generationContext.Feature.Tags.Select(t => stripLeadingAtSign(t.Name)).Any(isIgnoreTag) ||
-            tags.Any(isIgnoreTag);
-
         private static string stripLeadingAtSign(string s) => s.StartsWith("@") ? s.Substring(1) : s;
 
         private static bool isIgnoreTag(string tag) => tag.Equals(IGNORE_TAG, StringComparison.OrdinalIgnoreCase);

src/xRetry.SpecFlow/xRetry.SpecFlow.nuspec

@@ -13,7 +13,7 @@
     <tags>SpecFlow XUnit Retry Test-Automation</tags>
 
     <dependencies>
-      <dependency id="SpecFlow.xUnit" version="[3.9.8,3.10.0)" />
+      <dependency id="SpecFlow.xUnit" version="[3.9.50,3.10.0)" />
       <dependency id="xRetry" version="[$version$]" />
     </dependencies>
   </metadata>

src/xRetry/RetryFactAttribute.cs

@@ -7,7 +7,7 @@ namespace xRetry
 {
     /// <summary>
     /// Attribute that is applied to a method to indicate that it is a fact that should be run
-    /// by the test runner up to MaxRetries times, until it succeeds.
+    /// by the test runner up to <see cref="MaxRetries"/> times, until it succeeds.
     /// </summary>
     [XunitTestCaseDiscoverer("xRetry.RetryFactDiscoverer", "xRetry")]
     [AttributeUsage(AttributeTargets.Method)]
@@ -37,7 +37,7 @@ public RetryFactAttribute(params Type[] skipOnExceptions)
         /// <summary>
         /// Ctor (full)
         /// </summary>
-        /// <param name="maxRetries">The number of times to run a test for until it succeeds</param>
+        /// <param name="maxRetries">The number of times to attempt to run a test for until it succeeds</param>
         /// <param name="delayBetweenRetriesMs">The amount of time (in ms) to wait between each test run attempt</param>
         /// <param name="skipOnExceptions">Mark the test as skipped when this type of exception is encountered</param>
         public RetryFactAttribute(

src/xRetry/RetryTheoryAttribute.cs

@@ -5,7 +5,7 @@ namespace xRetry
 {
     /// <summary>
     /// Attribute that is applied to a method to indicate that it is a theory that should be run
-    /// by the test runner up to MaxRetries times, until it succeeds.
+    /// by the test runner up to <see cref="RetryFactAttribute.MaxRetries"/> times, until it succeeds.
     /// </summary>
     [XunitTestCaseDiscoverer("xRetry.RetryTheoryDiscoverer", "xRetry")]
     [AttributeUsage(AttributeTargets.Method)]

test/UnitTests/RetryFactAttributeTests.cs

@@ -56,5 +56,18 @@ public void FullCtr_Exceptions_ShouldSave()
         [Fact]
         public void Ctor_NonExceptionTypes_ShouldThrow() =>
             Assert.Throws<ArgumentException>(() => new RetryFactAttribute(typeof(RetryFactAttributeTests)));
+
+        [Theory]
+        [InlineData(0)]
+        [InlineData(-1)]
+        [InlineData(-1337)]
+        public void Ctor_LessThanOneMaxRetries_ShouldThrow(int maxRetries) =>
+            Assert.Throws<ArgumentOutOfRangeException>(() => new RetryFactAttribute(maxRetries));
+
+        [Theory]
+        [InlineData(-1)]
+        [InlineData(-1337)]
+        public void Ctor_NegativeDelayBetweenRetries_ShouldThrow(int delayBetweenRetriesMs) =>
+            Assert.Throws<ArgumentOutOfRangeException>(() => new RetryFactAttribute(delayBetweenRetriesMs: delayBetweenRetriesMs));
     }
 }

test/UnitTests/RetryTheoryAttributeTests.cs

@@ -0,0 +1,73 @@
+using System;
+using AutoFixture;
+using FluentAssertions;
+using xRetry;
+using Xunit;
+
+namespace UnitTests
+{
+    public class RetryTheoryAttributeTests
+    {
+        [Fact]
+        public void Ctor_Empty_NoSkipOnExceptions()
+        {
+            // Arrange & Act
+            RetryTheoryAttribute attr = new RetryTheoryAttribute();
+
+            // Assert
+            attr.SkipOnExceptions.Should().BeEmpty();
+        }
+
+        [Fact]
+        public void SkipOnExceptionsCtor_Exceptions_ShouldSave()
+        {
+            // Arrange
+            Type[] expected = new[]
+            {
+                typeof(ArgumentException),
+                typeof(ArgumentNullException)
+            };
+
+            // Act
+            RetryTheoryAttribute attr = new RetryTheoryAttribute(expected);
+
+            // Assert
+            attr.SkipOnExceptions.Should().BeEquivalentTo(expected);
+        }
+
+        [Fact]
+        public void FullCtr_Exceptions_ShouldSave()
+        {
+            // Arrange
+            Fixture fixture = new Fixture();
+            Type[] expected = new[]
+            {
+                typeof(ArgumentException),
+                typeof(ArgumentNullException)
+            };
+
+            // Act
+            RetryTheoryAttribute attr = new RetryTheoryAttribute(fixture.Create<int>(), fixture.Create<int>(), expected);
+
+            // Assert
+            attr.SkipOnExceptions.Should().BeEquivalentTo(expected);
+        }
+
+        [Fact]
+        public void Ctor_NonExceptionTypes_ShouldThrow() =>
+            Assert.Throws<ArgumentException>(() => new RetryTheoryAttribute(typeof(RetryFactAttributeTests)));
+
+        [Theory]
+        [InlineData(0)]
+        [InlineData(-1)]
+        [InlineData(-1337)]
+        public void Ctor_LessThanOneMaxRetries_ShouldThrow(int maxRetries) =>
+            Assert.Throws<ArgumentOutOfRangeException>(() => new RetryTheoryAttribute(maxRetries));
+
+        [Theory]
+        [InlineData(-1)]
+        [InlineData(-1337)]
+        public void Ctor_NegativeDelayBetweenRetries_ShouldThrow(int delayBetweenRetriesMs) =>
+            Assert.Throws<ArgumentOutOfRangeException>(() => new RetryTheoryAttribute(delayBetweenRetriesMs: delayBetweenRetriesMs));
+    }
+}

test/UnitTests/SpecFlow/Features/IgnoredFeature.feature

@@ -1,8 +1,8 @@
 @ignore
 Feature: Ignored Feature
-	In order to temorarily disable retriable features
+	In order to temorarily disable features containing retryable scenarios
 	As a QA engineer
-	I want to be able to ignore entire features that contain retriable tests
+	I want to be able to ignore entire features that contain retryable tests
 
 Scenario: Test is ignored
 	Then fail because this test should have been skipped

test/UnitTests/SpecFlow/Features/RetryFeature/IgnoredRetryFeature.feature

@@ -0,0 +1,27 @@
+@retry @ignore
+Feature: Ignored Retryable Feature
+	In order to temorarily disable retryable features
+	As a QA engineer
+	I want to be able to ignore entire features marked for retries
+
+Scenario: Test is ignored
+	Then fail because this test should have been skipped
+
+@retry
+Scenario: Explicit retry test is ignored
+	Then fail because this test should have been skipped
+
+Scenario Outline: Scenario outline test is ignored
+	Then fail because this test should have been skipped
+	Examples:
+	| n |
+	| 1 |
+	| 2 |
+
+@retry
+Scenario Outline: Explicit retry scenario outline test is ignored
+	Then fail because this test should have been skipped
+	Examples:
+	| n |
+	| 1 |
+	| 2 |