From 911d100f3f72be403b72e878bbb2ee0129a0ca9e Mon Sep 17 00:00:00 2001 From: Marcel <14852157+Marcel0024@users.noreply.github.com> Date: Thu, 11 Jul 2024 18:54:04 +0200 Subject: [PATCH] Restructured tests --- .../CocoCrawler.IntegrationTests.csproj | 4 +- .../{CookiesTest.cs => CookiesTests.cs} | 7 +--- ...ns.cs => ThrowOnBuilderExceptionsTests.cs} | 2 +- .../ConfigureEngine/UserAgentTests.cs | 5 +-- .../{VisitedLinks.cs => VisitedLinksTests.cs} | 2 +- .../{CsvOutput.cs => CsvOutputTests.cs} | 16 ++----- .../ExtractListAndPaginateTests.cs | 15 ++----- .../Responses/main-page.html | 0 .../Responses/page-2.html | 0 ...OpenLinksExtractObjectAndPaginateTests.cs} | 32 +++++++------- .../WireMockExtensions.cs | 42 +++++++++++++++++++ 11 files changed, 70 insertions(+), 55 deletions(-) rename Tests/CocoCrawler.IntegrationTests/ConfigureEngine/{CookiesTest.cs => CookiesTests.cs} (83%) rename Tests/CocoCrawler.IntegrationTests/ConfigureEngine/{ThrowOnBuilderExceptions.cs => ThrowOnBuilderExceptionsTests.cs} (98%) rename Tests/CocoCrawler.IntegrationTests/ConfigureEngine/{VisitedLinks.cs => VisitedLinksTests.cs} (98%) rename Tests/CocoCrawler.IntegrationTests/Outputs/{CsvOutput.cs => CsvOutputTests.cs} (82%) rename Tests/CocoCrawler.IntegrationTests/{ => Scenarios}/ExtractListAndPaginate/ExtractListAndPaginateTests.cs (70%) rename Tests/CocoCrawler.IntegrationTests/{ => Scenarios}/ExtractListAndPaginate/Responses/main-page.html (100%) rename Tests/CocoCrawler.IntegrationTests/{ => Scenarios}/ExtractListAndPaginate/Responses/page-2.html (100%) rename Tests/CocoCrawler.IntegrationTests/{OpenLinksExtractObjectAndPaginate/OpenLinksExtractObjectAndPaginate.cs => Scenarios/OpenLinksExtractObjectAndPaginate/OpenLinksExtractObjectAndPaginateTests.cs} (81%) create mode 100644 Tests/CocoCrawler.IntegrationTests/WireMockExtensions.cs diff --git a/Tests/CocoCrawler.IntegrationTests/CocoCrawler.IntegrationTests.csproj b/Tests/CocoCrawler.IntegrationTests/CocoCrawler.IntegrationTests.csproj index cbf362b..cd5ca28 100644 --- a/Tests/CocoCrawler.IntegrationTests/CocoCrawler.IntegrationTests.csproj +++ b/Tests/CocoCrawler.IntegrationTests/CocoCrawler.IntegrationTests.csproj @@ -33,10 +33,10 @@ - + Always - + Always diff --git a/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/CookiesTest.cs b/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/CookiesTests.cs similarity index 83% rename from Tests/CocoCrawler.IntegrationTests/ConfigureEngine/CookiesTest.cs rename to Tests/CocoCrawler.IntegrationTests/ConfigureEngine/CookiesTests.cs index 5242a4b..ba8f474 100644 --- a/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/CookiesTest.cs +++ b/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/CookiesTests.cs @@ -1,14 +1,12 @@ using CocoCrawler.Builders; using CocoCrawler.Scheduler; using FluentAssertions; -using WireMock.RequestBuilders; -using WireMock.ResponseBuilders; using WireMock.Server; namespace CocoCrawler.IntegrationTests.Engine; [Collection(nameof(BrowserCollection))] -public class CookiesTest +public class CookiesTests { private readonly WireMockServer _wireMockServer = WireMockServer.Start(); @@ -16,8 +14,7 @@ public class CookiesTest public async Task Cookies_Should_Be_Send_To_The_Client() { // Arange - _wireMockServer.Given(Request.Create().WithUrl($"{_wireMockServer.Url}/cookies")) - .RespondWith(Response.Create().WithSuccess()); + _wireMockServer.ReturnSuccessFor($"{_wireMockServer.Url}/cookies"); var crawlerEngine = await new CrawlerEngineBuilder() .AddPage($"{_wireMockServer.Url}/cookies", options => options.ExtractObject([new("No exist", "div.test")])) diff --git a/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/ThrowOnBuilderExceptions.cs b/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/ThrowOnBuilderExceptionsTests.cs similarity index 98% rename from Tests/CocoCrawler.IntegrationTests/ConfigureEngine/ThrowOnBuilderExceptions.cs rename to Tests/CocoCrawler.IntegrationTests/ConfigureEngine/ThrowOnBuilderExceptionsTests.cs index 950e156..16916c1 100644 --- a/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/ThrowOnBuilderExceptions.cs +++ b/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/ThrowOnBuilderExceptionsTests.cs @@ -6,7 +6,7 @@ namespace CocoCrawler.IntegrationTests.Engine; [Collection(nameof(BrowserCollection))] -public class ThrowOnBuilderExceptions +public class ThrowOnBuilderExceptionsTests { private readonly WireMockServer _wireMockServer = WireMockServer.Start(); diff --git a/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/UserAgentTests.cs b/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/UserAgentTests.cs index 70686df..cb4a9fc 100644 --- a/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/UserAgentTests.cs +++ b/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/UserAgentTests.cs @@ -1,8 +1,6 @@ using CocoCrawler.Builders; using CocoCrawler.Scheduler; using FluentAssertions; -using WireMock.RequestBuilders; -using WireMock.ResponseBuilders; using WireMock.Server; namespace CocoCrawler.IntegrationTests.Engine; @@ -16,8 +14,7 @@ public class UserAgentTests public async Task UserAgent_Should_Be_Overwritten() { // Arange - _wireMockServer.Given(Request.Create().WithUrl($"{_wireMockServer.Url}/useragent")) - .RespondWith(Response.Create().WithSuccess()); + _wireMockServer.ReturnSuccessFor($"{_wireMockServer.Url}/useragent"); var crawlerEngine = await new CrawlerEngineBuilder() .AddPage($"{_wireMockServer.Url}/useragent", options => options.ExtractObject([new("No exist", "div.test")])) diff --git a/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/VisitedLinks.cs b/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/VisitedLinksTests.cs similarity index 98% rename from Tests/CocoCrawler.IntegrationTests/ConfigureEngine/VisitedLinks.cs rename to Tests/CocoCrawler.IntegrationTests/ConfigureEngine/VisitedLinksTests.cs index 82cb347..d8992b3 100644 --- a/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/VisitedLinks.cs +++ b/Tests/CocoCrawler.IntegrationTests/ConfigureEngine/VisitedLinksTests.cs @@ -7,7 +7,7 @@ namespace CocoCrawler.IntegrationTests.ConfigureEngine; [Collection(nameof(BrowserCollection))] -public class VisitedLinks +public class VisitedLinksTests { private readonly WireMockServer _wireMockServer = WireMockServer.Start(); diff --git a/Tests/CocoCrawler.IntegrationTests/Outputs/CsvOutput.cs b/Tests/CocoCrawler.IntegrationTests/Outputs/CsvOutputTests.cs similarity index 82% rename from Tests/CocoCrawler.IntegrationTests/Outputs/CsvOutput.cs rename to Tests/CocoCrawler.IntegrationTests/Outputs/CsvOutputTests.cs index 93f4238..76fdd7c 100644 --- a/Tests/CocoCrawler.IntegrationTests/Outputs/CsvOutput.cs +++ b/Tests/CocoCrawler.IntegrationTests/Outputs/CsvOutputTests.cs @@ -1,25 +1,20 @@ using CocoCrawler.Builders; using CocoCrawler.Scheduler; using FluentAssertions; -using WireMock.RequestBuilders; -using WireMock.ResponseBuilders; using WireMock.Server; namespace CocoCrawler.IntegrationTests.Outputs; [Collection(nameof(BrowserCollection))] -public class CsvOutput +public class CsvOutputTests { private readonly WireMockServer _wireMockServer = WireMockServer.Start(); [Fact] - public async Task CsvOutput_ShouldCreateFile_OnCleanOnStartup() + public async Task CsvOutput_ShouldCreateFile_WhenCleanOnStartup() { // Arange - _wireMockServer.Given(Request.Create().WithUrl($"{_wireMockServer.Url}/main-page")) - .RespondWith(Response.Create() - .WithHeader("Content-Type", "text/xml; charset=utf-8") - .WithBody(GetPage())); + _wireMockServer.ReturnSuccessWithPage($"{_wireMockServer.Url}/main-page", GetPage()); var outputPath = Path.Combine("Outputs", "Really", "Deep", "Path", "resultstest1.csv"); @@ -53,10 +48,7 @@ public async Task CsvOutput_ShouldCreateFile_OnCleanOnStartup() public async Task CsvOutput_ShouldNotCreateFile_OnCleanOnStartupFalse() { // Arange - _wireMockServer.Given(Request.Create().WithUrl($"{_wireMockServer.Url}/main-page")) - .RespondWith(Response.Create() - .WithHeader("Content-Type", "text/xml; charset=utf-8") - .WithBody(GetPage())); + _wireMockServer.ReturnSuccessWithPage($"{_wireMockServer.Url}/main-page", GetPage()); var outputPath = Path.Combine("Outputs", "Really", "Deep", "Path", $"results-dont-clean-{Random.Shared.Next(0,100)}.csv"); diff --git a/Tests/CocoCrawler.IntegrationTests/ExtractListAndPaginate/ExtractListAndPaginateTests.cs b/Tests/CocoCrawler.IntegrationTests/Scenarios/ExtractListAndPaginate/ExtractListAndPaginateTests.cs similarity index 70% rename from Tests/CocoCrawler.IntegrationTests/ExtractListAndPaginate/ExtractListAndPaginateTests.cs rename to Tests/CocoCrawler.IntegrationTests/Scenarios/ExtractListAndPaginate/ExtractListAndPaginateTests.cs index fc0668b..2b939b0 100644 --- a/Tests/CocoCrawler.IntegrationTests/ExtractListAndPaginate/ExtractListAndPaginateTests.cs +++ b/Tests/CocoCrawler.IntegrationTests/Scenarios/ExtractListAndPaginate/ExtractListAndPaginateTests.cs @@ -1,11 +1,9 @@ using CocoCrawler.Builders; using CocoCrawler.Scheduler; using FluentAssertions; -using WireMock.RequestBuilders; -using WireMock.ResponseBuilders; using WireMock.Server; -namespace CocoCrawler.IntegrationTests.ExtractListAndPaginate; +namespace CocoCrawler.IntegrationTests.Scenarios.ExtractListAndPaginate; [Collection(nameof(BrowserCollection))] public class ExtractObjectAndPaginateTests @@ -16,15 +14,8 @@ public class ExtractObjectAndPaginateTests public async Task ExtractListAndPaginate_ShouldHaveDetailsInFile_OnHappyFlow() { // Arange - _wireMockServer.Given(Request.Create().WithUrl("http://localhost:9090/main-page")) - .RespondWith(Response.Create() - .WithHeader("Content-Type", "text/xml; charset=utf-8") - .WithBodyFromFile("ExtractListAndPaginate\\Responses\\main-page.html")); - - _wireMockServer.Given(Request.Create().WithUrl("http://localhost:9090/page-2")) - .RespondWith(Response.Create() - .WithHeader("Content-Type", "text/xml; charset=utf-8") - .WithBodyFromFile("ExtractListAndPaginate\\Responses\\page-2.html")); + _wireMockServer.ReturnSuccessWithBodyFromFile("http://localhost:9090/main-page", "Scenarios\\ExtractListAndPaginate\\Responses\\main-page.html"); + _wireMockServer.ReturnSuccessWithBodyFromFile("http://localhost:9090/page-2", "Scenarios\\ExtractListAndPaginate\\Responses\\page-2.html"); var crawlerEngine = await new CrawlerEngineBuilder() .AddPage("http://localhost:9090/main-page", options => options diff --git a/Tests/CocoCrawler.IntegrationTests/ExtractListAndPaginate/Responses/main-page.html b/Tests/CocoCrawler.IntegrationTests/Scenarios/ExtractListAndPaginate/Responses/main-page.html similarity index 100% rename from Tests/CocoCrawler.IntegrationTests/ExtractListAndPaginate/Responses/main-page.html rename to Tests/CocoCrawler.IntegrationTests/Scenarios/ExtractListAndPaginate/Responses/main-page.html diff --git a/Tests/CocoCrawler.IntegrationTests/ExtractListAndPaginate/Responses/page-2.html b/Tests/CocoCrawler.IntegrationTests/Scenarios/ExtractListAndPaginate/Responses/page-2.html similarity index 100% rename from Tests/CocoCrawler.IntegrationTests/ExtractListAndPaginate/Responses/page-2.html rename to Tests/CocoCrawler.IntegrationTests/Scenarios/ExtractListAndPaginate/Responses/page-2.html diff --git a/Tests/CocoCrawler.IntegrationTests/OpenLinksExtractObjectAndPaginate/OpenLinksExtractObjectAndPaginate.cs b/Tests/CocoCrawler.IntegrationTests/Scenarios/OpenLinksExtractObjectAndPaginate/OpenLinksExtractObjectAndPaginateTests.cs similarity index 81% rename from Tests/CocoCrawler.IntegrationTests/OpenLinksExtractObjectAndPaginate/OpenLinksExtractObjectAndPaginate.cs rename to Tests/CocoCrawler.IntegrationTests/Scenarios/OpenLinksExtractObjectAndPaginate/OpenLinksExtractObjectAndPaginateTests.cs index ac346ce..e688c33 100644 --- a/Tests/CocoCrawler.IntegrationTests/OpenLinksExtractObjectAndPaginate/OpenLinksExtractObjectAndPaginate.cs +++ b/Tests/CocoCrawler.IntegrationTests/Scenarios/OpenLinksExtractObjectAndPaginate/OpenLinksExtractObjectAndPaginateTests.cs @@ -1,39 +1,33 @@ using CocoCrawler.Builders; using CocoCrawler.Scheduler; using FluentAssertions; -using WireMock.RequestBuilders; -using WireMock.ResponseBuilders; using WireMock.Server; -namespace CocoCrawler.IntegrationTests.ExtractListAndPaginate; +namespace CocoCrawler.IntegrationTests.Scenarios.OpenLinksExtractObjectAndPaginate; [Collection(nameof(BrowserCollection))] -public class OpenLinksExtractObjectAndPaginate +public class OpenLinksExtractObjectAndPaginateTests { private readonly WireMockServer _wireMockServer = WireMockServer.Start(port: 9010); [Fact] - public async Task ExtractListAndPaginate_ShouldHaveDetailsInFile_OnHappyFlow() + public async Task OpenLinksExtractObjAndPaginate_ShouldHaveDetailsInFile_OnHappyFlow() { // Arange foreach (var index in Enumerable.Range(1, 10)) { - _wireMockServer.Given(Request.Create().WithUrl($"http://localhost:9010/main-page-{index}")) - .RespondWith(Response.Create() - .WithHeader("Content-Type", "text/xml; charset=utf-8") - .WithBody(GetListingHtmlPages(index))); + _wireMockServer.ReturnSuccessWithPage($"{_wireMockServer.Url}/main-page-{index}", GetListingHtmlPages(index)); } - foreach (var index in Enumerable.Range(1, 10 * 3)) + foreach (var index in Enumerable.Range(1, 10 * 3)) // 3 listing per page { - _wireMockServer.Given(Request.Create().WithUrl($"http://localhost:9010/content-page-{index}")) - .RespondWith(Response.Create() - .WithHeader("Content-Type", "text/xml; charset=utf-8") - .WithBody(GetContentPage(index))); + _wireMockServer.ReturnSuccessWithPage($"{_wireMockServer.Url}/content-page-{index}", GetContentPage(index)); } + var outputFile = Path.Combine("Scenarios", "OpenLinksExtractObjectAndPaginate", "Results", "resultstest1.csv"); + var crawlerEngine = await new CrawlerEngineBuilder() - .AddPage("http://localhost:9010/main-page-1", options => options + .AddPage($"{_wireMockServer.Url}/main-page-1", options => options .OpenLinks("div.content.test a.link", newPage => newPage .ExtractObject([ new("Title", "div.content.test div.title"), @@ -42,9 +36,9 @@ public async Task ExtractListAndPaginate_ShouldHaveDetailsInFile_OnHappyFlow() new("Link", "a", "href") ])) .AddPagination("div.pagination a:nth-last-child(1)") - .AddOutputToCsvFile("OpenLinksExtractObjectAndPaginate\\Results\\resultstest1.csv")) + .AddOutputToCsvFile(outputFile)) .ConfigureEngine(e => e - .WithScheduler(new InMemoryScheduler(totalSecondsTimeoutAfterJob: 5)) + .WithScheduler(new InMemoryScheduler(totalSecondsTimeoutAfterJob: 2)) .WithIgnoreUrls(["http://localhost:9010/content-page-6"]) .DisableParallelism()) .BuildAsync(); @@ -53,7 +47,7 @@ public async Task ExtractListAndPaginate_ShouldHaveDetailsInFile_OnHappyFlow() await crawlerEngine.RunAsync(); // Assert - var outputContents = File.ReadAllText("OpenLinksExtractObjectAndPaginate\\Results\\resultstest1.csv"); + var outputContents = File.ReadAllText(outputFile); var expect = @"Url,Title,Description,Amount,Link http://localhost:9010/content-page-1,Title 1,Description1,Amount 10,link1 @@ -88,6 +82,8 @@ public async Task ExtractListAndPaginate_ShouldHaveDetailsInFile_OnHappyFlow() "; outputContents.Should().BeEquivalentTo(expect); + + File.Delete(outputFile); } private static string GetContentPage(int index) diff --git a/Tests/CocoCrawler.IntegrationTests/WireMockExtensions.cs b/Tests/CocoCrawler.IntegrationTests/WireMockExtensions.cs new file mode 100644 index 0000000..7f35eb9 --- /dev/null +++ b/Tests/CocoCrawler.IntegrationTests/WireMockExtensions.cs @@ -0,0 +1,42 @@ +using WireMock.RequestBuilders; +using WireMock.ResponseBuilders; +using WireMock.Server; + +namespace CocoCrawler.IntegrationTests; + +internal static class WireMockExtensions +{ + internal static void ReturnSuccessFor(this WireMockServer wireMockServer, string url) + { + wireMockServer + .Given(Request.Create() + .UsingGet() + .WithUrl(url)) + .RespondWith(Response.Create() + .WithSuccess()); + } + + internal static void ReturnSuccessWithBodyFromFile(this WireMockServer wireMockServer, string url, string filePath) + { + wireMockServer + .Given(Request.Create() + .UsingGet() + .WithUrl(url)) + .RespondWith(Response.Create() + .WithHeader("Content-Type", "text/xml; charset=utf-8") + .WithSuccess() + .WithBodyFromFile(filePath)); + } + + internal static void ReturnSuccessWithPage(this WireMockServer wireMockServer, string url, string page) + { + wireMockServer + .Given(Request.Create() + .UsingGet() + .WithUrl(url)) + .RespondWith(Response.Create() + .WithHeader("Content-Type", "text/xml; charset=utf-8") + .WithSuccess() + .WithBody(page)); + } +}