Merge pull request #19 from ScoopInstaller/multiple-bucketsources

Support for multiple buckets sources

src/ScoopSearch.Indexer.Console/LoggingExtensions.cs

@@ -28,16 +28,21 @@ public static IHostBuilder ConfigureSerilog(this IHostBuilder @this, string logF
                 .Enrich.WithSensitiveDataMasking(options =>
                 {
                     options.MaskingOperators.Clear();
-                    options.MaskingOperators.Add(new TokensMaskingOperator(
-                        provider.GetRequiredService<IOptions<GitHubOptions>>().Value.Token,
-                        provider.GetRequiredService<IOptions<AzureSearchOptions>>().Value.AdminApiKey
-                    ));
+                    var tokens = new[]
+                        {
+                            provider.GetRequiredService<IOptions<GitHubOptions>>().Value.Token,
+                            provider.GetRequiredService<IOptions<AzureSearchOptions>>().Value.AdminApiKey
+                        }
+                        .Where(token => token != null)
+                        .Cast<string>()
+                        .ToArray();
+                    options.MaskingOperators.Add(new TokensMaskingOperator(tokens));
                 })
                 .WriteTo.File(new CompactJsonFormatter(), logFile)
                 .WriteTo.Logger(options => options
                     .MinimumLevel.Information()
                     // Exclude verbose HttpClient logs from the console
-                    .Filter.ByExcluding(_ => Matching.FromSource(typeof(HttpClient).FullName)(_) && _.Level < LogEventLevel.Warning)
+                    .Filter.ByExcluding(_ => Matching.FromSource(typeof(HttpClient).FullName!)(_) && _.Level < LogEventLevel.Warning)
                     .WriteTo.Console(outputTemplate: "[{Timestamp:HH:mm:ss} {Level:u3} {ThreadId}] {Message:lj}{NewLine}"));
         });
     }

src/ScoopSearch.Indexer.Console/Program.cs

@@ -1,33 +1,18 @@
-using Castle.DynamicProxy;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using ScoopSearch.Indexer;
 using ScoopSearch.Indexer.Console;
-using ScoopSearch.Indexer.Console.Interceptor;
-using ScoopSearch.Indexer.Git;
-using ScoopSearch.Indexer.GitHub;
-using ScoopSearch.Indexer.Indexer;
-using ScoopSearch.Indexer.Processor;
 
 const string LogFile = "output.txt";
+TimeSpan Timeout = TimeSpan.FromMinutes(30);
 
 using IHost host = Host.CreateDefaultBuilder(args)
     .ConfigureServices(services =>
     {
         services.RegisterScoopSearchIndexer();
-
-        // Decorate some classes with interceptors for logging purpose
-        services.AddSingleton<IAsyncInterceptor, TimingInterceptor>();
-        services.DecorateWithInterceptors<IGitRepositoryProvider, IAsyncInterceptor>();
-        services.DecorateWithInterceptors<IGitHubClient, IAsyncInterceptor>();
-        services.DecorateWithInterceptors<ISearchClient, IAsyncInterceptor>();
-        services.DecorateWithInterceptors<ISearchIndex, IAsyncInterceptor>();
-        services.DecorateWithInterceptors<IIndexingProcessor, IAsyncInterceptor>();
-        services.DecorateWithInterceptors<IFetchBucketsProcessor, IAsyncInterceptor>();
-        services.DecorateWithInterceptors<IFetchManifestsProcessor, IAsyncInterceptor>();
-        services.DecorateWithInterceptors<IScoopSearchIndexer, IAsyncInterceptor>();
     })
     .ConfigureSerilog(LogFile)
     .Build();
 
-await host.Services.GetRequiredService<IScoopSearchIndexer>().ExecuteAsync();
+var cancellationToken = new CancellationTokenSource(Timeout).Token;
+await host.Services.GetRequiredService<IScoopSearchIndexer>().ExecuteAsync(cancellationToken);

src/ScoopSearch.Indexer.Console/ScoopSearch.Indexer.Console.csproj

@@ -9,7 +9,7 @@
       <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
       <PackageReference Include="Serilog.Enrichers.Sensitive" Version="1.7.2" />
       <PackageReference Include="Serilog.Enrichers.Thread" Version="3.1.0" />
-      <PackageReference Include="Serilog.Extensions.Hosting" Version="5.0.1" />
+      <PackageReference Include="Serilog.Extensions.Hosting" Version="7.0.0" />
       <PackageReference Include="Serilog.Formatting.Compact" Version="1.1.0" />
       <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
       <PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />

src/ScoopSearch.Indexer.Tests/Buckets/Providers/GitHubBucketsProviderTests.cs

@@ -0,0 +1,75 @@
+using FluentAssertions;
+using Moq;
+using ScoopSearch.Indexer.Buckets.Providers;
+using ScoopSearch.Indexer.GitHub;
+using ScoopSearch.Indexer.Tests.Helpers;
+
+namespace ScoopSearch.Indexer.Tests.Buckets.Providers;
+
+public class GitHubBucketsProviderTests
+{
+    private readonly Mock<IGitHubClient> _gitHubClientMock;
+    private readonly GitHubBucketsProvider _sut;
+
+    public GitHubBucketsProviderTests()
+    {
+        _gitHubClientMock = new Mock<IGitHubClient>();
+        _sut = new GitHubBucketsProvider(_gitHubClientMock.Object);
+    }
+
+    [Theory]
+    [InlineData("http://foo/bar", false)]
+    [InlineData("https://foo/bar", false)]
+    [InlineData("http://www.google.fr/foo", false)]
+    [InlineData("https://www.google.fr/foo", false)]
+    [InlineData("http://github.com", true)]
+    [InlineData("https://github.com", true)]
+    [InlineData("http://www.github.com", true)]
+    [InlineData("https://www.github.com", true)]
+    [InlineData("http://www.GitHub.com", true)]
+    [InlineData("https://www.GitHub.com", true)]
+    public void IsCompatible_Succeeds(string input, bool expectedResult)
+    {
+        // Arrange
+        var uri = new Uri(input);
+
+        // Act
+        var result = _sut.IsCompatible(uri);
+
+        // Arrange
+        result.Should().Be(expectedResult);
+    }
+
+    [Fact]
+    public async void GetBucketAsync_ValidRepo_ReturnsBucket()
+    {
+        // Arrange
+        var cancellationToken = new CancellationToken();
+        var uri = Faker.CreateUri();
+        var gitHubRepo = Faker.CreateGitHubRepo().Generate();
+        _gitHubClientMock.Setup(x => x.GetRepositoryAsync(uri, cancellationToken)).ReturnsAsync(gitHubRepo);
+
+        // Act
+        var result = await _sut.GetBucketAsync(uri, cancellationToken);
+
+        // Assert
+        result.Should().NotBeNull();
+        result!.Uri.Should().Be(gitHubRepo.HtmlUri);
+        result.Stars.Should().Be(gitHubRepo.Stars);
+    }
+
+    [Fact]
+    public async void GetBucketAsync_InvalidRepo_ReturnsNull()
+    {
+        // Arrange
+        var cancellationToken = new CancellationToken();
+        var uri = Faker.CreateUri();
+        _gitHubClientMock.Setup(x => x.GetRepositoryAsync(uri, cancellationToken)).ReturnsAsync((GitHubRepo?)null);
+
+        // Act
+        var result = await _sut.GetBucketAsync(uri, cancellationToken);
+
+        // Assert
+        result.Should().BeNull();
+    }
+}

src/ScoopSearch.Indexer.Tests/Buckets/Sources/GitHubBucketsSourceTests.cs

@@ -0,0 +1,67 @@
+using FluentAssertions;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Moq;
+using ScoopSearch.Indexer.Buckets;
+using ScoopSearch.Indexer.Buckets.Sources;
+using ScoopSearch.Indexer.Configuration;
+using ScoopSearch.Indexer.GitHub;
+using ScoopSearch.Indexer.Tests.Helpers;
+using Xunit.Abstractions;
+
+namespace ScoopSearch.Indexer.Tests.Buckets.Sources;
+
+public class GitHubBucketsSourceTests
+{
+    private readonly Mock<IGitHubClient> _gitHubClientMock;
+    private readonly GitHubOptions _gitHubOptions;
+    private readonly XUnitLogger<GitHubBucketsSource> _logger;
+    private readonly GitHubBucketsSource _sut;
+
+    public GitHubBucketsSourceTests(ITestOutputHelper testOutputHelper)
+    {
+        _gitHubClientMock = new Mock<IGitHubClient>();
+        _gitHubOptions = new GitHubOptions();
+        _logger = new XUnitLogger<GitHubBucketsSource>(testOutputHelper);
+        _sut = new GitHubBucketsSource(_gitHubClientMock.Object, new OptionsWrapper<GitHubOptions>(_gitHubOptions), _logger);
+    }
+
+    [Fact]
+    public async void GetBucketsAsync_InvalidQueries_ReturnsEmpty()
+    {
+        // Arrange
+        var cancellationToken = new CancellationToken();
+        _gitHubOptions.BucketsSearchQueries = null;
+
+        // Act
+        var result = await _sut.GetBucketsAsync(cancellationToken).ToArrayAsync(cancellationToken);
+
+        // Arrange
+        result.Should().BeEmpty();
+        _logger.Should().Log(LogLevel.Warning, "No buckets search queries found in configuration");
+    }
+
+    [Fact]
+    public async void GetBucketsAsync_Succeeds()
+    {
+        // Arrange
+        var cancellationToken = new CancellationToken();
+        var input = new (string[] queries, GitHubRepo[] repos)[]
+        {
+            (new[] { "foo", "bar" }, new[] { Faker.CreateGitHubRepo().Generate() }),
+            (new[] { "bar", "foo" }, new[] { Faker.CreateGitHubRepo().Generate() }),
+        };
+        _gitHubOptions.BucketsSearchQueries = input.Select(x => x.queries).ToArray();
+        _gitHubClientMock.Setup(x => x.SearchRepositoriesAsync(input[0].queries, cancellationToken)).Returns(input[0].repos.ToAsyncEnumerable());
+        _gitHubClientMock.Setup(x => x.SearchRepositoriesAsync(input[1].queries, cancellationToken)).Returns(input[1].repos.ToAsyncEnumerable());
+
+        // Act
+        var result = await _sut.GetBucketsAsync(cancellationToken).ToArrayAsync(cancellationToken);
+
+        // Arrange
+        result.Should().BeEquivalentTo(
+            input.SelectMany(x => x.repos),
+            options => options
+                .WithMapping<Bucket>(x => x.HtmlUri, y => y.Uri));
+    }
+}

src/ScoopSearch.Indexer.Tests/Buckets/Sources/ManualBucketsListSourceTests.cs

@@ -0,0 +1,130 @@
+using System.Net;
+using CsvHelper;
+using FluentAssertions;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Moq;
+using Moq.Protected;
+using ScoopSearch.Indexer.Buckets;
+using ScoopSearch.Indexer.Buckets.Providers;
+using ScoopSearch.Indexer.Buckets.Sources;
+using ScoopSearch.Indexer.Configuration;
+using ScoopSearch.Indexer.Tests.Helpers;
+using Xunit.Abstractions;
+using Faker = ScoopSearch.Indexer.Tests.Helpers.Faker;
+using MissingFieldException = CsvHelper.MissingFieldException;
+
+namespace ScoopSearch.Indexer.Tests.Buckets.Sources;
+
+public class ManualBucketsListSourceTests
+{
+    private readonly Mock<IHttpClientFactory> _httpClientFactoryMock;
+    private readonly Mock<IBucketsProvider> _bucketsProviderMock;
+    private readonly BucketsOptions _bucketsOptions;
+    private readonly XUnitLogger<ManualBucketsListSource> _logger;
+    private readonly ManualBucketsListSource _sut;
+
+    public ManualBucketsListSourceTests(ITestOutputHelper testOutputHelper)
+    {
+        _httpClientFactoryMock = new Mock<IHttpClientFactory>();
+        _bucketsProviderMock = new Mock<IBucketsProvider>();
+        _bucketsOptions = new BucketsOptions();
+        _logger = new XUnitLogger<ManualBucketsListSource>(testOutputHelper);
+
+        _sut = new ManualBucketsListSource(
+            _httpClientFactoryMock.Object,
+            new[] {_bucketsProviderMock.Object },
+            new OptionsWrapper<BucketsOptions>(_bucketsOptions),
+            _logger);
+    }
+
+    [Fact]
+    public async void GetBucketsAsync_InvalidUri_ReturnsEmpty()
+    {
+        // Arrange
+        var cancellationToken = new CancellationToken();
+        _bucketsOptions.ManualBucketsListUrl = null;
+
+        // Act
+        var result = await _sut.GetBucketsAsync(cancellationToken).ToArrayAsync(cancellationToken);
+
+        // Arrange
+        result.Should().BeEmpty();
+        _logger.Should().Log(LogLevel.Warning, "No buckets list url found in configuration");
+    }
+
+    [Theory]
+    [MemberData(nameof(GetBucketsAsyncErrorsTestCases))]
+#pragma warning disable xUnit1026
+    public async void GetBucketsAsync_InvalidStatusCodeSucceeds<TExpectedException>(HttpStatusCode statusCode, string content, TExpectedException _)
+#pragma warning restore xUnit1026
+        where TExpectedException : Exception
+    {
+        // Arrange
+        _bucketsOptions.ManualBucketsListUrl = Faker.CreateUri();
+        var cancellationToken = new CancellationToken();
+        var httpMessageHandlerMock = new Mock<HttpMessageHandler>();
+        httpMessageHandlerMock
+            .Protected()
+            .Setup<Task<HttpResponseMessage>>(
+                "SendAsync",
+                ItExpr.Is<HttpRequestMessage>(x => x.RequestUri == _bucketsOptions.ManualBucketsListUrl),
+                ItExpr.IsAny<CancellationToken>())
+            .ReturnsAsync(new HttpResponseMessage() { StatusCode = statusCode, Content = new StringContent(content) });
+        _httpClientFactoryMock.Setup(x => x.CreateClient("Default")).Returns(new HttpClient(httpMessageHandlerMock.Object));
+
+        // Act
+        var result = async () => await _sut.GetBucketsAsync(cancellationToken).ToArrayAsync(cancellationToken);
+
+        // Assert
+        await result.Should().ThrowAsync<TExpectedException>();
+    }
+
+    public static IEnumerable<object[]> GetBucketsAsyncErrorsTestCases()
+    {
+        yield return new object[] { HttpStatusCode.NotFound, $"url", new HttpRequestException() };
+        yield return new object[] { HttpStatusCode.OK, "", new ReaderException(null) };
+        yield return new object[] { HttpStatusCode.OK, $"foo{Environment.NewLine}{Faker.CreateUrl()}", new MissingFieldException(null) };
+    }
+
+    [Theory]
+    [MemberData(nameof(GetBucketsAsyncTestCases))]
+    public async void GetBucketsAsync_Succeeds(string content, string repositoryUri, bool isCompatible, bool expectedBucket)
+    {
+        // Arrange
+        _bucketsOptions.ManualBucketsListUrl = Faker.CreateUri();
+        var cancellationToken = new CancellationToken();
+        var httpMessageHandlerMock = new Mock<HttpMessageHandler>();
+        _httpClientFactoryMock.Setup(x => x.CreateClient("Default")).Returns(new HttpClient(httpMessageHandlerMock.Object));
+
+        httpMessageHandlerMock
+            .Protected()
+            .Setup<Task<HttpResponseMessage>>(
+                "SendAsync",
+                ItExpr.Is<HttpRequestMessage>(x => x.RequestUri == _bucketsOptions.ManualBucketsListUrl),
+                ItExpr.IsAny<CancellationToken>())
+            .ReturnsAsync(new HttpResponseMessage() { StatusCode = HttpStatusCode.OK, Content = new StringContent(content) });
+        Bucket bucket = new Bucket(new Uri(repositoryUri), 123);
+        _bucketsProviderMock.Setup(x => x.IsCompatible(new Uri(repositoryUri))).Returns(isCompatible);
+        _bucketsProviderMock.Setup(x => x.GetBucketAsync(new Uri(repositoryUri), cancellationToken)).ReturnsAsync(bucket);
+
+        // Act
+        var result = await _sut.GetBucketsAsync(cancellationToken).ToArrayAsync(cancellationToken);
+
+        // Arrange
+        result.Should().HaveCount(expectedBucket ? 1 : 0);
+        if (expectedBucket)
+        {
+            result.Should().BeEquivalentTo(new[] { bucket });
+        }
+    }
+
+    public static IEnumerable<object[]> GetBucketsAsyncTestCases()
+    {
+        yield return new object[] { "url", Faker.CreateUrl(), true, false };
+        var url = Faker.CreateUrl();
+        yield return new object[] { $"url{Environment.NewLine}{url}", url, false, false };
+        yield return new object[] { $"url{Environment.NewLine}{url}", url, true, true };
+        yield return new object[] { $"url{Environment.NewLine}{url}.git", url, true, true };
+    }
+}