diff --git a/src/Elastic.Markdown/BuildContext.cs b/src/Elastic.Markdown/BuildContext.cs index 6d73b60..1e08ca3 100644 --- a/src/Elastic.Markdown/BuildContext.cs +++ b/src/Elastic.Markdown/BuildContext.cs @@ -17,6 +17,8 @@ public record BuildContext public IFileInfo ConfigurationPath { get; } + public GitConfiguration Git { get; } + public required DiagnosticsCollector Collector { get; init; } public bool Force { get; init; } @@ -54,6 +56,8 @@ public BuildContext(IFileSystem readFileSystem, IFileSystem writeFileSystem, str if (ConfigurationPath.FullName != SourcePath.FullName) SourcePath = ConfigurationPath.Directory!; + Git = GitConfiguration.Create(ReadFileSystem); + } diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 43dffce..6b847b7 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -2,7 +2,6 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information using System.IO.Abstractions; -using System.Security.Cryptography; using System.Text.Json; using System.Text.Json.Serialization; using Elastic.Markdown.IO; @@ -12,13 +11,20 @@ namespace Elastic.Markdown; [JsonSourceGenerationOptions(WriteIndented = true)] -[JsonSerializable(typeof(OutputState))] +[JsonSerializable(typeof(GenerationState))] +[JsonSerializable(typeof(LinkReference))] +[JsonSerializable(typeof(GitConfiguration))] internal partial class SourceGenerationContext : JsonSerializerContext; -public class OutputState +public record GenerationState { - public DateTimeOffset LastSeenChanges { get; set; } - public string[] Conflict { get; set; } = []; + [JsonPropertyName("last_seen_changes")] + public required DateTimeOffset LastSeenChanges { get; init; } + [JsonPropertyName("invalid_files")] + public required string[] InvalidFiles { get; init; } = []; + + [JsonPropertyName("git")] + public required GitConfiguration Git { get; init; } } public class DocumentationGenerator @@ -49,18 +55,13 @@ ILoggerFactory logger _logger.LogInformation($"Output directory: {docSet.OutputPath} Exists: {docSet.OutputPath.Exists}"); } - public OutputState? OutputState + public GenerationState? GetPreviousGenerationState() { - get - { - var stateFile = DocumentationSet.OutputStateFile; - stateFile.Refresh(); - if (!stateFile.Exists) return null; - var contents = stateFile.FileSystem.File.ReadAllText(stateFile.FullName); - return JsonSerializer.Deserialize(contents, SourceGenerationContext.Default.OutputState); - - - } + var stateFile = DocumentationSet.OutputStateFile; + stateFile.Refresh(); + if (!stateFile.Exists) return null; + var contents = stateFile.FileSystem.File.ReadAllText(stateFile.FullName); + return JsonSerializer.Deserialize(contents, SourceGenerationContext.Default.GenerationState); } @@ -69,26 +70,12 @@ public async Task ResolveDirectoryTree(Cancel ctx) => public async Task GenerateAll(Cancel ctx) { - if (Context.Force || OutputState == null) + var generationState = GetPreviousGenerationState(); + if (Context.Force || generationState == null) DocumentationSet.ClearOutputDirectory(); - _logger.LogInformation($"Last write source: {DocumentationSet.LastWrite}, output observed: {OutputState?.LastSeenChanges}"); - - var offendingFiles = new HashSet(OutputState?.Conflict ?? []); - var outputSeenChanges = OutputState?.LastSeenChanges ?? DateTimeOffset.MinValue; - if (offendingFiles.Count > 0) - { - _logger.LogInformation($"Reapplying changes since {DocumentationSet.LastWrite}"); - _logger.LogInformation($"Reapplying for {offendingFiles.Count} files with errors/warnings"); - } - else if (DocumentationSet.LastWrite > outputSeenChanges && OutputState != null) - _logger.LogInformation($"Using incremental build picking up changes since: {OutputState.LastSeenChanges}"); - else if (DocumentationSet.LastWrite <= outputSeenChanges && OutputState != null) - { - _logger.LogInformation($"No changes in source since last observed write {OutputState.LastSeenChanges} " - + "Pass --force to force a full regeneration"); + if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges)) return; - } _logger.LogInformation("Resolving tree"); await ResolveDirectoryTree(ctx); @@ -122,6 +109,7 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) => Context.Collector.Channel.TryComplete(); await GenerateDocumentationState(ctx); + await GenerateLinkReference(ctx); await Context.Collector.StopAsync(ctx); @@ -133,18 +121,58 @@ IFileInfo OutputFile(string relativePath) } + private bool CompilationNotNeeded(GenerationState? generationState, out HashSet offendingFiles, + out DateTimeOffset outputSeenChanges) + { + offendingFiles = new HashSet(generationState?.InvalidFiles ?? []); + outputSeenChanges = generationState?.LastSeenChanges ?? DateTimeOffset.MinValue; + if (generationState == null) + return false; + + if (Context.Git != generationState.Git) + { + _logger.LogInformation($"Full compilation: current git context: {Context.Git} differs from previous git context: {generationState.Git}"); + return false; + } + + if (offendingFiles.Count > 0) + { + _logger.LogInformation($"Incremental compilation. since: {DocumentationSet.LastWrite}"); + _logger.LogInformation($"Incremental compilation. {offendingFiles.Count} files with errors/warnings"); + } + else if (DocumentationSet.LastWrite > outputSeenChanges) + _logger.LogInformation($"Incremental compilation. since: {generationState.LastSeenChanges}"); + else if (DocumentationSet.LastWrite <= outputSeenChanges) + { + _logger.LogInformation($"No compilation: no changes since last observed: {generationState.LastSeenChanges}"); + _logger.LogInformation($"No compilation: no changes since last observed: {generationState.LastSeenChanges} " + + "Pass --force to force a full regeneration"); + return true; + } + + return false; + } + + private async Task GenerateLinkReference(Cancel ctx) + { + var file = DocumentationSet.LinkReferenceFile; + var state = LinkReference.Create(DocumentationSet); + var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.LinkReference); + await DocumentationSet.OutputPath.FileSystem.File.WriteAllBytesAsync(file.FullName, bytes, ctx); + } + private async Task GenerateDocumentationState(Cancel ctx) { var stateFile = DocumentationSet.OutputStateFile; _logger.LogInformation($"Writing documentation state {DocumentationSet.LastWrite} to {stateFile.FullName}"); var badFiles = Context.Collector.OffendingFiles.ToArray(); - var state = new OutputState + var state = new GenerationState { LastSeenChanges = DocumentationSet.LastWrite, - Conflict = badFiles - + InvalidFiles = badFiles, + Git = Context.Git }; - var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.OutputState); + var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.GenerationState); await DocumentationSet.OutputPath.FileSystem.File.WriteAllBytesAsync(stateFile.FullName, bytes, ctx); } diff --git a/src/Elastic.Markdown/Elastic.Markdown.csproj b/src/Elastic.Markdown/Elastic.Markdown.csproj index b6674eb..c9721f6 100644 --- a/src/Elastic.Markdown/Elastic.Markdown.csproj +++ b/src/Elastic.Markdown/Elastic.Markdown.csproj @@ -16,6 +16,7 @@ + diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 651bd8e..ecee2e3 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -13,6 +13,7 @@ public class DocumentationSet public BuildContext Context { get; } public string Name { get; } public IFileInfo OutputStateFile { get; } + public IFileInfo LinkReferenceFile { get; } public IDirectoryInfo SourcePath { get; } public IDirectoryInfo OutputPath { get; } @@ -34,6 +35,7 @@ public DocumentationSet(BuildContext context) Name = SourcePath.FullName; OutputStateFile = OutputPath.FileSystem.FileInfo.New(Path.Combine(OutputPath.FullName, ".doc.state")); + LinkReferenceFile = OutputPath.FileSystem.FileInfo.New(Path.Combine(OutputPath.FullName, "links.json")); Files = context.ReadFileSystem.Directory .EnumerateFiles(SourcePath.FullName, "*.*", SearchOption.AllDirectories) diff --git a/src/Elastic.Markdown/IO/GitConfiguration.cs b/src/Elastic.Markdown/IO/GitConfiguration.cs new file mode 100644 index 0000000..3e01d8f --- /dev/null +++ b/src/Elastic.Markdown/IO/GitConfiguration.cs @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using System.Text.Json.Serialization; +using IniParser; + +namespace Elastic.Markdown.IO; + +public record GitConfiguration +{ + [JsonPropertyName("branch")] + public required string Branch { get; init; } + [JsonPropertyName("remote")] + public required string Remote { get; init; } + [JsonPropertyName("ref")] + public required string Ref { get; init; } + + // manual read because libgit2sharp is not yet AOT ready + public static GitConfiguration Create(IFileSystem fileSystem) + { + // filesystem is not real so return a dummy + if (fileSystem is not FileSystem) + { + var fakeRef = Guid.NewGuid().ToString().Substring(0, 16); + return new GitConfiguration + { + Branch = $"test-{fakeRef}", + Remote = "elastic/docs-builder", + Ref = fakeRef, + }; + } + + var gitConfig = Git(".git/config"); + if (!gitConfig.Exists) + throw new Exception($"{Paths.Root.FullName} is not a git repository."); + + var head = Read(".git/HEAD").Replace("ref: ", string.Empty); + var gitRef = Read(".git/" + head); + var branch = head.Replace("refs/heads/", string.Empty); + + var ini = new FileIniDataParser(); + using var stream = gitConfig.OpenRead(); + using var streamReader = new StreamReader(stream); + var config = ini.ReadData(streamReader); + var remoteName = config[$"branch \"{branch}\""]["remote"]; + var remote = config[$"remote \"{remoteName}\""]["url"]; + + return new GitConfiguration + { + Ref = gitRef, + Branch = branch, + Remote = remote + }; + + IFileInfo Git(string path) => fileSystem.FileInfo.New(Path.Combine(Paths.Root.FullName, path)); + + string Read(string path) => + fileSystem.File.ReadAllText(Git(path).FullName).Trim(Environment.NewLine.ToCharArray()); + } +} diff --git a/src/Elastic.Markdown/IO/LinkReference.cs b/src/Elastic.Markdown/IO/LinkReference.cs new file mode 100644 index 0000000..b0f502e --- /dev/null +++ b/src/Elastic.Markdown/IO/LinkReference.cs @@ -0,0 +1,25 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using System.Text.Json.Serialization; +using IniParser; + +namespace Elastic.Markdown.IO; + +public record LinkReference +{ + [JsonPropertyName("origin")] + public required GitConfiguration Origin { get; init; } + [JsonPropertyName("links")] + public required string[] Links { get; init; } = []; + + public static LinkReference Create(DocumentationSet set) + { + var links = set.FlatMappedFiles.Values + .OfType() + .Select(m => m.RelativePath).ToArray(); + return new LinkReference { Origin = set.Context.Git, Links = links }; + } +} diff --git a/src/Elastic.Markdown/IO/Paths.cs b/src/Elastic.Markdown/IO/Paths.cs index a5788a4..8b63080 100644 --- a/src/Elastic.Markdown/IO/Paths.cs +++ b/src/Elastic.Markdown/IO/Paths.cs @@ -8,7 +8,8 @@ public static class Paths private static DirectoryInfo RootDirectoryInfo() { var directory = new DirectoryInfo(Directory.GetCurrentDirectory()); - while (directory != null && !directory.GetFiles("*.sln").Any()) + while (directory != null && + (directory.GetFiles("*.sln").Length == 0 || directory.GetDirectories(".git").Length == 0)) directory = directory.Parent; return directory ?? new DirectoryInfo(Directory.GetCurrentDirectory()); } diff --git a/tests/Elastic.Markdown.Tests/SiteMap/LinkReferenceTests.cs b/tests/Elastic.Markdown.Tests/SiteMap/LinkReferenceTests.cs new file mode 100644 index 0000000..38968cc --- /dev/null +++ b/tests/Elastic.Markdown.Tests/SiteMap/LinkReferenceTests.cs @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Markdown.IO; +using FluentAssertions; +using Xunit.Abstractions; + +namespace Elastic.Markdown.Tests.SiteMap; + +public class LinkReferenceTests(ITestOutputHelper output) : NavigationTestsBase(output) +{ + [Fact] + public void Create() + { + var reference = LinkReference.Create(Set); + + reference.Should().NotBeNull(); + } +} + +public class GitConfigurationTests(ITestOutputHelper output) : NavigationTestsBase(output) +{ + [Fact] + public void Create() + { + var git = GitConfiguration.Create(ReadFileSystem); + + git.Should().NotBeNull(); + git!.Branch.Should().NotBeNullOrWhiteSpace(); + // this validates we are not returning the test instance as were doing a real read + git.Branch.Should().NotContain(git.Ref); + git.Ref.Should().NotBeNullOrWhiteSpace(); + git.Remote.Should().NotBeNullOrWhiteSpace(); + } +} diff --git a/tests/Elastic.Markdown.Tests/SiteMap/NavigationTestsBase.cs b/tests/Elastic.Markdown.Tests/SiteMap/NavigationTestsBase.cs index 1559d8e..f1cd126 100644 --- a/tests/Elastic.Markdown.Tests/SiteMap/NavigationTestsBase.cs +++ b/tests/Elastic.Markdown.Tests/SiteMap/NavigationTestsBase.cs @@ -16,28 +16,29 @@ public class NavigationTestsBase : IAsyncLifetime protected NavigationTestsBase(ITestOutputHelper output) { var logger = new TestLoggerFactory(output); - var readFs = new FileSystem(); //use real IO to read docs. + ReadFileSystem = new FileSystem(); //use real IO to read docs. var writeFs = new MockFileSystem(new MockFileSystemOptions //use in memory mock fs to test generation { CurrentDirectory = Paths.Root.FullName }); - var context = new BuildContext(readFs, writeFs) + var context = new BuildContext(ReadFileSystem, writeFs) { Force = false, UrlPathPrefix = null, Collector = new DiagnosticsCollector(logger, []) }; - var set = new DocumentationSet(context); + Set = new DocumentationSet(context); - set.Files.Should().HaveCountGreaterThan(10); - Generator = new DocumentationGenerator(set, logger); + Set.Files.Should().HaveCountGreaterThan(10); + Generator = new DocumentationGenerator(Set, logger); } - public DocumentationGenerator Generator { get; } - - public ConfigurationFile Configuration { get; set; } = default!; + protected FileSystem ReadFileSystem { get; set; } + protected DocumentationSet Set { get; } + protected DocumentationGenerator Generator { get; } + protected ConfigurationFile Configuration { get; set; } = default!; public async Task InitializeAsync() {