Track git information as part of incremental builds and link references

src/Elastic.Markdown/BuildContext.cs

@@ -17,6 +17,8 @@ public record BuildContext
 
 	public IFileInfo ConfigurationPath { get; }
 
+	public GitConfiguration Git { get; }
+
 	public required DiagnosticsCollector Collector { get; init; }
 
 	public bool Force { get; init; }
@@ -54,6 +56,8 @@ public BuildContext(IFileSystem readFileSystem, IFileSystem writeFileSystem, str
 		if (ConfigurationPath.FullName != SourcePath.FullName)
 			SourcePath = ConfigurationPath.Directory!;
 
+		Git = GitConfiguration.Create(ReadFileSystem);
+
 
 	}
 

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -2,7 +2,6 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 using System.IO.Abstractions;
-using System.Security.Cryptography;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using Elastic.Markdown.IO;
@@ -12,13 +11,20 @@
 namespace Elastic.Markdown;
 
 [JsonSourceGenerationOptions(WriteIndented = true)]
-[JsonSerializable(typeof(OutputState))]
+[JsonSerializable(typeof(GenerationState))]
+[JsonSerializable(typeof(LinkReference))]
+[JsonSerializable(typeof(GitConfiguration))]
 internal partial class SourceGenerationContext : JsonSerializerContext;
 
-public class OutputState
+public record GenerationState
 {
-	public DateTimeOffset LastSeenChanges { get; set; }
-	public string[] Conflict { get; set; } = [];
+	[JsonPropertyName("last_seen_changes")]
+	public required DateTimeOffset LastSeenChanges { get; init; }
+	[JsonPropertyName("invalid_files")]
+	public required string[] InvalidFiles { get; init; } = [];
+
+	[JsonPropertyName("git")]
+	public required GitConfiguration Git { get; init; }
 }
 
 public class DocumentationGenerator
@@ -49,18 +55,13 @@ ILoggerFactory logger
 		_logger.LogInformation($"Output directory: {docSet.OutputPath} Exists: {docSet.OutputPath.Exists}");
 	}
 
-	public OutputState? OutputState
+	public GenerationState? GetPreviousGenerationState()
 	{
-		get
-		{
-			var stateFile = DocumentationSet.OutputStateFile;
-			stateFile.Refresh();
-			if (!stateFile.Exists) return null;
-			var contents = stateFile.FileSystem.File.ReadAllText(stateFile.FullName);
-			return JsonSerializer.Deserialize(contents, SourceGenerationContext.Default.OutputState);
-
-
-		}
+		var stateFile = DocumentationSet.OutputStateFile;
+		stateFile.Refresh();
+		if (!stateFile.Exists) return null;
+		var contents = stateFile.FileSystem.File.ReadAllText(stateFile.FullName);
+		return JsonSerializer.Deserialize(contents, SourceGenerationContext.Default.GenerationState);
 	}
 
 
@@ -69,26 +70,12 @@ public async Task ResolveDirectoryTree(Cancel ctx) =>
 
 	public async Task GenerateAll(Cancel ctx)
 	{
-		if (Context.Force || OutputState == null)
+		var generationState = GetPreviousGenerationState();
+		if (Context.Force || generationState == null)
 			DocumentationSet.ClearOutputDirectory();
 
-		_logger.LogInformation($"Last write source: {DocumentationSet.LastWrite}, output observed: {OutputState?.LastSeenChanges}");
-
-		var offendingFiles = new HashSet<string>(OutputState?.Conflict ?? []);
-		var outputSeenChanges = OutputState?.LastSeenChanges ?? DateTimeOffset.MinValue;
-		if (offendingFiles.Count > 0)
-		{
-			_logger.LogInformation($"Reapplying changes since {DocumentationSet.LastWrite}");
-			_logger.LogInformation($"Reapplying for {offendingFiles.Count} files with errors/warnings");
-		}
-		else if (DocumentationSet.LastWrite > outputSeenChanges && OutputState != null)
-			_logger.LogInformation($"Using incremental build picking up changes since: {OutputState.LastSeenChanges}");
-		else if (DocumentationSet.LastWrite <= outputSeenChanges && OutputState != null)
-		{
-			_logger.LogInformation($"No changes in source since last observed write {OutputState.LastSeenChanges} "
-			                       + "Pass --force to force a full regeneration");
+		if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges))
 			return;
-		}
 
 		_logger.LogInformation("Resolving tree");
 		await ResolveDirectoryTree(ctx);
@@ -122,6 +109,7 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 		Context.Collector.Channel.TryComplete();
 
 		await GenerateDocumentationState(ctx);
+		await GenerateLinkReference(ctx);
 
 		await Context.Collector.StopAsync(ctx);
 
@@ -133,18 +121,58 @@ IFileInfo OutputFile(string relativePath)
 
 	}
 
+	private bool CompilationNotNeeded(GenerationState? generationState, out HashSet<string> offendingFiles,
+		out DateTimeOffset outputSeenChanges)
+	{
+		offendingFiles = new HashSet<string>(generationState?.InvalidFiles ?? []);
+		outputSeenChanges = generationState?.LastSeenChanges ?? DateTimeOffset.MinValue;
+		if (generationState == null)
+			return false;
+
+		if (Context.Git != generationState.Git)
+		{
+			_logger.LogInformation($"Full compilation: current git context: {Context.Git} differs from previous git context: {generationState.Git}");
+			return false;
+		}
+
+		if (offendingFiles.Count > 0)
+		{
+			_logger.LogInformation($"Incremental compilation. since: {DocumentationSet.LastWrite}");
+			_logger.LogInformation($"Incremental compilation. {offendingFiles.Count} files with errors/warnings");
+		}
+		else if (DocumentationSet.LastWrite > outputSeenChanges)
+			_logger.LogInformation($"Incremental compilation. since: {generationState.LastSeenChanges}");
+		else if (DocumentationSet.LastWrite <= outputSeenChanges)
+		{
+			_logger.LogInformation($"No compilation: no changes since last observed: {generationState.LastSeenChanges}");
+			_logger.LogInformation($"No compilation: no changes since last observed: {generationState.LastSeenChanges} "
+			                       + "Pass --force to force a full regeneration");
+			return true;
+		}
+
+		return false;
+	}
+
+	private async Task GenerateLinkReference(Cancel ctx)
+	{
+		var file = DocumentationSet.LinkReferenceFile;
+		var state = LinkReference.Create(DocumentationSet);
+		var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.LinkReference);
+		await DocumentationSet.OutputPath.FileSystem.File.WriteAllBytesAsync(file.FullName, bytes, ctx);
+	}
+
 	private async Task GenerateDocumentationState(Cancel ctx)
 	{
 		var stateFile = DocumentationSet.OutputStateFile;
 		_logger.LogInformation($"Writing documentation state {DocumentationSet.LastWrite} to {stateFile.FullName}");
 		var badFiles = Context.Collector.OffendingFiles.ToArray();
-		var state = new OutputState
+		var state = new GenerationState
 		{
 			LastSeenChanges = DocumentationSet.LastWrite,
-			Conflict = badFiles
-
+			InvalidFiles = badFiles,
+			Git = Context.Git
 		};
-		var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.OutputState);
+		var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.GenerationState);
 		await DocumentationSet.OutputPath.FileSystem.File.WriteAllBytesAsync(stateFile.FullName, bytes, ctx);
 	}
 

src/Elastic.Markdown/Elastic.Markdown.csproj

@@ -16,6 +16,7 @@
 
   <ItemGroup>
     <PackageReference Include="DotNet.Glob" Version="3.1.3" />
+    <PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
     <PackageReference Include="Markdig" Version="0.37.0"/>
     <PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
     <PackageReference Include="RazorSlices" Version="0.8.1" />

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -13,6 +13,7 @@ public class DocumentationSet
 	public BuildContext Context { get; }
 	public string Name { get; }
 	public IFileInfo OutputStateFile { get; }
+	public IFileInfo LinkReferenceFile { get; }
 
 	public IDirectoryInfo SourcePath { get; }
 	public IDirectoryInfo OutputPath { get; }
@@ -34,6 +35,7 @@ public DocumentationSet(BuildContext context)
 
 		Name = SourcePath.FullName;
 		OutputStateFile = OutputPath.FileSystem.FileInfo.New(Path.Combine(OutputPath.FullName, ".doc.state"));
+		LinkReferenceFile = OutputPath.FileSystem.FileInfo.New(Path.Combine(OutputPath.FullName, "links.json"));
 
 		Files = context.ReadFileSystem.Directory
 			.EnumerateFiles(SourcePath.FullName, "*.*", SearchOption.AllDirectories)

src/Elastic.Markdown/IO/GitConfiguration.cs

@@ -0,0 +1,62 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+using System.Text.Json.Serialization;
+using IniParser;
+
+namespace Elastic.Markdown.IO;
+
+public record GitConfiguration
+{
+	[JsonPropertyName("branch")]
+	public required string Branch { get; init; }
+	[JsonPropertyName("remote")]
+	public required string Remote { get; init; }
+	[JsonPropertyName("ref")]
+	public required string Ref { get; init; }
+
+	// manual read because libgit2sharp is not yet AOT ready
+	public static GitConfiguration Create(IFileSystem fileSystem)
+	{
+		// filesystem is not real so return a dummy
+		if (fileSystem is not FileSystem)
+		{
+			var fakeRef = Guid.NewGuid().ToString().Substring(0, 16);
+			return new GitConfiguration
+			{
+				Branch = $"test-{fakeRef}",
+				Remote = "elastic/docs-builder",
+				Ref = fakeRef,
+			};
+		}
+
+		var gitConfig = Git(".git/config");
+		if (!gitConfig.Exists)
+			throw new Exception($"{Paths.Root.FullName} is not a git repository.");
+
+		var head = Read(".git/HEAD").Replace("ref: ", string.Empty);
+		var gitRef = Read(".git/" + head);
+		var branch = head.Replace("refs/heads/", string.Empty);
+
+		var ini = new FileIniDataParser();
+		using var stream = gitConfig.OpenRead();
+		using var streamReader = new StreamReader(stream);
+		var config = ini.ReadData(streamReader);
+		var remoteName = config[$"branch \"{branch}\""]["remote"];
+		var remote = config[$"remote \"{remoteName}\""]["url"];
+
+		return new GitConfiguration
+		{
+			Ref = gitRef,
+			Branch = branch,
+			Remote = remote
+		};
+
+		IFileInfo Git(string path) => fileSystem.FileInfo.New(Path.Combine(Paths.Root.FullName, path));
+
+		string Read(string path) =>
+			fileSystem.File.ReadAllText(Git(path).FullName).Trim(Environment.NewLine.ToCharArray());
+	}
+}

src/Elastic.Markdown/IO/LinkReference.cs

@@ -0,0 +1,25 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+using System.Text.Json.Serialization;
+using IniParser;
+
+namespace Elastic.Markdown.IO;
+
+public record LinkReference
+{
+	[JsonPropertyName("origin")]
+	public required GitConfiguration Origin { get; init; }
+	[JsonPropertyName("links")]
+	public required string[] Links { get; init; } = [];
+
+	public static LinkReference Create(DocumentationSet set)
+	{
+		var links = set.FlatMappedFiles.Values
+			.OfType<MarkdownFile>()
+			.Select(m => m.RelativePath).ToArray();
+		return new LinkReference { Origin = set.Context.Git, Links = links };
+	}
+}

src/Elastic.Markdown/IO/Paths.cs

@@ -8,7 +8,8 @@ public static class Paths
 	private static DirectoryInfo RootDirectoryInfo()
 	{
 		var directory = new DirectoryInfo(Directory.GetCurrentDirectory());
-		while (directory != null && !directory.GetFiles("*.sln").Any())
+		while (directory != null &&
+		       (directory.GetFiles("*.sln").Length == 0 || directory.GetDirectories(".git").Length == 0))
 			directory = directory.Parent;
 		return directory ?? new DirectoryInfo(Directory.GetCurrentDirectory());
 	}

tests/Elastic.Markdown.Tests/SiteMap/LinkReferenceTests.cs

@@ -0,0 +1,36 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Markdown.IO;
+using FluentAssertions;
+using Xunit.Abstractions;
+
+namespace Elastic.Markdown.Tests.SiteMap;
+
+public class LinkReferenceTests(ITestOutputHelper output) : NavigationTestsBase(output)
+{
+	[Fact]
+	public void Create()
+	{
+		var reference = LinkReference.Create(Set);
+
+		reference.Should().NotBeNull();
+	}
+}
+
+public class GitConfigurationTests(ITestOutputHelper output) : NavigationTestsBase(output)
+{
+	[Fact]
+	public void Create()
+	{
+		var git = GitConfiguration.Create(ReadFileSystem);
+
+		git.Should().NotBeNull();
+		git!.Branch.Should().NotBeNullOrWhiteSpace();
+		// this validates we are not returning the test instance as were doing a real read
+		git.Branch.Should().NotContain(git.Ref);
+		git.Ref.Should().NotBeNullOrWhiteSpace();
+		git.Remote.Should().NotBeNullOrWhiteSpace();
+	}
+}

tests/Elastic.Markdown.Tests/SiteMap/NavigationTestsBase.cs

@@ -16,28 +16,29 @@ public class NavigationTestsBase : IAsyncLifetime
 	protected NavigationTestsBase(ITestOutputHelper output)
 	{
 		var logger = new TestLoggerFactory(output);
-		var readFs = new FileSystem(); //use real IO to read docs.
+		ReadFileSystem = new FileSystem(); //use real IO to read docs.
 		var writeFs = new MockFileSystem(new MockFileSystemOptions //use in memory mock fs to test generation
 		{
 			CurrentDirectory = Paths.Root.FullName
 		});
-		var context = new BuildContext(readFs, writeFs)
+		var context = new BuildContext(ReadFileSystem, writeFs)
 		{
 			Force = false,
 			UrlPathPrefix = null,
 			Collector = new DiagnosticsCollector(logger, [])
 		};
 
-		var set = new DocumentationSet(context);
+		Set = new DocumentationSet(context);
 
-		set.Files.Should().HaveCountGreaterThan(10);
-		Generator = new DocumentationGenerator(set, logger);
+		Set.Files.Should().HaveCountGreaterThan(10);
+		Generator = new DocumentationGenerator(Set, logger);
 
 	}
 
-	public DocumentationGenerator Generator { get; }
-
-	public ConfigurationFile Configuration { get; set; } = default!;
+	protected FileSystem ReadFileSystem { get; set; }
+	protected DocumentationSet Set { get; }
+	protected DocumentationGenerator Generator { get; }
+	protected ConfigurationFile Configuration { get; set; } = default!;
 
 	public async Task InitializeAsync()
 	{