From f86d89e6a4ef6251e46327f22866c5bc788eb6ba Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 12 Nov 2024 18:42:04 +0100 Subject: [PATCH] Navigation powered by docset.yml (#61) --- docs/source/docset.yml | 12 +-- src/Elastic.Markdown/IO/ConfigurationFile.cs | 74 ++++++-------- src/Elastic.Markdown/IO/DocumentationFile.cs | 1 + .../IO/DocumentationFolder.cs | 99 ++++++++++--------- src/Elastic.Markdown/IO/DocumentationSet.cs | 15 +-- src/Elastic.Markdown/IO/ITocItem.cs | 11 +++ src/Elastic.Markdown/Slices/HtmlWriter.cs | 2 +- src/Elastic.Markdown/Slices/Index.cshtml | 2 +- src/Elastic.Markdown/Slices/_Layout.cshtml | 2 +- src/Elastic.Markdown/Slices/_ViewModels.cs | 4 +- src/docs-builder/Http/DocumentationWebHost.cs | 9 +- .../Http/ReloadGeneratorService.cs | 24 ++--- .../SiteMap/NavigationTests.cs | 4 +- 13 files changed, 130 insertions(+), 129 deletions(-) create mode 100644 src/Elastic.Markdown/IO/ITocItem.cs diff --git a/docs/source/docset.yml b/docs/source/docset.yml index 57ac05b..cf78b27 100644 --- a/docs/source/docset.yml +++ b/docs/source/docset.yml @@ -3,7 +3,6 @@ exclude: - '_*.md' toc: - file: index.md - - folder: markup - folder: elastic children: - file: index.md @@ -13,6 +12,10 @@ toc: - folder: search-labs children: - file: index.md + - file: install.md + children: + - file: install/cloud.md + - file: install/docker.md - file: chat.md children: - file: chat/req.md @@ -21,12 +24,9 @@ toc: children: - file: search/req.md - file: search/setup.md - - file: install.md - children: - - file: install/cloud.md - - file: install/docker.md + - folder: markup - folder: nested children: - folder: content - file: index.md - - folder: versioning \ No newline at end of file + - folder: versioning diff --git a/src/Elastic.Markdown/IO/ConfigurationFile.cs b/src/Elastic.Markdown/IO/ConfigurationFile.cs index 4a05e18..aa95aa8 100644 --- a/src/Elastic.Markdown/IO/ConfigurationFile.cs +++ b/src/Elastic.Markdown/IO/ConfigurationFile.cs @@ -21,7 +21,7 @@ public class ConfigurationFile : DocumentationFile public IReadOnlyCollection TableOfContents { get; } = []; public HashSet Files { get; } = new(StringComparer.OrdinalIgnoreCase); - public HashSet Folders { get; } = new(StringComparer.OrdinalIgnoreCase); + public HashSet ImplicitFolders { get; } = new(StringComparer.OrdinalIgnoreCase); public Glob[] Globs { get; } = []; public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildContext context) @@ -72,7 +72,7 @@ public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildCon break; } } - Globs = Folders.Select(f=> Glob.Parse($"{f}/*.md")).ToArray(); + Globs = ImplicitFolders.Select(f=> Glob.Parse($"{f}/*.md")).ToArray(); } private List ReadChildren(KeyValuePair entry, string parentPath) @@ -99,7 +99,8 @@ private List ReadChildren(KeyValuePair entry, stri { string? file = null; string? folder = null; - var found = false; + var fileFound = false; + var folderFound = false; IReadOnlyCollection? children = null; foreach (var entry in tocEntry.Children) { @@ -107,10 +108,10 @@ private List ReadChildren(KeyValuePair entry, stri switch (key) { case "file": - file = ReadFile(entry, parentPath); + file = ReadFile(entry, parentPath, out fileFound); break; case "folder": - folder = ReadString(entry); + folder = ReadFolder(entry, parentPath, out folderFound); parentPath += $"/{folder}"; break; case "children": @@ -120,30 +121,47 @@ private List ReadChildren(KeyValuePair entry, stri } if (file is not null) - return new TocFile(file, found, children ?? []); + return new TocFile($"{parentPath}/{file}".TrimStart('/'), fileFound, children ?? []); if (folder is not null) { if (children is null) - Folders.Add(parentPath.TrimStart('/')); + ImplicitFolders.Add(parentPath.TrimStart('/')); - return new TocFolder(folder, children ?? []); + return new TocFolder($"{parentPath}".TrimStart('/'), folderFound, children ?? []); } return null; } - private string? ReadFile(KeyValuePair entry, string parentPath) + private string? ReadFolder(KeyValuePair entry, string parentPath, out bool found) { - var file = ReadString(entry); - if (file is not null) + found = false; + var folder = ReadString(entry); + if (folder is not null) { - var path = Path.Combine(_rootPath.FullName, parentPath.TrimStart('/'), file); - if (!_context.ReadFileSystem.FileInfo.New(path).Exists) - EmitError($"File '{path}' does not exist", entry.Key); + var path = Path.Combine(_rootPath.FullName, parentPath.TrimStart('/'), folder); + if (!_context.ReadFileSystem.DirectoryInfo.New(path).Exists) + EmitError($"Directory '{path}' does not exist", entry.Key); + else + found = true; } + return folder; + } + private string? ReadFile(KeyValuePair entry, string parentPath, out bool found) + { + found = false; + var file = ReadString(entry); + if (file is null) return null; + + var path = Path.Combine(_rootPath.FullName, parentPath.TrimStart('/'), file); + if (!_context.ReadFileSystem.FileInfo.New(path).Exists) + EmitError($"File '{path}' does not exist", entry.Key); + else + found = true; Files.Add((parentPath + "/" + file).TrimStart('/')); + return file; } @@ -210,31 +228,3 @@ private void EmitWarning(string message, Mark? start = null, Mark? end = null, i } } -public interface ITocItem; - -public record TocFile(string Path, bool Found, IReadOnlyCollection Children) : ITocItem; - -public record TocFolder(string Path, IReadOnlyCollection Children) : ITocItem; - - -/* -exclude: - - notes.md - - '**ignore.md' -toc: -- file: index.md -- file: config.md -- file: search.md -children: -- file: search-part2.md -- folder: search -- folder: my-folder1 -exclude: -- '_*.md' -- folder: my-folder2 -children: -- file: subpath/file.md -- file: file.md -- pattern: *.md -- folder: sub/folder -*/ diff --git a/src/Elastic.Markdown/IO/DocumentationFile.cs b/src/Elastic.Markdown/IO/DocumentationFile.cs index 7486bb9..beff350 100644 --- a/src/Elastic.Markdown/IO/DocumentationFile.cs +++ b/src/Elastic.Markdown/IO/DocumentationFile.cs @@ -9,6 +9,7 @@ public abstract class DocumentationFile(IFileInfo sourceFile, IDirectoryInfo roo { public IFileInfo SourceFile { get; } = sourceFile; public string RelativePath { get; } = Path.GetRelativePath(rootPath.FullName, sourceFile.FullName); + public string RelativeFolder { get; } = Path.GetRelativePath(rootPath.FullName, sourceFile.Directory!.FullName); public FileInfo OutputFile(IDirectoryInfo outputPath) => new(Path.Combine(outputPath.FullName, RelativePath.Replace(".md", ".html"))); diff --git a/src/Elastic.Markdown/IO/DocumentationFolder.cs b/src/Elastic.Markdown/IO/DocumentationFolder.cs index a5e3b13..a82e375 100644 --- a/src/Elastic.Markdown/IO/DocumentationFolder.cs +++ b/src/Elastic.Markdown/IO/DocumentationFolder.cs @@ -1,6 +1,7 @@ // Licensed to Elasticsearch B.V under one or more agreements. // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information + using Markdig.Helpers; namespace Elastic.Markdown.IO; @@ -8,75 +9,77 @@ namespace Elastic.Markdown.IO; public class DocumentationFolder { public MarkdownFile? Index { get; } - private MarkdownFile[] Files { get; } - private DocumentationFolder[] Nested { get; } - public OrderedList FilesInOrder { get; private set; } - public OrderedList GroupsInOrder { get; private set; } + public List FilesInOrder { get; } = new(); + public List GroupsInOrder { get; } = new(); + + private HashSet OwnFiles { get; } + public int Level { get; } - public string? FolderName { get; } - public DocumentationFolder(Dictionary markdownFiles, int level, string folderName) + public DocumentationFolder(IReadOnlyCollection toc, + IDictionary lookup, + IDictionary folderLookup, + int level = 0, + MarkdownFile? index = null) { Level = level; - FolderName = folderName; - - var files = markdownFiles - .Where(k => k.Key.EndsWith(".md")).SelectMany(g => g.Value) - .Where(file => file.ParentFolders.Count == level) - .ToArray(); - - - Files = files - .Where(file => file.FileName != "index.md") - .ToArray(); - - FilesInOrder = new OrderedList(Files); - - Index = files.FirstOrDefault(f => f.FileName == "index.md"); + Index = index; - var newLevel = level + 1; - var groups = new List(); - foreach (var kv in markdownFiles.Where(kv=> !kv.Key.EndsWith(".md"))) + foreach (var tocItem in toc) { - var folder = kv.Key; - var folderFiles = kv.Value - .Where(file => file.ParentFolders.Count > level) - .Where(file => file.ParentFolders[level] == folder).ToArray(); - var mapped = folderFiles - .GroupBy(file => - { - var path = file.ParentFolders.Count > newLevel ? file.ParentFolders[newLevel] : file.FileName; - return path; - }) - .ToDictionary(k => k.Key, v => v.ToArray()); - var documentationGroup = new DocumentationFolder(mapped, newLevel, folder); - groups.Add(documentationGroup); + if (tocItem is TocFile file) + { + if (!lookup.TryGetValue(file.Path, out var d) || d is not MarkdownFile md) + continue; + if (file.Children.Count > 0 && d is MarkdownFile virtualIndex) + { + var group = new DocumentationFolder(file.Children, lookup, folderLookup, level + 1, virtualIndex); + GroupsInOrder.Add(group); + continue; + } + + FilesInOrder.Add(md); + if (file.Path.EndsWith("index.md") && d is MarkdownFile i) + Index ??= i; + } + else if (tocItem is TocFolder folder) + { + var children = folder.Children; + if (children.Count == 0 + && folderLookup.TryGetValue(folder.Path, out var documentationFiles)) + { + children = documentationFiles + .Select(d => new TocFile(d.RelativePath, true, [])) + .ToArray(); + } + + var group = new DocumentationFolder(children, lookup, folderLookup, level + 1); + GroupsInOrder.Add(group); + } } - Nested = groups.ToArray(); - GroupsInOrder = new OrderedList(Nested); + + Index ??= FilesInOrder.FirstOrDefault(); + if (Index != null) + FilesInOrder = FilesInOrder.Except(new[] { Index }).ToList(); + OwnFiles = [..FilesInOrder]; } public bool HoldsCurrent(MarkdownFile current) => - Index == current || Files.Contains(current) || Nested.Any(n => n.HoldsCurrent(current)); + Index == current || OwnFiles.Contains(current) || GroupsInOrder.Any(n => n.HoldsCurrent(current)); private bool _resolved; + public async Task Resolve(Cancel ctx = default) { if (_resolved) return; - await Parallel.ForEachAsync(Files, ctx, async (file, token) => await file.ParseAsync(token)); - await Parallel.ForEachAsync(Nested, ctx, async (group, token) => await group.Resolve(token)); + await Parallel.ForEachAsync(FilesInOrder, ctx, async (file, token) => await file.ParseAsync(token)); + await Parallel.ForEachAsync(GroupsInOrder, ctx, async (group, token) => await group.Resolve(token)); await (Index?.ParseAsync(ctx) ?? Task.CompletedTask); - var fileList = new OrderedList(); - var groupList = new OrderedList(); - - - FilesInOrder = fileList; - GroupsInOrder = groupList; _resolved = true; } } diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 4f66ed2..978094d 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -47,21 +47,14 @@ public DocumentationSet(IDirectoryInfo? sourcePath, IDirectoryInfo? outputPath, .ToList(); - LastWrite = Files.Max(f => f.SourceFile.LastWriteTimeUtc); FlatMappedFiles = Files.ToDictionary(file => file.RelativePath, file => file); + var folderFiles = Files + .GroupBy(file => file.RelativeFolder) + .ToDictionary(g=>g.Key, g=>g.ToArray()); - var markdownFiles = Files.OfType() - .Where(file => !file.RelativePath.StartsWith("_")) - .GroupBy(file => - { - var path = file.ParentFolders.Count >= 1 ? file.ParentFolders[0] : file.FileName; - return path; - }) - .ToDictionary(k => k.Key, v => v.ToArray()); - - Tree = new DocumentationFolder(markdownFiles, 0, ""); + Tree = new DocumentationFolder(Configuration.TableOfContents, FlatMappedFiles, folderFiles); } private DocumentationFile CreateMarkDownFile(IFileInfo file, BuildContext context) diff --git a/src/Elastic.Markdown/IO/ITocItem.cs b/src/Elastic.Markdown/IO/ITocItem.cs new file mode 100644 index 0000000..cbc0aa9 --- /dev/null +++ b/src/Elastic.Markdown/IO/ITocItem.cs @@ -0,0 +1,11 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Markdown.IO; + +public interface ITocItem; + +public record TocFile(string Path, bool Found, IReadOnlyCollection Children) : ITocItem; + +public record TocFolder(string Path, bool Found, IReadOnlyCollection Children) : ITocItem; diff --git a/src/Elastic.Markdown/Slices/HtmlWriter.cs b/src/Elastic.Markdown/Slices/HtmlWriter.cs index 8abb2d8..8f49719 100644 --- a/src/Elastic.Markdown/Slices/HtmlWriter.cs +++ b/src/Elastic.Markdown/Slices/HtmlWriter.cs @@ -50,7 +50,7 @@ public async Task RenderLayout(MarkdownFile markdown, Cancel ctx = defau PageTocItems = markdown.TableOfContents, Tree = DocumentationSet.Tree, CurrentDocument = markdown, - Navigation = navigationHtml, + NavigationHtml = navigationHtml, UrlPathPrefix = markdown.UrlPathPrefix }); return await slice.RenderAsync(cancellationToken: ctx); diff --git a/src/Elastic.Markdown/Slices/Index.cshtml b/src/Elastic.Markdown/Slices/Index.cshtml index 214a28d..f136c1d 100644 --- a/src/Elastic.Markdown/Slices/Index.cshtml +++ b/src/Elastic.Markdown/Slices/Index.cshtml @@ -7,7 +7,7 @@ PageTocItems = Model.PageTocItems, Tree = Model.Tree, CurrentDocument = Model.CurrentDocument, - Navigation = Model.Navigation, + NavigationHtml = Model.NavigationHtml, UrlPathPrefix = Model.UrlPathPrefix, }; } diff --git a/src/Elastic.Markdown/Slices/_Layout.cshtml b/src/Elastic.Markdown/Slices/_Layout.cshtml index dea1818..2b42c67 100644 --- a/src/Elastic.Markdown/Slices/_Layout.cshtml +++ b/src/Elastic.Markdown/Slices/_Layout.cshtml @@ -16,7 +16,7 @@
- @(new HtmlString(Model.Navigation)) + @(new HtmlString(Model.NavigationHtml)) @*@(await RenderPartialAsync(Elastic.Markdown.Slices.Layout._TocTree.Create(Model)))*@ @(await RenderPartialAsync(_TableOfContents.Create(Model)))
diff --git a/src/Elastic.Markdown/Slices/_ViewModels.cs b/src/Elastic.Markdown/Slices/_ViewModels.cs index 8b66369..59fb58b 100644 --- a/src/Elastic.Markdown/Slices/_ViewModels.cs +++ b/src/Elastic.Markdown/Slices/_ViewModels.cs @@ -12,7 +12,7 @@ public class IndexViewModel public required DocumentationFolder Tree { get; init; } public required IReadOnlyCollection PageTocItems { get; init; } public required MarkdownFile CurrentDocument { get; init; } - public required string Navigation { get; init; } + public required string NavigationHtml { get; init; } public required string? UrlPathPrefix { get; init; } } @@ -22,7 +22,7 @@ public class LayoutViewModel public required IReadOnlyCollection PageTocItems { get; init; } public required DocumentationFolder Tree { get; init; } public required MarkdownFile CurrentDocument { get; init; } - public required string Navigation { get; set; } + public required string NavigationHtml { get; set; } public required string? UrlPathPrefix { get; set; } diff --git a/src/docs-builder/Http/DocumentationWebHost.cs b/src/docs-builder/Http/DocumentationWebHost.cs index 75fb0bf..946493b 100644 --- a/src/docs-builder/Http/DocumentationWebHost.cs +++ b/src/docs-builder/Http/DocumentationWebHost.cs @@ -35,6 +35,7 @@ public DocumentationWebHost(string? path, ILoggerFactory logger, IFileSystem fil builder.Services.AddSingleton(_ => new ReloadableGeneratorState(sourcePath, null, context, logger)); builder.Services.AddHostedService(); builder.Services.AddSingleton(logger); + builder.Logging.SetMinimumLevel(LogLevel.Warning); _webApplication = builder.Build(); SetUpRoutes(); @@ -51,11 +52,11 @@ private void SetUpRoutes() }); _webApplication.UseRouting(); - _webApplication.MapGet("/", async (ReloadableGeneratorState holder, Cancel ctx) => - await ServeDocumentationFile(holder, "index.md", ctx)); + _webApplication.MapGet("/", (ReloadableGeneratorState holder, Cancel ctx) => + ServeDocumentationFile(holder, "index.md", ctx)); - _webApplication.MapGet("{**slug}", async (string slug, ReloadableGeneratorState holder, Cancel ctx) => - await ServeDocumentationFile(holder, slug, ctx)); + _webApplication.MapGet("{**slug}", (string slug, ReloadableGeneratorState holder, Cancel ctx) => + ServeDocumentationFile(holder, slug, ctx)); } private static async Task ServeDocumentationFile(ReloadableGeneratorState holder, string slug, Cancel ctx) diff --git a/src/docs-builder/Http/ReloadGeneratorService.cs b/src/docs-builder/Http/ReloadGeneratorService.cs index 85f3c34..118681f 100644 --- a/src/docs-builder/Http/ReloadGeneratorService.cs +++ b/src/docs-builder/Http/ReloadGeneratorService.cs @@ -21,15 +21,16 @@ public async Task StartAsync(Cancel ctx) { await ReloadableGenerator.ReloadAsync(ctx); - var watcher = new FileSystemWatcher(ReloadableGenerator.Generator.DocumentationSet.SourcePath.FullName); - - watcher.NotifyFilter = NotifyFilters.Attributes - | NotifyFilters.CreationTime - | NotifyFilters.DirectoryName - | NotifyFilters.FileName - | NotifyFilters.LastWrite - | NotifyFilters.Security - | NotifyFilters.Size; + var watcher = new FileSystemWatcher(ReloadableGenerator.Generator.DocumentationSet.SourcePath.FullName) + { + NotifyFilter = NotifyFilters.Attributes + | NotifyFilters.CreationTime + | NotifyFilters.DirectoryName + | NotifyFilters.FileName + | NotifyFilters.LastWrite + | NotifyFilters.Security + | NotifyFilters.Size + }; watcher.Changed += OnChanged; watcher.Created += OnCreated; @@ -37,7 +38,8 @@ public async Task StartAsync(Cancel ctx) watcher.Renamed += OnRenamed; watcher.Error += OnError; - watcher.Filter = "*.md"; + watcher.Filters.Add("*.md"); + watcher.Filters.Add("docset.yml"); watcher.IncludeSubdirectories = true; watcher.EnableRaisingEvents = true; _watcher = watcher; @@ -62,7 +64,7 @@ private void OnChanged(object sender, FileSystemEventArgs e) if (e.ChangeType != WatcherChangeTypes.Changed) return; - if (e.FullPath.EndsWith("index.md")) + if (e.FullPath.EndsWith("docset.yml")) Reload(); Logger.LogInformation($"Changed: {e.FullPath}"); diff --git a/tests/Elastic.Markdown.Tests/SiteMap/NavigationTests.cs b/tests/Elastic.Markdown.Tests/SiteMap/NavigationTests.cs index fe30d3a..9c98848 100644 --- a/tests/Elastic.Markdown.Tests/SiteMap/NavigationTests.cs +++ b/tests/Elastic.Markdown.Tests/SiteMap/NavigationTests.cs @@ -16,8 +16,8 @@ public void ParsesATableOfContents() => [Fact] public void ParsesNestedFoldersAndPrefixesPaths() { - Configuration.Folders.Should().NotBeNullOrEmpty(); - Configuration.Folders.Should() + Configuration.ImplicitFolders.Should().NotBeNullOrEmpty(); + Configuration.ImplicitFolders.Should() .Contain("markup") .And.Contain("elastic/observability"); }