diff --git a/.github/check-license-headers.sh b/.github/check-license-headers.sh index d80c95a..e872f18 100755 --- a/.github/check-license-headers.sh +++ b/.github/check-license-headers.sh @@ -42,4 +42,4 @@ if [[ $nErrors -eq 0 ]]; then exit 0 else exit 1 -fi \ No newline at end of file +fi diff --git a/build/Program.cs b/build/Program.cs index 42fd81b..c882713 100644 --- a/build/Program.cs +++ b/build/Program.cs @@ -25,6 +25,12 @@ dotnet publish {source} -c Release -o .artifacts/publish \ --self-contained true /p:PublishTrimmed=true /p:PublishSingleFile=false /p:PublishAot=true """; + + var generatorSource = "src/docs-generator/docs-generator.csproj"; + await $""" + dotnet publish {generatorSource} -c Release -o .artifacts/publish \ + --self-contained true /p:PublishTrimmed=true /p:PublishSingleFile=false /p:PublishAot=true + """; }); // this is manual for now and quite hacky. diff --git a/docs-builder.sln b/docs-builder.sln index b27cb67..50e4344 100644 --- a/docs-builder.sln +++ b/docs-builder.sln @@ -30,6 +30,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{67B576EE EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Markdown.Tests", "tests\Elastic.Markdown.Tests\Elastic.Markdown.Tests.csproj", "{B27C5107-128B-465A-B8F8-8985399E4CFB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-generator", "src\docs-generator\docs-generator.csproj", "{61904527-9753-4379-B546-56B6A29073AC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -60,10 +62,15 @@ Global {B27C5107-128B-465A-B8F8-8985399E4CFB}.Debug|Any CPU.Build.0 = Debug|Any CPU {B27C5107-128B-465A-B8F8-8985399E4CFB}.Release|Any CPU.ActiveCfg = Release|Any CPU {B27C5107-128B-465A-B8F8-8985399E4CFB}.Release|Any CPU.Build.0 = Release|Any CPU + {61904527-9753-4379-B546-56B6A29073AC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {61904527-9753-4379-B546-56B6A29073AC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {61904527-9753-4379-B546-56B6A29073AC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {61904527-9753-4379-B546-56B6A29073AC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} {01F05AD0-E0E0-401F-A7EC-905928E1E9F0} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} {B27C5107-128B-465A-B8F8-8985399E4CFB} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5} + {61904527-9753-4379-B546-56B6A29073AC} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} EndGlobalSection EndGlobal diff --git a/src/Elastic.Markdown/IO/MarkdownFile.cs b/src/Elastic.Markdown/IO/MarkdownFile.cs index 43e04ce..ece680f 100644 --- a/src/Elastic.Markdown/IO/MarkdownFile.cs +++ b/src/Elastic.Markdown/IO/MarkdownFile.cs @@ -85,7 +85,7 @@ private void ReadDocumentInstructions(MarkdownDocument document) } var contents = document - .Where(block => block is HeadingBlock { Level: 2 }) + .Where(block => block is HeadingBlock { Level: >= 2 }) .Cast() .Select(h => h.Inline?.FirstChild?.ToString()) .Where(title => !string.IsNullOrWhiteSpace(title)) diff --git a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs index e01ba40..524cfa8 100644 --- a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs +++ b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs @@ -68,7 +68,7 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice) includeFrom = context.Parser.SourcePath.FullName; var anchors = url.Split('#'); - var anchor = anchors.Length > 1 ? anchors[1] : null; + var anchor = anchors.Length > 1 ? anchors[1].Trim() : null; url = anchors[0]; if (!string.IsNullOrWhiteSpace(url)) @@ -87,7 +87,7 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice) if (link.FirstChild == null || !string.IsNullOrEmpty(anchor)) { var file = string.IsNullOrWhiteSpace(url) ? context.Path - : context.Build.ReadFileSystem.FileInfo.New(Path.Combine(context.Build.SourcePath.FullName, url)); + : context.Build.ReadFileSystem.FileInfo.New(Path.Combine(context.Build.SourcePath.FullName, url.TrimStart('/'))); var markdown = context.GetMarkdownFile?.Invoke(file); var title = markdown?.Title; diff --git a/src/docs-builder/Http/DocumentationWebHost.cs b/src/docs-builder/Http/DocumentationWebHost.cs index df3a33b..c9537ea 100644 --- a/src/docs-builder/Http/DocumentationWebHost.cs +++ b/src/docs-builder/Http/DocumentationWebHost.cs @@ -26,8 +26,7 @@ public class DocumentationWebHost public DocumentationWebHost(string? path, ILoggerFactory logger, IFileSystem fileSystem) { var builder = WebApplication.CreateSlimBuilder(); - var sourcePath = path != null ? fileSystem.DirectoryInfo.New(path) : null; - var context = new BuildContext(fileSystem) + var context = new BuildContext(fileSystem, fileSystem, path, null) { Collector = new ConsoleDiagnosticsCollector(logger) }; @@ -36,7 +35,7 @@ public DocumentationWebHost(string? path, ILoggerFactory logger, IFileSystem fil s.FolderToMonitor = context.SourcePath.FullName; s.ClientFileExtensions = ".md,.yml"; }); - builder.Services.AddSingleton(_ => new ReloadableGeneratorState(sourcePath, null, context, logger)); + builder.Services.AddSingleton(_ => new ReloadableGeneratorState(context.SourcePath, null, context, logger)); builder.Services.AddHostedService(); builder.Services.AddSingleton(logger); builder.Logging.SetMinimumLevel(LogLevel.Warning); diff --git a/src/docs-generator/Domain/Determinism.cs b/src/docs-generator/Domain/Determinism.cs new file mode 100644 index 0000000..1445614 --- /dev/null +++ b/src/docs-generator/Domain/Determinism.cs @@ -0,0 +1,34 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Bogus; + +namespace Documentation.Generator.Domain; + +public record Determinism +{ + public Determinism(int? seedFileSystem, int? seedContent) + { + var randomizer = new Randomizer(); + SeedFileSystem = seedFileSystem ?? randomizer.Int(1, int.MaxValue); + SeedContent = seedContent ?? randomizer.Int(1, int.MaxValue); + FileSystem = new Randomizer(SeedFileSystem); + Contents = new Randomizer(SeedContent); + + SectionProbability = Contents.Float(0.001f, Contents.Float(0.1f)); + FileProbability = FileSystem.Float(0.001f, Contents.Float(0.1f)); + } + + public int SeedFileSystem { get; } + public int SeedContent { get; } + + + public Randomizer FileSystem { get; } + public Randomizer Contents { get; } + + public float SectionProbability { get; } + public float FileProbability { get; } + + public static Determinism Random { get; set; } = new(null, null); +} diff --git a/src/docs-generator/Domain/Generators.cs b/src/docs-generator/Domain/Generators.cs new file mode 100644 index 0000000..5b512bb --- /dev/null +++ b/src/docs-generator/Domain/Generators.cs @@ -0,0 +1,50 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Slugify; +using Soenneker.Utils.AutoBogus; + +namespace Documentation.Generator.Domain; + +public static class Generators +{ + public static AutoFaker FolderName { get; } = new(); + public static AutoFaker
Section { get; } = new(); + public static AutoFaker File { get; } = new(); + public static SlugHelper Slug { get; } = new(); + + static Generators() + { + FolderName + .RuleFor(p => p.Folder, f => f.Lorem.Slug(1)); + + Section + .RuleFor(p => p.Paragraphs, f => f.Lorem.Paragraphs(f.Random.Number(1, 10))) + .RuleFor(p => p.Level, f => f.Random.Number(2, 4)); + + File + .Ignore(p => p.Links) + .Ignore(p => p.Directory) + .RuleFor(p => p.FileName, f => f.System.FileName("md")) + .RuleFor(p => p.IncludeInUpdate, f => f.Random.Float() <= Determinism.Random.FileProbability) + .RuleFor(p => p.Sections, f => Section.Generate(Determinism.Random.Contents.Number(1, 12)).ToArray()); + } + + public static IEnumerable CreateSubPaths(string parent, int maxDepth, int currentDepth) + { + yield return parent; + if (currentDepth == maxDepth) yield break; + var subFolders = FolderName.Generate(Determinism.Random.FileSystem.Number(0, 4)); + foreach (var subFolder in subFolders) + { + var path = $"{parent}/{subFolder.Folder}"; + yield return path; + var subPaths = CreateSubPaths(path, maxDepth, currentDepth + 1); + foreach (var p in subPaths) + yield return p; + } + } + + public static string[] FolderNames { get; set; } = []; +} diff --git a/src/docs-generator/Domain/MarkdownFile.cs b/src/docs-generator/Domain/MarkdownFile.cs new file mode 100644 index 0000000..0f732f3 --- /dev/null +++ b/src/docs-generator/Domain/MarkdownFile.cs @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Slugify; +using Soenneker.Utils.AutoBogus; + +namespace Documentation.Generator.Domain; + +public record MarkdownFile +{ + public required string FileName { get; init; } + public required Section[] Sections { get; init; } + public required string Title { get; init; } + + public string RelativePath => $"{Directory}/{FileName}"; + + public required bool IncludeInUpdate { get; init; } = true; + + public string Directory { get; set; } = string.Empty; + public List Links { get; set; } = []; + + public void RewriteLinksIntoSections() + { + var linksLength = Links.Count; + var sectionsLength = Sections.Length; + for (var i = 0; i < linksLength; i++) + { + var link = Links[i]; + var section = Sections[Determinism.Random.Contents.Number(0, sectionsLength - 1)]; + var words = section.Paragraphs.Split(" "); + var w = Determinism.Random.Contents.Number(0, words.Length - 1); + var word = words[w]; + words[w] = $"[{word}](/{link})"; + section.Paragraphs = string.Join(" ", words); + } + } + + + public string GetRandomLink() + { + var sectionLink = Determinism.Random.Contents.Bool(0.8f); + if (!sectionLink) return RelativePath; + var section = Sections[Determinism.Random.Contents.Number(0, Sections.Length - 1)]; + return $"{RelativePath}#{Generators.Slug.GenerateSlug(section.Header)}"; + + } +} + diff --git a/src/docs-generator/Domain/Path.cs b/src/docs-generator/Domain/Path.cs new file mode 100644 index 0000000..7211f9a --- /dev/null +++ b/src/docs-generator/Domain/Path.cs @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Documentation.Generator.Domain; + +public static class Paths +{ + private static DirectoryInfo RootDirectoryInfo() + { + var directory = new DirectoryInfo(Directory.GetCurrentDirectory()); + while (directory != null && + (directory.GetFiles("*.sln").Length == 0 || directory.GetDirectories(".git").Length == 0)) + directory = directory.Parent; + return directory ?? new DirectoryInfo(Directory.GetCurrentDirectory()); + } + + public static readonly DirectoryInfo Root = RootDirectoryInfo(); +} + +public record FolderName +{ + public required string Folder { get; init; } +} + +public record Folder +{ + public required string Path { get; init; } + public required MarkdownFile[] Files { get; init; } +} diff --git a/src/docs-generator/Domain/Section.cs b/src/docs-generator/Domain/Section.cs new file mode 100644 index 0000000..18558ca --- /dev/null +++ b/src/docs-generator/Domain/Section.cs @@ -0,0 +1,15 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Documentation.Generator.Domain; + +public record Section +{ + public required string Header { get; init; } + + public required int Level { get; init; } + + public required string Paragraphs { get; set; } + +} diff --git a/src/docs-generator/Program.cs b/src/docs-generator/Program.cs new file mode 100644 index 0000000..248565a --- /dev/null +++ b/src/docs-generator/Program.cs @@ -0,0 +1,156 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +// ReSharper disable RedundantLambdaParameterType + +using System.Security.Cryptography.X509Certificates; +using Bogus; +using ConsoleAppFramework; +using Documentation.Generator.Domain; +using Soenneker.Utils.AutoBogus; +using Soenneker.Utils.AutoBogus.Config; + +await ConsoleApp.RunAsync(args, async Task ( + int? seedFileSystem = null, + int? seedContent = null, + string? output = null, + bool? clear = null +) => +{ + var cleanOutputDirectory = clear ?? true; + var outputFolder = !string.IsNullOrWhiteSpace(output) + ? new DirectoryInfo(output) + : new DirectoryInfo(Path.Combine(Paths.Root.FullName, ".artifacts/docs/markdown")); + var stateFile = new FileInfo(Path.Combine(outputFolder.FullName, "generator.state")); + + LoadStateFromFile(stateFile, clear, ref seedFileSystem, ref cleanOutputDirectory); + + Determinism.Random = new Determinism(seedFileSystem, seedContent); + + Console.WriteLine($"Running generator with file seed: {Determinism.Random.SeedFileSystem} and content seed: {Determinism.Random.SeedContent}"); + + Generators.FolderName.UseSeed(Determinism.Random.SeedFileSystem); + Generators.File.UseSeed(Determinism.Random.SeedFileSystem); + Generators.Section.UseSeed(Determinism.Random.SeedContent); + + Generators.FolderNames = Generators.FolderName + .Generate(Determinism.Random.FileSystem.Number(3, 15)) + .SelectMany(p => Generators.CreateSubPaths(p.Folder, Determinism.Random.FileSystem.Number(0, 3), 0)) + .Distinct() + .ToArray(); + + var folders = new List(); + foreach (var folder in Generators.FolderNames) + { + var mdFolder = new Folder + { + Path = folder, + Files = Generators.File + .Generate(Determinism.Random.FileSystem.Number(1, 4)) + .Select(f => + { + f.Directory = folder; + return f; + }) + .ToArray() + }; + folders.Add(mdFolder); + } + + var files = folders.SelectMany(f => f.Files).ToArray(); + foreach (var folder in folders) + { + foreach (var file in folder.Files) + { + var length = Determinism.Random.Contents.Number(1, 10); + file.Links = Enumerable.Range(0, length) + .Select(i => files[Determinism.Random.Contents.Number(0, files.Length - 1)]) + .Select(f => f.GetRandomLink()) + .ToList(); + file.RewriteLinksIntoSections(); + } + } + + Console.WriteLine($"Writing to {outputFolder.FullName}"); + + if (outputFolder.Exists && cleanOutputDirectory) + Directory.Delete(outputFolder.FullName, true); + + var updateFiles = files + .Where(f => cleanOutputDirectory || f.IncludeInUpdate) + .ToArray(); + foreach (var file in updateFiles) + { + var directory = Path.Combine(outputFolder.FullName, file.Directory); + Console.WriteLine($"Writing to {directory}"); + Directory.CreateDirectory(directory); + + WriteMarkdownFile(outputFolder, file); + } + + var name = $"random-docset-{seedContent}-{seedFileSystem}"; + WriteIndexMarkdownFile(name, outputFolder); + + var docset = Path.Combine(outputFolder.FullName, "docset.yml"); + File.WriteAllText(docset, $"project: {name}{Environment.NewLine}"); + File.AppendAllText(docset, $"toc:{Environment.NewLine}"); + foreach (var folder in folders) + File.AppendAllText(docset, $" - folder: {folder.Path}{Environment.NewLine}"); + + File.AppendAllText(docset, $" - file: index.md{Environment.NewLine}"); + File.AppendAllText(docset, Environment.NewLine); + + File.WriteAllText(stateFile.FullName, $"{Determinism.Random.SeedFileSystem}|{Determinism.Random.SeedContent}"); + + return await Task.FromResult(0); +}); + +void WriteIndexMarkdownFile(string name, DirectoryInfo directoryInfo) +{ + var filePath = Path.Combine(directoryInfo.FullName, "index.md"); + File.WriteAllText(filePath, + $""" + --- + title: {name} Documentation Set + --- + + """); + File.AppendAllText(filePath, "This docset is generated using docs-generator"); + File.AppendAllText(filePath, Environment.NewLine); +} + +void WriteMarkdownFile(DirectoryInfo directoryInfo, MarkdownFile markdownFile) +{ + var filePath = Path.Combine(directoryInfo.FullName, markdownFile.RelativePath); + File.WriteAllText(filePath, + $""" + --- + title: {markdownFile.Title} + --- + + """); + foreach (var section in markdownFile.Sections) + { + File.AppendAllText(filePath, Environment.NewLine); + var header = new string('#', section.Level); + File.AppendAllText(filePath, $"{header} {section.Header}{Environment.NewLine}"); + File.AppendAllText(filePath, Environment.NewLine); + + File.AppendAllText(filePath, section.Paragraphs); + File.AppendAllText(filePath, Environment.NewLine); + } +} + +void LoadStateFromFile(FileInfo fileInfo, bool? clear, ref int? seedFs, ref bool cleanOutput) +{ + if (!fileInfo.Exists) return; + var state = File.ReadAllText(fileInfo.FullName).Split("|"); + if (state.Length != 2) return; + seedFs ??= int.TryParse(state[0], out var seed) ? seed : seedFs; + Console.WriteLine($"Seeding with {seedFs} from previous run {fileInfo.FullName}"); + cleanOutput = clear ?? false; +} diff --git a/src/docs-generator/docs-generator.csproj b/src/docs-generator/docs-generator.csproj new file mode 100644 index 0000000..8adb93b --- /dev/null +++ b/src/docs-generator/docs-generator.csproj @@ -0,0 +1,32 @@ + + + + net8.0 + Exe + docs-generator + Documentation.Generator + true + true + + true + true + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + +