Add infrastructure and diagnostics for unsupported directives

This PR also ensures tests log output correctly.

src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs

@@ -29,6 +29,8 @@ public void TryComplete(Exception? exception = null)
 		_ctxSource.Cancel();
 	}
 
+	public ValueTask<bool> WaitToWrite() => _channel.Writer.WaitToWriteAsync();
+
 	public void Write(Diagnostic diagnostic)
 	{
 		var written = _channel.Writer.TryWrite(diagnostic);
@@ -84,10 +86,18 @@ public class DiagnosticsCollector(ILoggerFactory loggerFactory, IReadOnlyCollect
 
 	public async Task StartAsync(Cancel ctx)
 	{
+		await Channel.WaitToWrite();
 		while (!Channel.CancellationToken.IsCancellationRequested)
 		{
-			while (await Channel.Reader.WaitToReadAsync(Channel.CancellationToken))
-				Drain();
+			try
+			{
+				while (await Channel.Reader.WaitToReadAsync(Channel.CancellationToken))
+					Drain();
+			}
+			catch
+			{
+				//ignore
+			}
 		}
 		Drain();
 

src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs

@@ -12,6 +12,8 @@ public static class ProcessorDiagnosticExtensions
 {
 	public static void EmitError(this InlineProcessor processor, int line, int column, int length, string message)
 	{
+		var context = processor.GetContext();
+		if (context.SkipValidation) return;
 		var d = new Diagnostic
 		{
 			Severity = Severity.Error,
@@ -21,6 +23,53 @@ public static void EmitError(this InlineProcessor processor, int line, int colum
 			Message = message,
 			Length = length
 		};
-		processor.GetBuildContext().Collector.Channel.Write(d);
+		context.Build.Collector.Channel.Write(d);
+	}
+
+
+	public static void EmitWarning(this BlockProcessor processor, int line, int column, int length, string message)
+	{
+		var context = processor.GetContext();
+		if (context.SkipValidation) return;
+		var d = new Diagnostic
+		{
+			Severity = Severity.Warning,
+			File = processor.GetContext().Path.FullName,
+			Column = column,
+			Line = line,
+			Message = message,
+			Length = length
+		};
+		context.Build.Collector.Channel.Write(d);
+	}
+
+	public static void EmitError(this ParserContext context, int line, int column, int length, string message)
+	{
+		if (context.SkipValidation) return;
+		var d = new Diagnostic
+		{
+			Severity = Severity.Error,
+			File = context.Path.FullName,
+			Column = column,
+			Line = line,
+			Message = message,
+			Length = length
+		};
+		context.Build.Collector.Channel.Write(d);
+	}
+
+	public static void EmitWarning(this ParserContext context, int line, int column, int length, string message)
+	{
+		if (context.SkipValidation) return;
+		var d = new Diagnostic
+		{
+			Severity = Severity.Warning,
+			File = context.Path.FullName,
+			Column = column,
+			Line = line,
+			Message = message,
+			Length = length
+		};
+		context.Build.Collector.Channel.Write(d);
 	}
 }

src/Elastic.Markdown/Myst/Directives/AdmonitionBlock.cs

@@ -25,7 +25,7 @@ public string Title
 		}
 	}
 
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
 		Classes = Properties.GetValueOrDefault("class");
 		CrossReferenceName = Properties.GetValueOrDefault("name");
@@ -38,9 +38,9 @@ public class DropdownBlock(DirectiveBlockParser parser, Dictionary<string, strin
 	: AdmonitionBlock(parser, "admonition", properties)
 {
 	// ReSharper disable once RedundantOverriddenMember
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
-		base.FinalizeAndValidate();
+		base.FinalizeAndValidate(context);
 		Classes = $"dropdown {Classes}";
 	}
 }

src/Elastic.Markdown/Myst/Directives/CardBlock.cs

@@ -14,7 +14,7 @@ public class CardBlock(DirectiveBlockParser parser, Dictionary<string, string> p
 
 	public string? Footer { get; set; }
 
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
 		Title = Arguments;
 		Link = Properties.GetValueOrDefault("link");

src/Elastic.Markdown/Myst/Directives/CodeBlock.cs

@@ -20,7 +20,7 @@ public string Language
 		}
 	}
 
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
 		Caption = Properties.GetValueOrDefault("caption");
 		CrossReferenceName = Properties.GetValueOrDefault("name");

src/Elastic.Markdown/Myst/Directives/DirectiveBlock.cs

@@ -66,7 +66,8 @@ public abstract class DirectiveBlock(DirectiveBlockParser parser, Dictionary<str
     /// <summary>
     /// Allows blocks to finalize setting properties once fully parsed
     /// </summary>
-    public abstract void FinalizeAndValidate();
+    /// <param name="context"></param>
+    public abstract void FinalizeAndValidate(ParserContext context);
 
 	protected void ParseBool(string key, Action<bool> setter)
 	{

src/Elastic.Markdown/Myst/Directives/DirectiveBlockParser.cs

@@ -5,6 +5,7 @@
 // This file is licensed under the BSD-Clause 2 license.
 // See the license.txt file in the project root for more information.
 
+using System.Collections.Frozen;
 using Markdig.Parsers;
 using Markdig.Syntax;
 using static System.StringSplitOptions;
@@ -37,6 +38,20 @@ public DirectiveBlockParser()
 
 	private readonly string[] _codeBlocks = [ "code", "code-block", "sourcecode"];
 
+	private readonly FrozenDictionary<string, int> _unsupportedBlocks = new Dictionary<string, int>
+	{
+		{ "bibliography", 5 },
+		{ "blockquote", 6 },
+		{ "csv-table", 9 },
+		{ "iframe", 14 },
+		{ "list-table", 17 },
+		{ "myst", 22 },
+		{ "topic", 24 },
+		{ "exercise", 30 },
+		{ "solution", 31 },
+		{ "toctree", 32 },
+	}.ToFrozenDictionary();
+
     protected override DirectiveBlock CreateFencedBlock(BlockProcessor processor)
     {
 	    _admonitionData = new Dictionary<string, string>();
@@ -108,14 +123,18 @@ protected override DirectiveBlock CreateFencedBlock(BlockProcessor processor)
 		    if (info.IndexOf($"{{{code}}}") > 0)
 			    return new CodeBlock(this, code, _admonitionData);
 	    }
+	    // TODO alternate lookup .NET 9
+	    var directive = info.ToString().Trim(['{', '}', '`']);
+	    if (_unsupportedBlocks.TryGetValue(directive, out var issueId))
+		    return new UnsupportedDirectiveBlock(this, directive, _admonitionData, issueId);
 
 	    return new UnknownDirectiveBlock(this, info.ToString(), _admonitionData);
     }
 
     public override bool Close(BlockProcessor processor, Block block)
     {
 	    if (block is DirectiveBlock directiveBlock)
-		    directiveBlock.FinalizeAndValidate();
+		    directiveBlock.FinalizeAndValidate(processor.GetContext());
 
 
 	    if (block is not TocTreeBlock toc)

src/Elastic.Markdown/Myst/Directives/ImageBlock.cs

@@ -56,7 +56,7 @@ public class ImageBlock(DirectiveBlockParser parser, Dictionary<string, string>
 
 	public string ImageUrl { get; private set; } = default!;
 
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
 		ImageUrl = Arguments ?? string.Empty; //todo validate
 		Classes = Properties.GetValueOrDefault("class");

src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs

@@ -28,7 +28,7 @@ public class IncludeBlock(DirectiveBlockParser parser, Dictionary<string, string
 
 	//TODO add all options from
 	//https://mystmd.org/guide/directives#directive-include
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
 		var includePath = Arguments; //todo validate
 		Literal |= bool.TryParse(Properties.GetValueOrDefault("literal"), out var b) && b;

src/Elastic.Markdown/Myst/Directives/MermaidBlock.cs

@@ -6,7 +6,7 @@ namespace Elastic.Markdown.Myst.Directives;
 public class MermaidBlock(DirectiveBlockParser parser, Dictionary<string, string> properties)
 	: DirectiveBlock(parser, properties)
 {
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
 	}
 }

src/Elastic.Markdown/Myst/Directives/SideBarBlock.cs

@@ -6,7 +6,7 @@ namespace Elastic.Markdown.Myst.Directives;
 public class SideBarBlock(DirectiveBlockParser parser, Dictionary<string, string> properties)
 	: DirectiveBlock(parser, properties)
 {
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
 	}
 }

src/Elastic.Markdown/Myst/Directives/TabSetBlock.cs

@@ -10,7 +10,7 @@ public class TabSetBlock(DirectiveBlockParser parser, Dictionary<string, string>
 	: DirectiveBlock(parser, properties)
 {
 	public int Index { get; set; }
-	public override void FinalizeAndValidate() => Index = FindIndex();
+	public override void FinalizeAndValidate(ParserContext context) => Index = FindIndex();
 
 	private int _index = -1;
 	public int FindIndex()
@@ -28,7 +28,7 @@ public class TabItemBlock(DirectiveBlockParser parser, Dictionary<string, string
 	public int Index { get; set; }
 	public int TabSetIndex { get; set; }
 
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
 		Title = Arguments ?? "Unnamed Tab";
 		Index = Parent!.IndexOf(this);
@@ -80,7 +80,7 @@ public class GridBlock(DirectiveBlockParser parser, Dictionary<string, string> p
 	public string? ClassRow { get; set; }
 
 
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
 		//todo we always assume 4 integers
 		if (!string.IsNullOrEmpty(Arguments))
@@ -124,17 +124,7 @@ private void ParseData(string data, Action<int, int, int, int> setter, bool allo
 public class GridItemCardBlock(DirectiveBlockParser parser, Dictionary<string, string> properties)
 	: DirectiveBlock(parser, properties)
 {
-	public override void FinalizeAndValidate()
-	{
-	}
-}
-
-public class UnknownDirectiveBlock(DirectiveBlockParser parser, string directive, Dictionary<string, string> properties)
-	: DirectiveBlock(parser, properties)
-{
-	public string Directive => directive;
-
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
 	}
 }

src/Elastic.Markdown/Myst/Directives/TocTreeBlock.cs

@@ -16,7 +16,7 @@ public class TocTreeBlock(DirectiveBlockParser parser, Dictionary<string, string
 {
 	public OrderedList<TocTreeLink> Links { get; } = new();
 
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
 	}
 }

src/Elastic.Markdown/Myst/Directives/UnknownDirectiveBlock.cs

@@ -0,0 +1,15 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Markdown.Myst.Directives;
+
+public class UnknownDirectiveBlock(DirectiveBlockParser parser, string directive, Dictionary<string, string> properties)
+	: DirectiveBlock(parser, properties)
+{
+	public string Directive => directive;
+
+	public override void FinalizeAndValidate(ParserContext context)
+	{
+	}
+}

src/Elastic.Markdown/Myst/Directives/UnsupportedDirectiveBlock.cs

@@ -0,0 +1,18 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Markdown.Diagnostics;
+
+namespace Elastic.Markdown.Myst.Directives;
+
+public class UnsupportedDirectiveBlock(DirectiveBlockParser parser, string directive, Dictionary<string, string> properties, int issueId)
+	: DirectiveBlock(parser, properties)
+{
+	public string Directive => directive;
+
+	public string IssueUrl => $"https://github.com/elastic/docs-builder/issues/{issueId}";
+
+	public override void FinalizeAndValidate(ParserContext context) =>
+		context.EmitWarning(line:1, column:1, length:2, message: $"Directive block '{directive}' is unsupported. See {IssueUrl} for more information.");
+}

src/Elastic.Markdown/Myst/Directives/VersionBlock.cs

@@ -21,7 +21,7 @@ public string Title
 		}
 	}
 
-	public override void FinalizeAndValidate()
+	public override void FinalizeAndValidate(ParserContext context)
 	{
 	}
 }

src/Elastic.Markdown/Myst/MarkdownParser.cs

@@ -36,30 +36,37 @@ public class MarkdownParser(IDirectoryInfo sourcePath, BuildContext context)
 	// TODO only scan for yaml front matter and toc information
 	public Task<MarkdownDocument> QuickParseAsync(IFileInfo path, Cancel ctx)
 	{
-		var context = new ParserContext(this, path, null, Context);
-		return ParseAsync(path, context, ctx);
+		var context = new ParserContext(this, path, null, Context)
+		{
+			SkipValidation = true
+		};
+		return ParseAsync(path, context, Pipeline, ctx);
 	}
 
 	public Task<MarkdownDocument> ParseAsync(IFileInfo path, YamlFrontMatter? matter, Cancel ctx)
 	{
 		var context = new ParserContext(this, path, matter, Context);
-		return ParseAsync(path, context, ctx);
+		return ParseAsync(path, context, Pipeline, ctx);
 	}
 
-	private async Task<MarkdownDocument> ParseAsync(IFileInfo path, MarkdownParserContext context, Cancel ctx)
+	private async Task<MarkdownDocument> ParseAsync(
+		IFileInfo path,
+		MarkdownParserContext context,
+		MarkdownPipeline pipeline,
+		Cancel ctx)
 	{
 		if (path.FileSystem is FileSystem)
 		{
 			//real IO optimize through UTF8 stream reader.
 			await using var streamReader = new Utf8StreamReader(path.FullName, fileOpenMode: FileOpenMode.Throughput);
 			var inputMarkdown = await streamReader.AsTextReader().ReadToEndAsync(ctx);
-			var markdownDocument = Markdig.Markdown.Parse(inputMarkdown, Pipeline, context);
+			var markdownDocument = Markdig.Markdown.Parse(inputMarkdown, pipeline, context);
 			return markdownDocument;
 		}
 		else
 		{
 			var inputMarkdown = await path.FileSystem.File.ReadAllTextAsync(path.FullName, ctx);
-			var markdownDocument = Markdig.Markdown.Parse(inputMarkdown, Pipeline, context);
+			var markdownDocument = Markdig.Markdown.Parse(inputMarkdown, pipeline, context);
 			return markdownDocument;
 		}
 	}

src/Elastic.Markdown/Myst/ParserContext.cs

@@ -48,4 +48,5 @@ public ParserContext(MarkdownParser markdownParser,
 	public IFileInfo Path { get; }
 	public YamlFrontMatter? FrontMatter { get; }
 	public BuildContext Build { get; }
+	public bool SkipValidation { get; init; }
 }