From 6ddd7a6e80cc8b19dfc53df7f6df7d73dd62fe8b Mon Sep 17 00:00:00 2001
From: Martijn Laarman <Mpdreamz@gmail.com>
Date: Thu, 19 Dec 2024 19:12:33 +0100
Subject: [PATCH] Add support for code callouts (#118)

---
 docs/source/markup/callout.md                 |  23 ++
 docs/source/markup/code.md                    | 137 ++++++++++--
 .../ProcessorDiagnosticExtensions.cs          |   8 +-
 .../Myst/CodeBlocks/CallOut.cs                |  16 ++
 .../Myst/CodeBlocks/CallOutParser.cs          |  19 ++
 .../Myst/CodeBlocks/EnhancedCodeBlock.cs      |  28 +++
 .../EnhancedCodeBlockHtmlRenderer.cs          |  97 +++++++++
 .../CodeBlocks/EnhancedCodeBlockParser.cs     | 203 ++++++++++++++++++
 .../EnhancedCodeMarkdownExtensions.cs         |  33 +++
 .../Myst/Directives/CodeBlock.cs              |  31 ---
 .../Myst/Directives/DirectiveBlock.cs         |  15 +-
 .../Myst/Directives/DirectiveBlockParser.cs   |  31 ++-
 .../Myst/Directives/DirectiveHtmlRenderer.cs  |  29 ++-
 .../Directives/DirectiveMarkdownExtension.cs  |   2 +-
 src/Elastic.Markdown/Myst/Directives/Role.cs  |   2 +
 src/Elastic.Markdown/Myst/MarkdownParser.cs   |   3 +-
 .../CodeBlocks/CallOutTests.cs                | 141 ++++++++++++
 .../{Directives => CodeBlocks}/CodeTests.cs   |  12 +-
 .../Inline/InlineImageTest.cs                 |   4 +-
 .../Inline/InlneBaseTests.cs                  |  19 ++
 .../Inline/SubstitutionTest.cs                |   8 +-
 21 files changed, 780 insertions(+), 81 deletions(-)
 create mode 100644 docs/source/markup/callout.md
 create mode 100644 src/Elastic.Markdown/Myst/CodeBlocks/CallOut.cs
 create mode 100644 src/Elastic.Markdown/Myst/CodeBlocks/CallOutParser.cs
 create mode 100644 src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlock.cs
 create mode 100644 src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockHtmlRenderer.cs
 create mode 100644 src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs
 create mode 100644 src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeMarkdownExtensions.cs
 delete mode 100644 src/Elastic.Markdown/Myst/Directives/CodeBlock.cs
 create mode 100644 tests/Elastic.Markdown.Tests/CodeBlocks/CallOutTests.cs
 rename tests/Elastic.Markdown.Tests/{Directives => CodeBlocks}/CodeTests.cs (80%)

diff --git a/docs/source/markup/callout.md b/docs/source/markup/callout.md
new file mode 100644
index 0000000..d310b3a
--- /dev/null
+++ b/docs/source/markup/callout.md
@@ -0,0 +1,23 @@
+---
+title: Callouts
+---
+
+You can use the regular markdown code block:
+
+```yaml
+project:
+  title: MyST Markdown 
+  github: https://github.com/jupyter-book/mystmd
+  license:
+    code: MIT
+    content: CC-BY-4.0 <1>
+  subject: MyST Markdown 
+```
+
+
+### C#
+
+```csharp
+var apiKey = new ApiKey("<API_KEY>"); // Set up the api key
+var client = new ElasticsearchClient("<CLOUD_ID>", apiKey); 
+```
diff --git a/docs/source/markup/code.md b/docs/source/markup/code.md
index d3cf2e5..8f85640 100644
--- a/docs/source/markup/code.md
+++ b/docs/source/markup/code.md
@@ -6,7 +6,7 @@ You can use the regular markdown code block:
 
 ```yaml
 project:
-  title: MyST Markdown
+  title: MyST Markdown 
   github: https://github.com/jupyter-book/mystmd
   license:
     code: MIT
@@ -26,13 +26,10 @@ project:
   subject: MyST Markdown
 ```
 
-This page also documents the [code directive](https://mystmd.org/guide/directives). It mentions `code-block` and `sourcecode` as aliases of the `code` directive. But `code-block` seems to behave differently. For example the `caption` option works for `code-block`, but not for `code`.
+For now we only support the `caption` option on the `{code}` or `{code-block}`
 
 ```{code-block} yaml
-:linenos:
 :caption: How to configure `license` of a project
-:name: myst.yml
-:emphasize-lines: 4, 5, 6
 project:
   title: MyST Markdown
   github: https://github.com/jupyter-book/mystmd
@@ -42,15 +39,125 @@ project:
   subject: MyST Markdown
 ```
 
-```{code-block} python
-    :caption: Code blocks can also have sidebars.
-    :linenos:
+## Code Callouts
+
+### YAML
+
+```yaml
+project:
+  title: MyST Markdown #1
+  github: https://github.com/jupyter-book/mystmd
+  license:
+    code: MIT
+    content: CC-BY-4.0
+  subject: MyST Markdown
+```
+
+### Java
+
+```java
+// Create the low-level client
+RestClient restClient = RestClient
+    .builder(HttpHost.create(serverUrl)) //1
+    .setDefaultHeaders(new Header[]{
+        new BasicHeader("Authorization", "ApiKey " + apiKey)
+    })
+    .build();
+```
+
+### Javascript
+
+```javascript
+const { Client } = require('@elastic/elasticsearch')
+const client = new Client({
+  cloud: {
+    id: '<cloud-id>' //1
+  },
+  auth: {
+    username: 'elastic',
+    password: 'changeme'
+  }
+})
+```
+
+### Ruby
+
+```ruby
+require 'elasticsearch'
+
+client = Elasticsearch::Client.new(
+  cloud_id: '<CloudID>'
+  user: '<Username>', #1
+  password: '<Password>',
+)
+```
+
+### Go
+
+```go
+cfg := elasticsearch.Config{
+        CloudID: "CLOUD_ID", //1
+        APIKey: "API_KEY"
+}
+es, err := elasticsearch.NewClient(cfg)
+```
+
+### C#
 
-    print("one")
-    print("two")
-    print("three")
-    print("four")
-    print("five")
-    print("six")
-    print("seven")
+```csharp
+var apiKey = new ApiKey("<API_KEY>"); //1
+var client = new ElasticsearchClient("<CLOUD_ID>", apiKey); 
 ```
+
+### PHP
+
+```php
+$hosts = [
+    '192.168.1.1:9200',         //1
+    '192.168.1.2',              // Just IP
+    'mydomain.server.com:9201', // Domain + Port
+    'mydomain2.server.com',     // Just Domain
+    'https://localhost',        // SSL to localhost
+    'https://192.168.1.3:9200'  // SSL to IP + Port
+];
+$client = ClientBuilder::create()           // Instantiate a new ClientBuilder
+                    ->setHosts($hosts)      // Set the hosts
+                    ->build();              // Build the client object
+```
+
+### Perl
+
+```perl
+my $e = Search::Elasticsearch->new( #1
+    nodes => [ 'https://my-test.es.us-central1.gcp.cloud.es.io' ],
+    elastic_cloud_api_key => 'insert here the API Key'
+);
+```
+### Python
+
+```python
+from elasticsearch import Elasticsearch
+
+ELASTIC_PASSWORD = "<password>" #1
+
+# Found in the 'Manage Deployment' page
+CLOUD_ID = "deployment-name:dXMtZWFzdDQuZ2Nw..."
+
+# Create the client instance
+client = Elasticsearch(
+    cloud_id=CLOUD_ID,
+    basic_auth=("elastic", ELASTIC_PASSWORD)
+)
+
+# Successful response!
+client.info()
+# {'name': 'instance-0000000000', 'cluster_name': ...}
+```
+### Rust
+
+```rust
+let url = Url::parse("https://example.com")?; //1
+let conn_pool = SingleNodeConnectionPool::new(url);
+let transport = TransportBuilder::new(conn_pool).disable_proxy().build()?;
+let client = Elasticsearch::new(transport);
+```
\ No newline at end of file
diff --git a/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs b/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs
index c2f690c..4fc5689 100644
--- a/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs
+++ b/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs
@@ -97,7 +97,7 @@ public static void EmitWarning(this BuildContext context, IFileInfo file, string
 		context.Collector.Channel.Write(d);
 	}
 
-	public static void EmitError(this DirectiveBlock block, string message, Exception? e = null)
+	public static void EmitError(this IBlockExtension block, string message, Exception? e = null)
 	{
 		if (block.SkipValidation) return;
 
@@ -107,13 +107,13 @@ public static void EmitError(this DirectiveBlock block, string message, Exceptio
 			File = block.CurrentFile.FullName,
 			Line = block.Line + 1,
 			Column = block.Column,
-			Length = block.Directive.Length + 5,
+			Length = block.OpeningLength + 5,
 			Message = message + (e != null ? Environment.NewLine + e : string.Empty),
 		};
 		block.Build.Collector.Channel.Write(d);
 	}
 
-	public static void EmitWarning(this DirectiveBlock block, string message)
+	public static void EmitWarning(this IBlockExtension block, string message)
 	{
 		if (block.SkipValidation) return;
 
@@ -123,7 +123,7 @@ public static void EmitWarning(this DirectiveBlock block, string message)
 			File = block.CurrentFile.FullName,
 			Line = block.Line + 1,
 			Column = block.Column,
-			Length = block.Directive.Length + 4,
+			Length = block.OpeningLength + 4,
 			Message = message
 		};
 		block.Build.Collector.Channel.Write(d);
diff --git a/src/Elastic.Markdown/Myst/CodeBlocks/CallOut.cs b/src/Elastic.Markdown/Myst/CodeBlocks/CallOut.cs
new file mode 100644
index 0000000..dcc841c
--- /dev/null
+++ b/src/Elastic.Markdown/Myst/CodeBlocks/CallOut.cs
@@ -0,0 +1,16 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Markdown.Myst.CodeBlocks;
+
+public record CallOut
+{
+	public required int Index { get; init; }
+	public required string Text { get; init; }
+	public required bool InlineCodeAnnotation { get; init; }
+
+	public required int SliceStart { get; init; }
+
+	public required int Line { get; init; }
+}
diff --git a/src/Elastic.Markdown/Myst/CodeBlocks/CallOutParser.cs b/src/Elastic.Markdown/Myst/CodeBlocks/CallOutParser.cs
new file mode 100644
index 0000000..66b740a
--- /dev/null
+++ b/src/Elastic.Markdown/Myst/CodeBlocks/CallOutParser.cs
@@ -0,0 +1,19 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Text.RegularExpressions;
+
+namespace Elastic.Markdown.Myst.CodeBlocks;
+
+public static partial class CallOutParser
+{
+	[GeneratedRegex(@"^.+\S+.*?\s<\d+>$", RegexOptions.IgnoreCase, "en-US")]
+	public static partial Regex CallOutNumber();
+
+	[GeneratedRegex(@"^.+\S+.*?\s(?:\/\/|#)\s[^""]+$", RegexOptions.IgnoreCase, "en-US")]
+	public static partial Regex MathInlineAnnotation();
+
+	[GeneratedRegex(@"\{\{[^\r\n}]+?\}\}", RegexOptions.IgnoreCase, "en-US")]
+	public static partial Regex MatchSubstitutions();
+}
diff --git a/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlock.cs b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlock.cs
new file mode 100644
index 0000000..b46b50b
--- /dev/null
+++ b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlock.cs
@@ -0,0 +1,28 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+using Elastic.Markdown.Myst.Directives;
+using Markdig.Parsers;
+using Markdig.Syntax;
+
+namespace Elastic.Markdown.Myst.CodeBlocks;
+
+public class EnhancedCodeBlock(BlockParser parser, ParserContext context)
+	: FencedCodeBlock(parser), IBlockExtension
+{
+	public BuildContext Build { get; } = context.Build;
+
+	public IFileInfo CurrentFile { get; } = context.Path;
+
+	public bool SkipValidation { get; } = context.SkipValidation;
+
+	public int OpeningLength => Info?.Length ?? 0 + 3;
+
+	public List<CallOut>? CallOuts { get; set; }
+
+	public bool InlineAnnotations { get; set; }
+
+	public string Language { get; set; } = "unknown";
+}
diff --git a/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockHtmlRenderer.cs b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockHtmlRenderer.cs
new file mode 100644
index 0000000..c3b724c
--- /dev/null
+++ b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockHtmlRenderer.cs
@@ -0,0 +1,97 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Markdown.Diagnostics;
+using Elastic.Markdown.Myst.Directives;
+using Elastic.Markdown.Slices.Directives;
+using Markdig.Renderers;
+using Markdig.Renderers.Html;
+using Markdig.Syntax;
+using RazorSlices;
+
+namespace Elastic.Markdown.Myst.CodeBlocks;
+
+public class EnhancedCodeBlockHtmlRenderer : HtmlObjectRenderer<EnhancedCodeBlock>
+{
+
+	private static void RenderRazorSlice<T>(RazorSlice<T> slice, HtmlRenderer renderer, EnhancedCodeBlock block)
+	{
+		var html = slice.RenderAsync().GetAwaiter().GetResult();
+		var blocks = html.Split("[CONTENT]", 2, StringSplitOptions.RemoveEmptyEntries);
+		renderer.Write(blocks[0]);
+		renderer.WriteLeafRawLines(block, true, false, false);
+		renderer.Write(blocks[1]);
+	}
+	protected override void Write(HtmlRenderer renderer, EnhancedCodeBlock block)
+	{
+		var callOuts = block.CallOuts ?? [];
+
+		var slice = Code.Create(new CodeViewModel
+		{
+			CrossReferenceName = string.Empty,// block.CrossReferenceName,
+			Language = block.Language,
+			Caption = string.Empty
+		});
+
+		RenderRazorSlice(slice, renderer, block);
+
+		if (!block.InlineAnnotations && callOuts.Count > 0)
+		{
+			var index = block.Parent!.IndexOf(block);
+			if (index == block.Parent!.Count - 1)
+				block.EmitError("Code block with annotations is not followed by any content, needs numbered list");
+			else
+			{
+				var siblingBlock = block.Parent[index + 1];
+				if (siblingBlock is not ListBlock)
+					block.EmitError("Code block with annotations is not followed by a list");
+				if (siblingBlock is ListBlock l && l.Count != callOuts.Count)
+				{
+					block.EmitError(
+						$"Code block has {callOuts.Count} callouts but the following list only has {l.Count}");
+				}
+				else if (siblingBlock is ListBlock listBlock)
+				{
+					block.Parent.Remove(listBlock);
+					renderer.WriteLine("<ol class=\"code-callouts\">");
+					foreach (var child in listBlock)
+					{
+						var listItem = (ListItemBlock)child;
+						var previousImplicit = renderer.ImplicitParagraph;
+						renderer.ImplicitParagraph = !listBlock.IsLoose;
+
+						renderer.EnsureLine();
+						if (renderer.EnableHtmlForBlock)
+						{
+							renderer.Write("<li");
+							renderer.WriteAttributes(listItem);
+							renderer.Write('>');
+						}
+
+						renderer.WriteChildren(listItem);
+
+						if (renderer.EnableHtmlForBlock)
+							renderer.WriteLine("</li>");
+
+						renderer.EnsureLine();
+						renderer.ImplicitParagraph = previousImplicit;
+					}
+					renderer.WriteLine("</ol>");
+				}
+			}
+		}
+		else if (block.InlineAnnotations)
+		{
+			renderer.WriteLine("<ol class=\"code-callouts\">");
+			foreach (var c in block.CallOuts ?? [])
+			{
+				renderer.WriteLine("<li>");
+				renderer.WriteLine(c.Text);
+				renderer.WriteLine("</li>");
+			}
+
+			renderer.WriteLine("</ol>");
+		}
+	}
+}
diff --git a/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs
new file mode 100644
index 0000000..ae77473
--- /dev/null
+++ b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs
@@ -0,0 +1,203 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Text.RegularExpressions;
+using Elastic.Markdown.Diagnostics;
+using Markdig.Helpers;
+using Markdig.Parsers;
+using Markdig.Syntax;
+
+namespace Elastic.Markdown.Myst.CodeBlocks;
+
+public class EnhancedCodeBlockParser : FencedBlockParserBase<EnhancedCodeBlock>
+{
+	private const string DefaultInfoPrefix = "language-";
+
+	/// <summary>
+	/// Initializes a new instance of the <see cref="FencedCodeBlockParser"/> class.
+	/// </summary>
+	public EnhancedCodeBlockParser()
+	{
+		OpeningCharacters = ['`'];
+		InfoPrefix = DefaultInfoPrefix;
+		InfoParser = RoundtripInfoParser;
+	}
+
+	protected override EnhancedCodeBlock CreateFencedBlock(BlockProcessor processor)
+	{
+		if (processor.Context is not ParserContext context)
+			throw new Exception("Expected parser context to be of type ParserContext");
+
+		var codeBlock = new EnhancedCodeBlock(this, context) { IndentCount = processor.Indent };
+
+		if (processor.TrackTrivia)
+		{
+			// mimic what internal method LinesBefore() does
+			codeBlock.LinesBefore = processor.LinesBefore;
+			processor.LinesBefore = null;
+
+			codeBlock.TriviaBefore = processor.UseTrivia(processor.Start - 1);
+			codeBlock.NewLine = processor.Line.NewLine;
+		}
+
+		return codeBlock;
+	}
+
+	public override BlockState TryContinue(BlockProcessor processor, Block block)
+	{
+		var result = base.TryContinue(processor, block);
+		if (result == BlockState.Continue && !processor.TrackTrivia)
+		{
+			var fence = (EnhancedCodeBlock)block;
+			// Remove any indent spaces
+			var c = processor.CurrentChar;
+			var indentCount = fence.IndentCount;
+			while (indentCount > 0 && c.IsSpace())
+			{
+				indentCount--;
+				c = processor.NextChar();
+			}
+		}
+
+		return result;
+	}
+
+	public override bool Close(BlockProcessor processor, Block block)
+	{
+		if (block is not EnhancedCodeBlock codeBlock)
+			return base.Close(processor, block);
+
+		if (processor.Context is not ParserContext context)
+			throw new Exception("Expected parser context to be of type ParserContext");
+
+		codeBlock.Language = (
+			(codeBlock.Info?.IndexOf("{") ?? -1) != -1
+				? codeBlock.Arguments
+				: codeBlock.Info
+		) ?? "unknown";
+
+		var lines = codeBlock.Lines;
+		var callOutIndex = 0;
+
+		var originatingLine = 0;
+		for (var index = 0; index < lines.Lines.Length; index++)
+		{
+			originatingLine++;
+			var line = lines.Lines[index];
+			var span = line.Slice.AsSpan();
+
+			if (ReplaceSubstitutions(context, span, out var replacement))
+			{
+				var s = new StringSlice(replacement);
+				lines.Lines[index] = new StringLine(ref s);
+				span = lines.Lines[index].Slice.AsSpan();
+			}
+
+			var matchClassicCallout = CallOutParser.CallOutNumber().EnumerateMatches(span);
+			var callOut = EnumerateAnnotations(matchClassicCallout, ref span, ref callOutIndex, originatingLine, false);
+
+			if (callOut is null)
+			{
+				var matchInline = CallOutParser.MathInlineAnnotation().EnumerateMatches(span);
+				callOut = EnumerateAnnotations(matchInline, ref span, ref callOutIndex, originatingLine,
+					true);
+			}
+
+			if (callOut is null)
+				continue;
+
+			codeBlock.CallOuts ??= [];
+			codeBlock.CallOuts.Add(callOut);
+		}
+
+		//update string slices to ignore call outs
+		if (codeBlock.CallOuts is not null)
+		{
+			foreach (var callout in codeBlock.CallOuts)
+			{
+				var line = lines.Lines[callout.Line - 1];
+
+				var newSpan = line.Slice.AsSpan()[..callout.SliceStart];
+				var s = new StringSlice(newSpan.ToString());
+				lines.Lines[callout.Line -1 ] = new StringLine(ref s);
+
+			}
+		}
+
+		var inlineAnnotations = codeBlock.CallOuts?.Where(c => c.InlineCodeAnnotation).Count() ?? 0;
+		var classicAnnotations = codeBlock.CallOuts?.Count - inlineAnnotations ?? 0;
+		if (inlineAnnotations > 0 && classicAnnotations > 0)
+			codeBlock.EmitError("Both inline and classic callouts are not supported");
+
+		if (inlineAnnotations > 0)
+			codeBlock.InlineAnnotations = true;
+
+		return base.Close(processor, block);
+	}
+
+	private static bool ReplaceSubstitutions(ParserContext context, ReadOnlySpan<char> span, out string? replacement)
+	{
+		replacement = null;
+		var substitutions = context.FrontMatter?.Properties ?? new();
+		if (substitutions.Count == 0)
+			return false;
+
+		var matchSubs = CallOutParser.MatchSubstitutions().EnumerateMatches(span);
+
+		var replaced = false;
+		foreach (var match in matchSubs)
+		{
+			if (match.Length == 0)
+				continue;
+
+			var spanMatch = span.Slice(match.Index, match.Length);
+			var key = spanMatch.Trim(['{', '}']);
+
+			// TODO: alternate lookup using span in c# 9
+			if (substitutions.TryGetValue(key.ToString(), out var value))
+			{
+				replacement ??= span.ToString();
+				replacement = replacement.Replace(spanMatch.ToString(), value);
+				replaced = true;
+			}
+
+		}
+
+		return replaced;
+	}
+
+	private static CallOut? EnumerateAnnotations(Regex.ValueMatchEnumerator matches,
+		ref ReadOnlySpan<char> span,
+		ref int callOutIndex,
+		int originatingLine,
+		bool inlineCodeAnnotation)
+	{
+		foreach (var match in matches)
+		{
+			if (match.Length == 0) continue;
+
+			var startIndex = span.LastIndexOf("<");
+			if (!inlineCodeAnnotation && startIndex <= 0) continue;
+			if (inlineCodeAnnotation)
+			{
+				startIndex = Math.Max(span.LastIndexOf("//"), span.LastIndexOf('#'));
+				if (startIndex <= 0)
+					continue;
+			}
+
+			callOutIndex++;
+			var callout = span.Slice(match.Index + startIndex, match.Length - startIndex);
+			return new CallOut
+			{
+				Index = callOutIndex,
+				Text = callout.TrimStart('/').TrimStart('#').TrimStart().ToString(),
+				InlineCodeAnnotation = inlineCodeAnnotation,
+				SliceStart = startIndex,
+				Line = originatingLine,
+			};
+		}
+
+		return null;
+	}
+}
diff --git a/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeMarkdownExtensions.cs b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeMarkdownExtensions.cs
new file mode 100644
index 0000000..9c6f003
--- /dev/null
+++ b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeMarkdownExtensions.cs
@@ -0,0 +1,33 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Markdown.Myst.Directives;
+using Markdig;
+using Markdig.Parsers;
+using Markdig.Renderers;
+using Markdig.Renderers.Html;
+
+namespace Elastic.Markdown.Myst.CodeBlocks;
+
+public static class EnhancedCodeBuilderExtensions
+{
+	public static MarkdownPipelineBuilder UseEnhancedCodeBlocks(this MarkdownPipelineBuilder pipeline)
+	{
+		pipeline.Extensions.AddIfNotAlready<EnhancedCodeBlockExtension>();
+		return pipeline;
+	}
+}
+
+/// <summary>
+/// Extension to allow custom containers.
+/// </summary>
+/// <seealso cref="IMarkdownExtension" />
+public class EnhancedCodeBlockExtension : IMarkdownExtension
+{
+	public void Setup(MarkdownPipelineBuilder pipeline) =>
+		pipeline.BlockParsers.Replace<FencedCodeBlockParser>(new EnhancedCodeBlockParser());
+
+	public void Setup(MarkdownPipeline pipeline, IMarkdownRenderer renderer) =>
+		renderer.ObjectRenderers.Replace<CodeBlockRenderer>(new EnhancedCodeBlockHtmlRenderer());
+}
diff --git a/src/Elastic.Markdown/Myst/Directives/CodeBlock.cs b/src/Elastic.Markdown/Myst/Directives/CodeBlock.cs
deleted file mode 100644
index b76ed7e..0000000
--- a/src/Elastic.Markdown/Myst/Directives/CodeBlock.cs
+++ /dev/null
@@ -1,31 +0,0 @@
-// Licensed to Elasticsearch B.V under one or more agreements.
-// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
-// See the LICENSE file in the project root for more information
-namespace Elastic.Markdown.Myst.Directives;
-
-public class CodeBlock(
-	DirectiveBlockParser parser,
-	string directive,
-	Dictionary<string, string> properties,
-	ParserContext context)
-	: DirectiveBlock(parser, properties, context)
-{
-	public override string Directive => directive;
-	public string? Caption { get; private set; }
-
-	public string Language
-	{
-		get
-		{
-			var language = (Directive is "code" or "code-block" or "sourcecode" ? Arguments : Info) ?? "unknown";
-			return language;
-
-		}
-	}
-
-	public override void FinalizeAndValidate(ParserContext context)
-	{
-		Caption = Properties.GetValueOrDefault("caption");
-		CrossReferenceName = Prop("name", "label");
-	}
-}
diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveBlock.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveBlock.cs
index 8740ece..4b2bf51 100644
--- a/src/Elastic.Markdown/Myst/Directives/DirectiveBlock.cs
+++ b/src/Elastic.Markdown/Myst/Directives/DirectiveBlock.cs
@@ -12,6 +12,17 @@
 
 namespace Elastic.Markdown.Myst.Directives;
 
+public interface IBlockExtension : IBlock
+{
+	BuildContext Build { get; }
+
+	bool SkipValidation { get; }
+
+	IFileInfo CurrentFile { get; }
+
+	int OpeningLength { get; }
+}
+
 /// <summary>
 /// A block custom container.
 /// </summary>
@@ -28,7 +39,7 @@ public abstract class DirectiveBlock(
 	Dictionary<string, string> properties,
 	ParserContext context
 	)
-	: ContainerBlock(parser), IFencedBlock
+	: ContainerBlock(parser), IFencedBlock, IBlockExtension
 {
 	protected IReadOnlyDictionary<string, string> Properties { get; } = properties;
 
@@ -38,6 +49,8 @@ ParserContext context
 
 	public bool SkipValidation { get; } = context.SkipValidation;
 
+	public int OpeningLength => Directive.Length;
+
 	public abstract string Directive { get; }
 
 	public string? CrossReferenceName  { get; protected set; }
diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveBlockParser.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveBlockParser.cs
index 852937d..e28a6f1 100644
--- a/src/Elastic.Markdown/Myst/Directives/DirectiveBlockParser.cs
+++ b/src/Elastic.Markdown/Myst/Directives/DirectiveBlockParser.cs
@@ -73,9 +73,6 @@ protected override DirectiveBlock CreateFencedBlock(BlockProcessor processor)
 		if (processor.Context is not ParserContext context)
 			throw new Exception("Expected parser context to be of type ParserContext");
 
-		if (info.IndexOf("{") == -1)
-		    return new CodeBlock(this, "raw", _admonitionData, context);
-
 	    // TODO alternate lookup .NET 9
 	    var directive = info.ToString().Trim(['{', '}', '`']);
 	    if (_unsupportedBlocks.TryGetValue(directive, out var issueId))
@@ -129,11 +126,6 @@ protected override DirectiveBlock CreateFencedBlock(BlockProcessor processor)
 			    return new VersionBlock(this, version, _admonitionData, context);
 	    }
 
-	    foreach (var code in _codeBlocks)
-	    {
-		    if (info.IndexOf($"{{{code}}}") > 0)
-			    return new CodeBlock(this, code, _admonitionData, context);
-	    }
 	    return new UnknownDirectiveBlock(this, info.ToString(), _admonitionData, context);
     }
 
@@ -145,6 +137,29 @@ public override bool Close(BlockProcessor processor, Block block)
 		return base.Close(processor, block);
     }
 
+    public override BlockState TryOpen(BlockProcessor processor)
+    {
+		if (processor.Context is not ParserContext context)
+			throw new Exception("Expected parser context to be of type ParserContext");
+
+        // We expect no indentation for a fenced code block.
+        if (processor.IsCodeIndent)
+            return BlockState.None;
+
+        var line = processor.Line;
+
+	    foreach (var code in _codeBlocks)
+	    {
+		    if (line.IndexOf($"{{{code}}}") > 0)
+			    return BlockState.None;
+	    }
+
+		if (line.IndexOf("{") == -1)
+		    return BlockState.None;
+
+	    return base.TryOpen(processor);
+    }
+
     public override BlockState TryContinue(BlockProcessor processor, Block block)
     {
 	    var line = processor.Line.AsSpan();
diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs
index df88297..4e68d17 100644
--- a/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs
+++ b/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs
@@ -6,6 +6,7 @@
 // See the license.txt file in the project root for more information.
 
 using Elastic.Markdown.Diagnostics;
+using Elastic.Markdown.Myst.CodeBlocks;
 using Elastic.Markdown.Myst.FrontMatter;
 using Elastic.Markdown.Myst.Settings;
 using Elastic.Markdown.Myst.Substitution;
@@ -53,9 +54,6 @@ protected override void Write(HtmlRenderer renderer, DirectiveBlock directiveBlo
 			case VersionBlock versionBlock:
 				WriteVersion(renderer, versionBlock);
 				return;
-			case CodeBlock codeBlock:
-				WriteCode(renderer, codeBlock);
-				return;
 			case TabSetBlock tabSet:
 				WriteTabSet(renderer, tabSet);
 				return;
@@ -164,13 +162,15 @@ private void WriteDropdown(HtmlRenderer renderer, DropdownBlock block)
 		RenderRazorSlice(slice, renderer, block);
 	}
 
-	private void WriteCode(HtmlRenderer renderer, CodeBlock block)
+	private void WriteCode(HtmlRenderer renderer, EnhancedCodeBlock block)
 	{
 		var slice = Code.Create(new CodeViewModel
 		{
-			CrossReferenceName = block.CrossReferenceName, Language = block.Language, Caption = block.Caption
+			CrossReferenceName = string.Empty,// block.CrossReferenceName,
+			Language = block.Language,
+			Caption = string.Empty
 		});
-		RenderRazorSliceRawContent(slice, renderer, block);
+		//RenderRazorSliceRawContent(slice, renderer, block);
 	}
 
 
@@ -317,13 +317,20 @@ void RenderLeaf(LeafBlock p)
 			renderer.EnableHtmlForInline = false;
 			foreach (var oo in p.Inline ?? [])
 			{
-
 				if (oo is SubstitutionLeaf sl)
 					renderer.Write(sl.Replacement);
-				if (oo is LiteralInline li)
+				else if (oo is LiteralInline li)
 					renderer.Write(li);
-				if (oo is LineBreakInline)
+				else if (oo is LineBreakInline)
 					renderer.WriteLine();
+				else if (oo is Role r)
+				{
+					renderer.Write(new string(r.DelimiterChar, r.DelimiterCount));
+					renderer.WriteChildren(r);
+				}
+
+				else
+					renderer.Write($"(LeafBlock: {oo.GetType().Name}");
 			}
 
 			renderer.EnableHtmlForInline = true;
@@ -342,6 +349,8 @@ void RenderListBlock(ListBlock l)
 					foreach (var lll in ll)
 						Render(lll);
 				}
+				else
+					renderer.Write($"(ListBlock: {l.GetType().Name}");
 			}
 		}
 
@@ -351,6 +360,8 @@ void Render(Block o)
 				RenderLeaf(p);
 			else if (o is ListBlock l)
 				RenderListBlock(l);
+			else
+				renderer.Write($"(Block: {o.GetType().Name}");
 		}
 	}
 }
diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveMarkdownExtension.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveMarkdownExtension.cs
index 8008c31..a5c2310 100644
--- a/src/Elastic.Markdown/Myst/Directives/DirectiveMarkdownExtension.cs
+++ b/src/Elastic.Markdown/Myst/Directives/DirectiveMarkdownExtension.cs
@@ -45,7 +45,7 @@ public void Setup(MarkdownPipelineBuilder pipeline)
 			inlineParser.TryCreateEmphasisInlineList.Add((emphasisChar, delimiterCount) =>
 			{
 				if (delimiterCount == 2 && emphasisChar == ':')
-					return new Role();
+					return new Role { DelimiterChar = ':', DelimiterCount = 2 };
 
 				return null;
 			});
diff --git a/src/Elastic.Markdown/Myst/Directives/Role.cs b/src/Elastic.Markdown/Myst/Directives/Role.cs
index e8f01ef..1c7ff92 100644
--- a/src/Elastic.Markdown/Myst/Directives/Role.cs
+++ b/src/Elastic.Markdown/Myst/Directives/Role.cs
@@ -9,6 +9,8 @@
 
 namespace Elastic.Markdown.Myst.Directives;
 
+
+//TODO evaluate if we need this
 /// <summary>
 /// An inline custom container
 /// </summary>
diff --git a/src/Elastic.Markdown/Myst/MarkdownParser.cs b/src/Elastic.Markdown/Myst/MarkdownParser.cs
index ee3ff9d..aed329a 100644
--- a/src/Elastic.Markdown/Myst/MarkdownParser.cs
+++ b/src/Elastic.Markdown/Myst/MarkdownParser.cs
@@ -5,6 +5,7 @@
 using System.IO.Abstractions;
 using Cysharp.IO;
 using Elastic.Markdown.IO;
+using Elastic.Markdown.Myst.CodeBlocks;
 using Elastic.Markdown.Myst.Comments;
 using Elastic.Markdown.Myst.Directives;
 using Elastic.Markdown.Myst.FrontMatter;
@@ -40,7 +41,6 @@ public class MarkdownParser(
 			.EnableTrackTrivia()
 			.UsePreciseSourceLocation()
 			.UseDiagnosticLinks()
-			.UseGenericAttributes()
 			.UseEmphasisExtras(EmphasisExtraOptions.Default)
 			.UseSoftlineBreakAsHardlineBreak()
 			.UseSubstitution()
@@ -49,6 +49,7 @@ public class MarkdownParser(
 			.UseGridTables()
 			.UsePipeTables()
 			.UseDirectives()
+			.UseEnhancedCodeBlocks()
 			.DisableHtml()
 			.Build();
 
diff --git a/tests/Elastic.Markdown.Tests/CodeBlocks/CallOutTests.cs b/tests/Elastic.Markdown.Tests/CodeBlocks/CallOutTests.cs
new file mode 100644
index 0000000..550aed3
--- /dev/null
+++ b/tests/Elastic.Markdown.Tests/CodeBlocks/CallOutTests.cs
@@ -0,0 +1,141 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Markdown.Myst.CodeBlocks;
+using Elastic.Markdown.Tests.Inline;
+using FluentAssertions;
+using JetBrains.Annotations;
+using Xunit.Abstractions;
+
+namespace Elastic.Markdown.Tests.CodeBlocks;
+
+public abstract class CodeBlockCallOutTests(
+	ITestOutputHelper output,
+	string language,
+	[LanguageInjection("csharp")] string code,
+	[LanguageInjection("markdown")] string? markdown = null
+)
+	: BlockTest<EnhancedCodeBlock>(output,
+$$"""
+```{{language}}
+{{code}}
+```
+{{markdown}}
+"""
+)
+{
+	[Fact]
+	public void ParsesAdmonitionBlock() => Block.Should().NotBeNull();
+
+	[Fact]
+	public void SetsLanguage() => Block!.Language.Should().Be("csharp");
+
+}
+
+public class MagicCalOuts(ITestOutputHelper output) : CodeBlockCallOutTests(output, "csharp",
+"""
+var x = 1; // this is a callout
+//this is not a callout
+var y = x - 2;
+var z = y - 2; // another callout
+"""
+	)
+{
+	[Fact]
+	public void ParsesMagicCallOuts() => Block!.CallOuts
+		.Should().NotBeNullOrEmpty()
+		.And.HaveCount(2)
+		.And.NotContain(c=>c.Text.Contains("not a callout"));
+
+	[Fact]
+	public void HasNoErrors() => Collector.Diagnostics.Should().HaveCount(0);
+}
+
+public class ClassicCallOutsRequiresContent(ITestOutputHelper output) : CodeBlockCallOutTests(output, "csharp",
+"""
+var x = 1; <1>
+var y = x - 2;
+var z = y - 2; <2>
+"""
+	)
+{
+	[Fact]
+	public void ParsesMagicCallOuts() => Block!.CallOuts
+		.Should().NotBeNullOrEmpty()
+		.And.HaveCount(2)
+		.And.OnlyContain(c=>c.Text.StartsWith("<"));
+
+	[Fact]
+	public void RequiresContentToFollow() => Collector.Diagnostics.Should().HaveCount(1)
+		.And.OnlyContain(c=> c.Message.StartsWith("Code block with annotations is not followed by any content"));
+}
+
+public class ClassicCallOutsNotFollowedByList(ITestOutputHelper output) : CodeBlockCallOutTests(output, "csharp",
+"""
+var x = 1; <1>
+var y = x - 2;
+var z = y - 2; <2>
+""",
+"""
+## hello world
+"""
+
+	)
+{
+	[Fact]
+	public void ParsesMagicCallOuts() => Block!.CallOuts
+		.Should().NotBeNullOrEmpty()
+		.And.HaveCount(2)
+		.And.OnlyContain(c=>c.Text.StartsWith("<"));
+
+	[Fact]
+	public void RequiresContentToFollow() => Collector.Diagnostics.Should().HaveCount(1)
+		.And.OnlyContain(c=> c.Message.StartsWith("Code block with annotations is not followed by a list"));
+}
+
+public class ClassicCallOutsFollowedByListWithWrongCoung(ITestOutputHelper output) : CodeBlockCallOutTests(output, "csharp",
+"""
+var x = 1; <1>
+var y = x - 2;
+var z = y - 2; <2>
+""",
+"""
+1. Only marking the first callout
+"""
+
+	)
+{
+	[Fact]
+	public void ParsesMagicCallOuts() => Block!.CallOuts
+		.Should().NotBeNullOrEmpty()
+		.And.HaveCount(2)
+		.And.OnlyContain(c=>c.Text.StartsWith("<"));
+
+	[Fact]
+	public void RequiresContentToFollow() => Collector.Diagnostics.Should().HaveCount(1)
+		.And.OnlyContain(c=> c.Message.StartsWith("Code block has 2 callouts but the following list only has 1"));
+}
+
+public class ClassicCallOutWithTheRightListItems(ITestOutputHelper output) : CodeBlockCallOutTests(output, "csharp",
+"""
+var x = 1; <1>
+var y = x - 2;
+var z = y - 2; <2>
+""",
+"""
+1. First callout
+2. Second callout
+"""
+
+	)
+{
+	[Fact]
+	public void ParsesMagicCallOuts() => Block!.CallOuts
+		.Should().NotBeNullOrEmpty()
+		.And.HaveCount(2)
+		.And.OnlyContain(c=>c.Text.StartsWith("<"));
+
+	[Fact]
+	public void HasNoErrors() => Collector.Diagnostics.Should().HaveCount(0);
+}
diff --git a/tests/Elastic.Markdown.Tests/Directives/CodeTests.cs b/tests/Elastic.Markdown.Tests/CodeBlocks/CodeTests.cs
similarity index 80%
rename from tests/Elastic.Markdown.Tests/Directives/CodeTests.cs
rename to tests/Elastic.Markdown.Tests/CodeBlocks/CodeTests.cs
index 3973ae8..f5f0707 100644
--- a/tests/Elastic.Markdown.Tests/Directives/CodeTests.cs
+++ b/tests/Elastic.Markdown.Tests/CodeBlocks/CodeTests.cs
@@ -1,13 +1,16 @@
 // Licensed to Elasticsearch B.V under one or more agreements.
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
-using Elastic.Markdown.Myst.Directives;
+
+using Elastic.Markdown.Myst.CodeBlocks;
+using Elastic.Markdown.Tests.Inline;
 using FluentAssertions;
 using Xunit.Abstractions;
 
-namespace Elastic.Markdown.Tests.Directives;
+namespace Elastic.Markdown.Tests.CodeBlocks;
 
-public abstract class CodeBlockTests(ITestOutputHelper output, string directive, string? language = null) : DirectiveTest<CodeBlock>(output,
+public abstract class CodeBlockTests(ITestOutputHelper output, string directive, string? language = null)
+	: BlockTest<EnhancedCodeBlock>(output,
 $$"""
 ```{{directive}} {{language}}
 var x = 1;
@@ -18,9 +21,6 @@ A regular paragraph.
 {
 	[Fact]
 	public void ParsesAdmonitionBlock() => Block.Should().NotBeNull();
-
-	[Fact]
-	public void SetsCorrectDirectiveType() => Block!.Directive.Should().Be(language != null ? directive.Trim('{','}') : "raw");
 }
 
 public class CodeBlockDirectiveTests(ITestOutputHelper output) : CodeBlockTests(output, "{code-block}", "csharp")
diff --git a/tests/Elastic.Markdown.Tests/Inline/InlineImageTest.cs b/tests/Elastic.Markdown.Tests/Inline/InlineImageTest.cs
index 173d3ee..977dfa0 100644
--- a/tests/Elastic.Markdown.Tests/Inline/InlineImageTest.cs
+++ b/tests/Elastic.Markdown.Tests/Inline/InlineImageTest.cs
@@ -9,7 +9,7 @@ namespace Elastic.Markdown.Tests.Inline;
 
 public class InlineImageTest(ITestOutputHelper output) : InlineTest<LinkInline>(output,
 """
-![Elasticsearch](/_static/img/observability.png){w=350px align=center}
+![Elasticsearch](/_static/img/observability.png)
 """
 )
 {
@@ -20,6 +20,6 @@ public class InlineImageTest(ITestOutputHelper output) : InlineTest<LinkInline>(
 	public void GeneratesAttributesInHtml() =>
 		// language=html
 		Html.Should().Contain(
-			"""<p><img src="/_static/img/observability.png" w="350px" align="center" alt="Elasticsearch" /></p>"""
+			"""<p><img src="/_static/img/observability.png" alt="Elasticsearch" /></p>"""
 		);
 }
diff --git a/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs b/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs
index 876b21a..2600991 100644
--- a/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs
+++ b/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs
@@ -31,6 +31,25 @@ public override async Task InitializeAsync()
 
 }
 
+public abstract class BlockTest<TDirective>(ITestOutputHelper output, [LanguageInjection("markdown")]string content)
+	: InlineTest(output, content)
+	where TDirective : Block
+{
+	protected TDirective? Block { get; private set; }
+
+	public override async Task InitializeAsync()
+	{
+		await base.InitializeAsync();
+		Block = Document
+			.Descendants<TDirective>()
+			.FirstOrDefault();
+	}
+
+	[Fact]
+	public void BlockIsNotNull() => Block.Should().NotBeNull();
+
+}
+
 public abstract class InlineTest<TDirective>(ITestOutputHelper output, [LanguageInjection("markdown")]string content)
 	: InlineTest(output, content)
 	where TDirective : ContainerInline
diff --git a/tests/Elastic.Markdown.Tests/Inline/SubstitutionTest.cs b/tests/Elastic.Markdown.Tests/Inline/SubstitutionTest.cs
index 7ea4b3c..e82debb 100644
--- a/tests/Elastic.Markdown.Tests/Inline/SubstitutionTest.cs
+++ b/tests/Elastic.Markdown.Tests/Inline/SubstitutionTest.cs
@@ -1,6 +1,8 @@
 // Licensed to Elasticsearch B.V under one or more agreements.
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
+
+using Elastic.Markdown.Myst.CodeBlocks;
 using Elastic.Markdown.Myst.Substitution;
 using FluentAssertions;
 using Xunit.Abstractions;
@@ -55,7 +57,7 @@ public void GeneratesAttributesInHtml() =>
 			.And.NotContain(
 				"""{{hello-world}}"""
 			)
-			.And.NotContain( // treated as attributes to the block
+			.And.Contain( // treated as attributes to the block
 				"""{substitution}"""
 			)
 			.And.Contain(
@@ -63,7 +65,7 @@ public void GeneratesAttributesInHtml() =>
 			);
 }
 
-public class SubstitutionInCodeBlockTest(ITestOutputHelper output) : LeafTest<SubstitutionLeaf>(output,
+public class SubstitutionInCodeBlockTest(ITestOutputHelper output) : BlockTest<EnhancedCodeBlock>(output,
 """
 ---
 sub:
@@ -81,7 +83,7 @@ cd elasticsearch-{{version}}/ <2>
 {
 
 	[Fact]
-	public void GeneratesAttributesInHtml() =>
+	public void ReplacesSubsInCode() =>
 		Html.Should().Contain("7.17.0");
 }