chore: Fix internal exception on TOC loading

src/Docfx.Build/TableOfContents/TocHelper.cs

@@ -6,15 +6,14 @@
 using Docfx.Common;
 using Docfx.DataContracts.Common;
 using Docfx.Plugins;
+using YamlDotNet.Core.Events;
+using YamlDotNet.Core;
+using Constants = Docfx.DataContracts.Common.Constants;
 
 namespace Docfx.Build.TableOfContents;
 
 public static class TocHelper
 {
- private static readonly YamlDeserializerWithFallback _deserializer =
- YamlDeserializerWithFallback.Create<List<TocItemViewModel>>()
- .WithFallback<TocItemViewModel>();
-
  internal static List<FileModel> ResolveToc(ImmutableList<FileModel> models)
  {
  var tocCache = new Dictionary<string, TocItemInfo>(FilePathComparer.OSPlatformSensitiveStringComparer);
@@ -63,21 +62,22 @@ public static TocItemViewModel LoadSingleToc(string file)
  var fileType = Utility.GetTocFileType(file);
  try
  {
- if (fileType == TocFileType.Markdown)
- {
- return new()
- {
- Items = MarkdownTocReader.LoadToc(EnvironmentContext.FileAbstractLayer.ReadAllText(file), file)
- };
- }
- else if (fileType == TocFileType.Yaml)
+ switch (fileType)
  {
- return _deserializer.Deserialize(file) switch
- {
- List<TocItemViewModel> vm => new() { Items = vm },
- TocItemViewModel root => root,
- _ => throw new NotSupportedException($"{file} is not a valid TOC file."),
- };
+ case TocFileType.Markdown:
+ return new()
+ {
+ Items = MarkdownTocReader.LoadToc(EnvironmentContext.FileAbstractLayer.ReadAllText(file), file)
+ };
+ case TocFileType.Yaml:
+ {
+ using var reader = EnvironmentContext.FileAbstractLayer.OpenReadText(file);
+ return IsListTocItems(reader)
+ ? new TocItemViewModel { Items = YamlUtility.Deserialize<List<TocItemViewModel>>(reader), }
+ : YamlUtility.Deserialize<TocItemViewModel>(reader);
+ }
+ default:
+ throw new NotSupportedException($"{file} is not a valid TOC file, supported TOC files should be either \"{Constants.TableOfContents.MarkdownTocFileName}\" or \"{Constants.TableOfContents.YamlTocFileName}\".");
  }
  }
  catch (Exception e)
@@ -86,7 +86,45 @@ public static TocItemViewModel LoadSingleToc(string file)
  Logger.LogError(message, code: ErrorCodes.Toc.InvalidTocFile);
  throw new DocumentException(message, e);
  }
+ }
+
+ private static bool IsListTocItems(StreamReader reader)
+ {
+ try
+ {
+ var scanner = new Scanner(reader, skipComments: true);
+ var parser = new Parser(scanner);
+
+ return parser.TryConsume<StreamStart>(out var _)
+ && parser.TryConsume<DocumentStart>(out var _)
+ && parser.TryConsume<SequenceStart>(out var _);
+ }
+ finally
+ {
+ ResetStreamReader(reader);
+ }
+ }
+
+ private static void ResetStreamReader(StreamReader reader)
+ {
+ // Reset stream position and discard StreamReader's buffered data.
+ reader.BaseStream.Seek(0, SeekOrigin.Begin);
+ reader.DiscardBufferedData();
+
+ // Gets current encoding's BOM bytes.
+ var bomBytes = reader.CurrentEncoding.Preamble;
+ if (bomBytes.IsEmpty)
+ return;
+
+ // Try to skip BOM bytes. (StreamReader don't skip BOM bytes automatically when calling DiscardBufferedData) 
+ Span<byte> bytes = stackalloc byte[bomBytes.Length];
+ reader.BaseStream.Read(bytes);
+
+ if (bomBytes.SequenceEqual(bytes))
+ return;
 
- throw new NotSupportedException($"{file} is not a valid TOC file, supported TOC files should be either \"{Constants.TableOfContents.MarkdownTocFileName}\" or \"{Constants.TableOfContents.YamlTocFileName}\".");
+ // Reset stream position. If header bytes are not matched to expected BOM bytes. 
+ // This logics are required for StreamReader that is created with `detectEncodingFromByteOrderMarks: true`.
+ reader.BaseStream.Seek(0, SeekOrigin.Begin);
  }
 }

test/Docfx.Build.Tests/TocHelperTest.cs

@@ -0,0 +1,124 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Text;
+using Docfx.Common;
+using Docfx.DataContracts.Common;
+using FluentAssertions;
+using Xunit;
+
+namespace Docfx.Build.TableOfContents.Tests;
+
+public class TocHelperTest
+{
+ [Fact]
+ public void TestItemDeserialization()
+ {
+ // Arrange
+ var item = new TocItemViewModel
+ {
+ Items =
+ [
+ new TocItemViewModel { Uid = "item1" },
+ new TocItemViewModel { Uid = "item2" }
+ ],
+ };
+
+ var yaml = ToYaml(item);
+ var filePath = Path.Combine(Path.GetTempPath(), "toc.yml");
+ File.WriteAllText(filePath, yaml, new UTF8Encoding(false));
+
+ try
+ {
+ // Act
+ var result = TocHelper.LoadSingleToc(filePath);
+
+ // Assert
+ result.Should().BeEquivalentTo(item);
+ }
+ finally
+ {
+ File.Delete(filePath);
+ }
+ }
+
+ [Fact]
+ public void TestListDeserialization()
+ {
+ // Arrange
+ var items = new TocItemViewModel[]
+ {
+ new TocItemViewModel { Uid = "item1" },
+ new TocItemViewModel { Uid = "item2" },
+ };
+
+ var yaml = ToYaml(items);
+ var filePath = Path.Combine(Path.GetTempPath(), "toc.yml");
+ File.WriteAllText(filePath, yaml);
+
+ try
+ {
+ // Act
+ var result = TocHelper.LoadSingleToc(filePath);
+
+ // Assert
+ result.Uid.Should().BeNull();
+ result.Href.Should().BeNull();
+ result.Items.Should().BeEquivalentTo(items);
+ }
+ finally
+ {
+ File.Delete(filePath);
+ }
+ }
+
+ [Fact]
+ public void TestItemDeserializationWithEncoding()
+ {
+ // Arrange
+ var item = new TocItemViewModel
+ {
+ Items =
+ [
+ new TocItemViewModel { Uid = "item1" },
+ new TocItemViewModel { Uid = "item2" }
+ ],
+ };
+
+ var yaml = ToYaml(item);
+
+ foreach (var encoding in Encodings)
+ {
+ var filePath = Path.Combine(Path.GetTempPath(), "toc.yml");
+ File.WriteAllText(filePath, yaml, encoding);
+
+ try
+ {
+ // Act
+ var result = TocHelper.LoadSingleToc(filePath);
+
+ // Assert
+ result.Should().BeEquivalentTo(item);
+ }
+ finally
+ {
+ File.Delete(filePath);
+ }
+ }
+ }
+
+ private static readonly Encoding[] Encodings =
+ [
+ new UTF8Encoding(false),
+ new UTF8Encoding(true),
+ Encoding.Unicode,
+ Encoding.BigEndianUnicode,
+ ];
+
+ private static string ToYaml<T>(T model)
+ {
+ using StringWriter sw = new StringWriter();
+ YamlUtility.Serialize(sw, model);
+ return sw.ToString();
+ }
+}

test/docfx.Tests/Api.verified.cs

@@ -2123,13 +2123,6 @@ public static class XrefUtility
  {
  public static bool TryGetXrefStringValue(this Docfx.Plugins.XRefSpec spec, string key, out string value) { }
  }
- public class YamlDeserializerWithFallback
- {
- public object Deserialize(System.Func<System.IO.TextReader> reader) { }
- public object Deserialize(string filePath) { }
- public Docfx.Common.YamlDeserializerWithFallback WithFallback<T>() { }
- public static Docfx.Common.YamlDeserializerWithFallback Create<T>() { }
- }
  public static class YamlMime
  {
  public const string ManagedReference = "YamlMime:ManagedReference";