Number of fixes (#46)

* Fixes

Includes test to verify #44 is fixed

* More fixes

src/Basic.CompilerLog.UnitTests/CompilerLogFixture.cs

@@ -151,7 +151,7 @@ partial class Util {
                     internal static Regex GetRegex() => null!;
                 }
                 """;
-            File.WriteAllText(Path.Combine(scratchPath, "Class1.cs"), program, TestBase.DefaultEncoding);
+            File.WriteAllText(Path.Combine(scratchPath, "Class 1.cs"), program, TestBase.DefaultEncoding);
             Assert.True(DotnetUtil.Command("build -bl", scratchPath).Succeeded);
             return Path.Combine(scratchPath, "msbuild.binlog");
         });
@@ -169,7 +169,7 @@ partial class Util {
         AllComplogs = allCompLogs;
         string WithBuild(string name, Func<string, string> action)
         {
-            var scratchPath = Path.Combine(StorageDirectory, "scratch");
+            var scratchPath = Path.Combine(StorageDirectory, "scratch dir");
             Directory.CreateDirectory(scratchPath);
             var binlogFilePath = action(scratchPath);
             var complogFilePath = Path.Combine(ComplogDirectory, name);

src/Basic.CompilerLog.UnitTests/ExportUtilTests.cs

@@ -4,6 +4,7 @@
 using System.Diagnostics.Contracts;
 using System.Linq;
 using System.Runtime.InteropServices;
+using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
 using Xunit;
@@ -24,7 +25,7 @@ public ExportUtilTests(ITestOutputHelper testOutputHelper, CompilerLogFixture fi
 
     private void TestExport(int expectedCount, Action<string>? callback = null)
     {
-        using var scratchDir = new TempDir();
+        using var scratchDir = new TempDir("export test");
         var binlogFilePath = Path.Combine(RootDirectory, "msbuild.binlog");
         var compilerLogFilePath = Path.Combine(scratchDir.DirectoryPath, "build.complog");
         var diagnosticList = CompilerLogUtil.ConvertBinaryLog(binlogFilePath, compilerLogFilePath);
@@ -37,7 +38,7 @@ private void TestExport(int expectedCount, Action<string>? callback = null)
         TestExport(compilerLogFilePath, expectedCount, callback: callback);
     }
 
-    private void TestExport(string compilerLogFilePath, int expectedCount, bool includeAnalyzers = true, Action<string>? callback = null)
+    private void TestExport(string compilerLogFilePath, int? expectedCount, bool includeAnalyzers = true, Action<string>? callback = null)
     {
         using var reader = CompilerLogReader.Create(compilerLogFilePath);
 #if NETCOREAPP
@@ -52,7 +53,7 @@ private void TestExport(string compilerLogFilePath, int expectedCount, bool incl
             count++;
             TestOutputHelper.WriteLine($"Testing export for {compilerCall.ProjectFileName} - {compilerCall.TargetFramework}");
             using var tempDir = new TempDir();
-            exportUtil.ExportRsp(compilerCall, tempDir.DirectoryPath, sdkDirs);
+            exportUtil.Export(compilerCall, tempDir.DirectoryPath, sdkDirs);
 
             // Now run the generated build.cmd and see if it succeeds;
             var buildResult = RunBuildCmd(tempDir.DirectoryPath);
@@ -69,7 +70,15 @@ private void TestExport(string compilerLogFilePath, int expectedCount, bool incl
 
             callback?.Invoke(tempDir.DirectoryPath);
         }
-        Assert.Equal(expectedCount, count);
+
+        if (expectedCount is { } ec)
+        {
+            Assert.Equal(ec, count);
+        }
+        else
+        {
+            Assert.True(count> 0);
+        }
     }
 
     [Fact]
@@ -170,6 +179,17 @@ This is an awesome resource
         TestExport(1);
     }
 
+    [Fact]
+    public void ConsoleWithSpaceInSourceName()
+    {
+        RunDotNet($"new console --name example --output .");
+        File.WriteAllText(Path.Combine(RootDirectory, "code file.cs"), """
+            class C { }
+            """);
+        RunDotNet("build -bl");
+        TestExport(1);
+    }
+
     [Fact]
     public void ContentWin32Elements()
     {
@@ -215,4 +235,15 @@ public void StrongNameKey()
         RunDotNet("build -bl");
         TestExport(1);
     }
+
+    [Theory]
+    [InlineData(true)]
+    [InlineData(false)]
+    public void AllCompilerLogs(bool includeAnalyzers)
+    {
+        foreach (var complogPath in Fixture.AllComplogs)
+        {
+            TestExport(complogPath, expectedCount: null, includeAnalyzers);
+        }
+    }
 }

src/Basic.CompilerLog.Util/CompilerLogBuilder.cs

@@ -25,11 +25,14 @@ namespace Basic.CompilerLog.Util;
 internal sealed class CompilerLogBuilder : IDisposable
 {
     // GUIDs specified in https://github.com/dotnet/runtime/blob/main/docs/design/specs/PortablePdb-Metadata.md#document-table-0x30
-    public static readonly Guid HashAlgorithmSha1 = unchecked(new Guid((int)0xff1816ec, (short)0xaa5e, 0x4d10, 0x87, 0xf7, 0x6f, 0x49, 0x63, 0x83, 0x34, 0x60));
-    public static readonly Guid HashAlgorithmSha256 = unchecked(new Guid((int)0x8829d00f, 0x11b8, 0x4213, 0x87, 0x8b, 0x77, 0x0e, 0x85, 0x97, 0xac, 0x16));
+    internal static readonly Guid HashAlgorithmSha1 = unchecked(new Guid((int)0xff1816ec, (short)0xaa5e, 0x4d10, 0x87, 0xf7, 0x6f, 0x49, 0x63, 0x83, 0x34, 0x60));
+    internal static readonly Guid HashAlgorithmSha256 = unchecked(new Guid((int)0x8829d00f, 0x11b8, 0x4213, 0x87, 0x8b, 0x77, 0x0e, 0x85, 0x97, 0xac, 0x16));
 
     // https://github.com/dotnet/runtime/blob/main/docs/design/specs/PortablePdb-Metadata.md#embedded-source-c-and-vb-compilers
-    public static readonly Guid EmbeddedSourceGuid = new Guid("0E8A571B-6926-466E-B4AD-8AB04611F5FE");
+    internal static readonly Guid EmbeddedSourceGuid = new Guid("0E8A571B-6926-466E-B4AD-8AB04611F5FE");
+
+    internal static readonly Guid LanguageTypeCSharp = new Guid("{3f5162f8-07c6-11d3-9053-00c04fa302a1}");
+    internal static readonly Guid LanguageTypeBasic = new Guid("{3a12d0b8-c26c-11d0-b442-00a0244a1dd2}");
 
     private readonly Dictionary<Guid, (string FileName, AssemblyName AssemblyName)> _mvidToRefInfoMap = new();
     private readonly Dictionary<string, Guid> _assemblyPathToMvidMap = new(PathUtil.Comparer);
@@ -235,12 +238,15 @@ private void AddGeneratedFiles(StreamWriter compilationWriter, CommandLineArgume
             return;
         }
 
+        var (languageGuid, languageExtension) = compilerCall.IsCSharp
+            ? (LanguageTypeCSharp, ".cs")
+            : (LanguageTypeBasic, ".vb");
         MetadataReaderProvider? pdbReaderProvider = null;
         try
         {
             using var reader = OpenFileForRead(assemblyFilePath);
             using var peReader = new PEReader(reader);
-            if (!peReader.TryOpenAssociatedPortablePdb(assemblyFilePath, OpenFileForRead, out pdbReaderProvider, out var pdbPath))
+            if (!peReader.TryOpenAssociatedPortablePdb(assemblyFilePath, OpenPortablePdbFile, out pdbReaderProvider, out var pdbPath))
             {
                 Diagnostics.Add($"Can't find portable pdb file for {compilerCall.GetDiagnosticName()}");
                 return;
@@ -249,7 +255,7 @@ private void AddGeneratedFiles(StreamWriter compilationWriter, CommandLineArgume
             var pdbReader = pdbReaderProvider!.GetMetadataReader();
             foreach (var documentHandle in pdbReader.Documents.Skip(args.SourceFiles.Length))
             {
-                if (GetContentStream(documentHandle) is {} tuple)
+                if (GetContentStream(documentHandle) is { } tuple)
                 {
                     var contentHash = AddContent(tuple.Stream);
                     compilationWriter.WriteLine($"generated:{contentHash}:{tuple.Name}");
@@ -259,7 +265,22 @@ private void AddGeneratedFiles(StreamWriter compilationWriter, CommandLineArgume
             (string Name, MemoryStream Stream)? GetContentStream(DocumentHandle documentHandle)
             {
                 var document = pdbReader.GetDocument(documentHandle);
+                if (pdbReader.GetGuid(document.Language) != languageGuid)
+                {
+                    return null;
+                }
+
                 var name = pdbReader.GetString(document.Name);
+
+                // A #line directive can be used to embed a file into the PDB. There is no way to differentiate
+                // between a file embedded this way and one generated from a source generator. For the moment
+                // using a simple hueristic to detect a generated file vs. say a .xaml file that was embedded
+                // https://github.com/jaredpar/basic-compilerlog/issues/45
+                if (Path.GetExtension(name) != languageExtension)
+                {
+                    return null;
+                }
+
                 foreach (var cdiHandle in pdbReader.GetCustomDebugInformation(documentHandle))
                 {
                     var cdi = pdbReader.GetCustomDebugInformation(cdiHandle);
@@ -304,6 +325,7 @@ private void AddGeneratedFiles(StreamWriter compilationWriter, CommandLineArgume
                         stream = decompressed;
                     }
 
+                    stream.Position = 0;
                     return (name, stream);
                 }
 
@@ -319,6 +341,18 @@ private void AddGeneratedFiles(StreamWriter compilationWriter, CommandLineArgume
         {
             pdbReaderProvider?.Dispose();
         }
+
+        // Similar to OpenFileForRead but don't throw here on file missing as it's expected that some files 
+        // will not have PDBs beside them.
+        static Stream? OpenPortablePdbFile(string filePath)
+        {
+            if (!File.Exists(filePath))
+            {
+                return null;
+            }
+
+            return new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
+        }
     }
 
     /// <summary>

src/Basic.CompilerLog.Util/ExportUtil.cs

@@ -148,7 +148,22 @@ public ExportUtil(CompilerLogReader reader, bool includeAnalyzers = true)
         IncludeAnalyzers = includeAnalyzers;
     }
 
-    public void ExportRsp(CompilerCall compilerCall, string destinationDir, IEnumerable<string> sdkDirectories)
+    public void ExportAll(string destinationDir, IEnumerable<string> sdkDirectories, Func<CompilerCall, bool>? predicate = null)
+    {
+        predicate ??= static _ => true;
+        for (int  i = 0; i < Reader.Count ; i++)
+        {
+            var compilerCall = Reader.ReadCompilerCall(i);
+            if (predicate(compilerCall))
+            {
+                var dir = Path.Combine(destinationDir, i.ToString());
+                Directory.CreateDirectory(dir);
+                Export(compilerCall, dir, sdkDirectories);
+            }
+        }
+    }
+
+    public void Export(CompilerCall compilerCall, string destinationDir, IEnumerable<string> sdkDirectories)
     {
         if (!Path.IsPathRooted(destinationDir))
         {

src/Basic.CompilerLog/FilterOptionSet.cs

@@ -11,7 +11,8 @@ internal sealed class FilterOptionSet : OptionSet
     internal FilterOptionSet()
     {
         Add("include", "include all compilation kinds", i => { if (i != null) IncludeAllKinds = true; });
-        Add("targetframework=", "include only compilations for the target framework (allows multiple)", TargetFrameworks.Add);
+        Add("targetframework=", "", TargetFrameworks.Add, hidden: true);
+        Add("framework=", "include only compilations for the target framework (allows multiple)", TargetFrameworks.Add);
         Add("n|projectName=", "include only compilations with the project name", (string n) => ProjectName = n);
         Add("h|help", "print help", h => { if (h != null) Help = true; });
     }

src/Basic.CompilerLog/Program.cs

@@ -314,7 +314,7 @@ int RunExport(IEnumerable<string> args)
         {
             var compilerCall = compilerCalls[i];
             var exportDir = GetOutputPath(baseOutputPath, compilerCalls, i, "export");
-            exportUtil.ExportRsp(compilerCall, exportDir, sdkDirs);
+            exportUtil.Export(compilerCall, exportDir, sdkDirs);
         }
 
         return ExitSuccess;

src/Scratch/Program.cs

@@ -1,5 +1,7 @@
-using Basic.CompilerLog;
+using System.Diagnostics;
+using Basic.CompilerLog;
 using Basic.CompilerLog.Util;
+using BenchmarkDotNet.Environments;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
@@ -10,11 +12,12 @@
 
 #pragma warning disable 8321
 
-var filePath = @"c:\users\jaredpar\temp\console\msbuild.binlog";
+// var filePath = @"c:\users\jaredpar\temp\console\msbuild.binlog";
+// var filePath = @"C:\Users\jaredpar\code\MudBlazor\src\msbuild.binlog";
 // var filePath = @"C:\Users\jaredpar\code\roslyn\src\Compilers\Core\Portable\msbuild.binlog";
 // var filePath = @"C:\Users\jaredpar\Downloads\Roslyn.complog";
 // var filePath = @"C:\Users\jaredpar\code\wt\ros2\artifacts\log\Debug\Build.binlog";
-// var filePath = @"C:\Users\jaredpar\code\roslyn\artifacts\log\Debug\Build.binlog";
+var filePath = @"C:\Users\jaredpar\code\roslyn\artifacts\log\Debug\Build.binlog";
 //var filePath = @"C:\Users\jaredpar\code\roslyn\src\Compilers\CSharp\csc\msbuild.binlog";
 
 //TestDiagnostics(filePath);
@@ -23,8 +26,7 @@
 
 // await SolutionScratchAsync(filePath);
 
-var reader = CompilerLogReader.Create(filePath);
-var all = reader.ReadAllCompilationData();
+VerifyAll(filePath);
 Console.WriteLine("Done");
 
 /*
@@ -101,16 +103,47 @@ static void C(Assembly assembly) { }
 
 }
 
+void VerifyAll(string logPath, BasicAnalyzerHostOptions? options = null)
+{
+    var exportDest = @"c:\users\jaredpar\temp\export";
+    EmptyDirectory(exportDest);
+
+    options ??= BasicAnalyzerHostOptions.None;
+    using var reader = CompilerLogReader.Create(logPath, options);
+    var exportUtil = new ExportUtil(reader, includeAnalyzers: options.Value.Kind != BasicAnalyzerKind.None);
+    var sdkDirs = DotnetUtil.GetSdkDirectories();
+    int failedCount = 0;
+    foreach (var compilationData in reader.ReadAllCompilationData())
+    {
+        Console.Write($"{compilationData.CompilerCall.GetDiagnosticName()} ...");
+        var result = compilationData.EmitToMemory();
+        if (result.Success)
+        {
+            Console.WriteLine("Success");
+            continue;
+        }
+
+        Console.WriteLine("Error");
+        foreach (var item in result.Diagnostics)
+        {
+            Console.WriteLine(item.GetMessage());
+        }
+
+        var dest = Path.Combine(exportDest, failedCount.ToString());
+        Console.WriteLine($"Exporting to {dest}");
+        exportUtil.Export(compilationData.CompilerCall, dest, sdkDirs);
+        failedCount++;
+    }
+}
 
-void ExportTest()
+void ExportTest(CompilerLogReader reader)
 {
     var dest = @"c:\users\jaredpar\temp\export";
     EmptyDirectory(dest);
 
     var d = DotnetUtil.GetSdkDirectories();
-    var reader = CompilerLogReader.Create(filePath!);
     var util = new ExportUtil(reader);
-    util.ExportRsp(reader.ReadCompilerCall(0), dest, DotnetUtil.GetSdkDirectories());
+    util.ExportAll(dest, d);
 }
 
 static void EmptyDirectory(string directory)