Reapply "Load analyzers and generators in isolated ALCs in our OOP process"

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer/HostWorkspace/LanguageServerWorkspaceFactory.cs

@@ -57,7 +57,12 @@ public LanguageServerWorkspaceFactory(
  public async Task InitializeSolutionLevelAnalyzersAsync(ImmutableArray<string> analyzerPaths)
  {
  var references = new List<AnalyzerFileReference>();
- var analyzerLoader = Workspace.Services.GetRequiredService<IAnalyzerAssemblyLoaderProvider>().SharedShadowCopyLoader;
+ var loaderProvider = Workspace.Services.GetRequiredService<IAnalyzerAssemblyLoaderProvider>();
+
+ // Load all analyzers into a fresh shadow copied load context. In the future, if we want to support reloading
+ // of solution-level analyzer references, we should just need to listen for changes to those analyzer paths and
+ // then call back into this method to update the solution accordingly.
+ var analyzerLoader = loaderProvider.CreateNewShadowCopyLoader();
 
  foreach (var analyzerPath in analyzerPaths)
  {

src/VisualStudio/Core/Test.Next/Remote/SerializationValidator.cs

@@ -110,7 +110,8 @@ public async Task<T> GetValueAsync<T>(Checksum checksum)
 
  public async Task<Solution> GetSolutionAsync(SolutionAssetStorage.Scope scope)
  {
- var solutionInfo = await new AssetProvider(this).CreateSolutionInfoAsync(scope.SolutionChecksum, CancellationToken.None).ConfigureAwait(false);
+ var solutionInfo = await new AssetProvider(this).CreateSolutionInfoAsync(
+ scope.SolutionChecksum, this.Services.SolutionServices, CancellationToken.None).ConfigureAwait(false);
 
  var workspace = new AdhocWorkspace(Services.HostServices);
  return workspace.AddSolution(solutionInfo);

src/VisualStudio/Core/Test.Next/Services/SolutionServiceTests.cs

@@ -15,6 +15,7 @@
 using Microsoft.CodeAnalysis.Diagnostics;
 using Microsoft.CodeAnalysis.Editor.Test;
 using Microsoft.CodeAnalysis.Formatting;
+using Microsoft.CodeAnalysis.Host;
 using Microsoft.CodeAnalysis.Remote;
 using Microsoft.CodeAnalysis.Remote.Testing;
 using Microsoft.CodeAnalysis.Serialization;
@@ -446,7 +447,10 @@ public async Task TestRemoteWorkspace()
  await Verify(remoteWorkspace, currentSolution, remoteSolution3);
 
  // move to new solution backward
- var solutionInfo2 = await assetProvider.CreateSolutionInfoAsync(await solution1.CompilationState.GetChecksumAsync(CancellationToken.None), CancellationToken.None);
+ var solutionInfo2 = await assetProvider.CreateSolutionInfoAsync(
+ await solution1.CompilationState.GetChecksumAsync(CancellationToken.None),
+ remoteWorkspace.Services.SolutionServices,
+ CancellationToken.None);
  var solution2 = remoteWorkspace.GetTestAccessor().CreateSolutionFromInfo(solutionInfo2);
 
  // move to new solution forward

src/Workspaces/Core/Portable/Serialization/SerializerService_Reference.cs

@@ -8,7 +8,6 @@
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Reflection.Metadata;
-using System.Threading;
 using Microsoft.CodeAnalysis.Diagnostics;
 using Microsoft.CodeAnalysis.Host;
 using Microsoft.CodeAnalysis.Shared.Extensions;
@@ -32,7 +31,7 @@ internal partial class SerializerService
  private static readonly object s_analyzerImageReferenceMapGate = new();
  private static IBidirectionalMap<AnalyzerImageReference, Guid> s_analyzerImageReferenceMap = BidirectionalMap<AnalyzerImageReference, Guid>.Empty;
 
- private static bool TryGetAnalyzerImageReferenceGuid(AnalyzerImageReference imageReference, out Guid guid)
+ public static bool TryGetAnalyzerImageReferenceGuid(AnalyzerImageReference imageReference, out Guid guid)
  {
  lock (s_analyzerImageReferenceMapGate)
  return s_analyzerImageReferenceMap.TryGetValue(imageReference, out guid);
@@ -54,15 +53,24 @@ private static Checksum CreateChecksum(MetadataReference reference)
 
  protected virtual Checksum CreateChecksum(AnalyzerReference reference)
  {
+#if NET
+ // If we're in the oop side and we're being asked to produce our local checksum (so we can compare it to the
+ // host checksum), then we want to just defer to the underlying analyzer reference of our isolated reference.
+ // This underlying reference corresponds to the reference that the host has, and we do not want to make any
+ // changes as long as they're both in agreement.
+ if (reference is IsolatedAnalyzerFileReference { UnderlyingAnalyzerFileReference: var underlyingReference })
+ reference = underlyingReference;
+#endif
+
  using var stream = SerializableBytes.CreateWritableStream();
 
  using (var writer = new ObjectWriter(stream, leaveOpen: true))
  {
  switch (reference)
  {
- case AnalyzerFileReference file:
- writer.WriteString(file.FullPath);
- writer.WriteGuid(TryGetAnalyzerFileReferenceMvid(file));
+ case AnalyzerFileReference fileReference:
+ writer.WriteString(fileReference.FullPath);
+ writer.WriteGuid(TryGetAnalyzerFileReferenceMvid(fileReference));
  break;
 
  case AnalyzerImageReference analyzerImageReference:
@@ -109,11 +117,11 @@ protected virtual void WriteAnalyzerReferenceTo(AnalyzerReference reference, Obj
  {
  switch (reference)
  {
- case AnalyzerFileReference file:
+ case AnalyzerFileReference fileReference:
  writer.WriteString(nameof(AnalyzerFileReference));
- writer.WriteString(file.FullPath);
+ writer.WriteString(fileReference.FullPath);
 
- // Note: it is intentional that we are not writing the MVID of the analyzer file reference over (even
+ // Note: it is intentional that we are not writing the MVID of the analyzer file reference over in (even
  // though we mixed it into the checksum). We don't actually need the data on the other side as it will
  // be read out from the file itself. So the flow is as follows when an analyzer-file-reference changes:
  //
@@ -150,8 +158,10 @@ protected virtual AnalyzerReference ReadAnalyzerReferenceFrom(ObjectReader reade
  switch (reader.ReadString())
  {
  case nameof(AnalyzerFileReference):
- var fullPath = reader.ReadRequiredString();
- return new AnalyzerFileReference(fullPath, _analyzerLoaderProvider.SharedShadowCopyLoader);
+ // Rehydrate the analyzer file reference with the simple shared shadow copy loader. Note: we won't
+ // actually use this instance we create. Instead, the caller will use create an IsolatedAssemblyReferenceSet
+ // from these to ensure that all the types can be safely loaded into their own ALC.
+ return new AnalyzerFileReference(reader.ReadRequiredString(), _analyzerLoaderProvider.SharedShadowCopyLoader);
 
  case nameof(AnalyzerImageReference):
  var guid = reader.ReadGuid();
@@ -286,7 +296,7 @@ private PortableExecutableReference ReadPortableExecutableReferenceFrom(ObjectRe
  // so that we can put xml doc comment as part of snapshot. but until we believe that is necessary,
  // it will go with simpler approach
  var documentProvider = filePath != null && _documentationService != null ?
- _documentationService.GetDocumentationProvider(filePath) : XmlDocumentationProvider.Default;
+ _documentationService.GetDocumentationProvider(filePath) : DocumentationProvider.Default;
 
  return new SerializedPortableExecutableReference(
  properties, filePath, metadata, storageHandles, documentProvider);
@@ -492,7 +502,7 @@ private static void WriteUnresolvedAnalyzerReferenceTo(AnalyzerReference referen
  }
  }
 
- private static Guid TryGetAnalyzerFileReferenceMvid(AnalyzerFileReference file)
+ public static Guid TryGetAnalyzerFileReferenceMvid(AnalyzerFileReference file)
  {
  try
  {

src/Workspaces/Core/Portable/Workspace/Host/Metadata/IAnalyzerAssemblyLoaderProvider.cs

@@ -8,12 +8,26 @@
 using System.Composition;
 using System.IO;
 using Microsoft.CodeAnalysis.Host.Mef;
+using System.Threading;
+
+#if NET
+using Microsoft.CodeAnalysis.Diagnostics;
+using System.Runtime.Loader;
+#endif
 
 namespace Microsoft.CodeAnalysis.Host;
 
 internal interface IAnalyzerAssemblyLoaderProvider : IWorkspaceService
 {
  IAnalyzerAssemblyLoaderInternal SharedShadowCopyLoader { get; }
+
+#if NET
+ /// <summary>
+ /// Creates a fresh shadow copying loader that will load all <see cref="AnalyzerReference"/>s and <see
+ /// cref="ISourceGenerator"/>s in a fresh <see cref="AssemblyLoadContext"/>.
+ /// </summary>
+ IAnalyzerAssemblyLoaderInternal CreateNewShadowCopyLoader();
+#endif
 }
 
 /// <summary>
@@ -28,13 +42,13 @@ internal abstract class AbstractAnalyzerAssemblyLoaderProvider : IAnalyzerAssemb
  public AbstractAnalyzerAssemblyLoaderProvider(IEnumerable<IAnalyzerAssemblyResolver> externalResolvers)
  {
  _externalResolvers = externalResolvers.ToImmutableArray();
- _shadowCopyLoader = new(CreateShadowCopyLoader);
+ _shadowCopyLoader = new(CreateNewShadowCopyLoader);
  }
 
  public IAnalyzerAssemblyLoaderInternal SharedShadowCopyLoader
  => _shadowCopyLoader.Value;
 
- private IAnalyzerAssemblyLoaderInternal CreateShadowCopyLoader()
+ public IAnalyzerAssemblyLoaderInternal CreateNewShadowCopyLoader()
  => this.WrapLoader(DefaultAnalyzerAssemblyLoader.CreateNonLockingLoader(
  Path.Combine(Path.GetTempPath(), nameof(Roslyn), "AnalyzerAssemblyLoader"),
  _externalResolvers));

src/Workspaces/Core/Portable/Workspace/IsolatedAnalyzerFileReference.cs

@@ -0,0 +1,114 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#if NET
+
+using System;
+using System.Collections.Immutable;
+using System.Runtime.CompilerServices;
+using Microsoft.CodeAnalysis.Diagnostics;
+using System.Runtime.Loader;
+using Roslyn.Utilities;
+
+namespace Microsoft.CodeAnalysis;
+
+/// <summary>
+/// Wrapper around a real <see cref="AnalyzerFileReference"/>. An "isolated" analyzer reference is an analyzer
+/// reference associated with an <see cref="AssemblyLoadContext"/> that is connected to a set of other "isolated"
+/// analyzer references. This allows for loading the analyzers and generators from it in a way that is associated with
+/// that load context, keeping them separate from other analyzers and generators loaded in other load contexts, while
+/// also allowing all of those instances to be collected when no longer needed. Being isolated means that if any of the
+/// underlying assembly references change, that they can be loaded side by side with the prior references. This enables
+/// functionality like live reloading of analyzers and generators when they change on disk. Note: this is only
+/// supported on .Net Core, and not .Net Framework, as only the former has <see cref="AssemblyLoadContext"/>s.
+/// </summary>
+/// <remarks>
+/// The purpose of this type is to allow passing out a <see cref="AnalyzerReference"/> to the rest of the system that
+/// then ensures that as long as it is alive (or any <see cref="DiagnosticAnalyzer"/> or <see cref="ISourceGenerator"/>
+/// it passes out is alive), that the <see cref="IsolatedAnalyzerReferenceSet"/> (and its corresponding <see
+/// cref="AssemblyLoadContext"/>) is kept alive as well.
+/// </remarks>
+internal sealed class IsolatedAnalyzerFileReference(
+ IsolatedAnalyzerReferenceSet isolatedAnalyzerReferenceSet,
+ AnalyzerFileReference underlyingAnalyzerReference)
+ : AnalyzerReference
+{
+ /// <summary>
+ /// Conditional weak tables that ensure that as long as a particular <see cref="DiagnosticAnalyzer"/> or <see
+ /// cref="ISourceGenerator"/> is alive, that the corresponding <see cref="IsolatedAnalyzerReferenceSet"/> (and its
+ /// corresponding <see cref="AssemblyLoadContext"/> is kept alive.
+ /// </summary>
+ private static readonly ConditionalWeakTable<DiagnosticAnalyzer, IsolatedAnalyzerReferenceSet> s_analyzerToPinnedReferenceSet = [];
+
+ /// <inheritdoc cref="s_analyzerToPinnedReferenceSet"/>
+ private static readonly ConditionalWeakTable<ISourceGenerator, IsolatedAnalyzerReferenceSet> s_generatorToPinnedReferenceSet = [];
+
+ /// <summary>
+ /// We keep a strong reference here. As long as this <see cref="IsolatedAnalyzerFileReference"/> is passed out and
+ /// held onto (say by a Project instance), it should keep the IsolatedAssemblyReferenceSet (and its ALC) alive.
+ /// </summary>
+ private readonly IsolatedAnalyzerReferenceSet _isolatedAnalyzerReferenceSet = isolatedAnalyzerReferenceSet;
+
+ /// <summary>
+ /// The actual real <see cref="AnalyzerReference"/> we defer our operations to.
+ /// </summary>
+ public readonly AnalyzerFileReference UnderlyingAnalyzerFileReference = underlyingAnalyzerReference;
+
+ public override string Display => UnderlyingAnalyzerFileReference.Display;
+ public override string? FullPath => UnderlyingAnalyzerFileReference.FullPath;
+ public override object Id => UnderlyingAnalyzerFileReference.Id;
+
+ public override ImmutableArray<DiagnosticAnalyzer> GetAnalyzers(string language)
+ => PinAnalyzers(static (reference, language) => reference.GetAnalyzers(language), language);
+
+ public override ImmutableArray<DiagnosticAnalyzer> GetAnalyzersForAllLanguages()
+ => PinAnalyzers(static (reference, _) => reference.GetAnalyzersForAllLanguages(), default(VoidResult));
+
+ [Obsolete]
+ public override ImmutableArray<ISourceGenerator> GetGenerators()
+ => PinGenerators(static (reference, _) => reference.GetGenerators(), default(VoidResult));
+
+ public override ImmutableArray<ISourceGenerator> GetGenerators(string language)
+ => PinGenerators(static (reference, language) => reference.GetGenerators(language), language);
+
+ public override ImmutableArray<ISourceGenerator> GetGeneratorsForAllLanguages()
+ => PinGenerators(static (reference, _) => reference.GetGeneratorsForAllLanguages(), default(VoidResult));
+
+ private ImmutableArray<DiagnosticAnalyzer> PinAnalyzers<TArg>(Func<AnalyzerReference, TArg, ImmutableArray<DiagnosticAnalyzer>> getItems, TArg arg)
+ => PinItems(s_analyzerToPinnedReferenceSet, getItems, arg);
+
+ private ImmutableArray<ISourceGenerator> PinGenerators<TArg>(Func<AnalyzerReference, TArg, ImmutableArray<ISourceGenerator>> getItems, TArg arg)
+ => PinItems(s_generatorToPinnedReferenceSet, getItems, arg);
+
+ private ImmutableArray<TItem> PinItems<TItem, TArg>(
+ ConditionalWeakTable<TItem, IsolatedAnalyzerReferenceSet> table,
+ Func<AnalyzerReference, TArg, ImmutableArray<TItem>> getItems,
+ TArg arg)
+ where TItem : class
+ {
+ // Keep a reference from each generator to the IsolatedAssemblyReferenceSet. This will ensure it (and the ALC
+ // it points at) stays alive as long as the generator instance stays alive.
+ var items = getItems(this.UnderlyingAnalyzerFileReference, arg);
+
+ foreach (var item in items)
+ table.TryAdd(item, _isolatedAnalyzerReferenceSet);
+
+ // Note: we want to keep ourselves alive during this call so that neither we nor our reference set get GC'ed
+ // while we're computing the items.
+ GC.KeepAlive(this);
+
+ return items;
+ }
+
+ public override bool Equals(object? obj)
+ => ReferenceEquals(this, obj);
+
+ public override int GetHashCode()
+ => RuntimeHelpers.GetHashCode(this);
+
+ public override string ToString()
+ => $"{nameof(IsolatedAnalyzerFileReference)}({UnderlyingAnalyzerFileReference})";
+}
+
+#endif