Report error for native PDB limit

src/Compilers/CSharp/Test/Emit2/PDB/PDBTests.cs

@@ -22,6 +22,7 @@
 using Roslyn.Test.PdbUtilities;
 using Roslyn.Test.Utilities;
 using Roslyn.Test.Utilities.TestGenerators;
+using Roslyn.Utilities;
 using Xunit;
 
 namespace Microsoft.CodeAnalysis.CSharp.UnitTests.PDB
@@ -367,6 +368,83 @@ public void SymWriterErrors4()
             Assert.False(result.Success);
         }
 
+        [ConditionalTheory(typeof(WindowsOnly), Reason = ConditionalSkipReason.NativePdbRequiresDesktop)]
+        [WorkItem("https://github.com/dotnet/roslyn/issues/75237")]
+        [InlineData(0x9_999)]
+        [InlineData(0x9_000)]
+        public void NativeWriterLimit_Under(int length)
+        {
+            CompileWithMockedCustomMetadata(length).Diagnostics.Verify();
+        }
+
+        [ConditionalTheory(typeof(WindowsOnly), Reason = ConditionalSkipReason.NativePdbRequiresDesktop)]
+        [WorkItem("https://github.com/dotnet/roslyn/issues/75237")]
+        [InlineData(0x10_000)]
+        [InlineData(0x20_000)]
+        public void NativeWriterLimit_Over(int length)
+        {
+            CompileWithMockedCustomMetadata(length).Diagnostics.Verify(
+                // error CS0041: Unexpected error writing debug information -- 'Insufficient memory to continue the execution of the program.'
+                Diagnostic(ErrorCode.FTL_DebugEmitFailure).WithArguments(new OutOfMemoryException().Message).WithLocation(1, 1));
+        }
+
+        private static EmitResult CompileWithMockedCustomMetadata(int length)
+        {
+            var comp = CreateCompilation("""
+                class C
+                {
+                    void M() { }
+                }
+                """);
+            return comp.Emit(
+                peStream: new MemoryStream(),
+                metadataPEStream: null,
+                pdbStream: new MemoryStream(),
+                xmlDocumentationStream: null,
+                cancellationToken: default,
+                win32Resources: null,
+                manifestResources: null,
+                options: null,
+                debugEntryPoint: null,
+                sourceLinkStream: null,
+                embeddedTexts: null,
+                rebuildData: null,
+                testData: new CompilationTestData
+                {
+                    SymWriterFactory = metadataProvider =>
+                    {
+                        var writer = SymWriterTestUtilities.CreateUnmanagedWriter(metadataProvider);
+                        return new CustomMetadataSymUnmanagedWriter(writer, new byte[length]);
+                    },
+                });
+        }
+
+        [ConditionalFact(typeof(WindowsOnly), Reason = ConditionalSkipReason.NativePdbRequiresDesktop)]
+        [WorkItem("https://github.com/dotnet/roslyn/issues/75237")]
+        public void NativeWriterLimit_EndToEnd()
+        {
+            var locals = Enumerable.Range(0, 14_000)
+                .Select(i => $"""
+                    var local{i} = {i};
+                    M2(local{i});
+                    """)
+                .Join(Environment.NewLine);
+            var source = $$"""
+                namespace N;
+                class C
+                {
+                    void M1()
+                    {
+                        {{locals}}
+                    }
+                    void M2(int x) { }
+                }
+                """;
+            CreateCompilation(source, options: TestOptions.DebugDll).VerifyEmitDiagnostics(
+                // error CS0041: Unexpected error writing debug information -- 'Cannot emit native PDB for method 'N.C.M1()' because its debug metadata size 69096 is over the limit 65536.'
+                Diagnostic(ErrorCode.FTL_DebugEmitFailure).WithArguments(string.Format(CodeAnalysisResources.SymWriterMetadataOverLimit, "N.C.M1()", 69096, 65536)).WithLocation(1, 1));
+        }
+
         [WorkItem(1067635, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/1067635")]
         [Fact]
         public void SuppressDynamicAndEncCDIForWinRT()

src/Compilers/Core/Portable/CodeAnalysisResources.resx

@@ -507,6 +507,9 @@
   <data name="SymWriterDoesNotSupportSourceLink" xml:space="preserve">
     <value>Windows PDB writer doesn't support SourceLink feature: '{0}'</value>
   </data>
+  <data name="SymWriterMetadataOverLimit" xml:space="preserve">
+    <value>Cannot emit native PDB for method '{0}' because its debug metadata size {1} is over the limit {2}.</value>
+  </data>
   <data name="RuleSetBadAttributeValue" xml:space="preserve">
     <value>The attribute {0} has an invalid value of {1}.</value>
   </data>

src/Compilers/Core/Portable/NativePdbWriter/PdbWriter.cs

@@ -148,6 +148,16 @@ public void SerializeDebugInfo(IMethodBody methodBody, StandaloneSignatureHandle
 
             if (blob.Length > 0)
             {
+                const int limit = 0x10_000;
+                if (blob.Length > limit)
+                {
+                    throw new SymUnmanagedWriterException(string.Format(
+                        CodeAnalysisResources.SymWriterMetadataOverLimit,
+                        methodBody.MethodDefinition,
+                        blob.Length,
+                        limit));
+                }
+
                 _symWriter.DefineCustomMetadata(blob);
             }
 

src/Compilers/VisualBasic/Test/Emit/PDB/PDBTests.vb

@@ -208,6 +208,31 @@ End Class
             result.Diagnostics.Verify(Diagnostic(ERRID.ERR_DebugEntryPointNotSourceMethodDefinition))
         End Sub
 
+        <ConditionalFact(GetType(WindowsOnly), Reason:=ConditionalSkipReason.NativePdbRequiresDesktop)>
+        <WorkItem("https://github.com/dotnet/roslyn/issues/75237")>
+        Public Sub NativeWriterLimit()
+            Dim locals = Enumerable.Range(0, 14_000).
+                Select(Function(i) $"
+Dim local{i} As Integer = {i}
+M2(local{i})
+").
+                Join(Environment.NewLine)
+            Dim source = $"
+Namespace N
+Class C
+    Shared Sub M1()
+        {locals}
+    End Sub
+    Shared Sub M2(x As Integer)
+    End Sub
+End Class
+End Namespace
+"
+            ' Cannot emit native PDB for method 'Public Shared Sub M1()' because its debug metadata size 69328 is over the limit 65536.
+            CreateCompilation(source, options:=TestOptions.DebugDll).VerifyEmitDiagnostics(
+                Diagnostic(ERRID.ERR_PDBWritingFailed).WithArguments(String.Format(CodeAnalysisResources.SymWriterMetadataOverLimit, "Public Shared Sub M1()", 69328, 65536)).WithLocation(1, 1))
+        End Sub
+
 #End Region
 
         <ConditionalFact(GetType(WindowsOnly), Reason:=ConditionalSkipReason.NativePdbRequiresDesktop)>

src/Test/PdbUtilities/Writer/CustomMetadataSymUnmanagedWriter.cs

@@ -0,0 +1,17 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.DiaSymReader;
+
+namespace Roslyn.Test.PdbUtilities;
+
+internal sealed class CustomMetadataSymUnmanagedWriter(SymUnmanagedWriter target, byte[] customMetadata) : DelegatingSymUnmanagedWriter(target)
+{
+    private readonly byte[] _customMetadata = customMetadata;
+
+    public override void DefineCustomMetadata(byte[] metadata)
+    {
+        base.DefineCustomMetadata(_customMetadata);
+    }
+}