diff --git a/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapDumpGraphReader.cs b/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapDumpGraphReader.cs index af5b45ebdb..be291f223c 100644 --- a/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapDumpGraphReader.cs +++ b/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapDumpGraphReader.cs @@ -1,4 +1,4 @@ -using Graphs; +using Graphs; using Microsoft.Diagnostics.Tracing; using Microsoft.Diagnostics.Tracing.Parsers; using Microsoft.Diagnostics.Tracing.Parsers.Clr; @@ -182,6 +182,12 @@ internal void SetupCallbacks(MemoryGraph memoryGraph, TraceEventDispatcher sourc } }; + source.Clr.GCGenAwareStart += delegate (GenAwareBeginTraceData data) + { + m_seenStart = true; + m_ignoreEvents = false; + }; + source.Clr.GCStart += delegate (GCStartTraceData data) { // If this GC is not part of a heap dump, ignore it. @@ -231,8 +237,6 @@ internal void SetupCallbacks(MemoryGraph memoryGraph, TraceEventDispatcher sourc } }; - - source.Clr.GCStop += delegate (GCEndTraceData data) { if (m_ignoreEvents || data.ProcessID != m_processId) @@ -262,6 +266,17 @@ internal void SetupCallbacks(MemoryGraph memoryGraph, TraceEventDispatcher sourc } }; + source.Clr.GCGenAwareEnd += delegate (GenAwareEndTraceData data) + { + m_ignoreEvents = true; + if (m_nodeBlocks.Count == 0 && m_typeBlocks.Count == 0 && m_edgeBlocks.Count == 0) + { + m_log.WriteLine("Found no node events, looking for another GC"); + m_seenStart = false; + return; + } + }; + source.Clr.TypeBulkType += delegate (GCBulkTypeTraceData data) { // Don't check m_ignoreEvents here, as BulkType events can be emitted by other events...such as the GC allocation event. @@ -474,6 +489,9 @@ internal void SetupCallbacks(MemoryGraph memoryGraph, TraceEventDispatcher sourc case 3: segment.Gen3End = end; break; + case 4: + segment.Gen4End = end; + break; default: throw new Exception("Invalid generation in GCGenerationRangeTraceData"); } diff --git a/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapInfo.cs b/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapInfo.cs index 27f01b9c16..8f71b0808f 100644 --- a/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapInfo.cs +++ b/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapInfo.cs @@ -52,6 +52,11 @@ public int GenerationFor(Address obj) } } + if (obj < m_lastSegment.Gen4End) + { + return 4; + } + if (obj < m_lastSegment.Gen3End) { return 3; @@ -107,7 +112,7 @@ void IFastSerializable.FromStream(Deserializer deserializer) #endregion } -public class GCHeapDumpSegment : IFastSerializable +public class GCHeapDumpSegment : IFastSerializable, IFastSerializableVersion { public Address Start { get; internal set; } public Address End { get; internal set; } @@ -115,6 +120,13 @@ public class GCHeapDumpSegment : IFastSerializable public Address Gen1End { get; internal set; } public Address Gen2End { get; internal set; } public Address Gen3End { get; internal set; } + public Address Gen4End { get; internal set; } + + public int Version => 1; + + public int MinimumVersionCanRead => 0; + + public int MinimumReaderVersion => 1; #region private void IFastSerializable.ToStream(Serializer serializer) @@ -125,6 +137,7 @@ void IFastSerializable.ToStream(Serializer serializer) serializer.Write((long)Gen1End); serializer.Write((long)Gen2End); serializer.Write((long)Gen3End); + serializer.Write((long)Gen4End); } void IFastSerializable.FromStream(Deserializer deserializer) @@ -135,6 +148,10 @@ void IFastSerializable.FromStream(Deserializer deserializer) Gen1End = (Address)deserializer.ReadInt64(); Gen2End = (Address)deserializer.ReadInt64(); Gen3End = (Address)deserializer.ReadInt64(); + if (deserializer.VersionBeingRead >= 1) + { + Gen4End = (Address)deserializer.ReadInt64(); + } } #endregion -} \ No newline at end of file +} diff --git a/src/Tools/dotnet-gcdump/DotNetHeapDump/GCHeapDump.cs b/src/Tools/dotnet-gcdump/DotNetHeapDump/GCHeapDump.cs index 424be99dab..abfdcd047e 100644 --- a/src/Tools/dotnet-gcdump/DotNetHeapDump/GCHeapDump.cs +++ b/src/Tools/dotnet-gcdump/DotNetHeapDump/GCHeapDump.cs @@ -1,11 +1,10 @@ -using FastSerialization; +using FastSerialization; using Graphs; using Microsoft.Diagnostics.Utilities; using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Security; using System.Text.RegularExpressions; using System.Xml; using Address = System.UInt64; @@ -18,11 +17,11 @@ public class GCHeapDump : IFastSerializable, IFastSerializableVersion { public GCHeapDump(string inputFileName) : - this(new Deserializer(inputFileName)) + this(new Deserializer(inputFileName, new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.FourBytes })) { } public GCHeapDump(Stream inputStream, string streamName) : - this(new Deserializer(inputStream, streamName)) + this(new Deserializer(inputStream, streamName, new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.FourBytes })) { } /// @@ -110,7 +109,7 @@ public static Dictionary GetProcessesWithGCHeaps() var ret = new Dictionary(); // Do the 64 bit processes first, then do us - if (System.Environment.Is64BitOperatingSystem && !System.Environment.Is64BitProcess) + if (EnvironmentUtilities.Is64BitOperatingSystem && !EnvironmentUtilities.Is64BitProcess) { GetProcessesWithGCHeapsFromHeapDump(ret); } @@ -193,7 +192,7 @@ public static Dictionary GetProcessesWithGCHeaps() private void Write(string outputFileName) { Debug.Assert(MemoryGraph != null); - var serializer = new Serializer(outputFileName, this); + var serializer = new Serializer(new IOStreamStreamWriter(outputFileName, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.FourBytes }), this); serializer.Close(); } @@ -848,7 +847,7 @@ internal static void WriteGCDumpToXml(GCHeapDump gcDump, StreamWriter writer) writer.WriteLine("{0}", gcDump.TimeCollected); if (!string.IsNullOrWhiteSpace(gcDump.CollectionLog)) { - writer.WriteLine("{0}", SecurityElement.Escape(gcDump.CollectionLog)); + writer.WriteLine("{0}", XmlUtilities.XmlEscape(gcDump.CollectionLog)); } if (!string.IsNullOrWhiteSpace(gcDump.MachineName)) @@ -858,7 +857,7 @@ internal static void WriteGCDumpToXml(GCHeapDump gcDump, StreamWriter writer) if (!string.IsNullOrWhiteSpace(gcDump.ProcessName)) { - writer.WriteLine("{0}", SecurityElement.Escape(gcDump.ProcessName)); + writer.WriteLine("{0}", XmlUtilities.XmlEscape(gcDump.ProcessName)); } if (gcDump.ProcessID != 0) @@ -883,7 +882,7 @@ internal static void WriteGCDumpToXml(GCHeapDump gcDump, StreamWriter writer) for (int i = 0; i < gcDump.CountMultipliersByType.Length; i++) { writer.WriteLine("", i, - SecurityElement.Escape(gcDump.MemoryGraph.GetType((NodeTypeIndex)i, typeStorage).Name), + XmlUtilities.XmlEscape(gcDump.MemoryGraph.GetType((NodeTypeIndex)i, typeStorage).Name), gcDump.CountMultipliersByType[i]); } @@ -1087,3 +1086,4 @@ private static float FetchFloat(XmlReader reader, string attributeName, float de } + diff --git a/src/Tools/dotnet-gcdump/DotNetHeapDump/Graph.cs b/src/Tools/dotnet-gcdump/DotNetHeapDump/Graph.cs index 23bed8b0e3..305c3b25a5 100644 --- a/src/Tools/dotnet-gcdump/DotNetHeapDump/Graph.cs +++ b/src/Tools/dotnet-gcdump/DotNetHeapDump/Graph.cs @@ -7,11 +7,8 @@ using System.IO; using System.Text; using System.Text.RegularExpressions; -using System.Security; using Address = System.UInt64; -// Copy of version in Microsoft/PerfView - // Graph contains generic Graph-Node traversal algorithms (spanning tree etc). namespace Graphs { @@ -43,8 +40,8 @@ namespace Graphs /// nodes with the code:Graph.AllocNodeStorage call /// /// Thus the basic flow is you call code:Graph.AllocNodeStorage to allocate storage, then call code:Graph.GetRoot - /// to get your first node. If you need to provide additional information about the nodes, you can allocate an auxiliary - /// array of Size code:Graph.NodeIndexLimit to hold it (for example a 'visited' bit). Then repeatedly call + /// to get your first node. If you need to 'hang' additional information off he nodes, you allocate an array + /// of Size code:Graph.NodeIndexLimit to hold it (for example a 'visited' bit). Then repeatedly call /// code:Node.GetFirstChild, code:Node.GetNextChild to get the children of a node to traverse the graph. /// /// OVERHEAD @@ -72,7 +69,7 @@ public class Graph : IFastSerializable, IFastSerializableVersion /// Given an arbitrary code:NodeIndex that identifies the node, Get a code:Node object. /// /// This routine does not allocated but uses the space passed in by 'storage. - /// 'storage' should be allocated with coode:AllocNodeStorage, and should be agressively reused. + /// 'storage' should be allocated with coode:AllocNodeStorage, and should be aggressively reused. /// public Node GetNode(NodeIndex nodeIndex, Node storage) { @@ -89,7 +86,7 @@ public Node GetNode(NodeIndex nodeIndex, Node storage) /// Given an arbitrary code:NodeTypeIndex that identifies the nodeId of the node, Get a code:NodeType object. /// /// This routine does not allocated but overwrites the space passed in by 'storage'. - /// 'storage' should be allocated with coode:AllocNodeTypeStorage, and should be agressively reused. + /// 'storage' should be allocated with coode:AllocNodeTypeStorage, and should be aggressively reused. /// /// Note that this routine does not get used much, instead Node.GetType is normal way of getting the nodeId. /// @@ -123,9 +120,9 @@ public virtual NodeType AllocTypeNodeStorage() /// public NodeIndex NodeIndexLimit { get { return (NodeIndex)m_nodes.Count; } } /// - /// Same as NodeIndexLimit, just cast to an integer. + /// Same as NodeIndexLimit. /// - public int NodeCount { get { return m_nodes.Count; } } + public long NodeCount { get { return m_nodes.Count; } } /// /// It is expected that users will want additional information associated with TYPES of the nodes of the graph. They can /// do this by allocating an array of code:NodeTypeIndexLimit and then indexing this by code:NodeTypeIndex @@ -164,8 +161,11 @@ public virtual NodeType AllocTypeNodeStorage() /// /// TODO I can eliminate the need for AllowReading. /// - public Graph(int expectedNodeCount) + /// if isVeryLargeGraph argument is true, then StreamLabels will be serialized as longs + /// too acommodate for the extra size of the graph's stream representation. + public Graph(int expectedNodeCount, bool isVeryLargeGraph = false) { + m_isVeryLargeGraph = isVeryLargeGraph; m_expectedNodeCount = expectedNodeCount; m_types = new GrowableArray(Math.Max(expectedNodeCount / 100, 2000)); m_nodes = new SegmentedList(SegmentSize, m_expectedNodeCount); @@ -407,7 +407,7 @@ public string HistogramByTypeXml(long minSize = 0) } sw.WriteLine(" ", - SecurityElement.Escape(GetType(sizeAndCount.TypeIdx, typeStorage).Name), sizeAndCount.Size, sizeAndCount.Count); + XmlUtilities.XmlEscape(GetType(sizeAndCount.TypeIdx, typeStorage).Name), sizeAndCount.Size, sizeAndCount.Count); } sw.WriteLine(""); return sw.ToString(); @@ -465,7 +465,8 @@ private void ClearWorker() RootIndex = NodeIndex.Invalid; if (m_writer == null) { - m_writer = new SegmentedMemoryStreamWriter(m_expectedNodeCount * 8); + m_writer = new SegmentedMemoryStreamWriter(m_expectedNodeCount * 8, + m_isVeryLargeGraph ? new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.EightBytes } : null); } m_totalSize = 0; @@ -512,8 +513,16 @@ public virtual void ToStream(Serializer serializer) serializer.Write(m_types[i].ModuleName); } - // Write out the Nodes - serializer.Write(m_nodes.Count); + // Write out the Nodes + if (m_isVeryLargeGraph) + { + serializer.Write(m_nodes.Count); + } + else + { + serializer.Write((int)m_nodes.Count); + } + for (int i = 0; i < m_nodes.Count; i++) { serializer.Write((int)m_nodes[i]); @@ -551,7 +560,7 @@ public virtual void ToStream(Serializer serializer) // You can place tagged values in here always adding right before the WriteTaggedEnd // for any new fields added after version 1 - serializer.WriteTaggedEnd(); // This insures tagged things don't read junk after the region. + serializer.WriteTaggedEnd(); // This ensures tagged things don't read junk after the region. }); } } @@ -574,10 +583,10 @@ public void FromStream(Deserializer deserializer) } // Read in the Nodes - int nodeCount = deserializer.ReadInt(); + long nodeCount = m_isVeryLargeGraph ? deserializer.ReadInt64() : deserializer.ReadInt(); m_nodes = new SegmentedList(SegmentSize, nodeCount); - for (int i = 0; i < nodeCount; i++) + for (long i = 0; i < nodeCount; i++) { m_nodes.Add((StreamLabel)(uint)deserializer.ReadInt()); } @@ -585,7 +594,9 @@ public void FromStream(Deserializer deserializer) // Read in the Blob stream. // TODO be lazy about reading in the blobs. int blobCount = deserializer.ReadInt(); - SegmentedMemoryStreamWriter writer = new SegmentedMemoryStreamWriter(blobCount); + SegmentedMemoryStreamWriter writer = new SegmentedMemoryStreamWriter(blobCount, + m_isVeryLargeGraph ? new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.EightBytes } : null); + while (8 <= blobCount) { writer.Write(deserializer.ReadInt64()); @@ -644,7 +655,7 @@ public void FromStream(Deserializer deserializer) } } - private int m_expectedNodeCount; // Initial guess at graph Size. + private long m_expectedNodeCount; // Initial guess at graph Size. private long m_totalSize; // Total Size of all the nodes in the graph. internal int m_totalRefs; // Total Number of references in the graph internal GrowableArray m_types; // We expect only thousands of these @@ -656,6 +667,7 @@ public void FromStream(Deserializer deserializer) // There should not be any of these left as long as every node referenced // by another node has a definition. internal SegmentedMemoryStreamWriter m_writer; // Used only during construction to serialize the nodes. + protected bool m_isVeryLargeGraph; #endregion } @@ -796,7 +808,7 @@ public virtual void WriteXml(TextWriter writer, bool includeChildren = true, str } writer.Write("{0}", prefix, (int)Index, SecurityElement.Escape(Name)); + writer.WriteLine("{0}", prefix, (int)Index, XmlUtilities.XmlEscape(Name)); } #region private protected internal NodeType(Graph graph) @@ -1062,7 +1074,7 @@ public class Module : IFastSerializable /// public DateTime BuildTime; // From in the PE header /// - /// The name of hte PDB file assoicated with this module. Ma bye null if unknown + /// The name of hte PDB file associated with this module. Ma bye null if unknown /// public string PdbName; /// @@ -1244,7 +1256,7 @@ public static void DumpNormalized(this MemoryGraph graph, TextWriter writer) node = graph.GetNode(graph.RootIndex, nodeStorage); writer.WriteLine("", - SecurityElement.Escape(node.GetType(typeStorage).Name), + XmlUtilities.XmlEscape(node.GetType(typeStorage).Name), graph.NodeIndexLimit, graph.NodeTypeIndexLimit, graph.TotalSize, @@ -1259,7 +1271,7 @@ public static void DumpNormalized(this MemoryGraph graph, TextWriter writer) node = graph.GetNode(nodeIdx, nodeStorage); string name = node.GetType(typeStorage).Name; - writer.Write(" ", graph.GetAddress(nodeIdx), node.Size, SecurityElement.Escape(name)); + writer.Write(" ", graph.GetAddress(nodeIdx), node.Size, XmlUtilities.XmlEscape(name)); bool isRoot = graph.GetAddress(node.Index) == 0; int childCnt = 0; for (var childIndex = node.GetFirstChildIndex(); childIndex != NodeIndex.Invalid; childIndex = node.GetNextChildIndex()) @@ -1373,7 +1385,7 @@ public RefGraph(Graph graph) /// Given an arbitrary code:NodeIndex that identifies the node, Get a code:Node object. /// /// This routine does not allocated but uses the space passed in by 'storage. - /// 'storage' should be allocated with coode:AllocNodeStorage, and should be agressively reused. + /// 'storage' should be allocated with coode:AllocNodeStorage, and should be aggressively reused. /// public RefNode GetNode(NodeIndex nodeIndex, RefNode storage) { @@ -1826,8 +1838,8 @@ private void AddOrphansToQueue(PriorityQueue nodesToVisit) /// /// A helper for AddOrphansToQueue, so we only add orphans that are not reachable from other orphans. /// - /// Mark all decendents (but not nodeIndex itself) as being visited. Any arcs that form - /// cycles are ignored, so nodeIndex is guarenteed to NOT be marked. + /// Mark all descendants (but not nodeIndex itself) as being visited. Any arcs that form + /// cycles are ignored, so nodeIndex is guaranteed to NOT be marked. /// private void MarkDecendentsIgnoringCycles(NodeIndex nodeIndex, int recursionCount) { @@ -1932,7 +1944,7 @@ private void SetTypePriorities(string priorityPats) var m = Regex.Match(priorityPatArray[i], @"(.*)->(-?\d+.?\d*)"); if (!m.Success) { - if (string.IsNullOrWhiteSpace(priorityPatArray[i])) + if (StringUtilities.IsNullOrWhiteSpace(priorityPatArray[i])) { continue; } @@ -2390,7 +2402,7 @@ private void VisitNode(NodeIndex nodeIdx, bool mustAdd, bool dontAddAncestors) stats.TotalMetric += node.Size; } - // Also insure that if there are a large number of types, that we sample them at least some. + // Also ensure that if there are a large number of types, that we sample them at least some. if (stats.SampleCount == 0 && !mustAdd && (m_numDistictTypesWithSamples + .5F) * m_filteringRatio <= m_numDistictTypes) { mustAdd = true; @@ -2603,9 +2615,8 @@ private void ValidateStats(bool allNodesVisited, bool completed = false) if (allNodesVisited) { Debug.Assert(total == m_graph.NodeCount); - // TODO The assert should be Debug.Assert(totalSize == m_graph.TotalSize); - // but we have to give a 1% error margin to get things passing. Fix this. - Debug.Assert(Math.Abs(totalSize - m_graph.TotalSize) / totalSize < .01); + // TODO FIX NOW enable Debug.Assert(totalSize == m_graph.TotalSize); + Debug.Assert(Math.Abs(totalSize - m_graph.TotalSize) / totalSize < .01); // TODO FIX NOW lame, replace with assert above } Debug.Assert(sampleTotal == m_newGraph.NodeCount); } @@ -2623,7 +2634,7 @@ private class SampleStats /// /// This value goes in the m_newIndex[]. If we accept the node into the sampled graph, we put the node - /// index in the NET graph in m_newIndex. If we reject the node we use the special RegjectedNode value + /// index in the NET graph in m_newIndex. If we reject the node we use the special RejectedNode value /// below /// private const NodeIndex RejectedNode = (NodeIndex)(-2); diff --git a/src/Tools/dotnet-gcdump/DotNetHeapDump/MemoryGraph.cs b/src/Tools/dotnet-gcdump/DotNetHeapDump/MemoryGraph.cs index 98e8335963..0ea6808755 100644 --- a/src/Tools/dotnet-gcdump/DotNetHeapDump/MemoryGraph.cs +++ b/src/Tools/dotnet-gcdump/DotNetHeapDump/MemoryGraph.cs @@ -1,18 +1,26 @@ -using FastSerialization; +using FastSerialization; using System.Collections.Generic; using System.Diagnostics; using Address = System.UInt64; -// Copy of version in Microsoft/PerfView - namespace Graphs { public class MemoryGraph : Graph, IFastSerializable { - public MemoryGraph(int expectedSize) - : base(expectedSize) + public MemoryGraph(int expectedSize, bool isVeryLargeGraph = false) + : base(expectedSize, isVeryLargeGraph) { - m_addressToNodeIndex = new Dictionary(expectedSize); + // If we have too many addresses we will reach the Dictionary's internal array's size limit and throw. + // Therefore use a new implementation of it that is similar in performance but that can handle the extra load. + if (isVeryLargeGraph) + { + m_addressToNodeIndex = new SegmentedDictionary(expectedSize); + } + else + { + m_addressToNodeIndex = new Dictionary(expectedSize); + } + m_nodeAddresses = new SegmentedList
(SegmentSize, expectedSize); } @@ -113,15 +121,23 @@ public bool IsInGraph(Address objectAddress) /// THis table maps the ID that CLRProfiler uses (an address), to the NodeIndex we have assigned to it. /// It is only needed while the file is being read in. /// - protected Dictionary m_addressToNodeIndex; // This field is only used during construction + protected IDictionary m_addressToNodeIndex; // This field is only used during construction #endregion #region private void IFastSerializable.ToStream(Serializer serializer) { base.ToStream(serializer); - // Write out the Memory addresses of each object - serializer.Write(m_nodeAddresses.Count); + // Write out the Memory addresses of each object + if (m_isVeryLargeGraph) + { + serializer.Write(m_nodeAddresses.Count); + } + else + { + serializer.Write((int)m_nodeAddresses.Count); + } + for (int i = 0; i < m_nodeAddresses.Count; i++) { serializer.Write((long)m_nodeAddresses[i]); @@ -134,10 +150,10 @@ void IFastSerializable.FromStream(Deserializer deserializer) { base.FromStream(deserializer); // Read in the Memory addresses of each object - int addressCount = deserializer.ReadInt(); + long addressCount = m_isVeryLargeGraph ? deserializer.ReadInt64() : deserializer.ReadInt(); m_nodeAddresses = new SegmentedList
(SegmentSize, addressCount); - for (int i = 0; i < addressCount; i++) + for (long i = 0; i < addressCount; i++) { m_nodeAddresses.Add((Address)deserializer.ReadInt64()); }