Skip to content

Commit

Permalink
Remove unneeded files after etw profiler (#1540)
Browse files Browse the repository at this point in the history
  • Loading branch information
@WojciechNagorski
WojciechNagorski authored Oct 1, 2020
1 parent e4d37d0 commit 10abc4f
Show file tree
Hide file tree
Showing 4 changed files with 192 additions and 156 deletions.
322 changes: 170 additions & 152 deletions src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Diagnosers;
using BenchmarkDotNet.Engines;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
Expand Down Expand Up @@ -41,208 +42,225 @@ public NativeMemoryLogParser(string etlFilePath, BenchmarkCase benchmarkCase, IL
functionName = nameof(EngineParameters.WorkloadActionUnroll);
}

//Code is inspired by https://github.com/Microsoft/perfview/blob/master/src/PerfView/PerfViewData.cs#L5719-L5944
public IEnumerable<Metric> Parse()
{
using (var traceLog = new TraceLog(TraceLog.CreateFromEventTraceLogFile(etlFilePath)))
{
var stackSource = new MutableTraceEventStackSource(traceLog);
var eventSource = traceLog.Events.GetSource();

var bdnEventsParser = new EngineEventLogParser(eventSource);

var start = false;
var isFirstActualStartEnd = false;
var etlxFilePath = TraceLog.CreateFromEventTraceLogFile(etlFilePath);

long totalOperation = 0;
long countOfAllocatedObject = 0;

bdnEventsParser.WorkloadActualStart += data =>
try
{
using (var traceLog = new TraceLog(etlxFilePath))
{
if (!isFirstActualStartEnd)
{
start = true;
}
return Parse(traceLog);
}
}
finally
{
etlxFilePath.DeleteFileIfExists();
}
}

totalOperation = data.TotalOperations;
};
bdnEventsParser.WorkloadActualStop += data =>
{
start = false;
isFirstActualStartEnd = true;
};
//Code is inspired by https://github.com/Microsoft/perfview/blob/master/src/PerfView/PerfViewData.cs#L5719-L5944
private IEnumerable<Metric> Parse(TraceLog traceLog)
{
var stackSource = new MutableTraceEventStackSource(traceLog);
var eventSource = traceLog.Events.GetSource();

var heapParser = new HeapTraceProviderTraceEventParser(eventSource);
// We index by heap address and then within the heap we remember the allocation stack
var heaps = new Dictionary<Address, Dictionary<Address, long>>();
Dictionary<Address, long> lastHeapAllocs = null;
var bdnEventsParser = new EngineEventLogParser(eventSource);

Address lastHeapHandle = 0;
var start = false;
var isFirstActualStartEnd = false;

long nativeLeakSize = 0;
long totalAllocation = 0;
long totalOperation = 0;
long countOfAllocatedObject = 0;

heapParser.HeapTraceAlloc += delegate(HeapAllocTraceData data)
bdnEventsParser.WorkloadActualStart += data =>
{
if (!isFirstActualStartEnd)
{
if (!start)
{
return;
}
start = true;
}

var call = data.CallStackIndex();
var frameIndex = stackSource.GetCallStack(call, data);
totalOperation = data.TotalOperations;
};
bdnEventsParser.WorkloadActualStop += data =>
{
start = false;
isFirstActualStartEnd = true;
};

if (!IsCallStackIn(frameIndex))
{
return;
}
var heapParser = new HeapTraceProviderTraceEventParser(eventSource);
// We index by heap address and then within the heap we remember the allocation stack
var heaps = new Dictionary<Address, Dictionary<Address, long>>();
Dictionary<Address, long> lastHeapAllocs = null;

var allocs = lastHeapAllocs;
if (data.HeapHandle != lastHeapHandle)
{
allocs = CreateHeapCache(data.HeapHandle, heaps, ref lastHeapAllocs, ref lastHeapHandle);
}
Address lastHeapHandle = 0;

allocs[data.AllocAddress] = data.AllocSize;
long nativeLeakSize = 0;
long totalAllocation = 0;

checked
{
countOfAllocatedObject++;
nativeLeakSize += data.AllocSize;
totalAllocation += data.AllocSize;
}
heapParser.HeapTraceAlloc += delegate(HeapAllocTraceData data)
{
if (!start)
{
return;
}

bool IsCallStackIn(StackSourceCallStackIndex index)
{
while (index != StackSourceCallStackIndex.Invalid)
{
var frame = stackSource.GetFrameIndex(index);
var name = stackSource.GetFrameName(frame, false);
var call = data.CallStackIndex();
var frameIndex = stackSource.GetCallStack(call, data);

if (name.StartsWith(moduleName, StringComparison.Ordinal) &&
name.IndexOf(functionName, StringComparison.Ordinal) > 0)
{
return true;
}
if (!IsCallStackIn(frameIndex))
{
return;
}

index = stackSource.GetCallerIndex(index);
}
var allocs = lastHeapAllocs;
if (data.HeapHandle != lastHeapHandle)
{
allocs = CreateHeapCache(data.HeapHandle, heaps, ref lastHeapAllocs, ref lastHeapHandle);
}

return false;
}
};
allocs[data.AllocAddress] = data.AllocSize;

heapParser.HeapTraceFree += delegate(HeapFreeTraceData data)
checked
{
if (!start)
{
return;
}
countOfAllocatedObject++;
nativeLeakSize += data.AllocSize;
totalAllocation += data.AllocSize;
}

var allocs = lastHeapAllocs;
if (data.HeapHandle != lastHeapHandle)
bool IsCallStackIn(StackSourceCallStackIndex index)
{
while (index != StackSourceCallStackIndex.Invalid)
{
allocs = CreateHeapCache(data.HeapHandle, heaps, ref lastHeapAllocs, ref lastHeapHandle);
}
var frame = stackSource.GetFrameIndex(index);
var name = stackSource.GetFrameName(frame, false);

if (allocs.TryGetValue(data.FreeAddress, out long alloc))
{
nativeLeakSize -= alloc;
if (name.StartsWith(moduleName, StringComparison.Ordinal) &&
name.IndexOf(functionName, StringComparison.Ordinal) > 0)
{
return true;
}

allocs.Remove(data.FreeAddress);
index = stackSource.GetCallerIndex(index);
}
};

heapParser.HeapTraceReAlloc += delegate(HeapReallocTraceData data)
return false;
}
};

heapParser.HeapTraceFree += delegate(HeapFreeTraceData data)
{
if (!start)
{
if (!start)
{
return;
}
// Reallocs that actually move stuff will cause a Alloc and delete event
// so there is nothing to do for those events. But when the address is
// the same we need to resize.
if (data.OldAllocAddress != data.NewAllocAddress)
{
return;
}
return;
}

var allocs = lastHeapAllocs;
if (data.HeapHandle != lastHeapHandle)
{
allocs = CreateHeapCache(data.HeapHandle, heaps, ref lastHeapAllocs, ref lastHeapHandle);
}
var allocs = lastHeapAllocs;
if (data.HeapHandle != lastHeapHandle)
{
allocs = CreateHeapCache(data.HeapHandle, heaps, ref lastHeapAllocs, ref lastHeapHandle);
}

if (allocs.TryGetValue(data.OldAllocAddress, out long alloc))
{
// Free
nativeLeakSize -= alloc;
if (allocs.TryGetValue(data.FreeAddress, out long alloc))
{
nativeLeakSize -= alloc;

allocs.Remove(data.OldAllocAddress);
allocs.Remove(data.FreeAddress);
}
};

// Alloc
allocs[data.NewAllocAddress] = data.NewAllocSize;
heapParser.HeapTraceReAlloc += delegate(HeapReallocTraceData data)
{
if (!start)
{
return;
}
// Reallocs that actually move stuff will cause a Alloc and delete event
// so there is nothing to do for those events. But when the address is
// the same we need to resize.
if (data.OldAllocAddress != data.NewAllocAddress)
{
return;
}

checked
{
nativeLeakSize += data.NewAllocSize;
}
}
};
var allocs = lastHeapAllocs;
if (data.HeapHandle != lastHeapHandle)
{
allocs = CreateHeapCache(data.HeapHandle, heaps, ref lastHeapAllocs, ref lastHeapHandle);
}

heapParser.HeapTraceDestroy += delegate(HeapTraceData data)
if (allocs.TryGetValue(data.OldAllocAddress, out long alloc))
{
if (!start)
{
return;
}
// Free
nativeLeakSize -= alloc;

// Heap is dieing, kill all objects in it.
var allocs = lastHeapAllocs;
if (data.HeapHandle != lastHeapHandle)
{
allocs = CreateHeapCache(data.HeapHandle, heaps, ref lastHeapAllocs, ref lastHeapHandle);
}
allocs.Remove(data.OldAllocAddress);

foreach (var alloc in allocs.Values)
// Alloc
allocs[data.NewAllocAddress] = data.NewAllocSize;

checked
{
nativeLeakSize -= alloc;
nativeLeakSize += data.NewAllocSize;
}
};

eventSource.Process();

logger.WriteLine();
logger.WriteLineHeader(LogSeparator);
logger.WriteLineInfo($"{benchmarkCase.DisplayInfo}");
logger.WriteLineHeader(LogSeparator);
}
};

if (totalOperation == 0)
heapParser.HeapTraceDestroy += delegate(HeapTraceData data)
{
if (!start)
{
logger.WriteLine($"Something went wrong. The trace file {etlFilePath} does not contain BenchmarkDotNet engine events.");
yield break;
return;
}

var memoryAllocatedPerOperation = totalAllocation / totalOperation;
var memoryLeakPerOperation = nativeLeakSize / totalOperation;

logger.WriteLine($"Native memory allocated per single operation: {SizeValue.FromBytes(memoryAllocatedPerOperation).ToString(SizeUnit.B, benchmarkCase.Config.CultureInfo)}");
logger.WriteLine($"Count of allocated object: {countOfAllocatedObject / totalOperation}");

if (nativeLeakSize != 0)
// Heap is dieing, kill all objects in it.
var allocs = lastHeapAllocs;
if (data.HeapHandle != lastHeapHandle)
{
logger.WriteLine($"Native memory leak per single operation: {SizeValue.FromBytes(memoryLeakPerOperation).ToString(SizeUnit.B, benchmarkCase.Config.CultureInfo)}");
allocs = CreateHeapCache(data.HeapHandle, heaps, ref lastHeapAllocs, ref lastHeapHandle);
}

var heapInfoList = heaps.Select(h => new { Address = h.Key, h.Value.Count, types = h.Value.Values });
foreach (var item in heapInfoList.Where(p => p.Count > 0))
foreach (var alloc in allocs.Values)
{
logger.WriteLine($"Count of not deallocated object: {item.Count / totalOperation}");
nativeLeakSize -= alloc;
}
};

eventSource.Process();

yield return new Metric(new AllocatedNativeMemoryDescriptor(), memoryAllocatedPerOperation);
yield return new Metric(new NativeMemoryLeakDescriptor(), memoryLeakPerOperation);
logger.WriteLine();
logger.WriteLineHeader(LogSeparator);
logger.WriteLineInfo($"{benchmarkCase.DisplayInfo}");
logger.WriteLineHeader(LogSeparator);

if (totalOperation == 0)
{
logger.WriteLine($"Something went wrong. The trace file {etlFilePath} does not contain BenchmarkDotNet engine events.");
return Enumerable.Empty<Metric>();
}

var memoryAllocatedPerOperation = totalAllocation / totalOperation;
var memoryLeakPerOperation = nativeLeakSize / totalOperation;

logger.WriteLine($"Native memory allocated per single operation: {SizeValue.FromBytes(memoryAllocatedPerOperation).ToString(SizeUnit.B, benchmarkCase.Config.CultureInfo)}");
logger.WriteLine($"Count of allocated object: {countOfAllocatedObject / totalOperation}");

if (nativeLeakSize != 0)
{
logger.WriteLine($"Native memory leak per single operation: {SizeValue.FromBytes(memoryLeakPerOperation).ToString(SizeUnit.B, benchmarkCase.Config.CultureInfo)}");
}

var heapInfoList = heaps.Select(h => new { Address = h.Key, h.Value.Count, types = h.Value.Values });
foreach (var item in heapInfoList.Where(p => p.Count > 0))
{
logger.WriteLine($"Count of not deallocated object: {item.Count / totalOperation}");
}

return new[]
{
new Metric(new AllocatedNativeMemoryDescriptor(), memoryAllocatedPerOperation),
new Metric(new NativeMemoryLeakDescriptor(), memoryLeakPerOperation)
};
}

private static Dictionary<Address, long> CreateHeapCache(Address heapHandle, Dictionary<Address, Dictionary<Address, long>> heaps, ref Dictionary<Address, long> lastHeapAllocs, ref Address lastHeapHandle)
Expand Down
Loading

0 comments on commit 10abc4f

Please sign in to comment.