Support SourceLink in Microsoft PDBs

This allows the 'Goto Source' functionality to work on PDBs (Microsoft or Portable) that uses SourceLink to describe
how to fetch the source.  This is the mechanism used in .NET Core and thus Goto Source 'Just works' on .NET Core V2.0.3
or later.

Author: vancem

src/Directory.Build.props

@@ -21,7 +21,7 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <PerfViewVersion>2.0.1.0</PerfViewVersion>
+    <PerfViewVersion>2.0.2.0</PerfViewVersion>
   </PropertyGroup>
 
   <!-- versions of dependencies that more than one project use -->

src/PerfView/SupportFiles/UsersGuide.htm

@@ -8094,6 +8094,19 @@ <h2>
         </ul>
     </li>
     -->
+        <li>
+            Version 2.0.2 1/12/18
+            <ul>
+                <li>
+                    Added support for SourceLink for 'Goto Source' functionality.   
+                    Symlink is a technique of finding source files by placing a mapping from built time file name to URL into the
+                    symbol file so that the source code can be fetched by url at debug/profiling time.  
+                    .NET Core annotates all its symbol files this way.   The result is that 'Goto Source' on .NET Core assemblies
+                    (that is the framework and ASP.NET) just work in PerfView (it will bring up the relevant source).  
+                </li>
+
+            </ul>
+        </li>
         <li>
             Version 2.0.1 1/8/18
             <ul>
@@ -8106,8 +8119,9 @@ <h2>
         <li>
             Version 2.0.0 1/5/18
             <ul>
-                <li> Officially update the version number to 2.0 in preparation for signing and releasing officially.  
-                     Only the version number update happens here.   
+                <li>
+                    Officially update the version number to 2.0 in preparation for signing and releasing officially.
+                    Only the version number update happens here.
                 </li>
 
             </ul>

src/TraceEvent/Symbols/NativeSymbolModule.cs

@@ -395,7 +395,12 @@ public override string Url
             {
                 get
                 {
-                    string target, command;
+                    string target = base.Url; // See if it is in sourceLink information.
+                    if (target != null)
+                        return target;
+
+                    // Use srcsrv information 
+                    string command;
                     GetSourceServerTargetAndCommand(out target, out command);
 
                     if (!string.IsNullOrEmpty(target) && Uri.IsWellFormedUriString(target, UriKind.Absolute))
@@ -426,8 +431,13 @@ public override string Url
             /// 
             /// If what is at the end is a valid URL it is looked up.   
             /// </summary>
-            public override  string GetSourceFromSrcServer()
+            protected override string GetSourceFromSrcServer()
             {
+                // Try getting the source from the source server using SourceLink information.  
+                var ret = base.GetSourceFromSrcServer();
+                if (ret != null)
+                    return ret;
+
                 var cacheDir = _symbolModule.SymbolReader.SourceCacheDirectory;
 
                 string target, fetchCmdStr;
@@ -627,7 +637,7 @@ private void GetSourceServerTargetAndCommand(out string target, out string comma
 
                 _log.WriteLine("*** Looking up {0} using source server", BuildTimeFilePath);
 
-                var srcServerPdb = (_symbolModule as NativeSymbolModule).PdbForSourceServer as NativeSymbolModule;
+                NativeSymbolModule srcServerPdb = (_symbolModule as NativeSymbolModule).PdbForSourceServer as NativeSymbolModule;
                 if (srcServerPdb == null)
                 {
                     _log.WriteLine("*** Could not find PDB to look up source server information");
@@ -886,6 +896,33 @@ internal string GetSrcSrvStream()
             }
         }
 
+        protected override string GetSourceLinkJson()
+        {
+            // To avoid the expensive match below, don't even try to get SourceLink data if you have srcsrv data.  
+            // This at least avoids the cost on many OS PDBs.  
+            // You can remove this when we can look up SourceLink information easily.  
+            uint len = 0;
+            m_source.getStreamSize("srcsrv", out len);
+            if (len != 0)
+            {
+                m_reader.m_log.WriteLine("Has srcsrv information, skipping looking for SourceLink information for {0}", SymbolFilePath);
+                return null;
+            }
+
+            // TODO We should be using msdia APIs to fetch this. 
+            // I don't know exactly which ones.  In the mean time we grep for something in the PDB that looks like the SourceLink json blob. 
+            string allData = File.ReadAllText(SymbolFilePath);
+            Match m = Regex.Match(allData, "({\\s*\"documents\"\\s*:\\s*{[ -~]*?}\\s*})", RegexOptions.Singleline);
+            if (m.Success)
+            {
+                string ret = m.Groups[1].Value;
+                m_reader.m_log.WriteLine("Found SourceLink Infomation for {0}\r\nData:    {1}", SymbolFilePath, ret.Replace("\r\n", " "));
+                return ret;
+            }
+            m_reader.m_log.WriteLine("Failed to look up SourceLink Infomation for {0}.", SymbolFilePath);
+            return null;
+        }
+
         // returns the path of the PDB that has source server information in it (which for NGEN images is the PDB for the managed image)
         internal ManagedSymbolModule PdbForSourceServer
         {

src/TraceEvent/Symbols/PortableSymbolModule.cs

@@ -17,8 +17,6 @@ public PortableSymbolModule(SymbolReader reader, Stream stream, string pdbFileNa
             _stream = stream;
             _provider = MetadataReaderProvider.FromPortablePdbStream(_stream);
             _metaData = _provider.GetMetadataReader();
-
-            InitializeFileToUrlMap();
         }
 
         public override Guid PdbGuid
@@ -58,68 +56,7 @@ public override SourceLocation SourceLocationForManagedCode(uint methodMetadataT
 
         #region private 
 
-        private string GetUrlForFilePath(string buildTimeFilePath)
-        {
-            if (_fileToUrlMap != null)
-            {
-                foreach (Tuple<string, string> map in _fileToUrlMap)
-                {
-                    string path = map.Item1;
-                    string urlReplacement = map.Item2;
-
-                    if (buildTimeFilePath.StartsWith(path, StringComparison.OrdinalIgnoreCase))
-                    {
-                        string tail = buildTimeFilePath.Substring(path.Length, buildTimeFilePath.Length - path.Length).Replace('\\', '/');
-                        return urlReplacement.Replace("*", tail);
-                    }
-                }
-            }
-            return null;
-        }
-
-        /// <summary>
-        /// Looks up SourceLink information (if present) and initializes _fileToUrlMap from it
-        /// </summary>  
-        private void InitializeFileToUrlMap()
-        {
-            string sourceLinkJson = GetSourceLinkJson();
-            if (sourceLinkJson == null)
-                return;
-
-            // TODO this is not right for corner cases (e.g. file paths with " or , } in them)
-            Match m = Regex.Match(sourceLinkJson, @"documents.?\s*:\s*{(.*?)}", RegexOptions.Singleline);
-            if (m.Success)
-            {
-                string mappings = m.Groups[1].Value;
-                while (!string.IsNullOrWhiteSpace(mappings))
-                {
-                    m = Regex.Match(m.Groups[1].Value, "^\\s*\"(.*?)\"\\s*:\\s*\"(.*?)\"\\s*,?(.*)", RegexOptions.Singleline);
-                    if (m.Success)
-                    {
-                        if (_fileToUrlMap == null)
-                            _fileToUrlMap = new List<Tuple<string, string>>();
-                        string pathSpec = m.Groups[1].Value.Replace("\\\\", "\\");
-                        if (pathSpec.EndsWith("*"))
-                        {
-                            pathSpec = pathSpec.Substring(0, pathSpec.Length - 1);      // Remove the *
-                            _fileToUrlMap.Add(new Tuple<string, string>(pathSpec, m.Groups[2].Value));
-                        }
-                        else
-                            _log.WriteLine("Warning: {0} does not end in *, skipping this mapping.", pathSpec);
-                        mappings = m.Groups[3].Value;
-                    }
-                    else
-                    {
-                        _log.WriteLine("Error: Could not parse SourceLink Mapping: {0}", mappings);
-                        break;
-                    }
-                }
-            }
-            else
-                _log.WriteLine("Error: Could not parse SourceLink Json: {0}", sourceLinkJson);
-        }
-
-        private string GetSourceLinkJson()
+        protected override string GetSourceLinkJson()
         {
             foreach(CustomDebugInformationHandle customDebugInformationHandle in _metaData.CustomDebugInformation)
             {
@@ -162,8 +99,6 @@ internal PortablePdbSourceFile(Document sourceFileDocument, PortableSymbolModule
                 _log.WriteLine("Opened Portable Pdb Source File: {0}", BuildTimeFilePath);
             }
 
-            public override string Url { get { return _portablePdb.GetUrlForFilePath(BuildTimeFilePath); } }
-
             #region private 
             private static Guid HashAlgorithmSha1 = Guid.Parse("ff1816ec-aa5e-4d10-87f7-6f4963833460");
             private static Guid HashAlgorithmSha256 = Guid.Parse("8829d00f-11b8-4213-878b-770e8597ac16");
@@ -179,7 +114,7 @@ internal PortablePdbSourceFile(Document sourceFileDocument, PortableSymbolModule
         // Needed by other things to look up data
         internal MetadataReader _metaData;
 
-        List<Tuple<string, string>> _fileToUrlMap;      // Used by SourceLink to map build paths to URLs (see GetUrlForFilePath)
+
         MetadataReaderProvider _provider;
         Stream _stream;
         #endregion

src/TraceEvent/Symbols/SymbolReader.cs

@@ -1419,13 +1419,95 @@ public abstract class ManagedSymbolModule
         /// </summary>
         public abstract SourceLocation SourceLocationForManagedCode(uint methodMetadataToken, int ilOffset);
 
+        /// <summary>
+        /// If the symbol file format supports SourceLink JSON this routine should be overriden
+        /// to return it.  
+        /// </summary>
+        protected virtual string GetSourceLinkJson() { return null; }
+
         #region private 
+
         protected ManagedSymbolModule(SymbolReader reader, string path) { _pdbPath = path; _reader = reader; }
 
         internal TextWriter _log { get { return _reader.m_log; } }
 
+        /// <summary>
+        /// Return a URL for 'buildTimeFilePath' using the source link mapping (that 'GetSourceLinkJson' fetched)
+        /// Returns null if there is URL using the SourceLink 
+        /// </summary>
+        /// <param name="buildTimeFilePath"></param>
+        /// <returns></returns>
+        internal string GetUrlForFilePathUsingSourceLink(string buildTimeFilePath)
+        {
+            if (!_sourceLinkMappingInited)
+            {
+                _sourceLinkMappingInited = true;
+                string sourceLinkJson = GetSourceLinkJson();
+                if (sourceLinkJson != null)
+                    _sourceLinkMapping = ParseSourceLinkJson(sourceLinkJson);
+            }
+
+            if (_sourceLinkMapping != null)
+            {
+                foreach (Tuple<string, string> map in _sourceLinkMapping)
+                {
+                    string path = map.Item1;
+                    string urlReplacement = map.Item2;
+
+                    if (buildTimeFilePath.StartsWith(path, StringComparison.OrdinalIgnoreCase))
+                    {
+                        string tail = buildTimeFilePath.Substring(path.Length, buildTimeFilePath.Length - path.Length).Replace('\\', '/');
+                        return urlReplacement.Replace("*", tail);
+                    }
+                }
+            }
+            return null;
+        }
+
+        /// <summary>
+        /// Parses SourceLink information and returns a list of filepath -> url Prefix tuples.  
+        /// </summary>  
+        private List<Tuple<string, string>> ParseSourceLinkJson(string sourceLinkJson)
+        {
+            List<Tuple<string, string>> ret = null;
+            // TODO this is not right for corner cases (e.g. file paths with " or , } in them)
+            Match m = Regex.Match(sourceLinkJson, @"documents.?\s*:\s*{(.*?)}", RegexOptions.Singleline);
+            if (m.Success)
+            {
+                string mappings = m.Groups[1].Value;
+                while (!string.IsNullOrWhiteSpace(mappings))
+                {
+                    m = Regex.Match(m.Groups[1].Value, "^\\s*\"(.*?)\"\\s*:\\s*\"(.*?)\"\\s*,?(.*)", RegexOptions.Singleline);
+                    if (m.Success)
+                    {
+                        if (ret == null)
+                            ret = new List<Tuple<string, string>>();
+                        string pathSpec = m.Groups[1].Value.Replace("\\\\", "\\");
+                        if (pathSpec.EndsWith("*"))
+                        {
+                            pathSpec = pathSpec.Substring(0, pathSpec.Length - 1);      // Remove the *
+                            ret.Add(new Tuple<string, string>(pathSpec, m.Groups[2].Value));
+                        }
+                        else
+                            _log.WriteLine("Warning: {0} does not end in *, skipping this mapping.", pathSpec);
+                        mappings = m.Groups[3].Value;
+                    }
+                    else
+                    {
+                        _log.WriteLine("Error: Could not parse SourceLink Mapping: {0}", mappings);
+                        break;
+                    }
+                }
+            }
+            else
+                _log.WriteLine("Error: Could not parse SourceLink Json: {0}", sourceLinkJson);
+            return ret;
+        }
+
         string _pdbPath;
         SymbolReader _reader;
+        List<Tuple<string, string>> _sourceLinkMapping;      // Used by SourceLink to map build paths to URLs (see GetUrlForFilePath)
+        bool _sourceLinkMappingInited;                       // Lazy init flag. 
         #endregion
     }
 
@@ -1489,36 +1571,7 @@ public abstract class SourceFile
         /// can be used to fetch it with HTTP Get), then return that Url.   If no such publishing 
         /// point exists this property will return null.   
         /// </summary>
-        public virtual string Url { get { return null; } }
-
-        /// <summary>
-        /// Look up the source from the source server.  Returns null if it can't find the source
-        /// By default this simply uses the Url to look it up on the web.   If 'Url' returns null
-        /// so does this.   
-        /// </summary>
-        public virtual string GetSourceFromSrcServer()
-        {
-            // Search the SourceLink url location 
-            string url = Url;
-            if (url != null)
-            {
-                HttpClient httpClient = new HttpClient();
-                HttpResponseMessage response = httpClient.GetAsync(url).Result;
-
-                response.EnsureSuccessStatusCode();
-                Stream content = response.Content.ReadAsStreamAsync().Result;
-                string cachedLocation = GetCachePathForUrl(url);
-                if (cachedLocation != null)
-                {
-                    using (FileStream file = File.Create(_filePath))
-                        content.CopyTo(file);
-                    return cachedLocation;
-                }
-                else
-                    _log.WriteLine("Warning: SourceCache not set, giving up fetching source from the network.");
-            }
-            return null;
-        }
+        public virtual string Url { get { return _symbolModule.GetUrlForFilePathUsingSourceLink(BuildTimeFilePath); } }
 
         /// <summary>
         /// This may fetch things from the source server, and thus can be very slow, which is why it is not a property. 
@@ -1609,10 +1662,40 @@ public virtual string GetSourceFile(bool requireChecksumMatch = false)
 
         protected TextWriter _log { get { return _symbolModule._log; } }
 
+        /// <summary>
+        /// Look up the source from the source server.  Returns null if it can't find the source
+        /// By default this simply uses the Url to look it up on the web.   If 'Url' returns null
+        /// so does this.   
+        /// </summary>
+        protected virtual string GetSourceFromSrcServer()
+        {
+            // Search the SourceLink url location 
+            string url = Url;
+            if (url != null)
+            {
+                HttpClient httpClient = new HttpClient();
+                HttpResponseMessage response = httpClient.GetAsync(url).Result;
+
+                response.EnsureSuccessStatusCode();
+                Stream content = response.Content.ReadAsStreamAsync().Result;
+                string cachedLocation = GetCachePathForUrl(url);
+                if (cachedLocation != null)
+                {
+                    Directory.CreateDirectory(Path.GetDirectoryName(cachedLocation));
+                    using (FileStream file = File.Create(cachedLocation))
+                        content.CopyTo(file);
+                    return cachedLocation;
+                }
+                else
+                    _log.WriteLine("Warning: SourceCache not set, giving up fetching source from the network.");
+            }
+            return null;
+        }
+
         /// <summary>
         /// Given 'fileName' which is a path to a file (which may  not exist), set 
         /// _filePath and _checksumMatches appropriately.    Namely _filePath should
-        /// always be the 'best' candidate for the soruce file path (matching checksum
+        /// always be the 'best' candidate for the source file path (matching checksum
         /// wins, otherwise first existing file wins).  
         /// 
         /// Returns true if we have a perfect match (no additional probing needed).