From 989ef58d2425d675fdbafad24e046b468d89faf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksey=20Kliger=20=28=CE=BBgeek=29?= Date: Sat, 21 Jan 2023 18:57:09 -0500 Subject: [PATCH] [mono][wasm] Bundle assemblies as WebCIL (#79416) Define a new container format for .NET assemblies that looks less like a Windows PE file. Use it for bundling assemblies in wasm projects. * Implement WebCIL loader It will try to look for WebCIL formatted images instread of normal .dll files * Checkpoint works on wasm sample; add design doc * Push .dll->.webcil probing lower in the bundle logic * Also convert satellite assemblies and implement satellite matching * [wasm] don't leak .webcil image names to the debugger In particular this will make source and breakpoint URLs look like `dotnet://foo.dll/Foo.cs` which means that grabbing PDBs via source link will work, etc. * Add PE DebugTableDirectory to webcil This is used to retrieve the PPDB data and/or the PDB checksum from an image. Refactor mono_has_pdb_checksum to support webcil in addition to PE images * Implement a WebcilReader for BorwserDebugProxy like PEReader This needs some improvements: - add support for reading CodeView and EmbeddedPDB data - copy/paste less from the WebcilWriter task - copy/paste less from PEReader (will require moving WebcilReader to SRM) * [debug] Match bundled pdbs if we're looking up .webcil files The pdbs are registered by wasm with a notional .dll filename. if the debugger does a lookup using a .webcil name instead, allow the match * Adjust debug directory entries when writing webcil files the PE COFF debug directory entries contain a 'pointer' field which is an offset from the start of the file. When writing the webcil file, the header is typically smaller than a PE file, so the offsets are wrong. Adjust the offsets by the size of the file. We assume (and assert) the debug directory entries actually point at some PE COFF sections in the PE file (as opposed to somewhere past the end of the known PE data). When writing, we initially just copy all the sections directly, then seek to where the debug directory entries are, and overwrite them with updated entries that have the correct 'pointer' * Fix bug in WebcilWriter Stream.CopyTo takes a buffer size, not the number of bytes to copy. * bugfix: the debug directory is at pe_debug_rva not at the CLI header * skip debug fixups if there's no debug directory * WebcilReader: implement CodeView and Emebedded PPDB support * [WBT] Add UseWebcil option (default to true) * rename WebcilWriter -> WebcilConverter [NFC] * fixup AssemblyLoadedEventTest * hack: no extension on assembly for breakpoint * pass normal .dll name for MainAssemblyName in config let the runtime deal with it - bundle matching will resolve it to the .webcil file * Wasm.Debugger.Tests: give CI 10 more minutes * Add Microsoft.NET.WebAssembly.Webcil assembly project Mark it as shipping, but not shipping a nuget package. The idea is that it will be shipped along with the WasmAppBuilder msbuild task, and with the BrowserDebugProxy tool. * Move WebcilConverter to Microsoft.NET.WebAssembly.Webcil * Move WebcilReader to Microsoft.NET.WebAssembly.Webcil delete the duplicated utility classes * make the webcil magic and version longer * Code style improvements from review * Improve some exception messages, when possible * Suggestings from code review * Add WasmEnableWebcil msbuild property. Off by default * Build non-wasm runtimes without .webcil support * Run WBT twice: with and without webcil This is a total of 4 runs: with and without workloads x with and without webcil * do the cartesian product correctly in msbuild * also add webcil to template projects * environment variable has to be non-null and "true" We set it to "false" sometimes * Fix wasm work items They should be the same whether or not webcil is used. Just the WorkloadItemPrefix should be used to change the name. * Update src/libraries/sendtohelix-wasm.targets * PInvokeTableGeneratorTests: don't try to use the net472 WasmAppBuilder Look for the default target framework subdirectory under the tasks directory in the runtime pack when trying to find the tasks dll. In particular don't try to load the net472 version on modern .NET * PInvokeTableGeneratorTests: Add more diagnostic output if tasksDir is not found * simplify prefix comparison in bundled_assembly_match * WasmAppBuilder improve logging Just emit a single Normal importance message about webcil; details as Low importance. * Add missing using Co-authored-by: Ankit Jain Co-authored-by: Larry Ewing --- docs/design/mono/webcil.md | 111 ++++++ .../Directory.Build.props | 11 + .../src/Common/IsExternalInit.cs | 7 + .../Microsoft.NET.WebAssembly.Webcil.csproj | 24 ++ .../src/Webcil/Internal/Constants.cs | 10 + .../src/Webcil/WebciHeader.cs | 33 ++ .../src/Webcil/WebcilConverter.cs | 342 ++++++++++++++++++ .../src/Webcil/WebcilReader.cs | 342 ++++++++++++++++++ .../src/Webcil/WebcilSectionHeader.cs | 26 ++ src/libraries/sendtohelix-wasm.targets | 3 +- src/libraries/sendtohelix.proj | 15 +- src/libraries/sendtohelixhelp.proj | 3 +- src/mono/cmake/config.h.in | 3 + src/mono/cmake/options.cmake | 1 + src/mono/mono.proj | 1 + src/mono/mono/metadata/CMakeLists.txt | 2 + src/mono/mono/metadata/assembly.c | 32 +- src/mono/mono/metadata/image.c | 65 +++- src/mono/mono/metadata/mono-debug.c | 22 +- src/mono/mono/metadata/reflection.c | 2 +- src/mono/mono/metadata/webcil-loader.c | 170 +++++++++ src/mono/mono/metadata/webcil-loader.h | 17 + src/mono/mono/mini/monovm.c | 16 + ...NETCore.BrowserDebugHost.Transport.pkgproj | 1 + .../wasm/Wasm.Build.Tests/BuildEnvironment.cs | 5 + .../wasm/Wasm.Build.Tests/BuildTestBase.cs | 27 +- .../Wasm.Build.Tests/EnvironmentVariables.cs | 1 + .../PInvokeTableGeneratorTests.cs | 18 +- .../Wasm.Build.Tests/Wasm.Build.Tests.csproj | 3 + .../data/RunScriptTemplate.cmd | 5 + .../data/RunScriptTemplate.sh | 6 + src/mono/wasm/build/WasmApp.targets | 5 + .../BrowserDebugProxy.csproj | 4 + .../debugger/BrowserDebugProxy/DebugStore.cs | 125 ++++++- .../BrowserDebugProxy/MonoSDBHelper.cs | 10 +- .../DebuggerTestSuite/DebuggerTestBase.cs | 2 +- .../debugger/DebuggerTestSuite/MonoJsTests.cs | 2 +- .../Wasm.Debugger.Tests/wasm.helix.targets | 4 +- src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 38 +- .../WasmAppBuilder/WasmAppBuilder.csproj | 11 + src/tasks/WasmAppBuilder/WebcilConverter.cs | 45 +++ 41 files changed, 1508 insertions(+), 62 deletions(-) create mode 100644 docs/design/mono/webcil.md create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Common/IsExternalInit.cs create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilSectionHeader.cs create mode 100644 src/mono/mono/metadata/webcil-loader.c create mode 100644 src/mono/mono/metadata/webcil-loader.h create mode 100644 src/tasks/WasmAppBuilder/WebcilConverter.cs diff --git a/docs/design/mono/webcil.md b/docs/design/mono/webcil.md new file mode 100644 index 00000000000000..3bcb7d6365353b --- /dev/null +++ b/docs/design/mono/webcil.md @@ -0,0 +1,111 @@ +# WebCIL assembly format + +## Version + +This is version 0.0 of the Webcil format. + +## Motivation + +When deploying the .NET runtime to the browser using WebAssembly, we have received some reports from +customers that certain users are unable to use their apps because firewalls and anti-virus software +may prevent browsers from downloading or caching assemblies with a .DLL extension and PE contents. + +This document defines a new container format for ECMA-335 assemblies +that uses the `.webcil` extension and uses a new WebCIL container +format. + + +## Specification + +As our starting point we take section II.25.1 "Structure of the +runtime file format" from ECMA-335 6th Edition. + +| | +|--------| +| PE Headers | +| CLI Header | +| CLI Data | +| Native Image Sections | +| | + + + +A Webcil file follows a similar structure + + +| | +|--------| +| Webcil Headers | +| CLI Header | +| CLI Data | +| | + +## Webcil Headers + +The Webcil headers consist of a Webcil header followed by a sequence of section headers. +(All multi-byte integers are in little endian format). + +### Webcil Header + +``` c +struct WebcilHeader { + uint8_t id[4]; // 'W' 'b' 'I' 'L' + // 4 bytes + uint16_t version_major; // 0 + uint16_t version_minor; // 0 + // 8 bytes + uint16_t coff_sections; + uint16_t reserved0; // 0 + // 12 bytes + + uint32_t pe_cli_header_rva; + uint32_t pe_cli_header_size; + // 20 bytes + + uint32_t pe_debug_rva; + uint32_t pe_debug_size; + // 28 bytes +}; +``` + +The Webcil header starts with the magic characters 'W' 'b' 'I' 'L' followed by the version in major +minor format (must be 0 and 0). Then a count of the section headers and two reserved bytes. + +The next pairs of integers are a subset of the PE Header data directory specifying the RVA and size +of the CLI header, as well as the directory entry for the PE debug directory. + + +### Section header table + +Immediately following the Webcil header is a sequence (whose length is given by `coff_sections` +above) of section headers giving their virtual address and virtual size, as well as the offset in +the Webcil file and the size in the file. This is a subset of the PE section header that includes +enough information to correctly interpret the RVAs from the webcil header and from the .NET +metadata. Other information (such as the section names) are not included. + +``` c +struct SectionHeader { + uint32_t st_virtual_size; + uint32_t st_virtual_address; + uint32_t st_raw_data_size; + uint32_t st_raw_data_ptr; +}; +``` + +### Sections + +Immediately following the section table are the sections. These are copied verbatim from the PE file. + +## Rationale + +The intention is to include only the information necessary for the runtime to locate the metadata +root, and to resolve the RVA references in the metadata (for locating data declarations and method IL). + +A goal is for the files not to be executable by .NET Framework. + +Unlike PE files, mixing native and managed code is not a goal. + +Lossless conversion from Webcil back to PE is not intended to be supported. The format is being +documented in order to support diagnostic tooling and utilities such as decompilers, disassemblers, +file identification utilities, dependency analyzers, etc. + diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props b/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props new file mode 100644 index 00000000000000..6bda1e7da2544e --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props @@ -0,0 +1,11 @@ + + + + true + + false + + false + + diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Common/IsExternalInit.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Common/IsExternalInit.cs new file mode 100644 index 00000000000000..d7be6913103471 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Common/IsExternalInit.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Runtime.CompilerServices +{ + internal sealed class IsExternalInit { } +} diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj new file mode 100644 index 00000000000000..6c66737e5535d2 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj @@ -0,0 +1,24 @@ + + + $(NetCoreAppToolCurrent);$(NetFrameworkToolCurrent) + Abstractions for modifying .NET webcil binary images + true + true + true + false + + + + + + + + + + + + + + + + diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs new file mode 100644 index 00000000000000..2d486645d23b66 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.NET.WebAssembly.Webcil.Internal; + +internal static unsafe class Constants +{ + public const int WC_VERSION_MAJOR = 0; + public const int WC_VERSION_MINOR = 0; +} diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs new file mode 100644 index 00000000000000..33dcc85791fff4 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +namespace Microsoft.NET.WebAssembly.Webcil; + +/// +/// The header of a WebCIL file. +/// +/// +/// +/// The header is a subset of the PE, COFF and CLI headers that are needed by the mono runtime to load managed assemblies. +/// +[StructLayout(LayoutKind.Sequential, Pack = 1)] +public unsafe struct WebcilHeader +{ + public fixed byte id[4]; // 'W' 'b' 'I' 'L' + // 4 bytes + public ushort version_major; // 0 + public ushort version_minor; // 0 + // 8 bytes + + public ushort coff_sections; + public ushort reserved0; // 0 + // 12 bytes + public uint pe_cli_header_rva; + public uint pe_cli_header_size; + // 20 bytes + public uint pe_debug_rva; + public uint pe_debug_size; + // 28 bytes +} diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs new file mode 100644 index 00000000000000..421b62439e190d --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs @@ -0,0 +1,342 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Collections.Immutable; +using System.Reflection.PortableExecutable; +using System.Runtime.InteropServices; + +namespace Microsoft.NET.WebAssembly.Webcil; + +/// +/// Reads a .NET assembly in a normal PE COFF file and writes it out as a Webcil file +/// +public class WebcilConverter +{ + + // Interesting stuff we've learned about the input PE file + public record PEFileInfo( + // The sections in the PE file + ImmutableArray SectionHeaders, + // The location of the debug directory entries + DirectoryEntry DebugTableDirectory, + // The file offset of the sections, following the section directory + FilePosition SectionStart, + // The debug directory entries + ImmutableArray DebugDirectoryEntries + ); + + // Intersting stuff we know about the webcil file we're writing + public record WCFileInfo( + // The header of the webcil file + WebcilHeader Header, + // The section directory of the webcil file + ImmutableArray SectionHeaders, + // The file offset of the sections, following the section directory + FilePosition SectionStart + ); + + private readonly string _inputPath; + private readonly string _outputPath; + + private string InputPath => _inputPath; + + private WebcilConverter(string inputPath, string outputPath) + { + _inputPath = inputPath; + _outputPath = outputPath; + } + + public static WebcilConverter FromPortableExecutable(string inputPath, string outputPath) + => new WebcilConverter(inputPath, outputPath); + + public void ConvertToWebcil() + { + using var inputStream = File.Open(_inputPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + PEFileInfo peInfo; + WCFileInfo wcInfo; + using (var peReader = new PEReader(inputStream, PEStreamOptions.LeaveOpen)) + { + GatherInfo(peReader, out wcInfo, out peInfo); + } + + using var outputStream = File.Open(_outputPath, FileMode.Create, FileAccess.Write); + WriteHeader(outputStream, wcInfo.Header); + WriteSectionHeaders(outputStream, wcInfo.SectionHeaders); + CopySections(outputStream, inputStream, peInfo.SectionHeaders); + if (wcInfo.Header.pe_debug_size != 0 && wcInfo.Header.pe_debug_rva != 0) + { + var wcDebugDirectoryEntries = FixupDebugDirectoryEntries(peInfo, wcInfo); + OverwriteDebugDirectoryEntries(outputStream, wcInfo, wcDebugDirectoryEntries); + } + } + + public record struct FilePosition(int Position) + { + public static implicit operator FilePosition(int position) => new(position); + + public static FilePosition operator +(FilePosition left, int right) => new(left.Position + right); + } + + private static unsafe int SizeOfHeader() + { + return sizeof(WebcilHeader); + } + + public unsafe void GatherInfo(PEReader peReader, out WCFileInfo wcInfo, out PEFileInfo peInfo) + { + var headers = peReader.PEHeaders; + var peHeader = headers.PEHeader!; + var coffHeader = headers.CoffHeader!; + var sections = headers.SectionHeaders; + WebcilHeader header; + header.id[0] = (byte)'W'; + header.id[1] = (byte)'b'; + header.id[2] = (byte)'I'; + header.id[3] = (byte)'L'; + header.version_major = Internal.Constants.WC_VERSION_MAJOR; + header.version_minor = Internal.Constants.WC_VERSION_MINOR; + header.coff_sections = (ushort)coffHeader.NumberOfSections; + header.reserved0 = 0; + header.pe_cli_header_rva = (uint)peHeader.CorHeaderTableDirectory.RelativeVirtualAddress; + header.pe_cli_header_size = (uint)peHeader.CorHeaderTableDirectory.Size; + header.pe_debug_rva = (uint)peHeader.DebugTableDirectory.RelativeVirtualAddress; + header.pe_debug_size = (uint)peHeader.DebugTableDirectory.Size; + + // current logical position in the output file + FilePosition pos = SizeOfHeader(); + // position of the current section in the output file + // initially it's after all the section headers + FilePosition curSectionPos = pos + sizeof(WebcilSectionHeader) * coffHeader.NumberOfSections; + // The first WC section is immediately after the section directory + FilePosition firstWCSection = curSectionPos; + + FilePosition firstPESection = 0; + + ImmutableArray.Builder headerBuilder = ImmutableArray.CreateBuilder(coffHeader.NumberOfSections); + foreach (var sectionHeader in sections) + { + // The first section is the one with the lowest file offset + if (firstPESection.Position == 0) + { + firstPESection = sectionHeader.PointerToRawData; + } + else + { + firstPESection = Math.Min(firstPESection.Position, sectionHeader.PointerToRawData); + } + + var newHeader = new WebcilSectionHeader + ( + virtualSize: sectionHeader.VirtualSize, + virtualAddress: sectionHeader.VirtualAddress, + sizeOfRawData: sectionHeader.SizeOfRawData, + pointerToRawData: curSectionPos.Position + ); + + pos += sizeof(WebcilSectionHeader); + curSectionPos += sectionHeader.SizeOfRawData; + headerBuilder.Add(newHeader); + } + + ImmutableArray debugDirectoryEntries = peReader.ReadDebugDirectory(); + + peInfo = new PEFileInfo(SectionHeaders: sections, + DebugTableDirectory: peHeader.DebugTableDirectory, + SectionStart: firstPESection, + DebugDirectoryEntries: debugDirectoryEntries); + + wcInfo = new WCFileInfo(Header: header, + SectionHeaders: headerBuilder.MoveToImmutable(), + SectionStart: firstWCSection); + } + + private static void WriteHeader(Stream s, WebcilHeader header) + { + WriteStructure(s, header); + } + + private static void WriteSectionHeaders(Stream s, ImmutableArray sectionsHeaders) + { + // FIXME: fixup endianness + if (!BitConverter.IsLittleEndian) + throw new NotImplementedException(); + foreach (var sectionHeader in sectionsHeaders) + { + WriteSectionHeader(s, sectionHeader); + } + } + + private static void WriteSectionHeader(Stream s, WebcilSectionHeader sectionHeader) + { + WriteStructure(s, sectionHeader); + } + +#if NETCOREAPP2_1_OR_GREATER + private static void WriteStructure(Stream s, T structure) + where T : unmanaged + { + // FIXME: fixup endianness + if (!BitConverter.IsLittleEndian) + throw new NotImplementedException(); + unsafe + { + byte* p = (byte*)&structure; + s.Write(new ReadOnlySpan(p, sizeof(T))); + } + } +#else + private static void WriteStructure(Stream s, T structure) + where T : unmanaged + { + // FIXME: fixup endianness + if (!BitConverter.IsLittleEndian) + throw new NotImplementedException(); + int size = Marshal.SizeOf(); + byte[] buffer = new byte[size]; + IntPtr ptr = IntPtr.Zero; + try + { + ptr = Marshal.AllocHGlobal(size); + Marshal.StructureToPtr(structure, ptr, false); + Marshal.Copy(ptr, buffer, 0, size); + } + finally + { + Marshal.FreeHGlobal(ptr); + } + s.Write(buffer, 0, size); + } +#endif + + private static void CopySections(Stream outStream, Stream inputStream, ImmutableArray peSections) + { + // endianness: ok, we're just copying from one stream to another + foreach (var peHeader in peSections) + { + var buffer = new byte[peHeader.SizeOfRawData]; + inputStream.Seek(peHeader.PointerToRawData, SeekOrigin.Begin); + ReadExactly(inputStream, buffer); + outStream.Write(buffer, 0, buffer.Length); + } + } + +#if NETCOREAPP2_1_OR_GREATER + private static void ReadExactly(Stream s, Span buffer) + { + s.ReadExactly(buffer); + } +#else + private static void ReadExactly(Stream s, byte[] buffer) + { + int offset = 0; + while (offset < buffer.Length) + { + int read = s.Read(buffer, offset, buffer.Length - offset); + if (read == 0) + throw new EndOfStreamException(); + offset += read; + } + } +#endif + + private static FilePosition GetPositionOfRelativeVirtualAddress(ImmutableArray wcSections, uint relativeVirtualAddress) + { + foreach (var section in wcSections) + { + if (relativeVirtualAddress >= section.VirtualAddress && relativeVirtualAddress < section.VirtualAddress + section.VirtualSize) + { + FilePosition pos = section.PointerToRawData + ((int)relativeVirtualAddress - section.VirtualAddress); + return pos; + } + } + + throw new InvalidOperationException("relative virtual address not in any section"); + } + + // Given a physical file offset, return the section and the offset within the section. + private (WebcilSectionHeader section, int offset) GetSectionFromFileOffset(ImmutableArray peSections, FilePosition fileOffset) + { + foreach (var section in peSections) + { + if (fileOffset.Position >= section.PointerToRawData && fileOffset.Position < section.PointerToRawData + section.SizeOfRawData) + { + return (section, fileOffset.Position - section.PointerToRawData); + } + } + + throw new InvalidOperationException($"file offset not in any section (Webcil) for {InputPath}"); + } + + private void GetSectionFromFileOffset(ImmutableArray sections, FilePosition fileOffset) + { + foreach (var section in sections) + { + if (fileOffset.Position >= section.PointerToRawData && fileOffset.Position < section.PointerToRawData + section.SizeOfRawData) + { + return; + } + } + + throw new InvalidOperationException($"file offset {fileOffset.Position} not in any section (PE) for {InputPath}"); + } + + // Make a new set of debug directory entries that + // have their data pointers adjusted to be relative to the start of the webcil file. + // This is necessary because the debug directory entires in the PE file are relative to the start of the PE file, + // and a PE header is bigger than a webcil header. + private ImmutableArray FixupDebugDirectoryEntries(PEFileInfo peInfo, WCFileInfo wcInfo) + { + int dataPointerAdjustment = peInfo.SectionStart.Position - wcInfo.SectionStart.Position; + ImmutableArray entries = peInfo.DebugDirectoryEntries; + ImmutableArray.Builder newEntries = ImmutableArray.CreateBuilder(entries.Length); + foreach (var entry in entries) + { + DebugDirectoryEntry newEntry; + if (entry.Type == DebugDirectoryEntryType.Reproducible || entry.DataPointer == 0 || entry.DataSize == 0) + { + // this entry doesn't have an associated data pointer, so just copy it + newEntry = entry; + } + else + { + // the "DataPointer" field is a file offset in the PE file, adjust the entry wit the corresponding offset in the Webcil file + var newDataPointer = entry.DataPointer - dataPointerAdjustment; + newEntry = new DebugDirectoryEntry(entry.Stamp, entry.MajorVersion, entry.MinorVersion, entry.Type, entry.DataSize, entry.DataRelativeVirtualAddress, newDataPointer); + GetSectionFromFileOffset(peInfo.SectionHeaders, entry.DataPointer); + // validate that the new entry is in some section + GetSectionFromFileOffset(wcInfo.SectionHeaders, newDataPointer); + } + newEntries.Add(newEntry); + } + return newEntries.MoveToImmutable(); + } + + private static void OverwriteDebugDirectoryEntries(Stream s, WCFileInfo wcInfo, ImmutableArray entries) + { + FilePosition debugDirectoryPos = GetPositionOfRelativeVirtualAddress(wcInfo.SectionHeaders, wcInfo.Header.pe_debug_rva); + using var writer = new BinaryWriter(s, System.Text.Encoding.UTF8, leaveOpen: true); + writer.Seek(debugDirectoryPos.Position, SeekOrigin.Begin); + foreach (var entry in entries) + { + WriteDebugDirectoryEntry(writer, entry); + } + // TODO check that we overwrite with the same size as the original + + // restore the stream position + writer.Seek(0, SeekOrigin.End); + } + + private static void WriteDebugDirectoryEntry(BinaryWriter writer, DebugDirectoryEntry entry) + { + writer.Write((uint)0); // Characteristics + writer.Write(entry.Stamp); + writer.Write(entry.MajorVersion); + writer.Write(entry.MinorVersion); + writer.Write((uint)entry.Type); + writer.Write(entry.DataSize); + writer.Write(entry.DataRelativeVirtualAddress); + writer.Write(entry.DataPointer); + } +} diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs new file mode 100644 index 00000000000000..6782ebf4a5aae6 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs @@ -0,0 +1,342 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using System.IO; +using System.Reflection; +using System.Runtime.InteropServices; + +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; + +namespace Microsoft.NET.WebAssembly.Webcil; + + +public sealed class WebcilReader : IDisposable +{ + // WISH: + // This should be implemented in terms of System.Reflection.Internal.MemoryBlockProvider like the PEReader, + // but the memory block classes are internal to S.R.M. + + private readonly Stream _stream; + private WebcilHeader _header; + private DirectoryEntry _corHeaderMetadataDirectory; + private MetadataReaderProvider? _metadataReaderProvider; + private ImmutableArray? _sections; + + private string? InputPath { get; } + + public WebcilReader(Stream stream) + { + this._stream = stream; + if (!stream.CanRead || !stream.CanSeek) + { + throw new ArgumentException("Stream must be readable and seekable", nameof(stream)); + } + if (!ReadHeader()) + { + throw new BadImageFormatException("Stream does not contain a valid Webcil file", nameof(stream)); + } + if (!ReadCorHeader()) + { + throw new BadImageFormatException("Stream does not contain a valid COR header in the Webcil file", nameof(stream)); + } + } + + public WebcilReader (Stream stream, string inputPath) : this(stream) + { + InputPath = inputPath; + } + + private unsafe bool ReadHeader() + { + WebcilHeader header; + var buffer = new byte[Marshal.SizeOf()]; + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + return false; + } + if (!BitConverter.IsLittleEndian) + { + throw new NotImplementedException("TODO: implement big endian support"); + } + fixed (byte* p = buffer) + { + header = *(WebcilHeader*)p; + } + if (header.id[0] != 'W' || header.id[1] != 'b' + || header.id[2] != 'I' || header.id[3] != 'L' + || header.version_major != Internal.Constants.WC_VERSION_MAJOR + || header.version_minor != Internal.Constants.WC_VERSION_MINOR) + { + return false; + } + _header = header; + return true; + } + + private unsafe bool ReadCorHeader() + { + // we can't construct CorHeader because it's constructor is internal + // but we don't care, really, we only want the metadata directory entry + var pos = TranslateRVA(_header.pe_cli_header_rva); + if (_stream.Seek(pos, SeekOrigin.Begin) != pos) + { + return false; + } + using var reader = new BinaryReader(_stream, System.Text.Encoding.UTF8, leaveOpen: true); + reader.ReadInt32(); // byte count + reader.ReadUInt16(); // major version + reader.ReadUInt16(); // minor version + _corHeaderMetadataDirectory = new DirectoryEntry(reader.ReadInt32(), reader.ReadInt32()); + return true; + } + + public MetadataReaderProvider GetMetadataReaderProvider() + { + // FIXME threading + if (_metadataReaderProvider == null) + { + long pos = TranslateRVA((uint)_corHeaderMetadataDirectory.RelativeVirtualAddress); + if (_stream.Seek(pos, SeekOrigin.Begin) != pos) + { + throw new BadImageFormatException("Could not seek to metadata in ", InputPath); + } + _metadataReaderProvider = MetadataReaderProvider.FromMetadataStream(_stream, MetadataStreamOptions.LeaveOpen); + } + return _metadataReaderProvider; + } + + public MetadataReader GetMetadataReader() => GetMetadataReaderProvider().GetMetadataReader(); + + public ImmutableArray ReadDebugDirectory() + { + var debugRVA = _header.pe_debug_rva; + if (debugRVA == 0) + { + return ImmutableArray.Empty; + } + var debugSize = _header.pe_debug_size; + if (debugSize == 0) + { + return ImmutableArray.Empty; + } + var debugOffset = TranslateRVA(debugRVA); + _stream.Seek(debugOffset, SeekOrigin.Begin); + var buffer = new byte[debugSize]; + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + throw new BadImageFormatException("Could not read debug directory", InputPath); + } + unsafe + { + fixed (byte* p = buffer) + { + return ReadDebugDirectoryEntries(new BlobReader(p, buffer.Length)); + } + } + } + + // FIXME: copied from DebugDirectoryEntry.Size + internal const int DebugDirectoryEntrySize = + sizeof(uint) + // Characteristics + sizeof(uint) + // TimeDataStamp + sizeof(uint) + // Version + sizeof(uint) + // Type + sizeof(uint) + // SizeOfData + sizeof(uint) + // AddressOfRawData + sizeof(uint); // PointerToRawData + + + // FIXME: copy-pasted from PEReader + private static ImmutableArray ReadDebugDirectoryEntries(BlobReader reader) + { + int entryCount = reader.Length / DebugDirectoryEntrySize; + var builder = ImmutableArray.CreateBuilder(entryCount); + for (int i = 0; i < entryCount; i++) + { + // Reserved, must be zero. + int characteristics = reader.ReadInt32(); + if (characteristics != 0) + { + throw new BadImageFormatException(); + } + + uint stamp = reader.ReadUInt32(); + ushort majorVersion = reader.ReadUInt16(); + ushort minorVersion = reader.ReadUInt16(); + + var type = (DebugDirectoryEntryType)reader.ReadInt32(); + + int dataSize = reader.ReadInt32(); + int dataRva = reader.ReadInt32(); + int dataPointer = reader.ReadInt32(); + + builder.Add(new DebugDirectoryEntry(stamp, majorVersion, minorVersion, type, dataSize, dataRva, dataPointer)); + } + + return builder.MoveToImmutable(); + } + + public CodeViewDebugDirectoryData ReadCodeViewDebugDirectoryData(DebugDirectoryEntry entry) + { + var pos = entry.DataPointer; + var buffer = new byte[entry.DataSize]; + if (_stream.Seek(pos, SeekOrigin.Begin) != pos) + { + throw new BadImageFormatException("Could not seek to CodeView debug directory data", nameof(_stream)); + } + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + throw new BadImageFormatException("Could not read CodeView debug directory data", nameof(_stream)); + } + unsafe + { + fixed (byte* p = buffer) + { + return DecodeCodeViewDebugDirectoryData(new BlobReader(p, buffer.Length)); + } + } + } + + private static CodeViewDebugDirectoryData DecodeCodeViewDebugDirectoryData(BlobReader reader) + { + // FIXME: copy-pasted from PEReader.DecodeCodeViewDebugDirectoryData + + if (reader.ReadByte() != (byte)'R' || + reader.ReadByte() != (byte)'S' || + reader.ReadByte() != (byte)'D' || + reader.ReadByte() != (byte)'S') + { + throw new BadImageFormatException("Unexpected CodeView data signature"); + } + + Guid guid = reader.ReadGuid(); + int age = reader.ReadInt32(); + string path = ReadUtf8NullTerminated(reader)!; + + return MakeCodeViewDebugDirectoryData(guid, age, path); + } + + private static string? ReadUtf8NullTerminated(BlobReader reader) + { + var mi = typeof(BlobReader).GetMethod("ReadUtf8NullTerminated", BindingFlags.NonPublic | BindingFlags.Instance); + if (mi == null) + { + throw new InvalidOperationException("Could not find BlobReader.ReadUtf8NullTerminated"); + } + return (string?)mi.Invoke(reader, null); + } + + private static CodeViewDebugDirectoryData MakeCodeViewDebugDirectoryData(Guid guid, int age, string path) + { + var types = new Type[] { typeof(Guid), typeof(int), typeof(string) }; + var mi = typeof(CodeViewDebugDirectoryData).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, types, null); + if (mi == null) + { + throw new InvalidOperationException("Could not find CodeViewDebugDirectoryData constructor"); + } + return (CodeViewDebugDirectoryData)mi.Invoke(new object[] { guid, age, path }); + } + + public MetadataReaderProvider ReadEmbeddedPortablePdbDebugDirectoryData(DebugDirectoryEntry entry) + { + var pos = entry.DataPointer; + var buffer = new byte[entry.DataSize]; + if (_stream.Seek(pos, SeekOrigin.Begin) != pos) + { + throw new BadImageFormatException("Could not seek to Embedded Portable PDB debug directory data", nameof(_stream)); + } + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + throw new BadImageFormatException("Could not read Embedded Portable PDB debug directory data", nameof(_stream)); + } + unsafe + { + fixed (byte* p = buffer) + { + return DecodeEmbeddedPortablePdbDirectoryData(new BlobReader(p, buffer.Length)); + } + } + } + + private const uint PortablePdbVersions_DebugDirectoryEmbeddedSignature = 0x4244504d; + private static MetadataReaderProvider DecodeEmbeddedPortablePdbDirectoryData(BlobReader reader) + { + // FIXME: inspired by PEReader.DecodeEmbeddedPortablePdbDebugDirectoryData + // but not using its internal utility classes. + + if (reader.ReadUInt32() != PortablePdbVersions_DebugDirectoryEmbeddedSignature) + { + throw new BadImageFormatException("Unexpected embedded portable PDB data signature"); + } + + int decompressedSize = reader.ReadInt32(); + + byte[] decompressedBuffer; + + byte[] compressedBuffer = reader.ReadBytes(reader.RemainingBytes); + + using (var compressedStream = new MemoryStream(compressedBuffer, writable: false)) + using (var deflateStream = new System.IO.Compression.DeflateStream(compressedStream, System.IO.Compression.CompressionMode.Decompress, leaveOpen: true)) + { +#if NETCOREAPP1_1_OR_GREATER + decompressedBuffer = GC.AllocateUninitializedArray(decompressedSize); +#else + decompressedBuffer = new byte[decompressedSize]; +#endif + using (var decompressedStream = new MemoryStream(decompressedBuffer, writable: true)) + { + deflateStream.CopyTo(decompressedStream); + } + } + + + return MetadataReaderProvider.FromPortablePdbStream(new MemoryStream(decompressedBuffer, writable: false)); + + } + + private long TranslateRVA(uint rva) + { + if (_sections == null) + { + _sections = ReadSections(); + } + foreach (var section in _sections.Value) + { + if (rva >= section.VirtualAddress && rva < section.VirtualAddress + section.VirtualSize) + { + return section.PointerToRawData + (rva - section.VirtualAddress); + } + } + throw new BadImageFormatException("RVA not found in any section", nameof(_stream)); + } + + private static long SectionDirectoryOffset => Marshal.SizeOf(); + + private unsafe ImmutableArray ReadSections() + { + var sections = ImmutableArray.CreateBuilder(_header.coff_sections); + var buffer = new byte[Marshal.SizeOf()]; + _stream.Seek(SectionDirectoryOffset, SeekOrigin.Begin); + for (int i = 0; i < _header.coff_sections; i++) + { + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + throw new BadImageFormatException("Stream does not contain a valid Webcil file", nameof(_stream)); + } + fixed (byte* p = buffer) + { + // FIXME endianness + sections.Add(*(WebcilSectionHeader*)p); + } + } + return sections.MoveToImmutable(); + } + + public void Dispose() + { + _stream.Dispose(); + } +} diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilSectionHeader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilSectionHeader.cs new file mode 100644 index 00000000000000..8571fb8ac325c4 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilSectionHeader.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +namespace Microsoft.NET.WebAssembly.Webcil; + +/// +/// This is the Webcil analog of System.Reflection.PortableExecutable.SectionHeader, but with fewer fields +/// +[StructLayout(LayoutKind.Sequential, Pack = 1)] +public readonly struct WebcilSectionHeader +{ + public readonly int VirtualSize; + public readonly int VirtualAddress; + public readonly int SizeOfRawData; + public readonly int PointerToRawData; + + public WebcilSectionHeader(int virtualSize, int virtualAddress, int sizeOfRawData, int pointerToRawData) + { + VirtualSize = virtualSize; + VirtualAddress = virtualAddress; + SizeOfRawData = sizeOfRawData; + PointerToRawData = pointerToRawData; + } +} diff --git a/src/libraries/sendtohelix-wasm.targets b/src/libraries/sendtohelix-wasm.targets index e1afc40d2cb1f4..0e9db2b6f861d8 100644 --- a/src/libraries/sendtohelix-wasm.targets +++ b/src/libraries/sendtohelix-wasm.targets @@ -236,6 +236,7 @@ $(RepositoryEngineeringDir)testing\scenarios\BuildWasmAppsJobsList.txt Workloads- NoWorkload- + $(WorkItemPrefix)Webcil- @@ -260,7 +261,7 @@ $(_workItemTimeout) - + $(_BuildWasmAppsPayloadArchive) $(HelixCommand) $(_workItemTimeout) diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj index 65d5ca651db72b..71619fc40466a4 100644 --- a/src/libraries/sendtohelix.proj +++ b/src/libraries/sendtohelix.proj @@ -78,12 +78,21 @@ - + <_TestUsingWorkloadsValues Include="true;false" /> - + <_TestUsingWebcilValues Include="true;false" /> + + + <_TestUsingCrossProductValuesTemp Include="@(_TestUsingWorkloadsValues)"> + %(_TestUsingWorkloadsValues.Identity) + + <_TestUsingCrossProductValues Include="@(_TestUsingCrossProductValuesTemp)"> + %(_TestUsingWebcilValues.Identity) + + <_BuildWasmAppsProjectsToBuild Include="$(PerScenarioProjectFile)"> - $(_PropertiesToPass);Scenario=BuildWasmApps;TestArchiveRuntimeFile=$(TestArchiveRuntimeFile);TestUsingWorkloads=%(_TestUsingWorkloadsValues.Identity) + $(_PropertiesToPass);Scenario=BuildWasmApps;TestArchiveRuntimeFile=$(TestArchiveRuntimeFile);TestUsingWorkloads=%(_TestUsingCrossProductValues.Workloads);TestUsingWebcil=%(_TestUsingCrossProductValues.Webcil) %(_BuildWasmAppsProjectsToBuild.AdditionalProperties);NeedsToBuildWasmAppsOnHelix=$(NeedsToBuildWasmAppsOnHelix) diff --git a/src/libraries/sendtohelixhelp.proj b/src/libraries/sendtohelixhelp.proj index 6f0b7704b575fc..bfb6447e647b6b 100644 --- a/src/libraries/sendtohelixhelp.proj +++ b/src/libraries/sendtohelixhelp.proj @@ -136,6 +136,7 @@ + @@ -287,7 +288,7 @@ + Text="Scenario: $(Scenario), TestUsingWorkloads: $(TestUsingWorkloads), TestUsingWebcil: $(TestUsingWebcil)" /> diff --git a/src/mono/cmake/config.h.in b/src/mono/cmake/config.h.in index ca8e02ac6fd888..d0b5d79ff2097f 100644 --- a/src/mono/cmake/config.h.in +++ b/src/mono/cmake/config.h.in @@ -933,6 +933,9 @@ /* Enable System.WeakAttribute support */ #cmakedefine ENABLE_WEAK_ATTR 1 +/* Enable WebCIL image loader */ +#cmakedefine ENABLE_WEBCIL 1 + #if defined(ENABLE_LLVM) && defined(HOST_WIN32) && defined(TARGET_WIN32) && (!defined(TARGET_AMD64) || !defined(_MSC_VER)) #error LLVM for host=Windows and target=Windows is only supported on x64 MSVC build. #endif diff --git a/src/mono/cmake/options.cmake b/src/mono/cmake/options.cmake index f00430fc9500d6..202a578db4fc4a 100644 --- a/src/mono/cmake/options.cmake +++ b/src/mono/cmake/options.cmake @@ -57,6 +57,7 @@ option (ENABLE_SIGALTSTACK "Enable support for using sigaltstack for SIGSEGV and option (USE_MALLOC_FOR_MEMPOOLS "Use malloc for each single mempool allocation, so tools like Valgrind can run better") option (STATIC_COMPONENTS "Compile mono runtime components as static (not dynamic) libraries") option (DISABLE_WASM_USER_THREADS "Disable creation of user managed threads on WebAssembly, only allow runtime internal managed and native threads") +option (ENABLE_WEBCIL "Enable the WebCIL loader") set (MONO_GC "sgen" CACHE STRING "Garbage collector implementation (sgen or boehm). Default: sgen") set (GC_SUSPEND "default" CACHE STRING "GC suspend method (default, preemptive, coop, hybrid)") diff --git a/src/mono/mono.proj b/src/mono/mono.proj index b4289a0b76f8b0..43a084f07a5a04 100644 --- a/src/mono/mono.proj +++ b/src/mono/mono.proj @@ -405,6 +405,7 @@ <_MonoCMakeArgs Include="-DDISABLE_ICALL_TABLES=1"/> <_MonoCMakeArgs Include="-DENABLE_ICALL_EXPORT=1"/> <_MonoCMakeArgs Include="-DENABLE_LAZY_GC_THREAD_CREATION=1"/> + <_MonoCMakeArgs Include="-DENABLE_WEBCIL=1"/> <_MonoCFLAGS Include="-fexceptions"/> <_MonoCFLAGS Condition="'$(MonoWasmThreads)' == 'true'" Include="-pthread"/> <_MonoCFLAGS Condition="'$(MonoWasmThreads)' == 'true'" Include="-D_GNU_SOURCE=1" /> diff --git a/src/mono/mono/metadata/CMakeLists.txt b/src/mono/mono/metadata/CMakeLists.txt index 67f7a66175b6e1..797637a566cbc7 100644 --- a/src/mono/mono/metadata/CMakeLists.txt +++ b/src/mono/mono/metadata/CMakeLists.txt @@ -78,6 +78,8 @@ set(metadata_common_sources icall-eventpipe.c image.c image-internals.h + webcil-loader.h + webcil-loader.c jit-info.h jit-info.c loader.c diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 60205d8f351f40..adb2ae0d5306ef 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -720,6 +720,7 @@ search_bundle_for_assembly (MonoAssemblyLoadContext *alc, MonoAssemblyName *anam if (!image && !g_str_has_suffix (aname->name, ".dll")) { char *name = g_strdup_printf ("%s.dll", aname->name); image = mono_assembly_open_from_bundle (alc, name, &status, aname->culture); + g_free (name); } if (image) { mono_assembly_request_prepare_load (&req, alc); @@ -1449,6 +1450,25 @@ absolute_dir (const gchar *filename) return res; } +static gboolean +bundled_assembly_match (const char *bundled_name, const char *name) +{ +#ifndef ENABLE_WEBCIL + return strcmp (bundled_name, name) == 0; +#else + if (strcmp (bundled_name, name) == 0) + return TRUE; + /* if they want a .dll and we have the matching .webcil, return it */ + if (g_str_has_suffix (bundled_name, ".webcil") && g_str_has_suffix (name, ".dll")) { + size_t bprefix = strlen (bundled_name) - 7; + size_t nprefix = strlen (name) - 4; + if (bprefix == nprefix && strncmp (bundled_name, name, bprefix) == 0) + return TRUE; + } + return FALSE; +#endif +} + static MonoImage * open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status, gboolean is_satellite) { @@ -1458,7 +1478,7 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M MonoImage *image = NULL; char *name = is_satellite ? g_strdup (filename) : g_path_get_basename (filename); for (int i = 0; !image && bundles [i]; ++i) { - if (strcmp (bundles [i]->name, name) == 0) { + if (bundled_assembly_match (bundles[i]->name, name)) { // Since bundled images don't exist on disk, don't give them a legit filename image = mono_image_open_from_data_internal (alc, (char*)bundles [i]->data, bundles [i]->size, FALSE, status, FALSE, name, NULL); break; @@ -1479,7 +1499,7 @@ open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, char *name = g_strdup (filename); for (int i = 0; !image && satellite_bundles [i]; ++i) { - if (strcmp (satellite_bundles [i]->name, name) == 0 && strcmp (satellite_bundles [i]->culture, culture) == 0) { + if (bundled_assembly_match (satellite_bundles[i]->name, name) && strcmp (satellite_bundles [i]->culture, culture) == 0) { char *bundle_name = g_strconcat (culture, "/", name, (const char *)NULL); image = mono_image_open_from_data_internal (alc, (char *)satellite_bundles [i]->data, satellite_bundles [i]->size, FALSE, status, FALSE, bundle_name, NULL); g_free (bundle_name); @@ -2710,6 +2730,14 @@ mono_assembly_load_corlib (void) corlib = mono_assembly_request_open (corlib_name, &req, &status); g_free (corlib_name); } +#ifdef ENABLE_WEBCIL + if (!corlib) { + /* Maybe its in a bundle */ + char *corlib_name = g_strdup_printf ("%s.webcil", MONO_ASSEMBLY_CORLIB_NAME); + corlib = mono_assembly_request_open (corlib_name, &req, &status); + g_free (corlib_name); + } +#endif g_assert (corlib); // exit the process if we weren't able to load corlib diff --git a/src/mono/mono/metadata/image.c b/src/mono/mono/metadata/image.c index 61fac42a5d4e9e..2eef0230f2a710 100644 --- a/src/mono/mono/metadata/image.c +++ b/src/mono/mono/metadata/image.c @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #ifdef HAVE_UNISTD_H @@ -259,6 +260,10 @@ mono_images_init (void) install_pe_loader (); +#ifdef ENABLE_WEBCIL + mono_webcil_loader_install (); +#endif + mutex_inited = TRUE; } @@ -923,26 +928,43 @@ do_load_header (MonoImage *image, MonoDotNetHeader *header, int offset) return offset; } -mono_bool -mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) +static int32_t +try_load_pe_cli_header (char *raw_data, uint32_t raw_data_len, MonoDotNetHeader *cli_header) { - MonoDotNetHeader cli_header; MonoMSDOSHeader msdos; - guint8 *data; int offset = 0; memcpy (&msdos, raw_data + offset, sizeof (msdos)); if (!(msdos.msdos_sig [0] == 'M' && msdos.msdos_sig [1] == 'Z')) { - return FALSE; + return -1; } msdos.pe_offset = GUINT32_FROM_LE (msdos.pe_offset); offset = msdos.pe_offset; - int ret = do_load_header_internal (raw_data, raw_data_len, &cli_header, offset, FALSE); - if ( ret >= 0 ) { + int32_t ret = do_load_header_internal (raw_data, raw_data_len, cli_header, offset, FALSE); + return ret; +} + +mono_bool +mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) +{ + guint8 *data; + MonoDotNetHeader cli_header; + gboolean is_pe = TRUE; + + int32_t ret = try_load_pe_cli_header (raw_data, raw_data_len, &cli_header); + +#ifdef ENABLE_WEBCIL + if (ret == -1) { + ret = mono_webcil_load_cli_header (raw_data, raw_data_len, 0, &cli_header); + is_pe = FALSE; + } +#endif + + if (ret > 0) { MonoPEDirEntry *debug_dir_entry = (MonoPEDirEntry *) &cli_header.datadir.pe_debug; ImageDebugDirectory debug_dir; if (!debug_dir_entry->size) @@ -951,28 +973,39 @@ mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) const int top = cli_header.coff.coff_sections; guint32 addr = debug_dir_entry->rva; int i = 0; + gboolean found = FALSE; for (i = 0; i < top; i++){ - MonoSectionTable t; + MonoSectionTable t = {0,}; - if (ret + sizeof (MonoSectionTable) > raw_data_len) { - return FALSE; - } + if (G_LIKELY (is_pe)) { + if (ret + sizeof (MonoSectionTable) > raw_data_len) + return FALSE; - memcpy (&t, raw_data + ret, sizeof (MonoSectionTable)); - ret += sizeof (MonoSectionTable); + memcpy (&t, raw_data + ret, sizeof (MonoSectionTable)); + ret += sizeof (MonoSectionTable); #if G_BYTE_ORDER != G_LITTLE_ENDIAN - t.st_virtual_address = GUINT32_FROM_LE (t.st_virtual_address); - t.st_raw_data_size = GUINT32_FROM_LE (t.st_raw_data_size); - t.st_raw_data_ptr = GUINT32_FROM_LE (t.st_raw_data_ptr); + t.st_virtual_address = GUINT32_FROM_LE (t.st_virtual_address); + t.st_raw_data_size = GUINT32_FROM_LE (t.st_raw_data_size); + t.st_raw_data_ptr = GUINT32_FROM_LE (t.st_raw_data_ptr); #endif + } +#ifdef ENABLE_WEBCIL + else { + ret = mono_webcil_load_section_table (raw_data, raw_data_len, ret, &t); + if (ret == -1) + return FALSE; + } +#endif /* consistency checks here */ if ((addr >= t.st_virtual_address) && (addr < t.st_virtual_address + t.st_raw_data_size)){ addr = addr - t.st_virtual_address + t.st_raw_data_ptr; + found = TRUE; break; } } + g_assert (found); for (guint32 idx = 0; idx < debug_dir_entry->size / sizeof (ImageDebugDirectory); ++idx) { data = (guint8 *) ((ImageDebugDirectory *) (raw_data + addr) + idx); debug_dir.characteristics = read32(data); diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 5c81475ccb2443..958b33657d43a9 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1097,13 +1097,31 @@ mono_register_symfile_for_assembly (const char *assembly_name, const mono_byte * bundled_symfiles = bsymfile; } +static gboolean +bsymfile_match (BundledSymfile *bsymfile, const char *assembly_name) +{ + if (!strcmp (bsymfile->aname, assembly_name)) + return TRUE; +#ifdef ENABLE_WEBCIL + const char *p = strstr (assembly_name, ".webcil"); + /* if assembly_name ends with .webcil, check if aname matches, with a .dll extension instead */ + if (p && *(p + 7) == 0) { + size_t n = p - assembly_name; + if (!strncmp (bsymfile->aname, assembly_name, n) + && !strcmp (bsymfile->aname + n, ".dll")) + return TRUE; + } +#endif + return FALSE; +} + static MonoDebugHandle * open_symfile_from_bundle (MonoImage *image) { BundledSymfile *bsymfile; for (bsymfile = bundled_symfiles; bsymfile; bsymfile = bsymfile->next) { - if (strcmp (bsymfile->aname, image->module_name)) + if (!bsymfile_match (bsymfile, image->module_name)) continue; return mono_debug_open_image (image, bsymfile->raw_contents, bsymfile->size); @@ -1117,7 +1135,7 @@ mono_get_symfile_bytes_from_bundle (const char *assembly_name, int *size) { BundledSymfile *bsymfile; for (bsymfile = bundled_symfiles; bsymfile; bsymfile = bsymfile->next) { - if (strcmp (bsymfile->aname, assembly_name)) + if (!bsymfile_match (bsymfile, assembly_name)) continue; *size = bsymfile->size; return bsymfile->raw_contents; diff --git a/src/mono/mono/metadata/reflection.c b/src/mono/mono/metadata/reflection.c index 487a31f4ec317e..ad4d9795b6877a 100644 --- a/src/mono/mono/metadata/reflection.c +++ b/src/mono/mono/metadata/reflection.c @@ -1285,7 +1285,7 @@ method_body_object_construct (MonoClass *unused_class, MonoMethod *method, gpoin if ((method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) || (method->flags & METHOD_ATTRIBUTE_ABSTRACT) || (method->iflags & METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL) || - (image->raw_data && image->raw_data [1] != 'Z') || + (image->raw_data && (image->raw_data [1] != 'Z' && image->raw_data [1] != 'b')) || (method->iflags & METHOD_IMPL_ATTRIBUTE_RUNTIME)) return MONO_HANDLE_CAST (MonoReflectionMethodBody, NULL_HANDLE); diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c new file mode 100644 index 00000000000000..1323c0f3ff1373 --- /dev/null +++ b/src/mono/mono/metadata/webcil-loader.c @@ -0,0 +1,170 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +#include + +#include + +#include "mono/metadata/metadata-internals.h" +#include "mono/metadata/webcil-loader.h" + +/* keep in sync with webcil-writer */ +enum { + MONO_WEBCIL_VERSION_MAJOR = 0, + MONO_WEBCIL_VERSION_MINOR = 0, +}; + +typedef struct MonoWebCilHeader { + uint8_t id[4]; // 'W' 'b' 'I' 'L' + // 4 bytes + uint16_t version_major; // 0 + uint16_t version_minor; // 0 + // 8 bytes + uint16_t coff_sections; + uint16_t reserved0; // 0 + // 12 bytes + + uint32_t pe_cli_header_rva; + uint32_t pe_cli_header_size; + // 20 bytes + + uint32_t pe_debug_rva; + uint32_t pe_debug_size; + // 28 bytes +} MonoWebCilHeader; + +static gboolean +webcil_image_match (MonoImage *image) +{ + if (image->raw_data_len >= sizeof (MonoWebCilHeader)) { + return image->raw_data[0] == 'W' && image->raw_data[1] == 'b' && image->raw_data[2] == 'I' && image->raw_data[3] == 'L'; + } + return FALSE; +} + +/* + * Fills the MonoDotNetHeader with data from the given raw_data+offset + * by reading the webcil header. + * most of MonoDotNetHeader is unused and left uninitialized (assumed zero); + */ +static int32_t +do_load_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header) +{ + MonoWebCilHeader wcheader; + if (offset + sizeof (MonoWebCilHeader) > raw_data_len) + return -1; + memcpy (&wcheader, raw_data + offset, sizeof (wcheader)); + + if (!(wcheader.id [0] == 'W' && wcheader.id [1] == 'b' && wcheader.id[2] == 'I' && wcheader.id[3] == 'L' && + GUINT16_FROM_LE (wcheader.version_major) == MONO_WEBCIL_VERSION_MAJOR && GUINT16_FROM_LE (wcheader.version_minor) == MONO_WEBCIL_VERSION_MINOR)) + return -1; + + memset (header, 0, sizeof(MonoDotNetHeader)); + header->coff.coff_sections = GUINT16_FROM_LE (wcheader.coff_sections); + header->datadir.pe_cli_header.rva = GUINT32_FROM_LE (wcheader.pe_cli_header_rva); + header->datadir.pe_cli_header.size = GUINT32_FROM_LE (wcheader.pe_cli_header_size); + header->datadir.pe_debug.rva = GUINT32_FROM_LE (wcheader.pe_debug_rva); + header->datadir.pe_debug.size = GUINT32_FROM_LE (wcheader.pe_debug_size); + + offset += sizeof (wcheader); + return offset; +} + +int32_t +mono_webcil_load_section_table (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoSectionTable *t) +{ + /* WebCIL section table entries are a subset of a PE section + * header. Initialize just the parts we have. + */ + uint32_t st [4]; + + if (G_UNLIKELY (offset < 0)) + return offset; + if ((uint32_t)offset > raw_data_len) + return -1; + memcpy (st, raw_data + offset, sizeof (st)); + t->st_virtual_size = GUINT32_FROM_LE (st [0]); + t->st_virtual_address = GUINT32_FROM_LE (st [1]); + t->st_raw_data_size = GUINT32_FROM_LE (st [2]); + t->st_raw_data_ptr = GUINT32_FROM_LE (st [3]); + offset += sizeof(st); + return offset; +} + + +static gboolean +webcil_image_load_pe_data (MonoImage *image) +{ + MonoCLIImageInfo *iinfo; + MonoDotNetHeader *header; + int32_t offset = 0; + int top; + + iinfo = image->image_info; + header = &iinfo->cli_header; + + offset = do_load_header (image->raw_data, image->raw_data_len, offset, header); + if (offset == -1) + goto invalid_image; + + top = iinfo->cli_header.coff.coff_sections; + + iinfo->cli_section_count = top; + iinfo->cli_section_tables = g_new0 (MonoSectionTable, top); + iinfo->cli_sections = g_new0 (void *, top); + + for (int i = 0; i < top; i++) { + MonoSectionTable *t = &iinfo->cli_section_tables [i]; + offset = mono_webcil_load_section_table (image->raw_data, image->raw_data_len, offset, t); + if (offset == -1) + goto invalid_image; + } + + return TRUE; + +invalid_image: + return FALSE; + +} + +static gboolean +webcil_image_load_cli_data (MonoImage *image) +{ + MonoCLIImageInfo *iinfo; + + iinfo = image->image_info; + + if (!mono_image_load_cli_header (image, iinfo)) + return FALSE; + + if (!mono_image_load_metadata (image, iinfo)) + return FALSE; + + return TRUE; +} + +static gboolean +webcil_image_load_tables (MonoImage *image) +{ + return TRUE; +} + +static const MonoImageLoader webcil_loader = { + webcil_image_match, + webcil_image_load_pe_data, + webcil_image_load_cli_data, + webcil_image_load_tables, +}; + +void +mono_webcil_loader_install (void) +{ + mono_install_image_loader (&webcil_loader); +} + +int32_t +mono_webcil_load_cli_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header) +{ + return do_load_header (raw_data, raw_data_len, offset, header); +} diff --git a/src/mono/mono/metadata/webcil-loader.h b/src/mono/mono/metadata/webcil-loader.h new file mode 100644 index 00000000000000..c95c2c5690409e --- /dev/null +++ b/src/mono/mono/metadata/webcil-loader.h @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +#ifndef _MONO_METADATA_WEBCIL_LOADER_H +#define _MONO_METADATA_WEBCIL_LOADER_H + +void +mono_webcil_loader_install (void); + +int32_t +mono_webcil_load_cli_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header); + +int32_t +mono_webcil_load_section_table (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoSectionTable *t); + +#endif /*_MONO_METADATA_WEBCIL_LOADER_H*/ diff --git a/src/mono/mono/mini/monovm.c b/src/mono/mono/mini/monovm.c index 144594276f2187..63e25581ac7a61 100644 --- a/src/mono/mono/mini/monovm.c +++ b/src/mono/mono/mini/monovm.c @@ -131,6 +131,22 @@ mono_core_preload_hook (MonoAssemblyLoadContext *alc, MonoAssemblyName *aname, c if (result) break; } +#ifdef ENABLE_WEBCIL + else { + /* /path/foo.dll -> /path/foo.webcil */ + size_t n = strlen (fullpath) - 4; + char *fullpath2 = g_malloc (n + 8); + g_strlcpy (fullpath2, fullpath, n + 1); + g_strlcpy (fullpath2 + n, ".webcil", 8); + if (g_file_test (fullpath2, G_FILE_TEST_IS_REGULAR)) { + MonoImageOpenStatus status; + result = mono_assembly_request_open (fullpath2, &req, &status); + } + g_free (fullpath2); + if (result) + break; + } +#endif } } diff --git a/src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj b/src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj index 7a6b8326bcfb31..2119425f85f0c4 100644 --- a/src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj +++ b/src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj @@ -14,6 +14,7 @@ <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\BrowserDebugHost.dll" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\BrowserDebugHost.runtimeconfig.json" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\BrowserDebugProxy.dll" /> + <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.NET.WebAssembly.Webcil.dll" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.CodeAnalysis.CSharp.dll" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.CodeAnalysis.dll" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Newtonsoft.Json.dll" /> diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs b/src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs index f40ddec9aa4a8b..6d88eb47a25635 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs @@ -24,11 +24,14 @@ public class BuildEnvironment public string WorkloadPacksDir { get; init; } public string BuiltNuGetsPath { get; init; } + public bool UseWebcil { get; init; } + public static readonly string RelativeTestAssetsPath = @"..\testassets\"; public static readonly string TestAssetsPath = Path.Combine(AppContext.BaseDirectory, "testassets"); public static readonly string TestDataPath = Path.Combine(AppContext.BaseDirectory, "data"); public static readonly string TmpPath = Path.Combine(AppContext.BaseDirectory, "wbt"); + private static readonly Dictionary s_runtimePackVersions = new(); public BuildEnvironment() @@ -86,6 +89,8 @@ public BuildEnvironment() DirectoryBuildTargetsContents = s_directoryBuildTargetsForLocal; } + UseWebcil = EnvironmentVariables.UseWebcil; + if (EnvironmentVariables.BuiltNuGetsPath is null || !Directory.Exists(EnvironmentVariables.BuiltNuGetsPath)) throw new Exception($"Cannot find 'BUILT_NUGETS_PATH={EnvironmentVariables.BuiltNuGetsPath}'"); diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs index 0b4777ae7ac31a..9ebc210e803b59 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs @@ -50,6 +50,7 @@ public abstract class BuildTestBase : IClassFixture s_buildEnv.IsWorkload; public static bool IsNotUsingWorkloads => !s_buildEnv.IsWorkload; + public static bool UseWebcil => s_buildEnv.UseWebcil; public static string GetNuGetConfigPathFor(string targetFramework) => Path.Combine(BuildEnvironment.TestDataPath, "nuget8.config"); // for now - we are still using net7, but with // targetFramework == "net7.0" ? "nuget7.config" : "nuget8.config"); @@ -71,6 +72,8 @@ static BuildTestBase() Console.WriteLine (""); Console.WriteLine ($"=============================================================================================="); Console.WriteLine ($"=============== Running with {(s_buildEnv.IsWorkload ? "Workloads" : "No workloads")} ==============="); + if (UseWebcil) + Console.WriteLine($"=============== Using .webcil ==============="); Console.WriteLine ($"=============================================================================================="); Console.WriteLine (""); } @@ -336,6 +339,10 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp extraProperties += $"\n{RuntimeInformation.IsOSPlatform(OSPlatform.Windows)}\n"; } + if (UseWebcil) { + extraProperties += "true\n"; + } + string projectContents = projectTemplate .Replace("##EXTRA_PROPERTIES##", extraProperties) .Replace("##EXTRA_ITEMS##", extraItems) @@ -423,7 +430,8 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp options.HasV8Script, options.TargetFramework ?? DefaultTargetFramework, options.HasIcudt, - options.DotnetWasmFromRuntimePack ?? !buildArgs.AOT); + options.DotnetWasmFromRuntimePack ?? !buildArgs.AOT, + UseWebcil); } if (options.UseCache) @@ -487,6 +495,8 @@ public string CreateWasmTemplateProject(string id, string template = "wasmbrowse string projectfile = Path.Combine(_projectDir!, $"{id}.csproj"); if (runAnalyzers) AddItemsPropertiesToProject(projectfile, "true"); + if (UseWebcil) + AddItemsPropertiesToProject(projectfile, "true"); return projectfile; } @@ -499,7 +509,10 @@ public string CreateBlazorWasmTemplateProject(string id) .ExecuteWithCapturedOutput("new blazorwasm") .EnsureSuccessful(); - return Path.Combine(_projectDir!, $"{id}.csproj"); + string projectFile = Path.Combine(_projectDir!, $"{id}.csproj"); + if (UseWebcil) + AddItemsPropertiesToProject(projectFile, "true"); + return projectFile; } protected (CommandResult, string) BlazorBuild(BlazorBuildOptions options, params string[] extraArgs) @@ -550,6 +563,7 @@ public string CreateBlazorWasmTemplateProject(string id) label, // same as the command name $"-bl:{logPath}", $"-p:Configuration={config}", + UseWebcil ? "-p:WasmEnableWebcil=true" : string.Empty, "-p:BlazorEnableCompression=false", "-nr:false", setWasmDevel ? "-p:_WasmDevel=true" : string.Empty @@ -616,7 +630,8 @@ protected static void AssertBasicAppBundle(string bundleDir, bool hasV8Script, string targetFramework, bool hasIcudt = true, - bool dotnetWasmFromRuntimePack = true) + bool dotnetWasmFromRuntimePack = true, + bool useWebcil = true) { AssertFilesExist(bundleDir, new [] { @@ -632,7 +647,9 @@ protected static void AssertBasicAppBundle(string bundleDir, AssertFilesExist(bundleDir, new[] { "icudt.dat" }, expectToExist: hasIcudt); string managedDir = Path.Combine(bundleDir, "managed"); - AssertFilesExist(managedDir, new[] { $"{projectName}.dll" }); + string bundledMainAppAssembly = + useWebcil ? $"{projectName}.webcil" : $"{projectName}.dll"; + AssertFilesExist(managedDir, new[] { bundledMainAppAssembly }); bool is_debug = config == "Debug"; if (is_debug) @@ -1094,7 +1111,7 @@ public record BuildProjectOptions bool CreateProject = true, bool Publish = true, bool BuildOnlyAfterPublish = true, - bool HasV8Script = true, + bool HasV8Script = true, string? Verbosity = null, string? Label = null, string? TargetFramework = null, diff --git a/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs b/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs index 328006786d09c6..91f6fadb51f5b5 100644 --- a/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs +++ b/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs @@ -18,5 +18,6 @@ internal static class EnvironmentVariables internal static readonly string? BuiltNuGetsPath = Environment.GetEnvironmentVariable("BUILT_NUGETS_PATH"); internal static readonly string? BrowserPathForTests = Environment.GetEnvironmentVariable("BROWSER_PATH_FOR_TESTS"); internal static readonly bool ShowBuildOutput = Environment.GetEnvironmentVariable("SHOW_BUILD_OUTPUT") is not null; + internal static readonly bool UseWebcil = Environment.GetEnvironmentVariable("USE_WEBCIL_FOR_TESTS") is "true"; } } diff --git a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs index 5473667f294c9c..0168840ad99bf3 100644 --- a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs @@ -466,15 +466,29 @@ public static void Main() string tasksDir = Path.Combine(s_buildEnv.WorkloadPacksDir, "Microsoft.NET.Runtime.WebAssembly.Sdk", s_buildEnv.GetRuntimePackVersion(DefaultTargetFramework), - "tasks"); - if (!Directory.Exists(tasksDir)) + "tasks", + BuildTestBase.DefaultTargetFramework); // not net472! + if (!Directory.Exists(tasksDir)) { + string? tasksDirParent = Path.GetDirectoryName (tasksDir); + if (!string.IsNullOrEmpty (tasksDirParent)) { + if (!Directory.Exists(tasksDirParent)) { + _testOutput.WriteLine($"Expected {tasksDirParent} to exist and contain TFM subdirectories"); + } + _testOutput.WriteLine($"runtime pack tasks dir {tasksDir} contains subdirectories:"); + foreach (string subdir in Directory.EnumerateDirectories(tasksDirParent)) { + _testOutput.WriteLine($" - {subdir}"); + } + } throw new DirectoryNotFoundException($"Could not find tasks directory {tasksDir}"); + } string? taskPath = Directory.EnumerateFiles(tasksDir, "WasmAppBuilder.dll", SearchOption.AllDirectories) .FirstOrDefault(); if (string.IsNullOrEmpty(taskPath)) throw new FileNotFoundException($"Could not find WasmAppBuilder.dll in {tasksDir}"); + _testOutput.WriteLine ("Using WasmAppBuilder.dll from {0}", taskPath); + projectCode = projectCode .Replace("###WasmPInvokeModule###", AddAssembly("System.Private.CoreLib") + AddAssembly("System.Runtime") + AddAssembly(libraryBuildArgs.ProjectName)) .Replace("###WasmAppBuilder###", taskPath); diff --git a/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj b/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj index cf7a20b5623910..133140d6960957 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj +++ b/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj @@ -66,6 +66,9 @@ + + + diff --git a/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd b/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd index f38746bf131515..23e3dbf4ce470c 100644 --- a/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd +++ b/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd @@ -104,6 +104,11 @@ if [%TEST_USING_WORKLOADS%] == [true] ( set _DIR_NAME=dotnet-none set SDK_HAS_WORKLOAD_INSTALLED=false ) +if [%TEST_USING_WEBCIL%] == [true] ( + set USE_WEBCIL_FOR_TESTS=true +) else ( + set USE_WEBCIL_FOR_TESTS=false +) if [%HELIX_CORRELATION_PAYLOAD%] NEQ [] ( robocopy /mt /np /nfl /NDL /nc /e %BASE_DIR%\%_DIR_NAME% %EXECUTION_DIR%\%_DIR_NAME% diff --git a/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh b/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh index 1f520aebd65f66..70187b9c5e58e8 100644 --- a/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh +++ b/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh @@ -79,6 +79,12 @@ function set_env_vars() export SDK_HAS_WORKLOAD_INSTALLED=false fi + if [ "x$TEST_USING_WEBCIL" = "xtrue" ]; then + export USE_WEBCIL_FOR_TESTS=true + else + export USE_WEBCIL_FOR_TESTS=false + fi + local _SDK_DIR= if [[ -n "$HELIX_WORKITEM_UPLOAD_ROOT" ]]; then cp -r $BASE_DIR/$_DIR_NAME $EXECUTION_DIR diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index 497eb57e7168f2..9f5647e8537907 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -67,6 +67,7 @@ - $(WasmAotProfilePath) - Path to an AOT profile file. - $(WasmEnableExceptionHandling) - Enable support for the WASM Exception Handling feature. - $(WasmEnableSIMD) - Enable support for the WASM SIMD feature. + - $(WasmEnableWebcil) - Enable conversion of assembly .dlls to .webcil Public items: - @(WasmExtraFilesToDeploy) - Files to copy to $(WasmAppDir). @@ -117,6 +118,9 @@ -1 + + + false @@ -367,6 +371,7 @@ DebugLevel="$(WasmDebugLevel)" IncludeThreadsWorker="$(_WasmAppIncludeThreadsWorker)" PThreadPoolSize="$(_WasmPThreadPoolSize)" + UseWebcil="$(WasmEnableWebcil)" > diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj b/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj index c05515cdd10624..eb93b1a34e23f1 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj +++ b/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj @@ -14,4 +14,8 @@ + + + + diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs index 1e0b19dd435ee8..d053921f054152 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs @@ -21,6 +21,7 @@ using System.Reflection; using System.Diagnostics; using System.Text; +using Microsoft.NET.WebAssembly.Webcil; namespace Microsoft.WebAssembly.Diagnostics { @@ -854,7 +855,7 @@ internal sealed class AssemblyInfo private readonly List sources = new List(); internal string Url { get; } //The caller must keep the PEReader alive and undisposed throughout the lifetime of the metadata reader - internal PEReader peReader; + private readonly IDisposable peReaderOrWebcilReader; internal MetadataReader asmMetadataReader { get; } internal MetadataReader pdbMetadataReader { get; set; } internal List> enCMetadataReader = new List>(); @@ -867,36 +868,54 @@ internal sealed class AssemblyInfo private readonly Dictionary _documentIdToSourceFileTable = new Dictionary(); - public AssemblyInfo(ILogger logger) + public static AssemblyInfo FromBytes(MonoProxy monoProxy, SessionId sessionId, byte[] assembly, byte[] pdb, ILogger logger, CancellationToken token) { - debugId = -1; - this.id = Interlocked.Increment(ref next_id); - this.logger = logger; + // First try to read it as a PE file, otherwise try it as a WebCIL file + using var asmStream = new MemoryStream(assembly); + try + { + var peReader = new PEReader(asmStream); + if (!peReader.HasMetadata) + throw new BadImageFormatException(); + return FromPEReader(monoProxy, sessionId, peReader, pdb, logger, token); + } + catch (BadImageFormatException) + { + // This is a WebAssembly file + asmStream.Seek(0, SeekOrigin.Begin); + var webcilReader = new WebcilReader(asmStream); + return FromWebcilReader(monoProxy, sessionId, webcilReader, pdb, logger, token); + } + } + + public static AssemblyInfo WithoutDebugInfo(ILogger logger) + { + return new AssemblyInfo(logger); } - public unsafe AssemblyInfo(MonoProxy monoProxy, SessionId sessionId, byte[] assembly, byte[] pdb, ILogger logger, CancellationToken token) + private AssemblyInfo(ILogger logger) { debugId = -1; this.id = Interlocked.Increment(ref next_id); this.logger = logger; - using var asmStream = new MemoryStream(assembly); - peReader = new PEReader(asmStream); + } + + private static AssemblyInfo FromPEReader(MonoProxy monoProxy, SessionId sessionId, PEReader peReader, byte[] pdb, ILogger logger, CancellationToken token) + { var entries = peReader.ReadDebugDirectory(); + CodeViewDebugDirectoryData? codeViewData = null; if (entries.Length > 0) { var codeView = entries[0]; if (codeView.Type == DebugDirectoryEntryType.CodeView) { - CodeViewDebugDirectoryData codeViewData = peReader.ReadCodeViewDebugDirectoryData(codeView); - PdbAge = codeViewData.Age; - PdbGuid = codeViewData.Guid; - PdbName = codeViewData.Path; - CodeViewInformationAvailable = true; + codeViewData = peReader.ReadCodeViewDebugDirectoryData(codeView); } } - asmMetadataReader = PEReaderExtensions.GetMetadataReader(peReader); - var asmDef = asmMetadataReader.GetAssemblyDefinition(); - Name = asmDef.GetAssemblyName().Name + ".dll"; + var asmMetadataReader = PEReaderExtensions.GetMetadataReader(peReader); + string name = ReadAssemblyName(asmMetadataReader); + + MetadataReader pdbMetadataReader = null; if (pdb != null) { var pdbStream = new MemoryStream(pdb); @@ -907,7 +926,7 @@ public unsafe AssemblyInfo(MonoProxy monoProxy, SessionId sessionId, byte[] asse } catch (BadImageFormatException) { - monoProxy.SendLog(sessionId, $"Warning: Unable to read debug information of: {Name} (use DebugType=Portable/Embedded)", token); + monoProxy.SendLog(sessionId, $"Warning: Unable to read debug information of: {name} (use DebugType=Portable/Embedded)", token); } } else @@ -918,6 +937,74 @@ public unsafe AssemblyInfo(MonoProxy monoProxy, SessionId sessionId, byte[] asse pdbMetadataReader = peReader.ReadEmbeddedPortablePdbDebugDirectoryData(embeddedPdbEntry).GetMetadataReader(); } } + + var assemblyInfo = new AssemblyInfo(peReader, name, asmMetadataReader, codeViewData, pdbMetadataReader, logger); + return assemblyInfo; + } + + private static AssemblyInfo FromWebcilReader(MonoProxy monoProxy, SessionId sessionId, WebcilReader wcReader, byte[] pdb, ILogger logger, CancellationToken token) + { + var entries = wcReader.ReadDebugDirectory(); + CodeViewDebugDirectoryData? codeViewData = null; + if (entries.Length > 0) + { + var codeView = entries[0]; + if (codeView.Type == DebugDirectoryEntryType.CodeView) + { + codeViewData = wcReader.ReadCodeViewDebugDirectoryData(codeView); + } + } + var asmMetadataReader = wcReader.GetMetadataReader(); + string name = ReadAssemblyName(asmMetadataReader); + + MetadataReader pdbMetadataReader = null; + if (pdb != null) + { + var pdbStream = new MemoryStream(pdb); + try + { + // MetadataReaderProvider.FromPortablePdbStream takes ownership of the stream + pdbMetadataReader = MetadataReaderProvider.FromPortablePdbStream(pdbStream).GetMetadataReader(); + } + catch (BadImageFormatException) + { + monoProxy.SendLog(sessionId, $"Warning: Unable to read debug information of: {name} (use DebugType=Portable/Embedded)", token); + } + } + else + { + var embeddedPdbEntry = entries.FirstOrDefault(e => e.Type == DebugDirectoryEntryType.EmbeddedPortablePdb); + if (embeddedPdbEntry.DataSize != 0) + { + pdbMetadataReader = wcReader.ReadEmbeddedPortablePdbDebugDirectoryData(embeddedPdbEntry).GetMetadataReader(); + } + } + + var assemblyInfo = new AssemblyInfo(wcReader, name, asmMetadataReader, codeViewData, pdbMetadataReader, logger); + return assemblyInfo; + } + + private static string ReadAssemblyName(MetadataReader asmMetadataReader) + { + var asmDef = asmMetadataReader.GetAssemblyDefinition(); + return asmDef.GetAssemblyName().Name + ".dll"; + } + + private unsafe AssemblyInfo(IDisposable owningReader, string name, MetadataReader asmMetadataReader, CodeViewDebugDirectoryData? codeViewData, MetadataReader pdbMetadataReader, ILogger logger) + : this(logger) + { + peReaderOrWebcilReader = owningReader; + if (codeViewData != null) + { + PdbAge = codeViewData.Value.Age; + PdbGuid = codeViewData.Value.Guid; + PdbName = codeViewData.Value.Path; + CodeViewInformationAvailable = true; + } + this.asmMetadataReader = asmMetadataReader; + Name = name; + logger.LogTrace($"Info: loading AssemblyInfo with name {Name}"); + this.pdbMetadataReader = pdbMetadataReader; Populate(); } @@ -1410,7 +1497,7 @@ public IEnumerable Add(SessionId id, byte[] assembly_data, byte[] pd AssemblyInfo assembly; try { - assembly = new AssemblyInfo(monoProxy, id, assembly_data, pdb_data, logger, token); + assembly = AssemblyInfo.FromBytes(monoProxy, id, assembly_data, pdb_data, logger, token); } catch (Exception e) { @@ -1504,7 +1591,7 @@ public async IAsyncEnumerable Load(SessionId id, string[] loaded_fil logger.LogDebug($"Bytes from assembly {step.Url} is NULL"); continue; } - assembly = new AssemblyInfo(monoProxy, id, bytes[0], bytes[1], logger, token); + assembly = AssemblyInfo.FromBytes(monoProxy, id, bytes[0], bytes[1], logger, token); } catch (Exception e) { diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs b/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs index 055c44880842e1..29d6356acb09e0 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs @@ -855,7 +855,7 @@ public async Task GetAssemblyInfo(int assemblyId, CancellationToke asm = store.GetAssemblyByName(assemblyName); if (asm == null) { - asm = new AssemblyInfo(logger); + asm = AssemblyInfo.WithoutDebugInfo(logger); logger.LogDebug($"Created assembly without debug information: {assemblyName}"); } } @@ -1205,7 +1205,13 @@ public async Task GetAssemblyName(int assembly_id, CancellationToken tok commandParamsWriter.Write(assembly_id); using var retDebuggerCmdReader = await SendDebuggerAgentCommand(CmdAssembly.GetLocation, commandParamsWriter, token); - return retDebuggerCmdReader.ReadString(); + string result = retDebuggerCmdReader.ReadString(); + if (result.EndsWith(".webcil")) { + /* don't leak .webcil names to the debugger - work in terms of the original .dlls */ + string baseName = result.Substring(0, result.Length - 7); + result = baseName + ".dll"; + } + return result; } public async Task GetFullAssemblyName(int assemblyId, CancellationToken token) diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs b/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs index e2ca58fe62054a..d5cac7e9a3d4f0 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs @@ -275,7 +275,7 @@ public async Task WaitForScriptParsedEventsAsync(params string[] paths) // sets breakpoint by method name and line offset internal async Task CheckInspectLocalsAtBreakpointSite(string type, string method, int line_offset, string bp_function_name, string eval_expression, - Func locals_fn = null, Func wait_for_event_fn = null, bool use_cfo = false, string assembly = "debugger-test.dll", int col = 0) + Func locals_fn = null, Func wait_for_event_fn = null, bool use_cfo = false, string assembly = "debugger-test", int col = 0) { UseCallFunctionOnBeforeGetProperties = use_cfo; diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs b/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs index 93d7221c51f7ee..139daae4ae3262 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs @@ -136,7 +136,7 @@ async Task AssemblyLoadedEventTest(string asm_name, string asm_path, string pdb_ : await Task.FromResult(ProtocolEventHandlerReturn.KeepHandler); }); - byte[] bytes = File.ReadAllBytes(asm_path); + byte[] bytes = File.Exists(asm_path) ? File.ReadAllBytes(asm_path) : File.ReadAllBytes(Path.ChangeExtension(asm_path, ".webcil")); // hack! string asm_base64 = Convert.ToBase64String(bytes); string pdb_base64 = String.Empty; diff --git a/src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets b/src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets index e874cc4239df46..b9a4726d94f5cf 100644 --- a/src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets +++ b/src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets @@ -3,8 +3,8 @@ true $(DebuggerHost)- true - <_DebuggerTestsWorkItemTimeout Condition="'$(Scenario)' == 'WasmDebuggerTests'">00:20:00 - <_DebuggerTestsWorkItemTimeout Condition="'$(Scenario)' == 'WasmDebuggerTests' and '$(BrowserHost)' == 'windows'">00:30:00 + <_DebuggerTestsWorkItemTimeout Condition="'$(Scenario)' == 'WasmDebuggerTests'">00:30:00 + <_DebuggerTestsWorkItemTimeout Condition="'$(Scenario)' == 'WasmDebuggerTests' and '$(BrowserHost)' == 'windows'">00:40:00 $(HelixExtensionTargets);_AddWorkItemsForWasmDebuggerTests diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index 2fead7b9860bfd..8dd76eeab79525 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -48,6 +48,7 @@ public class WasmAppBuilder : Task public string? MainHTMLPath { get; set; } public bool IncludeThreadsWorker {get; set; } public int PThreadPoolSize {get; set; } + public bool UseWebcil { get; set; } // // Extra json elements to add to mono-config.json @@ -187,9 +188,22 @@ private bool ExecuteInternal () var asmRootPath = Path.Combine(AppDir, config.AssemblyRootFolder); Directory.CreateDirectory(AppDir!); Directory.CreateDirectory(asmRootPath); + if (UseWebcil) + Log.LogMessage (MessageImportance.Normal, "Converting assemblies to Webcil"); foreach (var assembly in _assemblies) { - FileCopyChecked(assembly, Path.Combine(asmRootPath, Path.GetFileName(assembly)), "Assemblies"); + if (UseWebcil) + { + var tmpWebcil = Path.GetTempFileName(); + var webcilWriter = Microsoft.WebAssembly.Build.Tasks.WebcilConverter.FromPortableExecutable(inputPath: assembly, outputPath: tmpWebcil, logger: Log); + webcilWriter.ConvertToWebcil(); + var finalWebcil = Path.ChangeExtension(assembly, ".webcil"); + FileCopyChecked(tmpWebcil, Path.Combine(asmRootPath, Path.GetFileName(finalWebcil)), "Assemblies"); + } + else + { + FileCopyChecked(assembly, Path.Combine(asmRootPath, Path.GetFileName(assembly)), "Assemblies"); + } if (DebugLevel != 0) { var pdb = assembly; @@ -234,7 +248,10 @@ private bool ExecuteInternal () foreach (var assembly in _assemblies) { - config.Assets.Add(new AssemblyEntry(Path.GetFileName(assembly))); + string assemblyPath = assembly; + if (UseWebcil) + assemblyPath = Path.ChangeExtension(assemblyPath, ".webcil"); + config.Assets.Add(new AssemblyEntry(Path.GetFileName(assemblyPath))); if (DebugLevel != 0) { var pdb = assembly; pdb = Path.ChangeExtension(pdb, ".pdb"); @@ -261,8 +278,21 @@ private bool ExecuteInternal () string name = Path.GetFileName(fullPath); string directory = Path.Combine(AppDir, config.AssemblyRootFolder, culture); Directory.CreateDirectory(directory); - FileCopyChecked(fullPath, Path.Combine(directory, name), "SatelliteAssemblies"); - config.Assets.Add(new SatelliteAssemblyEntry(name, culture)); + if (UseWebcil) + { + var tmpWebcil = Path.GetTempFileName(); + var webcilWriter = Microsoft.WebAssembly.Build.Tasks.WebcilConverter.FromPortableExecutable(inputPath: fullPath, outputPath: tmpWebcil, logger: Log); + webcilWriter.ConvertToWebcil(); + var finalWebcil = Path.ChangeExtension(name, ".webcil"); + FileCopyChecked(tmpWebcil, Path.Combine(directory, finalWebcil), "SatelliteAssemblies"); + config.Assets.Add(new SatelliteAssemblyEntry(finalWebcil, culture)); + } + else + { + FileCopyChecked(fullPath, Path.Combine(directory, name), "SatelliteAssemblies"); + config.Assets.Add(new SatelliteAssemblyEntry(name, culture)); + } + } } diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj b/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj index c18167fcd21494..3fd2f17c30a253 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj @@ -7,6 +7,7 @@ $(NoWarn),CS8604,CS8602 false true + true @@ -20,6 +21,13 @@ + + + + + + + @@ -29,6 +37,9 @@ TargetPath="tasks\$(TargetFrameworkForNETCoreTasks)" /> + + +/// Reads a .NET assembly in a normal PE COFF file and writes it out as a Webcil file +/// +public class WebcilConverter +{ + private readonly string _inputPath; + private readonly string _outputPath; + + private readonly NET.WebAssembly.Webcil.WebcilConverter _converter; + + private TaskLoggingHelper Log { get; } + private WebcilConverter(NET.WebAssembly.Webcil.WebcilConverter converter, string inputPath, string outputPath, TaskLoggingHelper logger) + { + _converter = converter; + _inputPath = inputPath; + _outputPath = outputPath; + Log = logger; + } + + public static WebcilConverter FromPortableExecutable(string inputPath, string outputPath, TaskLoggingHelper logger) + { + var converter = NET.WebAssembly.Webcil.WebcilConverter.FromPortableExecutable(inputPath, outputPath); + return new WebcilConverter(converter, inputPath, outputPath, logger); + } + + public void ConvertToWebcil() + { + Log.LogMessage(MessageImportance.Low, $"Converting to Webcil: input {_inputPath} output: {_outputPath}"); + _converter.ConvertToWebcil(); + } + +}