From c213405924833640cb486556147a8a8f506bafa8 Mon Sep 17 00:00:00 2001 From: yowl Date: Fri, 23 Aug 2024 18:59:13 -0500 Subject: [PATCH] [NativeAOT-LLVM] add support for building WASI and browser debug & release on Linux (#2605) * Add Linux WASI and browser jobs hard code container in matrix update platform name * remove container from runtimelab.yml * reinstate and rename script extension as emcc does not use `.sh` on linux * update platform name * linux install of wasmtime * Set NATIVEAOT_CI_WASM_BUILD_EMSDK_PATH Remove wasi http implementation as it fails with Caused by: 0: component imports instance `wasi:http/types@0.2.1`, but a matching implementation was not found in the linker 1: instance export `fields` has the wrong type 2: resource implementation is missing * remove NATIVEAOT_CI_WASM_BUILD_EMSDK_PATH, use EMSDK * log EMSDK and use Write-Output * Add CI parameter * use InstallDir/emsdk * Update eng/testing/FindWasmHostExecutable.sh Co-authored-by: SingleAccretion <62474226+SingleAccretion@users.noreply.github.com> * Remove libtinfo dependency * Change container, dropp ROOTFS env * take upstream #106569 * Update src/tests/build.sh - thanks Co-authored-by: SingleAccretion <62474226+SingleAccretion@users.noreply.github.com> --------- Co-authored-by: SingleAccretion <62474226+SingleAccretion@users.noreply.github.com> --- build.sh | 7 ++++ eng/native/tryrun.cmake | 2 +- eng/pipelines/common/global-build-job.yml | 19 +++++---- eng/pipelines/common/platform-matrix.yml | 42 +++++++++++++++++++ eng/pipelines/runtimelab.yml | 6 +++ .../runtimelab/install-emscripten.ps1 | 11 ++++- eng/pipelines/runtimelab/install-llvm.ps1 | 26 +++++++++++- eng/pipelines/runtimelab/install-pwsh.sh | 4 +- eng/pipelines/runtimelab/install-python.sh | 3 ++ eng/pipelines/runtimelab/install-wasi-sdk.ps1 | 25 +++++++++-- eng/pipelines/runtimelab/install-wasmtime.ps1 | 34 +++++++++++++-- .../runtimelab-post-build-steps.yml | 17 +++++--- eng/testing/FindWasmHostExecutable.sh | 27 ++++++++++++ eng/testing/FindWasmHostExecutableAndRun.sh | 14 +++++++ src/coreclr/jit/CMakeLists.txt | 2 +- src/coreclr/jit/compiler.cpp | 12 ++++-- .../Microsoft.NETCore.Native.Unix.targets | 8 +++- .../Microsoft.NETCore.Native.targets | 7 ++-- src/coreclr/scripts/coreclr_arguments.py | 2 +- .../src/System.Net.Http.csproj | 8 +++- .../src/System/Net/Http/HttpClientHandler.cs | 9 ++-- .../CMakeLists.txt | 9 +--- src/tests/Common/CLRTest.Execute.Bash.targets | 6 +-- src/tests/Common/scripts/nativeaottest.sh | 13 ++++-- src/tests/build.sh | 14 ++++++- .../SmokeTests/HelloWasm/HelloWasm.csproj | 2 +- src/tests/run.py | 4 +- src/tests/run.sh | 3 ++ 28 files changed, 278 insertions(+), 58 deletions(-) create mode 100755 eng/pipelines/runtimelab/install-python.sh create mode 100755 eng/testing/FindWasmHostExecutable.sh create mode 100755 eng/testing/FindWasmHostExecutableAndRun.sh diff --git a/build.sh b/build.sh index da2ba1a158ec..205b3326c0cb 100755 --- a/build.sh +++ b/build.sh @@ -29,5 +29,12 @@ if is_cygwin_or_mingw; then scriptroot=$(cygpath -d "$scriptroot") powershell -c "$scriptroot\\build.cmd" $@ else + if [[ "$*" == *"wasm"* && "$*" == *"-ci"* ]]; then + # This is a bit of a workaround for the fact that the pipelines do not have a great + # way of preserving the environment between scripts. Set by install-emscripten.ps1. + if [[ -n $EMSDK ]]; then + source $EMSDK/emsdk_env.sh + fi + fi "$scriptroot/eng/build.sh" $@ fi diff --git a/eng/native/tryrun.cmake b/eng/native/tryrun.cmake index c0fe823dd78f..0ea10e065579 100644 --- a/eng/native/tryrun.cmake +++ b/eng/native/tryrun.cmake @@ -74,7 +74,7 @@ if(DARWIN) else() message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm64 or x64 is supported for OSX cross build!") endif() -elseif(TARGET_ARCH_NAME MATCHES "^(armel|arm|armv6|arm64|loongarch64|riscv64|s390x|ppc64le|x86|x64)$" OR FREEBSD OR ILLUMOS OR TIZEN OR HAIKU) +elseif(TARGET_ARCH_NAME MATCHES "^(armel|arm|armv6|arm64|loongarch64|riscv64|s390x|ppc64le|x86|x64|wasm)$" OR FREEBSD OR ILLUMOS OR TIZEN OR HAIKU) set_cache_value(HAS_POSIX_SEMAPHORES_EXITCODE 0) set_cache_value(HAVE_CLOCK_MONOTONIC_COARSE_EXITCODE 0) set_cache_value(HAVE_CLOCK_MONOTONIC_EXITCODE 0) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 006806496984..b068861190cf 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -210,22 +210,25 @@ jobs: # Install Wasm dependencies: emscripten, LLVM, NodeJS - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.archType, 'wasm')) }}: - - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-emscripten.ps1 $(Build.SourcesDirectory)/wasm-tools - displayName: Install/activate emscripten - - script: call $(Build.SourcesDirectory)/eng/pipelines/runtimelab/set-cmake-path.cmd - displayName: Set CMake path # Install Powershell on OSes that don't come with it by default - ${{ if ne(parameters.hostedOs, 'windows') }}: - script: $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-pwsh.sh $(Build.SourcesDirectory)/wasm-tools displayName: Install Powershell 7 + - script: $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-python.sh $(Build.SourcesDirectory)/wasm-tools + displayName: set PYTHON3 + - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-emscripten.ps1 $(Build.SourcesDirectory)/wasm-tools -CI + displayName: Install/activate emscripten + - ${{ if eq(parameters.hostedOs, 'windows') }}: + - script: call $(Build.SourcesDirectory)/eng/pipelines/runtimelab/set-cmake-path.cmd + displayName: Set CMake path - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-llvm.ps1 -CloneDir $(Build.SourcesDirectory)/wasm-tools -Configs ${{ parameters.buildConfig }} -CI displayName: Install/build LLVM - - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-nodejs.ps1 $(Build.SourcesDirectory)\wasm-tools + - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-nodejs.ps1 $(Build.SourcesDirectory)/wasm-tools displayName: Install NodeJS - - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.platform, 'wasi_wasm_win')) }}: - # Install Wasi Wasm dependencies: wasi-sdk, wasmer - - script: call $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-wasi-sdk.cmd $(Build.SourcesDirectory)\wasm-tools + - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), or(eq(parameters.platform, 'wasi_wasm_win'), eq(parameters.platform, 'wasi_wasm_linux_x64_naot_llvm'))) }}: + # Install Wasi Wasm dependencies: wasi-sdk, wasmtime + - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-wasi-sdk.ps1 -CI -InstallDir $(Build.SourcesDirectory)/wasm-tools displayName: Install wasi-sdk - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-wasmtime.ps1 -CI -InstallDir $(Build.SourcesDirectory)/wasm-tools displayName: Install wasmtime diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 23370eabe21a..b4364ae67fc6 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -479,6 +479,48 @@ jobs: buildConfig: ${{ parameters.buildConfig }} ${{ insert }}: ${{ parameters.jobParameters }} +# Browser WebAssembly Linux X64 for NAOT-LLVM + +- ${{ if containsValue(parameters.platforms, 'Browser_wasm_linux_x64_naot_llvm') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: browser + archType: wasm + targetRid: browser-wasm + platform: Browser_wasm_linux_x64_naot_llvm + shouldContinueOnError: ${{ parameters.shouldContinueOnError }} + container: linux_x64_sanitizer + jobParameters: + hostedOs: linux + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# WASI WebAssembly Linux X64 for NAOT-LLVM + +- ${{ if containsValue(parameters.platforms, 'wasi_wasm_linux_x64_naot_llvm') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: wasi + archType: wasm + targetRid: wasi-wasm + platform: wasi_wasm_linux_x64_naot_llvm + shouldContinueOnError: ${{ parameters.shouldContinueOnError }} + container: linux_x64_sanitizer + jobParameters: + hostedOs: linux + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ insert }}: ${{ parameters.jobParameters }} + # Browser WebAssembly - ${{ if containsValue(parameters.platforms, 'browser_wasm') }}: diff --git a/eng/pipelines/runtimelab.yml b/eng/pipelines/runtimelab.yml index e8bea9140f53..c0d9d8dcc3b7 100644 --- a/eng/pipelines/runtimelab.yml +++ b/eng/pipelines/runtimelab.yml @@ -63,6 +63,9 @@ extends: - windows_x64 - Browser_wasm_win - wasi_wasm_win + # Use a different name to differentiate from upstream as we need the -sanitizer image to get build tools that are not present in the stock wasi-wasm image. + - Browser_wasm_linux_x64_naot_llvm + - wasi_wasm_linux_x64_naot_llvm jobParameters: timeoutInMinutes: 300 buildArgs: -s clr.aot+libs+nativeaot.packages -c debug -rc $(_BuildConfig) @@ -104,6 +107,9 @@ extends: - windows_x64 - Browser_wasm_win - wasi_wasm_win + # Use a different name to differentiate from upstream as we need the -sanitizer image to get build tools that are not present in the stock wasi-wasm image. + - wasi_wasm_linux_x64_naot_llvm + - Browser_wasm_linux_x64_naot_llvm jobParameters: timeoutInMinutes: 300 buildArgs: -s clr.aot+libs+nativeaot.packages -c $(_BuildConfig) /p:ArchiveTests=true diff --git a/eng/pipelines/runtimelab/install-emscripten.ps1 b/eng/pipelines/runtimelab/install-emscripten.ps1 index 4359f3376c47..7c2911ca7899 100755 --- a/eng/pipelines/runtimelab/install-emscripten.ps1 +++ b/eng/pipelines/runtimelab/install-emscripten.ps1 @@ -1,5 +1,6 @@ param( - $InstallDir + $InstallDir, + [switch]$CI ) $ErrorActionPreference="Stop" @@ -17,4 +18,10 @@ git checkout ca7b40ae222a2d8763b6ac845388744b0e57cfb7 ./emsdk install 3.1.56 ./emsdk activate 3.1.56 -Write-Host "##vso[task.setvariable variable=EMSDK]$env:EMSDK" + +if ($CI) +{ + Write-Host "Setting EMSDK to '$InstallDir/emsdk'" + Write-Output "##vso[task.setvariable variable=EMSDK]$InstallDir/emsdk" +} + diff --git a/eng/pipelines/runtimelab/install-llvm.ps1 b/eng/pipelines/runtimelab/install-llvm.ps1 index fc5119c90925..fb6559db339b 100644 --- a/eng/pipelines/runtimelab/install-llvm.ps1 +++ b/eng/pipelines/runtimelab/install-llvm.ps1 @@ -51,6 +51,23 @@ else git clone https://github.com/llvm/llvm-project --branch $LlvmProjectTag $DepthOption } + +# Set the compiler for CI on non-Windows +if (!$IsWindows) { + $RepoDir = Split-path $PSScriptRoot | Split-Path | Split-Path + + bash -c "build_arch=amd64 compiler=clang source $RepoDir/eng/common/native/init-compiler.sh && set | grep -e CC -e CXX -e LDFLAGS" | + ForEach-Object { + if ($CI) + { + # Split the "=" line into the variable's name and value. + $name, $value = $_ -split '=', 2 + # Define it as a process-level environment variable in PowerShell. + Set-Content ENV:$name $value + } + } +} + # There is no [C/c]hecked LLVM config, so change to Debug foreach ($Config in $Configs | % { if ($_ -eq "Checked") { "Debug" } else { $_ } } | Select-Object -Unique) { @@ -71,7 +88,7 @@ foreach ($Config in $Configs | % { if ($_ -eq "Checked") { "Debug" } else { $_ } $CmakeGenerator = "Unix Makefiles" } - $CmakeConfigureCommandLine = "-G", "$CmakeGenerator", "-DLLVM_INCLUDE_BENCHMARKS=OFF" + $CmakeConfigureCommandLine = "-G", "$CmakeGenerator", "-DLLVM_INCLUDE_BENCHMARKS=OFF", "-DLLVM_ENABLE_TERMINFO=0" $CmakeConfigureCommandLine += "-S", $SourceDirName, "-B", $BuildDirPath if ($Config -eq "Release") { @@ -90,6 +107,11 @@ foreach ($Config in $Configs | % { if ($_ -eq "Checked") { "Debug" } else { $_ } } } $CmakeConfigureCommandLine += "-DCMAKE_BUILD_TYPE=$LlvmConfig" + + if (!$IsWindows) + { + $CmakeConfigureCommandLine += "-DCMAKE_SYSROOT=/crossrootfs/x64", "-DCMAKE_INSTALL_PREFIX=/usr/local/llvm-cross" + } Write-Host "Invoking CMake configure: 'cmake $CmakeConfigureCommandLine'" cmake @CmakeConfigureCommandLine @@ -118,6 +140,8 @@ foreach ($Config in $Configs | % { if ($_ -eq "Checked") { "Debug" } else { $_ } if ($CI) { Write-Output "##vso[task.setvariable variable=$LlvmCmakeConfigEnvVarName]$LlvmCmakeConfigPath" + # We need LLVM_DIR for Linux + Write-Output "##vso[task.setvariable variable=LLVM_DIR]$LlvmCmakeConfigPath" } else { diff --git a/eng/pipelines/runtimelab/install-pwsh.sh b/eng/pipelines/runtimelab/install-pwsh.sh index 5050e3a9a055..f6591dc14a64 100755 --- a/eng/pipelines/runtimelab/install-pwsh.sh +++ b/eng/pipelines/runtimelab/install-pwsh.sh @@ -15,5 +15,5 @@ tar zxf powershell.tar.gz -C powershell7 # Set execute permissions chmod +x powershell7/pwsh -echo setting PATH -echo ##vso[task.setvariable variable=PATH]$PATH:$1/powershell7 +echo setting PATH $PATH:$1/powershell7 +echo '##vso[task.setvariable variable=PATH]'$PATH:$1/powershell7 diff --git a/eng/pipelines/runtimelab/install-python.sh b/eng/pipelines/runtimelab/install-python.sh new file mode 100755 index 000000000000..498ab3cbc78f --- /dev/null +++ b/eng/pipelines/runtimelab/install-python.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +echo '##vso[task.setvariable variable=EMSDK_PYTHON]'/usr/bin/python3 \ No newline at end of file diff --git a/eng/pipelines/runtimelab/install-wasi-sdk.ps1 b/eng/pipelines/runtimelab/install-wasi-sdk.ps1 index 62c2136cfe35..d798a5bfd6ad 100644 --- a/eng/pipelines/runtimelab/install-wasi-sdk.ps1 +++ b/eng/pipelines/runtimelab/install-wasi-sdk.ps1 @@ -1,5 +1,10 @@ -$WasiSdkVersion = 24 +param( + $InstallDir, + [switch]$CI +) +$WasiSdkVersion = 24 +Set-Location -Path $InstallDir $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" @@ -11,13 +16,27 @@ if ($WasiSdkVersion -lt [int]$UpstreamWasiSdkVersion) exit } -$WasiSdkHost = "x86_64-windows" +if ($IsWindows) +{ + $WasiSdkHost = "x86_64-windows" +} +else +{ + $WasiSdkHost = "x86_64-linux" +} $WasiSdkDirName = "wasi-sdk-$WasiSdkVersion.0-$WasiSdkHost" $WasiSdkGzFile = "$WasiSdkDirName.tar.gz" -Invoke-WebRequest -Uri "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$WasiSdkVersion/$WasiSdkGzFile" -OutFile $WasiSdkGzFile + +Invoke-WebRequest -Uri https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$WasiSdkVersion/$WasiSdkGzFile -OutFile $WasiSdkGzFile tar -xzf $WasiSdkGzFile mv $WasiSdkDirName wasi-sdk # The upstream build expects this sentinel to exist, otherwise it tries to use a provisioned SDK. $WasiSdkVersion > wasi-sdk/"VERSION$("$WasiSdkVersion".ToUpper())" + +if ($CI) +{ + Write-Host "Setting WASI_SDK_PATH to '$InstallDir/wasi-sdk'" + Write-Output "##vso[task.setvariable variable=WASI_SDK_PATH]$InstallDir/wasi-sdk" +} diff --git a/eng/pipelines/runtimelab/install-wasmtime.ps1 b/eng/pipelines/runtimelab/install-wasmtime.ps1 index 4c4b87b8b07d..5bfe099d52f2 100644 --- a/eng/pipelines/runtimelab/install-wasmtime.ps1 +++ b/eng/pipelines/runtimelab/install-wasmtime.ps1 @@ -10,11 +10,37 @@ $ProgressPreference = "SilentlyContinue" Set-Location $InstallDir $WasmtimeVersion = "v21.0.1" -$WasmtimeFolderName = "wasmtime-$WasmtimeVersion-x86_64-windows" -Invoke-WebRequest -Uri https://github.com/bytecodealliance/wasmtime/releases/download/v21.0.1/$WasmtimeFolderName.zip -OutFile wasmtime.zip -Expand-Archive -LiteralPath wasmtime.zip -DestinationPath . + +if (!(Test-Path variable:global:IsWindows)) +{ + $IsWindows = [Environment]::OSVersion.Platform -eq [PlatformID]::Win32NT +} + +if ($IsWIndows) +{ + $WasmtimeBaseName = "wasmtime-$WasmtimeVersion-x86_64-windows" + $WasmtimeArchive = "$WasmtimeBaseName.zip" +} +else +{ + $WasmtimeBaseName = "wasmtime-$WasmtimeVersion-x86_64-linux" + $WasmtimeArchive = "$WasmtimeBaseName.tar.xz" +} + +Invoke-WebRequest -Uri https://github.com/bytecodealliance/wasmtime/releases/download/$WasmtimeVersion/$WasmtimeArchive -OutFile $WasmtimeArchive +if ($IsWIndows) +{ + Expand-Archive -LiteralPath $WasmtimeArchive -DestinationPath . +} +else +{ + New-Item -ItemType Directory -Force -Path $WasmtimeBaseName + tar -xf $WasmtimeArchive -C $WasmtimeBaseName +} if ($CI) { - Write-Output "##vso[task.prependpath]$pwd/$WasmtimeFolderName" + Write-Host "Setting WASMTIME_EXECUTABLE to '$pwd/$WasmtimeBaseName/$WasmtimeBaseName/wasmtime'" + Write-Output "##vso[task.setvariable variable=WASMTIME_EXECUTABLE]$pwd/$WasmtimeBaseName/$WasmtimeBaseName/wasmtime" + Write-Output "##vso[task.prependpath]$pwd/$WasmtimeBaseName" } diff --git a/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml b/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml index 29ff0de5cc2b..3bdd31ae5aaa 100644 --- a/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml +++ b/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml @@ -18,7 +18,7 @@ steps: # Now we need to build the cross-targeting compilers, RyuJit and ILC. Likewise with packages. - ${{ if eq(parameters.archType, 'wasm') }}: - - script: $(Build.SourcesDirectory)/build$(scriptExt) clr.wasmjit+clr.aot -c $(buildConfigUpper) $(_officialBuildParameter) -ci + - script: $(Build.SourcesDirectory)/build$(scriptExt) clr.wasmjit+clr.aot -c $(buildConfigUpper) $(_officialBuildParameter) -ci -cross displayName: Build the ILC and RyuJit cross-compilers - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.platform, 'browser_wasm_win')) }}: @@ -34,9 +34,17 @@ steps: displayName: Build WebAssembly tests - ${{ elseif eq(parameters.platform, 'wasi_wasm_win') }}: - script: | - call $(Build.SourcesDirectory)\wasm-tools\emsdk\emsdk_env $(Build.SourcesDirectory)/src/tests/build$(scriptExt) nativeaot $(buildConfigUpper) ${{ parameters.archType }} wasi tree nativeaot /p:LibrariesConfiguration=${{ parameters.librariesConfiguration }} displayName: Build WebAssembly tests + - ${{ elseif eq(parameters.platform, 'Browser_wasm_linux_x64_naot_llvm') }}: + - script: | + source $(Build.SourcesDirectory)/wasm-tools/emsdk/emsdk_env.sh + $(Build.SourcesDirectory)/src/tests/build$(scriptExt) nativeaot $(buildConfigUpper) -browser tree nativeaot /p:LibrariesConfiguration=${{ parameters.librariesConfiguration }} + displayName: Build WebAssembly tests + - ${{ elseif eq(parameters.platform, 'wasi_wasm_linux_x64_naot_llvm') }}: + - script: | + $(Build.SourcesDirectory)/src/tests/build$(scriptExt) nativeaot $(buildConfigUpper) -wasi tree nativeaot /p:LibrariesConfiguration=${{ parameters.librariesConfiguration }} + displayName: Build WebAssembly tests - ${{ elseif eq(parameters.osGroup, 'windows') }}: - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) nativeaot $(buildConfigUpper) ${{ parameters.archType }} ${{ parameters.testFilter }} /p:NativeAotMultimodule=true /p:LibrariesConfiguration=${{ parameters.librariesConfiguration }} @@ -52,15 +60,14 @@ steps: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) runnativeaottests $(buildConfigUpper) ${{ parameters.archType }} ${{ parameters.osGroup }} displayName: Run WebAssembly tests in single file mode - ${{ elseif eq(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) runnativeaottests $(buildConfigUpper) ${{ parameters.archType }} + - script: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) runnativeaottests $(buildConfigUpper) ${{ parameters.archType }} ${{ parameters.osGroup }} displayName: Run tests in single file mode - ${{ else }}: - - script: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) --runnativeaottests $(buildConfigUpper) ${{ parameters.archType }} + - script: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) --runnativeaottests $(buildConfigUpper) ${{ parameters.archType }} ${{ parameters.osGroup }} displayName: Run tests in single file mode - ${{ if eq(parameters.archType, 'wasm') }}: - script: | - call $(Build.SourcesDirectory)\wasm-tools\emsdk\emsdk_env $(Build.SourcesDirectory)/build$(scriptExt) libs.tests -test -a ${{ parameters.archType }} -os ${{ parameters.osGroup }} -lc ${{ parameters.librariesConfiguration }} -rc $(buildConfigUpper) /p:TestNativeAot=true /p:RunSmokeTestsOnly=true displayName: Build and run WebAssembly libraries tests diff --git a/eng/testing/FindWasmHostExecutable.sh b/eng/testing/FindWasmHostExecutable.sh new file mode 100755 index 000000000000..2a3291fa1152 --- /dev/null +++ b/eng/testing/FindWasmHostExecutable.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +exename=$(basename "$1" .wasm) +exename=$(basename "$exename" .dll) +exename=$(basename "$exename" .js) +dirname=$(dirname "$1") + +node="node --stack_trace_limit=100" +WASM_HOST_ARGS_SEPERATOR="" + +if [ -e "${dirname}/${exename}.js" ]; then + WASM_HOST_EXECUTABLE=$node + WASM_BINARY_TO_EXECUTE="${dirname}/${exename}.js" +elif [ -e "${dirname}/main.js" ]; then + WASM_HOST_EXECUTABLE=$node + WASM_BINARY_TO_EXECUTE="${dirname}/main.js" +elif [ -e "${dirname}/${exename}.mjs" ]; then + WASM_HOST_EXECUTABLE=$node + WASM_BINARY_TO_EXECUTE="${dirname}/${exename}.mjs" +elif [ -e "${dirname}/main.mjs" ]; then + WASM_HOST_EXECUTABLE=$node + WASM_BINARY_TO_EXECUTE="${dirname}/main.mjs" +elif [ -e "${dirname}/${exename}.wasm" ]; then + WASM_HOST_EXECUTABLE=$WASMTIME_EXECUTABLE + WASM_HOST_ARGS_SEPERATOR="--" + WASM_BINARY_TO_EXECUTE="${dirname}/${exename}.wasm" +fi diff --git a/eng/testing/FindWasmHostExecutableAndRun.sh b/eng/testing/FindWasmHostExecutableAndRun.sh new file mode 100755 index 000000000000..1f84e2a45333 --- /dev/null +++ b/eng/testing/FindWasmHostExecutableAndRun.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +source $SCRIPT_DIR/FindWasmHostExecutable.sh "$1" + +if [ -n "${WASM_HOST_EXECUTABLE}" ]; then + shift + echo $WASM_HOST_EXECUTABLE "$WASM_BINARY_TO_EXECUTE" $WASM_HOST_ARGS_SEPERATOR "$@" + $WASM_HOST_EXECUTABLE "$WASM_BINARY_TO_EXECUTE" $WASM_HOST_ARGS_SEPERATOR "$@" +else + echo WASM_HOST_EXECUTABLE not set. + exit 1 +fi diff --git a/src/coreclr/jit/CMakeLists.txt b/src/coreclr/jit/CMakeLists.txt index b68cad425119..8d5fbc921202 100644 --- a/src/coreclr/jit/CMakeLists.txt +++ b/src/coreclr/jit/CMakeLists.txt @@ -92,6 +92,7 @@ function(create_standalone_jit) target_include_directories(${TARGETDETAILS_TARGET} PRIVATE "../tools/Common/JitInterface") # Only look at "our" LLVM with LLVM_CMAKE_CONFIG. + set(LLVM_DIR $ENV{LLVM_CMAKE_CONFIG}) find_package(LLVM REQUIRED CONFIG PATHS $ENV{LLVM_CMAKE_CONFIG} NO_DEFAULT_PATH) target_include_directories(${TARGETDETAILS_TARGET} PRIVATE ${LLVM_INCLUDE_DIRS}) separate_arguments(LLVM_DEFINITIONS) @@ -728,7 +729,6 @@ if (CLR_CMAKE_BUILD_LLVM_JIT) list(FILTER JIT_SOURCES EXCLUDE REGEX lsrabuild\.cpp) list(FILTER JIT_SOURCES EXCLUDE REGEX regalloc\.cpp) - # LLVM headers require C++17. create_standalone_jit(TARGET clrjit_universal_wasm32_${ARCH_HOST_NAME} OS universal ARCH wasm32 CMAKE_CXX_STANDARD 17 DESTINATIONS .) install_clr(TARGETS clrjit_universal_wasm32_${ARCH_HOST_NAME} DESTINATIONS . COMPONENT wasmjit) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index a88bd007482d..516b774f3627 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -1262,8 +1262,12 @@ void DisplayNowayAssertMap() fout = _wfopen(strJitMeasureNowayAssertFile, W("a")); if (fout == nullptr) { +#if !defined(HOST_WINDOWS) + // TODO: how do we print a `const char16_t*` portably? +#else fprintf(jitstdout(), "Failed to open JitMeasureNowayAssertFile \"%ws\"\n", strJitMeasureNowayAssertFile); +#endif return; } } @@ -1295,7 +1299,7 @@ void DisplayNowayAssertMap() for (i = 0; i < count; i++) { - fprintf(fout, "%u, %s, %u, \"%s\"\n", nacp[i].count, nacp[i].fl.m_file, nacp[i].fl.m_line, + fprintf(fout, "%zu, %s, %u, \"%s\"\n", nacp[i].count, nacp[i].fl.m_file, nacp[i].fl.m_line, nacp[i].fl.m_condStr); } @@ -3414,7 +3418,7 @@ void Compiler::compInitOptions(JitFlags* jitFlags) compMaxUncheckedOffsetForNullObject = (size_t)JitConfig.JitMaxUncheckedOffset(); if (verbose) { - printf("STRESS_NULL_OBJECT_CHECK: compMaxUncheckedOffsetForNullObject=0x%X\n", + printf("STRESS_NULL_OBJECT_CHECK: compMaxUncheckedOffsetForNullObject=0x%zX\n", compMaxUncheckedOffsetForNullObject); } } @@ -6650,7 +6654,7 @@ int Compiler::compCompile(CORINFO_MODULE_HANDLE classPtr, } endErrorTrap() // ERROR TRAP: End - return param.result; + return param.result; } #if defined(DEBUG) @@ -9674,7 +9678,7 @@ void dumpConvertedVarSet(Compiler* comp, VARSET_VALARG_TP vars) bool first = true; printf("{"); - for (size_t varNum = 0; varNum < comp->lvaCount; varNum++) + for (unsigned varNum = 0; varNum < comp->lvaCount; varNum++) { if (pVarNumSet[varNum] == 1) { diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets index 19c6ce7a4b37..d2696ffbcad6 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets @@ -24,7 +24,7 @@ The .NET Foundation licenses this file to you under the MIT license. lld bfd 1572864 - true + true @@ -136,7 +136,6 @@ The .NET Foundation licenses this file to you under the MIT license. - @@ -151,6 +150,11 @@ The .NET Foundation licenses this file to you under the MIT license. + + + + + diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets index 7400c13fa252..0e8735e56dce 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets @@ -431,11 +431,12 @@ The .NET Foundation licenses this file to you under the MIT license. $(CompileWasmArgs) -mllvm -enable-emscripten-cxx-exceptions - .bat + .bat + "$(EMSDK)/upstream/bin/clang++" "$(EmscriptenSdkToolsPath)bin/clang++" - "$(EMSDK)/upstream/emscripten/emcc$(ScriptExt)" - "$(EmscriptenSdkToolsPath)emscripten/emcc$(ScriptExt)" + "$(EMSDK)/upstream/emscripten/emcc$(EmccScriptExt)" + "$(EmscriptenSdkToolsPath)emscripten/emcc$(EmccScriptExt)" diff --git a/src/coreclr/scripts/coreclr_arguments.py b/src/coreclr/scripts/coreclr_arguments.py index 7f40d076acdb..59db0fb92c83 100644 --- a/src/coreclr/scripts/coreclr_arguments.py +++ b/src/coreclr/scripts/coreclr_arguments.py @@ -283,7 +283,7 @@ def check_and_return_default_product_location(product_location): self.target_os = self.host_os if self.arch == "wasm" and self.target_os != "wasi": - self.target_os = "Browser" # assume we need to differentiate between running wasm tests on Windows and *nix so leave host_os as the os where the tests are running + self.target_os = "browser" # assume we need to differentiate between running wasm tests on Windows and *nix so leave host_os as the os where the tests are running self.verify(args, "build_type", diff --git a/src/libraries/System.Net.Http/src/System.Net.Http.csproj b/src/libraries/System.Net.Http/src/System.Net.Http.csproj index c6af468c2532..03e55dc3b04f 100644 --- a/src/libraries/System.Net.Http/src/System.Net.Http.csproj +++ b/src/libraries/System.Net.Http/src/System.Net.Http.csproj @@ -24,6 +24,11 @@ true + + + $(DefineConstants);NATIVE_AOT + + @@ -456,7 +461,8 @@ Link="Common\System\Net\Http\HttpHandlerDefaults.cs" /> - + + diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/HttpClientHandler.cs b/src/libraries/System.Net.Http/src/System/Net/Http/HttpClientHandler.cs index afba32455bc2..02c333177ba2 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/HttpClientHandler.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/HttpClientHandler.cs @@ -10,7 +10,8 @@ using System.Threading; using System.Threading.Tasks; using System.Diagnostics.Metrics; -#if TARGET_WASI +// TODO-LLVM: This is not upstreamable and should be deleted when https://github.com/dotnet/runtimelab/pull/2614 is merged +#if TARGET_WASI && !NATIVE_AOT using System.Diagnostics; using System.Net.Http.Metrics; using HttpHandlerType = System.Net.Http.WasiHttpHandler; @@ -28,7 +29,8 @@ public partial class HttpClientHandler : HttpMessageHandler { private readonly HttpHandlerType _underlyingHandler; -#if TARGET_BROWSER || TARGET_WASI +// TODO-LLVM: This is not upstreamable and !NATIVE_AOT should be reverted when https://github.com/dotnet/runtimelab/pull/2614 is merged +#if TARGET_BROWSER || (TARGET_WASI && !NATIVE_AOT) private IMeterFactory? _meterFactory; private HttpMessageHandler? _firstHandler; // DiagnosticsHandler or MetricsHandler, depending on global configuration. @@ -98,7 +100,8 @@ protected override void Dispose(bool disposing) [CLSCompliant(false)] public IMeterFactory? MeterFactory { -#if TARGET_BROWSER || TARGET_WASI +// TODO-LLVM: This is not upstreamable and !NATIVE_AOT should be reverted when https://github.com/dotnet/runtimelab/pull/2614 is merged +#if TARGET_BROWSER || (TARGET_WASI && !NATIVE_AOT) get => _meterFactory; set { diff --git a/src/native/libs/System.IO.Compression.Native/CMakeLists.txt b/src/native/libs/System.IO.Compression.Native/CMakeLists.txt index 5295371f2332..190770581404 100644 --- a/src/native/libs/System.IO.Compression.Native/CMakeLists.txt +++ b/src/native/libs/System.IO.Compression.Native/CMakeLists.txt @@ -186,12 +186,7 @@ else () endif () if((NOT CLR_CMAKE_USE_SYSTEM_ZLIB) AND STATIC_LIBS_ONLY) - if (CLR_CMAKE_TARGET_UNIX) - # zlib on Unix needs to be installed in the same location as System.IO.Compression.Native so that we can then treat is as a 'z' native library. - install_static_library(zlib ${STATIC_LIB_DESTINATION} nativeaot) - else() - install_static_library(zlib aotsdk nativeaot) - endif() -endif() + install_static_library(zlib aotsdk nativeaot) +endif () install (TARGETS System.IO.Compression.Native-Static DESTINATION ${STATIC_LIB_DESTINATION} COMPONENT libs) diff --git a/src/tests/Common/CLRTest.Execute.Bash.targets b/src/tests/Common/CLRTest.Execute.Bash.targets index 23205302d972..bc86aa93c494 100644 --- a/src/tests/Common/CLRTest.Execute.Bash.targets +++ b/src/tests/Common/CLRTest.Execute.Bash.targets @@ -341,7 +341,7 @@ fi "$CORE_ROOT/watchdog" $_WatcherTimeoutMins - - + - + /dev/null && pwd) +source $SCRIPT_DIR/../../../../eng/testing/FindWasmHostExecutable.sh "$1/native/$2" + +if [ -n "${WASM_HOST_EXECUTABLE}" ]; then + $WASM_HOST_EXECUTABLE "$WASM_BINARY_TO_EXECUTE" "${@:3}" +else + exename=$(basename $2 .dll) + chmod +x $1/native/$exename + $_DebuggerFullPath $1/native/$exename "${@:3}" +fi diff --git a/src/tests/build.sh b/src/tests/build.sh index b0ae9998c1cb..57de24ff9f03 100755 --- a/src/tests/build.sh +++ b/src/tests/build.sh @@ -60,7 +60,7 @@ build_Tests() export MSBUILDDEBUGPATH if [[ "$__SkipNative" != 1 && "$__BuildTestWrappersOnly" != 1 && "$__GenerateLayoutOnly" != 1 && "$__CopyNativeTestBinaries" != 1 && \ - "$__TargetOS" != "browser" && "$__TargetOS" != "android" && "$__TargetOS" != "ios" && "$__TargetOS" != "iossimulator" && "$__TargetOS" != "tvos" && "$__TargetOS" != "tvossimulator" ]]; then + "$__TargetOS" != "android" && "$__TargetOS" != "ios" && "$__TargetOS" != "iossimulator" && "$__TargetOS" != "tvos" && "$__TargetOS" != "tvossimulator" ]]; then build_native "$__TargetOS" "$__TargetArch" "$__TestDir" "$__NativeTestIntermediatesDir" "install" "$__CMakeArgs" "CoreCLR test component" if [[ "$?" -ne 0 ]]; then @@ -167,6 +167,8 @@ usage_list+=("-test:xxx - Only build the specified test project ^(relative or ab usage_list+=("-dir:xxx - Build all test projects in the given directory ^(relative or absolute directory under src\tests^)."); usage_list+=("-tree:xxx - Build all test projects in the given subtree ^(relative or absolute directory under src\tests^)."); usage_list+=("-log:xxx - Base file name to use for log files (used in lab pipelines that build tests in multiple steps to retain logs for each step).") +usage_list+=("-browser - Target Wasm/browser.") +usage_list+=("-wasi - Target Wasm/WASI.") usage_list+=("") usage_list+=("Any unrecognized arguments will be passed directly to MSBuild.") @@ -318,6 +320,16 @@ handle_arguments_local() { fi ;; + browser|-browser) + __TargetOS=browser + __TargetArch=wasm + ;; + + wasi|-wasi) + __TargetOS=wasi + __TargetArch=wasm + ;; + *) __UnprocessedBuildArgs+=("$1") ;; diff --git a/src/tests/nativeaot/SmokeTests/HelloWasm/HelloWasm.csproj b/src/tests/nativeaot/SmokeTests/HelloWasm/HelloWasm.csproj index 89e13156aabb..6421623312f1 100644 --- a/src/tests/nativeaot/SmokeTests/HelloWasm/HelloWasm.csproj +++ b/src/tests/nativeaot/SmokeTests/HelloWasm/HelloWasm.csproj @@ -31,7 +31,7 @@ - --js-library $(MSBuildProjectDirectory)\dotnet_support.js --pre-js $(MSBuildProjectDirectory)\Microsoft.JSInterop.js --pre-js $(MSBuildProjectDirectory)\shell.js --post-js $(MSBuildProjectDirectory)\HelloWasm.js + --js-library $(MSBuildProjectDirectory)/dotnet_support.js --pre-js $(MSBuildProjectDirectory)/Microsoft.JSInterop.js --pre-js $(MSBuildProjectDirectory)/shell.js --post-js $(MSBuildProjectDirectory)/HelloWasm.js diff --git a/src/tests/run.py b/src/tests/run.py index 0c3e8ee856d0..d3c92c5a2a7d 100644 --- a/src/tests/run.py +++ b/src/tests/run.py @@ -900,8 +900,8 @@ def setup_args(args): target_os = coreclr_setup_args.host_os # using the arg target_os always breaks the OSX tests - if coreclr_setup_args.target_os == "Browser": - target_os = "Browser" + if coreclr_setup_args.target_os == "browser": + target_os = "browser" if coreclr_setup_args.target_os == "wasi": target_os = "wasi" diff --git a/src/tests/run.sh b/src/tests/run.sh index 5a3a1c3065aa..0062a49bc1e6 100755 --- a/src/tests/run.sh +++ b/src/tests/run.sh @@ -106,6 +106,9 @@ do android) buildOS="android" ;; + browser) + buildOS="browser" + ;; wasi) buildOS="wasi" ;;