Use dotnet test (#48686)

This change is migrating us away from the xUnit console runner and onto the `dotnet test` command instead. This has the following advantages: 1. xunit console runner is not a supported product hence we really need to move off it anyways. 2. Can take advantage of all the crash dump and blame analysis that they are doing in the dotnet test tool 3. Simplifies our story a bit because it means we are dealing with a single way to invoke tests across all the different configurations, OS we run. It's dotnet test all the time. This is true even when we are running .NET Framework tests. 4. One step closer to removing all of the restore logic from our test jobs
dotnet · Oct 20, 2020 · 83e4631 · 83e4631
1 parent d347dae
commit 83e4631
Show file tree

Hide file tree

Showing 19 changed files with 313 additions and 422 deletions.
diff --git a/eng/Versions.props b/eng/Versions.props
@@ -231,6 +231,7 @@
     <xunitanalyzersVersion>0.10.0</xunitanalyzersVersion>
     <xunitassertVersion>$(xunitVersion)</xunitassertVersion>
     <XunitCombinatorialVersion>1.3.2</XunitCombinatorialVersion>
+    <XUnitXmlTestLoggerVersion>2.1.26</XUnitXmlTestLoggerVersion>
     <xunitextensibilitycoreVersion>$(xunitVersion)</xunitextensibilitycoreVersion>
     <xunitrunnerconsoleVersion>2.4.1-pre.build.4059</xunitrunnerconsoleVersion>
     <xunitrunnerwpfVersion>1.0.51</xunitrunnerwpfVersion>
@@ -266,5 +267,7 @@
       rather explicitly override it.
     -->
     <UsingToolMicrosoftNetCompilers Condition="'$(BootstrapBuildPath)' == ''">true</UsingToolMicrosoftNetCompilers>
+
+    <UseVSTestRunner>true</UseVSTestRunner>
   </PropertyGroup>
 </Project>
diff --git a/eng/build.ps1 b/eng/build.ps1
@@ -352,12 +352,13 @@ function TestUsingOptimizedRunner() {
     ExitWithExitCode 1
   }
 
-  $xunitDir = Join-Path (Get-PackageDir "xunit.runner.console") "tools\net472"
-  $args = "`"$xunitDir`""
-  $args += " `"-out:$testResultsDir`""
-  $args += " `"-logs:$LogDir`""
-  $args += " `"-secondaryLogs:$secondaryLogDir`""
-  $args += " -tfm:net472"
+  $dotnetExe = Join-Path $dotnet "dotnet.exe"
+  $args += " --dotnet `"$dotnetExe`""
+  $args += " --out `"$testResultsDir`""
+  $args += " --logs `"$LogDir`""
+  $args += " --secondaryLogs `"$secondaryLogDir`""
+  $args += " --tfm net472"
+  $args += " --html"
 
   if ($testDesktop -or $testIOperation) {
     if ($test32) {
@@ -373,10 +374,10 @@ function TestUsingOptimizedRunner() {
     }
 
     $dlls += @(Get-ChildItem -Recurse -Include "*.IntegrationTests.dll" $binDir)
-    $args += " -testVsi"
+    $args += " --testVsi"
   } else {
     $dlls = Get-ChildItem -Recurse -Include "*.IntegrationTests.dll" $binDir
-    $args += " -trait:Feature=NetCore"
+    $args += " --trait:Feature=NetCore"
   }
 
   # Exclude out the multi-targetted netcore app projects
@@ -396,26 +397,28 @@ function TestUsingOptimizedRunner() {
   $dlls = $dlls | ?{ -not (($_.FullName -match ".*\\$excludedConfiguration\\.*") -or ($_.FullName -match ".*/$excludedConfiguration/.*")) }
 
   if ($ci) {
-    $args += " -xml"
     if ($testVsi) {
-      $args += " -timeout:110"
+      $args += " --timeout 110"
     } else {
-      $args += " -timeout:90"
+      $args += " --timeout 90"
     }
   }
 
-  $procdumpPath = Ensure-ProcDump
-  $args += " -procdumppath:$procDumpPath"
   if ($procdump) {
-    $args += " -useprocdump";
+    $procdumpFilePath = Ensure-ProcDump
+    $args += " --procdumppath $procDumpFilePath"
+    $args += " --useprocdump";
   }
 
   if ($test64) {
-    $args += " -test64"
+    $args += " --platform x64"
+  }
+  else {
+    $args += " --platform x86"
   }
 
   if ($sequential) {
-    $args += " -sequential"
+    $args += " --sequential"
   }
 
   foreach ($dll in $dlls) {
@@ -525,7 +528,7 @@ function Ensure-ProcDump() {
     Unzip $zipFilePath $outDir
   }
 
-  return $outDir
+  return $filePath
 }
 
 # Setup the CI machine for running our integration tests.

diff --git a/eng/targets/XUnit.targets b/eng/targets/XUnit.targets
@@ -6,6 +6,10 @@
     <PrepareForBuildDependsOn>$(PrepareForBuildDependsOn);AddDefaultTestAppConfig</PrepareForBuildDependsOn>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="XunitXml.TestLogger" Version="$(XunitXmlTestLoggerVersion)" />
+  </ItemGroup>
+
   <ItemGroup>
     <!-- Include BannedSymbols covering all test projects -->
     <AdditionalFiles Include="$(MSBuildThisFileDirectory)..\config\test\BannedSymbols.txt" />

diff --git a/...es/VisualBasicTest/Diagnostics/AddImport/AddImportTestsWithAddImportDiagnosticProvider.vb b/...es/VisualBasicTest/Diagnostics/AddImport/AddImportTestsWithAddImportDiagnosticProvider.vb
@@ -33,8 +33,7 @@ End Class", TestHost.InProcess)
         End Function
 
         <WorkItem(829970, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/829970")>
-        <Theory>
-        <CombinatorialData>
+        <Fact>
         Public Async Function TestUnknownIdentifierGenericName() As Task
             Await TestAsync(
 "Class C
@@ -54,8 +53,7 @@ End Class", TestHost.InProcess)
         End Function
 
         <WorkItem(829970, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/829970")>
-        <Theory>
-        <CombinatorialData>
+        <Fact>
         Public Async Function TestUnknownIdentifierAddNamespaceImport() As Task
             Await TestAsync(
 "Class Class1
@@ -69,8 +67,7 @@ End Class", TestHost.InProcess)
         End Function
 
         <WorkItem(829970, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/829970")>
-        <Theory>
-        <CombinatorialData>
+        <Fact>
         Public Async Function TestUnknownAttributeInModule() As Task
             Await TestAsync(
 "Module Goo
@@ -108,8 +105,7 @@ Class MultiDictionary(Of K, V)
 End Class")
         End Function
 
-        <Theory>
-        <CombinatorialData>
+        <Fact>
         <WorkItem(1744, "https://github.com/dotnet/roslyn/issues/1744")>
         Public Async Function TestImportIncompleteSub() As Task
             Await TestAsync(
@@ -140,8 +136,7 @@ End Namespace", TestHost.InProcess)
         End Function
 
         <WorkItem(1239, "https://github.com/dotnet/roslyn/issues/1239")>
-        <Theory>
-        <CombinatorialData>
+        <Fact>
         Public Async Function TestImportIncompleteSub2() As Task
             Await TestAsync(
 "Imports System.Linq