[rel/17.6] Filter out known platform sources

src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs

@@ -663,7 +663,7 @@ private static void WarnAboutNotFoundRuntimeProvidersOrThrowWhenNoneAreFound(Lis
             throw new ArgumentException(null, nameof(testRuntimeProviders));
 
         // Throw when we did not find any runtime provider for any of the provided sources.
-        var shouldThrow = testRuntimeProviders.All(runtimeProvider => runtimeProvider == null);
+        var shouldThrow = testRuntimeProviders.All(runtimeProvider => runtimeProvider.Type == null);
 
         var missingRuntimeProviders = testRuntimeProviders.Where(p => p.Type == null);
         if (missingRuntimeProviders.Any())

src/vstest.console/CommandLine/CommandLineOptions.cs

@@ -8,6 +8,7 @@
 using System.Linq;
 
 using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors;
+using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers;
 using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces;
@@ -292,7 +293,8 @@ public void AddSource(string source)
                 string.Format(CultureInfo.CurrentCulture, CommandLineResources.InvalidArgument, source), ex);
         }
         // Add the matching files to source list
-        _sources = _sources.Union(matchingFiles).ToList();
+        var filteredFiles = KnownPlatformSourceFilter.FilterKnownPlatformSources(matchingFiles);
+        _sources = _sources.Union(filteredFiles).ToList();
     }
 
     /// <summary>

src/vstest.console/TestPlatformHelpers/TestRequestManager.cs

@@ -167,7 +167,7 @@ public void DiscoverTests(
         EqtTrace.Info("TestRequestManager.DiscoverTests: Discovery tests started.");
 
         // TODO: Normalize rest of the data on the request as well
-        discoveryPayload.Sources = discoveryPayload.Sources?.Distinct().ToList() ?? new List<string>();
+        discoveryPayload.Sources = KnownPlatformSourceFilter.FilterKnownPlatformSources(discoveryPayload.Sources?.Distinct().ToList());
         discoveryPayload.RunSettings ??= "<RunSettings></RunSettings>";
 
         var runsettings = discoveryPayload.RunSettings;
@@ -275,6 +275,11 @@ public void RunTests(
     {
         EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
         testRunRequestPayload.RunSettings ??= "<RunSettings></RunSettings>";
+        if (testRunRequestPayload.Sources != null)
+        {
+            testRunRequestPayload.Sources = KnownPlatformSourceFilter.FilterKnownPlatformSources(testRunRequestPayload.Sources);
+        }
+
         var runsettings = testRunRequestPayload.RunSettings;
 
         if (testRunRequestPayload.TestPlatformOptions != null)
@@ -1465,3 +1470,63 @@ private static List<string> GetSources(TestRunRequestPayload testRunRequestPaylo
         return sources;
     }
 }
+
+internal static class KnownPlatformSourceFilter
+{
+
+    // Running tests on AzureDevops, many projects use the default filter
+    // which includes all *test*.dll, this includes many of the TestPlatform dlls,
+    // which we cannot run, and don't want to attempt to run.
+    // The default filter also filters out !*TestAdapter*.dll but it is easy to forget
+    // so we skip the most used adapters here as well.
+    private static readonly HashSet<string> KnownPlatformSources = new(new string[]
+    {
+        "Microsoft.TestPlatform.CommunicationUtilities.dll",
+        "Microsoft.TestPlatform.CoreUtilities.dll",
+        "Microsoft.TestPlatform.CrossPlatEngine.dll",
+        "Microsoft.TestPlatform.PlatformAbstractions.dll",
+        "Microsoft.TestPlatform.Utilities.dll",
+        "Microsoft.VisualStudio.TestPlatform.Common.dll",
+        "Microsoft.VisualStudio.TestPlatform.ObjectModel.dll",
+        "testhost.dll",
+        "Microsoft.TestPlatform.AdapterUtilities.dll",
+
+        // NUnit
+        "NUnit3.TestAdapter.dll",
+
+        // XUnit
+        "xunit.runner.visualstudio.testadapter.dll",
+        "xunit.runner.visualstudio.dotnetcore.testadapter.dll",
+
+        // MSTest
+        "Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.dll",
+        "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll",
+        "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.dll",
+        "Microsoft.VisualStudio.TestPlatform.TestFramework.dll",
+        "Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll",
+    }, StringComparer.OrdinalIgnoreCase);
+
+
+    internal static List<string> FilterKnownPlatformSources(List<string>? sources)
+    {
+        if (sources == null)
+        {
+            return new List<string>();
+        }
+
+        var filteredSources = new List<string>();
+        foreach (string source in sources)
+        {
+            if (KnownPlatformSources.Contains(Path.GetFileName(source)))
+            {
+                EqtTrace.Info($"TestRequestManager.FilterKnownPlatformSources: Known platform dll was provided in sources, removing it '{source}'");
+            }
+            else
+            {
+                filteredSources.Add(source);
+            }
+        }
+
+        return filteredSources;
+    }
+}

test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs

@@ -9,6 +9,9 @@
 using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions;
 
 using TestPlatform.TestUtilities;
+using System.Linq;
+using Microsoft.VisualStudio.TestPlatform.Common;
+using FluentAssertions;
 
 namespace Microsoft.TestPlatform.AcceptanceTests;
 
@@ -396,4 +399,100 @@ public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereA
         // Returning 1 because of failing test in test assembly (SimpleTestProject2.dll)
         ExitCodeEquals(1);
     }
+
+    [TestMethod]
+    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
+    [NetCoreTargetFrameworkDataSource]
+    public void RunXunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
+    {
+        // This is the default filter of AzDo VSTest task:
+        //     testAssemblyVer2: |
+        //       **\*test *.dll
+        //       ! * *\*TestAdapter.dll
+        //       ! * *\obj\**
+        // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
+        // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
+        // or deps.json, and fails the run.
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var testAssemblyPath = _testEnvironment.GetTestAsset("XUTestProject.dll");
+        var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");
+
+        string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
+        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
+        var fails = this.StdErrWithWhiteSpace.Split('\n').Where(s => !s.IsNullOrWhiteSpace()).Select(s => s.Trim()).ToList();
+        fails.Should().HaveCount(2, "because there is 1 failed test, and one message that tests failed.");
+        fails.Last().Should().Be("Test Run Failed.");
+    }
+
+    [TestMethod]
+    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
+    [NetCoreTargetFrameworkDataSource()]
+    public void RunMstestTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
+    {
+        // This is the default filter of AzDo VSTest task:
+        //     testAssemblyVer2: |
+        //       **\*test *.dll
+        //       ! * *\*TestAdapter.dll
+        //       ! * *\obj\**
+        // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
+        // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
+        // or deps.json, and fails the run.
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var testAssemblyPath = _testEnvironment.GetTestAsset("SimpleTestProject.dll");
+        var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");
+
+        string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
+        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
+
+        StdErrHasTestRunFailedMessageButNoOtherError();
+    }
+
+    [TestMethod]
+    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
+    [NetCoreTargetFrameworkDataSource()]
+    public void RunNunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
+    {
+        // This is the default filter of AzDo VSTest task:
+        //     testAssemblyVer2: |
+        //       **\*test *.dll
+        //       ! * *\*TestAdapter.dll
+        //       ! * *\obj\**
+        // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
+        // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
+        // or deps.json, and fails the run.
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var testAssemblyPath = _testEnvironment.GetTestAsset("NUTestProject.dll");
+        var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");
+
+        string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
+        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
+
+        StdErrHasTestRunFailedMessageButNoOtherError();
+    }
+
+    [TestMethod]
+    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    public void RunTestsWhenProvidingJustPlatformDllsFailsTheRun(RunnerInfo runnerInfo)
+    {
+        // This is the default filter of AzDo VSTest task:
+        //     testAssemblyVer2: |
+        //       **\*test *.dll
+        //       ! * *\*TestAdapter.dll
+        //       ! * *\obj\**
+        // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
+        // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
+        // or deps.json, and fails the run.
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var xunitAssemblyPath = _testEnvironment.GetTestAsset("SimpleTestProject.dll");
+        var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(xunitAssemblyPath)!, "*test*.dll").Where(f => !f.EndsWith("SimpleTestProject.dll"));
+
+        string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Select(s => s.AddDoubleQuote()));
+        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
+
+        StdErr.Should().Be("No test source files were specified. ", "because all platform files we provided were filtered out");
+    }
 }