From e30805bcc42d32cbe2e4c92166ac45309c231bb6 Mon Sep 17 00:00:00 2001 From: Val Menn Date: Thu, 2 Apr 2015 14:09:19 -0700 Subject: [PATCH 1/2] Added build script for Mac/Linux Added a Perl script to build using xbuild, rebuild using generated MSBuild, and run tests. The last two steps are optional. The outputs can be redirected (this would include binaries, packages, and logs). In order to be able to redirect, made some project file fixes. --- build.pl | 244 ++++++++++++++++++ dir.props | 10 +- dir.targets | 6 +- ...Microsoft.Build.Framework.UnitTests.csproj | 2 +- ...Microsoft.Build.Utilities.UnitTests.csproj | 2 +- src/XMakeBuildEngine/Microsoft.Build.csproj | 2 +- .../Microsoft.Build.Engine.UnitTests.csproj | 2 +- ...Microsoft.Build.Engine.OM.UnitTests.csproj | 2 +- .../UnitTests/CommandLineSwitches_Tests.cs | 2 +- ...crosoft.Build.CommandLine.UnitTests.csproj | 2 +- .../Microsoft.Build.Tasks.UnitTests.csproj | 2 +- 11 files changed, 262 insertions(+), 14 deletions(-) create mode 100644 build.pl diff --git a/build.pl b/build.pl new file mode 100644 index 00000000000..0f607be51c8 --- /dev/null +++ b/build.pl @@ -0,0 +1,244 @@ +# !/usr/bin/perl + +use strict; +use Getopt::Long; +use Data::Dumper; +use File::Path qw(make_path); +use File::Spec::Functions qw(rel2abs catfile); +use File::Basename; +use FindBin qw($RealBin); +use Cwd qw(abs_path); +use POSIX; + +#set the timestamp in case we need to create a directory +use constant DATETIME=>strftime("%Y-%m-%d_%H-%M-%S", localtime); + +# The source solution +my $solutionToBuild = catfile(catfile($RealBin, 'src'), 'MSBuild.sln'); + +my $usage = <<"USAGE"; +Usage build.pl [-root=] [-fullbuild] [-verify] [-tests] [-all] [-quiet] [-silent] + +The script can build MSBuild.exe using mono, verify by +rebuilding with the generated MSBuild.exe and run tests. +If the output root is not specified, the bin and packages +subdirectories are created in the source tree (as well as +bin-verify and packages-verify if verification build runs). +If output root is provided, a dated subdirectory is created +under the output root and the same subdirectories appear +there. The tests are run either on the xbuild generated +binaries or, if verification is done, on the verification +binaries. + +Options: + root - specifies the output root. The root is the + source tree by default. + fullBuild - do a rebuild instead of the incremental build. + This only makes a difference without the root + options, as the rooted build are always clean. + verify - rebuild the source with the binaries generated + by the xbuild. + tests - run the tests. + all - shorthand for -verify and -tests flags. + quiet - don't show any build output, only summaries + silent - show nothing, not even summaries + +USAGE + + +# Quick check if we are in the right place and that mono is installed +# (just checking for solution and xbuild presence) +die ("Solution $solutionToBuild does not exist\n") unless -e $solutionToBuild; +die ("xbuild was not found") unless -e "/usr/bin/xbuild"; + +my $buildRoot; +my $runTests; +my $verification; +my $quiet; +my $silent; +my $fullBuild; +my $allSteps; +my $help; + +die $usage unless GetOptions( + 'root=s' => \$buildRoot, + 'verify' => \$verification, + 'tests' => \$runTests, + 'quiet' => \$quiet, + 'silent' => \$silent, + 'fullBuild' => \$fullBuild, + 'all' => \$allSteps, + 'help' => \$help + ); + +if ($help) { + print $usage; + exit (0); +} + +# The all steps flag override some other +if ($allSteps) { + $verification = 1; + $runTests = 1; +} + +# We need nunit-console to run tests +my $nunitConsole; +if ($runTests) { + # Find the nunit console program + my $n = `which nunit-console`; + chomp $n; + + die ("Tests are requested, but nunit-console was not found") unless -e "/usr/bin/nunit-console"; + + # Resolve any links + $nunitConsole = abs_path($n); + # Use version 4 if found + $n = $nunitConsole . '4'; + $nunitConsole = $n if -e $n; +} + + +# Find the location where we're going to store results +# If no root is specifed, use the script location (we'll create +# bin, packages, bin-verify, packages-verify directories and the +# logs there. +# If root is specified, make a dated subdirectory there. That +# will be our root. +if (!$buildRoot) { + $buildRoot = $RealBin; +} +else { + $buildRoot = catfile(rel2abs($buildRoot), DATETIME); + # Just make sure it's not a file + die ("Verification root '$buildRoot' exists and is a file") if -f $buildRoot; + make_path($buildRoot); +} + +# Make some paths +my $binRoot = catfile($buildRoot, "bin", ""); +my $verBinRoot = catfile($buildRoot, "bin-verify", ""); +my $packageRoot = catfile ($buildRoot, "packages", ""); +my $verPackageRoot = catfile ($buildRoot, "packages-verify", ""); + +# The regex to parse build logs +my $extractRegex = qr!^\s*Build binary target directory: '($buildRoot.+?)/?'!; + +my $msbuildPath; +my $exitCode; +my $errorCount; + +# Run the first build +($exitCode, $errorCount, $msbuildPath) = runbuild('xbuild', '', '/'); + +die ("Build with xbuild failed (code $exitCode)") unless $exitCode == 0; +die ("Build with xbuild failed (error count $errorCount") unless $errorCount == 0; +die ("Build succeeded, but MSBuild.exe binary was not found") unless -e $msbuildPath; + +# Use the MSBuild.exe we created and rebuild (if requested) +if ($verification) { + my $MSBuildProgram = catfile($msbuildPath, 'MSBuild.exe'); + $MSBuildProgram = 'mono ' . $MSBuildProgram if $^O == "MSWin32"; + my $newMSBuildPath; + ($exitCode, $errorCount, $newMSBuildPath) = runbuild($MSBuildProgram, '-verify', '-'); + die ("Build with msbuild failed (code $exitCode)") unless $exitCode == 0; + die ("Build with msbuild failed (error count $errorCount") unless $errorCount == 0; + print "New MSBuild path: $newMSBuildPath\n"; + $msbuildPath = $newMSBuildPath if -e $newMSBuildPath; +} + +if ($runTests) { + # Get the dlls for testing + my @file = glob catfile($msbuildPath, '*UnitTest*.dll'); + runtests (@file); +} + +# This functions runs a build. +# runbuild(program, suffix, switch) +# program -- the build (xbuild or msbuild) +# suffix -- appended to log and output directory names +# switch -- either - or / +sub runbuild { + die ('runbuild sub was not called correctly') unless @_ == 3; + my ($program, $suffix, $switch) = @_; + + # Get paths of output directories and the log + my $binDir = catfile($buildRoot, "bin$suffix", ""); + my $packagesDir = catfile ($buildRoot, "packages$suffix", ""); + my $logFile = catfile($buildRoot, "MSBuild${suffix}.log"); + + # If we need to rebuild, add a switch for the task + my $rebuildSwitch = $fullBuild ? "${switch}t:Rebuild " : ""; + + # Generate and print the command we run + my $command = "$program ${switch}nologo ${switch}v:q ${switch}tv:4.0 $rebuildSwitch " . + "${switch}p:Configuration=Debug-MONO " . + "${switch}p:BinDir=$binDir ${switch}p:PackagesDir=$packagesDir " . + "${switch}fl \"${switch}flp:LogFile=$logFile;V=diag\" $solutionToBuild"; + print $command . "\n" unless $silent; + + # Run build, parsing it's output to count errors and warnings + # Harakiri if can't run + open(BUILD_OUTPUT, "$command 2>&1 |") or die "Cannot run $program, error $!"; + my $warningCount = 0; + my $errorCount = 0; + for () { + print $_ unless ($quiet || $silent); + m/:\s+error / && ($errorCount++, next); + m/:\s+warning / && ($warningCount++, next); + } + + die "Failed to run $program, exit code $!" if $! != 0; + close BUILD_OUTPUT; + my $exitCode = $? >> 8; + + # Search the log for the full output path + my $msbuildPath; + if (open LOG, '<', $logFile) { + m/$extractRegex/ && ($msbuildPath = $1, last) for ; + close (LOG); + } + else { + # It's not an error if the path cannot be found. At worst, we cannot verify + print "Warning: Cannot open log file $logFile: $!" if $! && !$silent; + } + + print "Errors: $errorCount, Warnings $warningCount\n" unless $silent; + return ($exitCode, $errorCount, $msbuildPath); +} + +# This function runs the test. It gets the list of dlls to test. +sub runtests { + my @files = @_; + + # Create directory for output + my $testResultsDir = catfile($buildRoot, 'TestResults'); + make_path($testResultsDir); + + # Output file names in that directory + my $xmlResultFile = catfile($testResultsDir, 'Results.xml'); + my $outputFile = catfile($testResultsDir, 'TestOutput.txt'); + + # Build the command to run the test + my $command = "$nunitConsole -xml:$xmlResultFile " . join (' ', @files); + print $command . "\n" unless $silent; + + # Run it silently + system("$command 2>&1 >$outputFile"); + + # Count the passed/failed tests by readin the output XML file + my $testsFailed = 0; + my $testsSucceeded = 0; + + my $testRegex = qr!^\s*; + close (LOG); + } + else { + print "Warning: Cannot open log file $xmlResultFile: $!" if $! && !$silent; + } + my $testsRan = $testsSucceeded + $testsFailed; + print "Tests ran: $testsRan, tests succeeded: $testsSucceeded, tests failed: $testsFailed\n" unless $silent; +} \ No newline at end of file diff --git a/dir.props b/dir.props index 4755b9bdd6e..b69d77eb64d 100644 --- a/dir.props +++ b/dir.props @@ -10,19 +10,19 @@ $(ProjectDir)bin$([System.IO.Path]::DirectorySeparatorChar) $(BinDir)tests$([System.IO.Path]::DirectorySeparatorChar) $(ProjectDir)packages$([System.IO.Path]::DirectorySeparatorChar) - $(PackagesDir)Microsoft.DotNet.BuildTools.$(MSBuildToolsVersion)$([System.IO.Path]::DirectorySeparatorChar)lib$([System.IO.Path]::DirectorySeparatorChar) + $([System.IO.Path]::Combine($(PackagesDir)Microsoft.DotNet.BuildTools.$(MSBuildToolsVersion),"lib")$([System.IO.Path]::DirectorySeparatorChar) $(PackagesDir) $(PackagesDir)NuGet.exe - $(SourceDir).nuget$([System.IO.Path]::DirectorySeparatorChar)NuGet.Config + $([System.IO.Path]::Combine("$(SourceDir).nuget","NuGet.Config")) -ConfigFile "$(NuGetConfigFile)" "$(NuGetToolPath)" $(NugetRestoreCommand) install - $(NugetRestoreCommand) -OutputDirectory "$(PackagesDir.TrimEnd('\/'.ToCharArray()))" + $(NugetRestoreCommand) -OutputDirectory "$(PackagesDir)" $(NugetRestoreCommand) $(NuGetConfigCommandLine) mono $(NuGetRestoreCommand) @@ -67,10 +67,10 @@ - $(ProjectDir)bin$([System.IO.Path]::DirectorySeparatorChar) + $(BinDir) $([System.IO.Path]::Combine($(BaseOutputPath)$(OS),$(Configuration)))$([System.IO.Path]::DirectorySeparatorChar) $([System.IO.Path]::Combine($(BaseOutputPath)$(Platform),$(OS),$(Configuration)))$([System.IO.Path]::DirectorySeparatorChar) - $(BaseOutputPathWithConfig)$([System.IO.Path]::DirectorySeparatorChar) + $(BaseOutputPathWithConfig) $(BaseOutputPath)obj$([System.IO.Path]::DirectorySeparatorChar) $([System.IO.Path]::Combine($(BaseIntermediateOutputPath)$(MSBuildProjectName),$(OS),$(Configuration)))$([System.IO.Path]::DirectorySeparatorChar) diff --git a/dir.targets b/dir.targets index 9e888718577..7a2c9ed7e82 100644 --- a/dir.targets +++ b/dir.targets @@ -41,7 +41,11 @@ Needed to avoid the IntialTargets from having an Output which ends up getting added to the output references when you have a project to project reference. --> - + + + + + - ..\..\..\packages\NUnit.2.6.4\lib\nunit.framework.dll + $(PackagesDir)NUnit.2.6.4\lib\nunit.framework.dll True diff --git a/src/Utilities/UnitTests/Microsoft.Build.Utilities.UnitTests.csproj b/src/Utilities/UnitTests/Microsoft.Build.Utilities.UnitTests.csproj index c7333bdd249..d9533501063 100644 --- a/src/Utilities/UnitTests/Microsoft.Build.Utilities.UnitTests.csproj +++ b/src/Utilities/UnitTests/Microsoft.Build.Utilities.UnitTests.csproj @@ -61,7 +61,7 @@ - ..\..\..\packages\NUnit.2.6.4\lib\nunit.framework.dll + $(PackagesDir)NUnit.2.6.4\lib\nunit.framework.dll True diff --git a/src/XMakeBuildEngine/Microsoft.Build.csproj b/src/XMakeBuildEngine/Microsoft.Build.csproj index 5405da2103a..e61dd260140 100644 --- a/src/XMakeBuildEngine/Microsoft.Build.csproj +++ b/src/XMakeBuildEngine/Microsoft.Build.csproj @@ -652,7 +652,7 @@ False - ..\..\packages\Microsoft.Tpl.Dataflow.4.5.24\lib\portable-net45+win8+wpa81\System.Threading.Tasks.Dataflow.dll + $(PackagesDir)Microsoft.Tpl.Dataflow.4.5.24\lib\portable-net45+win8+wpa81\System.Threading.Tasks.Dataflow.dll True diff --git a/src/XMakeBuildEngine/UnitTests/Microsoft.Build.Engine.UnitTests.csproj b/src/XMakeBuildEngine/UnitTests/Microsoft.Build.Engine.UnitTests.csproj index faca6bb7b8c..b4131a26618 100644 --- a/src/XMakeBuildEngine/UnitTests/Microsoft.Build.Engine.UnitTests.csproj +++ b/src/XMakeBuildEngine/UnitTests/Microsoft.Build.Engine.UnitTests.csproj @@ -201,7 +201,7 @@ Microsoft.Build - ..\..\..\packages\NUnit.2.6.4\lib\nunit.framework.dll + $(PackagesDir)NUnit.2.6.4\lib\nunit.framework.dll True diff --git a/src/XMakeBuildEngine/UnitTestsPublicOM/Microsoft.Build.Engine.OM.UnitTests.csproj b/src/XMakeBuildEngine/UnitTestsPublicOM/Microsoft.Build.Engine.OM.UnitTests.csproj index 37fe7ac7918..e98c6d302a4 100644 --- a/src/XMakeBuildEngine/UnitTestsPublicOM/Microsoft.Build.Engine.OM.UnitTests.csproj +++ b/src/XMakeBuildEngine/UnitTestsPublicOM/Microsoft.Build.Engine.OM.UnitTests.csproj @@ -136,7 +136,7 @@ Microsoft.Build - ..\..\..\packages\NUnit.2.6.4\lib\nunit.framework.dll + $(PackagesDir)NUnit.2.6.4\lib\nunit.framework.dll True diff --git a/src/XMakeCommandLine/UnitTests/CommandLineSwitches_Tests.cs b/src/XMakeCommandLine/UnitTests/CommandLineSwitches_Tests.cs index f6d8c8c5a43..6dea0ab741b 100644 --- a/src/XMakeCommandLine/UnitTests/CommandLineSwitches_Tests.cs +++ b/src/XMakeCommandLine/UnitTests/CommandLineSwitches_Tests.cs @@ -19,7 +19,7 @@ namespace Microsoft.Build.UnitTests public class CommandLineSwitchesTests { [TestFixtureSetUp] - public static void Setup(TestContext testContext) + public static void Setup() { // Make sure resources are initialized MSBuildApp.Initialize(); diff --git a/src/XMakeCommandLine/UnitTests/Microsoft.Build.CommandLine.UnitTests.csproj b/src/XMakeCommandLine/UnitTests/Microsoft.Build.CommandLine.UnitTests.csproj index 1ecdd83dca3..7e527169941 100644 --- a/src/XMakeCommandLine/UnitTests/Microsoft.Build.CommandLine.UnitTests.csproj +++ b/src/XMakeCommandLine/UnitTests/Microsoft.Build.CommandLine.UnitTests.csproj @@ -44,7 +44,7 @@ - ..\..\..\packages\NUnit.2.6.4\lib\nunit.framework.dll + $(PackagesDir)NUnit.2.6.4\lib\nunit.framework.dll True diff --git a/src/XMakeTasks/UnitTests/Microsoft.Build.Tasks.UnitTests.csproj b/src/XMakeTasks/UnitTests/Microsoft.Build.Tasks.UnitTests.csproj index f50d8adb43a..ddbb388342e 100644 --- a/src/XMakeTasks/UnitTests/Microsoft.Build.Tasks.UnitTests.csproj +++ b/src/XMakeTasks/UnitTests/Microsoft.Build.Tasks.UnitTests.csproj @@ -132,7 +132,7 @@ - ..\..\..\packages\NUnit.2.6.4\lib\nunit.framework.dll + $(PackagesDir)NUnit.2.6.4\lib\nunit.framework.dll True From a35e3a2790251cea8ff27fbec5db2eb2c972dfb0 Mon Sep 17 00:00:00 2001 From: Val Menn Date: Thu, 2 Apr 2015 16:09:09 -0700 Subject: [PATCH 2/2] Trimmed backslash before a double quote. --- dir.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dir.props b/dir.props index b69d77eb64d..b31322f9070 100644 --- a/dir.props +++ b/dir.props @@ -22,7 +22,7 @@ "$(NuGetToolPath)" $(NugetRestoreCommand) install - $(NugetRestoreCommand) -OutputDirectory "$(PackagesDir)" + $(NugetRestoreCommand) -OutputDirectory "$(PackagesDir.TrimEnd('\'))" $(NugetRestoreCommand) $(NuGetConfigCommandLine) mono $(NuGetRestoreCommand)