diff --git a/tests/sampletester/BaselineTest/AppDelegate.cs b/tests/sampletester/BaselineTest/AppDelegate.cs new file mode 100644 index 000000000000..25802298148d --- /dev/null +++ b/tests/sampletester/BaselineTest/AppDelegate.cs @@ -0,0 +1,28 @@ +using Foundation; +using UIKit; + +namespace BaselineTest { + [Register ("AppDelegate")] + public class AppDelegate : UIResponder, IUIApplicationDelegate { + UIWindow window; + UIViewController vc; + + [Export ("application:didFinishLaunchingWithOptions:")] + public bool FinishedLaunching (UIApplication application, NSDictionary launchOptions) + { + window = new UIWindow (UIScreen.MainScreen.Bounds); + vc = new UIViewController (); + vc.View.BackgroundColor = UIColor.Green; + window.RootViewController = vc; + window.MakeKeyAndVisible (); + + return true; + } + + static void Main (string [] args) + { + UIApplication.Main (args, null, typeof (AppDelegate)); + } + } +} + diff --git a/tests/sampletester/BaselineTest/BaselineTest.csproj b/tests/sampletester/BaselineTest/BaselineTest.csproj new file mode 100644 index 000000000000..699748ec15ab --- /dev/null +++ b/tests/sampletester/BaselineTest/BaselineTest.csproj @@ -0,0 +1,72 @@ + + + + Debug + iPhoneSimulator + {E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F} + {FEACFBD2-3405-455C-9665-78FE426C6842};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + {edc1b0fa-90cd-4038-8fad-98fe74adb368} + Exe + BaselineTest + BaselineTest + Resources + true + NSUrlSessionHandler + PackageReference + + + true + full + false + bin\iPhoneSimulator\Debug + DEBUG + prompt + 4 + x86_64 + None + true + + + none + true + bin\iPhoneSimulator\Release + prompt + 4 + None + x86_64 + + + true + full + false + bin\iPhone\Debug + DEBUG + prompt + 4 + ARM64 + iPhone Developer + true + + + none + true + bin\iPhone\Release + prompt + 4 + ARM64 + iPhone Developer + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/sampletester/BaselineTest/BaselineTest.sln b/tests/sampletester/BaselineTest/BaselineTest.sln new file mode 100644 index 000000000000..b2b66729db02 --- /dev/null +++ b/tests/sampletester/BaselineTest/BaselineTest.sln @@ -0,0 +1,23 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaselineTest", "BaselineTest.csproj", "{E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|iPhoneSimulator = Debug|iPhoneSimulator + Release|iPhoneSimulator = Release|iPhoneSimulator + Debug|iPhone = Debug|iPhone + Release|iPhone = Release|iPhone + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Debug|iPhoneSimulator.ActiveCfg = Debug|iPhoneSimulator + {E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Debug|iPhoneSimulator.Build.0 = Debug|iPhoneSimulator + {E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Release|iPhoneSimulator.ActiveCfg = Release|iPhoneSimulator + {E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Release|iPhoneSimulator.Build.0 = Release|iPhoneSimulator + {E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Debug|iPhone.ActiveCfg = Debug|iPhone + {E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Debug|iPhone.Build.0 = Debug|iPhone + {E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Release|iPhone.ActiveCfg = Release|iPhone + {E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Release|iPhone.Build.0 = Release|iPhone + EndGlobalSection +EndGlobal diff --git a/tests/sampletester/BaselineTest/Info.plist b/tests/sampletester/BaselineTest/Info.plist new file mode 100644 index 000000000000..bba9b4ffd48a --- /dev/null +++ b/tests/sampletester/BaselineTest/Info.plist @@ -0,0 +1,48 @@ + + + + + CFBundleDisplayName + BaselineTest + CFBundleIdentifier + com.xamarin.BaselineTest + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1.0 + LSRequiresIPhoneOS + + MinimumOSVersion + 8.0 + UIDeviceFamily + + 1 + 2 + + UILaunchStoryboardName + LaunchScreen + UIMainStoryboardFile~ipad + Main + UIRequiredDeviceCapabilities + + armv7 + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + XSAppIconAssets + Assets.xcassets/AppIcon.appiconset + CFBundleName + BaselineTest + + diff --git a/tests/sampletester/BaselineTest/LaunchScreen.storyboard b/tests/sampletester/BaselineTest/LaunchScreen.storyboard new file mode 100644 index 000000000000..f18534b49b24 --- /dev/null +++ b/tests/sampletester/BaselineTest/LaunchScreen.storyboard @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/sampletester/Configuration.cs b/tests/sampletester/Configuration.cs index a209f1422160..dd2ca8c24a65 100644 --- a/tests/sampletester/Configuration.cs +++ b/tests/sampletester/Configuration.cs @@ -52,5 +52,57 @@ static void CreateGlobalConfig (string root) } "); } + + static string tested_hash; + public static string TestedHash { + get { + lock (lock_obj) { + if (tested_hash != null) + return tested_hash; + + tested_hash = GetCurrentHash (Environment.CurrentDirectory); + return tested_hash; + } + } + } + + public static string GetCurrentHash (string directory) + { + return ProcessHelper.RunProcess ("git", "log -1 --pretty=%H", directory).Trim (); + } + + public static string GetCurrentRemoteUrl (string directory) + { + return ProcessHelper.RunProcess ("git", "remote get-url origin", directory).Trim (); + } + + static string mono_version; + public static string MonoVersion { + get { + lock (lock_obj) { + if (mono_version != null) + return mono_version; + + // We only care about the first line + mono_version = ProcessHelper.RunProcess ("mono", "--version").Split (new char [] { '\n' }, StringSplitOptions.RemoveEmptyEntries) [0].Trim (); + } + + return mono_version; + } + } + + static string sw_version; + public static string OSVersion { + get { + lock (lock_obj) { + if (sw_version != null) + return sw_version; + + sw_version = ProcessHelper.RunProcess ("sw_vers").Replace ('\n', ';').Replace ((char) 9, ' '); + } + + return sw_version; + } + } } } diff --git a/tests/sampletester/ProcessHelper.cs b/tests/sampletester/ProcessHelper.cs index 2022b2c1a1f0..5af428be36fd 100644 --- a/tests/sampletester/ProcessHelper.cs +++ b/tests/sampletester/ProcessHelper.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Text; using System.Text.RegularExpressions; +using System.Xml; using NUnit.Framework; @@ -28,6 +29,11 @@ static string LogDirectory { } public static void AssertRunProcess (string filename, string[] arguments, TimeSpan timeout, string workingDirectory, Dictionary environment_variables, string message) + { + AssertRunProcess (filename, arguments, timeout, workingDirectory, environment_variables, message, out _); + } + + public static void AssertRunProcess (string filename, string[] arguments, TimeSpan timeout, string workingDirectory, Dictionary environment_variables, string message, out string logfile) { var exitCode = 0; var output = new List (); @@ -49,7 +55,7 @@ public static void AssertRunProcess (string filename, string[] arguments, TimeSp output_callback ($"Exit code: {exitCode} Timed out: {timed_out} Total duration: {watch.Elapsed.ToString ()}"); // Write execution log to disk (and print the path) - var logfile = Path.Combine (LogDirectory, $"{filename}-{Interlocked.Increment (ref counter)}.log"); + logfile = Path.Combine (LogDirectory, $"{filename}-{Interlocked.Increment (ref counter)}.log"); File.WriteAllLines (logfile, output); TestContext.AddTestAttachment (logfile, $"Execution log for {filename}"); Console.WriteLine ("Execution log for {0}: {1}", filename, logfile); @@ -79,8 +85,8 @@ public static void BuildSolution (string solution, string platform, string confi nuget_args.Add ("sln"); // replaced later nuget_args.Add ("-Verbosity"); nuget_args.Add ("detailed"); + var slndir = Path.GetDirectoryName (solution); if (!solution.EndsWith (".sln", StringComparison.Ordinal)) { - var slndir = Path.GetDirectoryName (solution); while ((solutions = Directory.GetFiles (slndir, "*.sln", SearchOption.TopDirectoryOnly)).Length == 0 && slndir.Length > 1) slndir = Path.GetDirectoryName (slndir); nuget_args.Add ("-SolutionDir"); @@ -89,7 +95,7 @@ public static void BuildSolution (string solution, string platform, string confi foreach (var sln in solutions) { nuget_args [1] = sln; // replacing here - AssertRunProcess ("nuget", nuget_args.ToArray (), TimeSpan.FromMinutes (2), Configuration.SampleRootDirectory, environment_variables, "nuget restore"); + AssertRunProcess ("nuget", nuget_args.ToArray (), TimeSpan.FromMinutes (5), Configuration.SampleRootDirectory, environment_variables, "nuget restore"); } // msbuild @@ -102,6 +108,201 @@ public static void BuildSolution (string solution, string platform, string confi sb.Add (solution); if (!string.IsNullOrEmpty (target)) sb.Add ($"/t:{target}"); - AssertRunProcess ("msbuild", sb.ToArray (), TimeSpan.FromMinutes (5), Configuration.SampleRootDirectory, environment_variables, "build"); + + sb.Add ($"/verbosity:diag"); + environment_variables ["MTOUCH_ENV_OPTIONS"] = "--time --time --time --time -vvvv"; + environment_variables ["MMP_ENV_OPTIONS"] = "--time --time --time --time -vvvv"; + + var watch = Stopwatch.StartNew (); + var failed = false; + string msbuild_logfile; + try { + AssertRunProcess ("msbuild", sb.ToArray (), TimeSpan.FromMinutes (5), Configuration.SampleRootDirectory, environment_variables, "build", out msbuild_logfile); + } catch { + failed = true; + throw; + } finally { + watch.Stop (); + } + + // Write performance data to disk + var subdirs = Directory.GetDirectories (slndir, "*", SearchOption.AllDirectories); + + // First figure out which .app subdirectory is the actual .app. This is a bit more complicated than it would seem... + var apps = subdirs.Where ((v) => { + var names = v.Substring (slndir.Length).Split (Path.DirectorySeparatorChar); + if (names.Length < 2) + return false; + + if (!names [names.Length - 1].EndsWith (".app", StringComparison.Ordinal)) + return false; + + if (names.Any ((v2) => v2 == "copySceneKitAssets")) + return false; + + var bin_idx = Array.IndexOf (names, "bin"); + var conf_idx = Array.IndexOf (names, configuration); + if (bin_idx < 0 || conf_idx < 0) + return false; + + if (bin_idx > conf_idx) + return false; + + if (platform.Length > 0) { + var platform_idx = Array.IndexOf (names, platform); + if (platform_idx < 0) + return false; + + if (bin_idx > platform_idx) + return false; + } + + return true; + }).ToArray (); + + if (apps.Length > 1) { + // Found more than one .app subdirectory, use additional logic to choose between them. + var filtered_apps = apps.Where ((v) => { + // If one .app is a subdirectory of another .app, we don't care about the former. + if (apps.Any ((v2) => v2.Length < v.Length && v.StartsWith (v2, StringComparison.Ordinal))) + return false; + + // If one .app is contained within another .app, we don't care about the former. + var vname = Path.GetFileName (v); + var otherApps = apps.Where ((v2) => v != v2); + if (otherApps.Any ((v2) => { + var otherSubdirs = subdirs.Where ((v3) => v3.StartsWith (v2, StringComparison.Ordinal)); + return otherSubdirs.Any ((v3) => Path.GetFileName (v3) == vname); + })) + return false; + + return true; + }).ToArray (); + if (apps.Length == 0) + Assert.Fail ($"Filtered away all the .apps, from:\n\t{string.Join ("\n\t", apps)}"); + apps = filtered_apps; + } + + if (apps.Length > 1) { + Assert.Fail ($"Found more than one .app directory:\n\t{string.Join ("\n\t", apps)}"); + } else if (apps.Length == 0) { + Assert.Fail ($"Found no .app directories for platform: {platform} configuration: {configuration} target: {target}. All directories:\n\t{string.Join ("\n\t", subdirs)}"); + } + + var logfile = Path.Combine (LogDirectory, $"{Path.GetFileNameWithoutExtension (solution)}-perfdata-{Interlocked.Increment (ref counter)}.xml"); + + var xmlSettings = new XmlWriterSettings { + Indent = true, + }; + var xml = XmlWriter.Create (logfile, xmlSettings); + xml.WriteStartDocument (true); + xml.WriteStartElement ("performance"); + xml.WriteStartElement ("sample-build"); + xml.WriteAttributeString ("mono-version", Configuration.MonoVersion); + xml.WriteAttributeString ("os-version", Configuration.OSVersion); + xml.WriteAttributeString ("xamarin-macios-hash", Configuration.TestedHash); + xml.WriteAttributeString ("sample-repository", Configuration.GetCurrentRemoteUrl (slndir)); + xml.WriteAttributeString ("sample-hash", Configuration.GetCurrentHash (slndir)); + xml.WriteAttributeString ("agent-machinename", Environment.GetEnvironmentVariable ("AGENT_MACHINENAME")); + xml.WriteAttributeString ("agent-name", Environment.GetEnvironmentVariable ("AGENT_NAME")); + foreach (var app in apps) { + xml.WriteStartElement ("test"); + xml.WriteAttributeString ("name", TestContext.CurrentContext.Test.FullName); + xml.WriteAttributeString ("result", failed ? "failed" : "success"); + if (platform.Length > 0) + xml.WriteAttributeString ("platform", platform); + xml.WriteAttributeString ("configuration", configuration); + if (!failed) { + xml.WriteAttributeString ("duration", watch.ElapsedTicks.ToString ()); + xml.WriteAttributeString ("duration-formatted", watch.Elapsed.ToString ()); + + var files = Directory.GetFiles (app, "*", SearchOption.AllDirectories).OrderBy ((v) => v).ToArray (); + var lengths = files.Select ((v) => new FileInfo (v).Length).ToArray (); + var total_size = lengths.Sum (); + + xml.WriteAttributeString ("total-size", total_size.ToString ()); + var appstart = Path.GetDirectoryName (app).Length; + for (var i = 0; i < files.Length; i++) { + xml.WriteStartElement ("file"); + xml.WriteAttributeString ("name", files [i].Substring (appstart + 1)); + xml.WriteAttributeString ("size", lengths [i].ToString ()); + xml.WriteEndElement (); + } + + if (File.Exists (msbuild_logfile)) { + var lines = File.ReadAllLines (msbuild_logfile); + var target_perf_summary = new List (); + var task_perf_summary = new List (); + var timestamps = new List (); + for (var i = lines.Length - 1; i >= 0; i--) { + if (lines [i].EndsWith ("Target Performance Summary:", StringComparison.Ordinal)) { + for (var k = i + 1; k < lines.Length && lines [k].EndsWith ("calls", StringComparison.Ordinal); k++) { + target_perf_summary.Add (lines [k].Substring (18).Trim ()); + } + } else if (lines [i].EndsWith ("Task Performance Summary:", StringComparison.Ordinal)) { + for (var k = i + 1; k < lines.Length && lines [k].EndsWith ("calls", StringComparison.Ordinal); k++) { + task_perf_summary.Add (lines [k].Substring (18).Trim ()); + } + } else if (lines [i].Contains ("!Timestamp")) { + timestamps.Add (lines [i]); + } + } + foreach (var tps in target_perf_summary) { + var split = tps.Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + xml.WriteStartElement ("target"); + xml.WriteAttributeString ("name", split [2]); + xml.WriteAttributeString ("ms", split [0]); + xml.WriteAttributeString ("calls", split [3]); + xml.WriteEndElement (); + } + foreach (var tps in task_perf_summary) { + var split = tps.Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + xml.WriteStartElement ("task"); + xml.WriteAttributeString ("name", split [2]); + xml.WriteAttributeString ("ms", split [0]); + xml.WriteAttributeString ("calls", split [3]); + xml.WriteEndElement (); + } + foreach (var ts in timestamps) { + // Sample line: + // 15:04:50.4609520: !Timestamp Setup: 28 ms (TaskId:137) + var splitFirst = ts.Split (new char [] { ':' }, StringSplitOptions.RemoveEmptyEntries); + var splitSecondA = splitFirst [3].Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + var splitSecondB = splitFirst [4].Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + var name = string.Join (" ", splitSecondA.Skip (1)); + var level = splitSecondA [0].Count ((v) => v == '!').ToString (); + var ms = splitSecondB [0]; + xml.WriteStartElement ("timestamp"); + xml.WriteAttributeString ("name", name); + xml.WriteAttributeString ("level", level); + xml.WriteAttributeString ("ms", ms); + xml.WriteEndElement (); + } + } + + xml.WriteEndElement (); + } + + xml.WriteEndElement (); // sample-build + xml.WriteEndElement (); // performance + xml.WriteEndDocument (); + xml.Dispose (); + + TestContext.AddTestAttachment (logfile, $"Performance data"); + } + } + + internal static string RunProcess (string filename, string arguments = "", string working_directory = null) + { + using (var p = Process.Start (filename, arguments)) { + p.StartInfo.RedirectStandardOutput = true; + p.StartInfo.UseShellExecute = false; + if (!string.IsNullOrEmpty (working_directory)) + p.StartInfo.WorkingDirectory = working_directory; + p.Start (); + var output = p.StandardOutput.ReadToEnd (); + p.WaitForExit (); + return output; + } } } diff --git a/tests/sampletester/SampleTester.cs b/tests/sampletester/SampleTester.cs index 2bdbd28cc9a1..6c83f706e179 100644 --- a/tests/sampletester/SampleTester.cs +++ b/tests/sampletester/SampleTester.cs @@ -247,4 +247,16 @@ string CloneRepo () return GitHub.CloneRepository (Org, Repository, Hash); } } + + [TestFixture] + public class BaselineTester { + [Test] + public void DeviceDebug () + { + var sln = Path.Combine (Configuration.SourceRoot, "tests", "sampletester", "BaselineTest", "BaselineTest.sln"); + GitHub.CleanRepository (Path.GetDirectoryName (sln)); + ProcessHelper.BuildSolution (sln, "iPhone", "Debug", new Dictionary ()); + } + + } } diff --git a/tests/sampletester/Samples.cs b/tests/sampletester/Samples.cs index 6ec67817268a..f74342a1c1d9 100644 --- a/tests/sampletester/Samples.cs +++ b/tests/sampletester/Samples.cs @@ -8,7 +8,7 @@ public class IosSampleTester : SampleTester { const string ORG = "xamarin"; const string REPO = "ios-samples"; // monotouch-samples redirects to ios-samples const string CATEGORY = "iossamples"; // categories can't contain dashes - const string HASH = "bffa511ecb8f74b2d4a42418a130d0c83c9723cf"; + const string HASH = "150f0e6167e2133182924114a3b36e8384f632f1"; static Dictionary test_data = new Dictionary { // Build solution instead of csproj diff --git a/tests/sampletester/sampletester.sln b/tests/sampletester/sampletester.sln index 6a85f8fa7094..42f569df45cc 100644 --- a/tests/sampletester/sampletester.sln +++ b/tests/sampletester/sampletester.sln @@ -13,5 +13,13 @@ Global {7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Debug|Any CPU.Build.0 = Debug|Any CPU {7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Release|Any CPU.ActiveCfg = Release|Any CPU {7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Release|Any CPU.Build.0 = Release|Any CPU + {7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Debug|iPhoneSimulator.ActiveCfg = Debug|Any CPU + {7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Debug|iPhoneSimulator.Build.0 = Debug|Any CPU + {7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Release|iPhoneSimulator.ActiveCfg = Release|Any CPU + {7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Release|iPhoneSimulator.Build.0 = Release|Any CPU + {7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Debug|iPhone.ActiveCfg = Debug|Any CPU + {7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Debug|iPhone.Build.0 = Debug|Any CPU + {7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Release|iPhone.ActiveCfg = Release|Any CPU + {7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Release|iPhone.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/tools/devops/build-samples.sh b/tools/devops/build-samples.sh index e678483fc9e1..50e3b95b87eb 100755 --- a/tools/devops/build-samples.sh +++ b/tools/devops/build-samples.sh @@ -1,8 +1,6 @@ #!/bin/bash -eux -# we want verbose output from mtouch and mlaunch -echo 123456789 > ~/.mtouch-verbosity -echo 123456789 > ~/.mlaunch-verbosity +cd "$(dirname "${BASH_SOURCE[0]}")/../.." make -C tests test-system.config make -C tests/sampletester TESTS_USE_SYSTEM=1 \ No newline at end of file diff --git a/tools/devops/build-samples.yml b/tools/devops/build-samples.yml index ebb27f094b9c..af18cdc2666a 100644 --- a/tools/devops/build-samples.yml +++ b/tools/devops/build-samples.yml @@ -13,6 +13,23 @@ variables: name_filter_gr: '^[G-Rg-r].*$' name_filter_rest: '^[^A-Ra-r].*$' +resources: + repositories: + - repository: xamarin-macios-data + type: github + name: xamarin/xamarin-macios-data + ref: refs/heads/master + endpoint: xamarin + - repository: maccore + type: github + name: xamarin/maccore + ref: refs/heads/master + endpoint: xamarin + +### +### Tell GitHub we're starting working on this commit +### + jobs: - job: ReportStartToGitHub displayName: Set pending GitHub status @@ -22,6 +39,13 @@ jobs: - bash: ./jenkins/add-commit-status.sh "--token=$(github-pat)" "--hash=$BUILD_SOURCEVERSION" "--state=pending" --target-url="$AZURE_BUILD_URL" --description="$BUILD_DEFINITIONNAME" --context="$BUILD_DEFINITIONNAME" displayName: Set pending GitHub status +### +### Run the sample tests. +### +### They're split over multiple bots to make them run faster (and not hit the +### max job time duration for a single job). +### + - job: macOS dependsOn: ReportStartToGitHub displayName: Build samples @@ -73,32 +97,35 @@ jobs: steps: - checkout: self + - checkout: xamarin-macios-data + persistCredentials: true + - checkout: maccore persistCredentials: true - - bash: ./tools/devops/system-info.sh + - bash: ./xamarin-macios/tools/devops/system-info.sh displayName: System Info - - bash: ./tools/devops/fetch-maccore.sh - displayName: Fetch maccore + - bash: ./xamarin-macios/tools/devops/fetch-maccore.sh + displayName: Fetch correct maccore hash - task: provisionator@2 displayName: Xcode inputs: provisionator_uri: '$(provisionator-uri)' github_token: '$(github-pat)' - provisioning_script: $(System.DefaultWorkingDirectory)/tools/devops/provision-xcode.csx + provisioning_script: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/provision-xcode.csx - task: provisionator@2 displayName: Provision XI/XM/Mono/Objective-Sharpie inputs: provisionator_uri: '$(provisionator-uri)' github_token: '$(github-pat)' - provisioning_script: $(System.DefaultWorkingDirectory)/tools/devops/build-samples.csx + provisioning_script: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/build-samples.csx - - bash: ./tools/devops/system-info.sh + - bash: ./xamarin-macios/tools/devops/system-info.sh displayName: System Info post provisioning - - bash: ./tools/devops/build-samples.sh + - bash: ./xamarin-macios/tools/devops/build-samples.sh displayName: Run tests - task: PublishTestResults@2 @@ -111,14 +138,58 @@ jobs: publishRunAttachments: true mergeTestResults: true + - bash: ./xamarin-macios/tools/devops/prepare-performance-data.sh + displayName: Prepare performance data + condition: always() + + - publish: logs + displayName: 'Upload perf data' + condition: always() + artifact: logs-$(System.JobId) + - bash: echo "##vso[task.setvariable variable=JobStatus;isOutput=true]$AGENT_JOBSTATUS" name: ExportedVariables displayName: Export status condition: always() +### +### Push performance data to the xamarin-macios-data repository +### + +- job: PublishPerformanceData + displayName: Publish Performance Data + condition: always() + dependsOn: macOS + pool: + name: '$(macOSVersion)' + steps: + - checkout: self + - checkout: xamarin-macios-data + persistCredentials: true + + - task: DownloadPipelineArtifact@2 + displayName: Download performance data + inputs: + source: current + path: logs + + - bash: ./xamarin-macios/tools/devops/push-performance-data.sh + displayName: Publish perf data + + - bash: echo "##vso[task.setvariable variable=JobStatus;isOutput=true]$AGENT_JOBSTATUS" + name: ExportedVariables + displayName: Export status + condition: always() + +### +### Report final results to GitHub +### + - job: ReportResultsToGitHub displayName: Report status/results to GitHub - dependsOn: macOS + dependsOn: + - PublishPerformanceData + - macOS condition: always() pool: name: '$(macOSVersion)' @@ -133,6 +204,7 @@ jobs: jobResultReleaseiPhoneSimulator: $[ dependencies.macOS.outputs['Release_iPhoneSimulator.ExportedVariables.JobStatus'] ] jobResultDebugMac: $[ dependencies.macOS.outputs['Debug_Mac.ExportedVariables.JobStatus'] ] jobResultReleaseMac: $[ dependencies.macOS.outputs['Release_Mac.ExportedVariables.JobStatus'] ] + jobResultPublishPerformanceData: $[ dependencies.PublishPerformanceData.outputs['ExportedVariables.JobStatus'] ] steps: - bash: | ./tools/devops/build-samples-report-to-github.sh "$(github-pat)" \ @@ -141,6 +213,7 @@ jobs: "Release_iPhone_AF" "Release_iPhone_GR" "Release_iPhone_SZ" \ "Release_iPhoneSimulator" \ "Debug_Mac" \ - "Release_Mac" + "Release_Mac" \ + "PublishPerformanceData" displayName: Report results to GitHub as comment / status condition: always() diff --git a/tools/devops/fetch-maccore.sh b/tools/devops/fetch-maccore.sh index cdfae75a8212..cecdb7c19eec 100755 --- a/tools/devops/fetch-maccore.sh +++ b/tools/devops/fetch-maccore.sh @@ -1,12 +1,17 @@ #!/bin/bash -eux +# maccore is already checked out, but our script does a different remote +# ('xamarin' vs 'origin'), so add the different remote, and at the same time +# use https for the repository (instead of git@), since GitHub auth on Azure +# Devops only works with https. + +cd "$(dirname "${BASH_SOURCE[0]}")" +cd "$(git rev-parse --show-toplevel)/../maccore" +git remote add -f xamarin https://github.com/xamarin/maccore +cd ../xamarin-macios + # Make sure we've enabled our xamarin bits ./configure --enable-xamarin -# grab Azure Devop's authorization token from the current repo, and use add it to the global git configuration -AUTH=$(git config -l | grep AUTHORIZATION | sed 's/.*AUTHORIZATION: //') -git config --global http.extraheader "AUTHORIZATION: $AUTH" - -# fetch maccore -# the github auth we use only works with https, so change maccore's url to be https:// instead of git@ -make reset-maccore MACCORE_MODULE="$(grep ^MACCORE_MODULE mk/xamarin.mk | sed -e 's/.*:= //' -e 's_git@github.com:_https://github.com/_' -e 's/[.]git//')" V=1 +# fetch the hash we want +make reset-maccore V=1 diff --git a/tools/devops/prepare-performance-data.sh b/tools/devops/prepare-performance-data.sh new file mode 100755 index 000000000000..7d3764d38140 --- /dev/null +++ b/tools/devops/prepare-performance-data.sh @@ -0,0 +1,14 @@ +#!/bin/bash -ex + +DIR=perf-data/samples/$BUILD_SOURCEBRANCHNAME/$BUILD_SOURCEVERSION/$SYSTEM_JOBID +mkdir -p "$DIR" + +XMLS=(xamarin-macios/tests/sampletester/bin/Debug/tmp-test-dir/execution-logs/*.xml) +if ! test -f "${XMLS[0]}"; then + echo "##vso[task.logissue type=warning]Could not find any performance data to publish" + exit 0 +fi +cp -c xamarin-macios/tests/sampletester/bin/Debug/tmp-test-dir/execution-logs/*.xml "$DIR/" + +mkdir -p logs +zip -9r "logs/execution-logs-$SYSTEM_JOBID.zip" perf-data diff --git a/tools/devops/push-performance-data.sh b/tools/devops/push-performance-data.sh new file mode 100755 index 000000000000..801f3341e06c --- /dev/null +++ b/tools/devops/push-performance-data.sh @@ -0,0 +1,39 @@ +#!/bin/bash -ex + +cd xamarin-macios-data +git checkout master +cp -cr ../logs/ ./ + +mv ./*/*.zip . +for zip in ./*.zip; do + unzip "$zip" +done + +# Merge each individual xml file into one big xml file +DIR=perf-data/samples/$BUILD_SOURCEBRANCHNAME/$BUILD_SOURCEVERSION +cd "$DIR" +# Merge the xml files from each bot into a big per-bot xml file. Don't merge +# the xml from all the bots together into a single enormous xml file, because +# it'll be close to GitHub's size limit per file (limit is 100mb, the enormous +# xml file would be ~80mb now), and might very well pass that one day. +for job in *-*-*-*-*; do +{ + echo '' + echo '' + find "$job" -name '*perfdata*.xml' -print0 | xargs -0 -n 1 tail -n +2 | grep -F -v -e '' -e '' + echo '' +} > "data-$job.xml" +done + +# Add the big xml files to git +git add data-*.xml +git commit -m "Add performance data for $BUILD_SOURCEBRANCHNAME/$BUILD_SOURCEVERSION." + +# Push! +# Try to push 5 times, just in case someone else pushed first. +COUNTER=5 +while [[ $COUNTER -gt 0 ]]; do + if git push; then break; fi + git pull + (( COUNTER-- )) +done diff --git a/tools/devops/system-info.sh b/tools/devops/system-info.sh index a753fb8412cf..455544f3f0dd 100755 --- a/tools/devops/system-info.sh +++ b/tools/devops/system-info.sh @@ -10,5 +10,7 @@ ls -lad /Applications/Xcode* xcode-select -p mono --version env | sort +uptime +ps aux -exit 0 \ No newline at end of file +exit 0 diff --git a/tools/mmp/driver.cs b/tools/mmp/driver.cs index 4409a8592769..75b708a5aec3 100644 --- a/tools/mmp/driver.cs +++ b/tools/mmp/driver.cs @@ -358,6 +358,13 @@ static void Main2 (string [] args) AddSharedOptions (App, os); + var extra_args = Environment.GetEnvironmentVariable ("MMP_ENV_OPTIONS"); + if (!string.IsNullOrEmpty (extra_args)) { + var l = new List (args); + l.AddRange (extra_args.Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries)); + args = l.ToArray (); + } + try { App.RootAssemblies.AddRange (os.Parse (args)); }