Skip to content

Commit c36a92f

Browse files
BMurriBMurri
committed
Call the runner directly from the task command-line (#735)
1 parent 63aa105 commit c36a92f

20 files changed

+226
-302
lines changed

‎src/CommonUtilities/Models/NodeTask.cs

+2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ public class NodeTask
1919
public string? MetricsFilename { get; set; }
2020
public string? InputsMetricsFormat { get; set; }
2121
public string? OutputsMetricsFormat { get; set; }
22+
public List<string>? TimestampMetricsFormats { get; set; }
23+
public List<string>? BashScriptMetricsFormats { get; set; }
2224
public RuntimeOptions RuntimeOptions { get; set; } = null!;
2325
}
2426

‎src/Tes.Runner.Test/Commands/CommandLauncherTests.cs

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using Moq;
55
using Tes.Runner.Exceptions;
6+
using Tes.RunnerCLI;
67
using Tes.RunnerCLI.Commands;
78

89
namespace Tes.Runner.Test.Commands

‎src/Tes.Runner.Test/Docker/DockerExecutorTests.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ public override FileInfo GetSharedFile(string name)
157157
return file;
158158
}
159159

160-
public override Task NodeCleanupAsync()
160+
public override Task NodeCleanupPreviousTasksAsync()
161161
{
162162
throw new NotSupportedException();
163163
}

‎src/Tes.Runner/Executor.cs

+12-1
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,18 @@ private async ValueTask AppendMetrics(string? metricsFormat, long bytesTransferr
9595
{
9696
if (!string.IsNullOrWhiteSpace(tesNodeTask.MetricsFilename) && !string.IsNullOrWhiteSpace(metricsFormat))
9797
{
98-
await new MetricsFormatter(tesNodeTask.MetricsFilename, metricsFormat).Write(bytesTransferred);
98+
await new MetricsFormatter(tesNodeTask.MetricsFilename, metricsFormat).WriteSize(bytesTransferred);
99+
}
100+
}
101+
102+
public async ValueTask AppendMetrics()
103+
{
104+
foreach (var bashMetric in tesNodeTask.BashScriptMetricsFormats ?? [])
105+
{
106+
if (!string.IsNullOrWhiteSpace(tesNodeTask.MetricsFilename) && !string.IsNullOrWhiteSpace(bashMetric))
107+
{
108+
await new MetricsFormatter(tesNodeTask.MetricsFilename, bashMetric).WriteWithBash();
109+
}
99110
}
100111
}
101112

‎src/Tes.Runner/Host/AzureBatchRunnerHost.cs

+6-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public override FileInfo GetSharedFile(string path)
3030
return new(fullPath);
3131
}
3232

33-
public override Task NodeCleanupAsync()
33+
public override Task NodeCleanupPreviousTasksAsync()
3434
{
3535
var rootDir = Environment.GetEnvironmentVariable(NodeRootDir) ?? throw new InvalidOperationException("Root node directory is unknown.");
3636
var taskDir = Environment.GetEnvironmentVariable(NodeTaskDir) ?? throw new InvalidOperationException("Task directory is unknown.");
@@ -54,5 +54,10 @@ public override Task NodeCleanupAsync()
5454

5555
return Task.CompletedTask;
5656
}
57+
58+
//public override void WriteMetric(string key, string value)
59+
//{
60+
// throw new NotImplementedException();
61+
//}
5762
}
5863
}

‎src/Tes.Runner/Host/RunnerHost.cs

+21-10
Original file line numberDiff line numberDiff line change
@@ -8,31 +8,35 @@ namespace Tes.Runner.Host
88
public interface IRunnerHost
99
{
1010
/// <summary>
11-
/// tbd
11+
/// Metadata of shared metadata file.
1212
/// </summary>
1313
/// <param name="name"></param>
1414
/// <returns></returns>
1515
FileInfo GetSharedFile(string name);
1616

1717
/// <summary>
18-
/// tbd
18+
/// Saves content of shared metadata file.
1919
/// </summary>
20-
/// <param name="name"></param>
21-
/// <param name="content"></param>
20+
/// <param name="name">File name.</param>
21+
/// <param name="content">File content.</param>
2222
void WriteSharedFile(string name, ReadOnlySpan<byte> content);
2323

2424
/// <summary>
25-
/// tbd
25+
/// Loads content of shared metadata file.
2626
/// </summary>
27-
/// <param name="name"></param>
28-
/// <returns></returns>
27+
/// <param name="name">File name.</param>
28+
/// <returns>File content.</returns>
2929
IMemoryOwner<byte>? ReadSharedFile(string name);
3030

3131
/// <summary>
32-
/// tbd
32+
/// Ensure previous tasks working and metadata directories are removed.
3333
/// </summary>
3434
/// <returns></returns>
35-
Task NodeCleanupAsync();
35+
Task NodeCleanupPreviousTasksAsync();
36+
37+
//void WriteMetric(string key);
38+
39+
//void WriteMetric(string key, string value);
3640
}
3741

3842
internal abstract class RunnerHost : IRunnerHost
@@ -41,7 +45,14 @@ internal abstract class RunnerHost : IRunnerHost
4145
public abstract FileInfo GetSharedFile(string name);
4246

4347
/// <inheritdoc/>
44-
public abstract Task NodeCleanupAsync();
48+
public abstract Task NodeCleanupPreviousTasksAsync();
49+
50+
///// <inheritdoc/>
51+
//public abstract void WriteMetric(string key, string value);
52+
53+
///// <inheritdoc/>
54+
//public void WriteMetric(string key)
55+
// => WriteMetric(key, DateTimeOffset.UtcNow.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'sszzz", System.Globalization.CultureInfo.InvariantCulture));
4556

4657
/// <inheritdoc/>
4758
IMemoryOwner<byte>? IRunnerHost.ReadSharedFile(string name)

‎src/Tes.Runner/Transfer/MetricsFormatter.cs

+20-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ namespace Tes.Runner
66
internal class MetricsFormatter
77
{
88
public const string FileSizeToken = "Size";
9+
public const string TimestampToken = "Time";
910

1011
private readonly string _metricsFile;
1112
private readonly string _fileLogFormat;
@@ -19,10 +20,28 @@ public MetricsFormatter(string metricsFile, string fileLogFormat)
1920
_fileLogFormat = fileLogFormat;
2021
}
2122

22-
public Task Write(long bytesTransfered)
23+
public Task WriteSize(long bytesTransfered)
2324
{
2425
var text = _fileLogFormat.Replace($"{{{FileSizeToken}}}", $"{bytesTransfered:d1}");
2526
return File.AppendAllTextAsync(_metricsFile, text + Environment.NewLine);
2627
}
28+
29+
public Task WriteTime()
30+
{
31+
var text = _fileLogFormat.Replace($"{{{TimestampToken}}}", DateTime.UtcNow.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'sszzz", System.Globalization.CultureInfo.InvariantCulture));
32+
return File.AppendAllTextAsync(_metricsFile, text + Environment.NewLine);
33+
}
34+
35+
public async Task WriteWithBash()
36+
{
37+
var process = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo("bash", ["-c", $"({_fileLogFormat}) >> {_metricsFile}"]) { UseShellExecute = true });
38+
39+
await (process?.WaitForExitAsync() ?? Task.CompletedTask);
40+
41+
if (process?.ExitCode != 0)
42+
{
43+
throw new Exception("Failed to obtain or write metric.");
44+
}
45+
}
2746
}
2847
}

‎src/Tes.RunnerCLI/Commands/CommandHandlers.cs

+6-1
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,11 @@ internal static async Task<int> ExecuteRootCommandAsync(
5454

5555
await ExecuteAllOperationsAsSubProcessesAsync(nodeTask, file, blockSize, writers, readers, bufferCapacity, apiVersion, dockerUri);
5656

57+
{
58+
await using var executor = await Executor.CreateExecutorAsync(nodeTask, apiVersion);
59+
await executor.AppendMetrics();
60+
}
61+
5762
await eventsPublisher.PublishTaskCompletionEventAsync(nodeTask, duration.Elapsed,
5863
EventsPublisher.SuccessStatus, errorMessage: string.Empty);
5964
}
@@ -87,7 +92,7 @@ await eventsPublisher.PublishTaskCompletionEventAsync(nodeTask, duration.Elapsed
8792
private static async Task RootCommandNodeCleanupAsync(Runner.Models.NodeTask nodeTask, Uri dockerUri)
8893
{
8994
await new DockerExecutor(dockerUri).NodeCleanupAsync(new(nodeTask.ImageName, nodeTask.ImageTag, default, default, default, new()));
90-
await Executor.RunnerHost.NodeCleanupAsync();
95+
await Executor.RunnerHost.NodeCleanupPreviousTasksAsync();
9196
}
9297

9398
/// <summary>

‎src/Tes.RunnerCLI/Commands/ProcessExitCode.cs

-26
This file was deleted.

‎src/Tes.RunnerCLI/ProcessExitCode.cs

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace Tes.RunnerCLI
5+
{
6+
public enum ProcessExitCode
7+
{
8+
Success = 0,
9+
UncategorizedError = 10,
10+
IdentityUnavailable = 30,
11+
12+
13+
/*
14+
consider going down from 88 towards 78 (not inclusive) for "general" codes
15+
16+
-- section boundaries (based on nice even base-2 boundaries) --
17+
18+
88
19+
20+
96
21+
22+
112
23+
24+
120
25+
26+
----------
27+
reserved (these can be used as appropriate, but only when there's no need to disambiguate)
28+
---
29+
long-standing exit status convention for normal termination, i.e. not by signal:
30+
31+
Exit status 0: success
32+
Exit status 1: "failure", as defined by the program
33+
Exit status 2: command line usage error
34+
35+
there are common conventions when a program starts another:
36+
37+
126 if the other program was found but could not be run
38+
127 if the other program could not be found
39+
128 plus the signal number if the process terminated abnormally (note: POSIX only specifies "greater than 128" and that kill -l exitstatus can be used to find the signal's short name)
40+
41+
Most binaries (especially shells) seem to use 1-63 based on disordered spot-checking of man pages
42+
43+
64-78 sysexits (search for sysexits.h)
44+
126-128 conventions for subprocess handling
45+
129-255 exits due to "unhandled" signals
46+
numbers above 255 are not possible in Linux (and will be wrapped).
47+
*/
48+
49+
// TODO Implement all process exit codes
50+
//AzureStorageNotResponding = 80,
51+
//AzureStorageNotAuthorizedManagedIdentityCredential = 81,
52+
//AzureStorageNotAuthorizedDefaultCredential = 82,
53+
//AzureStorageNotAuthorizedInvalidSASToken = 83,
54+
//AzureStorageNotAuthorizedSASTokenExpired = 84,
55+
//TerraAPINotResponding = 90,
56+
//TerraAPINotAuthorizedManagedIdentityCredential = 91,
57+
//TerraAPINotAuthorizedDefaultCredential = 92,
58+
//TerraAPIBadRequest = 93
59+
}
60+
}

‎src/Tes.RunnerCLI/Program.cs

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Licensed under the MIT License.
33

44
using System.CommandLine;
5+
using Tes.RunnerCLI;
56
using Tes.RunnerCLI.Commands;
67

78
return await StartUpAsync(args);

‎src/TesApi.Tests/BatchNodeScriptBuilderTests.cs

-65
This file was deleted.

‎src/TesApi.Tests/Runner/TaskExecutionScriptingManagerTests.cs

+6-6
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Licensed under the MIT License.
33

44
using System;
5+
using System.Collections.Generic;
56
using System.IO;
67
using System.Threading;
78
using System.Threading.Tasks;
@@ -60,12 +61,11 @@ public void SetUp()
6061
[TestMethod]
6162
public void ParseBatchRunCommand_ReturnsRunCommandWithScriptDownloadAndExecution()
6263
{
63-
var scriptName = "batch_script";
64-
var scriptUrl = $"https://foo.bar/{scriptName}";
64+
var scriptName = "node_task";
6565
var nodeTaskUrl = $"https://foo.bar/{scriptName}";
66-
var scriptAssets = new BatchScriptAssetsInfo(new(scriptUrl), new(nodeTaskUrl), scriptName);
66+
var scriptAssets = new BatchScriptAssetsInfo(new(nodeTaskUrl), new Dictionary<string, string>().AsReadOnly());
6767

68-
var expectedCommand = $"/bin/bash -c \"wget {WgetOptions} -O ${BatchNodeScriptBuilder.BatchTaskDirEnvVarName}/{scriptName} '{scriptUrl}' && chmod +x ${BatchNodeScriptBuilder.BatchTaskDirEnvVarName}/{scriptName} && ${BatchNodeScriptBuilder.BatchTaskDirEnvVarName}/{scriptName}\"";
68+
var expectedCommand = $"/usr/bin/env -S \"{BatchScheduler.BatchNodeSharedEnvVar}/{BatchScheduler.NodeTaskRunnerFilename} -i '{nodeTaskUrl}'\"";
6969

7070
var runCommand = taskExecutionScriptingManager.ParseBatchRunCommand(scriptAssets);
7171

@@ -87,8 +87,8 @@ public async Task PrepareBatchScript_ValidTask_CreatesAndUploadsExpectedAssetsTo
8787
It.IsAny<NodeTaskConversionOptions>(), It.IsAny<CancellationToken>()),
8888
Times.Once);
8989

90-
//it should upload three assets/blobs: server task, node task definition and script
91-
storageAccessProviderMock.Verify(p => p.UploadBlobAsync(It.IsAny<Uri>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Exactly(3));
90+
//it should upload two assets/blobs: server task and node task definition
91+
storageAccessProviderMock.Verify(p => p.UploadBlobAsync(It.IsAny<Uri>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Exactly(2));
9292
}
9393

9494
private static TesTask GetTestTesTask()

‎src/TesApi.Tests/TestServices/TestServiceProvider.cs

-2
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,6 @@ internal TestServiceProvider(
8484
.AddTransient<ILogger<PriceApiBatchSkuInformationProvider>>(_ => NullLogger<PriceApiBatchSkuInformationProvider>.Instance)
8585
.AddTransient<ILogger<TaskToNodeTaskConverter>>(_ => NullLogger<TaskToNodeTaskConverter>.Instance)
8686
.AddTransient<ILogger<TaskExecutionScriptingManager>>(_ => NullLogger<TaskExecutionScriptingManager>.Instance)
87-
.AddTransient<ILogger<BatchNodeScriptBuilder>>(_ => NullLogger<BatchNodeScriptBuilder>.Instance)
8887
.AddTransient<ILogger<CachingWithRetriesAzureProxy>>(_ => NullLogger<CachingWithRetriesAzureProxy>.Instance)
8988
.AddSingleton<CachingRetryPolicyBuilder>()
9089
.AddTransient<IActionIdentityProvider, DefaultActionIdentityProvider>()
@@ -96,7 +95,6 @@ internal TestServiceProvider(
9695
.AddSingleton<IBatchQuotaVerifier, BatchQuotaVerifier>()
9796
.AddSingleton<TaskToNodeTaskConverter>()
9897
.AddSingleton<TaskExecutionScriptingManager>()
99-
.AddSingleton<BatchNodeScriptBuilder>()
10098
.IfThenElse(additionalActions is null, s => { }, s => additionalActions(s))
10199
.BuildServiceProvider();
102100

0 commit comments

Comments
 (0)