Skip to content

Commit 27da88b

Browse files
giventocodeBMurri
giventocode
and
BMurri
authored
[245] Wildcard support and refactoring of runner task outputs and inputs expansion logic (#261)
* operation resolver * renamed fullfilename to path inputs and outputs * optimizer uses expanded outputs. * path prefix * ref json updated * formattting * Test formatting * doc update * file provider tests, root directory handling. * documentation update * removed commented out code. --------- Co-authored-by: Blair L Murri <BMurri@users.noreply.github.com>
1 parent 1f42bdc commit 27da88b

22 files changed

+710
-194
lines changed

‎src/CommonUtilities/Models/NodeTask.cs

+3-3
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,16 @@ public class NodeTask
1919

2020
public class FileOutput
2121
{
22-
public string? FullFileName { get; set; }
22+
public string? Path { get; set; }
2323
public string? TargetUrl { get; set; }
2424
public SasResolutionStrategy? SasStrategy { get; set; }
2525
public FileType? FileType { get; set; }
26-
public bool? Required { get; set; }
26+
public string? PathPrefix { get; set; }
2727
}
2828

2929
public class FileInput
3030
{
31-
public string? FullFileName { get; set; }
31+
public string? Path { get; set; }
3232
public string? SourceUrl { get; set; }
3333
public SasResolutionStrategy? SasStrategy { get; set; }
3434
}

‎src/Tes.Runner.Test/ResolutionPolicyHandlerTests.cs

+6-8
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT License.
33

4-
using Moq;
54
using Tes.Runner.Models;
65
using Tes.Runner.Storage;
76

@@ -19,7 +18,6 @@ public class ResolutionPolicyHandlerTests
1918
public void SetUp()
2019
{
2120
resolutionPolicyHandler = new ResolutionPolicyHandler();
22-
new Mock<ISasResolutionStrategy>();
2321
}
2422

2523
[TestMethod]
@@ -44,9 +42,9 @@ public async Task ApplyResolutionPolicyAsync_WhenTestTaskOutputsIsNotEmpty_Retur
4442
{
4543
var testTaskOutputs = new List<FileOutput>
4644
{
47-
new FileOutput(){FullFileName = "file", TargetUrl = "http://foo.bar", SasStrategy = SasResolutionStrategy.None, Required = true},
48-
new FileOutput(){FullFileName = "file1", TargetUrl = "http://foo1.bar", SasStrategy = SasResolutionStrategy.None, Required = true},
49-
new FileOutput(){FullFileName = "file2", TargetUrl = "http://foo2.bar", SasStrategy = SasResolutionStrategy.None, Required = true}
45+
new FileOutput(){Path = "file", TargetUrl = "http://foo.bar", SasStrategy = SasResolutionStrategy.None},
46+
new FileOutput(){Path = "file1", TargetUrl = "http://foo1.bar", SasStrategy = SasResolutionStrategy.None},
47+
new FileOutput(){Path = "file2", TargetUrl = "http://foo2.bar", SasStrategy = SasResolutionStrategy.None}
5048
};
5149
var result = await resolutionPolicyHandler.ApplyResolutionPolicyAsync(testTaskOutputs);
5250
Assert.IsNotNull(result);
@@ -74,9 +72,9 @@ public async Task ApplyResolutionPolicyAsync_WhenTestTaskInputsIsNotEmpty_Return
7472
{
7573
var testTaskInputs = new List<FileInput>
7674
{
77-
new FileInput(){FullFileName = "file", SourceUrl = "http://foo.bar", SasStrategy = SasResolutionStrategy.None},
78-
new FileInput(){FullFileName = "file1", SourceUrl = "http://foo1.bar", SasStrategy = SasResolutionStrategy.None},
79-
new FileInput(){FullFileName = "file2", SourceUrl = "http://foo2.bar", SasStrategy = SasResolutionStrategy.None}
75+
new FileInput(){Path = "file", SourceUrl = "http://foo.bar", SasStrategy = SasResolutionStrategy.None},
76+
new FileInput(){Path = "file1", SourceUrl = "http://foo1.bar", SasStrategy = SasResolutionStrategy.None},
77+
new FileInput(){Path = "file2", SourceUrl = "http://foo2.bar", SasStrategy = SasResolutionStrategy.None}
8078
};
8179
var result = await resolutionPolicyHandler.ApplyResolutionPolicyAsync(testTaskInputs);
8280
Assert.IsNotNull(result);

‎src/Tes.Runner.Test/RunnerTestUtils.cs

+24-5
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,30 @@ public static async Task<string> CreateTempFileAsync()
2020
return file;
2121
}
2222

23+
public static DirectoryInfo CreateTempFilesInDirectory(string dirStructure, string filePrefix)
24+
{
25+
var parentDirName = Guid.NewGuid().ToString();
26+
var root = Directory.CreateDirectory($"{parentDirName}/{dirStructure}");
27+
28+
while (root.Parent != null)
29+
{
30+
var fileName = $"{root.FullName}/{filePrefix}{Guid.NewGuid()}.tmp";
31+
32+
using var fs = File.Create(fileName);
33+
34+
fs.Close();
35+
36+
if (root.Name.Equals(parentDirName, StringComparison.InvariantCultureIgnoreCase))
37+
{
38+
return root;
39+
}
40+
41+
root = root.Parent;
42+
}
43+
44+
throw new Exception("Could not find root directory");
45+
}
46+
2347
public static string CalculateMd5(string file)
2448
{
2549
using var md5 = MD5.Create();
@@ -52,11 +76,6 @@ public static async Task<List<T>> ReadAllPipelineBuffersAsync<T>(IAsyncEnumerabl
5276
}
5377
return pipelineBuffers;
5478
}
55-
public static string AddRandomDataAndReturnMd5(byte[] data)
56-
{
57-
Random.NextBytes(data);
58-
return CalculateMd5Hash(data);
59-
}
6079

6180
public static string CalculateMd5Hash(byte[] data)
6281
{

‎src/Tes.Runner.Test/Storage/CloudProviderSchemeConverterTests.cs

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,26 @@
11
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT License.
33

4-
namespace Tes.Runner.Storage.Tests
4+
using Tes.Runner.Storage;
5+
6+
namespace Tes.Runner.Test.Storage
57
{
6-
[TestClass()]
8+
[TestClass]
9+
[TestCategory("Unit")]
710
public class CloudProviderSchemeConverterTests
811
{
912
[DataTestMethod]
1013
[DataRow(@"https://blob.name", @"https://blob.name")]
1114
[DataRow(@"s3://broad-references/hg38/v0/Homo_sapiens_assembly38.dict", @"https://broad-references.s3.amazonaws.com/hg38/v0/Homo_sapiens_assembly38.dict")]
1215
[DataRow(@"gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/ENCSR356KRQ/fastq_subsampled/rep2/pair1/ENCFF641SFZ.subsampled.400.fastq.gz", @"https://storage.googleapis.com/encode-pipeline-test-samples/encode-atac-seq-pipeline/ENCSR356KRQ/fastq_subsampled/rep2/pair1/ENCFF641SFZ.subsampled.400.fastq.gz")]
13-
public async Task CreateSasTokenWithStrategyAsyncTest_URIProvided_MatchesExpected(string sourceURI, string expectedURI)
16+
public async Task CreateSasTokenWithStrategyAsyncTest_URIProvided_MatchesExpected(string sourceUri, string expectedUri)
1417
{
1518
var provider = new CloudProviderSchemeConverter();
1619

17-
var result = await provider.CreateSasTokenWithStrategyAsync(sourceURI);
20+
var result = await provider.CreateSasTokenWithStrategyAsync(sourceUri);
1821

1922
Assert.IsNotNull(result);
20-
Assert.AreEqual(result.AbsoluteUri, new Uri(expectedURI).AbsoluteUri);
23+
Assert.AreEqual(result.AbsoluteUri, new Uri(expectedUri).AbsoluteUri);
2124
}
2225
}
2326
}

‎src/Tes.Runner.Test/Storage/FileOperationResolverTests.cs

+185
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using Moq;
5+
using Tes.Runner.Models;
6+
using Tes.Runner.Storage;
7+
using Tes.Runner.Transfer;
8+
9+
namespace Tes.Runner.Test.Storage
10+
{
11+
[TestClass]
12+
[TestCategory("Unit")]
13+
public class FileOperationResolverTests
14+
{
15+
private ResolutionPolicyHandler resolutionPolicyHandler = null!;
16+
private Mock<IFileInfoProvider> fileInfoProvider = null!;
17+
private FileOutput singleFileOutput = null!;
18+
private FileOutput directoryFileOutput = null!;
19+
private FileOutput patternFileOutput = null!;
20+
21+
private FileInput singleFileInput = null!;
22+
23+
[TestInitialize]
24+
public void SetUp()
25+
{
26+
resolutionPolicyHandler = new ResolutionPolicyHandler();
27+
fileInfoProvider = new Mock<IFileInfoProvider>();
28+
29+
fileInfoProvider.Setup(x => x.GetExpandedFileName(It.IsAny<string>())).Returns<string>(x => x);
30+
fileInfoProvider.Setup(x => x.FileExists(It.IsAny<string>())).Returns(true);
31+
32+
singleFileInput = new FileInput
33+
{
34+
Path = "/foo/bar",
35+
SourceUrl = "https://foo.bar/cont/foo/bar?sig=sasToken",
36+
SasStrategy = SasResolutionStrategy.None,
37+
};
38+
39+
singleFileOutput = new FileOutput
40+
{
41+
Path = "/foo/bar",
42+
TargetUrl = "https://foo.bar/cont/foo/bar?sig=sasToken",
43+
SasStrategy = SasResolutionStrategy.None,
44+
FileType = FileType.File
45+
};
46+
47+
directoryFileOutput = new FileOutput
48+
{
49+
Path = "/root",
50+
TargetUrl = "https://foo.bar/cont?sig=sasToken",
51+
SasStrategy = SasResolutionStrategy.None,
52+
FileType = FileType.Directory
53+
};
54+
55+
patternFileOutput = new FileOutput
56+
{
57+
Path = "/data/*.foo",
58+
TargetUrl = "https://foo.bar/cont?sig=sasToken",
59+
SasStrategy = SasResolutionStrategy.None,
60+
PathPrefix = "/prefix",
61+
FileType = FileType.File
62+
};
63+
}
64+
65+
[TestMethod]
66+
public async Task ResolveOutputsAsync_FileOutputProvided_FileOperationIsResolved()
67+
{
68+
var nodeTask = new NodeTask()
69+
{
70+
Outputs = new List<FileOutput>
71+
{
72+
singleFileOutput
73+
}
74+
};
75+
76+
var fileOperationInfoResolver = new FileOperationResolver(nodeTask, resolutionPolicyHandler, fileInfoProvider.Object);
77+
var resolvedOutputs = await fileOperationInfoResolver.ResolveOutputsAsync();
78+
79+
Assert.AreEqual(1, resolvedOutputs?.Count);
80+
Assert.IsTrue(resolvedOutputs?.Any(r => r.FullFilePath.Equals(singleFileOutput.Path, StringComparison.InvariantCultureIgnoreCase)));
81+
Assert.IsTrue(resolvedOutputs?.Any(r => r.TargetUri.ToString().Equals(singleFileOutput.TargetUrl, StringComparison.InvariantCultureIgnoreCase)));
82+
}
83+
84+
[TestMethod]
85+
public async Task ResolveOutputsAsync_PatternOutputProvided_FileOperationsAreResolved()
86+
{
87+
var nodeTask = new NodeTask()
88+
{
89+
Outputs = new List<FileOutput>
90+
{
91+
patternFileOutput
92+
}
93+
};
94+
95+
fileInfoProvider.Setup(x => x.GetFilesBySearchPattern(It.IsAny<string>(), It.IsAny<string>()))
96+
.Returns(new List<string> { "/prefix/data/foo.foo", "/prefix/data/bar.foo" }.ToArray);
97+
98+
var fileOperationInfoResolver = new FileOperationResolver(nodeTask, resolutionPolicyHandler, fileInfoProvider.Object);
99+
var resolvedOutputs = await fileOperationInfoResolver.ResolveOutputsAsync();
100+
101+
Assert.AreEqual(2, resolvedOutputs?.Count);
102+
Assert.IsTrue(resolvedOutputs!.Any(r => r.FullFilePath.Equals("/prefix/data/foo.foo", StringComparison.OrdinalIgnoreCase)));
103+
Assert.IsTrue(resolvedOutputs!.Any(r => r.FullFilePath.Equals("/prefix/data/bar.foo", StringComparison.OrdinalIgnoreCase)));
104+
Assert.IsTrue(resolvedOutputs!.Any(r => r.TargetUri.ToString().Equals(@"https://foo.bar/cont/data/foo.foo?sig=sasToken", StringComparison.OrdinalIgnoreCase)));
105+
Assert.IsTrue(resolvedOutputs!.Any(r => r.TargetUri.ToString().Equals(@"https://foo.bar/cont/data/bar.foo?sig=sasToken", StringComparison.OrdinalIgnoreCase)));
106+
}
107+
108+
[TestMethod]
109+
public async Task ResolveOutputsAsync_DirectoryOutputProvided_TargetUrlDoesNotContainRootDirInPathProperty()
110+
{
111+
var nodeTask = new NodeTask()
112+
{
113+
Outputs = new List<FileOutput>
114+
{
115+
directoryFileOutput
116+
}
117+
};
118+
var rootDir = directoryFileOutput.Path;
119+
var file1 = "/dir1/file1.tmp";
120+
var file2 = "/dir1/dir2/file2.tmp";
121+
122+
fileInfoProvider.Setup(x => x.GetAllFilesInDirectory(It.IsAny<string>()))
123+
.Returns(new List<string> { $"{rootDir}{file1}", $"{rootDir}{file2}" }.ToArray);
124+
125+
var fileOperationInfoResolver = new FileOperationResolver(nodeTask, resolutionPolicyHandler, fileInfoProvider.Object);
126+
var resolvedOutputs = await fileOperationInfoResolver.ResolveOutputsAsync();
127+
128+
129+
Assert.AreEqual(2, resolvedOutputs?.Count);
130+
Assert.IsTrue(resolvedOutputs!.Any(r => r.FullFilePath.Equals($"{rootDir}{file1}", StringComparison.OrdinalIgnoreCase)));
131+
Assert.IsTrue(resolvedOutputs!.Any(r => r.FullFilePath.Equals($"{rootDir}{file2}", StringComparison.OrdinalIgnoreCase)));
132+
Assert.IsTrue(resolvedOutputs!.Any(r => r.TargetUri.AbsoluteUri.ToString().Equals($@"https://foo.bar/cont{file1}?sig=sasToken", StringComparison.OrdinalIgnoreCase)));
133+
Assert.IsTrue(resolvedOutputs!.Any(r => r.TargetUri.AbsoluteUri.ToString().Equals($@"https://foo.bar/cont{file2}?sig=sasToken", StringComparison.OrdinalIgnoreCase)));
134+
}
135+
136+
[TestMethod]
137+
public async Task ResolveOutputsAsync_PatternAndFileOutputProvided_FileOperationsAreResolved()
138+
{
139+
var nodeTask = new NodeTask()
140+
{
141+
Outputs = new List<FileOutput>
142+
{
143+
patternFileOutput,
144+
singleFileOutput
145+
}
146+
};
147+
148+
fileInfoProvider.Setup(x => x.GetFilesBySearchPattern(It.IsAny<string>(), It.IsAny<string>()))
149+
.Returns(new List<string> { "/prefix/data/foo.foo", "/prefix/data/bar.foo" }.ToArray);
150+
151+
var fileOperationInfoResolver = new FileOperationResolver(nodeTask, resolutionPolicyHandler, fileInfoProvider.Object);
152+
var resolvedOutputs = await fileOperationInfoResolver.ResolveOutputsAsync();
153+
154+
Assert.AreEqual(3, resolvedOutputs?.Count);
155+
Assert.IsTrue(resolvedOutputs!.Any(r => r.FullFilePath.Equals("/prefix/data/foo.foo", StringComparison.OrdinalIgnoreCase)));
156+
Assert.IsTrue(resolvedOutputs!.Any(r => r.FullFilePath.Equals("/prefix/data/bar.foo", StringComparison.OrdinalIgnoreCase)));
157+
Assert.IsTrue(resolvedOutputs!.Any(r => r.FullFilePath.Equals("/prefix/data/bar.foo", StringComparison.OrdinalIgnoreCase)));
158+
Assert.IsTrue(resolvedOutputs!.Any(r => r.TargetUri.ToString().Equals(@"https://foo.bar/cont/data/foo.foo?sig=sasToken", StringComparison.OrdinalIgnoreCase)));
159+
Assert.IsTrue(resolvedOutputs!.Any(r => r.TargetUri.ToString().Equals(@"https://foo.bar/cont/data/bar.foo?sig=sasToken", StringComparison.OrdinalIgnoreCase)));
160+
Assert.IsTrue(resolvedOutputs?.Any(r => r.FullFilePath.Equals(singleFileOutput.Path, StringComparison.InvariantCultureIgnoreCase)));
161+
Assert.IsTrue(resolvedOutputs?.Any(r => r.TargetUri.ToString().Equals(singleFileOutput.TargetUrl, StringComparison.InvariantCultureIgnoreCase)));
162+
}
163+
164+
[TestMethod]
165+
public async Task ResolveInputsAsync_FileInputProvided_FileOperationsAreResolved()
166+
{
167+
var nodeTask = new NodeTask()
168+
{
169+
Inputs = new List<FileInput>
170+
{
171+
singleFileInput
172+
}
173+
};
174+
175+
var fileOperationInfoResolver = new FileOperationResolver(nodeTask, resolutionPolicyHandler, fileInfoProvider.Object);
176+
177+
var resolvedInputs = await fileOperationInfoResolver.ResolveInputsAsync();
178+
179+
Assert.AreEqual(1, resolvedInputs?.Count);
180+
Assert.IsTrue(resolvedInputs?.Any(r => r.FullFilePath.Equals(singleFileInput.Path, StringComparison.InvariantCultureIgnoreCase)));
181+
Assert.IsTrue(resolvedInputs?.Any(r => r.SourceUrl.ToString().Equals(singleFileInput.SourceUrl, StringComparison.InvariantCultureIgnoreCase)));
182+
}
183+
184+
}
185+
}

0 commit comments

Comments
 (0)