[275] Removes default storage account dependency when running in Terra (

#285) * initial changes * removed dependency on default storage account when running on terra * additional tests * removed commented code. * default storage provider tests and fixes * return null instead of throwing when blob doessn't exist * test refactoring, download operation does not eat exception * fixed filename * removed unnecessary return value for blob upload function * added app id property * use batchscheduling prefix in internal TES paths for Terra * removed app id from the Terra options * terra options
microsoft · Jul 14, 2023 · 0f5f4c6 · 0f5f4c6
1 parent 8693d86
commit 0f5f4c6
Show file tree

Hide file tree

Showing 12 changed files with 381 additions and 59 deletions.
diff --git a/src/TesApi.Tests/BatchSchedulerTests.cs b/src/TesApi.Tests/BatchSchedulerTests.cs
@@ -7,6 +7,7 @@
 using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Azure.Batch;
 using Microsoft.Azure.Batch.Common;
@@ -1488,6 +1489,9 @@ private static Action<Mock<ContainerRegistryProvider>> GetContainerRegistryInfoP
         private static Action<Mock<IAzureProxy>> GetMockAzureProxy(AzureProxyReturnValues azureProxyReturnValues)
             => azureProxy =>
             {
+                azureProxy.Setup(a => a.BlobExistsAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()))
+                    .ReturnsAsync(true);
+
                 azureProxy.Setup(a => a.GetActivePoolsAsync(It.IsAny<string>()))
                     .Returns(AsyncEnumerable.Empty<CloudPool>());
 

diff --git a/src/TesApi.Tests/ConfigurationUtilsTests.cs b/src/TesApi.Tests/ConfigurationUtilsTests.cs
@@ -104,6 +104,7 @@ private static void PrepareAzureProxy(Mock<IAzureProxy> azureProxy)
                 }
              };
 
+            azureProxy.Setup(a => a.BlobExistsAsync(It.IsAny<Uri>(), It.IsAny<System.Threading.CancellationToken>())).Returns(Task.FromResult(true));
             azureProxy.Setup(a => a.DownloadBlobAsync(It.IsAny<Uri>(), It.IsAny<System.Threading.CancellationToken>())).Returns(Task.FromResult(allowedVmSizesFileContent));
             azureProxy.Setup(a => a.GetStorageAccountInfoAsync("defaultstorageaccount", It.IsAny<System.Threading.CancellationToken>())).Returns(Task.FromResult(storageAccountInfos["defaultstorageaccount"]));
             azureProxy.Setup(a => a.GetStorageAccountKeyAsync(It.IsAny<StorageAccountInfo>(), It.IsAny<System.Threading.CancellationToken>())).Returns(Task.FromResult("Key1"));

diff --git a/src/TesApi.Tests/Storage/DefaultStorageAccessProviderTests.cs b/src/TesApi.Tests/Storage/DefaultStorageAccessProviderTests.cs
@@ -0,0 +1,103 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+using Tes.Models;
+using TesApi.Web;
+using TesApi.Web.Options;
+using TesApi.Web.Storage;
+
+namespace TesApi.Tests.Storage
+{
+    [TestClass, TestCategory("Unit")]
+    public class DefaultStorageAccessProviderTests
+    {
+        private DefaultStorageAccessProvider defaultStorageAccessProvider;
+        private Mock<IAzureProxy> azureProxyMock;
+        private StorageOptions storageOptions;
+        private StorageAccountInfo storageAccountInfo;
+        private const string DefaultStorageAccountName = "defaultstorage";
+        private const string StorageAccountBlobEndpoint = $"https://{DefaultStorageAccountName}.blob.windows.net";
+
+        [TestInitialize]
+        public void Setup()
+        {
+            azureProxyMock = new Mock<IAzureProxy>();
+            storageOptions = new StorageOptions() { DefaultAccountName = DefaultStorageAccountName };
+            var subscriptionId = Guid.NewGuid().ToString();
+            storageAccountInfo = new StorageAccountInfo()
+            {
+                BlobEndpoint = StorageAccountBlobEndpoint,
+                Name = DefaultStorageAccountName,
+                Id = $"/subscriptions/{subscriptionId}/resourceGroups/mrg/providers/Microsoft.Storage/storageAccounts/{DefaultStorageAccountName}",
+                SubscriptionId = subscriptionId
+            };
+            azureProxyMock.Setup(p => p.GetStorageAccountKeyAsync(It.IsAny<StorageAccountInfo>(), It.IsAny<CancellationToken>())).ReturnsAsync(GenerateRandomTestAzureStorageKey());
+            azureProxyMock.Setup(p => p.GetStorageAccountInfoAsync(It.Is<string>(s => s.Equals(DefaultStorageAccountName)), It.IsAny<CancellationToken>())).ReturnsAsync(storageAccountInfo);
+            defaultStorageAccessProvider = new DefaultStorageAccessProvider(NullLogger<DefaultStorageAccessProvider>.Instance, Options.Create(storageOptions), azureProxyMock.Object);
+        }
+
+        [TestMethod]
+        [DataRow("script/foo.sh")]
+        [DataRow("/script/foo.sh")]
+        public async Task GetInternalTesTaskBlobUrlAsync_BlobPathIsProvided_ReturnsValidURLWithDefaultStorageAccountTesInternalContainerAndTaskId(
+            string blobName)
+        {
+            var task = new TesTask { Name = "taskName", Id = Guid.NewGuid().ToString() };
+            var url = await defaultStorageAccessProvider.GetInternalTesTaskBlobUrlAsync(task, blobName, CancellationToken.None);
+
+            Assert.IsNotNull(url);
+            var uri = new Uri(url);
+            Assert.AreEqual($"{StorageAccountBlobEndpoint}{StorageAccessProvider.TesExecutionsPathPrefix}/{task.Id}/{blobName.TrimStart('/')}", ToHostWithAbsolutePathOnly(uri));
+        }
+
+        private static string ToHostWithAbsolutePathOnly(Uri uri)
+        {
+            return $"{uri.Scheme}://{uri.Host}{uri.AbsolutePath}";
+        }
+
+        [TestMethod]
+        [DataRow("script/foo.sh")]
+        [DataRow("/script/foo.sh")]
+        public async Task GetInternalTesTaskBlobUrlAsync_BlobPathAndInternalPathPrefixIsProvided_ReturnsValidURLWithDefaultStorageAccountAndInternalPathPrefixAppended(
+            string blobName)
+        {
+            var internalPathPrefix = "internalPathPrefix";
+
+            var task = new TesTask { Name = "taskName", Id = Guid.NewGuid().ToString() };
+            task.Resources = new TesResources();
+            task.Resources.BackendParameters = new Dictionary<string, string>
+            {
+                { TesResources.SupportedBackendParameters.internal_path_prefix.ToString(), internalPathPrefix }
+            };
+            var url = await defaultStorageAccessProvider.GetInternalTesTaskBlobUrlAsync(task, blobName, CancellationToken.None);
+
+            Assert.IsNotNull(url);
+            var uri = new Uri(url);
+            Assert.AreEqual($"{StorageAccountBlobEndpoint}/{internalPathPrefix}/{blobName.TrimStart('/')}", ToHostWithAbsolutePathOnly(uri));
+        }
+
+        private static string GenerateRandomTestAzureStorageKey()
+        {
+            const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+            var length = 64;
+            var random = new Random();
+            var result = new StringBuilder(length);
+
+            for (int i = 0; i < length; i++)
+            {
+                result.Append(chars[random.Next(chars.Length)]);
+            }
+
+            return result.ToString();
+        }
+    }
+}
diff --git a/src/TesApi.Tests/TerraApiStubData.cs b/src/TesApi.Tests/TerraApiStubData.cs
@@ -15,14 +15,17 @@ public class TerraApiStubData
     public const string WsmApiHost = "https://wsm.host";
     public const string ResourceGroup = "mrg-terra-dev-previ-20191228";
     public const string WorkspaceAccountName = "lzaccount1";
-    public const string WorkspaceStorageContainerName = "sc-ef9fed44-dba6-4825-868c-b00208522382";
     public const string SasToken = "SASTOKENSTUB=";
+    private const string WorkspaceIdValue = "41aa9346-670f-4206-8b6f-6b921a564bdd";
+
+    public const string WorkspaceStorageContainerName = $"sc-{WorkspaceIdValue}";
     public const string WsmGetSasResponseStorageUrl = $"https://{WorkspaceAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}";
 
     public Guid LandingZoneId { get; } = Guid.NewGuid();
     public Guid SubscriptionId { get; } = Guid.NewGuid();
-    public Guid WorkspaceId { get; } = Guid.NewGuid();
     public Guid ContainerResourceId { get; } = Guid.NewGuid();
+    public Guid WorkspaceId { get; } = Guid.Parse(WorkspaceIdValue);
+
     public string BatchAccountName => "lzee170c71b6cf678cfca744";
     public string Region => "westus3";
     public string BatchAccountId =>
@@ -310,7 +313,7 @@ public ApiCreateBatchPoolRequest GetApiCreateBatchPoolRequest()
             Common = new ApiCommon(),
             AzureBatchPool = new ApiAzureBatchPool()
             {
-                UserAssignedIdentities = new ApiUserAssignedIdentity[]
+                UserAssignedIdentities = new[]
                 {
                     new ApiUserAssignedIdentity()
                     {

diff --git a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs
@@ -2,16 +2,19 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
+using Tes.Models;
 using TesApi.Web;
 using TesApi.Web.Management.Clients;
 using TesApi.Web.Management.Configuration;
 using TesApi.Web.Management.Models.Terra;
+using TesApi.Web.Options;
 using TesApi.Web.Storage;
 
 namespace TesApi.Tests
@@ -29,6 +32,7 @@ public class TerraStorageAccessProviderTests
         private TerraApiStubData terraApiStubData;
         private Mock<IOptions<TerraOptions>> optionsMock;
         private TerraOptions terraOptions;
+        private BatchSchedulingOptions batchSchedulingOptions;
 
         [TestInitialize]
         public void SetUp()
@@ -37,10 +41,11 @@ public void SetUp()
             wsmApiClientMock = new Mock<TerraWsmApiClient>();
             optionsMock = new Mock<IOptions<TerraOptions>>();
             terraOptions = terraApiStubData.GetTerraOptions();
+            batchSchedulingOptions = new BatchSchedulingOptions() { Prefix = Guid.NewGuid().ToString() };
             optionsMock.Setup(o => o.Value).Returns(terraOptions);
             azureProxyMock = new Mock<IAzureProxy>();
             terraStorageAccessProvider = new TerraStorageAccessProvider(NullLogger<TerraStorageAccessProvider>.Instance,
-                optionsMock.Object, azureProxyMock.Object, wsmApiClientMock.Object);
+                optionsMock.Object, azureProxyMock.Object, wsmApiClientMock.Object, batchSchedulingOptions);
         }
 
         [TestMethod]
@@ -124,5 +129,57 @@ public async Task GetMappedSasUrlFromWsmAsync_WithOrWithOutBlobName_ReturnsValid
 
             Assert.AreEqual(uri.AbsolutePath, $"/{TerraApiStubData.WorkspaceStorageContainerName}/blobName");
         }
+
+        [TestMethod]
+        [DataRow("script/foo.sh")]
+        [DataRow("/script/foo.sh")]
+        public async Task GetInternalTesBlobUrlAsync_BlobPathIsProvided_ReturnsValidURLWithWsmContainerAndTesPrefixAppended(
+            string blobName)
+        {
+            SetUpTerraApiClient();
+
+            var url = await terraStorageAccessProvider.GetInternalTesBlobUrlAsync(blobName, CancellationToken.None);
+
+            Assert.IsNotNull(url);
+            var uri = new Uri(url);
+            Assert.AreEqual($"/{TerraApiStubData.WorkspaceStorageContainerName}/{batchSchedulingOptions.Prefix}{StorageAccessProvider.TesExecutionsPathPrefix}/{blobName.TrimStart('/')}", uri.AbsolutePath);
+        }
+
+        [TestMethod]
+        [DataRow("script/foo.sh")]
+        [DataRow("/script/foo.sh")]
+        public async Task GetInternalTesTaskBlobUrlAsync_BlobPathIsProvided_ReturnsValidURLWithWsmContainerTaskIdAndTesPrefixAppended(
+            string blobName)
+        {
+            SetUpTerraApiClient();
+            var task = new TesTask { Name = "taskName", Id = Guid.NewGuid().ToString() };
+            var url = await terraStorageAccessProvider.GetInternalTesTaskBlobUrlAsync(task, blobName, CancellationToken.None);
+
+            Assert.IsNotNull(url);
+            var uri = new Uri(url);
+            Assert.AreEqual($"/{TerraApiStubData.WorkspaceStorageContainerName}/{batchSchedulingOptions.Prefix}{StorageAccessProvider.TesExecutionsPathPrefix}/{task.Id}/{blobName.TrimStart('/')}", uri.AbsolutePath);
+        }
+
+        [TestMethod]
+        [DataRow("script/foo.sh")]
+        [DataRow("/script/foo.sh")]
+        public async Task GetInternalTesTaskBlobUrlAsync_BlobPathAndInternalPathPrefixIsProvided_ReturnsValidURLWithWsmContainerTaskIdAndInternalPathPrefixAppended(
+            string blobName)
+        {
+            var internalPathPrefix = "internalPathPrefix";
+
+            SetUpTerraApiClient();
+            var task = new TesTask { Name = "taskName", Id = Guid.NewGuid().ToString() };
+            task.Resources = new TesResources();
+            task.Resources.BackendParameters = new Dictionary<string, string>
+            {
+                { TesResources.SupportedBackendParameters.internal_path_prefix.ToString(), internalPathPrefix }
+            };
+            var url = await terraStorageAccessProvider.GetInternalTesTaskBlobUrlAsync(task, blobName, CancellationToken.None);
+
+            Assert.IsNotNull(url);
+            var uri = new Uri(url);
+            Assert.AreEqual($"/{TerraApiStubData.WorkspaceStorageContainerName}/{internalPathPrefix}/{blobName.TrimStart('/')}", uri.AbsolutePath);
+        }
     }
 }