diff --git a/src/TesApi.Tests/Integration/TerraWsmApiClientIntegrationTests.cs b/src/TesApi.Tests/Integration/TerraWsmApiClientIntegrationTests.cs new file mode 100644 index 000000000..f28f3cb79 --- /dev/null +++ b/src/TesApi.Tests/Integration/TerraWsmApiClientIntegrationTests.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TesApi.Web.Management; +using TesApi.Web.Management.Clients; +using TesApi.Web.Management.Configuration; + +namespace TesApi.Tests.Integration +{ + [TestClass, TestCategory("TerraIntegration")] + [Ignore] + public class TerraWsmApiClientIntegrationTests + { + private TerraWsmApiClient wsmApiClient = null!; + private TestTerraEnvInfo envInfo = null!; + + [TestInitialize] + public void Setup() + { + envInfo = new TestTerraEnvInfo(); + + var terraOptions = Options.Create(new TerraOptions() + { + WsmApiHost = envInfo.WsmApiHost + }); + var retryOptions = Options.Create(new RetryPolicyOptions()); + var memoryCache = new MemoryCache(new MemoryCacheOptions()); + + wsmApiClient = new TerraWsmApiClient(new TestEnvTokenCredential(), terraOptions, + new CacheAndRetryHandler(memoryCache, retryOptions), TestLoggerFactory.Create()); + + } + + [TestMethod] + public async Task GetContainerResourcesAsync_CallsUsingTheWSIdFromContainerName_ReturnsContainerInformation() + { + var workspaceId = Guid.Parse(envInfo.WorkspaceContainerName.Replace("sc-", "")); + + var results = await wsmApiClient.GetContainerResourcesAsync(workspaceId, 0, 100, CancellationToken.None); + + Assert.IsNotNull(results); + Assert.IsTrue(results.Resources.Any(i => i.ResourceAttributes.AzureStorageContainer.StorageContainerName.Equals(envInfo.WorkspaceContainerName, StringComparison.OrdinalIgnoreCase))); + } + } + + public static class TestLoggerFactory + { + private static readonly ILoggerFactory SLogFactory = LoggerFactory.Create(builder => + { + builder + .AddSystemdConsole(options => + { + options.IncludeScopes = true; + options.TimestampFormat = "yyyy-MM-dd HH:mm:ss.fff "; + options.UseUtcTimestamp = true; + }); + + builder.SetMinimumLevel(LogLevel.Trace); + }); + + + public static ILogger Create() + { + return SLogFactory.CreateLogger(); + } + } +} diff --git a/src/TesApi.Tests/Integration/TestEnvTokenCredential.cs b/src/TesApi.Tests/Integration/TestEnvTokenCredential.cs new file mode 100644 index 000000000..9236db902 --- /dev/null +++ b/src/TesApi.Tests/Integration/TestEnvTokenCredential.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; + +namespace TesApi.Tests.Integration +{ + internal class TestEnvTokenCredential : TokenCredential + { + public const string TerraTokenEnvVariableName = "TERRA_AUTH_TOKEN"; + + public override ValueTask GetTokenAsync(TokenRequestContext requestContext, CancellationToken cancellationToken) + { + var token = GetTokenFromEnvVariable(); + + return new ValueTask(new AccessToken(token, DateTimeOffset.MaxValue)); + } + + private static string GetTokenFromEnvVariable() + { + var token = Environment.GetEnvironmentVariable(TerraTokenEnvVariableName); + + if (string.IsNullOrEmpty(token)) + { + throw new InvalidOperationException($"Environment variable {TerraTokenEnvVariableName} is not set."); + } + + return token; + } + + public override AccessToken GetToken(TokenRequestContext requestContext, CancellationToken cancellationToken) + { + var token = GetTokenFromEnvVariable(); + + return new AccessToken(token, DateTimeOffset.MaxValue); + } + } +} diff --git a/src/TesApi.Tests/Integration/TestTerraEnvInfo.cs b/src/TesApi.Tests/Integration/TestTerraEnvInfo.cs new file mode 100644 index 000000000..07cc521da --- /dev/null +++ b/src/TesApi.Tests/Integration/TestTerraEnvInfo.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace TesApi.Tests.Integration +{ + internal class TestTerraEnvInfo + { + private const string LzStorageAccountNameEnvVarName = "TERRA_LZ_STORAGE_ACCOUNT"; + private const string WorkspaceContainerNameEnvVarName = "TERRA_WS_STORAGE_CONTAINER"; + private const string WsmApiHostEnvVarName = "TERRA_WSM_API_HOST"; + + private readonly string lzStorageAccountName; + private readonly string workspaceContainerName; + private readonly string wsmApiHost; + + public string LzStorageAccountName => lzStorageAccountName; + public string WorkspaceContainerName => workspaceContainerName; + public string WsmApiHost => wsmApiHost; + + public TestTerraEnvInfo() + { + lzStorageAccountName = Environment.GetEnvironmentVariable(LzStorageAccountNameEnvVarName); + workspaceContainerName = Environment.GetEnvironmentVariable(WorkspaceContainerNameEnvVarName); + wsmApiHost = Environment.GetEnvironmentVariable(WsmApiHostEnvVarName); + + if (string.IsNullOrEmpty(lzStorageAccountName)) + { + throw new InvalidOperationException( + $"The environment variable {LzStorageAccountNameEnvVarName} is not set"); + } + + if (string.IsNullOrEmpty(workspaceContainerName)) + { + throw new InvalidOperationException( + $"The environment variable {WorkspaceContainerNameEnvVarName} is not set"); + } + + if (string.IsNullOrEmpty(wsmApiHost)) + { + throw new InvalidOperationException( + $"The environment variable {WsmApiHostEnvVarName} is not set"); + } + } + } +} diff --git a/src/TesApi.Tests/TerraApiStubData.cs b/src/TesApi.Tests/TerraApiStubData.cs index 480c03607..c85428169 100644 --- a/src/TesApi.Tests/TerraApiStubData.cs +++ b/src/TesApi.Tests/TerraApiStubData.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; using System.Text.Json; using TesApi.Web.Management.Configuration; using TesApi.Web.Management.Models.Terra; @@ -13,10 +14,10 @@ public class TerraApiStubData public const string LandingZoneApiHost = "https://landingzone.host"; public const string WsmApiHost = "https://wsm.host"; public const string ResourceGroup = "mrg-terra-dev-previ-20191228"; - public const string WorkspaceAccountName = "fooaccount"; - public const string WorkspaceContainerName = "foocontainer"; + public const string WorkspaceAccountName = "lzaccount1"; + public const string WorkspaceStorageContainerName = "sc-ef9fed44-dba6-4825-868c-b00208522382"; public const string SasToken = "SASTOKENSTUB="; - public const string WsmGetSasResponseStorageUrl = $"https://bloburl.foo/{WorkspaceContainerName}"; + public const string WsmGetSasResponseStorageUrl = $"https://{WorkspaceAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}"; public Guid LandingZoneId { get; } = Guid.NewGuid(); public Guid SubscriptionId { get; } = Guid.NewGuid(); @@ -28,6 +29,11 @@ public class TerraApiStubData $"/subscriptions/{SubscriptionId}/resourceGroups/{ResourceGroup}/providers/Microsoft.Batch/batchAccounts/{BatchAccountName}"; public string PoolId => "poolId"; + + public Guid GetWorkspaceIdFromContainerName(string containerName) + { + return Guid.Parse(containerName.Replace("sc-", "")); + } public LandingZoneResourcesApiResponse GetResourceApiResponse() { return JsonSerializer.Deserialize(GetResourceApiResponseInJson()); @@ -50,7 +56,7 @@ public TerraOptions GetTerraOptions() LandingZoneApiHost = LandingZoneApiHost, WsmApiHost = WsmApiHost, WorkspaceStorageAccountName = WorkspaceAccountName, - WorkspaceStorageContainerName = WorkspaceContainerName, + WorkspaceStorageContainerName = WorkspaceStorageContainerName, WorkspaceStorageContainerResourceId = ContainerResourceId.ToString() }; } @@ -182,6 +188,44 @@ public string GetResourceApiResponseInJson() }}"; } + public string GetContainerResourcesApiResponseInJson() + { + return $@"{{ + ""resources"": [ + {{ + ""metadata"": {{ + ""workspaceId"": ""{WorkspaceId}"", + ""resourceId"": ""{ContainerResourceId}"", + ""name"": ""{WorkspaceStorageContainerName}"", + ""resourceType"": ""AZURE_STORAGE_CONTAINER"", + ""stewardshipType"": ""CONTROLLED"", + ""cloudPlatform"": ""AZURE"", + ""cloningInstructions"": ""COPY_NOTHING"", + ""controlledResourceMetadata"": {{ + ""accessScope"": ""SHARED_ACCESS"", + ""managedBy"": ""USER"", + ""privateResourceUser"": {{}}, + ""privateResourceState"": ""NOT_APPLICABLE"", + ""region"": ""southcentralus"" + }}, + ""resourceLineage"": [], + ""properties"": [], + ""createdBy"": ""user@foo.com"", + ""createdDate"": ""2023-02-09T01:48:46.040052Z"", + ""lastUpdatedBy"": ""user@foo.com"", + ""lastUpdatedDate"": ""2023-02-09T01:48:48.345442Z"", + ""state"": ""READY"" + }}, + ""resourceAttributes"": {{ + ""azureStorageContainer"": {{ + ""storageContainerName"": ""{WorkspaceStorageContainerName}"" + }} + }} + }} + ] +}}"; + } + public string GetResourceQuotaApiResponseInJson() { return $@"{{ @@ -292,4 +336,29 @@ public ApiCreateBatchPoolResponse GetApiCreateBatchPoolResponse() ResourceId = new Guid() }; } + + public WsmListContainerResourcesResponse GetWsmContainerResourcesApiResponse() + { + return new WsmListContainerResourcesResponse() + { + Resources = new List() + { + new Resource() + { + Metadata = new Metadata() + { + ResourceId = ContainerResourceId.ToString(), + Name = WorkspaceStorageContainerName + }, + ResourceAttributes = new ResourceAttributes() + { + AzureStorageContainer = new AzureStorageContainer() + { + StorageContainerName = WorkspaceStorageContainerName + } + } + } + } + }; + } } diff --git a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs index 882b4318f..a919af61c 100644 --- a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs +++ b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; @@ -20,7 +21,7 @@ namespace TesApi.Tests public class TerraStorageAccessProviderTests { private const string WorkspaceStorageAccountName = TerraApiStubData.WorkspaceAccountName; - private const string WorkspaceStorageContainerName = TerraApiStubData.WorkspaceContainerName; + private const string WorkspaceStorageContainerName = TerraApiStubData.WorkspaceStorageContainerName; private Mock wsmApiClientMock; private Mock azureProxyMock; @@ -49,48 +50,47 @@ public void SetUp() [DataRow("/foo/bar", false)] [DataRow("foo/bar", false)] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}", false)] - [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/foo", true)] + [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/foo", false)] [DataRow($"https://bar.blob.core.windows.net/{WorkspaceStorageContainerName}", true)] public async Task IsHttpPublicAsync_StringScenario(string input, bool expectedResult) { - var result = await terraStorageAccessProvider.IsPublicHttpUrlAsync(input, System.Threading.CancellationToken.None); + var result = await terraStorageAccessProvider.IsPublicHttpUrlAsync(input, CancellationToken.None); Assert.AreEqual(expectedResult, result); } [TestMethod] - [DataRow($"{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}")] - [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}")] - [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}/")] - [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}/dir/blobName")] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}")] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}/dir/blob")] public async Task MapLocalPathToSasUrlAsync_ValidInput(string input) { - wsmApiClientMock.Setup(a => a.GetSasTokenAsync(terraApiStubData.WorkspaceId, - terraApiStubData.ContainerResourceId, It.IsAny(), It.IsAny())) - .ReturnsAsync(terraApiStubData.GetWsmSasTokenApiResponse()); + SetUpTerraApiClient(); - var result = await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input, System.Threading.CancellationToken.None); + var result = await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input, CancellationToken.None); Assert.IsNotNull(terraApiStubData.GetWsmSasTokenApiResponse().Url, result); } + private void SetUpTerraApiClient() + { + wsmApiClientMock.Setup(a => a.GetSasTokenAsync( + terraApiStubData.GetWorkspaceIdFromContainerName(WorkspaceStorageContainerName), + terraApiStubData.ContainerResourceId, It.IsAny(), It.IsAny())) + .ReturnsAsync(terraApiStubData.GetWsmSasTokenApiResponse()); + wsmApiClientMock.Setup(a => + a.GetContainerResourcesAsync(It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny())) + .ReturnsAsync(terraApiStubData.GetWsmContainerResourcesApiResponse()); + } + [TestMethod] - [DataRow($"{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] - [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] - [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "/", TerraApiStubData.WsmGetSasResponseStorageUrl)] - [DataRow($"/cromwell-executions/test", "", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/cromwell-executions/test")] - [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "/dir/blobName", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/dir/blobName")] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}", "/dir/blob", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/dir/blob")] public async Task MapLocalPathToSasUrlAsync_GetContainerSasIsTrue(string input, string blobPath, string expected) { - wsmApiClientMock.Setup(a => a.GetSasTokenAsync(terraApiStubData.WorkspaceId, - terraApiStubData.ContainerResourceId, It.IsAny(), It.IsAny())) - .ReturnsAsync(terraApiStubData.GetWsmSasTokenApiResponse()); + SetUpTerraApiClient(); - var result = await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input + blobPath, System.Threading.CancellationToken.None, true); + var result = await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input + blobPath, CancellationToken.None, true); Assert.IsNotNull(result); Assert.AreEqual($"{expected}?sv={TerraApiStubData.SasToken}", result); @@ -101,12 +101,12 @@ public async Task MapLocalPathToSasUrlAsync_GetContainerSasIsTrue(string input, [DataRow($"/bar/{WorkspaceStorageContainerName}")] [DataRow($"/foo/bar/")] [DataRow($"/foo/bar/dir/blobName")] - [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/foo")] [DataRow($"https://bar.blob.core.windows.net/{WorkspaceStorageContainerName}/")] - [ExpectedException(typeof(Exception))] - public async Task MapLocalPathToSasUrlAsync_InvalidInputs(string input) + [DataRow($"https://bar.blob.core.windows.net/container/")] + [ExpectedException(typeof(InvalidOperationException))] + public async Task MapLocalPathToSasUrlAsync_InvalidStorageAccountInputs(string input) { - await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input, System.Threading.CancellationToken.None); + await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input, CancellationToken.None); } [TestMethod] @@ -114,16 +114,15 @@ public async Task MapLocalPathToSasUrlAsync_InvalidInputs(string input) [DataRow("blobName")] public async Task GetMappedSasUrlFromWsmAsync_WithOrWithOutBlobName_ReturnsValidURLWithBlobName(string responseBlobName) { - wsmApiClientMock.Setup(a => a.GetSasTokenAsync(terraApiStubData.WorkspaceId, - terraApiStubData.ContainerResourceId, It.IsAny(), It.IsAny())) - .ReturnsAsync(terraApiStubData.GetWsmSasTokenApiResponse(responseBlobName)); + SetUpTerraApiClient(); - var url = await terraStorageAccessProvider.GetMappedSasUrlFromWsmAsync("blobName", System.Threading.CancellationToken.None); + var blobInfo = new TerraBlobInfo(terraApiStubData.GetWorkspaceIdFromContainerName(WorkspaceStorageContainerName), terraApiStubData.ContainerResourceId, TerraApiStubData.WorkspaceStorageContainerName, "blobName"); + var url = await terraStorageAccessProvider.GetMappedSasUrlFromWsmAsync(blobInfo, CancellationToken.None); Assert.IsNotNull(url); var uri = new Uri(url); - Assert.AreEqual(uri.AbsolutePath, $"/{TerraApiStubData.WorkspaceContainerName}/blobName"); + Assert.AreEqual(uri.AbsolutePath, $"/{TerraApiStubData.WorkspaceStorageContainerName}/blobName"); } } } diff --git a/src/TesApi.Tests/TerraWsmApiClientTests.cs b/src/TesApi.Tests/TerraWsmApiClientTests.cs index c86af5d00..0fb20864f 100644 --- a/src/TesApi.Tests/TerraWsmApiClientTests.cs +++ b/src/TesApi.Tests/TerraWsmApiClientTests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Linq; using System.Net; using System.Net.Http; using System.Threading.Tasks; @@ -110,6 +111,25 @@ public async Task GetSasTokenAsync_ValidRequest_ReturnsPayload() Assert.IsTrue(!string.IsNullOrEmpty(apiResponse.Url)); } + [TestMethod] + public async Task GetContainerResourcesAsync_ValidRequest_ReturnsPayload() + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(terraApiStubData.GetContainerResourcesApiResponseInJson()) + }; + + cacheAndRetryHandler.Setup(c => c.ExecuteWithRetryAsync(It.IsAny>>(), It.IsAny())) + .ReturnsAsync(response); + + var apiResponse = await terraWsmApiClient.GetContainerResourcesAsync(terraApiStubData.WorkspaceId, + offset: 0, limit: 10, System.Threading.CancellationToken.None); + + Assert.IsNotNull(apiResponse); + Assert.AreEqual(1, apiResponse.Resources.Count); + Assert.IsTrue(apiResponse.Resources.Any(r => r.Metadata.ResourceId.ToString().Equals(terraApiStubData.ContainerResourceId.ToString(), StringComparison.OrdinalIgnoreCase))); + } + [TestMethod] public async Task DeleteBatchPoolAsync_204Response_Succeeds() { diff --git a/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs b/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs index eb95f3119..a0ec24410 100644 --- a/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs +++ b/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs @@ -45,6 +45,36 @@ public TerraWsmApiClient(TokenCredential tokenCredential, IOptions /// protected TerraWsmApiClient() { } + /// + /// Returns storage containers in the workspace. + /// + /// Terra workspace id + /// Number of items to skip before starting to collect the result + /// Maximum number of items to return + /// A for controlling the lifetime of the asynchronous operation. + /// + public virtual async Task GetContainerResourcesAsync(Guid workspaceId, int offset, int limit, CancellationToken cancellationToken) + { + var url = GetContainerResourcesApiUrl(workspaceId, offset, limit); + + var response = await HttpSendRequestWithRetryPolicyAsync(() => new HttpRequestMessage(HttpMethod.Get, url), + cancellationToken, setAuthorizationHeader: true); + + return await GetApiResponseContentAsync(response, cancellationToken); + } + + private Uri GetContainerResourcesApiUrl(Guid workspaceId, int offset, int limit) + { + var segments = "/resources"; + + var builder = GetWsmUriBuilder(workspaceId, segments); + + //TODO: add support for resource and stewardship parameters if required later + builder.Query = $"offset={offset}&limit={limit}&resource=AZURE_STORAGE_CONTAINER&stewardship=CONTROLLED"; + + return builder.Uri; + } + /// /// Returns the SAS token of a container or blob for WSM managed storage account. /// diff --git a/src/TesApi.Web/Management/Models/Terra/WorkspaceManagerAPI.csproj b/src/TesApi.Web/Management/Models/Terra/WorkspaceManagerAPI.csproj new file mode 100644 index 000000000..8964818ae --- /dev/null +++ b/src/TesApi.Web/Management/Models/Terra/WorkspaceManagerAPI.csproj @@ -0,0 +1,23 @@ + + + + netstandard2.0 + true + annotations + + + + 11.0 + true + https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json + + + + + + + + + + + diff --git a/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs b/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs new file mode 100644 index 000000000..0125bc661 --- /dev/null +++ b/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs @@ -0,0 +1,212 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace TesApi.Web.Management.Models.Terra +{ + /// + /// Azure storage container WSM resource + /// + public class AzureStorageContainer + { + /// + /// Storage container name + /// + [JsonPropertyName("storageContainerName")] + public string StorageContainerName { get; set; } + } + + /// + /// Controlled resource metadata + /// + public class ControlledResourceMetadata + { + /// + /// Access scope + /// + [JsonPropertyName("accessScope")] + public string AccessScope { get; set; } + + /// + /// Managed by + /// + [JsonPropertyName("managedBy")] + public string ManagedBy { get; set; } + + /// + /// Private resource user + /// + [JsonPropertyName("privateResourceUser")] + public PrivateResourceUser PrivateResourceUser { get; set; } + + /// + /// Private resource state + /// + [JsonPropertyName("privateResourceState")] + public string PrivateResourceState { get; set; } + + /// + /// Resource region + /// + [JsonPropertyName("region")] + public string Region { get; set; } + } + + /// + /// WSM resource metadata + /// + public class Metadata + { + /// + /// WSM Workspace ID + /// + [JsonPropertyName("workspaceId")] + public string WorkspaceId { get; set; } + + /// + /// Resource ID + /// + [JsonPropertyName("resourceId")] + public string ResourceId { get; set; } + + /// + /// Resource name + /// + [JsonPropertyName("name")] + public string Name { get; set; } + + /// + /// Resource type + /// + [JsonPropertyName("resourceType")] + public string ResourceType { get; set; } + + /// + /// Stewardship type + /// + [JsonPropertyName("stewardshipType")] + public string StewardshipType { get; set; } + + /// + /// Cloud platform + /// + [JsonPropertyName("cloudPlatform")] + public string CloudPlatform { get; set; } + + /// + /// + /// + [JsonPropertyName("cloningInstructions")] + public string CloningInstructions { get; set; } + + /// + /// Controlled resource metadata + /// + [JsonPropertyName("controlledResourceMetadata")] + public ControlledResourceMetadata ControlledResourceMetadata { get; set; } + + /// + /// Resource linage + /// + [JsonPropertyName("resourceLineage")] + public List ResourceLineage { get; set; } + + /// + /// Additional properties + /// + [JsonPropertyName("properties")] + public List Properties { get; set; } + + /// + /// Created by + /// + [JsonPropertyName("createdBy")] + public string CreatedBy { get; set; } + + /// + /// Creation date + /// + [JsonPropertyName("createdDate")] + public DateTime CreatedDate { get; set; } + + /// + /// Last updated by + /// + [JsonPropertyName("lastUpdatedBy")] + public string LastUpdatedBy { get; set; } + + /// + /// Last updated date + /// + [JsonPropertyName("lastUpdatedDate")] + public DateTime LastUpdatedDate { get; set; } + + /// + /// Resource state + /// + [JsonPropertyName("state")] + public string State { get; set; } + } + + /// + /// Private resource user + /// + public class PrivateResourceUser + { + /// + /// Iam Role + /// + [JsonPropertyName("privateResourceIamRole")] + public object PrivateResourceIamRole { get; set; } + + /// + /// email of the workspace user to grant access + /// + [JsonPropertyName("userName")] + public string UserName { get; set; } + } + + /// + /// WSM resource + /// + public class Resource + { + /// + /// Metadata + /// + [JsonPropertyName("metadata")] + public Metadata Metadata { get; set; } + /// + /// Resource attributes + /// + [JsonPropertyName("resourceAttributes")] + public ResourceAttributes ResourceAttributes { get; set; } + } + + /// + /// WSM resource attributes + /// + public class ResourceAttributes + { + /// + /// Azure storage container + /// + [JsonPropertyName("azureStorageContainer")] + public AzureStorageContainer AzureStorageContainer { get; set; } + } + + /// + /// Response to get storage container resources from a workspace + /// + public class WsmListContainerResourcesResponse + { + /// + /// List of resources in the workspace + /// + [JsonPropertyName("resources")] + public List Resources { get; set; } + } +} diff --git a/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs b/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs index c8f805040..b3208c2f5 100644 --- a/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs +++ b/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs @@ -2,6 +2,8 @@ // Licensed under the MIT License. using System; +using System.Linq; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using System.Web; @@ -14,7 +16,7 @@ namespace TesApi.Web.Storage { /// - /// Provides methods for blob storage access by using local path references in form of /storageaccount/container/blobpath + /// Provides methods for blob storage access for Terra /// public class TerraStorageAccessProvider : StorageAccessProvider { @@ -22,9 +24,10 @@ public class TerraStorageAccessProvider : StorageAccessProvider private readonly TerraWsmApiClient terraWsmApiClient; private const string SasBlobPermissions = "racw"; private const string SasContainerPermissions = "racwl"; + private const string LzStorageAccountNamePattern = "lz[0-9a-f]*"; /// - /// Provides methods for blob storage access by using local path references in form of /storageaccount/container/blobpath + /// Provides methods for blob storage access for Terra /// for Terra /// /// Logger @@ -58,7 +61,7 @@ public override Task IsPublicHttpUrlAsync(string uriString, CancellationTo if (StorageAccountUrlSegments.TryCreate(uriString, out var parts)) { - if (IsTerraWorkspaceContainer(parts.ContainerName) && IsTerraWorkspaceStorageAccount(parts.AccountName)) + if (IsTerraWorkspaceStorageAccount(parts.AccountName)) { return Task.FromResult(false); } @@ -72,60 +75,105 @@ public override async Task MapLocalPathToSasUrlAsync(string path, Cancel { ArgumentException.ThrowIfNullOrEmpty(path); - var normalizedPath = path; if (!TryParseHttpUrlFromInput(path, out _)) - { // if it is a local path, add the leading slash if missing. - normalizedPath = $"/{path.TrimStart('/')}"; + { + throw new InvalidOperationException("The path must be a valid HTTP URL"); } + var terraBlobInfo = await GetTerraBlobInfoFromContainerNameAsync(path, cancellationToken); + if (getContainerSas) { - return await MapAndGetSasContainerUrlFromWsmAsync(normalizedPath, cancellationToken); + return await GetMappedSasContainerUrlFromWsmAsync(terraBlobInfo, cancellationToken); } - if (IsKnownExecutionFilePath(normalizedPath)) - { - return await GetMappedSasUrlFromWsmAsync(normalizedPath, cancellationToken); - } + return await GetMappedSasUrlFromWsmAsync(terraBlobInfo, cancellationToken); + } - if (!StorageAccountUrlSegments.TryCreate(normalizedPath, out var segments)) + /// + /// Creates a Terra Blob Info from the container name in the path. The path must be a Terra managed storage URL. + /// This method assumes that the container name contains the workspace ID and validates that the storage container is a Terra workspace resource. + /// The BlobName property contains the blob name segment without a leading slash. + /// + /// + /// + /// Returns a Terra Blob Info + /// This method will throw if the path is not a valid Terra blob storage url. + private async Task GetTerraBlobInfoFromContainerNameAsync(string path, CancellationToken cancellationToken) + { + if (!StorageAccountUrlSegments.TryCreate(path, out var segments)) { - throw new Exception( + throw new InvalidOperationException( "Invalid path provided. The path must be a valid blob storage url or a path with the following format: /accountName/container"); } - CheckIfAccountAndContainerAreWorkspaceStorage(segments.AccountName, segments.ContainerName); + CheckIfAccountIsTerraStorageAccount(segments.AccountName); + + Logger.LogInformation($"Getting Workspace ID from the Container Name: {segments.ContainerName}"); + + var workspaceId = ToWorkspaceId(segments.ContainerName); - return await GetMappedSasUrlFromWsmAsync(segments.BlobName, cancellationToken); + Logger.LogInformation($"Workspace ID to use: {segments.ContainerName}"); + + var wsmContainerResourceId = await GetWsmContainerResourceIdAsync(workspaceId, segments.ContainerName, cancellationToken); + + return new TerraBlobInfo(workspaceId, wsmContainerResourceId, segments.ContainerName, segments.BlobName.TrimStart('/')); } - private async Task MapAndGetSasContainerUrlFromWsmAsync(string inputPath, CancellationToken cancellationToken) + private async Task GetWsmContainerResourceIdAsync(Guid workspaceId, string containerName, CancellationToken cancellationToken) { - if (IsKnownExecutionFilePath(inputPath)) + Logger.LogInformation($"Getting container resource information from WSM. Workspace ID: {workspaceId} Container Name: {containerName}"); + + try { - return await GetMappedSasContainerUrlFromWsmAsync(inputPath, cancellationToken); - } + //the goal is to get all containers, therefore the limit is set to 10000 which is a reasonable unreachable number of storage containers in a workspace. + var response = + await terraWsmApiClient.GetContainerResourcesAsync(workspaceId, offset: 0, limit: 10000, cancellationToken); + + var metadata = response.Resources.Single(r => + r.ResourceAttributes.AzureStorageContainer.StorageContainerName.Equals(containerName, + StringComparison.OrdinalIgnoreCase)).Metadata; - if (!StorageAccountUrlSegments.TryCreate(inputPath, out var withContainerSegments)) + Logger.LogInformation($"Found the resource id for storage container resource. Resource ID: {metadata.ResourceId} Container Name: {containerName}"); + + return Guid.Parse(metadata.ResourceId); + } + catch (Exception e) { - throw new Exception( - "Invalid path provided. The path must be a valid blob storage url or a path with the following format: /accountName/container"); + Logger.LogError(e, "Failed to call WSM to obtain the storage container resource ID"); + throw; } + } + - return await GetMappedSasContainerUrlFromWsmAsync(withContainerSegments.BlobName, cancellationToken); + private Guid ToWorkspaceId(string segmentsContainerName) + { + try + { + ArgumentException.ThrowIfNullOrEmpty(segmentsContainerName); + + var guidString = segmentsContainerName.Substring(3); // remove the sc- prefix + + return Guid.Parse(guidString); // throws if not a guid + } + catch (Exception e) + { + Logger.LogError(e, $"Failed to get the workspace ID from the container name. The name provided is not a valid GUID. Container Name: {segmentsContainerName}"); + throw; + } } - private async Task GetMappedSasContainerUrlFromWsmAsync(string pathToAppend, CancellationToken cancellationToken) + private async Task GetMappedSasContainerUrlFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) { //an empty blob name gets a container Sas token - var tokenInfo = await GetSasTokenFromWsmAsync(CreateTokenParamsFromOptions(blobName: string.Empty, SasContainerPermissions), cancellationToken); + var tokenInfo = await GetSasTokenForContainerFromWsmAsync(blobInfo, cancellationToken); var urlBuilder = new UriBuilder(tokenInfo.Url); - if (!string.IsNullOrEmpty(pathToAppend.TrimStart('/'))) + if (!string.IsNullOrEmpty(blobInfo.BlobName)) { - urlBuilder.Path += $"/{pathToAppend.TrimStart('/')}"; + urlBuilder.Path += $"/{blobInfo.BlobName}"; } return urlBuilder.Uri.ToString(); @@ -134,26 +182,22 @@ private async Task GetMappedSasContainerUrlFromWsmAsync(string pathToApp /// /// Returns a Url with a SAS token for the given input /// - /// + /// /// A for controlling the lifetime of the asynchronous operation. - /// SAS Token URL - public async Task GetMappedSasUrlFromWsmAsync(string blobName, CancellationToken cancellationToken) + /// URL with a SAS token + public async Task GetMappedSasUrlFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) { - var normalizedBlobName = blobName.TrimStart('/'); - - var tokenParams = CreateTokenParamsFromOptions(normalizedBlobName, SasBlobPermissions); - - var tokenInfo = await GetSasTokenFromWsmAsync(tokenParams, cancellationToken); + var tokenInfo = await GetSasTokenFromWsmAsync(blobInfo, cancellationToken); Logger.LogInformation($"Successfully obtained the Sas Url from Terra. Wsm resource id:{terraOptions.WorkspaceStorageContainerResourceId}"); var uriBuilder = new UriBuilder(tokenInfo.Url); - if (normalizedBlobName != string.Empty) + if (blobInfo.BlobName != string.Empty) { - if (!uriBuilder.Path.Contains(normalizedBlobName, StringComparison.OrdinalIgnoreCase)) + if (!uriBuilder.Path.Contains(blobInfo.BlobName, StringComparison.OrdinalIgnoreCase)) { - uriBuilder.Path += $"/{normalizedBlobName}"; + uriBuilder.Path += $"/{blobInfo.BlobName}"; } } @@ -166,33 +210,57 @@ private SasTokenApiParameters CreateTokenParamsFromOptions(string blobName, stri terraOptions.SasTokenExpirationInSeconds, sasPermissions, blobName); - private async Task GetSasTokenFromWsmAsync(SasTokenApiParameters tokenParams, CancellationToken cancellationToken) + + private async Task GetSasTokenFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) { + var tokenParams = CreateTokenParamsFromOptions(blobInfo.BlobName, SasBlobPermissions); + Logger.LogInformation( - $"Getting Sas Url from Terra. Wsm resource id:{terraOptions.WorkspaceStorageContainerResourceId}"); + $"Getting Sas Url from Terra. Wsm workspace id:{blobInfo.WorkspaceId}"); + return await terraWsmApiClient.GetSasTokenAsync( - Guid.Parse(terraOptions.WorkspaceId), - Guid.Parse(terraOptions.WorkspaceStorageContainerResourceId), + blobInfo.WorkspaceId, + blobInfo.WsmContainerResourceId, tokenParams, cancellationToken); } - private void CheckIfAccountAndContainerAreWorkspaceStorage(string accountName, string containerName) + private async Task GetSasTokenForContainerFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) { - if (!IsTerraWorkspaceStorageAccount(accountName)) - { - throw new Exception($"The account name does not match the configuration for Terra."); - } + // an empty blob name gets a container Sas token + var tokenParams = CreateTokenParamsFromOptions(blobName: "", SasContainerPermissions); + + Logger.LogInformation( + $"Getting Sas container Url from Terra. Wsm workspace id:{blobInfo.WorkspaceId}"); - if (!IsTerraWorkspaceContainer(containerName)) + return await terraWsmApiClient.GetSasTokenAsync( + blobInfo.WorkspaceId, + blobInfo.WsmContainerResourceId, + tokenParams, cancellationToken); + } + + + private void CheckIfAccountIsTerraStorageAccount(string accountName) + { + if (!IsTerraWorkspaceStorageAccount(accountName)) { - throw new Exception($"The container name does not match the configuration for Terra"); + throw new InvalidOperationException($"The account name does not match the configuration for Terra."); } } - private bool IsTerraWorkspaceContainer(string value) - => terraOptions.WorkspaceStorageContainerName.Equals(value, StringComparison.OrdinalIgnoreCase); - private bool IsTerraWorkspaceStorageAccount(string value) - => terraOptions.WorkspaceStorageAccountName.Equals(value, StringComparison.OrdinalIgnoreCase); + { + var match = Regex.Match(value, LzStorageAccountNamePattern); + + return match.Success; + } } + + /// + /// Contains the Terra and Azure Storage container properties where the blob is contained. + /// + /// + /// + /// + /// + public record TerraBlobInfo(Guid WorkspaceId, Guid WsmContainerResourceId, string WsmContainerName, string BlobName); } diff --git a/src/TesApi.Web/TesApi.Web.csproj b/src/TesApi.Web/TesApi.Web.csproj index 9e3f0021d..305af8fbf 100644 --- a/src/TesApi.Web/TesApi.Web.csproj +++ b/src/TesApi.Web/TesApi.Web.csproj @@ -148,4 +148,4 @@ - + \ No newline at end of file