From 94944cc7acafee392aeab1beea739a5612cc3da5 Mon Sep 17 00:00:00 2001 From: giventocode <3589801+giventocode@users.noreply.github.com> Date: Mon, 19 Jun 2023 18:34:43 -0400 Subject: [PATCH 1/8] initial commit --- .../TerraStorageAccessProviderTests.cs | 2 +- .../Terra/WorkspaceResourcesApiResponse.cs | 371 ++++++++++++++++++ .../Storage/TerraStorageAccessProvider.cs | 61 ++- 3 files changed, 425 insertions(+), 9 deletions(-) create mode 100644 src/TesApi.Web/Management/Models/Terra/WorkspaceResourcesApiResponse.cs diff --git a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs index 882b4318f..910312679 100644 --- a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs +++ b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs @@ -49,7 +49,7 @@ public void SetUp() [DataRow("/foo/bar", false)] [DataRow("foo/bar", false)] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}", false)] - [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/foo", true)] + [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/foo", false)] [DataRow($"https://bar.blob.core.windows.net/{WorkspaceStorageContainerName}", true)] public async Task IsHttpPublicAsync_StringScenario(string input, bool expectedResult) { diff --git a/src/TesApi.Web/Management/Models/Terra/WorkspaceResourcesApiResponse.cs b/src/TesApi.Web/Management/Models/Terra/WorkspaceResourcesApiResponse.cs new file mode 100644 index 000000000..56a8c52d5 --- /dev/null +++ b/src/TesApi.Web/Management/Models/Terra/WorkspaceResourcesApiResponse.cs @@ -0,0 +1,371 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System; + +namespace TesApi.Web.Management.Models.Terra +{ + public class WorkspaceResourcesApiResponse + { + } + + // Root myDeserializedClass = JsonConvert.DeserializeObject(myJsonResponse); + public class ApplicationPackage + { + public string id { get; set; } + public string version { get; set; } + } + + public class AutoScale + { + public string formula { get; set; } + public int evaluationInterval { get; set; } + } + + public class AutoUser + { + public string scope { get; set; } + public string elevationLevel { get; set; } + } + + public class AwsS3StorageFolder + { + public string bucketName { get; set; } + public string prefix { get; set; } + } + + public class AwsSageMakerNotebook + { + public string instanceName { get; set; } + public string instanceType { get; set; } + } + + public class AzureBatchPool + { + public string id { get; set; } + public string vmSize { get; set; } + public string displayName { get; set; } + public DeploymentConfiguration deploymentConfiguration { get; set; } + public List userAssignedIdentities { get; set; } + public ScaleSettings scaleSettings { get; set; } + public StartTask startTask { get; set; } + public List applicationPackages { get; set; } + public NetworkConfiguration networkConfiguration { get; set; } + } + + public class AzureDatabase + { + public string databaseName { get; set; } + public string databaseOwner { get; set; } + } + + public class AzureDisk + { + public string diskName { get; set; } + public string region { get; set; } + } + + public class AzureManagedIdentity + { + public string managedIdentityName { get; set; } + } + + public class AzureStorageContainer + { + public string storageContainerName { get; set; } + } + + public class AzureVm + { + public string vmName { get; set; } + public string region { get; set; } + public string vmSize { get; set; } + public string vmImage { get; set; } + public string diskId { get; set; } + } + + public class CloudServiceConfiguration + { + public string osFamily { get; set; } + public string osVersion { get; set; } + } + + public class ContainerSettings + { + public string containerRunOptions { get; set; } + public string imageName { get; set; } + public Registry registry { get; set; } + public string workingDirectory { get; set; } + } + + public class ControlledResourceMetadata + { + public string accessScope { get; set; } + public string managedBy { get; set; } + public PrivateResourceUser privateResourceUser { get; set; } + public string privateResourceState { get; set; } + public string region { get; set; } + } + + public class DeploymentConfiguration + { + public VirtualMachineConfiguration virtualMachineConfiguration { get; set; } + public CloudServiceConfiguration cloudServiceConfiguration { get; set; } + } + + public class EndpointConfiguration + { + public List inboundNatPools { get; set; } + } + + public class EnvironmentSetting + { + public string name { get; set; } + public string value { get; set; } + } + + public class ErrorReport + { + public string message { get; set; } + public int statusCode { get; set; } + public List causes { get; set; } + } + + public class FixedScale + { + public int resizeTimeout { get; set; } + public int targetDedicatedNodes { get; set; } + public int targetLowPriorityNodes { get; set; } + public string nodeDeallocationOption { get; set; } + } + + public class FlexibleResource + { + public string typeNamespace { get; set; } + public string type { get; set; } + public string data { get; set; } + } + + public class GcpAiNotebookInstance + { + public string projectId { get; set; } + public string location { get; set; } + public string instanceId { get; set; } + } + + public class GcpBqDataset + { + public string projectId { get; set; } + public string datasetId { get; set; } + } + + public class GcpBqDataTable + { + public string projectId { get; set; } + public string datasetId { get; set; } + public string dataTableId { get; set; } + } + + public class GcpDataRepoSnapshot + { + public string instanceName { get; set; } + public string snapshot { get; set; } + } + + public class GcpGcsBucket + { + public string bucketName { get; set; } + } + + public class GcpGcsObject + { + public string bucketName { get; set; } + public string fileName { get; set; } + } + + public class GitRepo + { + public string gitRepoUrl { get; set; } + } + + public class IdentityReference + { + public string resourceId { get; set; } + } + + public class ImageReference + { + public string publisher { get; set; } + public string offer { get; set; } + public string sku { get; set; } + public string version { get; set; } + public string id { get; set; } + } + + public class InboundNatPool + { + public string name { get; set; } + public string protocol { get; set; } + public int backendPort { get; set; } + public int frontendPortRangeStart { get; set; } + public int frontendPortRangeEnd { get; set; } + public List networkSecurityGroupRules { get; set; } + } + + public class Metadata + { + public string workspaceId { get; set; } + public string resourceId { get; set; } + public string name { get; set; } + public string description { get; set; } + public string resourceType { get; set; } + public string stewardshipType { get; set; } + public string cloudPlatform { get; set; } + public string cloningInstructions { get; set; } + public ControlledResourceMetadata controlledResourceMetadata { get; set; } + public List resourceLineage { get; set; } + public List properties { get; set; } + public string createdBy { get; set; } + public DateTime createdDate { get; set; } + public string lastUpdatedBy { get; set; } + public DateTime lastUpdatedDate { get; set; } + public string state { get; set; } + public ErrorReport errorReport { get; set; } + public string jobId { get; set; } + } + + public class NetworkConfiguration + { + public string subnetId { get; set; } + public string dynamicVNetAssignmentScope { get; set; } + public EndpointConfiguration endpointConfiguration { get; set; } + public PublicIpAddressConfiguration publicIpAddressConfiguration { get; set; } + } + + public class NetworkSecurityGroupRule + { + public int priority { get; set; } + public string access { get; set; } + public string sourceAddressPrefix { get; set; } + public List sourcePortRanges { get; set; } + } + + public class PrivateResourceUser + { + public string userName { get; set; } + public string privateResourceIamRole { get; set; } + } + + public class Property + { + public string key { get; set; } + public string value { get; set; } + } + + public class PublicIpAddressConfiguration + { + public string provision { get; set; } + public List ipAddressIds { get; set; } + } + + public class Registry + { + public string userName { get; set; } + public string password { get; set; } + public string registryServer { get; set; } + public IdentityReference identityReference { get; set; } + } + + public class Resource + { + public Metadata metadata { get; set; } + public ResourceAttributes resourceAttributes { get; set; } + } + + public class ResourceAttributes + { + public GcpBqDataset gcpBqDataset { get; set; } + public GcpBqDataTable gcpBqDataTable { get; set; } + public GcpDataRepoSnapshot gcpDataRepoSnapshot { get; set; } + public GcpGcsBucket gcpGcsBucket { get; set; } + public GcpGcsObject gcpGcsObject { get; set; } + public GcpAiNotebookInstance gcpAiNotebookInstance { get; set; } + public AzureManagedIdentity azureManagedIdentity { get; set; } + public AzureDatabase azureDatabase { get; set; } + public AzureDisk azureDisk { get; set; } + public AzureStorageContainer azureStorageContainer { get; set; } + public AzureVm azureVm { get; set; } + public AzureBatchPool azureBatchPool { get; set; } + public AwsS3StorageFolder awsS3StorageFolder { get; set; } + public AwsSageMakerNotebook awsSageMakerNotebook { get; set; } + public GitRepo gitRepo { get; set; } + public TerraWorkspace terraWorkspace { get; set; } + public FlexibleResource flexibleResource { get; set; } + } + + public class ResourceFile + { + public string autoStorageContainerName { get; set; } + public string storageContainerUrl { get; set; } + public string httpUrl { get; set; } + public string blobPrefix { get; set; } + public string filePath { get; set; } + public string fileMode { get; set; } + public IdentityReference identityReference { get; set; } + } + + public class ResourceLineage + { + public string sourceWorkspaceId { get; set; } + public string sourceResourceId { get; set; } + } + + public class Root + { + public List resources { get; set; } + } + + public class ScaleSettings + { + public FixedScale fixedScale { get; set; } + public AutoScale autoScale { get; set; } + } + + public class StartTask + { + public string commandLine { get; set; } + public List resourceFiles { get; set; } + public List environmentSettings { get; set; } + public UserIdentity userIdentity { get; set; } + public int maxTaskRetryCount { get; set; } + public bool waitForSuccess { get; set; } + public ContainerSettings containerSettings { get; set; } + } + + public class TerraWorkspace + { + public string referencedWorkspaceId { get; set; } + } + + public class UserAssignedIdentity + { + public string name { get; set; } + public string clientId { get; set; } + public string resourceGroupName { get; set; } + } + + public class UserIdentity + { + public string userName { get; set; } + public AutoUser autoUser { get; set; } + } + + public class VirtualMachineConfiguration + { + public ImageReference imageReference { get; set; } + public string nodeAgentSkuId { get; set; } + } + + +} diff --git a/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs b/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs index c8f805040..81666a3bb 100644 --- a/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs +++ b/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs @@ -58,7 +58,7 @@ public override Task IsPublicHttpUrlAsync(string uriString, CancellationTo if (StorageAccountUrlSegments.TryCreate(uriString, out var parts)) { - if (IsTerraWorkspaceContainer(parts.ContainerName) && IsTerraWorkspaceStorageAccount(parts.AccountName)) + if (IsTerraWorkspaceStorageAccount(parts.AccountName)) { return Task.FromResult(false); } @@ -95,11 +95,44 @@ public override async Task MapLocalPathToSasUrlAsync(string path, Cancel "Invalid path provided. The path must be a valid blob storage url or a path with the following format: /accountName/container"); } - CheckIfAccountAndContainerAreWorkspaceStorage(segments.AccountName, segments.ContainerName); + CheckIfAccountIsTerraStorageAccount(segments.AccountName); return await GetMappedSasUrlFromWsmAsync(segments.BlobName, cancellationToken); } + public TerraBlobInfo GetTerraBlobInfo(string normalizedPath) + { + if (!StorageAccountUrlSegments.TryCreate(normalizedPath, out var segments)) + { + throw new Exception( + "Invalid path provided. The path must be a valid blob storage url or a path with the following format: /accountName/container"); + } + + CheckIfAccountIsTerraStorageAccount(segments.AccountName); + + var workspaceId = ToWorkspaceId(segments.ContainerName); + + var wsmContainerResourceId = await GetWsmContainerResourceIdAsync(workspaceId, segments.ContainerName); + + return new TerraBlobInfo(workspaceId,wsmContainerResourceId, segments.BlobName); + } + + private async Task GetWsmContainerResourceIdAsync(string workspaceId, string segmentsContainerName) + { + //terraWsmApiClient. + } + + private string ToWorkspaceId(string segmentsContainerName) + { + ArgumentException.ThrowIfNullOrEmpty(segmentsContainerName); + + var guidString = segmentsContainerName.Substring(3); // remove the sc- prefix + + var guid = Guid.Parse(guidString); // throws if not a guid + + return guid.ToString(); + } + private async Task MapAndGetSasContainerUrlFromWsmAsync(string inputPath, CancellationToken cancellationToken) { if (IsKnownExecutionFilePath(inputPath)) @@ -176,17 +209,26 @@ private async Task GetSasTokenFromWsmAsync(SasTokenApiPa tokenParams, cancellationToken); } - private void CheckIfAccountAndContainerAreWorkspaceStorage(string accountName, string containerName) + private async Task GetSasTokenFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) + { + var tokenParams = CreateTokenParamsFromOptions(blobInfo.BlobName, SasBlobPermissions); + + Logger.LogInformation( + $"Getting Sas Url from Terra. Wsm workspace id:{terraOptions.WorkspaceStorageContainerResourceId}"); + + return await terraWsmApiClient.GetSasTokenAsync( + Guid.Parse(terraOptions.WorkspaceId), + Guid.Parse(terraOptions.WorkspaceStorageContainerResourceId), + tokenParams, cancellationToken); + } + + + private void CheckIfAccountIsTerraStorageAccount(string accountName) { if (!IsTerraWorkspaceStorageAccount(accountName)) { throw new Exception($"The account name does not match the configuration for Terra."); } - - if (!IsTerraWorkspaceContainer(containerName)) - { - throw new Exception($"The container name does not match the configuration for Terra"); - } } private bool IsTerraWorkspaceContainer(string value) @@ -195,4 +237,7 @@ private bool IsTerraWorkspaceContainer(string value) private bool IsTerraWorkspaceStorageAccount(string value) => terraOptions.WorkspaceStorageAccountName.Equals(value, StringComparison.OrdinalIgnoreCase); } + + public record TerraBlobInfo(string WorkspaceId, string WsmContainerResourceId, string BlobName); + } From 2ebebe8996420b4a7a67e920d40a1403b0d9ae83 Mon Sep 17 00:00:00 2001 From: giventocode <3589801+giventocode@users.noreply.github.com> Date: Thu, 22 Jun 2023 19:53:53 -0400 Subject: [PATCH 2/8] code simplification and refactoring --- src/TesApi.Tests/TerraApiStubData.cs | 28 +- .../TerraStorageAccessProviderTests.cs | 30 +- .../Management/Clients/TerraWsmApiClient.cs | 30 ++ .../Models/Terra/WorkspaceManagerAPI.csproj | 23 ++ .../Terra/WorkspaceResourcesApiResponse.cs | 371 ------------------ .../WsmListContainerResourcesResponse.cs | 212 ++++++++++ .../Storage/TerraStorageAccessProvider.cs | 178 +++++---- src/TesApi.Web/TesApi.Web.csproj | 10 +- 8 files changed, 413 insertions(+), 469 deletions(-) create mode 100644 src/TesApi.Web/Management/Models/Terra/WorkspaceManagerAPI.csproj delete mode 100644 src/TesApi.Web/Management/Models/Terra/WorkspaceResourcesApiResponse.cs create mode 100644 src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs diff --git a/src/TesApi.Tests/TerraApiStubData.cs b/src/TesApi.Tests/TerraApiStubData.cs index 480c03607..4130dcfff 100644 --- a/src/TesApi.Tests/TerraApiStubData.cs +++ b/src/TesApi.Tests/TerraApiStubData.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; using System.Text.Json; using TesApi.Web.Management.Configuration; using TesApi.Web.Management.Models.Terra; @@ -14,7 +15,7 @@ public class TerraApiStubData public const string WsmApiHost = "https://wsm.host"; public const string ResourceGroup = "mrg-terra-dev-previ-20191228"; public const string WorkspaceAccountName = "fooaccount"; - public const string WorkspaceContainerName = "foocontainer"; + public const string WorkspaceContainerName = "sc-ef9fed44-dba6-4825-868c-b00208522382"; public const string SasToken = "SASTOKENSTUB="; public const string WsmGetSasResponseStorageUrl = $"https://bloburl.foo/{WorkspaceContainerName}"; @@ -292,4 +293,29 @@ public ApiCreateBatchPoolResponse GetApiCreateBatchPoolResponse() ResourceId = new Guid() }; } + + public WsmListContainerResourcesResponse GetWsmContainerResourcesApiResponse() + { + return new WsmListContainerResourcesResponse() + { + Resources = new List() + { + new Resource() + { + Metadata = new Metadata() + { + ResourceId = Guid.NewGuid().ToString() + + }, + ResourceAttributes = new ResourceAttributes() + { + AzureStorageContainer = new AzureStorageContainer() + { + StorageContainerName = WorkspaceContainerName + } + } + } + } + }; + } } diff --git a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs index 910312679..aae2c0a32 100644 --- a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs +++ b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; @@ -53,7 +54,7 @@ public void SetUp() [DataRow($"https://bar.blob.core.windows.net/{WorkspaceStorageContainerName}", true)] public async Task IsHttpPublicAsync_StringScenario(string input, bool expectedResult) { - var result = await terraStorageAccessProvider.IsPublicHttpUrlAsync(input, System.Threading.CancellationToken.None); + var result = await terraStorageAccessProvider.IsPublicHttpUrlAsync(input, CancellationToken.None); Assert.AreEqual(expectedResult, result); } @@ -67,11 +68,13 @@ public async Task IsHttpPublicAsync_StringScenario(string input, bool expectedRe [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}/dir/blob")] public async Task MapLocalPathToSasUrlAsync_ValidInput(string input) { - wsmApiClientMock.Setup(a => a.GetSasTokenAsync(terraApiStubData.WorkspaceId, - terraApiStubData.ContainerResourceId, It.IsAny(), It.IsAny())) + wsmApiClientMock.Setup(a => a.GetSasTokenAsync(It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(terraApiStubData.GetWsmSasTokenApiResponse()); + wsmApiClientMock.Setup(a=>a.GetContainerResourcesAsync(It.IsAny(),It.IsAny(),It.IsAny(),It.IsAny())) + .ReturnsAsync(terraApiStubData.GetWsmContainerResourcesApiResponse()); - var result = await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input, System.Threading.CancellationToken.None); + var result = await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input, CancellationToken.None); Assert.IsNotNull(terraApiStubData.GetWsmSasTokenApiResponse().Url, result); } @@ -80,33 +83,31 @@ public async Task MapLocalPathToSasUrlAsync_ValidInput(string input) [DataRow($"{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "/", TerraApiStubData.WsmGetSasResponseStorageUrl)] - [DataRow($"/cromwell-executions/test", "", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/cromwell-executions/test")] + [DataRow("/cromwell-executions/test", "", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/cromwell-executions/test")] [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "/dir/blobName", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/dir/blobName")] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}", "/dir/blob", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/dir/blob")] public async Task MapLocalPathToSasUrlAsync_GetContainerSasIsTrue(string input, string blobPath, string expected) { wsmApiClientMock.Setup(a => a.GetSasTokenAsync(terraApiStubData.WorkspaceId, - terraApiStubData.ContainerResourceId, It.IsAny(), It.IsAny())) + terraApiStubData.ContainerResourceId, It.IsAny(), It.IsAny())) .ReturnsAsync(terraApiStubData.GetWsmSasTokenApiResponse()); - var result = await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input + blobPath, System.Threading.CancellationToken.None, true); + var result = await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input + blobPath, CancellationToken.None, true); Assert.IsNotNull(result); Assert.AreEqual($"{expected}?sv={TerraApiStubData.SasToken}", result); } [TestMethod] - [DataRow($"{WorkspaceStorageAccountName}/foo")] [DataRow($"/bar/{WorkspaceStorageContainerName}")] - [DataRow($"/foo/bar/")] - [DataRow($"/foo/bar/dir/blobName")] - [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/foo")] + [DataRow("/foo/bar/")] + [DataRow("/foo/bar/dir/blobName")] [DataRow($"https://bar.blob.core.windows.net/{WorkspaceStorageContainerName}/")] [ExpectedException(typeof(Exception))] public async Task MapLocalPathToSasUrlAsync_InvalidInputs(string input) { - await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input, System.Threading.CancellationToken.None); + await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input, CancellationToken.None); } [TestMethod] @@ -115,10 +116,11 @@ public async Task MapLocalPathToSasUrlAsync_InvalidInputs(string input) public async Task GetMappedSasUrlFromWsmAsync_WithOrWithOutBlobName_ReturnsValidURLWithBlobName(string responseBlobName) { wsmApiClientMock.Setup(a => a.GetSasTokenAsync(terraApiStubData.WorkspaceId, - terraApiStubData.ContainerResourceId, It.IsAny(), It.IsAny())) + terraApiStubData.ContainerResourceId, It.IsAny(), It.IsAny())) .ReturnsAsync(terraApiStubData.GetWsmSasTokenApiResponse(responseBlobName)); - var url = await terraStorageAccessProvider.GetMappedSasUrlFromWsmAsync("blobName", System.Threading.CancellationToken.None); + var blobInfo = new TerraBlobInfo(terraApiStubData.WorkspaceId,terraApiStubData.ContainerResourceId, TerraApiStubData.WorkspaceContainerName, "blobName"); + var url = await terraStorageAccessProvider.GetMappedSasUrlFromWsmAsync(blobInfo, CancellationToken.None); Assert.IsNotNull(url); var uri = new Uri(url); diff --git a/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs b/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs index eb95f3119..795ead650 100644 --- a/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs +++ b/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs @@ -45,6 +45,36 @@ public TerraWsmApiClient(TokenCredential tokenCredential, IOptions /// protected TerraWsmApiClient() { } + /// + /// Returns the SAS token of a container or blob for WSM managed storage account. + /// + /// Terra workspace id + /// Number of items to skip before starting to collect the result + /// Maximum number of items to return + /// A for controlling the lifetime of the asynchronous operation. + /// + public virtual async Task GetContainerResourcesAsync(Guid workspaceId, int offset, int limit, CancellationToken cancellationToken) + { + var url = GetContainerResourcesApiUrl(workspaceId, offset, limit); + + var response = await HttpSendRequestWithRetryPolicyAsync(() => new HttpRequestMessage(HttpMethod.Post, url), + cancellationToken, setAuthorizationHeader: true); + + return await GetApiResponseContentAsync(response, cancellationToken); + } + + private Uri GetContainerResourcesApiUrl(Guid workspaceId, int offset, int limit) + { + var segments = "/resources"; + + var builder = GetWsmUriBuilder(workspaceId, segments); + + //TODO: add support for resource and stewardship parameters if required later + builder.Query = $"offset={offset}&limit={limit}&resource=AZURE_STORAGE_CONTAINER&stewardship=CONTROLLED"; + + return builder.Uri; + } + /// /// Returns the SAS token of a container or blob for WSM managed storage account. /// diff --git a/src/TesApi.Web/Management/Models/Terra/WorkspaceManagerAPI.csproj b/src/TesApi.Web/Management/Models/Terra/WorkspaceManagerAPI.csproj new file mode 100644 index 000000000..8964818ae --- /dev/null +++ b/src/TesApi.Web/Management/Models/Terra/WorkspaceManagerAPI.csproj @@ -0,0 +1,23 @@ + + + + netstandard2.0 + true + annotations + + + + 11.0 + true + https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json + + + + + + + + + + + diff --git a/src/TesApi.Web/Management/Models/Terra/WorkspaceResourcesApiResponse.cs b/src/TesApi.Web/Management/Models/Terra/WorkspaceResourcesApiResponse.cs deleted file mode 100644 index 56a8c52d5..000000000 --- a/src/TesApi.Web/Management/Models/Terra/WorkspaceResourcesApiResponse.cs +++ /dev/null @@ -1,371 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Collections.Generic; -using System; - -namespace TesApi.Web.Management.Models.Terra -{ - public class WorkspaceResourcesApiResponse - { - } - - // Root myDeserializedClass = JsonConvert.DeserializeObject(myJsonResponse); - public class ApplicationPackage - { - public string id { get; set; } - public string version { get; set; } - } - - public class AutoScale - { - public string formula { get; set; } - public int evaluationInterval { get; set; } - } - - public class AutoUser - { - public string scope { get; set; } - public string elevationLevel { get; set; } - } - - public class AwsS3StorageFolder - { - public string bucketName { get; set; } - public string prefix { get; set; } - } - - public class AwsSageMakerNotebook - { - public string instanceName { get; set; } - public string instanceType { get; set; } - } - - public class AzureBatchPool - { - public string id { get; set; } - public string vmSize { get; set; } - public string displayName { get; set; } - public DeploymentConfiguration deploymentConfiguration { get; set; } - public List userAssignedIdentities { get; set; } - public ScaleSettings scaleSettings { get; set; } - public StartTask startTask { get; set; } - public List applicationPackages { get; set; } - public NetworkConfiguration networkConfiguration { get; set; } - } - - public class AzureDatabase - { - public string databaseName { get; set; } - public string databaseOwner { get; set; } - } - - public class AzureDisk - { - public string diskName { get; set; } - public string region { get; set; } - } - - public class AzureManagedIdentity - { - public string managedIdentityName { get; set; } - } - - public class AzureStorageContainer - { - public string storageContainerName { get; set; } - } - - public class AzureVm - { - public string vmName { get; set; } - public string region { get; set; } - public string vmSize { get; set; } - public string vmImage { get; set; } - public string diskId { get; set; } - } - - public class CloudServiceConfiguration - { - public string osFamily { get; set; } - public string osVersion { get; set; } - } - - public class ContainerSettings - { - public string containerRunOptions { get; set; } - public string imageName { get; set; } - public Registry registry { get; set; } - public string workingDirectory { get; set; } - } - - public class ControlledResourceMetadata - { - public string accessScope { get; set; } - public string managedBy { get; set; } - public PrivateResourceUser privateResourceUser { get; set; } - public string privateResourceState { get; set; } - public string region { get; set; } - } - - public class DeploymentConfiguration - { - public VirtualMachineConfiguration virtualMachineConfiguration { get; set; } - public CloudServiceConfiguration cloudServiceConfiguration { get; set; } - } - - public class EndpointConfiguration - { - public List inboundNatPools { get; set; } - } - - public class EnvironmentSetting - { - public string name { get; set; } - public string value { get; set; } - } - - public class ErrorReport - { - public string message { get; set; } - public int statusCode { get; set; } - public List causes { get; set; } - } - - public class FixedScale - { - public int resizeTimeout { get; set; } - public int targetDedicatedNodes { get; set; } - public int targetLowPriorityNodes { get; set; } - public string nodeDeallocationOption { get; set; } - } - - public class FlexibleResource - { - public string typeNamespace { get; set; } - public string type { get; set; } - public string data { get; set; } - } - - public class GcpAiNotebookInstance - { - public string projectId { get; set; } - public string location { get; set; } - public string instanceId { get; set; } - } - - public class GcpBqDataset - { - public string projectId { get; set; } - public string datasetId { get; set; } - } - - public class GcpBqDataTable - { - public string projectId { get; set; } - public string datasetId { get; set; } - public string dataTableId { get; set; } - } - - public class GcpDataRepoSnapshot - { - public string instanceName { get; set; } - public string snapshot { get; set; } - } - - public class GcpGcsBucket - { - public string bucketName { get; set; } - } - - public class GcpGcsObject - { - public string bucketName { get; set; } - public string fileName { get; set; } - } - - public class GitRepo - { - public string gitRepoUrl { get; set; } - } - - public class IdentityReference - { - public string resourceId { get; set; } - } - - public class ImageReference - { - public string publisher { get; set; } - public string offer { get; set; } - public string sku { get; set; } - public string version { get; set; } - public string id { get; set; } - } - - public class InboundNatPool - { - public string name { get; set; } - public string protocol { get; set; } - public int backendPort { get; set; } - public int frontendPortRangeStart { get; set; } - public int frontendPortRangeEnd { get; set; } - public List networkSecurityGroupRules { get; set; } - } - - public class Metadata - { - public string workspaceId { get; set; } - public string resourceId { get; set; } - public string name { get; set; } - public string description { get; set; } - public string resourceType { get; set; } - public string stewardshipType { get; set; } - public string cloudPlatform { get; set; } - public string cloningInstructions { get; set; } - public ControlledResourceMetadata controlledResourceMetadata { get; set; } - public List resourceLineage { get; set; } - public List properties { get; set; } - public string createdBy { get; set; } - public DateTime createdDate { get; set; } - public string lastUpdatedBy { get; set; } - public DateTime lastUpdatedDate { get; set; } - public string state { get; set; } - public ErrorReport errorReport { get; set; } - public string jobId { get; set; } - } - - public class NetworkConfiguration - { - public string subnetId { get; set; } - public string dynamicVNetAssignmentScope { get; set; } - public EndpointConfiguration endpointConfiguration { get; set; } - public PublicIpAddressConfiguration publicIpAddressConfiguration { get; set; } - } - - public class NetworkSecurityGroupRule - { - public int priority { get; set; } - public string access { get; set; } - public string sourceAddressPrefix { get; set; } - public List sourcePortRanges { get; set; } - } - - public class PrivateResourceUser - { - public string userName { get; set; } - public string privateResourceIamRole { get; set; } - } - - public class Property - { - public string key { get; set; } - public string value { get; set; } - } - - public class PublicIpAddressConfiguration - { - public string provision { get; set; } - public List ipAddressIds { get; set; } - } - - public class Registry - { - public string userName { get; set; } - public string password { get; set; } - public string registryServer { get; set; } - public IdentityReference identityReference { get; set; } - } - - public class Resource - { - public Metadata metadata { get; set; } - public ResourceAttributes resourceAttributes { get; set; } - } - - public class ResourceAttributes - { - public GcpBqDataset gcpBqDataset { get; set; } - public GcpBqDataTable gcpBqDataTable { get; set; } - public GcpDataRepoSnapshot gcpDataRepoSnapshot { get; set; } - public GcpGcsBucket gcpGcsBucket { get; set; } - public GcpGcsObject gcpGcsObject { get; set; } - public GcpAiNotebookInstance gcpAiNotebookInstance { get; set; } - public AzureManagedIdentity azureManagedIdentity { get; set; } - public AzureDatabase azureDatabase { get; set; } - public AzureDisk azureDisk { get; set; } - public AzureStorageContainer azureStorageContainer { get; set; } - public AzureVm azureVm { get; set; } - public AzureBatchPool azureBatchPool { get; set; } - public AwsS3StorageFolder awsS3StorageFolder { get; set; } - public AwsSageMakerNotebook awsSageMakerNotebook { get; set; } - public GitRepo gitRepo { get; set; } - public TerraWorkspace terraWorkspace { get; set; } - public FlexibleResource flexibleResource { get; set; } - } - - public class ResourceFile - { - public string autoStorageContainerName { get; set; } - public string storageContainerUrl { get; set; } - public string httpUrl { get; set; } - public string blobPrefix { get; set; } - public string filePath { get; set; } - public string fileMode { get; set; } - public IdentityReference identityReference { get; set; } - } - - public class ResourceLineage - { - public string sourceWorkspaceId { get; set; } - public string sourceResourceId { get; set; } - } - - public class Root - { - public List resources { get; set; } - } - - public class ScaleSettings - { - public FixedScale fixedScale { get; set; } - public AutoScale autoScale { get; set; } - } - - public class StartTask - { - public string commandLine { get; set; } - public List resourceFiles { get; set; } - public List environmentSettings { get; set; } - public UserIdentity userIdentity { get; set; } - public int maxTaskRetryCount { get; set; } - public bool waitForSuccess { get; set; } - public ContainerSettings containerSettings { get; set; } - } - - public class TerraWorkspace - { - public string referencedWorkspaceId { get; set; } - } - - public class UserAssignedIdentity - { - public string name { get; set; } - public string clientId { get; set; } - public string resourceGroupName { get; set; } - } - - public class UserIdentity - { - public string userName { get; set; } - public AutoUser autoUser { get; set; } - } - - public class VirtualMachineConfiguration - { - public ImageReference imageReference { get; set; } - public string nodeAgentSkuId { get; set; } - } - - -} diff --git a/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs b/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs new file mode 100644 index 000000000..799d8b864 --- /dev/null +++ b/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs @@ -0,0 +1,212 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System; +using System.Text.Json.Serialization; + +namespace TesApi.Web.Management.Models.Terra +{ + /// + /// Azure storage container WSM resource + /// + public class AzureStorageContainer + { + /// + /// Storage container name + /// + [JsonPropertyName("storageContainerName")] + public string StorageContainerName { get; set; } + } + + /// + /// Controlled resource metadata + /// + public class ControlledResourceMetadata + { + /// + /// Access scope + /// + [JsonPropertyName("accessScope")] + public string AccessScope { get; set; } + + /// + /// Managed by + /// + [JsonPropertyName("managedBy")] + public string ManagedBy { get; set; } + + /// + /// Private resource user + /// + [JsonPropertyName("privateResourceUser")] + public PrivateResourceUser PrivateResourceUser { get; set; } + + /// + /// Private resource state + /// + [JsonPropertyName("privateResourceState")] + public string PrivateResourceState { get; set; } + + /// + /// Resource region + /// + [JsonPropertyName("region")] + public string Region { get; set; } + } + + /// + /// WSM resource metadata + /// + public class Metadata + { + /// + /// WSM Workspace ID + /// + [JsonPropertyName("workspaceId")] + public string WorkspaceId { get; set; } + + /// + /// Resource ID + /// + [JsonPropertyName("resourceId")] + public string ResourceId { get; set; } + + /// + /// Resource name + /// + [JsonPropertyName("name")] + public string Name { get; set; } + + /// + /// Resource type + /// + [JsonPropertyName("resourceType")] + public string ResourceType { get; set; } + + /// + /// Stewardship type + /// + [JsonPropertyName("stewardshipType")] + public string StewardshipType { get; set; } + + /// + /// Cloud platform + /// + [JsonPropertyName("cloudPlatform")] + public string CloudPlatform { get; set; } + + /// + /// + /// + [JsonPropertyName("cloningInstructions")] + public string CloningInstructions { get; set; } + + /// + /// Controlled resource metadata + /// + [JsonPropertyName("controlledResourceMetadata")] + public ControlledResourceMetadata ControlledResourceMetadata { get; set; } + + /// + /// Resource linage + /// + [JsonPropertyName("resourceLineage")] + public List ResourceLineage { get; set; } + + /// + /// Additional properties + /// + [JsonPropertyName("properties")] + public Dictionary Properties { get; set; } + + /// + /// Created by + /// + [JsonPropertyName("createdBy")] + public string CreatedBy { get; set; } + + /// + /// Creation date + /// + [JsonPropertyName("createdDate")] + public DateTime CreatedDate { get; set; } + + /// + /// Last updated by + /// + [JsonPropertyName("lastUpdatedBy")] + public string LastUpdatedBy { get; set; } + + /// + /// Last updated date + /// + [JsonPropertyName("lastUpdatedDate")] + public DateTime LastUpdatedDate { get; set; } + + /// + /// Resource state + /// + [JsonPropertyName("state")] + public string State { get; set; } + } + + /// + /// Private resource user + /// + public class PrivateResourceUser + { + /// + /// Iam Role + /// + [JsonPropertyName("privateResourceIamRole")] + public object PrivateResourceIamRole { get; set; } + + /// + /// email of the workspace user to grant access + /// + [JsonPropertyName("userName")] + public string UserName { get; set; } + } + + /// + /// WSM resource + /// + public class Resource + { + /// + /// Metadata + /// + [JsonPropertyName("metadata")] + public Metadata Metadata { get; set; } + /// + /// Resource attributes + /// + [JsonPropertyName("resourceAttributes")] + public ResourceAttributes ResourceAttributes { get; set; } + } + + /// + /// WSM resource attributes + /// + public class ResourceAttributes + { + /// + /// Azure storage container + /// + [JsonPropertyName("azureStorageContainer")] + public AzureStorageContainer AzureStorageContainer { get; set; } + } + + /// + /// Response to get storage container resources from a workspace + /// + public class WsmListContainerResourcesResponse + { + /// + /// List of resources in the workspace + /// + [JsonPropertyName("resources")] + public List Resources { get; set; } + } +} diff --git a/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs b/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs index 81666a3bb..2208ec5b2 100644 --- a/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs +++ b/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs @@ -2,6 +2,8 @@ // Licensed under the MIT License. using System; +using System.Linq; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using System.Web; @@ -14,7 +16,7 @@ namespace TesApi.Web.Storage { /// - /// Provides methods for blob storage access by using local path references in form of /storageaccount/container/blobpath + /// Provides methods for blob storage access for Terra /// public class TerraStorageAccessProvider : StorageAccessProvider { @@ -22,9 +24,10 @@ public class TerraStorageAccessProvider : StorageAccessProvider private readonly TerraWsmApiClient terraWsmApiClient; private const string SasBlobPermissions = "racw"; private const string SasContainerPermissions = "racwl"; + private const string LzStorageAccountNamePattern = "lz[0-9a-f]*"; /// - /// Provides methods for blob storage access by using local path references in form of /storageaccount/container/blobpath + /// Provides methods for blob storage access for Terra /// for Terra /// /// Logger @@ -72,93 +75,106 @@ public override async Task MapLocalPathToSasUrlAsync(string path, Cancel { ArgumentException.ThrowIfNullOrEmpty(path); - var normalizedPath = path; if (!TryParseHttpUrlFromInput(path, out _)) - { // if it is a local path, add the leading slash if missing. - normalizedPath = $"/{path.TrimStart('/')}"; - } - - if (getContainerSas) { - return await MapAndGetSasContainerUrlFromWsmAsync(normalizedPath, cancellationToken); + throw new InvalidOperationException("The path must be a valid HTTP URL"); } + + var terraBlobInfo = await GetTerraBlobInfoFromContainerNameAsync(path); - if (IsKnownExecutionFilePath(normalizedPath)) - { - return await GetMappedSasUrlFromWsmAsync(normalizedPath, cancellationToken); - } - if (!StorageAccountUrlSegments.TryCreate(normalizedPath, out var segments)) + if (getContainerSas) { - throw new Exception( - "Invalid path provided. The path must be a valid blob storage url or a path with the following format: /accountName/container"); + return await GetMappedSasContainerUrlFromWsmAsync(terraBlobInfo,cancellationToken); } - CheckIfAccountIsTerraStorageAccount(segments.AccountName); - - return await GetMappedSasUrlFromWsmAsync(segments.BlobName, cancellationToken); + return await GetMappedSasUrlFromWsmAsync(terraBlobInfo, cancellationToken); } - - public TerraBlobInfo GetTerraBlobInfo(string normalizedPath) + + /// + /// Creates a Terra Blob Info from the container name in the path. The path must be a Terra managed storage URL. + /// This method assumes that the container name contains the workspace ID and validates that the storage container is a Terra workspace resource. + /// The BlobName property contains the blob name without a leading slash. + /// + /// + /// Returns a Terra Blob Info + /// This method will throw if the path is not a valid Terra blob storage url or a normalized path. + public async Task GetTerraBlobInfoFromContainerNameAsync(string path) { - if (!StorageAccountUrlSegments.TryCreate(normalizedPath, out var segments)) + if (!StorageAccountUrlSegments.TryCreate(path, out var segments)) { - throw new Exception( + throw new InvalidOperationException( "Invalid path provided. The path must be a valid blob storage url or a path with the following format: /accountName/container"); } CheckIfAccountIsTerraStorageAccount(segments.AccountName); + Logger.LogInformation($"Getting Workspace ID from the Container Name: {segments.ContainerName}"); + var workspaceId = ToWorkspaceId(segments.ContainerName); + Logger.LogInformation($"Workspace ID to use: {segments.ContainerName}"); + var wsmContainerResourceId = await GetWsmContainerResourceIdAsync(workspaceId, segments.ContainerName); - return new TerraBlobInfo(workspaceId,wsmContainerResourceId, segments.BlobName); + return new TerraBlobInfo(workspaceId, wsmContainerResourceId, segments.ContainerName, segments.BlobName.TrimStart('/')); } - private async Task GetWsmContainerResourceIdAsync(string workspaceId, string segmentsContainerName) + private async Task GetWsmContainerResourceIdAsync(Guid workspaceId, string containerName) { - //terraWsmApiClient. - } + Logger.LogInformation($"Getting container resource information from WSM. Workspace ID: {workspaceId} Container Name: {containerName}"); - private string ToWorkspaceId(string segmentsContainerName) - { - ArgumentException.ThrowIfNullOrEmpty(segmentsContainerName); + try + { + //the goal is to get all containers, therefore the limit is set to 10000 which is a reasonable unreachable number of storage containers in a workspace. + var response = + await terraWsmApiClient.GetContainerResourcesAsync(workspaceId, offset: 0, limit: 10000, + cancellationToken: default); - var guidString = segmentsContainerName.Substring(3); // remove the sc- prefix + var metadata = response.Resources.Single(r => + r.ResourceAttributes.AzureStorageContainer.StorageContainerName.Equals(containerName, + StringComparison.OrdinalIgnoreCase)).Metadata; - var guid = Guid.Parse(guidString); // throws if not a guid + Logger.LogInformation($"Found the resource id for storage container resource. Resource ID: {metadata.ResourceId} Container Name: {containerName}"); - return guid.ToString(); + return Guid.Parse(metadata.ResourceId); + } + catch (Exception e) + { + Logger.LogError(e, "Failed to call WSM to obtain the storage container resource ID"); + throw; + } } - private async Task MapAndGetSasContainerUrlFromWsmAsync(string inputPath, CancellationToken cancellationToken) + + private Guid ToWorkspaceId(string segmentsContainerName) { - if (IsKnownExecutionFilePath(inputPath)) + try { - return await GetMappedSasContainerUrlFromWsmAsync(inputPath, cancellationToken); - } + ArgumentException.ThrowIfNullOrEmpty(segmentsContainerName); + + var guidString = segmentsContainerName.Substring(3); // remove the sc- prefix - if (!StorageAccountUrlSegments.TryCreate(inputPath, out var withContainerSegments)) + return Guid.Parse(guidString); // throws if not a guid + } + catch (Exception e) { - throw new Exception( - "Invalid path provided. The path must be a valid blob storage url or a path with the following format: /accountName/container"); + Logger.LogError(e, $"Failed to get the workspace ID from the container name. The name provided is not a valid GUID. Container Name: {segmentsContainerName}"); + throw; } - - return await GetMappedSasContainerUrlFromWsmAsync(withContainerSegments.BlobName, cancellationToken); } - private async Task GetMappedSasContainerUrlFromWsmAsync(string pathToAppend, CancellationToken cancellationToken) + private async Task GetMappedSasContainerUrlFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) { //an empty blob name gets a container Sas token - var tokenInfo = await GetSasTokenFromWsmAsync(CreateTokenParamsFromOptions(blobName: string.Empty, SasContainerPermissions), cancellationToken); + var tokenInfo = await GetSasTokenForContainerFromWsmAsync(blobInfo, cancellationToken); var urlBuilder = new UriBuilder(tokenInfo.Url); - if (!string.IsNullOrEmpty(pathToAppend.TrimStart('/'))) + if (!string.IsNullOrEmpty(blobInfo.BlobName)) { - urlBuilder.Path += $"/{pathToAppend.TrimStart('/')}"; + urlBuilder.Path += $"/{blobInfo.BlobName}"; } return urlBuilder.Uri.ToString(); @@ -167,29 +183,25 @@ private async Task GetMappedSasContainerUrlFromWsmAsync(string pathToApp /// /// Returns a Url with a SAS token for the given input /// - /// + /// /// A for controlling the lifetime of the asynchronous operation. - /// SAS Token URL - public async Task GetMappedSasUrlFromWsmAsync(string blobName, CancellationToken cancellationToken) + /// URL with a SAS token + public async Task GetMappedSasUrlFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) { - var normalizedBlobName = blobName.TrimStart('/'); - - var tokenParams = CreateTokenParamsFromOptions(normalizedBlobName, SasBlobPermissions); - - var tokenInfo = await GetSasTokenFromWsmAsync(tokenParams, cancellationToken); - + var tokenInfo = await GetSasTokenFromWsmAsync(blobInfo, cancellationToken); + Logger.LogInformation($"Successfully obtained the Sas Url from Terra. Wsm resource id:{terraOptions.WorkspaceStorageContainerResourceId}"); - + var uriBuilder = new UriBuilder(tokenInfo.Url); - - if (normalizedBlobName != string.Empty) + + if (blobInfo.BlobName != string.Empty) { - if (!uriBuilder.Path.Contains(normalizedBlobName, StringComparison.OrdinalIgnoreCase)) + if (!uriBuilder.Path.Contains(blobInfo.BlobName, StringComparison.OrdinalIgnoreCase)) { - uriBuilder.Path += $"/{normalizedBlobName}"; + uriBuilder.Path += $"/{blobInfo.BlobName}"; } } - + return uriBuilder.Uri.ToString(); } @@ -199,26 +211,31 @@ private SasTokenApiParameters CreateTokenParamsFromOptions(string blobName, stri terraOptions.SasTokenExpirationInSeconds, sasPermissions, blobName); - private async Task GetSasTokenFromWsmAsync(SasTokenApiParameters tokenParams, CancellationToken cancellationToken) + + private async Task GetSasTokenFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) { + var tokenParams = CreateTokenParamsFromOptions(blobInfo.BlobName, SasBlobPermissions); + Logger.LogInformation( - $"Getting Sas Url from Terra. Wsm resource id:{terraOptions.WorkspaceStorageContainerResourceId}"); + $"Getting Sas Url from Terra. Wsm workspace id:{blobInfo.WorkspaceId}"); + return await terraWsmApiClient.GetSasTokenAsync( - Guid.Parse(terraOptions.WorkspaceId), - Guid.Parse(terraOptions.WorkspaceStorageContainerResourceId), + blobInfo.WorkspaceId, + blobInfo.WsmContainerResourceId, tokenParams, cancellationToken); } - private async Task GetSasTokenFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) + private async Task GetSasTokenForContainerFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) { - var tokenParams = CreateTokenParamsFromOptions(blobInfo.BlobName, SasBlobPermissions); - + // an empty blob name gets a container Sas token + var tokenParams = CreateTokenParamsFromOptions(blobName:"", SasContainerPermissions); + Logger.LogInformation( - $"Getting Sas Url from Terra. Wsm workspace id:{terraOptions.WorkspaceStorageContainerResourceId}"); + $"Getting Sas container Url from Terra. Wsm workspace id:{blobInfo.WorkspaceId}"); return await terraWsmApiClient.GetSasTokenAsync( - Guid.Parse(terraOptions.WorkspaceId), - Guid.Parse(terraOptions.WorkspaceStorageContainerResourceId), + blobInfo.WorkspaceId, + blobInfo.WsmContainerResourceId, tokenParams, cancellationToken); } @@ -231,13 +248,20 @@ private void CheckIfAccountIsTerraStorageAccount(string accountName) } } - private bool IsTerraWorkspaceContainer(string value) - => terraOptions.WorkspaceStorageContainerName.Equals(value, StringComparison.OrdinalIgnoreCase); - private bool IsTerraWorkspaceStorageAccount(string value) - => terraOptions.WorkspaceStorageAccountName.Equals(value, StringComparison.OrdinalIgnoreCase); - } + { + var match = Regex.Match(value, LzStorageAccountNamePattern); - public record TerraBlobInfo(string WorkspaceId, string WsmContainerResourceId, string BlobName); + return match.Success; + } + } + /// + /// Contains the Terra attributes related to the blob. + /// + /// + /// + /// + /// + public record TerraBlobInfo(Guid WorkspaceId, Guid WsmContainerResourceId, string WsmContainerName, string BlobName); } diff --git a/src/TesApi.Web/TesApi.Web.csproj b/src/TesApi.Web/TesApi.Web.csproj index 9e3f0021d..b9c11ca75 100644 --- a/src/TesApi.Web/TesApi.Web.csproj +++ b/src/TesApi.Web/TesApi.Web.csproj @@ -63,9 +63,7 @@ - + False None Never @@ -105,12 +103,12 @@ - + - <__ProjectNotReferenced Include="@(_MSBuildProjectReferenceExistent)" Condition=" '%(Filename)' == 'Tes.RunnerCLI' "/> - <_MSBuildProjectReferenceExistent Remove="@(__ProjectNotReferenced)"/> + <__ProjectNotReferenced Include="@(_MSBuildProjectReferenceExistent)" Condition=" '%(Filename)' == 'Tes.RunnerCLI' " /> + <_MSBuildProjectReferenceExistent Remove="@(__ProjectNotReferenced)" /> <_MSBuildProjectReferenceExistent Include="@(__ProjectNotReferenced)"> %(__ProjectNotReferenced.AdditionalProperties);PublishDir=$(__TesRunnerPublishDir);%(__ProjectNotReferenced.SetConfiguration);%(__ProjectNotReferenced.SetPlatform);%(__ProjectNotReferenced.SetTargetFramework) From 7441abae9522c9929ed94164f9ba8ddf8e1a8b9e Mon Sep 17 00:00:00 2001 From: giventocode <3589801+giventocode@users.noreply.github.com> Date: Mon, 26 Jun 2023 12:11:10 -0400 Subject: [PATCH 3/8] test refactoring --- src/TesApi.Tests/TerraApiStubData.cs | 4 ++-- src/TesApi.Tests/TerraStorageAccessProviderTests.cs | 10 +++++----- src/TesApi.Web/Storage/TerraStorageAccessProvider.cs | 7 +++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/TesApi.Tests/TerraApiStubData.cs b/src/TesApi.Tests/TerraApiStubData.cs index 4130dcfff..48e07b39b 100644 --- a/src/TesApi.Tests/TerraApiStubData.cs +++ b/src/TesApi.Tests/TerraApiStubData.cs @@ -14,10 +14,10 @@ public class TerraApiStubData public const string LandingZoneApiHost = "https://landingzone.host"; public const string WsmApiHost = "https://wsm.host"; public const string ResourceGroup = "mrg-terra-dev-previ-20191228"; - public const string WorkspaceAccountName = "fooaccount"; + public const string WorkspaceAccountName = "lzaccount1"; public const string WorkspaceContainerName = "sc-ef9fed44-dba6-4825-868c-b00208522382"; public const string SasToken = "SASTOKENSTUB="; - public const string WsmGetSasResponseStorageUrl = $"https://bloburl.foo/{WorkspaceContainerName}"; + public const string WsmGetSasResponseStorageUrl = $"https://{WorkspaceAccountName}.blob.core.windows.net/{WorkspaceContainerName}"; public Guid LandingZoneId { get; } = Guid.NewGuid(); public Guid SubscriptionId { get; } = Guid.NewGuid(); diff --git a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs index aae2c0a32..7bbdc1891 100644 --- a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs +++ b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs @@ -80,11 +80,11 @@ public async Task MapLocalPathToSasUrlAsync_ValidInput(string input) } [TestMethod] - [DataRow($"{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] - [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] - [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "/", TerraApiStubData.WsmGetSasResponseStorageUrl)] - [DataRow("/cromwell-executions/test", "", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/cromwell-executions/test")] - [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "/dir/blobName", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/dir/blobName")] + //[DataRow($"{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] + //[DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] + //[DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "/", TerraApiStubData.WsmGetSasResponseStorageUrl)] + //[DataRow("/cromwell-executions/test", "", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/cromwell-executions/test")] + //[DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "/dir/blobName", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/dir/blobName")] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}", "/dir/blob", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/dir/blob")] public async Task MapLocalPathToSasUrlAsync_GetContainerSasIsTrue(string input, string blobPath, string expected) diff --git a/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs b/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs index 2208ec5b2..f0bbd43bb 100644 --- a/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs +++ b/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs @@ -83,7 +83,6 @@ public override async Task MapLocalPathToSasUrlAsync(string path, Cancel var terraBlobInfo = await GetTerraBlobInfoFromContainerNameAsync(path); - if (getContainerSas) { return await GetMappedSasContainerUrlFromWsmAsync(terraBlobInfo,cancellationToken); @@ -95,11 +94,11 @@ public override async Task MapLocalPathToSasUrlAsync(string path, Cancel /// /// Creates a Terra Blob Info from the container name in the path. The path must be a Terra managed storage URL. /// This method assumes that the container name contains the workspace ID and validates that the storage container is a Terra workspace resource. - /// The BlobName property contains the blob name without a leading slash. + /// The BlobName property contains the blob name segment without a leading slash. /// /// /// Returns a Terra Blob Info - /// This method will throw if the path is not a valid Terra blob storage url or a normalized path. + /// This method will throw if the path is not a valid Terra blob storage url. public async Task GetTerraBlobInfoFromContainerNameAsync(string path) { if (!StorageAccountUrlSegments.TryCreate(path, out var segments)) @@ -257,7 +256,7 @@ private bool IsTerraWorkspaceStorageAccount(string value) } /// - /// Contains the Terra attributes related to the blob. + /// Contains the Terra and Azure Storage container properties where the blob is contained. /// /// /// From b0995b719c1487a203c775d79db1999f8ed444fe Mon Sep 17 00:00:00 2001 From: giventocode <3589801+giventocode@users.noreply.github.com> Date: Fri, 30 Jun 2023 13:26:50 -0400 Subject: [PATCH 4/8] cross wks support for Terra storage provider --- .../TerraWsmApiClientIntegrationTests.cs | 75 +++++++++++++++++++ .../Integration/TestEnvTokenCredential.cs | 41 ++++++++++ .../Integration/TestTerraEnvInfo.cs | 51 +++++++++++++ src/TesApi.Tests/TerraApiStubData.cs | 55 ++++++++++++-- .../TerraStorageAccessProviderTests.cs | 47 +++++------- src/TesApi.Tests/TerraWsmApiClientTests.cs | 20 +++++ .../Management/Clients/TerraWsmApiClient.cs | 4 +- .../WsmListContainerResourcesResponse.cs | 2 +- 8 files changed, 259 insertions(+), 36 deletions(-) create mode 100644 src/TesApi.Tests/Integration/TerraWsmApiClientIntegrationTests.cs create mode 100644 src/TesApi.Tests/Integration/TestEnvTokenCredential.cs create mode 100644 src/TesApi.Tests/Integration/TestTerraEnvInfo.cs diff --git a/src/TesApi.Tests/Integration/TerraWsmApiClientIntegrationTests.cs b/src/TesApi.Tests/Integration/TerraWsmApiClientIntegrationTests.cs new file mode 100644 index 000000000..ae04dcdbb --- /dev/null +++ b/src/TesApi.Tests/Integration/TerraWsmApiClientIntegrationTests.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TesApi.Web.Management; +using TesApi.Web.Management.Clients; +using TesApi.Web.Management.Configuration; + +namespace TesApi.Tests.Integration +{ + [TestClass, TestCategory("TerraIntegration")] + public class TerraWsmApiClientIntegrationTests + { + private TerraWsmApiClient wsmApiClient = null!; + private TestTerraEnvInfo envInfo = null!; + + [TestInitialize] + public void Setup() + { + envInfo = new TestTerraEnvInfo(); + + var terraOptions = Options.Create(new TerraOptions() + { + WsmApiHost = envInfo.WsmApiHost + }); + var retryOptions = Options.Create(new RetryPolicyOptions()); + var memoryCache = new MemoryCache(new MemoryCacheOptions()); + + wsmApiClient = new TerraWsmApiClient(new TestEnvTokenCredential(), terraOptions, + new CacheAndRetryHandler(memoryCache,retryOptions), TestLoggerFactory.Create()); + + } + + [TestMethod] + public async Task GetContainerResourcesAsync_CallsUsingTheWSIdFromContainerName_ReturnsContainerInformation() + { + var workspaceId = Guid.Parse(envInfo.WorkspaceContainerName.Replace("sc-", "")); + + var results = await wsmApiClient.GetContainerResourcesAsync(workspaceId, 0, 100, CancellationToken.None); + + Assert.IsNotNull(results); + Assert.IsTrue(results.Resources.Any(i=>i.ResourceAttributes.AzureStorageContainer.StorageContainerName.Equals(envInfo.WorkspaceContainerName,StringComparison.OrdinalIgnoreCase))); + } + } + + public static class TestLoggerFactory + { + private static readonly ILoggerFactory SLogFactory = LoggerFactory.Create(builder => + { + builder + .AddSystemdConsole(options => + { + options.IncludeScopes = true; + options.TimestampFormat = "yyyy-MM-dd HH:mm:ss.fff "; + options.UseUtcTimestamp = true; + }); + + builder.SetMinimumLevel(LogLevel.Trace); + }); + + + public static ILogger Create() + { + return SLogFactory.CreateLogger(); + } + } +} diff --git a/src/TesApi.Tests/Integration/TestEnvTokenCredential.cs b/src/TesApi.Tests/Integration/TestEnvTokenCredential.cs new file mode 100644 index 000000000..9236db902 --- /dev/null +++ b/src/TesApi.Tests/Integration/TestEnvTokenCredential.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; + +namespace TesApi.Tests.Integration +{ + internal class TestEnvTokenCredential : TokenCredential + { + public const string TerraTokenEnvVariableName = "TERRA_AUTH_TOKEN"; + + public override ValueTask GetTokenAsync(TokenRequestContext requestContext, CancellationToken cancellationToken) + { + var token = GetTokenFromEnvVariable(); + + return new ValueTask(new AccessToken(token, DateTimeOffset.MaxValue)); + } + + private static string GetTokenFromEnvVariable() + { + var token = Environment.GetEnvironmentVariable(TerraTokenEnvVariableName); + + if (string.IsNullOrEmpty(token)) + { + throw new InvalidOperationException($"Environment variable {TerraTokenEnvVariableName} is not set."); + } + + return token; + } + + public override AccessToken GetToken(TokenRequestContext requestContext, CancellationToken cancellationToken) + { + var token = GetTokenFromEnvVariable(); + + return new AccessToken(token, DateTimeOffset.MaxValue); + } + } +} diff --git a/src/TesApi.Tests/Integration/TestTerraEnvInfo.cs b/src/TesApi.Tests/Integration/TestTerraEnvInfo.cs new file mode 100644 index 000000000..b8bcc7d59 --- /dev/null +++ b/src/TesApi.Tests/Integration/TestTerraEnvInfo.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace TesApi.Tests.Integration +{ + internal class TestTerraEnvInfo + { + private const string LzStorageAccountNameEnvVarName = "TERRA_LZ_STORAGE_ACCOUNT"; + private const string WorkspaceContainerNameEnvVarName = "TERRA_WS_STORAGE_CONTAINER"; + private const string WsmApiHostEnvVarName = "TERRA_WSM_API_HOST"; + + private readonly string lzStorageAccountName; + private readonly string workspaceContainerName; + private readonly string wsmApiHost; + + public string LzStorageAccountName => lzStorageAccountName; + public string WorkspaceContainerName => workspaceContainerName; + public string WsmApiHost => wsmApiHost; + + public TestTerraEnvInfo() + { + lzStorageAccountName = Environment.GetEnvironmentVariable(LzStorageAccountNameEnvVarName); + workspaceContainerName = Environment.GetEnvironmentVariable(WorkspaceContainerNameEnvVarName); + wsmApiHost = Environment.GetEnvironmentVariable(WsmApiHostEnvVarName); + + if (string.IsNullOrEmpty(lzStorageAccountName)) + { + throw new InvalidOperationException( + $"The environment variable {LzStorageAccountNameEnvVarName} is not set"); + } + + if (string.IsNullOrEmpty(workspaceContainerName)) + { + throw new InvalidOperationException( + $"The environment variable {WorkspaceContainerNameEnvVarName} is not set"); + } + + if (string.IsNullOrEmpty(wsmApiHost)) + { + throw new InvalidOperationException( + $"The environment variable {WsmApiHostEnvVarName} is not set"); + } + } + } +} diff --git a/src/TesApi.Tests/TerraApiStubData.cs b/src/TesApi.Tests/TerraApiStubData.cs index 48e07b39b..305c238f9 100644 --- a/src/TesApi.Tests/TerraApiStubData.cs +++ b/src/TesApi.Tests/TerraApiStubData.cs @@ -15,9 +15,9 @@ public class TerraApiStubData public const string WsmApiHost = "https://wsm.host"; public const string ResourceGroup = "mrg-terra-dev-previ-20191228"; public const string WorkspaceAccountName = "lzaccount1"; - public const string WorkspaceContainerName = "sc-ef9fed44-dba6-4825-868c-b00208522382"; + public const string WorkspaceStorageContainerName = "sc-ef9fed44-dba6-4825-868c-b00208522382"; public const string SasToken = "SASTOKENSTUB="; - public const string WsmGetSasResponseStorageUrl = $"https://{WorkspaceAccountName}.blob.core.windows.net/{WorkspaceContainerName}"; + public const string WsmGetSasResponseStorageUrl = $"https://{WorkspaceAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}"; public Guid LandingZoneId { get; } = Guid.NewGuid(); public Guid SubscriptionId { get; } = Guid.NewGuid(); @@ -29,6 +29,11 @@ public class TerraApiStubData $"/subscriptions/{SubscriptionId}/resourceGroups/{ResourceGroup}/providers/Microsoft.Batch/batchAccounts/{BatchAccountName}"; public string PoolId => "poolId"; + + public Guid GetWorkspaceIdFromContainerName(string containerName) + { + return Guid.Parse(containerName.Replace("sc-","")); + } public LandingZoneResourcesApiResponse GetResourceApiResponse() { return JsonSerializer.Deserialize(GetResourceApiResponseInJson()); @@ -51,7 +56,7 @@ public TerraOptions GetTerraOptions() LandingZoneApiHost = LandingZoneApiHost, WsmApiHost = WsmApiHost, WorkspaceStorageAccountName = WorkspaceAccountName, - WorkspaceStorageContainerName = WorkspaceContainerName, + WorkspaceStorageContainerName = WorkspaceStorageContainerName, WorkspaceStorageContainerResourceId = ContainerResourceId.ToString() }; } @@ -183,6 +188,44 @@ public string GetResourceApiResponseInJson() }}"; } + public string GetContainerResourcesApiResponseInJson() + { + return $@"{{ + ""resources"": [ + {{ + ""metadata"": {{ + ""workspaceId"": ""{WorkspaceId}"", + ""resourceId"": ""{ContainerResourceId}"", + ""name"": ""{WorkspaceStorageContainerName}"", + ""resourceType"": ""AZURE_STORAGE_CONTAINER"", + ""stewardshipType"": ""CONTROLLED"", + ""cloudPlatform"": ""AZURE"", + ""cloningInstructions"": ""COPY_NOTHING"", + ""controlledResourceMetadata"": {{ + ""accessScope"": ""SHARED_ACCESS"", + ""managedBy"": ""USER"", + ""privateResourceUser"": {{}}, + ""privateResourceState"": ""NOT_APPLICABLE"", + ""region"": ""southcentralus"" + }}, + ""resourceLineage"": [], + ""properties"": [], + ""createdBy"": ""user@foo.com"", + ""createdDate"": ""2023-02-09T01:48:46.040052Z"", + ""lastUpdatedBy"": ""user@foo.com"", + ""lastUpdatedDate"": ""2023-02-09T01:48:48.345442Z"", + ""state"": ""READY"" + }}, + ""resourceAttributes"": {{ + ""azureStorageContainer"": {{ + ""storageContainerName"": ""{WorkspaceStorageContainerName}"" + }} + }} + }} + ] +}}"; + } + public string GetResourceQuotaApiResponseInJson() { return $@"{{ @@ -304,14 +347,14 @@ public WsmListContainerResourcesResponse GetWsmContainerResourcesApiResponse() { Metadata = new Metadata() { - ResourceId = Guid.NewGuid().ToString() - + ResourceId = ContainerResourceId.ToString(), + Name = WorkspaceStorageContainerName }, ResourceAttributes = new ResourceAttributes() { AzureStorageContainer = new AzureStorageContainer() { - StorageContainerName = WorkspaceContainerName + StorageContainerName = WorkspaceStorageContainerName } } } diff --git a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs index 7bbdc1891..a5878291e 100644 --- a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs +++ b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs @@ -21,7 +21,7 @@ namespace TesApi.Tests public class TerraStorageAccessProviderTests { private const string WorkspaceStorageAccountName = TerraApiStubData.WorkspaceAccountName; - private const string WorkspaceStorageContainerName = TerraApiStubData.WorkspaceContainerName; + private const string WorkspaceStorageContainerName = TerraApiStubData.WorkspaceStorageContainerName; private Mock wsmApiClientMock; private Mock azureProxyMock; @@ -60,38 +60,35 @@ public async Task IsHttpPublicAsync_StringScenario(string input, bool expectedRe } [TestMethod] - [DataRow($"{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}")] - [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}")] - [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}/")] - [DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}/dir/blobName")] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}")] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}/dir/blob")] public async Task MapLocalPathToSasUrlAsync_ValidInput(string input) { - wsmApiClientMock.Setup(a => a.GetSasTokenAsync(It.IsAny(), - It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(terraApiStubData.GetWsmSasTokenApiResponse()); - wsmApiClientMock.Setup(a=>a.GetContainerResourcesAsync(It.IsAny(),It.IsAny(),It.IsAny(),It.IsAny())) - .ReturnsAsync(terraApiStubData.GetWsmContainerResourcesApiResponse()); + SetUpTerraApiClient(); var result = await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input, CancellationToken.None); Assert.IsNotNull(terraApiStubData.GetWsmSasTokenApiResponse().Url, result); } + private void SetUpTerraApiClient() + { + wsmApiClientMock.Setup(a => a.GetSasTokenAsync( + terraApiStubData.GetWorkspaceIdFromContainerName(WorkspaceStorageContainerName), + terraApiStubData.ContainerResourceId, It.IsAny(), It.IsAny())) + .ReturnsAsync(terraApiStubData.GetWsmSasTokenApiResponse()); + wsmApiClientMock.Setup(a => + a.GetContainerResourcesAsync(It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny())) + .ReturnsAsync(terraApiStubData.GetWsmContainerResourcesApiResponse()); + } + [TestMethod] - //[DataRow($"{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] - //[DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] - //[DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "/", TerraApiStubData.WsmGetSasResponseStorageUrl)] - //[DataRow("/cromwell-executions/test", "", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/cromwell-executions/test")] - //[DataRow($"/{WorkspaceStorageAccountName}/{WorkspaceStorageContainerName}", "/dir/blobName", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/dir/blobName")] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}", "", TerraApiStubData.WsmGetSasResponseStorageUrl)] [DataRow($"https://{WorkspaceStorageAccountName}.blob.core.windows.net/{WorkspaceStorageContainerName}", "/dir/blob", $"{TerraApiStubData.WsmGetSasResponseStorageUrl}/dir/blob")] public async Task MapLocalPathToSasUrlAsync_GetContainerSasIsTrue(string input, string blobPath, string expected) { - wsmApiClientMock.Setup(a => a.GetSasTokenAsync(terraApiStubData.WorkspaceId, - terraApiStubData.ContainerResourceId, It.IsAny(), It.IsAny())) - .ReturnsAsync(terraApiStubData.GetWsmSasTokenApiResponse()); + SetUpTerraApiClient(); var result = await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input + blobPath, CancellationToken.None, true); @@ -100,12 +97,10 @@ public async Task MapLocalPathToSasUrlAsync_GetContainerSasIsTrue(string input, } [TestMethod] - [DataRow($"/bar/{WorkspaceStorageContainerName}")] - [DataRow("/foo/bar/")] - [DataRow("/foo/bar/dir/blobName")] [DataRow($"https://bar.blob.core.windows.net/{WorkspaceStorageContainerName}/")] + [DataRow($"https://bar.blob.core.windows.net/container/")] [ExpectedException(typeof(Exception))] - public async Task MapLocalPathToSasUrlAsync_InvalidInputs(string input) + public async Task MapLocalPathToSasUrlAsync_InvalidStorageAccountInputs(string input) { await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input, CancellationToken.None); } @@ -115,17 +110,15 @@ public async Task MapLocalPathToSasUrlAsync_InvalidInputs(string input) [DataRow("blobName")] public async Task GetMappedSasUrlFromWsmAsync_WithOrWithOutBlobName_ReturnsValidURLWithBlobName(string responseBlobName) { - wsmApiClientMock.Setup(a => a.GetSasTokenAsync(terraApiStubData.WorkspaceId, - terraApiStubData.ContainerResourceId, It.IsAny(), It.IsAny())) - .ReturnsAsync(terraApiStubData.GetWsmSasTokenApiResponse(responseBlobName)); + SetUpTerraApiClient(); - var blobInfo = new TerraBlobInfo(terraApiStubData.WorkspaceId,terraApiStubData.ContainerResourceId, TerraApiStubData.WorkspaceContainerName, "blobName"); + var blobInfo = new TerraBlobInfo(terraApiStubData.GetWorkspaceIdFromContainerName(WorkspaceStorageContainerName), terraApiStubData.ContainerResourceId, TerraApiStubData.WorkspaceStorageContainerName, "blobName"); var url = await terraStorageAccessProvider.GetMappedSasUrlFromWsmAsync(blobInfo, CancellationToken.None); Assert.IsNotNull(url); var uri = new Uri(url); - Assert.AreEqual(uri.AbsolutePath, $"/{TerraApiStubData.WorkspaceContainerName}/blobName"); + Assert.AreEqual(uri.AbsolutePath, $"/{TerraApiStubData.WorkspaceStorageContainerName}/blobName"); } } } diff --git a/src/TesApi.Tests/TerraWsmApiClientTests.cs b/src/TesApi.Tests/TerraWsmApiClientTests.cs index c86af5d00..7215e325f 100644 --- a/src/TesApi.Tests/TerraWsmApiClientTests.cs +++ b/src/TesApi.Tests/TerraWsmApiClientTests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Linq; using System.Net; using System.Net.Http; using System.Threading.Tasks; @@ -110,6 +111,25 @@ public async Task GetSasTokenAsync_ValidRequest_ReturnsPayload() Assert.IsTrue(!string.IsNullOrEmpty(apiResponse.Url)); } + [TestMethod] + public async Task GetContainerResourcesAsync_ValidRequest_ReturnsPayload() + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(terraApiStubData.GetContainerResourcesApiResponseInJson()) + }; + + cacheAndRetryHandler.Setup(c => c.ExecuteWithRetryAsync(It.IsAny>>(), It.IsAny())) + .ReturnsAsync(response); + + var apiResponse = await terraWsmApiClient.GetContainerResourcesAsync(terraApiStubData.WorkspaceId, + offset:0, limit:10, System.Threading.CancellationToken.None); + + Assert.IsNotNull(apiResponse); + Assert.AreEqual(1, apiResponse.Resources.Count); + Assert.IsTrue(apiResponse.Resources.Any(r=>r.Metadata.ResourceId.ToString().Equals(terraApiStubData.ContainerResourceId.ToString(), StringComparison.OrdinalIgnoreCase))); + } + [TestMethod] public async Task DeleteBatchPoolAsync_204Response_Succeeds() { diff --git a/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs b/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs index 795ead650..deba2192b 100644 --- a/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs +++ b/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs @@ -46,7 +46,7 @@ public TerraWsmApiClient(TokenCredential tokenCredential, IOptions protected TerraWsmApiClient() { } /// - /// Returns the SAS token of a container or blob for WSM managed storage account. + /// Returns storage containers in the workspace. /// /// Terra workspace id /// Number of items to skip before starting to collect the result @@ -57,7 +57,7 @@ public virtual async Task GetContainerResourc { var url = GetContainerResourcesApiUrl(workspaceId, offset, limit); - var response = await HttpSendRequestWithRetryPolicyAsync(() => new HttpRequestMessage(HttpMethod.Post, url), + var response = await HttpSendRequestWithRetryPolicyAsync(() => new HttpRequestMessage(HttpMethod.Get, url), cancellationToken, setAuthorizationHeader: true); return await GetApiResponseContentAsync(response, cancellationToken); diff --git a/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs b/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs index 799d8b864..cb8f08da3 100644 --- a/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs +++ b/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs @@ -118,7 +118,7 @@ public class Metadata /// Additional properties /// [JsonPropertyName("properties")] - public Dictionary Properties { get; set; } + public List Properties { get; set; } /// /// Created by From 8db9de477afec3357830b7cee311a2388cdcccba Mon Sep 17 00:00:00 2001 From: giventocode <3589801+giventocode@users.noreply.github.com> Date: Fri, 30 Jun 2023 13:29:23 -0400 Subject: [PATCH 5/8] code formatting --- .../TerraWsmApiClientIntegrationTests.cs | 8 ++++---- .../Integration/TestTerraEnvInfo.cs | 2 +- src/TesApi.Tests/TerraApiStubData.cs | 2 +- src/TesApi.Tests/TerraWsmApiClientTests.cs | 4 ++-- .../Management/Clients/TerraWsmApiClient.cs | 2 +- .../Terra/WsmListContainerResourcesResponse.cs | 14 +++++++------- .../Storage/TerraStorageAccessProvider.cs | 18 +++++++++--------- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/TesApi.Tests/Integration/TerraWsmApiClientIntegrationTests.cs b/src/TesApi.Tests/Integration/TerraWsmApiClientIntegrationTests.cs index ae04dcdbb..f28f3cb79 100644 --- a/src/TesApi.Tests/Integration/TerraWsmApiClientIntegrationTests.cs +++ b/src/TesApi.Tests/Integration/TerraWsmApiClientIntegrationTests.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.VisualStudio.TestTools.UnitTesting; using TesApi.Web.Management; @@ -17,6 +16,7 @@ namespace TesApi.Tests.Integration { [TestClass, TestCategory("TerraIntegration")] + [Ignore] public class TerraWsmApiClientIntegrationTests { private TerraWsmApiClient wsmApiClient = null!; @@ -35,7 +35,7 @@ public void Setup() var memoryCache = new MemoryCache(new MemoryCacheOptions()); wsmApiClient = new TerraWsmApiClient(new TestEnvTokenCredential(), terraOptions, - new CacheAndRetryHandler(memoryCache,retryOptions), TestLoggerFactory.Create()); + new CacheAndRetryHandler(memoryCache, retryOptions), TestLoggerFactory.Create()); } @@ -47,7 +47,7 @@ public async Task GetContainerResourcesAsync_CallsUsingTheWSIdFromContainerName_ var results = await wsmApiClient.GetContainerResourcesAsync(workspaceId, 0, 100, CancellationToken.None); Assert.IsNotNull(results); - Assert.IsTrue(results.Resources.Any(i=>i.ResourceAttributes.AzureStorageContainer.StorageContainerName.Equals(envInfo.WorkspaceContainerName,StringComparison.OrdinalIgnoreCase))); + Assert.IsTrue(results.Resources.Any(i => i.ResourceAttributes.AzureStorageContainer.StorageContainerName.Equals(envInfo.WorkspaceContainerName, StringComparison.OrdinalIgnoreCase))); } } @@ -62,7 +62,7 @@ public static class TestLoggerFactory options.TimestampFormat = "yyyy-MM-dd HH:mm:ss.fff "; options.UseUtcTimestamp = true; }); - + builder.SetMinimumLevel(LogLevel.Trace); }); diff --git a/src/TesApi.Tests/Integration/TestTerraEnvInfo.cs b/src/TesApi.Tests/Integration/TestTerraEnvInfo.cs index b8bcc7d59..07cc521da 100644 --- a/src/TesApi.Tests/Integration/TestTerraEnvInfo.cs +++ b/src/TesApi.Tests/Integration/TestTerraEnvInfo.cs @@ -28,7 +28,7 @@ public TestTerraEnvInfo() lzStorageAccountName = Environment.GetEnvironmentVariable(LzStorageAccountNameEnvVarName); workspaceContainerName = Environment.GetEnvironmentVariable(WorkspaceContainerNameEnvVarName); wsmApiHost = Environment.GetEnvironmentVariable(WsmApiHostEnvVarName); - + if (string.IsNullOrEmpty(lzStorageAccountName)) { throw new InvalidOperationException( diff --git a/src/TesApi.Tests/TerraApiStubData.cs b/src/TesApi.Tests/TerraApiStubData.cs index 305c238f9..c85428169 100644 --- a/src/TesApi.Tests/TerraApiStubData.cs +++ b/src/TesApi.Tests/TerraApiStubData.cs @@ -32,7 +32,7 @@ public class TerraApiStubData public Guid GetWorkspaceIdFromContainerName(string containerName) { - return Guid.Parse(containerName.Replace("sc-","")); + return Guid.Parse(containerName.Replace("sc-", "")); } public LandingZoneResourcesApiResponse GetResourceApiResponse() { diff --git a/src/TesApi.Tests/TerraWsmApiClientTests.cs b/src/TesApi.Tests/TerraWsmApiClientTests.cs index 7215e325f..0fb20864f 100644 --- a/src/TesApi.Tests/TerraWsmApiClientTests.cs +++ b/src/TesApi.Tests/TerraWsmApiClientTests.cs @@ -123,11 +123,11 @@ public async Task GetContainerResourcesAsync_ValidRequest_ReturnsPayload() .ReturnsAsync(response); var apiResponse = await terraWsmApiClient.GetContainerResourcesAsync(terraApiStubData.WorkspaceId, - offset:0, limit:10, System.Threading.CancellationToken.None); + offset: 0, limit: 10, System.Threading.CancellationToken.None); Assert.IsNotNull(apiResponse); Assert.AreEqual(1, apiResponse.Resources.Count); - Assert.IsTrue(apiResponse.Resources.Any(r=>r.Metadata.ResourceId.ToString().Equals(terraApiStubData.ContainerResourceId.ToString(), StringComparison.OrdinalIgnoreCase))); + Assert.IsTrue(apiResponse.Resources.Any(r => r.Metadata.ResourceId.ToString().Equals(terraApiStubData.ContainerResourceId.ToString(), StringComparison.OrdinalIgnoreCase))); } [TestMethod] diff --git a/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs b/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs index deba2192b..a0ec24410 100644 --- a/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs +++ b/src/TesApi.Web/Management/Clients/TerraWsmApiClient.cs @@ -71,7 +71,7 @@ private Uri GetContainerResourcesApiUrl(Guid workspaceId, int offset, int limit) //TODO: add support for resource and stewardship parameters if required later builder.Query = $"offset={offset}&limit={limit}&resource=AZURE_STORAGE_CONTAINER&stewardship=CONTROLLED"; - + return builder.Uri; } diff --git a/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs b/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs index cb8f08da3..0125bc661 100644 --- a/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs +++ b/src/TesApi.Web/Management/Models/Terra/WsmListContainerResourcesResponse.cs @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.Collections.Generic; using System; +using System.Collections.Generic; using System.Text.Json.Serialization; namespace TesApi.Web.Management.Models.Terra @@ -95,31 +95,31 @@ public class Metadata /// [JsonPropertyName("cloudPlatform")] public string CloudPlatform { get; set; } - + /// /// /// [JsonPropertyName("cloningInstructions")] public string CloningInstructions { get; set; } - + /// /// Controlled resource metadata /// [JsonPropertyName("controlledResourceMetadata")] public ControlledResourceMetadata ControlledResourceMetadata { get; set; } - + /// /// Resource linage /// [JsonPropertyName("resourceLineage")] public List ResourceLineage { get; set; } - + /// /// Additional properties /// [JsonPropertyName("properties")] public List Properties { get; set; } - + /// /// Created by /// @@ -161,7 +161,7 @@ public class PrivateResourceUser /// [JsonPropertyName("privateResourceIamRole")] public object PrivateResourceIamRole { get; set; } - + /// /// email of the workspace user to grant access /// diff --git a/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs b/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs index f0bbd43bb..f47f7a2b6 100644 --- a/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs +++ b/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs @@ -80,17 +80,17 @@ public override async Task MapLocalPathToSasUrlAsync(string path, Cancel { throw new InvalidOperationException("The path must be a valid HTTP URL"); } - + var terraBlobInfo = await GetTerraBlobInfoFromContainerNameAsync(path); if (getContainerSas) { - return await GetMappedSasContainerUrlFromWsmAsync(terraBlobInfo,cancellationToken); + return await GetMappedSasContainerUrlFromWsmAsync(terraBlobInfo, cancellationToken); } return await GetMappedSasUrlFromWsmAsync(terraBlobInfo, cancellationToken); } - + /// /// Creates a Terra Blob Info from the container name in the path. The path must be a Terra managed storage URL. /// This method assumes that the container name contains the workspace ID and validates that the storage container is a Terra workspace resource. @@ -188,11 +188,11 @@ private async Task GetMappedSasContainerUrlFromWsmAsync(TerraBlobInfo bl public async Task GetMappedSasUrlFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) { var tokenInfo = await GetSasTokenFromWsmAsync(blobInfo, cancellationToken); - + Logger.LogInformation($"Successfully obtained the Sas Url from Terra. Wsm resource id:{terraOptions.WorkspaceStorageContainerResourceId}"); - + var uriBuilder = new UriBuilder(tokenInfo.Url); - + if (blobInfo.BlobName != string.Empty) { if (!uriBuilder.Path.Contains(blobInfo.BlobName, StringComparison.OrdinalIgnoreCase)) @@ -200,7 +200,7 @@ public async Task GetMappedSasUrlFromWsmAsync(TerraBlobInfo blobInfo, Ca uriBuilder.Path += $"/{blobInfo.BlobName}"; } } - + return uriBuilder.Uri.ToString(); } @@ -214,7 +214,7 @@ private SasTokenApiParameters CreateTokenParamsFromOptions(string blobName, stri private async Task GetSasTokenFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) { var tokenParams = CreateTokenParamsFromOptions(blobInfo.BlobName, SasBlobPermissions); - + Logger.LogInformation( $"Getting Sas Url from Terra. Wsm workspace id:{blobInfo.WorkspaceId}"); @@ -227,7 +227,7 @@ private async Task GetSasTokenFromWsmAsync(TerraBlobInfo private async Task GetSasTokenForContainerFromWsmAsync(TerraBlobInfo blobInfo, CancellationToken cancellationToken) { // an empty blob name gets a container Sas token - var tokenParams = CreateTokenParamsFromOptions(blobName:"", SasContainerPermissions); + var tokenParams = CreateTokenParamsFromOptions(blobName: "", SasContainerPermissions); Logger.LogInformation( $"Getting Sas container Url from Terra. Wsm workspace id:{blobInfo.WorkspaceId}"); From 6f688d5fac5b0f80be6b3603b0ceeb2ce53c1261 Mon Sep 17 00:00:00 2001 From: giventocode <3589801+giventocode@users.noreply.github.com> Date: Fri, 30 Jun 2023 18:14:57 -0400 Subject: [PATCH 6/8] pr feedback --- .../TerraStorageAccessProviderTests.cs | 7 ++++++- .../Storage/TerraStorageAccessProvider.cs | 14 +++++++------- src/TesApi.Web/TesApi.Web.csproj | 12 +++++++----- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs index a5878291e..e9bdd7b79 100644 --- a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs +++ b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs @@ -97,9 +97,14 @@ public async Task MapLocalPathToSasUrlAsync_GetContainerSasIsTrue(string input, } [TestMethod] + [DataRow($"{WorkspaceStorageAccountName}/foo")] + [DataRow($"/bar/{WorkspaceStorageContainerName}")] + [DataRow($"/foo/bar/")] + [DataRow($"/foo/bar/dir/blobName")] [DataRow($"https://bar.blob.core.windows.net/{WorkspaceStorageContainerName}/")] [DataRow($"https://bar.blob.core.windows.net/container/")] - [ExpectedException(typeof(Exception))] + [ExpectedException(typeof(InvalidOperationException))] + //[ExpectedException(typeof(Exception))] public async Task MapLocalPathToSasUrlAsync_InvalidStorageAccountInputs(string input) { await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input, CancellationToken.None); diff --git a/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs b/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs index f47f7a2b6..b3208c2f5 100644 --- a/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs +++ b/src/TesApi.Web/Storage/TerraStorageAccessProvider.cs @@ -81,7 +81,7 @@ public override async Task MapLocalPathToSasUrlAsync(string path, Cancel throw new InvalidOperationException("The path must be a valid HTTP URL"); } - var terraBlobInfo = await GetTerraBlobInfoFromContainerNameAsync(path); + var terraBlobInfo = await GetTerraBlobInfoFromContainerNameAsync(path, cancellationToken); if (getContainerSas) { @@ -97,9 +97,10 @@ public override async Task MapLocalPathToSasUrlAsync(string path, Cancel /// The BlobName property contains the blob name segment without a leading slash. /// /// + /// /// Returns a Terra Blob Info /// This method will throw if the path is not a valid Terra blob storage url. - public async Task GetTerraBlobInfoFromContainerNameAsync(string path) + private async Task GetTerraBlobInfoFromContainerNameAsync(string path, CancellationToken cancellationToken) { if (!StorageAccountUrlSegments.TryCreate(path, out var segments)) { @@ -115,12 +116,12 @@ public async Task GetTerraBlobInfoFromContainerNameAsync(string p Logger.LogInformation($"Workspace ID to use: {segments.ContainerName}"); - var wsmContainerResourceId = await GetWsmContainerResourceIdAsync(workspaceId, segments.ContainerName); + var wsmContainerResourceId = await GetWsmContainerResourceIdAsync(workspaceId, segments.ContainerName, cancellationToken); return new TerraBlobInfo(workspaceId, wsmContainerResourceId, segments.ContainerName, segments.BlobName.TrimStart('/')); } - private async Task GetWsmContainerResourceIdAsync(Guid workspaceId, string containerName) + private async Task GetWsmContainerResourceIdAsync(Guid workspaceId, string containerName, CancellationToken cancellationToken) { Logger.LogInformation($"Getting container resource information from WSM. Workspace ID: {workspaceId} Container Name: {containerName}"); @@ -128,8 +129,7 @@ private async Task GetWsmContainerResourceIdAsync(Guid workspaceId, string { //the goal is to get all containers, therefore the limit is set to 10000 which is a reasonable unreachable number of storage containers in a workspace. var response = - await terraWsmApiClient.GetContainerResourcesAsync(workspaceId, offset: 0, limit: 10000, - cancellationToken: default); + await terraWsmApiClient.GetContainerResourcesAsync(workspaceId, offset: 0, limit: 10000, cancellationToken); var metadata = response.Resources.Single(r => r.ResourceAttributes.AzureStorageContainer.StorageContainerName.Equals(containerName, @@ -243,7 +243,7 @@ private void CheckIfAccountIsTerraStorageAccount(string accountName) { if (!IsTerraWorkspaceStorageAccount(accountName)) { - throw new Exception($"The account name does not match the configuration for Terra."); + throw new InvalidOperationException($"The account name does not match the configuration for Terra."); } } diff --git a/src/TesApi.Web/TesApi.Web.csproj b/src/TesApi.Web/TesApi.Web.csproj index b9c11ca75..305af8fbf 100644 --- a/src/TesApi.Web/TesApi.Web.csproj +++ b/src/TesApi.Web/TesApi.Web.csproj @@ -63,7 +63,9 @@ - + False None Never @@ -103,12 +105,12 @@ - + - <__ProjectNotReferenced Include="@(_MSBuildProjectReferenceExistent)" Condition=" '%(Filename)' == 'Tes.RunnerCLI' " /> - <_MSBuildProjectReferenceExistent Remove="@(__ProjectNotReferenced)" /> + <__ProjectNotReferenced Include="@(_MSBuildProjectReferenceExistent)" Condition=" '%(Filename)' == 'Tes.RunnerCLI' "/> + <_MSBuildProjectReferenceExistent Remove="@(__ProjectNotReferenced)"/> <_MSBuildProjectReferenceExistent Include="@(__ProjectNotReferenced)"> %(__ProjectNotReferenced.AdditionalProperties);PublishDir=$(__TesRunnerPublishDir);%(__ProjectNotReferenced.SetConfiguration);%(__ProjectNotReferenced.SetPlatform);%(__ProjectNotReferenced.SetTargetFramework) @@ -146,4 +148,4 @@ - + \ No newline at end of file From 0ca6a3a83b9c24027c31ad619e63e3154c8fa957 Mon Sep 17 00:00:00 2001 From: giventocode <3589801+giventocode@users.noreply.github.com> Date: Fri, 30 Jun 2023 18:16:46 -0400 Subject: [PATCH 7/8] removed extra line in project file --- src/TesApi.Web/TesApi.Web.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/TesApi.Web/TesApi.Web.csproj b/src/TesApi.Web/TesApi.Web.csproj index 305af8fbf..c89bf666d 100644 --- a/src/TesApi.Web/TesApi.Web.csproj +++ b/src/TesApi.Web/TesApi.Web.csproj @@ -147,5 +147,4 @@ - \ No newline at end of file From cda15509051f34d86c8ce33e6a635f2572272b82 Mon Sep 17 00:00:00 2001 From: giventocode <3589801+giventocode@users.noreply.github.com> Date: Fri, 30 Jun 2023 18:22:11 -0400 Subject: [PATCH 8/8] minor formatting --- src/TesApi.Tests/TerraStorageAccessProviderTests.cs | 1 - src/TesApi.Web/TesApi.Web.csproj | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs index e9bdd7b79..a919af61c 100644 --- a/src/TesApi.Tests/TerraStorageAccessProviderTests.cs +++ b/src/TesApi.Tests/TerraStorageAccessProviderTests.cs @@ -104,7 +104,6 @@ public async Task MapLocalPathToSasUrlAsync_GetContainerSasIsTrue(string input, [DataRow($"https://bar.blob.core.windows.net/{WorkspaceStorageContainerName}/")] [DataRow($"https://bar.blob.core.windows.net/container/")] [ExpectedException(typeof(InvalidOperationException))] - //[ExpectedException(typeof(Exception))] public async Task MapLocalPathToSasUrlAsync_InvalidStorageAccountInputs(string input) { await terraStorageAccessProvider.MapLocalPathToSasUrlAsync(input, CancellationToken.None); diff --git a/src/TesApi.Web/TesApi.Web.csproj b/src/TesApi.Web/TesApi.Web.csproj index c89bf666d..305af8fbf 100644 --- a/src/TesApi.Web/TesApi.Web.csproj +++ b/src/TesApi.Web/TesApi.Web.csproj @@ -147,4 +147,5 @@ + \ No newline at end of file